2158 files changed, 133075 insertions, 63715 deletions

diff --git a/Documentation/ABI/testing/ima_policy b/Documentation/ABI/testing/ima_policy
index 6cd6daefaae..98694661354 100644
--- a/Documentation/ABI/testing/ima_policy
+++ b/Documentation/ABI/testing/ima_policy
@@ -12,11 +12,14 @@ Description:
 		then closing the file.  The new policy takes effect after
 		the file ima/policy is closed.
 
+		IMA appraisal, if configured, uses these file measurements
+		for local measurement appraisal.
+
 		rule format: action [condition ...]
 
-		action: measure | dont_measure
+		action: measure | dont_measure | appraise | dont_appraise | audit
 		condition:= base | lsm
-			base:	[[func=] [mask=] [fsmagic=] [uid=]]
+			base:	[[func=] [mask=] [fsmagic=] [uid=] [fowner]]
 			lsm:	[[subj_user=] [subj_role=] [subj_type=]
 				 [obj_user=] [obj_role=] [obj_type=]]
 
@@ -24,36 +27,50 @@ Description:
 			mask:= [MAY_READ] [MAY_WRITE] [MAY_APPEND] [MAY_EXEC]
 			fsmagic:= hex value
 			uid:= decimal value
+			fowner:=decimal value
 		lsm:  	are LSM specific
 
 		default policy:
 			# PROC_SUPER_MAGIC
 			dont_measure fsmagic=0x9fa0
+			dont_appraise fsmagic=0x9fa0
 			# SYSFS_MAGIC
 			dont_measure fsmagic=0x62656572
+			dont_appraise fsmagic=0x62656572
 			# DEBUGFS_MAGIC
 			dont_measure fsmagic=0x64626720
+			dont_appraise fsmagic=0x64626720
 			# TMPFS_MAGIC
 			dont_measure fsmagic=0x01021994
+			dont_appraise fsmagic=0x01021994
+			# RAMFS_MAGIC
+			dont_measure fsmagic=0x858458f6
+			dont_appraise fsmagic=0x858458f6
 			# SECURITYFS_MAGIC
 			dont_measure fsmagic=0x73636673
+			dont_appraise fsmagic=0x73636673
 
 			measure func=BPRM_CHECK
 			measure func=FILE_MMAP mask=MAY_EXEC
 			measure func=FILE_CHECK mask=MAY_READ uid=0
+			appraise fowner=0
 
 		The default policy measures all executables in bprm_check,
 		all files mmapped executable in file_mmap, and all files
-		open for read by root in do_filp_open.
+		open for read by root in do_filp_open.  The default appraisal
+		policy appraises all files owned by root.
 
 		Examples of LSM specific definitions:
 
 		SELinux:
 			# SELINUX_MAGIC
-			dont_measure fsmagic=0xF97CFF8C
+			dont_measure fsmagic=0xf97cff8c
+			dont_appraise fsmagic=0xf97cff8c
 
 			dont_measure obj_type=var_log_t
+			dont_appraise obj_type=var_log_t
 			dont_measure obj_type=auditd_log_t
+			dont_appraise obj_type=auditd_log_t
 			measure subj_user=system_u func=FILE_CHECK mask=MAY_READ
 			measure subj_role=system_r func=FILE_CHECK mask=MAY_READ
 
diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 5dab36448b4..6943133afcb 100644
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -176,3 +176,14 @@ Description:	Disable L3 cache indices
 		All AMD processors with L3 caches provide this functionality.
 		For details, see BKDGs at
 		http://developer.amd.com/documentation/guides/Pages/default.aspx
+
+
+What:		/sys/devices/system/cpu/cpufreq/boost
+Date:		August 2012
+Contact:	Linux kernel mailing list <linux-kernel@vger.kernel.org>
+Description:	Processor frequency boosting control
+
+		This switch controls the boost setting for the whole system.
+		Boosting allows the CPU and the firmware to run at a frequency
+		beyound it's nominal limit.
+		More details can be found in Documentation/cpu-freq/boost.txt
diff --git a/Documentation/ABI/testing/sysfs-driver-ppi b/Documentation/ABI/testing/sysfs-driver-ppi
new file mode 100644
index 00000000000..97a003ee058
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-ppi
@@ -0,0 +1,70 @@
+What:		/sys/devices/pnp0/<bus-num>/ppi/
+Date:		August 2012
+Kernel Version:	3.6
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This folder includes the attributes related with PPI (Physical
+		Presence Interface). Only if TPM is supported by BIOS, this
+		folder makes sence. The folder path can be got by command
+		'find /sys/ -name 'pcrs''. For the detail information of PPI,
+		please refer to the PPI specification from
+		http://www.trustedcomputinggroup.org/
+
+What:		/sys/devices/pnp0/<bus-num>/ppi/version
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows the version of the PPI supported by the
+		platform.
+		This file is readonly.
+
+What:		/sys/devices/pnp0/<bus-num>/ppi/request
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows the request for an operation to be
+		executed in the pre-OS environment. It is the only input from
+		the OS to the pre-OS environment. The request should be an
+		integer value range from 1 to 160, and 0 means no request.
+		This file can be read and written.
+
+What:		/sys/devices/pnp0/00:<bus-num>/ppi/response
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows the response to the most recent operation
+		request it acted upon. The format is "<request> <response num>
+		: <response description>".
+		This file is readonly.
+
+What:		/sys/devices/pnp0/<bus-num>/ppi/transition_action
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows the platform-specific action that should
+		take place in order to transition to the BIOS for execution of
+		a requested operation. The format is "<action num>: <action
+		description>".
+		This file is readonly.
+
+What:		/sys/devices/pnp0/<bus-num>/ppi/tcg_operations
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows whether it is allowed to request an
+		operation to be executed in the pre-OS environment by the BIOS
+		for the requests defined by TCG, i.e. requests from 1 to 22.
+		The format is "<request> <status num>: <status description>".
+		This attribute is only supported by PPI version 1.2+.
+		This file is readonly.
+
+What:		/sys/devices/pnp0/<bus-num>/ppi/vs_operations
+Date:		August 2012
+Contact:	xiaoyan.zhang@intel.com
+Description:
+		This attribute shows whether it is allowed to request an
+		operation to be executed in the pre-OS environment by the BIOS
+		for the verdor specific requests, i.e. requests from 128 to
+		255. The format is same with tcg_operations. This attribute
+		is also only supported by PPI version 1.2+.
+		This file is readonly.
diff --git a/Documentation/CodingStyle b/Documentation/CodingStyle
index cb9258b8fd3..495e5ba1634 100644
--- a/Documentation/CodingStyle
+++ b/Documentation/CodingStyle
@@ -454,6 +454,16 @@ The preferred style for long (multi-line) comments is:
 	 * with beginning and ending almost-blank lines.
 	 */
 
+For files in net/ and drivers/net/ the preferred style for long (multi-line)
+comments is a little different.
+
+	/* The preferred comment style for files in net/ and drivers/net
+	 * looks like this.
+	 *
+	 * It is nearly the same as the generally preferred comment style,
+	 * but there is no initial almost-blank line.
+	 */
+
 It's also important to comment data, whether they are basic types or derived
 types.  To this end, use just one data declaration per line (no commas for
 multiple data declarations).  This leaves you room for a small comment on each
diff --git a/Documentation/DocBook/drm.tmpl b/Documentation/DocBook/drm.tmpl
index 196b8b9dba1..b0300529ab1 100644
--- a/Documentation/DocBook/drm.tmpl
+++ b/Documentation/DocBook/drm.tmpl
@@ -6,11 +6,36 @@
   <bookinfo>
     <title>Linux DRM Developer's Guide</title>
 
+    <authorgroup>
+      <author>
+	<firstname>Jesse</firstname>
+	<surname>Barnes</surname>
+	<contrib>Initial version</contrib>
+	<affiliation>
+	  <orgname>Intel Corporation</orgname>
+	  <address>
+	    <email>jesse.barnes@intel.com</email>
+	  </address>
+	</affiliation>
+      </author>
+      <author>
+	<firstname>Laurent</firstname>
+	<surname>Pinchart</surname>
+	<contrib>Driver internals</contrib>
+	<affiliation>
+	  <orgname>Ideas on board SPRL</orgname>
+	  <address>
+	    <email>laurent.pinchart@ideasonboard.com</email>
+	  </address>
+	</affiliation>
+      </author>
+    </authorgroup>
+
     <copyright>
       <year>2008-2009</year>
-      <holder>
-	Intel Corporation (Jesse Barnes &lt;jesse.barnes@intel.com&gt;)
-      </holder>
+      <year>2012</year>
+      <holder>Intel Corporation</holder>
+      <holder>Laurent Pinchart</holder>
     </copyright>
 
     <legalnotice>
@@ -20,6 +45,17 @@
 	the kernel source COPYING file.
       </para>
     </legalnotice>
+
+    <revhistory>
+      <!-- Put document revisions here, newest first. -->
+      <revision>
+	<revnumber>1.0</revnumber>
+	<date>2012-07-13</date>
+	<authorinitials>LP</authorinitials>
+	<revremark>Added extensive documentation about driver internals.
+	</revremark>
+      </revision>
+    </revhistory>
   </bookinfo>
 
 <toc></toc>
@@ -72,342 +108,361 @@
       submission &amp; fencing, suspend/resume support, and DMA
       services.
     </para>
-    <para>
-      The core of every DRM driver is struct drm_driver.  Drivers
-      typically statically initialize a drm_driver structure,
-      then pass it to drm_init() at load time.
-    </para>
 
   <!-- Internals: driver init -->
 
   <sect1>
-    <title>Driver initialization</title>
-    <para>
-      Before calling the DRM initialization routines, the driver must
-      first create and fill out a struct drm_driver structure.
-    </para>
-    <programlisting>
-      static struct drm_driver driver = {
-	/* Don't use MTRRs here; the Xserver or userspace app should
-	 * deal with them for Intel hardware.
-	 */
-	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_REQUIRE_AGP |
-	    DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_MODESET,
-	.load = i915_driver_load,
-	.unload = i915_driver_unload,
-	.firstopen = i915_driver_firstopen,
-	.lastclose = i915_driver_lastclose,
-	.preclose = i915_driver_preclose,
-	.save = i915_save,
-	.restore = i915_restore,
-	.device_is_agp = i915_driver_device_is_agp,
-	.get_vblank_counter = i915_get_vblank_counter,
-	.enable_vblank = i915_enable_vblank,
-	.disable_vblank = i915_disable_vblank,
-	.irq_preinstall = i915_driver_irq_preinstall,
-	.irq_postinstall = i915_driver_irq_postinstall,
-	.irq_uninstall = i915_driver_irq_uninstall,
-	.irq_handler = i915_driver_irq_handler,
-	.reclaim_buffers = drm_core_reclaim_buffers,
-	.get_map_ofs = drm_core_get_map_ofs,
-	.get_reg_ofs = drm_core_get_reg_ofs,
-	.fb_probe = intelfb_probe,
-	.fb_remove = intelfb_remove,
-	.fb_resize = intelfb_resize,
-	.master_create = i915_master_create,
-	.master_destroy = i915_master_destroy,
-#if defined(CONFIG_DEBUG_FS)
-	.debugfs_init = i915_debugfs_init,
-	.debugfs_cleanup = i915_debugfs_cleanup,
-#endif
-	.gem_init_object = i915_gem_init_object,
-	.gem_free_object = i915_gem_free_object,
-	.gem_vm_ops = &amp;i915_gem_vm_ops,
-	.ioctls = i915_ioctls,
-	.fops = {
-		.owner = THIS_MODULE,
-		.open = drm_open,
-		.release = drm_release,
-		.ioctl = drm_ioctl,
-		.mmap = drm_mmap,
-		.poll = drm_poll,
-		.fasync = drm_fasync,
-#ifdef CONFIG_COMPAT
-		.compat_ioctl = i915_compat_ioctl,
-#endif
-		.llseek = noop_llseek,
-		},
-	.pci_driver = {
-		.name = DRIVER_NAME,
-		.id_table = pciidlist,
-		.probe = probe,
-		.remove = __devexit_p(drm_cleanup_pci),
-		},
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.date = DRIVER_DATE,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
-      };
-    </programlisting>
-    <para>
-      In the example above, taken from the i915 DRM driver, the driver
-      sets several flags indicating what core features it supports;
-      we go over the individual callbacks in later sections.  Since
-      flags indicate which features your driver supports to the DRM
-      core, you need to set most of them prior to calling drm_init().  Some,
-      like DRIVER_MODESET can be set later based on user supplied parameters,
-      but that's the exception rather than the rule.
-    </para>
-    <variablelist>
-      <title>Driver flags</title>
-      <varlistentry>
-	<term>DRIVER_USE_AGP</term>
-	<listitem><para>
-	    Driver uses AGP interface
-	</para></listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_REQUIRE_AGP</term>
-	<listitem><para>
-	    Driver needs AGP interface to function.
-	</para></listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_USE_MTRR</term>
-	<listitem>
-	  <para>
-	    Driver uses MTRR interface for mapping memory.  Deprecated.
-	  </para>
-	</listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_PCI_DMA</term>
-	<listitem><para>
-	    Driver is capable of PCI DMA.  Deprecated.
-	</para></listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_SG</term>
-	<listitem><para>
-	    Driver can perform scatter/gather DMA.  Deprecated.
-	</para></listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_HAVE_DMA</term>
-	<listitem><para>Driver supports DMA.  Deprecated.</para></listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_HAVE_IRQ</term><term>DRIVER_IRQ_SHARED</term>
-	<listitem>
-	  <para>
-	    DRIVER_HAVE_IRQ indicates whether the driver has an IRQ
-	    handler.  DRIVER_IRQ_SHARED indicates whether the device &amp;
-	    handler support shared IRQs (note that this is required of
-	    PCI drivers).
-	  </para>
-	</listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_DMA_QUEUE</term>
-	<listitem>
-	  <para>
-	    Should be set if the driver queues DMA requests and completes them
-	    asynchronously.  Deprecated.
-	  </para>
-	</listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_FB_DMA</term>
-	<listitem>
-	  <para>
-	    Driver supports DMA to/from the framebuffer.  Deprecated.
-	  </para>
-	</listitem>
-      </varlistentry>
-      <varlistentry>
-	<term>DRIVER_MODESET</term>
-	<listitem>
-	  <para>
-	    Driver supports mode setting interfaces.
-	  </para>
-	</listitem>
-      </varlistentry>
-    </variablelist>
-    <para>
-      In this specific case, the driver requires AGP and supports
-      IRQs.  DMA, as discussed later, is handled by device-specific ioctls
-      in this case.  It also supports the kernel mode setting APIs, though
-      unlike in the actual i915 driver source, this example unconditionally
-      exports KMS capability.
+    <title>Driver Initialization</title>
+    <para>
+      At the core of every DRM driver is a <structname>drm_driver</structname>
+      structure. Drivers typically statically initialize a drm_driver structure,
+      and then pass it to one of the <function>drm_*_init()</function> functions
+      to register it with the DRM subsystem.
     </para>
-  </sect1>
-
-  <!-- Internals: driver load -->
-
-  <sect1>
-    <title>Driver load</title>
-    <para>
-      In the previous section, we saw what a typical drm_driver
-      structure might look like.  One of the more important fields in
-      the structure is the hook for the load function.
-    </para>
-    <programlisting>
-      static struct drm_driver driver = {
-      	...
-      	.load = i915_driver_load,
-        ...
-      };
-    </programlisting>
-    <para>
-      The load function has many responsibilities: allocating a driver
-      private structure, specifying supported performance counters,
-      configuring the device (e.g. mapping registers &amp; command
-      buffers), initializing the memory manager, and setting up the
-      initial output configuration.
-    </para>
-    <para>
-      If compatibility is a concern (e.g. with drivers converted over
-      to the new interfaces from the old ones), care must be taken to
-      prevent device initialization and control that is incompatible with
-      currently active userspace drivers.  For instance, if user
-      level mode setting drivers are in use, it would be problematic
-      to perform output discovery &amp; configuration at load time.
-      Likewise, if user-level drivers unaware of memory management are
-      in use, memory management and command buffer setup may need to
-      be omitted.  These requirements are driver-specific, and care
-      needs to be taken to keep both old and new applications and
-      libraries working.  The i915 driver supports the "modeset"
-      module parameter to control whether advanced features are
-      enabled at load time or in legacy fashion.
+    <para>
+      The <structname>drm_driver</structname> structure contains static
+      information that describes the driver and features it supports, and
+      pointers to methods that the DRM core will call to implement the DRM API.
+      We will first go through the <structname>drm_driver</structname> static
+      information fields, and will then describe individual operations in
+      details as they get used in later sections.
     </para>
-
     <sect2>
-      <title>Driver private &amp; performance counters</title>
-      <para>
-	The driver private hangs off the main drm_device structure and
-	can be used for tracking various device-specific bits of
-	information, like register offsets, command buffer status,
-	register state for suspend/resume, etc.  At load time, a
-	driver may simply allocate one and set drm_device.dev_priv
-	appropriately; it should be freed and drm_device.dev_priv set
-	to NULL when the driver is unloaded.
-      </para>
+      <title>Driver Information</title>
+      <sect3>
+        <title>Driver Features</title>
+        <para>
+          Drivers inform the DRM core about their requirements and supported
+          features by setting appropriate flags in the
+          <structfield>driver_features</structfield> field. Since those flags
+          influence the DRM core behaviour since registration time, most of them
+          must be set to registering the <structname>drm_driver</structname>
+          instance.
+        </para>
+        <synopsis>u32 driver_features;</synopsis>
+        <variablelist>
+          <title>Driver Feature Flags</title>
+          <varlistentry>
+            <term>DRIVER_USE_AGP</term>
+            <listitem><para>
+              Driver uses AGP interface, the DRM core will manage AGP resources.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_REQUIRE_AGP</term>
+            <listitem><para>
+              Driver needs AGP interface to function. AGP initialization failure
+              will become a fatal error.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_USE_MTRR</term>
+            <listitem><para>
+              Driver uses MTRR interface for mapping memory, the DRM core will
+              manage MTRR resources. Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_PCI_DMA</term>
+            <listitem><para>
+              Driver is capable of PCI DMA, mapping of PCI DMA buffers to
+              userspace will be enabled. Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_SG</term>
+            <listitem><para>
+              Driver can perform scatter/gather DMA, allocation and mapping of
+              scatter/gather buffers will be enabled. Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_HAVE_DMA</term>
+            <listitem><para>
+              Driver supports DMA, the userspace DMA API will be supported.
+              Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_HAVE_IRQ</term><term>DRIVER_IRQ_SHARED</term>
+            <listitem><para>
+              DRIVER_HAVE_IRQ indicates whether the driver has an IRQ handler. The
+              DRM core will automatically register an interrupt handler when the
+              flag is set. DRIVER_IRQ_SHARED indicates whether the device &amp;
+              handler support shared IRQs (note that this is required of PCI
+              drivers).
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_IRQ_VBL</term>
+            <listitem><para>Unused. Deprecated.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_DMA_QUEUE</term>
+            <listitem><para>
+              Should be set if the driver queues DMA requests and completes them
+              asynchronously.  Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_FB_DMA</term>
+            <listitem><para>
+              Driver supports DMA to/from the framebuffer, mapping of frambuffer
+              DMA buffers to userspace will be supported. Deprecated.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_IRQ_VBL2</term>
+            <listitem><para>Unused. Deprecated.</para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_GEM</term>
+            <listitem><para>
+              Driver use the GEM memory manager.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_MODESET</term>
+            <listitem><para>
+              Driver supports mode setting interfaces (KMS).
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRIVER_PRIME</term>
+            <listitem><para>
+              Driver implements DRM PRIME buffer sharing.
+            </para></listitem>
+          </varlistentry>
+        </variablelist>
+      </sect3>
+      <sect3>
+        <title>Major, Minor and Patchlevel</title>
+        <synopsis>int major;
+int minor;
+int patchlevel;</synopsis>
+        <para>
+          The DRM core identifies driver versions by a major, minor and patch
+          level triplet. The information is printed to the kernel log at
+          initialization time and passed to userspace through the
+          DRM_IOCTL_VERSION ioctl.
+        </para>
+        <para>
+          The major and minor numbers are also used to verify the requested driver
+          API version passed to DRM_IOCTL_SET_VERSION. When the driver API changes
+          between minor versions, applications can call DRM_IOCTL_SET_VERSION to
+          select a specific version of the API. If the requested major isn't equal
+          to the driver major, or the requested minor is larger than the driver
+          minor, the DRM_IOCTL_SET_VERSION call will return an error. Otherwise
+          the driver's set_version() method will be called with the requested
+          version.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Name, Description and Date</title>
+        <synopsis>char *name;
+char *desc;
+char *date;</synopsis>
+        <para>
+          The driver name is printed to the kernel log at initialization time,
+          used for IRQ registration and passed to userspace through
+          DRM_IOCTL_VERSION.
+        </para>
+        <para>
+          The driver description is a purely informative string passed to
+          userspace through the DRM_IOCTL_VERSION ioctl and otherwise unused by
+          the kernel.
+        </para>
+        <para>
+          The driver date, formatted as YYYYMMDD, is meant to identify the date of
+          the latest modification to the driver. However, as most drivers fail to
+          update it, its value is mostly useless. The DRM core prints it to the
+          kernel log at initialization time and passes it to userspace through the
+          DRM_IOCTL_VERSION ioctl.
+        </para>
+      </sect3>
+    </sect2>
+    <sect2>
+      <title>Driver Load</title>
       <para>
-	The DRM supports several counters which may be used for rough
-	performance characterization.  Note that the DRM stat counter
-	system is not often used by applications, and supporting
-	additional counters is completely optional.
+        The <methodname>load</methodname> method is the driver and device
+        initialization entry point. The method is responsible for allocating and
+        initializing driver private data, specifying supported performance
+        counters, performing resource allocation and mapping (e.g. acquiring
+        clocks, mapping registers or allocating command buffers), initializing
+        the memory manager (<xref linkend="drm-memory-management"/>), installing
+        the IRQ handler (<xref linkend="drm-irq-registration"/>), setting up
+        vertical blanking handling (<xref linkend="drm-vertical-blank"/>), mode
+	setting (<xref linkend="drm-mode-setting"/>) and initial output
+	configuration (<xref linkend="drm-kms-init"/>).
       </para>
+      <note><para>
+        If compatibility is a concern (e.g. with drivers converted over from
+        User Mode Setting to Kernel Mode Setting), care must be taken to prevent
+        device initialization and control that is incompatible with currently
+        active userspace drivers. For instance, if user level mode setting
+        drivers are in use, it would be problematic to perform output discovery
+        &amp; configuration at load time. Likewise, if user-level drivers
+        unaware of memory management are in use, memory management and command
+        buffer setup may need to be omitted. These requirements are
+        driver-specific, and care needs to be taken to keep both old and new
+        applications and libraries working.
+      </para></note>
+      <synopsis>int (*load) (struct drm_device *, unsigned long flags);</synopsis>
       <para>
-	These interfaces are deprecated and should not be used.  If performance
-	monitoring is desired, the developer should investigate and
-	potentially enhance the kernel perf and tracing infrastructure to export
-	GPU related performance information for consumption by performance
-	monitoring tools and applications.
+        The method takes two arguments, a pointer to the newly created
+	<structname>drm_device</structname> and flags. The flags are used to
+	pass the <structfield>driver_data</structfield> field of the device id
+	corresponding to the device passed to <function>drm_*_init()</function>.
+	Only PCI devices currently use this, USB and platform DRM drivers have
+	their <methodname>load</methodname> method called with flags to 0.
       </para>
+      <sect3>
+        <title>Driver Private &amp; Performance Counters</title>
+        <para>
+          The driver private hangs off the main
+          <structname>drm_device</structname> structure and can be used for
+          tracking various device-specific bits of information, like register
+          offsets, command buffer status, register state for suspend/resume, etc.
+          At load time, a driver may simply allocate one and set
+          <structname>drm_device</structname>.<structfield>dev_priv</structfield>
+          appropriately; it should be freed and
+          <structname>drm_device</structname>.<structfield>dev_priv</structfield>
+          set to NULL when the driver is unloaded.
+        </para>
+        <para>
+          DRM supports several counters which were used for rough performance
+          characterization. This stat counter system is deprecated and should not
+          be used. If performance monitoring is desired, the developer should
+          investigate and potentially enhance the kernel perf and tracing
+          infrastructure to export GPU related performance information for
+          consumption by performance monitoring tools and applications.
+        </para>
+      </sect3>
+      <sect3 id="drm-irq-registration">
+        <title>IRQ Registration</title>
+        <para>
+          The DRM core tries to facilitate IRQ handler registration and
+          unregistration by providing <function>drm_irq_install</function> and
+          <function>drm_irq_uninstall</function> functions. Those functions only
+          support a single interrupt per device.
+        </para>
+  <!--!Fdrivers/char/drm/drm_irq.c drm_irq_install-->
+        <para>
+          Both functions get the device IRQ by calling
+          <function>drm_dev_to_irq</function>. This inline function will call a
+          bus-specific operation to retrieve the IRQ number. For platform devices,
+          <function>platform_get_irq</function>(..., 0) is used to retrieve the
+          IRQ number.
+        </para>
+        <para>
+          <function>drm_irq_install</function> starts by calling the
+          <methodname>irq_preinstall</methodname> driver operation. The operation
+          is optional and must make sure that the interrupt will not get fired by
+          clearing all pending interrupt flags or disabling the interrupt.
+        </para>
+        <para>
+          The IRQ will then be requested by a call to
+          <function>request_irq</function>. If the DRIVER_IRQ_SHARED driver
+          feature flag is set, a shared (IRQF_SHARED) IRQ handler will be
+          requested.
+        </para>
+        <para>
+          The IRQ handler function must be provided as the mandatory irq_handler
+          driver operation. It will get passed directly to
+          <function>request_irq</function> and thus has the same prototype as all
+          IRQ handlers. It will get called with a pointer to the DRM device as the
+          second argument.
+        </para>
+        <para>
+          Finally the function calls the optional
+          <methodname>irq_postinstall</methodname> driver operation. The operation
+          usually enables interrupts (excluding the vblank interrupt, which is
+          enabled separately), but drivers may choose to enable/disable interrupts
+          at a different time.
+        </para>
+        <para>
+          <function>drm_irq_uninstall</function> is similarly used to uninstall an
+          IRQ handler. It starts by waking up all processes waiting on a vblank
+          interrupt to make sure they don't hang, and then calls the optional
+          <methodname>irq_uninstall</methodname> driver operation. The operation
+          must disable all hardware interrupts. Finally the function frees the IRQ
+          by calling <function>free_irq</function>.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Memory Manager Initialization</title>
+        <para>
+          Every DRM driver requires a memory manager which must be initialized at
+          load time. DRM currently contains two memory managers, the Translation
+          Table Manager (TTM) and the Graphics Execution Manager (GEM).
+          This document describes the use of the GEM memory manager only. See
+          <xref linkend="drm-memory-management"/> for details.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Miscellaneous Device Configuration</title>
+        <para>
+          Another task that may be necessary for PCI devices during configuration
+          is mapping the video BIOS. On many devices, the VBIOS describes device
+          configuration, LCD panel timings (if any), and contains flags indicating
+          device state. Mapping the BIOS can be done using the pci_map_rom() call,
+          a convenience function that takes care of mapping the actual ROM,
+          whether it has been shadowed into memory (typically at address 0xc0000)
+          or exists on the PCI device in the ROM BAR. Note that after the ROM has
+          been mapped and any necessary information has been extracted, it should
+          be unmapped; on many devices, the ROM address decoder is shared with
+          other BARs, so leaving it mapped could cause undesired behaviour like
+          hangs or memory corruption.
+  <!--!Fdrivers/pci/rom.c pci_map_rom-->
+        </para>
+      </sect3>
     </sect2>
+  </sect1>
 
-    <sect2>
-      <title>Configuring the device</title>
-      <para>
-	Obviously, device configuration is device-specific.
-	However, there are several common operations: finding a
-	device's PCI resources, mapping them, and potentially setting
-	up an IRQ handler.
-      </para>
-      <para>
-	Finding &amp; mapping resources is fairly straightforward.  The
-	DRM wrapper functions, drm_get_resource_start() and
-	drm_get_resource_len(), may be used to find BARs on the given
-	drm_device struct.  Once those values have been retrieved, the
-	driver load function can call drm_addmap() to create a new
-	mapping for the BAR in question.  Note that you probably want a
-	drm_local_map_t in your driver private structure to track any
-	mappings you create.
-<!-- !Fdrivers/gpu/drm/drm_bufs.c drm_get_resource_* -->
-<!-- !Finclude/drm/drmP.h drm_local_map_t -->
-      </para>
-      <para>
-	if compatibility with other operating systems isn't a concern
-	(DRM drivers can run under various BSD variants and OpenSolaris),
-	native Linux calls may be used for the above, e.g. pci_resource_*
-	and iomap*/iounmap.  See the Linux device driver book for more
-	info.
-      </para>
-      <para>
-	Once you have a register map, you may use the DRM_READn() and
-	DRM_WRITEn() macros to access the registers on your device, or
-	use driver-specific versions to offset into your MMIO space
-	relative to a driver-specific base pointer (see I915_READ for
-	an example).
-      </para>
-      <para>
-	If your device supports interrupt generation, you may want to
-	set up an interrupt handler when the driver is loaded.  This
-	is done using the drm_irq_install() function.  If your device
-	supports vertical blank interrupts, it should call
-	drm_vblank_init() to initialize the core vblank handling code before
-	enabling interrupts on your device.  This ensures the vblank related
-	structures are allocated and allows the core to handle vblank events.
-      </para>
-<!--!Fdrivers/char/drm/drm_irq.c drm_irq_install-->
-      <para>
-	Once your interrupt handler is registered (it uses your
-	drm_driver.irq_handler as the actual interrupt handling
-	function), you can safely enable interrupts on your device,
-	assuming any other state your interrupt handler uses is also
-	initialized.
-      </para>
-      <para>
-	Another task that may be necessary during configuration is
-	mapping the video BIOS.  On many devices, the VBIOS describes
-	device configuration, LCD panel timings (if any), and contains
-	flags indicating device state.  Mapping the BIOS can be done
-	using the pci_map_rom() call, a convenience function that
-	takes care of mapping the actual ROM, whether it has been
-	shadowed into memory (typically at address 0xc0000) or exists
-	on the PCI device in the ROM BAR.  Note that after the ROM
-	has been mapped and any necessary information has been extracted,
-	it should be unmapped; on many devices, the ROM address decoder is
-	shared with other BARs, so leaving it mapped could cause
-	undesired behavior like hangs or memory corruption.
-<!--!Fdrivers/pci/rom.c pci_map_rom-->
-      </para>
-    </sect2>
+  <!-- Internals: memory management -->
 
+  <sect1 id="drm-memory-management">
+    <title>Memory management</title>
+    <para>
+      Modern Linux systems require large amount of graphics memory to store
+      frame buffers, textures, vertices and other graphics-related data. Given
+      the very dynamic nature of many of that data, managing graphics memory
+      efficiently is thus crucial for the graphics stack and plays a central
+      role in the DRM infrastructure.
+    </para>
+    <para>
+      The DRM core includes two memory managers, namely Translation Table Maps
+      (TTM) and Graphics Execution Manager (GEM). TTM was the first DRM memory
+      manager to be developed and tried to be a one-size-fits-them all
+      solution. It provides a single userspace API to accomodate the need of
+      all hardware, supporting both Unified Memory Architecture (UMA) devices
+      and devices with dedicated video RAM (i.e. most discrete video cards).
+      This resulted in a large, complex piece of code that turned out to be
+      hard to use for driver development.
+    </para>
+    <para>
+      GEM started as an Intel-sponsored project in reaction to TTM's
+      complexity. Its design philosophy is completely different: instead of
+      providing a solution to every graphics memory-related problems, GEM
+      identified common code between drivers and created a support library to
+      share it. GEM has simpler initialization and execution requirements than
+      TTM, but has no video RAM management capabitilies and is thus limited to
+      UMA devices.
+    </para>
     <sect2>
-      <title>Memory manager initialization</title>
-      <para>
-	In order to allocate command buffers, cursor memory, scanout
-	buffers, etc., as well as support the latest features provided
-	by packages like Mesa and the X.Org X server, your driver
-	should support a memory manager.
-      </para>
+      <title>The Translation Table Manager (TTM)</title>
       <para>
-	If your driver supports memory management (it should!), you
-	need to set that up at load time as well.  How you initialize
-	it depends on which memory manager you're using: TTM or GEM.
+	TTM design background and information belongs here.
       </para>
       <sect3>
 	<title>TTM initialization</title>
-	<para>
-	  TTM (for Translation Table Manager) manages video memory and
-	  aperture space for graphics devices. TTM supports both UMA devices
-	  and devices with dedicated video RAM (VRAM), i.e. most discrete
-	  graphics devices.  If your device has dedicated RAM, supporting
-	  TTM is desirable.  TTM also integrates tightly with your
-	  driver-specific buffer execution function.  See the radeon
-	  driver for examples.
-	</para>
-	<para>
-	  The core TTM structure is the ttm_bo_driver struct.  It contains
-	  several fields with function pointers for initializing the TTM,
-	  allocating and freeing memory, waiting for command completion
-	  and fence synchronization, and memory migration.  See the
-	  radeon_ttm.c file for an example of usage.
+        <warning><para>This section is outdated.</para></warning>
+        <para>
+          Drivers wishing to support TTM must fill out a drm_bo_driver
+          structure. The structure contains several fields with function
+          pointers for initializing the TTM, allocating and freeing memory,
+          waiting for command completion and fence synchronization, and memory
+          migration. See the radeon_ttm.c file for an example of usage.
 	</para>
 	<para>
 	  The ttm_global_reference structure is made up of several fields:
@@ -445,82 +500,1081 @@
 	  count for the TTM, which will call your initialization function.
 	</para>
       </sect3>
+    </sect2>
+    <sect2 id="drm-gem">
+      <title>The Graphics Execution Manager (GEM)</title>
+      <para>
+        The GEM design approach has resulted in a memory manager that doesn't
+        provide full coverage of all (or even all common) use cases in its
+        userspace or kernel API. GEM exposes a set of standard memory-related
+        operations to userspace and a set of helper functions to drivers, and let
+        drivers implement hardware-specific operations with their own private API.
+      </para>
+      <para>
+        The GEM userspace API is described in the
+        <ulink url="http://lwn.net/Articles/283798/"><citetitle>GEM - the Graphics
+        Execution Manager</citetitle></ulink> article on LWN. While slightly
+        outdated, the document provides a good overview of the GEM API principles.
+        Buffer allocation and read and write operations, described as part of the
+        common GEM API, are currently implemented using driver-specific ioctls.
+      </para>
+      <para>
+        GEM is data-agnostic. It manages abstract buffer objects without knowing
+        what individual buffers contain. APIs that require knowledge of buffer
+        contents or purpose, such as buffer allocation or synchronization
+        primitives, are thus outside of the scope of GEM and must be implemented
+        using driver-specific ioctls.
+      </para>
+      <para>
+	On a fundamental level, GEM involves several operations:
+	<itemizedlist>
+	  <listitem>Memory allocation and freeing</listitem>
+	  <listitem>Command execution</listitem>
+	  <listitem>Aperture management at command execution time</listitem>
+	</itemizedlist>
+	Buffer object allocation is relatively straightforward and largely
+        provided by Linux's shmem layer, which provides memory to back each
+        object.
+      </para>
+      <para>
+        Device-specific operations, such as command execution, pinning, buffer
+	read &amp; write, mapping, and domain ownership transfers are left to
+        driver-specific ioctls.
+      </para>
+      <sect3>
+        <title>GEM Initialization</title>
+        <para>
+          Drivers that use GEM must set the DRIVER_GEM bit in the struct
+          <structname>drm_driver</structname>
+          <structfield>driver_features</structfield> field. The DRM core will
+          then automatically initialize the GEM core before calling the
+          <methodname>load</methodname> operation. Behind the scene, this will
+          create a DRM Memory Manager object which provides an address space
+          pool for object allocation.
+        </para>
+        <para>
+          In a KMS configuration, drivers need to allocate and initialize a
+          command ring buffer following core GEM initialization if required by
+          the hardware. UMA devices usually have what is called a "stolen"
+          memory region, which provides space for the initial framebuffer and
+          large, contiguous memory regions required by the device. This space is
+          typically not managed by GEM, and must be initialized separately into
+          its own DRM MM object.
+        </para>
+      </sect3>
       <sect3>
-	<title>GEM initialization</title>
-	<para>
-	  GEM is an alternative to TTM, designed specifically for UMA
-	  devices.  It has simpler initialization and execution requirements
-	  than TTM, but has no VRAM management capability.  Core GEM
-	  is initialized by calling drm_mm_init() to create
-	  a GTT DRM MM object, which provides an address space pool for
-	  object allocation.  In a KMS configuration, the driver
-	  needs to allocate and initialize a command ring buffer following
-	  core GEM initialization.  A UMA device usually has what is called a
-	  "stolen" memory region, which provides space for the initial
-	  framebuffer and large, contiguous memory regions required by the
-	  device.  This space is not typically managed by GEM, and it must
-	  be initialized separately into its own DRM MM object.
-	</para>
-	<para>
-	  Initialization is driver-specific. In the case of Intel
-	  integrated graphics chips like 965GM, GEM initialization can
-	  be done by calling the internal GEM init function,
-	  i915_gem_do_init().  Since the 965GM is a UMA device
-	  (i.e. it doesn't have dedicated VRAM), GEM manages
-	  making regular RAM available for GPU operations.  Memory set
-	  aside by the BIOS (called "stolen" memory by the i915
-	  driver) is managed by the DRM memrange allocator; the
-	  rest of the aperture is managed by GEM.
-	  <programlisting>
-	    /* Basic memrange allocator for stolen space (aka vram) */
-	    drm_memrange_init(&amp;dev_priv->vram, 0, prealloc_size);
-	    /* Let GEM Manage from end of prealloc space to end of aperture */
-	    i915_gem_do_init(dev, prealloc_size, agp_size);
-	  </programlisting>
-<!--!Edrivers/char/drm/drm_memrange.c-->
-	</para>
-	<para>
-	  Once the memory manager has been set up, we may allocate the
-	  command buffer.  In the i915 case, this is also done with a
-	  GEM function, i915_gem_init_ringbuffer().
-	</para>
+        <title>GEM Objects Creation</title>
+        <para>
+          GEM splits creation of GEM objects and allocation of the memory that
+          backs them in two distinct operations.
+        </para>
+        <para>
+          GEM objects are represented by an instance of struct
+          <structname>drm_gem_object</structname>. Drivers usually need to extend
+          GEM objects with private information and thus create a driver-specific
+          GEM object structure type that embeds an instance of struct
+          <structname>drm_gem_object</structname>.
+        </para>
+        <para>
+          To create a GEM object, a driver allocates memory for an instance of its
+          specific GEM object type and initializes the embedded struct
+          <structname>drm_gem_object</structname> with a call to
+          <function>drm_gem_object_init</function>. The function takes a pointer to
+          the DRM device, a pointer to the GEM object and the buffer object size
+          in bytes.
+        </para>
+        <para>
+          GEM uses shmem to allocate anonymous pageable memory.
+          <function>drm_gem_object_init</function> will create an shmfs file of
+          the requested size and store it into the struct
+          <structname>drm_gem_object</structname> <structfield>filp</structfield>
+          field. The memory is used as either main storage for the object when the
+          graphics hardware uses system memory directly or as a backing store
+          otherwise.
+        </para>
+        <para>
+          Drivers are responsible for the actual physical pages allocation by
+          calling <function>shmem_read_mapping_page_gfp</function> for each page.
+          Note that they can decide to allocate pages when initializing the GEM
+          object, or to delay allocation until the memory is needed (for instance
+          when a page fault occurs as a result of a userspace memory access or
+          when the driver needs to start a DMA transfer involving the memory).
+        </para>
+        <para>
+          Anonymous pageable memory allocation is not always desired, for instance
+          when the hardware requires physically contiguous system memory as is
+          often the case in embedded devices. Drivers can create GEM objects with
+          no shmfs backing (called private GEM objects) by initializing them with
+          a call to <function>drm_gem_private_object_init</function> instead of
+          <function>drm_gem_object_init</function>. Storage for private GEM
+          objects must be managed by drivers.
+        </para>
+        <para>
+          Drivers that do not need to extend GEM objects with private information
+          can call the <function>drm_gem_object_alloc</function> function to
+          allocate and initialize a struct <structname>drm_gem_object</structname>
+          instance. The GEM core will call the optional driver
+          <methodname>gem_init_object</methodname> operation after initializing
+          the GEM object with <function>drm_gem_object_init</function>.
+          <synopsis>int (*gem_init_object) (struct drm_gem_object *obj);</synopsis>
+        </para>
+        <para>
+          No alloc-and-init function exists for private GEM objects.
+        </para>
+      </sect3>
+      <sect3>
+        <title>GEM Objects Lifetime</title>
+        <para>
+          All GEM objects are reference-counted by the GEM core. References can be
+          acquired and release by <function>calling drm_gem_object_reference</function>
+          and <function>drm_gem_object_unreference</function> respectively. The
+          caller must hold the <structname>drm_device</structname>
+          <structfield>struct_mutex</structfield> lock. As a convenience, GEM
+          provides the <function>drm_gem_object_reference_unlocked</function> and
+          <function>drm_gem_object_unreference_unlocked</function> functions that
+          can be called without holding the lock.
+        </para>
+        <para>
+          When the last reference to a GEM object is released the GEM core calls
+          the <structname>drm_driver</structname>
+          <methodname>gem_free_object</methodname> operation. That operation is
+          mandatory for GEM-enabled drivers and must free the GEM object and all
+          associated resources.
+        </para>
+        <para>
+          <synopsis>void (*gem_free_object) (struct drm_gem_object *obj);</synopsis>
+          Drivers are responsible for freeing all GEM object resources, including
+          the resources created by the GEM core. If an mmap offset has been
+          created for the object (in which case
+          <structname>drm_gem_object</structname>::<structfield>map_list</structfield>::<structfield>map</structfield>
+          is not NULL) it must be freed by a call to
+          <function>drm_gem_free_mmap_offset</function>. The shmfs backing store
+          must be released by calling <function>drm_gem_object_release</function>
+          (that function can safely be called if no shmfs backing store has been
+          created).
+        </para>
+      </sect3>
+      <sect3>
+        <title>GEM Objects Naming</title>
+        <para>
+          Communication between userspace and the kernel refers to GEM objects
+          using local handles, global names or, more recently, file descriptors.
+          All of those are 32-bit integer values; the usual Linux kernel limits
+          apply to the file descriptors.
+        </para>
+        <para>
+          GEM handles are local to a DRM file. Applications get a handle to a GEM
+          object through a driver-specific ioctl, and can use that handle to refer
+          to the GEM object in other standard or driver-specific ioctls. Closing a
+          DRM file handle frees all its GEM handles and dereferences the
+          associated GEM objects.
+        </para>
+        <para>
+          To create a handle for a GEM object drivers call
+          <function>drm_gem_handle_create</function>. The function takes a pointer
+          to the DRM file and the GEM object and returns a locally unique handle.
+          When the handle is no longer needed drivers delete it with a call to
+          <function>drm_gem_handle_delete</function>. Finally the GEM object
+          associated with a handle can be retrieved by a call to
+          <function>drm_gem_object_lookup</function>.
+        </para>
+        <para>
+          Handles don't take ownership of GEM objects, they only take a reference
+          to the object that will be dropped when the handle is destroyed. To
+          avoid leaking GEM objects, drivers must make sure they drop the
+          reference(s) they own (such as the initial reference taken at object
+          creation time) as appropriate, without any special consideration for the
+          handle. For example, in the particular case of combined GEM object and
+          handle creation in the implementation of the
+          <methodname>dumb_create</methodname> operation, drivers must drop the
+          initial reference to the GEM object before returning the handle.
+        </para>
+        <para>
+          GEM names are similar in purpose to handles but are not local to DRM
+          files. They can be passed between processes to reference a GEM object
+          globally. Names can't be used directly to refer to objects in the DRM
+          API, applications must convert handles to names and names to handles
+          using the DRM_IOCTL_GEM_FLINK and DRM_IOCTL_GEM_OPEN ioctls
+          respectively. The conversion is handled by the DRM core without any
+          driver-specific support.
+        </para>
+        <para>
+          Similar to global names, GEM file descriptors are also used to share GEM
+          objects across processes. They offer additional security: as file
+          descriptors must be explictly sent over UNIX domain sockets to be shared
+          between applications, they can't be guessed like the globally unique GEM
+          names.
+        </para>
+        <para>
+          Drivers that support GEM file descriptors, also known as the DRM PRIME
+          API, must set the DRIVER_PRIME bit in the struct
+          <structname>drm_driver</structname>
+          <structfield>driver_features</structfield> field, and implement the
+          <methodname>prime_handle_to_fd</methodname> and
+          <methodname>prime_fd_to_handle</methodname> operations.
+        </para>
+        <para>
+          <synopsis>int (*prime_handle_to_fd)(struct drm_device *dev,
+                            struct drm_file *file_priv, uint32_t handle,
+                            uint32_t flags, int *prime_fd);
+  int (*prime_fd_to_handle)(struct drm_device *dev,
+                            struct drm_file *file_priv, int prime_fd,
+                            uint32_t *handle);</synopsis>
+          Those two operations convert a handle to a PRIME file descriptor and
+          vice versa. Drivers must use the kernel dma-buf buffer sharing framework
+          to manage the PRIME file descriptors.
+        </para>
+        <para>
+          While non-GEM drivers must implement the operations themselves, GEM
+          drivers must use the <function>drm_gem_prime_handle_to_fd</function>
+          and <function>drm_gem_prime_fd_to_handle</function> helper functions.
+          Those helpers rely on the driver
+          <methodname>gem_prime_export</methodname> and
+          <methodname>gem_prime_import</methodname> operations to create a dma-buf
+          instance from a GEM object (dma-buf exporter role) and to create a GEM
+          object from a dma-buf instance (dma-buf importer role).
+        </para>
+        <para>
+          <synopsis>struct dma_buf * (*gem_prime_export)(struct drm_device *dev,
+                                       struct drm_gem_object *obj,
+                                       int flags);
+  struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
+                                              struct dma_buf *dma_buf);</synopsis>
+          These two operations are mandatory for GEM drivers that support DRM
+          PRIME.
+        </para>
+      </sect3>
+      <sect3 id="drm-gem-objects-mapping">
+        <title>GEM Objects Mapping</title>
+        <para>
+          Because mapping operations are fairly heavyweight GEM favours
+          read/write-like access to buffers, implemented through driver-specific
+          ioctls, over mapping buffers to userspace. However, when random access
+          to the buffer is needed (to perform software rendering for instance),
+          direct access to the object can be more efficient.
+        </para>
+        <para>
+          The mmap system call can't be used directly to map GEM objects, as they
+          don't have their own file handle. Two alternative methods currently
+          co-exist to map GEM objects to userspace. The first method uses a
+          driver-specific ioctl to perform the mapping operation, calling
+          <function>do_mmap</function> under the hood. This is often considered
+          dubious, seems to be discouraged for new GEM-enabled drivers, and will
+          thus not be described here.
+        </para>
+        <para>
+          The second method uses the mmap system call on the DRM file handle.
+          <synopsis>void *mmap(void *addr, size_t length, int prot, int flags, int fd,
+             off_t offset);</synopsis>
+          DRM identifies the GEM object to be mapped by a fake offset passed
+          through the mmap offset argument. Prior to being mapped, a GEM object
+          must thus be associated with a fake offset. To do so, drivers must call
+          <function>drm_gem_create_mmap_offset</function> on the object. The
+          function allocates a fake offset range from a pool and stores the
+          offset divided by PAGE_SIZE in
+          <literal>obj-&gt;map_list.hash.key</literal>. Care must be taken not to
+          call <function>drm_gem_create_mmap_offset</function> if a fake offset
+          has already been allocated for the object. This can be tested by
+          <literal>obj-&gt;map_list.map</literal> being non-NULL.
+        </para>
+        <para>
+          Once allocated, the fake offset value
+          (<literal>obj-&gt;map_list.hash.key &lt;&lt; PAGE_SHIFT</literal>)
+          must be passed to the application in a driver-specific way and can then
+          be used as the mmap offset argument.
+        </para>
+        <para>
+          The GEM core provides a helper method <function>drm_gem_mmap</function>
+          to handle object mapping. The method can be set directly as the mmap
+          file operation handler. It will look up the GEM object based on the
+          offset value and set the VMA operations to the
+          <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
+          field. Note that <function>drm_gem_mmap</function> doesn't map memory to
+          userspace, but relies on the driver-provided fault handler to map pages
+          individually.
+        </para>
+        <para>
+          To use <function>drm_gem_mmap</function>, drivers must fill the struct
+          <structname>drm_driver</structname> <structfield>gem_vm_ops</structfield>
+          field with a pointer to VM operations.
+        </para>
+        <para>
+          <synopsis>struct vm_operations_struct *gem_vm_ops
+
+  struct vm_operations_struct {
+          void (*open)(struct vm_area_struct * area);
+          void (*close)(struct vm_area_struct * area);
+          int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
+  };</synopsis>
+        </para>
+        <para>
+          The <methodname>open</methodname> and <methodname>close</methodname>
+          operations must update the GEM object reference count. Drivers can use
+          the <function>drm_gem_vm_open</function> and
+          <function>drm_gem_vm_close</function> helper functions directly as open
+          and close handlers.
+        </para>
+        <para>
+          The fault operation handler is responsible for mapping individual pages
+          to userspace when a page fault occurs. Depending on the memory
+          allocation scheme, drivers can allocate pages at fault time, or can
+          decide to allocate memory for the GEM object at the time the object is
+          created.
+        </para>
+        <para>
+          Drivers that want to map the GEM object upfront instead of handling page
+          faults can implement their own mmap file operation handler.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Dumb GEM Objects</title>
+        <para>
+          The GEM API doesn't standardize GEM objects creation and leaves it to
+          driver-specific ioctls. While not an issue for full-fledged graphics
+          stacks that include device-specific userspace components (in libdrm for
+          instance), this limit makes DRM-based early boot graphics unnecessarily
+          complex.
+        </para>
+        <para>
+          Dumb GEM objects partly alleviate the problem by providing a standard
+          API to create dumb buffers suitable for scanout, which can then be used
+          to create KMS frame buffers.
+        </para>
+        <para>
+          To support dumb GEM objects drivers must implement the
+          <methodname>dumb_create</methodname>,
+          <methodname>dumb_destroy</methodname> and
+          <methodname>dumb_map_offset</methodname> operations.
+        </para>
+        <itemizedlist>
+          <listitem>
+            <synopsis>int (*dumb_create)(struct drm_file *file_priv, struct drm_device *dev,
+                     struct drm_mode_create_dumb *args);</synopsis>
+            <para>
+              The <methodname>dumb_create</methodname> operation creates a GEM
+              object suitable for scanout based on the width, height and depth
+              from the struct <structname>drm_mode_create_dumb</structname>
+              argument. It fills the argument's <structfield>handle</structfield>,
+              <structfield>pitch</structfield> and <structfield>size</structfield>
+              fields with a handle for the newly created GEM object and its line
+              pitch and size in bytes.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>int (*dumb_destroy)(struct drm_file *file_priv, struct drm_device *dev,
+                      uint32_t handle);</synopsis>
+            <para>
+              The <methodname>dumb_destroy</methodname> operation destroys a dumb
+              GEM object created by <methodname>dumb_create</methodname>.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>int (*dumb_map_offset)(struct drm_file *file_priv, struct drm_device *dev,
+                         uint32_t handle, uint64_t *offset);</synopsis>
+            <para>
+              The <methodname>dumb_map_offset</methodname> operation associates an
+              mmap fake offset with the GEM object given by the handle and returns
+              it. Drivers must use the
+              <function>drm_gem_create_mmap_offset</function> function to
+              associate the fake offset as described in
+              <xref linkend="drm-gem-objects-mapping"/>.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </sect3>
+      <sect3>
+        <title>Memory Coherency</title>
+        <para>
+          When mapped to the device or used in a command buffer, backing pages
+          for an object are flushed to memory and marked write combined so as to
+          be coherent with the GPU. Likewise, if the CPU accesses an object
+          after the GPU has finished rendering to the object, then the object
+          must be made coherent with the CPU's view of memory, usually involving
+          GPU cache flushing of various kinds. This core CPU&lt;-&gt;GPU
+          coherency management is provided by a device-specific ioctl, which
+          evaluates an object's current domain and performs any necessary
+          flushing or synchronization to put the object into the desired
+          coherency domain (note that the object may be busy, i.e. an active
+          render target; in that case, setting the domain blocks the client and
+          waits for rendering to complete before performing any necessary
+          flushing operations).
+        </para>
+      </sect3>
+      <sect3>
+        <title>Command Execution</title>
+        <para>
+	  Perhaps the most important GEM function for GPU devices is providing a
+          command execution interface to clients. Client programs construct
+          command buffers containing references to previously allocated memory
+          objects, and then submit them to GEM. At that point, GEM takes care to
+          bind all the objects into the GTT, execute the buffer, and provide
+          necessary synchronization between clients accessing the same buffers.
+          This often involves evicting some objects from the GTT and re-binding
+          others (a fairly expensive operation), and providing relocation
+          support which hides fixed GTT offsets from clients. Clients must take
+          care not to submit command buffers that reference more objects than
+          can fit in the GTT; otherwise, GEM will reject them and no rendering
+          will occur. Similarly, if several objects in the buffer require fence
+          registers to be allocated for correct rendering (e.g. 2D blits on
+          pre-965 chips), care must be taken not to require more fence registers
+          than are available to the client. Such resource management should be
+          abstracted from the client in libdrm.
+        </para>
       </sect3>
     </sect2>
+  </sect1>
+
+  <!-- Internals: mode setting -->
 
+  <sect1 id="drm-mode-setting">
+    <title>Mode Setting</title>
+    <para>
+      Drivers must initialize the mode setting core by calling
+      <function>drm_mode_config_init</function> on the DRM device. The function
+      initializes the <structname>drm_device</structname>
+      <structfield>mode_config</structfield> field and never fails. Once done,
+      mode configuration must be setup by initializing the following fields.
+    </para>
+    <itemizedlist>
+      <listitem>
+        <synopsis>int min_width, min_height;
+int max_width, max_height;</synopsis>
+        <para>
+	  Minimum and maximum width and height of the frame buffers in pixel
+	  units.
+	</para>
+      </listitem>
+      <listitem>
+        <synopsis>struct drm_mode_config_funcs *funcs;</synopsis>
+	<para>Mode setting functions.</para>
+      </listitem>
+    </itemizedlist>
     <sect2>
-      <title>Output configuration</title>
+      <title>Frame Buffer Creation</title>
+      <synopsis>struct drm_framebuffer *(*fb_create)(struct drm_device *dev,
+				     struct drm_file *file_priv,
+				     struct drm_mode_fb_cmd2 *mode_cmd);</synopsis>
       <para>
-	The final initialization task is output configuration.  This involves:
-	<itemizedlist>
-	  <listitem>
-	    Finding and initializing the CRTCs, encoders, and connectors
-	    for the device.
-	  </listitem>
-	  <listitem>
-	    Creating an initial configuration.
-	  </listitem>
-	  <listitem>
-	    Registering a framebuffer console driver.
-	  </listitem>
-	</itemizedlist>
+        Frame buffers are abstract memory objects that provide a source of
+        pixels to scanout to a CRTC. Applications explicitly request the
+        creation of frame buffers through the DRM_IOCTL_MODE_ADDFB(2) ioctls and
+        receive an opaque handle that can be passed to the KMS CRTC control,
+        plane configuration and page flip functions.
+      </para>
+      <para>
+        Frame buffers rely on the underneath memory manager for low-level memory
+        operations. When creating a frame buffer applications pass a memory
+        handle (or a list of memory handles for multi-planar formats) through
+        the <parameter>drm_mode_fb_cmd2</parameter> argument. This document
+        assumes that the driver uses GEM, those handles thus reference GEM
+        objects.
+      </para>
+      <para>
+        Drivers must first validate the requested frame buffer parameters passed
+        through the mode_cmd argument. In particular this is where invalid
+        sizes, pixel formats or pitches can be caught.
+      </para>
+      <para>
+        If the parameters are deemed valid, drivers then create, initialize and
+        return an instance of struct <structname>drm_framebuffer</structname>.
+        If desired the instance can be embedded in a larger driver-specific
+        structure. The new instance is initialized with a call to
+        <function>drm_framebuffer_init</function> which takes a pointer to DRM
+        frame buffer operations (struct
+        <structname>drm_framebuffer_funcs</structname>). Frame buffer operations are
+        <itemizedlist>
+          <listitem>
+            <synopsis>int (*create_handle)(struct drm_framebuffer *fb,
+		     struct drm_file *file_priv, unsigned int *handle);</synopsis>
+            <para>
+              Create a handle to the frame buffer underlying memory object. If
+              the frame buffer uses a multi-plane format, the handle will
+              reference the memory object associated with the first plane.
+            </para>
+            <para>
+              Drivers call <function>drm_gem_handle_create</function> to create
+              the handle.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>void (*destroy)(struct drm_framebuffer *framebuffer);</synopsis>
+            <para>
+              Destroy the frame buffer object and frees all associated
+              resources. Drivers must call
+              <function>drm_framebuffer_cleanup</function> to free resources
+              allocated by the DRM core for the frame buffer object, and must
+              make sure to unreference all memory objects associated with the
+              frame buffer. Handles created by the
+              <methodname>create_handle</methodname> operation are released by
+              the DRM core.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>int (*dirty)(struct drm_framebuffer *framebuffer,
+	     struct drm_file *file_priv, unsigned flags, unsigned color,
+	     struct drm_clip_rect *clips, unsigned num_clips);</synopsis>
+            <para>
+              This optional operation notifies the driver that a region of the
+              frame buffer has changed in response to a DRM_IOCTL_MODE_DIRTYFB
+              ioctl call.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </para>
+      <para>
+        After initializing the <structname>drm_framebuffer</structname>
+        instance drivers must fill its <structfield>width</structfield>,
+        <structfield>height</structfield>, <structfield>pitches</structfield>,
+        <structfield>offsets</structfield>, <structfield>depth</structfield>,
+        <structfield>bits_per_pixel</structfield> and
+        <structfield>pixel_format</structfield> fields from the values passed
+        through the <parameter>drm_mode_fb_cmd2</parameter> argument. They
+        should call the <function>drm_helper_mode_fill_fb_struct</function>
+        helper function to do so.
+      </para>
+    </sect2>
+    <sect2>
+      <title>Output Polling</title>
+      <synopsis>void (*output_poll_changed)(struct drm_device *dev);</synopsis>
+      <para>
+        This operation notifies the driver that the status of one or more
+        connectors has changed. Drivers that use the fb helper can just call the
+        <function>drm_fb_helper_hotplug_event</function> function to handle this
+        operation.
+      </para>
+    </sect2>
+  </sect1>
+
+  <!-- Internals: kms initialization and cleanup -->
+
+  <sect1 id="drm-kms-init">
+    <title>KMS Initialization and Cleanup</title>
+    <para>
+      A KMS device is abstracted and exposed as a set of planes, CRTCs, encoders
+      and connectors. KMS drivers must thus create and initialize all those
+      objects at load time after initializing mode setting.
+    </para>
+    <sect2>
+      <title>CRTCs (struct <structname>drm_crtc</structname>)</title>
+      <para>
+        A CRTC is an abstraction representing a part of the chip that contains a
+	pointer to a scanout buffer. Therefore, the number of CRTCs available
+	determines how many independent scanout buffers can be active at any
+	given time. The CRTC structure contains several fields to support this:
+	a pointer to some video memory (abstracted as a frame buffer object), a
+	display mode, and an (x, y) offset into the video memory to support
+	panning or configurations where one piece of video memory spans multiple
+	CRTCs.
       </para>
       <sect3>
-	<title>Output discovery and initialization</title>
-	<para>
-	  Several core functions exist to create CRTCs, encoders, and
-	  connectors, namely: drm_crtc_init(), drm_connector_init(), and
-	  drm_encoder_init(), along with several "helper" functions to
-	  perform common tasks.
-	</para>
-	<para>
-	  Connectors should be registered with sysfs once they've been
-	  detected and initialized, using the
-	  drm_sysfs_connector_add() function.  Likewise, when they're
-	  removed from the system, they should be destroyed with
-	  drm_sysfs_connector_remove().
-	</para>
-	<programlisting>
-<![CDATA[
+        <title>CRTC Initialization</title>
+        <para>
+          A KMS device must create and register at least one struct
+          <structname>drm_crtc</structname> instance. The instance is allocated
+          and zeroed by the driver, possibly as part of a larger structure, and
+          registered with a call to <function>drm_crtc_init</function> with a
+          pointer to CRTC functions.
+        </para>
+      </sect3>
+      <sect3>
+        <title>CRTC Operations</title>
+        <sect4>
+          <title>Set Configuration</title>
+          <synopsis>int (*set_config)(struct drm_mode_set *set);</synopsis>
+          <para>
+            Apply a new CRTC configuration to the device. The configuration
+            specifies a CRTC, a frame buffer to scan out from, a (x,y) position in
+            the frame buffer, a display mode and an array of connectors to drive
+            with the CRTC if possible.
+          </para>
+          <para>
+            If the frame buffer specified in the configuration is NULL, the driver
+            must detach all encoders connected to the CRTC and all connectors
+            attached to those encoders and disable them.
+          </para>
+          <para>
+            This operation is called with the mode config lock held.
+          </para>
+          <note><para>
+            FIXME: How should set_config interact with DPMS? If the CRTC is
+            suspended, should it be resumed?
+          </para></note>
+        </sect4>
+        <sect4>
+          <title>Page Flipping</title>
+          <synopsis>int (*page_flip)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                   struct drm_pending_vblank_event *event);</synopsis>
+          <para>
+            Schedule a page flip to the given frame buffer for the CRTC. This
+            operation is called with the mode config mutex held.
+          </para>
+          <para>
+            Page flipping is a synchronization mechanism that replaces the frame
+            buffer being scanned out by the CRTC with a new frame buffer during
+            vertical blanking, avoiding tearing. When an application requests a page
+            flip the DRM core verifies that the new frame buffer is large enough to
+            be scanned out by  the CRTC in the currently configured mode and then
+            calls the CRTC <methodname>page_flip</methodname> operation with a
+            pointer to the new frame buffer.
+          </para>
+          <para>
+            The <methodname>page_flip</methodname> operation schedules a page flip.
+            Once any pending rendering targetting the new frame buffer has
+            completed, the CRTC will be reprogrammed to display that frame buffer
+            after the next vertical refresh. The operation must return immediately
+            without waiting for rendering or page flip to complete and must block
+            any new rendering to the frame buffer until the page flip completes.
+          </para>
+          <para>
+            If a page flip is already pending, the
+            <methodname>page_flip</methodname> operation must return
+            -<errorname>EBUSY</errorname>.
+          </para>
+          <para>
+            To synchronize page flip to vertical blanking the driver will likely
+            need to enable vertical blanking interrupts. It should call
+            <function>drm_vblank_get</function> for that purpose, and call
+            <function>drm_vblank_put</function> after the page flip completes.
+          </para>
+          <para>
+            If the application has requested to be notified when page flip completes
+            the <methodname>page_flip</methodname> operation will be called with a
+            non-NULL <parameter>event</parameter> argument pointing to a
+            <structname>drm_pending_vblank_event</structname> instance. Upon page
+            flip completion the driver must fill the
+            <parameter>event</parameter>::<structfield>event</structfield>
+            <structfield>sequence</structfield>, <structfield>tv_sec</structfield>
+            and <structfield>tv_usec</structfield> fields with the associated
+            vertical blanking count and timestamp, add the event to the
+            <parameter>drm_file</parameter> list of events to be signaled, and wake
+            up any waiting process. This can be performed with
+            <programlisting><![CDATA[
+            struct timeval now;
+
+            event->event.sequence = drm_vblank_count_and_time(..., &now);
+            event->event.tv_sec = now.tv_sec;
+            event->event.tv_usec = now.tv_usec;
+
+            spin_lock_irqsave(&dev->event_lock, flags);
+            list_add_tail(&event->base.link, &event->base.file_priv->event_list);
+            wake_up_interruptible(&event->base.file_priv->event_wait);
+            spin_unlock_irqrestore(&dev->event_lock, flags);
+            ]]></programlisting>
+          </para>
+          <note><para>
+            FIXME: Could drivers that don't need to wait for rendering to complete
+            just add the event to <literal>dev-&gt;vblank_event_list</literal> and
+            let the DRM core handle everything, as for "normal" vertical blanking
+            events?
+          </para></note>
+          <para>
+            While waiting for the page flip to complete, the
+            <literal>event-&gt;base.link</literal> list head can be used freely by
+            the driver to store the pending event in a driver-specific list.
+          </para>
+          <para>
+            If the file handle is closed before the event is signaled, drivers must
+            take care to destroy the event in their
+            <methodname>preclose</methodname> operation (and, if needed, call
+            <function>drm_vblank_put</function>).
+          </para>
+        </sect4>
+        <sect4>
+          <title>Miscellaneous</title>
+          <itemizedlist>
+            <listitem>
+              <synopsis>void (*gamma_set)(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
+                        uint32_t start, uint32_t size);</synopsis>
+              <para>
+                Apply a gamma table to the device. The operation is optional.
+              </para>
+            </listitem>
+            <listitem>
+              <synopsis>void (*destroy)(struct drm_crtc *crtc);</synopsis>
+              <para>
+                Destroy the CRTC when not needed anymore. See
+                <xref linkend="drm-kms-init"/>.
+              </para>
+            </listitem>
+          </itemizedlist>
+        </sect4>
+      </sect3>
+    </sect2>
+    <sect2>
+      <title>Planes (struct <structname>drm_plane</structname>)</title>
+      <para>
+        A plane represents an image source that can be blended with or overlayed
+	on top of a CRTC during the scanout process. Planes are associated with
+	a frame buffer to crop a portion of the image memory (source) and
+	optionally scale it to a destination size. The result is then blended
+	with or overlayed on top of a CRTC.
+      </para>
+      <sect3>
+        <title>Plane Initialization</title>
+        <para>
+          Planes are optional. To create a plane, a KMS drivers allocates and
+          zeroes an instances of struct <structname>drm_plane</structname>
+          (possibly as part of a larger structure) and registers it with a call
+          to <function>drm_plane_init</function>. The function takes a bitmask
+          of the CRTCs that can be associated with the plane, a pointer to the
+          plane functions and a list of format supported formats.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Plane Operations</title>
+        <itemizedlist>
+          <listitem>
+            <synopsis>int (*update_plane)(struct drm_plane *plane, struct drm_crtc *crtc,
+                        struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+                        unsigned int crtc_w, unsigned int crtc_h,
+                        uint32_t src_x, uint32_t src_y,
+                        uint32_t src_w, uint32_t src_h);</synopsis>
+            <para>
+              Enable and configure the plane to use the given CRTC and frame buffer.
+            </para>
+            <para>
+              The source rectangle in frame buffer memory coordinates is given by
+              the <parameter>src_x</parameter>, <parameter>src_y</parameter>,
+              <parameter>src_w</parameter> and <parameter>src_h</parameter>
+              parameters (as 16.16 fixed point values). Devices that don't support
+              subpixel plane coordinates can ignore the fractional part.
+            </para>
+            <para>
+              The destination rectangle in CRTC coordinates is given by the
+              <parameter>crtc_x</parameter>, <parameter>crtc_y</parameter>,
+              <parameter>crtc_w</parameter> and <parameter>crtc_h</parameter>
+              parameters (as integer values). Devices scale the source rectangle to
+              the destination rectangle. If scaling is not supported, and the source
+              rectangle size doesn't match the destination rectangle size, the
+              driver must return a -<errorname>EINVAL</errorname> error.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>int (*disable_plane)(struct drm_plane *plane);</synopsis>
+            <para>
+              Disable the plane. The DRM core calls this method in response to a
+              DRM_IOCTL_MODE_SETPLANE ioctl call with the frame buffer ID set to 0.
+              Disabled planes must not be processed by the CRTC.
+            </para>
+          </listitem>
+          <listitem>
+            <synopsis>void (*destroy)(struct drm_plane *plane);</synopsis>
+            <para>
+              Destroy the plane when not needed anymore. See
+              <xref linkend="drm-kms-init"/>.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </sect3>
+    </sect2>
+    <sect2>
+      <title>Encoders (struct <structname>drm_encoder</structname>)</title>
+      <para>
+        An encoder takes pixel data from a CRTC and converts it to a format
+	suitable for any attached connectors. On some devices, it may be
+	possible to have a CRTC send data to more than one encoder. In that
+	case, both encoders would receive data from the same scanout buffer,
+	resulting in a "cloned" display configuration across the connectors
+	attached to each encoder.
+      </para>
+      <sect3>
+        <title>Encoder Initialization</title>
+        <para>
+          As for CRTCs, a KMS driver must create, initialize and register at
+          least one struct <structname>drm_encoder</structname> instance. The
+          instance is allocated and zeroed by the driver, possibly as part of a
+          larger structure.
+        </para>
+        <para>
+          Drivers must initialize the struct <structname>drm_encoder</structname>
+          <structfield>possible_crtcs</structfield> and
+          <structfield>possible_clones</structfield> fields before registering the
+          encoder. Both fields are bitmasks of respectively the CRTCs that the
+          encoder can be connected to, and sibling encoders candidate for cloning.
+        </para>
+        <para>
+          After being initialized, the encoder must be registered with a call to
+          <function>drm_encoder_init</function>. The function takes a pointer to
+          the encoder functions and an encoder type. Supported types are
+          <itemizedlist>
+            <listitem>
+              DRM_MODE_ENCODER_DAC for VGA and analog on DVI-I/DVI-A
+              </listitem>
+            <listitem>
+              DRM_MODE_ENCODER_TMDS for DVI, HDMI and (embedded) DisplayPort
+            </listitem>
+            <listitem>
+              DRM_MODE_ENCODER_LVDS for display panels
+            </listitem>
+            <listitem>
+              DRM_MODE_ENCODER_TVDAC for TV output (Composite, S-Video, Component,
+              SCART)
+            </listitem>
+            <listitem>
+              DRM_MODE_ENCODER_VIRTUAL for virtual machine displays
+            </listitem>
+          </itemizedlist>
+        </para>
+        <para>
+          Encoders must be attached to a CRTC to be used. DRM drivers leave
+          encoders unattached at initialization time. Applications (or the fbdev
+          compatibility layer when implemented) are responsible for attaching the
+          encoders they want to use to a CRTC.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Encoder Operations</title>
+        <itemizedlist>
+          <listitem>
+            <synopsis>void (*destroy)(struct drm_encoder *encoder);</synopsis>
+            <para>
+              Called to destroy the encoder when not needed anymore. See
+              <xref linkend="drm-kms-init"/>.
+            </para>
+          </listitem>
+        </itemizedlist>
+      </sect3>
+    </sect2>
+    <sect2>
+      <title>Connectors (struct <structname>drm_connector</structname>)</title>
+      <para>
+        A connector is the final destination for pixel data on a device, and
+	usually connects directly to an external display device like a monitor
+	or laptop panel. A connector can only be attached to one encoder at a
+	time. The connector is also the structure where information about the
+	attached display is kept, so it contains fields for display data, EDID
+	data, DPMS &amp; connection status, and information about modes
+	supported on the attached displays.
+      </para>
+      <sect3>
+        <title>Connector Initialization</title>
+        <para>
+          Finally a KMS driver must create, initialize, register and attach at
+          least one struct <structname>drm_connector</structname> instance. The
+          instance is created as other KMS objects and initialized by setting the
+          following fields.
+        </para>
+        <variablelist>
+          <varlistentry>
+            <term><structfield>interlace_allowed</structfield></term>
+            <listitem><para>
+              Whether the connector can handle interlaced modes.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term><structfield>doublescan_allowed</structfield></term>
+            <listitem><para>
+              Whether the connector can handle doublescan.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term><structfield>display_info
+            </structfield></term>
+            <listitem><para>
+              Display information is filled from EDID information when a display
+              is detected. For non hot-pluggable displays such as flat panels in
+              embedded systems, the driver should initialize the
+              <structfield>display_info</structfield>.<structfield>width_mm</structfield>
+              and
+              <structfield>display_info</structfield>.<structfield>height_mm</structfield>
+              fields with the physical size of the display.
+            </para></listitem>
+          </varlistentry>
+          <varlistentry>
+            <term id="drm-kms-connector-polled"><structfield>polled</structfield></term>
+            <listitem><para>
+              Connector polling mode, a combination of
+              <variablelist>
+                <varlistentry>
+                  <term>DRM_CONNECTOR_POLL_HPD</term>
+                  <listitem><para>
+                    The connector generates hotplug events and doesn't need to be
+                    periodically polled. The CONNECT and DISCONNECT flags must not
+                    be set together with the HPD flag.
+                  </para></listitem>
+                </varlistentry>
+                <varlistentry>
+                  <term>DRM_CONNECTOR_POLL_CONNECT</term>
+                  <listitem><para>
+                    Periodically poll the connector for connection.
+                  </para></listitem>
+                </varlistentry>
+                <varlistentry>
+                  <term>DRM_CONNECTOR_POLL_DISCONNECT</term>
+                  <listitem><para>
+                    Periodically poll the connector for disconnection.
+                  </para></listitem>
+                </varlistentry>
+              </variablelist>
+              Set to 0 for connectors that don't support connection status
+              discovery.
+            </para></listitem>
+          </varlistentry>
+        </variablelist>
+        <para>
+          The connector is then registered with a call to
+          <function>drm_connector_init</function> with a pointer to the connector
+          functions and a connector type, and exposed through sysfs with a call to
+          <function>drm_sysfs_connector_add</function>.
+        </para>
+        <para>
+          Supported connector types are
+          <itemizedlist>
+            <listitem>DRM_MODE_CONNECTOR_VGA</listitem>
+            <listitem>DRM_MODE_CONNECTOR_DVII</listitem>
+            <listitem>DRM_MODE_CONNECTOR_DVID</listitem>
+            <listitem>DRM_MODE_CONNECTOR_DVIA</listitem>
+            <listitem>DRM_MODE_CONNECTOR_Composite</listitem>
+            <listitem>DRM_MODE_CONNECTOR_SVIDEO</listitem>
+            <listitem>DRM_MODE_CONNECTOR_LVDS</listitem>
+            <listitem>DRM_MODE_CONNECTOR_Component</listitem>
+            <listitem>DRM_MODE_CONNECTOR_9PinDIN</listitem>
+            <listitem>DRM_MODE_CONNECTOR_DisplayPort</listitem>
+            <listitem>DRM_MODE_CONNECTOR_HDMIA</listitem>
+            <listitem>DRM_MODE_CONNECTOR_HDMIB</listitem>
+            <listitem>DRM_MODE_CONNECTOR_TV</listitem>
+            <listitem>DRM_MODE_CONNECTOR_eDP</listitem>
+            <listitem>DRM_MODE_CONNECTOR_VIRTUAL</listitem>
+          </itemizedlist>
+        </para>
+        <para>
+          Connectors must be attached to an encoder to be used. For devices that
+          map connectors to encoders 1:1, the connector should be attached at
+          initialization time with a call to
+          <function>drm_mode_connector_attach_encoder</function>. The driver must
+          also set the <structname>drm_connector</structname>
+          <structfield>encoder</structfield> field to point to the attached
+          encoder.
+        </para>
+        <para>
+          Finally, drivers must initialize the connectors state change detection
+          with a call to <function>drm_kms_helper_poll_init</function>. If at
+          least one connector is pollable but can't generate hotplug interrupts
+          (indicated by the DRM_CONNECTOR_POLL_CONNECT and
+          DRM_CONNECTOR_POLL_DISCONNECT connector flags), a delayed work will
+          automatically be queued to periodically poll for changes. Connectors
+          that can generate hotplug interrupts must be marked with the
+          DRM_CONNECTOR_POLL_HPD flag instead, and their interrupt handler must
+          call <function>drm_helper_hpd_irq_event</function>. The function will
+          queue a delayed work to check the state of all connectors, but no
+          periodic polling will be done.
+        </para>
+      </sect3>
+      <sect3>
+        <title>Connector Operations</title>
+        <note><para>
+          Unless otherwise state, all operations are mandatory.
+        </para></note>
+        <sect4>
+          <title>DPMS</title>
+          <synopsis>void (*dpms)(struct drm_connector *connector, int mode);</synopsis>
+          <para>
+            The DPMS operation sets the power state of a connector. The mode
+            argument is one of
+            <itemizedlist>
+              <listitem><para>DRM_MODE_DPMS_ON</para></listitem>
+              <listitem><para>DRM_MODE_DPMS_STANDBY</para></listitem>
+              <listitem><para>DRM_MODE_DPMS_SUSPEND</para></listitem>
+              <listitem><para>DRM_MODE_DPMS_OFF</para></listitem>
+            </itemizedlist>
+          </para>
+          <para>
+            In all but DPMS_ON mode the encoder to which the connector is attached
+            should put the display in low-power mode by driving its signals
+            appropriately. If more than one connector is attached to the encoder
+            care should be taken not to change the power state of other displays as
+            a side effect. Low-power mode should be propagated to the encoders and
+            CRTCs when all related connectors are put in low-power mode.
+          </para>
+        </sect4>
+        <sect4>
+          <title>Modes</title>
+          <synopsis>int (*fill_modes)(struct drm_connector *connector, uint32_t max_width,
+                      uint32_t max_height);</synopsis>
+          <para>
+            Fill the mode list with all supported modes for the connector. If the
+            <parameter>max_width</parameter> and <parameter>max_height</parameter>
+            arguments are non-zero, the implementation must ignore all modes wider
+            than <parameter>max_width</parameter> or higher than
+            <parameter>max_height</parameter>.
+          </para>
+          <para>
+            The connector must also fill in this operation its
+            <structfield>display_info</structfield>
+            <structfield>width_mm</structfield> and
+            <structfield>height_mm</structfield> fields with the connected display
+            physical size in millimeters. The fields should be set to 0 if the value
+            isn't known or is not applicable (for instance for projector devices).
+          </para>
+        </sect4>
+        <sect4>
+          <title>Connection Status</title>
+          <para>
+            The connection status is updated through polling or hotplug events when
+            supported (see <xref linkend="drm-kms-connector-polled"/>). The status
+            value is reported to userspace through ioctls and must not be used
+            inside the driver, as it only gets initialized by a call to
+            <function>drm_mode_getconnector</function> from userspace.
+          </para>
+          <synopsis>enum drm_connector_status (*detect)(struct drm_connector *connector,
+                                        bool force);</synopsis>
+          <para>
+            Check to see if anything is attached to the connector. The
+            <parameter>force</parameter> parameter is set to false whilst polling or
+            to true when checking the connector due to user request.
+            <parameter>force</parameter> can be used by the driver to avoid
+            expensive, destructive operations during automated probing.
+          </para>
+          <para>
+            Return connector_status_connected if something is connected to the
+            connector, connector_status_disconnected if nothing is connected and
+            connector_status_unknown if the connection state isn't known.
+          </para>
+          <para>
+            Drivers should only return connector_status_connected if the connection
+            status has really been probed as connected. Connectors that can't detect
+            the connection status, or failed connection status probes, should return
+            connector_status_unknown.
+          </para>
+        </sect4>
+        <sect4>
+          <title>Miscellaneous</title>
+          <itemizedlist>
+            <listitem>
+              <synopsis>void (*destroy)(struct drm_connector *connector);</synopsis>
+              <para>
+                Destroy the connector when not needed anymore. See
+                <xref linkend="drm-kms-init"/>.
+              </para>
+            </listitem>
+          </itemizedlist>
+        </sect4>
+      </sect3>
+    </sect2>
+    <sect2>
+      <title>Cleanup</title>
+      <para>
+        The DRM core manages its objects' lifetime. When an object is not needed
+	anymore the core calls its destroy function, which must clean up and
+	free every resource allocated for the object. Every
+	<function>drm_*_init</function> call must be matched with a
+	corresponding <function>drm_*_cleanup</function> call to cleanup CRTCs
+	(<function>drm_crtc_cleanup</function>), planes
+	(<function>drm_plane_cleanup</function>), encoders
+	(<function>drm_encoder_cleanup</function>) and connectors
+	(<function>drm_connector_cleanup</function>). Furthermore, connectors
+	that have been added to sysfs must be removed by a call to
+	<function>drm_sysfs_connector_remove</function> before calling
+	<function>drm_connector_cleanup</function>.
+      </para>
+      <para>
+        Connectors state change detection must be cleanup up with a call to
+	<function>drm_kms_helper_poll_fini</function>.
+      </para>
+    </sect2>
+    <sect2>
+      <title>Output discovery and initialization example</title>
+      <programlisting><![CDATA[
 void intel_crt_init(struct drm_device *dev)
 {
 	struct drm_connector *connector;
@@ -556,252 +1610,741 @@ void intel_crt_init(struct drm_device *dev)
 	drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
 	drm_sysfs_connector_add(connector);
-}
-]]>
-	</programlisting>
-	<para>
-	  In the example above (again, taken from the i915 driver), a
-	  CRT connector and encoder combination is created.  A device-specific
-	  i2c bus is also created for fetching EDID data and
-	  performing monitor detection.  Once the process is complete,
-	  the new connector is registered with sysfs to make its
-	  properties available to applications.
-	</para>
-	<sect4>
-	  <title>Helper functions and core functions</title>
-	  <para>
-	    Since many PC-class graphics devices have similar display output
-	    designs, the DRM provides a set of helper functions to make
-	    output management easier.  The core helper routines handle
-	    encoder re-routing and the disabling of unused functions following
-	    mode setting.  Using the helpers is optional, but recommended for
-	    devices with PC-style architectures (i.e. a set of display planes
-	    for feeding pixels to encoders which are in turn routed to
-	    connectors).  Devices with more complex requirements needing
-	    finer grained management may opt to use the core callbacks
-	    directly.
-	  </para>
-	  <para>
-	    [Insert typical diagram here.]  [Insert OMAP style config here.]
-	  </para>
-	</sect4>
-	<para>
-	  Each encoder object needs to provide:
-	  <itemizedlist>
-	    <listitem>
-	      A DPMS (basically on/off) function.
-	    </listitem>
-	    <listitem>
-	      A mode-fixup function (for converting requested modes into
-	      native hardware timings).
-	    </listitem>
-	    <listitem>
-	      Functions (prepare, set, and commit) for use by the core DRM
-	      helper functions.
-	    </listitem>
-	  </itemizedlist>
-	  Connector helpers need to provide functions (mode-fetch, validity,
-	  and encoder-matching) for returning an ideal encoder for a given
-	  connector.  The core connector functions include a DPMS callback,
-	  save/restore routines (deprecated), detection, mode probing,
-	  property handling, and cleanup functions.
-	</para>
-<!--!Edrivers/char/drm/drm_crtc.h-->
-<!--!Edrivers/char/drm/drm_crtc.c-->
-<!--!Edrivers/char/drm/drm_crtc_helper.c-->
-      </sect3>
+}]]></programlisting>
+      <para>
+        In the example above (taken from the i915 driver), a CRTC, connector and
+        encoder combination is created. A device-specific i2c bus is also
+        created for fetching EDID data and performing monitor detection. Once
+        the process is complete, the new connector is registered with sysfs to
+        make its properties available to applications.
+      </para>
     </sect2>
   </sect1>
 
-  <!-- Internals: vblank handling -->
+  <!-- Internals: mid-layer helper functions -->
 
   <sect1>
-    <title>VBlank event handling</title>
+    <title>Mid-layer Helper Functions</title>
     <para>
-      The DRM core exposes two vertical blank related ioctls:
-      <variablelist>
-        <varlistentry>
-          <term>DRM_IOCTL_WAIT_VBLANK</term>
-          <listitem>
-            <para>
-              This takes a struct drm_wait_vblank structure as its argument,
-              and it is used to block or request a signal when a specified
-              vblank event occurs.
-            </para>
-          </listitem>
-        </varlistentry>
-        <varlistentry>
-          <term>DRM_IOCTL_MODESET_CTL</term>
-          <listitem>
-            <para>
-              This should be called by application level drivers before and
-              after mode setting, since on many devices the vertical blank
-              counter is reset at that time.  Internally, the DRM snapshots
-              the last vblank count when the ioctl is called with the
-              _DRM_PRE_MODESET command, so that the counter won't go backwards
-              (which is dealt with when _DRM_POST_MODESET is used).
-            </para>
-          </listitem>
-        </varlistentry>
-      </variablelist>
-<!--!Edrivers/char/drm/drm_irq.c-->
+      The CRTC, encoder and connector functions provided by the drivers
+      implement the DRM API. They're called by the DRM core and ioctl handlers
+      to handle device state changes and configuration request. As implementing
+      those functions often requires logic not specific to drivers, mid-layer
+      helper functions are available to avoid duplicating boilerplate code.
+    </para>
+    <para>
+      The DRM core contains one mid-layer implementation. The mid-layer provides
+      implementations of several CRTC, encoder and connector functions (called
+      from the top of the mid-layer) that pre-process requests and call
+      lower-level functions provided by the driver (at the bottom of the
+      mid-layer). For instance, the
+      <function>drm_crtc_helper_set_config</function> function can be used to
+      fill the struct <structname>drm_crtc_funcs</structname>
+      <structfield>set_config</structfield> field. When called, it will split
+      the <methodname>set_config</methodname> operation in smaller, simpler
+      operations and call the driver to handle them.
     </para>
     <para>
-      To support the functions above, the DRM core provides several
-      helper functions for tracking vertical blank counters, and
-      requires drivers to provide several callbacks:
-      get_vblank_counter(), enable_vblank() and disable_vblank().  The
-      core uses get_vblank_counter() to keep the counter accurate
-      across interrupt disable periods.  It should return the current
-      vertical blank event count, which is often tracked in a device
-      register.  The enable and disable vblank callbacks should enable
-      and disable vertical blank interrupts, respectively.  In the
-      absence of DRM clients waiting on vblank events, the core DRM
-      code uses the disable_vblank() function to disable
-      interrupts, which saves power.  They are re-enabled again when
-      a client calls the vblank wait ioctl above.
+      To use the mid-layer, drivers call <function>drm_crtc_helper_add</function>,
+      <function>drm_encoder_helper_add</function> and
+      <function>drm_connector_helper_add</function> functions to install their
+      mid-layer bottom operations handlers, and fill the
+      <structname>drm_crtc_funcs</structname>,
+      <structname>drm_encoder_funcs</structname> and
+      <structname>drm_connector_funcs</structname> structures with pointers to
+      the mid-layer top API functions. Installing the mid-layer bottom operation
+      handlers is best done right after registering the corresponding KMS object.
     </para>
     <para>
-      A device that doesn't provide a count register may simply use an
-      internal atomic counter incremented on every vertical blank
-      interrupt (and then treat the enable_vblank() and disable_vblank()
-      callbacks as no-ops).
+      The mid-layer is not split between CRTC, encoder and connector operations.
+      To use it, a driver must provide bottom functions for all of the three KMS
+      entities.
     </para>
+    <sect2>
+      <title>Helper Functions</title>
+      <itemizedlist>
+        <listitem>
+          <synopsis>int drm_crtc_helper_set_config(struct drm_mode_set *set);</synopsis>
+          <para>
+            The <function>drm_crtc_helper_set_config</function> helper function
+            is a CRTC <methodname>set_config</methodname> implementation. It
+            first tries to locate the best encoder for each connector by calling
+            the connector <methodname>best_encoder</methodname> helper
+            operation.
+          </para>
+          <para>
+            After locating the appropriate encoders, the helper function will
+            call the <methodname>mode_fixup</methodname> encoder and CRTC helper
+            operations to adjust the requested mode, or reject it completely in
+            which case an error will be returned to the application. If the new
+            configuration after mode adjustment is identical to the current
+            configuration the helper function will return without performing any
+            other operation.
+          </para>
+          <para>
+            If the adjusted mode is identical to the current mode but changes to
+            the frame buffer need to be applied, the
+            <function>drm_crtc_helper_set_config</function> function will call
+            the CRTC <methodname>mode_set_base</methodname> helper operation. If
+            the adjusted mode differs from the current mode, or if the
+            <methodname>mode_set_base</methodname> helper operation is not
+            provided, the helper function performs a full mode set sequence by
+            calling the <methodname>prepare</methodname>,
+            <methodname>mode_set</methodname> and
+            <methodname>commit</methodname> CRTC and encoder helper operations,
+            in that order.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>void drm_helper_connector_dpms(struct drm_connector *connector, int mode);</synopsis>
+          <para>
+            The <function>drm_helper_connector_dpms</function> helper function
+            is a connector <methodname>dpms</methodname> implementation that
+            tracks power state of connectors. To use the function, drivers must
+            provide <methodname>dpms</methodname> helper operations for CRTCs
+            and encoders to apply the DPMS state to the device.
+          </para>
+          <para>
+            The mid-layer doesn't track the power state of CRTCs and encoders.
+            The <methodname>dpms</methodname> helper operations can thus be
+            called with a mode identical to the currently active mode.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
+                                            uint32_t maxX, uint32_t maxY);</synopsis>
+          <para>
+            The <function>drm_helper_probe_single_connector_modes</function> helper
+            function is a connector <methodname>fill_modes</methodname>
+            implementation that updates the connection status for the connector
+            and then retrieves a list of modes by calling the connector
+            <methodname>get_modes</methodname> helper operation.
+          </para>
+          <para>
+            The function filters out modes larger than
+            <parameter>max_width</parameter> and <parameter>max_height</parameter>
+            if specified. It then calls the connector
+            <methodname>mode_valid</methodname> helper operation for  each mode in
+            the probed list to check whether the mode is valid for the connector.
+          </para>
+        </listitem>
+      </itemizedlist>
+    </sect2>
+    <sect2>
+      <title>CRTC Helper Operations</title>
+      <itemizedlist>
+        <listitem id="drm-helper-crtc-mode-fixup">
+          <synopsis>bool (*mode_fixup)(struct drm_crtc *crtc,
+                       const struct drm_display_mode *mode,
+                       struct drm_display_mode *adjusted_mode);</synopsis>
+          <para>
+            Let CRTCs adjust the requested mode or reject it completely. This
+            operation returns true if the mode is accepted (possibly after being
+            adjusted) or false if it is rejected.
+          </para>
+          <para>
+            The <methodname>mode_fixup</methodname> operation should reject the
+            mode if it can't reasonably use it. The definition of "reasonable"
+            is currently fuzzy in this context. One possible behaviour would be
+            to set the adjusted mode to the panel timings when a fixed-mode
+            panel is used with hardware capable of scaling. Another behaviour
+            would be to accept any input mode and adjust it to the closest mode
+            supported by the hardware (FIXME: This needs to be clarified).
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>int (*mode_set_base)(struct drm_crtc *crtc, int x, int y,
+                     struct drm_framebuffer *old_fb)</synopsis>
+          <para>
+            Move the CRTC on the current frame buffer (stored in
+            <literal>crtc-&gt;fb</literal>) to position (x,y). Any of the frame
+            buffer, x position or y position may have been modified.
+          </para>
+          <para>
+            This helper operation is optional. If not provided, the
+            <function>drm_crtc_helper_set_config</function> function will fall
+            back to the <methodname>mode_set</methodname> helper operation.
+          </para>
+          <note><para>
+            FIXME: Why are x and y passed as arguments, as they can be accessed
+            through <literal>crtc-&gt;x</literal> and
+            <literal>crtc-&gt;y</literal>?
+          </para></note>
+        </listitem>
+        <listitem>
+          <synopsis>void (*prepare)(struct drm_crtc *crtc);</synopsis>
+          <para>
+            Prepare the CRTC for mode setting. This operation is called after
+            validating the requested mode. Drivers use it to perform
+            device-specific operations required before setting the new mode.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>int (*mode_set)(struct drm_crtc *crtc, struct drm_display_mode *mode,
+                struct drm_display_mode *adjusted_mode, int x, int y,
+                struct drm_framebuffer *old_fb);</synopsis>
+          <para>
+            Set a new mode, position and frame buffer. Depending on the device
+            requirements, the mode can be stored internally by the driver and
+            applied in the <methodname>commit</methodname> operation, or
+            programmed to the hardware immediately.
+          </para>
+          <para>
+            The <methodname>mode_set</methodname> operation returns 0 on success
+	    or a negative error code if an error occurs.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>void (*commit)(struct drm_crtc *crtc);</synopsis>
+          <para>
+            Commit a mode. This operation is called after setting the new mode.
+            Upon return the device must use the new mode and be fully
+            operational.
+          </para>
+        </listitem>
+      </itemizedlist>
+    </sect2>
+    <sect2>
+      <title>Encoder Helper Operations</title>
+      <itemizedlist>
+        <listitem>
+          <synopsis>bool (*mode_fixup)(struct drm_encoder *encoder,
+                       const struct drm_display_mode *mode,
+                       struct drm_display_mode *adjusted_mode);</synopsis>
+          <note><para>
+            FIXME: The mode argument be const, but the i915 driver modifies
+            mode-&gt;clock in <function>intel_dp_mode_fixup</function>.
+          </para></note>
+          <para>
+            Let encoders adjust the requested mode or reject it completely. This
+            operation returns true if the mode is accepted (possibly after being
+            adjusted) or false if it is rejected. See the
+            <link linkend="drm-helper-crtc-mode-fixup">mode_fixup CRTC helper
+            operation</link> for an explanation of the allowed adjustments.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>void (*prepare)(struct drm_encoder *encoder);</synopsis>
+          <para>
+            Prepare the encoder for mode setting. This operation is called after
+            validating the requested mode. Drivers use it to perform
+            device-specific operations required before setting the new mode.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>void (*mode_set)(struct drm_encoder *encoder,
+                 struct drm_display_mode *mode,
+                 struct drm_display_mode *adjusted_mode);</synopsis>
+          <para>
+            Set a new mode. Depending on the device requirements, the mode can
+            be stored internally by the driver and applied in the
+            <methodname>commit</methodname> operation, or programmed to the
+            hardware immediately.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>void (*commit)(struct drm_encoder *encoder);</synopsis>
+          <para>
+            Commit a mode. This operation is called after setting the new mode.
+            Upon return the device must use the new mode and be fully
+            operational.
+          </para>
+        </listitem>
+      </itemizedlist>
+    </sect2>
+    <sect2>
+      <title>Connector Helper Operations</title>
+      <itemizedlist>
+        <listitem>
+          <synopsis>struct drm_encoder *(*best_encoder)(struct drm_connector *connector);</synopsis>
+          <para>
+            Return a pointer to the best encoder for the connecter. Device that
+            map connectors to encoders 1:1 simply return the pointer to the
+            associated encoder. This operation is mandatory.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>int (*get_modes)(struct drm_connector *connector);</synopsis>
+          <para>
+            Fill the connector's <structfield>probed_modes</structfield> list
+            by parsing EDID data with <function>drm_add_edid_modes</function> or
+            calling <function>drm_mode_probed_add</function> directly for every
+            supported mode and return the number of modes it has detected. This
+            operation is mandatory.
+          </para>
+          <para>
+            When adding modes manually the driver creates each mode with a call to
+            <function>drm_mode_create</function> and must fill the following fields.
+            <itemizedlist>
+              <listitem>
+                <synopsis>__u32 type;</synopsis>
+                <para>
+                  Mode type bitmask, a combination of
+                  <variablelist>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_BUILTIN</term>
+                      <listitem><para>not used?</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_CLOCK_C</term>
+                      <listitem><para>not used?</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_CRTC_C</term>
+                      <listitem><para>not used?</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>
+        DRM_MODE_TYPE_PREFERRED - The preferred mode for the connector
+                      </term>
+                      <listitem>
+                        <para>not used?</para>
+                      </listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_DEFAULT</term>
+                      <listitem><para>not used?</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_USERDEF</term>
+                      <listitem><para>not used?</para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_TYPE_DRIVER</term>
+                      <listitem>
+                        <para>
+                          The mode has been created by the driver (as opposed to
+                          to user-created modes).
+                        </para>
+                      </listitem>
+                    </varlistentry>
+                  </variablelist>
+                  Drivers must set the DRM_MODE_TYPE_DRIVER bit for all modes they
+                  create, and set the DRM_MODE_TYPE_PREFERRED bit for the preferred
+                  mode.
+                </para>
+              </listitem>
+              <listitem>
+                <synopsis>__u32 clock;</synopsis>
+                <para>Pixel clock frequency in kHz unit</para>
+              </listitem>
+              <listitem>
+                <synopsis>__u16 hdisplay, hsync_start, hsync_end, htotal;
+    __u16 vdisplay, vsync_start, vsync_end, vtotal;</synopsis>
+                <para>Horizontal and vertical timing information</para>
+                <screen><![CDATA[
+             Active                 Front           Sync           Back
+             Region                 Porch                          Porch
+    <-----------------------><----------------><-------------><-------------->
+
+      //////////////////////|
+     ////////////////////// |
+    //////////////////////  |..................               ................
+                                               _______________
+
+    <----- [hv]display ----->
+    <------------- [hv]sync_start ------------>
+    <--------------------- [hv]sync_end --------------------->
+    <-------------------------------- [hv]total ----------------------------->
+]]></screen>
+              </listitem>
+              <listitem>
+                <synopsis>__u16 hskew;
+    __u16 vscan;</synopsis>
+                <para>Unknown</para>
+              </listitem>
+              <listitem>
+                <synopsis>__u32 flags;</synopsis>
+                <para>
+                  Mode flags, a combination of
+                  <variablelist>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_PHSYNC</term>
+                      <listitem><para>
+                        Horizontal sync is active high
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_NHSYNC</term>
+                      <listitem><para>
+                        Horizontal sync is active low
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_PVSYNC</term>
+                      <listitem><para>
+                        Vertical sync is active high
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_NVSYNC</term>
+                      <listitem><para>
+                        Vertical sync is active low
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_INTERLACE</term>
+                      <listitem><para>
+                        Mode is interlaced
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_DBLSCAN</term>
+                      <listitem><para>
+                        Mode uses doublescan
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_CSYNC</term>
+                      <listitem><para>
+                        Mode uses composite sync
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_PCSYNC</term>
+                      <listitem><para>
+                        Composite sync is active high
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_NCSYNC</term>
+                      <listitem><para>
+                        Composite sync is active low
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_HSKEW</term>
+                      <listitem><para>
+                        hskew provided (not used?)
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_BCAST</term>
+                      <listitem><para>
+                        not used?
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_PIXMUX</term>
+                      <listitem><para>
+                        not used?
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_DBLCLK</term>
+                      <listitem><para>
+                        not used?
+                      </para></listitem>
+                    </varlistentry>
+                    <varlistentry>
+                      <term>DRM_MODE_FLAG_CLKDIV2</term>
+                      <listitem><para>
+                        ?
+                      </para></listitem>
+                    </varlistentry>
+                  </variablelist>
+                </para>
+                <para>
+                  Note that modes marked with the INTERLACE or DBLSCAN flags will be
+                  filtered out by
+                  <function>drm_helper_probe_single_connector_modes</function> if
+                  the connector's <structfield>interlace_allowed</structfield> or
+                  <structfield>doublescan_allowed</structfield> field is set to 0.
+                </para>
+              </listitem>
+              <listitem>
+                <synopsis>char name[DRM_DISPLAY_MODE_LEN];</synopsis>
+                <para>
+                  Mode name. The driver must call
+                  <function>drm_mode_set_name</function> to fill the mode name from
+                  <structfield>hdisplay</structfield>,
+                  <structfield>vdisplay</structfield> and interlace flag after
+                  filling the corresponding fields.
+                </para>
+              </listitem>
+            </itemizedlist>
+          </para>
+          <para>
+            The <structfield>vrefresh</structfield> value is computed by
+            <function>drm_helper_probe_single_connector_modes</function>.
+          </para>
+          <para>
+            When parsing EDID data, <function>drm_add_edid_modes</function> fill the
+            connector <structfield>display_info</structfield>
+            <structfield>width_mm</structfield> and
+            <structfield>height_mm</structfield> fields. When creating modes
+            manually the <methodname>get_modes</methodname> helper operation must
+            set the <structfield>display_info</structfield>
+            <structfield>width_mm</structfield> and
+            <structfield>height_mm</structfield> fields if they haven't been set
+            already (for instance at initilization time when a fixed-size panel is
+            attached to the connector). The mode <structfield>width_mm</structfield>
+            and <structfield>height_mm</structfield> fields are only used internally
+            during EDID parsing and should not be set when creating modes manually.
+          </para>
+        </listitem>
+        <listitem>
+          <synopsis>int (*mode_valid)(struct drm_connector *connector,
+		  struct drm_display_mode *mode);</synopsis>
+          <para>
+            Verify whether a mode is valid for the connector. Return MODE_OK for
+            supported modes and one of the enum drm_mode_status values (MODE_*)
+            for unsupported modes. This operation is mandatory.
+          </para>
+          <para>
+            As the mode rejection reason is currently not used beside for
+            immediately removing the unsupported mode, an implementation can
+            return MODE_BAD regardless of the exact reason why the mode is not
+            valid.
+          </para>
+          <note><para>
+            Note that the <methodname>mode_valid</methodname> helper operation is
+            only called for modes detected by the device, and
+            <emphasis>not</emphasis> for modes set by the user through the CRTC
+            <methodname>set_config</methodname> operation.
+          </para></note>
+        </listitem>
+      </itemizedlist>
+    </sect2>
   </sect1>
 
-  <sect1>
-    <title>Memory management</title>
+  <!-- Internals: vertical blanking -->
+
+  <sect1 id="drm-vertical-blank">
+    <title>Vertical Blanking</title>
+    <para>
+      Vertical blanking plays a major role in graphics rendering. To achieve
+      tear-free display, users must synchronize page flips and/or rendering to
+      vertical blanking. The DRM API offers ioctls to perform page flips
+      synchronized to vertical blanking and wait for vertical blanking.
+    </para>
+    <para>
+      The DRM core handles most of the vertical blanking management logic, which
+      involves filtering out spurious interrupts, keeping race-free blanking
+      counters, coping with counter wrap-around and resets and keeping use
+      counts. It relies on the driver to generate vertical blanking interrupts
+      and optionally provide a hardware vertical blanking counter. Drivers must
+      implement the following operations.
+    </para>
+    <itemizedlist>
+      <listitem>
+        <synopsis>int (*enable_vblank) (struct drm_device *dev, int crtc);
+void (*disable_vblank) (struct drm_device *dev, int crtc);</synopsis>
+        <para>
+	  Enable or disable vertical blanking interrupts for the given CRTC.
+	</para>
+      </listitem>
+      <listitem>
+        <synopsis>u32 (*get_vblank_counter) (struct drm_device *dev, int crtc);</synopsis>
+        <para>
+	  Retrieve the value of the vertical blanking counter for the given
+	  CRTC. If the hardware maintains a vertical blanking counter its value
+	  should be returned. Otherwise drivers can use the
+	  <function>drm_vblank_count</function> helper function to handle this
+	  operation.
+	</para>
+      </listitem>
+    </itemizedlist>
     <para>
-      The memory manager lies at the heart of many DRM operations; it
-      is required to support advanced client features like OpenGL
-      pbuffers.  The DRM currently contains two memory managers: TTM
-      and GEM.
+      Drivers must initialize the vertical blanking handling core with a call to
+      <function>drm_vblank_init</function> in their
+      <methodname>load</methodname> operation. The function will set the struct
+      <structname>drm_device</structname>
+      <structfield>vblank_disable_allowed</structfield> field to 0. This will
+      keep vertical blanking interrupts enabled permanently until the first mode
+      set operation, where <structfield>vblank_disable_allowed</structfield> is
+      set to 1. The reason behind this is not clear. Drivers can set the field
+      to 1 after <function>calling drm_vblank_init</function> to make vertical
+      blanking interrupts dynamically managed from the beginning.
     </para>
+    <para>
+      Vertical blanking interrupts can be enabled by the DRM core or by drivers
+      themselves (for instance to handle page flipping operations). The DRM core
+      maintains a vertical blanking use count to ensure that the interrupts are
+      not disabled while a user still needs them. To increment the use count,
+      drivers call <function>drm_vblank_get</function>. Upon return vertical
+      blanking interrupts are guaranteed to be enabled.
+    </para>
+    <para>
+      To decrement the use count drivers call
+      <function>drm_vblank_put</function>. Only when the use count drops to zero
+      will the DRM core disable the vertical blanking interrupts after a delay
+      by scheduling a timer. The delay is accessible through the vblankoffdelay
+      module parameter or the <varname>drm_vblank_offdelay</varname> global
+      variable and expressed in milliseconds. Its default value is 5000 ms.
+    </para>
+    <para>
+      When a vertical blanking interrupt occurs drivers only need to call the
+      <function>drm_handle_vblank</function> function to account for the
+      interrupt.
+    </para>
+    <para>
+      Resources allocated by <function>drm_vblank_init</function> must be freed
+      with a call to <function>drm_vblank_cleanup</function> in the driver
+      <methodname>unload</methodname> operation handler.
+    </para>
+  </sect1>
+
+  <!-- Internals: open/close, file operations and ioctls -->
 
+  <sect1>
+    <title>Open/Close, File Operations and IOCTLs</title>
     <sect2>
-      <title>The Translation Table Manager (TTM)</title>
+      <title>Open and Close</title>
+      <synopsis>int (*firstopen) (struct drm_device *);
+void (*lastclose) (struct drm_device *);
+int (*open) (struct drm_device *, struct drm_file *);
+void (*preclose) (struct drm_device *, struct drm_file *);
+void (*postclose) (struct drm_device *, struct drm_file *);</synopsis>
+      <abstract>Open and close handlers. None of those methods are mandatory.
+      </abstract>
       <para>
-	TTM was developed by Tungsten Graphics, primarily by Thomas
-	Hellström, and is intended to be a flexible, high performance
-	graphics memory manager.
+        The <methodname>firstopen</methodname> method is called by the DRM core
+	when an application opens a device that has no other opened file handle.
+	Similarly the <methodname>lastclose</methodname> method is called when
+	the last application holding a file handle opened on the device closes
+	it. Both methods are mostly used for UMS (User Mode Setting) drivers to
+	acquire and release device resources which should be done in the
+	<methodname>load</methodname> and <methodname>unload</methodname>
+	methods for KMS drivers.
       </para>
       <para>
-	Drivers wishing to support TTM must fill out a drm_bo_driver
-	structure.
+        Note that the <methodname>lastclose</methodname> method is also called
+	at module unload time or, for hot-pluggable devices, when the device is
+	unplugged. The <methodname>firstopen</methodname> and
+	<methodname>lastclose</methodname> calls can thus be unbalanced.
       </para>
       <para>
-	TTM design background and information belongs here.
+        The <methodname>open</methodname> method is called every time the device
+	is opened by an application. Drivers can allocate per-file private data
+	in this method and store them in the struct
+	<structname>drm_file</structname> <structfield>driver_priv</structfield>
+	field. Note that the <methodname>open</methodname> method is called
+	before <methodname>firstopen</methodname>.
+      </para>
+      <para>
+        The close operation is split into <methodname>preclose</methodname> and
+	<methodname>postclose</methodname> methods. Drivers must stop and
+	cleanup all per-file operations in the <methodname>preclose</methodname>
+	method. For instance pending vertical blanking and page flip events must
+	be cancelled. No per-file operation is allowed on the file handle after
+	returning from the <methodname>preclose</methodname> method.
+      </para>
+      <para>
+        Finally the <methodname>postclose</methodname> method is called as the
+	last step of the close operation, right before calling the
+	<methodname>lastclose</methodname> method if no other open file handle
+	exists for the device. Drivers that have allocated per-file private data
+	in the <methodname>open</methodname> method should free it here.
+      </para>
+      <para>
+        The <methodname>lastclose</methodname> method should restore CRTC and
+	plane properties to default value, so that a subsequent open of the
+	device will not inherit state from the previous user.
       </para>
     </sect2>
-
     <sect2>
-      <title>The Graphics Execution Manager (GEM)</title>
+      <title>File Operations</title>
+      <synopsis>const struct file_operations *fops</synopsis>
+      <abstract>File operations for the DRM device node.</abstract>
       <para>
-	GEM is an Intel project, authored by Eric Anholt and Keith
-	Packard.  It provides simpler interfaces than TTM, and is well
-	suited for UMA devices.
+        Drivers must define the file operations structure that forms the DRM
+	userspace API entry point, even though most of those operations are
+	implemented in the DRM core. The <methodname>open</methodname>,
+	<methodname>release</methodname> and <methodname>ioctl</methodname>
+	operations are handled by
+	<programlisting>
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+  #ifdef CONFIG_COMPAT
+	.compat_ioctl = drm_compat_ioctl,
+  #endif
+        </programlisting>
       </para>
       <para>
-	GEM-enabled drivers must provide gem_init_object() and
-	gem_free_object() callbacks to support the core memory
-	allocation routines.  They should also provide several driver-specific
-	ioctls to support command execution, pinning, buffer
-	read &amp; write, mapping, and domain ownership transfers.
+        Drivers that implement private ioctls that requires 32/64bit
+	compatibility support must provide their own
+	<methodname>compat_ioctl</methodname> handler that processes private
+	ioctls and calls <function>drm_compat_ioctl</function> for core ioctls.
       </para>
       <para>
-	On a fundamental level, GEM involves several operations:
-	<itemizedlist>
-	  <listitem>Memory allocation and freeing</listitem>
-	  <listitem>Command execution</listitem>
-	  <listitem>Aperture management at command execution time</listitem>
-	</itemizedlist>
-	Buffer object allocation is relatively
-	straightforward and largely provided by Linux's shmem layer, which
-	provides memory to back each object.  When mapped into the GTT
-	or used in a command buffer, the backing pages for an object are
-	flushed to memory and marked write combined so as to be coherent
-	with the GPU.  Likewise, if the CPU accesses an object after the GPU
-	has finished rendering to the object, then the object must be made
-	coherent with the CPU's view
-	of memory, usually involving GPU cache flushing of various kinds.
-	This core CPU&lt;-&gt;GPU coherency management is provided by a
-	device-specific ioctl, which evaluates an object's current domain and
-	performs any necessary flushing or synchronization to put the object
-	into the desired coherency domain (note that the object may be busy,
-	i.e. an active render target; in that case, setting the domain
-	blocks the client and waits for rendering to complete before
-	performing any necessary flushing operations).
-      </para>
-      <para>
-	Perhaps the most important GEM function is providing a command
-	execution interface to clients.  Client programs construct command
-	buffers containing references to previously allocated memory objects,
-	and then submit them to GEM.  At that point, GEM takes care to bind
-	all the objects into the GTT, execute the buffer, and provide
-	necessary synchronization between clients accessing the same buffers.
-	This often involves evicting some objects from the GTT and re-binding
-	others (a fairly expensive operation), and providing relocation
-	support which hides fixed GTT offsets from clients.  Clients must
-	take care not to submit command buffers that reference more objects
-	than can fit in the GTT; otherwise, GEM will reject them and no rendering
-	will occur.  Similarly, if several objects in the buffer require
-	fence registers to be allocated for correct rendering (e.g. 2D blits
-	on pre-965 chips), care must be taken not to require more fence
-	registers than are available to the client.  Such resource management
-	should be abstracted from the client in libdrm.
+        The <methodname>read</methodname> and <methodname>poll</methodname>
+	operations provide support for reading DRM events and polling them. They
+	are implemented by
+	<programlisting>
+	.poll = drm_poll,
+	.read = drm_read,
+	.fasync = drm_fasync,
+	.llseek = no_llseek,
+	</programlisting>
+      </para>
+      <para>
+        The memory mapping implementation varies depending on how the driver
+	manages memory. Pre-GEM drivers will use <function>drm_mmap</function>,
+	while GEM-aware drivers will use <function>drm_gem_mmap</function>. See
+	<xref linkend="drm-gem"/>.
+	<programlisting>
+	.mmap = drm_gem_mmap,
+	</programlisting>
+      </para>
+      <para>
+        No other file operation is supported by the DRM API.
+      </para>
+    </sect2>
+    <sect2>
+      <title>IOCTLs</title>
+      <synopsis>struct drm_ioctl_desc *ioctls;
+int num_ioctls;</synopsis>
+      <abstract>Driver-specific ioctls descriptors table.</abstract>
+      <para>
+        Driver-specific ioctls numbers start at DRM_COMMAND_BASE. The ioctls
+	descriptors table is indexed by the ioctl number offset from the base
+	value. Drivers can use the DRM_IOCTL_DEF_DRV() macro to initialize the
+	table entries.
+      </para>
+      <para>
+        <programlisting>DRM_IOCTL_DEF_DRV(ioctl, func, flags)</programlisting>
+	<para>
+	  <parameter>ioctl</parameter> is the ioctl name. Drivers must define
+	  the DRM_##ioctl and DRM_IOCTL_##ioctl macros to the ioctl number
+	  offset from DRM_COMMAND_BASE and the ioctl number respectively. The
+	  first macro is private to the device while the second must be exposed
+	  to userspace in a public header.
+	</para>
+	<para>
+	  <parameter>func</parameter> is a pointer to the ioctl handler function
+	  compatible with the <type>drm_ioctl_t</type> type.
+	  <programlisting>typedef int drm_ioctl_t(struct drm_device *dev, void *data,
+		struct drm_file *file_priv);</programlisting>
+	</para>
+	<para>
+	  <parameter>flags</parameter> is a bitmask combination of the following
+	  values. It restricts how the ioctl is allowed to be called.
+	  <itemizedlist>
+	    <listitem><para>
+	      DRM_AUTH - Only authenticated callers allowed
+	    </para></listitem>
+	    <listitem><para>
+	      DRM_MASTER - The ioctl can only be called on the master file
+	      handle
+	    </para></listitem>
+            <listitem><para>
+	      DRM_ROOT_ONLY - Only callers with the SYSADMIN capability allowed
+	    </para></listitem>
+            <listitem><para>
+	      DRM_CONTROL_ALLOW - The ioctl can only be called on a control
+	      device
+	    </para></listitem>
+            <listitem><para>
+	      DRM_UNLOCKED - The ioctl handler will be called without locking
+	      the DRM global mutex
+	    </para></listitem>
+	  </itemizedlist>
+	</para>
       </para>
     </sect2>
-
-  </sect1>
-
-  <!-- Output management -->
-  <sect1>
-    <title>Output management</title>
-    <para>
-      At the core of the DRM output management code is a set of
-      structures representing CRTCs, encoders, and connectors.
-    </para>
-    <para>
-      A CRTC is an abstraction representing a part of the chip that
-      contains a pointer to a scanout buffer.  Therefore, the number
-      of CRTCs available determines how many independent scanout
-      buffers can be active at any given time.  The CRTC structure
-      contains several fields to support this: a pointer to some video
-      memory, a display mode, and an (x, y) offset into the video
-      memory to support panning or configurations where one piece of
-      video memory spans multiple CRTCs.
-    </para>
-    <para>
-      An encoder takes pixel data from a CRTC and converts it to a
-      format suitable for any attached connectors.  On some devices,
-      it may be possible to have a CRTC send data to more than one
-      encoder.  In that case, both encoders would receive data from
-      the same scanout buffer, resulting in a "cloned" display
-      configuration across the connectors attached to each encoder.
-    </para>
-    <para>
-      A connector is the final destination for pixel data on a device,
-      and usually connects directly to an external display device like
-      a monitor or laptop panel.  A connector can only be attached to
-      one encoder at a time.  The connector is also the structure
-      where information about the attached display is kept, so it
-      contains fields for display data, EDID data, DPMS &amp;
-      connection status, and information about modes supported on the
-      attached displays.
-    </para>
-<!--!Edrivers/char/drm/drm_crtc.c-->
-  </sect1>
-
-  <sect1>
-    <title>Framebuffer management</title>
-    <para>
-      Clients need to provide a framebuffer object which provides a source
-      of pixels for a CRTC to deliver to the encoder(s) and ultimately the
-      connector(s). A framebuffer is fundamentally a driver-specific memory
-      object, made into an opaque handle by the DRM's addfb() function.
-      Once a framebuffer has been created this way, it may be passed to the
-      KMS mode setting routines for use in a completed configuration.
-    </para>
   </sect1>
 
   <sect1>
@@ -812,15 +2355,24 @@ void intel_crt_init(struct drm_device *dev)
     </para>
   </sect1>
 
+  <!-- Internals: suspend/resume -->
+
   <sect1>
-    <title>Suspend/resume</title>
+    <title>Suspend/Resume</title>
+    <para>
+      The DRM core provides some suspend/resume code, but drivers wanting full
+      suspend/resume support should provide save() and restore() functions.
+      These are called at suspend, hibernate, or resume time, and should perform
+      any state save or restore required by your device across suspend or
+      hibernate states.
+    </para>
+    <synopsis>int (*suspend) (struct drm_device *, pm_message_t state);
+int (*resume) (struct drm_device *);</synopsis>
     <para>
-      The DRM core provides some suspend/resume code, but drivers
-      wanting full suspend/resume support should provide save() and
-      restore() functions.  These are called at suspend,
-      hibernate, or resume time, and should perform any state save or
-      restore required by your device across suspend or hibernate
-      states.
+      Those are legacy suspend and resume methods. New driver should use the
+      power management interface provided by their bus type (usually through
+      the struct <structname>device_driver</structname> dev_pm_ops) and set
+      these methods to NULL.
     </para>
   </sect1>
 
@@ -833,6 +2385,35 @@ void intel_crt_init(struct drm_device *dev)
   </sect1>
   </chapter>
 
+<!-- TODO
+
+- Add a glossary
+- Document the struct_mutex catch-all lock
+- Document connector properties
+
+- Why is the load method optional?
+- What are drivers supposed to set the initial display state to, and how?
+  Connector's DPMS states are not initialized and are thus equal to
+  DRM_MODE_DPMS_ON. The fbcon compatibility layer calls
+  drm_helper_disable_unused_functions(), which disables unused encoders and
+  CRTCs, but doesn't touch the connectors' DPMS state, and
+  drm_helper_connector_dpms() in reaction to fbdev blanking events. Do drivers
+  that don't implement (or just don't use) fbcon compatibility need to call
+  those functions themselves?
+- KMS drivers must call drm_vblank_pre_modeset() and drm_vblank_post_modeset()
+  around mode setting. Should this be done in the DRM core?
+- vblank_disable_allowed is set to 1 in the first drm_vblank_post_modeset()
+  call and never set back to 0. It seems to be safe to permanently set it to 1
+  in drm_vblank_init() for KMS driver, and it might be safe for UMS drivers as
+  well. This should be investigated.
+- crtc and connector .save and .restore operations are only used internally in
+  drivers, should they be removed from the core?
+- encoder mid-layer .save and .restore operations are only used internally in
+  drivers, should they be removed from the core?
+- encoder mid-layer .detect operation is only used internally in drivers,
+  should it be removed from the core?
+-->
+
   <!-- External interfaces -->
 
   <chapter id="drmExternals">
@@ -853,6 +2434,42 @@ void intel_crt_init(struct drm_device *dev)
       Cover generic ioctls and sysfs layout here.  We only need high-level
       info, since man pages should cover the rest.
     </para>
+
+  <!-- External: vblank handling -->
+
+    <sect1>
+      <title>VBlank event handling</title>
+      <para>
+        The DRM core exposes two vertical blank related ioctls:
+        <variablelist>
+          <varlistentry>
+            <term>DRM_IOCTL_WAIT_VBLANK</term>
+            <listitem>
+              <para>
+                This takes a struct drm_wait_vblank structure as its argument,
+                and it is used to block or request a signal when a specified
+                vblank event occurs.
+              </para>
+            </listitem>
+          </varlistentry>
+          <varlistentry>
+            <term>DRM_IOCTL_MODESET_CTL</term>
+            <listitem>
+              <para>
+                This should be called by application level drivers before and
+                after mode setting, since on many devices the vertical blank
+                counter is reset at that time.  Internally, the DRM snapshots
+                the last vblank count when the ioctl is called with the
+                _DRM_PRE_MODESET command, so that the counter won't go backwards
+                (which is dealt with when _DRM_POST_MODESET is used).
+              </para>
+            </listitem>
+          </varlistentry>
+        </variablelist>
+<!--!Edrivers/char/drm/drm_irq.c-->
+      </para>
+    </sect1>
+
   </chapter>
 
   <!-- API reference -->
diff --git a/Documentation/aoe/aoe.txt b/Documentation/aoe/aoe.txt
index 5f5aa16047f..bfc9cb19abc 100644
--- a/Documentation/aoe/aoe.txt
+++ b/Documentation/aoe/aoe.txt
@@ -1,8 +1,16 @@
-The EtherDrive (R) HOWTO for users of 2.6 kernels is found at ...
+ATA over Ethernet is a network protocol that provides simple access to
+block storage on the LAN.
 
-  http://www.coraid.com/SUPPORT/EtherDrive-HBA  
+  http://support.coraid.com/documents/AoEr11.txt
 
-  It has many tips and hints!
+The EtherDrive (R) HOWTO for 2.6 and 3.x kernels is found at ...
+
+  http://support.coraid.com/support/linux/EtherDrive-2.6-HOWTO.html
+
+It has many tips and hints!  Please see, especially, recommended
+tunings for virtual memory:
+
+  http://support.coraid.com/support/linux/EtherDrive-2.6-HOWTO-5.html#ss5.19
 
 The aoetools are userland programs that are designed to work with this
 driver.  The aoetools are on sourceforge.
@@ -23,20 +31,12 @@ CREATING DEVICE NODES
   There is a udev-install.sh script that shows how to install these
   rules on your system.
 
-  If you are not using udev, two scripts are provided in
-  Documentation/aoe as examples of static device node creation for
-  using the aoe driver.
-
-    rm -rf /dev/etherd
-    sh Documentation/aoe/mkdevs.sh /dev/etherd
-
-  ... or to make just one shelf's worth of block device nodes ...
-
-    sh Documentation/aoe/mkshelf.sh /dev/etherd 0
-
   There is also an autoload script that shows how to edit
   /etc/modprobe.d/aoe.conf to ensure that the aoe module is loaded when
-  necessary.
+  necessary.  Preloading the aoe module is preferable to autoloading,
+  however, because AoE discovery takes a few seconds.  It can be
+  confusing when an AoE device is not present the first time the a
+  command is run but appears a second later.
 
 USING DEVICE NODES
 
@@ -51,9 +51,9 @@ USING DEVICE NODES
   "echo > /dev/etherd/discover" tells the driver to find out what AoE
   devices are available.
 
-  These character devices may disappear and be replaced by sysfs
-  counterparts.  Using the commands in aoetools insulates users from
-  these implementation details.
+  In the future these character devices may disappear and be replaced
+  by sysfs counterparts.  Using the commands in aoetools insulates
+  users from these implementation details.
 
   The block devices are named like this:
 
@@ -76,8 +76,8 @@ USING SYSFS
   The netif attribute is the network interface on the localhost
   through which we are communicating with the remote AoE device.
 
-  There is a script in this directory that formats this information
-  in a convenient way.  Users with aoetools can use the aoe-stat
+  There is a script in this directory that formats this information in
+  a convenient way.  Users with aoetools should use the aoe-stat
   command.
 
   root@makki root# sh Documentation/aoe/status.sh 
@@ -121,3 +121,21 @@ DRIVER OPTIONS
   usage example for the module parameter.
 
     modprobe aoe_iflist="eth1 eth3"
+
+  The aoe_deadsecs module parameter determines the maximum number of
+  seconds that the driver will wait for an AoE device to provide a
+  response to an AoE command.  After aoe_deadsecs seconds have
+  elapsed, the AoE device will be marked as "down".
+
+  The aoe_maxout module parameter has a default of 128.  This is the
+  maximum number of unresponded packets that will be sent to an AoE
+  target at one time.
+
+  The aoe_dyndevs module parameter defaults to 1, meaning that the
+  driver will assign a block device minor number to a discovered AoE
+  target based on the order of its discovery.  With dynamic minor
+  device numbers in use, a greater range of AoE shelf and slot
+  addresses can be supported.  Users with udev will never have to
+  think about minor numbers.  Using aoe_dyndevs=0 allows device nodes
+  to be pre-created using a static minor-number scheme with the
+  aoe-mkshelf script in the aoetools.
diff --git a/Documentation/aoe/mkdevs.sh b/Documentation/aoe/mkdevs.sh
deleted file mode 100644
index 44c0ab70243..00000000000
--- a/Documentation/aoe/mkdevs.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-
-n_shelves=${n_shelves:-10}
-n_partitions=${n_partitions:-16}
-
-if test "$#" != "1"; then
-	echo "Usage: sh `basename $0` {dir}" 1>&2
-	echo "       n_partitions=16 sh `basename $0` {dir}" 1>&2
-	exit 1
-fi
-dir=$1
-
-MAJOR=152
-
-echo "Creating AoE devnode files in $dir ..."
-
-set -e
-
-mkdir -p $dir
-
-# (Status info is in sysfs.  See status.sh.)
-# rm -f $dir/stat
-# mknod -m 0400 $dir/stat c $MAJOR 1
-rm -f $dir/err
-mknod -m 0400 $dir/err c $MAJOR 2
-rm -f $dir/discover
-mknod -m 0200 $dir/discover c $MAJOR 3
-rm -f $dir/interfaces
-mknod -m 0200 $dir/interfaces c $MAJOR 4
-rm -f $dir/revalidate
-mknod -m 0200 $dir/revalidate c $MAJOR 5
-rm -f $dir/flush
-mknod -m 0200 $dir/flush c $MAJOR 6
-
-export n_partitions
-mkshelf=`echo $0 | sed 's!mkdevs!mkshelf!'`
-i=0
-while test $i -lt $n_shelves; do
-	sh -xc "sh $mkshelf $dir $i"
-	i=`expr $i + 1`
-done
diff --git a/Documentation/aoe/mkshelf.sh b/Documentation/aoe/mkshelf.sh
deleted file mode 100644
index 32615814271..00000000000
--- a/Documentation/aoe/mkshelf.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#! /bin/sh
-
-if test "$#" != "2"; then
-	echo "Usage: sh `basename $0` {dir} {shelfaddress}" 1>&2
-	echo "       n_partitions=16 sh `basename $0` {dir} {shelfaddress}" 1>&2
-	exit 1
-fi
-n_partitions=${n_partitions:-16}
-dir=$1
-shelf=$2
-nslots=16
-maxslot=`echo $nslots 1 - p | dc`
-MAJOR=152
-
-set -e
-
-minor=`echo $nslots \* $shelf \* $n_partitions | bc`
-endp=`echo $n_partitions - 1 | bc`
-for slot in `seq 0 $maxslot`; do
-	for part in `seq 0 $endp`; do
-		name=e$shelf.$slot
-		test "$part" != "0" && name=${name}p$part
-		rm -f $dir/$name
-		mknod -m 0660 $dir/$name b $MAJOR $minor
-
-		minor=`expr $minor + 1`
-	done
-done
diff --git a/Documentation/aoe/status.sh b/Documentation/aoe/status.sh
index 751f3be514b..eeec7baae57 100644
--- a/Documentation/aoe/status.sh
+++ b/Documentation/aoe/status.sh
@@ -1,5 +1,8 @@
 #! /bin/sh
 # collate and present sysfs information about AoE storage
+#
+# A more complete version of this script is aoe-stat, in the
+# aoetools.
 
 set -e
 format="%8s\t%8s\t%8s\n"
diff --git a/Documentation/cpu-freq/boost.txt b/Documentation/cpu-freq/boost.txt
new file mode 100644
index 00000000000..9b4edfcf486
--- /dev/null
+++ b/Documentation/cpu-freq/boost.txt
@@ -0,0 +1,93 @@
+Processor boosting control
+
+	- information for users -
+
+Quick guide for the impatient:
+--------------------
+/sys/devices/system/cpu/cpufreq/boost
+controls the boost setting for the whole system. You can read and write
+that file with either "0" (boosting disabled) or "1" (boosting allowed).
+Reading or writing 1 does not mean that the system is boosting at this
+very moment, but only that the CPU _may_ raise the frequency at it's
+discretion.
+--------------------
+
+Introduction
+-------------
+Some CPUs support a functionality to raise the operating frequency of
+some cores in a multi-core package if certain conditions apply, mostly
+if the whole chip is not fully utilized and below it's intended thermal
+budget. This is done without operating system control by a combination
+of hardware and firmware.
+On Intel CPUs this is called "Turbo Boost", AMD calls it "Turbo-Core",
+in technical documentation "Core performance boost". In Linux we use
+the term "boost" for convenience.
+
+Rationale for disable switch
+----------------------------
+
+Though the idea is to just give better performance without any user
+intervention, sometimes the need arises to disable this functionality.
+Most systems offer a switch in the (BIOS) firmware to disable the
+functionality at all, but a more fine-grained and dynamic control would
+be desirable:
+1. While running benchmarks, reproducible results are important. Since
+   the boosting functionality depends on the load of the whole package,
+   single thread performance can vary. By explicitly disabling the boost
+   functionality at least for the benchmark's run-time the system will run
+   at a fixed frequency and results are reproducible again.
+2. To examine the impact of the boosting functionality it is helpful
+   to do tests with and without boosting.
+3. Boosting means overclocking the processor, though under controlled
+   conditions. By raising the frequency and the voltage the processor
+   will consume more power than without the boosting, which may be
+   undesirable for instance for mobile users. Disabling boosting may
+   save power here, though this depends on the workload.
+
+
+User controlled switch
+----------------------
+
+To allow the user to toggle the boosting functionality, the acpi-cpufreq
+driver exports a sysfs knob to disable it. There is a file:
+/sys/devices/system/cpu/cpufreq/boost
+which can either read "0" (boosting disabled) or "1" (boosting enabled).
+Reading the file is always supported, even if the processor does not
+support boosting. In this case the file will be read-only and always
+reads as "0". Explicitly changing the permissions and writing to that
+file anyway will return EINVAL.
+
+On supported CPUs one can write either a "0" or a "1" into this file.
+This will either disable the boost functionality on all cores in the
+whole system (0) or will allow the hardware to boost at will (1).
+
+Writing a "1" does not explicitly boost the system, but just allows the
+CPU (and the firmware) to boost at their discretion. Some implementations
+take external factors like the chip's temperature into account, so
+boosting once does not necessarily mean that it will occur every time
+even using the exact same software setup.
+
+
+AMD legacy cpb switch
+---------------------
+The AMD powernow-k8 driver used to support a very similar switch to
+disable or enable the "Core Performance Boost" feature of some AMD CPUs.
+This switch was instantiated in each CPU's cpufreq directory
+(/sys/devices/system/cpu[0-9]*/cpufreq) and was called "cpb".
+Though the per CPU existence hints at a more fine grained control, the
+actual implementation only supported a system-global switch semantics,
+which was simply reflected into each CPU's file. Writing a 0 or 1 into it
+would pull the other CPUs to the same state.
+For compatibility reasons this file and its behavior is still supported
+on AMD CPUs, though it is now protected by a config switch
+(X86_ACPI_CPUFREQ_CPB). On Intel CPUs this file will never be created,
+even with the config option set.
+This functionality is considered legacy and will be removed in some future
+kernel version.
+
+More fine grained boosting control
+----------------------------------
+
+Technically it is possible to switch the boosting functionality at least
+on a per package basis, for some CPUs even per core. Currently the driver
+does not support it, but this may be implemented in the future.
diff --git a/Documentation/cpuidle/sysfs.txt b/Documentation/cpuidle/sysfs.txt
index 9d28a3406e7..b6f44f490ed 100644
--- a/Documentation/cpuidle/sysfs.txt
+++ b/Documentation/cpuidle/sysfs.txt
@@ -76,9 +76,17 @@ total 0
 
 
 * desc : Small description about the idle state (string)
-* disable : Option to disable this idle state (bool)
+* disable : Option to disable this idle state (bool) -> see note below
 * latency : Latency to exit out of this idle state (in microseconds)
 * name : Name of the idle state (string)
 * power : Power consumed while in this idle state (in milliwatts)
 * time : Total time spent in this idle state (in microseconds)
 * usage : Number of times this state was entered (count)
+
+Note:
+The behavior and the effect of the disable variable depends on the
+implementation of a particular governor. In the ladder governor, for
+example, it is not coherent, i.e. if one is disabling a light state,
+then all deeper states are disabled as well, but the disable variable
+does not reflect it. Likewise, if one enables a deep state but a lighter
+state still is disabled, then this has no effect.
diff --git a/Documentation/devicetree/bindings/arm/calxeda/combophy.txt b/Documentation/devicetree/bindings/arm/calxeda/combophy.txt
new file mode 100644
index 00000000000..6622bdb2e8b
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/calxeda/combophy.txt
@@ -0,0 +1,17 @@
+Calxeda Highbank Combination Phys for SATA
+
+Properties:
+- compatible : Should be "calxeda,hb-combophy"
+- #phy-cells: Should be 1.
+- reg : Address and size for Combination Phy registers.
+- phydev: device ID for programming the combophy.
+
+Example:
+
+	combophy5: combo-phy@fff5d000 {
+		compatible = "calxeda,hb-combophy";
+		#phy-cells = <1>;
+		reg = <0xfff5d000 0x1000>;
+		phydev = <31>;
+	};
+
diff --git a/Documentation/devicetree/bindings/ata/ahci-platform.txt b/Documentation/devicetree/bindings/ata/ahci-platform.txt
index 8bb8a76d42e..b519f9b699c 100644
--- a/Documentation/devicetree/bindings/ata/ahci-platform.txt
+++ b/Documentation/devicetree/bindings/ata/ahci-platform.txt
@@ -8,9 +8,18 @@ Required properties:
 - interrupts        : <interrupt mapping for SATA IRQ>
 - reg               : <registers mapping>
 
+Optional properties:
+- calxeda,port-phys: phandle-combophy and lane assignment, which maps each
+			SATA port to a combophy and a lane within that
+			combophy
+- dma-coherent      : Present if dma operations are coherent
+
 Example:
         sata@ffe08000 {
 		compatible = "calxeda,hb-ahci";
                 reg = <0xffe08000 0x1000>;
                 interrupts = <115>;
+		calxeda,port-phys = <&combophy5 0 &combophy0 0 &combophy0 1
+					&combophy0 2 &combophy0 3>;
+
         };
diff --git a/Documentation/devicetree/bindings/ata/pata-arasan.txt b/Documentation/devicetree/bindings/ata/pata-arasan.txt
new file mode 100644
index 00000000000..95ec7f825ed
--- /dev/null
+++ b/Documentation/devicetree/bindings/ata/pata-arasan.txt
@@ -0,0 +1,17 @@
+* ARASAN PATA COMPACT FLASH CONTROLLER
+
+Required properties:
+- compatible: "arasan,cf-spear1340"
+- reg: Address range of the CF registers
+- interrupt-parent: Should be the phandle for the interrupt controller
+  that services interrupts for this device
+- interrupt: Should contain the CF interrupt number
+
+Example:
+
+	cf@fc000000 {
+		compatible = "arasan,cf-spear1340";
+		reg = <0xfc000000 0x1000>;
+		interrupt-parent = <&vic1>;
+		interrupts = <12>;
+	};
diff --git a/Documentation/devicetree/bindings/cpufreq/cpufreq-cpu0.txt b/Documentation/devicetree/bindings/cpufreq/cpufreq-cpu0.txt
new file mode 100644
index 00000000000..4416ccc3347
--- /dev/null
+++ b/Documentation/devicetree/bindings/cpufreq/cpufreq-cpu0.txt
@@ -0,0 +1,55 @@
+Generic CPU0 cpufreq driver
+
+It is a generic cpufreq driver for CPU0 frequency management.  It
+supports both uniprocessor (UP) and symmetric multiprocessor (SMP)
+systems which share clock and voltage across all CPUs.
+
+Both required and optional properties listed below must be defined
+under node /cpus/cpu@0.
+
+Required properties:
+- operating-points: Refer to Documentation/devicetree/bindings/power/opp.txt
+  for details
+
+Optional properties:
+- clock-latency: Specify the possible maximum transition latency for clock,
+  in unit of nanoseconds.
+- voltage-tolerance: Specify the CPU voltage tolerance in percentage.
+
+Examples:
+
+cpus {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	cpu@0 {
+		compatible = "arm,cortex-a9";
+		reg = <0>;
+		next-level-cache = <&L2>;
+		operating-points = <
+			/* kHz    uV */
+			792000  1100000
+			396000  950000
+			198000  850000
+		>;
+		transition-latency = <61036>; /* two CLK32 periods */
+	};
+
+	cpu@1 {
+		compatible = "arm,cortex-a9";
+		reg = <1>;
+		next-level-cache = <&L2>;
+	};
+
+	cpu@2 {
+		compatible = "arm,cortex-a9";
+		reg = <2>;
+		next-level-cache = <&L2>;
+	};
+
+	cpu@3 {
+		compatible = "arm,cortex-a9";
+		reg = <3>;
+		next-level-cache = <&L2>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
index bf57ecd5d73..bd7ce120bc1 100644
--- a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
+++ b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@@ -9,6 +9,7 @@ Copyright (C) 2008-2011 Freescale Semiconductor Inc.
    -Run Time Integrity Check (RTIC) Node
    -Run Time Integrity Check (RTIC) Memory Node
    -Secure Non-Volatile Storage (SNVS) Node
+   -Secure Non-Volatile Storage (SNVS) Low Power (LP) RTC Node
    -Full Example
 
 NOTE: the SEC 4 is also known as Freescale's Cryptographic Accelerator
@@ -294,6 +295,27 @@ Secure Non-Volatile Storage (SNVS) Node
           address and length of the SEC4 configuration
           registers.
 
+   - #address-cells
+       Usage: required
+       Value type: <u32>
+       Definition: A standard property.  Defines the number of cells
+           for representing physical addresses in child nodes.  Must
+           have a value of 1.
+
+   - #size-cells
+       Usage: required
+       Value type: <u32>
+       Definition: A standard property.  Defines the number of cells
+           for representing the size of physical addresses in
+           child nodes.  Must have a value of 1.
+
+   - ranges
+       Usage: required
+       Value type: <prop-encoded-array>
+       Definition: A standard property.  Specifies the physical address
+           range of the SNVS register space.  A triplet that includes
+           the child address, parent address, & length.
+
    - interrupts
       Usage: required
       Value type: <prop_encoded-array>
@@ -314,11 +336,34 @@ EXAMPLE
 	sec_mon@314000 {
 		compatible = "fsl,sec-v4.0-mon";
 		reg = <0x314000 0x1000>;
+		ranges = <0 0x314000 0x1000>;
 		interrupt-parent = <&mpic>;
 		interrupts = <93 2>;
 	};
 
 =====================================================================
+Secure Non-Volatile Storage (SNVS) Low Power (LP) RTC Node
+
+  A SNVS child node that defines SNVS LP RTC.
+
+  - compatible
+      Usage: required
+      Value type: <string>
+      Definition: Must include "fsl,sec-v4.0-mon-rtc-lp".
+
+  - reg
+      Usage: required
+      Value type: <prop-encoded-array>
+      Definition: A standard property.  Specifies the physical
+          address and length of the SNVS LP configuration registers.
+
+EXAMPLE
+	sec_mon_rtc_lp@314000 {
+		compatible = "fsl,sec-v4.0-mon-rtc-lp";
+		reg = <0x34 0x58>;
+	};
+
+=====================================================================
 FULL EXAMPLE
 
 	crypto: crypto@300000 {
@@ -390,8 +435,14 @@ FULL EXAMPLE
 	sec_mon: sec_mon@314000 {
 		compatible = "fsl,sec-v4.0-mon";
 		reg = <0x314000 0x1000>;
+		ranges = <0 0x314000 0x1000>;
 		interrupt-parent = <&mpic>;
 		interrupts = <93 2>;
+
+		sec_mon_rtc_lp@34 {
+			compatible = "fsl,sec-v4.0-mon-rtc-lp";
+			reg = <0x34 0x58>;
+		};
 	};
 
 =====================================================================
diff --git a/Documentation/devicetree/bindings/dma/arm-pl330.txt b/Documentation/devicetree/bindings/dma/arm-pl330.txt
index a4cd273b2a6..36e27d54260 100644
--- a/Documentation/devicetree/bindings/dma/arm-pl330.txt
+++ b/Documentation/devicetree/bindings/dma/arm-pl330.txt
@@ -9,6 +9,9 @@ Required properties:
     region.
   - interrupts: interrupt number to the cpu.
 
+Optional properties:
+- dma-coherent      : Present if dma operations are coherent
+
 Example:
 
 	pdma0: pdma@12680000 {
diff --git a/Documentation/devicetree/bindings/interrupt-controller/interrupts.txt b/Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
new file mode 100644
index 00000000000..72a06c0ab1d
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/interrupts.txt
@@ -0,0 +1,95 @@
+Specifying interrupt information for devices
+============================================
+
+1) Interrupt client nodes
+-------------------------
+
+Nodes that describe devices which generate interrupts must contain an
+"interrupts" property. This property must contain a list of interrupt
+specifiers, one per output interrupt. The format of the interrupt specifier is
+determined by the interrupt controller to which the interrupts are routed; see
+section 2 below for details.
+
+The "interrupt-parent" property is used to specify the controller to which
+interrupts are routed and contains a single phandle referring to the interrupt
+controller node. This property is inherited, so it may be specified in an
+interrupt client node or in any of its parent nodes.
+
+2) Interrupt controller nodes
+-----------------------------
+
+A device is marked as an interrupt controller with the "interrupt-controller"
+property. This is a empty, boolean property. An additional "#interrupt-cells"
+property defines the number of cells needed to specify a single interrupt.
+
+It is the responsibility of the interrupt controller's binding to define the
+length and format of the interrupt specifier. The following two variants are
+commonly used:
+
+  a) one cell
+  -----------
+  The #interrupt-cells property is set to 1 and the single cell defines the
+  index of the interrupt within the controller.
+
+  Example:
+
+	vic: intc@10140000 {
+		compatible = "arm,versatile-vic";
+		interrupt-controller;
+		#interrupt-cells = <1>;
+		reg = <0x10140000 0x1000>;
+	};
+
+	sic: intc@10003000 {
+		compatible = "arm,versatile-sic";
+		interrupt-controller;
+		#interrupt-cells = <1>;
+		reg = <0x10003000 0x1000>;
+		interrupt-parent = <&vic>;
+		interrupts = <31>; /* Cascaded to vic */
+	};
+
+  b) two cells
+  ------------
+  The #interrupt-cells property is set to 2 and the first cell defines the
+  index of the interrupt within the controller, while the second cell is used
+  to specify any of the following flags:
+    - bits[3:0] trigger type and level flags
+        1 = low-to-high edge triggered
+        2 = high-to-low edge triggered
+        4 = active high level-sensitive
+        8 = active low level-sensitive
+
+  Example:
+
+	i2c@7000c000 {
+		gpioext: gpio-adnp@41 {
+			compatible = "ad,gpio-adnp";
+			reg = <0x41>;
+
+			interrupt-parent = <&gpio>;
+			interrupts = <160 1>;
+
+			gpio-controller;
+			#gpio-cells = <1>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			nr-gpios = <64>;
+		};
+
+		sx8634@2b {
+			compatible = "smtc,sx8634";
+			reg = <0x2b>;
+
+			interrupt-parent = <&gpioext>;
+			interrupts = <3 0x8>;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			threshold = <0x40>;
+			sensitivity = <7>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/mfd/88pm860x.txt b/Documentation/devicetree/bindings/mfd/88pm860x.txt
new file mode 100644
index 00000000000..63f3ee33759
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/88pm860x.txt
@@ -0,0 +1,85 @@
+* Marvell 88PM860x Power Management IC
+
+Required parent device properties:
+- compatible : "marvell,88pm860x"
+- reg : the I2C slave address for the 88pm860x chip
+- interrupts : IRQ line for the 88pm860x chip
+- interrupt-controller: describes the 88pm860x as an interrupt controller (has its own domain)
+- #interrupt-cells : should be 1.
+		- The cell is the 88pm860x local IRQ number
+
+Optional parent device properties:
+- marvell,88pm860x-irq-read-clr: inicates whether interrupt status is cleared by read
+- marvell,88pm860x-slave-addr: 88pm860x are two chips solution. <reg> stores the I2C address
+				of one chip, and this property stores the I2C address of
+				another chip.
+
+88pm860x consists of a large and varied group of sub-devices:
+
+Device			 Supply Names	 Description
+------			 ------------	 -----------
+88pm860x-onkey		:		: On key
+88pm860x-rtc		:		: RTC
+88pm8607		:		: Regulators
+88pm860x-backlight	:		: Backlight
+88pm860x-led		:		: Led
+88pm860x-touch		:		: Touchscreen
+
+Example:
+
+	pmic: 88pm860x@34 {
+		compatible = "marvell,88pm860x";
+		reg = <0x34>;
+		interrupts = <4>;
+		interrupt-parent = <&intc>;
+		interrupt-controller;
+		#interrupt-cells = <1>;
+
+		marvell,88pm860x-irq-read-clr;
+		marvell,88pm860x-slave-addr = <0x11>;
+
+		regulators {
+			BUCK1 {
+				regulator-min-microvolt = <1000000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+			LDO1 {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+		rtc {
+			marvell,88pm860x-vrtc = <1>;
+		};
+		touch {
+			marvell,88pm860x-gpadc-prebias = <1>;
+			marvell,88pm860x-gpadc-slot-cycle = <1>;
+			marvell,88pm860x-tsi-prebias = <6>;
+			marvell,88pm860x-pen-prebias = <16>;
+			marvell,88pm860x-pen-prechg = <2>;
+			marvell,88pm860x-resistor-X = <300>;
+		};
+		backlights {
+			backlight-0 {
+				marvell,88pm860x-iset = <4>;
+				marvell,88pm860x-pwm = <3>;
+			};
+			backlight-2 {
+			};
+		};
+		leds {
+			led0-red {
+				marvell,88pm860x-iset = <12>;
+			};
+			led0-green {
+				marvell,88pm860x-iset = <12>;
+			};
+			led0-blue {
+				marvell,88pm860x-iset = <12>;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/mfd/syscon.txt b/Documentation/devicetree/bindings/mfd/syscon.txt
new file mode 100644
index 00000000000..fe8150bb324
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/syscon.txt
@@ -0,0 +1,20 @@
+* System Controller Registers R/W driver
+
+System controller node represents a register region containing a set
+of miscellaneous registers. The registers are not cohesive enough to
+represent as any specific type of device. The typical use-case is for
+some other node's driver, or platform-specific code, to acquire a
+reference to the syscon node (e.g. by phandle, node path, or search
+using a specific compatible value), interrogate the node (or associated
+OS driver) to determine the location of the registers, and access the
+registers directly.
+
+Required properties:
+- compatible: Should contain "syscon".
+- reg: the register region can be accessed from syscon
+
+Examples:
+gpr: iomuxc-gpr@020e0000 {
+	compatible = "fsl,imx6q-iomuxc-gpr", "syscon";
+	reg = <0x020e0000 0x38>;
+};
diff --git a/Documentation/devicetree/bindings/mfd/tps65910.txt b/Documentation/devicetree/bindings/mfd/tps65910.txt
index db03599ae4d..2e3304888ff 100644
--- a/Documentation/devicetree/bindings/mfd/tps65910.txt
+++ b/Documentation/devicetree/bindings/mfd/tps65910.txt
@@ -59,6 +59,8 @@ Optional properties:
   in TPS6591X datasheet)
 - ti,en-gpio-sleep: enable sleep control for gpios
   There should be 9 entries here, one for each gpio.
+- ti,system-power-controller: Telling whether or not this pmic is controlling
+  the system power.
 
 Regulator Optional properties:
 - ti,regulator-ext-sleep-control: enable external sleep
@@ -79,6 +81,8 @@ Example:
 		#interrupt-cells = <2>;
 		interrupt-controller;
 
+		ti,system-power-controller;
+
 		ti,vmbch-threshold = 0;
 		ti,vmbch2-threshold = 0;
 		ti,en-ck32k-xtal;
diff --git a/Documentation/devicetree/bindings/mfd/twl4030-audio.txt b/Documentation/devicetree/bindings/mfd/twl4030-audio.txt
new file mode 100644
index 00000000000..414d2ae0adf
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/twl4030-audio.txt
@@ -0,0 +1,46 @@
+Texas Instruments TWL family (twl4030) audio module
+
+The audio module inside the TWL family consist of an audio codec and a vibra
+driver.
+
+Required properties:
+- compatible : must be "ti,twl4030-audio"
+
+Optional properties, nodes:
+
+Audio functionality:
+- codec { }: Need to be present if the audio functionality is used. Within this
+	     section the following options can be used:
+- ti,digimic_delay: Delay need after enabling the digimic to reduce artifacts
+		    from the start of the recorded sample (in ms)
+-ti,ramp_delay_value: HS ramp delay configuration to reduce pop noise
+-ti,hs_extmute: Use external mute for HS pop reduction
+-ti,hs_extmute_gpio: Use external GPIO to control the external mute
+-ti,offset_cncl_path: Offset cancellation path selection, refer to TRM for the
+		      valid values.
+
+Vibra functionality
+- ti,enable-vibra: Need to be set to <1> if the vibra functionality is used. if
+		   missing or it is 0, the vibra functionality is disabled.
+
+Example:
+&i2c1 {
+	clock-frequency = <2600000>;
+
+	twl: twl@48 {
+		reg = <0x48>;
+		interrupts = <7>; /* SYS_NIRQ cascaded to intc */
+		interrupt-parent = <&intc>;
+
+		twl_audio: audio {
+			compatible = "ti,twl4030-audio";
+
+			ti,enable-vibra = <1>;
+
+			codec {
+				ti,ramp_delay_value = <3>;
+			};
+
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/mfd/twl6040.txt b/Documentation/devicetree/bindings/mfd/twl6040.txt
index c855240f3a0..0f5dd709d75 100644
--- a/Documentation/devicetree/bindings/mfd/twl6040.txt
+++ b/Documentation/devicetree/bindings/mfd/twl6040.txt
@@ -1,7 +1,7 @@
 Texas Instruments TWL6040 family
 
-The TWL6040s are 8-channel high quality low-power audio codecs providing audio
-and vibra functionality on OMAP4+ platforms.
+The TWL6040s are 8-channel high quality low-power audio codecs providing audio,
+vibra and GPO functionality on OMAP4+ platforms.
 They are connected ot the host processor via i2c for commands, McPDM for audio
 data and commands.
 
@@ -10,6 +10,8 @@ Required properties:
 - reg: must be 0x4b for i2c address
 - interrupts: twl6040 has one interrupt line connecteded to the main SoC
 - interrupt-parent: The parent interrupt controller
+- gpio-controller:
+- #gpio-cells = <1>: twl6040 provides GPO lines.
 - twl6040,audpwron-gpio: Power on GPIO line for the twl6040
 
 - vio-supply: Regulator for the twl6040 VIO supply
@@ -37,7 +39,6 @@ Example:
 &i2c1 {
 	twl6040: twl@4b {
 		compatible = "ti,twl6040";
-		reg = <0x4b>;
 
 		interrupts = <0 119 4>;
 		interrupt-parent = <&gic>;
@@ -60,3 +61,5 @@ Example:
 		};
 	};
 };
+
+/include/ "twl6040.dtsi"
diff --git a/Documentation/devicetree/bindings/net/calxeda-xgmac.txt b/Documentation/devicetree/bindings/net/calxeda-xgmac.txt
index 411727a3f82..c8ae996bd8f 100644
--- a/Documentation/devicetree/bindings/net/calxeda-xgmac.txt
+++ b/Documentation/devicetree/bindings/net/calxeda-xgmac.txt
@@ -6,6 +6,9 @@ Required properties:
 - interrupts : Should contain 3 xgmac interrupts. The 1st is main interrupt.
   The 2nd is pwr mgt interrupt. The 3rd is low power state interrupt.
 
+Optional properties:
+- dma-coherent      : Present if dma operations are coherent
+
 Example:
 
 ethernet@fff50000 {
diff --git a/Documentation/devicetree/bindings/power/opp.txt b/Documentation/devicetree/bindings/power/opp.txt
new file mode 100644
index 00000000000..74499e5033f
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/opp.txt
@@ -0,0 +1,25 @@
+* Generic OPP Interface
+
+SoCs have a standard set of tuples consisting of frequency and
+voltage pairs that the device will support per voltage domain. These
+are called Operating Performance Points or OPPs.
+
+Properties:
+- operating-points: An array of 2-tuples items, and each item consists
+  of frequency and voltage like <freq-kHz vol-uV>.
+	freq: clock frequency in kHz
+	vol: voltage in microvolt
+
+Examples:
+
+cpu@0 {
+	compatible = "arm,cortex-a9";
+	reg = <0>;
+	next-level-cache = <&L2>;
+	operating-points = <
+		/* kHz    uV */
+		792000  1100000
+		396000  950000
+		198000  850000
+	>;
+};
diff --git a/Documentation/devicetree/bindings/pwm/mxs-pwm.txt b/Documentation/devicetree/bindings/pwm/mxs-pwm.txt
index b16f4a57d11..11963e4d6bc 100644
--- a/Documentation/devicetree/bindings/pwm/mxs-pwm.txt
+++ b/Documentation/devicetree/bindings/pwm/mxs-pwm.txt
@@ -11,7 +11,7 @@ Example:
 
 pwm: pwm@80064000 {
 	compatible = "fsl,imx28-pwm", "fsl,imx23-pwm";
-	reg = <0x80064000 2000>;
+	reg = <0x80064000 0x2000>;
 	#pwm-cells = <2>;
 	fsl,pwm-number = <8>;
 };
diff --git a/Documentation/devicetree/bindings/regulator/88pm860x.txt b/Documentation/devicetree/bindings/regulator/88pm860x.txt
new file mode 100644
index 00000000000..1267b3e1a2c
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/88pm860x.txt
@@ -0,0 +1,30 @@
+Marvell 88PM860x regulator
+
+Required properties:
+- compatible: "marvell,88pm860x"
+- reg: I2C slave address
+- regulators: A node that houses a sub-node for each regulator within the
+  device. Each sub-node is identified using the regulator-compatible
+  property, with valid values listed below.
+
+Example:
+
+	pmic: 88pm860x@34 {
+		compatible = "marvell,88pm860x";
+		reg = <0x34>;
+
+		regulators {
+			BUCK1 {
+			        regulator-min-microvolt = <1000000>;
+			        regulator-max-microvolt = <1500000>;
+			        regulator-boot-on;
+			        regulator-always-on;
+			};
+			BUCK3 {
+			        regulator-min-microvolt = <1000000>;
+			        regulator-max-microvolt = <3000000>;
+			        regulator-boot-on;
+			        regulator-always-on;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/regulator/max8907.txt b/Documentation/devicetree/bindings/regulator/max8907.txt
new file mode 100644
index 00000000000..371eccd1cd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/max8907.txt
@@ -0,0 +1,69 @@
+MAX8907 regulator
+
+Required properties:
+- compatible: "maxim,max8907"
+- reg: I2C slave address
+- interrupts: The interrupt output of the controller
+- mbatt-supply: The input supply for MBATT, BBAT, SDBY, VRTC.
+- in-v1-supply: The input supply for SD1.
+- in-v2-supply: The input supply for SD2.
+- in-v3-supply: The input supply for SD3.
+- in1-supply: The input supply for LDO1.
+...
+- in20-supply: The input supply for LDO20.
+- regulators: A node that houses a sub-node for each regulator within the
+  device. Each sub-node is identified using the node's name (or the deprecated
+  regulator-compatible property if present), with valid values listed below.
+  The content of each sub-node is defined by the standard binding for
+  regulators; see regulator.txt.
+
+Optional properties:
+- maxim,system-power-controller: Boolean property indicating that the PMIC
+  controls the overall system power.
+
+The valid names for regulators are:
+
+  sd1, sd2, sd3, ldo1, ldo2, ldo3, ldo4, ldo5, ldo6, ldo7, ldo8, ldo9, ldo10,
+  ldo11, ldo12, ldo13, ldo14, ldo15, ldo16, ldo17, ldo18, ldo19, ldo20, out5v,
+  out33v, bbat, sdby, vrtc.
+
+Example:
+
+		max8907@3c {
+			compatible = "maxim,max8907";
+			reg = <0x3c>;
+			interrupts = <0 86 0x4>;
+
+			maxim,system-power-controller;
+
+			mbatt-supply = <&some_reg>;
+			in-v1-supply = <&mbatt_reg>;
+			...
+			in1-supply = <&mbatt_reg>;
+			...
+
+			regulators {
+				mbatt_reg: mbatt {
+					regulator-name = "vbat_pmu";
+					regulator-min-microvolt = <5000000>;
+					regulator-max-microvolt = <5000000>;
+					regulator-always-on;
+				};
+
+				sd1 {
+					regulator-name = "nvvdd_sv1,vdd_cpu_pmu";
+					regulator-min-microvolt = <1000000>;
+					regulator-max-microvolt = <1000000>;
+					regulator-always-on;
+				};
+
+				sd2 {
+					regulator-name = "nvvdd_sv2,vdd_core";
+					regulator-min-microvolt = <1200000>;
+					regulator-max-microvolt = <1200000>;
+					regulator-always-on;
+				};
+...
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/regulator/tps6586x.txt b/Documentation/devicetree/bindings/regulator/tps6586x.txt
index 07b9ef6e49d..8b40cac24d9 100644
--- a/Documentation/devicetree/bindings/regulator/tps6586x.txt
+++ b/Documentation/devicetree/bindings/regulator/tps6586x.txt
@@ -22,6 +22,10 @@ Required properties:
 - vinldo678-supply: The input supply for the LDO6, LDO7 and LDO8
 - vinldo9-supply: The input supply for the LDO9
 
+Optional properties:
+- ti,system-power-controller: Telling whether or not this pmic is controlling
+  the system power.
+
 Each regulator is defined using the standard binding for regulators.
 
 Note: LDO5 and LDO_RTC is supplied by SYS regulator internally and driver
@@ -37,6 +41,8 @@ Example:
 		#gpio-cells = <2>;
 		gpio-controller;
 
+		ti,system-power-controller;
+
 		sys-supply = <&some_reg>;
 		vin-sm0-supply = <&some_reg>;
 		vin-sm1-supply = <&some_reg>;
diff --git a/Documentation/devicetree/bindings/rtc/snvs-rtc.txt b/Documentation/devicetree/bindings/rtc/snvs-rtc.txt
new file mode 100644
index 00000000000..fb61ed77ada
--- /dev/null
+++ b/Documentation/devicetree/bindings/rtc/snvs-rtc.txt
@@ -0,0 +1 @@
+See Documentation/devicetree/bindings/crypto/fsl-sec4.txt for details.
diff --git a/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt b/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt
new file mode 100644
index 00000000000..c58573b5b1a
--- /dev/null
+++ b/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt
@@ -0,0 +1,35 @@
+* Freescale i.MX UART controller
+
+Required properties:
+- compatible : should be "fsl,imx21-uart"
+- reg : Address and length of the register set for the device
+- interrupts : Should contain UART interrupt number
+
+Optional properties:
+- fsl,uart-has-rtscts: indicate that RTS/CTS signals are used
+
+Note: Each uart controller should have an alias correctly numbered
+in "aliases" node.
+
+Example:
+
+- From imx51.dtsi:
+aliases {
+	serial0 = &uart1;
+	serial1 = &uart2;
+	serial2 = &uart3;
+};
+
+uart1: serial@73fbc000 {
+	compatible = "fsl,imx51-uart", "fsl,imx21-uart";
+	reg = <0x73fbc000 0x4000>;
+	interrupts = <31>;
+	status = "disabled";
+}
+
+- From imx51-babbage.dts:
+uart1: serial@73fbc000 {
+	fsl,uart-has-rtscts;
+	status = "okay";
+};
+
diff --git a/Documentation/devicetree/bindings/video/backlight/88pm860x.txt b/Documentation/devicetree/bindings/video/backlight/88pm860x.txt
new file mode 100644
index 00000000000..261df279931
--- /dev/null
+++ b/Documentation/devicetree/bindings/video/backlight/88pm860x.txt
@@ -0,0 +1,15 @@
+88pm860x-backlight bindings
+
+Optional properties:
+  - marvell,88pm860x-iset: Current supplies on backlight device.
+  - marvell,88pm860x-pwm: PWM frequency on backlight device.
+
+Example:
+
+	backlights {
+		backlight-0 {
+			marvell,88pm860x-iset = <4>;
+			marvell,88pm860x-pwm = <3>;
+		};
+		backlight-2 {
+		};
diff --git a/Documentation/filesystems/jfs.txt b/Documentation/filesystems/jfs.txt
index 26ebde77e82..f7433355394 100644
--- a/Documentation/filesystems/jfs.txt
+++ b/Documentation/filesystems/jfs.txt
@@ -3,6 +3,7 @@ IBM's Journaled File System (JFS) for Linux
 JFS Homepage:  http://jfs.sourceforge.net/
 
 The following mount options are supported:
+(*) == default
 
 iocharset=name	Character set to use for converting from Unicode to
 		ASCII.  The default is to do no conversion.  Use
@@ -21,12 +22,12 @@ nointegrity	Do not write to the journal.  The primary use of this option
 		from backup media.  The integrity of the volume is not
 		guaranteed if the system abnormally abends.
 
-integrity	Default.  Commit metadata changes to the journal.  Use this
-		option to remount a volume where the nointegrity option was
+integrity(*)	Commit metadata changes to the journal.  Use this option to
+		remount a volume where the nointegrity option was
 		previously specified in order to restore normal behavior.
 
 errors=continue		Keep going on a filesystem error.
-errors=remount-ro	Default. Remount the filesystem read-only on an error.
+errors=remount-ro(*)	Remount the filesystem read-only on an error.
 errors=panic		Panic and halt the machine if an error occurs.
 
 uid=value	Override on-disk uid with specified value
@@ -35,7 +36,17 @@ umask=value	Override on-disk umask with specified octal value.  For
 		directories, the execute bit will be set if the corresponding
 		read bit is set.
 
-Please send bugs, comments, cards and letters to shaggy@linux.vnet.ibm.com.
+discard=minlen	This enables/disables the use of discard/TRIM commands.
+discard		The discard/TRIM commands are sent to the underlying
+nodiscard(*)	block device when blocks are freed. This is useful for SSD
+		devices and sparse/thinly-provisioned LUNs.  The FITRIM ioctl
+		command is also available together with the nodiscard option.
+		The value of minlen specifies the minimum blockcount, when
+		a TRIM command to the block device is considered usefull.
+		When no value is given to the discard option, it defaults to
+		64 blocks, which means 256KiB in JFS.
+		The minlen value of discard overrides the minlen value given
+		on an FITRIM ioctl().
 
 The JFS mailing list can be subscribed to by using the link labeled
 "Mail list Subscribe" at our web page http://jfs.sourceforge.net/
diff --git a/Documentation/kbuild/makefiles.txt b/Documentation/kbuild/makefiles.txt
index ab0a984530d..ec9ae670869 100644
--- a/Documentation/kbuild/makefiles.txt
+++ b/Documentation/kbuild/makefiles.txt
@@ -45,7 +45,7 @@ This document describes the Linux kernel Makefiles.
 
 	=== 7 Kbuild syntax for exported headers
 		--- 7.1 header-y
-		--- 7.2 objhdr-y
+		--- 7.2 genhdr-y
 		--- 7.3 destination-y
 		--- 7.4 generic-y
 
@@ -1282,15 +1282,15 @@ See subsequent chapter for the syntax of the Kbuild file.
 
 	Subdirectories are visited before their parent directories.
 
-	--- 7.2 objhdr-y
+	--- 7.2 genhdr-y
 
-	objhdr-y specifies generated files to be exported.
+	genhdr-y specifies generated files to be exported.
 	Generated files are special as they need to be looked
 	up in another directory when doing 'make O=...' builds.
 
 		Example:
 			#include/linux/Kbuild
-			objhdr-y += version.h
+			genhdr-y += version.h
 
 	--- 7.3 destination-y
 
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index df43807bb5d..f777fa96243 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -1051,6 +1051,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
 	ihash_entries=	[KNL]
 			Set number of hash buckets for inode cache.
 
+	ima_appraise=	[IMA] appraise integrity measurements
+			Format: { "off" | "enforce" | "fix" }
+			default: "enforce"
+
+	ima_appraise_tcb [IMA]
+			The builtin appraise policy appraises all files
+			owned by uid=0.
+
 	ima_audit=	[IMA]
 			Format: { "0" | "1" }
 			0 -- integrity auditing messages. (Default)
@@ -1350,6 +1358,9 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
 			* nohrst, nosrst, norst: suppress hard, soft
                           and both resets.
 
+			* rstonce: only attempt one reset during
+			  hot-unplug link recovery
+
 			* dump_id: dump IDENTIFY data.
 
 			If there are multiple matching configurations changing
diff --git a/Documentation/printk-formats.txt b/Documentation/printk-formats.txt
index 7561d7ed8e1..8ffb274367c 100644
--- a/Documentation/printk-formats.txt
+++ b/Documentation/printk-formats.txt
@@ -69,6 +69,7 @@ MAC/FDDI addresses:
 	%pMR	05:04:03:02:01:00
 	%pMF	00-01-02-03-04-05
 	%pm	000102030405
+	%pmR	050403020100
 
 	For printing 6-byte MAC/FDDI addresses in hex notation. The 'M' and 'm'
 	specifiers result in a printed address with ('M') or without ('m') byte
diff --git a/Documentation/remoteproc.txt b/Documentation/remoteproc.txt
index 23a09b884bc..e6469fdcf89 100644
--- a/Documentation/remoteproc.txt
+++ b/Documentation/remoteproc.txt
@@ -129,6 +129,13 @@ int dummy_rproc_example(struct rproc *my_rproc)
 
       Returns 0 on success and -EINVAL if @rproc isn't valid.
 
+  void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type)
+    - Report a crash in a remoteproc
+      This function must be called every time a crash is detected by the
+      platform specific rproc implementation. This should not be called from a
+      non-remoteproc driver. This function can be called from atomic/interrupt
+      context.
+
 5. Implementation callbacks
 
 These callbacks should be provided by platform-specific remoteproc
diff --git a/Documentation/rtc.txt b/Documentation/rtc.txt
index 250160469d8..32aa4002de4 100644
--- a/Documentation/rtc.txt
+++ b/Documentation/rtc.txt
@@ -119,8 +119,9 @@ three different userspace interfaces:
     *	/sys/class/rtc/rtcN ... sysfs attributes support readonly
 	access to some RTC attributes.
 
-    *	/proc/driver/rtc ... the first RTC (rtc0) may expose itself
-	using a procfs interface.  More information is (currently) shown
+    *	/proc/driver/rtc ... the system clock RTC may expose itself
+	using a procfs interface. If there is no RTC for the system clock,
+	rtc0 is used by default. More information is (currently) shown
 	here than through sysfs.
 
 The RTC Class framework supports a wide variety of RTCs, ranging from those
diff --git a/Documentation/scsi/ChangeLog.megaraid_sas b/Documentation/scsi/ChangeLog.megaraid_sas
index 80441ab608e..3a3079411a3 100644
--- a/Documentation/scsi/ChangeLog.megaraid_sas
+++ b/Documentation/scsi/ChangeLog.megaraid_sas
@@ -1,3 +1,13 @@
+Release Date    : Tue. Jun 17, 2012 17:00:00 PST 2012 -
+			(emaild-id:megaraidlinux@lsi.com)
+			Adam Radford/Kashyap Desai
+Current Version : 00.00.06.18-rc1
+Old Version     : 00.00.06.15-rc1
+    1. Fix Copyright dates.
+    2. Add throttlequeuedepth module parameter.
+    3. Add resetwaittime module parameter.
+    4. Move poll_aen_lock initializer.
+-------------------------------------------------------------------------------
 Release Date    : Mon. Mar 19, 2012 17:00:00 PST 2012 -
 			(emaild-id:megaraidlinux@lsi.com)
 			Adam Radford
diff --git a/Documentation/scsi/LICENSE.qla2xxx b/Documentation/scsi/LICENSE.qla2xxx
index ce0fdf349a8..27a91cf43d6 100644
--- a/Documentation/scsi/LICENSE.qla2xxx
+++ b/Documentation/scsi/LICENSE.qla2xxx
@@ -1,4 +1,4 @@
-Copyright (c) 2003-2011 QLogic Corporation
+Copyright (c) 2003-2012 QLogic Corporation
 QLogic Linux FC-FCoE Driver
 
 This program includes a device driver for Linux 3.x.
diff --git a/Documentation/scsi/LICENSE.qla4xxx b/Documentation/scsi/LICENSE.qla4xxx
index ab899591ecb..78c169f0d7c 100644
--- a/Documentation/scsi/LICENSE.qla4xxx
+++ b/Documentation/scsi/LICENSE.qla4xxx
@@ -1,4 +1,4 @@
-Copyright (c) 2003-2011 QLogic Corporation
+Copyright (c) 2003-2012 QLogic Corporation
 QLogic Linux iSCSI Driver
 
 This program includes a device driver for Linux 3.x.
diff --git a/Documentation/scsi/st.txt b/Documentation/scsi/st.txt
index 685bf3582ab..f346abbdd6f 100644
--- a/Documentation/scsi/st.txt
+++ b/Documentation/scsi/st.txt
@@ -112,10 +112,8 @@ attempted).
 
 MINOR NUMBERS
 
-The tape driver currently supports 128 drives by default. This number
-can be increased by editing st.h and recompiling the driver if
-necessary. The upper limit is 2^17 drives if 4 modes for each drive
-are used.
+The tape driver currently supports up to 2^17 drives if 4 modes for
+each drive are used.
 
 The minor numbers consist of the following bit fields:
 
diff --git a/Documentation/security/Smack.txt b/Documentation/security/Smack.txt
index a416479b8a1..8a177e4b6e2 100644
--- a/Documentation/security/Smack.txt
+++ b/Documentation/security/Smack.txt
@@ -28,12 +28,11 @@ Smack kernels use the CIPSO IP option. Some network
 configurations are intolerant of IP options and can impede
 access to systems that use them as Smack does.
 
-The current git repositories for Smack user space are:
+The current git repository for Smack user space is:
 
-	git@gitorious.org:meego-platform-security/smackutil.git
-	git@gitorious.org:meego-platform-security/libsmack.git
+	git://github.com/smack-team/smack.git
 
-These should make and install on most modern distributions.
+This should make and install on most modern distributions.
 There are three commands included in smackutil:
 
 smackload  - properly formats data for writing to /smack/load
@@ -194,6 +193,9 @@ onlycap
 	these capabilities are effective at for processes with any
 	label. The value is set by writing the desired label to the
 	file or cleared by writing "-" to the file.
+revoke-subject
+	Writing a Smack label here sets the access to '-' for all access
+	rules with that subject label.
 
 You can add access rules in /etc/smack/accesses. They take the form:
 
diff --git a/Documentation/smsc_ece1099.txt b/Documentation/smsc_ece1099.txt
new file mode 100644
index 00000000000..6b492e82b43
--- /dev/null
+++ b/Documentation/smsc_ece1099.txt
@@ -0,0 +1,56 @@
+What is smsc-ece1099?
+----------------------
+
+The ECE1099 is a 40-Pin 3.3V Keyboard Scan Expansion
+or GPIO Expansion device. The device supports a keyboard
+scan matrix of 23x8. The device is connected to a Master
+via the SMSC BC-Link interface or via the SMBus.
+Keypad scan Input(KSI) and Keypad Scan Output(KSO) signals
+are multiplexed with GPIOs.
+
+Interrupt generation
+--------------------
+
+Interrupts can be generated by an edge detection on a GPIO
+pin or an edge detection on one of the bus interface pins.
+Interrupts can also be detected on the keyboard scan interface.
+The bus interrupt pin (BC_INT# or SMBUS_INT#) is asserted if
+any bit in one of the Interrupt Status registers is 1 and
+the corresponding Interrupt Mask bit is also 1.
+
+In order for software to determine which device is the source
+of an interrupt, it should first read the Group Interrupt Status Register
+to determine which Status register group is a source for the interrupt.
+Software should read both the Status register and the associated Mask register,
+then AND the two values together. Bits that are 1 in the result of the AND
+are active interrupts. Software clears an interrupt by writing a 1 to the
+corresponding bit in the Status register.
+
+Communication Protocol
+----------------------
+
+- SMbus slave Interface
+	The host processor communicates with the ECE1099 device
+	through a series of read/write registers via the SMBus
+	interface. SMBus is a serial communication protocol between
+	a computer host and its peripheral devices. The SMBus data
+	rate is 10KHz minimum to 400 KHz maximum
+
+- Slave Bus Interface
+	The ECE1099 device SMBus implementation is a subset of the
+	SMBus interface to the host. The device is a slave-only SMBus device.
+	The implementation in the device is a subset of SMBus since it
+	only supports four protocols.
+
+	The Write Byte, Read Byte, Send Byte, and Receive Byte protocols are the
+	only valid SMBus protocols for the device.
+
+- BC-LinkTM Interface
+	The BC-Link is a proprietary bus that allows communication
+	between a Master device and a Companion device. The Master
+	device uses this serial bus to read and write registers
+	located on the Companion device. The bus comprises three signals,
+	BC_CLK, BC_DAT and BC_INT#. The Master device always provides the
+	clock, BC_CLK, and the Companion device is the source for an
+	independent asynchronous interrupt signal, BC_INT#. The ECE1099
+	supports BC-Link speeds up to 24MHz.
diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
index 6d78841fd41..2907ba6c360 100644
--- a/Documentation/sysctl/kernel.txt
+++ b/Documentation/sysctl/kernel.txt
@@ -181,6 +181,8 @@ core_pattern is used to specify a core dumpfile pattern name.
 	%p	pid
 	%u	uid
 	%g	gid
+	%d	dump mode, matches PR_SET_DUMPABLE and
+		/proc/sys/fs/suid_dumpable
 	%s	signal number
 	%t	UNIX time of dump
 	%h	hostname
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index bf33aaa4c59..f6ec3a92e62 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -857,7 +857,8 @@ struct kvm_userspace_memory_region {
 };
 
 /* for kvm_memory_region::flags */
-#define KVM_MEM_LOG_DIRTY_PAGES  1UL
+#define KVM_MEM_LOG_DIRTY_PAGES	(1UL << 0)
+#define KVM_MEM_READONLY	(1UL << 1)
 
 This ioctl allows the user to create or modify a guest physical memory
 slot.  When changing an existing slot, it may be moved in the guest
@@ -873,14 +874,17 @@ It is recommended that the lower 21 bits of guest_phys_addr and userspace_addr
 be identical.  This allows large pages in the guest to be backed by large
 pages in the host.
 
-The flags field supports just one flag, KVM_MEM_LOG_DIRTY_PAGES, which
-instructs kvm to keep track of writes to memory within the slot.  See
-the KVM_GET_DIRTY_LOG ioctl.
+The flags field supports two flag, KVM_MEM_LOG_DIRTY_PAGES, which instructs
+kvm to keep track of writes to memory within the slot.  See KVM_GET_DIRTY_LOG
+ioctl.  The KVM_CAP_READONLY_MEM capability indicates the availability of the
+KVM_MEM_READONLY flag.  When this flag is set for a memory region, KVM only
+allows read accesses.  Writes will be posted to userspace as KVM_EXIT_MMIO
+exits.
 
-When the KVM_CAP_SYNC_MMU capability, changes in the backing of the memory
-region are automatically reflected into the guest.  For example, an mmap()
-that affects the region will be made visible immediately.  Another example
-is madvise(MADV_DROP).
+When the KVM_CAP_SYNC_MMU capability is available, changes in the backing of
+the memory region are automatically reflected into the guest.  For example, an
+mmap() that affects the region will be made visible immediately.  Another
+example is madvise(MADV_DROP).
 
 It is recommended to use this API instead of the KVM_SET_MEMORY_REGION ioctl.
 The KVM_SET_MEMORY_REGION does not allow fine grained control over memory
@@ -1946,6 +1950,19 @@ the guest using the specified gsi pin.  The irqfd is removed using
 the KVM_IRQFD_FLAG_DEASSIGN flag, specifying both kvm_irqfd.fd
 and kvm_irqfd.gsi.
 
+With KVM_CAP_IRQFD_RESAMPLE, KVM_IRQFD supports a de-assert and notify
+mechanism allowing emulation of level-triggered, irqfd-based
+interrupts.  When KVM_IRQFD_FLAG_RESAMPLE is set the user must pass an
+additional eventfd in the kvm_irqfd.resamplefd field.  When operating
+in resample mode, posting of an interrupt through kvm_irq.fd asserts
+the specified gsi in the irqchip.  When the irqchip is resampled, such
+as from an EOI, the gsi is de-asserted and the user is notifed via
+kvm_irqfd.resamplefd.  It is the user's responsibility to re-queue
+the interrupt if the device making use of it still requires service.
+Note that closing the resamplefd is not sufficient to disable the
+irqfd.  The KVM_IRQFD_FLAG_RESAMPLE is only necessary on assignment
+and need not be specified with KVM_IRQFD_FLAG_DEASSIGN.
+
 4.76 KVM_PPC_ALLOCATE_HTAB
 
 Capability: KVM_CAP_PPC_ALLOC_HTAB
diff --git a/Documentation/virtual/kvm/hypercalls.txt b/Documentation/virtual/kvm/hypercalls.txt
new file mode 100644
index 00000000000..ea113b5d87a
--- /dev/null
+++ b/Documentation/virtual/kvm/hypercalls.txt
@@ -0,0 +1,66 @@
+Linux KVM Hypercall:
+===================
+X86:
+ KVM Hypercalls have a three-byte sequence of either the vmcall or the vmmcall
+ instruction. The hypervisor can replace it with instructions that are
+ guaranteed to be supported.
+
+ Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively.
+ The hypercall number should be placed in rax and the return value will be
+ placed in rax.  No other registers will be clobbered unless explicitly stated
+ by the particular hypercall.
+
+S390:
+  R2-R7 are used for parameters 1-6. In addition, R1 is used for hypercall
+  number. The return value is written to R2.
+
+  S390 uses diagnose instruction as hypercall (0x500) along with hypercall
+  number in R1.
+
+ PowerPC:
+  It uses R3-R10 and hypercall number in R11. R4-R11 are used as output registers.
+  Return value is placed in R3.
+
+  KVM hypercalls uses 4 byte opcode, that are patched with 'hypercall-instructions'
+  property inside the device tree's /hypervisor node.
+  For more information refer to Documentation/virtual/kvm/ppc-pv.txt
+
+KVM Hypercalls Documentation
+===========================
+The template for each hypercall is:
+1. Hypercall name.
+2. Architecture(s)
+3. Status (deprecated, obsolete, active)
+4. Purpose
+
+1. KVM_HC_VAPIC_POLL_IRQ
+------------------------
+Architecture: x86
+Status: active
+Purpose: Trigger guest exit so that the host can check for pending
+interrupts on reentry.
+
+2. KVM_HC_MMU_OP
+------------------------
+Architecture: x86
+Status: deprecated.
+Purpose: Support MMU operations such as writing to PTE,
+flushing TLB, release PT.
+
+3. KVM_HC_FEATURES
+------------------------
+Architecture: PPC
+Status: active
+Purpose: Expose hypercall availability to the guest. On x86 platforms, cpuid
+used to enumerate which hypercalls are available. On PPC, either device tree
+based lookup ( which is also what EPAPR dictates) OR KVM specific enumeration
+mechanism (which is this hypercall) can be used.
+
+4. KVM_HC_PPC_MAP_MAGIC_PAGE
+------------------------
+Architecture: PPC
+Status: active
+Purpose: To enable communication between the hypervisor and guest there is a
+shared page that contains parts of supervisor visible register state.
+The guest can map this shared page to access its supervisor register through
+memory using this hypercall.
diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
index 73047104858..6d470ae7b07 100644
--- a/Documentation/virtual/kvm/msr.txt
+++ b/Documentation/virtual/kvm/msr.txt
@@ -34,9 +34,12 @@ MSR_KVM_WALL_CLOCK_NEW:   0x4b564d00
 		time information and check that they are both equal and even.
 		An odd version indicates an in-progress update.
 
-		sec: number of seconds for wallclock.
+		sec: number of seconds for wallclock at time of boot.
 
-		nsec: number of nanoseconds for wallclock.
+		nsec: number of nanoseconds for wallclock at time of boot.
+
+	In order to get the current wallclock time, the system_time from
+	MSR_KVM_SYSTEM_TIME_NEW needs to be added.
 
 	Note that although MSRs are per-CPU entities, the effect of this
 	particular MSR is global.
@@ -82,20 +85,25 @@ MSR_KVM_SYSTEM_TIME_NEW:  0x4b564d01
 		time at the time this structure was last updated. Unit is
 		nanoseconds.
 
-		tsc_to_system_mul: a function of the tsc frequency. One has
-		to multiply any tsc-related quantity by this value to get
-		a value in nanoseconds, besides dividing by 2^tsc_shift
+		tsc_to_system_mul: multiplier to be used when converting
+		tsc-related quantity to nanoseconds
 
-		tsc_shift: cycle to nanosecond divider, as a power of two, to
-		allow for shift rights. One has to shift right any tsc-related
-		quantity by this value to get a value in nanoseconds, besides
-		multiplying by tsc_to_system_mul.
+		tsc_shift: shift to be used when converting tsc-related
+		quantity to nanoseconds. This shift will ensure that
+		multiplication with tsc_to_system_mul does not overflow.
+		A positive value denotes a left shift, a negative value
+		a right shift.
 
-		With this information, guests can derive per-CPU time by
-		doing:
+		The conversion from tsc to nanoseconds involves an additional
+		right shift by 32 bits. With this information, guests can
+		derive per-CPU time by doing:
 
 			time = (current_tsc - tsc_timestamp)
-			time = (time * tsc_to_system_mul) >> tsc_shift
+			if (tsc_shift >= 0)
+				time <<= tsc_shift;
+			else
+				time >>= -tsc_shift;
+			time = (time * tsc_to_system_mul) >> 32
 			time = time + system_time
 
 		flags: bits in this field indicate extended capabilities
diff --git a/Documentation/virtual/kvm/ppc-pv.txt b/Documentation/virtual/kvm/ppc-pv.txt
index 4911cf95c67..4cd076febb0 100644
--- a/Documentation/virtual/kvm/ppc-pv.txt
+++ b/Documentation/virtual/kvm/ppc-pv.txt
@@ -174,3 +174,25 @@ following:
 That way we can inject an arbitrary amount of code as replacement for a single
 instruction. This allows us to check for pending interrupts when setting EE=1
 for example.
+
+Hypercall ABIs in KVM on PowerPC
+=================================
+1) KVM hypercalls (ePAPR)
+
+These are ePAPR compliant hypercall implementation (mentioned above). Even
+generic hypercalls are implemented here, like the ePAPR idle hcall. These are
+available on all targets.
+
+2) PAPR hypercalls
+
+PAPR hypercalls are needed to run server PowerPC PAPR guests (-M pseries in QEMU).
+These are the same hypercalls that pHyp, the POWER hypervisor implements. Some of
+them are handled in the kernel, some are handled in user space. This is only
+available on book3s_64.
+
+3) OSI hypercalls
+
+Mac-on-Linux is another user of KVM on PowerPC, which has its own hypercall (long
+before KVM). This is supported to maintain compatibility. All these hypercalls get
+forwarded to user space. This is only useful on book3s_32, but can be used with
+book3s_64 as well.
diff --git a/MAINTAINERS b/MAINTAINERS
index 78336396a43..0976bd1381b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -572,7 +572,7 @@ F:	drivers/net/appletalk/
 F:	net/appletalk/
 
 ARASAN COMPACT FLASH PATA CONTROLLER
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 L:	linux-ide@vger.kernel.org
 S:	Maintained
 F:	include/linux/pata_arasan_cf_data.h
@@ -760,6 +760,7 @@ S:	Maintained
 T:	git git://git.pengutronix.de/git/imx/linux-2.6.git
 F:	arch/arm/mach-imx/
 F:	arch/arm/plat-mxc/
+F:	arch/arm/configs/imx*_defconfig
 
 ARM/FREESCALE IMX6
 M:	Shawn Guo <shawn.guo@linaro.org>
@@ -1245,7 +1246,7 @@ F:	include/linux/i2c/at24.h
 
 ATA OVER ETHERNET (AOE) DRIVER
 M:	"Ed L. Cashin" <ecashin@coraid.com>
-W:	http://www.coraid.com/support/linux
+W:	http://support.coraid.com/support/linux
 S:	Supported
 F:	Documentation/aoe/
 F:	drivers/block/aoe/
@@ -1650,7 +1651,6 @@ F:	drivers/bcma/
 F:	include/linux/bcma/
 
 BROCADE BFA FC SCSI DRIVER
-M:	Jing Huang <huangj@brocade.com>
 M:	Krishna C Gudipati <kgudipat@brocade.com>
 L:	linux-scsi@vger.kernel.org
 S:	Supported
@@ -2632,6 +2632,18 @@ T:	git git://git.alsa-project.org/alsa-kernel.git
 S:	Maintained
 F:	sound/usb/misc/ua101.c
 
+EXTENSIBLE FIRMWARE INTERFACE (EFI)
+M:	Matt Fleming <matt.fleming@intel.com>
+L:	linux-efi@vger.kernel.org
+S:	Maintained
+F:	Documentation/x86/efi-stub.txt
+F:	arch/ia64/kernel/efi.c
+F:	arch/x86/boot/compressed/eboot.[ch]
+F:	arch/x86/include/asm/efi.h
+F:	arch/x86/platform/efi/*
+F:	drivers/firmware/efivars.c
+F:	include/linux/efi*.h
+
 EFIFB FRAMEBUFFER DRIVER
 L:	linux-fbdev@vger.kernel.org
 M:	Peter Jones <pjones@redhat.com>
@@ -3112,6 +3124,7 @@ T:	git git://git.secretlab.ca/git/linux-2.6.git
 F:	Documentation/gpio.txt
 F:	drivers/gpio/
 F:	include/linux/gpio*
+F:	include/asm-generic/gpio.h
 
 GRE DEMULTIPLEXER DRIVER
 M:	Dmitry Kozlov <xeb@mail.ru>
@@ -3427,6 +3440,18 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux.git
 S:	Maintained
 F:	arch/ia64/
 
+IBM Power in-Nest Crypto Acceleration
+M:	Kent Yoder <key@linux.vnet.ibm.com>
+L:	linux-crypto@vger.kernel.org
+S:	Supported
+F:	drivers/crypto/nx/
+
+IBM Power 842 compression accelerator
+M:	Robert Jennings <rcj@linux.vnet.ibm.com>
+S:	Supported
+F:	drivers/crypto/nx/nx-842.c
+F:	include/linux/nx842.h
+
 IBM Power Linux RAID adapter
 M:	Brian King <brking@us.ibm.com>
 S:	Supported
@@ -3438,6 +3463,13 @@ L:	netdev@vger.kernel.org
 S:	Supported
 F:	drivers/net/ethernet/ibm/ibmveth.*
 
+IBM Power Virtual SCSI/FC Device Drivers
+M:	Robert Jennings <rcj@linux.vnet.ibm.com>
+L:	linux-scsi@vger.kernel.org
+S:	Supported
+F:	drivers/scsi/ibmvscsi/
+X:	drivers/scsi/ibmvscsi/ibmvstgt.c
+
 IBM ServeRAID RAID DRIVER
 P:	Jack Hammer
 M:	Dave Jeffery <ipslinux@adaptec.com>
@@ -5049,6 +5081,7 @@ S:	Maintained
 F:	Documentation/devicetree
 F:	drivers/of
 F:	include/linux/of*.h
+F:	scripts/dtc
 K:	of_get_property
 K:	of_match_table
 
@@ -5234,7 +5267,7 @@ F:	include/linux/i2c-algo-pca.h
 F:	include/linux/i2c-pca-platform.h
 
 PCDP - PRIMARY CONSOLE AND DEBUG PORT
-M:	Khalid Aziz <khalid.aziz@hp.com>
+M:	Khalid Aziz <khalid@gonehiking.org>
 S:	Maintained
 F:	drivers/firmware/pcdp.*
 
@@ -5523,7 +5556,7 @@ S:	Maintained
 W:	http://linuxptp.sourceforge.net/
 F:	Documentation/ABI/testing/sysfs-ptp
 F:	Documentation/ptp/*
-F:	drivers/net/gianfar_ptp.c
+F:	drivers/net/ethernet/freescale/gianfar_ptp.c
 F:	drivers/net/phy/dp83640*
 F:	drivers/ptp/*
 F:	include/linux/ptp_cl*
@@ -5955,7 +5988,7 @@ S:	Maintained
 F:	drivers/tty/serial
 
 SYNOPSYS DESIGNWARE DMAC DRIVER
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 S:	Maintained
 F:	include/linux/dw_dmac.h
 F:	drivers/dma/dw_dmac_regs.h
@@ -6103,7 +6136,7 @@ S:	Maintained
 F:	drivers/mmc/host/sdhci-s3c.c
 
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) ST SPEAR DRIVER
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 L:	spear-devel@list.st.com
 L:	linux-mmc@vger.kernel.org
 S:	Maintained
@@ -6468,7 +6501,7 @@ S:	Maintained
 F:	include/linux/compiler.h
 
 SPEAR PLATFORM SUPPORT
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 M:	Shiraz Hashim <shiraz.hashim@st.com>
 L:	spear-devel@list.st.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -6477,7 +6510,7 @@ S:	Maintained
 F:	arch/arm/plat-spear/
 
 SPEAR13XX MACHINE SUPPORT
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 M:	Shiraz Hashim <shiraz.hashim@st.com>
 L:	spear-devel@list.st.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -6486,7 +6519,7 @@ S:	Maintained
 F:	arch/arm/mach-spear13xx/
 
 SPEAR3XX MACHINE SUPPORT
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 M:	Shiraz Hashim <shiraz.hashim@st.com>
 L:	spear-devel@list.st.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -6497,7 +6530,7 @@ F:	arch/arm/mach-spear3xx/
 SPEAR6XX MACHINE SUPPORT
 M:	Rajeev Kumar <rajeev-dlh.kumar@st.com>
 M:	Shiraz Hashim <shiraz.hashim@st.com>
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 L:	spear-devel@list.st.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 W:	http://www.st.com/spear
@@ -6505,7 +6538,7 @@ S:	Maintained
 F:	arch/arm/mach-spear6xx/
 
 SPEAR CLOCK FRAMEWORK SUPPORT
-M:     Viresh Kumar <viresh.linux@gmail.com>
+M:	Viresh Kumar <viresh.linux@gmail.com>
 L:	spear-devel@list.st.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 W:	http://www.st.com/spear
diff --git a/Makefile b/Makefile
index 846dd760785..86eb6acb397 100644
--- a/Makefile
+++ b/Makefile
@@ -350,12 +350,22 @@ AFLAGS_KERNEL	=
 CFLAGS_GCOV	= -fprofile-arcs -ftest-coverage
 
 
+# Use USERINCLUDE when you must reference the UAPI directories only.
+USERINCLUDE    := \
+		-I$(srctree)/arch/$(hdr-arch)/include/uapi \
+		-Iarch/$(hdr-arch)/include/generated/uapi \
+		-I$(srctree)/include/uapi \
+		-Iinclude/generated/uapi \
+                -include $(srctree)/include/linux/kconfig.h
+
 # Use LINUXINCLUDE when you must reference the include/ directory.
 # Needed to be compatible with the O= option
-LINUXINCLUDE    := -I$(srctree)/arch/$(hdr-arch)/include \
-                   -Iarch/$(hdr-arch)/include/generated -Iinclude \
-                   $(if $(KBUILD_SRC), -I$(srctree)/include) \
-                   -include $(srctree)/include/linux/kconfig.h
+LINUXINCLUDE    := \
+		-I$(srctree)/arch/$(hdr-arch)/include \
+		-Iarch/$(hdr-arch)/include/generated \
+		$(if $(KBUILD_SRC), -I$(srctree)/include) \
+		-Iinclude \
+		$(USERINCLUDE)
 
 KBUILD_CPPFLAGS := -D__KERNEL__
 
@@ -437,9 +447,11 @@ asm-generic:
 # Detect when mixed targets is specified, and make a second invocation
 # of make so .config is not included in this case either (for *config).
 
+version_h := include/generated/uapi/linux/version.h
+
 no-dot-config-targets := clean mrproper distclean \
 			 cscope gtags TAGS tags help %docs check% coccicheck \
-			 include/linux/version.h headers_% archheaders archscripts \
+			 $(version_h) headers_% archheaders archscripts \
 			 kernelversion %src-pkg
 
 config-targets := 0
@@ -809,7 +821,7 @@ endif
 # prepare2 creates a makefile if using a separate output directory
 prepare2: prepare3 outputmakefile asm-generic
 
-prepare1: prepare2 include/linux/version.h include/generated/utsrelease.h \
+prepare1: prepare2 $(version_h) include/generated/utsrelease.h \
                    include/config/auto.conf
 	$(cmd_crmodverdir)
 
@@ -842,7 +854,7 @@ define filechk_version.h
 	echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))';)
 endef
 
-include/linux/version.h: $(srctree)/Makefile FORCE
+$(version_h): $(srctree)/Makefile FORCE
 	$(call filechk,version.h)
 
 include/generated/utsrelease.h: include/config/kernel.release FORCE
@@ -887,7 +899,7 @@ PHONY += archscripts
 archscripts:
 
 PHONY += __headers
-__headers: include/linux/version.h scripts_basic asm-generic archheaders archscripts FORCE
+__headers: $(version_h) scripts_basic asm-generic archheaders archscripts FORCE
 	$(Q)$(MAKE) $(build)=scripts build_unifdef
 
 PHONY += headers_install_all
@@ -896,10 +908,10 @@ headers_install_all:
 
 PHONY += headers_install
 headers_install: __headers
-	$(if $(wildcard $(srctree)/arch/$(hdr-arch)/include/asm/Kbuild),, \
-	$(error Headers not exportable for the $(SRCARCH) architecture))
-	$(Q)$(MAKE) $(hdr-inst)=include
-	$(Q)$(MAKE) $(hdr-inst)=arch/$(hdr-arch)/include/asm $(hdr-dst)
+	$(if $(wildcard $(srctree)/arch/$(hdr-arch)/include/uapi/asm/Kbuild),, \
+	  $(error Headers not exportable for the $(SRCARCH) architecture))
+	$(Q)$(MAKE) $(hdr-inst)=include/uapi
+	$(Q)$(MAKE) $(hdr-inst)=arch/$(hdr-arch)/include/uapi/asm $(hdr-dst)
 
 PHONY += headers_check_all
 headers_check_all: headers_install_all
@@ -907,8 +919,8 @@ headers_check_all: headers_install_all
 
 PHONY += headers_check
 headers_check: headers_install
-	$(Q)$(MAKE) $(hdr-inst)=include HDRCHECK=1
-	$(Q)$(MAKE) $(hdr-inst)=arch/$(hdr-arch)/include/asm $(hdr-dst) HDRCHECK=1
+	$(Q)$(MAKE) $(hdr-inst)=include/uapi HDRCHECK=1
+	$(Q)$(MAKE) $(hdr-inst)=arch/$(hdr-arch)/include/uapi/asm $(hdr-dst) HDRCHECK=1
 
 # ---------------------------------------------------------------------------
 # Modules
@@ -997,8 +1009,7 @@ CLEAN_DIRS  += $(MODVERDIR)
 # Directories & files removed with 'make mrproper'
 MRPROPER_DIRS  += include/config usr/include include/generated          \
                   arch/*/include/generated
-MRPROPER_FILES += .config .config.old .version .old_version             \
-                  include/linux/version.h                               \
+MRPROPER_FILES += .config .config.old .version .old_version $(version_h) \
 		  Module.symvers tags TAGS cscope* GPATH GTAGS GRTAGS GSYMS
 
 # clean - Delete most, but leave enough to build external modules
diff --git a/arch/alpha/include/uapi/asm/Kbuild b/arch/alpha/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/alpha/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c
index 9503a4be40f..63e77e3944c 100644
--- a/arch/alpha/kernel/osf_sys.c
+++ b/arch/alpha/kernel/osf_sys.c
@@ -145,27 +145,24 @@ SYSCALL_DEFINE4(osf_getdirentries, unsigned int, fd,
 		long __user *, basep)
 {
 	int error;
-	struct file *file;
+	struct fd arg = fdget(fd);
 	struct osf_dirent_callback buf;
 
-	error = -EBADF;
-	file = fget(fd);
-	if (!file)
-		goto out;
+	if (!arg.file)
+		return -EBADF;
 
 	buf.dirent = dirent;
 	buf.basep = basep;
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, osf_filldir, &buf);
+	error = vfs_readdir(arg.file, osf_filldir, &buf);
 	if (error >= 0)
 		error = buf.error;
 	if (count != buf.count)
 		error = count - buf.count;
 
-	fput(file);
- out:
+	fdput(arg);
 	return error;
 }
 
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 770da51242c..b86e57ef146 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -254,6 +254,7 @@ core-$(CONFIG_VFP)		+= arch/arm/vfp/
 # If we have a machine-specific directory, then include it in the build.
 core-y				+= arch/arm/kernel/ arch/arm/mm/ arch/arm/common/
 core-y				+= arch/arm/net/
+core-y				+= arch/arm/crypto/
 core-y				+= $(machdirs) $(platdirs)
 
 drivers-$(CONFIG_OPROFILE)      += arch/arm/oprofile/
diff --git a/arch/arm/boot/dts/highbank.dts b/arch/arm/boot/dts/highbank.dts
index 9fecf1ae777..0c6fc34821f 100644
--- a/arch/arm/boot/dts/highbank.dts
+++ b/arch/arm/boot/dts/highbank.dts
@@ -121,6 +121,10 @@
 			compatible = "calxeda,hb-ahci";
 			reg = <0xffe08000 0x10000>;
 			interrupts = <0 83 4>;
+			calxeda,port-phys = <&combophy5 0 &combophy0 0
+					     &combophy0 1 &combophy0 2
+					     &combophy0 3>;
+			dma-coherent;
 		};
 
 		sdhci@ffe0e000 {
@@ -306,5 +310,19 @@
 			reg = <0xfff51000 0x1000>;
 			interrupts = <0 80 4  0 81 4  0 82 4>;
 		};
+
+		combophy0: combo-phy@fff58000 {
+			compatible = "calxeda,hb-combophy";
+			#phy-cells = <1>;
+			reg = <0xfff58000 0x1000>;
+			phydev = <5>;
+		};
+
+		combophy5: combo-phy@fff5d000 {
+			compatible = "calxeda,hb-combophy";
+			#phy-cells = <1>;
+			reg = <0xfff5d000 0x1000>;
+			phydev = <31>;
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi
index 35e5895ba3d..f3990b04fec 100644
--- a/arch/arm/boot/dts/imx6q.dtsi
+++ b/arch/arm/boot/dts/imx6q.dtsi
@@ -400,8 +400,8 @@
 				#clock-cells = <1>;
 			};
 
-			anatop@020c8000 {
-				compatible = "fsl,imx6q-anatop";
+			anatop: anatop@020c8000 {
+				compatible = "fsl,imx6q-anatop", "syscon", "simple-bus";
 				reg = <0x020c8000 0x1000>;
 				interrupts = <0 49 0x04 0 54 0x04 0 127 0x04>;
 
@@ -531,6 +531,11 @@
 				interrupts = <0 89 0x04 0 90 0x04>;
 			};
 
+			gpr: iomuxc-gpr@020e0000 {
+				compatible = "fsl,imx6q-iomuxc-gpr", "syscon";
+				reg = <0x020e0000 0x38>;
+			};
+
 			iomuxc@020e0000 {
 				compatible = "fsl,imx6q-iomuxc";
 				reg = <0x020e0000 0x4000>;
diff --git a/arch/arm/boot/dts/pxa910-dkb.dts b/arch/arm/boot/dts/pxa910-dkb.dts
index e92be5a474e..595492aa505 100644
--- a/arch/arm/boot/dts/pxa910-dkb.dts
+++ b/arch/arm/boot/dts/pxa910-dkb.dts
@@ -29,6 +29,143 @@
 			};
 			twsi1: i2c@d4011000 {
 				status = "okay";
+
+				pmic: 88pm860x@34 {
+					compatible = "marvell,88pm860x";
+					reg = <0x34>;
+					interrupts = <4>;
+					interrupt-parent = <&intc>;
+					interrupt-controller;
+					#interrupt-cells = <1>;
+
+					marvell,88pm860x-irq-read-clr;
+					marvell,88pm860x-slave-addr = <0x11>;
+
+					regulators {
+						BUCK1 {
+							regulator-min-microvolt = <1000000>;
+							regulator-max-microvolt = <1500000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						BUCK2 {
+							regulator-min-microvolt = <1000000>;
+							regulator-max-microvolt = <1500000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						BUCK3 {
+							regulator-min-microvolt = <1000000>;
+							regulator-max-microvolt = <3000000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO1 {
+							regulator-min-microvolt = <1200000>;
+							regulator-max-microvolt = <2800000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO2 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO3 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO4 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-always-on;
+						};
+						LDO5 {
+							regulator-min-microvolt = <2900000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO6 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO7 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <2900000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO8 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <2900000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO9 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO10 {
+							regulator-min-microvolt = <1200000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-boot-on;
+							regulator-always-on;
+						};
+						LDO12 {
+							regulator-min-microvolt = <1200000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-always-on;
+						};
+						LDO13 {
+							regulator-min-microvolt = <1200000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-always-on;
+						};
+						LDO14 {
+							regulator-min-microvolt = <1800000>;
+							regulator-max-microvolt = <3300000>;
+							regulator-always-on;
+						};
+					};
+					rtc {
+						marvell,88pm860x-vrtc = <1>;
+					};
+					touch {
+						marvell,88pm860x-gpadc-prebias = <1>;
+						marvell,88pm860x-gpadc-slot-cycle = <1>;
+						marvell,88pm860x-tsi-prebias = <6>;
+						marvell,88pm860x-pen-prebias = <16>;
+						marvell,88pm860x-pen-prechg = <2>;
+						marvell,88pm860x-resistor-X = <300>;
+					};
+					backlights {
+						backlight-0 {
+							marvell,88pm860x-iset = <4>;
+							marvell,88pm860x-pwm = <3>;
+						};
+						backlight-2 {
+						};
+					};
+					leds {
+						led0-red {
+							marvell,88pm860x-iset = <12>;
+						};
+						led0-green {
+							marvell,88pm860x-iset = <12>;
+						};
+						led0-blue {
+							marvell,88pm860x-iset = <12>;
+						};
+					};
+				};
 			};
 			rtc: rtc@d4010000 {
 				status = "okay";
diff --git a/arch/arm/boot/dts/pxa910.dtsi b/arch/arm/boot/dts/pxa910.dtsi
index a3be44d86bc..825aaca3303 100644
--- a/arch/arm/boot/dts/pxa910.dtsi
+++ b/arch/arm/boot/dts/pxa910.dtsi
@@ -120,6 +120,8 @@
 
 			twsi1: i2c@d4011000 {
 				compatible = "mrvl,mmp-twsi";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xd4011000 0x1000>;
 				interrupts = <7>;
 				mrvl,i2c-fast-mode;
@@ -128,6 +130,8 @@
 
 			twsi2: i2c@d4037000 {
 				compatible = "mrvl,mmp-twsi";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xd4037000 0x1000>;
 				interrupts = <54>;
 				status = "disabled";
diff --git a/arch/arm/configs/sam9_l9260_defconfig b/arch/arm/configs/sam9_l9260_defconfig
index ecf2531523a..b4384af1bea 100644
--- a/arch/arm/configs/sam9_l9260_defconfig
+++ b/arch/arm/configs/sam9_l9260_defconfig
@@ -39,7 +39,7 @@ CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_ATMEL=y
 CONFIG_MTD_NAND_PLATFORM=y
 CONFIG_MTD_UBI=y
-CONFIG_MTD_UBI_BEB_RESERVE=3
+CONFIG_MTD_UBI_BEB_LIMIT=25
 CONFIG_MTD_UBI_GLUEBI=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
new file mode 100644
index 00000000000..a2c83851bc9
--- /dev/null
+++ b/arch/arm/crypto/Makefile
@@ -0,0 +1,9 @@
+#
+# Arch-specific CryptoAPI modules.
+#
+
+obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
+obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
+
+aes-arm-y  := aes-armv4.o aes_glue.o
+sha1-arm-y := sha1-armv4-large.o sha1_glue.o
diff --git a/arch/arm/crypto/aes-armv4.S b/arch/arm/crypto/aes-armv4.S
new file mode 100644
index 00000000000..e59b1d505d6
--- /dev/null
+++ b/arch/arm/crypto/aes-armv4.S
@@ -0,0 +1,1112 @@
+#define __ARM_ARCH__ __LINUX_ARM_ARCH__
+@ ====================================================================
+@ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
+@ project. The module is, however, dual licensed under OpenSSL and
+@ CRYPTOGAMS licenses depending on where you obtain it. For further
+@ details see http://www.openssl.org/~appro/cryptogams/.
+@ ====================================================================
+
+@ AES for ARMv4
+
+@ January 2007.
+@
+@ Code uses single 1K S-box and is >2 times faster than code generated
+@ by gcc-3.4.1. This is thanks to unique feature of ARMv4 ISA, which
+@ allows to merge logical or arithmetic operation with shift or rotate
+@ in one instruction and emit combined result every cycle. The module
+@ is endian-neutral. The performance is ~42 cycles/byte for 128-bit
+@ key [on single-issue Xscale PXA250 core].
+
+@ May 2007.
+@
+@ AES_set_[en|de]crypt_key is added.
+
+@ July 2010.
+@
+@ Rescheduling for dual-issue pipeline resulted in 12% improvement on
+@ Cortex A8 core and ~25 cycles per byte processed with 128-bit key.
+
+@ February 2011.
+@
+@ Profiler-assisted and platform-specific optimization resulted in 16%
+@ improvement on Cortex A8 core and ~21.5 cycles per byte.
+
+@ A little glue here to select the correct code below for the ARM CPU
+@ that is being targetted.
+
+.text
+.code	32
+
+.type	AES_Te,%object
+.align	5
+AES_Te:
+.word	0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d
+.word	0xfff2f20d, 0xd66b6bbd, 0xde6f6fb1, 0x91c5c554
+.word	0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d
+.word	0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a
+.word	0x8fcaca45, 0x1f82829d, 0x89c9c940, 0xfa7d7d87
+.word	0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b
+.word	0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea
+.word	0x239c9cbf, 0x53a4a4f7, 0xe4727296, 0x9bc0c05b
+.word	0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a
+.word	0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f
+.word	0x6834345c, 0x51a5a5f4, 0xd1e5e534, 0xf9f1f108
+.word	0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f
+.word	0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e
+.word	0x30181828, 0x379696a1, 0x0a05050f, 0x2f9a9ab5
+.word	0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d
+.word	0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f
+.word	0x1209091b, 0x1d83839e, 0x582c2c74, 0x341a1a2e
+.word	0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb
+.word	0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce
+.word	0x5229297b, 0xdde3e33e, 0x5e2f2f71, 0x13848497
+.word	0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c
+.word	0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed
+.word	0xd46a6abe, 0x8dcbcb46, 0x67bebed9, 0x7239394b
+.word	0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a
+.word	0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16
+.word	0x864343c5, 0x9a4d4dd7, 0x66333355, 0x11858594
+.word	0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81
+.word	0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3
+.word	0xa25151f3, 0x5da3a3fe, 0x804040c0, 0x058f8f8a
+.word	0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504
+.word	0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163
+.word	0x20101030, 0xe5ffff1a, 0xfdf3f30e, 0xbfd2d26d
+.word	0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f
+.word	0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739
+.word	0x93c4c457, 0x55a7a7f2, 0xfc7e7e82, 0x7a3d3d47
+.word	0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395
+.word	0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f
+.word	0x44222266, 0x542a2a7e, 0x3b9090ab, 0x0b888883
+.word	0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c
+.word	0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76
+.word	0xdbe0e03b, 0x64323256, 0x743a3a4e, 0x140a0a1e
+.word	0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4
+.word	0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6
+.word	0x399191a8, 0x319595a4, 0xd3e4e437, 0xf279798b
+.word	0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7
+.word	0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0
+.word	0xd86c6cb4, 0xac5656fa, 0xf3f4f407, 0xcfeaea25
+.word	0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818
+.word	0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72
+.word	0x381c1c24, 0x57a6a6f1, 0x73b4b4c7, 0x97c6c651
+.word	0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21
+.word	0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85
+.word	0xe0707090, 0x7c3e3e42, 0x71b5b5c4, 0xcc6666aa
+.word	0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12
+.word	0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0
+.word	0x17868691, 0x99c1c158, 0x3a1d1d27, 0x279e9eb9
+.word	0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133
+.word	0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7
+.word	0x2d9b9bb6, 0x3c1e1e22, 0x15878792, 0xc9e9e920
+.word	0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a
+.word	0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17
+.word	0x65bfbfda, 0xd7e6e631, 0x844242c6, 0xd06868b8
+.word	0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11
+.word	0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a
+@ Te4[256]
+.byte	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
+.byte	0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
+.byte	0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0
+.byte	0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0
+.byte	0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc
+.byte	0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15
+.byte	0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a
+.byte	0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75
+.byte	0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0
+.byte	0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84
+.byte	0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b
+.byte	0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf
+.byte	0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85
+.byte	0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8
+.byte	0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5
+.byte	0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2
+.byte	0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17
+.byte	0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73
+.byte	0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88
+.byte	0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb
+.byte	0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c
+.byte	0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79
+.byte	0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9
+.byte	0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08
+.byte	0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6
+.byte	0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a
+.byte	0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e
+.byte	0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e
+.byte	0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94
+.byte	0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf
+.byte	0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68
+.byte	0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16
+@ rcon[]
+.word	0x01000000, 0x02000000, 0x04000000, 0x08000000
+.word	0x10000000, 0x20000000, 0x40000000, 0x80000000
+.word	0x1B000000, 0x36000000, 0, 0, 0, 0, 0, 0
+.size	AES_Te,.-AES_Te
+
+@ void AES_encrypt(const unsigned char *in, unsigned char *out,
+@ 		 const AES_KEY *key) {
+.global AES_encrypt
+.type   AES_encrypt,%function
+.align	5
+AES_encrypt:
+	sub	r3,pc,#8		@ AES_encrypt
+	stmdb   sp!,{r1,r4-r12,lr}
+	mov	r12,r0		@ inp
+	mov	r11,r2
+	sub	r10,r3,#AES_encrypt-AES_Te	@ Te
+#if __ARM_ARCH__<7
+	ldrb	r0,[r12,#3]	@ load input data in endian-neutral
+	ldrb	r4,[r12,#2]	@ manner...
+	ldrb	r5,[r12,#1]
+	ldrb	r6,[r12,#0]
+	orr	r0,r0,r4,lsl#8
+	ldrb	r1,[r12,#7]
+	orr	r0,r0,r5,lsl#16
+	ldrb	r4,[r12,#6]
+	orr	r0,r0,r6,lsl#24
+	ldrb	r5,[r12,#5]
+	ldrb	r6,[r12,#4]
+	orr	r1,r1,r4,lsl#8
+	ldrb	r2,[r12,#11]
+	orr	r1,r1,r5,lsl#16
+	ldrb	r4,[r12,#10]
+	orr	r1,r1,r6,lsl#24
+	ldrb	r5,[r12,#9]
+	ldrb	r6,[r12,#8]
+	orr	r2,r2,r4,lsl#8
+	ldrb	r3,[r12,#15]
+	orr	r2,r2,r5,lsl#16
+	ldrb	r4,[r12,#14]
+	orr	r2,r2,r6,lsl#24
+	ldrb	r5,[r12,#13]
+	ldrb	r6,[r12,#12]
+	orr	r3,r3,r4,lsl#8
+	orr	r3,r3,r5,lsl#16
+	orr	r3,r3,r6,lsl#24
+#else
+	ldr	r0,[r12,#0]
+	ldr	r1,[r12,#4]
+	ldr	r2,[r12,#8]
+	ldr	r3,[r12,#12]
+#ifdef __ARMEL__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+#endif
+#endif
+	bl	_armv4_AES_encrypt
+
+	ldr	r12,[sp],#4		@ pop out
+#if __ARM_ARCH__>=7
+#ifdef __ARMEL__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+#endif
+	str	r0,[r12,#0]
+	str	r1,[r12,#4]
+	str	r2,[r12,#8]
+	str	r3,[r12,#12]
+#else
+	mov	r4,r0,lsr#24		@ write output in endian-neutral
+	mov	r5,r0,lsr#16		@ manner...
+	mov	r6,r0,lsr#8
+	strb	r4,[r12,#0]
+	strb	r5,[r12,#1]
+	mov	r4,r1,lsr#24
+	strb	r6,[r12,#2]
+	mov	r5,r1,lsr#16
+	strb	r0,[r12,#3]
+	mov	r6,r1,lsr#8
+	strb	r4,[r12,#4]
+	strb	r5,[r12,#5]
+	mov	r4,r2,lsr#24
+	strb	r6,[r12,#6]
+	mov	r5,r2,lsr#16
+	strb	r1,[r12,#7]
+	mov	r6,r2,lsr#8
+	strb	r4,[r12,#8]
+	strb	r5,[r12,#9]
+	mov	r4,r3,lsr#24
+	strb	r6,[r12,#10]
+	mov	r5,r3,lsr#16
+	strb	r2,[r12,#11]
+	mov	r6,r3,lsr#8
+	strb	r4,[r12,#12]
+	strb	r5,[r12,#13]
+	strb	r6,[r12,#14]
+	strb	r3,[r12,#15]
+#endif
+#if __ARM_ARCH__>=5
+	ldmia	sp!,{r4-r12,pc}
+#else
+	ldmia   sp!,{r4-r12,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
+#endif
+.size	AES_encrypt,.-AES_encrypt
+
+.type   _armv4_AES_encrypt,%function
+.align	2
+_armv4_AES_encrypt:
+	str	lr,[sp,#-4]!		@ push lr
+	ldmia	r11!,{r4-r7}
+	eor	r0,r0,r4
+	ldr	r12,[r11,#240-16]
+	eor	r1,r1,r5
+	eor	r2,r2,r6
+	eor	r3,r3,r7
+	sub	r12,r12,#1
+	mov	lr,#255
+
+	and	r7,lr,r0
+	and	r8,lr,r0,lsr#8
+	and	r9,lr,r0,lsr#16
+	mov	r0,r0,lsr#24
+.Lenc_loop:
+	ldr	r4,[r10,r7,lsl#2]	@ Te3[s0>>0]
+	and	r7,lr,r1,lsr#16	@ i0
+	ldr	r5,[r10,r8,lsl#2]	@ Te2[s0>>8]
+	and	r8,lr,r1
+	ldr	r6,[r10,r9,lsl#2]	@ Te1[s0>>16]
+	and	r9,lr,r1,lsr#8
+	ldr	r0,[r10,r0,lsl#2]	@ Te0[s0>>24]
+	mov	r1,r1,lsr#24
+
+	ldr	r7,[r10,r7,lsl#2]	@ Te1[s1>>16]
+	ldr	r8,[r10,r8,lsl#2]	@ Te3[s1>>0]
+	ldr	r9,[r10,r9,lsl#2]	@ Te2[s1>>8]
+	eor	r0,r0,r7,ror#8
+	ldr	r1,[r10,r1,lsl#2]	@ Te0[s1>>24]
+	and	r7,lr,r2,lsr#8	@ i0
+	eor	r5,r5,r8,ror#8
+	and	r8,lr,r2,lsr#16	@ i1
+	eor	r6,r6,r9,ror#8
+	and	r9,lr,r2
+	ldr	r7,[r10,r7,lsl#2]	@ Te2[s2>>8]
+	eor	r1,r1,r4,ror#24
+	ldr	r8,[r10,r8,lsl#2]	@ Te1[s2>>16]
+	mov	r2,r2,lsr#24
+
+	ldr	r9,[r10,r9,lsl#2]	@ Te3[s2>>0]
+	eor	r0,r0,r7,ror#16
+	ldr	r2,[r10,r2,lsl#2]	@ Te0[s2>>24]
+	and	r7,lr,r3		@ i0
+	eor	r1,r1,r8,ror#8
+	and	r8,lr,r3,lsr#8	@ i1
+	eor	r6,r6,r9,ror#16
+	and	r9,lr,r3,lsr#16	@ i2
+	ldr	r7,[r10,r7,lsl#2]	@ Te3[s3>>0]
+	eor	r2,r2,r5,ror#16
+	ldr	r8,[r10,r8,lsl#2]	@ Te2[s3>>8]
+	mov	r3,r3,lsr#24
+
+	ldr	r9,[r10,r9,lsl#2]	@ Te1[s3>>16]
+	eor	r0,r0,r7,ror#24
+	ldr	r7,[r11],#16
+	eor	r1,r1,r8,ror#16
+	ldr	r3,[r10,r3,lsl#2]	@ Te0[s3>>24]
+	eor	r2,r2,r9,ror#8
+	ldr	r4,[r11,#-12]
+	eor	r3,r3,r6,ror#8
+
+	ldr	r5,[r11,#-8]
+	eor	r0,r0,r7
+	ldr	r6,[r11,#-4]
+	and	r7,lr,r0
+	eor	r1,r1,r4
+	and	r8,lr,r0,lsr#8
+	eor	r2,r2,r5
+	and	r9,lr,r0,lsr#16
+	eor	r3,r3,r6
+	mov	r0,r0,lsr#24
+
+	subs	r12,r12,#1
+	bne	.Lenc_loop
+
+	add	r10,r10,#2
+
+	ldrb	r4,[r10,r7,lsl#2]	@ Te4[s0>>0]
+	and	r7,lr,r1,lsr#16	@ i0
+	ldrb	r5,[r10,r8,lsl#2]	@ Te4[s0>>8]
+	and	r8,lr,r1
+	ldrb	r6,[r10,r9,lsl#2]	@ Te4[s0>>16]
+	and	r9,lr,r1,lsr#8
+	ldrb	r0,[r10,r0,lsl#2]	@ Te4[s0>>24]
+	mov	r1,r1,lsr#24
+
+	ldrb	r7,[r10,r7,lsl#2]	@ Te4[s1>>16]
+	ldrb	r8,[r10,r8,lsl#2]	@ Te4[s1>>0]
+	ldrb	r9,[r10,r9,lsl#2]	@ Te4[s1>>8]
+	eor	r0,r7,r0,lsl#8
+	ldrb	r1,[r10,r1,lsl#2]	@ Te4[s1>>24]
+	and	r7,lr,r2,lsr#8	@ i0
+	eor	r5,r8,r5,lsl#8
+	and	r8,lr,r2,lsr#16	@ i1
+	eor	r6,r9,r6,lsl#8
+	and	r9,lr,r2
+	ldrb	r7,[r10,r7,lsl#2]	@ Te4[s2>>8]
+	eor	r1,r4,r1,lsl#24
+	ldrb	r8,[r10,r8,lsl#2]	@ Te4[s2>>16]
+	mov	r2,r2,lsr#24
+
+	ldrb	r9,[r10,r9,lsl#2]	@ Te4[s2>>0]
+	eor	r0,r7,r0,lsl#8
+	ldrb	r2,[r10,r2,lsl#2]	@ Te4[s2>>24]
+	and	r7,lr,r3		@ i0
+	eor	r1,r1,r8,lsl#16
+	and	r8,lr,r3,lsr#8	@ i1
+	eor	r6,r9,r6,lsl#8
+	and	r9,lr,r3,lsr#16	@ i2
+	ldrb	r7,[r10,r7,lsl#2]	@ Te4[s3>>0]
+	eor	r2,r5,r2,lsl#24
+	ldrb	r8,[r10,r8,lsl#2]	@ Te4[s3>>8]
+	mov	r3,r3,lsr#24
+
+	ldrb	r9,[r10,r9,lsl#2]	@ Te4[s3>>16]
+	eor	r0,r7,r0,lsl#8
+	ldr	r7,[r11,#0]
+	ldrb	r3,[r10,r3,lsl#2]	@ Te4[s3>>24]
+	eor	r1,r1,r8,lsl#8
+	ldr	r4,[r11,#4]
+	eor	r2,r2,r9,lsl#16
+	ldr	r5,[r11,#8]
+	eor	r3,r6,r3,lsl#24
+	ldr	r6,[r11,#12]
+
+	eor	r0,r0,r7
+	eor	r1,r1,r4
+	eor	r2,r2,r5
+	eor	r3,r3,r6
+
+	sub	r10,r10,#2
+	ldr	pc,[sp],#4		@ pop and return
+.size	_armv4_AES_encrypt,.-_armv4_AES_encrypt
+
+.global private_AES_set_encrypt_key
+.type   private_AES_set_encrypt_key,%function
+.align	5
+private_AES_set_encrypt_key:
+_armv4_AES_set_encrypt_key:
+	sub	r3,pc,#8		@ AES_set_encrypt_key
+	teq	r0,#0
+	moveq	r0,#-1
+	beq	.Labrt
+	teq	r2,#0
+	moveq	r0,#-1
+	beq	.Labrt
+
+	teq	r1,#128
+	beq	.Lok
+	teq	r1,#192
+	beq	.Lok
+	teq	r1,#256
+	movne	r0,#-1
+	bne	.Labrt
+
+.Lok:	stmdb   sp!,{r4-r12,lr}
+	sub	r10,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024	@ Te4
+
+	mov	r12,r0		@ inp
+	mov	lr,r1			@ bits
+	mov	r11,r2			@ key
+
+#if __ARM_ARCH__<7
+	ldrb	r0,[r12,#3]	@ load input data in endian-neutral
+	ldrb	r4,[r12,#2]	@ manner...
+	ldrb	r5,[r12,#1]
+	ldrb	r6,[r12,#0]
+	orr	r0,r0,r4,lsl#8
+	ldrb	r1,[r12,#7]
+	orr	r0,r0,r5,lsl#16
+	ldrb	r4,[r12,#6]
+	orr	r0,r0,r6,lsl#24
+	ldrb	r5,[r12,#5]
+	ldrb	r6,[r12,#4]
+	orr	r1,r1,r4,lsl#8
+	ldrb	r2,[r12,#11]
+	orr	r1,r1,r5,lsl#16
+	ldrb	r4,[r12,#10]
+	orr	r1,r1,r6,lsl#24
+	ldrb	r5,[r12,#9]
+	ldrb	r6,[r12,#8]
+	orr	r2,r2,r4,lsl#8
+	ldrb	r3,[r12,#15]
+	orr	r2,r2,r5,lsl#16
+	ldrb	r4,[r12,#14]
+	orr	r2,r2,r6,lsl#24
+	ldrb	r5,[r12,#13]
+	ldrb	r6,[r12,#12]
+	orr	r3,r3,r4,lsl#8
+	str	r0,[r11],#16
+	orr	r3,r3,r5,lsl#16
+	str	r1,[r11,#-12]
+	orr	r3,r3,r6,lsl#24
+	str	r2,[r11,#-8]
+	str	r3,[r11,#-4]
+#else
+	ldr	r0,[r12,#0]
+	ldr	r1,[r12,#4]
+	ldr	r2,[r12,#8]
+	ldr	r3,[r12,#12]
+#ifdef __ARMEL__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+#endif
+	str	r0,[r11],#16
+	str	r1,[r11,#-12]
+	str	r2,[r11,#-8]
+	str	r3,[r11,#-4]
+#endif
+
+	teq	lr,#128
+	bne	.Lnot128
+	mov	r12,#10
+	str	r12,[r11,#240-16]
+	add	r6,r10,#256			@ rcon
+	mov	lr,#255
+
+.L128_loop:
+	and	r5,lr,r3,lsr#24
+	and	r7,lr,r3,lsr#16
+	ldrb	r5,[r10,r5]
+	and	r8,lr,r3,lsr#8
+	ldrb	r7,[r10,r7]
+	and	r9,lr,r3
+	ldrb	r8,[r10,r8]
+	orr	r5,r5,r7,lsl#24
+	ldrb	r9,[r10,r9]
+	orr	r5,r5,r8,lsl#16
+	ldr	r4,[r6],#4			@ rcon[i++]
+	orr	r5,r5,r9,lsl#8
+	eor	r5,r5,r4
+	eor	r0,r0,r5			@ rk[4]=rk[0]^...
+	eor	r1,r1,r0			@ rk[5]=rk[1]^rk[4]
+	str	r0,[r11],#16
+	eor	r2,r2,r1			@ rk[6]=rk[2]^rk[5]
+	str	r1,[r11,#-12]
+	eor	r3,r3,r2			@ rk[7]=rk[3]^rk[6]
+	str	r2,[r11,#-8]
+	subs	r12,r12,#1
+	str	r3,[r11,#-4]
+	bne	.L128_loop
+	sub	r2,r11,#176
+	b	.Ldone
+
+.Lnot128:
+#if __ARM_ARCH__<7
+	ldrb	r8,[r12,#19]
+	ldrb	r4,[r12,#18]
+	ldrb	r5,[r12,#17]
+	ldrb	r6,[r12,#16]
+	orr	r8,r8,r4,lsl#8
+	ldrb	r9,[r12,#23]
+	orr	r8,r8,r5,lsl#16
+	ldrb	r4,[r12,#22]
+	orr	r8,r8,r6,lsl#24
+	ldrb	r5,[r12,#21]
+	ldrb	r6,[r12,#20]
+	orr	r9,r9,r4,lsl#8
+	orr	r9,r9,r5,lsl#16
+	str	r8,[r11],#8
+	orr	r9,r9,r6,lsl#24
+	str	r9,[r11,#-4]
+#else
+	ldr	r8,[r12,#16]
+	ldr	r9,[r12,#20]
+#ifdef __ARMEL__
+	rev	r8,r8
+	rev	r9,r9
+#endif
+	str	r8,[r11],#8
+	str	r9,[r11,#-4]
+#endif
+
+	teq	lr,#192
+	bne	.Lnot192
+	mov	r12,#12
+	str	r12,[r11,#240-24]
+	add	r6,r10,#256			@ rcon
+	mov	lr,#255
+	mov	r12,#8
+
+.L192_loop:
+	and	r5,lr,r9,lsr#24
+	and	r7,lr,r9,lsr#16
+	ldrb	r5,[r10,r5]
+	and	r8,lr,r9,lsr#8
+	ldrb	r7,[r10,r7]
+	and	r9,lr,r9
+	ldrb	r8,[r10,r8]
+	orr	r5,r5,r7,lsl#24
+	ldrb	r9,[r10,r9]
+	orr	r5,r5,r8,lsl#16
+	ldr	r4,[r6],#4			@ rcon[i++]
+	orr	r5,r5,r9,lsl#8
+	eor	r9,r5,r4
+	eor	r0,r0,r9			@ rk[6]=rk[0]^...
+	eor	r1,r1,r0			@ rk[7]=rk[1]^rk[6]
+	str	r0,[r11],#24
+	eor	r2,r2,r1			@ rk[8]=rk[2]^rk[7]
+	str	r1,[r11,#-20]
+	eor	r3,r3,r2			@ rk[9]=rk[3]^rk[8]
+	str	r2,[r11,#-16]
+	subs	r12,r12,#1
+	str	r3,[r11,#-12]
+	subeq	r2,r11,#216
+	beq	.Ldone
+
+	ldr	r7,[r11,#-32]
+	ldr	r8,[r11,#-28]
+	eor	r7,r7,r3			@ rk[10]=rk[4]^rk[9]
+	eor	r9,r8,r7			@ rk[11]=rk[5]^rk[10]
+	str	r7,[r11,#-8]
+	str	r9,[r11,#-4]
+	b	.L192_loop
+
+.Lnot192:
+#if __ARM_ARCH__<7
+	ldrb	r8,[r12,#27]
+	ldrb	r4,[r12,#26]
+	ldrb	r5,[r12,#25]
+	ldrb	r6,[r12,#24]
+	orr	r8,r8,r4,lsl#8
+	ldrb	r9,[r12,#31]
+	orr	r8,r8,r5,lsl#16
+	ldrb	r4,[r12,#30]
+	orr	r8,r8,r6,lsl#24
+	ldrb	r5,[r12,#29]
+	ldrb	r6,[r12,#28]
+	orr	r9,r9,r4,lsl#8
+	orr	r9,r9,r5,lsl#16
+	str	r8,[r11],#8
+	orr	r9,r9,r6,lsl#24
+	str	r9,[r11,#-4]
+#else
+	ldr	r8,[r12,#24]
+	ldr	r9,[r12,#28]
+#ifdef __ARMEL__
+	rev	r8,r8
+	rev	r9,r9
+#endif
+	str	r8,[r11],#8
+	str	r9,[r11,#-4]
+#endif
+
+	mov	r12,#14
+	str	r12,[r11,#240-32]
+	add	r6,r10,#256			@ rcon
+	mov	lr,#255
+	mov	r12,#7
+
+.L256_loop:
+	and	r5,lr,r9,lsr#24
+	and	r7,lr,r9,lsr#16
+	ldrb	r5,[r10,r5]
+	and	r8,lr,r9,lsr#8
+	ldrb	r7,[r10,r7]
+	and	r9,lr,r9
+	ldrb	r8,[r10,r8]
+	orr	r5,r5,r7,lsl#24
+	ldrb	r9,[r10,r9]
+	orr	r5,r5,r8,lsl#16
+	ldr	r4,[r6],#4			@ rcon[i++]
+	orr	r5,r5,r9,lsl#8
+	eor	r9,r5,r4
+	eor	r0,r0,r9			@ rk[8]=rk[0]^...
+	eor	r1,r1,r0			@ rk[9]=rk[1]^rk[8]
+	str	r0,[r11],#32
+	eor	r2,r2,r1			@ rk[10]=rk[2]^rk[9]
+	str	r1,[r11,#-28]
+	eor	r3,r3,r2			@ rk[11]=rk[3]^rk[10]
+	str	r2,[r11,#-24]
+	subs	r12,r12,#1
+	str	r3,[r11,#-20]
+	subeq	r2,r11,#256
+	beq	.Ldone
+
+	and	r5,lr,r3
+	and	r7,lr,r3,lsr#8
+	ldrb	r5,[r10,r5]
+	and	r8,lr,r3,lsr#16
+	ldrb	r7,[r10,r7]
+	and	r9,lr,r3,lsr#24
+	ldrb	r8,[r10,r8]
+	orr	r5,r5,r7,lsl#8
+	ldrb	r9,[r10,r9]
+	orr	r5,r5,r8,lsl#16
+	ldr	r4,[r11,#-48]
+	orr	r5,r5,r9,lsl#24
+
+	ldr	r7,[r11,#-44]
+	ldr	r8,[r11,#-40]
+	eor	r4,r4,r5			@ rk[12]=rk[4]^...
+	ldr	r9,[r11,#-36]
+	eor	r7,r7,r4			@ rk[13]=rk[5]^rk[12]
+	str	r4,[r11,#-16]
+	eor	r8,r8,r7			@ rk[14]=rk[6]^rk[13]
+	str	r7,[r11,#-12]
+	eor	r9,r9,r8			@ rk[15]=rk[7]^rk[14]
+	str	r8,[r11,#-8]
+	str	r9,[r11,#-4]
+	b	.L256_loop
+
+.Ldone:	mov	r0,#0
+	ldmia   sp!,{r4-r12,lr}
+.Labrt:	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
+.size	private_AES_set_encrypt_key,.-private_AES_set_encrypt_key
+
+.global private_AES_set_decrypt_key
+.type   private_AES_set_decrypt_key,%function
+.align	5
+private_AES_set_decrypt_key:
+	str	lr,[sp,#-4]!            @ push lr
+#if 0
+	@ kernel does both of these in setkey so optimise this bit out by
+	@ expecting the key to already have the enc_key work done (see aes_glue.c)
+	bl	_armv4_AES_set_encrypt_key
+#else
+	mov	r0,#0
+#endif
+	teq	r0,#0
+	ldrne	lr,[sp],#4              @ pop lr
+	bne	.Labrt
+
+	stmdb   sp!,{r4-r12}
+
+	ldr	r12,[r2,#240]	@ AES_set_encrypt_key preserves r2,
+	mov	r11,r2			@ which is AES_KEY *key
+	mov	r7,r2
+	add	r8,r2,r12,lsl#4
+
+.Linv:	ldr	r0,[r7]
+	ldr	r1,[r7,#4]
+	ldr	r2,[r7,#8]
+	ldr	r3,[r7,#12]
+	ldr	r4,[r8]
+	ldr	r5,[r8,#4]
+	ldr	r6,[r8,#8]
+	ldr	r9,[r8,#12]
+	str	r0,[r8],#-16
+	str	r1,[r8,#16+4]
+	str	r2,[r8,#16+8]
+	str	r3,[r8,#16+12]
+	str	r4,[r7],#16
+	str	r5,[r7,#-12]
+	str	r6,[r7,#-8]
+	str	r9,[r7,#-4]
+	teq	r7,r8
+	bne	.Linv
+	ldr	r0,[r11,#16]!		@ prefetch tp1
+	mov	r7,#0x80
+	mov	r8,#0x1b
+	orr	r7,r7,#0x8000
+	orr	r8,r8,#0x1b00
+	orr	r7,r7,r7,lsl#16
+	orr	r8,r8,r8,lsl#16
+	sub	r12,r12,#1
+	mvn	r9,r7
+	mov	r12,r12,lsl#2	@ (rounds-1)*4
+
+.Lmix:	and	r4,r0,r7
+	and	r1,r0,r9
+	sub	r4,r4,r4,lsr#7
+	and	r4,r4,r8
+	eor	r1,r4,r1,lsl#1	@ tp2
+
+	and	r4,r1,r7
+	and	r2,r1,r9
+	sub	r4,r4,r4,lsr#7
+	and	r4,r4,r8
+	eor	r2,r4,r2,lsl#1	@ tp4
+
+	and	r4,r2,r7
+	and	r3,r2,r9
+	sub	r4,r4,r4,lsr#7
+	and	r4,r4,r8
+	eor	r3,r4,r3,lsl#1	@ tp8
+
+	eor	r4,r1,r2
+	eor	r5,r0,r3		@ tp9
+	eor	r4,r4,r3		@ tpe
+	eor	r4,r4,r1,ror#24
+	eor	r4,r4,r5,ror#24	@ ^= ROTATE(tpb=tp9^tp2,8)
+	eor	r4,r4,r2,ror#16
+	eor	r4,r4,r5,ror#16	@ ^= ROTATE(tpd=tp9^tp4,16)
+	eor	r4,r4,r5,ror#8	@ ^= ROTATE(tp9,24)
+
+	ldr	r0,[r11,#4]		@ prefetch tp1
+	str	r4,[r11],#4
+	subs	r12,r12,#1
+	bne	.Lmix
+
+	mov	r0,#0
+#if __ARM_ARCH__>=5
+	ldmia	sp!,{r4-r12,pc}
+#else
+	ldmia   sp!,{r4-r12,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
+#endif
+.size	private_AES_set_decrypt_key,.-private_AES_set_decrypt_key
+
+.type	AES_Td,%object
+.align	5
+AES_Td:
+.word	0x51f4a750, 0x7e416553, 0x1a17a4c3, 0x3a275e96
+.word	0x3bab6bcb, 0x1f9d45f1, 0xacfa58ab, 0x4be30393
+.word	0x2030fa55, 0xad766df6, 0x88cc7691, 0xf5024c25
+.word	0x4fe5d7fc, 0xc52acbd7, 0x26354480, 0xb562a38f
+.word	0xdeb15a49, 0x25ba1b67, 0x45ea0e98, 0x5dfec0e1
+.word	0xc32f7502, 0x814cf012, 0x8d4697a3, 0x6bd3f9c6
+.word	0x038f5fe7, 0x15929c95, 0xbf6d7aeb, 0x955259da
+.word	0xd4be832d, 0x587421d3, 0x49e06929, 0x8ec9c844
+.word	0x75c2896a, 0xf48e7978, 0x99583e6b, 0x27b971dd
+.word	0xbee14fb6, 0xf088ad17, 0xc920ac66, 0x7dce3ab4
+.word	0x63df4a18, 0xe51a3182, 0x97513360, 0x62537f45
+.word	0xb16477e0, 0xbb6bae84, 0xfe81a01c, 0xf9082b94
+.word	0x70486858, 0x8f45fd19, 0x94de6c87, 0x527bf8b7
+.word	0xab73d323, 0x724b02e2, 0xe31f8f57, 0x6655ab2a
+.word	0xb2eb2807, 0x2fb5c203, 0x86c57b9a, 0xd33708a5
+.word	0x302887f2, 0x23bfa5b2, 0x02036aba, 0xed16825c
+.word	0x8acf1c2b, 0xa779b492, 0xf307f2f0, 0x4e69e2a1
+.word	0x65daf4cd, 0x0605bed5, 0xd134621f, 0xc4a6fe8a
+.word	0x342e539d, 0xa2f355a0, 0x058ae132, 0xa4f6eb75
+.word	0x0b83ec39, 0x4060efaa, 0x5e719f06, 0xbd6e1051
+.word	0x3e218af9, 0x96dd063d, 0xdd3e05ae, 0x4de6bd46
+.word	0x91548db5, 0x71c45d05, 0x0406d46f, 0x605015ff
+.word	0x1998fb24, 0xd6bde997, 0x894043cc, 0x67d99e77
+.word	0xb0e842bd, 0x07898b88, 0xe7195b38, 0x79c8eedb
+.word	0xa17c0a47, 0x7c420fe9, 0xf8841ec9, 0x00000000
+.word	0x09808683, 0x322bed48, 0x1e1170ac, 0x6c5a724e
+.word	0xfd0efffb, 0x0f853856, 0x3daed51e, 0x362d3927
+.word	0x0a0fd964, 0x685ca621, 0x9b5b54d1, 0x24362e3a
+.word	0x0c0a67b1, 0x9357e70f, 0xb4ee96d2, 0x1b9b919e
+.word	0x80c0c54f, 0x61dc20a2, 0x5a774b69, 0x1c121a16
+.word	0xe293ba0a, 0xc0a02ae5, 0x3c22e043, 0x121b171d
+.word	0x0e090d0b, 0xf28bc7ad, 0x2db6a8b9, 0x141ea9c8
+.word	0x57f11985, 0xaf75074c, 0xee99ddbb, 0xa37f60fd
+.word	0xf701269f, 0x5c72f5bc, 0x44663bc5, 0x5bfb7e34
+.word	0x8b432976, 0xcb23c6dc, 0xb6edfc68, 0xb8e4f163
+.word	0xd731dcca, 0x42638510, 0x13972240, 0x84c61120
+.word	0x854a247d, 0xd2bb3df8, 0xaef93211, 0xc729a16d
+.word	0x1d9e2f4b, 0xdcb230f3, 0x0d8652ec, 0x77c1e3d0
+.word	0x2bb3166c, 0xa970b999, 0x119448fa, 0x47e96422
+.word	0xa8fc8cc4, 0xa0f03f1a, 0x567d2cd8, 0x223390ef
+.word	0x87494ec7, 0xd938d1c1, 0x8ccaa2fe, 0x98d40b36
+.word	0xa6f581cf, 0xa57ade28, 0xdab78e26, 0x3fadbfa4
+.word	0x2c3a9de4, 0x5078920d, 0x6a5fcc9b, 0x547e4662
+.word	0xf68d13c2, 0x90d8b8e8, 0x2e39f75e, 0x82c3aff5
+.word	0x9f5d80be, 0x69d0937c, 0x6fd52da9, 0xcf2512b3
+.word	0xc8ac993b, 0x10187da7, 0xe89c636e, 0xdb3bbb7b
+.word	0xcd267809, 0x6e5918f4, 0xec9ab701, 0x834f9aa8
+.word	0xe6956e65, 0xaaffe67e, 0x21bccf08, 0xef15e8e6
+.word	0xbae79bd9, 0x4a6f36ce, 0xea9f09d4, 0x29b07cd6
+.word	0x31a4b2af, 0x2a3f2331, 0xc6a59430, 0x35a266c0
+.word	0x744ebc37, 0xfc82caa6, 0xe090d0b0, 0x33a7d815
+.word	0xf104984a, 0x41ecdaf7, 0x7fcd500e, 0x1791f62f
+.word	0x764dd68d, 0x43efb04d, 0xccaa4d54, 0xe49604df
+.word	0x9ed1b5e3, 0x4c6a881b, 0xc12c1fb8, 0x4665517f
+.word	0x9d5eea04, 0x018c355d, 0xfa877473, 0xfb0b412e
+.word	0xb3671d5a, 0x92dbd252, 0xe9105633, 0x6dd64713
+.word	0x9ad7618c, 0x37a10c7a, 0x59f8148e, 0xeb133c89
+.word	0xcea927ee, 0xb761c935, 0xe11ce5ed, 0x7a47b13c
+.word	0x9cd2df59, 0x55f2733f, 0x1814ce79, 0x73c737bf
+.word	0x53f7cdea, 0x5ffdaa5b, 0xdf3d6f14, 0x7844db86
+.word	0xcaaff381, 0xb968c43e, 0x3824342c, 0xc2a3405f
+.word	0x161dc372, 0xbce2250c, 0x283c498b, 0xff0d9541
+.word	0x39a80171, 0x080cb3de, 0xd8b4e49c, 0x6456c190
+.word	0x7bcb8461, 0xd532b670, 0x486c5c74, 0xd0b85742
+@ Td4[256]
+.byte	0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
+.byte	0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
+.byte	0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
+.byte	0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
+.byte	0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
+.byte	0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
+.byte	0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
+.byte	0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
+.byte	0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
+.byte	0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
+.byte	0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
+.byte	0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
+.byte	0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
+.byte	0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
+.byte	0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
+.byte	0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
+.byte	0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
+.byte	0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
+.byte	0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
+.byte	0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
+.byte	0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
+.byte	0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
+.byte	0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
+.byte	0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
+.byte	0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
+.byte	0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
+.byte	0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
+.byte	0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
+.byte	0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
+.byte	0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
+.byte	0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
+.byte	0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
+.size	AES_Td,.-AES_Td
+
+@ void AES_decrypt(const unsigned char *in, unsigned char *out,
+@ 		 const AES_KEY *key) {
+.global AES_decrypt
+.type   AES_decrypt,%function
+.align	5
+AES_decrypt:
+	sub	r3,pc,#8		@ AES_decrypt
+	stmdb   sp!,{r1,r4-r12,lr}
+	mov	r12,r0		@ inp
+	mov	r11,r2
+	sub	r10,r3,#AES_decrypt-AES_Td		@ Td
+#if __ARM_ARCH__<7
+	ldrb	r0,[r12,#3]	@ load input data in endian-neutral
+	ldrb	r4,[r12,#2]	@ manner...
+	ldrb	r5,[r12,#1]
+	ldrb	r6,[r12,#0]
+	orr	r0,r0,r4,lsl#8
+	ldrb	r1,[r12,#7]
+	orr	r0,r0,r5,lsl#16
+	ldrb	r4,[r12,#6]
+	orr	r0,r0,r6,lsl#24
+	ldrb	r5,[r12,#5]
+	ldrb	r6,[r12,#4]
+	orr	r1,r1,r4,lsl#8
+	ldrb	r2,[r12,#11]
+	orr	r1,r1,r5,lsl#16
+	ldrb	r4,[r12,#10]
+	orr	r1,r1,r6,lsl#24
+	ldrb	r5,[r12,#9]
+	ldrb	r6,[r12,#8]
+	orr	r2,r2,r4,lsl#8
+	ldrb	r3,[r12,#15]
+	orr	r2,r2,r5,lsl#16
+	ldrb	r4,[r12,#14]
+	orr	r2,r2,r6,lsl#24
+	ldrb	r5,[r12,#13]
+	ldrb	r6,[r12,#12]
+	orr	r3,r3,r4,lsl#8
+	orr	r3,r3,r5,lsl#16
+	orr	r3,r3,r6,lsl#24
+#else
+	ldr	r0,[r12,#0]
+	ldr	r1,[r12,#4]
+	ldr	r2,[r12,#8]
+	ldr	r3,[r12,#12]
+#ifdef __ARMEL__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+#endif
+#endif
+	bl	_armv4_AES_decrypt
+
+	ldr	r12,[sp],#4		@ pop out
+#if __ARM_ARCH__>=7
+#ifdef __ARMEL__
+	rev	r0,r0
+	rev	r1,r1
+	rev	r2,r2
+	rev	r3,r3
+#endif
+	str	r0,[r12,#0]
+	str	r1,[r12,#4]
+	str	r2,[r12,#8]
+	str	r3,[r12,#12]
+#else
+	mov	r4,r0,lsr#24		@ write output in endian-neutral
+	mov	r5,r0,lsr#16		@ manner...
+	mov	r6,r0,lsr#8
+	strb	r4,[r12,#0]
+	strb	r5,[r12,#1]
+	mov	r4,r1,lsr#24
+	strb	r6,[r12,#2]
+	mov	r5,r1,lsr#16
+	strb	r0,[r12,#3]
+	mov	r6,r1,lsr#8
+	strb	r4,[r12,#4]
+	strb	r5,[r12,#5]
+	mov	r4,r2,lsr#24
+	strb	r6,[r12,#6]
+	mov	r5,r2,lsr#16
+	strb	r1,[r12,#7]
+	mov	r6,r2,lsr#8
+	strb	r4,[r12,#8]
+	strb	r5,[r12,#9]
+	mov	r4,r3,lsr#24
+	strb	r6,[r12,#10]
+	mov	r5,r3,lsr#16
+	strb	r2,[r12,#11]
+	mov	r6,r3,lsr#8
+	strb	r4,[r12,#12]
+	strb	r5,[r12,#13]
+	strb	r6,[r12,#14]
+	strb	r3,[r12,#15]
+#endif
+#if __ARM_ARCH__>=5
+	ldmia	sp!,{r4-r12,pc}
+#else
+	ldmia   sp!,{r4-r12,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
+#endif
+.size	AES_decrypt,.-AES_decrypt
+
+.type   _armv4_AES_decrypt,%function
+.align	2
+_armv4_AES_decrypt:
+	str	lr,[sp,#-4]!		@ push lr
+	ldmia	r11!,{r4-r7}
+	eor	r0,r0,r4
+	ldr	r12,[r11,#240-16]
+	eor	r1,r1,r5
+	eor	r2,r2,r6
+	eor	r3,r3,r7
+	sub	r12,r12,#1
+	mov	lr,#255
+
+	and	r7,lr,r0,lsr#16
+	and	r8,lr,r0,lsr#8
+	and	r9,lr,r0
+	mov	r0,r0,lsr#24
+.Ldec_loop:
+	ldr	r4,[r10,r7,lsl#2]	@ Td1[s0>>16]
+	and	r7,lr,r1		@ i0
+	ldr	r5,[r10,r8,lsl#2]	@ Td2[s0>>8]
+	and	r8,lr,r1,lsr#16
+	ldr	r6,[r10,r9,lsl#2]	@ Td3[s0>>0]
+	and	r9,lr,r1,lsr#8
+	ldr	r0,[r10,r0,lsl#2]	@ Td0[s0>>24]
+	mov	r1,r1,lsr#24
+
+	ldr	r7,[r10,r7,lsl#2]	@ Td3[s1>>0]
+	ldr	r8,[r10,r8,lsl#2]	@ Td1[s1>>16]
+	ldr	r9,[r10,r9,lsl#2]	@ Td2[s1>>8]
+	eor	r0,r0,r7,ror#24
+	ldr	r1,[r10,r1,lsl#2]	@ Td0[s1>>24]
+	and	r7,lr,r2,lsr#8	@ i0
+	eor	r5,r8,r5,ror#8
+	and	r8,lr,r2		@ i1
+	eor	r6,r9,r6,ror#8
+	and	r9,lr,r2,lsr#16
+	ldr	r7,[r10,r7,lsl#2]	@ Td2[s2>>8]
+	eor	r1,r1,r4,ror#8
+	ldr	r8,[r10,r8,lsl#2]	@ Td3[s2>>0]
+	mov	r2,r2,lsr#24
+
+	ldr	r9,[r10,r9,lsl#2]	@ Td1[s2>>16]
+	eor	r0,r0,r7,ror#16
+	ldr	r2,[r10,r2,lsl#2]	@ Td0[s2>>24]
+	and	r7,lr,r3,lsr#16	@ i0
+	eor	r1,r1,r8,ror#24
+	and	r8,lr,r3,lsr#8	@ i1
+	eor	r6,r9,r6,ror#8
+	and	r9,lr,r3		@ i2
+	ldr	r7,[r10,r7,lsl#2]	@ Td1[s3>>16]
+	eor	r2,r2,r5,ror#8
+	ldr	r8,[r10,r8,lsl#2]	@ Td2[s3>>8]
+	mov	r3,r3,lsr#24
+
+	ldr	r9,[r10,r9,lsl#2]	@ Td3[s3>>0]
+	eor	r0,r0,r7,ror#8
+	ldr	r7,[r11],#16
+	eor	r1,r1,r8,ror#16
+	ldr	r3,[r10,r3,lsl#2]	@ Td0[s3>>24]
+	eor	r2,r2,r9,ror#24
+
+	ldr	r4,[r11,#-12]
+	eor	r0,r0,r7
+	ldr	r5,[r11,#-8]
+	eor	r3,r3,r6,ror#8
+	ldr	r6,[r11,#-4]
+	and	r7,lr,r0,lsr#16
+	eor	r1,r1,r4
+	and	r8,lr,r0,lsr#8
+	eor	r2,r2,r5
+	and	r9,lr,r0
+	eor	r3,r3,r6
+	mov	r0,r0,lsr#24
+
+	subs	r12,r12,#1
+	bne	.Ldec_loop
+
+	add	r10,r10,#1024
+
+	ldr	r5,[r10,#0]		@ prefetch Td4
+	ldr	r6,[r10,#32]
+	ldr	r4,[r10,#64]
+	ldr	r5,[r10,#96]
+	ldr	r6,[r10,#128]
+	ldr	r4,[r10,#160]
+	ldr	r5,[r10,#192]
+	ldr	r6,[r10,#224]
+
+	ldrb	r0,[r10,r0]		@ Td4[s0>>24]
+	ldrb	r4,[r10,r7]		@ Td4[s0>>16]
+	and	r7,lr,r1		@ i0
+	ldrb	r5,[r10,r8]		@ Td4[s0>>8]
+	and	r8,lr,r1,lsr#16
+	ldrb	r6,[r10,r9]		@ Td4[s0>>0]
+	and	r9,lr,r1,lsr#8
+
+	ldrb	r7,[r10,r7]		@ Td4[s1>>0]
+	ldrb	r1,[r10,r1,lsr#24]	@ Td4[s1>>24]
+	ldrb	r8,[r10,r8]		@ Td4[s1>>16]
+	eor	r0,r7,r0,lsl#24
+	ldrb	r9,[r10,r9]		@ Td4[s1>>8]
+	eor	r1,r4,r1,lsl#8
+	and	r7,lr,r2,lsr#8	@ i0
+	eor	r5,r5,r8,lsl#8
+	and	r8,lr,r2		@ i1
+	ldrb	r7,[r10,r7]		@ Td4[s2>>8]
+	eor	r6,r6,r9,lsl#8
+	ldrb	r8,[r10,r8]		@ Td4[s2>>0]
+	and	r9,lr,r2,lsr#16
+
+	ldrb	r2,[r10,r2,lsr#24]	@ Td4[s2>>24]
+	eor	r0,r0,r7,lsl#8
+	ldrb	r9,[r10,r9]		@ Td4[s2>>16]
+	eor	r1,r8,r1,lsl#16
+	and	r7,lr,r3,lsr#16	@ i0
+	eor	r2,r5,r2,lsl#16
+	and	r8,lr,r3,lsr#8	@ i1
+	ldrb	r7,[r10,r7]		@ Td4[s3>>16]
+	eor	r6,r6,r9,lsl#16
+	ldrb	r8,[r10,r8]		@ Td4[s3>>8]
+	and	r9,lr,r3		@ i2
+
+	ldrb	r9,[r10,r9]		@ Td4[s3>>0]
+	ldrb	r3,[r10,r3,lsr#24]	@ Td4[s3>>24]
+	eor	r0,r0,r7,lsl#16
+	ldr	r7,[r11,#0]
+	eor	r1,r1,r8,lsl#8
+	ldr	r4,[r11,#4]
+	eor	r2,r9,r2,lsl#8
+	ldr	r5,[r11,#8]
+	eor	r3,r6,r3,lsl#24
+	ldr	r6,[r11,#12]
+
+	eor	r0,r0,r7
+	eor	r1,r1,r4
+	eor	r2,r2,r5
+	eor	r3,r3,r6
+
+	sub	r10,r10,#1024
+	ldr	pc,[sp],#4		@ pop and return
+.size	_armv4_AES_decrypt,.-_armv4_AES_decrypt
+.asciz	"AES for ARMv4, CRYPTOGAMS by <appro@openssl.org>"
+.align	2
diff --git a/arch/arm/crypto/aes_glue.c b/arch/arm/crypto/aes_glue.c
new file mode 100644
index 00000000000..59f7877ead6
--- /dev/null
+++ b/arch/arm/crypto/aes_glue.c
@@ -0,0 +1,108 @@
+/*
+ * Glue Code for the asm optimized version of the AES Cipher Algorithm
+ */
+
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <crypto/aes.h>
+
+#define AES_MAXNR 14
+
+typedef struct {
+	unsigned int rd_key[4 *(AES_MAXNR + 1)];
+	int rounds;
+} AES_KEY;
+
+struct AES_CTX {
+	AES_KEY enc_key;
+	AES_KEY dec_key;
+};
+
+asmlinkage void AES_encrypt(const u8 *in, u8 *out, AES_KEY *ctx);
+asmlinkage void AES_decrypt(const u8 *in, u8 *out, AES_KEY *ctx);
+asmlinkage int private_AES_set_decrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
+asmlinkage int private_AES_set_encrypt_key(const unsigned char *userKey, const int bits, AES_KEY *key);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+	struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
+	AES_encrypt(src, dst, &ctx->enc_key);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+	struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
+	AES_decrypt(src, dst, &ctx->dec_key);
+}
+
+static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+		unsigned int key_len)
+{
+	struct AES_CTX *ctx = crypto_tfm_ctx(tfm);
+
+	switch (key_len) {
+	case AES_KEYSIZE_128:
+		key_len = 128;
+		break;
+	case AES_KEYSIZE_192:
+		key_len = 192;
+		break;
+	case AES_KEYSIZE_256:
+		key_len = 256;
+		break;
+	default:
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	if (private_AES_set_encrypt_key(in_key, key_len, &ctx->enc_key) == -1) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+	/* private_AES_set_decrypt_key expects an encryption key as input */
+	ctx->dec_key = ctx->enc_key;
+	if (private_AES_set_decrypt_key(in_key, key_len, &ctx->dec_key) == -1) {
+		tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static struct crypto_alg aes_alg = {
+	.cra_name		= "aes",
+	.cra_driver_name	= "aes-asm",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct AES_CTX),
+	.cra_module		= THIS_MODULE,
+	.cra_list		= LIST_HEAD_INIT(aes_alg.cra_list),
+	.cra_u	= {
+		.cipher	= {
+			.cia_min_keysize	= AES_MIN_KEY_SIZE,
+			.cia_max_keysize	= AES_MAX_KEY_SIZE,
+			.cia_setkey			= aes_set_key,
+			.cia_encrypt		= aes_encrypt,
+			.cia_decrypt		= aes_decrypt
+		}
+	}
+};
+
+static int __init aes_init(void)
+{
+	return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+	crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm (ASM)");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("aes");
+MODULE_ALIAS("aes-asm");
+MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");
diff --git a/arch/arm/crypto/sha1-armv4-large.S b/arch/arm/crypto/sha1-armv4-large.S
new file mode 100644
index 00000000000..7050ab133b9
--- /dev/null
+++ b/arch/arm/crypto/sha1-armv4-large.S
@@ -0,0 +1,503 @@
+#define __ARM_ARCH__ __LINUX_ARM_ARCH__
+@ ====================================================================
+@ Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
+@ project. The module is, however, dual licensed under OpenSSL and
+@ CRYPTOGAMS licenses depending on where you obtain it. For further
+@ details see http://www.openssl.org/~appro/cryptogams/.
+@ ====================================================================
+
+@ sha1_block procedure for ARMv4.
+@
+@ January 2007.
+
+@ Size/performance trade-off
+@ ====================================================================
+@ impl		size in bytes	comp cycles[*]	measured performance
+@ ====================================================================
+@ thumb		304		3212		4420
+@ armv4-small	392/+29%	1958/+64%	2250/+96%
+@ armv4-compact	740/+89%	1552/+26%	1840/+22%
+@ armv4-large	1420/+92%	1307/+19%	1370/+34%[***]
+@ full unroll	~5100/+260%	~1260/+4%	~1300/+5%
+@ ====================================================================
+@ thumb		= same as 'small' but in Thumb instructions[**] and
+@		  with recurring code in two private functions;
+@ small		= detached Xload/update, loops are folded;
+@ compact	= detached Xload/update, 5x unroll;
+@ large		= interleaved Xload/update, 5x unroll;
+@ full unroll	= interleaved Xload/update, full unroll, estimated[!];
+@
+@ [*]	Manually counted instructions in "grand" loop body. Measured
+@	performance is affected by prologue and epilogue overhead,
+@	i-cache availability, branch penalties, etc.
+@ [**]	While each Thumb instruction is twice smaller, they are not as
+@	diverse as ARM ones: e.g., there are only two arithmetic
+@	instructions with 3 arguments, no [fixed] rotate, addressing
+@	modes are limited. As result it takes more instructions to do
+@	the same job in Thumb, therefore the code is never twice as
+@	small and always slower.
+@ [***]	which is also ~35% better than compiler generated code. Dual-
+@	issue Cortex A8 core was measured to process input block in
+@	~990 cycles.
+
+@ August 2010.
+@
+@ Rescheduling for dual-issue pipeline resulted in 13% improvement on
+@ Cortex A8 core and in absolute terms ~870 cycles per input block
+@ [or 13.6 cycles per byte].
+
+@ February 2011.
+@
+@ Profiler-assisted and platform-specific optimization resulted in 10%
+@ improvement on Cortex A8 core and 12.2 cycles per byte.
+
+.text
+
+.global	sha1_block_data_order
+.type	sha1_block_data_order,%function
+
+.align	2
+sha1_block_data_order:
+	stmdb	sp!,{r4-r12,lr}
+	add	r2,r1,r2,lsl#6	@ r2 to point at the end of r1
+	ldmia	r0,{r3,r4,r5,r6,r7}
+.Lloop:
+	ldr	r8,.LK_00_19
+	mov	r14,sp
+	sub	sp,sp,#15*4
+	mov	r5,r5,ror#30
+	mov	r6,r6,ror#30
+	mov	r7,r7,ror#30		@ [6]
+.L_00_15:
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r7,r8,r7,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r5,r6			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r7,r8,r7,ror#2			@ E+=K_00_19
+	eor	r10,r5,r6			@ F_xx_xx
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r4,r10,ror#2
+	add	r7,r7,r9			@ E+=X[i]
+	eor	r10,r10,r6,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r7,r7,r10			@ E+=F_00_19(B,C,D)
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r6,r8,r6,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r4,r5			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r6,r8,r6,ror#2			@ E+=K_00_19
+	eor	r10,r4,r5			@ F_xx_xx
+	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r3,r10,ror#2
+	add	r6,r6,r9			@ E+=X[i]
+	eor	r10,r10,r5,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r6,r6,r10			@ E+=F_00_19(B,C,D)
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r5,r8,r5,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r3,r4			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r5,r8,r5,ror#2			@ E+=K_00_19
+	eor	r10,r3,r4			@ F_xx_xx
+	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r7,r10,ror#2
+	add	r5,r5,r9			@ E+=X[i]
+	eor	r10,r10,r4,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r5,r5,r10			@ E+=F_00_19(B,C,D)
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r4,r8,r4,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r7,r3			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r4,r8,r4,ror#2			@ E+=K_00_19
+	eor	r10,r7,r3			@ F_xx_xx
+	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r6,r10,ror#2
+	add	r4,r4,r9			@ E+=X[i]
+	eor	r10,r10,r3,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r4,r4,r10			@ E+=F_00_19(B,C,D)
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r3,r8,r3,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r6,r7			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r3,r8,r3,ror#2			@ E+=K_00_19
+	eor	r10,r6,r7			@ F_xx_xx
+	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r5,r10,ror#2
+	add	r3,r3,r9			@ E+=X[i]
+	eor	r10,r10,r7,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r3,r3,r10			@ E+=F_00_19(B,C,D)
+	teq	r14,sp
+	bne	.L_00_15		@ [((11+4)*5+2)*3]
+#if __ARM_ARCH__<7
+	ldrb	r10,[r1,#2]
+	ldrb	r9,[r1,#3]
+	ldrb	r11,[r1,#1]
+	add	r7,r8,r7,ror#2			@ E+=K_00_19
+	ldrb	r12,[r1],#4
+	orr	r9,r9,r10,lsl#8
+	eor	r10,r5,r6			@ F_xx_xx
+	orr	r9,r9,r11,lsl#16
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+	orr	r9,r9,r12,lsl#24
+#else
+	ldr	r9,[r1],#4			@ handles unaligned
+	add	r7,r8,r7,ror#2			@ E+=K_00_19
+	eor	r10,r5,r6			@ F_xx_xx
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+#ifdef __ARMEL__
+	rev	r9,r9				@ byte swap
+#endif
+#endif
+	and	r10,r4,r10,ror#2
+	add	r7,r7,r9			@ E+=X[i]
+	eor	r10,r10,r6,ror#2		@ F_00_19(B,C,D)
+	str	r9,[r14,#-4]!
+	add	r7,r7,r10			@ E+=F_00_19(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r4,r5			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r3,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r6,r6,r9			@ E+=X[i]
+	eor	r10,r10,r5,ror#2		@ F_00_19(B,C,D)
+	add	r6,r6,r10			@ E+=F_00_19(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r3,r4			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r7,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r5,r5,r9			@ E+=X[i]
+	eor	r10,r10,r4,ror#2		@ F_00_19(B,C,D)
+	add	r5,r5,r10			@ E+=F_00_19(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r7,r3			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r6,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r4,r4,r9			@ E+=X[i]
+	eor	r10,r10,r3,ror#2		@ F_00_19(B,C,D)
+	add	r4,r4,r10			@ E+=F_00_19(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r6,r7			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r5,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r3,r3,r9			@ E+=X[i]
+	eor	r10,r10,r7,ror#2		@ F_00_19(B,C,D)
+	add	r3,r3,r10			@ E+=F_00_19(B,C,D)
+
+	ldr	r8,.LK_20_39		@ [+15+16*4]
+	sub	sp,sp,#25*4
+	cmn	sp,#0			@ [+3], clear carry to denote 20_39
+.L_20_39_or_60_79:
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r7,r8,r7,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r5,r6			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	eor r10,r4,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r7,r7,r9			@ E+=X[i]
+	add	r7,r7,r10			@ E+=F_20_39(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r4,r5			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	eor r10,r3,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r6,r6,r9			@ E+=X[i]
+	add	r6,r6,r10			@ E+=F_20_39(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r3,r4			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	eor r10,r7,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r5,r5,r9			@ E+=X[i]
+	add	r5,r5,r10			@ E+=F_20_39(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r7,r3			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	eor r10,r6,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r4,r4,r9			@ E+=X[i]
+	add	r4,r4,r10			@ E+=F_20_39(B,C,D)
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r6,r7			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	eor r10,r5,r10,ror#2					@ F_xx_xx
+						@ F_xx_xx
+	add	r3,r3,r9			@ E+=X[i]
+	add	r3,r3,r10			@ E+=F_20_39(B,C,D)
+	teq	r14,sp			@ preserve carry
+	bne	.L_20_39_or_60_79	@ [+((12+3)*5+2)*4]
+	bcs	.L_done			@ [+((12+3)*5+2)*4], spare 300 bytes
+
+	ldr	r8,.LK_40_59
+	sub	sp,sp,#20*4		@ [+2]
+.L_40_59:
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r7,r8,r7,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r5,r6			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r7,r7,r3,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r4,r10,ror#2					@ F_xx_xx
+	and r11,r5,r6					@ F_xx_xx
+	add	r7,r7,r9			@ E+=X[i]
+	add	r7,r7,r10			@ E+=F_40_59(B,C,D)
+	add	r7,r7,r11,ror#2
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r6,r8,r6,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r4,r5			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r6,r6,r7,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r3,r10,ror#2					@ F_xx_xx
+	and r11,r4,r5					@ F_xx_xx
+	add	r6,r6,r9			@ E+=X[i]
+	add	r6,r6,r10			@ E+=F_40_59(B,C,D)
+	add	r6,r6,r11,ror#2
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r5,r8,r5,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r3,r4			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r5,r5,r6,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r7,r10,ror#2					@ F_xx_xx
+	and r11,r3,r4					@ F_xx_xx
+	add	r5,r5,r9			@ E+=X[i]
+	add	r5,r5,r10			@ E+=F_40_59(B,C,D)
+	add	r5,r5,r11,ror#2
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r4,r8,r4,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r7,r3			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r4,r4,r5,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r6,r10,ror#2					@ F_xx_xx
+	and r11,r7,r3					@ F_xx_xx
+	add	r4,r4,r9			@ E+=X[i]
+	add	r4,r4,r10			@ E+=F_40_59(B,C,D)
+	add	r4,r4,r11,ror#2
+	ldr	r9,[r14,#15*4]
+	ldr	r10,[r14,#13*4]
+	ldr	r11,[r14,#7*4]
+	add	r3,r8,r3,ror#2			@ E+=K_xx_xx
+	ldr	r12,[r14,#2*4]
+	eor	r9,r9,r10
+	eor	r11,r11,r12			@ 1 cycle stall
+	eor	r10,r6,r7			@ F_xx_xx
+	mov	r9,r9,ror#31
+	add	r3,r3,r4,ror#27			@ E+=ROR(A,27)
+	eor	r9,r9,r11,ror#31
+	str	r9,[r14,#-4]!
+	and r10,r5,r10,ror#2					@ F_xx_xx
+	and r11,r6,r7					@ F_xx_xx
+	add	r3,r3,r9			@ E+=X[i]
+	add	r3,r3,r10			@ E+=F_40_59(B,C,D)
+	add	r3,r3,r11,ror#2
+	teq	r14,sp
+	bne	.L_40_59		@ [+((12+5)*5+2)*4]
+
+	ldr	r8,.LK_60_79
+	sub	sp,sp,#20*4
+	cmp	sp,#0			@ set carry to denote 60_79
+	b	.L_20_39_or_60_79	@ [+4], spare 300 bytes
+.L_done:
+	add	sp,sp,#80*4		@ "deallocate" stack frame
+	ldmia	r0,{r8,r9,r10,r11,r12}
+	add	r3,r8,r3
+	add	r4,r9,r4
+	add	r5,r10,r5,ror#2
+	add	r6,r11,r6,ror#2
+	add	r7,r12,r7,ror#2
+	stmia	r0,{r3,r4,r5,r6,r7}
+	teq	r1,r2
+	bne	.Lloop			@ [+18], total 1307
+
+#if __ARM_ARCH__>=5
+	ldmia	sp!,{r4-r12,pc}
+#else
+	ldmia	sp!,{r4-r12,lr}
+	tst	lr,#1
+	moveq	pc,lr			@ be binary compatible with V4, yet
+	.word	0xe12fff1e			@ interoperable with Thumb ISA:-)
+#endif
+.align	2
+.LK_00_19:	.word	0x5a827999
+.LK_20_39:	.word	0x6ed9eba1
+.LK_40_59:	.word	0x8f1bbcdc
+.LK_60_79:	.word	0xca62c1d6
+.size	sha1_block_data_order,.-sha1_block_data_order
+.asciz	"SHA1 block transform for ARMv4, CRYPTOGAMS by <appro@openssl.org>"
+.align	2
diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c
new file mode 100644
index 00000000000..76cd976230b
--- /dev/null
+++ b/arch/arm/crypto/sha1_glue.c
@@ -0,0 +1,179 @@
+/*
+ * Cryptographic API.
+ * Glue code for the SHA1 Secure Hash Algorithm assembler implementation
+ *
+ * This file is based on sha1_generic.c and sha1_ssse3_glue.c
+ *
+ * Copyright (c) Alan Smithee.
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
+ * Copyright (c) Mathias Krause <minipli@googlemail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/cryptohash.h>
+#include <linux/types.h>
+#include <crypto/sha.h>
+#include <asm/byteorder.h>
+
+struct SHA1_CTX {
+	uint32_t h0,h1,h2,h3,h4;
+	u64 count;
+	u8 data[SHA1_BLOCK_SIZE];
+};
+
+asmlinkage void sha1_block_data_order(struct SHA1_CTX *digest,
+		const unsigned char *data, unsigned int rounds);
+
+
+static int sha1_init(struct shash_desc *desc)
+{
+	struct SHA1_CTX *sctx = shash_desc_ctx(desc);
+	memset(sctx, 0, sizeof(*sctx));
+	sctx->h0 = SHA1_H0;
+	sctx->h1 = SHA1_H1;
+	sctx->h2 = SHA1_H2;
+	sctx->h3 = SHA1_H3;
+	sctx->h4 = SHA1_H4;
+	return 0;
+}
+
+
+static int __sha1_update(struct SHA1_CTX *sctx, const u8 *data,
+			       unsigned int len, unsigned int partial)
+{
+	unsigned int done = 0;
+
+	sctx->count += len;
+
+	if (partial) {
+		done = SHA1_BLOCK_SIZE - partial;
+		memcpy(sctx->data + partial, data, done);
+		sha1_block_data_order(sctx, sctx->data, 1);
+	}
+
+	if (len - done >= SHA1_BLOCK_SIZE) {
+		const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE;
+		sha1_block_data_order(sctx, data + done, rounds);
+		done += rounds * SHA1_BLOCK_SIZE;
+	}
+
+	memcpy(sctx->data, data + done, len - done);
+	return 0;
+}
+
+
+static int sha1_update(struct shash_desc *desc, const u8 *data,
+			     unsigned int len)
+{
+	struct SHA1_CTX *sctx = shash_desc_ctx(desc);
+	unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
+	int res;
+
+	/* Handle the fast case right here */
+	if (partial + len < SHA1_BLOCK_SIZE) {
+		sctx->count += len;
+		memcpy(sctx->data + partial, data, len);
+		return 0;
+	}
+	res = __sha1_update(sctx, data, len, partial);
+	return res;
+}
+
+
+/* Add padding and return the message digest. */
+static int sha1_final(struct shash_desc *desc, u8 *out)
+{
+	struct SHA1_CTX *sctx = shash_desc_ctx(desc);
+	unsigned int i, index, padlen;
+	__be32 *dst = (__be32 *)out;
+	__be64 bits;
+	static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
+
+	bits = cpu_to_be64(sctx->count << 3);
+
+	/* Pad out to 56 mod 64 and append length */
+	index = sctx->count % SHA1_BLOCK_SIZE;
+	padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index);
+	/* We need to fill a whole block for __sha1_update() */
+	if (padlen <= 56) {
+		sctx->count += padlen;
+		memcpy(sctx->data + index, padding, padlen);
+	} else {
+		__sha1_update(sctx, padding, padlen, index);
+	}
+	__sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56);
+
+	/* Store state in digest */
+	for (i = 0; i < 5; i++)
+		dst[i] = cpu_to_be32(((u32 *)sctx)[i]);
+
+	/* Wipe context */
+	memset(sctx, 0, sizeof(*sctx));
+	return 0;
+}
+
+
+static int sha1_export(struct shash_desc *desc, void *out)
+{
+	struct SHA1_CTX *sctx = shash_desc_ctx(desc);
+	memcpy(out, sctx, sizeof(*sctx));
+	return 0;
+}
+
+
+static int sha1_import(struct shash_desc *desc, const void *in)
+{
+	struct SHA1_CTX *sctx = shash_desc_ctx(desc);
+	memcpy(sctx, in, sizeof(*sctx));
+	return 0;
+}
+
+
+static struct shash_alg alg = {
+	.digestsize	=	SHA1_DIGEST_SIZE,
+	.init		=	sha1_init,
+	.update		=	sha1_update,
+	.final		=	sha1_final,
+	.export		=	sha1_export,
+	.import		=	sha1_import,
+	.descsize	=	sizeof(struct SHA1_CTX),
+	.statesize	=	sizeof(struct SHA1_CTX),
+	.base		=	{
+		.cra_name	=	"sha1",
+		.cra_driver_name=	"sha1-asm",
+		.cra_priority	=	150,
+		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
+		.cra_blocksize	=	SHA1_BLOCK_SIZE,
+		.cra_module	=	THIS_MODULE,
+	}
+};
+
+
+static int __init sha1_mod_init(void)
+{
+	return crypto_register_shash(&alg);
+}
+
+
+static void __exit sha1_mod_fini(void)
+{
+	crypto_unregister_shash(&alg);
+}
+
+
+module_init(sha1_mod_init);
+module_exit(sha1_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (ARM)");
+MODULE_ALIAS("sha1");
+MODULE_AUTHOR("David McCullough <ucdevel@gmail.com>");
diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h
index 05112380dc5..8dcd9c702d9 100644
--- a/arch/arm/include/asm/barrier.h
+++ b/arch/arm/include/asm/barrier.h
@@ -44,10 +44,9 @@
 #define rmb()		dsb()
 #define wmb()		mb()
 #else
-#include <asm/memory.h>
-#define mb()	do { if (arch_is_coherent()) dmb(); else barrier(); } while (0)
-#define rmb()	do { if (arch_is_coherent()) dmb(); else barrier(); } while (0)
-#define wmb()	do { if (arch_is_coherent()) dmb(); else barrier(); } while (0)
+#define mb()		barrier()
+#define rmb()		barrier()
+#define wmb()		barrier()
 #endif
 
 #ifndef CONFIG_SMP
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index 5c44dcb0987..23004847bb0 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -13,6 +13,7 @@
 
 #define DMA_ERROR_CODE	(~0)
 extern struct dma_map_ops arm_dma_ops;
+extern struct dma_map_ops arm_coherent_dma_ops;
 
 static inline struct dma_map_ops *get_dma_ops(struct device *dev)
 {
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h
index 5f6ddcc5645..73cf03aa981 100644
--- a/arch/arm/include/asm/memory.h
+++ b/arch/arm/include/asm/memory.h
@@ -275,14 +275,6 @@ static inline __deprecated void *bus_to_virt(unsigned long x)
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 #define virt_addr_valid(kaddr)	((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory)
 
-/*
- * Optional coherency support.  Currently used only by selected
- * Intel XSC3-based systems.
- */
-#ifndef arch_is_coherent
-#define arch_is_coherent()		0
-#endif
-
 #endif
 
 #include <asm-generic/memory_model.h>
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index ecf901902e4..812a4944e78 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -19,7 +19,7 @@
 
 #ifndef CONFIG_MMU
 
-#include "page-nommu.h"
+#include <asm/page-nommu.h>
 
 #else
 
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 41dc31f834c..08c12312a1f 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -16,7 +16,7 @@
 #ifndef CONFIG_MMU
 
 #include <asm-generic/4level-fixup.h>
-#include "pgtable-nommu.h"
+#include <asm/pgtable-nommu.h>
 
 #else
 
diff --git a/arch/arm/include/asm/vfpmacros.h b/arch/arm/include/asm/vfpmacros.h
index 3d5fc41ae8d..a7aadbd9a6d 100644
--- a/arch/arm/include/asm/vfpmacros.h
+++ b/arch/arm/include/asm/vfpmacros.h
@@ -5,7 +5,7 @@
  */
 #include <asm/hwcap.h>
 
-#include "vfp.h"
+#include <asm/vfp.h>
 
 @ Macros to allow building with old toolkits (with no VFP support)
 	.macro	VFPFMRX, rd, sysreg, cond
diff --git a/arch/arm/include/uapi/asm/Kbuild b/arch/arm/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/arm/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index aa4ffe6e5ec..dea7a925c7e 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -24,6 +24,7 @@
 #include <linux/percpu.h>
 #include <linux/clockchips.h>
 #include <linux/completion.h>
+#include <linux/cpufreq.h>
 
 #include <linux/atomic.h>
 #include <asm/smp.h>
@@ -650,3 +651,56 @@ int setup_profiling_timer(unsigned int multiplier)
 {
 	return -EINVAL;
 }
+
+#ifdef CONFIG_CPU_FREQ
+
+static DEFINE_PER_CPU(unsigned long, l_p_j_ref);
+static DEFINE_PER_CPU(unsigned long, l_p_j_ref_freq);
+static unsigned long global_l_p_j_ref;
+static unsigned long global_l_p_j_ref_freq;
+
+static int cpufreq_callback(struct notifier_block *nb,
+					unsigned long val, void *data)
+{
+	struct cpufreq_freqs *freq = data;
+	int cpu = freq->cpu;
+
+	if (freq->flags & CPUFREQ_CONST_LOOPS)
+		return NOTIFY_OK;
+
+	if (!per_cpu(l_p_j_ref, cpu)) {
+		per_cpu(l_p_j_ref, cpu) =
+			per_cpu(cpu_data, cpu).loops_per_jiffy;
+		per_cpu(l_p_j_ref_freq, cpu) = freq->old;
+		if (!global_l_p_j_ref) {
+			global_l_p_j_ref = loops_per_jiffy;
+			global_l_p_j_ref_freq = freq->old;
+		}
+	}
+
+	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
+	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
+	    (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
+		loops_per_jiffy = cpufreq_scale(global_l_p_j_ref,
+						global_l_p_j_ref_freq,
+						freq->new);
+		per_cpu(cpu_data, cpu).loops_per_jiffy =
+			cpufreq_scale(per_cpu(l_p_j_ref, cpu),
+					per_cpu(l_p_j_ref_freq, cpu),
+					freq->new);
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block cpufreq_notifier = {
+	.notifier_call  = cpufreq_callback,
+};
+
+static int __init register_cpufreq_notifier(void)
+{
+	return cpufreq_register_notifier(&cpufreq_notifier,
+						CPUFREQ_TRANSITION_NOTIFIER);
+}
+core_initcall(register_cpufreq_notifier);
+
+#endif
diff --git a/arch/arm/mach-at91/clock.c b/arch/arm/mach-at91/clock.c
index 188c82971eb..33361505c0c 100644
--- a/arch/arm/mach-at91/clock.c
+++ b/arch/arm/mach-at91/clock.c
@@ -625,7 +625,7 @@ fail:
 	return 0;
 }
 
-static struct clk *const standard_pmc_clocks[] __initdata = {
+static struct clk *const standard_pmc_clocks[] __initconst = {
 	/* four primary clocks */
 	&clk32k,
 	&main_clk,
diff --git a/arch/arm/mach-davinci/board-tnetv107x-evm.c b/arch/arm/mach-davinci/board-tnetv107x-evm.c
index ac4e003ad86..be3099733b1 100644
--- a/arch/arm/mach-davinci/board-tnetv107x-evm.c
+++ b/arch/arm/mach-davinci/board-tnetv107x-evm.c
@@ -88,7 +88,7 @@ static struct davinci_mmc_config mmc_config = {
 	.version	= MMC_CTLR_VERSION_1,
 };
 
-static const short sdio1_pins[] __initdata = {
+static const short sdio1_pins[] __initconst = {
 	TNETV107X_SDIO1_CLK_1,		TNETV107X_SDIO1_CMD_1,
 	TNETV107X_SDIO1_DATA0_1,	TNETV107X_SDIO1_DATA1_1,
 	TNETV107X_SDIO1_DATA2_1,	TNETV107X_SDIO1_DATA3_1,
@@ -96,12 +96,12 @@ static const short sdio1_pins[] __initdata = {
 	-1
 };
 
-static const short uart1_pins[] __initdata = {
+static const short uart1_pins[] __initconst = {
 	TNETV107X_UART1_RD,		TNETV107X_UART1_TD,
 	-1
 };
 
-static const short ssp_pins[] __initdata = {
+static const short ssp_pins[] __initconst = {
 	TNETV107X_SSP0_0, TNETV107X_SSP0_1, TNETV107X_SSP0_2,
 	TNETV107X_SSP1_0, TNETV107X_SSP1_1, TNETV107X_SSP1_2,
 	TNETV107X_SSP1_3, -1
diff --git a/arch/arm/mach-davinci/da830.c b/arch/arm/mach-davinci/da830.c
index deee5c2da75..510648e0394 100644
--- a/arch/arm/mach-davinci/da830.c
+++ b/arch/arm/mach-davinci/da830.c
@@ -838,7 +838,7 @@ static const struct mux_config da830_pins[] = {
 #endif
 };
 
-const short da830_emif25_pins[] __initdata = {
+const short da830_emif25_pins[] __initconst = {
 	DA830_EMA_D_0, DA830_EMA_D_1, DA830_EMA_D_2, DA830_EMA_D_3,
 	DA830_EMA_D_4, DA830_EMA_D_5, DA830_EMA_D_6, DA830_EMA_D_7,
 	DA830_EMA_D_8, DA830_EMA_D_9, DA830_EMA_D_10, DA830_EMA_D_11,
@@ -853,19 +853,19 @@ const short da830_emif25_pins[] __initdata = {
 	-1
 };
 
-const short da830_spi0_pins[] __initdata = {
+const short da830_spi0_pins[] __initconst = {
 	DA830_SPI0_SOMI_0, DA830_SPI0_SIMO_0, DA830_SPI0_CLK, DA830_NSPI0_ENA,
 	DA830_NSPI0_SCS_0,
 	-1
 };
 
-const short da830_spi1_pins[] __initdata = {
+const short da830_spi1_pins[] __initconst = {
 	DA830_SPI1_SOMI_0, DA830_SPI1_SIMO_0, DA830_SPI1_CLK, DA830_NSPI1_ENA,
 	DA830_NSPI1_SCS_0,
 	-1
 };
 
-const short da830_mmc_sd_pins[] __initdata = {
+const short da830_mmc_sd_pins[] __initconst = {
 	DA830_MMCSD_DAT_0, DA830_MMCSD_DAT_1, DA830_MMCSD_DAT_2,
 	DA830_MMCSD_DAT_3, DA830_MMCSD_DAT_4, DA830_MMCSD_DAT_5,
 	DA830_MMCSD_DAT_6, DA830_MMCSD_DAT_7, DA830_MMCSD_CLK,
@@ -873,32 +873,32 @@ const short da830_mmc_sd_pins[] __initdata = {
 	-1
 };
 
-const short da830_uart0_pins[] __initdata = {
+const short da830_uart0_pins[] __initconst = {
 	DA830_NUART0_CTS, DA830_NUART0_RTS, DA830_UART0_RXD, DA830_UART0_TXD,
 	-1
 };
 
-const short da830_uart1_pins[] __initdata = {
+const short da830_uart1_pins[] __initconst = {
 	DA830_UART1_RXD, DA830_UART1_TXD,
 	-1
 };
 
-const short da830_uart2_pins[] __initdata = {
+const short da830_uart2_pins[] __initconst = {
 	DA830_UART2_RXD, DA830_UART2_TXD,
 	-1
 };
 
-const short da830_usb20_pins[] __initdata = {
+const short da830_usb20_pins[] __initconst = {
 	DA830_USB0_DRVVBUS, DA830_USB_REFCLKIN,
 	-1
 };
 
-const short da830_usb11_pins[] __initdata = {
+const short da830_usb11_pins[] __initconst = {
 	DA830_USB_REFCLKIN,
 	-1
 };
 
-const short da830_uhpi_pins[] __initdata = {
+const short da830_uhpi_pins[] __initconst = {
 	DA830_UHPI_HD_0, DA830_UHPI_HD_1, DA830_UHPI_HD_2, DA830_UHPI_HD_3,
 	DA830_UHPI_HD_4, DA830_UHPI_HD_5, DA830_UHPI_HD_6, DA830_UHPI_HD_7,
 	DA830_UHPI_HD_8, DA830_UHPI_HD_9, DA830_UHPI_HD_10, DA830_UHPI_HD_11,
@@ -909,14 +909,14 @@ const short da830_uhpi_pins[] __initdata = {
 	-1
 };
 
-const short da830_cpgmac_pins[] __initdata = {
+const short da830_cpgmac_pins[] __initconst = {
 	DA830_RMII_TXD_0, DA830_RMII_TXD_1, DA830_RMII_TXEN, DA830_RMII_CRS_DV,
 	DA830_RMII_RXD_0, DA830_RMII_RXD_1, DA830_RMII_RXER, DA830_MDIO_CLK,
 	DA830_MDIO_D,
 	-1
 };
 
-const short da830_emif3c_pins[] __initdata = {
+const short da830_emif3c_pins[] __initconst = {
 	DA830_EMB_SDCKE, DA830_EMB_CLK_GLUE, DA830_EMB_CLK, DA830_NEMB_CS_0,
 	DA830_NEMB_CAS, DA830_NEMB_RAS, DA830_NEMB_WE, DA830_EMB_BA_1,
 	DA830_EMB_BA_0, DA830_EMB_A_0, DA830_EMB_A_1, DA830_EMB_A_2,
@@ -935,7 +935,7 @@ const short da830_emif3c_pins[] __initdata = {
 	-1
 };
 
-const short da830_mcasp0_pins[] __initdata = {
+const short da830_mcasp0_pins[] __initconst = {
 	DA830_AHCLKX0, DA830_ACLKX0, DA830_AFSX0,
 	DA830_AHCLKR0, DA830_ACLKR0, DA830_AFSR0, DA830_AMUTE0,
 	DA830_AXR0_0, DA830_AXR0_1, DA830_AXR0_2, DA830_AXR0_3,
@@ -945,7 +945,7 @@ const short da830_mcasp0_pins[] __initdata = {
 	-1
 };
 
-const short da830_mcasp1_pins[] __initdata = {
+const short da830_mcasp1_pins[] __initconst = {
 	DA830_AHCLKX1, DA830_ACLKX1, DA830_AFSX1,
 	DA830_AHCLKR1, DA830_ACLKR1, DA830_AFSR1, DA830_AMUTE1,
 	DA830_AXR1_0, DA830_AXR1_1, DA830_AXR1_2, DA830_AXR1_3,
@@ -954,24 +954,24 @@ const short da830_mcasp1_pins[] __initdata = {
 	-1
 };
 
-const short da830_mcasp2_pins[] __initdata = {
+const short da830_mcasp2_pins[] __initconst = {
 	DA830_AHCLKX2, DA830_ACLKX2, DA830_AFSX2,
 	DA830_AHCLKR2, DA830_ACLKR2, DA830_AFSR2, DA830_AMUTE2,
 	DA830_AXR2_0, DA830_AXR2_1, DA830_AXR2_2, DA830_AXR2_3,
 	-1
 };
 
-const short da830_i2c0_pins[] __initdata = {
+const short da830_i2c0_pins[] __initconst = {
 	DA830_I2C0_SDA, DA830_I2C0_SCL,
 	-1
 };
 
-const short da830_i2c1_pins[] __initdata = {
+const short da830_i2c1_pins[] __initconst = {
 	DA830_I2C1_SCL, DA830_I2C1_SDA,
 	-1
 };
 
-const short da830_lcdcntl_pins[] __initdata = {
+const short da830_lcdcntl_pins[] __initconst = {
 	DA830_LCD_D_0, DA830_LCD_D_1, DA830_LCD_D_2, DA830_LCD_D_3,
 	DA830_LCD_D_4, DA830_LCD_D_5, DA830_LCD_D_6, DA830_LCD_D_7,
 	DA830_LCD_D_8, DA830_LCD_D_9, DA830_LCD_D_10, DA830_LCD_D_11,
@@ -981,34 +981,34 @@ const short da830_lcdcntl_pins[] __initdata = {
 	-1
 };
 
-const short da830_pwm_pins[] __initdata = {
+const short da830_pwm_pins[] __initconst = {
 	DA830_ECAP0_APWM0, DA830_ECAP1_APWM1, DA830_EPWM0B, DA830_EPWM0A,
 	DA830_EPWMSYNCI, DA830_EPWMSYNC0, DA830_ECAP2_APWM2, DA830_EHRPWMGLUETZ,
 	DA830_EPWM2B, DA830_EPWM2A, DA830_EPWM1B, DA830_EPWM1A,
 	-1
 };
 
-const short da830_ecap0_pins[] __initdata = {
+const short da830_ecap0_pins[] __initconst = {
 	DA830_ECAP0_APWM0,
 	-1
 };
 
-const short da830_ecap1_pins[] __initdata = {
+const short da830_ecap1_pins[] __initconst = {
 	DA830_ECAP1_APWM1,
 	-1
 };
 
-const short da830_ecap2_pins[] __initdata = {
+const short da830_ecap2_pins[] __initconst = {
 	DA830_ECAP2_APWM2,
 	-1
 };
 
-const short da830_eqep0_pins[] __initdata = {
+const short da830_eqep0_pins[] __initconst = {
 	DA830_EQEP0I, DA830_EQEP0S, DA830_EQEP0A, DA830_EQEP0B,
 	-1
 };
 
-const short da830_eqep1_pins[] __initdata = {
+const short da830_eqep1_pins[] __initconst = {
 	DA830_EQEP1I, DA830_EQEP1S, DA830_EQEP1A, DA830_EQEP1B,
 	-1
 };
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c
index b44dc844e15..6676dee7104 100644
--- a/arch/arm/mach-davinci/da850.c
+++ b/arch/arm/mach-davinci/da850.c
@@ -576,17 +576,17 @@ static const struct mux_config da850_pins[] = {
 #endif
 };
 
-const short da850_i2c0_pins[] __initdata = {
+const short da850_i2c0_pins[] __initconst = {
 	DA850_I2C0_SDA, DA850_I2C0_SCL,
 	-1
 };
 
-const short da850_i2c1_pins[] __initdata = {
+const short da850_i2c1_pins[] __initconst = {
 	DA850_I2C1_SCL, DA850_I2C1_SDA,
 	-1
 };
 
-const short da850_lcdcntl_pins[] __initdata = {
+const short da850_lcdcntl_pins[] __initconst = {
 	DA850_LCD_D_0, DA850_LCD_D_1, DA850_LCD_D_2, DA850_LCD_D_3,
 	DA850_LCD_D_4, DA850_LCD_D_5, DA850_LCD_D_6, DA850_LCD_D_7,
 	DA850_LCD_D_8, DA850_LCD_D_9, DA850_LCD_D_10, DA850_LCD_D_11,
diff --git a/arch/arm/mach-highbank/highbank.c b/arch/arm/mach-highbank/highbank.c
index af1da34ccf9..40e36a50304 100644
--- a/arch/arm/mach-highbank/highbank.c
+++ b/arch/arm/mach-highbank/highbank.c
@@ -15,6 +15,7 @@
  */
 #include <linux/clk.h>
 #include <linux/clkdev.h>
+#include <linux/dma-mapping.h>
 #include <linux/io.h>
 #include <linux/irq.h>
 #include <linux/irqdomain.h>
@@ -23,6 +24,7 @@
 #include <linux/of_platform.h>
 #include <linux/of_address.h>
 #include <linux/smp.h>
+#include <linux/amba/bus.h>
 
 #include <asm/cacheflush.h>
 #include <asm/smp_plat.h>
@@ -149,11 +151,61 @@ static void highbank_power_off(void)
 		cpu_do_idle();
 }
 
+static int highbank_platform_notifier(struct notifier_block *nb,
+				  unsigned long event, void *__dev)
+{
+	struct resource *res;
+	int reg = -1;
+	struct device *dev = __dev;
+
+	if (event != BUS_NOTIFY_ADD_DEVICE)
+		return NOTIFY_DONE;
+
+	if (of_device_is_compatible(dev->of_node, "calxeda,hb-ahci"))
+		reg = 0xc;
+	else if (of_device_is_compatible(dev->of_node, "calxeda,hb-sdhci"))
+		reg = 0x18;
+	else if (of_device_is_compatible(dev->of_node, "arm,pl330"))
+		reg = 0x20;
+	else if (of_device_is_compatible(dev->of_node, "calxeda,hb-xgmac")) {
+		res = platform_get_resource(to_platform_device(dev),
+					    IORESOURCE_MEM, 0);
+		if (res) {
+			if (res->start == 0xfff50000)
+				reg = 0;
+			else if (res->start == 0xfff51000)
+				reg = 4;
+		}
+	}
+
+	if (reg < 0)
+		return NOTIFY_DONE;
+
+	if (of_property_read_bool(dev->of_node, "dma-coherent")) {
+		writel(0xff31, sregs_base + reg);
+		set_dma_ops(dev, &arm_coherent_dma_ops);
+	} else
+		writel(0, sregs_base + reg);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block highbank_amba_nb = {
+	.notifier_call = highbank_platform_notifier,
+};
+
+static struct notifier_block highbank_platform_nb = {
+	.notifier_call = highbank_platform_notifier,
+};
+
 static void __init highbank_init(void)
 {
 	pm_power_off = highbank_power_off;
 	highbank_pm_init();
 
+	bus_register_notifier(&platform_bus_type, &highbank_platform_nb);
+	bus_register_notifier(&amba_bustype, &highbank_amba_nb);
+
 	of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
 }
 
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index 3a2042fb971..32197c117af 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -758,7 +758,7 @@ config SOC_IMX6Q
 	select HAVE_IMX_MMDC
 	select HAVE_IMX_SRC
 	select HAVE_SMP
-	select MFD_ANATOP
+	select MFD_SYSCON
 	select PINCTRL
 	select PINCTRL_IMX6Q
 
diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c
index 36979d3dfe3..47c91f7185d 100644
--- a/arch/arm/mach-imx/mach-imx6q.c
+++ b/arch/arm/mach-imx/mach-imx6q.c
@@ -23,8 +23,9 @@
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/phy.h>
+#include <linux/regmap.h>
 #include <linux/micrel_phy.h>
-#include <linux/mfd/anatop.h>
+#include <linux/mfd/syscon.h>
 #include <asm/cpuidle.h>
 #include <asm/smp_twd.h>
 #include <asm/hardware/cache-l2x0.h>
@@ -118,20 +119,7 @@ static void __init imx6q_sabrelite_init(void)
 
 static void __init imx6q_usb_init(void)
 {
-	struct device_node *np;
-	struct platform_device *pdev = NULL;
-	struct anatop *adata = NULL;
-
-	np = of_find_compatible_node(NULL, NULL, "fsl,imx6q-anatop");
-	if (np)
-		pdev = of_find_device_by_node(np);
-	if (pdev)
-		adata = platform_get_drvdata(pdev);
-	if (!adata) {
-		if (np)
-			of_node_put(np);
-		return;
-	}
+	struct regmap *anatop;
 
 #define HW_ANADIG_USB1_CHRG_DETECT		0x000001b0
 #define HW_ANADIG_USB2_CHRG_DETECT		0x00000210
@@ -139,20 +127,21 @@ static void __init imx6q_usb_init(void)
 #define BM_ANADIG_USB_CHRG_DETECT_EN_B		0x00100000
 #define BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B	0x00080000
 
-	/*
-	 * The external charger detector needs to be disabled,
-	 * or the signal at DP will be poor
-	 */
-	anatop_write_reg(adata, HW_ANADIG_USB1_CHRG_DETECT,
-			BM_ANADIG_USB_CHRG_DETECT_EN_B
-			| BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B,
-			~0);
-	anatop_write_reg(adata, HW_ANADIG_USB2_CHRG_DETECT,
-			BM_ANADIG_USB_CHRG_DETECT_EN_B |
-			BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B,
-			~0);
-
-	of_node_put(np);
+	anatop = syscon_regmap_lookup_by_compatible("fsl,imx6q-anatop");
+	if (!IS_ERR(anatop)) {
+		/*
+		 * The external charger detector needs to be disabled,
+		 * or the signal at DP will be poor
+		 */
+		regmap_write(anatop, HW_ANADIG_USB1_CHRG_DETECT,
+				BM_ANADIG_USB_CHRG_DETECT_EN_B
+				| BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B);
+		regmap_write(anatop, HW_ANADIG_USB2_CHRG_DETECT,
+				BM_ANADIG_USB_CHRG_DETECT_EN_B |
+				BM_ANADIG_USB_CHRG_DETECT_CHK_CHRG_B);
+	} else {
+		pr_warn("failed to find fsl,imx6q-anatop regmap\n");
+	}
 }
 
 static void __init imx6q_init_machine(void)
diff --git a/arch/arm/mach-msm/board-qsd8x50.c b/arch/arm/mach-msm/board-qsd8x50.c
index a344a373928..2448fcf09eb 100644
--- a/arch/arm/mach-msm/board-qsd8x50.c
+++ b/arch/arm/mach-msm/board-qsd8x50.c
@@ -37,8 +37,8 @@
 #include "devices.h"
 #include "common.h"
 
-static const resource_size_t qsd8x50_surf_smc91x_base __initdata = 0x70000300;
-static const unsigned        qsd8x50_surf_smc91x_gpio __initdata = 156;
+static const resource_size_t qsd8x50_surf_smc91x_base __initconst = 0x70000300;
+static const unsigned        qsd8x50_surf_smc91x_gpio __initconst = 156;
 
 /* Leave smc91x resources empty here, as we'll fill them in
  * at run-time: they vary from board to board, and the true
diff --git a/arch/arm/mach-omap2/display.c b/arch/arm/mach-omap2/display.c
index af1ed7d24a1..e470c6e50ac 100644
--- a/arch/arm/mach-omap2/display.c
+++ b/arch/arm/mach-omap2/display.c
@@ -76,14 +76,14 @@ struct omap_dss_hwmod_data {
 	const int id;
 };
 
-static const struct omap_dss_hwmod_data omap2_dss_hwmod_data[] __initdata = {
+static const struct omap_dss_hwmod_data omap2_dss_hwmod_data[] __initconst = {
 	{ "dss_core", "omapdss_dss", -1 },
 	{ "dss_dispc", "omapdss_dispc", -1 },
 	{ "dss_rfbi", "omapdss_rfbi", -1 },
 	{ "dss_venc", "omapdss_venc", -1 },
 };
 
-static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initdata = {
+static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initconst = {
 	{ "dss_core", "omapdss_dss", -1 },
 	{ "dss_dispc", "omapdss_dispc", -1 },
 	{ "dss_rfbi", "omapdss_rfbi", -1 },
@@ -91,7 +91,7 @@ static const struct omap_dss_hwmod_data omap3_dss_hwmod_data[] __initdata = {
 	{ "dss_dsi1", "omapdss_dsi", 0 },
 };
 
-static const struct omap_dss_hwmod_data omap4_dss_hwmod_data[] __initdata = {
+static const struct omap_dss_hwmod_data omap4_dss_hwmod_data[] __initconst = {
 	{ "dss_core", "omapdss_dss", -1 },
 	{ "dss_dispc", "omapdss_dispc", -1 },
 	{ "dss_rfbi", "omapdss_rfbi", -1 },
diff --git a/arch/arm/mach-omap2/usb-host.c b/arch/arm/mach-omap2/usb-host.c
index ac95daaa470..3c434498e12 100644
--- a/arch/arm/mach-omap2/usb-host.c
+++ b/arch/arm/mach-omap2/usb-host.c
@@ -33,10 +33,12 @@
 #ifdef CONFIG_MFD_OMAP_USB_HOST
 
 #define OMAP_USBHS_DEVICE	"usbhs_omap"
+#define OMAP_USBTLL_DEVICE	"usbhs_tll"
 #define	USBHS_UHH_HWMODNAME	"usb_host_hs"
 #define USBHS_TLL_HWMODNAME	"usb_tll_hs"
 
 static struct usbhs_omap_platform_data		usbhs_data;
+static struct usbtll_omap_platform_data		usbtll_data;
 static struct ehci_hcd_omap_platform_data	ehci_data;
 static struct ohci_hcd_omap_platform_data	ohci_data;
 
@@ -485,13 +487,14 @@ void __init setup_4430ohci_io_mux(const enum usbhs_omap_port_mode *port_mode)
 
 void __init usbhs_init(const struct usbhs_omap_board_data *pdata)
 {
-	struct omap_hwmod	*oh[2];
+	struct omap_hwmod	*uhh_hwm, *tll_hwm;
 	struct platform_device	*pdev;
 	int			bus_id = -1;
 	int			i;
 
 	for (i = 0; i < OMAP3_HS_USB_PORTS; i++) {
 		usbhs_data.port_mode[i] = pdata->port_mode[i];
+		usbtll_data.port_mode[i] = pdata->port_mode[i];
 		ohci_data.port_mode[i] = pdata->port_mode[i];
 		ehci_data.port_mode[i] = pdata->port_mode[i];
 		ehci_data.reset_gpio_port[i] = pdata->reset_gpio_port[i];
@@ -510,25 +513,35 @@ void __init usbhs_init(const struct usbhs_omap_board_data *pdata)
 		setup_4430ohci_io_mux(pdata->port_mode);
 	}
 
-	oh[0] = omap_hwmod_lookup(USBHS_UHH_HWMODNAME);
-	if (!oh[0]) {
+	uhh_hwm = omap_hwmod_lookup(USBHS_UHH_HWMODNAME);
+	if (!uhh_hwm) {
 		pr_err("Could not look up %s\n", USBHS_UHH_HWMODNAME);
 		return;
 	}
 
-	oh[1] = omap_hwmod_lookup(USBHS_TLL_HWMODNAME);
-	if (!oh[1]) {
+	tll_hwm = omap_hwmod_lookup(USBHS_TLL_HWMODNAME);
+	if (!tll_hwm) {
 		pr_err("Could not look up %s\n", USBHS_TLL_HWMODNAME);
 		return;
 	}
 
-	pdev = omap_device_build_ss(OMAP_USBHS_DEVICE, bus_id, oh, 2,
-				(void *)&usbhs_data, sizeof(usbhs_data),
+	pdev = omap_device_build(OMAP_USBTLL_DEVICE, bus_id, tll_hwm,
+				&usbtll_data, sizeof(usbtll_data),
 				omap_uhhtll_latency,
 				ARRAY_SIZE(omap_uhhtll_latency), false);
 	if (IS_ERR(pdev)) {
-		pr_err("Could not build hwmod devices %s,%s\n",
-			USBHS_UHH_HWMODNAME, USBHS_TLL_HWMODNAME);
+		pr_err("Could not build hwmod device %s\n",
+		       USBHS_TLL_HWMODNAME);
+		return;
+	}
+
+	pdev = omap_device_build(OMAP_USBHS_DEVICE, bus_id, uhh_hwm,
+				&usbhs_data, sizeof(usbhs_data),
+				omap_uhhtll_latency,
+				ARRAY_SIZE(omap_uhhtll_latency), false);
+	if (IS_ERR(pdev)) {
+		pr_err("Could not build hwmod devices %s\n",
+		       USBHS_UHH_HWMODNAME);
 		return;
 	}
 }
diff --git a/arch/arm/mach-shmobile/Makefile b/arch/arm/mach-shmobile/Makefile
index 0df5ae6740c..fe2c97c179d 100644
--- a/arch/arm/mach-shmobile/Makefile
+++ b/arch/arm/mach-shmobile/Makefile
@@ -3,7 +3,7 @@
 #
 
 # Common objects
-obj-y				:= timer.o console.o clock.o common.o
+obj-y				:= timer.o console.o clock.o
 
 # CPU objects
 obj-$(CONFIG_ARCH_SH7367)	+= setup-sh7367.o clock-sh7367.o intc-sh7367.o
diff --git a/arch/arm/mach-shmobile/board-ap4evb.c b/arch/arm/mach-shmobile/board-ap4evb.c
index bc3b5da59e2..790dc68c431 100644
--- a/arch/arm/mach-shmobile/board-ap4evb.c
+++ b/arch/arm/mach-shmobile/board-ap4evb.c
@@ -1231,6 +1231,15 @@ static struct i2c_board_info i2c1_devices[] = {
 #define USCCR1		IOMEM(0xE6058144)
 static void __init ap4evb_init(void)
 {
+	struct pm_domain_device domain_devices[] = {
+		{ "A4LC", &lcdc1_device, },
+		{ "A4LC", &lcdc_device, },
+		{ "A4MP", &fsi_device, },
+		{ "A3SP", &sh_mmcif_device, },
+		{ "A3SP", &sdhi0_device, },
+		{ "A3SP", &sdhi1_device, },
+		{ "A4R", &ceu_device, },
+	};
 	u32 srcr4;
 	struct clk *clk;
 
@@ -1463,14 +1472,8 @@ static void __init ap4evb_init(void)
 
 	platform_add_devices(ap4evb_devices, ARRAY_SIZE(ap4evb_devices));
 
-	rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4mp, &fsi_device);
-
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sh_mmcif_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &ceu_device);
+	rmobile_add_devices_to_domains(domain_devices,
+				       ARRAY_SIZE(domain_devices));
 
 	hdmi_init_pm_clock();
 	fsi_init_pm_clock();
@@ -1485,6 +1488,6 @@ MACHINE_START(AP4EVB, "ap4evb")
 	.init_irq	= sh7372_init_irq,
 	.handle_irq	= shmobile_handle_irq_intc,
 	.init_machine	= ap4evb_init,
-	.init_late	= shmobile_init_late,
+	.init_late	= sh7372_pm_init_late,
 	.timer		= &shmobile_timer,
 MACHINE_END
diff --git a/arch/arm/mach-shmobile/board-armadillo800eva.c b/arch/arm/mach-shmobile/board-armadillo800eva.c
index c6593d39427..2912eab3b96 100644
--- a/arch/arm/mach-shmobile/board-armadillo800eva.c
+++ b/arch/arm/mach-shmobile/board-armadillo800eva.c
@@ -1209,10 +1209,10 @@ static void __init eva_init(void)
 
 	eva_clock_init();
 
-	rmobile_add_device_to_domain(&r8a7740_pd_a4lc, &lcdc0_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a4lc, &hdmi_lcdc_device);
+	rmobile_add_device_to_domain("A4LC", &lcdc0_device);
+	rmobile_add_device_to_domain("A4LC", &hdmi_lcdc_device);
 	if (usb)
-		rmobile_add_device_to_domain(&r8a7740_pd_a3sp, usb);
+		rmobile_add_device_to_domain("A3SP", usb);
 }
 
 static void __init eva_earlytimer_init(void)
diff --git a/arch/arm/mach-shmobile/board-mackerel.c b/arch/arm/mach-shmobile/board-mackerel.c
index 62783b5d881..0c27c810cf9 100644
--- a/arch/arm/mach-shmobile/board-mackerel.c
+++ b/arch/arm/mach-shmobile/board-mackerel.c
@@ -1412,6 +1412,22 @@ static struct i2c_board_info i2c1_devices[] = {
 #define USCCR1		IOMEM(0xE6058144)
 static void __init mackerel_init(void)
 {
+	struct pm_domain_device domain_devices[] = {
+		{ "A4LC", &lcdc_device, },
+		{ "A4LC", &hdmi_lcdc_device, },
+		{ "A4LC", &meram_device, },
+		{ "A4MP", &fsi_device, },
+		{ "A3SP", &usbhs0_device, },
+		{ "A3SP", &usbhs1_device, },
+		{ "A3SP", &nand_flash_device, },
+		{ "A3SP", &sh_mmcif_device, },
+		{ "A3SP", &sdhi0_device, },
+#if !defined(CONFIG_MMC_SH_MMCIF) && !defined(CONFIG_MMC_SH_MMCIF_MODULE)
+		{ "A3SP", &sdhi1_device, },
+#endif
+		{ "A3SP", &sdhi2_device, },
+		{ "A4R", &ceu_device, },
+	};
 	u32 srcr4;
 	struct clk *clk;
 
@@ -1626,20 +1642,8 @@ static void __init mackerel_init(void)
 
 	platform_add_devices(mackerel_devices, ARRAY_SIZE(mackerel_devices));
 
-	rmobile_add_device_to_domain(&sh7372_pd_a4lc, &lcdc_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4lc, &hdmi_lcdc_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4lc, &meram_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4mp, &fsi_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usbhs0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usbhs1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &nand_flash_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sh_mmcif_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi0_device);
-#if !defined(CONFIG_MMC_SH_MMCIF) && !defined(CONFIG_MMC_SH_MMCIF_MODULE)
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi1_device);
-#endif
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &sdhi2_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &ceu_device);
+	rmobile_add_devices_to_domains(domain_devices,
+				       ARRAY_SIZE(domain_devices));
 
 	hdmi_init_pm_clock();
 	sh7372_pm_init();
@@ -1653,6 +1657,6 @@ MACHINE_START(MACKEREL, "mackerel")
 	.init_irq	= sh7372_init_irq,
 	.handle_irq	= shmobile_handle_irq_intc,
 	.init_machine	= mackerel_init,
-	.init_late	= shmobile_init_late,
+	.init_late	= sh7372_pm_init_late,
 	.timer		= &shmobile_timer,
 MACHINE_END
diff --git a/arch/arm/mach-shmobile/common.c b/arch/arm/mach-shmobile/common.c
deleted file mode 100644
index 608aba9d60d..00000000000
--- a/arch/arm/mach-shmobile/common.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- */
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <mach/common.h>
-
-void __init shmobile_init_late(void)
-{
-	shmobile_suspend_init();
-	shmobile_cpuidle_init();
-}
diff --git a/arch/arm/mach-shmobile/cpuidle.c b/arch/arm/mach-shmobile/cpuidle.c
index 7b541e911ab..9e050268cde 100644
--- a/arch/arm/mach-shmobile/cpuidle.c
+++ b/arch/arm/mach-shmobile/cpuidle.c
@@ -16,51 +16,38 @@
 #include <asm/cpuidle.h>
 #include <asm/io.h>
 
-static void shmobile_enter_wfi(void)
+int shmobile_enter_wfi(struct cpuidle_device *dev, struct cpuidle_driver *drv,
+		       int index)
 {
 	cpu_do_idle();
-}
-
-void (*shmobile_cpuidle_modes[CPUIDLE_STATE_MAX])(void) = {
-	shmobile_enter_wfi, /* regular sleep mode */
-};
-
-static int shmobile_cpuidle_enter(struct cpuidle_device *dev,
-				  struct cpuidle_driver *drv,
-				  int index)
-{
-	shmobile_cpuidle_modes[index]();
-
-	return index;
+	return 0;
 }
 
 static struct cpuidle_device shmobile_cpuidle_dev;
-static struct cpuidle_driver shmobile_cpuidle_driver = {
+static struct cpuidle_driver shmobile_cpuidle_default_driver = {
 	.name			= "shmobile_cpuidle",
 	.owner			= THIS_MODULE,
 	.en_core_tk_irqen	= 1,
 	.states[0]		= ARM_CPUIDLE_WFI_STATE,
+	.states[0].enter	= shmobile_enter_wfi,
 	.safe_state_index	= 0, /* C1 */
 	.state_count		= 1,
 };
 
-void (*shmobile_cpuidle_setup)(struct cpuidle_driver *drv);
+static struct cpuidle_driver *cpuidle_drv = &shmobile_cpuidle_default_driver;
+
+void shmobile_cpuidle_set_driver(struct cpuidle_driver *drv)
+{
+	cpuidle_drv = drv;
+}
 
 int shmobile_cpuidle_init(void)
 {
 	struct cpuidle_device *dev = &shmobile_cpuidle_dev;
-	struct cpuidle_driver *drv = &shmobile_cpuidle_driver;
-	int i;
-
-	for (i = 0; i < CPUIDLE_STATE_MAX; i++)
-		drv->states[i].enter = shmobile_cpuidle_enter;
-
-	if (shmobile_cpuidle_setup)
-		shmobile_cpuidle_setup(drv);
 
-	cpuidle_register_driver(drv);
+	cpuidle_register_driver(cpuidle_drv);
 
-	dev->state_count = drv->state_count;
+	dev->state_count = cpuidle_drv->state_count;
 	cpuidle_register_device(dev);
 
 	return 0;
diff --git a/arch/arm/mach-shmobile/include/mach/common.h b/arch/arm/mach-shmobile/include/mach/common.h
index f80f9c54939..ed77ab8c914 100644
--- a/arch/arm/mach-shmobile/include/mach/common.h
+++ b/arch/arm/mach-shmobile/include/mach/common.h
@@ -13,8 +13,10 @@ extern int shmobile_clk_init(void);
 extern void shmobile_handle_irq_intc(struct pt_regs *);
 extern struct platform_suspend_ops shmobile_suspend_ops;
 struct cpuidle_driver;
-extern void (*shmobile_cpuidle_modes[])(void);
-extern void (*shmobile_cpuidle_setup)(struct cpuidle_driver *drv);
+struct cpuidle_device;
+extern int shmobile_enter_wfi(struct cpuidle_device *dev,
+			      struct cpuidle_driver *drv, int index);
+extern void shmobile_cpuidle_set_driver(struct cpuidle_driver *drv);
 
 extern void sh7367_init_irq(void);
 extern void sh7367_map_io(void);
@@ -75,8 +77,6 @@ extern void r8a7740_meram_workaround(void);
 
 extern void r8a7779_register_twd(void);
 
-extern void shmobile_init_late(void);
-
 #ifdef CONFIG_SUSPEND
 int shmobile_suspend_init(void);
 #else
@@ -100,4 +100,10 @@ static inline int shmobile_cpu_is_dead(unsigned int cpu) { return 1; }
 
 extern void shmobile_smp_init_cpus(unsigned int ncores);
 
+static inline void shmobile_init_late(void)
+{
+	shmobile_suspend_init();
+	shmobile_cpuidle_init();
+}
+
 #endif /* __ARCH_MACH_COMMON_H */
diff --git a/arch/arm/mach-shmobile/include/mach/pm-rmobile.h b/arch/arm/mach-shmobile/include/mach/pm-rmobile.h
index 5a402840fe2..690553a0688 100644
--- a/arch/arm/mach-shmobile/include/mach/pm-rmobile.h
+++ b/arch/arm/mach-shmobile/include/mach/pm-rmobile.h
@@ -12,6 +12,8 @@
 
 #include <linux/pm_domain.h>
 
+#define DEFAULT_DEV_LATENCY_NS	250000
+
 struct platform_device;
 
 struct rmobile_pm_domain {
@@ -29,16 +31,33 @@ struct rmobile_pm_domain *to_rmobile_pd(struct generic_pm_domain *d)
 	return container_of(d, struct rmobile_pm_domain, genpd);
 }
 
+struct pm_domain_device {
+	const char *domain_name;
+	struct platform_device *pdev;
+};
+
 #ifdef CONFIG_PM
-extern void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd);
-extern void rmobile_add_device_to_domain(struct rmobile_pm_domain *rmobile_pd,
-					struct platform_device *pdev);
-extern void rmobile_pm_add_subdomain(struct rmobile_pm_domain *rmobile_pd,
-				    struct rmobile_pm_domain *rmobile_sd);
+extern void rmobile_init_domains(struct rmobile_pm_domain domains[], int num);
+extern void rmobile_add_device_to_domain_td(const char *domain_name,
+					    struct platform_device *pdev,
+					    struct gpd_timing_data *td);
+
+static inline void rmobile_add_device_to_domain(const char *domain_name,
+						struct platform_device *pdev)
+{
+	rmobile_add_device_to_domain_td(domain_name, pdev, NULL);
+}
+
+extern void rmobile_add_devices_to_domains(struct pm_domain_device data[],
+					   int size);
 #else
-#define rmobile_init_pm_domain(pd) do { } while (0)
-#define rmobile_add_device_to_domain(pd, pdev) do { } while (0)
-#define rmobile_pm_add_subdomain(pd, sd) do { } while (0)
+
+#define rmobile_init_domains(domains, num) do { } while (0)
+#define rmobile_add_device_to_domain_td(name, pdev, td) do { } while (0)
+#define rmobile_add_device_to_domain(name, pdev) do { } while (0)
+
+static inline void rmobile_add_devices_to_domains(struct pm_domain_device d[],
+						  int size) {}
 #endif /* CONFIG_PM */
 
 #endif /* PM_RMOBILE_H */
diff --git a/arch/arm/mach-shmobile/include/mach/r8a7740.h b/arch/arm/mach-shmobile/include/mach/r8a7740.h
index 7143147780d..59d252f4cf9 100644
--- a/arch/arm/mach-shmobile/include/mach/r8a7740.h
+++ b/arch/arm/mach-shmobile/include/mach/r8a7740.h
@@ -607,9 +607,9 @@ enum {
 };
 
 #ifdef CONFIG_PM
-extern struct rmobile_pm_domain r8a7740_pd_a4s;
-extern struct rmobile_pm_domain r8a7740_pd_a3sp;
-extern struct rmobile_pm_domain r8a7740_pd_a4lc;
+extern void __init r8a7740_init_pm_domains(void);
+#else
+static inline void r8a7740_init_pm_domains(void) {}
 #endif /* CONFIG_PM */
 
 #endif /* __ASM_R8A7740_H__ */
diff --git a/arch/arm/mach-shmobile/include/mach/r8a7779.h b/arch/arm/mach-shmobile/include/mach/r8a7779.h
index f504c5e81b4..499f52d2a4a 100644
--- a/arch/arm/mach-shmobile/include/mach/r8a7779.h
+++ b/arch/arm/mach-shmobile/include/mach/r8a7779.h
@@ -347,17 +347,9 @@ extern int r8a7779_sysc_power_down(struct r8a7779_pm_ch *r8a7779_ch);
 extern int r8a7779_sysc_power_up(struct r8a7779_pm_ch *r8a7779_ch);
 
 #ifdef CONFIG_PM
-extern struct r8a7779_pm_domain r8a7779_sh4a;
-extern struct r8a7779_pm_domain r8a7779_sgx;
-extern struct r8a7779_pm_domain r8a7779_vdp1;
-extern struct r8a7779_pm_domain r8a7779_impx3;
-
-extern void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd);
-extern void r8a7779_add_device_to_domain(struct r8a7779_pm_domain *r8a7779_pd,
-					struct platform_device *pdev);
+extern void __init r8a7779_init_pm_domains(void);
 #else
-#define r8a7779_init_pm_domain(pd) do { } while (0)
-#define r8a7779_add_device_to_domain(pd, pdev) do { } while (0)
+static inline void r8a7779_init_pm_domains(void) {}
 #endif /* CONFIG_PM */
 
 extern struct smp_operations r8a7779_smp_ops;
diff --git a/arch/arm/mach-shmobile/include/mach/sh7372.h b/arch/arm/mach-shmobile/include/mach/sh7372.h
index b59048e6d8f..eb98b45c508 100644
--- a/arch/arm/mach-shmobile/include/mach/sh7372.h
+++ b/arch/arm/mach-shmobile/include/mach/sh7372.h
@@ -478,21 +478,17 @@ extern struct clk sh7372_fsibck_clk;
 extern struct clk sh7372_fsidiva_clk;
 extern struct clk sh7372_fsidivb_clk;
 
-#ifdef CONFIG_PM
-extern struct rmobile_pm_domain sh7372_pd_a4lc;
-extern struct rmobile_pm_domain sh7372_pd_a4mp;
-extern struct rmobile_pm_domain sh7372_pd_d4;
-extern struct rmobile_pm_domain sh7372_pd_a4r;
-extern struct rmobile_pm_domain sh7372_pd_a3rv;
-extern struct rmobile_pm_domain sh7372_pd_a3ri;
-extern struct rmobile_pm_domain sh7372_pd_a4s;
-extern struct rmobile_pm_domain sh7372_pd_a3sp;
-extern struct rmobile_pm_domain sh7372_pd_a3sg;
-#endif /* CONFIG_PM */
-
 extern void sh7372_intcs_suspend(void);
 extern void sh7372_intcs_resume(void);
 extern void sh7372_intca_suspend(void);
 extern void sh7372_intca_resume(void);
 
+#ifdef CONFIG_PM
+extern void __init sh7372_init_pm_domains(void);
+#else
+static inline void sh7372_init_pm_domains(void) {}
+#endif
+
+extern void __init sh7372_pm_init_late(void);
+
 #endif /* __ASM_SH7372_H__ */
diff --git a/arch/arm/mach-shmobile/pm-r8a7740.c b/arch/arm/mach-shmobile/pm-r8a7740.c
index 893504d012a..21e5316d2d8 100644
--- a/arch/arm/mach-shmobile/pm-r8a7740.c
+++ b/arch/arm/mach-shmobile/pm-r8a7740.c
@@ -21,14 +21,6 @@ static int r8a7740_pd_a4s_suspend(void)
 	return -EBUSY;
 }
 
-struct rmobile_pm_domain r8a7740_pd_a4s = {
-	.genpd.name	= "A4S",
-	.bit_shift	= 10,
-	.gov		= &pm_domain_always_on_gov,
-	.no_debug	= true,
-	.suspend	= r8a7740_pd_a4s_suspend,
-};
-
 static int r8a7740_pd_a3sp_suspend(void)
 {
 	/*
@@ -38,17 +30,31 @@ static int r8a7740_pd_a3sp_suspend(void)
 	return console_suspend_enabled ? 0 : -EBUSY;
 }
 
-struct rmobile_pm_domain r8a7740_pd_a3sp = {
-	.genpd.name	= "A3SP",
-	.bit_shift	= 11,
-	.gov		= &pm_domain_always_on_gov,
-	.no_debug	= true,
-	.suspend	= r8a7740_pd_a3sp_suspend,
+static struct rmobile_pm_domain r8a7740_pm_domains[] = {
+	{
+		.genpd.name	= "A4S",
+		.bit_shift	= 10,
+		.gov		= &pm_domain_always_on_gov,
+		.no_debug	= true,
+		.suspend	= r8a7740_pd_a4s_suspend,
+	},
+	{
+		.genpd.name	= "A3SP",
+		.bit_shift	= 11,
+		.gov		= &pm_domain_always_on_gov,
+		.no_debug	= true,
+		.suspend	= r8a7740_pd_a3sp_suspend,
+	},
+	{
+		.genpd.name	= "A4LC",
+		.bit_shift	= 1,
+	},
 };
 
-struct rmobile_pm_domain r8a7740_pd_a4lc = {
-	.genpd.name	= "A4LC",
-	.bit_shift	= 1,
-};
+void __init r8a7740_init_pm_domains(void)
+{
+	rmobile_init_domains(r8a7740_pm_domains, ARRAY_SIZE(r8a7740_pm_domains));
+	pm_genpd_add_subdomain_names("A4S", "A3SP");
+}
 
 #endif /* CONFIG_PM */
diff --git a/arch/arm/mach-shmobile/pm-r8a7779.c b/arch/arm/mach-shmobile/pm-r8a7779.c
index a18a4ae16d2..d50a8e9b94a 100644
--- a/arch/arm/mach-shmobile/pm-r8a7779.c
+++ b/arch/arm/mach-shmobile/pm-r8a7779.c
@@ -183,7 +183,7 @@ static bool pd_active_wakeup(struct device *dev)
 	return true;
 }
 
-void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd)
+static void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd)
 {
 	struct generic_pm_domain *genpd = &r8a7779_pd->genpd;
 
@@ -199,43 +199,44 @@ void r8a7779_init_pm_domain(struct r8a7779_pm_domain *r8a7779_pd)
 		pd_power_up(&r8a7779_pd->genpd);
 }
 
-void r8a7779_add_device_to_domain(struct r8a7779_pm_domain *r8a7779_pd,
-				 struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-
-	pm_genpd_add_device(&r8a7779_pd->genpd, dev);
-	if (pm_clk_no_clocks(dev))
-		pm_clk_add(dev, NULL);
-}
-
-struct r8a7779_pm_domain r8a7779_sh4a = {
-	.ch = {
-		.chan_offs = 0x80, /* PWRSR1 .. PWRER1 */
-		.isr_bit = 16, /* SH4A */
-	}
-};
-
-struct r8a7779_pm_domain r8a7779_sgx = {
-	.ch = {
-		.chan_offs = 0xc0, /* PWRSR2 .. PWRER2 */
-		.isr_bit = 20, /* SGX */
-	}
+static struct r8a7779_pm_domain r8a7779_pm_domains[] = {
+	{
+		.genpd.name = "SH4A",
+		.ch = {
+			.chan_offs = 0x80, /* PWRSR1 .. PWRER1 */
+			.isr_bit = 16, /* SH4A */
+		},
+	},
+	{
+		.genpd.name = "SGX",
+		.ch = {
+			.chan_offs = 0xc0, /* PWRSR2 .. PWRER2 */
+			.isr_bit = 20, /* SGX */
+		},
+	},
+	{
+		.genpd.name = "VDP1",
+		.ch = {
+			.chan_offs = 0x100, /* PWRSR3 .. PWRER3 */
+			.isr_bit = 21, /* VDP */
+		},
+	},
+	{
+		.genpd.name = "IMPX3",
+		.ch = {
+			.chan_offs = 0x140, /* PWRSR4 .. PWRER4 */
+			.isr_bit = 24, /* IMP */
+		},
+	},
 };
 
-struct r8a7779_pm_domain r8a7779_vdp1 = {
-	.ch = {
-		.chan_offs = 0x100, /* PWRSR3 .. PWRER3 */
-		.isr_bit = 21, /* VDP */
-	}
-};
+void __init r8a7779_init_pm_domains(void)
+{
+	int j;
 
-struct r8a7779_pm_domain r8a7779_impx3 = {
-	.ch = {
-		.chan_offs = 0x140, /* PWRSR4 .. PWRER4 */
-		.isr_bit = 24, /* IMP */
-	}
-};
+	for (j = 0; j < ARRAY_SIZE(r8a7779_pm_domains); j++)
+		r8a7779_init_pm_domain(&r8a7779_pm_domains[j]);
+}
 
 #endif /* CONFIG_PM */
 
diff --git a/arch/arm/mach-shmobile/pm-rmobile.c b/arch/arm/mach-shmobile/pm-rmobile.c
index 32e177275e4..1fc05d9453d 100644
--- a/arch/arm/mach-shmobile/pm-rmobile.c
+++ b/arch/arm/mach-shmobile/pm-rmobile.c
@@ -134,7 +134,7 @@ static int rmobile_pd_start_dev(struct device *dev)
 	return ret;
 }
 
-void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd)
+static void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd)
 {
 	struct generic_pm_domain *genpd = &rmobile_pd->genpd;
 	struct dev_power_governor *gov = rmobile_pd->gov;
@@ -149,19 +149,38 @@ void rmobile_init_pm_domain(struct rmobile_pm_domain *rmobile_pd)
 	__rmobile_pd_power_up(rmobile_pd, false);
 }
 
-void rmobile_add_device_to_domain(struct rmobile_pm_domain *rmobile_pd,
-				 struct platform_device *pdev)
+void rmobile_init_domains(struct rmobile_pm_domain domains[], int num)
+{
+	int j;
+
+	for (j = 0; j < num; j++)
+		rmobile_init_pm_domain(&domains[j]);
+}
+
+void rmobile_add_device_to_domain_td(const char *domain_name,
+				     struct platform_device *pdev,
+				     struct gpd_timing_data *td)
 {
 	struct device *dev = &pdev->dev;
 
-	pm_genpd_add_device(&rmobile_pd->genpd, dev);
+	__pm_genpd_name_add_device(domain_name, dev, td);
 	if (pm_clk_no_clocks(dev))
 		pm_clk_add(dev, NULL);
 }
 
-void rmobile_pm_add_subdomain(struct rmobile_pm_domain *rmobile_pd,
-			     struct rmobile_pm_domain *rmobile_sd)
+void rmobile_add_devices_to_domains(struct pm_domain_device data[],
+				    int size)
 {
-	pm_genpd_add_subdomain(&rmobile_pd->genpd, &rmobile_sd->genpd);
+	struct gpd_timing_data latencies = {
+		.stop_latency_ns = DEFAULT_DEV_LATENCY_NS,
+		.start_latency_ns = DEFAULT_DEV_LATENCY_NS,
+		.save_state_latency_ns = DEFAULT_DEV_LATENCY_NS,
+		.restore_state_latency_ns = DEFAULT_DEV_LATENCY_NS,
+	};
+	int j;
+
+	for (j = 0; j < size; j++)
+		rmobile_add_device_to_domain_td(data[j].domain_name,
+						data[j].pdev, &latencies);
 }
 #endif /* CONFIG_PM */
diff --git a/arch/arm/mach-shmobile/pm-sh7372.c b/arch/arm/mach-shmobile/pm-sh7372.c
index 162121842a2..a0826a48dd0 100644
--- a/arch/arm/mach-shmobile/pm-sh7372.c
+++ b/arch/arm/mach-shmobile/pm-sh7372.c
@@ -21,6 +21,7 @@
 #include <linux/irq.h>
 #include <linux/bitrev.h>
 #include <linux/console.h>
+#include <asm/cpuidle.h>
 #include <asm/io.h>
 #include <asm/tlbflush.h>
 #include <asm/suspend.h>
@@ -72,20 +73,7 @@
 
 #ifdef CONFIG_PM
 
-struct rmobile_pm_domain sh7372_pd_a4lc = {
-	.genpd.name = "A4LC",
-	.bit_shift = 1,
-};
-
-struct rmobile_pm_domain sh7372_pd_a4mp = {
-	.genpd.name = "A4MP",
-	.bit_shift = 2,
-};
-
-struct rmobile_pm_domain sh7372_pd_d4 = {
-	.genpd.name = "D4",
-	.bit_shift = 3,
-};
+#define PM_DOMAIN_ON_OFF_LATENCY_NS	250000
 
 static int sh7372_a4r_pd_suspend(void)
 {
@@ -94,39 +82,25 @@ static int sh7372_a4r_pd_suspend(void)
 	return 0;
 }
 
-struct rmobile_pm_domain sh7372_pd_a4r = {
-	.genpd.name = "A4R",
-	.bit_shift = 5,
-	.suspend = sh7372_a4r_pd_suspend,
-	.resume = sh7372_intcs_resume,
-};
+static bool a4s_suspend_ready;
 
-struct rmobile_pm_domain sh7372_pd_a3rv = {
-	.genpd.name = "A3RV",
-	.bit_shift = 6,
-};
-
-struct rmobile_pm_domain sh7372_pd_a3ri = {
-	.genpd.name = "A3RI",
-	.bit_shift = 8,
-};
-
-static int sh7372_pd_a4s_suspend(void)
+static int sh7372_a4s_pd_suspend(void)
 {
 	/*
 	 * The A4S domain contains the CPU core and therefore it should
-	 * only be turned off if the CPU is in use.
+	 * only be turned off if the CPU is not in use.  This may happen
+	 * during system suspend, when SYSC is going to be used for generating
+	 * resume signals and a4s_suspend_ready is set to let
+	 * sh7372_enter_suspend() know that it can turn A4S off.
 	 */
+	a4s_suspend_ready = true;
 	return -EBUSY;
 }
 
-struct rmobile_pm_domain sh7372_pd_a4s = {
-	.genpd.name = "A4S",
-	.bit_shift = 10,
-	.gov = &pm_domain_always_on_gov,
-	.no_debug = true,
-	.suspend = sh7372_pd_a4s_suspend,
-};
+static void sh7372_a4s_pd_resume(void)
+{
+	a4s_suspend_ready = false;
+}
 
 static int sh7372_a3sp_pd_suspend(void)
 {
@@ -137,18 +111,80 @@ static int sh7372_a3sp_pd_suspend(void)
 	return console_suspend_enabled ? 0 : -EBUSY;
 }
 
-struct rmobile_pm_domain sh7372_pd_a3sp = {
-	.genpd.name = "A3SP",
-	.bit_shift = 11,
-	.gov = &pm_domain_always_on_gov,
-	.no_debug = true,
-	.suspend = sh7372_a3sp_pd_suspend,
+static struct rmobile_pm_domain sh7372_pm_domains[] = {
+	{
+		.genpd.name = "A4LC",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 1,
+	},
+	{
+		.genpd.name = "A4MP",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 2,
+	},
+	{
+		.genpd.name = "D4",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 3,
+	},
+	{
+		.genpd.name = "A4R",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 5,
+		.suspend = sh7372_a4r_pd_suspend,
+		.resume = sh7372_intcs_resume,
+	},
+	{
+		.genpd.name = "A3RV",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 6,
+	},
+	{
+		.genpd.name = "A3RI",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 8,
+	},
+	{
+		.genpd.name = "A4S",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 10,
+		.gov = &pm_domain_always_on_gov,
+		.no_debug = true,
+		.suspend = sh7372_a4s_pd_suspend,
+		.resume = sh7372_a4s_pd_resume,
+	},
+	{
+		.genpd.name = "A3SP",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 11,
+		.gov = &pm_domain_always_on_gov,
+		.no_debug = true,
+		.suspend = sh7372_a3sp_pd_suspend,
+	},
+	{
+		.genpd.name = "A3SG",
+		.genpd.power_on_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.genpd.power_off_latency_ns = PM_DOMAIN_ON_OFF_LATENCY_NS,
+		.bit_shift = 13,
+	},
 };
 
-struct rmobile_pm_domain sh7372_pd_a3sg = {
-	.genpd.name = "A3SG",
-	.bit_shift = 13,
-};
+void __init sh7372_init_pm_domains(void)
+{
+	rmobile_init_domains(sh7372_pm_domains, ARRAY_SIZE(sh7372_pm_domains));
+	pm_genpd_add_subdomain_names("A4LC", "A3RV");
+	pm_genpd_add_subdomain_names("A4R", "A4LC");
+	pm_genpd_add_subdomain_names("A4S", "A3SG");
+	pm_genpd_add_subdomain_names("A4S", "A3SP");
+}
 
 #endif /* CONFIG_PM */
 
@@ -304,6 +340,21 @@ static void sh7372_enter_a3sm_common(int pllc0_on)
 	sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc));
 	sh7372_enter_sysc(pllc0_on, 1 << 12);
 }
+
+static void sh7372_enter_a4s_common(int pllc0_on)
+{
+	sh7372_intca_suspend();
+	sh7372_set_reset_vector(SMFRAM);
+	sh7372_enter_sysc(pllc0_on, 1 << 10);
+	sh7372_intca_resume();
+}
+
+static void sh7372_pm_setup_smfram(void)
+{
+	memcpy((void *)SMFRAM, sh7372_resume_core_standby_sysc, 0x100);
+}
+#else
+static inline void sh7372_pm_setup_smfram(void) {}
 #endif /* CONFIG_SUSPEND || CONFIG_CPU_IDLE */
 
 #ifdef CONFIG_CPU_IDLE
@@ -313,7 +364,8 @@ static int sh7372_do_idle_core_standby(unsigned long unused)
 	return 0;
 }
 
-static void sh7372_enter_core_standby(void)
+static int sh7372_enter_core_standby(struct cpuidle_device *dev,
+				     struct cpuidle_driver *drv, int index)
 {
 	sh7372_set_reset_vector(__pa(sh7372_resume_core_standby_sysc));
 
@@ -324,83 +376,102 @@ static void sh7372_enter_core_standby(void)
 
 	 /* disable reset vector translation */
 	__raw_writel(0, SBAR);
+
+	return 1;
 }
 
-static void sh7372_enter_a3sm_pll_on(void)
+static int sh7372_enter_a3sm_pll_on(struct cpuidle_device *dev,
+				    struct cpuidle_driver *drv, int index)
 {
 	sh7372_enter_a3sm_common(1);
+	return 2;
 }
 
-static void sh7372_enter_a3sm_pll_off(void)
+static int sh7372_enter_a3sm_pll_off(struct cpuidle_device *dev,
+				     struct cpuidle_driver *drv, int index)
 {
 	sh7372_enter_a3sm_common(0);
+	return 3;
 }
 
-static void sh7372_cpuidle_setup(struct cpuidle_driver *drv)
+static int sh7372_enter_a4s(struct cpuidle_device *dev,
+			    struct cpuidle_driver *drv, int index)
 {
-	struct cpuidle_state *state = &drv->states[drv->state_count];
-
-	snprintf(state->name, CPUIDLE_NAME_LEN, "C2");
-	strncpy(state->desc, "Core Standby Mode", CPUIDLE_DESC_LEN);
-	state->exit_latency = 10;
-	state->target_residency = 20 + 10;
-	state->flags = CPUIDLE_FLAG_TIME_VALID;
-	shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_core_standby;
-	drv->state_count++;
-
-	state = &drv->states[drv->state_count];
-	snprintf(state->name, CPUIDLE_NAME_LEN, "C3");
-	strncpy(state->desc, "A3SM PLL ON", CPUIDLE_DESC_LEN);
-	state->exit_latency = 20;
-	state->target_residency = 30 + 20;
-	state->flags = CPUIDLE_FLAG_TIME_VALID;
-	shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_a3sm_pll_on;
-	drv->state_count++;
-
-	state = &drv->states[drv->state_count];
-	snprintf(state->name, CPUIDLE_NAME_LEN, "C4");
-	strncpy(state->desc, "A3SM PLL OFF", CPUIDLE_DESC_LEN);
-	state->exit_latency = 120;
-	state->target_residency = 30 + 120;
-	state->flags = CPUIDLE_FLAG_TIME_VALID;
-	shmobile_cpuidle_modes[drv->state_count] = sh7372_enter_a3sm_pll_off;
-	drv->state_count++;
+	unsigned long msk, msk2;
+
+	if (!sh7372_sysc_valid(&msk, &msk2))
+		return sh7372_enter_a3sm_pll_off(dev, drv, index);
+
+	sh7372_setup_sysc(msk, msk2);
+	sh7372_enter_a4s_common(0);
+	return 4;
 }
 
+static struct cpuidle_driver sh7372_cpuidle_driver = {
+	.name			= "sh7372_cpuidle",
+	.owner			= THIS_MODULE,
+	.en_core_tk_irqen	= 1,
+	.state_count		= 5,
+	.safe_state_index	= 0, /* C1 */
+	.states[0] = ARM_CPUIDLE_WFI_STATE,
+	.states[0].enter = shmobile_enter_wfi,
+	.states[1] = {
+		.name = "C2",
+		.desc = "Core Standby Mode",
+		.exit_latency = 10,
+		.target_residency = 20 + 10,
+		.flags = CPUIDLE_FLAG_TIME_VALID,
+		.enter = sh7372_enter_core_standby,
+	},
+	.states[2] = {
+		.name = "C3",
+		.desc = "A3SM PLL ON",
+		.exit_latency = 20,
+		.target_residency = 30 + 20,
+		.flags = CPUIDLE_FLAG_TIME_VALID,
+		.enter = sh7372_enter_a3sm_pll_on,
+	},
+	.states[3] = {
+		.name = "C4",
+		.desc = "A3SM PLL OFF",
+		.exit_latency = 120,
+		.target_residency = 30 + 120,
+		.flags = CPUIDLE_FLAG_TIME_VALID,
+		.enter = sh7372_enter_a3sm_pll_off,
+	},
+	.states[4] = {
+		.name = "C5",
+		.desc = "A4S PLL OFF",
+		.exit_latency = 240,
+		.target_residency = 30 + 240,
+		.flags = CPUIDLE_FLAG_TIME_VALID,
+		.enter = sh7372_enter_a4s,
+		.disabled = true,
+	},
+};
+
 static void sh7372_cpuidle_init(void)
 {
-	shmobile_cpuidle_setup = sh7372_cpuidle_setup;
+	shmobile_cpuidle_set_driver(&sh7372_cpuidle_driver);
 }
 #else
 static void sh7372_cpuidle_init(void) {}
 #endif
 
 #ifdef CONFIG_SUSPEND
-static void sh7372_enter_a4s_common(int pllc0_on)
-{
-	sh7372_intca_suspend();
-	memcpy((void *)SMFRAM, sh7372_resume_core_standby_sysc, 0x100);
-	sh7372_set_reset_vector(SMFRAM);
-	sh7372_enter_sysc(pllc0_on, 1 << 10);
-	sh7372_intca_resume();
-}
-
 static int sh7372_enter_suspend(suspend_state_t suspend_state)
 {
 	unsigned long msk, msk2;
 
 	/* check active clocks to determine potential wakeup sources */
-	if (sh7372_sysc_valid(&msk, &msk2)) {
-		if (!console_suspend_enabled &&
-		    sh7372_pd_a4s.genpd.status == GPD_STATE_POWER_OFF) {
-			/* convert INTC mask/sense to SYSC mask/sense */
-			sh7372_setup_sysc(msk, msk2);
-
-			/* enter A4S sleep with PLLC0 off */
-			pr_debug("entering A4S\n");
-			sh7372_enter_a4s_common(0);
-			return 0;
-		}
+	if (sh7372_sysc_valid(&msk, &msk2) && a4s_suspend_ready) {
+		/* convert INTC mask/sense to SYSC mask/sense */
+		sh7372_setup_sysc(msk, msk2);
+
+		/* enter A4S sleep with PLLC0 off */
+		pr_debug("entering A4S\n");
+		sh7372_enter_a4s_common(0);
+		return 0;
 	}
 
 	/* default to enter A3SM sleep with PLLC0 off */
@@ -426,7 +497,7 @@ static int sh7372_pm_notifier_fn(struct notifier_block *notifier,
 		 * executed during system suspend and resume, respectively, so
 		 * that those functions don't crash while accessing the INTCS.
 		 */
-		pm_genpd_poweron(&sh7372_pd_a4r.genpd);
+		pm_genpd_name_poweron("A4R");
 		break;
 	case PM_POST_SUSPEND:
 		pm_genpd_poweroff_unused();
@@ -455,6 +526,14 @@ void __init sh7372_pm_init(void)
 	/* do not convert A3SM, A3SP, A3SG, A4R power down into A4S */
 	__raw_writel(0, PDNSEL);
 
+	sh7372_pm_setup_smfram();
+
 	sh7372_suspend_init();
 	sh7372_cpuidle_init();
 }
+
+void __init sh7372_pm_init_late(void)
+{
+	shmobile_init_late();
+	pm_genpd_name_attach_cpuidle("A4S", 4);
+}
diff --git a/arch/arm/mach-shmobile/setup-r8a7740.c b/arch/arm/mach-shmobile/setup-r8a7740.c
index 78948a9dba0..11bb1d98419 100644
--- a/arch/arm/mach-shmobile/setup-r8a7740.c
+++ b/arch/arm/mach-shmobile/setup-r8a7740.c
@@ -673,12 +673,7 @@ void __init r8a7740_add_standard_devices(void)
 	r8a7740_i2c_workaround(&i2c0_device);
 	r8a7740_i2c_workaround(&i2c1_device);
 
-	/* PM domain */
-	rmobile_init_pm_domain(&r8a7740_pd_a4s);
-	rmobile_init_pm_domain(&r8a7740_pd_a3sp);
-	rmobile_init_pm_domain(&r8a7740_pd_a4lc);
-
-	rmobile_pm_add_subdomain(&r8a7740_pd_a4s, &r8a7740_pd_a3sp);
+	r8a7740_init_pm_domains();
 
 	/* add devices */
 	platform_add_devices(r8a7740_early_devices,
@@ -688,16 +683,16 @@ void __init r8a7740_add_standard_devices(void)
 
 	/* add devices to PM domain  */
 
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif0_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif1_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif2_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif3_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif4_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif5_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif6_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scif7_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&scifb_device);
-	rmobile_add_device_to_domain(&r8a7740_pd_a3sp,	&i2c1_device);
+	rmobile_add_device_to_domain("A3SP",	&scif0_device);
+	rmobile_add_device_to_domain("A3SP",	&scif1_device);
+	rmobile_add_device_to_domain("A3SP",	&scif2_device);
+	rmobile_add_device_to_domain("A3SP",	&scif3_device);
+	rmobile_add_device_to_domain("A3SP",	&scif4_device);
+	rmobile_add_device_to_domain("A3SP",	&scif5_device);
+	rmobile_add_device_to_domain("A3SP",	&scif6_device);
+	rmobile_add_device_to_domain("A3SP",	&scif7_device);
+	rmobile_add_device_to_domain("A3SP",	&scifb_device);
+	rmobile_add_device_to_domain("A3SP",	&i2c1_device);
 }
 
 static void __init r8a7740_earlytimer_init(void)
diff --git a/arch/arm/mach-shmobile/setup-r8a7779.c b/arch/arm/mach-shmobile/setup-r8a7779.c
index e98e46f6cf5..2917668f009 100644
--- a/arch/arm/mach-shmobile/setup-r8a7779.c
+++ b/arch/arm/mach-shmobile/setup-r8a7779.c
@@ -251,10 +251,7 @@ void __init r8a7779_add_standard_devices(void)
 #endif
 	r8a7779_pm_init();
 
-	r8a7779_init_pm_domain(&r8a7779_sh4a);
-	r8a7779_init_pm_domain(&r8a7779_sgx);
-	r8a7779_init_pm_domain(&r8a7779_vdp1);
-	r8a7779_init_pm_domain(&r8a7779_impx3);
+	r8a7779_init_pm_domains();
 
 	platform_add_devices(r8a7779_early_devices,
 			    ARRAY_SIZE(r8a7779_early_devices));
diff --git a/arch/arm/mach-shmobile/setup-sh7372.c b/arch/arm/mach-shmobile/setup-sh7372.c
index 838a87be1d5..a07954fbcd2 100644
--- a/arch/arm/mach-shmobile/setup-sh7372.c
+++ b/arch/arm/mach-shmobile/setup-sh7372.c
@@ -1001,21 +1001,34 @@ static struct platform_device *sh7372_late_devices[] __initdata = {
 
 void __init sh7372_add_standard_devices(void)
 {
-	rmobile_init_pm_domain(&sh7372_pd_a4lc);
-	rmobile_init_pm_domain(&sh7372_pd_a4mp);
-	rmobile_init_pm_domain(&sh7372_pd_d4);
-	rmobile_init_pm_domain(&sh7372_pd_a4r);
-	rmobile_init_pm_domain(&sh7372_pd_a3rv);
-	rmobile_init_pm_domain(&sh7372_pd_a3ri);
-	rmobile_init_pm_domain(&sh7372_pd_a4s);
-	rmobile_init_pm_domain(&sh7372_pd_a3sp);
-	rmobile_init_pm_domain(&sh7372_pd_a3sg);
-
-	rmobile_pm_add_subdomain(&sh7372_pd_a4lc, &sh7372_pd_a3rv);
-	rmobile_pm_add_subdomain(&sh7372_pd_a4r, &sh7372_pd_a4lc);
-
-	rmobile_pm_add_subdomain(&sh7372_pd_a4s, &sh7372_pd_a3sg);
-	rmobile_pm_add_subdomain(&sh7372_pd_a4s, &sh7372_pd_a3sp);
+	struct pm_domain_device domain_devices[] = {
+		{ "A3RV", &vpu_device, },
+		{ "A4MP", &spu0_device, },
+		{ "A4MP", &spu1_device, },
+		{ "A3SP", &scif0_device, },
+		{ "A3SP", &scif1_device, },
+		{ "A3SP", &scif2_device, },
+		{ "A3SP", &scif3_device, },
+		{ "A3SP", &scif4_device, },
+		{ "A3SP", &scif5_device, },
+		{ "A3SP", &scif6_device, },
+		{ "A3SP", &iic1_device, },
+		{ "A3SP", &dma0_device, },
+		{ "A3SP", &dma1_device, },
+		{ "A3SP", &dma2_device, },
+		{ "A3SP", &usb_dma0_device, },
+		{ "A3SP", &usb_dma1_device, },
+		{ "A4R", &iic0_device, },
+		{ "A4R", &veu0_device, },
+		{ "A4R", &veu1_device, },
+		{ "A4R", &veu2_device, },
+		{ "A4R", &veu3_device, },
+		{ "A4R", &jpu_device, },
+		{ "A4R", &tmu00_device, },
+		{ "A4R", &tmu01_device, },
+	};
+
+	sh7372_init_pm_domains();
 
 	platform_add_devices(sh7372_early_devices,
 			    ARRAY_SIZE(sh7372_early_devices));
@@ -1023,30 +1036,8 @@ void __init sh7372_add_standard_devices(void)
 	platform_add_devices(sh7372_late_devices,
 			    ARRAY_SIZE(sh7372_late_devices));
 
-	rmobile_add_device_to_domain(&sh7372_pd_a3rv, &vpu_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4mp, &spu0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4mp, &spu1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif2_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif3_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif4_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif5_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &scif6_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &iic1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &dma2_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usb_dma0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a3sp, &usb_dma1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &iic0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu0_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu1_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu2_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &veu3_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &jpu_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &tmu00_device);
-	rmobile_add_device_to_domain(&sh7372_pd_a4r, &tmu01_device);
+	rmobile_add_devices_to_domains(domain_devices,
+				       ARRAY_SIZE(domain_devices));
 }
 
 static void __init sh7372_earlytimer_init(void)
diff --git a/arch/arm/mach-u300/i2c.c b/arch/arm/mach-u300/i2c.c
index 0d4620ed853..96800aa1316 100644
--- a/arch/arm/mach-u300/i2c.c
+++ b/arch/arm/mach-u300/i2c.c
@@ -9,7 +9,7 @@
  */
 #include <linux/kernel.h>
 #include <linux/i2c.h>
-#include <linux/mfd/abx500.h>
+#include <linux/mfd/ab3100.h>
 #include <linux/regulator/machine.h>
 #include <linux/amba/bus.h>
 #include <mach/irqs.h>
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 13f555d6249..477a2d23ddf 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -73,11 +73,18 @@ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
 	     unsigned long offset, size_t size, enum dma_data_direction dir,
 	     struct dma_attrs *attrs)
 {
-	if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
+	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 		__dma_page_cpu_to_dev(page, offset, size, dir);
 	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 }
 
+static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
+	     unsigned long offset, size_t size, enum dma_data_direction dir,
+	     struct dma_attrs *attrs)
+{
+	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
+}
+
 /**
  * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
@@ -96,7 +103,7 @@ static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
 		size_t size, enum dma_data_direction dir,
 		struct dma_attrs *attrs)
 {
-	if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
+	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 		__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
 				      handle & ~PAGE_MASK, size, dir);
 }
@@ -106,8 +113,7 @@ static void arm_dma_sync_single_for_cpu(struct device *dev,
 {
 	unsigned int offset = handle & (PAGE_SIZE - 1);
 	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
-	if (!arch_is_coherent())
-		__dma_page_dev_to_cpu(page, offset, size, dir);
+	__dma_page_dev_to_cpu(page, offset, size, dir);
 }
 
 static void arm_dma_sync_single_for_device(struct device *dev,
@@ -115,8 +121,7 @@ static void arm_dma_sync_single_for_device(struct device *dev,
 {
 	unsigned int offset = handle & (PAGE_SIZE - 1);
 	struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
-	if (!arch_is_coherent())
-		__dma_page_cpu_to_dev(page, offset, size, dir);
+	__dma_page_cpu_to_dev(page, offset, size, dir);
 }
 
 static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
@@ -138,6 +143,22 @@ struct dma_map_ops arm_dma_ops = {
 };
 EXPORT_SYMBOL(arm_dma_ops);
 
+static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
+	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
+static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
+				  dma_addr_t handle, struct dma_attrs *attrs);
+
+struct dma_map_ops arm_coherent_dma_ops = {
+	.alloc			= arm_coherent_dma_alloc,
+	.free			= arm_coherent_dma_free,
+	.mmap			= arm_dma_mmap,
+	.get_sgtable		= arm_dma_get_sgtable,
+	.map_page		= arm_coherent_dma_map_page,
+	.map_sg			= arm_dma_map_sg,
+	.set_dma_mask		= arm_dma_set_mask,
+};
+EXPORT_SYMBOL(arm_coherent_dma_ops);
+
 static u64 get_coherent_dma_mask(struct device *dev)
 {
 	u64 mask = (u64)arm_dma_limit;
@@ -586,7 +607,7 @@ static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
 
 
 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
-			 gfp_t gfp, pgprot_t prot, const void *caller)
+			 gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
 {
 	u64 mask = get_coherent_dma_mask(dev);
 	struct page *page;
@@ -619,7 +640,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 	*handle = DMA_ERROR_CODE;
 	size = PAGE_ALIGN(size);
 
-	if (arch_is_coherent() || nommu())
+	if (is_coherent || nommu())
 		addr = __alloc_simple_buffer(dev, size, gfp, &page);
 	else if (gfp & GFP_ATOMIC)
 		addr = __alloc_from_pool(size, &page);
@@ -647,7 +668,20 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 	if (dma_alloc_from_coherent(dev, size, handle, &memory))
 		return memory;
 
-	return __dma_alloc(dev, size, handle, gfp, prot,
+	return __dma_alloc(dev, size, handle, gfp, prot, false,
+			   __builtin_return_address(0));
+}
+
+static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
+	dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
+{
+	pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
+	void *memory;
+
+	if (dma_alloc_from_coherent(dev, size, handle, &memory))
+		return memory;
+
+	return __dma_alloc(dev, size, handle, gfp, prot, true,
 			   __builtin_return_address(0));
 }
 
@@ -684,8 +718,9 @@ int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 /*
  * Free a buffer as defined by the above mapping.
  */
-void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
-		  dma_addr_t handle, struct dma_attrs *attrs)
+static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
+			   dma_addr_t handle, struct dma_attrs *attrs,
+			   bool is_coherent)
 {
 	struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
 
@@ -694,7 +729,7 @@ void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 
 	size = PAGE_ALIGN(size);
 
-	if (arch_is_coherent() || nommu()) {
+	if (is_coherent || nommu()) {
 		__dma_free_buffer(page, size);
 	} else if (__free_from_pool(cpu_addr, size)) {
 		return;
@@ -710,6 +745,18 @@ void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
 	}
 }
 
+void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
+		  dma_addr_t handle, struct dma_attrs *attrs)
+{
+	__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
+}
+
+static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
+				  dma_addr_t handle, struct dma_attrs *attrs)
+{
+	__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
+}
+
 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
 		 void *cpu_addr, dma_addr_t handle, size_t size,
 		 struct dma_attrs *attrs)
@@ -1012,11 +1059,12 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t
 		if (!pages[i])
 			goto error;
 
-		if (order)
+		if (order) {
 			split_page(pages[i], order);
-		j = 1 << order;
-		while (--j)
-			pages[i + j] = pages[i] + j;
+			j = 1 << order;
+			while (--j)
+				pages[i + j] = pages[i] + j;
+		}
 
 		__dma_clear_buffer(pages[i], PAGE_SIZE << order);
 		i += 1 << order;
@@ -1303,7 +1351,8 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
  */
 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
 			  size_t size, dma_addr_t *handle,
-			  enum dma_data_direction dir, struct dma_attrs *attrs)
+			  enum dma_data_direction dir, struct dma_attrs *attrs,
+			  bool is_coherent)
 {
 	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
 	dma_addr_t iova, iova_base;
@@ -1322,8 +1371,8 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
 		phys_addr_t phys = page_to_phys(sg_page(s));
 		unsigned int len = PAGE_ALIGN(s->offset + s->length);
 
-		if (!arch_is_coherent() &&
-		    !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
+		if (!is_coherent &&
+			!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
 
 		ret = iommu_map(mapping->domain, iova, phys, len, 0);
@@ -1341,20 +1390,9 @@ fail:
 	return ret;
 }
 
-/**
- * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
- * @dev: valid struct device pointer
- * @sg: list of buffers
- * @nents: number of buffers to map
- * @dir: DMA transfer direction
- *
- * Map a set of buffers described by scatterlist in streaming mode for DMA.
- * The scatter gather list elements are merged together (if possible) and
- * tagged with the appropriate dma address and length. They are obtained via
- * sg_dma_{address,length}.
- */
-int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
-		     enum dma_data_direction dir, struct dma_attrs *attrs)
+static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
+		     enum dma_data_direction dir, struct dma_attrs *attrs,
+		     bool is_coherent)
 {
 	struct scatterlist *s = sg, *dma = sg, *start = sg;
 	int i, count = 0;
@@ -1370,7 +1408,7 @@ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 
 		if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
 			if (__map_sg_chunk(dev, start, size, &dma->dma_address,
-			    dir, attrs) < 0)
+			    dir, attrs, is_coherent) < 0)
 				goto bad_mapping;
 
 			dma->dma_address += offset;
@@ -1383,7 +1421,8 @@ int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 		}
 		size += s->length;
 	}
-	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs) < 0)
+	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
+		is_coherent) < 0)
 		goto bad_mapping;
 
 	dma->dma_address += offset;
@@ -1398,17 +1437,44 @@ bad_mapping:
 }
 
 /**
- * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
+ * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
  * @dev: valid struct device pointer
  * @sg: list of buffers
- * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
- * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ * @nents: number of buffers to map
+ * @dir: DMA transfer direction
  *
- * Unmap a set of streaming mode DMA translations.  Again, CPU access
- * rules concerning calls here are the same as for dma_unmap_single().
+ * Map a set of i/o coherent buffers described by scatterlist in streaming
+ * mode for DMA. The scatter gather list elements are merged together (if
+ * possible) and tagged with the appropriate dma address and length. They are
+ * obtained via sg_dma_{address,length}.
  */
-void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
-			enum dma_data_direction dir, struct dma_attrs *attrs)
+int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
+}
+
+/**
+ * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to map
+ * @dir: DMA transfer direction
+ *
+ * Map a set of buffers described by scatterlist in streaming mode for DMA.
+ * The scatter gather list elements are merged together (if possible) and
+ * tagged with the appropriate dma address and length. They are obtained via
+ * sg_dma_{address,length}.
+ */
+int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
+}
+
+static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
+		int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
+		bool is_coherent)
 {
 	struct scatterlist *s;
 	int i;
@@ -1417,7 +1483,7 @@ void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 		if (sg_dma_len(s))
 			__iommu_remove_mapping(dev, sg_dma_address(s),
 					       sg_dma_len(s));
-		if (!arch_is_coherent() &&
+		if (!is_coherent &&
 		    !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 			__dma_page_dev_to_cpu(sg_page(s), s->offset,
 					      s->length, dir);
@@ -1425,6 +1491,38 @@ void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
 }
 
 /**
+ * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
+ * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ *
+ * Unmap a set of streaming mode DMA translations.  Again, CPU access
+ * rules concerning calls here are the same as for dma_unmap_single().
+ */
+void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
+		int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
+}
+
+/**
+ * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
+ * @dev: valid struct device pointer
+ * @sg: list of buffers
+ * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
+ * @dir: DMA transfer direction (same as was passed to dma_map_sg)
+ *
+ * Unmap a set of streaming mode DMA translations.  Again, CPU access
+ * rules concerning calls here are the same as for dma_unmap_single().
+ */
+void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
+			enum dma_data_direction dir, struct dma_attrs *attrs)
+{
+	__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
+}
+
+/**
  * arm_iommu_sync_sg_for_cpu
  * @dev: valid struct device pointer
  * @sg: list of buffers
@@ -1438,8 +1536,7 @@ void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 	int i;
 
 	for_each_sg(sg, s, nents, i)
-		if (!arch_is_coherent())
-			__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
+		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
 
 }
 
@@ -1457,22 +1554,21 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 	int i;
 
 	for_each_sg(sg, s, nents, i)
-		if (!arch_is_coherent())
-			__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
+		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
 }
 
 
 /**
- * arm_iommu_map_page
+ * arm_coherent_iommu_map_page
  * @dev: valid struct device pointer
  * @page: page that buffer resides in
  * @offset: offset into page for start of buffer
  * @size: size of buffer to map
  * @dir: DMA transfer direction
  *
- * IOMMU aware version of arm_dma_map_page()
+ * Coherent IOMMU aware version of arm_dma_map_page()
  */
-static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
+static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
 	     unsigned long offset, size_t size, enum dma_data_direction dir,
 	     struct dma_attrs *attrs)
 {
@@ -1480,9 +1576,6 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
 	dma_addr_t dma_addr;
 	int ret, len = PAGE_ALIGN(size + offset);
 
-	if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
-		__dma_page_cpu_to_dev(page, offset, size, dir);
-
 	dma_addr = __alloc_iova(mapping, len);
 	if (dma_addr == DMA_ERROR_CODE)
 		return dma_addr;
@@ -1498,6 +1591,51 @@ fail:
 }
 
 /**
+ * arm_iommu_map_page
+ * @dev: valid struct device pointer
+ * @page: page that buffer resides in
+ * @offset: offset into page for start of buffer
+ * @size: size of buffer to map
+ * @dir: DMA transfer direction
+ *
+ * IOMMU aware version of arm_dma_map_page()
+ */
+static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
+	     unsigned long offset, size_t size, enum dma_data_direction dir,
+	     struct dma_attrs *attrs)
+{
+	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
+		__dma_page_cpu_to_dev(page, offset, size, dir);
+
+	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
+}
+
+/**
+ * arm_coherent_iommu_unmap_page
+ * @dev: valid struct device pointer
+ * @handle: DMA address of buffer
+ * @size: size of buffer (same as passed to dma_map_page)
+ * @dir: DMA transfer direction (same as passed to dma_map_page)
+ *
+ * Coherent IOMMU aware version of arm_dma_unmap_page()
+ */
+static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
+		size_t size, enum dma_data_direction dir,
+		struct dma_attrs *attrs)
+{
+	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+	dma_addr_t iova = handle & PAGE_MASK;
+	int offset = handle & ~PAGE_MASK;
+	int len = PAGE_ALIGN(size + offset);
+
+	if (!iova)
+		return;
+
+	iommu_unmap(mapping->domain, iova, len);
+	__free_iova(mapping, iova, len);
+}
+
+/**
  * arm_iommu_unmap_page
  * @dev: valid struct device pointer
  * @handle: DMA address of buffer
@@ -1519,7 +1657,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
 	if (!iova)
 		return;
 
-	if (!arch_is_coherent() && !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
+	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 		__dma_page_dev_to_cpu(page, offset, size, dir);
 
 	iommu_unmap(mapping->domain, iova, len);
@@ -1537,8 +1675,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
 	if (!iova)
 		return;
 
-	if (!arch_is_coherent())
-		__dma_page_dev_to_cpu(page, offset, size, dir);
+	__dma_page_dev_to_cpu(page, offset, size, dir);
 }
 
 static void arm_iommu_sync_single_for_device(struct device *dev,
@@ -1572,6 +1709,19 @@ struct dma_map_ops iommu_ops = {
 	.sync_sg_for_device	= arm_iommu_sync_sg_for_device,
 };
 
+struct dma_map_ops iommu_coherent_ops = {
+	.alloc		= arm_iommu_alloc_attrs,
+	.free		= arm_iommu_free_attrs,
+	.mmap		= arm_iommu_mmap_attrs,
+	.get_sgtable	= arm_iommu_get_sgtable,
+
+	.map_page	= arm_coherent_iommu_map_page,
+	.unmap_page	= arm_coherent_iommu_unmap_page,
+
+	.map_sg		= arm_coherent_iommu_map_sg,
+	.unmap_sg	= arm_coherent_iommu_unmap_sg,
+};
+
 /**
  * arm_iommu_create_mapping
  * @bus: pointer to the bus holding the client device (for IOMMU calls)
@@ -1665,7 +1815,7 @@ int arm_iommu_attach_device(struct device *dev,
 	dev->archdata.mapping = mapping;
 	set_dma_ops(dev, &iommu_ops);
 
-	pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev));
+	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
 	return 0;
 }
 
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 18144e6a311..941dfb9e9a7 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -423,17 +423,6 @@ static void __init build_mem_type_table(void)
 	vecs_pgprot = kern_pgprot = user_pgprot = cp->pte;
 
 	/*
-	 * Enable CPU-specific coherency if supported.
-	 * (Only available on XSC3 at the moment.)
-	 */
-	if (arch_is_coherent() && cpu_is_xsc3()) {
-		mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
-		mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED;
-		mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED;
-		mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S;
-		mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED;
-	}
-	/*
 	 * ARMv6 and above have extended page tables.
 	 */
 	if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
diff --git a/arch/arm/plat-omap/include/plat/usb.h b/arch/arm/plat-omap/include/plat/usb.h
index bd20588c356..87ee140fefa 100644
--- a/arch/arm/plat-omap/include/plat/usb.h
+++ b/arch/arm/plat-omap/include/plat/usb.h
@@ -4,6 +4,7 @@
 #define	__ASM_ARCH_OMAP_USB_H
 
 #include <linux/io.h>
+#include <linux/platform_device.h>
 #include <linux/usb/musb.h>
 
 #define OMAP3_HS_USB_PORTS	3
@@ -63,6 +64,10 @@ struct usbhs_omap_platform_data {
 	struct ehci_hcd_omap_platform_data	*ehci_data;
 	struct ohci_hcd_omap_platform_data	*ohci_data;
 };
+
+struct usbtll_omap_platform_data {
+	enum usbhs_omap_port_mode		port_mode[OMAP3_HS_USB_PORTS];
+};
 /*-------------------------------------------------------------------------*/
 
 struct omap_musb_board_data {
@@ -81,6 +86,8 @@ enum musb_interface    {MUSB_INTERFACE_ULPI, MUSB_INTERFACE_UTMI};
 extern void usb_musb_init(struct omap_musb_board_data *board_data);
 
 extern void usbhs_init(const struct usbhs_omap_board_data *pdata);
+extern int omap_tll_enable(void);
+extern int omap_tll_disable(void);
 
 extern int omap4430_phy_power(struct device *dev, int ID, int on);
 extern int omap4430_phy_set_clk(struct device *dev, int on);
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index a670a33ad73..37e610dc084 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -55,6 +55,7 @@ typedef s64		compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -130,6 +131,64 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		/* The padding is the same size as AArch64. */
+		int _pad[128/sizeof(int) - 3];
+
+		/* kill() */
+		struct {
+			compat_pid_t _pid;	/* sender's pid */
+			__compat_uid32_t _uid;	/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;	/* timer id */
+			int _overrun;		/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;       /* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			compat_pid_t _pid;	/* sender's pid */
+			__compat_uid32_t _uid;	/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			compat_pid_t _pid;	/* which child */
+			__compat_uid32_t _uid;	/* sender's uid */
+			int _status;		/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			compat_uptr_t _addr; /* faulting insn/memory ref. */
+			short _addr_lsb; /* LSB of the reported address */
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			compat_long_t _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -139,7 +198,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/arm64/include/uapi/asm/Kbuild b/arch/arm64/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/arm64/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index ac74c2f261e..0790a87a434 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -30,59 +30,6 @@
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
 
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		/* The padding is the same size as AArch64. */
-		int _pad[SI_PAD_SIZE];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;       /* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;	/* which child */
-			__compat_uid32_t _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			compat_uptr_t _addr; /* faulting insn/memory ref. */
-			short _addr_lsb; /* LSB of the reported address */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			compat_long_t _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
 struct compat_sigaction {
 	compat_uptr_t			sa_handler;
 	compat_ulong_t			sa_flags;
diff --git a/arch/avr32/include/asm/elf.h b/arch/avr32/include/asm/elf.h
index 3b3159b710d..e2c32873980 100644
--- a/arch/avr32/include/asm/elf.h
+++ b/arch/avr32/include/asm/elf.h
@@ -102,6 +102,7 @@ typedef struct user_fpu_struct elf_fpregset_t;
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK)))
 
 #endif /* __ASM_AVR32_ELF_H */
diff --git a/arch/avr32/include/uapi/asm/Kbuild b/arch/avr32/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/avr32/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/blackfin/include/asm/elf.h b/arch/blackfin/include/asm/elf.h
index e6c6812a9ab..14bc98ff668 100644
--- a/arch/blackfin/include/asm/elf.h
+++ b/arch/blackfin/include/asm/elf.h
@@ -132,6 +132,7 @@ do {											\
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif
diff --git a/arch/blackfin/include/uapi/asm/Kbuild b/arch/blackfin/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/blackfin/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/c6x/include/asm/elf.h b/arch/c6x/include/asm/elf.h
index f4552db20b4..32b997126ad 100644
--- a/arch/c6x/include/asm/elf.h
+++ b/arch/c6x/include/asm/elf.h
@@ -77,7 +77,8 @@ do {								\
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 /* C6X specific section types */
 #define SHT_C6000_UNWIND	0x70000001
diff --git a/arch/c6x/include/uapi/asm/Kbuild b/arch/c6x/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/c6x/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig
index e92215428a3..72bd5ae50a8 100644
--- a/arch/cris/Kconfig
+++ b/arch/cris/Kconfig
@@ -138,11 +138,6 @@ config CRIS_MACH_ARTPEC3
 
 endchoice
 
-config ETRAX_VCS_SIM
-	bool "VCS Simulator"
-	help
-	  Setup hardware to be run in the VCS simulator.
-
 config ETRAX_ARCH_V10
        bool
        default y if ETRAX100LX || ETRAX100LX_V2
diff --git a/arch/cris/Makefile b/arch/cris/Makefile
index 29c2ceb38a7..39dc7d00083 100644
--- a/arch/cris/Makefile
+++ b/arch/cris/Makefile
@@ -23,7 +23,9 @@ mach-$(CONFIG_ETRAXFS) := fs
 
 ifneq ($(arch-y),)
 SARCH := arch-$(arch-y)
-inc := -Iarch/cris/include/$(SARCH)
+inc := -Iarch/cris/include/uapi/$(SARCH)
+inc += -Iarch/cris/include/$(SARCH)
+inc += -Iarch/cris/include/uapi/$(SARCH)/arch
 inc += -Iarch/cris/include/$(SARCH)/arch
 else
 SARCH :=
diff --git a/arch/cris/arch-v32/drivers/axisflashmap.c b/arch/cris/arch-v32/drivers/axisflashmap.c
index b34438e026b..1b6ad624720 100644
--- a/arch/cris/arch-v32/drivers/axisflashmap.c
+++ b/arch/cris/arch-v32/drivers/axisflashmap.c
@@ -329,7 +329,6 @@ static int __init init_axis_flash(void)
 	}
 #endif
 
-#ifndef CONFIG_ETRAX_VCS_SIM
 	main_mtd = flash_probe();
 	if (main_mtd)
 		printk(KERN_INFO "%s: 0x%08x bytes of NOR flash memory.\n",
@@ -603,34 +602,7 @@ static int __init init_axis_flash(void)
 					"partition %d\n", part);
 		}
 	}
-#endif /* CONFIG_EXTRAX_VCS_SIM */
 
-#ifdef CONFIG_ETRAX_VCS_SIM
-	/* For simulator, always use a RAM partition.
-	 * The rootfs will be found after the kernel in RAM,
-	 * with romfs_start and romfs_end indicating location and size.
-	 */
-	struct mtd_info *mtd_ram;
-
-	mtd_ram = kmalloc(sizeof(struct mtd_info), GFP_KERNEL);
-	if (!mtd_ram) {
-		panic("axisflashmap: Couldn't allocate memory for "
-		      "mtd_info!\n");
-	}
-
-	printk(KERN_INFO "axisflashmap: Adding RAM partition for romfs, "
-	       "at %u, size %u\n",
-	       (unsigned) romfs_start, (unsigned) romfs_length);
-
-	err = mtdram_init_device(mtd_ram, (void *)romfs_start,
-				 romfs_length, "romfs");
-	if (err) {
-		panic("axisflashmap: Could not initialize MTD RAM "
-		      "device!\n");
-	}
-#endif /* CONFIG_EXTRAX_VCS_SIM */
-
-#ifndef CONFIG_ETRAX_VCS_SIM
 	if (aux_mtd) {
 		aux_partition.size = aux_mtd->size;
 		err = mtd_device_register(aux_mtd, &aux_partition, 1);
@@ -639,7 +611,6 @@ static int __init init_axis_flash(void)
 			      "aux mtd device!\n");
 
 	}
-#endif /* CONFIG_EXTRAX_VCS_SIM */
 
 	return err;
 }
diff --git a/arch/cris/arch-v32/drivers/pci/bios.c b/arch/cris/arch-v32/drivers/pci/bios.c
index 5b1ee82f63c..e3dfc72d0cf 100644
--- a/arch/cris/arch-v32/drivers/pci/bios.c
+++ b/arch/cris/arch-v32/drivers/pci/bios.c
@@ -97,28 +97,3 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 		pcibios_enable_irq(dev);
 	return 0;
 }
-
-int pcibios_assign_resources(void)
-{
-	struct pci_dev *dev = NULL;
-	int idx;
-	struct resource *r;
-
-	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
-		int class = dev->class >> 8;
-
-		/* Don't touch classless devices and host bridges */
-		if (!class || class == PCI_CLASS_BRIDGE_HOST)
-			continue;
-
-		for(idx=0; idx<6; idx++) {
-			r = &dev->resource[idx];
-
-			if (!r->start && r->end)
-				pci_assign_resource(dev, idx);
-		}
-	}
-	return 0;
-}
-
-EXPORT_SYMBOL(pcibios_assign_resources);
diff --git a/arch/cris/arch-v32/kernel/head.S b/arch/cris/arch-v32/kernel/head.S
index 5d502b9ab56..51e34165ece 100644
--- a/arch/cris/arch-v32/kernel/head.S
+++ b/arch/cris/arch-v32/kernel/head.S
@@ -36,13 +36,6 @@
 	.global nand_boot
 	.global swapper_pg_dir
 
-	;; Dummy section to make it bootable with current VCS simulator
-#ifdef CONFIG_ETRAX_VCS_SIM
-	.section ".boot", "ax"
-	ba tstart
-	nop
-#endif
-
 	.text
 tstart:
 	;; This is the entry point of the kernel. The CPU is currently in
@@ -75,17 +68,10 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */
 		| REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4)	\
 		| REG_FIELD(mmu, rw_mm_kbase_hi, base_d, 5)     \
 		| REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0
-#elif !defined(CONFIG_ETRAX_VCS_SIM)
-	move.d	REG_FIELD(mmu, rw_mm_kbase_hi, base_e, 8)	\
-		| REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4)	\
-		| REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0
 #else
-	;; Map the virtual DRAM to the RW eprom area at address 0.
-	;; Also map 0xa for the hook calls,
 	move.d	REG_FIELD(mmu, rw_mm_kbase_hi, base_e, 8)	\
 		| REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 4)	\
-		| REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb)   \
-		| REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0xa), $r0
+		| REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb), $r0
 #endif
 
 	;; Temporary map of 0x40 -> 0x40 and 0x00 -> 0x00.
@@ -126,27 +112,6 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */
 		| REG_STATE(mmu, rw_mm_cfg, seg_2, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_1, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_0, linear), $r2
-#elif !defined(CONFIG_ETRAX_VCS_SIM)
-	move.d	REG_STATE(mmu, rw_mm_cfg, we, on)		\
-		| REG_STATE(mmu, rw_mm_cfg, acc, on)		\
-		| REG_STATE(mmu, rw_mm_cfg, ex, on)		\
-		| REG_STATE(mmu, rw_mm_cfg, inv, on)		\
-		| REG_STATE(mmu, rw_mm_cfg, seg_f, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_e, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_d, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_c, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_b, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_a, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_9, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_8, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_7, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_6, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_5, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_4, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_3, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_2, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_1, page)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_0, linear), $r2
 #else
 	move.d	REG_STATE(mmu, rw_mm_cfg, we, on)		\
 		| REG_STATE(mmu, rw_mm_cfg, acc, on)		\
@@ -157,7 +122,7 @@ secondary_cpu_entry: /* Entry point for secondary CPUs */
 		| REG_STATE(mmu, rw_mm_cfg, seg_d, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_c, linear)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_b, linear)	\
-		| REG_STATE(mmu, rw_mm_cfg, seg_a, linear)	\
+		| REG_STATE(mmu, rw_mm_cfg, seg_a, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_9, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_8, page)	\
 		| REG_STATE(mmu, rw_mm_cfg, seg_7, page)	\
@@ -226,7 +191,6 @@ master_cpu:
 	move.d secondary_cpu_entry, $r1
 	move.d $r1, [$r0]
 #endif
-#ifndef CONFIG_ETRAX_VCS_SIM
 	; Check if starting from DRAM (network->RAM boot or unpacked
 	; compressed kernel), or directly from flash.
 	lapcq	., $r0
@@ -234,7 +198,6 @@ master_cpu:
 	cmp.d	0x10000, $r0	; Arbitrary, something above this code.
 	blo	_inflash0
 	nop
-#endif
 
 	jump	_inram		; Jump to cached RAM.
 	nop
@@ -326,7 +289,6 @@ move_cramfs:
 	move.d	romfs_length, $r1
 	move.d	$r0, [$r1]
 
-#ifndef CONFIG_ETRAX_VCS_SIM
 	;; The kernel could have been unpacked to DRAM by the loader, but
 	;; the cramfs image could still be in the flash immediately
 	;; following the compressed kernel image. The loader passes the address
@@ -335,10 +297,6 @@ move_cramfs:
 	cmp.d	0x0ffffff8, $r9
 	bhs	_no_romfs_in_flash ; R9 points outside the flash area.
 	nop
-#else
-	ba _no_romfs_in_flash
-	nop
-#endif
 	;; cramfs rootfs might to be in flash. Check for it.
 	move.d	[$r9], $r0	; cramfs_super.magic
 	cmp.d	CRAMFS_MAGIC, $r0
@@ -396,7 +354,6 @@ _no_romfs_in_flash:
 	move.d	romfs_length, $r3
 	move.d	$r2, [$r3]	; store size at romfs_length
 
-#ifndef CONFIG_ETRAX_VCS_SIM
 	add.d	$r2, $r0	; copy from end and downwards
 	add.d	$r2, $r1
 
@@ -410,7 +367,6 @@ _no_romfs_in_flash:
 	subq	1, $r2
 	bne	1b
 	nop
-#endif
 
 4:
 	;; BSS move done.
@@ -455,7 +411,6 @@ no_command_line:
 	move.d	etrax_irv, $r1	; Set the exception base register and pointer.
 	move.d	$r0, [$r1]
 
-#ifndef CONFIG_ETRAX_VCS_SIM
 	;; Clear the BSS region from _bss_start to _end.
 	move.d	__bss_start, $r0
 	move.d	_end, $r1
@@ -463,15 +418,6 @@ no_command_line:
 	cmp.d	$r1, $r0
 	blo 1b
 	nop
-#endif
-
-#ifdef CONFIG_ETRAX_VCS_SIM
-	/* Set the watchdog timeout to something big. Will be removed when */
-	/* watchdog can be disabled with command line option */
-	move.d  0x7fffffff, $r10
-	jsr     CPU_WATCHDOG_TIMEOUT
-	nop
-#endif
 
 	; Initialize registers to increase determinism
 	move.d __bss_start, $r0
diff --git a/arch/cris/arch-v32/kernel/kgdb.c b/arch/cris/arch-v32/kernel/kgdb.c
index 8c1d35cdf00..b06813aeb12 100644
--- a/arch/cris/arch-v32/kernel/kgdb.c
+++ b/arch/cris/arch-v32/kernel/kgdb.c
@@ -381,23 +381,9 @@ static int read_register(char regno, unsigned int *valptr);
 /* Serial port, reads one character. ETRAX 100 specific. from debugport.c */
 int getDebugChar(void);
 
-#ifdef CONFIG_ETRAX_VCS_SIM
-int getDebugChar(void)
-{
-  return socketread();
-}
-#endif
-
 /* Serial port, writes one character. ETRAX 100 specific. from debugport.c */
 void putDebugChar(int val);
 
-#ifdef CONFIG_ETRAX_VCS_SIM
-void putDebugChar(int val)
-{
-  socketwrite((char *)&val, 1);
-}
-#endif
-
 /* Returns the integer equivalent of a hexadecimal character. */
 static int hex(char ch);
 
diff --git a/arch/cris/arch-v32/mach-a3/Makefile b/arch/cris/arch-v32/mach-a3/Makefile
index 41fa6a6893a..d366e089198 100644
--- a/arch/cris/arch-v32/mach-a3/Makefile
+++ b/arch/cris/arch-v32/mach-a3/Makefile
@@ -1,10 +1,8 @@
-# $Id: Makefile,v 1.3 2007/03/13 11:57:46 starvik Exp $
 #
 # Makefile for the linux kernel.
 #
 
 obj-y   := dma.o pinmux.o io.o arbiter.o
-obj-$(CONFIG_ETRAX_VCS_SIM) += vcs_hook.o
 obj-$(CONFIG_CPU_FREQ)   += cpufreq.o
 
 clean:
diff --git a/arch/cris/arch-v32/mach-a3/vcs_hook.c b/arch/cris/arch-v32/mach-a3/vcs_hook.c
deleted file mode 100644
index 58b1a5469fd..00000000000
--- a/arch/cris/arch-v32/mach-a3/vcs_hook.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Simulator hook mechanism
- */
-
-#include "vcs_hook.h"
-#include <asm/io.h>
-#include <stdarg.h>
-
-#define HOOK_TRIG_ADDR      0xb7000000
-#define HOOK_MEM_BASE_ADDR  0xce000000
-
-static volatile unsigned *hook_base;
-
-#define HOOK_DATA(offset) hook_base[offset]
-#define VHOOK_DATA(offset) hook_base[offset]
-#define HOOK_TRIG(funcid) \
-	do { \
-		*((unsigned *) HOOK_TRIG_ADDR) = funcid; \
-	} while (0)
-#define HOOK_DATA_BYTE(offset) ((unsigned char *)hook_base)[offset]
-
-static void hook_init(void)
-{
-	static int first = 1;
-	if (first) {
-		first = 0;
-		hook_base = ioremap(HOOK_MEM_BASE_ADDR, 8192);
-	}
-}
-
-static unsigned hook_trig(unsigned id)
-{
-	unsigned ret;
-
-	/* preempt_disable(); */
-
-	/* Dummy read from mem to make sure data has propagated to memory
-	 * before trigging */
-	ret = *hook_base;
-
-	/* trigger hook */
-	HOOK_TRIG(id);
-
-	/* wait for call to finish */
-	while (VHOOK_DATA(0) > 0) ;
-
-	/* extract return value */
-
-	ret = VHOOK_DATA(1);
-
-	return ret;
-}
-
-int hook_call(unsigned id, unsigned pcnt, ...)
-{
-	va_list ap;
-	int i;
-	unsigned ret;
-
-	hook_init();
-
-	HOOK_DATA(0) = id;
-
-	va_start(ap, pcnt);
-	for (i = 1; i <= pcnt; i++)
-		HOOK_DATA(i) = va_arg(ap, unsigned);
-	va_end(ap);
-
-	ret = hook_trig(id);
-
-	return ret;
-}
-
-int hook_call_str(unsigned id, unsigned size, const char *str)
-{
-	int i;
-	unsigned ret;
-
-	hook_init();
-
-	HOOK_DATA(0) = id;
-	HOOK_DATA(1) = size;
-
-	for (i = 0; i < size; i++)
-		HOOK_DATA_BYTE(8 + i) = str[i];
-	HOOK_DATA_BYTE(8 + i) = 0;
-
-	ret = hook_trig(id);
-
-	return ret;
-}
-
-void print_str(const char *str)
-{
-	int i;
-	/* find null at end of string */
-	for (i = 1; str[i]; i++) ;
-	hook_call(hook_print_str, i, str);
-}
-
-void CPU_WATCHDOG_TIMEOUT(unsigned t)
-{
-}
diff --git a/arch/cris/arch-v32/mach-a3/vcs_hook.h b/arch/cris/arch-v32/mach-a3/vcs_hook.h
deleted file mode 100644
index 8b73d0e8392..00000000000
--- a/arch/cris/arch-v32/mach-a3/vcs_hook.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Simulator hook call mechanism
- */
-
-#ifndef __hook_h__
-#define __hook_h__
-
-int hook_call(unsigned id, unsigned pcnt, ...);
-int hook_call_str(unsigned id, unsigned size, const char *str);
-
-enum hook_ids {
-  hook_debug_on = 1,
-  hook_debug_off,
-  hook_stop_sim_ok,
-  hook_stop_sim_fail,
-  hook_alloc_shared,
-  hook_ptr_shared,
-  hook_free_shared,
-  hook_file2shared,
-  hook_cmp_shared,
-  hook_print_params,
-  hook_sim_time,
-  hook_stop_sim,
-  hook_kick_dog,
-  hook_dog_timeout,
-  hook_rand,
-  hook_srand,
-  hook_rand_range,
-  hook_print_str,
-  hook_print_hex,
-  hook_cmp_offset_shared,
-  hook_fill_random_shared,
-  hook_alloc_random_data,
-  hook_calloc_random_data,
-  hook_print_int,
-  hook_print_uint,
-  hook_fputc,
-  hook_init_fd,
-  hook_sbrk,
-  hook_print_context_descr,
-  hook_print_data_descr,
-  hook_print_group_descr,
-  hook_fill_shared,
-  hook_sl_srand,
-  hook_sl_rand_irange,
-  hook_sl_rand_urange,
-  hook_sl_sh_malloc_aligned,
-  hook_sl_sh_calloc_aligned,
-  hook_sl_sh_alloc_random_data,
-  hook_sl_sh_file2mem,
-  hook_sl_vera_mbox_handle,
-  hook_sl_vera_mbox_put,
-  hook_sl_vera_mbox_get,
-  hook_sl_system,
-  hook_sl_sh_hexdump
-};
-
-#endif
diff --git a/arch/cris/arch-v32/mach-fs/Makefile b/arch/cris/arch-v32/mach-fs/Makefile
index 41fa6a6893a..d366e089198 100644
--- a/arch/cris/arch-v32/mach-fs/Makefile
+++ b/arch/cris/arch-v32/mach-fs/Makefile
@@ -1,10 +1,8 @@
-# $Id: Makefile,v 1.3 2007/03/13 11:57:46 starvik Exp $
 #
 # Makefile for the linux kernel.
 #
 
 obj-y   := dma.o pinmux.o io.o arbiter.o
-obj-$(CONFIG_ETRAX_VCS_SIM) += vcs_hook.o
 obj-$(CONFIG_CPU_FREQ)   += cpufreq.o
 
 clean:
diff --git a/arch/cris/arch-v32/mach-fs/vcs_hook.c b/arch/cris/arch-v32/mach-fs/vcs_hook.c
deleted file mode 100644
index b11594ae0cb..00000000000
--- a/arch/cris/arch-v32/mach-fs/vcs_hook.c
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Call simulator hook. This is the part running in the
- * simulated program.
- */
-
-#include "vcs_hook.h"
-#include <stdarg.h>
-#include <arch-v32/hwregs/reg_map.h>
-#include <arch-v32/hwregs/intr_vect_defs.h>
-
-#define HOOK_TRIG_ADDR     0xb7000000	/* hook cvlog model reg address */
-#define HOOK_MEM_BASE_ADDR 0xa0000000	/* csp4 (shared mem) base addr */
-
-#define HOOK_DATA(offset) ((unsigned *)HOOK_MEM_BASE_ADDR)[offset]
-#define VHOOK_DATA(offset) ((volatile unsigned *)HOOK_MEM_BASE_ADDR)[offset]
-#define HOOK_TRIG(funcid) \
-	do { \
-		*((unsigned *) HOOK_TRIG_ADDR) = funcid; \
-	} while (0)
-#define HOOK_DATA_BYTE(offset) ((unsigned char *)HOOK_MEM_BASE_ADDR)[offset]
-
-int hook_call(unsigned id, unsigned pcnt, ...)
-{
-	va_list ap;
-	unsigned i;
-	unsigned ret;
-#ifdef USING_SOS
-	PREEMPT_OFF_SAVE();
-#endif
-
-	/* pass parameters */
-	HOOK_DATA(0) = id;
-
-	/* Have to make hook_print_str a special case since we call with a
-	 * parameter of byte type. Should perhaps be a separate
-	 * hook_call. */
-
-	if (id == hook_print_str) {
-		int i;
-		char *str;
-
-		HOOK_DATA(1) = pcnt;
-
-		va_start(ap, pcnt);
-		str = (char *)va_arg(ap, unsigned);
-
-		for (i = 0; i != pcnt; i++)
-			HOOK_DATA_BYTE(8 + i) = str[i];
-
-		HOOK_DATA_BYTE(8 + i) = 0;	/* null byte */
-	} else {
-		va_start(ap, pcnt);
-		for (i = 1; i <= pcnt; i++)
-			HOOK_DATA(i) = va_arg(ap, unsigned);
-		va_end(ap);
-	}
-
-	/* read from mem to make sure data has propagated to memory before
-	 * trigging */
-	ret = *((volatile unsigned *)HOOK_MEM_BASE_ADDR);
-
-	/* trigger hook */
-	HOOK_TRIG(id);
-
-	/* wait for call to finish */
-	while (VHOOK_DATA(0) > 0) ;
-
-	/* extract return value */
-
-	ret = VHOOK_DATA(1);
-
-#ifdef USING_SOS
-	PREEMPT_RESTORE();
-#endif
-	return ret;
-}
-
-unsigned hook_buf(unsigned i)
-{
-	return (HOOK_DATA(i));
-}
-
-void print_str(const char *str)
-{
-	int i;
-	/* find null at end of string */
-	for (i = 1; str[i]; i++) ;
-	hook_call(hook_print_str, i, str);
-}
-
-void CPU_KICK_DOG(void)
-{
-	(void)hook_call(hook_kick_dog, 0);
-}
-
-void CPU_WATCHDOG_TIMEOUT(unsigned t)
-{
-	(void)hook_call(hook_dog_timeout, 1, t);
-}
-
diff --git a/arch/cris/arch-v32/mach-fs/vcs_hook.h b/arch/cris/arch-v32/mach-fs/vcs_hook.h
deleted file mode 100644
index c000b9fece4..00000000000
--- a/arch/cris/arch-v32/mach-fs/vcs_hook.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Call simulator hook functions
- */
-
-#ifndef HOOK_H
-#define HOOK_H
-
-int hook_call(unsigned id, unsigned pcnt, ...);
-
-enum hook_ids {
-  hook_debug_on = 1,
-  hook_debug_off,
-  hook_stop_sim_ok,
-  hook_stop_sim_fail,
-  hook_alloc_shared,
-  hook_ptr_shared,
-  hook_free_shared,
-  hook_file2shared,
-  hook_cmp_shared,
-  hook_print_params,
-  hook_sim_time,
-  hook_stop_sim,
-  hook_kick_dog,
-  hook_dog_timeout,
-  hook_rand,
-  hook_srand,
-  hook_rand_range,
-  hook_print_str,
-  hook_print_hex,
-  hook_cmp_offset_shared,
-  hook_fill_random_shared,
-  hook_alloc_random_data,
-  hook_calloc_random_data,
-  hook_print_int,
-  hook_print_uint,
-  hook_fputc,
-  hook_init_fd,
-  hook_sbrk
-
-};
-
-#endif
diff --git a/arch/cris/arch-v32/mm/init.c b/arch/cris/arch-v32/mm/init.c
index 0768bc409ca..3deca5253d9 100644
--- a/arch/cris/arch-v32/mm/init.c
+++ b/arch/cris/arch-v32/mm/init.c
@@ -73,11 +73,7 @@ void __init cris_mmu_init(void)
 #endif
 		       REG_STATE(mmu, rw_mm_cfg, seg_c, linear) |
 		       REG_STATE(mmu, rw_mm_cfg, seg_b, linear) |
-#ifndef CONFIG_ETRAX_VCS_SIM
                        REG_STATE(mmu, rw_mm_cfg, seg_a, page)   |
-#else
-		       REG_STATE(mmu, rw_mm_cfg, seg_a, linear) |
-#endif
 		       REG_STATE(mmu, rw_mm_cfg, seg_9, page)   |
 		       REG_STATE(mmu, rw_mm_cfg, seg_8, page)   |
 		       REG_STATE(mmu, rw_mm_cfg, seg_7, page)   |
@@ -100,11 +96,7 @@ void __init cris_mmu_init(void)
 #endif
                          REG_FIELD(mmu, rw_mm_kbase_hi, base_c, 0x4) |
 			 REG_FIELD(mmu, rw_mm_kbase_hi, base_b, 0xb) |
-#ifndef CONFIG_ETRAX_VCS_SIM
 			 REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0x0) |
-#else
-                         REG_FIELD(mmu, rw_mm_kbase_hi, base_a, 0xa) |
-#endif
 			 REG_FIELD(mmu, rw_mm_kbase_hi, base_9, 0x0) |
 			 REG_FIELD(mmu, rw_mm_kbase_hi, base_8, 0x0));
 
diff --git a/arch/cris/include/arch-v10/arch/sv_addr_ag.h b/arch/cris/include/arch-v10/arch/sv_addr_ag.h
index e4a6b68b898..5517f04153a 100644
--- a/arch/cris/include/arch-v10/arch/sv_addr_ag.h
+++ b/arch/cris/include/arch-v10/arch/sv_addr_ag.h
@@ -114,7 +114,7 @@
 
 /*------------------------------------------------------------*/
 
-#include "sv_addr.agh"
+#include <arch/sv_addr.agh>
 
 #if __test_sv_addr__
 /* IO_MASK( R_BUS_CONFIG , CE ) */
diff --git a/arch/cris/include/arch-v10/arch/svinto.h b/arch/cris/include/arch-v10/arch/svinto.h
index 0881a1af7ce..da5c1527265 100644
--- a/arch/cris/include/arch-v10/arch/svinto.h
+++ b/arch/cris/include/arch-v10/arch/svinto.h
@@ -1,7 +1,7 @@
 #ifndef _ASM_CRIS_SVINTO_H
 #define _ASM_CRIS_SVINTO_H
 
-#include "sv_addr_ag.h"
+#include <arch/sv_addr_ag.h>
 
 extern unsigned int genconfig_shadow; /* defined and set in head.S */
 
diff --git a/arch/cris/include/arch-v32/arch/dma.h b/arch/cris/include/arch-v32/arch/dma.h
index 61906153a9a..6f92f4f23f2 100644
--- a/arch/cris/include/arch-v32/arch/dma.h
+++ b/arch/cris/include/arch-v32/arch/dma.h
@@ -1 +1 @@
-#include "mach/dma.h"
+#include <mach/dma.h>
diff --git a/arch/cris/include/arch-v32/arch/hwregs/dma.h b/arch/cris/include/arch-v32/arch/hwregs/dma.h
index 3ce322b5c73..52bf67907f2 100644
--- a/arch/cris/include/arch-v32/arch/hwregs/dma.h
+++ b/arch/cris/include/arch-v32/arch/hwregs/dma.h
@@ -7,7 +7,7 @@
 #define dma_h
 
 /* registers */ /* Really needed, since both are listed in sw.list? */
-#include "dma_defs.h"
+#include <arch/hwregs/dma_defs.h>
 
 
 /* descriptors */
diff --git a/arch/cris/include/arch-v32/arch/page.h b/arch/cris/include/arch-v32/arch/page.h
index 20f1b4806bf..e5b5aab52de 100644
--- a/arch/cris/include/arch-v32/arch/page.h
+++ b/arch/cris/include/arch-v32/arch/page.h
@@ -11,13 +11,8 @@
  * selected bit it's possible to convert between KSEG_x and 0x40000000 where the
  * DRAM really resides. DRAM is virtually at 0xc.
  */
-#ifndef CONFIG_ETRAX_VCS_SIM
 #define __pa(x) ((unsigned long)(x) & 0x7fffffff)
 #define __va(x) ((void *)((unsigned long)(x) | 0x80000000))
-#else
-#define __pa(x) ((unsigned long)(x) & 0x3fffffff)
-#define __va(x) ((void *)((unsigned long)(x) | 0xc0000000))
-#endif
 
 #define VM_STACK_DEFAULT_FLAGS	(VM_READ | VM_WRITE | \
 				 VM_MAYREAD | VM_MAYWRITE)
diff --git a/arch/cris/include/arch-v32/arch/processor.h b/arch/cris/include/arch-v32/arch/processor.h
index 9603c907fbc..a024b7d32fe 100644
--- a/arch/cris/include/arch-v32/arch/processor.h
+++ b/arch/cris/include/arch-v32/arch/processor.h
@@ -21,13 +21,9 @@ struct thread_struct {
 
 /*
  * User-space process size. This is hardcoded into a few places, so don't
- * changed it unless everything's clear!
+ * change it unless everything's clear!
  */
-#ifndef CONFIG_ETRAX_VCS_SIM
 #define TASK_SIZE	(0xB0000000UL)
-#else
-#define TASK_SIZE	(0xA0000000UL)
-#endif
 
 /* CCS I=1, enable interrupts. */
 #define INIT_THREAD { 0, 0, (1 << I_CCS_BITNR) }
diff --git a/arch/cris/include/arch-v32/mach-fs/mach/startup.inc b/arch/cris/include/arch-v32/mach-fs/mach/startup.inc
index dd1abbdcbc7..96c3b0fb62c 100644
--- a/arch/cris/include/arch-v32/mach-fs/mach/startup.inc
+++ b/arch/cris/include/arch-v32/mach-fs/mach/startup.inc
@@ -71,12 +71,6 @@
 	move.d   REG_ADDR(bif_core, regi_bif_core, rw_grp4_cfg), $r0
 	move.d   CONFIG_ETRAX_MEM_GRP4_CONFIG, $r1
 	move.d   $r1, [$r0]
-#ifdef CONFIG_ETRAX_VCS_SIM
-	;; Set up minimal flash waitstates
-	move.d 0, $r10
-	move.d REG_ADDR(bif_core, regi_bif_core, rw_grp1_cfg), $r11
-	move.d $r10, [$r11]
-#endif
 	.endm
 
 #endif
diff --git a/arch/cris/include/asm/elf.h b/arch/cris/include/asm/elf.h
index 8a3d8e2b33c..8182f2dc89d 100644
--- a/arch/cris/include/asm/elf.h
+++ b/arch/cris/include/asm/elf.h
@@ -86,6 +86,7 @@ typedef unsigned long elf_fpregset_t;
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif
diff --git a/arch/cris/include/asm/pci.h b/arch/cris/include/asm/pci.h
index 9f1cd56da28..146da904cdd 100644
--- a/arch/cris/include/asm/pci.h
+++ b/arch/cris/include/asm/pci.h
@@ -19,7 +19,6 @@ extern unsigned long pci_mem_start;
 
 void pcibios_config_init(void);
 struct pci_bus * pcibios_scan_root(int bus);
-int pcibios_assign_resources(void);
 
 void pcibios_set_master(struct pci_dev *dev);
 void pcibios_penalize_isa_irq(int irq);
diff --git a/arch/cris/include/uapi/arch-v10/arch/Kbuild b/arch/cris/include/uapi/arch-v10/arch/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/arch/cris/include/uapi/arch-v10/arch/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/arch/cris/include/uapi/arch-v32/arch/Kbuild b/arch/cris/include/uapi/arch-v32/arch/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/arch/cris/include/uapi/arch-v32/arch/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/arch/cris/include/uapi/asm/Kbuild b/arch/cris/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..f50236ae9ca
--- /dev/null
+++ b/arch/cris/include/uapi/asm/Kbuild
@@ -0,0 +1,5 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
+header-y += arch-v10/
+header-y += arch-v32/
diff --git a/arch/frv/include/asm/elf.h b/arch/frv/include/asm/elf.h
index c3819804a74..9ccbc80f0b1 100644
--- a/arch/frv/include/asm/elf.h
+++ b/arch/frv/include/asm/elf.h
@@ -137,6 +137,7 @@ do {											\
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif
diff --git a/arch/frv/include/uapi/asm/Kbuild b/arch/frv/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/frv/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/frv/kernel/pm.c b/arch/frv/kernel/pm.c
index 5fa3889d858..0b579927439 100644
--- a/arch/frv/kernel/pm.c
+++ b/arch/frv/kernel/pm.c
@@ -153,23 +153,22 @@ static int user_atoi(char __user *ubuf, size_t len)
 static int sysctl_pm_do_suspend(ctl_table *ctl, int write,
 				void __user *buffer, size_t *lenp, loff_t *fpos)
 {
-	int retval, mode;
+	int mode;
 
 	if (*lenp <= 0)
 		return -EIO;
 
 	mode = user_atoi(buffer, *lenp);
-	if ((mode != 1) && (mode != 5))
-		return -EINVAL;
+	switch (mode) {
+	case 1:
+	    return pm_do_suspend();
 
-	if (retval == 0) {
-		if (mode == 5)
-		    retval = pm_do_bus_sleep();
-		else
-		    retval = pm_do_suspend();
-	}
+	case 5:
+	    return pm_do_bus_sleep();
 
-	return retval;
+	default:
+	    return -EINVAL;
+	}
 }
 
 static int try_set_cmode(int new_cmode)
diff --git a/arch/frv/kernel/setup.c b/arch/frv/kernel/setup.c
index 75cf7f4b2fa..1f1e5efb338 100644
--- a/arch/frv/kernel/setup.c
+++ b/arch/frv/kernel/setup.c
@@ -184,7 +184,7 @@ static struct clock_cmode __pminitdata clock_cmodes_fr555[16] = {
 	[6]	= {	_x1,	_x1_5,	_x1_5,	_x4_5,	_x0_375	},
 };
 
-static const struct clock_cmode __pminitdata *clock_cmodes;
+static const struct clock_cmode __pminitconst *clock_cmodes;
 static int __pminitdata clock_doubled;
 
 static struct uart_port __pminitdata __frv_uart0 = {
diff --git a/arch/frv/mb93090-mb00/pci-irq.c b/arch/frv/mb93090-mb00/pci-irq.c
index 20f6497b2cd..c677b9d81d3 100644
--- a/arch/frv/mb93090-mb00/pci-irq.c
+++ b/arch/frv/mb93090-mb00/pci-irq.c
@@ -28,7 +28,7 @@
  *
  */
 
-static const uint8_t __initdata pci_bus0_irq_routing[32][4] = {
+static const uint8_t __initconst pci_bus0_irq_routing[32][4] = {
 	[0 ] = { IRQ_FPGA_MB86943_PCI_INTA },
 	[16] = { IRQ_FPGA_RTL8029_INTA },
 	[17] = { IRQ_FPGA_PCI_INTC, IRQ_FPGA_PCI_INTD, IRQ_FPGA_PCI_INTA, IRQ_FPGA_PCI_INTB },
diff --git a/arch/h8300/include/asm/elf.h b/arch/h8300/include/asm/elf.h
index c24fa250d65..41193c396bf 100644
--- a/arch/h8300/include/asm/elf.h
+++ b/arch/h8300/include/asm/elf.h
@@ -54,7 +54,8 @@ typedef unsigned long elf_fpregset_t;
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #define R_H8_NONE       0
 #define R_H8_DIR32      1
diff --git a/arch/h8300/include/uapi/asm/Kbuild b/arch/h8300/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/h8300/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/h8300/kernel/sys_h8300.c b/arch/h8300/kernel/sys_h8300.c
index aaf5e5a48f9..4bdc7311784 100644
--- a/arch/h8300/kernel/sys_h8300.c
+++ b/arch/h8300/kernel/sys_h8300.c
@@ -51,6 +51,7 @@ asmlinkage void syscall_print(void *dummy,...)
  * Do a system call from kernel instead of calling sys_execve so we
  * end up with proper pt_regs.
  */
+asmlinkage
 int kernel_execve(const char *filename,
 		  const char *const argv[],
 		  const char *const envp[])
diff --git a/arch/h8300/kernel/timer/itu.c b/arch/h8300/kernel/timer/itu.c
index a2ae5e95213..0a8b5cd5bf3 100644
--- a/arch/h8300/kernel/timer/itu.c
+++ b/arch/h8300/kernel/timer/itu.c
@@ -62,7 +62,7 @@ static struct irqaction itu_irq = {
 	.flags		= IRQF_DISABLED | IRQF_TIMER,
 };
 
-static const int __initdata divide_rate[] = {1, 2, 4, 8};
+static const int __initconst divide_rate[] = {1, 2, 4, 8};
 
 void __init h8300_timer_setup(void)
 {
diff --git a/arch/h8300/kernel/timer/timer16.c b/arch/h8300/kernel/timer/timer16.c
index ae0d3816113..462d9f58171 100644
--- a/arch/h8300/kernel/timer/timer16.c
+++ b/arch/h8300/kernel/timer/timer16.c
@@ -57,7 +57,7 @@ static struct irqaction timer16_irq = {
 	.flags		= IRQF_DISABLED | IRQF_TIMER,
 };
 
-static const int __initdata divide_rate[] = {1, 2, 4, 8};
+static const int __initconst divide_rate[] = {1, 2, 4, 8};
 
 void __init h8300_timer_setup(void)
 {
diff --git a/arch/h8300/kernel/timer/timer8.c b/arch/h8300/kernel/timer/timer8.c
index 7a1533fad47..505f3415b40 100644
--- a/arch/h8300/kernel/timer/timer8.c
+++ b/arch/h8300/kernel/timer/timer8.c
@@ -77,7 +77,7 @@ static struct irqaction timer8_irq = {
 	.flags		= IRQF_DISABLED | IRQF_TIMER,
 };
 
-static const int __initdata divide_rate[] = {8, 64, 8192};
+static const int __initconst divide_rate[] = {8, 64, 8192};
 
 void __init h8300_timer_setup(void)
 {
diff --git a/arch/h8300/kernel/timer/tpu.c b/arch/h8300/kernel/timer/tpu.c
index 2193a2e2859..0350f6204ec 100644
--- a/arch/h8300/kernel/timer/tpu.c
+++ b/arch/h8300/kernel/timer/tpu.c
@@ -66,7 +66,7 @@ static struct irqaction tpu_irq = {
 	.flags		= IRQF_DISABLED | IRQF_TIMER,
 };
 
-static const int __initdata divide_rate[] = {
+static const int __initconst divide_rate[] = {
 #if CONFIG_H8300_TPU_CH == 0
 	1,4,16,64,0,0,0,0,
 #elif (CONFIG_H8300_TPU_CH == 1) || (CONFIG_H8300_TPU_CH == 5)
diff --git a/arch/h8300/platform/h8300h/irq.c b/arch/h8300/platform/h8300h/irq.c
index bc4f51bceef..0a50353e09d 100644
--- a/arch/h8300/platform/h8300h/irq.c
+++ b/arch/h8300/platform/h8300h/irq.c
@@ -14,14 +14,14 @@
 #include <asm/gpio-internal.h>
 #include <asm/regs306x.h>
 
-const int __initdata h8300_saved_vectors[] = {
+const int __initconst h8300_saved_vectors[] = {
 #if defined(CONFIG_GDB_DEBUG)
 	TRAP3_VEC,	/* TRAPA #3 is GDB breakpoint */
 #endif
 	-1,
 };
 
-const h8300_vector __initdata h8300_trap_table[] = {
+const h8300_vector __initconst h8300_trap_table[] = {
 	0, 0, 0, 0, 0, 0, 0, 0,
 	system_call,
 	0,
diff --git a/arch/h8300/platform/h8s/irq.c b/arch/h8300/platform/h8s/irq.c
index 7b5f29febc0..f3a5511c16b 100644
--- a/arch/h8300/platform/h8s/irq.c
+++ b/arch/h8300/platform/h8s/irq.c
@@ -18,7 +18,7 @@
 #include <asm/regs267x.h>
 
 /* saved vector list */
-const int __initdata h8300_saved_vectors[]={
+const int __initconst h8300_saved_vectors[] = {
 #if defined(CONFIG_GDB_DEBUG)
 	TRACE_VEC,
 	TRAP3_VEC,
@@ -27,7 +27,7 @@ const int __initdata h8300_saved_vectors[]={
 };
 
 /* trap entry table */
-const H8300_VECTOR __initdata h8300_trap_table[] = {
+const H8300_VECTOR __initconst h8300_trap_table[] = {
 	0,0,0,0,0,
 	trace_break,  /* TRACE */
 	0,0,
diff --git a/arch/hexagon/include/asm/elf.h b/arch/hexagon/include/asm/elf.h
index 37976a0d365..82b499621e0 100644
--- a/arch/hexagon/include/asm/elf.h
+++ b/arch/hexagon/include/asm/elf.h
@@ -217,7 +217,8 @@ do {					\
 #define ELF_PLATFORM  (NULL)
 
 #ifdef __KERNEL__
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 #endif
 
 #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1
diff --git a/arch/hexagon/include/uapi/asm/Kbuild b/arch/hexagon/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/hexagon/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/ia64/include/asm/xen/interface.h b/arch/ia64/include/asm/xen/interface.h
index 09d5f7fd9db..3d52a5bbd85 100644
--- a/arch/ia64/include/asm/xen/interface.h
+++ b/arch/ia64/include/asm/xen/interface.h
@@ -67,6 +67,10 @@
 #define set_xen_guest_handle(hnd, val)	do { (hnd).p = val; } while (0)
 
 #ifndef __ASSEMBLY__
+/* Explicitly size integers that represent pfns in the public interface
+ * with Xen so that we could have one ABI that works for 32 and 64 bit
+ * guests. */
+typedef unsigned long xen_pfn_t;
 /* Guest handles for primitive C types. */
 __DEFINE_GUEST_HANDLE(uchar, unsigned char);
 __DEFINE_GUEST_HANDLE(uint, unsigned int);
@@ -79,7 +83,6 @@ DEFINE_GUEST_HANDLE(void);
 DEFINE_GUEST_HANDLE(uint64_t);
 DEFINE_GUEST_HANDLE(uint32_t);
 
-typedef unsigned long xen_pfn_t;
 DEFINE_GUEST_HANDLE(xen_pfn_t);
 #define PRI_xen_pfn	"lx"
 #endif
@@ -265,6 +268,8 @@ typedef struct xen_callback xen_callback_t;
 
 #endif /* !__ASSEMBLY__ */
 
+#include <asm/pvclock-abi.h>
+
 /* Size of the shared_info area (this is not related to page size).  */
 #define XSI_SHIFT			14
 #define XSI_SIZE			(1 << XSI_SHIFT)
diff --git a/arch/ia64/include/uapi/asm/Kbuild b/arch/ia64/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/ia64/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 5a5c22245de..f388b4e18a3 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -2306,7 +2306,7 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t
 	 * partially initialize the vma for the sampling buffer
 	 */
 	vma->vm_mm	     = mm;
-	vma->vm_file	     = filp;
+	vma->vm_file	     = get_file(filp);
 	vma->vm_flags	     = VM_READ| VM_MAYREAD |VM_RESERVED;
 	vma->vm_page_prot    = PAGE_READONLY; /* XXX may need to change */
 
@@ -2345,8 +2345,6 @@ pfm_smpl_buffer_alloc(struct task_struct *task, struct file *filp, pfm_context_t
 		goto error;
 	}
 
-	get_file(filp);
-
 	/*
 	 * now insert the vma in the vm list for the process, must be
 	 * done with mmap lock held
@@ -4782,7 +4780,7 @@ recheck:
 asmlinkage long
 sys_perfmonctl (int fd, int cmd, void __user *arg, int count)
 {
-	struct file *file = NULL;
+	struct fd f = {NULL, 0};
 	pfm_context_t *ctx = NULL;
 	unsigned long flags = 0UL;
 	void *args_k = NULL;
@@ -4879,17 +4877,17 @@ restart_args:
 
 	ret = -EBADF;
 
-	file = fget(fd);
-	if (unlikely(file == NULL)) {
+	f = fdget(fd);
+	if (unlikely(f.file == NULL)) {
 		DPRINT(("invalid fd %d\n", fd));
 		goto error_args;
 	}
-	if (unlikely(PFM_IS_FILE(file) == 0)) {
+	if (unlikely(PFM_IS_FILE(f.file) == 0)) {
 		DPRINT(("fd %d not related to perfmon\n", fd));
 		goto error_args;
 	}
 
-	ctx = file->private_data;
+	ctx = f.file->private_data;
 	if (unlikely(ctx == NULL)) {
 		DPRINT(("no context for fd %d\n", fd));
 		goto error_args;
@@ -4919,8 +4917,8 @@ abort_locked:
 	if (call_made && PFM_CMD_RW_ARG(cmd) && copy_to_user(arg, args_k, base_sz*count)) ret = -EFAULT;
 
 error_args:
-	if (file)
-		fput(file);
+	if (f.file)
+		fdput(f);
 
 	kfree(args_k);
 
diff --git a/arch/ia64/kvm/kvm-ia64.c b/arch/ia64/kvm/kvm-ia64.c
index bd77cb507c1..8b3a9c0e771 100644
--- a/arch/ia64/kvm/kvm-ia64.c
+++ b/arch/ia64/kvm/kvm-ia64.c
@@ -924,6 +924,16 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	return 0;
 }
 
+int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event)
+{
+	if (!irqchip_in_kernel(kvm))
+		return -ENXIO;
+
+	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
+					irq_event->irq, irq_event->level);
+	return 0;
+}
+
 long kvm_arch_vm_ioctl(struct file *filp,
 		unsigned int ioctl, unsigned long arg)
 {
@@ -963,29 +973,6 @@ long kvm_arch_vm_ioctl(struct file *filp,
 			goto out;
 		}
 		break;
-	case KVM_IRQ_LINE_STATUS:
-	case KVM_IRQ_LINE: {
-		struct kvm_irq_level irq_event;
-
-		r = -EFAULT;
-		if (copy_from_user(&irq_event, argp, sizeof irq_event))
-			goto out;
-		r = -ENXIO;
-		if (irqchip_in_kernel(kvm)) {
-			__s32 status;
-			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
-				    irq_event.irq, irq_event.level);
-			if (ioctl == KVM_IRQ_LINE_STATUS) {
-				r = -EFAULT;
-				irq_event.status = status;
-				if (copy_to_user(argp, &irq_event,
-							sizeof irq_event))
-					goto out;
-			}
-			r = 0;
-		}
-		break;
-		}
 	case KVM_GET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip chip;
@@ -1626,11 +1613,17 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 	return;
 }
 
-void kvm_arch_flush_shadow(struct kvm *kvm)
+void kvm_arch_flush_shadow_all(struct kvm *kvm)
 {
 	kvm_flush_remote_tlbs(kvm);
 }
 
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
+{
+	kvm_arch_flush_shadow_all();
+}
+
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
diff --git a/arch/ia64/xen/irq_xen.c b/arch/ia64/xen/irq_xen.c
index 3bb12230721..01f479ee1c4 100644
--- a/arch/ia64/xen/irq_xen.c
+++ b/arch/ia64/xen/irq_xen.c
@@ -433,7 +433,7 @@ xen_resend_irq(unsigned int vector)
 	(void)resend_irq_on_evtchn(vector);
 }
 
-const struct pv_irq_ops xen_irq_ops __initdata = {
+const struct pv_irq_ops xen_irq_ops __initconst = {
 	.register_ipi = xen_register_ipi,
 
 	.assign_irq_vector = xen_assign_irq_vector,
diff --git a/arch/ia64/xen/irq_xen.h b/arch/ia64/xen/irq_xen.h
index 26110f330c8..1778517b90f 100644
--- a/arch/ia64/xen/irq_xen.h
+++ b/arch/ia64/xen/irq_xen.h
@@ -27,7 +27,7 @@ extern void (*late_time_init)(void);
 extern char xen_event_callback;
 void __init xen_init_IRQ(void);
 
-extern const struct pv_irq_ops xen_irq_ops __initdata;
+extern const struct pv_irq_ops xen_irq_ops __initconst;
 extern void xen_smp_intr_init(void);
 extern void xen_send_ipi(int cpu, int vec);
 
diff --git a/arch/m32r/include/asm/elf.h b/arch/m32r/include/asm/elf.h
index b8da7d0574d..70896161c63 100644
--- a/arch/m32r/include/asm/elf.h
+++ b/arch/m32r/include/asm/elf.h
@@ -128,6 +128,7 @@ typedef elf_fpreg_t elf_fpregset_t;
    intent than poking at uname or /proc/cpuinfo.  */
 #define ELF_PLATFORM	(NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif  /* _ASM_M32R__ELF_H */
diff --git a/arch/m32r/include/uapi/asm/Kbuild b/arch/m32r/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/m32r/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/m68k/include/asm/cacheflush.h b/arch/m68k/include/asm/cacheflush.h
index a70d7319630..4fc738209bd 100644
--- a/arch/m68k/include/asm/cacheflush.h
+++ b/arch/m68k/include/asm/cacheflush.h
@@ -1,5 +1,5 @@
 #ifdef __uClinux__
-#include "cacheflush_no.h"
+#include <asm/cacheflush_no.h>
 #else
-#include "cacheflush_mm.h"
+#include <asm/cacheflush_mm.h>
 #endif
diff --git a/arch/m68k/include/asm/elf.h b/arch/m68k/include/asm/elf.h
index e9b7cda5974..f83c1d0a87c 100644
--- a/arch/m68k/include/asm/elf.h
+++ b/arch/m68k/include/asm/elf.h
@@ -113,6 +113,7 @@ typedef struct user_m68kfp_struct elf_fpregset_t;
 
 #define ELF_PLATFORM  (NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif
diff --git a/arch/m68k/include/asm/io.h b/arch/m68k/include/asm/io.h
index c7210ba184e..c70cc915500 100644
--- a/arch/m68k/include/asm/io.h
+++ b/arch/m68k/include/asm/io.h
@@ -1,5 +1,5 @@
 #ifdef __uClinux__
-#include "io_no.h"
+#include <asm/io_no.h>
 #else
-#include "io_mm.h"
+#include <asm/io_mm.h>
 #endif
diff --git a/arch/m68k/include/asm/m68360.h b/arch/m68k/include/asm/m68360.h
index eb7d39ef285..4664180a3ab 100644
--- a/arch/m68k/include/asm/m68360.h
+++ b/arch/m68k/include/asm/m68360.h
@@ -1,7 +1,7 @@
-#include "m68360_regs.h"
-#include "m68360_pram.h"
-#include "m68360_quicc.h"
-#include "m68360_enet.h"
+#include <asm/m68360_regs.h>
+#include <asm/m68360_pram.h>
+#include <asm/m68360_quicc.h>
+#include <asm/m68360_enet.h>
 
 #ifdef CONFIG_M68360
 
diff --git a/arch/m68k/include/asm/m68360_enet.h b/arch/m68k/include/asm/m68360_enet.h
index c36f4d05920..4d04037c78a 100644
--- a/arch/m68k/include/asm/m68360_enet.h
+++ b/arch/m68k/include/asm/m68360_enet.h
@@ -10,7 +10,7 @@
 #ifndef __ETHER_H
 #define __ETHER_H
 
-#include "quicc_simple.h"
+#include <asm/quicc_simple.h>
 
 /*
  * transmit BD's
diff --git a/arch/m68k/include/asm/page.h b/arch/m68k/include/asm/page.h
index 98baa82a861..7c360dac00b 100644
--- a/arch/m68k/include/asm/page.h
+++ b/arch/m68k/include/asm/page.h
@@ -43,9 +43,9 @@ extern unsigned long _ramend;
 #endif /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_MMU
-#include "page_mm.h"
+#include <asm/page_mm.h>
 #else
-#include "page_no.h"
+#include <asm/page_no.h>
 #endif
 
 #include <asm-generic/getorder.h>
diff --git a/arch/m68k/include/asm/pgtable.h b/arch/m68k/include/asm/pgtable.h
index ee6759eb445..a3d733b524d 100644
--- a/arch/m68k/include/asm/pgtable.h
+++ b/arch/m68k/include/asm/pgtable.h
@@ -1,5 +1,5 @@
 #ifdef __uClinux__
-#include "pgtable_no.h"
+#include <asm/pgtable_no.h>
 #else
-#include "pgtable_mm.h"
+#include <asm/pgtable_mm.h>
 #endif
diff --git a/arch/m68k/include/asm/q40_master.h b/arch/m68k/include/asm/q40_master.h
index 3907a09d4fc..fc5b36278d0 100644
--- a/arch/m68k/include/asm/q40_master.h
+++ b/arch/m68k/include/asm/q40_master.h
@@ -60,7 +60,7 @@
 #define Q40_RTC_WRITE  128
 
 /* define some Q40 specific ints */
-#include "q40ints.h"
+#include <asm/q40ints.h>
 
 /* misc defs */
 #define DAC_LEFT  ((unsigned char *)0xff008000)
diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h
index 38f92dbb9a4..639c731568b 100644
--- a/arch/m68k/include/asm/uaccess.h
+++ b/arch/m68k/include/asm/uaccess.h
@@ -1,5 +1,5 @@
 #ifdef __uClinux__
-#include "uaccess_no.h"
+#include <asm/uaccess_no.h>
 #else
-#include "uaccess_mm.h"
+#include <asm/uaccess_mm.h>
 #endif
diff --git a/arch/m68k/include/uapi/asm/Kbuild b/arch/m68k/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/m68k/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/microblaze/include/asm/elf.h b/arch/microblaze/include/asm/elf.h
index 834849f59ae..640ddd4b6a9 100644
--- a/arch/microblaze/include/asm/elf.h
+++ b/arch/microblaze/include/asm/elf.h
@@ -116,7 +116,8 @@ do {							\
 } while (0)
 
 #ifdef __KERNEL__
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK)))
 #endif
 
 #endif /* __uClinux__ */
diff --git a/arch/microblaze/include/asm/mmu_context.h b/arch/microblaze/include/asm/mmu_context.h
index 24eab1674d3..0ccd8c402cd 100644
--- a/arch/microblaze/include/asm/mmu_context.h
+++ b/arch/microblaze/include/asm/mmu_context.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_MMU
-# include "mmu_context_mm.h"
+# include <asm/mmu_context_mm.h>
 #else
 # include <asm-generic/mmu_context.h>
 #endif
diff --git a/arch/microblaze/include/uapi/asm/Kbuild b/arch/microblaze/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/microblaze/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/mips/bcm63xx/boards/board_bcm963xx.c b/arch/mips/bcm63xx/boards/board_bcm963xx.c
index feb05258a4d..dd18e4b761a 100644
--- a/arch/mips/bcm63xx/boards/board_bcm963xx.c
+++ b/arch/mips/bcm63xx/boards/board_bcm963xx.c
@@ -632,7 +632,7 @@ static struct board_info __initdata board_DWVS0 = {
 /*
  * all boards
  */
-static const struct board_info __initdata *bcm963xx_boards[] = {
+static const struct board_info __initconst *bcm963xx_boards[] = {
 #ifdef CONFIG_BCM63XX_CPU_6328
 	&board_96328avng,
 #endif
diff --git a/arch/mips/include/asm/compat-signal.h b/arch/mips/include/asm/compat-signal.h
index 368a99e5c3e..6599a901b63 100644
--- a/arch/mips/include/asm/compat-signal.h
+++ b/arch/mips/include/asm/compat-signal.h
@@ -10,68 +10,6 @@
 
 #include <asm/uaccess.h>
 
-#define SI_PAD_SIZE32   ((SI_MAX_SIZE/sizeof(int)) - 3)
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_code;
-	int si_errno;
-
-	union {
-		int _pad[SI_PAD_SIZE32];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			compat_uid_t _uid;	/* sender's uid */
-		} _kill;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;	/* which child */
-			compat_uid_t _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* IRIX SIGCHLD */
-		struct {
-			compat_pid_t _pid;	/* which child */
-			compat_clock_t _utime;
-			int _status;		/* exit code */
-			compat_clock_t _stime;
-		} _irix_sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			s32 _addr; /* faulting insn/memory ref. */
-		} _sigfault;
-
-		/* SIGPOLL, SIGXFSZ (To do ...)  */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		/* POSIX.1b timers */
-		struct {
-			timer_t _tid;		/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;/* same as below */
-			int _sys_private;       /* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			compat_uid_t _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-	} _sifields;
-} compat_siginfo_t;
-
 static inline int __copy_conv_sigset_to_user(compat_sigset_t __user *d,
 	const sigset_t *s)
 {
diff --git a/arch/mips/include/asm/compat.h b/arch/mips/include/asm/compat.h
index b77df0366ee..58277e0e9cd 100644
--- a/arch/mips/include/asm/compat.h
+++ b/arch/mips/include/asm/compat.h
@@ -43,6 +43,7 @@ typedef s64		compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -124,6 +125,73 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_code;
+	int si_errno;
+
+	union {
+		int _pad[SI_PAD_SIZE32];
+
+		/* kill() */
+		struct {
+			compat_pid_t _pid;	/* sender's pid */
+			__compat_uid_t _uid;	/* sender's uid */
+		} _kill;
+
+		/* SIGCHLD */
+		struct {
+			compat_pid_t _pid;	/* which child */
+			__compat_uid_t _uid;	/* sender's uid */
+			int _status;		/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* IRIX SIGCHLD */
+		struct {
+			compat_pid_t _pid;	/* which child */
+			compat_clock_t _utime;
+			int _status;		/* exit code */
+			compat_clock_t _stime;
+		} _irix_sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			s32 _addr; /* faulting insn/memory ref. */
+		} _sigfault;
+
+		/* SIGPOLL, SIGXFSZ (To do ...)  */
+		struct {
+			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+
+		/* POSIX.1b timers */
+		struct {
+			timer_t _tid;		/* timer id */
+			int _overrun;		/* overrun count */
+			compat_sigval_t _sigval;/* same as below */
+			int _sys_private;       /* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			compat_pid_t _pid;	/* sender's pid */
+			__compat_uid_t _uid;	/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -133,7 +201,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h
index 9203d90e610..03a54df5fb8 100644
--- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h
+++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_io.h
@@ -1,7 +1,7 @@
 #ifndef BCM63XX_IO_H_
 #define BCM63XX_IO_H_
 
-#include "bcm63xx_cpu.h"
+#include <asm/mach-bcm63xx/bcm63xx_cpu.h>
 
 /*
  * Physical memory map, RAM is mapped at 0x0.
diff --git a/arch/mips/include/asm/mach-pnx833x/gpio.h b/arch/mips/include/asm/mach-pnx833x/gpio.h
index ed3a88da70f..f192acf4a8a 100644
--- a/arch/mips/include/asm/mach-pnx833x/gpio.h
+++ b/arch/mips/include/asm/mach-pnx833x/gpio.h
@@ -30,7 +30,7 @@
    - including locking between different uses
 */
 
-#include "pnx833x.h"
+#include <asm/mach-pnx833x/pnx833x.h>
 
 #define SET_REG_BIT(reg, bit)		do { (reg |= (1 << (bit))); } while (0)
 #define CLEAR_REG_BIT(reg, bit)		do { (reg &= ~(1 << (bit))); } while (0)
diff --git a/arch/mips/include/asm/octeon/cvmx-asm.h b/arch/mips/include/asm/octeon/cvmx-asm.h
index 5de5de95311..31eacc24b77 100644
--- a/arch/mips/include/asm/octeon/cvmx-asm.h
+++ b/arch/mips/include/asm/octeon/cvmx-asm.h
@@ -32,7 +32,7 @@
 #ifndef __CVMX_ASM_H__
 #define __CVMX_ASM_H__
 
-#include "octeon-model.h"
+#include <asm/octeon/octeon-model.h>
 
 /* other useful stuff */
 #define CVMX_SYNC asm volatile ("sync" : : : "memory")
diff --git a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h
index 614653b686a..fed91125317 100644
--- a/arch/mips/include/asm/octeon/cvmx-cmd-queue.h
+++ b/arch/mips/include/asm/octeon/cvmx-cmd-queue.h
@@ -76,7 +76,7 @@
 
 #include <linux/prefetch.h>
 
-#include "cvmx-fpa.h"
+#include <asm/octeon/cvmx-fpa.h>
 /**
  * By default we disable the max depth support. Most programs
  * don't use it and it slows down the command queue processing
diff --git a/arch/mips/include/asm/octeon/cvmx-fpa.h b/arch/mips/include/asm/octeon/cvmx-fpa.h
index 1f04f965873..541a1ae02b6 100644
--- a/arch/mips/include/asm/octeon/cvmx-fpa.h
+++ b/arch/mips/include/asm/octeon/cvmx-fpa.h
@@ -36,8 +36,8 @@
 #ifndef __CVMX_FPA_H__
 #define __CVMX_FPA_H__
 
-#include "cvmx-address.h"
-#include "cvmx-fpa-defs.h"
+#include <asm/octeon/cvmx-address.h>
+#include <asm/octeon/cvmx-fpa-defs.h>
 
 #define CVMX_FPA_NUM_POOLS      8
 #define CVMX_FPA_MIN_BLOCK_SIZE 128
diff --git a/arch/mips/include/asm/octeon/cvmx-helper-board.h b/arch/mips/include/asm/octeon/cvmx-helper-board.h
index 88527fa835c..442f508eaac 100644
--- a/arch/mips/include/asm/octeon/cvmx-helper-board.h
+++ b/arch/mips/include/asm/octeon/cvmx-helper-board.h
@@ -34,7 +34,7 @@
 #ifndef __CVMX_HELPER_BOARD_H__
 #define __CVMX_HELPER_BOARD_H__
 
-#include "cvmx-helper.h"
+#include <asm/octeon/cvmx-helper.h>
 
 typedef enum {
 	set_phy_link_flags_autoneg = 0x1,
diff --git a/arch/mips/include/asm/octeon/cvmx-helper.h b/arch/mips/include/asm/octeon/cvmx-helper.h
index 0ac6b9f412b..691c8142cd4 100644
--- a/arch/mips/include/asm/octeon/cvmx-helper.h
+++ b/arch/mips/include/asm/octeon/cvmx-helper.h
@@ -34,9 +34,9 @@
 #ifndef __CVMX_HELPER_H__
 #define __CVMX_HELPER_H__
 
-#include "cvmx-config.h"
-#include "cvmx-fpa.h"
-#include "cvmx-wqe.h"
+#include <asm/octeon/cvmx-config.h>
+#include <asm/octeon/cvmx-fpa.h>
+#include <asm/octeon/cvmx-wqe.h>
 
 typedef enum {
 	CVMX_HELPER_INTERFACE_MODE_DISABLED,
@@ -62,13 +62,13 @@ typedef union {
 } cvmx_helper_link_info_t;
 
 #include <asm/octeon/cvmx-helper-errata.h>
-#include "cvmx-helper-loop.h"
-#include "cvmx-helper-npi.h"
-#include "cvmx-helper-rgmii.h"
-#include "cvmx-helper-sgmii.h"
-#include "cvmx-helper-spi.h"
-#include "cvmx-helper-util.h"
-#include "cvmx-helper-xaui.h"
+#include <asm/octeon/cvmx-helper-loop.h>
+#include <asm/octeon/cvmx-helper-npi.h>
+#include <asm/octeon/cvmx-helper-rgmii.h>
+#include <asm/octeon/cvmx-helper-sgmii.h>
+#include <asm/octeon/cvmx-helper-spi.h>
+#include <asm/octeon/cvmx-helper-util.h>
+#include <asm/octeon/cvmx-helper-xaui.h>
 
 /**
  * cvmx_override_pko_queue_priority(int ipd_port, uint64_t
diff --git a/arch/mips/include/asm/octeon/cvmx-mdio.h b/arch/mips/include/asm/octeon/cvmx-mdio.h
index d88ab8d8e37..6f0cd182cec 100644
--- a/arch/mips/include/asm/octeon/cvmx-mdio.h
+++ b/arch/mips/include/asm/octeon/cvmx-mdio.h
@@ -35,7 +35,7 @@
 #ifndef __CVMX_MIO_H__
 #define __CVMX_MIO_H__
 
-#include "cvmx-smix-defs.h"
+#include <asm/octeon/cvmx-smix-defs.h>
 
 /**
  * PHY register 0 from the 802.3 spec
diff --git a/arch/mips/include/asm/octeon/cvmx-pip.h b/arch/mips/include/asm/octeon/cvmx-pip.h
index 78dbce8f2c5..9e739a64085 100644
--- a/arch/mips/include/asm/octeon/cvmx-pip.h
+++ b/arch/mips/include/asm/octeon/cvmx-pip.h
@@ -33,9 +33,9 @@
 #ifndef __CVMX_PIP_H__
 #define __CVMX_PIP_H__
 
-#include "cvmx-wqe.h"
-#include "cvmx-fpa.h"
-#include "cvmx-pip-defs.h"
+#include <asm/octeon/cvmx-wqe.h>
+#include <asm/octeon/cvmx-fpa.h>
+#include <asm/octeon/cvmx-pip-defs.h>
 
 #define CVMX_PIP_NUM_INPUT_PORTS                40
 #define CVMX_PIP_NUM_WATCHERS                   4
diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h
index de3412aada5..c6daeedf1f8 100644
--- a/arch/mips/include/asm/octeon/cvmx-pko.h
+++ b/arch/mips/include/asm/octeon/cvmx-pko.h
@@ -58,10 +58,10 @@
 #ifndef __CVMX_PKO_H__
 #define __CVMX_PKO_H__
 
-#include "cvmx-fpa.h"
-#include "cvmx-pow.h"
-#include "cvmx-cmd-queue.h"
-#include "cvmx-pko-defs.h"
+#include <asm/octeon/cvmx-fpa.h>
+#include <asm/octeon/cvmx-pow.h>
+#include <asm/octeon/cvmx-cmd-queue.h>
+#include <asm/octeon/cvmx-pko-defs.h>
 
 /* Adjust the command buffer size by 1 word so that in the case of using only
  * two word PKO commands no command words stradle buffers.  The useful values
diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h
index 999aefe3274..92742b241a5 100644
--- a/arch/mips/include/asm/octeon/cvmx-pow.h
+++ b/arch/mips/include/asm/octeon/cvmx-pow.h
@@ -53,8 +53,8 @@
 
 #include <asm/octeon/cvmx-pow-defs.h>
 
-#include "cvmx-scratch.h"
-#include "cvmx-wqe.h"
+#include <asm/octeon/cvmx-scratch.h>
+#include <asm/octeon/cvmx-wqe.h>
 
 /* Default to having all POW constancy checks turned on */
 #ifndef CVMX_ENABLE_POW_CHECKS
diff --git a/arch/mips/include/asm/octeon/cvmx-spi.h b/arch/mips/include/asm/octeon/cvmx-spi.h
index e814648953a..3bf53b537bc 100644
--- a/arch/mips/include/asm/octeon/cvmx-spi.h
+++ b/arch/mips/include/asm/octeon/cvmx-spi.h
@@ -32,7 +32,7 @@
 #ifndef __CVMX_SPI_H__
 #define __CVMX_SPI_H__
 
-#include "cvmx-gmxx-defs.h"
+#include <asm/octeon/cvmx-gmxx-defs.h>
 
 /* CSR typedefs have been moved to cvmx-csr-*.h */
 
diff --git a/arch/mips/include/asm/octeon/cvmx-spinlock.h b/arch/mips/include/asm/octeon/cvmx-spinlock.h
index 2fbf0871df1..a672abb1bc4 100644
--- a/arch/mips/include/asm/octeon/cvmx-spinlock.h
+++ b/arch/mips/include/asm/octeon/cvmx-spinlock.h
@@ -35,7 +35,7 @@
 #ifndef __CVMX_SPINLOCK_H__
 #define __CVMX_SPINLOCK_H__
 
-#include "cvmx-asm.h"
+#include <asm/octeon/cvmx-asm.h>
 
 /* Spinlocks for Octeon */
 
diff --git a/arch/mips/include/asm/octeon/cvmx-wqe.h b/arch/mips/include/asm/octeon/cvmx-wqe.h
index 653610953d2..df762389e27 100644
--- a/arch/mips/include/asm/octeon/cvmx-wqe.h
+++ b/arch/mips/include/asm/octeon/cvmx-wqe.h
@@ -40,7 +40,7 @@
 #ifndef __CVMX_WQE_H__
 #define __CVMX_WQE_H__
 
-#include "cvmx-packet.h"
+#include <asm/octeon/cvmx-packet.h>
 
 
 #define OCT_TAG_TYPE_STRING(x)						\
diff --git a/arch/mips/include/asm/octeon/cvmx.h b/arch/mips/include/asm/octeon/cvmx.h
index 740be97a325..db58beab6cb 100644
--- a/arch/mips/include/asm/octeon/cvmx.h
+++ b/arch/mips/include/asm/octeon/cvmx.h
@@ -52,24 +52,24 @@ enum cvmx_mips_space {
 #define CVMX_ADD_IO_SEG(add) CVMX_ADD_SEG(CVMX_IO_SEG, (add))
 #endif
 
-#include "cvmx-asm.h"
-#include "cvmx-packet.h"
-#include "cvmx-sysinfo.h"
-
-#include "cvmx-ciu-defs.h"
-#include "cvmx-gpio-defs.h"
-#include "cvmx-iob-defs.h"
-#include "cvmx-ipd-defs.h"
-#include "cvmx-l2c-defs.h"
-#include "cvmx-l2d-defs.h"
-#include "cvmx-l2t-defs.h"
-#include "cvmx-led-defs.h"
-#include "cvmx-mio-defs.h"
-#include "cvmx-pow-defs.h"
-
-#include "cvmx-bootinfo.h"
-#include "cvmx-bootmem.h"
-#include "cvmx-l2c.h"
+#include <asm/octeon/cvmx-asm.h>
+#include <asm/octeon/cvmx-packet.h>
+#include <asm/octeon/cvmx-sysinfo.h>
+
+#include <asm/octeon/cvmx-ciu-defs.h>
+#include <asm/octeon/cvmx-gpio-defs.h>
+#include <asm/octeon/cvmx-iob-defs.h>
+#include <asm/octeon/cvmx-ipd-defs.h>
+#include <asm/octeon/cvmx-l2c-defs.h>
+#include <asm/octeon/cvmx-l2d-defs.h>
+#include <asm/octeon/cvmx-l2t-defs.h>
+#include <asm/octeon/cvmx-led-defs.h>
+#include <asm/octeon/cvmx-mio-defs.h>
+#include <asm/octeon/cvmx-pow-defs.h>
+
+#include <asm/octeon/cvmx-bootinfo.h>
+#include <asm/octeon/cvmx-bootmem.h>
+#include <asm/octeon/cvmx-l2c.h>
 
 #ifndef CVMX_ENABLE_DEBUG_PRINTS
 #define CVMX_ENABLE_DEBUG_PRINTS 1
diff --git a/arch/mips/include/asm/octeon/octeon-model.h b/arch/mips/include/asm/octeon/octeon-model.h
index 4e338a4d942..23b895cb260 100644
--- a/arch/mips/include/asm/octeon/octeon-model.h
+++ b/arch/mips/include/asm/octeon/octeon-model.h
@@ -313,6 +313,6 @@ static inline int __octeon_is_model_runtime__(uint32_t model)
 const char *octeon_model_get_string(uint32_t chip_id);
 const char *octeon_model_get_string_buffer(uint32_t chip_id, char *buffer);
 
-#include "octeon-feature.h"
+#include <asm/octeon/octeon-feature.h>
 
 #endif /* __OCTEON_MODEL_H__ */
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h
index 1e2486e2357..c4a1b31966b 100644
--- a/arch/mips/include/asm/octeon/octeon.h
+++ b/arch/mips/include/asm/octeon/octeon.h
@@ -8,7 +8,7 @@
 #ifndef __ASM_OCTEON_OCTEON_H
 #define __ASM_OCTEON_OCTEON_H
 
-#include "cvmx.h"
+#include <asm/octeon/cvmx.h>
 
 extern uint64_t octeon_bootmem_alloc_range_phys(uint64_t size,
 						uint64_t alignment,
diff --git a/arch/mips/include/asm/sibyte/bcm1480_int.h b/arch/mips/include/asm/sibyte/bcm1480_int.h
index 6109557c14e..fffb224d229 100644
--- a/arch/mips/include/asm/sibyte/bcm1480_int.h
+++ b/arch/mips/include/asm/sibyte/bcm1480_int.h
@@ -34,7 +34,7 @@
 #ifndef _BCM1480_INT_H
 #define _BCM1480_INT_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  Interrupt Mapper Constants
diff --git a/arch/mips/include/asm/sibyte/bcm1480_l2c.h b/arch/mips/include/asm/sibyte/bcm1480_l2c.h
index fd75817f7ac..725d38cb9d1 100644
--- a/arch/mips/include/asm/sibyte/bcm1480_l2c.h
+++ b/arch/mips/include/asm/sibyte/bcm1480_l2c.h
@@ -33,7 +33,7 @@
 #ifndef _BCM1480_L2C_H
 #define _BCM1480_L2C_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Format of level 2 cache management address (Table 55)
diff --git a/arch/mips/include/asm/sibyte/bcm1480_mc.h b/arch/mips/include/asm/sibyte/bcm1480_mc.h
index f26a41a82b5..4307a758e3b 100644
--- a/arch/mips/include/asm/sibyte/bcm1480_mc.h
+++ b/arch/mips/include/asm/sibyte/bcm1480_mc.h
@@ -33,7 +33,7 @@
 #ifndef _BCM1480_MC_H
 #define _BCM1480_MC_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Memory Channel Configuration Register (Table 81)
diff --git a/arch/mips/include/asm/sibyte/bcm1480_regs.h b/arch/mips/include/asm/sibyte/bcm1480_regs.h
index b4077bb7261..84d168ddfeb 100644
--- a/arch/mips/include/asm/sibyte/bcm1480_regs.h
+++ b/arch/mips/include/asm/sibyte/bcm1480_regs.h
@@ -32,14 +32,14 @@
 #ifndef _BCM1480_REGS_H
 #define _BCM1480_REGS_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  Pull in the BCM1250's registers since a great deal of the 1480's
     *  functions are the same as the BCM1250.
     ********************************************************************* */
 
-#include "sb1250_regs.h"
+#include <asm/sibyte/sb1250_regs.h>
 
 
 /*  *********************************************************************
diff --git a/arch/mips/include/asm/sibyte/bcm1480_scd.h b/arch/mips/include/asm/sibyte/bcm1480_scd.h
index 25ef24cbb92..2af3706b964 100644
--- a/arch/mips/include/asm/sibyte/bcm1480_scd.h
+++ b/arch/mips/include/asm/sibyte/bcm1480_scd.h
@@ -32,13 +32,13 @@
 #ifndef _BCM1480_SCD_H
 #define _BCM1480_SCD_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  Pull in the BCM1250's SCD since lots of stuff is the same.
     ********************************************************************* */
 
-#include "sb1250_scd.h"
+#include <asm/sibyte/sb1250_scd.h>
 
 /*  *********************************************************************
     *  Some general notes:
diff --git a/arch/mips/include/asm/sibyte/sb1250_dma.h b/arch/mips/include/asm/sibyte/sb1250_dma.h
index bad56171d74..6c44dfb5287 100644
--- a/arch/mips/include/asm/sibyte/sb1250_dma.h
+++ b/arch/mips/include/asm/sibyte/sb1250_dma.h
@@ -36,7 +36,7 @@
 #define _SB1250_DMA_H
 
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  DMA Registers
diff --git a/arch/mips/include/asm/sibyte/sb1250_genbus.h b/arch/mips/include/asm/sibyte/sb1250_genbus.h
index 94e9c7c8e78..a96ded17bdc 100644
--- a/arch/mips/include/asm/sibyte/sb1250_genbus.h
+++ b/arch/mips/include/asm/sibyte/sb1250_genbus.h
@@ -34,7 +34,7 @@
 #ifndef _SB1250_GENBUS_H
 #define _SB1250_GENBUS_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Generic Bus Region Configuration Registers (Table 11-4)
diff --git a/arch/mips/include/asm/sibyte/sb1250_int.h b/arch/mips/include/asm/sibyte/sb1250_int.h
index f2850b4bcfd..dbea73ddd2f 100644
--- a/arch/mips/include/asm/sibyte/sb1250_int.h
+++ b/arch/mips/include/asm/sibyte/sb1250_int.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_INT_H
 #define _SB1250_INT_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  Interrupt Mapper Constants
diff --git a/arch/mips/include/asm/sibyte/sb1250_l2c.h b/arch/mips/include/asm/sibyte/sb1250_l2c.h
index 6554dcf05cf..b61a7491607 100644
--- a/arch/mips/include/asm/sibyte/sb1250_l2c.h
+++ b/arch/mips/include/asm/sibyte/sb1250_l2c.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_L2C_H
 #define _SB1250_L2C_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Level 2 Cache Tag register (Table 5-3)
diff --git a/arch/mips/include/asm/sibyte/sb1250_ldt.h b/arch/mips/include/asm/sibyte/sb1250_ldt.h
index 1e76cf13799..bf7f320d1a8 100644
--- a/arch/mips/include/asm/sibyte/sb1250_ldt.h
+++ b/arch/mips/include/asm/sibyte/sb1250_ldt.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_LDT_H
 #define _SB1250_LDT_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 #define K_LDT_VENDOR_SIBYTE	0x166D
 #define K_LDT_DEVICE_SB1250	0x0002
diff --git a/arch/mips/include/asm/sibyte/sb1250_mac.h b/arch/mips/include/asm/sibyte/sb1250_mac.h
index 77f78728423..cfc4d787088 100644
--- a/arch/mips/include/asm/sibyte/sb1250_mac.h
+++ b/arch/mips/include/asm/sibyte/sb1250_mac.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_MAC_H
 #define _SB1250_MAC_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  Ethernet MAC Registers
diff --git a/arch/mips/include/asm/sibyte/sb1250_mc.h b/arch/mips/include/asm/sibyte/sb1250_mc.h
index 1eb1b5a8873..15048dcaf22 100644
--- a/arch/mips/include/asm/sibyte/sb1250_mc.h
+++ b/arch/mips/include/asm/sibyte/sb1250_mc.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_MC_H
 #define _SB1250_MC_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Memory Channel Config Register (table 6-14)
diff --git a/arch/mips/include/asm/sibyte/sb1250_regs.h b/arch/mips/include/asm/sibyte/sb1250_regs.h
index 8f53ec817a5..29b9f0b26b3 100644
--- a/arch/mips/include/asm/sibyte/sb1250_regs.h
+++ b/arch/mips/include/asm/sibyte/sb1250_regs.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_REGS_H
 #define _SB1250_REGS_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 
 /*  *********************************************************************
diff --git a/arch/mips/include/asm/sibyte/sb1250_scd.h b/arch/mips/include/asm/sibyte/sb1250_scd.h
index e49c3e89b5e..615e165dbd2 100644
--- a/arch/mips/include/asm/sibyte/sb1250_scd.h
+++ b/arch/mips/include/asm/sibyte/sb1250_scd.h
@@ -32,7 +32,7 @@
 #ifndef _SB1250_SCD_H
 #define _SB1250_SCD_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*  *********************************************************************
     *  System control/debug registers
diff --git a/arch/mips/include/asm/sibyte/sb1250_smbus.h b/arch/mips/include/asm/sibyte/sb1250_smbus.h
index 04769923cf1..128d6b75b81 100644
--- a/arch/mips/include/asm/sibyte/sb1250_smbus.h
+++ b/arch/mips/include/asm/sibyte/sb1250_smbus.h
@@ -34,7 +34,7 @@
 #ifndef _SB1250_SMBUS_H
 #define _SB1250_SMBUS_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * SMBus Clock Frequency Register (Table 14-2)
diff --git a/arch/mips/include/asm/sibyte/sb1250_syncser.h b/arch/mips/include/asm/sibyte/sb1250_syncser.h
index d4b8558e0bf..274e9179d32 100644
--- a/arch/mips/include/asm/sibyte/sb1250_syncser.h
+++ b/arch/mips/include/asm/sibyte/sb1250_syncser.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_SYNCSER_H
 #define _SB1250_SYNCSER_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /*
  * Serial Mode Configuration Register
diff --git a/arch/mips/include/asm/sibyte/sb1250_uart.h b/arch/mips/include/asm/sibyte/sb1250_uart.h
index d835bf28014..bb99ecac581 100644
--- a/arch/mips/include/asm/sibyte/sb1250_uart.h
+++ b/arch/mips/include/asm/sibyte/sb1250_uart.h
@@ -33,7 +33,7 @@
 #ifndef _SB1250_UART_H
 #define _SB1250_UART_H
 
-#include "sb1250_defs.h"
+#include <asm/sibyte/sb1250_defs.h>
 
 /* **********************************************************************
    * DUART Registers
diff --git a/arch/mips/include/uapi/asm/Kbuild b/arch/mips/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/mips/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/mips/pci/pci-octeon.c b/arch/mips/pci/pci-octeon.c
index c5dfb2c87d4..4b0c347d7a8 100644
--- a/arch/mips/pci/pci-octeon.c
+++ b/arch/mips/pci/pci-octeon.c
@@ -58,7 +58,7 @@ union octeon_pci_address {
 	} s;
 };
 
-int __initdata (*octeon_pcibios_map_irq)(const struct pci_dev *dev,
+int __initconst (*octeon_pcibios_map_irq)(const struct pci_dev *dev,
 					 u8 slot, u8 pin);
 enum octeon_dma_bar_type octeon_dma_bar_type = OCTEON_DMA_BAR_TYPE_INVALID;
 
diff --git a/arch/mn10300/Makefile b/arch/mn10300/Makefile
index 33188b6e81e..a3d0fef3b12 100644
--- a/arch/mn10300/Makefile
+++ b/arch/mn10300/Makefile
@@ -26,7 +26,7 @@ CHECKFLAGS	+=
 PROCESSOR	:= unset
 UNIT		:= unset
 
-KBUILD_CFLAGS	+= -mam33 -mmem-funcs -DCPU=AM33
+KBUILD_CFLAGS	+= -mam33 -DCPU=AM33 $(call cc-option,-mmem-funcs,)
 KBUILD_AFLAGS	+= -mam33 -DCPU=AM33
 
 ifeq ($(CONFIG_MN10300_CURRENT_IN_E2),y)
diff --git a/arch/mn10300/include/asm/elf.h b/arch/mn10300/include/asm/elf.h
index 8157c9267f4..4ebd6b3a0a1 100644
--- a/arch/mn10300/include/asm/elf.h
+++ b/arch/mn10300/include/asm/elf.h
@@ -151,7 +151,8 @@ do {						\
 #define ELF_PLATFORM  (NULL)
 
 #ifdef __KERNEL__
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 #endif
 
 #endif /* _ASM_ELF_H */
diff --git a/arch/mn10300/include/uapi/asm/Kbuild b/arch/mn10300/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/mn10300/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/openrisc/include/asm/elf.h b/arch/openrisc/include/asm/elf.h
index a8fe2c51307..225a7ff320a 100644
--- a/arch/openrisc/include/asm/elf.h
+++ b/arch/openrisc/include/asm/elf.h
@@ -110,7 +110,8 @@ extern void dump_elf_thread(elf_greg_t *dest, struct pt_regs *pt);
 
 #define ELF_PLATFORM	(NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif /* __KERNEL__ */
 #endif
diff --git a/arch/openrisc/include/uapi/asm/Kbuild b/arch/openrisc/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/openrisc/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index 3ff21b536f2..b87438bb338 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -13,6 +13,7 @@ config PARISC
 	select HAVE_PERF_EVENTS
 	select GENERIC_ATOMIC64 if !64BIT
 	select HAVE_GENERIC_HARDIRQS
+	select BROKEN_RODATA
 	select GENERIC_IRQ_PROBE
 	select GENERIC_PCI_IOMAP
 	select IRQ_PER_CPU
diff --git a/arch/parisc/hpux/fs.c b/arch/parisc/hpux/fs.c
index c71eb6c7989..6785de7bd2a 100644
--- a/arch/parisc/hpux/fs.c
+++ b/arch/parisc/hpux/fs.c
@@ -109,33 +109,32 @@ Efault:
 
 int hpux_getdents(unsigned int fd, struct hpux_dirent __user *dirent, unsigned int count)
 {
-	struct file * file;
+	struct fd arg;
 	struct hpux_dirent __user * lastdirent;
 	struct getdents_callback buf;
-	int error = -EBADF;
+	int error;
 
-	file = fget(fd);
-	if (!file)
-		goto out;
+	arg = fdget(fd);
+	if (!arg.file)
+		return -EBADF;
 
 	buf.current_dir = dirent;
 	buf.previous = NULL;
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, filldir, &buf);
+	error = vfs_readdir(arg.file, filldir, &buf);
 	if (error >= 0)
 		error = buf.error;
 	lastdirent = buf.previous;
 	if (lastdirent) {
-		if (put_user(file->f_pos, &lastdirent->d_off))
+		if (put_user(arg.file->f_pos, &lastdirent->d_off))
 			error = -EFAULT;
 		else
 			error = count - buf.count;
 	}
 
-	fput(file);
-out:
+	fdput(arg);
 	return error;
 }
 
diff --git a/arch/parisc/include/asm/compat.h b/arch/parisc/include/asm/compat.h
index 760f331d4fa..db7a662691a 100644
--- a/arch/parisc/include/asm/compat.h
+++ b/arch/parisc/include/asm/compat.h
@@ -36,6 +36,7 @@ typedef s64	compat_s64;
 typedef u32	compat_uint_t;
 typedef u32	compat_ulong_t;
 typedef u64	compat_u64;
+typedef u32	compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t		tv_sec;
@@ -127,6 +128,63 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		int _pad[128/sizeof(int) - 3];
+
+		/* kill() */
+		struct {
+			unsigned int _pid;      /* sender's pid */
+			unsigned int _uid;      /* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;            /* timer id */
+			int _overrun;           /* overrun count */
+			char _pad[sizeof(unsigned int) - sizeof(int)];
+			compat_sigval_t _sigval;        /* same as below */
+			int _sys_private;       /* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			unsigned int _pid;      /* sender's pid */
+			unsigned int _uid;      /* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			unsigned int _pid;      /* which child */
+			unsigned int _uid;      /* sender's uid */
+			int _status;            /* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			unsigned int _addr;     /* faulting insn/memory ref. */
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int _band;      /* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -136,7 +194,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/parisc/include/uapi/asm/Kbuild b/arch/parisc/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/parisc/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/parisc/kernel/signal32.h b/arch/parisc/kernel/signal32.h
index c7800846422..08a88b5349a 100644
--- a/arch/parisc/kernel/signal32.h
+++ b/arch/parisc/kernel/signal32.h
@@ -55,58 +55,6 @@ struct k_sigaction32 {
 	struct compat_sigaction sa;
 };
 
-typedef struct compat_siginfo {
-        int si_signo;
-        int si_errno;
-        int si_code;
-
-        union {
-                int _pad[((128/sizeof(int)) - 3)];
-
-                /* kill() */
-                struct {
-                        unsigned int _pid;      /* sender's pid */
-                        unsigned int _uid;      /* sender's uid */
-                } _kill;
-
-                /* POSIX.1b timers */
-                struct {
-                        compat_timer_t _tid;            /* timer id */
-                        int _overrun;           /* overrun count */
-                        char _pad[sizeof(unsigned int) - sizeof(int)];
-                        compat_sigval_t _sigval;        /* same as below */
-                        int _sys_private;       /* not to be passed to user */
-                } _timer;
-
-                /* POSIX.1b signals */
-                struct {
-                        unsigned int _pid;      /* sender's pid */
-                        unsigned int _uid;      /* sender's uid */
-                        compat_sigval_t _sigval;
-                } _rt;
-
-                /* SIGCHLD */
-                struct {
-                        unsigned int _pid;      /* which child */
-                        unsigned int _uid;      /* sender's uid */
-                        int _status;            /* exit code */
-                        compat_clock_t _utime;
-                        compat_clock_t _stime;
-                } _sigchld;
-
-                /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-                struct {
-                        unsigned int _addr;     /* faulting insn/memory ref. */
-                } _sigfault;
-
-                /* SIGPOLL */
-                struct {
-                        int _band;      /* POLL_IN, POLL_OUT, POLL_MSG */
-                        int _fd;
-                } _sigpoll;
-        } _sifields;
-} compat_siginfo_t;
-
 int copy_siginfo_to_user32 (compat_siginfo_t __user *to, siginfo_t *from);
 int copy_siginfo_from_user32 (siginfo_t *to, compat_siginfo_t __user *from);
 
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 06b56245d78..de7c4c53f5c 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -486,7 +486,8 @@ CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
 # CONFIG_CRYPTO_ANSI_CPRNG is not set
 CONFIG_CRYPTO_HW=y
-CONFIG_CRYPTO_DEV_NX=m
+CONFIG_CRYPTO_DEV_NX=y
+CONFIG_CRYPTO_DEV_NX_ENCRYPT=m
 CONFIG_VIRTUALIZATION=y
 CONFIG_KVM_BOOK3S_64=m
 CONFIG_KVM_BOOK3S_64_HV=y
diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig
index 1f65b3c9b59..9f4a9368f51 100644
--- a/arch/powerpc/configs/pseries_defconfig
+++ b/arch/powerpc/configs/pseries_defconfig
@@ -369,7 +369,8 @@ CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_LZO=m
 # CONFIG_CRYPTO_ANSI_CPRNG is not set
 CONFIG_CRYPTO_HW=y
-CONFIG_CRYPTO_DEV_NX=m
+CONFIG_CRYPTO_DEV_NX=y
+CONFIG_CRYPTO_DEV_NX_ENCRYPT=m
 CONFIG_VIRTUALIZATION=y
 CONFIG_KVM_BOOK3S_64=m
 CONFIG_KVM_BOOK3S_64_HV=y
diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h
index efdc92618b3..dc2cf9c6d9e 100644
--- a/arch/powerpc/include/asm/bitops.h
+++ b/arch/powerpc/include/asm/bitops.h
@@ -288,6 +288,16 @@ static __inline__ int test_bit_le(unsigned long nr,
 	return (tmp[nr >> 3] >> (nr & 7)) & 1;
 }
 
+static inline void set_bit_le(int nr, void *addr)
+{
+	set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
+}
+
+static inline void clear_bit_le(int nr, void *addr)
+{
+	clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
+}
+
 static inline void __set_bit_le(int nr, void *addr)
 {
 	__set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h
index 88e602f6430..84fdf6857c3 100644
--- a/arch/powerpc/include/asm/compat.h
+++ b/arch/powerpc/include/asm/compat.h
@@ -38,6 +38,7 @@ typedef s64		compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -114,6 +115,64 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		int _pad[SI_PAD_SIZE32];
+
+		/* kill() */
+		struct {
+			compat_pid_t _pid;		/* sender's pid */
+			__compat_uid_t _uid;		/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;		/* timer id */
+			int _overrun;			/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;	/* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			compat_pid_t _pid;		/* sender's pid */
+			__compat_uid_t _uid;		/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			compat_pid_t _pid;		/* which child */
+			__compat_uid_t _uid;		/* sender's uid */
+			int _status;			/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
+		struct {
+			unsigned int _addr; /* faulting insn/memory ref. */
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -123,7 +182,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index a8bf5c673a3..28e8f5e5c63 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -53,6 +53,8 @@
 
 struct kvm;
 extern int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
+extern int kvm_unmap_hva_range(struct kvm *kvm,
+			       unsigned long start, unsigned long end);
 extern int kvm_age_hva(struct kvm *kvm, unsigned long hva);
 extern int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 extern void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
@@ -220,6 +222,7 @@ struct revmap_entry {
 #define KVMPPC_GOT_PAGE		0x80
 
 struct kvm_arch_memory_slot {
+	unsigned long *rmap;
 };
 
 struct kvm_arch {
diff --git a/arch/powerpc/include/asm/ps3.h b/arch/powerpc/include/asm/ps3.h
index 7f065e178ec..0e15db4d703 100644
--- a/arch/powerpc/include/asm/ps3.h
+++ b/arch/powerpc/include/asm/ps3.h
@@ -24,7 +24,7 @@
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/device.h>
-#include "cell-pmu.h"
+#include <asm/cell-pmu.h>
 
 union ps3_firmware_version {
 	u64 raw;
diff --git a/arch/powerpc/include/asm/siginfo.h b/arch/powerpc/include/asm/siginfo.h
index 49495b0534e..ccce3ef5cd8 100644
--- a/arch/powerpc/include/asm/siginfo.h
+++ b/arch/powerpc/include/asm/siginfo.h
@@ -10,7 +10,6 @@
 
 #ifdef __powerpc64__
 #    define __ARCH_SI_PREAMBLE_SIZE	(4 * sizeof(int))
-#    define SI_PAD_SIZE32		((SI_MAX_SIZE/sizeof(int)) - 3)
 #endif
 
 #include <asm-generic/siginfo.h>
diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h
index 559ae1ee670..84083876985 100644
--- a/arch/powerpc/include/asm/systbl.h
+++ b/arch/powerpc/include/asm/systbl.h
@@ -189,7 +189,7 @@ SYSCALL_SPU(getcwd)
 SYSCALL_SPU(capget)
 SYSCALL_SPU(capset)
 COMPAT_SYS(sigaltstack)
-SYSX_SPU(sys_sendfile64,compat_sys_sendfile,sys_sendfile)
+SYSX_SPU(sys_sendfile,compat_sys_sendfile_wrapper,sys_sendfile)
 SYSCALL(ni_syscall)
 SYSCALL(ni_syscall)
 PPC_SYS(vfork)
@@ -229,7 +229,7 @@ COMPAT_SYS_SPU(sched_setaffinity)
 COMPAT_SYS_SPU(sched_getaffinity)
 SYSCALL(ni_syscall)
 SYSCALL(ni_syscall)
-SYS32ONLY(sendfile64)
+SYSX(sys_ni_syscall,compat_sys_sendfile64_wrapper,sys_sendfile64)
 COMPAT_SYS_SPU(io_setup)
 SYSCALL_SPU(io_destroy)
 COMPAT_SYS_SPU(io_getevents)
diff --git a/arch/powerpc/include/asm/ucc_fast.h b/arch/powerpc/include/asm/ucc_fast.h
index 839aab8bf37..4644c840e2f 100644
--- a/arch/powerpc/include/asm/ucc_fast.h
+++ b/arch/powerpc/include/asm/ucc_fast.h
@@ -19,7 +19,7 @@
 #include <asm/immap_qe.h>
 #include <asm/qe.h>
 
-#include "ucc.h"
+#include <asm/ucc.h>
 
 /* Receive BD's status */
 #define R_E	0x80000000	/* buffer empty */
diff --git a/arch/powerpc/include/asm/ucc_slow.h b/arch/powerpc/include/asm/ucc_slow.h
index 0980e6ad335..cf131ffdb8d 100644
--- a/arch/powerpc/include/asm/ucc_slow.h
+++ b/arch/powerpc/include/asm/ucc_slow.h
@@ -20,7 +20,7 @@
 #include <asm/immap_qe.h>
 #include <asm/qe.h>
 
-#include "ucc.h"
+#include <asm/ucc.h>
 
 /* transmit BD's status */
 #define T_R	0x80000000	/* ready bit */
diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h
index bd377a36861..c683fa350ad 100644
--- a/arch/powerpc/include/asm/unistd.h
+++ b/arch/powerpc/include/asm/unistd.h
@@ -419,6 +419,7 @@
 #define __ARCH_WANT_COMPAT_SYS_TIME
 #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND
 #define __ARCH_WANT_SYS_NEWFSTATAT
+#define __ARCH_WANT_COMPAT_SYS_SENDFILE
 #endif
 
 /*
diff --git a/arch/powerpc/include/uapi/asm/Kbuild b/arch/powerpc/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/powerpc/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/powerpc/kernel/ppc32.h b/arch/powerpc/kernel/ppc32.h
index dc16aefe1dd..02fb0ee2609 100644
--- a/arch/powerpc/kernel/ppc32.h
+++ b/arch/powerpc/kernel/ppc32.h
@@ -16,57 +16,6 @@
 
 /* These are here to support 32-bit syscalls on a 64-bit kernel. */
 
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[SI_PAD_SIZE32];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			compat_uid_t _uid;		/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;			/* timer id */
-			int _overrun;			/* overrun count */
-			compat_sigval_t _sigval;		/* same as below */
-			int _sys_private;		/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			compat_uid_t _uid;		/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;		/* which child */
-			compat_uid_t _uid;		/* sender's uid */
-			int _status;			/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
-		struct {
-			unsigned int _addr; /* faulting insn/memory ref. */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
 #define __old_sigaction32	old_sigaction32
 
 struct __old_sigaction32 {
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 0794a3017b1..47834a3f493 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -705,6 +705,7 @@ static void __init early_cmdline_parse(void)
 #endif
 #define OV5_TYPE1_AFFINITY	0x80	/* Type 1 NUMA affinity */
 #define OV5_PFO_HW_RNG		0x80	/* PFO Random Number Generator */
+#define OV5_PFO_HW_842		0x40	/* PFO Compression Accelerator */
 #define OV5_PFO_HW_ENCR		0x20	/* PFO Encryption Accelerator */
 
 /* Option Vector 6: IBM PAPR hints */
@@ -774,8 +775,7 @@ static unsigned char ibm_architecture_vec[] = {
 	0,
 	0,
 	0,
-	OV5_PFO_HW_RNG | OV5_PFO_HW_ENCR,
-
+	OV5_PFO_HW_RNG | OV5_PFO_HW_ENCR | OV5_PFO_HW_842,
 	/* option vector 6: IBM PAPR hints */
 	4 - 2,				/* length */
 	0,
@@ -1624,6 +1624,63 @@ static void __init prom_instantiate_rtas(void)
 
 #ifdef CONFIG_PPC64
 /*
+ * Allocate room for and instantiate Stored Measurement Log (SML)
+ */
+static void __init prom_instantiate_sml(void)
+{
+	phandle ibmvtpm_node;
+	ihandle ibmvtpm_inst;
+	u32 entry = 0, size = 0;
+	u64 base;
+
+	prom_debug("prom_instantiate_sml: start...\n");
+
+	ibmvtpm_node = call_prom("finddevice", 1, 1, ADDR("/ibm,vtpm"));
+	prom_debug("ibmvtpm_node: %x\n", ibmvtpm_node);
+	if (!PHANDLE_VALID(ibmvtpm_node))
+		return;
+
+	ibmvtpm_inst = call_prom("open", 1, 1, ADDR("/ibm,vtpm"));
+	if (!IHANDLE_VALID(ibmvtpm_inst)) {
+		prom_printf("opening vtpm package failed (%x)\n", ibmvtpm_inst);
+		return;
+	}
+
+	if (call_prom_ret("call-method", 2, 2, &size,
+			  ADDR("sml-get-handover-size"),
+			  ibmvtpm_inst) != 0 || size == 0) {
+		prom_printf("SML get handover size failed\n");
+		return;
+	}
+
+	base = alloc_down(size, PAGE_SIZE, 0);
+	if (base == 0)
+		prom_panic("Could not allocate memory for sml\n");
+
+	prom_printf("instantiating sml at 0x%x...", base);
+
+	if (call_prom_ret("call-method", 4, 2, &entry,
+			  ADDR("sml-handover"),
+			  ibmvtpm_inst, size, base) != 0 || entry == 0) {
+		prom_printf("SML handover failed\n");
+		return;
+	}
+	prom_printf(" done\n");
+
+	reserve_mem(base, size);
+
+	prom_setprop(ibmvtpm_node, "/ibm,vtpm", "linux,sml-base",
+		     &base, sizeof(base));
+	prom_setprop(ibmvtpm_node, "/ibm,vtpm", "linux,sml-size",
+		     &size, sizeof(size));
+
+	prom_debug("sml base     = 0x%x\n", base);
+	prom_debug("sml size     = 0x%x\n", (long)size);
+
+	prom_debug("prom_instantiate_sml: end...\n");
+}
+
+/*
  * Allocate room for and initialize TCE tables
  */
 static void __init prom_initialize_tce_table(void)
@@ -2916,6 +2973,11 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4,
 		prom_instantiate_opal();
 #endif
 
+#ifdef CONFIG_PPC64
+	/* instantiate sml */
+	prom_instantiate_sml();
+#endif
+
 	/*
 	 * On non-powermacs, put all CPUs in spin-loops.
 	 *
diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c
index 81c570633ea..abd1112da54 100644
--- a/arch/powerpc/kernel/sys_ppc32.c
+++ b/arch/powerpc/kernel/sys_ppc32.c
@@ -143,48 +143,17 @@ long compat_sys_ipc(u32 call, u32 first, u32 second, u32 third, compat_uptr_t pt
  * proper conversion (sign extension) between the register representation of a signed int (msr in 32-bit mode)
  * and the register representation of a signed int (msr in 64-bit mode) is performed.
  */
-asmlinkage long compat_sys_sendfile(u32 out_fd, u32 in_fd, compat_off_t __user * offset, u32 count)
+asmlinkage long compat_sys_sendfile_wrapper(u32 out_fd, u32 in_fd,
+					    compat_off_t __user *offset, u32 count)
 {
-	mm_segment_t old_fs = get_fs();
-	int ret;
-	off_t of;
-	off_t __user *up;
-
-	if (offset && get_user(of, offset))
-		return -EFAULT;
-
-	/* The __user pointer cast is valid because of the set_fs() */		
-	set_fs(KERNEL_DS);
-	up = offset ? (off_t __user *) &of : NULL;
-	ret = sys_sendfile((int)out_fd, (int)in_fd, up, count);
-	set_fs(old_fs);
-	
-	if (offset && put_user(of, offset))
-		return -EFAULT;
-		
-	return ret;
+	return compat_sys_sendfile((int)out_fd, (int)in_fd, offset, count);
 }
 
-asmlinkage int compat_sys_sendfile64(int out_fd, int in_fd, compat_loff_t __user *offset, s32 count)
+asmlinkage long compat_sys_sendfile64_wrapper(u32 out_fd, u32 in_fd,
+					      compat_loff_t __user *offset, u32 count)
 {
-	mm_segment_t old_fs = get_fs();
-	int ret;
-	loff_t lof;
-	loff_t __user *up;
-	
-	if (offset && get_user(lof, offset))
-		return -EFAULT;
-		
-	/* The __user pointer cast is valid because of the set_fs() */		
-	set_fs(KERNEL_DS);
-	up = offset ? (loff_t __user *) &lof : NULL;
-	ret = sys_sendfile64(out_fd, in_fd, up, count);
-	set_fs(old_fs);
-	
-	if (offset && put_user(lof, offset))
-		return -EFAULT;
-		
-	return ret;
+	return sys_sendfile((int)out_fd, (int)in_fd,
+			    (off_t __user *)offset, count);
 }
 
 long compat_sys_execve(unsigned long a0, unsigned long a1, unsigned long a2,
diff --git a/arch/powerpc/kvm/44x_tlb.c b/arch/powerpc/kvm/44x_tlb.c
index 33aa715dab2..5dd3ab46997 100644
--- a/arch/powerpc/kvm/44x_tlb.c
+++ b/arch/powerpc/kvm/44x_tlb.c
@@ -319,7 +319,6 @@ void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr,
 	if (is_error_page(new_page)) {
 		printk(KERN_ERR "Couldn't get guest page for gfn %llx!\n",
 			(unsigned long long)gfn);
-		kvm_release_page_clean(new_page);
 		return;
 	}
 	hpaddr = page_to_phys(new_page);
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index d03eb6f7b05..d95d11322a1 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -705,7 +705,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 		goto out_unlock;
 	hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 
-	rmap = &memslot->rmap[gfn - memslot->base_gfn];
+	rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
 	lock_rmap(rmap);
 
 	/* Check if we might have been invalidated; let the guest retry if so */
@@ -756,9 +756,12 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	goto out_put;
 }
 
-static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
-			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
-					 unsigned long gfn))
+static int kvm_handle_hva_range(struct kvm *kvm,
+				unsigned long start,
+				unsigned long end,
+				int (*handler)(struct kvm *kvm,
+					       unsigned long *rmapp,
+					       unsigned long gfn))
 {
 	int ret;
 	int retval = 0;
@@ -767,15 +770,25 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 
 	slots = kvm_memslots(kvm);
 	kvm_for_each_memslot(memslot, slots) {
-		unsigned long start = memslot->userspace_addr;
-		unsigned long end;
+		unsigned long hva_start, hva_end;
+		gfn_t gfn, gfn_end;
 
-		end = start + (memslot->npages << PAGE_SHIFT);
-		if (hva >= start && hva < end) {
-			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
+		hva_start = max(start, memslot->userspace_addr);
+		hva_end = min(end, memslot->userspace_addr +
+					(memslot->npages << PAGE_SHIFT));
+		if (hva_start >= hva_end)
+			continue;
+		/*
+		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
+		 * {gfn, gfn+1, ..., gfn_end-1}.
+		 */
+		gfn = hva_to_gfn_memslot(hva_start, memslot);
+		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
+
+		for (; gfn < gfn_end; ++gfn) {
+			gfn_t gfn_offset = gfn - memslot->base_gfn;
 
-			ret = handler(kvm, &memslot->rmap[gfn_offset],
-				      memslot->base_gfn + gfn_offset);
+			ret = handler(kvm, &memslot->arch.rmap[gfn_offset], gfn);
 			retval |= ret;
 		}
 	}
@@ -783,6 +796,13 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 	return retval;
 }
 
+static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
+			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
+					 unsigned long gfn))
+{
+	return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
+}
+
 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 			   unsigned long gfn)
 {
@@ -850,6 +870,13 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 	return 0;
 }
 
+int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
+{
+	if (kvm->arch.using_mmu_notifiers)
+		kvm_handle_hva_range(kvm, start, end, kvm_unmap_rmapp);
+	return 0;
+}
+
 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 			 unsigned long gfn)
 {
@@ -1009,7 +1036,7 @@ long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
 	unsigned long *rmapp, *map;
 
 	preempt_disable();
-	rmapp = memslot->rmap;
+	rmapp = memslot->arch.rmap;
 	map = memslot->dirty_bitmap;
 	for (i = 0; i < memslot->npages; ++i) {
 		if (kvm_test_clear_dirty(kvm, rmapp))
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 5c70d19494f..fb0e821622d 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -84,7 +84,7 @@ static void remove_revmap_chain(struct kvm *kvm, long pte_index,
 	if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 		return;
 
-	rmap = real_vmalloc_addr(&memslot->rmap[gfn - memslot->base_gfn]);
+	rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]);
 	lock_rmap(rmap);
 
 	head = *rmap & KVMPPC_RMAP_INDEX;
@@ -180,7 +180,7 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
 	if (!slot_is_aligned(memslot, psize))
 		return H_PARAMETER;
 	slot_fn = gfn - memslot->base_gfn;
-	rmap = &memslot->rmap[slot_fn];
+	rmap = &memslot->arch.rmap[slot_fn];
 
 	if (!kvm->arch.using_mmu_notifiers) {
 		physp = kvm->arch.slot_phys[memslot->id];
@@ -197,7 +197,7 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags,
 		pa &= PAGE_MASK;
 	} else {
 		/* Translate to host virtual address */
-		hva = gfn_to_hva_memslot(memslot, gfn);
+		hva = __gfn_to_hva_memslot(memslot, gfn);
 
 		/* Look up the Linux PTE for the backing page */
 		pte_size = psize;
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
index a1baec340f7..05c28f59f77 100644
--- a/arch/powerpc/kvm/book3s_pr.c
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -242,10 +242,8 @@ static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
 	int i;
 
 	hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
-	if (is_error_page(hpage)) {
-		kvm_release_page_clean(hpage);
+	if (is_error_page(hpage))
 		return;
-	}
 
 	hpage_offset = pte->raddr & ~PAGE_MASK;
 	hpage_offset &= ~0xFFFULL;
diff --git a/arch/powerpc/kvm/e500_tlb.c b/arch/powerpc/kvm/e500_tlb.c
index a2b66717813..ff38b664195 100644
--- a/arch/powerpc/kvm/e500_tlb.c
+++ b/arch/powerpc/kvm/e500_tlb.c
@@ -520,11 +520,10 @@ static inline void kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
 
 	if (likely(!pfnmap)) {
 		unsigned long tsize_pages = 1 << (tsize + 10 - PAGE_SHIFT);
-		pfn = gfn_to_pfn_memslot(vcpu_e500->vcpu.kvm, slot, gfn);
+		pfn = gfn_to_pfn_memslot(slot, gfn);
 		if (is_error_pfn(pfn)) {
 			printk(KERN_ERR "Couldn't get real page for gfn %lx!\n",
 					(long)gfn);
-			kvm_release_pfn_clean(pfn);
 			return;
 		}
 
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 87f4dc88607..4d213b8b0fb 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -302,10 +302,18 @@ long kvm_arch_dev_ioctl(struct file *filp,
 void kvm_arch_free_memslot(struct kvm_memory_slot *free,
 			   struct kvm_memory_slot *dont)
 {
+	if (!dont || free->arch.rmap != dont->arch.rmap) {
+		vfree(free->arch.rmap);
+		free->arch.rmap = NULL;
+	}
 }
 
 int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
 {
+	slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap));
+	if (!slot->arch.rmap)
+		return -ENOMEM;
+
 	return 0;
 }
 
@@ -326,8 +334,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 	kvmppc_core_commit_memory_region(kvm, mem);
 }
 
+void kvm_arch_flush_shadow_all(struct kvm *kvm)
+{
+}
 
-void kvm_arch_flush_shadow(struct kvm *kvm)
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
 {
 }
 
diff --git a/arch/powerpc/platforms/40x/ppc40x_simple.c b/arch/powerpc/platforms/40x/ppc40x_simple.c
index 97612068fae..969dddcf332 100644
--- a/arch/powerpc/platforms/40x/ppc40x_simple.c
+++ b/arch/powerpc/platforms/40x/ppc40x_simple.c
@@ -50,7 +50,7 @@ machine_device_initcall(ppc40x_simple, ppc40x_device_probe);
  * Again, if your board needs to do things differently then create a
  * board.c file for it rather than adding it to this list.
  */
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"amcc,acadia",
 	"amcc,haleakala",
 	"amcc,kilauea",
diff --git a/arch/powerpc/platforms/512x/mpc5121_generic.c b/arch/powerpc/platforms/512x/mpc5121_generic.c
index 926731f1ff0..ca1ca666999 100644
--- a/arch/powerpc/platforms/512x/mpc5121_generic.c
+++ b/arch/powerpc/platforms/512x/mpc5121_generic.c
@@ -26,7 +26,7 @@
 /*
  * list of supported boards
  */
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"prt,prtlvt",
 	NULL
 };
diff --git a/arch/powerpc/platforms/52xx/lite5200.c b/arch/powerpc/platforms/52xx/lite5200.c
index 01ffa64d2aa..448d862bcf3 100644
--- a/arch/powerpc/platforms/52xx/lite5200.c
+++ b/arch/powerpc/platforms/52xx/lite5200.c
@@ -172,7 +172,7 @@ static void __init lite5200_setup_arch(void)
 	mpc52xx_setup_pci();
 }
 
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"fsl,lite5200",
 	"fsl,lite5200b",
 	NULL,
diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c
index 17d91b7da31..070d315dd6c 100644
--- a/arch/powerpc/platforms/52xx/media5200.c
+++ b/arch/powerpc/platforms/52xx/media5200.c
@@ -232,7 +232,7 @@ static void __init media5200_setup_arch(void)
 }
 
 /* list of the supported boards */
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"fsl,media5200",
 	NULL
 };
diff --git a/arch/powerpc/platforms/83xx/mpc837x_rdb.c b/arch/powerpc/platforms/83xx/mpc837x_rdb.c
index 16c9c9cbbb7..eca1f0960ff 100644
--- a/arch/powerpc/platforms/83xx/mpc837x_rdb.c
+++ b/arch/powerpc/platforms/83xx/mpc837x_rdb.c
@@ -60,7 +60,7 @@ static void __init mpc837x_rdb_setup_arch(void)
 
 machine_device_initcall(mpc837x_rdb, mpc83xx_declare_of_platform_devices);
 
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"fsl,mpc8377rdb",
 	"fsl,mpc8378rdb",
 	"fsl,mpc8379rdb",
diff --git a/arch/powerpc/platforms/85xx/tqm85xx.c b/arch/powerpc/platforms/85xx/tqm85xx.c
index 3e70a2035e5..b62fa87521a 100644
--- a/arch/powerpc/platforms/85xx/tqm85xx.c
+++ b/arch/powerpc/platforms/85xx/tqm85xx.c
@@ -125,7 +125,7 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_1520,
 
 machine_device_initcall(tqm85xx, mpc85xx_common_publish_devices);
 
-static const char *board[] __initdata = {
+static const char * const board[] __initconst = {
 	"tqc,tqm8540",
 	"tqc,tqm8541",
 	"tqc,tqm8548",
diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c
index 714bbfc3162..db4e638cf40 100644
--- a/arch/powerpc/platforms/cell/spu_syscalls.c
+++ b/arch/powerpc/platforms/cell/spu_syscalls.c
@@ -69,8 +69,6 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags,
 	umode_t, mode, int, neighbor_fd)
 {
 	long ret;
-	struct file *neighbor;
-	int fput_needed;
 	struct spufs_calls *calls;
 
 	calls = spufs_calls_get();
@@ -78,11 +76,11 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags,
 		return -ENOSYS;
 
 	if (flags & SPU_CREATE_AFFINITY_SPU) {
+		struct fd neighbor = fdget(neighbor_fd);
 		ret = -EBADF;
-		neighbor = fget_light(neighbor_fd, &fput_needed);
-		if (neighbor) {
-			ret = calls->create_thread(name, flags, mode, neighbor);
-			fput_light(neighbor, fput_needed);
+		if (neighbor.file) {
+			ret = calls->create_thread(name, flags, mode, neighbor.file);
+			fdput(neighbor);
 		}
 	} else
 		ret = calls->create_thread(name, flags, mode, NULL);
@@ -94,8 +92,7 @@ SYSCALL_DEFINE4(spu_create, const char __user *, name, unsigned int, flags,
 asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus)
 {
 	long ret;
-	struct file *filp;
-	int fput_needed;
+	struct fd arg;
 	struct spufs_calls *calls;
 
 	calls = spufs_calls_get();
@@ -103,10 +100,10 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus)
 		return -ENOSYS;
 
 	ret = -EBADF;
-	filp = fget_light(fd, &fput_needed);
-	if (filp) {
-		ret = calls->spu_run(filp, unpc, ustatus);
-		fput_light(filp, fput_needed);
+	arg = fdget(fd);
+	if (arg.file) {
+		ret = calls->spu_run(arg.file, unpc, ustatus);
+		fdput(arg);
 	}
 
 	spufs_calls_put(calls);
diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c
index c2c5b078ba8..657e3f233a6 100644
--- a/arch/powerpc/platforms/cell/spufs/coredump.c
+++ b/arch/powerpc/platforms/cell/spufs/coredump.c
@@ -106,6 +106,17 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd)
 	return total;
 }
 
+static int match_context(const void *v, struct file *file, unsigned fd)
+{
+	struct spu_context *ctx;
+	if (file->f_op != &spufs_context_fops)
+		return 0;
+	ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx;
+	if (ctx->flags & SPU_CREATE_NOSCHED)
+		return 0;
+	return fd + 1;
+}
+
 /*
  * The additional architecture-specific notes for Cell are various
  * context files in the spu context.
@@ -115,29 +126,18 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd)
  * internal functionality to dump them without needing to actually
  * open the files.
  */
+/*
+ * descriptor table is not shared, so files can't change or go away.
+ */
 static struct spu_context *coredump_next_context(int *fd)
 {
-	struct fdtable *fdt = files_fdtable(current->files);
 	struct file *file;
-	struct spu_context *ctx = NULL;
-
-	for (; *fd < fdt->max_fds; (*fd)++) {
-		if (!fd_is_open(*fd, fdt))
-			continue;
-
-		file = fcheck(*fd);
-
-		if (!file || file->f_op != &spufs_context_fops)
-			continue;
-
-		ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx;
-		if (ctx->flags & SPU_CREATE_NOSCHED)
-			continue;
-
-		break;
-	}
-
-	return ctx;
+	int n = iterate_fd(current->files, *fd, match_context, NULL);
+	if (!n)
+		return NULL;
+	*fd = n - 1;
+	file = fcheck(*fd);
+	return SPUFS_I(file->f_dentry->d_inode)->i_ctx;
 }
 
 int spufs_coredump_extra_notes_size(void)
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index e402a9dd4ed..da3c1a7dcd8 100644
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -216,7 +216,6 @@ static struct crypto_alg aes_alg = {
 	.cra_blocksize		=	AES_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
 	.cra_init               =       fallback_init_cip,
 	.cra_exit               =       fallback_exit_cip,
 	.cra_u			=	{
@@ -398,7 +397,6 @@ static struct crypto_alg ecb_aes_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
 	.cra_init		=	fallback_init_blk,
 	.cra_exit		=	fallback_exit_blk,
 	.cra_u			=	{
@@ -508,7 +506,6 @@ static struct crypto_alg cbc_aes_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
 	.cra_init		=	fallback_init_blk,
 	.cra_exit		=	fallback_exit_blk,
 	.cra_u			=	{
@@ -710,7 +707,6 @@ static struct crypto_alg xts_aes_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_xts_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(xts_aes_alg.cra_list),
 	.cra_init		=	xts_fallback_init,
 	.cra_exit		=	xts_fallback_exit,
 	.cra_u			=	{
@@ -832,7 +828,6 @@ static struct crypto_alg ctr_aes_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ctr_aes_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	AES_MIN_KEY_SIZE,
diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c
index 1eaa371ca3e..b49fb96f420 100644
--- a/arch/s390/crypto/des_s390.c
+++ b/arch/s390/crypto/des_s390.c
@@ -70,7 +70,6 @@ static struct crypto_alg des_alg = {
 	.cra_blocksize		=	DES_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(des_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	DES_KEY_SIZE,
@@ -163,7 +162,6 @@ static struct crypto_alg ecb_des_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ecb_des_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES_KEY_SIZE,
@@ -206,7 +204,6 @@ static struct crypto_alg cbc_des_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(cbc_des_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES_KEY_SIZE,
@@ -271,7 +268,6 @@ static struct crypto_alg des3_alg = {
 	.cra_blocksize		=	DES_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(des3_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	DES3_KEY_SIZE,
@@ -314,8 +310,6 @@ static struct crypto_alg ecb_des3_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(
-						ecb_des3_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES3_KEY_SIZE,
@@ -358,8 +352,6 @@ static struct crypto_alg cbc_des3_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(
-						cbc_des3_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES3_KEY_SIZE,
@@ -452,7 +444,6 @@ static struct crypto_alg ctr_des_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ctr_des_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES_KEY_SIZE,
@@ -496,7 +487,6 @@ static struct crypto_alg ctr_des3_alg = {
 	.cra_ctxsize		=	sizeof(struct s390_des_ctx),
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ctr_des3_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	DES3_KEY_SIZE,
diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c
index b1bd170f24b..1ebd3a15cca 100644
--- a/arch/s390/crypto/ghash_s390.c
+++ b/arch/s390/crypto/ghash_s390.c
@@ -135,7 +135,6 @@ static struct shash_alg ghash_alg = {
 		.cra_blocksize		= GHASH_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct ghash_ctx),
 		.cra_module		= THIS_MODULE,
-		.cra_list		= LIST_HEAD_INIT(ghash_alg.base.cra_list),
 	},
 };
 
diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c
index 124ec1a55cc..06ea69bd387 100644
--- a/arch/s390/hypfs/inode.c
+++ b/arch/s390/hypfs/inode.c
@@ -72,8 +72,6 @@ static void hypfs_remove(struct dentry *dentry)
 	struct dentry *parent;
 
 	parent = dentry->d_parent;
-	if (!parent || !parent->d_inode)
-		return;
 	mutex_lock(&parent->d_inode->i_mutex);
 	if (hypfs_positive(dentry)) {
 		if (S_ISDIR(dentry->d_inode->i_mode))
diff --git a/arch/s390/include/asm/compat.h b/arch/s390/include/asm/compat.h
index 234f1d859ce..a34a9d612fc 100644
--- a/arch/s390/include/asm/compat.h
+++ b/arch/s390/include/asm/compat.h
@@ -65,6 +65,7 @@ typedef s64		compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -144,6 +145,79 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+typedef struct compat_siginfo {
+	int	si_signo;
+	int	si_errno;
+	int	si_code;
+
+	union {
+		int _pad[128/sizeof(int) - 3];
+
+		/* kill() */
+		struct {
+			pid_t	_pid;	/* sender's pid */
+			uid_t	_uid;	/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;		/* timer id */
+			int _overrun;			/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;	/* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			pid_t			_pid;	/* sender's pid */
+			uid_t			_uid;	/* sender's uid */
+			compat_sigval_t		_sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			pid_t			_pid;	/* which child */
+			uid_t			_uid;	/* sender's uid */
+			int			_status;/* exit code */
+			compat_clock_t		_utime;
+			compat_clock_t		_stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			__u32	_addr;	/* faulting insn/memory ref. - pointer */
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int	_band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int	_fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
+/*
+ * How these fields are to be accessed.
+ */
+#define si_pid		_sifields._kill._pid
+#define si_uid		_sifields._kill._uid
+#define si_status	_sifields._sigchld._status
+#define si_utime	_sifields._sigchld._utime
+#define si_stime	_sifields._sigchld._stime
+#define si_value	_sifields._rt._sigval
+#define si_int		_sifields._rt._sigval.sival_int
+#define si_ptr		_sifields._rt._sigval.sival_ptr
+#define si_addr		_sifields._sigfault._addr
+#define si_band		_sifields._sigpoll._band
+#define si_fd		_sifields._sigpoll._fd
+#define si_tid		_sifields._timer._tid
+#define si_overrun	_sifields._timer._overrun
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -153,7 +227,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index f3e0aabfc6b..56831dfa919 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -159,6 +159,7 @@ extern unsigned long thread_saved_pc(struct task_struct *t);
 
 extern void show_code(struct pt_regs *regs);
 extern void print_fn_code(unsigned char *code, unsigned long len);
+extern int insn_to_mnemonic(unsigned char *instruction, char buf[8]);
 
 unsigned long get_wchan(struct task_struct *p);
 #define task_pt_regs(tsk) ((struct pt_regs *) \
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/s390/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/s390/kernel/compat_linux.h b/arch/s390/kernel/compat_linux.h
index 9635d759c2b..90887bd98cf 100644
--- a/arch/s390/kernel/compat_linux.h
+++ b/arch/s390/kernel/compat_linux.h
@@ -23,74 +23,6 @@ struct old_sigaction32 {
        __u32			sa_flags;
        __u32			sa_restorer;	/* Another 32 bit pointer */
 };
- 
-typedef struct compat_siginfo {
-	int	si_signo;
-	int	si_errno;
-	int	si_code;
-
-	union {
-		int _pad[((128/sizeof(int)) - 3)];
-
-		/* kill() */
-		struct {
-			pid_t	_pid;	/* sender's pid */
-			uid_t	_uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;		/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;       /* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			pid_t			_pid;	/* sender's pid */
-			uid_t			_uid;	/* sender's uid */
-			compat_sigval_t		_sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			pid_t			_pid;	/* which child */
-			uid_t			_uid;	/* sender's uid */
-			int			_status;/* exit code */
-			compat_clock_t		_utime;
-			compat_clock_t		_stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			__u32	_addr;	/* faulting insn/memory ref. - pointer */
-		} _sigfault;
-                          
-		/* SIGPOLL */
-		struct {
-			int	_band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int	_fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
-/*
- * How these fields are to be accessed.
- */
-#define si_pid		_sifields._kill._pid
-#define si_uid		_sifields._kill._uid
-#define si_status	_sifields._sigchld._status
-#define si_utime	_sifields._sigchld._utime
-#define si_stime	_sifields._sigchld._stime
-#define si_value	_sifields._rt._sigval
-#define si_int		_sifields._rt._sigval.sival_int
-#define si_ptr		_sifields._rt._sigval.sival_ptr
-#define si_addr		_sifields._sigfault._addr
-#define si_band		_sifields._sigpoll._band
-#define si_fd		_sifields._sigpoll._fd    
-#define si_tid		_sifields._timer._tid
-#define si_overrun	_sifields._timer._overrun
 
 /* asm/sigcontext.h */
 typedef union
diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c
index cc84a24c023..f00286bd2ef 100644
--- a/arch/s390/kernel/dis.c
+++ b/arch/s390/kernel/dis.c
@@ -1501,6 +1501,33 @@ static struct insn *find_insn(unsigned char *code)
 	return NULL;
 }
 
+/**
+ * insn_to_mnemonic - decode an s390 instruction
+ * @instruction: instruction to decode
+ * @buf: buffer to fill with mnemonic
+ *
+ * Decode the instruction at @instruction and store the corresponding
+ * mnemonic into @buf.
+ * @buf is left unchanged if the instruction could not be decoded.
+ * Returns:
+ *  %0 on success, %-ENOENT if the instruction was not found.
+ */
+int insn_to_mnemonic(unsigned char *instruction, char buf[8])
+{
+	struct insn *insn;
+
+	insn = find_insn(instruction);
+	if (!insn)
+		return -ENOENT;
+	if (insn->name[0] == '\0')
+		snprintf(buf, sizeof(buf), "%s",
+			 long_insn_name[(int) insn->name[1]]);
+	else
+		snprintf(buf, sizeof(buf), "%.5s", insn->name);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(insn_to_mnemonic);
+
 static int print_insn(char *buffer, unsigned char *code, unsigned long addr)
 {
 	struct insn *insn;
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 9b04a32e569..b58dd869cb3 100644
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -21,6 +21,7 @@ config KVM
 	depends on HAVE_KVM && EXPERIMENTAL
 	select PREEMPT_NOTIFIERS
 	select ANON_INODES
+	select HAVE_KVM_CPU_RELAX_INTERCEPT
 	---help---
 	  Support hosting paravirtualized guest machines using the SIE
 	  virtualization capability on the mainframe. This should work
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index c88bb779339..a390687feb1 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -14,6 +14,8 @@
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 #include "kvm-s390.h"
+#include "trace.h"
+#include "trace-s390.h"
 
 static int diag_release_pages(struct kvm_vcpu *vcpu)
 {
@@ -98,6 +100,7 @@ static int __diag_ipl_functions(struct kvm_vcpu *vcpu)
 	vcpu->run->exit_reason = KVM_EXIT_S390_RESET;
 	VCPU_EVENT(vcpu, 3, "requesting userspace resets %llx",
 	  vcpu->run->s390_reset_flags);
+	trace_kvm_s390_request_resets(vcpu->run->s390_reset_flags);
 	return -EREMOTE;
 }
 
@@ -105,6 +108,7 @@ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
 {
 	int code = (vcpu->arch.sie_block->ipb & 0xfff0000) >> 16;
 
+	trace_kvm_s390_handle_diag(vcpu, code);
 	switch (code) {
 	case 0x10:
 		return diag_release_pages(vcpu);
diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c
index adae539f12e..22798ec33fd 100644
--- a/arch/s390/kvm/intercept.c
+++ b/arch/s390/kvm/intercept.c
@@ -19,6 +19,8 @@
 
 #include "kvm-s390.h"
 #include "gaccess.h"
+#include "trace.h"
+#include "trace-s390.h"
 
 static int handle_lctlg(struct kvm_vcpu *vcpu)
 {
@@ -45,6 +47,7 @@ static int handle_lctlg(struct kvm_vcpu *vcpu)
 
 	VCPU_EVENT(vcpu, 5, "lctlg r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2,
 		   disp2);
+	trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, useraddr);
 
 	do {
 		rc = get_guest_u64(vcpu, useraddr,
@@ -82,6 +85,7 @@ static int handle_lctl(struct kvm_vcpu *vcpu)
 
 	VCPU_EVENT(vcpu, 5, "lctl r1:%x, r3:%x,b2:%x,d2:%x", reg1, reg3, base2,
 		   disp2);
+	trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, useraddr);
 
 	reg = reg1;
 	do {
@@ -135,6 +139,8 @@ static int handle_stop(struct kvm_vcpu *vcpu)
 	vcpu->stat.exit_stop_request++;
 	spin_lock_bh(&vcpu->arch.local_int.lock);
 
+	trace_kvm_s390_stop_request(vcpu->arch.local_int.action_bits);
+
 	if (vcpu->arch.local_int.action_bits & ACTION_RELOADVCPU_ON_STOP) {
 		vcpu->arch.local_int.action_bits &= ~ACTION_RELOADVCPU_ON_STOP;
 		rc = SIE_INTERCEPT_RERUNVCPU;
@@ -171,6 +177,7 @@ static int handle_validity(struct kvm_vcpu *vcpu)
 	int rc;
 
 	vcpu->stat.exit_validity++;
+	trace_kvm_s390_intercept_validity(vcpu, viwhy);
 	if (viwhy == 0x37) {
 		vmaddr = gmap_fault(vcpu->arch.sie_block->prefix,
 				    vcpu->arch.gmap);
@@ -213,6 +220,9 @@ static int handle_instruction(struct kvm_vcpu *vcpu)
 	intercept_handler_t handler;
 
 	vcpu->stat.exit_instruction++;
+	trace_kvm_s390_intercept_instruction(vcpu,
+					     vcpu->arch.sie_block->ipa,
+					     vcpu->arch.sie_block->ipb);
 	handler = instruction_handlers[vcpu->arch.sie_block->ipa >> 8];
 	if (handler)
 		return handler(vcpu);
@@ -222,6 +232,7 @@ static int handle_instruction(struct kvm_vcpu *vcpu)
 static int handle_prog(struct kvm_vcpu *vcpu)
 {
 	vcpu->stat.exit_program_interruption++;
+	trace_kvm_s390_intercept_prog(vcpu, vcpu->arch.sie_block->iprcc);
 	return kvm_s390_inject_program_int(vcpu, vcpu->arch.sie_block->iprcc);
 }
 
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index b7bc1aac8ed..ff1e2f8ef94 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -19,6 +19,7 @@
 #include <asm/uaccess.h>
 #include "kvm-s390.h"
 #include "gaccess.h"
+#include "trace-s390.h"
 
 static int psw_extint_disabled(struct kvm_vcpu *vcpu)
 {
@@ -130,6 +131,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 	case KVM_S390_INT_EMERGENCY:
 		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp emerg");
 		vcpu->stat.deliver_emergency_signal++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->emerg.code, 0);
 		rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1201);
 		if (rc == -EFAULT)
 			exception = 1;
@@ -152,6 +155,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 	case KVM_S390_INT_EXTERNAL_CALL:
 		VCPU_EVENT(vcpu, 4, "%s", "interrupt: sigp ext call");
 		vcpu->stat.deliver_external_call++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->extcall.code, 0);
 		rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x1202);
 		if (rc == -EFAULT)
 			exception = 1;
@@ -175,6 +180,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 		VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
 			   inti->ext.ext_params);
 		vcpu->stat.deliver_service_signal++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->ext.ext_params, 0);
 		rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2401);
 		if (rc == -EFAULT)
 			exception = 1;
@@ -198,6 +205,9 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 		VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
 			   inti->ext.ext_params, inti->ext.ext_params2);
 		vcpu->stat.deliver_virtio_interrupt++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->ext.ext_params,
+						 inti->ext.ext_params2);
 		rc = put_guest_u16(vcpu, __LC_EXT_INT_CODE, 0x2603);
 		if (rc == -EFAULT)
 			exception = 1;
@@ -229,6 +239,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 	case KVM_S390_SIGP_STOP:
 		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu stop");
 		vcpu->stat.deliver_stop_signal++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 0, 0);
 		__set_intercept_indicator(vcpu, inti);
 		break;
 
@@ -236,12 +248,16 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 		VCPU_EVENT(vcpu, 4, "interrupt: set prefix to %x",
 			   inti->prefix.address);
 		vcpu->stat.deliver_prefix_signal++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->prefix.address, 0);
 		kvm_s390_set_prefix(vcpu, inti->prefix.address);
 		break;
 
 	case KVM_S390_RESTART:
 		VCPU_EVENT(vcpu, 4, "%s", "interrupt: cpu restart");
 		vcpu->stat.deliver_restart_signal++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 0, 0);
 		rc = copy_to_guest(vcpu, offsetof(struct _lowcore,
 		  restart_old_psw), &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
 		if (rc == -EFAULT)
@@ -259,6 +275,8 @@ static void __do_deliver_interrupt(struct kvm_vcpu *vcpu,
 			   inti->pgm.code,
 			   table[vcpu->arch.sie_block->ipa >> 14]);
 		vcpu->stat.deliver_program_int++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
+						 inti->pgm.code, 0);
 		rc = put_guest_u16(vcpu, __LC_PGM_INT_CODE, inti->pgm.code);
 		if (rc == -EFAULT)
 			exception = 1;
@@ -405,9 +423,7 @@ no_timer:
 		set_current_state(TASK_INTERRUPTIBLE);
 		spin_unlock_bh(&vcpu->arch.local_int.lock);
 		spin_unlock(&vcpu->arch.local_int.float_int->lock);
-		vcpu_put(vcpu);
 		schedule();
-		vcpu_load(vcpu);
 		spin_lock(&vcpu->arch.local_int.float_int->lock);
 		spin_lock_bh(&vcpu->arch.local_int.lock);
 	}
@@ -515,6 +531,7 @@ int kvm_s390_inject_program_int(struct kvm_vcpu *vcpu, u16 code)
 	inti->pgm.code = code;
 
 	VCPU_EVENT(vcpu, 3, "inject: program check %d (from kernel)", code);
+	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, inti->type, code, 0, 1);
 	spin_lock_bh(&li->lock);
 	list_add(&inti->list, &li->list);
 	atomic_set(&li->active, 1);
@@ -556,6 +573,8 @@ int kvm_s390_inject_vm(struct kvm *kvm,
 		kfree(inti);
 		return -EINVAL;
 	}
+	trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
+				 2);
 
 	mutex_lock(&kvm->lock);
 	fi = &kvm->arch.float_int;
@@ -621,6 +640,8 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
 		kfree(inti);
 		return -EINVAL;
 	}
+	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, s390int->type, s390int->parm,
+				   s390int->parm64, 2);
 
 	mutex_lock(&vcpu->kvm->lock);
 	li = &vcpu->arch.local_int;
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index d470ccbfaba..ecced9d1898 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -32,6 +32,10 @@
 #include "kvm-s390.h"
 #include "gaccess.h"
 
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+#include "trace-s390.h"
+
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 
 struct kvm_stats_debugfs_item debugfs_entries[] = {
@@ -242,6 +246,7 @@ out_err:
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
+	trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 	if (!kvm_is_ucontrol(vcpu->kvm)) {
 		clear_bit(63 - vcpu->vcpu_id,
 			  (unsigned long *) &vcpu->kvm->arch.sca->mcn);
@@ -417,6 +422,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 		goto out_free_sie_block;
 	VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 		 vcpu->arch.sie_block);
+	trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 
 	return vcpu;
 out_free_sie_block:
@@ -607,18 +613,22 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 	local_irq_enable();
 	VCPU_EVENT(vcpu, 6, "entering sie flags %x",
 		   atomic_read(&vcpu->arch.sie_block->cpuflags));
+	trace_kvm_s390_sie_enter(vcpu,
+				 atomic_read(&vcpu->arch.sie_block->cpuflags));
 	rc = sie64a(vcpu->arch.sie_block, vcpu->run->s.regs.gprs);
 	if (rc) {
 		if (kvm_is_ucontrol(vcpu->kvm)) {
 			rc = SIE_INTERCEPT_UCONTROL;
 		} else {
 			VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
+			trace_kvm_s390_sie_fault(vcpu);
 			kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 			rc = 0;
 		}
 	}
 	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
 		   vcpu->arch.sie_block->icptcode);
+	trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
 	local_irq_disable();
 	kvm_guest_exit();
 	local_irq_enable();
@@ -959,7 +969,12 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 	return;
 }
 
-void kvm_arch_flush_shadow(struct kvm *kvm)
+void kvm_arch_flush_shadow_all(struct kvm *kvm)
+{
+}
+
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
 {
 }
 
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 310be61bead..d768906f15c 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -20,6 +20,7 @@
 #include <asm/sysinfo.h>
 #include "gaccess.h"
 #include "kvm-s390.h"
+#include "trace.h"
 
 static int handle_set_prefix(struct kvm_vcpu *vcpu)
 {
@@ -59,6 +60,7 @@ static int handle_set_prefix(struct kvm_vcpu *vcpu)
 	kvm_s390_set_prefix(vcpu, address);
 
 	VCPU_EVENT(vcpu, 5, "setting prefix to %x", address);
+	trace_kvm_s390_handle_prefix(vcpu, 1, address);
 out:
 	return 0;
 }
@@ -91,6 +93,7 @@ static int handle_store_prefix(struct kvm_vcpu *vcpu)
 	}
 
 	VCPU_EVENT(vcpu, 5, "storing prefix to %x", address);
+	trace_kvm_s390_handle_prefix(vcpu, 0, address);
 out:
 	return 0;
 }
@@ -119,6 +122,7 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu)
 	}
 
 	VCPU_EVENT(vcpu, 5, "storing cpu address to %llx", useraddr);
+	trace_kvm_s390_handle_stap(vcpu, useraddr);
 out:
 	return 0;
 }
@@ -164,9 +168,11 @@ static int handle_stfl(struct kvm_vcpu *vcpu)
 			   &facility_list, sizeof(facility_list));
 	if (rc == -EFAULT)
 		kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
-	else
+	else {
 		VCPU_EVENT(vcpu, 5, "store facility list value %x",
 			   facility_list);
+		trace_kvm_s390_handle_stfl(vcpu, facility_list);
+	}
 	return 0;
 }
 
@@ -278,6 +284,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 		kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 		goto out_mem;
 	}
+	trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2);
 	free_page(mem);
 	vcpu->arch.sie_block->gpsw.mask &= ~(3ul << 44);
 	vcpu->run->s.regs.gprs[0] = 0;
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index 56f80e1f98f..566ddf6e8df 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -18,6 +18,7 @@
 #include <asm/sigp.h>
 #include "gaccess.h"
 #include "kvm-s390.h"
+#include "trace.h"
 
 static int __sigp_sense(struct kvm_vcpu *vcpu, u16 cpu_addr,
 			u64 *reg)
@@ -344,6 +345,7 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu)
 	else
 		parameter = vcpu->run->s.regs.gprs[r1 + 1];
 
+	trace_kvm_s390_handle_sigp(vcpu, order_code, cpu_addr, parameter);
 	switch (order_code) {
 	case SIGP_SENSE:
 		vcpu->stat.instruction_sigp_sense++;
diff --git a/arch/s390/kvm/trace-s390.h b/arch/s390/kvm/trace-s390.h
new file mode 100644
index 00000000000..90fdf85b5ff
--- /dev/null
+++ b/arch/s390/kvm/trace-s390.h
@@ -0,0 +1,210 @@
+#if !defined(_TRACE_KVMS390_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_KVMS390_H
+
+#include <linux/tracepoint.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm-s390
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace-s390
+
+/*
+ * Trace point for the creation of the kvm instance.
+ */
+TRACE_EVENT(kvm_s390_create_vm,
+	    TP_PROTO(unsigned long type),
+	    TP_ARGS(type),
+
+	    TP_STRUCT__entry(
+		    __field(unsigned long, type)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->type = type;
+		    ),
+
+	    TP_printk("create vm%s",
+		      __entry->type & KVM_VM_S390_UCONTROL ? " (UCONTROL)" : "")
+	);
+
+/*
+ * Trace points for creation and destruction of vpcus.
+ */
+TRACE_EVENT(kvm_s390_create_vcpu,
+	    TP_PROTO(unsigned int id, struct kvm_vcpu *vcpu,
+		     struct kvm_s390_sie_block *sie_block),
+	    TP_ARGS(id, vcpu, sie_block),
+
+	    TP_STRUCT__entry(
+		    __field(unsigned int, id)
+		    __field(struct kvm_vcpu *, vcpu)
+		    __field(struct kvm_s390_sie_block *, sie_block)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->id = id;
+		    __entry->vcpu = vcpu;
+		    __entry->sie_block = sie_block;
+		    ),
+
+	    TP_printk("create cpu %d at %p, sie block at %p", __entry->id,
+		      __entry->vcpu, __entry->sie_block)
+	);
+
+TRACE_EVENT(kvm_s390_destroy_vcpu,
+	    TP_PROTO(unsigned int id),
+	    TP_ARGS(id),
+
+	    TP_STRUCT__entry(
+		    __field(unsigned int, id)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->id = id;
+		    ),
+
+	    TP_printk("destroy cpu %d", __entry->id)
+	);
+
+/*
+ * Trace points for injection of interrupts, either per machine or
+ * per vcpu.
+ */
+
+#define kvm_s390_int_type						\
+	{KVM_S390_SIGP_STOP, "sigp stop"},				\
+	{KVM_S390_PROGRAM_INT, "program interrupt"},			\
+	{KVM_S390_SIGP_SET_PREFIX, "sigp set prefix"},			\
+	{KVM_S390_RESTART, "sigp restart"},				\
+	{KVM_S390_INT_VIRTIO, "virtio interrupt"},			\
+	{KVM_S390_INT_SERVICE, "sclp interrupt"},			\
+	{KVM_S390_INT_EMERGENCY, "sigp emergency"},			\
+	{KVM_S390_INT_EXTERNAL_CALL, "sigp ext call"}
+
+TRACE_EVENT(kvm_s390_inject_vm,
+	    TP_PROTO(__u64 type, __u32 parm, __u64 parm64, int who),
+	    TP_ARGS(type, parm, parm64, who),
+
+	    TP_STRUCT__entry(
+		    __field(__u32, inttype)
+		    __field(__u32, parm)
+		    __field(__u64, parm64)
+		    __field(int, who)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->inttype = type & 0x00000000ffffffff;
+		    __entry->parm = parm;
+		    __entry->parm64 = parm64;
+		    __entry->who = who;
+		    ),
+
+	    TP_printk("inject%s: type:%x (%s) parm:%x parm64:%llx",
+		      (__entry->who == 1) ? " (from kernel)" :
+		      (__entry->who == 2) ? " (from user)" : "",
+		      __entry->inttype,
+		      __print_symbolic(__entry->inttype, kvm_s390_int_type),
+		      __entry->parm, __entry->parm64)
+	);
+
+TRACE_EVENT(kvm_s390_inject_vcpu,
+	    TP_PROTO(unsigned int id, __u64 type, __u32 parm, __u64 parm64, \
+		     int who),
+	    TP_ARGS(id, type, parm, parm64, who),
+
+	    TP_STRUCT__entry(
+		    __field(int, id)
+		    __field(__u32, inttype)
+		    __field(__u32, parm)
+		    __field(__u64, parm64)
+		    __field(int, who)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->id = id;
+		    __entry->inttype = type & 0x00000000ffffffff;
+		    __entry->parm = parm;
+		    __entry->parm64 = parm64;
+		    __entry->who = who;
+		    ),
+
+	    TP_printk("inject%s (vcpu %d): type:%x (%s) parm:%x parm64:%llx",
+		      (__entry->who == 1) ? " (from kernel)" :
+		      (__entry->who == 2) ? " (from user)" : "",
+		      __entry->id, __entry->inttype,
+		      __print_symbolic(__entry->inttype, kvm_s390_int_type),
+		      __entry->parm, __entry->parm64)
+	);
+
+/*
+ * Trace point for the actual delivery of interrupts.
+ */
+TRACE_EVENT(kvm_s390_deliver_interrupt,
+	    TP_PROTO(unsigned int id, __u64 type, __u32 data0, __u64 data1),
+	    TP_ARGS(id, type, data0, data1),
+
+	    TP_STRUCT__entry(
+		    __field(int, id)
+		    __field(__u32, inttype)
+		    __field(__u32, data0)
+		    __field(__u64, data1)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->id = id;
+		    __entry->inttype = type & 0x00000000ffffffff;
+		    __entry->data0 = data0;
+		    __entry->data1 = data1;
+		    ),
+
+	    TP_printk("deliver interrupt (vcpu %d): type:%x (%s) "	\
+		      "data:%08x %016llx",
+		      __entry->id, __entry->inttype,
+		      __print_symbolic(__entry->inttype, kvm_s390_int_type),
+		      __entry->data0, __entry->data1)
+	);
+
+/*
+ * Trace point for resets that may be requested from userspace.
+ */
+TRACE_EVENT(kvm_s390_request_resets,
+	    TP_PROTO(__u64 resets),
+	    TP_ARGS(resets),
+
+	    TP_STRUCT__entry(
+		    __field(__u64, resets)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->resets = resets;
+		    ),
+
+	    TP_printk("requesting userspace resets %llx",
+		      __entry->resets)
+	);
+
+/*
+ * Trace point for a vcpu's stop requests.
+ */
+TRACE_EVENT(kvm_s390_stop_request,
+	    TP_PROTO(unsigned int action_bits),
+	    TP_ARGS(action_bits),
+
+	    TP_STRUCT__entry(
+		    __field(unsigned int, action_bits)
+		    ),
+
+	    TP_fast_assign(
+		    __entry->action_bits = action_bits;
+		    ),
+
+	    TP_printk("stop request, action_bits = %08x",
+		      __entry->action_bits)
+	);
+
+
+#endif /* _TRACE_KVMS390_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/arch/s390/kvm/trace.h b/arch/s390/kvm/trace.h
new file mode 100644
index 00000000000..2b29e62351d
--- /dev/null
+++ b/arch/s390/kvm/trace.h
@@ -0,0 +1,341 @@
+#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_KVM_H
+
+#include <linux/tracepoint.h>
+#include <asm/sigp.h>
+#include <asm/debug.h>
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM kvm
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+/*
+ * Helpers for vcpu-specific tracepoints containing the same information
+ * as s390dbf VCPU_EVENTs.
+ */
+#define VCPU_PROTO_COMMON struct kvm_vcpu *vcpu
+#define VCPU_ARGS_COMMON vcpu
+#define VCPU_FIELD_COMMON __field(int, id)			\
+	__field(unsigned long, pswmask)				\
+	__field(unsigned long, pswaddr)
+#define VCPU_ASSIGN_COMMON do {						\
+	__entry->id = vcpu->vcpu_id;					\
+	__entry->pswmask = vcpu->arch.sie_block->gpsw.mask;		\
+	__entry->pswaddr = vcpu->arch.sie_block->gpsw.addr;		\
+	} while (0);
+#define VCPU_TP_PRINTK(p_str, p_args...)				\
+	TP_printk("%02d[%016lx-%016lx]: " p_str, __entry->id,		\
+		  __entry->pswmask, __entry->pswaddr, p_args)
+
+/*
+ * Tracepoints for SIE entry and exit.
+ */
+TRACE_EVENT(kvm_s390_sie_enter,
+	    TP_PROTO(VCPU_PROTO_COMMON, int cpuflags),
+	    TP_ARGS(VCPU_ARGS_COMMON, cpuflags),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(int, cpuflags)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->cpuflags = cpuflags;
+		    ),
+
+	    VCPU_TP_PRINTK("entering sie flags %x", __entry->cpuflags)
+	);
+
+TRACE_EVENT(kvm_s390_sie_fault,
+	    TP_PROTO(VCPU_PROTO_COMMON),
+	    TP_ARGS(VCPU_ARGS_COMMON),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    ),
+
+	    VCPU_TP_PRINTK("%s", "fault in sie instruction")
+	);
+
+#define sie_intercept_code				\
+	{0x04, "Instruction"},				\
+	{0x08, "Program interruption"},			\
+	{0x0C, "Instruction and program interuption"},	\
+	{0x10, "External request"},			\
+	{0x14, "External interruption"},		\
+	{0x18, "I/O request"},				\
+	{0x1C, "Wait state"},				\
+	{0x20, "Validity"},				\
+	{0x28, "Stop request"}
+
+TRACE_EVENT(kvm_s390_sie_exit,
+	    TP_PROTO(VCPU_PROTO_COMMON, u8 icptcode),
+	    TP_ARGS(VCPU_ARGS_COMMON, icptcode),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(u8, icptcode)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->icptcode = icptcode;
+		    ),
+
+	    VCPU_TP_PRINTK("exit sie icptcode %d (%s)", __entry->icptcode,
+			   __print_symbolic(__entry->icptcode,
+					    sie_intercept_code))
+	);
+
+/*
+ * Trace point for intercepted instructions.
+ */
+TRACE_EVENT(kvm_s390_intercept_instruction,
+	    TP_PROTO(VCPU_PROTO_COMMON, __u16 ipa, __u32 ipb),
+	    TP_ARGS(VCPU_ARGS_COMMON, ipa, ipb),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(__u64, instruction)
+		    __field(char, insn[8])
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->instruction = ((__u64)ipa << 48) |
+		    ((__u64)ipb << 16);
+		    ),
+
+	    VCPU_TP_PRINTK("intercepted instruction %016llx (%s)",
+			   __entry->instruction,
+			   insn_to_mnemonic((unsigned char *)
+					    &__entry->instruction,
+					 __entry->insn) ?
+			   "unknown" : __entry->insn)
+	);
+
+/*
+ * Trace point for intercepted program interruptions.
+ */
+TRACE_EVENT(kvm_s390_intercept_prog,
+	    TP_PROTO(VCPU_PROTO_COMMON, __u16 code),
+	    TP_ARGS(VCPU_ARGS_COMMON, code),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(__u16, code)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->code = code;
+		    ),
+
+	    VCPU_TP_PRINTK("intercepted program interruption %04x",
+			   __entry->code)
+	);
+
+/*
+ * Trace point for validity intercepts.
+ */
+TRACE_EVENT(kvm_s390_intercept_validity,
+	    TP_PROTO(VCPU_PROTO_COMMON, __u16 viwhy),
+	    TP_ARGS(VCPU_ARGS_COMMON, viwhy),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(__u16, viwhy)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->viwhy = viwhy;
+		    ),
+
+	    VCPU_TP_PRINTK("got validity intercept %04x", __entry->viwhy)
+	);
+
+/*
+ * Trace points for instructions that are of special interest.
+ */
+
+#define sigp_order_codes					\
+	{SIGP_SENSE, "sense"},					\
+	{SIGP_EXTERNAL_CALL, "external call"},			\
+	{SIGP_EMERGENCY_SIGNAL, "emergency signal"},		\
+	{SIGP_STOP, "stop"},					\
+	{SIGP_STOP_AND_STORE_STATUS, "stop and store status"},	\
+	{SIGP_SET_ARCHITECTURE, "set architecture"},		\
+	{SIGP_SET_PREFIX, "set prefix"},			\
+	{SIGP_SENSE_RUNNING, "sense running"},			\
+	{SIGP_RESTART, "restart"}
+
+TRACE_EVENT(kvm_s390_handle_sigp,
+	    TP_PROTO(VCPU_PROTO_COMMON, __u8 order_code, __u16 cpu_addr, \
+		     __u32 parameter),
+	    TP_ARGS(VCPU_ARGS_COMMON, order_code, cpu_addr, parameter),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(__u8, order_code)
+		    __field(__u16, cpu_addr)
+		    __field(__u32, parameter)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->order_code = order_code;
+		    __entry->cpu_addr = cpu_addr;
+		    __entry->parameter = parameter;
+		    ),
+
+	    VCPU_TP_PRINTK("handle sigp order %02x (%s), cpu address %04x, " \
+			   "parameter %08x", __entry->order_code,
+			   __print_symbolic(__entry->order_code,
+					    sigp_order_codes),
+			   __entry->cpu_addr, __entry->parameter)
+	);
+
+#define diagnose_codes				\
+	{0x10, "release pages"},		\
+	{0x44, "time slice end"},		\
+	{0x308, "ipl functions"},		\
+	{0x500, "kvm hypercall"},		\
+	{0x501, "kvm breakpoint"}
+
+TRACE_EVENT(kvm_s390_handle_diag,
+	    TP_PROTO(VCPU_PROTO_COMMON, __u16 code),
+	    TP_ARGS(VCPU_ARGS_COMMON, code),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(__u16, code)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->code = code;
+		    ),
+
+	    VCPU_TP_PRINTK("handle diagnose call %04x (%s)", __entry->code,
+			   __print_symbolic(__entry->code, diagnose_codes))
+	);
+
+TRACE_EVENT(kvm_s390_handle_lctl,
+	    TP_PROTO(VCPU_PROTO_COMMON, int g, int reg1, int reg3, u64 addr),
+	    TP_ARGS(VCPU_ARGS_COMMON, g, reg1, reg3, addr),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(int, g)
+		    __field(int, reg1)
+		    __field(int, reg3)
+		    __field(u64, addr)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->g = g;
+		    __entry->reg1 = reg1;
+		    __entry->reg3 = reg3;
+		    __entry->addr = addr;
+		    ),
+
+	    VCPU_TP_PRINTK("%s: loading cr %x-%x from %016llx",
+			   __entry->g ? "lctlg" : "lctl",
+			   __entry->reg1, __entry->reg3, __entry->addr)
+	);
+
+TRACE_EVENT(kvm_s390_handle_prefix,
+	    TP_PROTO(VCPU_PROTO_COMMON, int set, u32 address),
+	    TP_ARGS(VCPU_ARGS_COMMON, set, address),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(int, set)
+		    __field(u32, address)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->set = set;
+		    __entry->address = address;
+		    ),
+
+	    VCPU_TP_PRINTK("%s prefix to %08x",
+			   __entry->set ? "setting" : "storing",
+			   __entry->address)
+	);
+
+TRACE_EVENT(kvm_s390_handle_stap,
+	    TP_PROTO(VCPU_PROTO_COMMON, u64 address),
+	    TP_ARGS(VCPU_ARGS_COMMON, address),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(u64, address)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->address = address;
+		    ),
+
+	    VCPU_TP_PRINTK("storing cpu address to %016llx",
+			   __entry->address)
+	);
+
+TRACE_EVENT(kvm_s390_handle_stfl,
+	    TP_PROTO(VCPU_PROTO_COMMON, unsigned int facility_list),
+	    TP_ARGS(VCPU_ARGS_COMMON, facility_list),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(unsigned int, facility_list)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->facility_list = facility_list;
+		    ),
+
+	    VCPU_TP_PRINTK("store facility list value %08x",
+			   __entry->facility_list)
+	);
+
+TRACE_EVENT(kvm_s390_handle_stsi,
+	    TP_PROTO(VCPU_PROTO_COMMON, int fc, int sel1, int sel2, u64 addr),
+	    TP_ARGS(VCPU_ARGS_COMMON, fc, sel1, sel2, addr),
+
+	    TP_STRUCT__entry(
+		    VCPU_FIELD_COMMON
+		    __field(int, fc)
+		    __field(int, sel1)
+		    __field(int, sel2)
+		    __field(u64, addr)
+		    ),
+
+	    TP_fast_assign(
+		    VCPU_ASSIGN_COMMON
+		    __entry->fc = fc;
+		    __entry->sel1 = sel1;
+		    __entry->sel2 = sel2;
+		    __entry->addr = addr;
+		    ),
+
+	    VCPU_TP_PRINTK("STSI %d.%d.%d information stored to %016llx",
+			   __entry->fc, __entry->sel1, __entry->sel2,
+			   __entry->addr)
+	);
+
+#endif /* _TRACE_KVM_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/arch/score/Kconfig b/arch/score/Kconfig
index ba0f412920b..461c2374749 100644
--- a/arch/score/Kconfig
+++ b/arch/score/Kconfig
@@ -5,6 +5,7 @@ config SCORE
        select HAVE_GENERIC_HARDIRQS
        select GENERIC_IRQ_SHOW
        select GENERIC_IOMAP
+       select GENERIC_ATOMIC64
        select HAVE_MEMBLOCK
        select HAVE_MEMBLOCK_NODE_MAP
        select ARCH_DISCARD_MEMBLOCK
diff --git a/arch/score/include/asm/elf.h b/arch/score/include/asm/elf.h
index f478ce94181..5d566c7a0af 100644
--- a/arch/score/include/asm/elf.h
+++ b/arch/score/include/asm/elf.h
@@ -54,7 +54,7 @@ typedef elf_fpreg_t	elf_fpregset_t;
 
 #define SET_PERSONALITY(ex)					\
 do {								\
-	set_personality(PER_LINUX);				\
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \
 } while (0)
 
 struct task_struct;
diff --git a/arch/score/include/uapi/asm/Kbuild b/arch/score/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/score/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/score/kernel/sys_score.c b/arch/score/kernel/sys_score.c
index e478bf9a7e9..21e86797406 100644
--- a/arch/score/kernel/sys_score.c
+++ b/arch/score/kernel/sys_score.c
@@ -112,6 +112,7 @@ score_execve(struct pt_regs *regs)
  * Do a system call from kernel instead of calling sys_execve so we
  * end up with proper pt_regs.
  */
+asmlinkage
 int kernel_execve(const char *filename,
 		  const char *const argv[],
 		  const char *const envp[])
diff --git a/arch/sh/include/asm/bl_bit.h b/arch/sh/include/asm/bl_bit.h
index 45e6b9fc37a..06e4163c674 100644
--- a/arch/sh/include/asm/bl_bit.h
+++ b/arch/sh/include/asm/bl_bit.h
@@ -2,9 +2,9 @@
 #define __ASM_SH_BL_BIT_H
 
 #ifdef CONFIG_SUPERH32
-# include "bl_bit_32.h"
+# include <asm/bl_bit_32.h>
 #else
-# include "bl_bit_64.h"
+# include <asm/bl_bit_64.h>
 #endif
 
 #endif /* __ASM_SH_BL_BIT_H */
diff --git a/arch/sh/include/asm/cache_insns.h b/arch/sh/include/asm/cache_insns.h
index d25fbe53090..355cb06b7a3 100644
--- a/arch/sh/include/asm/cache_insns.h
+++ b/arch/sh/include/asm/cache_insns.h
@@ -3,9 +3,9 @@
 
 
 #ifdef CONFIG_SUPERH32
-# include "cache_insns_32.h"
+# include <asm/cache_insns_32.h>
 #else
-# include "cache_insns_64.h"
+# include <asm/cache_insns_64.h>
 #endif
 
 #endif /* __ASM_SH_CACHE_INSNS_H */
diff --git a/arch/sh/include/asm/checksum.h b/arch/sh/include/asm/checksum.h
index fc26d1f4b59..34ae2620452 100644
--- a/arch/sh/include/asm/checksum.h
+++ b/arch/sh/include/asm/checksum.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_SUPERH32
-# include "checksum_32.h"
+# include <asm/checksum_32.h>
 #else
 # include <asm-generic/checksum.h>
 #endif
diff --git a/arch/sh/include/asm/elf.h b/arch/sh/include/asm/elf.h
index f38112be67d..37924afa8d8 100644
--- a/arch/sh/include/asm/elf.h
+++ b/arch/sh/include/asm/elf.h
@@ -183,7 +183,8 @@ do {									\
 } while (0)
 #endif
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK)))
 
 #ifdef CONFIG_VSYSCALL
 /* vDSO has arch_setup_additional_pages */
diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h
index 0cf60a62881..73a23f4617a 100644
--- a/arch/sh/include/asm/io.h
+++ b/arch/sh/include/asm/io.h
@@ -134,7 +134,7 @@ __BUILD_MEMORY_STRING(__raw_, q, u64)
  * load/store instructions. sh_io_port_base is the virtual address to
  * which all ports are being mapped.
  */
-extern const unsigned long sh_io_port_base;
+extern unsigned long sh_io_port_base;
 
 static inline void __set_io_port_base(unsigned long pbase)
 {
diff --git a/arch/sh/include/asm/mmu_context.h b/arch/sh/include/asm/mmu_context.h
index 384c7471a37..21c5088788d 100644
--- a/arch/sh/include/asm/mmu_context.h
+++ b/arch/sh/include/asm/mmu_context.h
@@ -46,9 +46,9 @@
 #define MMU_VPN_MASK	0xfffff000
 
 #if defined(CONFIG_SUPERH32)
-#include "mmu_context_32.h"
+#include <asm/mmu_context_32.h>
 #else
-#include "mmu_context_64.h"
+#include <asm/mmu_context_64.h>
 #endif
 
 /*
diff --git a/arch/sh/include/asm/posix_types.h b/arch/sh/include/asm/posix_types.h
index 4eeb723aee7..f08449bcbde 100644
--- a/arch/sh/include/asm/posix_types.h
+++ b/arch/sh/include/asm/posix_types.h
@@ -1,13 +1,13 @@
 #ifdef __KERNEL__
 # ifdef CONFIG_SUPERH32
-#  include "posix_types_32.h"
+#  include <asm/posix_types_32.h>
 # else
-#  include "posix_types_64.h"
+#  include <asm/posix_types_64.h>
 # endif
 #else
 # ifdef __SH5__
-#  include "posix_types_64.h"
+#  include <asm/posix_types_64.h>
 # else
-#  include "posix_types_32.h"
+#  include <asm/posix_types_32.h>
 # endif
 #endif /* __KERNEL__ */
diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h
index 3d14aeaef57..5448f9bbf4a 100644
--- a/arch/sh/include/asm/processor.h
+++ b/arch/sh/include/asm/processor.h
@@ -175,9 +175,9 @@ extern unsigned int instruction_size(unsigned int insn);
 #endif /* __ASSEMBLY__ */
 
 #ifdef CONFIG_SUPERH32
-# include "processor_32.h"
+# include <asm/processor_32.h>
 #else
-# include "processor_64.h"
+# include <asm/processor_64.h>
 #endif
 
 #endif /* __ASM_SH_PROCESSOR_H */
diff --git a/arch/sh/include/asm/ptrace.h b/arch/sh/include/asm/ptrace.h
index c7b7e1ed194..a4a38dff997 100644
--- a/arch/sh/include/asm/ptrace.h
+++ b/arch/sh/include/asm/ptrace.h
@@ -25,9 +25,9 @@
 #define PT_TEXT_LEN		252
 
 #if defined(__SH5__) || defined(CONFIG_CPU_SH5)
-#include "ptrace_64.h"
+#include <asm/ptrace_64.h>
 #else
-#include "ptrace_32.h"
+#include <asm/ptrace_32.h>
 #endif
 
 #ifdef __KERNEL__
diff --git a/arch/sh/include/asm/string.h b/arch/sh/include/asm/string.h
index 8c1ea21dc0a..114011fa08a 100644
--- a/arch/sh/include/asm/string.h
+++ b/arch/sh/include/asm/string.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_SUPERH32
-# include "string_32.h"
+# include <asm/string_32.h>
 #else
-# include "string_64.h"
+# include <asm/string_64.h>
 #endif
diff --git a/arch/sh/include/asm/switch_to.h b/arch/sh/include/asm/switch_to.h
index 62b1941813e..bcd722fc834 100644
--- a/arch/sh/include/asm/switch_to.h
+++ b/arch/sh/include/asm/switch_to.h
@@ -11,9 +11,9 @@
 #define __ASM_SH_SWITCH_TO_H
 
 #ifdef CONFIG_SUPERH32
-# include "switch_to_32.h"
+# include <asm/switch_to_32.h>
 #else
-# include "switch_to_64.h"
+# include <asm/switch_to_64.h>
 #endif
 
 #endif /* __ASM_SH_SWITCH_TO_H */
diff --git a/arch/sh/include/asm/syscall.h b/arch/sh/include/asm/syscall.h
index aa7777bdc37..847128da6ea 100644
--- a/arch/sh/include/asm/syscall.h
+++ b/arch/sh/include/asm/syscall.h
@@ -4,9 +4,9 @@
 extern const unsigned long sys_call_table[];
 
 #ifdef CONFIG_SUPERH32
-# include "syscall_32.h"
+# include <asm/syscall_32.h>
 #else
-# include "syscall_64.h"
+# include <asm/syscall_64.h>
 #endif
 
 #endif /* __ASM_SH_SYSCALL_H */
diff --git a/arch/sh/include/asm/syscalls.h b/arch/sh/include/asm/syscalls.h
index 507725af2e5..3dbfef06f6b 100644
--- a/arch/sh/include/asm/syscalls.h
+++ b/arch/sh/include/asm/syscalls.h
@@ -11,9 +11,9 @@ asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
 			  unsigned long fd, unsigned long pgoff);
 
 #ifdef CONFIG_SUPERH32
-# include "syscalls_32.h"
+# include <asm/syscalls_32.h>
 #else
-# include "syscalls_64.h"
+# include <asm/syscalls_64.h>
 #endif
 
 #endif /* __KERNEL__ */
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index ec88bfcdf7c..e61d43d9f68 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -2,7 +2,7 @@
 #define __ASM_SH_TLB_H
 
 #ifdef CONFIG_SUPERH64
-# include "tlb_64.h"
+# include <asm/tlb_64.h>
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/arch/sh/include/asm/traps.h b/arch/sh/include/asm/traps.h
index afd9df8d064..9cc149a0dbd 100644
--- a/arch/sh/include/asm/traps.h
+++ b/arch/sh/include/asm/traps.h
@@ -4,9 +4,9 @@
 #include <linux/compiler.h>
 
 #ifdef CONFIG_SUPERH32
-# include "traps_32.h"
+# include <asm/traps_32.h>
 #else
-# include "traps_64.h"
+# include <asm/traps_64.h>
 #endif
 
 BUILD_TRAP_HANDLER(address_error);
diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h
index 8698a80ed00..9486376605f 100644
--- a/arch/sh/include/asm/uaccess.h
+++ b/arch/sh/include/asm/uaccess.h
@@ -97,9 +97,9 @@ struct __large_struct { unsigned long buf[100]; };
 })
 
 #ifdef CONFIG_SUPERH32
-# include "uaccess_32.h"
+# include <asm/uaccess_32.h>
 #else
-# include "uaccess_64.h"
+# include <asm/uaccess_64.h>
 #endif
 
 extern long strncpy_from_user(char *dest, const char __user *src, long count);
diff --git a/arch/sh/include/asm/unistd.h b/arch/sh/include/asm/unistd.h
index 7bc67076baa..307201a854f 100644
--- a/arch/sh/include/asm/unistd.h
+++ b/arch/sh/include/asm/unistd.h
@@ -1,8 +1,8 @@
 #ifdef __KERNEL__
 # ifdef CONFIG_SUPERH32
-#  include "unistd_32.h"
+#  include <asm/unistd_32.h>
 # else
-#  include "unistd_64.h"
+#  include <asm/unistd_64.h>
 # endif
 
 # define __ARCH_WANT_SYS_RT_SIGSUSPEND
@@ -40,8 +40,8 @@
 
 #else
 # ifdef __SH5__
-#  include "unistd_64.h"
+#  include <asm/unistd_64.h>
 # else
-#  include "unistd_32.h"
+#  include <asm/unistd_32.h>
 # endif
 #endif
diff --git a/arch/sh/include/mach-ecovec24/mach/romimage.h b/arch/sh/include/mach-ecovec24/mach/romimage.h
index d63ef51ec18..60f3e8af05f 100644
--- a/arch/sh/include/mach-ecovec24/mach/romimage.h
+++ b/arch/sh/include/mach-ecovec24/mach/romimage.h
@@ -6,7 +6,7 @@
  */
 
 #include <asm/romimage-macros.h>
-#include "partner-jet-setup.txt"
+#include <mach/partner-jet-setup.txt>
 
 	/* execute icbi after enabling cache */
 	mov.l	1f, r0
diff --git a/arch/sh/include/mach-kfr2r09/mach/romimage.h b/arch/sh/include/mach-kfr2r09/mach/romimage.h
index 7a883167c84..1afae21ced5 100644
--- a/arch/sh/include/mach-kfr2r09/mach/romimage.h
+++ b/arch/sh/include/mach-kfr2r09/mach/romimage.h
@@ -6,7 +6,7 @@
  */
 
 #include <asm/romimage-macros.h>
-#include "partner-jet-setup.txt"
+#include <mach/partner-jet-setup.txt>
 
 	/* execute icbi after enabling cache */
 	mov.l	1f, r0
diff --git a/arch/sh/include/uapi/asm/Kbuild b/arch/sh/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/sh/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/sh/kernel/ioport.c b/arch/sh/kernel/ioport.c
index e3ad6103e7c..cca14ba84a3 100644
--- a/arch/sh/kernel/ioport.c
+++ b/arch/sh/kernel/ioport.c
@@ -11,7 +11,7 @@
 #include <linux/module.h>
 #include <linux/io.h>
 
-const unsigned long sh_io_port_base __read_mostly = -1;
+unsigned long sh_io_port_base __read_mostly = -1;
 EXPORT_SYMBOL(sh_io_port_base);
 
 void __iomem *__ioport_map(unsigned long addr, unsigned int size)
diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h
index b8be20d42a0..cef99fbc0a2 100644
--- a/arch/sparc/include/asm/compat.h
+++ b/arch/sparc/include/asm/compat.h
@@ -36,6 +36,7 @@ typedef s64		compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64		compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -147,6 +148,65 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		int _pad[SI_PAD_SIZE32];
+
+		/* kill() */
+		struct {
+			compat_pid_t _pid;		/* sender's pid */
+			unsigned int _uid;		/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;		/* timer id */
+			int _overrun;			/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;	/* not to be passed to user */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			compat_pid_t _pid;		/* sender's pid */
+			unsigned int _uid;		/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			compat_pid_t _pid;		/* which child */
+			unsigned int _uid;		/* sender's uid */
+			int _status;			/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
+		struct {
+			u32 _addr; /* faulting insn/memory ref. */
+			int _trapno;
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -156,7 +216,6 @@ typedef u32		compat_sigset_word;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/sparc/include/asm/elf_32.h b/arch/sparc/include/asm/elf_32.h
index 2d4d755cba9..ac74a2c98e6 100644
--- a/arch/sparc/include/asm/elf_32.h
+++ b/arch/sparc/include/asm/elf_32.h
@@ -128,6 +128,7 @@ typedef struct {
 
 #define ELF_PLATFORM	(NULL)
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK)))
 
 #endif /* !(__ASMSPARC_ELF_H) */
diff --git a/arch/sparc/include/asm/oplib_32.h b/arch/sparc/include/asm/oplib_32.h
index 27517879a6c..c72f3045820 100644
--- a/arch/sparc/include/asm/oplib_32.h
+++ b/arch/sparc/include/asm/oplib_32.h
@@ -94,7 +94,7 @@ extern int prom_getprev(void);
 extern void prom_console_write_buf(const char *buf, int len);
 
 /* Prom's internal routines, don't use in kernel/boot code. */
-extern void prom_printf(const char *fmt, ...);
+extern __printf(1, 2) void prom_printf(const char *fmt, ...);
 extern void prom_write(const char *buf, unsigned int len);
 
 /* Multiprocessor operations... */
diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h
index 97a90475c31..a12dbe3b776 100644
--- a/arch/sparc/include/asm/oplib_64.h
+++ b/arch/sparc/include/asm/oplib_64.h
@@ -98,7 +98,7 @@ extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size);
 extern void prom_console_write_buf(const char *buf, int len);
 
 /* Prom's internal routines, don't use in kernel/boot code. */
-extern void prom_printf(const char *fmt, ...);
+extern __printf(1, 2) void prom_printf(const char *fmt, ...);
 extern void prom_write(const char *buf, unsigned int len);
 
 /* Multiprocessor operations... */
diff --git a/arch/sparc/include/asm/siginfo.h b/arch/sparc/include/asm/siginfo.h
index 215900fce21..dbc182c438b 100644
--- a/arch/sparc/include/asm/siginfo.h
+++ b/arch/sparc/include/asm/siginfo.h
@@ -3,7 +3,6 @@
 
 #if defined(__sparc__) && defined(__arch64__)
 
-#define SI_PAD_SIZE32	((SI_MAX_SIZE/sizeof(int)) - 3)
 #define __ARCH_SI_PREAMBLE_SIZE	(4 * sizeof(int))
 #define __ARCH_SI_BAND_T int
 
diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h
index fb269346480..d9a677c5192 100644
--- a/arch/sparc/include/asm/unistd.h
+++ b/arch/sparc/include/asm/unistd.h
@@ -447,6 +447,7 @@
 #else
 #define __ARCH_WANT_COMPAT_SYS_TIME
 #define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND
+#define __ARCH_WANT_COMPAT_SYS_SENDFILE
 #endif
 
 /*
diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..7518ad28696
--- /dev/null
+++ b/arch/sparc/include/uapi/asm/Kbuild
@@ -0,0 +1,5 @@
+# UAPI Header export list
+# User exported sparc header files
+
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c
index 1032df43ec9..c0a2de0fd62 100644
--- a/arch/sparc/kernel/hvapi.c
+++ b/arch/sparc/kernel/hvapi.c
@@ -194,7 +194,7 @@ void __init sun4v_hvapi_init(void)
 
 bad:
 	prom_printf("HVAPI: Cannot register API group "
-		    "%lx with major(%u) minor(%u)\n",
+		    "%lx with major(%lu) minor(%lu)\n",
 		    group, major, minor);
 	prom_halt();
 }
diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c
index 340c5b976d2..d397d7fc5c2 100644
--- a/arch/sparc/kernel/prom_64.c
+++ b/arch/sparc/kernel/prom_64.c
@@ -37,7 +37,7 @@ void * __init prom_early_alloc(unsigned long size)
 	void *ret;
 
 	if (!paddr) {
-		prom_printf("prom_early_alloc(%lu) failed\n");
+		prom_printf("prom_early_alloc(%lu) failed\n", size);
 		prom_halt();
 	}
 
diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c
index a53e0a5fd3a..53e48f721ce 100644
--- a/arch/sparc/kernel/signal32.c
+++ b/arch/sparc/kernel/signal32.c
@@ -54,58 +54,6 @@ struct signal_frame32 {
 	/* __siginfo_rwin_t * */u32 rwin_save;
 } __attribute__((aligned(8)));
 
-typedef struct compat_siginfo{
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[SI_PAD_SIZE32];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			unsigned int _uid;		/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;			/* timer id */
-			int _overrun;			/* overrun count */
-			compat_sigval_t _sigval;		/* same as below */
-			int _sys_private;		/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			unsigned int _uid;		/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;		/* which child */
-			unsigned int _uid;		/* sender's uid */
-			int _status;			/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
-		struct {
-			u32 _addr; /* faulting insn/memory ref. */
-			int _trapno;
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-}compat_siginfo_t;
-
 struct rt_signal_frame32 {
 	struct sparc_stackf32	ss;
 	compat_siginfo_t	info;
diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S
index d97f3eb72e0..44025f4ba41 100644
--- a/arch/sparc/kernel/sys32.S
+++ b/arch/sparc/kernel/sys32.S
@@ -90,7 +90,7 @@ SIGN1(sys32_mkdir, sys_mkdir, %o1)
 SIGN3(sys32_futex, compat_sys_futex, %o1, %o2, %o5)
 SIGN1(sys32_sysfs, compat_sys_sysfs, %o0)
 SIGN2(sys32_sendfile, compat_sys_sendfile, %o0, %o1)
-SIGN2(sys32_sendfile64, compat_sys_sendfile64, %o0, %o1)
+SIGN2(sys32_sendfile64, sys_sendfile, %o0, %o1)
 SIGN1(sys32_prctl, sys_prctl, %o0)
 SIGN1(sys32_sched_rr_get_interval, compat_sys_sched_rr_get_interval, %o0)
 SIGN2(sys32_waitpid, sys_waitpid, %o0, %o2)
diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c
index f7392336961..d862499eb01 100644
--- a/arch/sparc/kernel/sys_sparc32.c
+++ b/arch/sparc/kernel/sys_sparc32.c
@@ -506,52 +506,6 @@ long compat_sys_fadvise64_64(int fd,
 				advice);
 }
 
-asmlinkage long compat_sys_sendfile(int out_fd, int in_fd,
-				    compat_off_t __user *offset,
-				    compat_size_t count)
-{
-	mm_segment_t old_fs = get_fs();
-	int ret;
-	off_t of;
-	
-	if (offset && get_user(of, offset))
-		return -EFAULT;
-		
-	set_fs(KERNEL_DS);
-	ret = sys_sendfile(out_fd, in_fd,
-			   offset ? (off_t __user *) &of : NULL,
-			   count);
-	set_fs(old_fs);
-	
-	if (offset && put_user(of, offset))
-		return -EFAULT;
-		
-	return ret;
-}
-
-asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd,
-				      compat_loff_t __user *offset,
-				      compat_size_t count)
-{
-	mm_segment_t old_fs = get_fs();
-	int ret;
-	loff_t lof;
-	
-	if (offset && get_user(lof, offset))
-		return -EFAULT;
-		
-	set_fs(KERNEL_DS);
-	ret = sys_sendfile64(out_fd, in_fd,
-			     offset ? (loff_t __user *) &lof : NULL,
-			     count);
-	set_fs(old_fs);
-	
-	if (offset && put_user(lof, offset))
-		return -EFAULT;
-		
-	return ret;
-}
-
 /* This is just a version for 32-bit applications which does
  * not force O_LARGEFILE on.
  */
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index 3b05e669771..fa1f1d375ff 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -850,7 +850,7 @@ void __init cheetah_ecache_flush_init(void)
 	ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
 
 	if (ecache_flush_physbase == ~0UL) {
-		prom_printf("cheetah_ecache_flush_init: Cannot find %d byte "
+		prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
 			    "contiguous physical memory.\n",
 			    ecache_flush_size);
 		prom_halt();
diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
index 03eadf66b0d..2c20ad63ddb 100644
--- a/arch/sparc/lib/NG2memcpy.S
+++ b/arch/sparc/lib/NG2memcpy.S
@@ -14,7 +14,7 @@
 #define FPRS_FEF  0x04
 #ifdef MEMCPY_DEBUG
 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
-		     clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
+		     clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0;
 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
 #else
 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
@@ -182,13 +182,13 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	cmp		%g2, 0
 	tne		%xcc, 5
 	PREAMBLE
-	mov		%o0, GLOBAL_SPARE
+	mov		%o0, %o3
 	cmp		%o2, 0
 	be,pn		%XCC, 85f
-	 or		%o0, %o1, %o3
+	 or		%o0, %o1, GLOBAL_SPARE
 	cmp		%o2, 16
 	blu,a,pn	%XCC, 80f
-	 or		%o3, %o2, %o3
+	 or		GLOBAL_SPARE, %o2, GLOBAL_SPARE
 
 	/* 2 blocks (128 bytes) is the minimum we can do the block
 	 * copy with.  We need to ensure that we'll iterate at least
@@ -202,7 +202,7 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	 */
 	cmp		%o2, (4 * 64)
 	blu,pt		%XCC, 75f
-	 andcc		%o3, 0x7, %g0
+	 andcc		GLOBAL_SPARE, 0x7, %g0
 
 	/* %o0:	dst
 	 * %o1:	src
@@ -404,13 +404,13 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	 * over. If anything is left, we copy it one byte at a time.
 	 */
 	brz,pt		%o2, 85f
-	 sub		%o0, %o1, %o3
+	 sub		%o0, %o1, GLOBAL_SPARE
 	ba,a,pt		%XCC, 90f
 
 	.align		64
 75: /* 16 < len <= 64 */
 	bne,pn		%XCC, 75f
-	 sub		%o0, %o1, %o3
+	 sub		%o0, %o1, GLOBAL_SPARE
 
 72:
 	andn		%o2, 0xf, %o4
@@ -420,9 +420,9 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	add		%o1, 0x08, %o1
 	EX_LD(LOAD(ldx, %o1, %g1))
 	sub		%o1, 0x08, %o1
-	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
 	add		%o1, 0x8, %o1
-	EX_ST(STORE(stx, %g1, %o1 + %o3))
+	EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
 	bgu,pt		%XCC, 1b
 	 add		%o1, 0x8, %o1
 73:	andcc		%o2, 0x8, %g0
@@ -430,14 +430,14 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	 nop
 	sub		%o2, 0x8, %o2
 	EX_LD(LOAD(ldx, %o1, %o5))
-	EX_ST(STORE(stx, %o5, %o1 + %o3))
+	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
 	add		%o1, 0x8, %o1
 1:	andcc		%o2, 0x4, %g0
 	be,pt		%XCC, 1f
 	 nop
 	sub		%o2, 0x4, %o2
 	EX_LD(LOAD(lduw, %o1, %o5))
-	EX_ST(STORE(stw, %o5, %o1 + %o3))
+	EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
 	add		%o1, 0x4, %o1
 1:	cmp		%o2, 0
 	be,pt		%XCC, 85f
@@ -454,11 +454,11 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 
 1:	subcc		%g1, 1, %g1
 	EX_LD(LOAD(ldub, %o1, %o5))
-	EX_ST(STORE(stb, %o5, %o1 + %o3))
+	EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
 	bgu,pt		%icc, 1b
 	 add		%o1, 1, %o1
 
-2:	add		%o1, %o3, %o0
+2:	add		%o1, GLOBAL_SPARE, %o0
 	andcc		%o1, 0x7, %g1
 	bne,pt		%icc, 8f
 	 sll		%g1, 3, %g1
@@ -468,16 +468,16 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	 nop
 	ba,a,pt		%xcc, 73b
 
-8:	mov		64, %o3
+8:	mov		64, GLOBAL_SPARE
 	andn		%o1, 0x7, %o1
 	EX_LD(LOAD(ldx, %o1, %g2))
-	sub		%o3, %g1, %o3
+	sub		GLOBAL_SPARE, %g1, GLOBAL_SPARE
 	andn		%o2, 0x7, %o4
 	sllx		%g2, %g1, %g2
 1:	add		%o1, 0x8, %o1
 	EX_LD(LOAD(ldx, %o1, %g3))
 	subcc		%o4, 0x8, %o4
-	srlx		%g3, %o3, %o5
+	srlx		%g3, GLOBAL_SPARE, %o5
 	or		%o5, %g2, %o5
 	EX_ST(STORE(stx, %o5, %o0))
 	add		%o0, 0x8, %o0
@@ -489,32 +489,32 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
 	be,pn		%icc, 85f
 	 add		%o1, %g1, %o1
 	ba,pt		%xcc, 90f
-	 sub		%o0, %o1, %o3
+	 sub		%o0, %o1, GLOBAL_SPARE
 
 	.align		64
 80: /* 0 < len <= 16 */
-	andcc		%o3, 0x3, %g0
+	andcc		GLOBAL_SPARE, 0x3, %g0
 	bne,pn		%XCC, 90f
-	 sub		%o0, %o1, %o3
+	 sub		%o0, %o1, GLOBAL_SPARE
 
 1:
 	subcc		%o2, 4, %o2
 	EX_LD(LOAD(lduw, %o1, %g1))
-	EX_ST(STORE(stw, %g1, %o1 + %o3))
+	EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
 	bgu,pt		%XCC, 1b
 	 add		%o1, 4, %o1
 
 85:	retl
-	 mov		EX_RETVAL(GLOBAL_SPARE), %o0
+	 mov		EX_RETVAL(%o3), %o0
 
 	.align		32
 90:
 	subcc		%o2, 1, %o2
 	EX_LD(LOAD(ldub, %o1, %g1))
-	EX_ST(STORE(stb, %g1, %o1 + %o3))
+	EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
 	bgu,pt		%XCC, 90b
 	 add		%o1, 1, %o1
 	retl
-	 mov		EX_RETVAL(GLOBAL_SPARE), %o0
+	 mov		EX_RETVAL(%o3), %o0
 
 	.size		FUNC_NAME, .-FUNC_NAME
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 696bb095e0f..7a9b788c6ce 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -119,7 +119,8 @@ static void __init read_obp_memory(const char *property,
 
 	ret = prom_getproperty(node, property, (char *) regs, prop_size);
 	if (ret == -1) {
-		prom_printf("Couldn't get %s property from /memory.\n");
+		prom_printf("Couldn't get %s property from /memory.\n",
+				property);
 		prom_halt();
 	}
 
@@ -497,7 +498,7 @@ static void __init read_obp_translations(void)
 		prom_halt();
 	}
 	if (unlikely(n > sizeof(prom_trans))) {
-		prom_printf("prom_mappings: Size %Zd is too big.\n", n);
+		prom_printf("prom_mappings: Size %d is too big.\n", n);
 		prom_halt();
 	}
 
@@ -559,7 +560,7 @@ static void __init hypervisor_tlb_lock(unsigned long vaddr,
 	unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
 
 	if (ret != 0) {
-		prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: "
+		prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
 			    "errors with %lx\n", vaddr, 0, pte, mmu, ret);
 		prom_halt();
 	}
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index a8a58cad9d2..0f4f7191fbb 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -90,8 +90,8 @@ static void __init sbus_iommu_init(struct platform_device *op)
            it to us. */
         tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
 	if (!tmp) {
-		prom_printf("Unable to allocate iommu table [0x%08x]\n",
-			    IOMMU_NPTES*sizeof(iopte_t));
+		prom_printf("Unable to allocate iommu table [0x%lx]\n",
+			    IOMMU_NPTES * sizeof(iopte_t));
 		prom_halt();
 	}
 	iommu->page_table = (iopte_t *)tmp;
diff --git a/arch/tile/include/asm/compat.h b/arch/tile/include/asm/compat.h
index 6e74450ff0a..3063e6fc8da 100644
--- a/arch/tile/include/asm/compat.h
+++ b/arch/tile/include/asm/compat.h
@@ -110,6 +110,68 @@ struct compat_flock64 {
 
 typedef u32               compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+#define COMPAT_SI_PAD_SIZE	(128/sizeof(int) - 3)
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		int _pad[COMPAT_SI_PAD_SIZE];
+
+		/* kill() */
+		struct {
+			unsigned int _pid;	/* sender's pid */
+			unsigned int _uid;	/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;	/* timer id */
+			int _overrun;		/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;	/* not to be passed to user */
+			int _overrun_incr;	/* amount to add to overrun */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			unsigned int _pid;	/* sender's pid */
+			unsigned int _uid;	/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			unsigned int _pid;	/* which child */
+			unsigned int _uid;	/* sender's uid */
+			int _status;		/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			unsigned int _addr;	/* faulting insn/memory ref. */
+#ifdef __ARCH_SI_TRAPNO
+			int _trapno;	/* TRAP # which caused the signal */
+#endif
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
diff --git a/arch/tile/include/asm/elf.h b/arch/tile/include/asm/elf.h
index d16d006d660..f8ccf08f693 100644
--- a/arch/tile/include/asm/elf.h
+++ b/arch/tile/include/asm/elf.h
@@ -156,12 +156,12 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm,
 #undef SET_PERSONALITY
 #define SET_PERSONALITY(ex) \
 do { \
-	current->personality = PER_LINUX; \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \
 	current_thread_info()->status &= ~TS_COMPAT; \
 } while (0)
 #define COMPAT_SET_PERSONALITY(ex) \
 do { \
-	current->personality = PER_LINUX_32BIT; \
+	set_personality(PER_LINUX | (current->personality & (~PER_MASK))); \
 	current_thread_info()->status |= TS_COMPAT; \
 } while (0)
 
diff --git a/arch/tile/include/gxio/dma_queue.h b/arch/tile/include/gxio/dma_queue.h
index 00654feb7db..b9e45e37649 100644
--- a/arch/tile/include/gxio/dma_queue.h
+++ b/arch/tile/include/gxio/dma_queue.h
@@ -19,7 +19,7 @@
  * DMA queue management APIs shared between TRIO and mPIPE.
  */
 
-#include "common.h"
+#include <gxio/common.h>
 
 /* The credit counter lives in the high 32 bits. */
 #define DMA_QUEUE_CREDIT_SHIFT 32
diff --git a/arch/tile/include/gxio/mpipe.h b/arch/tile/include/gxio/mpipe.h
index 78c598618c9..b74f470ed11 100644
--- a/arch/tile/include/gxio/mpipe.h
+++ b/arch/tile/include/gxio/mpipe.h
@@ -21,8 +21,8 @@
  * resources.
  */
 
-#include "common.h"
-#include "dma_queue.h"
+#include <gxio/common.h>
+#include <gxio/dma_queue.h>
 
 #include <linux/time.h>
 
diff --git a/arch/tile/include/gxio/trio.h b/arch/tile/include/gxio/trio.h
index 77b80cdd46d..df10a662cc2 100644
--- a/arch/tile/include/gxio/trio.h
+++ b/arch/tile/include/gxio/trio.h
@@ -140,8 +140,8 @@
 
 #include <linux/types.h>
 
-#include "common.h"
-#include "dma_queue.h"
+#include <gxio/common.h>
+#include <gxio/dma_queue.h>
 
 #include <arch/trio_constants.h>
 #include <arch/trio.h>
diff --git a/arch/tile/include/gxio/usb_host.h b/arch/tile/include/gxio/usb_host.h
index a60a126e456..5eedec0e988 100644
--- a/arch/tile/include/gxio/usb_host.h
+++ b/arch/tile/include/gxio/usb_host.h
@@ -14,7 +14,7 @@
 #ifndef _GXIO_USB_H_
 #define _GXIO_USB_H_
 
-#include "common.h"
+#include <gxio/common.h>
 
 #include <hv/drv_usb_host_intf.h>
 #include <hv/iorpc.h>
diff --git a/arch/tile/include/hv/iorpc.h b/arch/tile/include/hv/iorpc.h
index 89c72a5d934..ddf1604482b 100644
--- a/arch/tile/include/hv/iorpc.h
+++ b/arch/tile/include/hv/iorpc.h
@@ -248,7 +248,7 @@
 #if defined(__HV__)
 #include <hv/hypervisor.h>
 #elif defined(__KERNEL__)
-#include "hypervisor.h"
+#include <hv/hypervisor.h>
 #include <linux/types.h>
 #else
 #include <stdint.h>
diff --git a/arch/tile/include/uapi/arch/Kbuild b/arch/tile/include/uapi/arch/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/arch/tile/include/uapi/arch/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/arch/tile/include/uapi/asm/Kbuild b/arch/tile/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/tile/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/tile/kernel/compat_signal.c b/arch/tile/kernel/compat_signal.c
index 474571b8408..7bc0859a9f5 100644
--- a/arch/tile/kernel/compat_signal.c
+++ b/arch/tile/kernel/compat_signal.c
@@ -55,63 +55,6 @@ struct compat_ucontext {
 	sigset_t	  uc_sigmask;	/* mask last for extensibility */
 };
 
-#define COMPAT_SI_PAD_SIZE	((SI_MAX_SIZE - 3 * sizeof(int)) / sizeof(int))
-
-struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[COMPAT_SI_PAD_SIZE];
-
-		/* kill() */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-			int _overrun_incr;	/* amount to add to overrun */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			unsigned int _addr;	/* faulting insn/memory ref. */
-#ifdef __ARCH_SI_TRAPNO
-			int _trapno;	/* TRAP # which caused the signal */
-#endif
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-};
-
 struct compat_rt_sigframe {
 	unsigned char save_area[C_ABI_SAVE_AREA_SIZE]; /* caller save area */
 	struct compat_siginfo info;
diff --git a/arch/um/Makefile b/arch/um/Makefile
index 097091059aa..133f7de2a13 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -66,7 +66,9 @@ USER_CFLAGS = $(patsubst $(KERNEL_DEFINES),,$(patsubst -D__KERNEL__,,\
 include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS)
 
 KBUILD_CPPFLAGS += -I$(srctree)/$(HOST_DIR)/include \
-		   -I$(HOST_DIR)/include/generated
+		   -I$(srctree)/$(HOST_DIR)/include/uapi \
+		   -I$(HOST_DIR)/include/generated \
+		   -I$(HOST_DIR)/include/generated/uapi
 
 # -Derrno=kernel_errno - This turns all kernel references to errno into
 # kernel_errno to separate them from the libc errno.  This allows -fno-common
diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c
index c17de0db673..9efeb6da48b 100644
--- a/arch/um/drivers/mconsole_kern.c
+++ b/arch/um/drivers/mconsole_kern.c
@@ -21,6 +21,9 @@
 #include <linux/un.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>
+#include <linux/fs.h>
+#include <linux/mount.h>
+#include <linux/file.h>
 #include <asm/uaccess.h>
 #include <asm/switch_to.h>
 
@@ -118,90 +121,38 @@ void mconsole_log(struct mc_request *req)
 	mconsole_reply(req, "", 0, 0);
 }
 
-/* This is a more convoluted version of mconsole_proc, which has some stability
- * problems; however, we need it fixed, because it is expected that UML users
- * mount HPPFS instead of procfs on /proc. And we want mconsole_proc to still
- * show the real procfs content, not the ones from hppfs.*/
-#if 0
 void mconsole_proc(struct mc_request *req)
 {
 	struct vfsmount *mnt = current->nsproxy->pid_ns->proc_mnt;
-	struct file *file;
-	int n;
-	char *ptr = req->request.data, *buf;
-	mm_segment_t old_fs = get_fs();
-
-	ptr += strlen("proc");
-	ptr = skip_spaces(ptr);
-
-	file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY);
-	if (IS_ERR(file)) {
-		mconsole_reply(req, "Failed to open file", 1, 0);
-		goto out;
-	}
-
-	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
-	if (buf == NULL) {
-		mconsole_reply(req, "Failed to allocate buffer", 1, 0);
-		goto out_fput;
-	}
-
-	if (file->f_op->read) {
-		do {
-			loff_t pos;
-			set_fs(KERNEL_DS);
-			n = vfs_read(file, buf, PAGE_SIZE - 1, &pos);
-			file_pos_write(file, pos);
-			set_fs(old_fs);
-			if (n >= 0) {
-				buf[n] = '\0';
-				mconsole_reply(req, buf, 0, (n > 0));
-			}
-			else {
-				mconsole_reply(req, "Read of file failed",
-					       1, 0);
-				goto out_free;
-			}
-		} while (n > 0);
-	}
-	else mconsole_reply(req, "", 0, 0);
-
- out_free:
-	kfree(buf);
- out_fput:
-	fput(file);
- out: ;
-}
-#endif
-
-void mconsole_proc(struct mc_request *req)
-{
-	char path[64];
 	char *buf;
 	int len;
-	int fd;
+	struct file *file;
 	int first_chunk = 1;
 	char *ptr = req->request.data;
 
 	ptr += strlen("proc");
 	ptr = skip_spaces(ptr);
-	snprintf(path, sizeof(path), "/proc/%s", ptr);
 
-	fd = sys_open(path, 0, 0);
-	if (fd < 0) {
+	file = file_open_root(mnt->mnt_root, mnt, ptr, O_RDONLY);
+	if (IS_ERR(file)) {
 		mconsole_reply(req, "Failed to open file", 1, 0);
-		printk(KERN_ERR "open %s: %d\n",path,fd);
+		printk(KERN_ERR "open /proc/%s: %ld\n", ptr, PTR_ERR(file));
 		goto out;
 	}
 
 	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 	if (buf == NULL) {
 		mconsole_reply(req, "Failed to allocate buffer", 1, 0);
-		goto out_close;
+		goto out_fput;
 	}
 
-	for (;;) {
-		len = sys_read(fd, buf, PAGE_SIZE-1);
+	do {
+		loff_t pos;
+		mm_segment_t old_fs = get_fs();
+		set_fs(KERNEL_DS);
+		len = vfs_read(file, buf, PAGE_SIZE - 1, &pos);
+		set_fs(old_fs);
+		file->f_pos = pos;
 		if (len < 0) {
 			mconsole_reply(req, "Read of file failed", 1, 0);
 			goto out_free;
@@ -211,22 +162,14 @@ void mconsole_proc(struct mc_request *req)
 			mconsole_reply(req, "\n", 0, 1);
 			first_chunk = 0;
 		}
-		if (len == PAGE_SIZE-1) {
-			buf[len] = '\0';
-			mconsole_reply(req, buf, 0, 1);
-		} else {
-			buf[len] = '\0';
-			mconsole_reply(req, buf, 0, 0);
-			break;
-		}
-	}
-
+		buf[len] = '\0';
+		mconsole_reply(req, buf, 0, (len > 0));
+	} while (len > 0);
  out_free:
 	kfree(buf);
- out_close:
-	sys_close(fd);
- out:
-	/* nothing */;
+ out_fput:
+	fput(file);
+ out: ;
 }
 
 #define UML_MCONSOLE_HELPTEXT \
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig
index b0a47433341..1e638e75a6b 100644
--- a/arch/unicore32/Kconfig
+++ b/arch/unicore32/Kconfig
@@ -6,6 +6,7 @@ config UNICORE32
 	select HAVE_DMA_ATTRS
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_BZIP2
+	select GENERIC_ATOMIC64
 	select HAVE_KERNEL_LZO
 	select HAVE_KERNEL_LZMA
 	select ARCH_HAVE_CUSTOM_GPIO_H
diff --git a/arch/unicore32/include/mach/PKUnity.h b/arch/unicore32/include/mach/PKUnity.h
index 8040d575ddd..46705afcbf5 100644
--- a/arch/unicore32/include/mach/PKUnity.h
+++ b/arch/unicore32/include/mach/PKUnity.h
@@ -15,7 +15,7 @@
 #error You must include hardware.h not PKUnity.h
 #endif
 
-#include "bitfield.h"
+#include <mach/bitfield.h>
 
 /*
  * Memory Definitions
@@ -32,7 +32,7 @@
  * 0x98000000 - 0x9FFFFFFF 128MB  PCI PCI-AHB MEM-mapping
  */
 #define PKUNITY_PCI_BASE		io_p2v(0x80000000) /* 0x80000000 - 0xBFFFFFFF 1GB */
-#include "regs-pci.h"
+#include <mach/regs-pci.h>
 
 #define PKUNITY_PCICFG_BASE		(PKUNITY_PCI_BASE + 0x0)
 #define PKUNITY_PCIBRI_BASE		(PKUNITY_PCI_BASE + 0x00010000)
@@ -50,18 +50,18 @@
 #define PKUNITY_ARBITER_BASE		(PKUNITY_AHB_BASE + 0x000000) /* AHB-2 */
 #define PKUNITY_DDR2CTRL_BASE		(PKUNITY_AHB_BASE + 0x100000) /* AHB-3 */
 #define PKUNITY_DMAC_BASE		(PKUNITY_AHB_BASE + 0x200000) /* AHB-4 */
-#include "regs-dmac.h"
+#include <mach/regs-dmac.h>
 #define PKUNITY_UMAL_BASE		(PKUNITY_AHB_BASE + 0x300000) /* AHB-5 */
-#include "regs-umal.h"
+#include <mach/regs-umal.h>
 #define PKUNITY_USB_BASE		(PKUNITY_AHB_BASE + 0x400000) /* AHB-6 */
 #define PKUNITY_SATA_BASE		(PKUNITY_AHB_BASE + 0x500000) /* AHB-7 */
 #define PKUNITY_SMC_BASE		(PKUNITY_AHB_BASE + 0x600000) /* AHB-8 */
 /* AHB-9 is for APB bridge */
 #define PKUNITY_MME_BASE		(PKUNITY_AHB_BASE + 0x700000) /* AHB-10 */
 #define PKUNITY_UNIGFX_BASE		(PKUNITY_AHB_BASE + 0x800000) /* AHB-11 */
-#include "regs-unigfx.h"
+#include <mach/regs-unigfx.h>
 #define PKUNITY_NAND_BASE		(PKUNITY_AHB_BASE + 0x900000) /* AHB-12 */
-#include "regs-nand.h"
+#include <mach/regs-nand.h>
 #define PKUNITY_H264D_BASE		(PKUNITY_AHB_BASE + 0xA00000) /* AHB-13 */
 #define PKUNITY_H264E_BASE		(PKUNITY_AHB_BASE + 0xB00000) /* AHB-14 */
 
@@ -72,27 +72,27 @@
 
 #define PKUNITY_UART0_BASE		(PKUNITY_APB_BASE + 0x000000) /* APB-0 */
 #define PKUNITY_UART1_BASE		(PKUNITY_APB_BASE + 0x100000) /* APB-1 */
-#include "regs-uart.h"
+#include <mach/regs-uart.h>
 #define PKUNITY_I2C_BASE		(PKUNITY_APB_BASE + 0x200000) /* APB-2 */
-#include "regs-i2c.h"
+#include <mach/regs-i2c.h>
 #define PKUNITY_SPI_BASE		(PKUNITY_APB_BASE + 0x300000) /* APB-3 */
-#include "regs-spi.h"
+#include <mach/regs-spi.h>
 #define PKUNITY_AC97_BASE		(PKUNITY_APB_BASE + 0x400000) /* APB-4 */
-#include "regs-ac97.h"
+#include <mach/regs-ac97.h>
 #define PKUNITY_GPIO_BASE		(PKUNITY_APB_BASE + 0x500000) /* APB-5 */
-#include "regs-gpio.h"
+#include <mach/regs-gpio.h>
 #define PKUNITY_INTC_BASE		(PKUNITY_APB_BASE + 0x600000) /* APB-6 */
-#include "regs-intc.h"
+#include <mach/regs-intc.h>
 #define PKUNITY_RTC_BASE		(PKUNITY_APB_BASE + 0x700000) /* APB-7 */
-#include "regs-rtc.h"
+#include <mach/regs-rtc.h>
 #define PKUNITY_OST_BASE		(PKUNITY_APB_BASE + 0x800000) /* APB-8 */
-#include "regs-ost.h"
+#include <mach/regs-ost.h>
 #define PKUNITY_RESETC_BASE		(PKUNITY_APB_BASE + 0x900000) /* APB-9 */
-#include "regs-resetc.h"
+#include <mach/regs-resetc.h>
 #define PKUNITY_PM_BASE			(PKUNITY_APB_BASE + 0xA00000) /* APB-10 */
-#include "regs-pm.h"
+#include <mach/regs-pm.h>
 #define PKUNITY_PS2_BASE		(PKUNITY_APB_BASE + 0xB00000) /* APB-11 */
-#include "regs-ps2.h"
+#include <mach/regs-ps2.h>
 #define PKUNITY_SDC_BASE		(PKUNITY_APB_BASE + 0xC00000) /* APB-12 */
-#include "regs-sdc.h"
+#include <mach/regs-sdc.h>
 
diff --git a/arch/unicore32/include/mach/hardware.h b/arch/unicore32/include/mach/hardware.h
index 930bea6e129..9e20b5d9ed5 100644
--- a/arch/unicore32/include/mach/hardware.h
+++ b/arch/unicore32/include/mach/hardware.h
@@ -15,7 +15,7 @@
 #ifndef __MACH_PUV3_HARDWARE_H__
 #define __MACH_PUV3_HARDWARE_H__
 
-#include "PKUnity.h"
+#include <mach/PKUnity.h>
 
 #ifndef __ASSEMBLY__
 #define io_p2v(x)	(void __iomem *)((x) - PKUNITY_MMIO_BASE)
diff --git a/arch/unicore32/include/mach/uncompress.h b/arch/unicore32/include/mach/uncompress.h
index 142d3e7958a..9be67c9d3b5 100644
--- a/arch/unicore32/include/mach/uncompress.h
+++ b/arch/unicore32/include/mach/uncompress.h
@@ -13,8 +13,8 @@
 #ifndef __MACH_PUV3_UNCOMPRESS_H__
 #define __MACH_PUV3_UNCOMPRESS_H__
 
-#include "hardware.h"
-#include "ocd.h"
+#include <mach/hardware.h>
+#include <mach/ocd.h>
 
 extern char input_data[];
 extern char input_data_end[];
diff --git a/arch/unicore32/include/uapi/asm/Kbuild b/arch/unicore32/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/unicore32/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7f9a395c525..b72777ff32a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -586,23 +586,18 @@ config PARAVIRT_TIME_ACCOUNTING
 
 source "arch/x86/xen/Kconfig"
 
-config KVM_CLOCK
-	bool "KVM paravirtualized clock"
-	select PARAVIRT
-	select PARAVIRT_CLOCK
-	---help---
-	  Turning on this option will allow you to run a paravirtualized clock
-	  when running over the KVM hypervisor. Instead of relying on a PIT
-	  (or probably other) emulation by the underlying device model, the host
-	  provides the guest with timing infrastructure such as time of day, and
-	  system time
-
 config KVM_GUEST
-	bool "KVM Guest support"
+	bool "KVM Guest support (including kvmclock)"
+	select PARAVIRT
 	select PARAVIRT
+	select PARAVIRT_CLOCK
+	default y if PARAVIRT_GUEST
 	---help---
 	  This option enables various optimizations for running under the KVM
-	  hypervisor.
+	  hypervisor. It includes a paravirtualized clock, so that instead
+	  of relying on a PIT (or probably other) emulation by the
+	  underlying device model, the host provides the guest with
+	  timing infrastructure such as time of day, and system time
 
 source "arch/x86/lguest/Kconfig"
 
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index f7535bedc33..ce03476d8c8 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -37,7 +37,7 @@ setup-y		+= video-bios.o
 targets		+= $(setup-y)
 hostprogs-y	:= mkcpustr tools/build
 
-HOST_EXTRACFLAGS += -I$(srctree)/tools/include $(LINUXINCLUDE) \
+HOST_EXTRACFLAGS += -I$(srctree)/tools/include $(USERINCLUDE) \
 	            -D__EXPORTED_HEADERS__
 
 $(obj)/cpu.o: $(obj)/cpustr.h
@@ -52,7 +52,7 @@ $(obj)/cpustr.h: $(obj)/mkcpustr FORCE
 
 # How to compile the 16-bit code.  Note we always compile for -march=i386,
 # that way we can complain to the user if the CPU is insufficient.
-KBUILD_CFLAGS	:= $(LINUXINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \
+KBUILD_CFLAGS	:= $(USERINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \
 		   -DDISABLE_BRANCH_PROFILING \
 		   -Wall -Wstrict-prototypes \
 		   -march=i386 -mregparm=3 \
diff --git a/arch/x86/boot/mkcpustr.c b/arch/x86/boot/mkcpustr.c
index 919257f526f..4579eff0ef4 100644
--- a/arch/x86/boot/mkcpustr.c
+++ b/arch/x86/boot/mkcpustr.c
@@ -15,6 +15,8 @@
 
 #include <stdio.h>
 
+#include "../include/asm/required-features.h"
+#include "../include/asm/cpufeature.h"
 #include "../kernel/cpu/capflags.c"
 
 int main(void)
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e908e5de82d..5bacb4a226a 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -12,6 +12,8 @@ obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o
 
 obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
 obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
+obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o
+obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o
 obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
@@ -32,6 +34,8 @@ serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o
 
 aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
 camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
+cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o
+cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o
 blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
 twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
 twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c
index 59b37deb8c8..aafe8ce0d65 100644
--- a/arch/x86/crypto/aes_glue.c
+++ b/arch/x86/crypto/aes_glue.c
@@ -40,7 +40,6 @@ static struct crypto_alg aes_alg = {
 	.cra_blocksize		= AES_BLOCK_SIZE,
 	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(aes_alg.cra_list),
 	.cra_u	= {
 		.cipher	= {
 			.cia_min_keysize	= AES_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 34fdcff4d2c..7c04d0da709 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -28,6 +28,9 @@
 #include <crypto/aes.h>
 #include <crypto/cryptd.h>
 #include <crypto/ctr.h>
+#include <crypto/b128ops.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
 #include <asm/cpu_device_id.h>
 #include <asm/i387.h>
 #include <asm/crypto/aes.h>
@@ -41,18 +44,10 @@
 #define HAS_CTR
 #endif
 
-#if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
-#define HAS_LRW
-#endif
-
 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
 #define HAS_PCBC
 #endif
 
-#if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
-#define HAS_XTS
-#endif
-
 /* This data is stored at the end of the crypto_tfm struct.
  * It's a type of per "session" data storage location.
  * This needs to be 16 byte aligned.
@@ -79,6 +74,16 @@ struct aesni_hash_subkey_req_data {
 #define AES_BLOCK_MASK	(~(AES_BLOCK_SIZE-1))
 #define RFC4106_HASH_SUBKEY_SIZE 16
 
+struct aesni_lrw_ctx {
+	struct lrw_table_ctx lrw_table;
+	u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
+};
+
+struct aesni_xts_ctx {
+	u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
+	u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
+};
+
 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
 			     unsigned int key_len);
 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
@@ -398,13 +403,6 @@ static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 #endif
 #endif
 
-#ifdef HAS_LRW
-static int ablk_lrw_init(struct crypto_tfm *tfm)
-{
-	return ablk_init_common(tfm, "fpu(lrw(__driver-aes-aesni))");
-}
-#endif
-
 #ifdef HAS_PCBC
 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 {
@@ -412,12 +410,160 @@ static int ablk_pcbc_init(struct crypto_tfm *tfm)
 }
 #endif
 
-#ifdef HAS_XTS
-static int ablk_xts_init(struct crypto_tfm *tfm)
+static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
 {
-	return ablk_init_common(tfm, "fpu(xts(__driver-aes-aesni))");
+	aesni_ecb_enc(ctx, blks, blks, nbytes);
+}
+
+static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
+{
+	aesni_ecb_dec(ctx, blks, blks, nbytes);
+}
+
+static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
+			    unsigned int keylen)
+{
+	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+	int err;
+
+	err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
+				 keylen - AES_BLOCK_SIZE);
+	if (err)
+		return err;
+
+	return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
+}
+
+static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	lrw_free_table(&ctx->lrw_table);
+}
+
+static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[8];
+	struct lrw_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.table_ctx = &ctx->lrw_table,
+		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
+		.crypt_fn = lrw_xts_encrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	kernel_fpu_begin();
+	ret = lrw_crypt(desc, dst, src, nbytes, &req);
+	kernel_fpu_end();
+
+	return ret;
+}
+
+static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[8];
+	struct lrw_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.table_ctx = &ctx->lrw_table,
+		.crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
+		.crypt_fn = lrw_xts_decrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	kernel_fpu_begin();
+	ret = lrw_crypt(desc, dst, src, nbytes, &req);
+	kernel_fpu_end();
+
+	return ret;
+}
+
+static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
+			    unsigned int keylen)
+{
+	struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	u32 *flags = &tfm->crt_flags;
+	int err;
+
+	/* key consists of keys of equal size concatenated, therefore
+	 * the length must be even
+	 */
+	if (keylen % 2) {
+		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	/* first half of xts-key is for crypt */
+	err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
+	if (err)
+		return err;
+
+	/* second half of xts-key is for tweak */
+	return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
+				  keylen / 2);
+}
+
+
+static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[8];
+	struct xts_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
+		.tweak_fn = XTS_TWEAK_CAST(aesni_enc),
+		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
+		.crypt_fn = lrw_xts_encrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	kernel_fpu_begin();
+	ret = xts_crypt(desc, dst, src, nbytes, &req);
+	kernel_fpu_end();
+
+	return ret;
+}
+
+static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[8];
+	struct xts_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
+		.tweak_fn = XTS_TWEAK_CAST(aesni_enc),
+		.crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
+		.crypt_fn = lrw_xts_decrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	kernel_fpu_begin();
+	ret = xts_crypt(desc, dst, src, nbytes, &req);
+	kernel_fpu_end();
+
+	return ret;
 }
-#endif
 
 #ifdef CONFIG_X86_64
 static int rfc4106_init(struct crypto_tfm *tfm)
@@ -1035,10 +1181,10 @@ static struct crypto_alg aesni_algs[] = { {
 	},
 #endif
 #endif
-#ifdef HAS_LRW
+#ifdef HAS_PCBC
 }, {
-	.cra_name		= "lrw(aes)",
-	.cra_driver_name	= "lrw-aes-aesni",
+	.cra_name		= "pcbc(aes)",
+	.cra_driver_name	= "pcbc-aes-aesni",
 	.cra_priority		= 400,
 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 	.cra_blocksize		= AES_BLOCK_SIZE,
@@ -1046,12 +1192,12 @@ static struct crypto_alg aesni_algs[] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_lrw_init,
+	.cra_init		= ablk_pcbc_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
 		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
+			.min_keysize	= AES_MIN_KEY_SIZE,
+			.max_keysize	= AES_MAX_KEY_SIZE,
 			.ivsize		= AES_BLOCK_SIZE,
 			.setkey		= ablk_set_key,
 			.encrypt	= ablk_encrypt,
@@ -1059,10 +1205,50 @@ static struct crypto_alg aesni_algs[] = { {
 		},
 	},
 #endif
-#ifdef HAS_PCBC
 }, {
-	.cra_name		= "pcbc(aes)",
-	.cra_driver_name	= "pcbc-aes-aesni",
+	.cra_name		= "__lrw-aes-aesni",
+	.cra_driver_name	= "__driver-lrw-aes-aesni",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct aesni_lrw_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_exit		= lrw_aesni_exit_tfm,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
+			.max_keysize	= AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
+			.ivsize		= AES_BLOCK_SIZE,
+			.setkey		= lrw_aesni_setkey,
+			.encrypt	= lrw_encrypt,
+			.decrypt	= lrw_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__xts-aes-aesni",
+	.cra_driver_name	= "__driver-xts-aes-aesni",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct aesni_xts_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= 2 * AES_MIN_KEY_SIZE,
+			.max_keysize	= 2 * AES_MAX_KEY_SIZE,
+			.ivsize		= AES_BLOCK_SIZE,
+			.setkey		= xts_aesni_setkey,
+			.encrypt	= xts_encrypt,
+			.decrypt	= xts_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "lrw(aes)",
+	.cra_driver_name	= "lrw-aes-aesni",
 	.cra_priority		= 400,
 	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 	.cra_blocksize		= AES_BLOCK_SIZE,
@@ -1070,20 +1256,18 @@ static struct crypto_alg aesni_algs[] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_pcbc_init,
+	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
 		.ablkcipher = {
-			.min_keysize	= AES_MIN_KEY_SIZE,
-			.max_keysize	= AES_MAX_KEY_SIZE,
+			.min_keysize	= AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
+			.max_keysize	= AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
 			.ivsize		= AES_BLOCK_SIZE,
 			.setkey		= ablk_set_key,
 			.encrypt	= ablk_encrypt,
 			.decrypt	= ablk_decrypt,
 		},
 	},
-#endif
-#ifdef HAS_XTS
 }, {
 	.cra_name		= "xts(aes)",
 	.cra_driver_name	= "xts-aes-aesni",
@@ -1094,7 +1278,7 @@ static struct crypto_alg aesni_algs[] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_init		= ablk_xts_init,
+	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
 		.ablkcipher = {
@@ -1106,7 +1290,6 @@ static struct crypto_alg aesni_algs[] = { {
 			.decrypt	= ablk_decrypt,
 		},
 	},
-#endif
 } };
 
 
@@ -1118,7 +1301,7 @@ MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
 
 static int __init aesni_init(void)
 {
-	int err, i;
+	int err;
 
 	if (!x86_match_cpu(aesni_cpu_id))
 		return -ENODEV;
@@ -1127,9 +1310,6 @@ static int __init aesni_init(void)
 	if (err)
 		return err;
 
-	for (i = 0; i < ARRAY_SIZE(aesni_algs); i++)
-		INIT_LIST_HEAD(&aesni_algs[i].cra_list);
-
 	return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
 }
 
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
index 7967474de8f..50ec333b70e 100644
--- a/arch/x86/crypto/blowfish_glue.c
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -367,7 +367,6 @@ static struct crypto_alg bf_algs[4] = { {
 	.cra_ctxsize		= sizeof(struct bf_ctx),
 	.cra_alignmask		= 0,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(bf_algs[0].cra_list),
 	.cra_u = {
 		.cipher = {
 			.cia_min_keysize	= BF_MIN_KEY_SIZE,
@@ -387,7 +386,6 @@ static struct crypto_alg bf_algs[4] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(bf_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= BF_MIN_KEY_SIZE,
@@ -407,7 +405,6 @@ static struct crypto_alg bf_algs[4] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(bf_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= BF_MIN_KEY_SIZE,
@@ -428,7 +425,6 @@ static struct crypto_alg bf_algs[4] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(bf_algs[3].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= BF_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index eeb2b3b743e..42ffd2bbab5 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -92,715 +92,715 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 
 /* camellia sboxes */
 const u64 camellia_sp10011110[256] = {
-	0x7000007070707000, 0x8200008282828200, 0x2c00002c2c2c2c00,
-	0xec0000ecececec00, 0xb30000b3b3b3b300, 0x2700002727272700,
-	0xc00000c0c0c0c000, 0xe50000e5e5e5e500, 0xe40000e4e4e4e400,
-	0x8500008585858500, 0x5700005757575700, 0x3500003535353500,
-	0xea0000eaeaeaea00, 0x0c00000c0c0c0c00, 0xae0000aeaeaeae00,
-	0x4100004141414100, 0x2300002323232300, 0xef0000efefefef00,
-	0x6b00006b6b6b6b00, 0x9300009393939300, 0x4500004545454500,
-	0x1900001919191900, 0xa50000a5a5a5a500, 0x2100002121212100,
-	0xed0000edededed00, 0x0e00000e0e0e0e00, 0x4f00004f4f4f4f00,
-	0x4e00004e4e4e4e00, 0x1d00001d1d1d1d00, 0x6500006565656500,
-	0x9200009292929200, 0xbd0000bdbdbdbd00, 0x8600008686868600,
-	0xb80000b8b8b8b800, 0xaf0000afafafaf00, 0x8f00008f8f8f8f00,
-	0x7c00007c7c7c7c00, 0xeb0000ebebebeb00, 0x1f00001f1f1f1f00,
-	0xce0000cececece00, 0x3e00003e3e3e3e00, 0x3000003030303000,
-	0xdc0000dcdcdcdc00, 0x5f00005f5f5f5f00, 0x5e00005e5e5e5e00,
-	0xc50000c5c5c5c500, 0x0b00000b0b0b0b00, 0x1a00001a1a1a1a00,
-	0xa60000a6a6a6a600, 0xe10000e1e1e1e100, 0x3900003939393900,
-	0xca0000cacacaca00, 0xd50000d5d5d5d500, 0x4700004747474700,
-	0x5d00005d5d5d5d00, 0x3d00003d3d3d3d00, 0xd90000d9d9d9d900,
-	0x0100000101010100, 0x5a00005a5a5a5a00, 0xd60000d6d6d6d600,
-	0x5100005151515100, 0x5600005656565600, 0x6c00006c6c6c6c00,
-	0x4d00004d4d4d4d00, 0x8b00008b8b8b8b00, 0x0d00000d0d0d0d00,
-	0x9a00009a9a9a9a00, 0x6600006666666600, 0xfb0000fbfbfbfb00,
-	0xcc0000cccccccc00, 0xb00000b0b0b0b000, 0x2d00002d2d2d2d00,
-	0x7400007474747400, 0x1200001212121200, 0x2b00002b2b2b2b00,
-	0x2000002020202000, 0xf00000f0f0f0f000, 0xb10000b1b1b1b100,
-	0x8400008484848400, 0x9900009999999900, 0xdf0000dfdfdfdf00,
-	0x4c00004c4c4c4c00, 0xcb0000cbcbcbcb00, 0xc20000c2c2c2c200,
-	0x3400003434343400, 0x7e00007e7e7e7e00, 0x7600007676767600,
-	0x0500000505050500, 0x6d00006d6d6d6d00, 0xb70000b7b7b7b700,
-	0xa90000a9a9a9a900, 0x3100003131313100, 0xd10000d1d1d1d100,
-	0x1700001717171700, 0x0400000404040400, 0xd70000d7d7d7d700,
-	0x1400001414141400, 0x5800005858585800, 0x3a00003a3a3a3a00,
-	0x6100006161616100, 0xde0000dededede00, 0x1b00001b1b1b1b00,
-	0x1100001111111100, 0x1c00001c1c1c1c00, 0x3200003232323200,
-	0x0f00000f0f0f0f00, 0x9c00009c9c9c9c00, 0x1600001616161600,
-	0x5300005353535300, 0x1800001818181800, 0xf20000f2f2f2f200,
-	0x2200002222222200, 0xfe0000fefefefe00, 0x4400004444444400,
-	0xcf0000cfcfcfcf00, 0xb20000b2b2b2b200, 0xc30000c3c3c3c300,
-	0xb50000b5b5b5b500, 0x7a00007a7a7a7a00, 0x9100009191919100,
-	0x2400002424242400, 0x0800000808080800, 0xe80000e8e8e8e800,
-	0xa80000a8a8a8a800, 0x6000006060606000, 0xfc0000fcfcfcfc00,
-	0x6900006969696900, 0x5000005050505000, 0xaa0000aaaaaaaa00,
-	0xd00000d0d0d0d000, 0xa00000a0a0a0a000, 0x7d00007d7d7d7d00,
-	0xa10000a1a1a1a100, 0x8900008989898900, 0x6200006262626200,
-	0x9700009797979700, 0x5400005454545400, 0x5b00005b5b5b5b00,
-	0x1e00001e1e1e1e00, 0x9500009595959500, 0xe00000e0e0e0e000,
-	0xff0000ffffffff00, 0x6400006464646400, 0xd20000d2d2d2d200,
-	0x1000001010101000, 0xc40000c4c4c4c400, 0x0000000000000000,
-	0x4800004848484800, 0xa30000a3a3a3a300, 0xf70000f7f7f7f700,
-	0x7500007575757500, 0xdb0000dbdbdbdb00, 0x8a00008a8a8a8a00,
-	0x0300000303030300, 0xe60000e6e6e6e600, 0xda0000dadadada00,
-	0x0900000909090900, 0x3f00003f3f3f3f00, 0xdd0000dddddddd00,
-	0x9400009494949400, 0x8700008787878700, 0x5c00005c5c5c5c00,
-	0x8300008383838300, 0x0200000202020200, 0xcd0000cdcdcdcd00,
-	0x4a00004a4a4a4a00, 0x9000009090909000, 0x3300003333333300,
-	0x7300007373737300, 0x6700006767676700, 0xf60000f6f6f6f600,
-	0xf30000f3f3f3f300, 0x9d00009d9d9d9d00, 0x7f00007f7f7f7f00,
-	0xbf0000bfbfbfbf00, 0xe20000e2e2e2e200, 0x5200005252525200,
-	0x9b00009b9b9b9b00, 0xd80000d8d8d8d800, 0x2600002626262600,
-	0xc80000c8c8c8c800, 0x3700003737373700, 0xc60000c6c6c6c600,
-	0x3b00003b3b3b3b00, 0x8100008181818100, 0x9600009696969600,
-	0x6f00006f6f6f6f00, 0x4b00004b4b4b4b00, 0x1300001313131300,
-	0xbe0000bebebebe00, 0x6300006363636300, 0x2e00002e2e2e2e00,
-	0xe90000e9e9e9e900, 0x7900007979797900, 0xa70000a7a7a7a700,
-	0x8c00008c8c8c8c00, 0x9f00009f9f9f9f00, 0x6e00006e6e6e6e00,
-	0xbc0000bcbcbcbc00, 0x8e00008e8e8e8e00, 0x2900002929292900,
-	0xf50000f5f5f5f500, 0xf90000f9f9f9f900, 0xb60000b6b6b6b600,
-	0x2f00002f2f2f2f00, 0xfd0000fdfdfdfd00, 0xb40000b4b4b4b400,
-	0x5900005959595900, 0x7800007878787800, 0x9800009898989800,
-	0x0600000606060600, 0x6a00006a6a6a6a00, 0xe70000e7e7e7e700,
-	0x4600004646464600, 0x7100007171717100, 0xba0000babababa00,
-	0xd40000d4d4d4d400, 0x2500002525252500, 0xab0000abababab00,
-	0x4200004242424200, 0x8800008888888800, 0xa20000a2a2a2a200,
-	0x8d00008d8d8d8d00, 0xfa0000fafafafa00, 0x7200007272727200,
-	0x0700000707070700, 0xb90000b9b9b9b900, 0x5500005555555500,
-	0xf80000f8f8f8f800, 0xee0000eeeeeeee00, 0xac0000acacacac00,
-	0x0a00000a0a0a0a00, 0x3600003636363600, 0x4900004949494900,
-	0x2a00002a2a2a2a00, 0x6800006868686800, 0x3c00003c3c3c3c00,
-	0x3800003838383800, 0xf10000f1f1f1f100, 0xa40000a4a4a4a400,
-	0x4000004040404000, 0x2800002828282800, 0xd30000d3d3d3d300,
-	0x7b00007b7b7b7b00, 0xbb0000bbbbbbbb00, 0xc90000c9c9c9c900,
-	0x4300004343434300, 0xc10000c1c1c1c100, 0x1500001515151500,
-	0xe30000e3e3e3e300, 0xad0000adadadad00, 0xf40000f4f4f4f400,
-	0x7700007777777700, 0xc70000c7c7c7c700, 0x8000008080808000,
-	0x9e00009e9e9e9e00,
+	0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
+	0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
+	0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
+	0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL,
+	0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL,
+	0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL,
+	0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL,
+	0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL,
+	0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL,
+	0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL,
+	0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL,
+	0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL,
+	0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL,
+	0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL,
+	0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL,
+	0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL,
+	0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL,
+	0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL,
+	0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL,
+	0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL,
+	0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL,
+	0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL,
+	0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL,
+	0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL,
+	0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL,
+	0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL,
+	0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL,
+	0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL,
+	0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL,
+	0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL,
+	0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL,
+	0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL,
+	0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL,
+	0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL,
+	0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL,
+	0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL,
+	0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL,
+	0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL,
+	0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL,
+	0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL,
+	0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL,
+	0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL,
+	0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL,
+	0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL,
+	0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL,
+	0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL,
+	0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL,
+	0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL,
+	0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL,
+	0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL,
+	0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL,
+	0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL,
+	0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL,
+	0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL,
+	0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL,
+	0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL,
+	0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL,
+	0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL,
+	0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL,
+	0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL,
+	0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL,
+	0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL,
+	0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL,
+	0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL,
+	0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL,
+	0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL,
+	0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL,
+	0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL,
+	0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL,
+	0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL,
+	0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL,
+	0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL,
+	0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL,
+	0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL,
+	0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL,
+	0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL,
+	0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL,
+	0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL,
+	0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL,
+	0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL,
+	0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL,
+	0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL,
+	0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL,
+	0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL,
+	0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL,
+	0x9e00009e9e9e9e00ULL,
 };
 
 const u64 camellia_sp22000222[256] = {
-	0xe0e0000000e0e0e0, 0x0505000000050505, 0x5858000000585858,
-	0xd9d9000000d9d9d9, 0x6767000000676767, 0x4e4e0000004e4e4e,
-	0x8181000000818181, 0xcbcb000000cbcbcb, 0xc9c9000000c9c9c9,
-	0x0b0b0000000b0b0b, 0xaeae000000aeaeae, 0x6a6a0000006a6a6a,
-	0xd5d5000000d5d5d5, 0x1818000000181818, 0x5d5d0000005d5d5d,
-	0x8282000000828282, 0x4646000000464646, 0xdfdf000000dfdfdf,
-	0xd6d6000000d6d6d6, 0x2727000000272727, 0x8a8a0000008a8a8a,
-	0x3232000000323232, 0x4b4b0000004b4b4b, 0x4242000000424242,
-	0xdbdb000000dbdbdb, 0x1c1c0000001c1c1c, 0x9e9e0000009e9e9e,
-	0x9c9c0000009c9c9c, 0x3a3a0000003a3a3a, 0xcaca000000cacaca,
-	0x2525000000252525, 0x7b7b0000007b7b7b, 0x0d0d0000000d0d0d,
-	0x7171000000717171, 0x5f5f0000005f5f5f, 0x1f1f0000001f1f1f,
-	0xf8f8000000f8f8f8, 0xd7d7000000d7d7d7, 0x3e3e0000003e3e3e,
-	0x9d9d0000009d9d9d, 0x7c7c0000007c7c7c, 0x6060000000606060,
-	0xb9b9000000b9b9b9, 0xbebe000000bebebe, 0xbcbc000000bcbcbc,
-	0x8b8b0000008b8b8b, 0x1616000000161616, 0x3434000000343434,
-	0x4d4d0000004d4d4d, 0xc3c3000000c3c3c3, 0x7272000000727272,
-	0x9595000000959595, 0xabab000000ababab, 0x8e8e0000008e8e8e,
-	0xbaba000000bababa, 0x7a7a0000007a7a7a, 0xb3b3000000b3b3b3,
-	0x0202000000020202, 0xb4b4000000b4b4b4, 0xadad000000adadad,
-	0xa2a2000000a2a2a2, 0xacac000000acacac, 0xd8d8000000d8d8d8,
-	0x9a9a0000009a9a9a, 0x1717000000171717, 0x1a1a0000001a1a1a,
-	0x3535000000353535, 0xcccc000000cccccc, 0xf7f7000000f7f7f7,
-	0x9999000000999999, 0x6161000000616161, 0x5a5a0000005a5a5a,
-	0xe8e8000000e8e8e8, 0x2424000000242424, 0x5656000000565656,
-	0x4040000000404040, 0xe1e1000000e1e1e1, 0x6363000000636363,
-	0x0909000000090909, 0x3333000000333333, 0xbfbf000000bfbfbf,
-	0x9898000000989898, 0x9797000000979797, 0x8585000000858585,
-	0x6868000000686868, 0xfcfc000000fcfcfc, 0xecec000000ececec,
-	0x0a0a0000000a0a0a, 0xdada000000dadada, 0x6f6f0000006f6f6f,
-	0x5353000000535353, 0x6262000000626262, 0xa3a3000000a3a3a3,
-	0x2e2e0000002e2e2e, 0x0808000000080808, 0xafaf000000afafaf,
-	0x2828000000282828, 0xb0b0000000b0b0b0, 0x7474000000747474,
-	0xc2c2000000c2c2c2, 0xbdbd000000bdbdbd, 0x3636000000363636,
-	0x2222000000222222, 0x3838000000383838, 0x6464000000646464,
-	0x1e1e0000001e1e1e, 0x3939000000393939, 0x2c2c0000002c2c2c,
-	0xa6a6000000a6a6a6, 0x3030000000303030, 0xe5e5000000e5e5e5,
-	0x4444000000444444, 0xfdfd000000fdfdfd, 0x8888000000888888,
-	0x9f9f0000009f9f9f, 0x6565000000656565, 0x8787000000878787,
-	0x6b6b0000006b6b6b, 0xf4f4000000f4f4f4, 0x2323000000232323,
-	0x4848000000484848, 0x1010000000101010, 0xd1d1000000d1d1d1,
-	0x5151000000515151, 0xc0c0000000c0c0c0, 0xf9f9000000f9f9f9,
-	0xd2d2000000d2d2d2, 0xa0a0000000a0a0a0, 0x5555000000555555,
-	0xa1a1000000a1a1a1, 0x4141000000414141, 0xfafa000000fafafa,
-	0x4343000000434343, 0x1313000000131313, 0xc4c4000000c4c4c4,
-	0x2f2f0000002f2f2f, 0xa8a8000000a8a8a8, 0xb6b6000000b6b6b6,
-	0x3c3c0000003c3c3c, 0x2b2b0000002b2b2b, 0xc1c1000000c1c1c1,
-	0xffff000000ffffff, 0xc8c8000000c8c8c8, 0xa5a5000000a5a5a5,
-	0x2020000000202020, 0x8989000000898989, 0x0000000000000000,
-	0x9090000000909090, 0x4747000000474747, 0xefef000000efefef,
-	0xeaea000000eaeaea, 0xb7b7000000b7b7b7, 0x1515000000151515,
-	0x0606000000060606, 0xcdcd000000cdcdcd, 0xb5b5000000b5b5b5,
-	0x1212000000121212, 0x7e7e0000007e7e7e, 0xbbbb000000bbbbbb,
-	0x2929000000292929, 0x0f0f0000000f0f0f, 0xb8b8000000b8b8b8,
-	0x0707000000070707, 0x0404000000040404, 0x9b9b0000009b9b9b,
-	0x9494000000949494, 0x2121000000212121, 0x6666000000666666,
-	0xe6e6000000e6e6e6, 0xcece000000cecece, 0xeded000000ededed,
-	0xe7e7000000e7e7e7, 0x3b3b0000003b3b3b, 0xfefe000000fefefe,
-	0x7f7f0000007f7f7f, 0xc5c5000000c5c5c5, 0xa4a4000000a4a4a4,
-	0x3737000000373737, 0xb1b1000000b1b1b1, 0x4c4c0000004c4c4c,
-	0x9191000000919191, 0x6e6e0000006e6e6e, 0x8d8d0000008d8d8d,
-	0x7676000000767676, 0x0303000000030303, 0x2d2d0000002d2d2d,
-	0xdede000000dedede, 0x9696000000969696, 0x2626000000262626,
-	0x7d7d0000007d7d7d, 0xc6c6000000c6c6c6, 0x5c5c0000005c5c5c,
-	0xd3d3000000d3d3d3, 0xf2f2000000f2f2f2, 0x4f4f0000004f4f4f,
-	0x1919000000191919, 0x3f3f0000003f3f3f, 0xdcdc000000dcdcdc,
-	0x7979000000797979, 0x1d1d0000001d1d1d, 0x5252000000525252,
-	0xebeb000000ebebeb, 0xf3f3000000f3f3f3, 0x6d6d0000006d6d6d,
-	0x5e5e0000005e5e5e, 0xfbfb000000fbfbfb, 0x6969000000696969,
-	0xb2b2000000b2b2b2, 0xf0f0000000f0f0f0, 0x3131000000313131,
-	0x0c0c0000000c0c0c, 0xd4d4000000d4d4d4, 0xcfcf000000cfcfcf,
-	0x8c8c0000008c8c8c, 0xe2e2000000e2e2e2, 0x7575000000757575,
-	0xa9a9000000a9a9a9, 0x4a4a0000004a4a4a, 0x5757000000575757,
-	0x8484000000848484, 0x1111000000111111, 0x4545000000454545,
-	0x1b1b0000001b1b1b, 0xf5f5000000f5f5f5, 0xe4e4000000e4e4e4,
-	0x0e0e0000000e0e0e, 0x7373000000737373, 0xaaaa000000aaaaaa,
-	0xf1f1000000f1f1f1, 0xdddd000000dddddd, 0x5959000000595959,
-	0x1414000000141414, 0x6c6c0000006c6c6c, 0x9292000000929292,
-	0x5454000000545454, 0xd0d0000000d0d0d0, 0x7878000000787878,
-	0x7070000000707070, 0xe3e3000000e3e3e3, 0x4949000000494949,
-	0x8080000000808080, 0x5050000000505050, 0xa7a7000000a7a7a7,
-	0xf6f6000000f6f6f6, 0x7777000000777777, 0x9393000000939393,
-	0x8686000000868686, 0x8383000000838383, 0x2a2a0000002a2a2a,
-	0xc7c7000000c7c7c7, 0x5b5b0000005b5b5b, 0xe9e9000000e9e9e9,
-	0xeeee000000eeeeee, 0x8f8f0000008f8f8f, 0x0101000000010101,
-	0x3d3d0000003d3d3d,
+	0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
+	0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
+	0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
+	0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL,
+	0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL,
+	0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL,
+	0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL,
+	0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL,
+	0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL,
+	0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL,
+	0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL,
+	0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL,
+	0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL,
+	0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL,
+	0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL,
+	0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL,
+	0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL,
+	0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL,
+	0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL,
+	0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL,
+	0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL,
+	0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL,
+	0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL,
+	0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL,
+	0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL,
+	0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL,
+	0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL,
+	0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL,
+	0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL,
+	0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL,
+	0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL,
+	0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL,
+	0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL,
+	0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL,
+	0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL,
+	0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL,
+	0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL,
+	0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL,
+	0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL,
+	0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL,
+	0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL,
+	0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL,
+	0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL,
+	0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL,
+	0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL,
+	0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL,
+	0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL,
+	0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL,
+	0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL,
+	0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL,
+	0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL,
+	0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL,
+	0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL,
+	0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL,
+	0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL,
+	0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL,
+	0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL,
+	0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL,
+	0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL,
+	0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL,
+	0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL,
+	0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL,
+	0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL,
+	0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL,
+	0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL,
+	0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL,
+	0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL,
+	0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL,
+	0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL,
+	0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL,
+	0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL,
+	0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL,
+	0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL,
+	0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL,
+	0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL,
+	0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL,
+	0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL,
+	0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL,
+	0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL,
+	0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL,
+	0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL,
+	0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL,
+	0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL,
+	0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL,
+	0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL,
+	0x3d3d0000003d3d3dULL,
 };
 
 const u64 camellia_sp03303033[256] = {
-	0x0038380038003838, 0x0041410041004141, 0x0016160016001616,
-	0x0076760076007676, 0x00d9d900d900d9d9, 0x0093930093009393,
-	0x0060600060006060, 0x00f2f200f200f2f2, 0x0072720072007272,
-	0x00c2c200c200c2c2, 0x00abab00ab00abab, 0x009a9a009a009a9a,
-	0x0075750075007575, 0x0006060006000606, 0x0057570057005757,
-	0x00a0a000a000a0a0, 0x0091910091009191, 0x00f7f700f700f7f7,
-	0x00b5b500b500b5b5, 0x00c9c900c900c9c9, 0x00a2a200a200a2a2,
-	0x008c8c008c008c8c, 0x00d2d200d200d2d2, 0x0090900090009090,
-	0x00f6f600f600f6f6, 0x0007070007000707, 0x00a7a700a700a7a7,
-	0x0027270027002727, 0x008e8e008e008e8e, 0x00b2b200b200b2b2,
-	0x0049490049004949, 0x00dede00de00dede, 0x0043430043004343,
-	0x005c5c005c005c5c, 0x00d7d700d700d7d7, 0x00c7c700c700c7c7,
-	0x003e3e003e003e3e, 0x00f5f500f500f5f5, 0x008f8f008f008f8f,
-	0x0067670067006767, 0x001f1f001f001f1f, 0x0018180018001818,
-	0x006e6e006e006e6e, 0x00afaf00af00afaf, 0x002f2f002f002f2f,
-	0x00e2e200e200e2e2, 0x0085850085008585, 0x000d0d000d000d0d,
-	0x0053530053005353, 0x00f0f000f000f0f0, 0x009c9c009c009c9c,
-	0x0065650065006565, 0x00eaea00ea00eaea, 0x00a3a300a300a3a3,
-	0x00aeae00ae00aeae, 0x009e9e009e009e9e, 0x00ecec00ec00ecec,
-	0x0080800080008080, 0x002d2d002d002d2d, 0x006b6b006b006b6b,
-	0x00a8a800a800a8a8, 0x002b2b002b002b2b, 0x0036360036003636,
-	0x00a6a600a600a6a6, 0x00c5c500c500c5c5, 0x0086860086008686,
-	0x004d4d004d004d4d, 0x0033330033003333, 0x00fdfd00fd00fdfd,
-	0x0066660066006666, 0x0058580058005858, 0x0096960096009696,
-	0x003a3a003a003a3a, 0x0009090009000909, 0x0095950095009595,
-	0x0010100010001010, 0x0078780078007878, 0x00d8d800d800d8d8,
-	0x0042420042004242, 0x00cccc00cc00cccc, 0x00efef00ef00efef,
-	0x0026260026002626, 0x00e5e500e500e5e5, 0x0061610061006161,
-	0x001a1a001a001a1a, 0x003f3f003f003f3f, 0x003b3b003b003b3b,
-	0x0082820082008282, 0x00b6b600b600b6b6, 0x00dbdb00db00dbdb,
-	0x00d4d400d400d4d4, 0x0098980098009898, 0x00e8e800e800e8e8,
-	0x008b8b008b008b8b, 0x0002020002000202, 0x00ebeb00eb00ebeb,
-	0x000a0a000a000a0a, 0x002c2c002c002c2c, 0x001d1d001d001d1d,
-	0x00b0b000b000b0b0, 0x006f6f006f006f6f, 0x008d8d008d008d8d,
-	0x0088880088008888, 0x000e0e000e000e0e, 0x0019190019001919,
-	0x0087870087008787, 0x004e4e004e004e4e, 0x000b0b000b000b0b,
-	0x00a9a900a900a9a9, 0x000c0c000c000c0c, 0x0079790079007979,
-	0x0011110011001111, 0x007f7f007f007f7f, 0x0022220022002222,
-	0x00e7e700e700e7e7, 0x0059590059005959, 0x00e1e100e100e1e1,
-	0x00dada00da00dada, 0x003d3d003d003d3d, 0x00c8c800c800c8c8,
-	0x0012120012001212, 0x0004040004000404, 0x0074740074007474,
-	0x0054540054005454, 0x0030300030003030, 0x007e7e007e007e7e,
-	0x00b4b400b400b4b4, 0x0028280028002828, 0x0055550055005555,
-	0x0068680068006868, 0x0050500050005050, 0x00bebe00be00bebe,
-	0x00d0d000d000d0d0, 0x00c4c400c400c4c4, 0x0031310031003131,
-	0x00cbcb00cb00cbcb, 0x002a2a002a002a2a, 0x00adad00ad00adad,
-	0x000f0f000f000f0f, 0x00caca00ca00caca, 0x0070700070007070,
-	0x00ffff00ff00ffff, 0x0032320032003232, 0x0069690069006969,
-	0x0008080008000808, 0x0062620062006262, 0x0000000000000000,
-	0x0024240024002424, 0x00d1d100d100d1d1, 0x00fbfb00fb00fbfb,
-	0x00baba00ba00baba, 0x00eded00ed00eded, 0x0045450045004545,
-	0x0081810081008181, 0x0073730073007373, 0x006d6d006d006d6d,
-	0x0084840084008484, 0x009f9f009f009f9f, 0x00eeee00ee00eeee,
-	0x004a4a004a004a4a, 0x00c3c300c300c3c3, 0x002e2e002e002e2e,
-	0x00c1c100c100c1c1, 0x0001010001000101, 0x00e6e600e600e6e6,
-	0x0025250025002525, 0x0048480048004848, 0x0099990099009999,
-	0x00b9b900b900b9b9, 0x00b3b300b300b3b3, 0x007b7b007b007b7b,
-	0x00f9f900f900f9f9, 0x00cece00ce00cece, 0x00bfbf00bf00bfbf,
-	0x00dfdf00df00dfdf, 0x0071710071007171, 0x0029290029002929,
-	0x00cdcd00cd00cdcd, 0x006c6c006c006c6c, 0x0013130013001313,
-	0x0064640064006464, 0x009b9b009b009b9b, 0x0063630063006363,
-	0x009d9d009d009d9d, 0x00c0c000c000c0c0, 0x004b4b004b004b4b,
-	0x00b7b700b700b7b7, 0x00a5a500a500a5a5, 0x0089890089008989,
-	0x005f5f005f005f5f, 0x00b1b100b100b1b1, 0x0017170017001717,
-	0x00f4f400f400f4f4, 0x00bcbc00bc00bcbc, 0x00d3d300d300d3d3,
-	0x0046460046004646, 0x00cfcf00cf00cfcf, 0x0037370037003737,
-	0x005e5e005e005e5e, 0x0047470047004747, 0x0094940094009494,
-	0x00fafa00fa00fafa, 0x00fcfc00fc00fcfc, 0x005b5b005b005b5b,
-	0x0097970097009797, 0x00fefe00fe00fefe, 0x005a5a005a005a5a,
-	0x00acac00ac00acac, 0x003c3c003c003c3c, 0x004c4c004c004c4c,
-	0x0003030003000303, 0x0035350035003535, 0x00f3f300f300f3f3,
-	0x0023230023002323, 0x00b8b800b800b8b8, 0x005d5d005d005d5d,
-	0x006a6a006a006a6a, 0x0092920092009292, 0x00d5d500d500d5d5,
-	0x0021210021002121, 0x0044440044004444, 0x0051510051005151,
-	0x00c6c600c600c6c6, 0x007d7d007d007d7d, 0x0039390039003939,
-	0x0083830083008383, 0x00dcdc00dc00dcdc, 0x00aaaa00aa00aaaa,
-	0x007c7c007c007c7c, 0x0077770077007777, 0x0056560056005656,
-	0x0005050005000505, 0x001b1b001b001b1b, 0x00a4a400a400a4a4,
-	0x0015150015001515, 0x0034340034003434, 0x001e1e001e001e1e,
-	0x001c1c001c001c1c, 0x00f8f800f800f8f8, 0x0052520052005252,
-	0x0020200020002020, 0x0014140014001414, 0x00e9e900e900e9e9,
-	0x00bdbd00bd00bdbd, 0x00dddd00dd00dddd, 0x00e4e400e400e4e4,
-	0x00a1a100a100a1a1, 0x00e0e000e000e0e0, 0x008a8a008a008a8a,
-	0x00f1f100f100f1f1, 0x00d6d600d600d6d6, 0x007a7a007a007a7a,
-	0x00bbbb00bb00bbbb, 0x00e3e300e300e3e3, 0x0040400040004040,
-	0x004f4f004f004f4f,
+	0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
+	0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
+	0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
+	0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL,
+	0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL,
+	0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL,
+	0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL,
+	0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL,
+	0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL,
+	0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL,
+	0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL,
+	0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL,
+	0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL,
+	0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL,
+	0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL,
+	0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL,
+	0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL,
+	0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL,
+	0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL,
+	0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL,
+	0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL,
+	0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL,
+	0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL,
+	0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL,
+	0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL,
+	0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL,
+	0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL,
+	0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL,
+	0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL,
+	0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL,
+	0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL,
+	0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL,
+	0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL,
+	0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL,
+	0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL,
+	0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL,
+	0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL,
+	0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL,
+	0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL,
+	0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL,
+	0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL,
+	0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL,
+	0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL,
+	0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL,
+	0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL,
+	0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL,
+	0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL,
+	0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL,
+	0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL,
+	0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL,
+	0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL,
+	0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL,
+	0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL,
+	0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL,
+	0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL,
+	0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL,
+	0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL,
+	0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL,
+	0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL,
+	0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL,
+	0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL,
+	0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL,
+	0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL,
+	0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL,
+	0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL,
+	0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL,
+	0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL,
+	0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL,
+	0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL,
+	0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL,
+	0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL,
+	0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL,
+	0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL,
+	0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL,
+	0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL,
+	0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL,
+	0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL,
+	0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL,
+	0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL,
+	0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL,
+	0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL,
+	0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL,
+	0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL,
+	0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL,
+	0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL,
+	0x004f4f004f004f4fULL,
 };
 
 const u64 camellia_sp00444404[256] = {
-	0x0000707070700070, 0x00002c2c2c2c002c, 0x0000b3b3b3b300b3,
-	0x0000c0c0c0c000c0, 0x0000e4e4e4e400e4, 0x0000575757570057,
-	0x0000eaeaeaea00ea, 0x0000aeaeaeae00ae, 0x0000232323230023,
-	0x00006b6b6b6b006b, 0x0000454545450045, 0x0000a5a5a5a500a5,
-	0x0000edededed00ed, 0x00004f4f4f4f004f, 0x00001d1d1d1d001d,
-	0x0000929292920092, 0x0000868686860086, 0x0000afafafaf00af,
-	0x00007c7c7c7c007c, 0x00001f1f1f1f001f, 0x00003e3e3e3e003e,
-	0x0000dcdcdcdc00dc, 0x00005e5e5e5e005e, 0x00000b0b0b0b000b,
-	0x0000a6a6a6a600a6, 0x0000393939390039, 0x0000d5d5d5d500d5,
-	0x00005d5d5d5d005d, 0x0000d9d9d9d900d9, 0x00005a5a5a5a005a,
-	0x0000515151510051, 0x00006c6c6c6c006c, 0x00008b8b8b8b008b,
-	0x00009a9a9a9a009a, 0x0000fbfbfbfb00fb, 0x0000b0b0b0b000b0,
-	0x0000747474740074, 0x00002b2b2b2b002b, 0x0000f0f0f0f000f0,
-	0x0000848484840084, 0x0000dfdfdfdf00df, 0x0000cbcbcbcb00cb,
-	0x0000343434340034, 0x0000767676760076, 0x00006d6d6d6d006d,
-	0x0000a9a9a9a900a9, 0x0000d1d1d1d100d1, 0x0000040404040004,
-	0x0000141414140014, 0x00003a3a3a3a003a, 0x0000dededede00de,
-	0x0000111111110011, 0x0000323232320032, 0x00009c9c9c9c009c,
-	0x0000535353530053, 0x0000f2f2f2f200f2, 0x0000fefefefe00fe,
-	0x0000cfcfcfcf00cf, 0x0000c3c3c3c300c3, 0x00007a7a7a7a007a,
-	0x0000242424240024, 0x0000e8e8e8e800e8, 0x0000606060600060,
-	0x0000696969690069, 0x0000aaaaaaaa00aa, 0x0000a0a0a0a000a0,
-	0x0000a1a1a1a100a1, 0x0000626262620062, 0x0000545454540054,
-	0x00001e1e1e1e001e, 0x0000e0e0e0e000e0, 0x0000646464640064,
-	0x0000101010100010, 0x0000000000000000, 0x0000a3a3a3a300a3,
-	0x0000757575750075, 0x00008a8a8a8a008a, 0x0000e6e6e6e600e6,
-	0x0000090909090009, 0x0000dddddddd00dd, 0x0000878787870087,
-	0x0000838383830083, 0x0000cdcdcdcd00cd, 0x0000909090900090,
-	0x0000737373730073, 0x0000f6f6f6f600f6, 0x00009d9d9d9d009d,
-	0x0000bfbfbfbf00bf, 0x0000525252520052, 0x0000d8d8d8d800d8,
-	0x0000c8c8c8c800c8, 0x0000c6c6c6c600c6, 0x0000818181810081,
-	0x00006f6f6f6f006f, 0x0000131313130013, 0x0000636363630063,
-	0x0000e9e9e9e900e9, 0x0000a7a7a7a700a7, 0x00009f9f9f9f009f,
-	0x0000bcbcbcbc00bc, 0x0000292929290029, 0x0000f9f9f9f900f9,
-	0x00002f2f2f2f002f, 0x0000b4b4b4b400b4, 0x0000787878780078,
-	0x0000060606060006, 0x0000e7e7e7e700e7, 0x0000717171710071,
-	0x0000d4d4d4d400d4, 0x0000abababab00ab, 0x0000888888880088,
-	0x00008d8d8d8d008d, 0x0000727272720072, 0x0000b9b9b9b900b9,
-	0x0000f8f8f8f800f8, 0x0000acacacac00ac, 0x0000363636360036,
-	0x00002a2a2a2a002a, 0x00003c3c3c3c003c, 0x0000f1f1f1f100f1,
-	0x0000404040400040, 0x0000d3d3d3d300d3, 0x0000bbbbbbbb00bb,
-	0x0000434343430043, 0x0000151515150015, 0x0000adadadad00ad,
-	0x0000777777770077, 0x0000808080800080, 0x0000828282820082,
-	0x0000ecececec00ec, 0x0000272727270027, 0x0000e5e5e5e500e5,
-	0x0000858585850085, 0x0000353535350035, 0x00000c0c0c0c000c,
-	0x0000414141410041, 0x0000efefefef00ef, 0x0000939393930093,
-	0x0000191919190019, 0x0000212121210021, 0x00000e0e0e0e000e,
-	0x00004e4e4e4e004e, 0x0000656565650065, 0x0000bdbdbdbd00bd,
-	0x0000b8b8b8b800b8, 0x00008f8f8f8f008f, 0x0000ebebebeb00eb,
-	0x0000cececece00ce, 0x0000303030300030, 0x00005f5f5f5f005f,
-	0x0000c5c5c5c500c5, 0x00001a1a1a1a001a, 0x0000e1e1e1e100e1,
-	0x0000cacacaca00ca, 0x0000474747470047, 0x00003d3d3d3d003d,
-	0x0000010101010001, 0x0000d6d6d6d600d6, 0x0000565656560056,
-	0x00004d4d4d4d004d, 0x00000d0d0d0d000d, 0x0000666666660066,
-	0x0000cccccccc00cc, 0x00002d2d2d2d002d, 0x0000121212120012,
-	0x0000202020200020, 0x0000b1b1b1b100b1, 0x0000999999990099,
-	0x00004c4c4c4c004c, 0x0000c2c2c2c200c2, 0x00007e7e7e7e007e,
-	0x0000050505050005, 0x0000b7b7b7b700b7, 0x0000313131310031,
-	0x0000171717170017, 0x0000d7d7d7d700d7, 0x0000585858580058,
-	0x0000616161610061, 0x00001b1b1b1b001b, 0x00001c1c1c1c001c,
-	0x00000f0f0f0f000f, 0x0000161616160016, 0x0000181818180018,
-	0x0000222222220022, 0x0000444444440044, 0x0000b2b2b2b200b2,
-	0x0000b5b5b5b500b5, 0x0000919191910091, 0x0000080808080008,
-	0x0000a8a8a8a800a8, 0x0000fcfcfcfc00fc, 0x0000505050500050,
-	0x0000d0d0d0d000d0, 0x00007d7d7d7d007d, 0x0000898989890089,
-	0x0000979797970097, 0x00005b5b5b5b005b, 0x0000959595950095,
-	0x0000ffffffff00ff, 0x0000d2d2d2d200d2, 0x0000c4c4c4c400c4,
-	0x0000484848480048, 0x0000f7f7f7f700f7, 0x0000dbdbdbdb00db,
-	0x0000030303030003, 0x0000dadadada00da, 0x00003f3f3f3f003f,
-	0x0000949494940094, 0x00005c5c5c5c005c, 0x0000020202020002,
-	0x00004a4a4a4a004a, 0x0000333333330033, 0x0000676767670067,
-	0x0000f3f3f3f300f3, 0x00007f7f7f7f007f, 0x0000e2e2e2e200e2,
-	0x00009b9b9b9b009b, 0x0000262626260026, 0x0000373737370037,
-	0x00003b3b3b3b003b, 0x0000969696960096, 0x00004b4b4b4b004b,
-	0x0000bebebebe00be, 0x00002e2e2e2e002e, 0x0000797979790079,
-	0x00008c8c8c8c008c, 0x00006e6e6e6e006e, 0x00008e8e8e8e008e,
-	0x0000f5f5f5f500f5, 0x0000b6b6b6b600b6, 0x0000fdfdfdfd00fd,
-	0x0000595959590059, 0x0000989898980098, 0x00006a6a6a6a006a,
-	0x0000464646460046, 0x0000babababa00ba, 0x0000252525250025,
-	0x0000424242420042, 0x0000a2a2a2a200a2, 0x0000fafafafa00fa,
-	0x0000070707070007, 0x0000555555550055, 0x0000eeeeeeee00ee,
-	0x00000a0a0a0a000a, 0x0000494949490049, 0x0000686868680068,
-	0x0000383838380038, 0x0000a4a4a4a400a4, 0x0000282828280028,
-	0x00007b7b7b7b007b, 0x0000c9c9c9c900c9, 0x0000c1c1c1c100c1,
-	0x0000e3e3e3e300e3, 0x0000f4f4f4f400f4, 0x0000c7c7c7c700c7,
-	0x00009e9e9e9e009e,
+	0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
+	0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
+	0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
+	0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL,
+	0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL,
+	0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL,
+	0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL,
+	0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL,
+	0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL,
+	0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL,
+	0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL,
+	0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL,
+	0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL,
+	0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL,
+	0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL,
+	0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL,
+	0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL,
+	0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL,
+	0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL,
+	0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL,
+	0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL,
+	0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL,
+	0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL,
+	0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL,
+	0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL,
+	0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL,
+	0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL,
+	0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL,
+	0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL,
+	0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL,
+	0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL,
+	0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL,
+	0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL,
+	0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL,
+	0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL,
+	0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL,
+	0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL,
+	0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL,
+	0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL,
+	0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL,
+	0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL,
+	0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL,
+	0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL,
+	0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL,
+	0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL,
+	0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL,
+	0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL,
+	0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL,
+	0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL,
+	0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL,
+	0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL,
+	0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL,
+	0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL,
+	0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL,
+	0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL,
+	0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL,
+	0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL,
+	0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL,
+	0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL,
+	0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL,
+	0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL,
+	0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL,
+	0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL,
+	0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL,
+	0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL,
+	0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL,
+	0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL,
+	0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL,
+	0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL,
+	0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL,
+	0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL,
+	0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL,
+	0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL,
+	0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL,
+	0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL,
+	0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL,
+	0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL,
+	0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL,
+	0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL,
+	0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL,
+	0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL,
+	0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL,
+	0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL,
+	0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL,
+	0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL,
+	0x00009e9e9e9e009eULL,
 };
 
 const u64 camellia_sp02220222[256] = {
-	0x00e0e0e000e0e0e0, 0x0005050500050505, 0x0058585800585858,
-	0x00d9d9d900d9d9d9, 0x0067676700676767, 0x004e4e4e004e4e4e,
-	0x0081818100818181, 0x00cbcbcb00cbcbcb, 0x00c9c9c900c9c9c9,
-	0x000b0b0b000b0b0b, 0x00aeaeae00aeaeae, 0x006a6a6a006a6a6a,
-	0x00d5d5d500d5d5d5, 0x0018181800181818, 0x005d5d5d005d5d5d,
-	0x0082828200828282, 0x0046464600464646, 0x00dfdfdf00dfdfdf,
-	0x00d6d6d600d6d6d6, 0x0027272700272727, 0x008a8a8a008a8a8a,
-	0x0032323200323232, 0x004b4b4b004b4b4b, 0x0042424200424242,
-	0x00dbdbdb00dbdbdb, 0x001c1c1c001c1c1c, 0x009e9e9e009e9e9e,
-	0x009c9c9c009c9c9c, 0x003a3a3a003a3a3a, 0x00cacaca00cacaca,
-	0x0025252500252525, 0x007b7b7b007b7b7b, 0x000d0d0d000d0d0d,
-	0x0071717100717171, 0x005f5f5f005f5f5f, 0x001f1f1f001f1f1f,
-	0x00f8f8f800f8f8f8, 0x00d7d7d700d7d7d7, 0x003e3e3e003e3e3e,
-	0x009d9d9d009d9d9d, 0x007c7c7c007c7c7c, 0x0060606000606060,
-	0x00b9b9b900b9b9b9, 0x00bebebe00bebebe, 0x00bcbcbc00bcbcbc,
-	0x008b8b8b008b8b8b, 0x0016161600161616, 0x0034343400343434,
-	0x004d4d4d004d4d4d, 0x00c3c3c300c3c3c3, 0x0072727200727272,
-	0x0095959500959595, 0x00ababab00ababab, 0x008e8e8e008e8e8e,
-	0x00bababa00bababa, 0x007a7a7a007a7a7a, 0x00b3b3b300b3b3b3,
-	0x0002020200020202, 0x00b4b4b400b4b4b4, 0x00adadad00adadad,
-	0x00a2a2a200a2a2a2, 0x00acacac00acacac, 0x00d8d8d800d8d8d8,
-	0x009a9a9a009a9a9a, 0x0017171700171717, 0x001a1a1a001a1a1a,
-	0x0035353500353535, 0x00cccccc00cccccc, 0x00f7f7f700f7f7f7,
-	0x0099999900999999, 0x0061616100616161, 0x005a5a5a005a5a5a,
-	0x00e8e8e800e8e8e8, 0x0024242400242424, 0x0056565600565656,
-	0x0040404000404040, 0x00e1e1e100e1e1e1, 0x0063636300636363,
-	0x0009090900090909, 0x0033333300333333, 0x00bfbfbf00bfbfbf,
-	0x0098989800989898, 0x0097979700979797, 0x0085858500858585,
-	0x0068686800686868, 0x00fcfcfc00fcfcfc, 0x00ececec00ececec,
-	0x000a0a0a000a0a0a, 0x00dadada00dadada, 0x006f6f6f006f6f6f,
-	0x0053535300535353, 0x0062626200626262, 0x00a3a3a300a3a3a3,
-	0x002e2e2e002e2e2e, 0x0008080800080808, 0x00afafaf00afafaf,
-	0x0028282800282828, 0x00b0b0b000b0b0b0, 0x0074747400747474,
-	0x00c2c2c200c2c2c2, 0x00bdbdbd00bdbdbd, 0x0036363600363636,
-	0x0022222200222222, 0x0038383800383838, 0x0064646400646464,
-	0x001e1e1e001e1e1e, 0x0039393900393939, 0x002c2c2c002c2c2c,
-	0x00a6a6a600a6a6a6, 0x0030303000303030, 0x00e5e5e500e5e5e5,
-	0x0044444400444444, 0x00fdfdfd00fdfdfd, 0x0088888800888888,
-	0x009f9f9f009f9f9f, 0x0065656500656565, 0x0087878700878787,
-	0x006b6b6b006b6b6b, 0x00f4f4f400f4f4f4, 0x0023232300232323,
-	0x0048484800484848, 0x0010101000101010, 0x00d1d1d100d1d1d1,
-	0x0051515100515151, 0x00c0c0c000c0c0c0, 0x00f9f9f900f9f9f9,
-	0x00d2d2d200d2d2d2, 0x00a0a0a000a0a0a0, 0x0055555500555555,
-	0x00a1a1a100a1a1a1, 0x0041414100414141, 0x00fafafa00fafafa,
-	0x0043434300434343, 0x0013131300131313, 0x00c4c4c400c4c4c4,
-	0x002f2f2f002f2f2f, 0x00a8a8a800a8a8a8, 0x00b6b6b600b6b6b6,
-	0x003c3c3c003c3c3c, 0x002b2b2b002b2b2b, 0x00c1c1c100c1c1c1,
-	0x00ffffff00ffffff, 0x00c8c8c800c8c8c8, 0x00a5a5a500a5a5a5,
-	0x0020202000202020, 0x0089898900898989, 0x0000000000000000,
-	0x0090909000909090, 0x0047474700474747, 0x00efefef00efefef,
-	0x00eaeaea00eaeaea, 0x00b7b7b700b7b7b7, 0x0015151500151515,
-	0x0006060600060606, 0x00cdcdcd00cdcdcd, 0x00b5b5b500b5b5b5,
-	0x0012121200121212, 0x007e7e7e007e7e7e, 0x00bbbbbb00bbbbbb,
-	0x0029292900292929, 0x000f0f0f000f0f0f, 0x00b8b8b800b8b8b8,
-	0x0007070700070707, 0x0004040400040404, 0x009b9b9b009b9b9b,
-	0x0094949400949494, 0x0021212100212121, 0x0066666600666666,
-	0x00e6e6e600e6e6e6, 0x00cecece00cecece, 0x00ededed00ededed,
-	0x00e7e7e700e7e7e7, 0x003b3b3b003b3b3b, 0x00fefefe00fefefe,
-	0x007f7f7f007f7f7f, 0x00c5c5c500c5c5c5, 0x00a4a4a400a4a4a4,
-	0x0037373700373737, 0x00b1b1b100b1b1b1, 0x004c4c4c004c4c4c,
-	0x0091919100919191, 0x006e6e6e006e6e6e, 0x008d8d8d008d8d8d,
-	0x0076767600767676, 0x0003030300030303, 0x002d2d2d002d2d2d,
-	0x00dedede00dedede, 0x0096969600969696, 0x0026262600262626,
-	0x007d7d7d007d7d7d, 0x00c6c6c600c6c6c6, 0x005c5c5c005c5c5c,
-	0x00d3d3d300d3d3d3, 0x00f2f2f200f2f2f2, 0x004f4f4f004f4f4f,
-	0x0019191900191919, 0x003f3f3f003f3f3f, 0x00dcdcdc00dcdcdc,
-	0x0079797900797979, 0x001d1d1d001d1d1d, 0x0052525200525252,
-	0x00ebebeb00ebebeb, 0x00f3f3f300f3f3f3, 0x006d6d6d006d6d6d,
-	0x005e5e5e005e5e5e, 0x00fbfbfb00fbfbfb, 0x0069696900696969,
-	0x00b2b2b200b2b2b2, 0x00f0f0f000f0f0f0, 0x0031313100313131,
-	0x000c0c0c000c0c0c, 0x00d4d4d400d4d4d4, 0x00cfcfcf00cfcfcf,
-	0x008c8c8c008c8c8c, 0x00e2e2e200e2e2e2, 0x0075757500757575,
-	0x00a9a9a900a9a9a9, 0x004a4a4a004a4a4a, 0x0057575700575757,
-	0x0084848400848484, 0x0011111100111111, 0x0045454500454545,
-	0x001b1b1b001b1b1b, 0x00f5f5f500f5f5f5, 0x00e4e4e400e4e4e4,
-	0x000e0e0e000e0e0e, 0x0073737300737373, 0x00aaaaaa00aaaaaa,
-	0x00f1f1f100f1f1f1, 0x00dddddd00dddddd, 0x0059595900595959,
-	0x0014141400141414, 0x006c6c6c006c6c6c, 0x0092929200929292,
-	0x0054545400545454, 0x00d0d0d000d0d0d0, 0x0078787800787878,
-	0x0070707000707070, 0x00e3e3e300e3e3e3, 0x0049494900494949,
-	0x0080808000808080, 0x0050505000505050, 0x00a7a7a700a7a7a7,
-	0x00f6f6f600f6f6f6, 0x0077777700777777, 0x0093939300939393,
-	0x0086868600868686, 0x0083838300838383, 0x002a2a2a002a2a2a,
-	0x00c7c7c700c7c7c7, 0x005b5b5b005b5b5b, 0x00e9e9e900e9e9e9,
-	0x00eeeeee00eeeeee, 0x008f8f8f008f8f8f, 0x0001010100010101,
-	0x003d3d3d003d3d3d,
+	0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
+	0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
+	0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
+	0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL,
+	0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL,
+	0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL,
+	0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL,
+	0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL,
+	0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL,
+	0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL,
+	0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL,
+	0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL,
+	0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL,
+	0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL,
+	0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL,
+	0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL,
+	0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL,
+	0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL,
+	0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL,
+	0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL,
+	0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL,
+	0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL,
+	0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL,
+	0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL,
+	0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL,
+	0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL,
+	0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL,
+	0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL,
+	0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL,
+	0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL,
+	0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL,
+	0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL,
+	0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL,
+	0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL,
+	0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL,
+	0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL,
+	0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL,
+	0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL,
+	0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL,
+	0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL,
+	0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL,
+	0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL,
+	0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL,
+	0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL,
+	0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL,
+	0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL,
+	0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL,
+	0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL,
+	0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL,
+	0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL,
+	0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL,
+	0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL,
+	0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL,
+	0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL,
+	0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL,
+	0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL,
+	0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL,
+	0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL,
+	0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL,
+	0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL,
+	0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL,
+	0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL,
+	0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL,
+	0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL,
+	0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL,
+	0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL,
+	0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL,
+	0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL,
+	0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL,
+	0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL,
+	0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL,
+	0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL,
+	0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL,
+	0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL,
+	0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL,
+	0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL,
+	0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL,
+	0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL,
+	0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL,
+	0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL,
+	0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL,
+	0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL,
+	0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL,
+	0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL,
+	0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL,
+	0x003d3d3d003d3d3dULL,
 };
 
 const u64 camellia_sp30333033[256] = {
-	0x3800383838003838, 0x4100414141004141, 0x1600161616001616,
-	0x7600767676007676, 0xd900d9d9d900d9d9, 0x9300939393009393,
-	0x6000606060006060, 0xf200f2f2f200f2f2, 0x7200727272007272,
-	0xc200c2c2c200c2c2, 0xab00ababab00abab, 0x9a009a9a9a009a9a,
-	0x7500757575007575, 0x0600060606000606, 0x5700575757005757,
-	0xa000a0a0a000a0a0, 0x9100919191009191, 0xf700f7f7f700f7f7,
-	0xb500b5b5b500b5b5, 0xc900c9c9c900c9c9, 0xa200a2a2a200a2a2,
-	0x8c008c8c8c008c8c, 0xd200d2d2d200d2d2, 0x9000909090009090,
-	0xf600f6f6f600f6f6, 0x0700070707000707, 0xa700a7a7a700a7a7,
-	0x2700272727002727, 0x8e008e8e8e008e8e, 0xb200b2b2b200b2b2,
-	0x4900494949004949, 0xde00dedede00dede, 0x4300434343004343,
-	0x5c005c5c5c005c5c, 0xd700d7d7d700d7d7, 0xc700c7c7c700c7c7,
-	0x3e003e3e3e003e3e, 0xf500f5f5f500f5f5, 0x8f008f8f8f008f8f,
-	0x6700676767006767, 0x1f001f1f1f001f1f, 0x1800181818001818,
-	0x6e006e6e6e006e6e, 0xaf00afafaf00afaf, 0x2f002f2f2f002f2f,
-	0xe200e2e2e200e2e2, 0x8500858585008585, 0x0d000d0d0d000d0d,
-	0x5300535353005353, 0xf000f0f0f000f0f0, 0x9c009c9c9c009c9c,
-	0x6500656565006565, 0xea00eaeaea00eaea, 0xa300a3a3a300a3a3,
-	0xae00aeaeae00aeae, 0x9e009e9e9e009e9e, 0xec00ececec00ecec,
-	0x8000808080008080, 0x2d002d2d2d002d2d, 0x6b006b6b6b006b6b,
-	0xa800a8a8a800a8a8, 0x2b002b2b2b002b2b, 0x3600363636003636,
-	0xa600a6a6a600a6a6, 0xc500c5c5c500c5c5, 0x8600868686008686,
-	0x4d004d4d4d004d4d, 0x3300333333003333, 0xfd00fdfdfd00fdfd,
-	0x6600666666006666, 0x5800585858005858, 0x9600969696009696,
-	0x3a003a3a3a003a3a, 0x0900090909000909, 0x9500959595009595,
-	0x1000101010001010, 0x7800787878007878, 0xd800d8d8d800d8d8,
-	0x4200424242004242, 0xcc00cccccc00cccc, 0xef00efefef00efef,
-	0x2600262626002626, 0xe500e5e5e500e5e5, 0x6100616161006161,
-	0x1a001a1a1a001a1a, 0x3f003f3f3f003f3f, 0x3b003b3b3b003b3b,
-	0x8200828282008282, 0xb600b6b6b600b6b6, 0xdb00dbdbdb00dbdb,
-	0xd400d4d4d400d4d4, 0x9800989898009898, 0xe800e8e8e800e8e8,
-	0x8b008b8b8b008b8b, 0x0200020202000202, 0xeb00ebebeb00ebeb,
-	0x0a000a0a0a000a0a, 0x2c002c2c2c002c2c, 0x1d001d1d1d001d1d,
-	0xb000b0b0b000b0b0, 0x6f006f6f6f006f6f, 0x8d008d8d8d008d8d,
-	0x8800888888008888, 0x0e000e0e0e000e0e, 0x1900191919001919,
-	0x8700878787008787, 0x4e004e4e4e004e4e, 0x0b000b0b0b000b0b,
-	0xa900a9a9a900a9a9, 0x0c000c0c0c000c0c, 0x7900797979007979,
-	0x1100111111001111, 0x7f007f7f7f007f7f, 0x2200222222002222,
-	0xe700e7e7e700e7e7, 0x5900595959005959, 0xe100e1e1e100e1e1,
-	0xda00dadada00dada, 0x3d003d3d3d003d3d, 0xc800c8c8c800c8c8,
-	0x1200121212001212, 0x0400040404000404, 0x7400747474007474,
-	0x5400545454005454, 0x3000303030003030, 0x7e007e7e7e007e7e,
-	0xb400b4b4b400b4b4, 0x2800282828002828, 0x5500555555005555,
-	0x6800686868006868, 0x5000505050005050, 0xbe00bebebe00bebe,
-	0xd000d0d0d000d0d0, 0xc400c4c4c400c4c4, 0x3100313131003131,
-	0xcb00cbcbcb00cbcb, 0x2a002a2a2a002a2a, 0xad00adadad00adad,
-	0x0f000f0f0f000f0f, 0xca00cacaca00caca, 0x7000707070007070,
-	0xff00ffffff00ffff, 0x3200323232003232, 0x6900696969006969,
-	0x0800080808000808, 0x6200626262006262, 0x0000000000000000,
-	0x2400242424002424, 0xd100d1d1d100d1d1, 0xfb00fbfbfb00fbfb,
-	0xba00bababa00baba, 0xed00ededed00eded, 0x4500454545004545,
-	0x8100818181008181, 0x7300737373007373, 0x6d006d6d6d006d6d,
-	0x8400848484008484, 0x9f009f9f9f009f9f, 0xee00eeeeee00eeee,
-	0x4a004a4a4a004a4a, 0xc300c3c3c300c3c3, 0x2e002e2e2e002e2e,
-	0xc100c1c1c100c1c1, 0x0100010101000101, 0xe600e6e6e600e6e6,
-	0x2500252525002525, 0x4800484848004848, 0x9900999999009999,
-	0xb900b9b9b900b9b9, 0xb300b3b3b300b3b3, 0x7b007b7b7b007b7b,
-	0xf900f9f9f900f9f9, 0xce00cecece00cece, 0xbf00bfbfbf00bfbf,
-	0xdf00dfdfdf00dfdf, 0x7100717171007171, 0x2900292929002929,
-	0xcd00cdcdcd00cdcd, 0x6c006c6c6c006c6c, 0x1300131313001313,
-	0x6400646464006464, 0x9b009b9b9b009b9b, 0x6300636363006363,
-	0x9d009d9d9d009d9d, 0xc000c0c0c000c0c0, 0x4b004b4b4b004b4b,
-	0xb700b7b7b700b7b7, 0xa500a5a5a500a5a5, 0x8900898989008989,
-	0x5f005f5f5f005f5f, 0xb100b1b1b100b1b1, 0x1700171717001717,
-	0xf400f4f4f400f4f4, 0xbc00bcbcbc00bcbc, 0xd300d3d3d300d3d3,
-	0x4600464646004646, 0xcf00cfcfcf00cfcf, 0x3700373737003737,
-	0x5e005e5e5e005e5e, 0x4700474747004747, 0x9400949494009494,
-	0xfa00fafafa00fafa, 0xfc00fcfcfc00fcfc, 0x5b005b5b5b005b5b,
-	0x9700979797009797, 0xfe00fefefe00fefe, 0x5a005a5a5a005a5a,
-	0xac00acacac00acac, 0x3c003c3c3c003c3c, 0x4c004c4c4c004c4c,
-	0x0300030303000303, 0x3500353535003535, 0xf300f3f3f300f3f3,
-	0x2300232323002323, 0xb800b8b8b800b8b8, 0x5d005d5d5d005d5d,
-	0x6a006a6a6a006a6a, 0x9200929292009292, 0xd500d5d5d500d5d5,
-	0x2100212121002121, 0x4400444444004444, 0x5100515151005151,
-	0xc600c6c6c600c6c6, 0x7d007d7d7d007d7d, 0x3900393939003939,
-	0x8300838383008383, 0xdc00dcdcdc00dcdc, 0xaa00aaaaaa00aaaa,
-	0x7c007c7c7c007c7c, 0x7700777777007777, 0x5600565656005656,
-	0x0500050505000505, 0x1b001b1b1b001b1b, 0xa400a4a4a400a4a4,
-	0x1500151515001515, 0x3400343434003434, 0x1e001e1e1e001e1e,
-	0x1c001c1c1c001c1c, 0xf800f8f8f800f8f8, 0x5200525252005252,
-	0x2000202020002020, 0x1400141414001414, 0xe900e9e9e900e9e9,
-	0xbd00bdbdbd00bdbd, 0xdd00dddddd00dddd, 0xe400e4e4e400e4e4,
-	0xa100a1a1a100a1a1, 0xe000e0e0e000e0e0, 0x8a008a8a8a008a8a,
-	0xf100f1f1f100f1f1, 0xd600d6d6d600d6d6, 0x7a007a7a7a007a7a,
-	0xbb00bbbbbb00bbbb, 0xe300e3e3e300e3e3, 0x4000404040004040,
-	0x4f004f4f4f004f4f,
+	0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
+	0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
+	0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
+	0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL,
+	0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL,
+	0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL,
+	0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL,
+	0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL,
+	0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL,
+	0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL,
+	0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL,
+	0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL,
+	0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL,
+	0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL,
+	0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL,
+	0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL,
+	0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL,
+	0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL,
+	0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL,
+	0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL,
+	0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL,
+	0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL,
+	0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL,
+	0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL,
+	0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL,
+	0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL,
+	0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL,
+	0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL,
+	0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL,
+	0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL,
+	0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL,
+	0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL,
+	0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL,
+	0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL,
+	0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL,
+	0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL,
+	0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL,
+	0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL,
+	0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL,
+	0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL,
+	0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL,
+	0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL,
+	0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL,
+	0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL,
+	0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL,
+	0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL,
+	0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL,
+	0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL,
+	0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL,
+	0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL,
+	0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL,
+	0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL,
+	0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL,
+	0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL,
+	0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL,
+	0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL,
+	0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL,
+	0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL,
+	0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL,
+	0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL,
+	0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL,
+	0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL,
+	0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL,
+	0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL,
+	0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL,
+	0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL,
+	0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL,
+	0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL,
+	0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL,
+	0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL,
+	0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL,
+	0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL,
+	0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL,
+	0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL,
+	0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL,
+	0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL,
+	0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL,
+	0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL,
+	0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL,
+	0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL,
+	0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL,
+	0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL,
+	0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL,
+	0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL,
+	0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL,
+	0x4f004f4f4f004f4fULL,
 };
 
 const u64 camellia_sp44044404[256] = {
-	0x7070007070700070, 0x2c2c002c2c2c002c, 0xb3b300b3b3b300b3,
-	0xc0c000c0c0c000c0, 0xe4e400e4e4e400e4, 0x5757005757570057,
-	0xeaea00eaeaea00ea, 0xaeae00aeaeae00ae, 0x2323002323230023,
-	0x6b6b006b6b6b006b, 0x4545004545450045, 0xa5a500a5a5a500a5,
-	0xeded00ededed00ed, 0x4f4f004f4f4f004f, 0x1d1d001d1d1d001d,
-	0x9292009292920092, 0x8686008686860086, 0xafaf00afafaf00af,
-	0x7c7c007c7c7c007c, 0x1f1f001f1f1f001f, 0x3e3e003e3e3e003e,
-	0xdcdc00dcdcdc00dc, 0x5e5e005e5e5e005e, 0x0b0b000b0b0b000b,
-	0xa6a600a6a6a600a6, 0x3939003939390039, 0xd5d500d5d5d500d5,
-	0x5d5d005d5d5d005d, 0xd9d900d9d9d900d9, 0x5a5a005a5a5a005a,
-	0x5151005151510051, 0x6c6c006c6c6c006c, 0x8b8b008b8b8b008b,
-	0x9a9a009a9a9a009a, 0xfbfb00fbfbfb00fb, 0xb0b000b0b0b000b0,
-	0x7474007474740074, 0x2b2b002b2b2b002b, 0xf0f000f0f0f000f0,
-	0x8484008484840084, 0xdfdf00dfdfdf00df, 0xcbcb00cbcbcb00cb,
-	0x3434003434340034, 0x7676007676760076, 0x6d6d006d6d6d006d,
-	0xa9a900a9a9a900a9, 0xd1d100d1d1d100d1, 0x0404000404040004,
-	0x1414001414140014, 0x3a3a003a3a3a003a, 0xdede00dedede00de,
-	0x1111001111110011, 0x3232003232320032, 0x9c9c009c9c9c009c,
-	0x5353005353530053, 0xf2f200f2f2f200f2, 0xfefe00fefefe00fe,
-	0xcfcf00cfcfcf00cf, 0xc3c300c3c3c300c3, 0x7a7a007a7a7a007a,
-	0x2424002424240024, 0xe8e800e8e8e800e8, 0x6060006060600060,
-	0x6969006969690069, 0xaaaa00aaaaaa00aa, 0xa0a000a0a0a000a0,
-	0xa1a100a1a1a100a1, 0x6262006262620062, 0x5454005454540054,
-	0x1e1e001e1e1e001e, 0xe0e000e0e0e000e0, 0x6464006464640064,
-	0x1010001010100010, 0x0000000000000000, 0xa3a300a3a3a300a3,
-	0x7575007575750075, 0x8a8a008a8a8a008a, 0xe6e600e6e6e600e6,
-	0x0909000909090009, 0xdddd00dddddd00dd, 0x8787008787870087,
-	0x8383008383830083, 0xcdcd00cdcdcd00cd, 0x9090009090900090,
-	0x7373007373730073, 0xf6f600f6f6f600f6, 0x9d9d009d9d9d009d,
-	0xbfbf00bfbfbf00bf, 0x5252005252520052, 0xd8d800d8d8d800d8,
-	0xc8c800c8c8c800c8, 0xc6c600c6c6c600c6, 0x8181008181810081,
-	0x6f6f006f6f6f006f, 0x1313001313130013, 0x6363006363630063,
-	0xe9e900e9e9e900e9, 0xa7a700a7a7a700a7, 0x9f9f009f9f9f009f,
-	0xbcbc00bcbcbc00bc, 0x2929002929290029, 0xf9f900f9f9f900f9,
-	0x2f2f002f2f2f002f, 0xb4b400b4b4b400b4, 0x7878007878780078,
-	0x0606000606060006, 0xe7e700e7e7e700e7, 0x7171007171710071,
-	0xd4d400d4d4d400d4, 0xabab00ababab00ab, 0x8888008888880088,
-	0x8d8d008d8d8d008d, 0x7272007272720072, 0xb9b900b9b9b900b9,
-	0xf8f800f8f8f800f8, 0xacac00acacac00ac, 0x3636003636360036,
-	0x2a2a002a2a2a002a, 0x3c3c003c3c3c003c, 0xf1f100f1f1f100f1,
-	0x4040004040400040, 0xd3d300d3d3d300d3, 0xbbbb00bbbbbb00bb,
-	0x4343004343430043, 0x1515001515150015, 0xadad00adadad00ad,
-	0x7777007777770077, 0x8080008080800080, 0x8282008282820082,
-	0xecec00ececec00ec, 0x2727002727270027, 0xe5e500e5e5e500e5,
-	0x8585008585850085, 0x3535003535350035, 0x0c0c000c0c0c000c,
-	0x4141004141410041, 0xefef00efefef00ef, 0x9393009393930093,
-	0x1919001919190019, 0x2121002121210021, 0x0e0e000e0e0e000e,
-	0x4e4e004e4e4e004e, 0x6565006565650065, 0xbdbd00bdbdbd00bd,
-	0xb8b800b8b8b800b8, 0x8f8f008f8f8f008f, 0xebeb00ebebeb00eb,
-	0xcece00cecece00ce, 0x3030003030300030, 0x5f5f005f5f5f005f,
-	0xc5c500c5c5c500c5, 0x1a1a001a1a1a001a, 0xe1e100e1e1e100e1,
-	0xcaca00cacaca00ca, 0x4747004747470047, 0x3d3d003d3d3d003d,
-	0x0101000101010001, 0xd6d600d6d6d600d6, 0x5656005656560056,
-	0x4d4d004d4d4d004d, 0x0d0d000d0d0d000d, 0x6666006666660066,
-	0xcccc00cccccc00cc, 0x2d2d002d2d2d002d, 0x1212001212120012,
-	0x2020002020200020, 0xb1b100b1b1b100b1, 0x9999009999990099,
-	0x4c4c004c4c4c004c, 0xc2c200c2c2c200c2, 0x7e7e007e7e7e007e,
-	0x0505000505050005, 0xb7b700b7b7b700b7, 0x3131003131310031,
-	0x1717001717170017, 0xd7d700d7d7d700d7, 0x5858005858580058,
-	0x6161006161610061, 0x1b1b001b1b1b001b, 0x1c1c001c1c1c001c,
-	0x0f0f000f0f0f000f, 0x1616001616160016, 0x1818001818180018,
-	0x2222002222220022, 0x4444004444440044, 0xb2b200b2b2b200b2,
-	0xb5b500b5b5b500b5, 0x9191009191910091, 0x0808000808080008,
-	0xa8a800a8a8a800a8, 0xfcfc00fcfcfc00fc, 0x5050005050500050,
-	0xd0d000d0d0d000d0, 0x7d7d007d7d7d007d, 0x8989008989890089,
-	0x9797009797970097, 0x5b5b005b5b5b005b, 0x9595009595950095,
-	0xffff00ffffff00ff, 0xd2d200d2d2d200d2, 0xc4c400c4c4c400c4,
-	0x4848004848480048, 0xf7f700f7f7f700f7, 0xdbdb00dbdbdb00db,
-	0x0303000303030003, 0xdada00dadada00da, 0x3f3f003f3f3f003f,
-	0x9494009494940094, 0x5c5c005c5c5c005c, 0x0202000202020002,
-	0x4a4a004a4a4a004a, 0x3333003333330033, 0x6767006767670067,
-	0xf3f300f3f3f300f3, 0x7f7f007f7f7f007f, 0xe2e200e2e2e200e2,
-	0x9b9b009b9b9b009b, 0x2626002626260026, 0x3737003737370037,
-	0x3b3b003b3b3b003b, 0x9696009696960096, 0x4b4b004b4b4b004b,
-	0xbebe00bebebe00be, 0x2e2e002e2e2e002e, 0x7979007979790079,
-	0x8c8c008c8c8c008c, 0x6e6e006e6e6e006e, 0x8e8e008e8e8e008e,
-	0xf5f500f5f5f500f5, 0xb6b600b6b6b600b6, 0xfdfd00fdfdfd00fd,
-	0x5959005959590059, 0x9898009898980098, 0x6a6a006a6a6a006a,
-	0x4646004646460046, 0xbaba00bababa00ba, 0x2525002525250025,
-	0x4242004242420042, 0xa2a200a2a2a200a2, 0xfafa00fafafa00fa,
-	0x0707000707070007, 0x5555005555550055, 0xeeee00eeeeee00ee,
-	0x0a0a000a0a0a000a, 0x4949004949490049, 0x6868006868680068,
-	0x3838003838380038, 0xa4a400a4a4a400a4, 0x2828002828280028,
-	0x7b7b007b7b7b007b, 0xc9c900c9c9c900c9, 0xc1c100c1c1c100c1,
-	0xe3e300e3e3e300e3, 0xf4f400f4f4f400f4, 0xc7c700c7c7c700c7,
-	0x9e9e009e9e9e009e,
+	0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
+	0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
+	0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
+	0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL,
+	0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL,
+	0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL,
+	0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL,
+	0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL,
+	0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL,
+	0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL,
+	0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL,
+	0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL,
+	0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL,
+	0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL,
+	0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL,
+	0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL,
+	0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL,
+	0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL,
+	0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL,
+	0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL,
+	0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL,
+	0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL,
+	0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL,
+	0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL,
+	0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL,
+	0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL,
+	0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL,
+	0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL,
+	0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL,
+	0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL,
+	0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL,
+	0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL,
+	0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL,
+	0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL,
+	0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL,
+	0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL,
+	0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL,
+	0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL,
+	0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL,
+	0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL,
+	0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL,
+	0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL,
+	0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL,
+	0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL,
+	0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL,
+	0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL,
+	0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL,
+	0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL,
+	0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL,
+	0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL,
+	0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL,
+	0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL,
+	0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL,
+	0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL,
+	0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL,
+	0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL,
+	0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL,
+	0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL,
+	0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL,
+	0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL,
+	0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL,
+	0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL,
+	0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL,
+	0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL,
+	0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL,
+	0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL,
+	0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL,
+	0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL,
+	0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL,
+	0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL,
+	0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL,
+	0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL,
+	0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL,
+	0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL,
+	0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL,
+	0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL,
+	0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL,
+	0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL,
+	0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL,
+	0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL,
+	0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL,
+	0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL,
+	0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL,
+	0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL,
+	0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL,
+	0x9e9e009e9e9e009eULL,
 };
 
 const u64 camellia_sp11101110[256] = {
-	0x7070700070707000, 0x8282820082828200, 0x2c2c2c002c2c2c00,
-	0xececec00ececec00, 0xb3b3b300b3b3b300, 0x2727270027272700,
-	0xc0c0c000c0c0c000, 0xe5e5e500e5e5e500, 0xe4e4e400e4e4e400,
-	0x8585850085858500, 0x5757570057575700, 0x3535350035353500,
-	0xeaeaea00eaeaea00, 0x0c0c0c000c0c0c00, 0xaeaeae00aeaeae00,
-	0x4141410041414100, 0x2323230023232300, 0xefefef00efefef00,
-	0x6b6b6b006b6b6b00, 0x9393930093939300, 0x4545450045454500,
-	0x1919190019191900, 0xa5a5a500a5a5a500, 0x2121210021212100,
-	0xededed00ededed00, 0x0e0e0e000e0e0e00, 0x4f4f4f004f4f4f00,
-	0x4e4e4e004e4e4e00, 0x1d1d1d001d1d1d00, 0x6565650065656500,
-	0x9292920092929200, 0xbdbdbd00bdbdbd00, 0x8686860086868600,
-	0xb8b8b800b8b8b800, 0xafafaf00afafaf00, 0x8f8f8f008f8f8f00,
-	0x7c7c7c007c7c7c00, 0xebebeb00ebebeb00, 0x1f1f1f001f1f1f00,
-	0xcecece00cecece00, 0x3e3e3e003e3e3e00, 0x3030300030303000,
-	0xdcdcdc00dcdcdc00, 0x5f5f5f005f5f5f00, 0x5e5e5e005e5e5e00,
-	0xc5c5c500c5c5c500, 0x0b0b0b000b0b0b00, 0x1a1a1a001a1a1a00,
-	0xa6a6a600a6a6a600, 0xe1e1e100e1e1e100, 0x3939390039393900,
-	0xcacaca00cacaca00, 0xd5d5d500d5d5d500, 0x4747470047474700,
-	0x5d5d5d005d5d5d00, 0x3d3d3d003d3d3d00, 0xd9d9d900d9d9d900,
-	0x0101010001010100, 0x5a5a5a005a5a5a00, 0xd6d6d600d6d6d600,
-	0x5151510051515100, 0x5656560056565600, 0x6c6c6c006c6c6c00,
-	0x4d4d4d004d4d4d00, 0x8b8b8b008b8b8b00, 0x0d0d0d000d0d0d00,
-	0x9a9a9a009a9a9a00, 0x6666660066666600, 0xfbfbfb00fbfbfb00,
-	0xcccccc00cccccc00, 0xb0b0b000b0b0b000, 0x2d2d2d002d2d2d00,
-	0x7474740074747400, 0x1212120012121200, 0x2b2b2b002b2b2b00,
-	0x2020200020202000, 0xf0f0f000f0f0f000, 0xb1b1b100b1b1b100,
-	0x8484840084848400, 0x9999990099999900, 0xdfdfdf00dfdfdf00,
-	0x4c4c4c004c4c4c00, 0xcbcbcb00cbcbcb00, 0xc2c2c200c2c2c200,
-	0x3434340034343400, 0x7e7e7e007e7e7e00, 0x7676760076767600,
-	0x0505050005050500, 0x6d6d6d006d6d6d00, 0xb7b7b700b7b7b700,
-	0xa9a9a900a9a9a900, 0x3131310031313100, 0xd1d1d100d1d1d100,
-	0x1717170017171700, 0x0404040004040400, 0xd7d7d700d7d7d700,
-	0x1414140014141400, 0x5858580058585800, 0x3a3a3a003a3a3a00,
-	0x6161610061616100, 0xdedede00dedede00, 0x1b1b1b001b1b1b00,
-	0x1111110011111100, 0x1c1c1c001c1c1c00, 0x3232320032323200,
-	0x0f0f0f000f0f0f00, 0x9c9c9c009c9c9c00, 0x1616160016161600,
-	0x5353530053535300, 0x1818180018181800, 0xf2f2f200f2f2f200,
-	0x2222220022222200, 0xfefefe00fefefe00, 0x4444440044444400,
-	0xcfcfcf00cfcfcf00, 0xb2b2b200b2b2b200, 0xc3c3c300c3c3c300,
-	0xb5b5b500b5b5b500, 0x7a7a7a007a7a7a00, 0x9191910091919100,
-	0x2424240024242400, 0x0808080008080800, 0xe8e8e800e8e8e800,
-	0xa8a8a800a8a8a800, 0x6060600060606000, 0xfcfcfc00fcfcfc00,
-	0x6969690069696900, 0x5050500050505000, 0xaaaaaa00aaaaaa00,
-	0xd0d0d000d0d0d000, 0xa0a0a000a0a0a000, 0x7d7d7d007d7d7d00,
-	0xa1a1a100a1a1a100, 0x8989890089898900, 0x6262620062626200,
-	0x9797970097979700, 0x5454540054545400, 0x5b5b5b005b5b5b00,
-	0x1e1e1e001e1e1e00, 0x9595950095959500, 0xe0e0e000e0e0e000,
-	0xffffff00ffffff00, 0x6464640064646400, 0xd2d2d200d2d2d200,
-	0x1010100010101000, 0xc4c4c400c4c4c400, 0x0000000000000000,
-	0x4848480048484800, 0xa3a3a300a3a3a300, 0xf7f7f700f7f7f700,
-	0x7575750075757500, 0xdbdbdb00dbdbdb00, 0x8a8a8a008a8a8a00,
-	0x0303030003030300, 0xe6e6e600e6e6e600, 0xdadada00dadada00,
-	0x0909090009090900, 0x3f3f3f003f3f3f00, 0xdddddd00dddddd00,
-	0x9494940094949400, 0x8787870087878700, 0x5c5c5c005c5c5c00,
-	0x8383830083838300, 0x0202020002020200, 0xcdcdcd00cdcdcd00,
-	0x4a4a4a004a4a4a00, 0x9090900090909000, 0x3333330033333300,
-	0x7373730073737300, 0x6767670067676700, 0xf6f6f600f6f6f600,
-	0xf3f3f300f3f3f300, 0x9d9d9d009d9d9d00, 0x7f7f7f007f7f7f00,
-	0xbfbfbf00bfbfbf00, 0xe2e2e200e2e2e200, 0x5252520052525200,
-	0x9b9b9b009b9b9b00, 0xd8d8d800d8d8d800, 0x2626260026262600,
-	0xc8c8c800c8c8c800, 0x3737370037373700, 0xc6c6c600c6c6c600,
-	0x3b3b3b003b3b3b00, 0x8181810081818100, 0x9696960096969600,
-	0x6f6f6f006f6f6f00, 0x4b4b4b004b4b4b00, 0x1313130013131300,
-	0xbebebe00bebebe00, 0x6363630063636300, 0x2e2e2e002e2e2e00,
-	0xe9e9e900e9e9e900, 0x7979790079797900, 0xa7a7a700a7a7a700,
-	0x8c8c8c008c8c8c00, 0x9f9f9f009f9f9f00, 0x6e6e6e006e6e6e00,
-	0xbcbcbc00bcbcbc00, 0x8e8e8e008e8e8e00, 0x2929290029292900,
-	0xf5f5f500f5f5f500, 0xf9f9f900f9f9f900, 0xb6b6b600b6b6b600,
-	0x2f2f2f002f2f2f00, 0xfdfdfd00fdfdfd00, 0xb4b4b400b4b4b400,
-	0x5959590059595900, 0x7878780078787800, 0x9898980098989800,
-	0x0606060006060600, 0x6a6a6a006a6a6a00, 0xe7e7e700e7e7e700,
-	0x4646460046464600, 0x7171710071717100, 0xbababa00bababa00,
-	0xd4d4d400d4d4d400, 0x2525250025252500, 0xababab00ababab00,
-	0x4242420042424200, 0x8888880088888800, 0xa2a2a200a2a2a200,
-	0x8d8d8d008d8d8d00, 0xfafafa00fafafa00, 0x7272720072727200,
-	0x0707070007070700, 0xb9b9b900b9b9b900, 0x5555550055555500,
-	0xf8f8f800f8f8f800, 0xeeeeee00eeeeee00, 0xacacac00acacac00,
-	0x0a0a0a000a0a0a00, 0x3636360036363600, 0x4949490049494900,
-	0x2a2a2a002a2a2a00, 0x6868680068686800, 0x3c3c3c003c3c3c00,
-	0x3838380038383800, 0xf1f1f100f1f1f100, 0xa4a4a400a4a4a400,
-	0x4040400040404000, 0x2828280028282800, 0xd3d3d300d3d3d300,
-	0x7b7b7b007b7b7b00, 0xbbbbbb00bbbbbb00, 0xc9c9c900c9c9c900,
-	0x4343430043434300, 0xc1c1c100c1c1c100, 0x1515150015151500,
-	0xe3e3e300e3e3e300, 0xadadad00adadad00, 0xf4f4f400f4f4f400,
-	0x7777770077777700, 0xc7c7c700c7c7c700, 0x8080800080808000,
-	0x9e9e9e009e9e9e00,
+	0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
+	0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
+	0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
+	0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL,
+	0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL,
+	0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL,
+	0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL,
+	0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL,
+	0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL,
+	0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL,
+	0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL,
+	0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL,
+	0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL,
+	0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL,
+	0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL,
+	0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL,
+	0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL,
+	0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL,
+	0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL,
+	0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL,
+	0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL,
+	0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL,
+	0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL,
+	0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL,
+	0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL,
+	0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL,
+	0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL,
+	0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL,
+	0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL,
+	0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL,
+	0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL,
+	0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL,
+	0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL,
+	0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL,
+	0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL,
+	0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL,
+	0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL,
+	0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL,
+	0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL,
+	0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL,
+	0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL,
+	0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL,
+	0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL,
+	0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL,
+	0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL,
+	0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL,
+	0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL,
+	0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL,
+	0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL,
+	0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL,
+	0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL,
+	0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL,
+	0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL,
+	0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL,
+	0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL,
+	0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL,
+	0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL,
+	0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL,
+	0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL,
+	0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL,
+	0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL,
+	0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL,
+	0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL,
+	0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL,
+	0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL,
+	0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL,
+	0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL,
+	0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL,
+	0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL,
+	0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL,
+	0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL,
+	0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL,
+	0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL,
+	0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL,
+	0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL,
+	0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL,
+	0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL,
+	0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL,
+	0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL,
+	0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL,
+	0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL,
+	0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL,
+	0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL,
+	0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL,
+	0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL,
+	0x9e9e9e009e9e9e00ULL,
 };
 
 /* key constants */
@@ -1601,7 +1601,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_ctxsize		= sizeof(struct camellia_ctx),
 	.cra_alignmask		= 0,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[0].cra_list),
 	.cra_u			= {
 		.cipher = {
 			.cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
@@ -1621,7 +1620,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= CAMELLIA_MIN_KEY_SIZE,
@@ -1641,7 +1639,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= CAMELLIA_MIN_KEY_SIZE,
@@ -1662,7 +1659,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[3].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= CAMELLIA_MIN_KEY_SIZE,
@@ -1683,7 +1679,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[4].cra_list),
 	.cra_exit		= lrw_exit_tfm,
 	.cra_u = {
 		.blkcipher = {
@@ -1707,7 +1702,6 @@ static struct crypto_alg camellia_algs[6] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(camellia_algs[5].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= CAMELLIA_MIN_KEY_SIZE * 2,
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
new file mode 100644
index 00000000000..a41a3aaba22
--- /dev/null
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -0,0 +1,376 @@
+/*
+ * Cast5 Cipher 16-way parallel algorithm (AVX/x86_64)
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+.file "cast5-avx-x86_64-asm_64.S"
+
+.extern cast5_s1
+.extern cast5_s2
+.extern cast5_s3
+.extern cast5_s4
+
+/* structure of crypto context */
+#define km	0
+#define kr	(16*4)
+#define rr	((16*4)+16)
+
+/* s-boxes */
+#define s1	cast5_s1
+#define s2	cast5_s2
+#define s3	cast5_s3
+#define s4	cast5_s4
+
+/**********************************************************************
+  16-way AVX cast5
+ **********************************************************************/
+#define CTX %rdi
+
+#define RL1 %xmm0
+#define RR1 %xmm1
+#define RL2 %xmm2
+#define RR2 %xmm3
+#define RL3 %xmm4
+#define RR3 %xmm5
+#define RL4 %xmm6
+#define RR4 %xmm7
+
+#define RX %xmm8
+
+#define RKM  %xmm9
+#define RKR  %xmm10
+#define RKRF %xmm11
+#define RKRR %xmm12
+
+#define R32  %xmm13
+#define R1ST %xmm14
+
+#define RTMP %xmm15
+
+#define RID1  %rbp
+#define RID1d %ebp
+#define RID2  %rsi
+#define RID2d %esi
+
+#define RGI1   %rdx
+#define RGI1bl %dl
+#define RGI1bh %dh
+#define RGI2   %rcx
+#define RGI2bl %cl
+#define RGI2bh %ch
+
+#define RGI3   %rax
+#define RGI3bl %al
+#define RGI3bh %ah
+#define RGI4   %rbx
+#define RGI4bl %bl
+#define RGI4bh %bh
+
+#define RFS1  %r8
+#define RFS1d %r8d
+#define RFS2  %r9
+#define RFS2d %r9d
+#define RFS3  %r10
+#define RFS3d %r10d
+
+
+#define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \
+	movzbl		src ## bh,     RID1d;    \
+	movzbl		src ## bl,     RID2d;    \
+	shrq $16,	src;                     \
+	movl		s1(, RID1, 4), dst ## d; \
+	op1		s2(, RID2, 4), dst ## d; \
+	movzbl		src ## bh,     RID1d;    \
+	movzbl		src ## bl,     RID2d;    \
+	interleave_op(il_reg);			 \
+	op2		s3(, RID1, 4), dst ## d; \
+	op3		s4(, RID2, 4), dst ## d;
+
+#define dummy(d) /* do nothing */
+
+#define shr_next(reg) \
+	shrq $16,	reg;
+
+#define F_head(a, x, gi1, gi2, op0) \
+	op0	a,	RKM,  x;                 \
+	vpslld	RKRF,	x,    RTMP;              \
+	vpsrld	RKRR,	x,    x;                 \
+	vpor	RTMP,	x,    x;                 \
+	\
+	vmovq		x,    gi1;               \
+	vpextrq $1,	x,    gi2;
+
+#define F_tail(a, x, gi1, gi2, op1, op2, op3) \
+	lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \
+	lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \
+	\
+	lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none);     \
+	shlq $32,	RFS2;                                      \
+	orq		RFS1, RFS2;                                \
+	lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none);     \
+	shlq $32,	RFS1;                                      \
+	orq		RFS1, RFS3;                                \
+	\
+	vmovq		RFS2, x;                                   \
+	vpinsrq $1,	RFS3, x, x;
+
+#define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \
+	F_head(b1, RX, RGI1, RGI2, op0);              \
+	F_head(b2, RX, RGI3, RGI4, op0);              \
+	\
+	F_tail(b1, RX, RGI1, RGI2, op1, op2, op3);    \
+	F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3);  \
+	\
+	vpxor		a1, RX,   a1;                 \
+	vpxor		a2, RTMP, a2;
+
+#define F1_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl)
+#define F2_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl)
+#define F3_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl)
+
+#define subround(a1, b1, a2, b2, f) \
+	F ## f ## _2(a1, b1, a2, b2);
+
+#define round(l, r, n, f) \
+	vbroadcastss 	(km+(4*n))(CTX), RKM;        \
+	vpand		R1ST,            RKR,  RKRF; \
+	vpsubq		RKRF,            R32,  RKRR; \
+	vpsrldq $1,	RKR,             RKR;        \
+	subround(l ## 1, r ## 1, l ## 2, r ## 2, f); \
+	subround(l ## 3, r ## 3, l ## 4, r ## 4, f);
+
+#define enc_preload_rkr() \
+	vbroadcastss	.L16_mask,                RKR;      \
+	/* add 16-bit rotation to key rotations (mod 32) */ \
+	vpxor		kr(CTX),                  RKR, RKR;
+
+#define dec_preload_rkr() \
+	vbroadcastss	.L16_mask,                RKR;      \
+	/* add 16-bit rotation to key rotations (mod 32) */ \
+	vpxor		kr(CTX),                  RKR, RKR; \
+	vpshufb		.Lbswap128_mask,          RKR, RKR;
+
+#define transpose_2x4(x0, x1, t0, t1) \
+	vpunpckldq		x1, x0, t0; \
+	vpunpckhdq		x1, x0, t1; \
+	\
+	vpunpcklqdq		t1, t0, x0; \
+	vpunpckhqdq		t1, t0, x1;
+
+#define inpack_blocks(in, x0, x1, t0, t1, rmask) \
+	vmovdqu (0*4*4)(in),	x0; \
+	vmovdqu (1*4*4)(in),	x1; \
+	vpshufb rmask, 	x0,	x0; \
+	vpshufb rmask, 	x1,	x1; \
+	\
+	transpose_2x4(x0, x1, t0, t1)
+
+#define outunpack_blocks(out, x0, x1, t0, t1, rmask) \
+	transpose_2x4(x0, x1, t0, t1) \
+	\
+	vpshufb rmask,	x0, x0;           \
+	vpshufb rmask,	x1, x1;           \
+	vmovdqu 	x0, (0*4*4)(out); \
+	vmovdqu		x1, (1*4*4)(out);
+
+#define outunpack_xor_blocks(out, x0, x1, t0, t1, rmask) \
+	transpose_2x4(x0, x1, t0, t1) \
+	\
+	vpshufb rmask,	x0, x0;               \
+	vpshufb rmask,	x1, x1;               \
+	vpxor		(0*4*4)(out), x0, x0; \
+	vmovdqu 	x0, (0*4*4)(out);     \
+	vpxor		(1*4*4)(out), x1, x1; \
+	vmovdqu	        x1, (1*4*4)(out);
+
+.data
+
+.align 16
+.Lbswap_mask:
+	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+.Lbswap128_mask:
+	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+.L16_mask:
+	.byte 16, 16, 16, 16
+.L32_mask:
+	.byte 32, 0, 0, 0
+.Lfirst_mask:
+	.byte 0x1f, 0, 0, 0
+
+.text
+
+.align 16
+.global __cast5_enc_blk_16way
+.type   __cast5_enc_blk_16way,@function;
+
+__cast5_enc_blk_16way:
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 *	%rcx: bool, if true: xor output
+	 */
+
+	pushq %rbp;
+	pushq %rbx;
+	pushq %rcx;
+
+	vmovdqa .Lbswap_mask, RKM;
+	vmovd .Lfirst_mask, R1ST;
+	vmovd .L32_mask, R32;
+	enc_preload_rkr();
+
+	leaq 1*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
+	inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
+	leaq 2*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
+	leaq 3*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
+
+	movq %rsi, %r11;
+
+	round(RL, RR, 0, 1);
+	round(RR, RL, 1, 2);
+	round(RL, RR, 2, 3);
+	round(RR, RL, 3, 1);
+	round(RL, RR, 4, 2);
+	round(RR, RL, 5, 3);
+	round(RL, RR, 6, 1);
+	round(RR, RL, 7, 2);
+	round(RL, RR, 8, 3);
+	round(RR, RL, 9, 1);
+	round(RL, RR, 10, 2);
+	round(RR, RL, 11, 3);
+
+	movzbl rr(CTX), %eax;
+	testl %eax, %eax;
+	jnz __skip_enc;
+
+	round(RL, RR, 12, 1);
+	round(RR, RL, 13, 2);
+	round(RL, RR, 14, 3);
+	round(RR, RL, 15, 1);
+
+__skip_enc:
+	popq %rcx;
+	popq %rbx;
+	popq %rbp;
+
+	vmovdqa .Lbswap_mask, RKM;
+	leaq 1*(2*4*4)(%r11), %rax;
+
+	testb %cl, %cl;
+	jnz __enc_xor16;
+
+	outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
+	outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
+	leaq 2*(2*4*4)(%r11), %rax;
+	outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
+	leaq 3*(2*4*4)(%r11), %rax;
+	outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
+
+	ret;
+
+__enc_xor16:
+	outunpack_xor_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
+	outunpack_xor_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
+	leaq 2*(2*4*4)(%r11), %rax;
+	outunpack_xor_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
+	leaq 3*(2*4*4)(%r11), %rax;
+	outunpack_xor_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
+
+	ret;
+
+.align 16
+.global cast5_dec_blk_16way
+.type   cast5_dec_blk_16way,@function;
+
+cast5_dec_blk_16way:
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 */
+
+	pushq %rbp;
+	pushq %rbx;
+
+	vmovdqa .Lbswap_mask, RKM;
+	vmovd .Lfirst_mask, R1ST;
+	vmovd .L32_mask, R32;
+	dec_preload_rkr();
+
+	leaq 1*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
+	inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
+	leaq 2*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
+	leaq 3*(2*4*4)(%rdx), %rax;
+	inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
+
+	movq %rsi, %r11;
+
+	movzbl rr(CTX), %eax;
+	testl %eax, %eax;
+	jnz __skip_dec;
+
+	round(RL, RR, 15, 1);
+	round(RR, RL, 14, 3);
+	round(RL, RR, 13, 2);
+	round(RR, RL, 12, 1);
+
+__dec_tail:
+	round(RL, RR, 11, 3);
+	round(RR, RL, 10, 2);
+	round(RL, RR, 9, 1);
+	round(RR, RL, 8, 3);
+	round(RL, RR, 7, 2);
+	round(RR, RL, 6, 1);
+	round(RL, RR, 5, 3);
+	round(RR, RL, 4, 2);
+	round(RL, RR, 3, 1);
+	round(RR, RL, 2, 3);
+	round(RL, RR, 1, 2);
+	round(RR, RL, 0, 1);
+
+	vmovdqa .Lbswap_mask, RKM;
+	popq %rbx;
+	popq %rbp;
+
+	leaq 1*(2*4*4)(%r11), %rax;
+	outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
+	outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
+	leaq 2*(2*4*4)(%r11), %rax;
+	outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
+	leaq 3*(2*4*4)(%r11), %rax;
+	outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
+
+	ret;
+
+__skip_dec:
+	vpsrldq $4, RKR, RKR;
+	jmp __dec_tail;
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
new file mode 100644
index 00000000000..e0ea14f9547
--- /dev/null
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -0,0 +1,530 @@
+/*
+ * Glue Code for the AVX assembler implemention of the Cast5 Cipher
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/hardirq.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <crypto/algapi.h>
+#include <crypto/cast5.h>
+#include <crypto/cryptd.h>
+#include <crypto/ctr.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
+
+#define CAST5_PARALLEL_BLOCKS 16
+
+asmlinkage void __cast5_enc_blk_16way(struct cast5_ctx *ctx, u8 *dst,
+				      const u8 *src, bool xor);
+asmlinkage void cast5_dec_blk_16way(struct cast5_ctx *ctx, u8 *dst,
+				    const u8 *src);
+
+static inline void cast5_enc_blk_xway(struct cast5_ctx *ctx, u8 *dst,
+				      const u8 *src)
+{
+	__cast5_enc_blk_16way(ctx, dst, src, false);
+}
+
+static inline void cast5_enc_blk_xway_xor(struct cast5_ctx *ctx, u8 *dst,
+					  const u8 *src)
+{
+	__cast5_enc_blk_16way(ctx, dst, src, true);
+}
+
+static inline void cast5_dec_blk_xway(struct cast5_ctx *ctx, u8 *dst,
+				      const u8 *src)
+{
+	cast5_dec_blk_16way(ctx, dst, src);
+}
+
+
+static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes)
+{
+	return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
+			      NULL, fpu_enabled, nbytes);
+}
+
+static inline void cast5_fpu_end(bool fpu_enabled)
+{
+	return glue_fpu_end(fpu_enabled);
+}
+
+static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
+		     bool enc)
+{
+	bool fpu_enabled = false;
+	struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	const unsigned int bsize = CAST5_BLOCK_SIZE;
+	unsigned int nbytes;
+	int err;
+
+	err = blkcipher_walk_virt(desc, walk);
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	while ((nbytes = walk->nbytes)) {
+		u8 *wsrc = walk->src.virt.addr;
+		u8 *wdst = walk->dst.virt.addr;
+
+		fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
+
+		/* Process multi-block batch */
+		if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
+			do {
+				if (enc)
+					cast5_enc_blk_xway(ctx, wdst, wsrc);
+				else
+					cast5_dec_blk_xway(ctx, wdst, wsrc);
+
+				wsrc += bsize * CAST5_PARALLEL_BLOCKS;
+				wdst += bsize * CAST5_PARALLEL_BLOCKS;
+				nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
+			} while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
+
+			if (nbytes < bsize)
+				goto done;
+		}
+
+		/* Handle leftovers */
+		do {
+			if (enc)
+				__cast5_encrypt(ctx, wdst, wsrc);
+			else
+				__cast5_decrypt(ctx, wdst, wsrc);
+
+			wsrc += bsize;
+			wdst += bsize;
+			nbytes -= bsize;
+		} while (nbytes >= bsize);
+
+done:
+		err = blkcipher_walk_done(desc, walk, nbytes);
+	}
+
+	cast5_fpu_end(fpu_enabled);
+	return err;
+}
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct blkcipher_walk walk;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	return ecb_crypt(desc, &walk, true);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct blkcipher_walk walk;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	return ecb_crypt(desc, &walk, false);
+}
+
+static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
+				  struct blkcipher_walk *walk)
+{
+	struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	const unsigned int bsize = CAST5_BLOCK_SIZE;
+	unsigned int nbytes = walk->nbytes;
+	u64 *src = (u64 *)walk->src.virt.addr;
+	u64 *dst = (u64 *)walk->dst.virt.addr;
+	u64 *iv = (u64 *)walk->iv;
+
+	do {
+		*dst = *src ^ *iv;
+		__cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
+		iv = dst;
+
+		src += 1;
+		dst += 1;
+		nbytes -= bsize;
+	} while (nbytes >= bsize);
+
+	*(u64 *)walk->iv = *iv;
+	return nbytes;
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while ((nbytes = walk.nbytes)) {
+		nbytes = __cbc_encrypt(desc, &walk);
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
+}
+
+static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
+				  struct blkcipher_walk *walk)
+{
+	struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	const unsigned int bsize = CAST5_BLOCK_SIZE;
+	unsigned int nbytes = walk->nbytes;
+	u64 *src = (u64 *)walk->src.virt.addr;
+	u64 *dst = (u64 *)walk->dst.virt.addr;
+	u64 ivs[CAST5_PARALLEL_BLOCKS - 1];
+	u64 last_iv;
+	int i;
+
+	/* Start of the last block. */
+	src += nbytes / bsize - 1;
+	dst += nbytes / bsize - 1;
+
+	last_iv = *src;
+
+	/* Process multi-block batch */
+	if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
+		do {
+			nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
+			src -= CAST5_PARALLEL_BLOCKS - 1;
+			dst -= CAST5_PARALLEL_BLOCKS - 1;
+
+			for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
+				ivs[i] = src[i];
+
+			cast5_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
+
+			for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
+				*(dst + (i + 1)) ^= *(ivs + i);
+
+			nbytes -= bsize;
+			if (nbytes < bsize)
+				goto done;
+
+			*dst ^= *(src - 1);
+			src -= 1;
+			dst -= 1;
+		} while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
+
+		if (nbytes < bsize)
+			goto done;
+	}
+
+	/* Handle leftovers */
+	for (;;) {
+		__cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
+
+		nbytes -= bsize;
+		if (nbytes < bsize)
+			break;
+
+		*dst ^= *(src - 1);
+		src -= 1;
+		dst -= 1;
+	}
+
+done:
+	*dst ^= *(u64 *)walk->iv;
+	*(u64 *)walk->iv = last_iv;
+
+	return nbytes;
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	bool fpu_enabled = false;
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	while ((nbytes = walk.nbytes)) {
+		fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
+		nbytes = __cbc_decrypt(desc, &walk);
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	cast5_fpu_end(fpu_enabled);
+	return err;
+}
+
+static void ctr_crypt_final(struct blkcipher_desc *desc,
+			    struct blkcipher_walk *walk)
+{
+	struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	u8 *ctrblk = walk->iv;
+	u8 keystream[CAST5_BLOCK_SIZE];
+	u8 *src = walk->src.virt.addr;
+	u8 *dst = walk->dst.virt.addr;
+	unsigned int nbytes = walk->nbytes;
+
+	__cast5_encrypt(ctx, keystream, ctrblk);
+	crypto_xor(keystream, src, nbytes);
+	memcpy(dst, keystream, nbytes);
+
+	crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
+}
+
+static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
+				struct blkcipher_walk *walk)
+{
+	struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	const unsigned int bsize = CAST5_BLOCK_SIZE;
+	unsigned int nbytes = walk->nbytes;
+	u64 *src = (u64 *)walk->src.virt.addr;
+	u64 *dst = (u64 *)walk->dst.virt.addr;
+	u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
+	__be64 ctrblocks[CAST5_PARALLEL_BLOCKS];
+	int i;
+
+	/* Process multi-block batch */
+	if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
+		do {
+			/* create ctrblks for parallel encrypt */
+			for (i = 0; i < CAST5_PARALLEL_BLOCKS; i++) {
+				if (dst != src)
+					dst[i] = src[i];
+
+				ctrblocks[i] = cpu_to_be64(ctrblk++);
+			}
+
+			cast5_enc_blk_xway_xor(ctx, (u8 *)dst,
+					       (u8 *)ctrblocks);
+
+			src += CAST5_PARALLEL_BLOCKS;
+			dst += CAST5_PARALLEL_BLOCKS;
+			nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
+		} while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
+
+		if (nbytes < bsize)
+			goto done;
+	}
+
+	/* Handle leftovers */
+	do {
+		if (dst != src)
+			*dst = *src;
+
+		ctrblocks[0] = cpu_to_be64(ctrblk++);
+
+		__cast5_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
+		*dst ^= ctrblocks[0];
+
+		src += 1;
+		dst += 1;
+		nbytes -= bsize;
+	} while (nbytes >= bsize);
+
+done:
+	*(__be64 *)walk->iv = cpu_to_be64(ctrblk);
+	return nbytes;
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		     struct scatterlist *src, unsigned int nbytes)
+{
+	bool fpu_enabled = false;
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt_block(desc, &walk, CAST5_BLOCK_SIZE);
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+	while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
+		fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
+		nbytes = __ctr_crypt(desc, &walk);
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	cast5_fpu_end(fpu_enabled);
+
+	if (walk.nbytes) {
+		ctr_crypt_final(desc, &walk);
+		err = blkcipher_walk_done(desc, &walk, 0);
+	}
+
+	return err;
+}
+
+
+static struct crypto_alg cast5_algs[6] = { {
+	.cra_name		= "__ecb-cast5-avx",
+	.cra_driver_name	= "__driver-ecb-cast5-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST5_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast5_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.setkey		= cast5_setkey,
+			.encrypt	= ecb_encrypt,
+			.decrypt	= ecb_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__cbc-cast5-avx",
+	.cra_driver_name	= "__driver-cbc-cast5-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST5_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast5_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.setkey		= cast5_setkey,
+			.encrypt	= cbc_encrypt,
+			.decrypt	= cbc_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__ctr-cast5-avx",
+	.cra_driver_name	= "__driver-ctr-cast5-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct cast5_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.ivsize		= CAST5_BLOCK_SIZE,
+			.setkey		= cast5_setkey,
+			.encrypt	= ctr_crypt,
+			.decrypt	= ctr_crypt,
+		},
+	},
+}, {
+	.cra_name		= "ecb(cast5)",
+	.cra_driver_name	= "ecb-cast5-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST5_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "cbc(cast5)",
+	.cra_driver_name	= "cbc-cast5-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST5_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.ivsize		= CAST5_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= __ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "ctr(cast5)",
+	.cra_driver_name	= "ctr-cast5-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST5_MIN_KEY_SIZE,
+			.max_keysize	= CAST5_MAX_KEY_SIZE,
+			.ivsize		= CAST5_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_encrypt,
+			.geniv		= "chainiv",
+		},
+	},
+} };
+
+static int __init cast5_init(void)
+{
+	u64 xcr0;
+
+	if (!cpu_has_avx || !cpu_has_osxsave) {
+		pr_info("AVX instructions are not detected.\n");
+		return -ENODEV;
+	}
+
+	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+		pr_info("AVX detected but unusable.\n");
+		return -ENODEV;
+	}
+
+	return crypto_register_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
+}
+
+static void __exit cast5_exit(void)
+{
+	crypto_unregister_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
+}
+
+module_init(cast5_init);
+module_exit(cast5_exit);
+
+MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("cast5");
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
new file mode 100644
index 00000000000..218d283772f
--- /dev/null
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -0,0 +1,383 @@
+/*
+ * Cast6 Cipher 8-way parallel algorithm (AVX/x86_64)
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+.file "cast6-avx-x86_64-asm_64.S"
+
+.extern cast6_s1
+.extern cast6_s2
+.extern cast6_s3
+.extern cast6_s4
+
+/* structure of crypto context */
+#define km	0
+#define kr	(12*4*4)
+
+/* s-boxes */
+#define s1	cast6_s1
+#define s2	cast6_s2
+#define s3	cast6_s3
+#define s4	cast6_s4
+
+/**********************************************************************
+  8-way AVX cast6
+ **********************************************************************/
+#define CTX %rdi
+
+#define RA1 %xmm0
+#define RB1 %xmm1
+#define RC1 %xmm2
+#define RD1 %xmm3
+
+#define RA2 %xmm4
+#define RB2 %xmm5
+#define RC2 %xmm6
+#define RD2 %xmm7
+
+#define RX  %xmm8
+
+#define RKM  %xmm9
+#define RKR  %xmm10
+#define RKRF %xmm11
+#define RKRR %xmm12
+#define R32  %xmm13
+#define R1ST %xmm14
+
+#define RTMP %xmm15
+
+#define RID1  %rbp
+#define RID1d %ebp
+#define RID2  %rsi
+#define RID2d %esi
+
+#define RGI1   %rdx
+#define RGI1bl %dl
+#define RGI1bh %dh
+#define RGI2   %rcx
+#define RGI2bl %cl
+#define RGI2bh %ch
+
+#define RGI3   %rax
+#define RGI3bl %al
+#define RGI3bh %ah
+#define RGI4   %rbx
+#define RGI4bl %bl
+#define RGI4bh %bh
+
+#define RFS1  %r8
+#define RFS1d %r8d
+#define RFS2  %r9
+#define RFS2d %r9d
+#define RFS3  %r10
+#define RFS3d %r10d
+
+
+#define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \
+	movzbl		src ## bh,     RID1d;    \
+	movzbl		src ## bl,     RID2d;    \
+	shrq $16,	src;                     \
+	movl		s1(, RID1, 4), dst ## d; \
+	op1		s2(, RID2, 4), dst ## d; \
+	movzbl		src ## bh,     RID1d;    \
+	movzbl		src ## bl,     RID2d;    \
+	interleave_op(il_reg);			 \
+	op2		s3(, RID1, 4), dst ## d; \
+	op3		s4(, RID2, 4), dst ## d;
+
+#define dummy(d) /* do nothing */
+
+#define shr_next(reg) \
+	shrq $16,	reg;
+
+#define F_head(a, x, gi1, gi2, op0) \
+	op0	a,	RKM,  x;                 \
+	vpslld	RKRF,	x,    RTMP;              \
+	vpsrld	RKRR,	x,    x;                 \
+	vpor	RTMP,	x,    x;                 \
+	\
+	vmovq		x,    gi1;               \
+	vpextrq $1,	x,    gi2;
+
+#define F_tail(a, x, gi1, gi2, op1, op2, op3) \
+	lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \
+	lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \
+	\
+	lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none);     \
+	shlq $32,	RFS2;                                      \
+	orq		RFS1, RFS2;                                \
+	lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none);     \
+	shlq $32,	RFS1;                                      \
+	orq		RFS1, RFS3;                                \
+	\
+	vmovq		RFS2, x;                                   \
+	vpinsrq $1,	RFS3, x, x;
+
+#define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \
+	F_head(b1, RX, RGI1, RGI2, op0);              \
+	F_head(b2, RX, RGI3, RGI4, op0);              \
+	\
+	F_tail(b1, RX, RGI1, RGI2, op1, op2, op3);    \
+	F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3);  \
+	\
+	vpxor		a1, RX,   a1;                 \
+	vpxor		a2, RTMP, a2;
+
+#define F1_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl)
+#define F2_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl)
+#define F3_2(a1, b1, a2, b2) \
+	F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl)
+
+#define qop(in, out, f) \
+	F ## f ## _2(out ## 1, in ## 1, out ## 2, in ## 2);
+
+#define get_round_keys(nn) \
+	vbroadcastss	(km+(4*(nn)))(CTX), RKM;        \
+	vpand		R1ST,               RKR,  RKRF; \
+	vpsubq		RKRF,               R32,  RKRR; \
+	vpsrldq $1,	RKR,                RKR;
+
+#define Q(n) \
+	get_round_keys(4*n+0); \
+	qop(RD, RC, 1);        \
+	\
+	get_round_keys(4*n+1); \
+	qop(RC, RB, 2);        \
+	\
+	get_round_keys(4*n+2); \
+	qop(RB, RA, 3);        \
+	\
+	get_round_keys(4*n+3); \
+	qop(RA, RD, 1);
+
+#define QBAR(n) \
+	get_round_keys(4*n+3); \
+	qop(RA, RD, 1);        \
+	\
+	get_round_keys(4*n+2); \
+	qop(RB, RA, 3);        \
+	\
+	get_round_keys(4*n+1); \
+	qop(RC, RB, 2);        \
+	\
+	get_round_keys(4*n+0); \
+	qop(RD, RC, 1);
+
+#define shuffle(mask) \
+	vpshufb		mask,            RKR, RKR;
+
+#define preload_rkr(n, do_mask, mask) \
+	vbroadcastss	.L16_mask,                RKR;      \
+	/* add 16-bit rotation to key rotations (mod 32) */ \
+	vpxor		(kr+n*16)(CTX),           RKR, RKR; \
+	do_mask(mask);
+
+#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+	vpunpckldq		x1, x0, t0; \
+	vpunpckhdq		x1, x0, t2; \
+	vpunpckldq		x3, x2, t1; \
+	vpunpckhdq		x3, x2, x3; \
+	\
+	vpunpcklqdq		t1, t0, x0; \
+	vpunpckhqdq		t1, t0, x1; \
+	vpunpcklqdq		x3, t2, x2; \
+	vpunpckhqdq		x3, t2, x3;
+
+#define inpack_blocks(in, x0, x1, x2, x3, t0, t1, t2, rmask) \
+	vmovdqu (0*4*4)(in),	x0; \
+	vmovdqu (1*4*4)(in),	x1; \
+	vmovdqu (2*4*4)(in),	x2; \
+	vmovdqu (3*4*4)(in),	x3; \
+	vpshufb rmask, x0,	x0; \
+	vpshufb rmask, x1,	x1; \
+	vpshufb rmask, x2,	x2; \
+	vpshufb rmask, x3,	x3; \
+	\
+	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
+
+#define outunpack_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
+	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+	\
+	vpshufb rmask,		x0, x0;       \
+	vpshufb rmask,		x1, x1;       \
+	vpshufb rmask,		x2, x2;       \
+	vpshufb rmask,		x3, x3;       \
+	vmovdqu x0,		(0*4*4)(out); \
+	vmovdqu	x1,		(1*4*4)(out); \
+	vmovdqu	x2,		(2*4*4)(out); \
+	vmovdqu	x3,		(3*4*4)(out);
+
+#define outunpack_xor_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
+	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+	\
+	vpshufb rmask,		x0, x0;       \
+	vpshufb rmask,		x1, x1;       \
+	vpshufb rmask,		x2, x2;       \
+	vpshufb rmask,		x3, x3;       \
+	vpxor (0*4*4)(out),	x0, x0;       \
+	vmovdqu	x0,		(0*4*4)(out); \
+	vpxor (1*4*4)(out),	x1, x1;       \
+	vmovdqu	x1,		(1*4*4)(out); \
+	vpxor (2*4*4)(out),	x2, x2;       \
+	vmovdqu x2,		(2*4*4)(out); \
+	vpxor (3*4*4)(out),	x3, x3;       \
+	vmovdqu x3,		(3*4*4)(out);
+
+.data
+
+.align 16
+.Lbswap_mask:
+	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+.Lrkr_enc_Q_Q_QBAR_QBAR:
+	.byte 0, 1, 2, 3, 4, 5, 6, 7, 11, 10, 9, 8, 15, 14, 13, 12
+.Lrkr_enc_QBAR_QBAR_QBAR_QBAR:
+	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+.Lrkr_dec_Q_Q_Q_Q:
+	.byte 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3
+.Lrkr_dec_Q_Q_QBAR_QBAR:
+	.byte 12, 13, 14, 15, 8, 9, 10, 11, 7, 6, 5, 4, 3, 2, 1, 0
+.Lrkr_dec_QBAR_QBAR_QBAR_QBAR:
+	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+.L16_mask:
+	.byte 16, 16, 16, 16
+.L32_mask:
+	.byte 32, 0, 0, 0
+.Lfirst_mask:
+	.byte 0x1f, 0, 0, 0
+
+.text
+
+.align 16
+.global __cast6_enc_blk_8way
+.type   __cast6_enc_blk_8way,@function;
+
+__cast6_enc_blk_8way:
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 *	%rcx: bool, if true: xor output
+	 */
+
+	pushq %rbp;
+	pushq %rbx;
+	pushq %rcx;
+
+	vmovdqa .Lbswap_mask, RKM;
+	vmovd .Lfirst_mask, R1ST;
+	vmovd .L32_mask, R32;
+
+	leaq (4*4*4)(%rdx), %rax;
+	inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+	inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+	movq %rsi, %r11;
+
+	preload_rkr(0, dummy, none);
+	Q(0);
+	Q(1);
+	Q(2);
+	Q(3);
+	preload_rkr(1, shuffle, .Lrkr_enc_Q_Q_QBAR_QBAR);
+	Q(4);
+	Q(5);
+	QBAR(6);
+	QBAR(7);
+	preload_rkr(2, shuffle, .Lrkr_enc_QBAR_QBAR_QBAR_QBAR);
+	QBAR(8);
+	QBAR(9);
+	QBAR(10);
+	QBAR(11);
+
+	popq %rcx;
+	popq %rbx;
+	popq %rbp;
+
+	vmovdqa .Lbswap_mask, RKM;
+	leaq (4*4*4)(%r11), %rax;
+
+	testb %cl, %cl;
+	jnz __enc_xor8;
+
+	outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+	outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+	ret;
+
+__enc_xor8:
+	outunpack_xor_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+	outunpack_xor_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+	ret;
+
+.align 16
+.global cast6_dec_blk_8way
+.type   cast6_dec_blk_8way,@function;
+
+cast6_dec_blk_8way:
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 */
+
+	pushq %rbp;
+	pushq %rbx;
+
+	vmovdqa .Lbswap_mask, RKM;
+	vmovd .Lfirst_mask, R1ST;
+	vmovd .L32_mask, R32;
+
+	leaq (4*4*4)(%rdx), %rax;
+	inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+	inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+	movq %rsi, %r11;
+
+	preload_rkr(2, shuffle, .Lrkr_dec_Q_Q_Q_Q);
+	Q(11);
+	Q(10);
+	Q(9);
+	Q(8);
+	preload_rkr(1, shuffle, .Lrkr_dec_Q_Q_QBAR_QBAR);
+	Q(7);
+	Q(6);
+	QBAR(5);
+	QBAR(4);
+	preload_rkr(0, shuffle, .Lrkr_dec_QBAR_QBAR_QBAR_QBAR);
+	QBAR(3);
+	QBAR(2);
+	QBAR(1);
+	QBAR(0);
+
+	popq %rbx;
+	popq %rbp;
+
+	vmovdqa .Lbswap_mask, RKM;
+	leaq (4*4*4)(%r11), %rax;
+	outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
+	outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
+
+	ret;
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
new file mode 100644
index 00000000000..15e5f85a501
--- /dev/null
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -0,0 +1,648 @@
+/*
+ * Glue Code for the AVX assembler implemention of the Cast6 Cipher
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/hardirq.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <crypto/algapi.h>
+#include <crypto/cast6.h>
+#include <crypto/cryptd.h>
+#include <crypto/b128ops.h>
+#include <crypto/ctr.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
+
+#define CAST6_PARALLEL_BLOCKS 8
+
+asmlinkage void __cast6_enc_blk_8way(struct cast6_ctx *ctx, u8 *dst,
+				     const u8 *src, bool xor);
+asmlinkage void cast6_dec_blk_8way(struct cast6_ctx *ctx, u8 *dst,
+				   const u8 *src);
+
+static inline void cast6_enc_blk_xway(struct cast6_ctx *ctx, u8 *dst,
+				      const u8 *src)
+{
+	__cast6_enc_blk_8way(ctx, dst, src, false);
+}
+
+static inline void cast6_enc_blk_xway_xor(struct cast6_ctx *ctx, u8 *dst,
+					  const u8 *src)
+{
+	__cast6_enc_blk_8way(ctx, dst, src, true);
+}
+
+static inline void cast6_dec_blk_xway(struct cast6_ctx *ctx, u8 *dst,
+				      const u8 *src)
+{
+	cast6_dec_blk_8way(ctx, dst, src);
+}
+
+
+static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
+{
+	u128 ivs[CAST6_PARALLEL_BLOCKS - 1];
+	unsigned int j;
+
+	for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
+		ivs[j] = src[j];
+
+	cast6_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
+
+	for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
+		u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
+}
+
+static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+{
+	be128 ctrblk;
+
+	u128_to_be128(&ctrblk, iv);
+	u128_inc(iv);
+
+	__cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
+	u128_xor(dst, src, (u128 *)&ctrblk);
+}
+
+static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
+				   u128 *iv)
+{
+	be128 ctrblks[CAST6_PARALLEL_BLOCKS];
+	unsigned int i;
+
+	for (i = 0; i < CAST6_PARALLEL_BLOCKS; i++) {
+		if (dst != src)
+			dst[i] = src[i];
+
+		u128_to_be128(&ctrblks[i], iv);
+		u128_inc(iv);
+	}
+
+	cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
+}
+
+static const struct common_glue_ctx cast6_enc = {
+	.num_funcs = 2,
+	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
+
+	.funcs = { {
+		.num_blocks = CAST6_PARALLEL_BLOCKS,
+		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_enc_blk_xway) }
+	}, {
+		.num_blocks = 1,
+		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
+	} }
+};
+
+static const struct common_glue_ctx cast6_ctr = {
+	.num_funcs = 2,
+	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
+
+	.funcs = { {
+		.num_blocks = CAST6_PARALLEL_BLOCKS,
+		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr_xway) }
+	}, {
+		.num_blocks = 1,
+		.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
+	} }
+};
+
+static const struct common_glue_ctx cast6_dec = {
+	.num_funcs = 2,
+	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
+
+	.funcs = { {
+		.num_blocks = CAST6_PARALLEL_BLOCKS,
+		.fn_u = { .ecb = GLUE_FUNC_CAST(cast6_dec_blk_xway) }
+	}, {
+		.num_blocks = 1,
+		.fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
+	} }
+};
+
+static const struct common_glue_ctx cast6_dec_cbc = {
+	.num_funcs = 2,
+	.fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
+
+	.funcs = { {
+		.num_blocks = CAST6_PARALLEL_BLOCKS,
+		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_decrypt_cbc_xway) }
+	}, {
+		.num_blocks = 1,
+		.fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
+	} }
+};
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
+				       dst, src, nbytes);
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
+				       nbytes);
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		     struct scatterlist *src, unsigned int nbytes)
+{
+	return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
+}
+
+static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
+{
+	return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
+			      NULL, fpu_enabled, nbytes);
+}
+
+static inline void cast6_fpu_end(bool fpu_enabled)
+{
+	glue_fpu_end(fpu_enabled);
+}
+
+struct crypt_priv {
+	struct cast6_ctx *ctx;
+	bool fpu_enabled;
+};
+
+static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+	const unsigned int bsize = CAST6_BLOCK_SIZE;
+	struct crypt_priv *ctx = priv;
+	int i;
+
+	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
+
+	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
+		cast6_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+		return;
+	}
+
+	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+		__cast6_encrypt(ctx->ctx, srcdst, srcdst);
+}
+
+static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+	const unsigned int bsize = CAST6_BLOCK_SIZE;
+	struct crypt_priv *ctx = priv;
+	int i;
+
+	ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
+
+	if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
+		cast6_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+		return;
+	}
+
+	for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+		__cast6_decrypt(ctx->ctx, srcdst, srcdst);
+}
+
+struct cast6_lrw_ctx {
+	struct lrw_table_ctx lrw_table;
+	struct cast6_ctx cast6_ctx;
+};
+
+static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
+			      unsigned int keylen)
+{
+	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+	int err;
+
+	err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
+			     &tfm->crt_flags);
+	if (err)
+		return err;
+
+	return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
+}
+
+static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[CAST6_PARALLEL_BLOCKS];
+	struct crypt_priv crypt_ctx = {
+		.ctx = &ctx->cast6_ctx,
+		.fpu_enabled = false,
+	};
+	struct lrw_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.table_ctx = &ctx->lrw_table,
+		.crypt_ctx = &crypt_ctx,
+		.crypt_fn = encrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	ret = lrw_crypt(desc, dst, src, nbytes, &req);
+	cast6_fpu_end(crypt_ctx.fpu_enabled);
+
+	return ret;
+}
+
+static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[CAST6_PARALLEL_BLOCKS];
+	struct crypt_priv crypt_ctx = {
+		.ctx = &ctx->cast6_ctx,
+		.fpu_enabled = false,
+	};
+	struct lrw_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.table_ctx = &ctx->lrw_table,
+		.crypt_ctx = &crypt_ctx,
+		.crypt_fn = decrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	ret = lrw_crypt(desc, dst, src, nbytes, &req);
+	cast6_fpu_end(crypt_ctx.fpu_enabled);
+
+	return ret;
+}
+
+static void lrw_exit_tfm(struct crypto_tfm *tfm)
+{
+	struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	lrw_free_table(&ctx->lrw_table);
+}
+
+struct cast6_xts_ctx {
+	struct cast6_ctx tweak_ctx;
+	struct cast6_ctx crypt_ctx;
+};
+
+static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
+			      unsigned int keylen)
+{
+	struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+	u32 *flags = &tfm->crt_flags;
+	int err;
+
+	/* key consists of keys of equal size concatenated, therefore
+	 * the length must be even
+	 */
+	if (keylen % 2) {
+		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+		return -EINVAL;
+	}
+
+	/* first half of xts-key is for crypt */
+	err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
+	if (err)
+		return err;
+
+	/* second half of xts-key is for tweak */
+	return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
+			      flags);
+}
+
+static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[CAST6_PARALLEL_BLOCKS];
+	struct crypt_priv crypt_ctx = {
+		.ctx = &ctx->crypt_ctx,
+		.fpu_enabled = false,
+	};
+	struct xts_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.tweak_ctx = &ctx->tweak_ctx,
+		.tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
+		.crypt_ctx = &crypt_ctx,
+		.crypt_fn = encrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	ret = xts_crypt(desc, dst, src, nbytes, &req);
+	cast6_fpu_end(crypt_ctx.fpu_enabled);
+
+	return ret;
+}
+
+static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+		       struct scatterlist *src, unsigned int nbytes)
+{
+	struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+	be128 buf[CAST6_PARALLEL_BLOCKS];
+	struct crypt_priv crypt_ctx = {
+		.ctx = &ctx->crypt_ctx,
+		.fpu_enabled = false,
+	};
+	struct xts_crypt_req req = {
+		.tbuf = buf,
+		.tbuflen = sizeof(buf),
+
+		.tweak_ctx = &ctx->tweak_ctx,
+		.tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
+		.crypt_ctx = &crypt_ctx,
+		.crypt_fn = decrypt_callback,
+	};
+	int ret;
+
+	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+	ret = xts_crypt(desc, dst, src, nbytes, &req);
+	cast6_fpu_end(crypt_ctx.fpu_enabled);
+
+	return ret;
+}
+
+static struct crypto_alg cast6_algs[10] = { {
+	.cra_name		= "__ecb-cast6-avx",
+	.cra_driver_name	= "__driver-ecb-cast6-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast6_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.setkey		= cast6_setkey,
+			.encrypt	= ecb_encrypt,
+			.decrypt	= ecb_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__cbc-cast6-avx",
+	.cra_driver_name	= "__driver-cbc-cast6-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast6_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.setkey		= cast6_setkey,
+			.encrypt	= cbc_encrypt,
+			.decrypt	= cbc_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__ctr-cast6-avx",
+	.cra_driver_name	= "__driver-ctr-cast6-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct cast6_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= cast6_setkey,
+			.encrypt	= ctr_crypt,
+			.decrypt	= ctr_crypt,
+		},
+	},
+}, {
+	.cra_name		= "__lrw-cast6-avx",
+	.cra_driver_name	= "__driver-lrw-cast6-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast6_lrw_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_exit		= lrw_exit_tfm,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE +
+					  CAST6_BLOCK_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE +
+					  CAST6_BLOCK_SIZE,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= lrw_cast6_setkey,
+			.encrypt	= lrw_encrypt,
+			.decrypt	= lrw_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "__xts-cast6-avx",
+	.cra_driver_name	= "__driver-xts-cast6-avx",
+	.cra_priority		= 0,
+	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast6_xts_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_blkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_u = {
+		.blkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
+			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= xts_cast6_setkey,
+			.encrypt	= xts_encrypt,
+			.decrypt	= xts_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "ecb(cast6)",
+	.cra_driver_name	= "ecb-cast6-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "cbc(cast6)",
+	.cra_driver_name	= "cbc-cast6-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= __ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "ctr(cast6)",
+	.cra_driver_name	= "ctr-cast6-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_encrypt,
+			.geniv		= "chainiv",
+		},
+	},
+}, {
+	.cra_name		= "lrw(cast6)",
+	.cra_driver_name	= "lrw-cast6-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE +
+					  CAST6_BLOCK_SIZE,
+			.max_keysize	= CAST6_MAX_KEY_SIZE +
+					  CAST6_BLOCK_SIZE,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+}, {
+	.cra_name		= "xts(cast6)",
+	.cra_driver_name	= "xts-cast6-avx",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= CAST6_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct async_helper_ctx),
+	.cra_alignmask		= 0,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= ablk_init,
+	.cra_exit		= ablk_exit,
+	.cra_u = {
+		.ablkcipher = {
+			.min_keysize	= CAST6_MIN_KEY_SIZE * 2,
+			.max_keysize	= CAST6_MAX_KEY_SIZE * 2,
+			.ivsize		= CAST6_BLOCK_SIZE,
+			.setkey		= ablk_set_key,
+			.encrypt	= ablk_encrypt,
+			.decrypt	= ablk_decrypt,
+		},
+	},
+} };
+
+static int __init cast6_init(void)
+{
+	u64 xcr0;
+
+	if (!cpu_has_avx || !cpu_has_osxsave) {
+		pr_info("AVX instructions are not detected.\n");
+		return -ENODEV;
+	}
+
+	xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+	if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+		pr_info("AVX detected but unusable.\n");
+		return -ENODEV;
+	}
+
+	return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
+}
+
+static void __exit cast6_exit(void)
+{
+	crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
+}
+
+module_init(cast6_init);
+module_exit(cast6_exit);
+
+MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("cast6");
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
index b4bf0a63b52..6759dd1135b 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -150,7 +150,6 @@ static struct shash_alg ghash_alg = {
 		.cra_blocksize		= GHASH_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct ghash_ctx),
 		.cra_module		= THIS_MODULE,
-		.cra_list		= LIST_HEAD_INIT(ghash_alg.base.cra_list),
 	},
 };
 
@@ -288,7 +287,6 @@ static struct ahash_alg ghash_async_alg = {
 			.cra_blocksize		= GHASH_BLOCK_SIZE,
 			.cra_type		= &crypto_ahash_type,
 			.cra_module		= THIS_MODULE,
-			.cra_list		= LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list),
 			.cra_init		= ghash_async_init_tfm,
 			.cra_exit		= ghash_async_exit_tfm,
 		},
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 4854f0f31e4..30b3927bd73 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -110,7 +110,7 @@ static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
 		nbytes -= bsize;
 	} while (nbytes >= bsize);
 
-	u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
+	*(u128 *)walk->iv = *iv;
 	return nbytes;
 }
 
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
index bccb76d8098..a3a3c0205c1 100644
--- a/arch/x86/crypto/salsa20_glue.c
+++ b/arch/x86/crypto/salsa20_glue.c
@@ -97,7 +97,6 @@ static struct crypto_alg alg = {
 	.cra_ctxsize        =   sizeof(struct salsa20_ctx),
 	.cra_alignmask      =	3,
 	.cra_module         =   THIS_MODULE,
-	.cra_list           =   LIST_HEAD_INIT(alg.cra_list),
 	.cra_u              =   {
 		.blkcipher = {
 			.setkey         =   setkey,
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index b36bdac237e..3f543a04cf1 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -390,7 +390,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[0].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -410,7 +409,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -430,7 +428,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -451,7 +448,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[3].cra_list),
 	.cra_exit		= lrw_exit_tfm,
 	.cra_u = {
 		.blkcipher = {
@@ -475,7 +471,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[4].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
@@ -496,7 +491,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[5].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -518,7 +512,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[6].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -541,7 +534,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[7].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -565,7 +557,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[8].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -590,7 +581,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[9].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index d679c8675f4..9107a9908c4 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -393,7 +393,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[0].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -413,7 +412,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -433,7 +431,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE,
@@ -454,7 +451,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[3].cra_list),
 	.cra_exit		= lrw_exit_tfm,
 	.cra_u = {
 		.blkcipher = {
@@ -478,7 +474,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[4].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= SERPENT_MIN_KEY_SIZE * 2,
@@ -499,7 +494,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[5].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -521,7 +515,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[6].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -544,7 +537,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[7].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -568,7 +560,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[8].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -593,7 +584,6 @@ static struct crypto_alg serpent_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(serpent_algs[9].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index 35f45574390..1585abb13dd 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -4,6 +4,8 @@
  * Copyright (C) 2012 Johannes Goetzfried
  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
  *
+ * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -47,16 +49,22 @@
 #define RC2 %xmm6
 #define RD2 %xmm7
 
-#define RX %xmm8
-#define RY %xmm9
+#define RX0 %xmm8
+#define RY0 %xmm9
+
+#define RX1 %xmm10
+#define RY1 %xmm11
 
-#define RK1 %xmm10
-#define RK2 %xmm11
+#define RK1 %xmm12
+#define RK2 %xmm13
 
-#define RID1  %rax
-#define RID1b %al
-#define RID2  %rbx
-#define RID2b %bl
+#define RT %xmm14
+#define RR %xmm15
+
+#define RID1  %rbp
+#define RID1d %ebp
+#define RID2  %rsi
+#define RID2d %esi
 
 #define RGI1   %rdx
 #define RGI1bl %dl
@@ -65,6 +73,13 @@
 #define RGI2bl %cl
 #define RGI2bh %ch
 
+#define RGI3   %rax
+#define RGI3bl %al
+#define RGI3bh %ah
+#define RGI4   %rbx
+#define RGI4bl %bl
+#define RGI4bh %bh
+
 #define RGS1  %r8
 #define RGS1d %r8d
 #define RGS2  %r9
@@ -73,89 +88,123 @@
 #define RGS3d %r10d
 
 
-#define lookup_32bit(t0, t1, t2, t3, src, dst) \
-	movb		src ## bl,        RID1b;     \
-	movb		src ## bh,        RID2b;     \
-	movl		t0(CTX, RID1, 4), dst ## d;  \
-	xorl		t1(CTX, RID2, 4), dst ## d;  \
+#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
+	movzbl		src ## bl,        RID1d;     \
+	movzbl		src ## bh,        RID2d;     \
 	shrq $16,	src;                         \
-	movb		src ## bl,        RID1b;     \
-	movb		src ## bh,        RID2b;     \
+	movl		t0(CTX, RID1, 4), dst ## d;  \
+	movl		t1(CTX, RID2, 4), RID2d;     \
+	movzbl		src ## bl,        RID1d;     \
+	xorl		RID2d,            dst ## d;  \
+	movzbl		src ## bh,        RID2d;     \
+	interleave_op(il_reg);			     \
 	xorl		t2(CTX, RID1, 4), dst ## d;  \
 	xorl		t3(CTX, RID2, 4), dst ## d;
 
-#define G(a, x, t0, t1, t2, t3) \
-	vmovq		a,    RGI1;               \
-	vpsrldq $8,	a,    x;                  \
-	vmovq		x,    RGI2;               \
+#define dummy(d) /* do nothing */
+
+#define shr_next(reg) \
+	shrq $16,	reg;
+
+#define G(gi1, gi2, x, t0, t1, t2, t3) \
+	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1);  \
+	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2);  \
+	\
+	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none);      \
+	shlq $32,	RGS2;                                        \
+	orq		RGS1, RGS2;                                  \
+	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none);      \
+	shlq $32,	RGS1;                                        \
+	orq		RGS1, RGS3;
+
+#define round_head_2(a, b, x1, y1, x2, y2) \
+	vmovq		b ## 1, RGI3;           \
+	vpextrq $1,	b ## 1, RGI4;           \
 	\
-	lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \
-	shrq $16,	RGI1;                     \
-	lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \
-	shlq $32,	RGS2;                     \
-	orq		RGS1, RGS2;               \
+	G(RGI1, RGI2, x1, s0, s1, s2, s3);      \
+	vmovq		a ## 2, RGI1;           \
+	vpextrq $1,	a ## 2, RGI2;           \
+	vmovq		RGS2, x1;               \
+	vpinsrq $1,	RGS3, x1, x1;           \
 	\
-	lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \
-	shrq $16,	RGI2;                     \
-	lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \
-	shlq $32,	RGS3;                     \
-	orq		RGS1, RGS3;               \
+	G(RGI3, RGI4, y1, s1, s2, s3, s0);      \
+	vmovq		b ## 2, RGI3;           \
+	vpextrq $1,	b ## 2, RGI4;           \
+	vmovq		RGS2, y1;               \
+	vpinsrq $1,	RGS3, y1, y1;           \
 	\
-	vmovq		RGS2, x;                  \
-	vpinsrq $1,	RGS3, x, x;
+	G(RGI1, RGI2, x2, s0, s1, s2, s3);      \
+	vmovq		RGS2, x2;               \
+	vpinsrq $1,	RGS3, x2, x2;           \
+	\
+	G(RGI3, RGI4, y2, s1, s2, s3, s0);      \
+	vmovq		RGS2, y2;               \
+	vpinsrq $1,	RGS3, y2, y2;
 
-#define encround(a, b, c, d, x, y) \
-	G(a, x, s0, s1, s2, s3);           \
-	G(b, y, s1, s2, s3, s0);           \
+#define encround_tail(a, b, c, d, x, y, prerotate) \
 	vpaddd			x, y,   x; \
+	vpaddd			x, RK1, RT;\
+	prerotate(b);			   \
+	vpxor			RT, c,  c; \
 	vpaddd			y, x,   y; \
-	vpaddd			x, RK1, x; \
 	vpaddd			y, RK2, y; \
-	vpxor			x, c,   c; \
-	vpsrld $1,		c, x;      \
+	vpsrld $1,		c, RT;     \
 	vpslld $(32 - 1),	c, c;      \
-	vpor			c, x,   c; \
-	vpslld $1,		d, x;      \
-	vpsrld $(32 - 1),	d, d;      \
-	vpor			d, x,   d; \
-	vpxor			d, y,   d;
-
-#define decround(a, b, c, d, x, y) \
-	G(a, x, s0, s1, s2, s3);           \
-	G(b, y, s1, s2, s3, s0);           \
+	vpor			c, RT,  c; \
+	vpxor			d, y,   d; \
+
+#define decround_tail(a, b, c, d, x, y, prerotate) \
 	vpaddd			x, y,   x; \
+	vpaddd			x, RK1, RT;\
+	prerotate(a);			   \
+	vpxor			RT, c,  c; \
 	vpaddd			y, x,   y; \
 	vpaddd			y, RK2, y; \
 	vpxor			d, y,   d; \
 	vpsrld $1,		d, y;      \
 	vpslld $(32 - 1),	d, d;      \
 	vpor			d, y,   d; \
-	vpslld $1,		c, y;      \
-	vpsrld $(32 - 1),	c, c;      \
-	vpor			c, y,   c; \
-	vpaddd			x, RK1, x; \
-	vpxor			x, c,   c;
-
-#define encrypt_round(n, a, b, c, d) \
-	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;           \
-	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;           \
-	encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
-	encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
-
-#define decrypt_round(n, a, b, c, d) \
-	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;           \
-	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;           \
-	decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
-	decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
+
+#define rotate_1l(x) \
+	vpslld $1,		x, RR;     \
+	vpsrld $(32 - 1),	x, x;      \
+	vpor			x, RR,  x;
+
+#define preload_rgi(c) \
+	vmovq			c, RGI1; \
+	vpextrq $1,		c, RGI2;
+
+#define encrypt_round(n, a, b, c, d, preload, prerotate) \
+	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
+	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
+	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
+	encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
+	preload(c ## 1);                                         \
+	encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
+
+#define decrypt_round(n, a, b, c, d, preload, prerotate) \
+	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
+	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
+	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
+	decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
+	preload(c ## 1);                                         \
+	decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
 
 #define encrypt_cycle(n) \
-	encrypt_round((2*n), RA, RB, RC, RD);       \
-	encrypt_round(((2*n) + 1), RC, RD, RA, RB);
+	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
+	encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);
+
+#define encrypt_cycle_last(n) \
+	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
+	encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);
 
 #define decrypt_cycle(n) \
-	decrypt_round(((2*n) + 1), RC, RD, RA, RB); \
-	decrypt_round((2*n), RA, RB, RC, RD);
+	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
+	decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);
 
+#define decrypt_cycle_last(n) \
+	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
+	decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);
 
 #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
 	vpunpckldq		x1, x0, t0; \
@@ -216,17 +265,20 @@ __twofish_enc_blk_8way:
 	 *	%rcx: bool, if true: xor output
 	 */
 
+	pushq %rbp;
 	pushq %rbx;
 	pushq %rcx;
 
 	vmovdqu w(CTX), RK1;
 
 	leaq (4*4*4)(%rdx), %rax;
-	inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
-	inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
+	inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
+	preload_rgi(RA1);
+	rotate_1l(RD1);
+	inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
+	rotate_1l(RD2);
 
-	xorq RID1, RID1;
-	xorq RID2, RID2;
+	movq %rsi, %r11;
 
 	encrypt_cycle(0);
 	encrypt_cycle(1);
@@ -235,26 +287,27 @@ __twofish_enc_blk_8way:
 	encrypt_cycle(4);
 	encrypt_cycle(5);
 	encrypt_cycle(6);
-	encrypt_cycle(7);
+	encrypt_cycle_last(7);
 
 	vmovdqu (w+4*4)(CTX), RK1;
 
 	popq %rcx;
 	popq %rbx;
+	popq %rbp;
 
-	leaq (4*4*4)(%rsi), %rax;
+	leaq (4*4*4)(%r11), %rax;
 
 	testb %cl, %cl;
 	jnz __enc_xor8;
 
-	outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
-	outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+	outunpack_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
+	outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
 
 	ret;
 
 __enc_xor8:
-	outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
-	outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+	outunpack_xor_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
+	outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
 
 	ret;
 
@@ -269,16 +322,19 @@ twofish_dec_blk_8way:
 	 *	%rdx: src
 	 */
 
+	pushq %rbp;
 	pushq %rbx;
 
 	vmovdqu (w+4*4)(CTX), RK1;
 
 	leaq (4*4*4)(%rdx), %rax;
-	inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
-	inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+	inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
+	preload_rgi(RC1);
+	rotate_1l(RA1);
+	inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
+	rotate_1l(RA2);
 
-	xorq RID1, RID1;
-	xorq RID2, RID2;
+	movq %rsi, %r11;
 
 	decrypt_cycle(7);
 	decrypt_cycle(6);
@@ -287,14 +343,15 @@ twofish_dec_blk_8way:
 	decrypt_cycle(3);
 	decrypt_cycle(2);
 	decrypt_cycle(1);
-	decrypt_cycle(0);
+	decrypt_cycle_last(0);
 
 	vmovdqu (w)(CTX), RK1;
 
 	popq %rbx;
+	popq %rbp;
 
-	leaq (4*4*4)(%rsi), %rax;
-	outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
-	outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
+	leaq (4*4*4)(%r11), %rax;
+	outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
+	outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
 
 	ret;
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index 782b67ddaf6..e7708b5442e 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -378,7 +378,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[0].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -398,7 +397,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -418,7 +416,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -439,7 +436,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[3].cra_list),
 	.cra_exit		= lrw_twofish_exit_tfm,
 	.cra_u = {
 		.blkcipher = {
@@ -463,7 +459,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[4].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE * 2,
@@ -484,7 +479,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[5].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -506,7 +500,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[6].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -529,7 +522,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[7].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -553,7 +545,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[8].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
@@ -578,7 +569,6 @@ static struct crypto_alg twofish_algs[10] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_ablkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(twofish_algs[9].cra_list),
 	.cra_init		= ablk_init,
 	.cra_exit		= ablk_exit,
 	.cra_u = {
diff --git a/arch/x86/crypto/twofish_glue.c b/arch/x86/crypto/twofish_glue.c
index 359ae084275..0a520230350 100644
--- a/arch/x86/crypto/twofish_glue.c
+++ b/arch/x86/crypto/twofish_glue.c
@@ -70,7 +70,6 @@ static struct crypto_alg alg = {
 	.cra_ctxsize		=	sizeof(struct twofish_ctx),
 	.cra_alignmask		=	0,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	TF_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 15f9347316c..aa3eb358b7e 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -342,7 +342,6 @@ static struct crypto_alg tf_algs[5] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(tf_algs[0].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -362,7 +361,6 @@ static struct crypto_alg tf_algs[5] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(tf_algs[1].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -383,7 +381,6 @@ static struct crypto_alg tf_algs[5] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(tf_algs[2].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE,
@@ -404,7 +401,6 @@ static struct crypto_alg tf_algs[5] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(tf_algs[3].cra_list),
 	.cra_exit		= lrw_twofish_exit_tfm,
 	.cra_u = {
 		.blkcipher = {
@@ -426,7 +422,6 @@ static struct crypto_alg tf_algs[5] = { {
 	.cra_alignmask		= 0,
 	.cra_type		= &crypto_blkcipher_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(tf_algs[4].cra_list),
 	.cra_u = {
 		.blkcipher = {
 			.min_keysize	= TF_MIN_KEY_SIZE * 2,
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index f9c0d3ba9e8..1595d681343 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -22,7 +22,3 @@ header-y += sigcontext32.h
 header-y += ucontext.h
 header-y += vm86.h
 header-y += vsyscall.h
-
-genhdr-y += unistd_32.h
-genhdr-y += unistd_64.h
-genhdr-y += unistd_x32.h
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index f34261296ff..33880342223 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -409,7 +409,7 @@ extern struct apic *apic;
  * to enforce the order with in them.
  */
 #define apic_driver(sym)					\
-	static struct apic *__apicdrivers_##sym __used		\
+	static const struct apic *__apicdrivers_##sym __used		\
 	__aligned(sizeof(struct apic *))			\
 	__section(.apicdrivers) = { &sym }
 
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 58cb6d4085f..250b8774c15 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -309,9 +309,9 @@ static inline void atomic_or_long(unsigned long *v1, unsigned long v2)
 #define smp_mb__after_atomic_inc()	barrier()
 
 #ifdef CONFIG_X86_32
-# include "atomic64_32.h"
+# include <asm/atomic64_32.h>
 #else
-# include "atomic64_64.h"
+# include <asm/atomic64_64.h>
 #endif
 
 #endif /* _ASM_X86_ATOMIC_H */
diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
index 7f8422a28a4..0fa67503391 100644
--- a/arch/x86/include/asm/calling.h
+++ b/arch/x86/include/asm/calling.h
@@ -46,7 +46,7 @@ For 32-bit we have the following conventions - kernel is built with
 
 */
 
-#include "dwarf2.h"
+#include <asm/dwarf2.h>
 
 /*
  * 64-bit system call stack frame layout defines and helpers,
diff --git a/arch/x86/include/asm/checksum.h b/arch/x86/include/asm/checksum.h
index 848850fd7d6..5f5bb0f9736 100644
--- a/arch/x86/include/asm/checksum.h
+++ b/arch/x86/include/asm/checksum.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "checksum_32.h"
+# include <asm/checksum_32.h>
 #else
-# include "checksum_64.h"
+# include <asm/checksum_64.h>
 #endif
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index 99480e55973..8d871eaddb6 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -138,9 +138,9 @@ extern void __add_wrong_size(void)
 	__raw_cmpxchg((ptr), (old), (new), (size), "")
 
 #ifdef CONFIG_X86_32
-# include "cmpxchg_32.h"
+# include <asm/cmpxchg_32.h>
 #else
-# include "cmpxchg_64.h"
+# include <asm/cmpxchg_64.h>
 #endif
 
 #ifdef __HAVE_ARCH_CMPXCHG
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index fedf32b73e6..59c6c401f79 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -41,6 +41,7 @@ typedef s64 __attribute__((aligned(4))) compat_s64;
 typedef u32		compat_uint_t;
 typedef u32		compat_ulong_t;
 typedef u64 __attribute__((aligned(4))) compat_u64;
+typedef u32		compat_uptr_t;
 
 struct compat_timespec {
 	compat_time_t	tv_sec;
@@ -124,6 +125,78 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32               compat_sigset_word;
 
+typedef union compat_sigval {
+	compat_int_t	sival_int;
+	compat_uptr_t	sival_ptr;
+} compat_sigval_t;
+
+typedef struct compat_siginfo {
+	int si_signo;
+	int si_errno;
+	int si_code;
+
+	union {
+		int _pad[128/sizeof(int) - 3];
+
+		/* kill() */
+		struct {
+			unsigned int _pid;	/* sender's pid */
+			unsigned int _uid;	/* sender's uid */
+		} _kill;
+
+		/* POSIX.1b timers */
+		struct {
+			compat_timer_t _tid;	/* timer id */
+			int _overrun;		/* overrun count */
+			compat_sigval_t _sigval;	/* same as below */
+			int _sys_private;	/* not to be passed to user */
+			int _overrun_incr;	/* amount to add to overrun */
+		} _timer;
+
+		/* POSIX.1b signals */
+		struct {
+			unsigned int _pid;	/* sender's pid */
+			unsigned int _uid;	/* sender's uid */
+			compat_sigval_t _sigval;
+		} _rt;
+
+		/* SIGCHLD */
+		struct {
+			unsigned int _pid;	/* which child */
+			unsigned int _uid;	/* sender's uid */
+			int _status;		/* exit code */
+			compat_clock_t _utime;
+			compat_clock_t _stime;
+		} _sigchld;
+
+		/* SIGCHLD (x32 version) */
+		struct {
+			unsigned int _pid;	/* which child */
+			unsigned int _uid;	/* sender's uid */
+			int _status;		/* exit code */
+			compat_s64 _utime;
+			compat_s64 _stime;
+		} _sigchld_x32;
+
+		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
+		struct {
+			unsigned int _addr;	/* faulting insn/memory ref. */
+		} _sigfault;
+
+		/* SIGPOLL */
+		struct {
+			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
+			int _fd;
+		} _sigpoll;
+
+		struct {
+			unsigned int _call_addr; /* calling insn */
+			int _syscall;	/* triggering system call number */
+			unsigned int _arch;	/* AUDIT_ARCH_* of syscall */
+		} _sigsys;
+	} _sifields;
+} compat_siginfo_t;
+
 #define COMPAT_OFF_T_MAX	0x7fffffff
 #define COMPAT_LOFF_T_MAX	0x7fffffffffffffffL
 
@@ -209,7 +282,6 @@ typedef struct user_regs_struct32 compat_elf_gregset_t;
  * as pointers because the syscall entry code will have
  * appropriately converted them already.
  */
-typedef	u32		compat_uptr_t;
 
 static inline void __user *compat_ptr(compat_uptr_t uptr)
 {
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 16cae425d1f..8c297aa53ee 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -4,7 +4,9 @@
 #ifndef _ASM_X86_CPUFEATURE_H
 #define _ASM_X86_CPUFEATURE_H
 
+#ifndef _ASM_X86_REQUIRED_FEATURES_H
 #include <asm/required-features.h>
+#endif
 
 #define NCAPINTS	10	/* N 32-bit words worth of info */
 
diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h
index b04cbdb138c..e6232773ce4 100644
--- a/arch/x86/include/asm/ia32.h
+++ b/arch/x86/include/asm/ia32.h
@@ -86,73 +86,6 @@ struct stat64 {
 	unsigned long long	st_ino;
 } __attribute__((packed));
 
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[((128 / sizeof(int)) - 3)];
-
-		/* kill() */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-			int _overrun_incr;	/* amount to add to overrun */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGCHLD (x32 version) */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_s64 _utime;
-			compat_s64 _stime;
-		} _sigchld_x32;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			unsigned int _addr;	/* faulting insn/memory ref. */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		struct {
-			unsigned int _call_addr; /* calling insn */
-			int _syscall;	/* triggering system call number */
-			unsigned int _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define IA32_STACK_TOP IA32_PAGE_OFFSET
 
 #ifdef __KERNEL__
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index 41e08cb6a09..a65ec29e6ff 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -41,6 +41,7 @@
 #define __KVM_HAVE_DEBUGREGS
 #define __KVM_HAVE_XSAVE
 #define __KVM_HAVE_XCRS
+#define __KVM_HAVE_READONLY_MEM
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index c764f43b71c..15f960c06ff 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -86,6 +86,19 @@ struct x86_instruction_info {
 
 struct x86_emulate_ops {
 	/*
+	 * read_gpr: read a general purpose register (rax - r15)
+	 *
+	 * @reg: gpr number.
+	 */
+	ulong (*read_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg);
+	/*
+	 * write_gpr: write a general purpose register (rax - r15)
+	 *
+	 * @reg: gpr number.
+	 * @val: value to write.
+	 */
+	void (*write_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val);
+	/*
 	 * read_std: Read bytes of standard (non-emulated/special) memory.
 	 *           Used for descriptor reading.
 	 *  @addr:  [IN ] Linear address from which to read.
@@ -200,8 +213,9 @@ typedef u32 __attribute__((vector_size(16))) sse128_t;
 
 /* Type, address-of, and value of an instruction's operand. */
 struct operand {
-	enum { OP_REG, OP_MEM, OP_IMM, OP_XMM, OP_MM, OP_NONE } type;
+	enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_MM, OP_NONE } type;
 	unsigned int bytes;
+	unsigned int count;
 	union {
 		unsigned long orig_val;
 		u64 orig_val64;
@@ -221,6 +235,7 @@ struct operand {
 		char valptr[sizeof(unsigned long) + 2];
 		sse128_t vec_val;
 		u64 mm_val;
+		void *data;
 	};
 };
 
@@ -236,14 +251,23 @@ struct read_cache {
 	unsigned long end;
 };
 
+/* Execution mode, passed to the emulator. */
+enum x86emul_mode {
+	X86EMUL_MODE_REAL,	/* Real mode.             */
+	X86EMUL_MODE_VM86,	/* Virtual 8086 mode.     */
+	X86EMUL_MODE_PROT16,	/* 16-bit protected mode. */
+	X86EMUL_MODE_PROT32,	/* 32-bit protected mode. */
+	X86EMUL_MODE_PROT64,	/* 64-bit (long) mode.    */
+};
+
 struct x86_emulate_ctxt {
-	struct x86_emulate_ops *ops;
+	const struct x86_emulate_ops *ops;
 
 	/* Register state before/after emulation. */
 	unsigned long eflags;
 	unsigned long eip; /* eip before instruction emulation */
 	/* Emulated execution mode, represented by an X86EMUL_MODE value. */
-	int mode;
+	enum x86emul_mode mode;
 
 	/* interruptibility state, as a result of execution of STI or MOV SS */
 	int interruptibility;
@@ -281,8 +305,10 @@ struct x86_emulate_ctxt {
 	bool rip_relative;
 	unsigned long _eip;
 	struct operand memop;
+	u32 regs_valid;  /* bitmaps of registers in _regs[] that can be read */
+	u32 regs_dirty;  /* bitmaps of registers in _regs[] that have been written */
 	/* Fields above regs are cleared together. */
-	unsigned long regs[NR_VCPU_REGS];
+	unsigned long _regs[NR_VCPU_REGS];
 	struct operand *memopp;
 	struct fetch_cache fetch;
 	struct read_cache io_read;
@@ -293,17 +319,6 @@ struct x86_emulate_ctxt {
 #define REPE_PREFIX	0xf3
 #define REPNE_PREFIX	0xf2
 
-/* Execution mode, passed to the emulator. */
-#define X86EMUL_MODE_REAL     0	/* Real mode.             */
-#define X86EMUL_MODE_VM86     1	/* Virtual 8086 mode.     */
-#define X86EMUL_MODE_PROT16   2	/* 16-bit protected mode. */
-#define X86EMUL_MODE_PROT32   4	/* 32-bit protected mode. */
-#define X86EMUL_MODE_PROT64   8	/* 64-bit (long) mode.    */
-
-/* any protected mode   */
-#define X86EMUL_MODE_PROT     (X86EMUL_MODE_PROT16|X86EMUL_MODE_PROT32| \
-			       X86EMUL_MODE_PROT64)
-
 /* CPUID vendors */
 #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541
 #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163
@@ -394,4 +409,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
 			 u16 tss_selector, int idt_index, int reason,
 			 bool has_error_code, u32 error_code);
 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq);
+void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt);
+void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt);
+
 #endif /* _ASM_X86_KVM_X86_EMULATE_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 1eaa6b05667..b2e11f45243 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -271,10 +271,24 @@ struct kvm_mmu {
 	union kvm_mmu_page_role base_role;
 	bool direct_map;
 
+	/*
+	 * Bitmap; bit set = permission fault
+	 * Byte index: page fault error code [4:1]
+	 * Bit index: pte permissions in ACC_* format
+	 */
+	u8 permissions[16];
+
 	u64 *pae_root;
 	u64 *lm_root;
 	u64 rsvd_bits_mask[2][4];
 
+	/*
+	 * Bitmap: bit set = last pte in walk
+	 * index[0:1]: level (zero-based)
+	 * index[2]: pte.ps
+	 */
+	u8 last_pte_bitmap;
+
 	bool nx;
 
 	u64 pdptrs[4]; /* pae */
@@ -398,12 +412,15 @@ struct kvm_vcpu_arch {
 	struct x86_emulate_ctxt emulate_ctxt;
 	bool emulate_regs_need_sync_to_vcpu;
 	bool emulate_regs_need_sync_from_vcpu;
+	int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
 
 	gpa_t time;
 	struct pvclock_vcpu_time_info hv_clock;
 	unsigned int hw_tsc_khz;
 	unsigned int time_offset;
 	struct page *time_page;
+	/* set guest stopped flag in pvclock flags field */
+	bool pvclock_set_guest_stopped_request;
 
 	struct {
 		u64 msr_val;
@@ -438,6 +455,7 @@ struct kvm_vcpu_arch {
 	unsigned long dr6;
 	unsigned long dr7;
 	unsigned long eff_db[KVM_NR_DB_REGS];
+	unsigned long guest_debug_dr7;
 
 	u64 mcg_cap;
 	u64 mcg_status;
@@ -484,14 +502,24 @@ struct kvm_vcpu_arch {
 };
 
 struct kvm_lpage_info {
-	unsigned long rmap_pde;
 	int write_count;
 };
 
 struct kvm_arch_memory_slot {
+	unsigned long *rmap[KVM_NR_PAGE_SIZES];
 	struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
 };
 
+struct kvm_apic_map {
+	struct rcu_head rcu;
+	u8 ldr_bits;
+	/* fields bellow are used to decode ldr values in different modes */
+	u32 cid_shift, cid_mask, lid_mask;
+	struct kvm_lapic *phys_map[256];
+	/* first index is cluster id second is cpu id in a cluster */
+	struct kvm_lapic *logical_map[16][16];
+};
+
 struct kvm_arch {
 	unsigned int n_used_mmu_pages;
 	unsigned int n_requested_mmu_pages;
@@ -509,6 +537,8 @@ struct kvm_arch {
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
 	int vapics_in_nmi_mode;
+	struct mutex apic_map_lock;
+	struct kvm_apic_map *apic_map;
 
 	unsigned int tss_addr;
 	struct page *apic_access_page;
@@ -602,8 +632,7 @@ struct kvm_x86_ops {
 	void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
 	void (*vcpu_put)(struct kvm_vcpu *vcpu);
 
-	void (*set_guest_debug)(struct kvm_vcpu *vcpu,
-				struct kvm_guest_debug *dbg);
+	void (*update_db_bp_intercept)(struct kvm_vcpu *vcpu);
 	int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata);
 	int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data);
 	u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
@@ -941,6 +970,7 @@ extern bool kvm_rebooting;
 
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
+int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
 int kvm_age_hva(struct kvm *kvm, unsigned long hva);
 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 2f7712e08b1..eb3e9d85e1f 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -102,21 +102,21 @@ struct kvm_vcpu_pv_apf_data {
 extern void kvmclock_init(void);
 extern int kvm_register_clock(char *txt);
 
-#ifdef CONFIG_KVM_CLOCK
+#ifdef CONFIG_KVM_GUEST
 bool kvm_check_and_clear_guest_paused(void);
 #else
 static inline bool kvm_check_and_clear_guest_paused(void)
 {
 	return false;
 }
-#endif /* CONFIG_KVMCLOCK */
+#endif /* CONFIG_KVM_GUEST */
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a
  * trap that we will then rewrite to the appropriate instruction.
  */
 #define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1"
 
-/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun
+/* For KVM hypercalls, a three-byte sequence of either the vmcall or the vmmcall
  * instruction.  The hypervisor may replace it with something else but only the
  * instructions are guaranteed to be supported.
  *
diff --git a/arch/x86/include/asm/mmzone.h b/arch/x86/include/asm/mmzone.h
index 64217ea16a3..d497bc425ca 100644
--- a/arch/x86/include/asm/mmzone.h
+++ b/arch/x86/include/asm/mmzone.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "mmzone_32.h"
+# include <asm/mmzone_32.h>
 #else
-# include "mmzone_64.h"
+# include <asm/mmzone_64.h>
 #endif
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 957ec87385a..fbee9714d9a 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -248,6 +248,9 @@
 
 #define MSR_IA32_PERF_STATUS		0x00000198
 #define MSR_IA32_PERF_CTL		0x00000199
+#define MSR_AMD_PSTATE_DEF_BASE		0xc0010064
+#define MSR_AMD_PERF_STATUS		0xc0010063
+#define MSR_AMD_PERF_CTL		0xc0010062
 
 #define MSR_IA32_MPERF			0x000000e7
 #define MSR_IA32_APERF			0x000000e8
diff --git a/arch/x86/include/asm/mutex.h b/arch/x86/include/asm/mutex.h
index a731b9c573a..7d3a4827539 100644
--- a/arch/x86/include/asm/mutex.h
+++ b/arch/x86/include/asm/mutex.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "mutex_32.h"
+# include <asm/mutex_32.h>
 #else
-# include "mutex_64.h"
+# include <asm/mutex_64.h>
 #endif
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index bfacd2ccf65..49119fcea2d 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -53,9 +53,9 @@ static inline int numa_cpu_node(int cpu)
 #endif	/* CONFIG_NUMA */
 
 #ifdef CONFIG_X86_32
-# include "numa_32.h"
+# include <asm/numa_32.h>
 #else
-# include "numa_64.h"
+# include <asm/numa_64.h>
 #endif
 
 #ifdef CONFIG_NUMA
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index df75d07571c..6e41b934392 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -141,7 +141,7 @@ void default_restore_msi_irqs(struct pci_dev *dev, int irq);
 #endif  /* __KERNEL__ */
 
 #ifdef CONFIG_X86_64
-#include "pci_64.h"
+#include <asm/pci_64.h>
 #endif
 
 /* implement the pci_ DMA API in terms of the generic device dma_ one */
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 49afb3f41eb..fc994846529 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -384,9 +384,9 @@ pte_t *populate_extra_pte(unsigned long vaddr);
 #endif	/* __ASSEMBLY__ */
 
 #ifdef CONFIG_X86_32
-# include "pgtable_32.h"
+# include <asm/pgtable_32.h>
 #else
-# include "pgtable_64.h"
+# include <asm/pgtable_64.h>
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index db8fec6d295..ec8a1fc9505 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -174,9 +174,9 @@
 #endif
 
 #ifdef CONFIG_X86_32
-# include "pgtable_32_types.h"
+# include <asm/pgtable_32_types.h>
 #else
-# include "pgtable_64_types.h"
+# include <asm/pgtable_64_types.h>
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/arch/x86/include/asm/posix_types.h b/arch/x86/include/asm/posix_types.h
index 7ef7c3020e5..bad3665c25f 100644
--- a/arch/x86/include/asm/posix_types.h
+++ b/arch/x86/include/asm/posix_types.h
@@ -1,15 +1,15 @@
 #ifdef __KERNEL__
 # ifdef CONFIG_X86_32
-#  include "posix_types_32.h"
+#  include <asm/posix_types_32.h>
 # else
-#  include "posix_types_64.h"
+#  include <asm/posix_types_64.h>
 # endif
 #else
 # ifdef __i386__
-#  include "posix_types_32.h"
+#  include <asm/posix_types_32.h>
 # elif defined(__ILP32__)
-#  include "posix_types_x32.h"
+#  include <asm/posix_types_x32.h>
 # else
-#  include "posix_types_64.h"
+#  include <asm/posix_types_64.h>
 # endif
 #endif
diff --git a/arch/x86/include/asm/seccomp.h b/arch/x86/include/asm/seccomp.h
index c62e58a5a90..0f3d7f09922 100644
--- a/arch/x86/include/asm/seccomp.h
+++ b/arch/x86/include/asm/seccomp.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "seccomp_32.h"
+# include <asm/seccomp_32.h>
 #else
-# include "seccomp_64.h"
+# include <asm/seccomp_64.h>
 #endif
diff --git a/arch/x86/include/asm/string.h b/arch/x86/include/asm/string.h
index 6dfd6d9373a..09224d7a586 100644
--- a/arch/x86/include/asm/string.h
+++ b/arch/x86/include/asm/string.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "string_32.h"
+# include <asm/string_32.h>
 #else
-# include "string_64.h"
+# include <asm/string_64.h>
 #endif
diff --git a/arch/x86/include/asm/suspend.h b/arch/x86/include/asm/suspend.h
index 9bd521fe457..2fab6c2c357 100644
--- a/arch/x86/include/asm/suspend.h
+++ b/arch/x86/include/asm/suspend.h
@@ -1,5 +1,5 @@
 #ifdef CONFIG_X86_32
-# include "suspend_32.h"
+# include <asm/suspend_32.h>
 #else
-# include "suspend_64.h"
+# include <asm/suspend_64.h>
 #endif
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index a91acfbb1a9..7ccf8d13153 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -589,9 +589,9 @@ extern struct movsl_mask {
 #define ARCH_HAS_NOCACHE_UACCESS 1
 
 #ifdef CONFIG_X86_32
-# include "uaccess_32.h"
+# include <asm/uaccess_32.h>
 #else
-# include "uaccess_64.h"
+# include <asm/uaccess_64.h>
 #endif
 
 #endif /* _ASM_X86_UACCESS_H */
diff --git a/arch/x86/include/asm/user.h b/arch/x86/include/asm/user.h
index 24532c7da3d..ccab4af1646 100644
--- a/arch/x86/include/asm/user.h
+++ b/arch/x86/include/asm/user.h
@@ -2,9 +2,9 @@
 #define _ASM_X86_USER_H
 
 #ifdef CONFIG_X86_32
-# include "user_32.h"
+# include <asm/user_32.h>
 #else
-# include "user_64.h"
+# include <asm/user_64.h>
 #endif
 
 #include <asm/types.h>
diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h
index cbf0c9d50b9..1707cfa928f 100644
--- a/arch/x86/include/asm/xen/interface.h
+++ b/arch/x86/include/asm/xen/interface.h
@@ -47,6 +47,10 @@
 #endif
 
 #ifndef __ASSEMBLY__
+/* Explicitly size integers that represent pfns in the public interface
+ * with Xen so that on ARM we can have one ABI that works for 32 and 64
+ * bit guests. */
+typedef unsigned long xen_pfn_t;
 /* Guest handles for primitive C types. */
 __DEFINE_GUEST_HANDLE(uchar, unsigned char);
 __DEFINE_GUEST_HANDLE(uint,  unsigned int);
@@ -57,6 +61,7 @@ DEFINE_GUEST_HANDLE(long);
 DEFINE_GUEST_HANDLE(void);
 DEFINE_GUEST_HANDLE(uint64_t);
 DEFINE_GUEST_HANDLE(uint32_t);
+DEFINE_GUEST_HANDLE(xen_pfn_t);
 #endif
 
 #ifndef HYPERVISOR_VIRT_START
@@ -116,11 +121,13 @@ struct arch_shared_info {
 #endif	/* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_32
-#include "interface_32.h"
+#include <asm/xen/interface_32.h>
 #else
-#include "interface_64.h"
+#include <asm/xen/interface_64.h>
 #endif
 
+#include <asm/pvclock-abi.h>
+
 #ifndef __ASSEMBLY__
 /*
  * The following is all CPU context. Note that the fpu_ctxt block is filled
diff --git a/arch/x86/include/asm/xen/swiotlb-xen.h b/arch/x86/include/asm/xen/swiotlb-xen.h
index 1be1ab7d6a4..ee52fcac6f7 100644
--- a/arch/x86/include/asm/xen/swiotlb-xen.h
+++ b/arch/x86/include/asm/xen/swiotlb-xen.h
@@ -5,10 +5,12 @@
 extern int xen_swiotlb;
 extern int __init pci_xen_swiotlb_detect(void);
 extern void __init pci_xen_swiotlb_init(void);
+extern int pci_xen_swiotlb_init_late(void);
 #else
 #define xen_swiotlb (0)
 static inline int __init pci_xen_swiotlb_detect(void) { return 0; }
 static inline void __init pci_xen_swiotlb_init(void) { }
+static inline int pci_xen_swiotlb_init_late(void) { return -ENXIO; }
 #endif
 
 #endif /* _ASM_X86_SWIOTLB_XEN_H */
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
index 7fcf6f3dbcc..f8fde90bc45 100644
--- a/arch/x86/include/asm/xor.h
+++ b/arch/x86/include/asm/xor.h
@@ -3,8 +3,8 @@
 # include <asm-generic/xor.h>
 #else
 #ifdef CONFIG_X86_32
-# include "xor_32.h"
+# include <asm/xor_32.h>
 #else
-# include "xor_64.h"
+# include <asm/xor_64.h>
 #endif
 #endif
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
index aabd5850bdb..f79cb7ec0e0 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/arch/x86/include/asm/xor_32.h
@@ -822,7 +822,7 @@ static struct xor_block_template xor_block_pIII_sse = {
 };
 
 /* Also try the AVX routines */
-#include "xor_avx.h"
+#include <asm/xor_avx.h>
 
 /* Also try the generic routines.  */
 #include <asm-generic/xor.h>
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
index 5fc06d0b7eb..87ac522c4af 100644
--- a/arch/x86/include/asm/xor_64.h
+++ b/arch/x86/include/asm/xor_64.h
@@ -306,7 +306,7 @@ static struct xor_block_template xor_block_sse = {
 
 
 /* Also try the AVX routines */
-#include "xor_avx.h"
+#include <asm/xor_avx.h>
 
 #undef XOR_TRY_TEMPLATES
 #define XOR_TRY_TEMPLATES			\
diff --git a/arch/x86/include/uapi/asm/Kbuild b/arch/x86/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..83b6e9a0dce
--- /dev/null
+++ b/arch/x86/include/uapi/asm/Kbuild
@@ -0,0 +1,6 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
+genhdr-y += unistd_32.h
+genhdr-y += unistd_64.h
+genhdr-y += unistd_x32.h
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 8d7a619718b..a48ea05157d 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -81,8 +81,7 @@ obj-$(CONFIG_DEBUG_RODATA_TEST)	+= test_rodata.o
 obj-$(CONFIG_DEBUG_NX_TEST)	+= test_nx.o
 obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
 
-obj-$(CONFIG_KVM_GUEST)		+= kvm.o
-obj-$(CONFIG_KVM_CLOCK)		+= kvmclock.o
+obj-$(CONFIG_KVM_GUEST)		+= kvm.o kvmclock.o
 obj-$(CONFIG_PARAVIRT)		+= paravirt.o paravirt_patch_$(BITS).o
 obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
 obj-$(CONFIG_PARAVIRT_CLOCK)	+= pvclock.o
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index bc552cff257..a65829ac2b9 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -30,7 +30,7 @@
 
 static int numachip_system __read_mostly;
 
-static struct apic apic_numachip __read_mostly;
+static const struct apic apic_numachip __read_mostly;
 
 static unsigned int get_apic_id(unsigned long x)
 {
@@ -199,7 +199,7 @@ static int numachip_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 	return 0;
 }
 
-static struct apic apic_numachip __refconst = {
+static const struct apic apic_numachip __refconst = {
 
 	.name				= "NumaConnect system",
 	.probe				= numachip_probe,
diff --git a/arch/x86/kernel/cpu/mkcapflags.pl b/arch/x86/kernel/cpu/mkcapflags.pl
index c7b3fe2d72e..091972ef49d 100644
--- a/arch/x86/kernel/cpu/mkcapflags.pl
+++ b/arch/x86/kernel/cpu/mkcapflags.pl
@@ -8,7 +8,10 @@
 open(IN, "< $in\0")   or die "$0: cannot open: $in: $!\n";
 open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n";
 
-print OUT "#include <asm/cpufeature.h>\n\n";
+print OUT "#ifndef _ASM_X86_CPUFEATURE_H\n";
+print OUT "#include <asm/cpufeature.h>\n";
+print OUT "#endif\n";
+print OUT "\n";
 print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n";
 
 %features = ();
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index c1d61ee4b4f..b3e5e51bc90 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -354,6 +354,7 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
 	kvm_pv_disable_apf();
+	kvm_disable_steal_time();
 }
 
 static int kvm_pv_reboot_notify(struct notifier_block *nb,
@@ -396,9 +397,7 @@ void kvm_disable_steal_time(void)
 #ifdef CONFIG_SMP
 static void __init kvm_smp_prepare_boot_cpu(void)
 {
-#ifdef CONFIG_KVM_CLOCK
 	WARN_ON(kvm_register_clock("primary cpu clock"));
-#endif
 	kvm_guest_cpu_init();
 	native_smp_prepare_boot_cpu();
 }
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index af6db6ec5b2..4929c1be0ac 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -225,7 +225,7 @@ static struct platform_device rtc_device = {
 static __init int add_rtc_cmos(void)
 {
 #ifdef CONFIG_PNP
-	static const char *ids[] __initconst =
+	static const char * const  const ids[] __initconst =
 	    { "PNP0b00", "PNP0b01", "PNP0b02", };
 	struct pnp_dev *dev;
 	struct pnp_id *id;
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 4f165479c45..d609be046b5 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -957,7 +957,7 @@ void __init setup_arch(char **cmdline_p)
 	initmem_init();
 	memblock_find_dma_reserve();
 
-#ifdef CONFIG_KVM_CLOCK
+#ifdef CONFIG_KVM_GUEST
 	kvmclock_init();
 #endif
 
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index a28f338843e..586f0005980 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -20,6 +20,7 @@ if VIRTUALIZATION
 config KVM
 	tristate "Kernel-based Virtual Machine (KVM) support"
 	depends on HAVE_KVM
+	depends on HIGH_RES_TIMERS
 	# for device assignment:
 	depends on PCI
 	# for TASKSTATS/TASK_DELAY_ACCT:
@@ -37,6 +38,7 @@ config KVM
 	select TASK_DELAY_ACCT
 	select PERF_EVENTS
 	select HAVE_KVM_MSI
+	select HAVE_KVM_CPU_RELAX_INTERCEPT
 	---help---
 	  Support hosting fully virtualized guest machines using hardware
 	  virtualization extensions.  You will need a fairly recent
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 4f579e8dcac..04d30401c5c 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -12,7 +12,7 @@ kvm-$(CONFIG_IOMMU_API)	+= $(addprefix ../../../virt/kvm/, iommu.o)
 kvm-$(CONFIG_KVM_ASYNC_PF)	+= $(addprefix ../../../virt/kvm/, async_pf.o)
 
 kvm-y			+= x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
-			   i8254.o timer.o cpuid.o pmu.o
+			   i8254.o cpuid.o pmu.o
 kvm-intel-y		+= vmx.o
 kvm-amd-y		+= svm.o
 
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 0595f1397b7..ec79e773342 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -316,7 +316,7 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 	}
 	case 7: {
 		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
-		/* Mask ebx against host capbability word 9 */
+		/* Mask ebx against host capability word 9 */
 		if (index == 0) {
 			entry->ebx &= kvm_supported_word9_x86_features;
 			cpuid_mask(&entry->ebx, 9);
@@ -397,8 +397,8 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 		break;
 	}
 	case KVM_CPUID_SIGNATURE: {
-		char signature[12] = "KVMKVMKVM\0\0";
-		u32 *sigptr = (u32 *)signature;
+		static const char signature[12] = "KVMKVMKVM\0\0";
+		const u32 *sigptr = (const u32 *)signature;
 		entry->eax = KVM_CPUID_FEATURES;
 		entry->ebx = sigptr[0];
 		entry->ecx = sigptr[1];
@@ -484,10 +484,10 @@ struct kvm_cpuid_param {
 	u32 func;
 	u32 idx;
 	bool has_leaf_count;
-	bool (*qualifier)(struct kvm_cpuid_param *param);
+	bool (*qualifier)(const struct kvm_cpuid_param *param);
 };
 
-static bool is_centaur_cpu(struct kvm_cpuid_param *param)
+static bool is_centaur_cpu(const struct kvm_cpuid_param *param)
 {
 	return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR;
 }
@@ -498,7 +498,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
 	struct kvm_cpuid_entry2 *cpuid_entries;
 	int limit, nent = 0, r = -E2BIG, i;
 	u32 func;
-	static struct kvm_cpuid_param param[] = {
+	static const struct kvm_cpuid_param param[] = {
 		{ .func = 0, .has_leaf_count = true },
 		{ .func = 0x80000000, .has_leaf_count = true },
 		{ .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true },
@@ -517,7 +517,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
 
 	r = 0;
 	for (i = 0; i < ARRAY_SIZE(param); i++) {
-		struct kvm_cpuid_param *ent = &param[i];
+		const struct kvm_cpuid_param *ent = &param[i];
 
 		if (ent->qualifier && !ent->qualifier(ent))
 			continue;
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index a3b57a27be8..39171cb307e 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -161,9 +161,9 @@ struct opcode {
 	u64 intercept : 8;
 	union {
 		int (*execute)(struct x86_emulate_ctxt *ctxt);
-		struct opcode *group;
-		struct group_dual *gdual;
-		struct gprefix *gprefix;
+		const struct opcode *group;
+		const struct group_dual *gdual;
+		const struct gprefix *gprefix;
 	} u;
 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
 };
@@ -202,6 +202,42 @@ struct gprefix {
 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
 #define EFLG_RESERVED_ONE_MASK 2
 
+static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+	if (!(ctxt->regs_valid & (1 << nr))) {
+		ctxt->regs_valid |= 1 << nr;
+		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
+	}
+	return ctxt->_regs[nr];
+}
+
+static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+	ctxt->regs_valid |= 1 << nr;
+	ctxt->regs_dirty |= 1 << nr;
+	return &ctxt->_regs[nr];
+}
+
+static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
+{
+	reg_read(ctxt, nr);
+	return reg_write(ctxt, nr);
+}
+
+static void writeback_registers(struct x86_emulate_ctxt *ctxt)
+{
+	unsigned reg;
+
+	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
+		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
+}
+
+static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
+{
+	ctxt->regs_dirty = 0;
+	ctxt->regs_valid = 0;
+}
+
 /*
  * Instruction emulation:
  * Most instructions are emulated directly via a fragment of inline assembly
@@ -374,8 +410,8 @@ struct gprefix {
 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex)			\
 	do {								\
 		unsigned long _tmp;					\
-		ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX];		\
-		ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX];		\
+		ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX);		\
+		ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX);		\
 									\
 		__asm__ __volatile__ (					\
 			_PRE_EFLAGS("0", "5", "1")			\
@@ -494,7 +530,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, in
 
 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
 {
-	masked_increment(&ctxt->regs[VCPU_REGS_RSP], stack_mask(ctxt), inc);
+	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
 }
 
 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
@@ -632,8 +668,6 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
 
 	la = seg_base(ctxt, addr.seg) + addr.ea;
 	switch (ctxt->mode) {
-	case X86EMUL_MODE_REAL:
-		break;
 	case X86EMUL_MODE_PROT64:
 		if (((signed long)la << 16) >> 16 != la)
 			return emulate_gp(ctxt, 0);
@@ -655,7 +689,7 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
 			if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
 				goto bad;
 		} else {
-			/* exapand-down segment */
+			/* expand-down segment */
 			if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
 				goto bad;
 			lim = desc.d ? 0xffffffff : 0xffff;
@@ -663,7 +697,10 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
 				goto bad;
 		}
 		cpl = ctxt->ops->cpl(ctxt);
-		rpl = sel & 3;
+		if (ctxt->mode == X86EMUL_MODE_REAL)
+			rpl = 0;
+		else
+			rpl = sel & 3;
 		cpl = max(cpl, rpl);
 		if (!(desc.type & 8)) {
 			/* data segment */
@@ -688,9 +725,9 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
 	return X86EMUL_CONTINUE;
 bad:
 	if (addr.seg == VCPU_SREG_SS)
-		return emulate_ss(ctxt, addr.seg);
+		return emulate_ss(ctxt, sel);
 	else
-		return emulate_gp(ctxt, addr.seg);
+		return emulate_gp(ctxt, sel);
 }
 
 static int linearize(struct x86_emulate_ctxt *ctxt,
@@ -786,14 +823,15 @@ static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
  * pointer into the block that addresses the relevant register.
  * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
  */
-static void *decode_register(u8 modrm_reg, unsigned long *regs,
+static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
 			     int highbyte_regs)
 {
 	void *p;
 
-	p = &regs[modrm_reg];
 	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
-		p = (unsigned char *)&regs[modrm_reg & 3] + 1;
+		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
+	else
+		p = reg_rmw(ctxt, modrm_reg);
 	return p;
 }
 
@@ -871,23 +909,23 @@ static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
 {
 	ctxt->ops->get_fpu(ctxt);
 	switch (reg) {
-	case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break;
-	case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break;
-	case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break;
-	case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break;
-	case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break;
-	case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break;
-	case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break;
-	case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break;
+	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
+	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
+	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
+	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
+	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
+	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
+	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
+	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
 #ifdef CONFIG_X86_64
-	case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break;
-	case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break;
-	case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break;
-	case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break;
-	case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break;
-	case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break;
-	case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break;
-	case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break;
+	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
+	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
+	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
+	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
+	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
+	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
+	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
+	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
 #endif
 	default: BUG();
 	}
@@ -899,23 +937,23 @@ static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
 {
 	ctxt->ops->get_fpu(ctxt);
 	switch (reg) {
-	case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break;
-	case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break;
-	case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break;
-	case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break;
-	case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break;
-	case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break;
-	case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break;
-	case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break;
+	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
+	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
+	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
+	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
+	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
+	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
+	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
+	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
 #ifdef CONFIG_X86_64
-	case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break;
-	case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break;
-	case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break;
-	case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break;
-	case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break;
-	case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break;
-	case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break;
-	case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break;
+	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
+	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
+	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
+	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
+	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
+	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
+	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
+	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
 #endif
 	default: BUG();
 	}
@@ -982,10 +1020,10 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
 
 	op->type = OP_REG;
 	if (ctxt->d & ByteOp) {
-		op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs);
+		op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
 		op->bytes = 1;
 	} else {
-		op->addr.reg = decode_register(reg, ctxt->regs, 0);
+		op->addr.reg = decode_register(ctxt, reg, 0);
 		op->bytes = ctxt->op_bytes;
 	}
 	fetch_register_operand(op);
@@ -1020,8 +1058,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
 	if (ctxt->modrm_mod == 3) {
 		op->type = OP_REG;
 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
-		op->addr.reg = decode_register(ctxt->modrm_rm,
-					       ctxt->regs, ctxt->d & ByteOp);
+		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
 		if (ctxt->d & Sse) {
 			op->type = OP_XMM;
 			op->bytes = 16;
@@ -1042,10 +1079,10 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
 	op->type = OP_MEM;
 
 	if (ctxt->ad_bytes == 2) {
-		unsigned bx = ctxt->regs[VCPU_REGS_RBX];
-		unsigned bp = ctxt->regs[VCPU_REGS_RBP];
-		unsigned si = ctxt->regs[VCPU_REGS_RSI];
-		unsigned di = ctxt->regs[VCPU_REGS_RDI];
+		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
+		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
+		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
+		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
 
 		/* 16-bit ModR/M decode. */
 		switch (ctxt->modrm_mod) {
@@ -1102,17 +1139,17 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
 			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
 				modrm_ea += insn_fetch(s32, ctxt);
 			else {
-				modrm_ea += ctxt->regs[base_reg];
+				modrm_ea += reg_read(ctxt, base_reg);
 				adjust_modrm_seg(ctxt, base_reg);
 			}
 			if (index_reg != 4)
-				modrm_ea += ctxt->regs[index_reg] << scale;
+				modrm_ea += reg_read(ctxt, index_reg) << scale;
 		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
 			if (ctxt->mode == X86EMUL_MODE_PROT64)
 				ctxt->rip_relative = 1;
 		} else {
 			base_reg = ctxt->modrm_rm;
-			modrm_ea += ctxt->regs[base_reg];
+			modrm_ea += reg_read(ctxt, base_reg);
 			adjust_modrm_seg(ctxt, base_reg);
 		}
 		switch (ctxt->modrm_mod) {
@@ -1179,24 +1216,21 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
 	int rc;
 	struct read_cache *mc = &ctxt->mem_read;
 
-	while (size) {
-		int n = min(size, 8u);
-		size -= n;
-		if (mc->pos < mc->end)
-			goto read_cached;
+	if (mc->pos < mc->end)
+		goto read_cached;
 
-		rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n,
-					      &ctxt->exception);
-		if (rc != X86EMUL_CONTINUE)
-			return rc;
-		mc->end += n;
+	WARN_ON((mc->end + size) >= sizeof(mc->data));
 
-	read_cached:
-		memcpy(dest, mc->data + mc->pos, n);
-		mc->pos += n;
-		dest += n;
-		addr += n;
-	}
+	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
+				      &ctxt->exception);
+	if (rc != X86EMUL_CONTINUE)
+		return rc;
+
+	mc->end += size;
+
+read_cached:
+	memcpy(dest, mc->data + mc->pos, size);
+	mc->pos += size;
 	return X86EMUL_CONTINUE;
 }
 
@@ -1253,10 +1287,10 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
 	if (rc->pos == rc->end) { /* refill pio read ahead */
 		unsigned int in_page, n;
 		unsigned int count = ctxt->rep_prefix ?
-			address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1;
+			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
 		in_page = (ctxt->eflags & EFLG_DF) ?
-			offset_in_page(ctxt->regs[VCPU_REGS_RDI]) :
-			PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]);
+			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
+			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
 		n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
 			count);
 		if (n == 0)
@@ -1267,8 +1301,15 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
 		rc->end = n * size;
 	}
 
-	memcpy(dest, rc->data + rc->pos, size);
-	rc->pos += size;
+	if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
+		ctxt->dst.data = rc->data + rc->pos;
+		ctxt->dst.type = OP_MEM_STR;
+		ctxt->dst.count = (rc->end - rc->pos) / size;
+		rc->pos = rc->end;
+	} else {
+		memcpy(dest, rc->data + rc->pos, size);
+		rc->pos += size;
+	}
 	return 1;
 }
 
@@ -1291,7 +1332,7 @@ static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
 				     u16 selector, struct desc_ptr *dt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 
 	if (selector & 1 << 2) {
 		struct desc_struct desc;
@@ -1355,19 +1396,15 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
 	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
 	ulong desc_addr;
 	int ret;
+	u16 dummy;
 
 	memset(&seg_desc, 0, sizeof seg_desc);
 
 	if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
 	    || ctxt->mode == X86EMUL_MODE_REAL) {
 		/* set real mode segment descriptor */
+		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
 		set_desc_base(&seg_desc, selector << 4);
-		set_desc_limit(&seg_desc, 0xffff);
-		seg_desc.type = 3;
-		seg_desc.p = 1;
-		seg_desc.s = 1;
-		if (ctxt->mode == X86EMUL_MODE_VM86)
-			seg_desc.dpl = 3;
 		goto load;
 	}
 
@@ -1396,7 +1433,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
 	err_code = selector & 0xfffc;
 	err_vec = GP_VECTOR;
 
-	/* can't load system descriptor into segment selecor */
+	/* can't load system descriptor into segment selector */
 	if (seg <= VCPU_SREG_GS && !seg_desc.s)
 		goto exception;
 
@@ -1516,6 +1553,14 @@ static int writeback(struct x86_emulate_ctxt *ctxt)
 		if (rc != X86EMUL_CONTINUE)
 			return rc;
 		break;
+	case OP_MEM_STR:
+		rc = segmented_write(ctxt,
+				ctxt->dst.addr.mem,
+				ctxt->dst.data,
+				ctxt->dst.bytes * ctxt->dst.count);
+		if (rc != X86EMUL_CONTINUE)
+			return rc;
+		break;
 	case OP_XMM:
 		write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
 		break;
@@ -1536,7 +1581,7 @@ static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
 	struct segmented_address addr;
 
 	rsp_increment(ctxt, -bytes);
-	addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt);
+	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
 	addr.seg = VCPU_SREG_SS;
 
 	return segmented_write(ctxt, addr, data, bytes);
@@ -1555,7 +1600,7 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt,
 	int rc;
 	struct segmented_address addr;
 
-	addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt);
+	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
 	addr.seg = VCPU_SREG_SS;
 	rc = segmented_read(ctxt, addr, dest, len);
 	if (rc != X86EMUL_CONTINUE)
@@ -1623,26 +1668,28 @@ static int em_enter(struct x86_emulate_ctxt *ctxt)
 	int rc;
 	unsigned frame_size = ctxt->src.val;
 	unsigned nesting_level = ctxt->src2.val & 31;
+	ulong rbp;
 
 	if (nesting_level)
 		return X86EMUL_UNHANDLEABLE;
 
-	rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt));
+	rbp = reg_read(ctxt, VCPU_REGS_RBP);
+	rc = push(ctxt, &rbp, stack_size(ctxt));
 	if (rc != X86EMUL_CONTINUE)
 		return rc;
-	assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP],
+	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
 		      stack_mask(ctxt));
-	assign_masked(&ctxt->regs[VCPU_REGS_RSP],
-		      ctxt->regs[VCPU_REGS_RSP] - frame_size,
+	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
+		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
 		      stack_mask(ctxt));
 	return X86EMUL_CONTINUE;
 }
 
 static int em_leave(struct x86_emulate_ctxt *ctxt)
 {
-	assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP],
+	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
 		      stack_mask(ctxt));
-	return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes);
+	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
 }
 
 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
@@ -1670,13 +1717,13 @@ static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
 
 static int em_pusha(struct x86_emulate_ctxt *ctxt)
 {
-	unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP];
+	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
 	int rc = X86EMUL_CONTINUE;
 	int reg = VCPU_REGS_RAX;
 
 	while (reg <= VCPU_REGS_RDI) {
 		(reg == VCPU_REGS_RSP) ?
-		(ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]);
+		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
 
 		rc = em_push(ctxt);
 		if (rc != X86EMUL_CONTINUE)
@@ -1705,7 +1752,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
 			--reg;
 		}
 
-		rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes);
+		rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
 		if (rc != X86EMUL_CONTINUE)
 			break;
 		--reg;
@@ -1713,9 +1760,9 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
 	return rc;
 }
 
-int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
+static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	int rc;
 	struct desc_ptr dt;
 	gva_t cs_addr;
@@ -1762,11 +1809,22 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
 	return rc;
 }
 
+int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
+{
+	int rc;
+
+	invalidate_registers(ctxt);
+	rc = __emulate_int_real(ctxt, irq);
+	if (rc == X86EMUL_CONTINUE)
+		writeback_registers(ctxt);
+	return rc;
+}
+
 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
 {
 	switch(ctxt->mode) {
 	case X86EMUL_MODE_REAL:
-		return emulate_int_real(ctxt, irq);
+		return __emulate_int_real(ctxt, irq);
 	case X86EMUL_MODE_VM86:
 	case X86EMUL_MODE_PROT16:
 	case X86EMUL_MODE_PROT32:
@@ -1973,14 +2031,14 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
 {
 	u64 old = ctxt->dst.orig_val64;
 
-	if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) ||
-	    ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) {
-		ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0);
-		ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32);
+	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
+	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
+		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
+		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
 		ctxt->eflags &= ~EFLG_ZF;
 	} else {
-		ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) |
-			(u32) ctxt->regs[VCPU_REGS_RBX];
+		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
+			(u32) reg_read(ctxt, VCPU_REGS_RBX);
 
 		ctxt->eflags |= EFLG_ZF;
 	}
@@ -2016,7 +2074,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
 {
 	/* Save real source value, then compare EAX against destination. */
 	ctxt->src.orig_val = ctxt->src.val;
-	ctxt->src.val = ctxt->regs[VCPU_REGS_RAX];
+	ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
 	emulate_2op_SrcV(ctxt, "cmp");
 
 	if (ctxt->eflags & EFLG_ZF) {
@@ -2025,7 +2083,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
 	} else {
 		/* Failure: write the value we saw to EAX. */
 		ctxt->dst.type = OP_REG;
-		ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX];
+		ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
 	}
 	return X86EMUL_CONTINUE;
 }
@@ -2050,12 +2108,6 @@ static void
 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
 			struct desc_struct *cs, struct desc_struct *ss)
 {
-	u16 selector;
-
-	memset(cs, 0, sizeof(struct desc_struct));
-	ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
-	memset(ss, 0, sizeof(struct desc_struct));
-
 	cs->l = 0;		/* will be adjusted later */
 	set_desc_base(cs, 0);	/* flat segment */
 	cs->g = 1;		/* 4kb granularity */
@@ -2065,6 +2117,7 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
 	cs->dpl = 0;		/* will be adjusted later */
 	cs->p = 1;
 	cs->d = 1;
+	cs->avl = 0;
 
 	set_desc_base(ss, 0);	/* flat segment */
 	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
@@ -2074,6 +2127,8 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
 	ss->d = 1;		/* 32bit stack segment */
 	ss->dpl = 0;
 	ss->p = 1;
+	ss->l = 0;
+	ss->avl = 0;
 }
 
 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
@@ -2089,7 +2144,7 @@ static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
 
 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	u32 eax, ebx, ecx, edx;
 
 	/*
@@ -2133,7 +2188,7 @@ static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
 
 static int em_syscall(struct x86_emulate_ctxt *ctxt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct desc_struct cs, ss;
 	u64 msr_data;
 	u16 cs_sel, ss_sel;
@@ -2165,10 +2220,10 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
 
-	ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip;
+	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
 	if (efer & EFER_LMA) {
 #ifdef CONFIG_X86_64
-		ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF;
+		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
 
 		ops->get_msr(ctxt,
 			     ctxt->mode == X86EMUL_MODE_PROT64 ?
@@ -2191,7 +2246,7 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
 
 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct desc_struct cs, ss;
 	u64 msr_data;
 	u16 cs_sel, ss_sel;
@@ -2228,6 +2283,8 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
 		if (msr_data == 0x0)
 			return emulate_gp(ctxt, 0);
 		break;
+	default:
+		break;
 	}
 
 	ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
@@ -2247,14 +2304,14 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
 	ctxt->_eip = msr_data;
 
 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
-	ctxt->regs[VCPU_REGS_RSP] = msr_data;
+	*reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
 
 	return X86EMUL_CONTINUE;
 }
 
 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct desc_struct cs, ss;
 	u64 msr_data;
 	int usermode;
@@ -2297,8 +2354,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt)
 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
 
-	ctxt->_eip = ctxt->regs[VCPU_REGS_RDX];
-	ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX];
+	ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
+	*reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
 
 	return X86EMUL_CONTINUE;
 }
@@ -2317,7 +2374,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
 					    u16 port, u16 len)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct desc_struct tr_seg;
 	u32 base3;
 	int r;
@@ -2367,14 +2424,14 @@ static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
 {
 	tss->ip = ctxt->_eip;
 	tss->flag = ctxt->eflags;
-	tss->ax = ctxt->regs[VCPU_REGS_RAX];
-	tss->cx = ctxt->regs[VCPU_REGS_RCX];
-	tss->dx = ctxt->regs[VCPU_REGS_RDX];
-	tss->bx = ctxt->regs[VCPU_REGS_RBX];
-	tss->sp = ctxt->regs[VCPU_REGS_RSP];
-	tss->bp = ctxt->regs[VCPU_REGS_RBP];
-	tss->si = ctxt->regs[VCPU_REGS_RSI];
-	tss->di = ctxt->regs[VCPU_REGS_RDI];
+	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
+	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
+	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
+	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
+	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
+	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
+	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
+	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
 
 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2390,14 +2447,14 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
 
 	ctxt->_eip = tss->ip;
 	ctxt->eflags = tss->flag | 2;
-	ctxt->regs[VCPU_REGS_RAX] = tss->ax;
-	ctxt->regs[VCPU_REGS_RCX] = tss->cx;
-	ctxt->regs[VCPU_REGS_RDX] = tss->dx;
-	ctxt->regs[VCPU_REGS_RBX] = tss->bx;
-	ctxt->regs[VCPU_REGS_RSP] = tss->sp;
-	ctxt->regs[VCPU_REGS_RBP] = tss->bp;
-	ctxt->regs[VCPU_REGS_RSI] = tss->si;
-	ctxt->regs[VCPU_REGS_RDI] = tss->di;
+	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
+	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
+	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
+	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
+	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
+	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
+	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
+	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
 
 	/*
 	 * SDM says that segment selectors are loaded before segment
@@ -2410,7 +2467,7 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
 
 	/*
-	 * Now load segment descriptors. If fault happenes at this stage
+	 * Now load segment descriptors. If fault happens at this stage
 	 * it is handled in a context of new task
 	 */
 	ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
@@ -2436,7 +2493,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
 			  u16 tss_selector, u16 old_tss_sel,
 			  ulong old_tss_base, struct desc_struct *new_desc)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct tss_segment_16 tss_seg;
 	int ret;
 	u32 new_tss_base = get_desc_base(new_desc);
@@ -2482,14 +2539,14 @@ static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
 	tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
 	tss->eip = ctxt->_eip;
 	tss->eflags = ctxt->eflags;
-	tss->eax = ctxt->regs[VCPU_REGS_RAX];
-	tss->ecx = ctxt->regs[VCPU_REGS_RCX];
-	tss->edx = ctxt->regs[VCPU_REGS_RDX];
-	tss->ebx = ctxt->regs[VCPU_REGS_RBX];
-	tss->esp = ctxt->regs[VCPU_REGS_RSP];
-	tss->ebp = ctxt->regs[VCPU_REGS_RBP];
-	tss->esi = ctxt->regs[VCPU_REGS_RSI];
-	tss->edi = ctxt->regs[VCPU_REGS_RDI];
+	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
+	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
+	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
+	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
+	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
+	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
+	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
+	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
 
 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2511,14 +2568,14 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
 	ctxt->eflags = tss->eflags | 2;
 
 	/* General purpose registers */
-	ctxt->regs[VCPU_REGS_RAX] = tss->eax;
-	ctxt->regs[VCPU_REGS_RCX] = tss->ecx;
-	ctxt->regs[VCPU_REGS_RDX] = tss->edx;
-	ctxt->regs[VCPU_REGS_RBX] = tss->ebx;
-	ctxt->regs[VCPU_REGS_RSP] = tss->esp;
-	ctxt->regs[VCPU_REGS_RBP] = tss->ebp;
-	ctxt->regs[VCPU_REGS_RSI] = tss->esi;
-	ctxt->regs[VCPU_REGS_RDI] = tss->edi;
+	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
+	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
+	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
+	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
+	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
+	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
+	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
+	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
 
 	/*
 	 * SDM says that segment selectors are loaded before segment
@@ -2583,7 +2640,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
 			  u16 tss_selector, u16 old_tss_sel,
 			  ulong old_tss_base, struct desc_struct *new_desc)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct tss_segment_32 tss_seg;
 	int ret;
 	u32 new_tss_base = get_desc_base(new_desc);
@@ -2627,7 +2684,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
 				   u16 tss_selector, int idt_index, int reason,
 				   bool has_error_code, u32 error_code)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	struct desc_struct curr_tss_desc, next_tss_desc;
 	int ret;
 	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
@@ -2652,7 +2709,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
 	 *
 	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
 	 * 2. Exception/IRQ/iret: No check is performed
-	 * 3. jmp/call to TSS: Check agains DPL of the TSS
+	 * 3. jmp/call to TSS: Check against DPL of the TSS
 	 */
 	if (reason == TASK_SWITCH_GATE) {
 		if (idt_index != -1) {
@@ -2693,7 +2750,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
 		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
 
 	/* set back link to prev task only if NT bit is set in eflags
-	   note that old_tss_sel is not used afetr this point */
+	   note that old_tss_sel is not used after this point */
 	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
 		old_tss_sel = 0xffff;
 
@@ -2733,26 +2790,28 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
 {
 	int rc;
 
+	invalidate_registers(ctxt);
 	ctxt->_eip = ctxt->eip;
 	ctxt->dst.type = OP_NONE;
 
 	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
 				     has_error_code, error_code);
 
-	if (rc == X86EMUL_CONTINUE)
+	if (rc == X86EMUL_CONTINUE) {
 		ctxt->eip = ctxt->_eip;
+		writeback_registers(ctxt);
+	}
 
 	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 }
 
-static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg,
-			    int reg, struct operand *op)
+static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
+		struct operand *op)
 {
-	int df = (ctxt->eflags & EFLG_DF) ? -1 : 1;
+	int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
 
-	register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes);
-	op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]);
-	op->addr.mem.seg = seg;
+	register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
+	op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
 }
 
 static int em_das(struct x86_emulate_ctxt *ctxt)
@@ -2927,7 +2986,7 @@ static int em_cwd(struct x86_emulate_ctxt *ctxt)
 {
 	ctxt->dst.type = OP_REG;
 	ctxt->dst.bytes = ctxt->src.bytes;
-	ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX];
+	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
 	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
 
 	return X86EMUL_CONTINUE;
@@ -2938,8 +2997,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
 	u64 tsc = 0;
 
 	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
-	ctxt->regs[VCPU_REGS_RAX] = (u32)tsc;
-	ctxt->regs[VCPU_REGS_RDX] = tsc >> 32;
+	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
+	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
 	return X86EMUL_CONTINUE;
 }
 
@@ -2947,10 +3006,10 @@ static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
 {
 	u64 pmc;
 
-	if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc))
+	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
 		return emulate_gp(ctxt, 0);
-	ctxt->regs[VCPU_REGS_RAX] = (u32)pmc;
-	ctxt->regs[VCPU_REGS_RDX] = pmc >> 32;
+	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
+	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
 	return X86EMUL_CONTINUE;
 }
 
@@ -2992,9 +3051,9 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
 {
 	u64 msr_data;
 
-	msr_data = (u32)ctxt->regs[VCPU_REGS_RAX]
-		| ((u64)ctxt->regs[VCPU_REGS_RDX] << 32);
-	if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data))
+	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
+		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
+	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
 		return emulate_gp(ctxt, 0);
 
 	return X86EMUL_CONTINUE;
@@ -3004,11 +3063,11 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
 {
 	u64 msr_data;
 
-	if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data))
+	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
 		return emulate_gp(ctxt, 0);
 
-	ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data;
-	ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32;
+	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
+	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
 	return X86EMUL_CONTINUE;
 }
 
@@ -3188,8 +3247,8 @@ static int em_lmsw(struct x86_emulate_ctxt *ctxt)
 
 static int em_loop(struct x86_emulate_ctxt *ctxt)
 {
-	register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
-	if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) &&
+	register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
+	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
 	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
 		jmp_rel(ctxt, ctxt->src.val);
 
@@ -3198,7 +3257,7 @@ static int em_loop(struct x86_emulate_ctxt *ctxt)
 
 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
 {
-	if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0)
+	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
 		jmp_rel(ctxt, ctxt->src.val);
 
 	return X86EMUL_CONTINUE;
@@ -3286,20 +3345,20 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt)
 {
 	u32 eax, ebx, ecx, edx;
 
-	eax = ctxt->regs[VCPU_REGS_RAX];
-	ecx = ctxt->regs[VCPU_REGS_RCX];
+	eax = reg_read(ctxt, VCPU_REGS_RAX);
+	ecx = reg_read(ctxt, VCPU_REGS_RCX);
 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
-	ctxt->regs[VCPU_REGS_RAX] = eax;
-	ctxt->regs[VCPU_REGS_RBX] = ebx;
-	ctxt->regs[VCPU_REGS_RCX] = ecx;
-	ctxt->regs[VCPU_REGS_RDX] = edx;
+	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
+	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
+	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
+	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
 	return X86EMUL_CONTINUE;
 }
 
 static int em_lahf(struct x86_emulate_ctxt *ctxt)
 {
-	ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL;
-	ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8;
+	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
+	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
 	return X86EMUL_CONTINUE;
 }
 
@@ -3456,7 +3515,7 @@ static int check_svme(struct x86_emulate_ctxt *ctxt)
 
 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
 {
-	u64 rax = ctxt->regs[VCPU_REGS_RAX];
+	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
 
 	/* Valid physical address? */
 	if (rax & 0xffff000000000000ULL)
@@ -3478,7 +3537,7 @@ static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
 {
 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
-	u64 rcx = ctxt->regs[VCPU_REGS_RCX];
+	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
 
 	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
 	    (rcx > 3))
@@ -3531,13 +3590,13 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
 		I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
 		I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
 
-static struct opcode group7_rm1[] = {
+static const struct opcode group7_rm1[] = {
 	DI(SrcNone | Priv, monitor),
 	DI(SrcNone | Priv, mwait),
 	N, N, N, N, N, N,
 };
 
-static struct opcode group7_rm3[] = {
+static const struct opcode group7_rm3[] = {
 	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
 	II(SrcNone  | Prot | VendorSpecific,	em_vmmcall,	vmmcall),
 	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
@@ -3548,13 +3607,13 @@ static struct opcode group7_rm3[] = {
 	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
 };
 
-static struct opcode group7_rm7[] = {
+static const struct opcode group7_rm7[] = {
 	N,
 	DIP(SrcNone, rdtscp, check_rdtsc),
 	N, N, N, N, N, N,
 };
 
-static struct opcode group1[] = {
+static const struct opcode group1[] = {
 	I(Lock, em_add),
 	I(Lock | PageTable, em_or),
 	I(Lock, em_adc),
@@ -3565,11 +3624,11 @@ static struct opcode group1[] = {
 	I(0, em_cmp),
 };
 
-static struct opcode group1A[] = {
+static const struct opcode group1A[] = {
 	I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
 };
 
-static struct opcode group3[] = {
+static const struct opcode group3[] = {
 	I(DstMem | SrcImm, em_test),
 	I(DstMem | SrcImm, em_test),
 	I(DstMem | SrcNone | Lock, em_not),
@@ -3580,13 +3639,13 @@ static struct opcode group3[] = {
 	I(SrcMem, em_idiv_ex),
 };
 
-static struct opcode group4[] = {
+static const struct opcode group4[] = {
 	I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
 	I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
 	N, N, N, N, N, N,
 };
 
-static struct opcode group5[] = {
+static const struct opcode group5[] = {
 	I(DstMem | SrcNone | Lock,		em_grp45),
 	I(DstMem | SrcNone | Lock,		em_grp45),
 	I(SrcMem | Stack,			em_grp45),
@@ -3596,7 +3655,7 @@ static struct opcode group5[] = {
 	I(SrcMem | Stack,			em_grp45), N,
 };
 
-static struct opcode group6[] = {
+static const struct opcode group6[] = {
 	DI(Prot,	sldt),
 	DI(Prot,	str),
 	II(Prot | Priv | SrcMem16, em_lldt, lldt),
@@ -3604,7 +3663,7 @@ static struct opcode group6[] = {
 	N, N, N, N,
 };
 
-static struct group_dual group7 = { {
+static const struct group_dual group7 = { {
 	II(Mov | DstMem | Priv,			em_sgdt, sgdt),
 	II(Mov | DstMem | Priv,			em_sidt, sidt),
 	II(SrcMem | Priv,			em_lgdt, lgdt),
@@ -3621,7 +3680,7 @@ static struct group_dual group7 = { {
 	EXT(0, group7_rm7),
 } };
 
-static struct opcode group8[] = {
+static const struct opcode group8[] = {
 	N, N, N, N,
 	I(DstMem | SrcImmByte,				em_bt),
 	I(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
@@ -3629,26 +3688,26 @@ static struct opcode group8[] = {
 	I(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
 };
 
-static struct group_dual group9 = { {
+static const struct group_dual group9 = { {
 	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
 }, {
 	N, N, N, N, N, N, N, N,
 } };
 
-static struct opcode group11[] = {
+static const struct opcode group11[] = {
 	I(DstMem | SrcImm | Mov | PageTable, em_mov),
 	X7(D(Undefined)),
 };
 
-static struct gprefix pfx_0f_6f_0f_7f = {
+static const struct gprefix pfx_0f_6f_0f_7f = {
 	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
 };
 
-static struct gprefix pfx_vmovntpx = {
+static const struct gprefix pfx_vmovntpx = {
 	I(0, em_mov), N, N, N,
 };
 
-static struct opcode opcode_table[256] = {
+static const struct opcode opcode_table[256] = {
 	/* 0x00 - 0x07 */
 	I6ALU(Lock, em_add),
 	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
@@ -3689,7 +3748,7 @@ static struct opcode opcode_table[256] = {
 	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
 	I(SrcImmByte | Mov | Stack, em_push),
 	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
-	I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */
+	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
 	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
 	/* 0x70 - 0x7F */
 	X16(D(SrcImmByte)),
@@ -3765,7 +3824,7 @@ static struct opcode opcode_table[256] = {
 	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
 };
 
-static struct opcode twobyte_table[256] = {
+static const struct opcode twobyte_table[256] = {
 	/* 0x00 - 0x0F */
 	G(0, group6), GD(0, &group7), N, N,
 	N, I(ImplicitOps | VendorSpecific, em_syscall),
@@ -3936,7 +3995,7 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
 	case OpAcc:
 		op->type = OP_REG;
 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
-		op->addr.reg = &ctxt->regs[VCPU_REGS_RAX];
+		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
 		fetch_register_operand(op);
 		op->orig_val = op->val;
 		break;
@@ -3944,19 +4003,20 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
 		op->type = OP_MEM;
 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
 		op->addr.mem.ea =
-			register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]);
+			register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
 		op->addr.mem.seg = VCPU_SREG_ES;
 		op->val = 0;
+		op->count = 1;
 		break;
 	case OpDX:
 		op->type = OP_REG;
 		op->bytes = 2;
-		op->addr.reg = &ctxt->regs[VCPU_REGS_RDX];
+		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
 		fetch_register_operand(op);
 		break;
 	case OpCL:
 		op->bytes = 1;
-		op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff;
+		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
 		break;
 	case OpImmByte:
 		rc = decode_imm(ctxt, op, 1, true);
@@ -3987,9 +4047,10 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
 		op->type = OP_MEM;
 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
 		op->addr.mem.ea =
-			register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]);
+			register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
 		op->addr.mem.seg = seg_override(ctxt);
 		op->val = 0;
+		op->count = 1;
 		break;
 	case OpImmFAddr:
 		op->type = OP_IMM;
@@ -4293,9 +4354,10 @@ static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
 		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
 }
 
+
 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 {
-	struct x86_emulate_ops *ops = ctxt->ops;
+	const struct x86_emulate_ops *ops = ctxt->ops;
 	int rc = X86EMUL_CONTINUE;
 	int saved_dst_type = ctxt->dst.type;
 
@@ -4356,7 +4418,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 	}
 
 	/* Instruction can only be executed in protected mode */
-	if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) {
+	if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
 		rc = emulate_ud(ctxt);
 		goto done;
 	}
@@ -4377,7 +4439,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
 
 	if (ctxt->rep_prefix && (ctxt->d & String)) {
 		/* All REP prefixes have the same first termination condition */
-		if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) {
+		if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
 			ctxt->eip = ctxt->_eip;
 			goto done;
 		}
@@ -4450,7 +4512,7 @@ special_insn:
 		ctxt->dst.val = ctxt->src.addr.mem.ea;
 		break;
 	case 0x90 ... 0x97: /* nop / xchg reg, rax */
-		if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX])
+		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
 			break;
 		rc = em_xchg(ctxt);
 		break;
@@ -4478,7 +4540,7 @@ special_insn:
 		rc = em_grp2(ctxt);
 		break;
 	case 0xd2 ... 0xd3:	/* Grp2 */
-		ctxt->src.val = ctxt->regs[VCPU_REGS_RCX];
+		ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
 		rc = em_grp2(ctxt);
 		break;
 	case 0xe9: /* jmp rel */
@@ -4524,23 +4586,27 @@ writeback:
 	ctxt->dst.type = saved_dst_type;
 
 	if ((ctxt->d & SrcMask) == SrcSI)
-		string_addr_inc(ctxt, seg_override(ctxt),
-				VCPU_REGS_RSI, &ctxt->src);
+		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
 
 	if ((ctxt->d & DstMask) == DstDI)
-		string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI,
-				&ctxt->dst);
+		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
 
 	if (ctxt->rep_prefix && (ctxt->d & String)) {
+		unsigned int count;
 		struct read_cache *r = &ctxt->io_read;
-		register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1);
+		if ((ctxt->d & SrcMask) == SrcSI)
+			count = ctxt->src.count;
+		else
+			count = ctxt->dst.count;
+		register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
+				-count);
 
 		if (!string_insn_completed(ctxt)) {
 			/*
 			 * Re-enter guest when pio read ahead buffer is empty
 			 * or, if it is not used, after each 1024 iteration.
 			 */
-			if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) &&
+			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
 			    (r->end == 0 || r->end != r->pos)) {
 				/*
 				 * Reset read cache. Usually happens before
@@ -4548,6 +4614,7 @@ writeback:
 				 * we have to do it here.
 				 */
 				ctxt->mem_read.end = 0;
+				writeback_registers(ctxt);
 				return EMULATION_RESTART;
 			}
 			goto done; /* skip rip writeback */
@@ -4562,6 +4629,9 @@ done:
 	if (rc == X86EMUL_INTERCEPTED)
 		return EMULATION_INTERCEPTED;
 
+	if (rc == X86EMUL_CONTINUE)
+		writeback_registers(ctxt);
+
 	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
 
 twobyte_insn:
@@ -4634,3 +4704,13 @@ twobyte_insn:
 cannot_emulate:
 	return EMULATION_FAILED;
 }
+
+void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
+{
+	invalidate_registers(ctxt);
+}
+
+void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
+{
+	writeback_registers(ctxt);
+}
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index adba28f88d1..11300d2fa71 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -108,7 +108,7 @@ static s64 __kpit_elapsed(struct kvm *kvm)
 	ktime_t remaining;
 	struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
 
-	if (!ps->pit_timer.period)
+	if (!ps->period)
 		return 0;
 
 	/*
@@ -120,9 +120,9 @@ static s64 __kpit_elapsed(struct kvm *kvm)
 	 * itself with the initial count and continues counting
 	 * from there.
 	 */
-	remaining = hrtimer_get_remaining(&ps->pit_timer.timer);
-	elapsed = ps->pit_timer.period - ktime_to_ns(remaining);
-	elapsed = mod_64(elapsed, ps->pit_timer.period);
+	remaining = hrtimer_get_remaining(&ps->timer);
+	elapsed = ps->period - ktime_to_ns(remaining);
+	elapsed = mod_64(elapsed, ps->period);
 
 	return elapsed;
 }
@@ -238,12 +238,12 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian)
 	int value;
 
 	spin_lock(&ps->inject_lock);
-	value = atomic_dec_return(&ps->pit_timer.pending);
+	value = atomic_dec_return(&ps->pending);
 	if (value < 0)
 		/* spurious acks can be generated if, for example, the
 		 * PIC is being reset.  Handle it gracefully here
 		 */
-		atomic_inc(&ps->pit_timer.pending);
+		atomic_inc(&ps->pending);
 	else if (value > 0)
 		/* in this case, we had multiple outstanding pit interrupts
 		 * that we needed to inject.  Reinject
@@ -261,28 +261,17 @@ void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
 	if (!kvm_vcpu_is_bsp(vcpu) || !pit)
 		return;
 
-	timer = &pit->pit_state.pit_timer.timer;
+	timer = &pit->pit_state.timer;
 	if (hrtimer_cancel(timer))
 		hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
 }
 
 static void destroy_pit_timer(struct kvm_pit *pit)
 {
-	hrtimer_cancel(&pit->pit_state.pit_timer.timer);
+	hrtimer_cancel(&pit->pit_state.timer);
 	flush_kthread_work(&pit->expired);
 }
 
-static bool kpit_is_periodic(struct kvm_timer *ktimer)
-{
-	struct kvm_kpit_state *ps = container_of(ktimer, struct kvm_kpit_state,
-						 pit_timer);
-	return ps->is_periodic;
-}
-
-static struct kvm_timer_ops kpit_ops = {
-	.is_periodic = kpit_is_periodic,
-};
-
 static void pit_do_work(struct kthread_work *work)
 {
 	struct kvm_pit *pit = container_of(work, struct kvm_pit, expired);
@@ -322,16 +311,16 @@ static void pit_do_work(struct kthread_work *work)
 
 static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
 {
-	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
-	struct kvm_pit *pt = ktimer->kvm->arch.vpit;
+	struct kvm_kpit_state *ps = container_of(data, struct kvm_kpit_state, timer);
+	struct kvm_pit *pt = ps->kvm->arch.vpit;
 
-	if (ktimer->reinject || !atomic_read(&ktimer->pending)) {
-		atomic_inc(&ktimer->pending);
+	if (ps->reinject || !atomic_read(&ps->pending)) {
+		atomic_inc(&ps->pending);
 		queue_kthread_work(&pt->worker, &pt->expired);
 	}
 
-	if (ktimer->t_ops->is_periodic(ktimer)) {
-		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
+	if (ps->is_periodic) {
+		hrtimer_add_expires_ns(&ps->timer, ps->period);
 		return HRTIMER_RESTART;
 	} else
 		return HRTIMER_NORESTART;
@@ -340,7 +329,6 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
 static void create_pit_timer(struct kvm *kvm, u32 val, int is_period)
 {
 	struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
-	struct kvm_timer *pt = &ps->pit_timer;
 	s64 interval;
 
 	if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)
@@ -351,19 +339,18 @@ static void create_pit_timer(struct kvm *kvm, u32 val, int is_period)
 	pr_debug("create pit timer, interval is %llu nsec\n", interval);
 
 	/* TODO The new value only affected after the retriggered */
-	hrtimer_cancel(&pt->timer);
+	hrtimer_cancel(&ps->timer);
 	flush_kthread_work(&ps->pit->expired);
-	pt->period = interval;
+	ps->period = interval;
 	ps->is_periodic = is_period;
 
-	pt->timer.function = pit_timer_fn;
-	pt->t_ops = &kpit_ops;
-	pt->kvm = ps->pit->kvm;
+	ps->timer.function = pit_timer_fn;
+	ps->kvm = ps->pit->kvm;
 
-	atomic_set(&pt->pending, 0);
+	atomic_set(&ps->pending, 0);
 	ps->irq_ack = 1;
 
-	hrtimer_start(&pt->timer, ktime_add_ns(ktime_get(), interval),
+	hrtimer_start(&ps->timer, ktime_add_ns(ktime_get(), interval),
 		      HRTIMER_MODE_ABS);
 }
 
@@ -639,7 +626,7 @@ void kvm_pit_reset(struct kvm_pit *pit)
 	}
 	mutex_unlock(&pit->pit_state.lock);
 
-	atomic_set(&pit->pit_state.pit_timer.pending, 0);
+	atomic_set(&pit->pit_state.pending, 0);
 	pit->pit_state.irq_ack = 1;
 }
 
@@ -648,7 +635,7 @@ static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask)
 	struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier);
 
 	if (!mask) {
-		atomic_set(&pit->pit_state.pit_timer.pending, 0);
+		atomic_set(&pit->pit_state.pending, 0);
 		pit->pit_state.irq_ack = 1;
 	}
 }
@@ -706,12 +693,11 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
 
 	pit_state = &pit->pit_state;
 	pit_state->pit = pit;
-	hrtimer_init(&pit_state->pit_timer.timer,
-		     CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
+	hrtimer_init(&pit_state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 	pit_state->irq_ack_notifier.gsi = 0;
 	pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq;
 	kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier);
-	pit_state->pit_timer.reinject = true;
+	pit_state->reinject = true;
 	mutex_unlock(&pit->pit_state.lock);
 
 	kvm_pit_reset(pit);
@@ -761,7 +747,7 @@ void kvm_free_pit(struct kvm *kvm)
 		kvm_unregister_irq_ack_notifier(kvm,
 				&kvm->arch.vpit->pit_state.irq_ack_notifier);
 		mutex_lock(&kvm->arch.vpit->pit_state.lock);
-		timer = &kvm->arch.vpit->pit_state.pit_timer.timer;
+		timer = &kvm->arch.vpit->pit_state.timer;
 		hrtimer_cancel(timer);
 		flush_kthread_work(&kvm->arch.vpit->expired);
 		kthread_stop(kvm->arch.vpit->worker_task);
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index fdf40425ea1..dd1b16b611b 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -24,8 +24,12 @@ struct kvm_kpit_channel_state {
 struct kvm_kpit_state {
 	struct kvm_kpit_channel_state channels[3];
 	u32 flags;
-	struct kvm_timer pit_timer;
 	bool is_periodic;
+	s64 period; 				/* unit: ns */
+	struct hrtimer timer;
+	atomic_t pending;			/* accumulated triggered timers */
+	bool reinject;
+	struct kvm *kvm;
 	u32    speaker_data_on;
 	struct mutex lock;
 	struct kvm_pit *pit;
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index 9fc9aa7ac70..848206df096 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -190,17 +190,17 @@ void kvm_pic_update_irq(struct kvm_pic *s)
 
 int kvm_pic_set_irq(struct kvm_pic *s, int irq, int irq_source_id, int level)
 {
-	int ret = -1;
+	int ret, irq_level;
+
+	BUG_ON(irq < 0 || irq >= PIC_NUM_PINS);
 
 	pic_lock(s);
-	if (irq >= 0 && irq < PIC_NUM_PINS) {
-		int irq_level = __kvm_irq_line_state(&s->irq_states[irq],
-						     irq_source_id, level);
-		ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level);
-		pic_update_irq(s);
-		trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr,
-				      s->pics[irq >> 3].imr, ret == 0);
-	}
+	irq_level = __kvm_irq_line_state(&s->irq_states[irq],
+					 irq_source_id, level);
+	ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level);
+	pic_update_irq(s);
+	trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr,
+			      s->pics[irq >> 3].imr, ret == 0);
 	pic_unlock(s);
 
 	return ret;
@@ -275,23 +275,20 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
 {
 	int irq, i;
 	struct kvm_vcpu *vcpu;
-	u8 irr = s->irr, isr = s->imr;
+	u8 edge_irr = s->irr & ~s->elcr;
 	bool found = false;
 
 	s->last_irr = 0;
-	s->irr = 0;
+	s->irr &= s->elcr;
 	s->imr = 0;
-	s->isr = 0;
 	s->priority_add = 0;
-	s->irq_base = 0;
-	s->read_reg_select = 0;
-	s->poll = 0;
 	s->special_mask = 0;
-	s->init_state = 0;
-	s->auto_eoi = 0;
-	s->rotate_on_auto_eoi = 0;
-	s->special_fully_nested_mode = 0;
-	s->init4 = 0;
+	s->read_reg_select = 0;
+	if (!s->init4) {
+		s->special_fully_nested_mode = 0;
+		s->auto_eoi = 0;
+	}
+	s->init_state = 1;
 
 	kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm)
 		if (kvm_apic_accept_pic_intr(vcpu)) {
@@ -304,7 +301,7 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
 		return;
 
 	for (irq = 0; irq < PIC_NUM_PINS/2; irq++)
-		if (irr & (1 << irq) || isr & (1 << irq))
+		if (edge_irr & (1 << irq))
 			pic_clear_isr(s, irq);
 }
 
@@ -316,40 +313,13 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val)
 	addr &= 1;
 	if (addr == 0) {
 		if (val & 0x10) {
-			u8 edge_irr = s->irr & ~s->elcr;
-			int i;
-			bool found = false;
-			struct kvm_vcpu *vcpu;
-
 			s->init4 = val & 1;
-			s->last_irr = 0;
-			s->irr &= s->elcr;
-			s->imr = 0;
-			s->priority_add = 0;
-			s->special_mask = 0;
-			s->read_reg_select = 0;
-			if (!s->init4) {
-				s->special_fully_nested_mode = 0;
-				s->auto_eoi = 0;
-			}
-			s->init_state = 1;
 			if (val & 0x02)
 				pr_pic_unimpl("single mode not supported");
 			if (val & 0x08)
 				pr_pic_unimpl(
-					"level sensitive irq not supported");
-
-			kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm)
-				if (kvm_apic_accept_pic_intr(vcpu)) {
-					found = true;
-					break;
-				}
-
-
-			if (found)
-				for (irq = 0; irq < PIC_NUM_PINS/2; irq++)
-					if (edge_irr & (1 << irq))
-						pic_clear_isr(s, irq);
+						"level sensitive irq not supported");
+			kvm_pic_reset(s);
 		} else if (val & 0x08) {
 			if (val & 0x04)
 				s->poll = 1;
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 2086f2bfba3..2d03568e949 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -70,7 +70,7 @@ struct kvm_pic {
 	struct kvm_io_device dev_slave;
 	struct kvm_io_device dev_eclr;
 	void (*ack_notifier)(void *opaque, int irq);
-	unsigned long irq_states[16];
+	unsigned long irq_states[PIC_NUM_PINS];
 };
 
 struct kvm_pic *kvm_create_pic(struct kvm *kvm);
diff --git a/arch/x86/kvm/kvm_timer.h b/arch/x86/kvm/kvm_timer.h
deleted file mode 100644
index 497dbaa366d..00000000000
--- a/arch/x86/kvm/kvm_timer.h
+++ /dev/null
@@ -1,18 +0,0 @@
-
-struct kvm_timer {
-	struct hrtimer timer;
-	s64 period; 				/* unit: ns */
-	u32 timer_mode_mask;
-	u64 tscdeadline;
-	atomic_t pending;			/* accumulated triggered timers */
-	bool reinject;
-	struct kvm_timer_ops *t_ops;
-	struct kvm *kvm;
-	struct kvm_vcpu *vcpu;
-};
-
-struct kvm_timer_ops {
-	bool (*is_periodic)(struct kvm_timer *);
-};
-
-enum hrtimer_restart kvm_timer_fn(struct hrtimer *data);
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index ce878788a39..c6e6b721b6e 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -34,6 +34,7 @@
 #include <asm/current.h>
 #include <asm/apicdef.h>
 #include <linux/atomic.h>
+#include <linux/jump_label.h>
 #include "kvm_cache_regs.h"
 #include "irq.h"
 #include "trace.h"
@@ -65,6 +66,7 @@
 #define APIC_DEST_NOSHORT		0x0
 #define APIC_DEST_MASK			0x800
 #define MAX_APIC_VECTOR			256
+#define APIC_VECTORS_PER_REG		32
 
 #define VEC_POS(v) ((v) & (32 - 1))
 #define REG_POS(v) (((v) >> 5) << 4)
@@ -72,11 +74,6 @@
 static unsigned int min_timer_period_us = 500;
 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
 
-static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off)
-{
-	return *((u32 *) (apic->regs + reg_off));
-}
-
 static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
 {
 	*((u32 *) (apic->regs + reg_off)) = val;
@@ -117,19 +114,23 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
 	return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
 }
 
-static inline int apic_hw_enabled(struct kvm_lapic *apic)
-{
-	return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
-}
+struct static_key_deferred apic_hw_disabled __read_mostly;
+struct static_key_deferred apic_sw_disabled __read_mostly;
 
-static inline int  apic_sw_enabled(struct kvm_lapic *apic)
+static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 {
-	return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED;
+	if ((kvm_apic_get_reg(apic, APIC_SPIV) ^ val) & APIC_SPIV_APIC_ENABLED) {
+		if (val & APIC_SPIV_APIC_ENABLED)
+			static_key_slow_dec_deferred(&apic_sw_disabled);
+		else
+			static_key_slow_inc(&apic_sw_disabled.key);
+	}
+	apic_set_reg(apic, APIC_SPIV, val);
 }
 
 static inline int apic_enabled(struct kvm_lapic *apic)
 {
-	return apic_sw_enabled(apic) &&	apic_hw_enabled(apic);
+	return kvm_apic_sw_enabled(apic) &&	kvm_apic_hw_enabled(apic);
 }
 
 #define LVT_MASK	\
@@ -139,36 +140,135 @@ static inline int apic_enabled(struct kvm_lapic *apic)
 	(LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 	 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 
+static inline int apic_x2apic_mode(struct kvm_lapic *apic)
+{
+	return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
+}
+
 static inline int kvm_apic_id(struct kvm_lapic *apic)
 {
-	return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
+	return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
+}
+
+static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
+{
+	u16 cid;
+	ldr >>= 32 - map->ldr_bits;
+	cid = (ldr >> map->cid_shift) & map->cid_mask;
+
+	BUG_ON(cid >= ARRAY_SIZE(map->logical_map));
+
+	return cid;
+}
+
+static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
+{
+	ldr >>= (32 - map->ldr_bits);
+	return ldr & map->lid_mask;
+}
+
+static void recalculate_apic_map(struct kvm *kvm)
+{
+	struct kvm_apic_map *new, *old = NULL;
+	struct kvm_vcpu *vcpu;
+	int i;
+
+	new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL);
+
+	mutex_lock(&kvm->arch.apic_map_lock);
+
+	if (!new)
+		goto out;
+
+	new->ldr_bits = 8;
+	/* flat mode is default */
+	new->cid_shift = 8;
+	new->cid_mask = 0;
+	new->lid_mask = 0xff;
+
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		struct kvm_lapic *apic = vcpu->arch.apic;
+		u16 cid, lid;
+		u32 ldr;
+
+		if (!kvm_apic_present(vcpu))
+			continue;
+
+		/*
+		 * All APICs have to be configured in the same mode by an OS.
+		 * We take advatage of this while building logical id loockup
+		 * table. After reset APICs are in xapic/flat mode, so if we
+		 * find apic with different setting we assume this is the mode
+		 * OS wants all apics to be in; build lookup table accordingly.
+		 */
+		if (apic_x2apic_mode(apic)) {
+			new->ldr_bits = 32;
+			new->cid_shift = 16;
+			new->cid_mask = new->lid_mask = 0xffff;
+		} else if (kvm_apic_sw_enabled(apic) &&
+				!new->cid_mask /* flat mode */ &&
+				kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) {
+			new->cid_shift = 4;
+			new->cid_mask = 0xf;
+			new->lid_mask = 0xf;
+		}
+
+		new->phys_map[kvm_apic_id(apic)] = apic;
+
+		ldr = kvm_apic_get_reg(apic, APIC_LDR);
+		cid = apic_cluster_id(new, ldr);
+		lid = apic_logical_id(new, ldr);
+
+		if (lid)
+			new->logical_map[cid][ffs(lid) - 1] = apic;
+	}
+out:
+	old = rcu_dereference_protected(kvm->arch.apic_map,
+			lockdep_is_held(&kvm->arch.apic_map_lock));
+	rcu_assign_pointer(kvm->arch.apic_map, new);
+	mutex_unlock(&kvm->arch.apic_map_lock);
+
+	if (old)
+		kfree_rcu(old, rcu);
+}
+
+static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id)
+{
+	apic_set_reg(apic, APIC_ID, id << 24);
+	recalculate_apic_map(apic->vcpu->kvm);
+}
+
+static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
+{
+	apic_set_reg(apic, APIC_LDR, id);
+	recalculate_apic_map(apic->vcpu->kvm);
 }
 
 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 {
-	return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
+	return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 }
 
 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
 {
-	return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
+	return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
 }
 
 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 {
-	return ((apic_get_reg(apic, APIC_LVTT) &
+	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
 		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT);
 }
 
 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 {
-	return ((apic_get_reg(apic, APIC_LVTT) &
+	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
 		apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC);
 }
 
 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 {
-	return ((apic_get_reg(apic, APIC_LVTT) &
+	return ((kvm_apic_get_reg(apic, APIC_LVTT) &
 		apic->lapic_timer.timer_mode_mask) ==
 			APIC_LVT_TIMER_TSCDEADLINE);
 }
@@ -184,7 +284,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 	struct kvm_cpuid_entry2 *feat;
 	u32 v = APIC_VERSION;
 
-	if (!irqchip_in_kernel(vcpu->kvm))
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return;
 
 	feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
@@ -193,12 +293,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 	apic_set_reg(apic, APIC_LVR, v);
 }
 
-static inline int apic_x2apic_mode(struct kvm_lapic *apic)
-{
-	return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
-}
-
-static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
+static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
 	LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
 	LVT_MASK | APIC_MODE_MASK,	/* LVTTHMR */
 	LVT_MASK | APIC_MODE_MASK,	/* LVTPC */
@@ -208,25 +303,30 @@ static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
 
 static int find_highest_vector(void *bitmap)
 {
-	u32 *word = bitmap;
-	int word_offset = MAX_APIC_VECTOR >> 5;
+	int vec;
+	u32 *reg;
 
-	while ((word_offset != 0) && (word[(--word_offset) << 2] == 0))
-		continue;
+	for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
+	     vec >= 0; vec -= APIC_VECTORS_PER_REG) {
+		reg = bitmap + REG_POS(vec);
+		if (*reg)
+			return fls(*reg) - 1 + vec;
+	}
 
-	if (likely(!word_offset && !word[0]))
-		return -1;
-	else
-		return fls(word[word_offset << 2]) - 1 + (word_offset << 5);
+	return -1;
 }
 
 static u8 count_vectors(void *bitmap)
 {
-	u32 *word = bitmap;
-	int word_offset;
+	int vec;
+	u32 *reg;
 	u8 count = 0;
-	for (word_offset = 0; word_offset < MAX_APIC_VECTOR >> 5; ++word_offset)
-		count += hweight32(word[word_offset << 2]);
+
+	for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
+		reg = bitmap + REG_POS(vec);
+		count += hweight32(*reg);
+	}
+
 	return count;
 }
 
@@ -285,7 +385,6 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 {
-	struct kvm_lapic *apic = vcpu->arch.apic;
 	int highest_irr;
 
 	/* This may race with setting of irr in __apic_accept_irq() and
@@ -293,9 +392,9 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 	 * will cause vmexit immediately and the value will be recalculated
 	 * on the next vmentry.
 	 */
-	if (!apic)
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return 0;
-	highest_irr = apic_find_highest_irr(apic);
+	highest_irr = apic_find_highest_irr(vcpu->arch.apic);
 
 	return highest_irr;
 }
@@ -378,8 +477,8 @@ static void apic_update_ppr(struct kvm_lapic *apic)
 	u32 tpr, isrv, ppr, old_ppr;
 	int isr;
 
-	old_ppr = apic_get_reg(apic, APIC_PROCPRI);
-	tpr = apic_get_reg(apic, APIC_TASKPRI);
+	old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI);
+	tpr = kvm_apic_get_reg(apic, APIC_TASKPRI);
 	isr = apic_find_highest_isr(apic);
 	isrv = (isr != -1) ? isr : 0;
 
@@ -415,13 +514,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
 	u32 logical_id;
 
 	if (apic_x2apic_mode(apic)) {
-		logical_id = apic_get_reg(apic, APIC_LDR);
+		logical_id = kvm_apic_get_reg(apic, APIC_LDR);
 		return logical_id & mda;
 	}
 
-	logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR));
+	logical_id = GET_APIC_LOGICAL_ID(kvm_apic_get_reg(apic, APIC_LDR));
 
-	switch (apic_get_reg(apic, APIC_DFR)) {
+	switch (kvm_apic_get_reg(apic, APIC_DFR)) {
 	case APIC_DFR_FLAT:
 		if (logical_id & mda)
 			result = 1;
@@ -433,7 +532,7 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
 		break;
 	default:
 		apic_debug("Bad DFR vcpu %d: %08x\n",
-			   apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR));
+			   apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR));
 		break;
 	}
 
@@ -478,6 +577,72 @@ int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 	return result;
 }
 
+bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
+		struct kvm_lapic_irq *irq, int *r)
+{
+	struct kvm_apic_map *map;
+	unsigned long bitmap = 1;
+	struct kvm_lapic **dst;
+	int i;
+	bool ret = false;
+
+	*r = -1;
+
+	if (irq->shorthand == APIC_DEST_SELF) {
+		*r = kvm_apic_set_irq(src->vcpu, irq);
+		return true;
+	}
+
+	if (irq->shorthand)
+		return false;
+
+	rcu_read_lock();
+	map = rcu_dereference(kvm->arch.apic_map);
+
+	if (!map)
+		goto out;
+
+	if (irq->dest_mode == 0) { /* physical mode */
+		if (irq->delivery_mode == APIC_DM_LOWEST ||
+				irq->dest_id == 0xff)
+			goto out;
+		dst = &map->phys_map[irq->dest_id & 0xff];
+	} else {
+		u32 mda = irq->dest_id << (32 - map->ldr_bits);
+
+		dst = map->logical_map[apic_cluster_id(map, mda)];
+
+		bitmap = apic_logical_id(map, mda);
+
+		if (irq->delivery_mode == APIC_DM_LOWEST) {
+			int l = -1;
+			for_each_set_bit(i, &bitmap, 16) {
+				if (!dst[i])
+					continue;
+				if (l < 0)
+					l = i;
+				else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0)
+					l = i;
+			}
+
+			bitmap = (l >= 0) ? 1 << l : 0;
+		}
+	}
+
+	for_each_set_bit(i, &bitmap, 16) {
+		if (!dst[i])
+			continue;
+		if (*r < 0)
+			*r = 0;
+		*r += kvm_apic_set_irq(dst[i]->vcpu, irq);
+	}
+
+	ret = true;
+out:
+	rcu_read_unlock();
+	return ret;
+}
+
 /*
  * Add a pending IRQ into lapic.
  * Return 1 if successfully added and 0 if discarded.
@@ -591,7 +756,7 @@ static int apic_set_eoi(struct kvm_lapic *apic)
 	apic_clear_isr(vector, apic);
 	apic_update_ppr(apic);
 
-	if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
+	if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
 	    kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
 		int trigger_mode;
 		if (apic_test_vector(vector, apic->regs + APIC_TMR))
@@ -606,8 +771,8 @@ static int apic_set_eoi(struct kvm_lapic *apic)
 
 static void apic_send_ipi(struct kvm_lapic *apic)
 {
-	u32 icr_low = apic_get_reg(apic, APIC_ICR);
-	u32 icr_high = apic_get_reg(apic, APIC_ICR2);
+	u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR);
+	u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2);
 	struct kvm_lapic_irq irq;
 
 	irq.vector = icr_low & APIC_VECTOR_MASK;
@@ -642,7 +807,7 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic)
 	ASSERT(apic != NULL);
 
 	/* if initial count is 0, current count should also be 0 */
-	if (apic_get_reg(apic, APIC_TMICT) == 0)
+	if (kvm_apic_get_reg(apic, APIC_TMICT) == 0)
 		return 0;
 
 	remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
@@ -696,13 +861,15 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
 
 		val = apic_get_tmcct(apic);
 		break;
-
+	case APIC_PROCPRI:
+		apic_update_ppr(apic);
+		val = kvm_apic_get_reg(apic, offset);
+		break;
 	case APIC_TASKPRI:
 		report_tpr_access(apic, false);
 		/* fall thru */
 	default:
-		apic_update_ppr(apic);
-		val = apic_get_reg(apic, offset);
+		val = kvm_apic_get_reg(apic, offset);
 		break;
 	}
 
@@ -719,7 +886,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
 {
 	unsigned char alignment = offset & 0xf;
 	u32 result;
-	/* this bitmask has a bit cleared for each reserver register */
+	/* this bitmask has a bit cleared for each reserved register */
 	static const u64 rmask = 0x43ff01ffffffe70cULL;
 
 	if ((alignment + len) > 4) {
@@ -754,7 +921,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
 
 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
 {
-	return apic_hw_enabled(apic) &&
+	return kvm_apic_hw_enabled(apic) &&
 	    addr >= apic->base_address &&
 	    addr < apic->base_address + LAPIC_MMIO_LENGTH;
 }
@@ -777,7 +944,7 @@ static void update_divide_count(struct kvm_lapic *apic)
 {
 	u32 tmp1, tmp2, tdcr;
 
-	tdcr = apic_get_reg(apic, APIC_TDCR);
+	tdcr = kvm_apic_get_reg(apic, APIC_TDCR);
 	tmp1 = tdcr & 0xf;
 	tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
 	apic->divide_count = 0x1 << (tmp2 & 0x7);
@@ -792,9 +959,9 @@ static void start_apic_timer(struct kvm_lapic *apic)
 	atomic_set(&apic->lapic_timer.pending, 0);
 
 	if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
-		/* lapic timer in oneshot or peroidic mode */
+		/* lapic timer in oneshot or periodic mode */
 		now = apic->lapic_timer.timer.base->get_time();
-		apic->lapic_timer.period = (u64)apic_get_reg(apic, APIC_TMICT)
+		apic->lapic_timer.period = (u64)kvm_apic_get_reg(apic, APIC_TMICT)
 			    * APIC_BUS_CYCLE_NS * apic->divide_count;
 
 		if (!apic->lapic_timer.period)
@@ -826,7 +993,7 @@ static void start_apic_timer(struct kvm_lapic *apic)
 			   "timer initial count 0x%x, period %lldns, "
 			   "expire @ 0x%016" PRIx64 ".\n", __func__,
 			   APIC_BUS_CYCLE_NS, ktime_to_ns(now),
-			   apic_get_reg(apic, APIC_TMICT),
+			   kvm_apic_get_reg(apic, APIC_TMICT),
 			   apic->lapic_timer.period,
 			   ktime_to_ns(ktime_add_ns(now,
 					apic->lapic_timer.period)));
@@ -858,7 +1025,7 @@ static void start_apic_timer(struct kvm_lapic *apic)
 
 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
 {
-	int nmi_wd_enabled = apic_lvt_nmi_mode(apic_get_reg(apic, APIC_LVT0));
+	int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0));
 
 	if (apic_lvt_nmi_mode(lvt0_val)) {
 		if (!nmi_wd_enabled) {
@@ -879,7 +1046,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
 	switch (reg) {
 	case APIC_ID:		/* Local APIC ID */
 		if (!apic_x2apic_mode(apic))
-			apic_set_reg(apic, APIC_ID, val);
+			kvm_apic_set_id(apic, val >> 24);
 		else
 			ret = 1;
 		break;
@@ -895,29 +1062,30 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
 
 	case APIC_LDR:
 		if (!apic_x2apic_mode(apic))
-			apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK);
+			kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
 		else
 			ret = 1;
 		break;
 
 	case APIC_DFR:
-		if (!apic_x2apic_mode(apic))
+		if (!apic_x2apic_mode(apic)) {
 			apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
-		else
+			recalculate_apic_map(apic->vcpu->kvm);
+		} else
 			ret = 1;
 		break;
 
 	case APIC_SPIV: {
 		u32 mask = 0x3ff;
-		if (apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
+		if (kvm_apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
 			mask |= APIC_SPIV_DIRECTED_EOI;
-		apic_set_reg(apic, APIC_SPIV, val & mask);
+		apic_set_spiv(apic, val & mask);
 		if (!(val & APIC_SPIV_APIC_ENABLED)) {
 			int i;
 			u32 lvt_val;
 
 			for (i = 0; i < APIC_LVT_NUM; i++) {
-				lvt_val = apic_get_reg(apic,
+				lvt_val = kvm_apic_get_reg(apic,
 						       APIC_LVTT + 0x10 * i);
 				apic_set_reg(apic, APIC_LVTT + 0x10 * i,
 					     lvt_val | APIC_LVT_MASKED);
@@ -946,7 +1114,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
 	case APIC_LVT1:
 	case APIC_LVTERR:
 		/* TODO: Check vector */
-		if (!apic_sw_enabled(apic))
+		if (!kvm_apic_sw_enabled(apic))
 			val |= APIC_LVT_MASKED;
 
 		val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
@@ -955,12 +1123,12 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
 		break;
 
 	case APIC_LVTT:
-		if ((apic_get_reg(apic, APIC_LVTT) &
+		if ((kvm_apic_get_reg(apic, APIC_LVTT) &
 		    apic->lapic_timer.timer_mode_mask) !=
 		   (val & apic->lapic_timer.timer_mode_mask))
 			hrtimer_cancel(&apic->lapic_timer.timer);
 
-		if (!apic_sw_enabled(apic))
+		if (!kvm_apic_sw_enabled(apic))
 			val |= APIC_LVT_MASKED;
 		val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
 		apic_set_reg(apic, APIC_LVTT, val);
@@ -1039,24 +1207,30 @@ static int apic_mmio_write(struct kvm_io_device *this,
 
 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
 {
-	struct kvm_lapic *apic = vcpu->arch.apic;
-
-	if (apic)
+	if (kvm_vcpu_has_lapic(vcpu))
 		apic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
 
 void kvm_free_lapic(struct kvm_vcpu *vcpu)
 {
+	struct kvm_lapic *apic = vcpu->arch.apic;
+
 	if (!vcpu->arch.apic)
 		return;
 
-	hrtimer_cancel(&vcpu->arch.apic->lapic_timer.timer);
+	hrtimer_cancel(&apic->lapic_timer.timer);
+
+	if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
+		static_key_slow_dec_deferred(&apic_hw_disabled);
+
+	if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED))
+		static_key_slow_dec_deferred(&apic_sw_disabled);
 
-	if (vcpu->arch.apic->regs)
-		free_page((unsigned long)vcpu->arch.apic->regs);
+	if (apic->regs)
+		free_page((unsigned long)apic->regs);
 
-	kfree(vcpu->arch.apic);
+	kfree(apic);
 }
 
 /*
@@ -1068,10 +1242,9 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu)
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
-	if (!apic)
-		return 0;
 
-	if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic))
+	if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) ||
+			apic_lvtt_period(apic))
 		return 0;
 
 	return apic->lapic_timer.tscdeadline;
@@ -1080,10 +1253,9 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
-	if (!apic)
-		return;
 
-	if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic))
+	if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) ||
+			apic_lvtt_period(apic))
 		return;
 
 	hrtimer_cancel(&apic->lapic_timer.timer);
@@ -1095,20 +1267,21 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (!apic)
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return;
+
 	apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
-		     | (apic_get_reg(apic, APIC_TASKPRI) & 4));
+		     | (kvm_apic_get_reg(apic, APIC_TASKPRI) & 4));
 }
 
 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
 {
-	struct kvm_lapic *apic = vcpu->arch.apic;
 	u64 tpr;
 
-	if (!apic)
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return 0;
-	tpr = (u64) apic_get_reg(apic, APIC_TASKPRI);
+
+	tpr = (u64) kvm_apic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
 
 	return (tpr & 0xf0) >> 4;
 }
@@ -1123,6 +1296,15 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 		return;
 	}
 
+	/* update jump label if enable bit changes */
+	if ((vcpu->arch.apic_base ^ value) & MSR_IA32_APICBASE_ENABLE) {
+		if (value & MSR_IA32_APICBASE_ENABLE)
+			static_key_slow_dec_deferred(&apic_hw_disabled);
+		else
+			static_key_slow_inc(&apic_hw_disabled.key);
+		recalculate_apic_map(vcpu->kvm);
+	}
+
 	if (!kvm_vcpu_is_bsp(apic->vcpu))
 		value &= ~MSR_IA32_APICBASE_BSP;
 
@@ -1130,7 +1312,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 	if (apic_x2apic_mode(apic)) {
 		u32 id = kvm_apic_id(apic);
 		u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf));
-		apic_set_reg(apic, APIC_LDR, ldr);
+		kvm_apic_set_ldr(apic, ldr);
 	}
 	apic->base_address = apic->vcpu->arch.apic_base &
 			     MSR_IA32_APICBASE_BASE;
@@ -1155,7 +1337,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 	/* Stop the timer in case it's a reset to an active apic */
 	hrtimer_cancel(&apic->lapic_timer.timer);
 
-	apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24);
+	kvm_apic_set_id(apic, vcpu->vcpu_id);
 	kvm_apic_set_version(apic->vcpu);
 
 	for (i = 0; i < APIC_LVT_NUM; i++)
@@ -1164,9 +1346,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 		     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
 
 	apic_set_reg(apic, APIC_DFR, 0xffffffffU);
-	apic_set_reg(apic, APIC_SPIV, 0xff);
+	apic_set_spiv(apic, 0xff);
 	apic_set_reg(apic, APIC_TASKPRI, 0);
-	apic_set_reg(apic, APIC_LDR, 0);
+	kvm_apic_set_ldr(apic, 0);
 	apic_set_reg(apic, APIC_ESR, 0);
 	apic_set_reg(apic, APIC_ICR, 0);
 	apic_set_reg(apic, APIC_ICR2, 0);
@@ -1183,7 +1365,8 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 	update_divide_count(apic);
 	atomic_set(&apic->lapic_timer.pending, 0);
 	if (kvm_vcpu_is_bsp(vcpu))
-		vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP;
+		kvm_lapic_set_base(vcpu,
+				vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
 	vcpu->arch.pv_eoi.msr_val = 0;
 	apic_update_ppr(apic);
 
@@ -1196,45 +1379,34 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
 		   vcpu->arch.apic_base, apic->base_address);
 }
 
-bool kvm_apic_present(struct kvm_vcpu *vcpu)
-{
-	return vcpu->arch.apic && apic_hw_enabled(vcpu->arch.apic);
-}
-
-int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
-{
-	return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic);
-}
-
 /*
  *----------------------------------------------------------------------
  * timer interface
  *----------------------------------------------------------------------
  */
 
-static bool lapic_is_periodic(struct kvm_timer *ktimer)
+static bool lapic_is_periodic(struct kvm_lapic *apic)
 {
-	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic,
-					      lapic_timer);
 	return apic_lvtt_period(apic);
 }
 
 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
 {
-	struct kvm_lapic *lapic = vcpu->arch.apic;
+	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT))
-		return atomic_read(&lapic->lapic_timer.pending);
+	if (kvm_vcpu_has_lapic(vcpu) && apic_enabled(apic) &&
+			apic_lvt_enabled(apic, APIC_LVTT))
+		return atomic_read(&apic->lapic_timer.pending);
 
 	return 0;
 }
 
 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
 {
-	u32 reg = apic_get_reg(apic, lvt_type);
+	u32 reg = kvm_apic_get_reg(apic, lvt_type);
 	int vector, mode, trig_mode;
 
-	if (apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
+	if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
 		vector = reg & APIC_VECTOR_MASK;
 		mode = reg & APIC_MODE_MASK;
 		trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
@@ -1251,15 +1423,40 @@ void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
 		kvm_apic_local_deliver(apic, APIC_LVT0);
 }
 
-static struct kvm_timer_ops lapic_timer_ops = {
-	.is_periodic = lapic_is_periodic,
-};
-
 static const struct kvm_io_device_ops apic_mmio_ops = {
 	.read     = apic_mmio_read,
 	.write    = apic_mmio_write,
 };
 
+static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
+{
+	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
+	struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
+	struct kvm_vcpu *vcpu = apic->vcpu;
+	wait_queue_head_t *q = &vcpu->wq;
+
+	/*
+	 * There is a race window between reading and incrementing, but we do
+	 * not care about potentially losing timer events in the !reinject
+	 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked
+	 * in vcpu_enter_guest.
+	 */
+	if (!atomic_read(&ktimer->pending)) {
+		atomic_inc(&ktimer->pending);
+		/* FIXME: this code should not know anything about vcpus */
+		kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
+	}
+
+	if (waitqueue_active(q))
+		wake_up_interruptible(q);
+
+	if (lapic_is_periodic(apic)) {
+		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
+		return HRTIMER_RESTART;
+	} else
+		return HRTIMER_NORESTART;
+}
+
 int kvm_create_lapic(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic;
@@ -1283,14 +1480,17 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
 
 	hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
 		     HRTIMER_MODE_ABS);
-	apic->lapic_timer.timer.function = kvm_timer_fn;
-	apic->lapic_timer.t_ops = &lapic_timer_ops;
-	apic->lapic_timer.kvm = vcpu->kvm;
-	apic->lapic_timer.vcpu = vcpu;
+	apic->lapic_timer.timer.function = apic_timer_fn;
 
-	apic->base_address = APIC_DEFAULT_PHYS_BASE;
-	vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE;
+	/*
+	 * APIC is created enabled. This will prevent kvm_lapic_set_base from
+	 * thinking that APIC satet has changed.
+	 */
+	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
+	kvm_lapic_set_base(vcpu,
+			APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE);
 
+	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
 	kvm_lapic_reset(vcpu);
 	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
 
@@ -1306,23 +1506,23 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
 	struct kvm_lapic *apic = vcpu->arch.apic;
 	int highest_irr;
 
-	if (!apic || !apic_enabled(apic))
+	if (!kvm_vcpu_has_lapic(vcpu) || !apic_enabled(apic))
 		return -1;
 
 	apic_update_ppr(apic);
 	highest_irr = apic_find_highest_irr(apic);
 	if ((highest_irr == -1) ||
-	    ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI)))
+	    ((highest_irr & 0xF0) <= kvm_apic_get_reg(apic, APIC_PROCPRI)))
 		return -1;
 	return highest_irr;
 }
 
 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
 {
-	u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0);
+	u32 lvt0 = kvm_apic_get_reg(vcpu->arch.apic, APIC_LVT0);
 	int r = 0;
 
-	if (!apic_hw_enabled(vcpu->arch.apic))
+	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
 		r = 1;
 	if ((lvt0 & APIC_LVT_MASKED) == 0 &&
 	    GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
@@ -1334,7 +1534,10 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (apic && atomic_read(&apic->lapic_timer.pending) > 0) {
+	if (!kvm_vcpu_has_lapic(vcpu))
+		return;
+
+	if (atomic_read(&apic->lapic_timer.pending) > 0) {
 		if (kvm_apic_local_deliver(apic, APIC_LVTT))
 			atomic_dec(&apic->lapic_timer.pending);
 	}
@@ -1354,12 +1557,17 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
 	return vector;
 }
 
-void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
+void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
+		struct kvm_lapic_state *s)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	apic->base_address = vcpu->arch.apic_base &
-			     MSR_IA32_APICBASE_BASE;
+	kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
+	/* set SPIV separately to get count of SW disabled APICs right */
+	apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
+	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
+	/* call kvm_apic_set_id() to put apic into apic_map */
+	kvm_apic_set_id(apic, kvm_apic_id(apic));
 	kvm_apic_set_version(vcpu);
 
 	apic_update_ppr(apic);
@@ -1374,13 +1582,12 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
 
 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
 {
-	struct kvm_lapic *apic = vcpu->arch.apic;
 	struct hrtimer *timer;
 
-	if (!apic)
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return;
 
-	timer = &apic->lapic_timer.timer;
+	timer = &vcpu->arch.apic->lapic_timer.timer;
 	if (hrtimer_cancel(timer))
 		hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
 }
@@ -1478,7 +1685,7 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
 	if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
 		return;
 
-	tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff;
+	tpr = kvm_apic_get_reg(apic, APIC_TASKPRI) & 0xff;
 	max_irr = apic_find_highest_irr(apic);
 	if (max_irr < 0)
 		max_irr = 0;
@@ -1537,7 +1744,7 @@ int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	if (!irqchip_in_kernel(vcpu->kvm))
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return 1;
 
 	/* if this is ICR write vector before command */
@@ -1551,7 +1758,7 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
 	struct kvm_lapic *apic = vcpu->arch.apic;
 	u32 low, high = 0;
 
-	if (!irqchip_in_kernel(vcpu->kvm))
+	if (!kvm_vcpu_has_lapic(vcpu))
 		return 1;
 
 	if (apic_reg_read(apic, reg, 4, &low))
@@ -1576,3 +1783,10 @@ int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
 	return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
 					 addr);
 }
+
+void kvm_lapic_init(void)
+{
+	/* do not patch jump label more than once per second */
+	jump_label_rate_limit(&apic_hw_disabled, HZ);
+	jump_label_rate_limit(&apic_sw_disabled, HZ);
+}
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 4af5405ae1e..e5ebf9f3571 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -2,10 +2,17 @@
 #define __KVM_X86_LAPIC_H
 
 #include "iodev.h"
-#include "kvm_timer.h"
 
 #include <linux/kvm_host.h>
 
+struct kvm_timer {
+	struct hrtimer timer;
+	s64 period; 				/* unit: ns */
+	u32 timer_mode_mask;
+	u64 tscdeadline;
+	atomic_t pending;			/* accumulated triggered timers */
+};
+
 struct kvm_lapic {
 	unsigned long base_address;
 	struct kvm_io_device dev;
@@ -45,11 +52,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda);
 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq);
 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type);
 
+bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
+		struct kvm_lapic_irq *irq, int *r);
+
 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu);
 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data);
-void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu);
-int kvm_lapic_enabled(struct kvm_vcpu *vcpu);
-bool kvm_apic_present(struct kvm_vcpu *vcpu);
+void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
+		struct kvm_lapic_state *s);
 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
 
 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
@@ -71,4 +80,48 @@ static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu)
 }
 
 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data);
+void kvm_lapic_init(void);
+
+static inline u32 kvm_apic_get_reg(struct kvm_lapic *apic, int reg_off)
+{
+	        return *((u32 *) (apic->regs + reg_off));
+}
+
+extern struct static_key kvm_no_apic_vcpu;
+
+static inline bool kvm_vcpu_has_lapic(struct kvm_vcpu *vcpu)
+{
+	if (static_key_false(&kvm_no_apic_vcpu))
+		return vcpu->arch.apic;
+	return true;
+}
+
+extern struct static_key_deferred apic_hw_disabled;
+
+static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
+{
+	if (static_key_false(&apic_hw_disabled.key))
+		return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
+	return MSR_IA32_APICBASE_ENABLE;
+}
+
+extern struct static_key_deferred apic_sw_disabled;
+
+static inline int kvm_apic_sw_enabled(struct kvm_lapic *apic)
+{
+	if (static_key_false(&apic_sw_disabled.key))
+		return kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED;
+	return APIC_SPIV_APIC_ENABLED;
+}
+
+static inline bool kvm_apic_present(struct kvm_vcpu *vcpu)
+{
+	return kvm_vcpu_has_lapic(vcpu) && kvm_apic_hw_enabled(vcpu->arch.apic);
+}
+
+static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
+{
+	return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic);
+}
+
 #endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 7fbd0d273ea..d289fee1ffb 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -556,6 +556,14 @@ static int mmu_spte_clear_track_bits(u64 *sptep)
 		return 0;
 
 	pfn = spte_to_pfn(old_spte);
+
+	/*
+	 * KVM does not hold the refcount of the page used by
+	 * kvm mmu, before reclaiming the page, we should
+	 * unmap it from mmu first.
+	 */
+	WARN_ON(!kvm_is_mmio_pfn(pfn) && !page_count(pfn_to_page(pfn)));
+
 	if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
 		kvm_set_pfn_accessed(pfn);
 	if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
@@ -960,13 +968,10 @@ static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
 static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
 				    struct kvm_memory_slot *slot)
 {
-	struct kvm_lpage_info *linfo;
-
-	if (likely(level == PT_PAGE_TABLE_LEVEL))
-		return &slot->rmap[gfn - slot->base_gfn];
+	unsigned long idx;
 
-	linfo = lpage_info_slot(gfn, slot, level);
-	return &linfo->rmap_pde;
+	idx = gfn_to_index(gfn, slot->base_gfn, level);
+	return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
 }
 
 /*
@@ -1173,7 +1178,8 @@ void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
 	unsigned long *rmapp;
 
 	while (mask) {
-		rmapp = &slot->rmap[gfn_offset + __ffs(mask)];
+		rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
+				      PT_PAGE_TABLE_LEVEL, slot);
 		__rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false);
 
 		/* clear the first set bit */
@@ -1200,7 +1206,7 @@ static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
 }
 
 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
-			   unsigned long data)
+			   struct kvm_memory_slot *slot, unsigned long data)
 {
 	u64 *sptep;
 	struct rmap_iterator iter;
@@ -1218,7 +1224,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
 }
 
 static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
-			     unsigned long data)
+			     struct kvm_memory_slot *slot, unsigned long data)
 {
 	u64 *sptep;
 	struct rmap_iterator iter;
@@ -1259,43 +1265,67 @@ static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
 	return 0;
 }
 
-static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
-			  unsigned long data,
-			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
-					 unsigned long data))
+static int kvm_handle_hva_range(struct kvm *kvm,
+				unsigned long start,
+				unsigned long end,
+				unsigned long data,
+				int (*handler)(struct kvm *kvm,
+					       unsigned long *rmapp,
+					       struct kvm_memory_slot *slot,
+					       unsigned long data))
 {
 	int j;
-	int ret;
-	int retval = 0;
+	int ret = 0;
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 
 	slots = kvm_memslots(kvm);
 
 	kvm_for_each_memslot(memslot, slots) {
-		unsigned long start = memslot->userspace_addr;
-		unsigned long end;
+		unsigned long hva_start, hva_end;
+		gfn_t gfn_start, gfn_end;
 
-		end = start + (memslot->npages << PAGE_SHIFT);
-		if (hva >= start && hva < end) {
-			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
-			gfn_t gfn = memslot->base_gfn + gfn_offset;
+		hva_start = max(start, memslot->userspace_addr);
+		hva_end = min(end, memslot->userspace_addr +
+					(memslot->npages << PAGE_SHIFT));
+		if (hva_start >= hva_end)
+			continue;
+		/*
+		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
+		 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
+		 */
+		gfn_start = hva_to_gfn_memslot(hva_start, memslot);
+		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 
-			ret = handler(kvm, &memslot->rmap[gfn_offset], data);
+		for (j = PT_PAGE_TABLE_LEVEL;
+		     j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
+			unsigned long idx, idx_end;
+			unsigned long *rmapp;
 
-			for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) {
-				struct kvm_lpage_info *linfo;
+			/*
+			 * {idx(page_j) | page_j intersects with
+			 *  [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
+			 */
+			idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
+			idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
 
-				linfo = lpage_info_slot(gfn, memslot,
-							PT_DIRECTORY_LEVEL + j);
-				ret |= handler(kvm, &linfo->rmap_pde, data);
-			}
-			trace_kvm_age_page(hva, memslot, ret);
-			retval |= ret;
+			rmapp = __gfn_to_rmap(gfn_start, j, memslot);
+
+			for (; idx <= idx_end; ++idx)
+				ret |= handler(kvm, rmapp++, memslot, data);
 		}
 	}
 
-	return retval;
+	return ret;
+}
+
+static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
+			  unsigned long data,
+			  int (*handler)(struct kvm *kvm, unsigned long *rmapp,
+					 struct kvm_memory_slot *slot,
+					 unsigned long data))
+{
+	return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
 }
 
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1303,13 +1333,18 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 	return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
 }
 
+int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
+{
+	return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
+}
+
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
 {
 	kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
 }
 
 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
-			 unsigned long data)
+			 struct kvm_memory_slot *slot, unsigned long data)
 {
 	u64 *sptep;
 	struct rmap_iterator uninitialized_var(iter);
@@ -1323,8 +1358,10 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 	 * This has some overhead, but not as much as the cost of swapping
 	 * out actively used pages or breaking up actively used hugepages.
 	 */
-	if (!shadow_accessed_mask)
-		return kvm_unmap_rmapp(kvm, rmapp, data);
+	if (!shadow_accessed_mask) {
+		young = kvm_unmap_rmapp(kvm, rmapp, slot, data);
+		goto out;
+	}
 
 	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
 	     sptep = rmap_get_next(&iter)) {
@@ -1336,12 +1373,14 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
 				 (unsigned long *)sptep);
 		}
 	}
-
+out:
+	/* @data has hva passed to kvm_age_hva(). */
+	trace_kvm_age_page(data, slot, young);
 	return young;
 }
 
 static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
-			      unsigned long data)
+			      struct kvm_memory_slot *slot, unsigned long data)
 {
 	u64 *sptep;
 	struct rmap_iterator iter;
@@ -1379,13 +1418,13 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
 
 	rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
 
-	kvm_unmap_rmapp(vcpu->kvm, rmapp, 0);
+	kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, 0);
 	kvm_flush_remote_tlbs(vcpu->kvm);
 }
 
 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
 {
-	return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp);
+	return kvm_handle_hva(kvm, hva, hva, kvm_age_rmapp);
 }
 
 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
@@ -2457,7 +2496,9 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 				rmap_recycle(vcpu, sptep, gfn);
 		}
 	}
-	kvm_release_pfn_clean(pfn);
+
+	if (!is_error_pfn(pfn))
+		kvm_release_pfn_clean(pfn);
 }
 
 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
@@ -2469,17 +2510,12 @@ static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
 				     bool no_dirty_log)
 {
 	struct kvm_memory_slot *slot;
-	unsigned long hva;
 
 	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
-	if (!slot) {
-		get_page(fault_page);
-		return page_to_pfn(fault_page);
-	}
+	if (!slot)
+		return KVM_PFN_ERR_FAULT;
 
-	hva = gfn_to_hva_memslot(slot, gfn);
-
-	return hva_to_pfn_atomic(vcpu->kvm, hva);
+	return gfn_to_pfn_memslot_atomic(slot, gfn);
 }
 
 static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
@@ -2580,11 +2616,6 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
 			sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
 					      iterator.level - 1,
 					      1, ACC_ALL, iterator.sptep);
-			if (!sp) {
-				pgprintk("nonpaging_map: ENOMEM\n");
-				kvm_release_pfn_clean(pfn);
-				return -ENOMEM;
-			}
 
 			mmu_spte_set(iterator.sptep,
 				     __pa(sp->spt)
@@ -2611,8 +2642,16 @@ static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *
 
 static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
 {
-	kvm_release_pfn_clean(pfn);
-	if (is_hwpoison_pfn(pfn)) {
+	/*
+	 * Do not cache the mmio info caused by writing the readonly gfn
+	 * into the spte otherwise read access on readonly gfn also can
+	 * caused mmio page fault and treat it as mmio access.
+	 * Return 1 to tell kvm to emulate it.
+	 */
+	if (pfn == KVM_PFN_ERR_RO_FAULT)
+		return 1;
+
+	if (pfn == KVM_PFN_ERR_HWPOISON) {
 		kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
 		return 0;
 	}
@@ -3236,8 +3275,6 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
 	if (!async)
 		return false; /* *pfn has correct page already */
 
-	put_page(pfn_to_page(*pfn));
-
 	if (!prefault && can_do_async_pf(vcpu)) {
 		trace_kvm_try_async_get_page(gva, gfn);
 		if (kvm_find_async_pf_gfn(vcpu, gfn)) {
@@ -3371,6 +3408,18 @@ static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
 	return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
 }
 
+static inline void protect_clean_gpte(unsigned *access, unsigned gpte)
+{
+	unsigned mask;
+
+	BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
+
+	mask = (unsigned)~ACC_WRITE_MASK;
+	/* Allow write access to dirty gptes */
+	mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK;
+	*access &= mask;
+}
+
 static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
 			   int *nr_present)
 {
@@ -3388,6 +3437,25 @@ static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
 	return false;
 }
 
+static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
+{
+	unsigned access;
+
+	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
+	access &= ~(gpte >> PT64_NX_SHIFT);
+
+	return access;
+}
+
+static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
+{
+	unsigned index;
+
+	index = level - 1;
+	index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
+	return mmu->last_pte_bitmap & (1 << index);
+}
+
 #define PTTYPE 64
 #include "paging_tmpl.h"
 #undef PTTYPE
@@ -3457,6 +3525,56 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
 	}
 }
 
+static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+	unsigned bit, byte, pfec;
+	u8 map;
+	bool fault, x, w, u, wf, uf, ff, smep;
+
+	smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
+	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
+		pfec = byte << 1;
+		map = 0;
+		wf = pfec & PFERR_WRITE_MASK;
+		uf = pfec & PFERR_USER_MASK;
+		ff = pfec & PFERR_FETCH_MASK;
+		for (bit = 0; bit < 8; ++bit) {
+			x = bit & ACC_EXEC_MASK;
+			w = bit & ACC_WRITE_MASK;
+			u = bit & ACC_USER_MASK;
+
+			/* Not really needed: !nx will cause pte.nx to fault */
+			x |= !mmu->nx;
+			/* Allow supervisor writes if !cr0.wp */
+			w |= !is_write_protection(vcpu) && !uf;
+			/* Disallow supervisor fetches of user code if cr4.smep */
+			x &= !(smep && u && !uf);
+
+			fault = (ff && !x) || (uf && !u) || (wf && !w);
+			map |= fault << bit;
+		}
+		mmu->permissions[byte] = map;
+	}
+}
+
+static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+	u8 map;
+	unsigned level, root_level = mmu->root_level;
+	const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */
+
+	if (root_level == PT32E_ROOT_LEVEL)
+		--root_level;
+	/* PT_PAGE_TABLE_LEVEL always terminates */
+	map = 1 | (1 << ps_set_index);
+	for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
+		if (level <= PT_PDPE_LEVEL
+		    && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
+			map |= 1 << (ps_set_index | (level - 1));
+	}
+	mmu->last_pte_bitmap = map;
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 					struct kvm_mmu *context,
 					int level)
@@ -3465,6 +3583,8 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 	context->root_level = level;
 
 	reset_rsvds_bits_mask(vcpu, context);
+	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	ASSERT(is_pae(vcpu));
 	context->new_cr3 = paging_new_cr3;
@@ -3493,6 +3613,8 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
 	context->root_level = PT32_ROOT_LEVEL;
 
 	reset_rsvds_bits_mask(vcpu, context);
+	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
 
 	context->new_cr3 = paging_new_cr3;
 	context->page_fault = paging32_page_fault;
@@ -3553,6 +3675,9 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
 		context->gva_to_gpa = paging32_gva_to_gpa;
 	}
 
+	update_permission_bitmask(vcpu, context);
+	update_last_pte_bitmap(vcpu, context);
+
 	return 0;
 }
 
@@ -3628,6 +3753,9 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
 		g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
 	}
 
+	update_permission_bitmask(vcpu, g_context);
+	update_last_pte_bitmap(vcpu, g_context);
+
 	return 0;
 }
 
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index e374db9af02..69871080e86 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -18,8 +18,10 @@
 #define PT_PCD_MASK (1ULL << 4)
 #define PT_ACCESSED_SHIFT 5
 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
-#define PT_DIRTY_MASK (1ULL << 6)
-#define PT_PAGE_SIZE_MASK (1ULL << 7)
+#define PT_DIRTY_SHIFT 6
+#define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
+#define PT_PAGE_SIZE_SHIFT 7
+#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
 #define PT_PAT_MASK (1ULL << 7)
 #define PT_GLOBAL_MASK (1ULL << 8)
 #define PT64_NX_SHIFT 63
@@ -88,17 +90,14 @@ static inline bool is_write_protection(struct kvm_vcpu *vcpu)
 	return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
 }
 
-static inline bool check_write_user_access(struct kvm_vcpu *vcpu,
-					   bool write_fault, bool user_fault,
-					   unsigned long pte)
+/*
+ * Will a fault with a given page-fault error code (pfec) cause a permission
+ * fault with the given access (in ACC_* format)?
+ */
+static inline bool permission_fault(struct kvm_mmu *mmu, unsigned pte_access,
+				    unsigned pfec)
 {
-	if (unlikely(write_fault && !is_writable_pte(pte)
-	      && (user_fault || is_write_protection(vcpu))))
-		return false;
-
-	if (unlikely(user_fault && !(pte & PT_USER_MASK)))
-		return false;
-
-	return true;
+	return (mmu->permissions[pfec >> 1] >> pte_access) & 1;
 }
+
 #endif
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index 7d7d0b9e23e..daff69e2115 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -116,10 +116,8 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
 	gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
 	pfn = gfn_to_pfn_atomic(vcpu->kvm, gfn);
 
-	if (is_error_pfn(pfn)) {
-		kvm_release_pfn_clean(pfn);
+	if (is_error_pfn(pfn))
 		return;
-	}
 
 	hpa =  pfn << PAGE_SHIFT;
 	if ((*sptep & PT64_BASE_ADDR_MASK) != hpa)
@@ -190,7 +188,6 @@ static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp)
 
 static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
 {
-	struct kvm_memory_slot *slot;
 	unsigned long *rmapp;
 	u64 *sptep;
 	struct rmap_iterator iter;
@@ -198,8 +195,7 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
 	if (sp->role.direct || sp->unsync || sp->role.invalid)
 		return;
 
-	slot = gfn_to_memslot(kvm, sp->gfn);
-	rmapp = &slot->rmap[sp->gfn - slot->base_gfn];
+	rmapp = gfn_to_rmap(kvm, sp->gfn, PT_PAGE_TABLE_LEVEL);
 
 	for (sptep = rmap_get_first(*rmapp, &iter); sptep;
 	     sptep = rmap_get_next(&iter)) {
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index bb7cf01cae7..714e2c01a6f 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -63,10 +63,12 @@
  */
 struct guest_walker {
 	int level;
+	unsigned max_level;
 	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
 	pt_element_t ptes[PT_MAX_FULL_LEVELS];
 	pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
 	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
+	pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS];
 	unsigned pt_access;
 	unsigned pte_access;
 	gfn_t gfn;
@@ -101,38 +103,41 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 	return (ret != orig_pte);
 }
 
-static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte,
-				   bool last)
+static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
+					     struct kvm_mmu *mmu,
+					     struct guest_walker *walker,
+					     int write_fault)
 {
-	unsigned access;
-
-	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
-	if (last && !is_dirty_gpte(gpte))
-		access &= ~ACC_WRITE_MASK;
-
-#if PTTYPE == 64
-	if (vcpu->arch.mmu.nx)
-		access &= ~(gpte >> PT64_NX_SHIFT);
-#endif
-	return access;
-}
-
-static bool FNAME(is_last_gpte)(struct guest_walker *walker,
-				struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-				pt_element_t gpte)
-{
-	if (walker->level == PT_PAGE_TABLE_LEVEL)
-		return true;
-
-	if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) &&
-	    (PTTYPE == 64 || is_pse(vcpu)))
-		return true;
+	unsigned level, index;
+	pt_element_t pte, orig_pte;
+	pt_element_t __user *ptep_user;
+	gfn_t table_gfn;
+	int ret;
+
+	for (level = walker->max_level; level >= walker->level; --level) {
+		pte = orig_pte = walker->ptes[level - 1];
+		table_gfn = walker->table_gfn[level - 1];
+		ptep_user = walker->ptep_user[level - 1];
+		index = offset_in_page(ptep_user) / sizeof(pt_element_t);
+		if (!(pte & PT_ACCESSED_MASK)) {
+			trace_kvm_mmu_set_accessed_bit(table_gfn, index, sizeof(pte));
+			pte |= PT_ACCESSED_MASK;
+		}
+		if (level == walker->level && write_fault && !is_dirty_gpte(pte)) {
+			trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
+			pte |= PT_DIRTY_MASK;
+		}
+		if (pte == orig_pte)
+			continue;
 
-	if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) &&
-	    (mmu->root_level == PT64_ROOT_LEVEL))
-		return true;
+		ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte);
+		if (ret)
+			return ret;
 
-	return false;
+		mark_page_dirty(vcpu->kvm, table_gfn);
+		walker->ptes[level] = pte;
+	}
+	return 0;
 }
 
 /*
@@ -142,21 +147,22 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 				    struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 				    gva_t addr, u32 access)
 {
+	int ret;
 	pt_element_t pte;
 	pt_element_t __user *uninitialized_var(ptep_user);
 	gfn_t table_gfn;
-	unsigned index, pt_access, uninitialized_var(pte_access);
+	unsigned index, pt_access, pte_access, accessed_dirty, shift;
 	gpa_t pte_gpa;
-	bool eperm, last_gpte;
 	int offset;
 	const int write_fault = access & PFERR_WRITE_MASK;
 	const int user_fault  = access & PFERR_USER_MASK;
 	const int fetch_fault = access & PFERR_FETCH_MASK;
 	u16 errcode = 0;
+	gpa_t real_gpa;
+	gfn_t gfn;
 
 	trace_kvm_mmu_pagetable_walk(addr, access);
 retry_walk:
-	eperm = false;
 	walker->level = mmu->root_level;
 	pte           = mmu->get_cr3(vcpu);
 
@@ -169,15 +175,21 @@ retry_walk:
 		--walker->level;
 	}
 #endif
+	walker->max_level = walker->level;
 	ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
 	       (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0);
 
-	pt_access = ACC_ALL;
+	accessed_dirty = PT_ACCESSED_MASK;
+	pt_access = pte_access = ACC_ALL;
+	++walker->level;
 
-	for (;;) {
+	do {
 		gfn_t real_gfn;
 		unsigned long host_addr;
 
+		pt_access &= pte_access;
+		--walker->level;
+
 		index = PT_INDEX(addr, walker->level);
 
 		table_gfn = gpte_to_gfn(pte);
@@ -199,6 +211,7 @@ retry_walk:
 		ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
 		if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte))))
 			goto error;
+		walker->ptep_user[walker->level - 1] = ptep_user;
 
 		trace_kvm_mmu_paging_element(pte, walker->level);
 
@@ -211,92 +224,48 @@ retry_walk:
 			goto error;
 		}
 
-		if (!check_write_user_access(vcpu, write_fault, user_fault,
-					  pte))
-			eperm = true;
-
-#if PTTYPE == 64
-		if (unlikely(fetch_fault && (pte & PT64_NX_MASK)))
-			eperm = true;
-#endif
-
-		last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
-		if (last_gpte) {
-			pte_access = pt_access &
-				     FNAME(gpte_access)(vcpu, pte, true);
-			/* check if the kernel is fetching from user page */
-			if (unlikely(pte_access & PT_USER_MASK) &&
-			    kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
-				if (fetch_fault && !user_fault)
-					eperm = true;
-		}
-
-		if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) {
-			int ret;
-			trace_kvm_mmu_set_accessed_bit(table_gfn, index,
-						       sizeof(pte));
-			ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index,
-						  pte, pte|PT_ACCESSED_MASK);
-			if (unlikely(ret < 0))
-				goto error;
-			else if (ret)
-				goto retry_walk;
-
-			mark_page_dirty(vcpu->kvm, table_gfn);
-			pte |= PT_ACCESSED_MASK;
-		}
+		accessed_dirty &= pte;
+		pte_access = pt_access & gpte_access(vcpu, pte);
 
 		walker->ptes[walker->level - 1] = pte;
+	} while (!is_last_gpte(mmu, walker->level, pte));
 
-		if (last_gpte) {
-			int lvl = walker->level;
-			gpa_t real_gpa;
-			gfn_t gfn;
-			u32 ac;
-
-			gfn = gpte_to_gfn_lvl(pte, lvl);
-			gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT;
-
-			if (PTTYPE == 32 &&
-			    walker->level == PT_DIRECTORY_LEVEL &&
-			    is_cpuid_PSE36())
-				gfn += pse36_gfn_delta(pte);
-
-			ac = write_fault | fetch_fault | user_fault;
+	if (unlikely(permission_fault(mmu, pte_access, access))) {
+		errcode |= PFERR_PRESENT_MASK;
+		goto error;
+	}
 
-			real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn),
-						      ac);
-			if (real_gpa == UNMAPPED_GVA)
-				return 0;
+	gfn = gpte_to_gfn_lvl(pte, walker->level);
+	gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT;
 
-			walker->gfn = real_gpa >> PAGE_SHIFT;
+	if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36())
+		gfn += pse36_gfn_delta(pte);
 
-			break;
-		}
+	real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access);
+	if (real_gpa == UNMAPPED_GVA)
+		return 0;
 
-		pt_access &= FNAME(gpte_access)(vcpu, pte, false);
-		--walker->level;
-	}
+	walker->gfn = real_gpa >> PAGE_SHIFT;
 
-	if (unlikely(eperm)) {
-		errcode |= PFERR_PRESENT_MASK;
-		goto error;
-	}
+	if (!write_fault)
+		protect_clean_gpte(&pte_access, pte);
 
-	if (write_fault && unlikely(!is_dirty_gpte(pte))) {
-		int ret;
+	/*
+	 * On a write fault, fold the dirty bit into accessed_dirty by shifting it one
+	 * place right.
+	 *
+	 * On a read fault, do nothing.
+	 */
+	shift = write_fault >> ilog2(PFERR_WRITE_MASK);
+	shift *= PT_DIRTY_SHIFT - PT_ACCESSED_SHIFT;
+	accessed_dirty &= pte >> shift;
 
-		trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
-		ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index,
-					  pte, pte|PT_DIRTY_MASK);
+	if (unlikely(!accessed_dirty)) {
+		ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker, write_fault);
 		if (unlikely(ret < 0))
 			goto error;
 		else if (ret)
 			goto retry_walk;
-
-		mark_page_dirty(vcpu->kvm, table_gfn);
-		pte |= PT_DIRTY_MASK;
-		walker->ptes[walker->level - 1] = pte;
 	}
 
 	walker->pt_access = pt_access;
@@ -368,12 +337,11 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 		return;
 
 	pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
-	pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, true);
+	pte_access = sp->role.access & gpte_access(vcpu, gpte);
+	protect_clean_gpte(&pte_access, gpte);
 	pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
-	if (mmu_invalid_pfn(pfn)) {
-		kvm_release_pfn_clean(pfn);
+	if (mmu_invalid_pfn(pfn))
 		return;
-	}
 
 	/*
 	 * we call mmu_set_spte() with host_writable = true because that
@@ -443,15 +411,13 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
 		if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
 			continue;
 
-		pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte,
-								  true);
+		pte_access = sp->role.access & gpte_access(vcpu, gpte);
+		protect_clean_gpte(&pte_access, gpte);
 		gfn = gpte_to_gfn(gpte);
 		pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
 				      pte_access & ACC_WRITE_MASK);
-		if (mmu_invalid_pfn(pfn)) {
-			kvm_release_pfn_clean(pfn);
+		if (mmu_invalid_pfn(pfn))
 			break;
-		}
 
 		mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
 			     NULL, PT_PAGE_TABLE_LEVEL, gfn,
@@ -798,7 +764,8 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 
 		gfn = gpte_to_gfn(gpte);
 		pte_access = sp->role.access;
-		pte_access &= FNAME(gpte_access)(vcpu, gpte, true);
+		pte_access &= gpte_access(vcpu, gpte);
+		protect_clean_gpte(&pte_access, gpte);
 
 		if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present))
 			continue;
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 9b7ec1150ab..cfc258a6bf9 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -1,5 +1,5 @@
 /*
- * Kernel-based Virtual Machine -- Performane Monitoring Unit support
+ * Kernel-based Virtual Machine -- Performance Monitoring Unit support
  *
  * Copyright 2011 Red Hat, Inc. and/or its affiliates.
  *
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index baead950d6c..d017df3899e 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -163,7 +163,7 @@ static DEFINE_PER_CPU(u64, current_tsc_ratio);
 
 #define MSR_INVALID			0xffffffffU
 
-static struct svm_direct_access_msrs {
+static const struct svm_direct_access_msrs {
 	u32 index;   /* Index of the MSR */
 	bool always; /* True if intercept is always on */
 } direct_access_msrs[] = {
@@ -400,7 +400,7 @@ struct svm_init_data {
 	int r;
 };
 
-static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
+static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 
 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 #define MSRS_RANGE_SIZE 2048
@@ -1146,7 +1146,6 @@ static void init_vmcb(struct vcpu_svm *svm)
 
 	svm_set_efer(&svm->vcpu, 0);
 	save->dr6 = 0xffff0ff0;
-	save->dr7 = 0x400;
 	kvm_set_rflags(&svm->vcpu, 2);
 	save->rip = 0x0000fff0;
 	svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
@@ -1643,7 +1642,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
 	mark_dirty(svm->vmcb, VMCB_SEG);
 }
 
-static void update_db_intercept(struct kvm_vcpu *vcpu)
+static void update_db_bp_intercept(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -1663,20 +1662,6 @@ static void update_db_intercept(struct kvm_vcpu *vcpu)
 		vcpu->guest_debug = 0;
 }
 
-static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
-{
-	struct vcpu_svm *svm = to_svm(vcpu);
-
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-		svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
-	else
-		svm->vmcb->save.dr7 = vcpu->arch.dr7;
-
-	mark_dirty(svm->vmcb, VMCB_DR);
-
-	update_db_intercept(vcpu);
-}
-
 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 {
 	if (sd->next_asid > sd->max_asid) {
@@ -1748,7 +1733,7 @@ static int db_interception(struct vcpu_svm *svm)
 		if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
 			svm->vmcb->save.rflags &=
 				~(X86_EFLAGS_TF | X86_EFLAGS_RF);
-		update_db_intercept(&svm->vcpu);
+		update_db_bp_intercept(&svm->vcpu);
 	}
 
 	if (svm->vcpu.guest_debug &
@@ -2063,7 +2048,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm)
 	if (svm->nested.intercept & 1ULL) {
 		/*
 		 * The #vmexit can't be emulated here directly because this
-		 * code path runs with irqs and preemtion disabled. A
+		 * code path runs with irqs and preemption disabled. A
 		 * #vmexit emulation might sleep. Only signal request for
 		 * the #vmexit here.
 		 */
@@ -2105,7 +2090,6 @@ static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
 	return kmap(page);
 
 error:
-	kvm_release_page_clean(page);
 	kvm_inject_gp(&svm->vcpu, 0);
 
 	return NULL;
@@ -2409,7 +2393,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
 {
 	/*
 	 * This function merges the msr permission bitmaps of kvm and the
-	 * nested vmcb. It is omptimized in that it only merges the parts where
+	 * nested vmcb. It is optimized in that it only merges the parts where
 	 * the kvm msr permission bitmap may contain zero bits
 	 */
 	int i;
@@ -3268,7 +3252,7 @@ static int pause_interception(struct vcpu_svm *svm)
 	return 1;
 }
 
-static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = {
+static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
 	[SVM_EXIT_READ_CR0]			= cr_interception,
 	[SVM_EXIT_READ_CR3]			= cr_interception,
 	[SVM_EXIT_READ_CR4]			= cr_interception,
@@ -3660,7 +3644,7 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
 	 */
 	svm->nmi_singlestep = true;
 	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
-	update_db_intercept(vcpu);
+	update_db_bp_intercept(vcpu);
 }
 
 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
@@ -3783,12 +3767,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu)
 	svm_complete_interrupts(svm);
 }
 
-#ifdef CONFIG_X86_64
-#define R "r"
-#else
-#define R "e"
-#endif
-
 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -3815,13 +3793,13 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 	local_irq_enable();
 
 	asm volatile (
-		"push %%"R"bp; \n\t"
-		"mov %c[rbx](%[svm]), %%"R"bx \n\t"
-		"mov %c[rcx](%[svm]), %%"R"cx \n\t"
-		"mov %c[rdx](%[svm]), %%"R"dx \n\t"
-		"mov %c[rsi](%[svm]), %%"R"si \n\t"
-		"mov %c[rdi](%[svm]), %%"R"di \n\t"
-		"mov %c[rbp](%[svm]), %%"R"bp \n\t"
+		"push %%" _ASM_BP "; \n\t"
+		"mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
+		"mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
+		"mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
+		"mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
+		"mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
+		"mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
 #ifdef CONFIG_X86_64
 		"mov %c[r8](%[svm]),  %%r8  \n\t"
 		"mov %c[r9](%[svm]),  %%r9  \n\t"
@@ -3834,20 +3812,20 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 
 		/* Enter guest mode */
-		"push %%"R"ax \n\t"
-		"mov %c[vmcb](%[svm]), %%"R"ax \n\t"
+		"push %%" _ASM_AX " \n\t"
+		"mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
 		__ex(SVM_VMLOAD) "\n\t"
 		__ex(SVM_VMRUN) "\n\t"
 		__ex(SVM_VMSAVE) "\n\t"
-		"pop %%"R"ax \n\t"
+		"pop %%" _ASM_AX " \n\t"
 
 		/* Save guest registers, load host registers */
-		"mov %%"R"bx, %c[rbx](%[svm]) \n\t"
-		"mov %%"R"cx, %c[rcx](%[svm]) \n\t"
-		"mov %%"R"dx, %c[rdx](%[svm]) \n\t"
-		"mov %%"R"si, %c[rsi](%[svm]) \n\t"
-		"mov %%"R"di, %c[rdi](%[svm]) \n\t"
-		"mov %%"R"bp, %c[rbp](%[svm]) \n\t"
+		"mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
+		"mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
+		"mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
+		"mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
+		"mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
+		"mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
 #ifdef CONFIG_X86_64
 		"mov %%r8,  %c[r8](%[svm]) \n\t"
 		"mov %%r9,  %c[r9](%[svm]) \n\t"
@@ -3858,7 +3836,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 		"mov %%r14, %c[r14](%[svm]) \n\t"
 		"mov %%r15, %c[r15](%[svm]) \n\t"
 #endif
-		"pop %%"R"bp"
+		"pop %%" _ASM_BP
 		:
 		: [svm]"a"(svm),
 		  [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
@@ -3879,9 +3857,11 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 		  [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
 #endif
 		: "cc", "memory"
-		, R"bx", R"cx", R"dx", R"si", R"di"
 #ifdef CONFIG_X86_64
+		, "rbx", "rcx", "rdx", "rsi", "rdi"
 		, "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
+#else
+		, "ebx", "ecx", "edx", "esi", "edi"
 #endif
 		);
 
@@ -3941,8 +3921,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 	mark_all_clean(svm->vmcb);
 }
 
-#undef R
-
 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -4069,7 +4047,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
 #define POST_MEM(exit) { .exit_code = (exit), \
 			.stage = X86_ICPT_POST_MEMACCESS, }
 
-static struct __x86_intercept {
+static const struct __x86_intercept {
 	u32 exit_code;
 	enum x86_intercept_stage stage;
 } x86_intercept_map[] = {
@@ -4260,7 +4238,7 @@ static struct kvm_x86_ops svm_x86_ops = {
 	.vcpu_load = svm_vcpu_load,
 	.vcpu_put = svm_vcpu_put,
 
-	.set_guest_debug = svm_guest_debug,
+	.update_db_bp_intercept = update_db_bp_intercept,
 	.get_msr = svm_get_msr,
 	.set_msr = svm_set_msr,
 	.get_segment_base = svm_get_segment_base,
diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c
deleted file mode 100644
index 6b85cc647f3..00000000000
--- a/arch/x86/kvm/timer.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Kernel-based Virtual Machine driver for Linux
- *
- * This module enables machines with Intel VT-x extensions to run virtual
- * machines without emulation or binary translation.
- *
- * timer support
- *
- * Copyright 2010 Red Hat, Inc. and/or its affiliates.
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- */
-
-#include <linux/kvm_host.h>
-#include <linux/kvm.h>
-#include <linux/hrtimer.h>
-#include <linux/atomic.h>
-#include "kvm_timer.h"
-
-enum hrtimer_restart kvm_timer_fn(struct hrtimer *data)
-{
-	struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
-	struct kvm_vcpu *vcpu = ktimer->vcpu;
-	wait_queue_head_t *q = &vcpu->wq;
-
-	/*
-	 * There is a race window between reading and incrementing, but we do
-	 * not care about potentially losing timer events in the !reinject
-	 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked
-	 * in vcpu_enter_guest.
-	 */
-	if (ktimer->reinject || !atomic_read(&ktimer->pending)) {
-		atomic_inc(&ktimer->pending);
-		/* FIXME: this code should not know anything about vcpus */
-		kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
-	}
-
-	if (waitqueue_active(q))
-		wake_up_interruptible(q);
-
-	if (ktimer->t_ops->is_periodic(ktimer)) {
-		hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
-		return HRTIMER_RESTART;
-	} else
-		return HRTIMER_NORESTART;
-}
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 851aa7c3b89..ad6b1dd06f8 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -127,6 +127,8 @@ module_param(ple_gap, int, S_IRUGO);
 static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 module_param(ple_window, int, S_IRUGO);
 
+extern const ulong vmx_return;
+
 #define NR_AUTOLOAD_MSRS 8
 #define VMCS02_POOL_SIZE 1
 
@@ -405,16 +407,16 @@ struct vcpu_vmx {
 	struct {
 		int vm86_active;
 		ulong save_rflags;
+		struct kvm_segment segs[8];
+	} rmode;
+	struct {
+		u32 bitmask; /* 4 bits per segment (1 bit per field) */
 		struct kvm_save_segment {
 			u16 selector;
 			unsigned long base;
 			u32 limit;
 			u32 ar;
-		} tr, es, ds, fs, gs;
-	} rmode;
-	struct {
-		u32 bitmask; /* 4 bits per segment (1 bit per field) */
-		struct kvm_save_segment seg[8];
+		} seg[8];
 	} segment_cache;
 	int vpid;
 	bool emulation_required;
@@ -450,7 +452,7 @@ static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
 #define FIELD64(number, name)	[number] = VMCS12_OFFSET(name), \
 				[number##_HIGH] = VMCS12_OFFSET(name)+4
 
-static unsigned short vmcs_field_to_offset_table[] = {
+static const unsigned short vmcs_field_to_offset_table[] = {
 	FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
 	FIELD(GUEST_ES_SELECTOR, guest_es_selector),
 	FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
@@ -596,10 +598,9 @@ static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
 static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
 {
 	struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT);
-	if (is_error_page(page)) {
-		kvm_release_page_clean(page);
+	if (is_error_page(page))
 		return NULL;
-	}
+
 	return page;
 }
 
@@ -667,7 +668,7 @@ static struct vmx_capability {
 		.ar_bytes = GUEST_##seg##_AR_BYTES,	   	\
 	}
 
-static struct kvm_vmx_segment_field {
+static const struct kvm_vmx_segment_field {
 	unsigned selector;
 	unsigned base;
 	unsigned limit;
@@ -1343,7 +1344,7 @@ static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 	guest_efer = vmx->vcpu.arch.efer;
 
 	/*
-	 * NX is emulated; LMA and LME handled by hardware; SCE meaninless
+	 * NX is emulated; LMA and LME handled by hardware; SCE meaningless
 	 * outside long mode
 	 */
 	ignore_bits = EFER_NX | EFER_SCE;
@@ -1995,7 +1996,7 @@ static __init void nested_vmx_setup_ctls_msrs(void)
 #endif
 		CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
 		CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING |
-		CPU_BASED_RDPMC_EXITING |
+		CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING |
 		CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
 	/*
 	 * We can allow some features even when not supported by the
@@ -2291,16 +2292,6 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 	}
 }
 
-static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
-{
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
-		vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
-	else
-		vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
-
-	update_exception_bitmap(vcpu);
-}
-
 static __init int cpu_has_kvm_support(void)
 {
 	return cpu_has_vmx();
@@ -2698,20 +2689,17 @@ static __exit void hardware_unsetup(void)
 	free_kvm_area();
 }
 
-static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
+static void fix_pmode_dataseg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save)
 {
-	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+	struct kvm_segment tmp = *save;
 
-	if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
-		vmcs_write16(sf->selector, save->selector);
-		vmcs_writel(sf->base, save->base);
-		vmcs_write32(sf->limit, save->limit);
-		vmcs_write32(sf->ar_bytes, save->ar);
-	} else {
-		u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
-			<< AR_DPL_SHIFT;
-		vmcs_write32(sf->ar_bytes, 0x93 | dpl);
+	if (!(vmcs_readl(sf->base) == tmp.base && tmp.s)) {
+		tmp.base = vmcs_readl(sf->base);
+		tmp.selector = vmcs_read16(sf->selector);
+		tmp.s = 1;
 	}
+	vmx_set_segment(vcpu, &tmp, seg);
 }
 
 static void enter_pmode(struct kvm_vcpu *vcpu)
@@ -2724,10 +2712,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
 
 	vmx_segment_cache_clear(vmx);
 
-	vmcs_write16(GUEST_TR_SELECTOR, vmx->rmode.tr.selector);
-	vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
-	vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
-	vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
+	vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
 
 	flags = vmcs_readl(GUEST_RFLAGS);
 	flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
@@ -2742,10 +2727,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
 	if (emulate_invalid_guest_state)
 		return;
 
-	fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es);
-	fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds);
-	fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs);
-	fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs);
+	fix_pmode_dataseg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
+	fix_pmode_dataseg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
+	fix_pmode_dataseg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
+	fix_pmode_dataseg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
 
 	vmx_segment_cache_clear(vmx);
 
@@ -2773,14 +2758,10 @@ static gva_t rmode_tss_base(struct kvm *kvm)
 	return kvm->arch.tss_addr;
 }
 
-static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
+static void fix_rmode_seg(int seg, struct kvm_segment *save)
 {
-	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 
-	save->selector = vmcs_read16(sf->selector);
-	save->base = vmcs_readl(sf->base);
-	save->limit = vmcs_read32(sf->limit);
-	save->ar = vmcs_read32(sf->ar_bytes);
 	vmcs_write16(sf->selector, save->base >> 4);
 	vmcs_write32(sf->base, save->base & 0xffff0);
 	vmcs_write32(sf->limit, 0xffff);
@@ -2800,9 +2781,16 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
 	if (enable_unrestricted_guest)
 		return;
 
+	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
+	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
+	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
+	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
+	vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
+
 	vmx->emulation_required = 1;
 	vmx->rmode.vm86_active = 1;
 
+
 	/*
 	 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
 	 * vcpu. Call it here with phys address pointing 16M below 4G.
@@ -2817,14 +2805,8 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
 
 	vmx_segment_cache_clear(vmx);
 
-	vmx->rmode.tr.selector = vmcs_read16(GUEST_TR_SELECTOR);
-	vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
 	vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
-
-	vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
 	vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
-
-	vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
 	vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
 
 	flags = vmcs_readl(GUEST_RFLAGS);
@@ -3117,35 +3099,24 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	struct kvm_save_segment *save;
 	u32 ar;
 
 	if (vmx->rmode.vm86_active
 	    && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
 		|| seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
-		|| seg == VCPU_SREG_GS)
-	    && !emulate_invalid_guest_state) {
-		switch (seg) {
-		case VCPU_SREG_TR: save = &vmx->rmode.tr; break;
-		case VCPU_SREG_ES: save = &vmx->rmode.es; break;
-		case VCPU_SREG_DS: save = &vmx->rmode.ds; break;
-		case VCPU_SREG_FS: save = &vmx->rmode.fs; break;
-		case VCPU_SREG_GS: save = &vmx->rmode.gs; break;
-		default: BUG();
-		}
-		var->selector = save->selector;
-		var->base = save->base;
-		var->limit = save->limit;
-		ar = save->ar;
+		|| seg == VCPU_SREG_GS)) {
+		*var = vmx->rmode.segs[seg];
 		if (seg == VCPU_SREG_TR
 		    || var->selector == vmx_read_guest_seg_selector(vmx, seg))
-			goto use_saved_rmode_seg;
+			return;
+		var->base = vmx_read_guest_seg_base(vmx, seg);
+		var->selector = vmx_read_guest_seg_selector(vmx, seg);
+		return;
 	}
 	var->base = vmx_read_guest_seg_base(vmx, seg);
 	var->limit = vmx_read_guest_seg_limit(vmx, seg);
 	var->selector = vmx_read_guest_seg_selector(vmx, seg);
 	ar = vmx_read_guest_seg_ar(vmx, seg);
-use_saved_rmode_seg:
 	if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
 		ar = 0;
 	var->type = ar & 15;
@@ -3227,23 +3198,21 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
 			    struct kvm_segment *var, int seg)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 	u32 ar;
 
 	vmx_segment_cache_clear(vmx);
 
 	if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
 		vmcs_write16(sf->selector, var->selector);
-		vmx->rmode.tr.selector = var->selector;
-		vmx->rmode.tr.base = var->base;
-		vmx->rmode.tr.limit = var->limit;
-		vmx->rmode.tr.ar = vmx_segment_access_rights(var);
+		vmx->rmode.segs[VCPU_SREG_TR] = *var;
 		return;
 	}
 	vmcs_writel(sf->base, var->base);
 	vmcs_write32(sf->limit, var->limit);
 	vmcs_write16(sf->selector, var->selector);
 	if (vmx->rmode.vm86_active && var->s) {
+		vmx->rmode.segs[seg] = *var;
 		/*
 		 * Hack real-mode segments into vm86 compatibility.
 		 */
@@ -3258,7 +3227,7 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
 	 * qemu binaries.
 	 *   IA32 arch specifies that at the time of processor reset the
 	 * "Accessed" bit in the AR field of segment registers is 1. And qemu
-	 * is setting it to 0 in the usedland code. This causes invalid guest
+	 * is setting it to 0 in the userland code. This causes invalid guest
 	 * state vmexit when "unrestricted guest" mode is turned on.
 	 *    Fix for this setup issue in cpu_reset is being pushed in the qemu
 	 * tree. Newer qemu binaries with that qemu fix would not need this
@@ -3288,16 +3257,10 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
 				     vmcs_readl(GUEST_CS_BASE) >> 4);
 			break;
 		case VCPU_SREG_ES:
-			fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
-			break;
 		case VCPU_SREG_DS:
-			fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
-			break;
 		case VCPU_SREG_GS:
-			fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
-			break;
 		case VCPU_SREG_FS:
-			fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs);
+			fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
 			break;
 		case VCPU_SREG_SS:
 			vmcs_write16(GUEST_SS_SELECTOR,
@@ -3351,9 +3314,9 @@ static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
 
 	if (var.base != (var.selector << 4))
 		return false;
-	if (var.limit != 0xffff)
+	if (var.limit < 0xffff)
 		return false;
-	if (ar != 0xf3)
+	if (((ar | (3 << AR_DPL_SHIFT)) & ~(AR_G_MASK | AR_DB_MASK)) != 0xf3)
 		return false;
 
 	return true;
@@ -3605,7 +3568,7 @@ out:
 
 static void seg_setup(int seg)
 {
-	struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
+	const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
 	unsigned int ar;
 
 	vmcs_write16(sf->selector, 0);
@@ -3770,8 +3733,7 @@ static void vmx_set_constant_host_state(void)
 	native_store_idt(&dt);
 	vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
 
-	asm("mov $.Lkvm_vmx_return, %0" : "=r"(tmpl));
-	vmcs_writel(HOST_RIP, tmpl); /* 22.2.5 */
+	vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
 
 	rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
 	vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
@@ -4005,8 +3967,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
 		kvm_rip_write(vcpu, 0);
 	kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
 
-	vmcs_writel(GUEST_DR7, 0x400);
-
 	vmcs_writel(GUEST_GDTR_BASE, 0);
 	vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
 
@@ -4456,7 +4416,7 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
 	hypercall[2] = 0xc1;
 }
 
-/* called to set cr0 as approriate for a mov-to-cr0 exit. */
+/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
 {
 	if (to_vmx(vcpu)->nested.vmxon &&
@@ -5701,7 +5661,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
  * to be done to userspace and return 0.
  */
-static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
+static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
 	[EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
 	[EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
@@ -6229,17 +6189,10 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
 					msrs[i].host);
 }
 
-#ifdef CONFIG_X86_64
-#define R "r"
-#define Q "q"
-#else
-#define R "e"
-#define Q "l"
-#endif
-
 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	unsigned long debugctlmsr;
 
 	if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) {
 		struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
@@ -6279,34 +6232,35 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 		vmx_set_interrupt_shadow(vcpu, 0);
 
 	atomic_switch_perf_msrs(vmx);
+	debugctlmsr = get_debugctlmsr();
 
 	vmx->__launched = vmx->loaded_vmcs->launched;
 	asm(
 		/* Store host registers */
-		"push %%"R"dx; push %%"R"bp;"
-		"push %%"R"cx \n\t" /* placeholder for guest rcx */
-		"push %%"R"cx \n\t"
-		"cmp %%"R"sp, %c[host_rsp](%0) \n\t"
+		"push %%" _ASM_DX "; push %%" _ASM_BP ";"
+		"push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
+		"push %%" _ASM_CX " \n\t"
+		"cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
 		"je 1f \n\t"
-		"mov %%"R"sp, %c[host_rsp](%0) \n\t"
+		"mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
 		__ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
 		"1: \n\t"
 		/* Reload cr2 if changed */
-		"mov %c[cr2](%0), %%"R"ax \n\t"
-		"mov %%cr2, %%"R"dx \n\t"
-		"cmp %%"R"ax, %%"R"dx \n\t"
+		"mov %c[cr2](%0), %%" _ASM_AX " \n\t"
+		"mov %%cr2, %%" _ASM_DX " \n\t"
+		"cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
 		"je 2f \n\t"
-		"mov %%"R"ax, %%cr2 \n\t"
+		"mov %%" _ASM_AX", %%cr2 \n\t"
 		"2: \n\t"
 		/* Check if vmlaunch of vmresume is needed */
 		"cmpl $0, %c[launched](%0) \n\t"
 		/* Load guest registers.  Don't clobber flags. */
-		"mov %c[rax](%0), %%"R"ax \n\t"
-		"mov %c[rbx](%0), %%"R"bx \n\t"
-		"mov %c[rdx](%0), %%"R"dx \n\t"
-		"mov %c[rsi](%0), %%"R"si \n\t"
-		"mov %c[rdi](%0), %%"R"di \n\t"
-		"mov %c[rbp](%0), %%"R"bp \n\t"
+		"mov %c[rax](%0), %%" _ASM_AX " \n\t"
+		"mov %c[rbx](%0), %%" _ASM_BX " \n\t"
+		"mov %c[rdx](%0), %%" _ASM_DX " \n\t"
+		"mov %c[rsi](%0), %%" _ASM_SI " \n\t"
+		"mov %c[rdi](%0), %%" _ASM_DI " \n\t"
+		"mov %c[rbp](%0), %%" _ASM_BP " \n\t"
 #ifdef CONFIG_X86_64
 		"mov %c[r8](%0),  %%r8  \n\t"
 		"mov %c[r9](%0),  %%r9  \n\t"
@@ -6317,24 +6271,24 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 		"mov %c[r14](%0), %%r14 \n\t"
 		"mov %c[r15](%0), %%r15 \n\t"
 #endif
-		"mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */
+		"mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
 
 		/* Enter guest mode */
-		"jne .Llaunched \n\t"
+		"jne 1f \n\t"
 		__ex(ASM_VMX_VMLAUNCH) "\n\t"
-		"jmp .Lkvm_vmx_return \n\t"
-		".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
-		".Lkvm_vmx_return: "
+		"jmp 2f \n\t"
+		"1: " __ex(ASM_VMX_VMRESUME) "\n\t"
+		"2: "
 		/* Save guest registers, load host registers, keep flags */
-		"mov %0, %c[wordsize](%%"R"sp) \n\t"
+		"mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
 		"pop %0 \n\t"
-		"mov %%"R"ax, %c[rax](%0) \n\t"
-		"mov %%"R"bx, %c[rbx](%0) \n\t"
-		"pop"Q" %c[rcx](%0) \n\t"
-		"mov %%"R"dx, %c[rdx](%0) \n\t"
-		"mov %%"R"si, %c[rsi](%0) \n\t"
-		"mov %%"R"di, %c[rdi](%0) \n\t"
-		"mov %%"R"bp, %c[rbp](%0) \n\t"
+		"mov %%" _ASM_AX ", %c[rax](%0) \n\t"
+		"mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
+		__ASM_SIZE(pop) " %c[rcx](%0) \n\t"
+		"mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
+		"mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
+		"mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
+		"mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
 #ifdef CONFIG_X86_64
 		"mov %%r8,  %c[r8](%0) \n\t"
 		"mov %%r9,  %c[r9](%0) \n\t"
@@ -6345,11 +6299,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 		"mov %%r14, %c[r14](%0) \n\t"
 		"mov %%r15, %c[r15](%0) \n\t"
 #endif
-		"mov %%cr2, %%"R"ax   \n\t"
-		"mov %%"R"ax, %c[cr2](%0) \n\t"
+		"mov %%cr2, %%" _ASM_AX "   \n\t"
+		"mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
 
-		"pop  %%"R"bp; pop  %%"R"dx \n\t"
+		"pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
 		"setbe %c[fail](%0) \n\t"
+		".pushsection .rodata \n\t"
+		".global vmx_return \n\t"
+		"vmx_return: " _ASM_PTR " 2b \n\t"
+		".popsection"
 	      : : "c"(vmx), "d"((unsigned long)HOST_RSP),
 		[launched]"i"(offsetof(struct vcpu_vmx, __launched)),
 		[fail]"i"(offsetof(struct vcpu_vmx, fail)),
@@ -6374,12 +6332,18 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 		[cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
 		[wordsize]"i"(sizeof(ulong))
 	      : "cc", "memory"
-		, R"ax", R"bx", R"di", R"si"
 #ifdef CONFIG_X86_64
+		, "rax", "rbx", "rdi", "rsi"
 		, "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
+#else
+		, "eax", "ebx", "edi", "esi"
 #endif
 	      );
 
+	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
+	if (debugctlmsr)
+		update_debugctlmsr(debugctlmsr);
+
 #ifndef CONFIG_X86_64
 	/*
 	 * The sysexit path does not restore ds/es, so we must set them to
@@ -6424,9 +6388,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 	vmx_complete_interrupts(vmx);
 }
 
-#undef R
-#undef Q
-
 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -7281,7 +7242,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
 	.vcpu_load = vmx_vcpu_load,
 	.vcpu_put = vmx_vcpu_put,
 
-	.set_guest_debug = set_guest_debug,
+	.update_db_bp_intercept = update_exception_bitmap,
 	.get_msr = vmx_get_msr,
 	.set_msr = vmx_set_msr,
 	.get_segment_base = vmx_get_segment_base,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 1f09552572f..1eefebe5d72 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -246,20 +246,14 @@ static void drop_user_return_notifiers(void *ignore)
 
 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 {
-	if (irqchip_in_kernel(vcpu->kvm))
-		return vcpu->arch.apic_base;
-	else
-		return vcpu->arch.apic_base;
+	return vcpu->arch.apic_base;
 }
 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 
 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
 {
 	/* TODO: reserve bits check */
-	if (irqchip_in_kernel(vcpu->kvm))
-		kvm_lapic_set_base(vcpu, data);
-	else
-		vcpu->arch.apic_base = data;
+	kvm_lapic_set_base(vcpu, data);
 }
 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 
@@ -698,6 +692,18 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 }
 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 
+static void kvm_update_dr7(struct kvm_vcpu *vcpu)
+{
+	unsigned long dr7;
+
+	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
+		dr7 = vcpu->arch.guest_debug_dr7;
+	else
+		dr7 = vcpu->arch.dr7;
+	kvm_x86_ops->set_dr7(vcpu, dr7);
+	vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK);
+}
+
 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 {
 	switch (dr) {
@@ -723,10 +729,7 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 		if (val & 0xffffffff00000000ULL)
 			return -1; /* #GP */
 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
-		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
-			kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
-			vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
-		}
+		kvm_update_dr7(vcpu);
 		break;
 	}
 
@@ -823,7 +826,7 @@ static u32 msrs_to_save[] = {
 
 static unsigned num_msrs_to_save;
 
-static u32 emulated_msrs[] = {
+static const u32 emulated_msrs[] = {
 	MSR_IA32_TSCDEADLINE,
 	MSR_IA32_MISC_ENABLE,
 	MSR_IA32_MCG_STATUS,
@@ -1097,7 +1100,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data)
 		 * For each generation, we track the original measured
 		 * nanosecond time, offset, and write, so if TSCs are in
 		 * sync, we can match exact offset, and if not, we can match
-		 * exact software computaion in compute_guest_tsc()
+		 * exact software computation in compute_guest_tsc()
 		 *
 		 * These values are tracked in kvm->arch.cur_xxx variables.
 		 */
@@ -1140,6 +1143,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 	unsigned long this_tsc_khz;
 	s64 kernel_ns, max_kernel_ns;
 	u64 tsc_timestamp;
+	u8 pvclock_flags;
 
 	/* Keep irq disabled to prevent changes to the clock */
 	local_irq_save(flags);
@@ -1221,7 +1225,14 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 	vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
 	vcpu->last_kernel_ns = kernel_ns;
 	vcpu->last_guest_tsc = tsc_timestamp;
-	vcpu->hv_clock.flags = 0;
+
+	pvclock_flags = 0;
+	if (vcpu->pvclock_set_guest_stopped_request) {
+		pvclock_flags |= PVCLOCK_GUEST_STOPPED;
+		vcpu->pvclock_set_guest_stopped_request = false;
+	}
+
+	vcpu->hv_clock.flags = pvclock_flags;
 
 	/*
 	 * The interface expects us to write an even number signaling that the
@@ -1504,7 +1515,7 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
 {
 	gpa_t gpa = data & ~0x3f;
 
-	/* Bits 2:5 are resrved, Should be zero */
+	/* Bits 2:5 are reserved, Should be zero */
 	if (data & 0x3c)
 		return 1;
 
@@ -1639,10 +1650,9 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 		vcpu->arch.time_page =
 				gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
 
-		if (is_error_page(vcpu->arch.time_page)) {
-			kvm_release_page_clean(vcpu->arch.time_page);
+		if (is_error_page(vcpu->arch.time_page))
 			vcpu->arch.time_page = NULL;
-		}
+
 		break;
 	}
 	case MSR_KVM_ASYNC_PF_EN:
@@ -1727,7 +1737,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 		 * Ignore all writes to this no longer documented MSR.
 		 * Writes are only relevant for old K7 processors,
 		 * all pre-dating SVM, but a recommended workaround from
-		 * AMD for these chips. It is possible to speicify the
+		 * AMD for these chips. It is possible to specify the
 		 * affected processor models on the command line, hence
 		 * the need to ignore the workaround.
 		 */
@@ -2177,6 +2187,8 @@ int kvm_dev_ioctl_check_extension(long ext)
 	case KVM_CAP_GET_TSC_KHZ:
 	case KVM_CAP_PCI_2_3:
 	case KVM_CAP_KVMCLOCK_CTRL:
+	case KVM_CAP_READONLY_MEM:
+	case KVM_CAP_IRQFD_RESAMPLE:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
@@ -2358,8 +2370,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
-	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
-	kvm_apic_post_state_restore(vcpu);
+	kvm_apic_post_state_restore(vcpu, s);
 	update_cr8_intercept(vcpu);
 
 	return 0;
@@ -2368,7 +2379,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
 				    struct kvm_interrupt *irq)
 {
-	if (irq->irq < 0 || irq->irq >= 256)
+	if (irq->irq < 0 || irq->irq >= KVM_NR_INTERRUPTS)
 		return -EINVAL;
 	if (irqchip_in_kernel(vcpu->kvm))
 		return -ENXIO;
@@ -2635,11 +2646,9 @@ static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
  */
 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
 {
-	struct pvclock_vcpu_time_info *src = &vcpu->arch.hv_clock;
 	if (!vcpu->arch.time_page)
 		return -EINVAL;
-	src->flags |= PVCLOCK_GUEST_STOPPED;
-	mark_page_dirty(vcpu->kvm, vcpu->arch.time >> PAGE_SHIFT);
+	vcpu->arch.pvclock_set_guest_stopped_request = true;
 	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 	return 0;
 }
@@ -3090,7 +3099,7 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm,
 	if (!kvm->arch.vpit)
 		return -ENXIO;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
-	kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
+	kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return 0;
 }
@@ -3173,6 +3182,16 @@ out:
 	return r;
 }
 
+int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event)
+{
+	if (!irqchip_in_kernel(kvm))
+		return -ENXIO;
+
+	irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
+					irq_event->irq, irq_event->level);
+	return 0;
+}
+
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg)
 {
@@ -3279,29 +3298,6 @@ long kvm_arch_vm_ioctl(struct file *filp,
 	create_pit_unlock:
 		mutex_unlock(&kvm->slots_lock);
 		break;
-	case KVM_IRQ_LINE_STATUS:
-	case KVM_IRQ_LINE: {
-		struct kvm_irq_level irq_event;
-
-		r = -EFAULT;
-		if (copy_from_user(&irq_event, argp, sizeof irq_event))
-			goto out;
-		r = -ENXIO;
-		if (irqchip_in_kernel(kvm)) {
-			__s32 status;
-			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
-					irq_event.irq, irq_event.level);
-			if (ioctl == KVM_IRQ_LINE_STATUS) {
-				r = -EFAULT;
-				irq_event.status = status;
-				if (copy_to_user(argp, &irq_event,
-							sizeof irq_event))
-					goto out;
-			}
-			r = 0;
-		}
-		break;
-	}
 	case KVM_GET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip *chip;
@@ -3689,20 +3685,17 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
 				gpa_t *gpa, struct x86_exception *exception,
 				bool write)
 {
-	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
+	u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
+		| (write ? PFERR_WRITE_MASK : 0);
 
-	if (vcpu_match_mmio_gva(vcpu, gva) &&
-		  check_write_user_access(vcpu, write, access,
-		  vcpu->arch.access)) {
+	if (vcpu_match_mmio_gva(vcpu, gva)
+	    && !permission_fault(vcpu->arch.walk_mmu, vcpu->arch.access, access)) {
 		*gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
 					(gva & (PAGE_SIZE - 1));
 		trace_vcpu_match_mmio(gva, *gpa, write, false);
 		return 1;
 	}
 
-	if (write)
-		access |= PFERR_WRITE_MASK;
-
 	*gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
 
 	if (*gpa == UNMAPPED_GVA)
@@ -3790,14 +3783,14 @@ static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
 	return X86EMUL_CONTINUE;
 }
 
-static struct read_write_emulator_ops read_emultor = {
+static const struct read_write_emulator_ops read_emultor = {
 	.read_write_prepare = read_prepare,
 	.read_write_emulate = read_emulate,
 	.read_write_mmio = vcpu_mmio_read,
 	.read_write_exit_mmio = read_exit_mmio,
 };
 
-static struct read_write_emulator_ops write_emultor = {
+static const struct read_write_emulator_ops write_emultor = {
 	.read_write_emulate = write_emulate,
 	.read_write_mmio = write_mmio,
 	.read_write_exit_mmio = write_exit_mmio,
@@ -3808,7 +3801,7 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
 				       unsigned int bytes,
 				       struct x86_exception *exception,
 				       struct kvm_vcpu *vcpu,
-				       struct read_write_emulator_ops *ops)
+				       const struct read_write_emulator_ops *ops)
 {
 	gpa_t gpa;
 	int handled, ret;
@@ -3857,7 +3850,7 @@ mmio:
 int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
 			void *val, unsigned int bytes,
 			struct x86_exception *exception,
-			struct read_write_emulator_ops *ops)
+			const struct read_write_emulator_ops *ops)
 {
 	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
 	gpa_t gpa;
@@ -3962,10 +3955,8 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
 		goto emul_write;
 
 	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
-	if (is_error_page(page)) {
-		kvm_release_page_clean(page);
+	if (is_error_page(page))
 		goto emul_write;
-	}
 
 	kaddr = kmap_atomic(page);
 	kaddr += offset_in_page(gpa);
@@ -4332,7 +4323,19 @@ static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
 	kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
 }
 
-static struct x86_emulate_ops emulate_ops = {
+static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
+{
+	return kvm_register_read(emul_to_vcpu(ctxt), reg);
+}
+
+static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
+{
+	kvm_register_write(emul_to_vcpu(ctxt), reg, val);
+}
+
+static const struct x86_emulate_ops emulate_ops = {
+	.read_gpr            = emulator_read_gpr,
+	.write_gpr           = emulator_write_gpr,
 	.read_std            = kvm_read_guest_virt_system,
 	.write_std           = kvm_write_guest_virt_system,
 	.fetch               = kvm_fetch_guest_virt,
@@ -4367,14 +4370,6 @@ static struct x86_emulate_ops emulate_ops = {
 	.get_cpuid           = emulator_get_cpuid,
 };
 
-static void cache_all_regs(struct kvm_vcpu *vcpu)
-{
-	kvm_register_read(vcpu, VCPU_REGS_RAX);
-	kvm_register_read(vcpu, VCPU_REGS_RSP);
-	kvm_register_read(vcpu, VCPU_REGS_RIP);
-	vcpu->arch.regs_dirty = ~0;
-}
-
 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
 {
 	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
@@ -4401,12 +4396,10 @@ static void inject_emulated_exception(struct kvm_vcpu *vcpu)
 		kvm_queue_exception(vcpu, ctxt->exception.vector);
 }
 
-static void init_decode_cache(struct x86_emulate_ctxt *ctxt,
-			      const unsigned long *regs)
+static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
 {
 	memset(&ctxt->twobyte, 0,
-	       (void *)&ctxt->regs - (void *)&ctxt->twobyte);
-	memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
+	       (void *)&ctxt->_regs - (void *)&ctxt->twobyte);
 
 	ctxt->fetch.start = 0;
 	ctxt->fetch.end = 0;
@@ -4421,14 +4414,6 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
 	int cs_db, cs_l;
 
-	/*
-	 * TODO: fix emulate.c to use guest_read/write_register
-	 * instead of direct ->regs accesses, can save hundred cycles
-	 * on Intel for instructions that don't read/change RSP, for
-	 * for example.
-	 */
-	cache_all_regs(vcpu);
-
 	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 
 	ctxt->eflags = kvm_get_rflags(vcpu);
@@ -4440,7 +4425,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
 							  X86EMUL_MODE_PROT16;
 	ctxt->guest_mode = is_guest_mode(vcpu);
 
-	init_decode_cache(ctxt, vcpu->arch.regs);
+	init_decode_cache(ctxt);
 	vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
 }
 
@@ -4460,7 +4445,6 @@ int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
 		return EMULATE_FAIL;
 
 	ctxt->eip = ctxt->_eip;
-	memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
 	kvm_rip_write(vcpu, ctxt->eip);
 	kvm_set_rflags(vcpu, ctxt->eflags);
 
@@ -4493,13 +4477,14 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu)
 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
 {
 	gpa_t gpa;
+	pfn_t pfn;
 
 	if (tdp_enabled)
 		return false;
 
 	/*
 	 * if emulation was due to access to shadowed page table
-	 * and it failed try to unshadow page and re-entetr the
+	 * and it failed try to unshadow page and re-enter the
 	 * guest to let CPU execute the instruction.
 	 */
 	if (kvm_mmu_unprotect_page_virt(vcpu, gva))
@@ -4510,8 +4495,17 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
 	if (gpa == UNMAPPED_GVA)
 		return true; /* let cpu generate fault */
 
-	if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT)))
+	/*
+	 * Do not retry the unhandleable instruction if it faults on the
+	 * readonly host memory, otherwise it will goto a infinite loop:
+	 * retry instruction -> write #PF -> emulation fail -> retry
+	 * instruction -> ...
+	 */
+	pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
+	if (!is_error_pfn(pfn)) {
+		kvm_release_pfn_clean(pfn);
 		return true;
+	}
 
 	return false;
 }
@@ -4560,6 +4554,9 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
 	return true;
 }
 
+static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
+static int complete_emulated_pio(struct kvm_vcpu *vcpu);
+
 int x86_emulate_instruction(struct kvm_vcpu *vcpu,
 			    unsigned long cr2,
 			    int emulation_type,
@@ -4608,7 +4605,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
 	   changes registers values  during IO operation */
 	if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
 		vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
-		memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs);
+		emulator_invalidate_register_cache(ctxt);
 	}
 
 restart:
@@ -4630,13 +4627,16 @@ restart:
 	} else if (vcpu->arch.pio.count) {
 		if (!vcpu->arch.pio.in)
 			vcpu->arch.pio.count = 0;
-		else
+		else {
 			writeback = false;
+			vcpu->arch.complete_userspace_io = complete_emulated_pio;
+		}
 		r = EMULATE_DO_MMIO;
 	} else if (vcpu->mmio_needed) {
 		if (!vcpu->mmio_is_write)
 			writeback = false;
 		r = EMULATE_DO_MMIO;
+		vcpu->arch.complete_userspace_io = complete_emulated_mmio;
 	} else if (r == EMULATION_RESTART)
 		goto restart;
 	else
@@ -4646,7 +4646,6 @@ restart:
 		toggle_interruptibility(vcpu, ctxt->interruptibility);
 		kvm_set_rflags(vcpu, ctxt->eflags);
 		kvm_make_request(KVM_REQ_EVENT, vcpu);
-		memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
 		kvm_rip_write(vcpu, ctxt->eip);
 	} else
@@ -4929,6 +4928,7 @@ int kvm_arch_init(void *opaque)
 	if (cpu_has_xsave)
 		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 
+	kvm_lapic_init();
 	return 0;
 
 out:
@@ -5499,6 +5499,24 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 	return r;
 }
 
+static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
+{
+	int r;
+	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
+	r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
+	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
+	if (r != EMULATE_DONE)
+		return 0;
+	return 1;
+}
+
+static int complete_emulated_pio(struct kvm_vcpu *vcpu)
+{
+	BUG_ON(!vcpu->arch.pio.count);
+
+	return complete_emulated_io(vcpu);
+}
+
 /*
  * Implements the following, as a state machine:
  *
@@ -5515,47 +5533,37 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
  *      copy data
  *      exit
  */
-static int complete_mmio(struct kvm_vcpu *vcpu)
+static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
 {
 	struct kvm_run *run = vcpu->run;
 	struct kvm_mmio_fragment *frag;
-	int r;
 
-	if (!(vcpu->arch.pio.count || vcpu->mmio_needed))
-		return 1;
+	BUG_ON(!vcpu->mmio_needed);
 
-	if (vcpu->mmio_needed) {
-		/* Complete previous fragment */
-		frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++];
-		if (!vcpu->mmio_is_write)
-			memcpy(frag->data, run->mmio.data, frag->len);
-		if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) {
-			vcpu->mmio_needed = 0;
-			if (vcpu->mmio_is_write)
-				return 1;
-			vcpu->mmio_read_completed = 1;
-			goto done;
-		}
-		/* Initiate next fragment */
-		++frag;
-		run->exit_reason = KVM_EXIT_MMIO;
-		run->mmio.phys_addr = frag->gpa;
+	/* Complete previous fragment */
+	frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++];
+	if (!vcpu->mmio_is_write)
+		memcpy(frag->data, run->mmio.data, frag->len);
+	if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) {
+		vcpu->mmio_needed = 0;
 		if (vcpu->mmio_is_write)
-			memcpy(run->mmio.data, frag->data, frag->len);
-		run->mmio.len = frag->len;
-		run->mmio.is_write = vcpu->mmio_is_write;
-		return 0;
-
-	}
-done:
-	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
-	r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
-	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
-	if (r != EMULATE_DONE)
-		return 0;
-	return 1;
+			return 1;
+		vcpu->mmio_read_completed = 1;
+		return complete_emulated_io(vcpu);
+	}
+	/* Initiate next fragment */
+	++frag;
+	run->exit_reason = KVM_EXIT_MMIO;
+	run->mmio.phys_addr = frag->gpa;
+	if (vcpu->mmio_is_write)
+		memcpy(run->mmio.data, frag->data, frag->len);
+	run->mmio.len = frag->len;
+	run->mmio.is_write = vcpu->mmio_is_write;
+	vcpu->arch.complete_userspace_io = complete_emulated_mmio;
+	return 0;
 }
 
+
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
@@ -5582,9 +5590,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		}
 	}
 
-	r = complete_mmio(vcpu);
-	if (r <= 0)
-		goto out;
+	if (unlikely(vcpu->arch.complete_userspace_io)) {
+		int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
+		vcpu->arch.complete_userspace_io = NULL;
+		r = cui(vcpu);
+		if (r <= 0)
+			goto out;
+	} else
+		WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
 
 	r = __vcpu_run(vcpu);
 
@@ -5602,12 +5615,11 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 		/*
 		 * We are here if userspace calls get_regs() in the middle of
 		 * instruction emulation. Registers state needs to be copied
-		 * back from emulation context to vcpu. Usrapace shouldn't do
+		 * back from emulation context to vcpu. Userspace shouldn't do
 		 * that usually, but some bad designed PV devices (vmware
 		 * backdoor interface) need this to work
 		 */
-		struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
-		memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
+		emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
 		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
 	}
 	regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
@@ -5747,7 +5759,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
 	if (ret)
 		return EMULATE_FAIL;
 
-	memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
 	kvm_rip_write(vcpu, ctxt->eip);
 	kvm_set_rflags(vcpu, ctxt->eflags);
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
@@ -5799,7 +5810,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 	if (mmu_reset_needed)
 		kvm_mmu_reset_context(vcpu);
 
-	max_bits = (sizeof sregs->interrupt_bitmap) << 3;
+	max_bits = KVM_NR_INTERRUPTS;
 	pending_vec = find_first_bit(
 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
 	if (pending_vec < max_bits) {
@@ -5859,13 +5870,12 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
 		for (i = 0; i < KVM_NR_DB_REGS; ++i)
 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
-		vcpu->arch.switch_db_regs =
-			(dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
+		vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
 	} else {
 		for (i = 0; i < KVM_NR_DB_REGS; i++)
 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
-		vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
 	}
+	kvm_update_dr7(vcpu);
 
 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
 		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
@@ -5877,7 +5887,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 	 */
 	kvm_set_rflags(vcpu, rflags);
 
-	kvm_x86_ops->set_guest_debug(vcpu, dbg);
+	kvm_x86_ops->update_db_bp_intercept(vcpu);
 
 	r = 0;
 
@@ -6023,7 +6033,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 	int r;
 
 	vcpu->arch.mtrr_state.have_fixed = 1;
-	vcpu_load(vcpu);
+	r = vcpu_load(vcpu);
+	if (r)
+		return r;
 	r = kvm_arch_vcpu_reset(vcpu);
 	if (r == 0)
 		r = kvm_mmu_setup(vcpu);
@@ -6034,9 +6046,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
+	int r;
 	vcpu->arch.apf.msr_val = 0;
 
-	vcpu_load(vcpu);
+	r = vcpu_load(vcpu);
+	BUG_ON(r);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 
@@ -6050,10 +6064,10 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
 	vcpu->arch.nmi_pending = 0;
 	vcpu->arch.nmi_injected = false;
 
-	vcpu->arch.switch_db_regs = 0;
 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
 	vcpu->arch.dr6 = DR6_FIXED_1;
 	vcpu->arch.dr7 = DR7_FIXED_1;
+	kvm_update_dr7(vcpu);
 
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 	vcpu->arch.apf.msr_val = 0;
@@ -6132,7 +6146,7 @@ int kvm_arch_hardware_enable(void *garbage)
 	 * as we reset last_host_tsc on all VCPUs to stop this from being
 	 * called multiple times (one for each physical CPU bringup).
 	 *
-	 * Platforms with unnreliable TSCs don't have to deal with this, they
+	 * Platforms with unreliable TSCs don't have to deal with this, they
 	 * will be compensated by the logic in vcpu_load, which sets the TSC to
 	 * catchup mode.  This will catchup all VCPUs to real time, but cannot
 	 * guarantee that they stay in perfect synchronization.
@@ -6185,6 +6199,8 @@ bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
 	return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
 }
 
+struct static_key kvm_no_apic_vcpu __read_mostly;
+
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	struct page *page;
@@ -6217,7 +6233,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 		r = kvm_create_lapic(vcpu);
 		if (r < 0)
 			goto fail_mmu_destroy;
-	}
+	} else
+		static_key_slow_inc(&kvm_no_apic_vcpu);
 
 	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
 				       GFP_KERNEL);
@@ -6257,6 +6274,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 	kvm_mmu_destroy(vcpu);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	free_page((unsigned long)vcpu->arch.pio_data);
+	if (!irqchip_in_kernel(vcpu->kvm))
+		static_key_slow_dec(&kvm_no_apic_vcpu);
 }
 
 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
@@ -6269,15 +6288,21 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
+	/* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
+	set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
+		&kvm->arch.irq_sources_bitmap);
 
 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
+	mutex_init(&kvm->arch.apic_map_lock);
 
 	return 0;
 }
 
 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 {
-	vcpu_load(vcpu);
+	int r;
+	r = vcpu_load(vcpu);
+	BUG_ON(r);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 }
@@ -6321,6 +6346,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 		put_page(kvm->arch.apic_access_page);
 	if (kvm->arch.ept_identity_pagetable)
 		put_page(kvm->arch.ept_identity_pagetable);
+	kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
 }
 
 void kvm_arch_free_memslot(struct kvm_memory_slot *free,
@@ -6328,10 +6354,18 @@ void kvm_arch_free_memslot(struct kvm_memory_slot *free,
 {
 	int i;
 
-	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
-		if (!dont || free->arch.lpage_info[i] != dont->arch.lpage_info[i]) {
-			kvm_kvfree(free->arch.lpage_info[i]);
-			free->arch.lpage_info[i] = NULL;
+	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
+		if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
+			kvm_kvfree(free->arch.rmap[i]);
+			free->arch.rmap[i] = NULL;
+		}
+		if (i == 0)
+			continue;
+
+		if (!dont || free->arch.lpage_info[i - 1] !=
+			     dont->arch.lpage_info[i - 1]) {
+			kvm_kvfree(free->arch.lpage_info[i - 1]);
+			free->arch.lpage_info[i - 1] = NULL;
 		}
 	}
 }
@@ -6340,23 +6374,30 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
 {
 	int i;
 
-	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
+	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
 		unsigned long ugfn;
 		int lpages;
-		int level = i + 2;
+		int level = i + 1;
 
 		lpages = gfn_to_index(slot->base_gfn + npages - 1,
 				      slot->base_gfn, level) + 1;
 
-		slot->arch.lpage_info[i] =
-			kvm_kvzalloc(lpages * sizeof(*slot->arch.lpage_info[i]));
-		if (!slot->arch.lpage_info[i])
+		slot->arch.rmap[i] =
+			kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
+		if (!slot->arch.rmap[i])
+			goto out_free;
+		if (i == 0)
+			continue;
+
+		slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
+					sizeof(*slot->arch.lpage_info[i - 1]));
+		if (!slot->arch.lpage_info[i - 1])
 			goto out_free;
 
 		if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
-			slot->arch.lpage_info[i][0].write_count = 1;
+			slot->arch.lpage_info[i - 1][0].write_count = 1;
 		if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
-			slot->arch.lpage_info[i][lpages - 1].write_count = 1;
+			slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
 		ugfn = slot->userspace_addr >> PAGE_SHIFT;
 		/*
 		 * If the gfn and userspace address are not aligned wrt each
@@ -6368,16 +6409,21 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
 			unsigned long j;
 
 			for (j = 0; j < lpages; ++j)
-				slot->arch.lpage_info[i][j].write_count = 1;
+				slot->arch.lpage_info[i - 1][j].write_count = 1;
 		}
 	}
 
 	return 0;
 
 out_free:
-	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
-		kvm_kvfree(slot->arch.lpage_info[i]);
-		slot->arch.lpage_info[i] = NULL;
+	for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
+		kvm_kvfree(slot->arch.rmap[i]);
+		slot->arch.rmap[i] = NULL;
+		if (i == 0)
+			continue;
+
+		kvm_kvfree(slot->arch.lpage_info[i - 1]);
+		slot->arch.lpage_info[i - 1] = NULL;
 	}
 	return -ENOMEM;
 }
@@ -6396,10 +6442,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
 		map_flags = MAP_SHARED | MAP_ANONYMOUS;
 
 	/*To keep backward compatibility with older userspace,
-	 *x86 needs to hanlde !user_alloc case.
+	 *x86 needs to handle !user_alloc case.
 	 */
 	if (!user_alloc) {
-		if (npages && !old.rmap) {
+		if (npages && !old.npages) {
 			unsigned long userspace_addr;
 
 			userspace_addr = vm_mmap(NULL, 0,
@@ -6427,7 +6473,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 
 	int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
 
-	if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
+	if (!user_alloc && !old.user_alloc && old.npages && !npages) {
 		int ret;
 
 		ret = vm_munmap(old.userspace_addr,
@@ -6446,14 +6492,28 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
 	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
 	spin_unlock(&kvm->mmu_lock);
+	/*
+	 * If memory slot is created, or moved, we need to clear all
+	 * mmio sptes.
+	 */
+	if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) {
+		kvm_mmu_zap_all(kvm);
+		kvm_reload_remote_mmus(kvm);
+	}
 }
 
-void kvm_arch_flush_shadow(struct kvm *kvm)
+void kvm_arch_flush_shadow_all(struct kvm *kvm)
 {
 	kvm_mmu_zap_all(kvm);
 	kvm_reload_remote_mmus(kvm);
 }
 
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot)
+{
+	kvm_arch_flush_shadow_all(kvm);
+}
+
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 3d1134ddb88..2b5219c12ac 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -124,4 +124,5 @@ int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
 
 extern u64 host_xcr0;
 
+extern struct static_key kvm_no_apic_vcpu;
 #endif
diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c
index b1e6c4b2e8e..54fcffed28e 100644
--- a/arch/x86/lib/insn.c
+++ b/arch/x86/lib/insn.c
@@ -18,7 +18,11 @@
  * Copyright (C) IBM Corporation, 2002, 2004, 2009
  */
 
+#ifdef __KERNEL__
 #include <linux/string.h>
+#else
+#include <string.h>
+#endif
 #include <asm/inat.h>
 #include <asm/insn.h>
 
diff --git a/arch/x86/syscalls/Makefile b/arch/x86/syscalls/Makefile
index 3236aebc828..f325af26107 100644
--- a/arch/x86/syscalls/Makefile
+++ b/arch/x86/syscalls/Makefile
@@ -1,7 +1,9 @@
 out := $(obj)/../include/generated/asm
+uapi := $(obj)/../include/generated/uapi/asm
 
 # Create output directory if not already present
-_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)')
+_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \
+	  $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)')
 
 syscall32 := $(srctree)/$(src)/syscall_32.tbl
 syscall64 := $(srctree)/$(src)/syscall_64.tbl
@@ -18,7 +20,7 @@ quiet_cmd_systbl = SYSTBL  $@
       cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@
 
 syshdr_abi_unistd_32 := i386
-$(out)/unistd_32.h: $(syscall32) $(syshdr)
+$(uapi)/unistd_32.h: $(syscall32) $(syshdr)
 	$(call if_changed,syshdr)
 
 syshdr_abi_unistd_32_ia32 := i386
@@ -28,11 +30,11 @@ $(out)/unistd_32_ia32.h: $(syscall32) $(syshdr)
 
 syshdr_abi_unistd_x32 := common,x32
 syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT
-$(out)/unistd_x32.h: $(syscall64) $(syshdr)
+$(uapi)/unistd_x32.h: $(syscall64) $(syshdr)
 	$(call if_changed,syshdr)
 
 syshdr_abi_unistd_64 := common,64
-$(out)/unistd_64.h: $(syscall64) $(syshdr)
+$(uapi)/unistd_64.h: $(syscall64) $(syshdr)
 	$(call if_changed,syshdr)
 
 syshdr_abi_unistd_64_x32 := x32
@@ -45,11 +47,12 @@ $(out)/syscalls_32.h: $(syscall32) $(systbl)
 $(out)/syscalls_64.h: $(syscall64) $(systbl)
 	$(call if_changed,systbl)
 
-syshdr-y			+= unistd_32.h unistd_64.h unistd_x32.h
+uapisyshdr-y			+= unistd_32.h unistd_64.h unistd_x32.h
 syshdr-y			+= syscalls_32.h
 syshdr-$(CONFIG_X86_64)		+= unistd_32_ia32.h unistd_64_x32.h
 syshdr-$(CONFIG_X86_64)		+= syscalls_64.h
 
-targets	+= $(syshdr-y)
+targets	+= $(uapisyshdr-y) $(syshdr-y)
 
-all: $(addprefix $(out)/,$(targets))
+all: $(addprefix $(uapi)/,$(uapisyshdr-y))
+all: $(addprefix $(out)/,$(syshdr-y))
diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile
index 733057b435b..bae601f900e 100644
--- a/arch/x86/tools/Makefile
+++ b/arch/x86/tools/Makefile
@@ -28,7 +28,7 @@ posttest: $(obj)/test_get_len vmlinux $(obj)/insn_sanity
 hostprogs-y	+= test_get_len insn_sanity
 
 # -I needed for generated C source and C source which in the kernel tree.
-HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/
+HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/
 
 HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/
 
diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index ec57bd3818a..7005ced5d1a 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -6,8 +6,9 @@
 
 #include <xen/xen.h>
 #include <xen/interface/physdev.h>
+#include "xen-ops.h"
 
-unsigned int xen_io_apic_read(unsigned apic, unsigned reg)
+static unsigned int xen_io_apic_read(unsigned apic, unsigned reg)
 {
 	struct physdev_apic apic_op;
 	int ret;
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 1fbe75a95f1..2d932c351f9 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -80,6 +80,8 @@
 #include "smp.h"
 #include "multicalls.h"
 
+#include <xen/events.h>
+
 EXPORT_SYMBOL_GPL(hypercall_page);
 
 DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
@@ -1288,7 +1290,6 @@ asmlinkage void __init xen_start_kernel(void)
 {
 	struct physdev_set_iopl set_iopl;
 	int rc;
-	pgd_t *pgd;
 
 	if (!xen_start_info)
 		return;
@@ -1380,8 +1381,6 @@ asmlinkage void __init xen_start_kernel(void)
 	acpi_numa = -1;
 #endif
 
-	pgd = (pgd_t *)xen_start_info->pt_base;
-
 	/* Don't do the full vcpu_info placement stuff until we have a
 	   possible map and a non-dummy shared_info. */
 	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
@@ -1390,7 +1389,7 @@ asmlinkage void __init xen_start_kernel(void)
 	early_boot_irqs_disabled = true;
 
 	xen_raw_console_write("mapping kernel into physical memory\n");
-	pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
+	xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages);
 
 	/* Allocate and initialize top and mid mfn levels for p2m structure */
 	xen_build_mfn_list_list();
@@ -1441,11 +1440,19 @@ asmlinkage void __init xen_start_kernel(void)
 		const struct dom0_vga_console_info *info =
 			(void *)((char *)xen_start_info +
 				 xen_start_info->console.dom0.info_off);
+		struct xen_platform_op op = {
+			.cmd = XENPF_firmware_info,
+			.interface_version = XENPF_INTERFACE_VERSION,
+			.u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS,
+		};
 
 		xen_init_vga(info, xen_start_info->console.dom0.info_size);
 		xen_start_info->console.domU.mfn = 0;
 		xen_start_info->console.domU.evtchn = 0;
 
+		if (HYPERVISOR_dom0_op(&op) == 0)
+			boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags;
+
 		xen_init_apic();
 
 		/* Make sure ACS will be enabled */
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 7a769b7526c..5a16824cc2b 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -84,6 +84,7 @@
  */
 DEFINE_SPINLOCK(xen_reservation_lock);
 
+#ifdef CONFIG_X86_32
 /*
  * Identity map, in addition to plain kernel map.  This needs to be
  * large enough to allocate page table pages to allocate the rest.
@@ -91,7 +92,7 @@ DEFINE_SPINLOCK(xen_reservation_lock);
  */
 #define LEVEL1_IDENT_ENTRIES	(PTRS_PER_PTE * 4)
 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
-
+#endif
 #ifdef CONFIG_X86_64
 /* l3 pud for userspace vsyscall mapping */
 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
@@ -1176,13 +1177,6 @@ static void xen_exit_mmap(struct mm_struct *mm)
 
 static void xen_post_allocator_init(void);
 
-static void __init xen_pagetable_init(void)
-{
-	paging_init();
-	xen_setup_shared_info();
-	xen_post_allocator_init();
-}
-
 static __init void xen_mapping_pagetable_reserve(u64 start, u64 end)
 {
 	/* reserve the range used */
@@ -1197,6 +1191,87 @@ static __init void xen_mapping_pagetable_reserve(u64 start, u64 end)
 	}
 }
 
+#ifdef CONFIG_X86_64
+static void __init xen_cleanhighmap(unsigned long vaddr,
+				    unsigned long vaddr_end)
+{
+	unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
+	pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr);
+
+	/* NOTE: The loop is more greedy than the cleanup_highmap variant.
+	 * We include the PMD passed in on _both_ boundaries. */
+	for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE));
+			pmd++, vaddr += PMD_SIZE) {
+		if (pmd_none(*pmd))
+			continue;
+		if (vaddr < (unsigned long) _text || vaddr > kernel_end)
+			set_pmd(pmd, __pmd(0));
+	}
+	/* In case we did something silly, we should crash in this function
+	 * instead of somewhere later and be confusing. */
+	xen_mc_flush();
+}
+#endif
+static void __init xen_pagetable_init(void)
+{
+#ifdef CONFIG_X86_64
+	unsigned long size;
+	unsigned long addr;
+#endif
+	paging_init();
+	xen_setup_shared_info();
+#ifdef CONFIG_X86_64
+	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+		unsigned long new_mfn_list;
+
+		size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
+
+		/* On 32-bit, we get zero so this never gets executed. */
+		new_mfn_list = xen_revector_p2m_tree();
+		if (new_mfn_list && new_mfn_list != xen_start_info->mfn_list) {
+			/* using __ka address and sticking INVALID_P2M_ENTRY! */
+			memset((void *)xen_start_info->mfn_list, 0xff, size);
+
+			/* We should be in __ka space. */
+			BUG_ON(xen_start_info->mfn_list < __START_KERNEL_map);
+			addr = xen_start_info->mfn_list;
+			/* We roundup to the PMD, which means that if anybody at this stage is
+			 * using the __ka address of xen_start_info or xen_start_info->shared_info
+			 * they are in going to crash. Fortunatly we have already revectored
+			 * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */
+			size = roundup(size, PMD_SIZE);
+			xen_cleanhighmap(addr, addr + size);
+
+			size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
+			memblock_free(__pa(xen_start_info->mfn_list), size);
+			/* And revector! Bye bye old array */
+			xen_start_info->mfn_list = new_mfn_list;
+		} else
+			goto skip;
+	}
+	/* At this stage, cleanup_highmap has already cleaned __ka space
+	 * from _brk_limit way up to the max_pfn_mapped (which is the end of
+	 * the ramdisk). We continue on, erasing PMD entries that point to page
+	 * tables - do note that they are accessible at this stage via __va.
+	 * For good measure we also round up to the PMD - which means that if
+	 * anybody is using __ka address to the initial boot-stack - and try
+	 * to use it - they are going to crash. The xen_start_info has been
+	 * taken care of already in xen_setup_kernel_pagetable. */
+	addr = xen_start_info->pt_base;
+	size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE);
+
+	xen_cleanhighmap(addr, addr + size);
+	xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base));
+#ifdef DEBUG
+	/* This is superflous and is not neccessary, but you know what
+	 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
+	 * anything at this stage. */
+	xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1);
+#endif
+skip:
+#endif
+	xen_post_allocator_init();
+}
 static void xen_write_cr2(unsigned long cr2)
 {
 	this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
@@ -1652,7 +1727,7 @@ static void set_page_prot(void *addr, pgprot_t prot)
 	if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
 		BUG();
 }
-
+#ifdef CONFIG_X86_32
 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
 {
 	unsigned pmdidx, pteidx;
@@ -1703,7 +1778,7 @@ static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
 
 	set_page_prot(pmd, PAGE_KERNEL_RO);
 }
-
+#endif
 void __init xen_setup_machphys_mapping(void)
 {
 	struct xen_machphys_mapping mapping;
@@ -1731,7 +1806,20 @@ static void convert_pfn_mfn(void *v)
 	for (i = 0; i < PTRS_PER_PTE; i++)
 		pte[i] = xen_make_pte(pte[i].pte);
 }
-
+static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
+				 unsigned long addr)
+{
+	if (*pt_base == PFN_DOWN(__pa(addr))) {
+		set_page_prot((void *)addr, PAGE_KERNEL);
+		clear_page((void *)addr);
+		(*pt_base)++;
+	}
+	if (*pt_end == PFN_DOWN(__pa(addr))) {
+		set_page_prot((void *)addr, PAGE_KERNEL);
+		clear_page((void *)addr);
+		(*pt_end)--;
+	}
+}
 /*
  * Set up the initial kernel pagetable.
  *
@@ -1743,11 +1831,13 @@ static void convert_pfn_mfn(void *v)
  * of the physical mapping once some sort of allocator has been set
  * up.
  */
-pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
-					 unsigned long max_pfn)
+void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 {
 	pud_t *l3;
 	pmd_t *l2;
+	unsigned long addr[3];
+	unsigned long pt_base, pt_end;
+	unsigned i;
 
 	/* max_pfn_mapped is the last pfn mapped in the initial memory
 	 * mappings. Considering that on Xen after the kernel mappings we
@@ -1755,32 +1845,53 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
 	 * set max_pfn_mapped to the last real pfn mapped. */
 	max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
 
+	pt_base = PFN_DOWN(__pa(xen_start_info->pt_base));
+	pt_end = pt_base + xen_start_info->nr_pt_frames;
+
 	/* Zap identity mapping */
 	init_level4_pgt[0] = __pgd(0);
 
 	/* Pre-constructed entries are in pfn, so convert to mfn */
+	/* L4[272] -> level3_ident_pgt
+	 * L4[511] -> level3_kernel_pgt */
 	convert_pfn_mfn(init_level4_pgt);
+
+	/* L3_i[0] -> level2_ident_pgt */
 	convert_pfn_mfn(level3_ident_pgt);
+	/* L3_k[510] -> level2_kernel_pgt
+	 * L3_i[511] -> level2_fixmap_pgt */
 	convert_pfn_mfn(level3_kernel_pgt);
 
+	/* We get [511][511] and have Xen's version of level2_kernel_pgt */
 	l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
 	l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
 
-	memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
-	memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
-
+	addr[0] = (unsigned long)pgd;
+	addr[1] = (unsigned long)l3;
+	addr[2] = (unsigned long)l2;
+	/* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
+	 * Both L4[272][0] and L4[511][511] have entries that point to the same
+	 * L2 (PMD) tables. Meaning that if you modify it in __va space
+	 * it will be also modified in the __ka space! (But if you just
+	 * modify the PMD table to point to other PTE's or none, then you
+	 * are OK - which is what cleanup_highmap does) */
+	copy_page(level2_ident_pgt, l2);
+	/* Graft it onto L4[511][511] */
+	copy_page(level2_kernel_pgt, l2);
+
+	/* Get [511][510] and graft that in level2_fixmap_pgt */
 	l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
 	l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
-	memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
-
-	/* Set up identity map */
-	xen_map_identity_early(level2_ident_pgt, max_pfn);
+	copy_page(level2_fixmap_pgt, l2);
+	/* Note that we don't do anything with level1_fixmap_pgt which
+	 * we don't need. */
 
 	/* Make pagetable pieces RO */
 	set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
 	set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
 	set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
 	set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
+	set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
 	set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
 	set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
 
@@ -1791,22 +1902,28 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
 	/* Unpin Xen-provided one */
 	pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
 
-	/* Switch over */
-	pgd = init_level4_pgt;
-
 	/*
 	 * At this stage there can be no user pgd, and no page
 	 * structure to attach it to, so make sure we just set kernel
 	 * pgd.
 	 */
 	xen_mc_batch();
-	__xen_write_cr3(true, __pa(pgd));
+	__xen_write_cr3(true, __pa(init_level4_pgt));
 	xen_mc_issue(PARAVIRT_LAZY_CPU);
 
-	memblock_reserve(__pa(xen_start_info->pt_base),
-			 xen_start_info->nr_pt_frames * PAGE_SIZE);
+	/* We can't that easily rip out L3 and L2, as the Xen pagetables are
+	 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ...  for
+	 * the initial domain. For guests using the toolstack, they are in:
+	 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
+	 * rip out the [L4] (pgd), but for guests we shave off three pages.
+	 */
+	for (i = 0; i < ARRAY_SIZE(addr); i++)
+		check_pt_base(&pt_base, &pt_end, addr[i]);
 
-	return pgd;
+	/* Our (by three pages) smaller Xen pagetable that we are using */
+	memblock_reserve(PFN_PHYS(pt_base), (pt_end - pt_base) * PAGE_SIZE);
+	/* Revector the xen_start_info */
+	xen_start_info = (struct start_info *)__va(__pa(xen_start_info));
 }
 #else	/* !CONFIG_X86_64 */
 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
@@ -1831,8 +1948,7 @@ static void __init xen_write_cr3_init(unsigned long cr3)
 	 */
 	swapper_kernel_pmd =
 		extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
-	memcpy(swapper_kernel_pmd, initial_kernel_pmd,
-	       sizeof(pmd_t) * PTRS_PER_PMD);
+	copy_page(swapper_kernel_pmd, initial_kernel_pmd);
 	swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
 		__pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
 	set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
@@ -1849,8 +1965,7 @@ static void __init xen_write_cr3_init(unsigned long cr3)
 	pv_mmu_ops.write_cr3 = &xen_write_cr3;
 }
 
-pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
-					 unsigned long max_pfn)
+void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
 {
 	pmd_t *kernel_pmd;
 
@@ -1862,11 +1977,11 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
 				  512*1024);
 
 	kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
-	memcpy(initial_kernel_pmd, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
+	copy_page(initial_kernel_pmd, kernel_pmd);
 
 	xen_map_identity_early(initial_kernel_pmd, max_pfn);
 
-	memcpy(initial_page_table, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
+	copy_page(initial_page_table, pgd);
 	initial_page_table[KERNEL_PGD_BOUNDARY] =
 		__pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
 
@@ -1882,8 +1997,6 @@ pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
 
 	memblock_reserve(__pa(xen_start_info->pt_base),
 			 xen_start_info->nr_pt_frames * PAGE_SIZE);
-
-	return initial_page_table;
 }
 #endif	/* CONFIG_X86_64 */
 
@@ -2333,6 +2446,9 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
 	unsigned long range;
 	int err = 0;
 
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return -EINVAL;
+
 	prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP);
 
 	BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) ==
@@ -2351,8 +2467,8 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
 		if (err)
 			goto out;
 
-		err = -EFAULT;
-		if (HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid) < 0)
+		err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid);
+		if (err < 0)
 			goto out;
 
 		nr -= batch;
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 72213da605f..95fb2aa5927 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -22,7 +22,7 @@
  *
  * P2M_PER_PAGE depends on the architecture, as a mfn is always
  * unsigned long (8 bytes on 64-bit, 4 bytes on 32), leading to
- * 512 and 1024 entries respectively. 
+ * 512 and 1024 entries respectively.
  *
  * In short, these structures contain the Machine Frame Number (MFN) of the PFN.
  *
@@ -139,11 +139,11 @@
  *      /    | ~0, ~0, ....  |
  *     |     \---------------/
  *     |
- *     p2m_missing             p2m_missing
- * /------------------\     /------------\
- * | [p2m_mid_missing]+---->| ~0, ~0, ~0 |
- * | [p2m_mid_missing]+---->| ..., ~0    |
- * \------------------/     \------------/
+ *   p2m_mid_missing           p2m_missing
+ * /-----------------\     /------------\
+ * | [p2m_missing]   +---->| ~0, ~0, ~0 |
+ * | [p2m_missing]   +---->| ..., ~0    |
+ * \-----------------/     \------------/
  *
  * where ~0 is INVALID_P2M_ENTRY. IDENTITY is (PFN | IDENTITY_BIT)
  */
@@ -396,7 +396,85 @@ void __init xen_build_dynamic_phys_to_machine(void)
 
 	m2p_override_init();
 }
+#ifdef CONFIG_X86_64
+#include <linux/bootmem.h>
+unsigned long __init xen_revector_p2m_tree(void)
+{
+	unsigned long va_start;
+	unsigned long va_end;
+	unsigned long pfn;
+	unsigned long pfn_free = 0;
+	unsigned long *mfn_list = NULL;
+	unsigned long size;
+
+	va_start = xen_start_info->mfn_list;
+	/*We copy in increments of P2M_PER_PAGE * sizeof(unsigned long),
+	 * so make sure it is rounded up to that */
+	size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
+	va_end = va_start + size;
+
+	/* If we were revectored already, don't do it again. */
+	if (va_start <= __START_KERNEL_map && va_start >= __PAGE_OFFSET)
+		return 0;
+
+	mfn_list = alloc_bootmem_align(size, PAGE_SIZE);
+	if (!mfn_list) {
+		pr_warn("Could not allocate space for a new P2M tree!\n");
+		return xen_start_info->mfn_list;
+	}
+	/* Fill it out with INVALID_P2M_ENTRY value */
+	memset(mfn_list, 0xFF, size);
+
+	for (pfn = 0; pfn < ALIGN(MAX_DOMAIN_PAGES, P2M_PER_PAGE); pfn += P2M_PER_PAGE) {
+		unsigned topidx = p2m_top_index(pfn);
+		unsigned mididx;
+		unsigned long *mid_p;
+
+		if (!p2m_top[topidx])
+			continue;
+
+		if (p2m_top[topidx] == p2m_mid_missing)
+			continue;
+
+		mididx = p2m_mid_index(pfn);
+		mid_p = p2m_top[topidx][mididx];
+		if (!mid_p)
+			continue;
+		if ((mid_p == p2m_missing) || (mid_p == p2m_identity))
+			continue;
+
+		if ((unsigned long)mid_p == INVALID_P2M_ENTRY)
+			continue;
+
+		/* The old va. Rebase it on mfn_list */
+		if (mid_p >= (unsigned long *)va_start && mid_p <= (unsigned long *)va_end) {
+			unsigned long *new;
+
+			if (pfn_free  > (size / sizeof(unsigned long))) {
+				WARN(1, "Only allocated for %ld pages, but we want %ld!\n",
+				     size / sizeof(unsigned long), pfn_free);
+				return 0;
+			}
+			new = &mfn_list[pfn_free];
+
+			copy_page(new, mid_p);
+			p2m_top[topidx][mididx] = &mfn_list[pfn_free];
+			p2m_top_mfn_p[topidx][mididx] = virt_to_mfn(&mfn_list[pfn_free]);
+
+			pfn_free += P2M_PER_PAGE;
 
+		}
+		/* This should be the leafs allocated for identity from _brk. */
+	}
+	return (unsigned long)mfn_list;
+
+}
+#else
+unsigned long __init xen_revector_p2m_tree(void)
+{
+	return 0;
+}
+#endif
 unsigned long get_phys_to_machine(unsigned long pfn)
 {
 	unsigned topidx, mididx, idx;
@@ -430,7 +508,7 @@ static void free_p2m_page(void *p)
 	free_page((unsigned long)p);
 }
 
-/* 
+/*
  * Fully allocate the p2m structure for a given pfn.  We need to check
  * that both the top and mid levels are allocated, and make sure the
  * parallel mfn tree is kept in sync.  We may race with other cpus, so
diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c
index 967633ad98c..969570491c3 100644
--- a/arch/x86/xen/pci-swiotlb-xen.c
+++ b/arch/x86/xen/pci-swiotlb-xen.c
@@ -8,6 +8,14 @@
 #include <xen/xen.h>
 #include <asm/iommu_table.h>
 
+
+#include <asm/xen/swiotlb-xen.h>
+#ifdef CONFIG_X86_64
+#include <asm/iommu.h>
+#include <asm/dma.h>
+#endif
+#include <linux/export.h>
+
 int xen_swiotlb __read_mostly;
 
 static struct dma_map_ops xen_swiotlb_dma_ops = {
@@ -34,34 +42,64 @@ static struct dma_map_ops xen_swiotlb_dma_ops = {
 int __init pci_xen_swiotlb_detect(void)
 {
 
+	if (!xen_pv_domain())
+		return 0;
+
 	/* If running as PV guest, either iommu=soft, or swiotlb=force will
 	 * activate this IOMMU. If running as PV privileged, activate it
 	 * irregardless.
 	 */
-	if ((xen_initial_domain() || swiotlb || swiotlb_force) &&
-	    (xen_pv_domain()))
+	if ((xen_initial_domain() || swiotlb || swiotlb_force))
 		xen_swiotlb = 1;
 
 	/* If we are running under Xen, we MUST disable the native SWIOTLB.
 	 * Don't worry about swiotlb_force flag activating the native, as
 	 * the 'swiotlb' flag is the only one turning it on. */
-	if (xen_pv_domain())
-		swiotlb = 0;
+	swiotlb = 0;
 
+#ifdef CONFIG_X86_64
+	/* pci_swiotlb_detect_4gb turns on native SWIOTLB if no_iommu == 0
+	 * (so no iommu=X command line over-writes).
+	 * Considering that PV guests do not want the *native SWIOTLB* but
+	 * only Xen SWIOTLB it is not useful to us so set no_iommu=1 here.
+	 */
+	if (max_pfn > MAX_DMA32_PFN)
+		no_iommu = 1;
+#endif
 	return xen_swiotlb;
 }
 
 void __init pci_xen_swiotlb_init(void)
 {
 	if (xen_swiotlb) {
-		xen_swiotlb_init(1);
+		xen_swiotlb_init(1, true /* early */);
 		dma_ops = &xen_swiotlb_dma_ops;
 
 		/* Make sure ACS will be enabled */
 		pci_request_acs();
 	}
 }
+
+int pci_xen_swiotlb_init_late(void)
+{
+	int rc;
+
+	if (xen_swiotlb)
+		return 0;
+
+	rc = xen_swiotlb_init(1, false /* late */);
+	if (rc)
+		return rc;
+
+	dma_ops = &xen_swiotlb_dma_ops;
+	/* Make sure ACS will be enabled */
+	pci_request_acs();
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late);
+
 IOMMU_INIT_FINISH(pci_xen_swiotlb_detect,
-		  0,
+		  NULL,
 		  pci_xen_swiotlb_init,
-		  0);
+		  NULL);
diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c
index ffcf2615640..0a7852483ff 100644
--- a/arch/x86/xen/platform-pci-unplug.c
+++ b/arch/x86/xen/platform-pci-unplug.c
@@ -24,6 +24,7 @@
 #include <linux/module.h>
 
 #include <xen/platform_pci.h>
+#include "xen-ops.h"
 
 #define XEN_PLATFORM_ERR_MAGIC -1
 #define XEN_PLATFORM_ERR_PROTOCOL -2
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index e2d62d697b5..8971a26d21a 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -432,6 +432,24 @@ char * __init xen_memory_setup(void)
 	 *  - mfn_list
 	 *  - xen_start_info
 	 * See comment above "struct start_info" in <xen/interface/xen.h>
+	 * We tried to make the the memblock_reserve more selective so
+	 * that it would be clear what region is reserved. Sadly we ran
+	 * in the problem wherein on a 64-bit hypervisor with a 32-bit
+	 * initial domain, the pt_base has the cr3 value which is not
+	 * neccessarily where the pagetable starts! As Jan put it: "
+	 * Actually, the adjustment turns out to be correct: The page
+	 * tables for a 32-on-64 dom0 get allocated in the order "first L1",
+	 * "first L2", "first L3", so the offset to the page table base is
+	 * indeed 2. When reading xen/include/public/xen.h's comment
+	 * very strictly, this is not a violation (since there nothing is said
+	 * that the first thing in the page table space is pointed to by
+	 * pt_base; I admit that this seems to be implied though, namely
+	 * do I think that it is implied that the page table space is the
+	 * range [pt_base, pt_base + nt_pt_frames), whereas that
+	 * range here indeed is [pt_base - 2, pt_base - 2 + nt_pt_frames),
+	 * which - without a priori knowledge - the kernel would have
+	 * difficulty to figure out)." - so lets just fall back to the
+	 * easy way and reserve the whole region.
 	 */
 	memblock_reserve(__pa(xen_start_info->mfn_list),
 			 xen_start_info->pt_base - xen_start_info->mfn_list);
diff --git a/arch/x86/xen/vga.c b/arch/x86/xen/vga.c
index 1cd7f4d11e2..6722e3733f0 100644
--- a/arch/x86/xen/vga.c
+++ b/arch/x86/xen/vga.c
@@ -35,6 +35,7 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size)
 			info->u.text_mode_3.font_height;
 		break;
 
+	case XEN_VGATYPE_EFI_LFB:
 	case XEN_VGATYPE_VESA_LFB:
 		if (size < offsetof(struct dom0_vga_console_info,
 				    u.vesa_lfb.gbl_caps))
@@ -54,6 +55,12 @@ void __init xen_init_vga(const struct dom0_vga_console_info *info, size_t size)
 		screen_info->blue_pos = info->u.vesa_lfb.blue_pos;
 		screen_info->rsvd_size = info->u.vesa_lfb.rsvd_size;
 		screen_info->rsvd_pos = info->u.vesa_lfb.rsvd_pos;
+
+		if (info->video_type == XEN_VGATYPE_EFI_LFB) {
+			screen_info->orig_video_isVGA = VIDEO_TYPE_EFI;
+			break;
+		}
+
 		if (size >= offsetof(struct dom0_vga_console_info,
 				     u.vesa_lfb.gbl_caps)
 		    + sizeof(info->u.vesa_lfb.gbl_caps))
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index aaa7291c925..7faed5869e5 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -28,9 +28,61 @@ ENTRY(startup_xen)
 	__FINIT
 
 .pushsection .text
-	.align PAGE_SIZE
+	.balign PAGE_SIZE
 ENTRY(hypercall_page)
-	.skip PAGE_SIZE
+#define NEXT_HYPERCALL(x) \
+	ENTRY(xen_hypercall_##x) \
+	.skip 32
+
+NEXT_HYPERCALL(set_trap_table)
+NEXT_HYPERCALL(mmu_update)
+NEXT_HYPERCALL(set_gdt)
+NEXT_HYPERCALL(stack_switch)
+NEXT_HYPERCALL(set_callbacks)
+NEXT_HYPERCALL(fpu_taskswitch)
+NEXT_HYPERCALL(sched_op_compat)
+NEXT_HYPERCALL(platform_op)
+NEXT_HYPERCALL(set_debugreg)
+NEXT_HYPERCALL(get_debugreg)
+NEXT_HYPERCALL(update_descriptor)
+NEXT_HYPERCALL(ni)
+NEXT_HYPERCALL(memory_op)
+NEXT_HYPERCALL(multicall)
+NEXT_HYPERCALL(update_va_mapping)
+NEXT_HYPERCALL(set_timer_op)
+NEXT_HYPERCALL(event_channel_op_compat)
+NEXT_HYPERCALL(xen_version)
+NEXT_HYPERCALL(console_io)
+NEXT_HYPERCALL(physdev_op_compat)
+NEXT_HYPERCALL(grant_table_op)
+NEXT_HYPERCALL(vm_assist)
+NEXT_HYPERCALL(update_va_mapping_otherdomain)
+NEXT_HYPERCALL(iret)
+NEXT_HYPERCALL(vcpu_op)
+NEXT_HYPERCALL(set_segment_base)
+NEXT_HYPERCALL(mmuext_op)
+NEXT_HYPERCALL(xsm_op)
+NEXT_HYPERCALL(nmi_op)
+NEXT_HYPERCALL(sched_op)
+NEXT_HYPERCALL(callback_op)
+NEXT_HYPERCALL(xenoprof_op)
+NEXT_HYPERCALL(event_channel_op)
+NEXT_HYPERCALL(physdev_op)
+NEXT_HYPERCALL(hvm_op)
+NEXT_HYPERCALL(sysctl)
+NEXT_HYPERCALL(domctl)
+NEXT_HYPERCALL(kexec_op)
+NEXT_HYPERCALL(tmem_op) /* 38 */
+ENTRY(xen_hypercall_rsvr)
+	.skip 320
+NEXT_HYPERCALL(mca) /* 48 */
+NEXT_HYPERCALL(arch_1)
+NEXT_HYPERCALL(arch_2)
+NEXT_HYPERCALL(arch_3)
+NEXT_HYPERCALL(arch_4)
+NEXT_HYPERCALL(arch_5)
+NEXT_HYPERCALL(arch_6)
+	.balign PAGE_SIZE
 .popsection
 
 	ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS,       .asciz "linux")
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index 202d4c15015..bb5a8105ea8 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -27,7 +27,7 @@ void xen_setup_mfn_list_list(void);
 void xen_setup_shared_info(void);
 void xen_build_mfn_list_list(void);
 void xen_setup_machphys_mapping(void);
-pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
+void xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn);
 void xen_reserve_top(void);
 extern unsigned long xen_max_p2m_pfn;
 
@@ -45,6 +45,7 @@ void xen_hvm_init_shared_info(void);
 void xen_unplug_emulated_devices(void);
 
 void __init xen_build_dynamic_phys_to_machine(void);
+unsigned long __init xen_revector_p2m_tree(void);
 
 void xen_init_irq_ops(void);
 void xen_setup_timer(int cpu);
diff --git a/arch/xtensa/include/asm/elf.h b/arch/xtensa/include/asm/elf.h
index 6e65eadaae1..5293312bc6a 100644
--- a/arch/xtensa/include/asm/elf.h
+++ b/arch/xtensa/include/asm/elf.h
@@ -189,7 +189,8 @@ typedef struct {
 #endif
 } elf_xtregs_t;
 
-#define SET_PERSONALITY(ex) set_personality(PER_LINUX_32BIT)
+#define SET_PERSONALITY(ex) \
+	set_personality(PER_LINUX_32BIT | (current->personality & (~PER_MASK)))
 
 struct task_struct;
 
diff --git a/arch/xtensa/include/uapi/asm/Kbuild b/arch/xtensa/include/uapi/asm/Kbuild
new file mode 100644
index 00000000000..baebb3da1d4
--- /dev/null
+++ b/arch/xtensa/include/uapi/asm/Kbuild
@@ -0,0 +1,3 @@
+# UAPI Header export list
+include include/uapi/asm-generic/Kbuild.asm
+
diff --git a/crypto/842.c b/crypto/842.c
new file mode 100644
index 00000000000..65c7a89cfa0
--- /dev/null
+++ b/crypto/842.c
@@ -0,0 +1,182 @@
+/*
+ * Cryptographic API for the 842 compression algorithm.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * Copyright (C) IBM Corporation, 2011
+ *
+ * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ *          Seth Jennings <sjenning@linux.vnet.ibm.com>
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/vmalloc.h>
+#include <linux/nx842.h>
+#include <linux/lzo.h>
+#include <linux/timer.h>
+
+static int nx842_uselzo;
+
+struct nx842_ctx {
+	void *nx842_wmem; /* working memory for 842/lzo */
+};
+
+enum nx842_crypto_type {
+	NX842_CRYPTO_TYPE_842,
+	NX842_CRYPTO_TYPE_LZO
+};
+
+#define NX842_SENTINEL 0xdeadbeef
+
+struct nx842_crypto_header {
+	unsigned int sentinel; /* debug */
+	enum nx842_crypto_type type;
+};
+
+static int nx842_init(struct crypto_tfm *tfm)
+{
+	struct nx842_ctx *ctx = crypto_tfm_ctx(tfm);
+	int wmemsize;
+
+	wmemsize = max_t(int, nx842_get_workmem_size(), LZO1X_MEM_COMPRESS);
+	ctx->nx842_wmem = kmalloc(wmemsize, GFP_NOFS);
+	if (!ctx->nx842_wmem)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void nx842_exit(struct crypto_tfm *tfm)
+{
+	struct nx842_ctx *ctx = crypto_tfm_ctx(tfm);
+
+	kfree(ctx->nx842_wmem);
+}
+
+static void nx842_reset_uselzo(unsigned long data)
+{
+	nx842_uselzo = 0;
+}
+
+static DEFINE_TIMER(failover_timer, nx842_reset_uselzo, 0, 0);
+
+static int nx842_crypto_compress(struct crypto_tfm *tfm, const u8 *src,
+			    unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct nx842_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct nx842_crypto_header *hdr;
+	unsigned int tmp_len = *dlen;
+	size_t lzodlen; /* needed for lzo */
+	int err;
+
+	*dlen = 0;
+	hdr = (struct nx842_crypto_header *)dst;
+	hdr->sentinel = NX842_SENTINEL; /* debug */
+	dst += sizeof(struct nx842_crypto_header);
+	tmp_len -= sizeof(struct nx842_crypto_header);
+	lzodlen = tmp_len;
+
+	if (likely(!nx842_uselzo)) {
+		err = nx842_compress(src, slen, dst, &tmp_len, ctx->nx842_wmem);
+
+		if (likely(!err)) {
+			hdr->type = NX842_CRYPTO_TYPE_842;
+			*dlen = tmp_len + sizeof(struct nx842_crypto_header);
+			return 0;
+		}
+
+		/* hardware failed */
+		nx842_uselzo = 1;
+
+		/* set timer to check for hardware again in 1 second */
+		mod_timer(&failover_timer, jiffies + msecs_to_jiffies(1000));
+	}
+
+	/* no hardware, use lzo */
+	err = lzo1x_1_compress(src, slen, dst, &lzodlen, ctx->nx842_wmem);
+	if (err != LZO_E_OK)
+		return -EINVAL;
+
+	hdr->type = NX842_CRYPTO_TYPE_LZO;
+	*dlen = lzodlen + sizeof(struct nx842_crypto_header);
+	return 0;
+}
+
+static int nx842_crypto_decompress(struct crypto_tfm *tfm, const u8 *src,
+			      unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct nx842_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct nx842_crypto_header *hdr;
+	unsigned int tmp_len = *dlen;
+	size_t lzodlen; /* needed for lzo */
+	int err;
+
+	*dlen = 0;
+	hdr = (struct nx842_crypto_header *)src;
+
+	if (unlikely(hdr->sentinel != NX842_SENTINEL))
+		return -EINVAL;
+
+	src += sizeof(struct nx842_crypto_header);
+	slen -= sizeof(struct nx842_crypto_header);
+
+	if (likely(hdr->type == NX842_CRYPTO_TYPE_842)) {
+		err = nx842_decompress(src, slen, dst, &tmp_len,
+			ctx->nx842_wmem);
+		if (err)
+			return -EINVAL;
+		*dlen = tmp_len;
+	} else if (hdr->type == NX842_CRYPTO_TYPE_LZO) {
+		lzodlen = tmp_len;
+		err = lzo1x_decompress_safe(src, slen, dst, &lzodlen);
+		if (err != LZO_E_OK)
+			return -EINVAL;
+		*dlen = lzodlen;
+	} else
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct crypto_alg alg = {
+	.cra_name		= "842",
+	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
+	.cra_ctxsize		= sizeof(struct nx842_ctx),
+	.cra_module		= THIS_MODULE,
+	.cra_init		= nx842_init,
+	.cra_exit		= nx842_exit,
+	.cra_u			= { .compress = {
+	.coa_compress		= nx842_crypto_compress,
+	.coa_decompress		= nx842_crypto_decompress } }
+};
+
+static int __init nx842_mod_init(void)
+{
+	del_timer(&failover_timer);
+	return crypto_register_alg(&alg);
+}
+
+static void __exit nx842_mod_exit(void)
+{
+	crypto_unregister_alg(&alg);
+}
+
+module_init(nx842_mod_init);
+module_exit(nx842_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("842 Compression Algorithm");
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 94f232f96d0..50402dc0ea3 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -460,6 +460,15 @@ config CRYPTO_SHA1_SPARC64
 	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
 	  using sparc64 crypto instructions, when available.
 
+config CRYPTO_SHA1_ARM
+	tristate "SHA1 digest algorithm (ARM-asm)"
+	depends on ARM
+	select CRYPTO_SHA1
+	select CRYPTO_HASH
+	help
+	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
+	  using optimized ARM assembler.
+
 config CRYPTO_SHA256
 	tristate "SHA224 and SHA256 digest algorithm"
 	select CRYPTO_HASH
@@ -609,6 +618,8 @@ config CRYPTO_AES_NI_INTEL
 	select CRYPTO_CRYPTD
 	select CRYPTO_ABLK_HELPER_X86
 	select CRYPTO_ALGAPI
+	select CRYPTO_LRW
+	select CRYPTO_XTS
 	help
 	  Use Intel AES-NI instructions for AES algorithm.
 
@@ -661,6 +672,30 @@ config CRYPTO_AES_SPARC64
 	  for some popular block cipher mode is supported too, including
 	  ECB and CBC.
 
+config CRYPTO_AES_ARM
+	tristate "AES cipher algorithms (ARM-asm)"
+	depends on ARM
+	select CRYPTO_ALGAPI
+	select CRYPTO_AES
+	help
+	  Use optimized AES assembler routines for ARM platforms.
+
+	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
+	  algorithm.
+
+	  Rijndael appears to be consistently a very good performer in
+	  both hardware and software across a wide range of computing
+	  environments regardless of its use in feedback or non-feedback
+	  modes. Its key setup time is excellent, and its key agility is
+	  good. Rijndael's very low memory requirements make it very well
+	  suited for restricted-space environments, in which it also
+	  demonstrates excellent performance. Rijndael's operations are
+	  among the easiest to defend against power and timing attacks.
+
+	  The AES specifies three key sizes: 128, 192 and 256 bits
+
+	  See <http://csrc.nist.gov/encryption/aes/> for more information.
+
 config CRYPTO_ANUBIS
 	tristate "Anubis cipher algorithm"
 	select CRYPTO_ALGAPI
@@ -781,6 +816,20 @@ config CRYPTO_CAST5
 	  The CAST5 encryption algorithm (synonymous with CAST-128) is
 	  described in RFC2144.
 
+config CRYPTO_CAST5_AVX_X86_64
+	tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)"
+	depends on X86 && 64BIT
+	select CRYPTO_ALGAPI
+	select CRYPTO_CRYPTD
+	select CRYPTO_ABLK_HELPER_X86
+	select CRYPTO_CAST5
+	help
+	  The CAST5 encryption algorithm (synonymous with CAST-128) is
+	  described in RFC2144.
+
+	  This module provides the Cast5 cipher algorithm that processes
+	  sixteen blocks parallel using the AVX instruction set.
+
 config CRYPTO_CAST6
 	tristate "CAST6 (CAST-256) cipher algorithm"
 	select CRYPTO_ALGAPI
@@ -788,6 +837,23 @@ config CRYPTO_CAST6
 	  The CAST6 encryption algorithm (synonymous with CAST-256) is
 	  described in RFC2612.
 
+config CRYPTO_CAST6_AVX_X86_64
+	tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)"
+	depends on X86 && 64BIT
+	select CRYPTO_ALGAPI
+	select CRYPTO_CRYPTD
+	select CRYPTO_ABLK_HELPER_X86
+	select CRYPTO_GLUE_HELPER_X86
+	select CRYPTO_CAST6
+	select CRYPTO_LRW
+	select CRYPTO_XTS
+	help
+	  The CAST6 encryption algorithm (synonymous with CAST-256) is
+	  described in RFC2612.
+
+	  This module provides the Cast6 cipher algorithm that processes
+	  eight blocks parallel using the AVX instruction set.
+
 config CRYPTO_DES
 	tristate "DES and Triple DES EDE cipher algorithms"
 	select CRYPTO_ALGAPI
@@ -796,6 +862,7 @@ config CRYPTO_DES
 
 config CRYPTO_DES_SPARC64
 	tristate "DES and Triple DES EDE cipher algorithms (SPARC64)"
+	depends on SPARC64
 	select CRYPTO_ALGAPI
 	select CRYPTO_DES
 	help
@@ -1105,6 +1172,15 @@ config CRYPTO_LZO
 	help
 	  This is the LZO algorithm.
 
+config CRYPTO_842
+	tristate "842 compression algorithm"
+	depends on CRYPTO_DEV_NX_COMPRESS
+	# 842 uses lzo if the hardware becomes unavailable
+	select LZO_COMPRESS
+	select LZO_DECOMPRESS
+	help
+	  This is the 842 algorithm.
+
 comment "Random Number Generation"
 
 config CRYPTO_ANSI_CPRNG
diff --git a/crypto/Makefile b/crypto/Makefile
index 30f33d67533..a301ad2b258 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -68,8 +68,8 @@ obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
 obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
 obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
 obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
-obj-$(CONFIG_CRYPTO_CAST5) += cast5.o
-obj-$(CONFIG_CRYPTO_CAST6) += cast6.o
+obj-$(CONFIG_CRYPTO_CAST5) += cast5_generic.o
+obj-$(CONFIG_CRYPTO_CAST6) += cast6_generic.o
 obj-$(CONFIG_CRYPTO_ARC4) += arc4.o
 obj-$(CONFIG_CRYPTO_TEA) += tea.o
 obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o
@@ -82,6 +82,7 @@ obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
 obj-$(CONFIG_CRYPTO_CRC32C) += crc32c.o
 obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
 obj-$(CONFIG_CRYPTO_LZO) += lzo.o
+obj-$(CONFIG_CRYPTO_842) += 842.o
 obj-$(CONFIG_CRYPTO_RNG2) += rng.o
 obj-$(CONFIG_CRYPTO_RNG2) += krng.o
 obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o
diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c
index a68c73dae15..47f2e5c7175 100644
--- a/crypto/aes_generic.c
+++ b/crypto/aes_generic.c
@@ -1448,7 +1448,6 @@ static struct crypto_alg aes_alg = {
 	.cra_ctxsize		=	sizeof(struct crypto_aes_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c
index 6ddd99e6114..c0bb3778f1a 100644
--- a/crypto/ansi_cprng.c
+++ b/crypto/ansi_cprng.c
@@ -382,26 +382,6 @@ static int cprng_reset(struct crypto_rng *tfm, u8 *seed, unsigned int slen)
 	return 0;
 }
 
-static struct crypto_alg rng_alg = {
-	.cra_name		= "stdrng",
-	.cra_driver_name	= "ansi_cprng",
-	.cra_priority		= 100,
-	.cra_flags		= CRYPTO_ALG_TYPE_RNG,
-	.cra_ctxsize		= sizeof(struct prng_context),
-	.cra_type		= &crypto_rng_type,
-	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(rng_alg.cra_list),
-	.cra_init		= cprng_init,
-	.cra_exit		= cprng_exit,
-	.cra_u			= {
-		.rng = {
-			.rng_make_random	= cprng_get_random,
-			.rng_reset		= cprng_reset,
-			.seedsize = DEFAULT_PRNG_KSZ + 2*DEFAULT_BLK_SZ,
-		}
-	}
-};
-
 #ifdef CONFIG_CRYPTO_FIPS
 static int fips_cprng_get_random(struct crypto_rng *tfm, u8 *rdata,
 			    unsigned int dlen)
@@ -438,8 +418,27 @@ static int fips_cprng_reset(struct crypto_rng *tfm, u8 *seed, unsigned int slen)
 out:
 	return rc;
 }
+#endif
 
-static struct crypto_alg fips_rng_alg = {
+static struct crypto_alg rng_algs[] = { {
+	.cra_name		= "stdrng",
+	.cra_driver_name	= "ansi_cprng",
+	.cra_priority		= 100,
+	.cra_flags		= CRYPTO_ALG_TYPE_RNG,
+	.cra_ctxsize		= sizeof(struct prng_context),
+	.cra_type		= &crypto_rng_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= cprng_init,
+	.cra_exit		= cprng_exit,
+	.cra_u			= {
+		.rng = {
+			.rng_make_random	= cprng_get_random,
+			.rng_reset		= cprng_reset,
+			.seedsize = DEFAULT_PRNG_KSZ + 2*DEFAULT_BLK_SZ,
+		}
+	}
+#ifdef CONFIG_CRYPTO_FIPS
+}, {
 	.cra_name		= "fips(ansi_cprng)",
 	.cra_driver_name	= "fips_ansi_cprng",
 	.cra_priority		= 300,
@@ -447,7 +446,6 @@ static struct crypto_alg fips_rng_alg = {
 	.cra_ctxsize		= sizeof(struct prng_context),
 	.cra_type		= &crypto_rng_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(rng_alg.cra_list),
 	.cra_init		= cprng_init,
 	.cra_exit		= cprng_exit,
 	.cra_u			= {
@@ -457,33 +455,18 @@ static struct crypto_alg fips_rng_alg = {
 			.seedsize = DEFAULT_PRNG_KSZ + 2*DEFAULT_BLK_SZ,
 		}
 	}
-};
 #endif
+} };
 
 /* Module initalization */
 static int __init prng_mod_init(void)
 {
-	int rc = 0;
-
-	rc = crypto_register_alg(&rng_alg);
-#ifdef CONFIG_CRYPTO_FIPS
-	if (rc)
-		goto out;
-
-	rc = crypto_register_alg(&fips_rng_alg);
-
-out:
-#endif
-	return rc;
+	return crypto_register_algs(rng_algs, ARRAY_SIZE(rng_algs));
 }
 
 static void __exit prng_mod_fini(void)
 {
-	crypto_unregister_alg(&rng_alg);
-#ifdef CONFIG_CRYPTO_FIPS
-	crypto_unregister_alg(&fips_rng_alg);
-#endif
-	return;
+	crypto_unregister_algs(rng_algs, ARRAY_SIZE(rng_algs));
 }
 
 MODULE_LICENSE("GPL");
diff --git a/crypto/anubis.c b/crypto/anubis.c
index 77530d571c9..008c8a4fb67 100644
--- a/crypto/anubis.c
+++ b/crypto/anubis.c
@@ -678,7 +678,6 @@ static struct crypto_alg anubis_alg = {
 	.cra_ctxsize		=	sizeof (struct anubis_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(anubis_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	ANUBIS_MIN_KEY_SIZE,
 	.cia_max_keysize	=	ANUBIS_MAX_KEY_SIZE,
diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c
index 6f269b5cfa3..8baf5447d35 100644
--- a/crypto/blowfish_generic.c
+++ b/crypto/blowfish_generic.c
@@ -115,7 +115,6 @@ static struct crypto_alg alg = {
 	.cra_ctxsize		=	sizeof(struct bf_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	BF_MIN_KEY_SIZE,
 	.cia_max_keysize	=	BF_MAX_KEY_SIZE,
diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c
index f7aaaaf8698..75efa205230 100644
--- a/crypto/camellia_generic.c
+++ b/crypto/camellia_generic.c
@@ -1072,7 +1072,6 @@ static struct crypto_alg camellia_alg = {
 	.cra_ctxsize		=	sizeof(struct camellia_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(camellia_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	CAMELLIA_MIN_KEY_SIZE,
diff --git a/crypto/cast5.c b/crypto/cast5_generic.c
index 4a230ddec87..bc525dbd8a4 100644
--- a/crypto/cast5.c
+++ b/crypto/cast5_generic.c
@@ -4,8 +4,8 @@
 * Derived from GnuPG implementation of cast5.
 *
 * Major Changes.
-* 	Complete conformance to rfc2144.
-* 	Supports key size from 40 to 128 bits.
+*	Complete conformance to rfc2144.
+*	Supports key size from 40 to 128 bits.
 *
 * Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
 * Copyright (C) 2003 Kartikey Mahendra Bhatt <kartik_me@hotmail.com>.
@@ -28,19 +28,10 @@
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
+#include <crypto/cast5.h>
 
-#define CAST5_BLOCK_SIZE 8
-#define CAST5_MIN_KEY_SIZE 5
-#define CAST5_MAX_KEY_SIZE 16
 
-struct cast5_ctx {
-	u32 Km[16];
-	u8 Kr[16];
-	int rr;	/* rr?number of rounds = 16:number of rounds = 12; (rfc 2144) */
-};
-
-
-static const u32 s1[256] = {
+const u32 cast5_s1[256] = {
 	0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f,
 	0x9c004dd3, 0x6003e540, 0xcf9fc949,
 	0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0,
@@ -106,7 +97,8 @@ static const u32 s1[256] = {
 	0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c,
 	0x5ac9f049, 0xdd8f0f00, 0x5c8165bf
 };
-static const u32 s2[256] = {
+EXPORT_SYMBOL_GPL(cast5_s1);
+const u32 cast5_s2[256] = {
 	0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a,
 	0xeec5207a, 0x55889c94, 0x72fc0651,
 	0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef,
@@ -172,7 +164,8 @@ static const u32 s2[256] = {
 	0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539,
 	0x73bfbe70, 0x83877605, 0x4523ecf1
 };
-static const u32 s3[256] = {
+EXPORT_SYMBOL_GPL(cast5_s2);
+const u32 cast5_s3[256] = {
 	0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff,
 	0x369fe44b, 0x8c1fc644, 0xaececa90,
 	0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806,
@@ -238,7 +231,8 @@ static const u32 s3[256] = {
 	0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636,
 	0xa133c501, 0xe9d3531c, 0xee353783
 };
-static const u32 s4[256] = {
+EXPORT_SYMBOL_GPL(cast5_s3);
+const u32 cast5_s4[256] = {
 	0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb,
 	0x64ad8c57, 0x85510443, 0xfa020ed1,
 	0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43,
@@ -304,6 +298,7 @@ static const u32 s4[256] = {
 	0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0,
 	0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2
 };
+EXPORT_SYMBOL_GPL(cast5_s4);
 static const u32 s5[256] = {
 	0x7ec90c04, 0x2c6e74b9, 0x9b0e66df, 0xa6337911, 0xb86a7fff,
 	0x1dd358f5, 0x44dd9d44, 0x1731167f,
@@ -569,17 +564,21 @@ static const u32 sb8[256] = {
 	0xeaee6801, 0x8db2a283, 0xea8bf59e
 };
 
+#define s1 cast5_s1
+#define s2 cast5_s2
+#define s3 cast5_s3
+#define s4 cast5_s4
+
 #define F1(D, m, r)  ((I = ((m) + (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]))
+	(((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]))
 #define F2(D, m, r)  ((I = ((m) ^ (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]))
+	(((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]))
 #define F3(D, m, r)  ((I = ((m) - (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]))
+	(((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]))
 
 
-static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf)
 {
-	struct cast5_ctx *c = crypto_tfm_ctx(tfm);
 	const __be32 *src = (const __be32 *)inbuf;
 	__be32 *dst = (__be32 *)outbuf;
 	u32 l, r, t;
@@ -628,10 +627,15 @@ static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
 	dst[0] = cpu_to_be32(r);
 	dst[1] = cpu_to_be32(l);
 }
+EXPORT_SYMBOL_GPL(__cast5_encrypt);
 
-static void cast5_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+{
+	__cast5_encrypt(crypto_tfm_ctx(tfm), outbuf, inbuf);
+}
+
+void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf)
 {
-	struct cast5_ctx *c = crypto_tfm_ctx(tfm);
 	const __be32 *src = (const __be32 *)inbuf;
 	__be32 *dst = (__be32 *)outbuf;
 	u32 l, r, t;
@@ -667,6 +671,12 @@ static void cast5_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
 	dst[0] = cpu_to_be32(r);
 	dst[1] = cpu_to_be32(l);
 }
+EXPORT_SYMBOL_GPL(__cast5_decrypt);
+
+static void cast5_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+{
+	__cast5_decrypt(crypto_tfm_ctx(tfm), outbuf, inbuf);
+}
 
 static void key_schedule(u32 *x, u32 *z, u32 *k)
 {
@@ -743,7 +753,7 @@ static void key_schedule(u32 *x, u32 *z, u32 *k)
 }
 
 
-static int cast5_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned key_len)
+int cast5_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key_len)
 {
 	struct cast5_ctx *c = crypto_tfm_ctx(tfm);
 	int i;
@@ -771,20 +781,22 @@ static int cast5_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned key_len)
 		c->Kr[i] = k[i] & 0x1f;
 	return 0;
 }
+EXPORT_SYMBOL_GPL(cast5_setkey);
 
 static struct crypto_alg alg = {
-	.cra_name 	= "cast5",
-	.cra_flags 	= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize 	= CAST5_BLOCK_SIZE,
-	.cra_ctxsize 	= sizeof(struct cast5_ctx),
-	.cra_alignmask	= 3,
-	.cra_module 	= THIS_MODULE,
-	.cra_list 	= LIST_HEAD_INIT(alg.cra_list),
-	.cra_u 		= {
+	.cra_name		= "cast5",
+	.cra_driver_name	= "cast5-generic",
+	.cra_priority		= 100,
+	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		= CAST5_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct cast5_ctx),
+	.cra_alignmask		= 3,
+	.cra_module		= THIS_MODULE,
+	.cra_u			= {
 		.cipher = {
 			.cia_min_keysize = CAST5_MIN_KEY_SIZE,
 			.cia_max_keysize = CAST5_MAX_KEY_SIZE,
-			.cia_setkey = cast5_setkey,
+			.cia_setkey  = cast5_setkey,
 			.cia_encrypt = cast5_encrypt,
 			.cia_decrypt = cast5_decrypt
 		}
@@ -806,4 +818,4 @@ module_exit(cast5_mod_fini);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Cast5 Cipher Algorithm");
-
+MODULE_ALIAS("cast5");
diff --git a/crypto/cast6.c b/crypto/cast6_generic.c
index e0c15a6c7c3..1acd2f1c48f 100644
--- a/crypto/cast6.c
+++ b/crypto/cast6_generic.c
@@ -25,24 +25,21 @@
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
+#include <crypto/cast6.h>
 
-#define CAST6_BLOCK_SIZE 16
-#define CAST6_MIN_KEY_SIZE 16
-#define CAST6_MAX_KEY_SIZE 32
-
-struct cast6_ctx {
-	u32 Km[12][4];
-	u8 Kr[12][4];
-};
+#define s1 cast6_s1
+#define s2 cast6_s2
+#define s3 cast6_s3
+#define s4 cast6_s4
 
 #define F1(D, r, m)  ((I = ((m) + (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]))
+	(((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]))
 #define F2(D, r, m)  ((I = ((m) ^ (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]))
+	(((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]))
 #define F3(D, r, m)  ((I = ((m) - (D))), (I = rol32(I, (r))),   \
-    (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]))
+	(((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]))
 
-static const u32 s1[256] = {
+const u32 cast6_s1[256] = {
 	0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f,
 	0x9c004dd3, 0x6003e540, 0xcf9fc949,
 	0xbfd4af27, 0x88bbbdb5, 0xe2034090, 0x98d09675, 0x6e63a0e0,
@@ -108,8 +105,9 @@ static const u32 s1[256] = {
 	0x1a69e783, 0x02cc4843, 0xa2f7c579, 0x429ef47d, 0x427b169c,
 	0x5ac9f049, 0xdd8f0f00, 0x5c8165bf
 };
+EXPORT_SYMBOL_GPL(cast6_s1);
 
-static const u32 s2[256] = {
+const u32 cast6_s2[256] = {
 	0x1f201094, 0xef0ba75b, 0x69e3cf7e, 0x393f4380, 0xfe61cf7a,
 	0xeec5207a, 0x55889c94, 0x72fc0651,
 	0xada7ef79, 0x4e1d7235, 0xd55a63ce, 0xde0436ba, 0x99c430ef,
@@ -175,8 +173,9 @@ static const u32 s2[256] = {
 	0x43d79572, 0x7e6dd07c, 0x06dfdf1e, 0x6c6cc4ef, 0x7160a539,
 	0x73bfbe70, 0x83877605, 0x4523ecf1
 };
+EXPORT_SYMBOL_GPL(cast6_s2);
 
-static const u32 s3[256] = {
+const u32 cast6_s3[256] = {
 	0x8defc240, 0x25fa5d9f, 0xeb903dbf, 0xe810c907, 0x47607fff,
 	0x369fe44b, 0x8c1fc644, 0xaececa90,
 	0xbeb1f9bf, 0xeefbcaea, 0xe8cf1950, 0x51df07ae, 0x920e8806,
@@ -242,8 +241,9 @@ static const u32 s3[256] = {
 	0xf7baefd5, 0x4142ed9c, 0xa4315c11, 0x83323ec5, 0xdfef4636,
 	0xa133c501, 0xe9d3531c, 0xee353783
 };
+EXPORT_SYMBOL_GPL(cast6_s3);
 
-static const u32 s4[256] = {
+const u32 cast6_s4[256] = {
 	0x9db30420, 0x1fb6e9de, 0xa7be7bef, 0xd273a298, 0x4a4f7bdb,
 	0x64ad8c57, 0x85510443, 0xfa020ed1,
 	0x7e287aff, 0xe60fb663, 0x095f35a1, 0x79ebf120, 0xfd059d43,
@@ -309,6 +309,7 @@ static const u32 s4[256] = {
 	0x7ae5290c, 0x3cb9536b, 0x851e20fe, 0x9833557e, 0x13ecf0b0,
 	0xd3ffb372, 0x3f85c5c1, 0x0aef7ed2
 };
+EXPORT_SYMBOL_GPL(cast6_s4);
 
 static const u32 Tm[24][8] = {
 	{ 0x5a827999, 0xc95c653a, 0x383650db, 0xa7103c7c, 0x15ea281d,
@@ -369,7 +370,7 @@ static const u8 Tr[4][8] = {
 };
 
 /* forward octave */
-static void W(u32 *key, unsigned int i)
+static inline void W(u32 *key, unsigned int i)
 {
 	u32 I;
 	key[6] ^= F1(key[7], Tr[i % 4][0], Tm[i][0]);
@@ -382,14 +383,12 @@ static void W(u32 *key, unsigned int i)
 	key[7] ^= F2(key[0], Tr[i % 4][7], Tm[i][7]);
 }
 
-static int cast6_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-			unsigned key_len)
+int __cast6_setkey(struct cast6_ctx *c, const u8 *in_key,
+		   unsigned key_len, u32 *flags)
 {
 	int i;
 	u32 key[8];
 	__be32 p_key[8]; /* padded key */
-	struct cast6_ctx *c = crypto_tfm_ctx(tfm);
-	u32 *flags = &tfm->crt_flags;
 
 	if (key_len % 4 != 0) {
 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
@@ -425,9 +424,17 @@ static int cast6_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(__cast6_setkey);
+
+int cast6_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen)
+{
+	return __cast6_setkey(crypto_tfm_ctx(tfm), key, keylen,
+			      &tfm->crt_flags);
+}
+EXPORT_SYMBOL_GPL(cast6_setkey);
 
 /*forward quad round*/
-static void Q(u32 *block, u8 *Kr, u32 *Km)
+static inline void Q(u32 *block, u8 *Kr, u32 *Km)
 {
 	u32 I;
 	block[2] ^= F1(block[3], Kr[0], Km[0]);
@@ -437,7 +444,7 @@ static void Q(u32 *block, u8 *Kr, u32 *Km)
 }
 
 /*reverse quad round*/
-static void QBAR(u32 *block, u8 *Kr, u32 *Km)
+static inline void QBAR(u32 *block, u8 *Kr, u32 *Km)
 {
 	u32 I;
 	block[3] ^= F1(block[0], Kr[3], Km[3]);
@@ -446,9 +453,8 @@ static void QBAR(u32 *block, u8 *Kr, u32 *Km)
 	block[2] ^= F1(block[3], Kr[0], Km[0]);
 }
 
-static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+void __cast6_encrypt(struct cast6_ctx *c, u8 *outbuf, const u8 *inbuf)
 {
-	struct cast6_ctx *c = crypto_tfm_ctx(tfm);
 	const __be32 *src = (const __be32 *)inbuf;
 	__be32 *dst = (__be32 *)outbuf;
 	u32 block[4];
@@ -478,10 +484,15 @@ static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
 	dst[2] = cpu_to_be32(block[2]);
 	dst[3] = cpu_to_be32(block[3]);
 }
+EXPORT_SYMBOL_GPL(__cast6_encrypt);
 
-static void cast6_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+{
+	__cast6_encrypt(crypto_tfm_ctx(tfm), outbuf, inbuf);
+}
+
+void __cast6_decrypt(struct cast6_ctx *c, u8 *outbuf, const u8 *inbuf)
 {
-	struct cast6_ctx *c = crypto_tfm_ctx(tfm);
 	const __be32 *src = (const __be32 *)inbuf;
 	__be32 *dst = (__be32 *)outbuf;
 	u32 block[4];
@@ -511,15 +522,22 @@ static void cast6_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
 	dst[2] = cpu_to_be32(block[2]);
 	dst[3] = cpu_to_be32(block[3]);
 }
+EXPORT_SYMBOL_GPL(__cast6_decrypt);
+
+static void cast6_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
+{
+	__cast6_decrypt(crypto_tfm_ctx(tfm), outbuf, inbuf);
+}
 
 static struct crypto_alg alg = {
 	.cra_name = "cast6",
+	.cra_driver_name = "cast6-generic",
+	.cra_priority = 100,
 	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize = CAST6_BLOCK_SIZE,
 	.cra_ctxsize = sizeof(struct cast6_ctx),
 	.cra_alignmask = 3,
 	.cra_module = THIS_MODULE,
-	.cra_list = LIST_HEAD_INIT(alg.cra_list),
 	.cra_u = {
 		  .cipher = {
 			     .cia_min_keysize = CAST6_MIN_KEY_SIZE,
@@ -545,3 +563,4 @@ module_exit(cast6_mod_fini);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Cast6 Cipher Algorithm");
+MODULE_ALIAS("cast6");
diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c
index 07a8a96d46f..fee7265cd35 100644
--- a/crypto/crypto_null.c
+++ b/crypto/crypto_null.c
@@ -94,18 +94,6 @@ static int skcipher_null_crypt(struct blkcipher_desc *desc,
 	return err;
 }
 
-static struct crypto_alg compress_null = {
-	.cra_name		=	"compress_null",
-	.cra_flags		=	CRYPTO_ALG_TYPE_COMPRESS,
-	.cra_blocksize		=	NULL_BLOCK_SIZE,
-	.cra_ctxsize		=	0,
-	.cra_module		=	THIS_MODULE,
-	.cra_list		=       LIST_HEAD_INIT(compress_null.cra_list),
-	.cra_u			=	{ .compress = {
-	.coa_compress 		=	null_compress,
-	.coa_decompress		=	null_compress } }
-};
-
 static struct shash_alg digest_null = {
 	.digestsize		=	NULL_DIGEST_SIZE,
 	.setkey   		=	null_hash_setkey,
@@ -122,22 +110,19 @@ static struct shash_alg digest_null = {
 	}
 };
 
-static struct crypto_alg cipher_null = {
+static struct crypto_alg null_algs[3] = { {
 	.cra_name		=	"cipher_null",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	NULL_BLOCK_SIZE,
 	.cra_ctxsize		=	0,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(cipher_null.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	NULL_KEY_SIZE,
 	.cia_max_keysize	=	NULL_KEY_SIZE,
 	.cia_setkey		= 	null_setkey,
 	.cia_encrypt		=	null_crypt,
 	.cia_decrypt		=	null_crypt } }
-};
-
-static struct crypto_alg skcipher_null = {
+}, {
 	.cra_name		=	"ecb(cipher_null)",
 	.cra_driver_name	=	"ecb-cipher_null",
 	.cra_priority		=	100,
@@ -146,7 +131,6 @@ static struct crypto_alg skcipher_null = {
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_ctxsize		=	0,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(skcipher_null.cra_list),
 	.cra_u			=	{ .blkcipher = {
 	.min_keysize		=	NULL_KEY_SIZE,
 	.max_keysize		=	NULL_KEY_SIZE,
@@ -154,7 +138,16 @@ static struct crypto_alg skcipher_null = {
 	.setkey			= 	null_setkey,
 	.encrypt		=	skcipher_null_crypt,
 	.decrypt		=	skcipher_null_crypt } }
-};
+}, {
+	.cra_name		=	"compress_null",
+	.cra_flags		=	CRYPTO_ALG_TYPE_COMPRESS,
+	.cra_blocksize		=	NULL_BLOCK_SIZE,
+	.cra_ctxsize		=	0,
+	.cra_module		=	THIS_MODULE,
+	.cra_u			=	{ .compress = {
+	.coa_compress		=	null_compress,
+	.coa_decompress		=	null_compress } }
+} };
 
 MODULE_ALIAS("compress_null");
 MODULE_ALIAS("digest_null");
@@ -164,40 +157,26 @@ static int __init crypto_null_mod_init(void)
 {
 	int ret = 0;
 
-	ret = crypto_register_alg(&cipher_null);
+	ret = crypto_register_algs(null_algs, ARRAY_SIZE(null_algs));
 	if (ret < 0)
 		goto out;
 
-	ret = crypto_register_alg(&skcipher_null);
-	if (ret < 0)
-		goto out_unregister_cipher;
-
 	ret = crypto_register_shash(&digest_null);
 	if (ret < 0)
-		goto out_unregister_skcipher;
+		goto out_unregister_algs;
 
-	ret = crypto_register_alg(&compress_null);
-	if (ret < 0)
-		goto out_unregister_digest;
+	return 0;
 
+out_unregister_algs:
+	crypto_unregister_algs(null_algs, ARRAY_SIZE(null_algs));
 out:
 	return ret;
-
-out_unregister_digest:
-	crypto_unregister_shash(&digest_null);
-out_unregister_skcipher:
-	crypto_unregister_alg(&skcipher_null);
-out_unregister_cipher:
-	crypto_unregister_alg(&cipher_null);
-	goto out;
 }
 
 static void __exit crypto_null_mod_fini(void)
 {
-	crypto_unregister_alg(&compress_null);
 	crypto_unregister_shash(&digest_null);
-	crypto_unregister_alg(&skcipher_null);
-	crypto_unregister_alg(&cipher_null);
+	crypto_unregister_algs(null_algs, ARRAY_SIZE(null_algs));
 }
 
 module_init(crypto_null_mod_init);
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index 6bba414d0c6..35d700a97d7 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -30,7 +30,7 @@
 
 #include "internal.h"
 
-DEFINE_MUTEX(crypto_cfg_mutex);
+static DEFINE_MUTEX(crypto_cfg_mutex);
 
 /* The crypto netlink socket */
 static struct sock *crypto_nlsk;
diff --git a/crypto/deflate.c b/crypto/deflate.c
index b0165ecad0c..b57d70eb156 100644
--- a/crypto/deflate.c
+++ b/crypto/deflate.c
@@ -199,7 +199,6 @@ static struct crypto_alg alg = {
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
 	.cra_ctxsize		= sizeof(struct deflate_ctx),
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(alg.cra_list),
 	.cra_init		= deflate_init,
 	.cra_exit		= deflate_exit,
 	.cra_u			= { .compress = {
diff --git a/crypto/des_generic.c b/crypto/des_generic.c
index 873818d48e8..f6cf63f8846 100644
--- a/crypto/des_generic.c
+++ b/crypto/des_generic.c
@@ -943,59 +943,44 @@ static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	d[1] = cpu_to_le32(L);
 }
 
-static struct crypto_alg des_alg = {
+static struct crypto_alg des_algs[2] = { {
 	.cra_name		=	"des",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	DES_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct des_ctx),
 	.cra_module		=	THIS_MODULE,
 	.cra_alignmask		=	3,
-	.cra_list		=	LIST_HEAD_INIT(des_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	DES_KEY_SIZE,
 	.cia_max_keysize	=	DES_KEY_SIZE,
 	.cia_setkey		=	des_setkey,
 	.cia_encrypt		=	des_encrypt,
 	.cia_decrypt		=	des_decrypt } }
-};
-
-static struct crypto_alg des3_ede_alg = {
+}, {
 	.cra_name		=	"des3_ede",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	DES3_EDE_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct des3_ede_ctx),
 	.cra_module		=	THIS_MODULE,
 	.cra_alignmask		=	3,
-	.cra_list		=	LIST_HEAD_INIT(des3_ede_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	DES3_EDE_KEY_SIZE,
 	.cia_max_keysize	=	DES3_EDE_KEY_SIZE,
 	.cia_setkey		=	des3_ede_setkey,
 	.cia_encrypt		=	des3_ede_encrypt,
 	.cia_decrypt		=	des3_ede_decrypt } }
-};
+} };
 
 MODULE_ALIAS("des3_ede");
 
 static int __init des_generic_mod_init(void)
 {
-	int ret = 0;
-
-	ret = crypto_register_alg(&des_alg);
-	if (ret < 0)
-		goto out;
-
-	ret = crypto_register_alg(&des3_ede_alg);
-	if (ret < 0)
-		crypto_unregister_alg(&des_alg);
-out:
-	return ret;
+	return crypto_register_algs(des_algs, ARRAY_SIZE(des_algs));
 }
 
 static void __exit des_generic_mod_fini(void)
 {
-	crypto_unregister_alg(&des3_ede_alg);
-	crypto_unregister_alg(&des_alg);
+	crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs));
 }
 
 module_init(des_generic_mod_init);
diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c
index c33107e340b..3b2cf569c68 100644
--- a/crypto/fcrypt.c
+++ b/crypto/fcrypt.c
@@ -396,7 +396,6 @@ static struct crypto_alg fcrypt_alg = {
 	.cra_ctxsize		=	sizeof(struct fcrypt_ctx),
 	.cra_module		=	THIS_MODULE,
 	.cra_alignmask		=	3,
-	.cra_list		=	LIST_HEAD_INIT(fcrypt_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	8,
 	.cia_max_keysize	=	8,
diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c
index 7835b8fc94d..9d3f0c69a86 100644
--- a/crypto/ghash-generic.c
+++ b/crypto/ghash-generic.c
@@ -153,7 +153,6 @@ static struct shash_alg ghash_alg = {
 		.cra_blocksize		= GHASH_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct ghash_ctx),
 		.cra_module		= THIS_MODULE,
-		.cra_list		= LIST_HEAD_INIT(ghash_alg.base.cra_list),
 		.cra_exit		= ghash_exit_tfm,
 	},
 };
diff --git a/crypto/khazad.c b/crypto/khazad.c
index 527e4e395fc..60e7cd66fac 100644
--- a/crypto/khazad.c
+++ b/crypto/khazad.c
@@ -853,7 +853,6 @@ static struct crypto_alg khazad_alg = {
 	.cra_ctxsize		=	sizeof (struct khazad_ctx),
 	.cra_alignmask		=	7,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(khazad_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	KHAZAD_KEY_SIZE,
 	.cia_max_keysize	=	KHAZAD_KEY_SIZE,
diff --git a/crypto/krng.c b/crypto/krng.c
index 4328bb3430e..a2d2b72fc13 100644
--- a/crypto/krng.c
+++ b/crypto/krng.c
@@ -35,7 +35,6 @@ static struct crypto_alg krng_alg = {
 	.cra_ctxsize		= 0,
 	.cra_type		= &crypto_rng_type,
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(krng_alg.cra_list),
 	.cra_u			= {
 		.rng = {
 			.rng_make_random	= krng_get_random,
diff --git a/crypto/lzo.c b/crypto/lzo.c
index b5e77077d75..1c2aa69c54b 100644
--- a/crypto/lzo.c
+++ b/crypto/lzo.c
@@ -81,7 +81,6 @@ static struct crypto_alg alg = {
 	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
 	.cra_ctxsize		= sizeof(struct lzo_ctx),
 	.cra_module		= THIS_MODULE,
-	.cra_list		= LIST_HEAD_INIT(alg.cra_list),
 	.cra_init		= lzo_init,
 	.cra_exit		= lzo_exit,
 	.cra_u			= { .compress = {
diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c
index eac10c11685..9a4770c0228 100644
--- a/crypto/salsa20_generic.c
+++ b/crypto/salsa20_generic.c
@@ -221,7 +221,6 @@ static struct crypto_alg alg = {
 	.cra_ctxsize        =   sizeof(struct salsa20_ctx),
 	.cra_alignmask      =	3,
 	.cra_module         =   THIS_MODULE,
-	.cra_list           =   LIST_HEAD_INIT(alg.cra_list),
 	.cra_u              =   {
 		.blkcipher = {
 			.setkey         =   setkey,
diff --git a/crypto/seed.c b/crypto/seed.c
index d3e422f6055..9c904d6d215 100644
--- a/crypto/seed.c
+++ b/crypto/seed.c
@@ -449,7 +449,6 @@ static struct crypto_alg seed_alg = {
 	.cra_ctxsize		=	sizeof(struct seed_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(seed_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	SEED_KEY_SIZE,
diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c
index 8f32cf35e5c..7ddbd7e8885 100644
--- a/crypto/serpent_generic.c
+++ b/crypto/serpent_generic.c
@@ -567,24 +567,6 @@ static void serpent_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	__serpent_decrypt(ctx, dst, src);
 }
 
-static struct crypto_alg serpent_alg = {
-	.cra_name		=	"serpent",
-	.cra_driver_name	=	"serpent-generic",
-	.cra_priority		=	100,
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		=	SERPENT_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct serpent_ctx),
-	.cra_alignmask		=	3,
-	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(serpent_alg.cra_list),
-	.cra_u			=	{ .cipher = {
-	.cia_min_keysize	=	SERPENT_MIN_KEY_SIZE,
-	.cia_max_keysize	=	SERPENT_MAX_KEY_SIZE,
-	.cia_setkey		=	serpent_setkey,
-	.cia_encrypt		=	serpent_encrypt,
-	.cia_decrypt		=	serpent_decrypt } }
-};
-
 static int tnepres_setkey(struct crypto_tfm *tfm, const u8 *key,
 			  unsigned int keylen)
 {
@@ -637,41 +619,44 @@ static void tnepres_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	d[3] = swab32(rd[0]);
 }
 
-static struct crypto_alg tnepres_alg = {
+static struct crypto_alg srp_algs[2] = { {
+	.cra_name		=	"serpent",
+	.cra_driver_name	=	"serpent-generic",
+	.cra_priority		=	100,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	SERPENT_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct serpent_ctx),
+	.cra_alignmask		=	3,
+	.cra_module		=	THIS_MODULE,
+	.cra_u			=	{ .cipher = {
+	.cia_min_keysize	=	SERPENT_MIN_KEY_SIZE,
+	.cia_max_keysize	=	SERPENT_MAX_KEY_SIZE,
+	.cia_setkey		=	serpent_setkey,
+	.cia_encrypt		=	serpent_encrypt,
+	.cia_decrypt		=	serpent_decrypt } }
+}, {
 	.cra_name		=	"tnepres",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	SERPENT_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct serpent_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(serpent_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	SERPENT_MIN_KEY_SIZE,
 	.cia_max_keysize	=	SERPENT_MAX_KEY_SIZE,
 	.cia_setkey		=	tnepres_setkey,
 	.cia_encrypt		=	tnepres_encrypt,
 	.cia_decrypt		=	tnepres_decrypt } }
-};
+} };
 
 static int __init serpent_mod_init(void)
 {
-	int ret = crypto_register_alg(&serpent_alg);
-
-	if (ret)
-		return ret;
-
-	ret = crypto_register_alg(&tnepres_alg);
-
-	if (ret)
-		crypto_unregister_alg(&serpent_alg);
-
-	return ret;
+	return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs));
 }
 
 static void __exit serpent_mod_fini(void)
 {
-	crypto_unregister_alg(&tnepres_alg);
-	crypto_unregister_alg(&serpent_alg);
+	crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs));
 }
 
 module_init(serpent_mod_init);
diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c
index c48459ebf05..c3ed4ec924e 100644
--- a/crypto/sha256_generic.c
+++ b/crypto/sha256_generic.c
@@ -336,7 +336,7 @@ static int sha256_import(struct shash_desc *desc, const void *in)
 	return 0;
 }
 
-static struct shash_alg sha256 = {
+static struct shash_alg sha256_algs[2] = { {
 	.digestsize	=	SHA256_DIGEST_SIZE,
 	.init		=	sha256_init,
 	.update		=	sha256_update,
@@ -352,9 +352,7 @@ static struct shash_alg sha256 = {
 		.cra_blocksize	=	SHA256_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg sha224 = {
+}, {
 	.digestsize	=	SHA224_DIGEST_SIZE,
 	.init		=	sha224_init,
 	.update		=	sha256_update,
@@ -367,29 +365,16 @@ static struct shash_alg sha224 = {
 		.cra_blocksize	=	SHA224_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
+} };
 
 static int __init sha256_generic_mod_init(void)
 {
-	int ret = 0;
-
-	ret = crypto_register_shash(&sha224);
-
-	if (ret < 0)
-		return ret;
-
-	ret = crypto_register_shash(&sha256);
-
-	if (ret < 0)
-		crypto_unregister_shash(&sha224);
-
-	return ret;
+	return crypto_register_shashes(sha256_algs, ARRAY_SIZE(sha256_algs));
 }
 
 static void __exit sha256_generic_mod_fini(void)
 {
-	crypto_unregister_shash(&sha224);
-	crypto_unregister_shash(&sha256);
+	crypto_unregister_shashes(sha256_algs, ARRAY_SIZE(sha256_algs));
 }
 
 module_init(sha256_generic_mod_init);
diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c
index dd30f40af9f..71fcf361102 100644
--- a/crypto/sha512_generic.c
+++ b/crypto/sha512_generic.c
@@ -242,7 +242,7 @@ static int sha384_final(struct shash_desc *desc, u8 *hash)
 	return 0;
 }
 
-static struct shash_alg sha512 = {
+static struct shash_alg sha512_algs[2] = { {
 	.digestsize	=	SHA512_DIGEST_SIZE,
 	.init		=	sha512_init,
 	.update		=	sha512_update,
@@ -254,9 +254,7 @@ static struct shash_alg sha512 = {
 		.cra_blocksize	=	SHA512_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg sha384 = {
+}, {
 	.digestsize	=	SHA384_DIGEST_SIZE,
 	.init		=	sha384_init,
 	.update		=	sha512_update,
@@ -268,24 +266,16 @@ static struct shash_alg sha384 = {
 		.cra_blocksize	=	SHA384_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
+} };
 
 static int __init sha512_generic_mod_init(void)
 {
-        int ret = 0;
-
-        if ((ret = crypto_register_shash(&sha384)) < 0)
-                goto out;
-        if ((ret = crypto_register_shash(&sha512)) < 0)
-                crypto_unregister_shash(&sha384);
-out:
-        return ret;
+	return crypto_register_shashes(sha512_algs, ARRAY_SIZE(sha512_algs));
 }
 
 static void __exit sha512_generic_mod_fini(void)
 {
-        crypto_unregister_shash(&sha384);
-        crypto_unregister_shash(&sha512);
+	crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs));
 }
 
 module_init(sha512_generic_mod_init);
diff --git a/crypto/shash.c b/crypto/shash.c
index 32067f47e6c..f426330f101 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -629,6 +629,42 @@ int crypto_unregister_shash(struct shash_alg *alg)
 }
 EXPORT_SYMBOL_GPL(crypto_unregister_shash);
 
+int crypto_register_shashes(struct shash_alg *algs, int count)
+{
+	int i, ret;
+
+	for (i = 0; i < count; i++) {
+		ret = crypto_register_shash(&algs[i]);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+
+err:
+	for (--i; i >= 0; --i)
+		crypto_unregister_shash(&algs[i]);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_register_shashes);
+
+int crypto_unregister_shashes(struct shash_alg *algs, int count)
+{
+	int i, ret;
+
+	for (i = count - 1; i >= 0; --i) {
+		ret = crypto_unregister_shash(&algs[i]);
+		if (ret)
+			pr_err("Failed to unregister %s %s: %d\n",
+			       algs[i].base.cra_driver_name,
+			       algs[i].base.cra_name, ret);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(crypto_unregister_shashes);
+
 int shash_register_instance(struct crypto_template *tmpl,
 			    struct shash_instance *inst)
 {
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 5cf2ccb1540..e87fa60f583 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -97,7 +97,6 @@ static int test_cipher_cycles(struct blkcipher_desc *desc, int enc,
 	int ret = 0;
 	int i;
 
-	local_bh_disable();
 	local_irq_disable();
 
 	/* Warm-up run. */
@@ -130,7 +129,6 @@ static int test_cipher_cycles(struct blkcipher_desc *desc, int enc,
 
 out:
 	local_irq_enable();
-	local_bh_enable();
 
 	if (ret == 0)
 		printk("1 operation in %lu cycles (%d bytes)\n",
@@ -300,7 +298,6 @@ static int test_hash_cycles_digest(struct hash_desc *desc,
 	int i;
 	int ret;
 
-	local_bh_disable();
 	local_irq_disable();
 
 	/* Warm-up run. */
@@ -327,7 +324,6 @@ static int test_hash_cycles_digest(struct hash_desc *desc,
 
 out:
 	local_irq_enable();
-	local_bh_enable();
 
 	if (ret)
 		return ret;
@@ -348,7 +344,6 @@ static int test_hash_cycles(struct hash_desc *desc, struct scatterlist *sg,
 	if (plen == blen)
 		return test_hash_cycles_digest(desc, sg, blen, out);
 
-	local_bh_disable();
 	local_irq_disable();
 
 	/* Warm-up run. */
@@ -391,7 +386,6 @@ static int test_hash_cycles(struct hash_desc *desc, struct scatterlist *sg,
 
 out:
 	local_irq_enable();
-	local_bh_enable();
 
 	if (ret)
 		return ret;
@@ -1037,10 +1031,16 @@ static int do_test(int m)
 
 	case 14:
 		ret += tcrypt_test("ecb(cast5)");
+		ret += tcrypt_test("cbc(cast5)");
+		ret += tcrypt_test("ctr(cast5)");
 		break;
 
 	case 15:
 		ret += tcrypt_test("ecb(cast6)");
+		ret += tcrypt_test("cbc(cast6)");
+		ret += tcrypt_test("ctr(cast6)");
+		ret += tcrypt_test("lrw(cast6)");
+		ret += tcrypt_test("xts(cast6)");
 		break;
 
 	case 16:
@@ -1112,6 +1112,9 @@ static int do_test(int m)
 	case 32:
 		ret += tcrypt_test("ecb(camellia)");
 		ret += tcrypt_test("cbc(camellia)");
+		ret += tcrypt_test("ctr(camellia)");
+		ret += tcrypt_test("lrw(camellia)");
+		ret += tcrypt_test("xts(camellia)");
 		break;
 	case 33:
 		ret += tcrypt_test("sha224");
@@ -1165,6 +1168,10 @@ static int do_test(int m)
 		ret += tcrypt_test("rfc4309(ccm(aes))");
 		break;
 
+	case 46:
+		ret += tcrypt_test("ghash");
+		break;
+
 	case 100:
 		ret += tcrypt_test("hmac(md5)");
 		break;
@@ -1359,6 +1366,44 @@ static int do_test(int m)
 				  speed_template_8);
 		break;
 
+	case 209:
+		test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
+				  speed_template_8_16);
+		break;
+
+	case 210:
+		test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
+				  speed_template_16_32);
+		test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
+				  speed_template_32_48);
+		test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
+				  speed_template_32_48);
+		test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
+				  speed_template_32_64);
+		test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
+				  speed_template_32_64);
+		break;
+
 	case 300:
 		/* fall through */
 
@@ -1639,6 +1684,44 @@ static int do_test(int m)
 				   speed_template_8);
 		break;
 
+	case 506:
+		test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
+				   speed_template_8_16);
+		break;
+
+	case 507:
+		test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
+				   speed_template_16_32);
+		test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
+				   speed_template_32_48);
+		test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
+				   speed_template_32_48);
+		test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
+				   speed_template_32_64);
+		test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
+				   speed_template_32_64);
+		break;
+
 	case 1000:
 		test_available();
 		break;
diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h
index 5be1fc8c1ab..cd2068524f3 100644
--- a/crypto/tcrypt.h
+++ b/crypto/tcrypt.h
@@ -47,6 +47,7 @@ static struct cipher_speed_template des3_speed_template[] = {
  */
 static u8 speed_template_8[] = {8, 0};
 static u8 speed_template_24[] = {24, 0};
+static u8 speed_template_8_16[] = {8, 16, 0};
 static u8 speed_template_8_32[] = {8, 32, 0};
 static u8 speed_template_16_32[] = {16, 32, 0};
 static u8 speed_template_16_24_32[] = {16, 24, 32, 0};
diff --git a/crypto/tea.c b/crypto/tea.c
index 412bc74f817..0a572323ee4 100644
--- a/crypto/tea.c
+++ b/crypto/tea.c
@@ -219,84 +219,55 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 	out[1] = cpu_to_le32(z);
 }
 
-static struct crypto_alg tea_alg = {
+static struct crypto_alg tea_algs[3] = { {
 	.cra_name		=	"tea",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	TEA_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof (struct tea_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(tea_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	TEA_KEY_SIZE,
 	.cia_max_keysize	=	TEA_KEY_SIZE,
 	.cia_setkey		= 	tea_setkey,
 	.cia_encrypt		=	tea_encrypt,
 	.cia_decrypt		=	tea_decrypt } }
-};
-
-static struct crypto_alg xtea_alg = {
+}, {
 	.cra_name		=	"xtea",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	XTEA_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof (struct xtea_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(xtea_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	XTEA_KEY_SIZE,
 	.cia_max_keysize	=	XTEA_KEY_SIZE,
 	.cia_setkey		= 	xtea_setkey,
 	.cia_encrypt		=	xtea_encrypt,
 	.cia_decrypt		=	xtea_decrypt } }
-};
-
-static struct crypto_alg xeta_alg = {
+}, {
 	.cra_name		=	"xeta",
 	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
 	.cra_blocksize		=	XTEA_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof (struct xtea_ctx),
 	.cra_alignmask		=	3,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(xtea_alg.cra_list),
 	.cra_u			=	{ .cipher = {
 	.cia_min_keysize	=	XTEA_KEY_SIZE,
 	.cia_max_keysize	=	XTEA_KEY_SIZE,
 	.cia_setkey		= 	xtea_setkey,
 	.cia_encrypt		=	xeta_encrypt,
 	.cia_decrypt		=	xeta_decrypt } }
-};
+} };
 
 static int __init tea_mod_init(void)
 {
-	int ret = 0;
-	
-	ret = crypto_register_alg(&tea_alg);
-	if (ret < 0)
-		goto out;
-
-	ret = crypto_register_alg(&xtea_alg);
-	if (ret < 0) {
-		crypto_unregister_alg(&tea_alg);
-		goto out;
-	}
-
-	ret = crypto_register_alg(&xeta_alg);
-	if (ret < 0) {
-		crypto_unregister_alg(&tea_alg);
-		crypto_unregister_alg(&xtea_alg);
-		goto out;
-	}
-
-out:	
-	return ret;
+	return crypto_register_algs(tea_algs, ARRAY_SIZE(tea_algs));
 }
 
 static void __exit tea_mod_fini(void)
 {
-	crypto_unregister_alg(&tea_alg);
-	crypto_unregister_alg(&xtea_alg);
-	crypto_unregister_alg(&xeta_alg);
+	crypto_unregister_algs(tea_algs, ARRAY_SIZE(tea_algs));
 }
 
 MODULE_ALIAS("xtea");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index a2ca7431760..941d75cd1f7 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -358,8 +358,9 @@ out_nobuf:
 	return ret;
 }
 
-static int test_aead(struct crypto_aead *tfm, int enc,
-		     struct aead_testvec *template, unsigned int tcount)
+static int __test_aead(struct crypto_aead *tfm, int enc,
+		       struct aead_testvec *template, unsigned int tcount,
+		       const bool diff_dst)
 {
 	const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm));
 	unsigned int i, j, k, n, temp;
@@ -367,15 +368,18 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 	char *q;
 	char *key;
 	struct aead_request *req;
-	struct scatterlist sg[8];
-	struct scatterlist asg[8];
-	const char *e;
+	struct scatterlist *sg;
+	struct scatterlist *asg;
+	struct scatterlist *sgout;
+	const char *e, *d;
 	struct tcrypt_result result;
 	unsigned int authsize;
 	void *input;
+	void *output;
 	void *assoc;
 	char iv[MAX_IVLEN];
 	char *xbuf[XBUFSIZE];
+	char *xoutbuf[XBUFSIZE];
 	char *axbuf[XBUFSIZE];
 
 	if (testmgr_alloc_buf(xbuf))
@@ -383,6 +387,21 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 	if (testmgr_alloc_buf(axbuf))
 		goto out_noaxbuf;
 
+	if (diff_dst && testmgr_alloc_buf(xoutbuf))
+		goto out_nooutbuf;
+
+	/* avoid "the frame size is larger than 1024 bytes" compiler warning */
+	sg = kmalloc(sizeof(*sg) * 8 * (diff_dst ? 3 : 2), GFP_KERNEL);
+	if (!sg)
+		goto out_nosg;
+	asg = &sg[8];
+	sgout = &asg[8];
+
+	if (diff_dst)
+		d = "-ddst";
+	else
+		d = "";
+
 	if (enc == ENCRYPT)
 		e = "encryption";
 	else
@@ -392,8 +411,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 
 	req = aead_request_alloc(tfm, GFP_KERNEL);
 	if (!req) {
-		printk(KERN_ERR "alg: aead: Failed to allocate request for "
-		       "%s\n", algo);
+		pr_err("alg: aead%s: Failed to allocate request for %s\n",
+		       d, algo);
 		goto out;
 	}
 
@@ -432,9 +451,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 			ret = crypto_aead_setkey(tfm, key,
 						 template[i].klen);
 			if (!ret == template[i].fail) {
-				printk(KERN_ERR "alg: aead: setkey failed on "
-				       "test %d for %s: flags=%x\n", j, algo,
-				       crypto_aead_get_flags(tfm));
+				pr_err("alg: aead%s: setkey failed on test %d for %s: flags=%x\n",
+				       d, j, algo, crypto_aead_get_flags(tfm));
 				goto out;
 			} else if (ret)
 				continue;
@@ -442,18 +460,26 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 			authsize = abs(template[i].rlen - template[i].ilen);
 			ret = crypto_aead_setauthsize(tfm, authsize);
 			if (ret) {
-				printk(KERN_ERR "alg: aead: Failed to set "
-				       "authsize to %u on test %d for %s\n",
-				       authsize, j, algo);
+				pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n",
+				       d, authsize, j, algo);
 				goto out;
 			}
 
 			sg_init_one(&sg[0], input,
 				    template[i].ilen + (enc ? authsize : 0));
 
+			if (diff_dst) {
+				output = xoutbuf[0];
+				sg_init_one(&sgout[0], output,
+					    template[i].ilen +
+						(enc ? authsize : 0));
+			} else {
+				output = input;
+			}
+
 			sg_init_one(&asg[0], assoc, template[i].alen);
 
-			aead_request_set_crypt(req, sg, sg,
+			aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
 					       template[i].ilen, iv);
 
 			aead_request_set_assoc(req, asg, template[i].alen);
@@ -466,10 +492,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 			case 0:
 				if (template[i].novrfy) {
 					/* verification was supposed to fail */
-					printk(KERN_ERR "alg: aead: %s failed "
-					       "on test %d for %s: ret was 0, "
-					       "expected -EBADMSG\n",
-					       e, j, algo);
+					pr_err("alg: aead%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n",
+					       d, e, j, algo);
 					/* so really, we got a bad message */
 					ret = -EBADMSG;
 					goto out;
@@ -489,15 +513,15 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 					continue;
 				/* fall through */
 			default:
-				printk(KERN_ERR "alg: aead: %s failed on test "
-				       "%d for %s: ret=%d\n", e, j, algo, -ret);
+				pr_err("alg: aead%s: %s failed on test %d for %s: ret=%d\n",
+				       d, e, j, algo, -ret);
 				goto out;
 			}
 
-			q = input;
+			q = output;
 			if (memcmp(q, template[i].result, template[i].rlen)) {
-				printk(KERN_ERR "alg: aead: Test %d failed on "
-				       "%s for %s\n", j, e, algo);
+				pr_err("alg: aead%s: Test %d failed on %s for %s\n",
+				       d, j, e, algo);
 				hexdump(q, template[i].rlen);
 				ret = -EINVAL;
 				goto out;
@@ -522,9 +546,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 
 			ret = crypto_aead_setkey(tfm, key, template[i].klen);
 			if (!ret == template[i].fail) {
-				printk(KERN_ERR "alg: aead: setkey failed on "
-				       "chunk test %d for %s: flags=%x\n", j,
-				       algo, crypto_aead_get_flags(tfm));
+				pr_err("alg: aead%s: setkey failed on chunk test %d for %s: flags=%x\n",
+				       d, j, algo, crypto_aead_get_flags(tfm));
 				goto out;
 			} else if (ret)
 				continue;
@@ -533,6 +556,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 
 			ret = -EINVAL;
 			sg_init_table(sg, template[i].np);
+			if (diff_dst)
+				sg_init_table(sgout, template[i].np);
 			for (k = 0, temp = 0; k < template[i].np; k++) {
 				if (WARN_ON(offset_in_page(IDX[k]) +
 					    template[i].tap[k] > PAGE_SIZE))
@@ -551,14 +576,26 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 					q[n] = 0;
 
 				sg_set_buf(&sg[k], q, template[i].tap[k]);
+
+				if (diff_dst) {
+					q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
+
+					memset(q, 0, template[i].tap[k]);
+					if (offset_in_page(q) + n < PAGE_SIZE)
+						q[n] = 0;
+
+					sg_set_buf(&sgout[k], q,
+						   template[i].tap[k]);
+				}
+
 				temp += template[i].tap[k];
 			}
 
 			ret = crypto_aead_setauthsize(tfm, authsize);
 			if (ret) {
-				printk(KERN_ERR "alg: aead: Failed to set "
-				       "authsize to %u on chunk test %d for "
-				       "%s\n", authsize, j, algo);
+				pr_err("alg: aead%s: Failed to set authsize to %u on chunk test %d for %s\n",
+				       d, authsize, j, algo);
 				goto out;
 			}
 
@@ -571,6 +608,9 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 				}
 
 				sg[k - 1].length += authsize;
+
+				if (diff_dst)
+					sgout[k - 1].length += authsize;
 			}
 
 			sg_init_table(asg, template[i].anp);
@@ -588,7 +628,7 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 				temp += template[i].atap[k];
 			}
 
-			aead_request_set_crypt(req, sg, sg,
+			aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
 					       template[i].ilen,
 					       iv);
 
@@ -602,10 +642,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 			case 0:
 				if (template[i].novrfy) {
 					/* verification was supposed to fail */
-					printk(KERN_ERR "alg: aead: %s failed "
-					       "on chunk test %d for %s: ret "
-					       "was 0, expected -EBADMSG\n",
-					       e, j, algo);
+					pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret was 0, expected -EBADMSG\n",
+					       d, e, j, algo);
 					/* so really, we got a bad message */
 					ret = -EBADMSG;
 					goto out;
@@ -625,32 +663,35 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 					continue;
 				/* fall through */
 			default:
-				printk(KERN_ERR "alg: aead: %s failed on "
-				       "chunk test %d for %s: ret=%d\n", e, j,
-				       algo, -ret);
+				pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret=%d\n",
+				       d, e, j, algo, -ret);
 				goto out;
 			}
 
 			ret = -EINVAL;
 			for (k = 0, temp = 0; k < template[i].np; k++) {
-				q = xbuf[IDX[k] >> PAGE_SHIFT] +
-				    offset_in_page(IDX[k]);
+				if (diff_dst)
+					q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
+				else
+					q = xbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
 
 				n = template[i].tap[k];
 				if (k == template[i].np - 1)
 					n += enc ? authsize : -authsize;
 
 				if (memcmp(q, template[i].result + temp, n)) {
-					printk(KERN_ERR "alg: aead: Chunk "
-					       "test %d failed on %s at page "
-					       "%u for %s\n", j, e, k, algo);
+					pr_err("alg: aead%s: Chunk test %d failed on %s at page %u for %s\n",
+					       d, j, e, k, algo);
 					hexdump(q, n);
 					goto out;
 				}
 
 				q += n;
 				if (k == template[i].np - 1 && !enc) {
-					if (memcmp(q, template[i].input +
+					if (!diff_dst &&
+						memcmp(q, template[i].input +
 						      temp + n, authsize))
 						n = authsize;
 					else
@@ -661,11 +702,8 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 						;
 				}
 				if (n) {
-					printk(KERN_ERR "alg: aead: Result "
-					       "buffer corruption in chunk "
-					       "test %d on %s at page %u for "
-					       "%s: %u bytes:\n", j, e, k,
-					       algo, n);
+					pr_err("alg: aead%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
+					       d, j, e, k, algo, n);
 					hexdump(q, n);
 					goto out;
 				}
@@ -679,6 +717,11 @@ static int test_aead(struct crypto_aead *tfm, int enc,
 
 out:
 	aead_request_free(req);
+	kfree(sg);
+out_nosg:
+	if (diff_dst)
+		testmgr_free_buf(xoutbuf);
+out_nooutbuf:
 	testmgr_free_buf(axbuf);
 out_noaxbuf:
 	testmgr_free_buf(xbuf);
@@ -686,6 +729,20 @@ out_noxbuf:
 	return ret;
 }
 
+static int test_aead(struct crypto_aead *tfm, int enc,
+		     struct aead_testvec *template, unsigned int tcount)
+{
+	int ret;
+
+	/* test 'dst == src' case */
+	ret = __test_aead(tfm, enc, template, tcount, false);
+	if (ret)
+		return ret;
+
+	/* test 'dst != src' case */
+	return __test_aead(tfm, enc, template, tcount, true);
+}
+
 static int test_cipher(struct crypto_cipher *tfm, int enc,
 		       struct cipher_testvec *template, unsigned int tcount)
 {
@@ -761,8 +818,9 @@ out_nobuf:
 	return ret;
 }
 
-static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
-			 struct cipher_testvec *template, unsigned int tcount)
+static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc,
+			   struct cipher_testvec *template, unsigned int tcount,
+			   const bool diff_dst)
 {
 	const char *algo =
 		crypto_tfm_alg_driver_name(crypto_ablkcipher_tfm(tfm));
@@ -770,16 +828,26 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 	char *q;
 	struct ablkcipher_request *req;
 	struct scatterlist sg[8];
-	const char *e;
+	struct scatterlist sgout[8];
+	const char *e, *d;
 	struct tcrypt_result result;
 	void *data;
 	char iv[MAX_IVLEN];
 	char *xbuf[XBUFSIZE];
+	char *xoutbuf[XBUFSIZE];
 	int ret = -ENOMEM;
 
 	if (testmgr_alloc_buf(xbuf))
 		goto out_nobuf;
 
+	if (diff_dst && testmgr_alloc_buf(xoutbuf))
+		goto out_nooutbuf;
+
+	if (diff_dst)
+		d = "-ddst";
+	else
+		d = "";
+
 	if (enc == ENCRYPT)
 	        e = "encryption";
 	else
@@ -789,8 +857,8 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 
 	req = ablkcipher_request_alloc(tfm, GFP_KERNEL);
 	if (!req) {
-		printk(KERN_ERR "alg: skcipher: Failed to allocate request "
-		       "for %s\n", algo);
+		pr_err("alg: skcipher%s: Failed to allocate request for %s\n",
+		       d, algo);
 		goto out;
 	}
 
@@ -804,7 +872,7 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 		else
 			memset(iv, 0, MAX_IVLEN);
 
-		if (!(template[i].np)) {
+		if (!(template[i].np) || (template[i].also_non_np)) {
 			j++;
 
 			ret = -EINVAL;
@@ -822,16 +890,21 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 			ret = crypto_ablkcipher_setkey(tfm, template[i].key,
 						       template[i].klen);
 			if (!ret == template[i].fail) {
-				printk(KERN_ERR "alg: skcipher: setkey failed "
-				       "on test %d for %s: flags=%x\n", j,
-				       algo, crypto_ablkcipher_get_flags(tfm));
+				pr_err("alg: skcipher%s: setkey failed on test %d for %s: flags=%x\n",
+				       d, j, algo,
+				       crypto_ablkcipher_get_flags(tfm));
 				goto out;
 			} else if (ret)
 				continue;
 
 			sg_init_one(&sg[0], data, template[i].ilen);
+			if (diff_dst) {
+				data = xoutbuf[0];
+				sg_init_one(&sgout[0], data, template[i].ilen);
+			}
 
-			ablkcipher_request_set_crypt(req, sg, sg,
+			ablkcipher_request_set_crypt(req, sg,
+						     (diff_dst) ? sgout : sg,
 						     template[i].ilen, iv);
 			ret = enc ?
 				crypto_ablkcipher_encrypt(req) :
@@ -850,16 +923,15 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 				}
 				/* fall through */
 			default:
-				printk(KERN_ERR "alg: skcipher: %s failed on "
-				       "test %d for %s: ret=%d\n", e, j, algo,
-				       -ret);
+				pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
+				       d, e, j, algo, -ret);
 				goto out;
 			}
 
 			q = data;
 			if (memcmp(q, template[i].result, template[i].rlen)) {
-				printk(KERN_ERR "alg: skcipher: Test %d "
-				       "failed on %s for %s\n", j, e, algo);
+				pr_err("alg: skcipher%s: Test %d failed on %s for %s\n",
+				       d, j, e, algo);
 				hexdump(q, template[i].rlen);
 				ret = -EINVAL;
 				goto out;
@@ -886,9 +958,8 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 			ret = crypto_ablkcipher_setkey(tfm, template[i].key,
 						       template[i].klen);
 			if (!ret == template[i].fail) {
-				printk(KERN_ERR "alg: skcipher: setkey failed "
-				       "on chunk test %d for %s: flags=%x\n",
-				       j, algo,
+				pr_err("alg: skcipher%s: setkey failed on chunk test %d for %s: flags=%x\n",
+				       d, j, algo,
 				       crypto_ablkcipher_get_flags(tfm));
 				goto out;
 			} else if (ret)
@@ -897,6 +968,8 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 			temp = 0;
 			ret = -EINVAL;
 			sg_init_table(sg, template[i].np);
+			if (diff_dst)
+				sg_init_table(sgout, template[i].np);
 			for (k = 0; k < template[i].np; k++) {
 				if (WARN_ON(offset_in_page(IDX[k]) +
 					    template[i].tap[k] > PAGE_SIZE))
@@ -913,11 +986,24 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 					q[template[i].tap[k]] = 0;
 
 				sg_set_buf(&sg[k], q, template[i].tap[k]);
+				if (diff_dst) {
+					q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
+
+					sg_set_buf(&sgout[k], q,
+						   template[i].tap[k]);
+
+					memset(q, 0, template[i].tap[k]);
+					if (offset_in_page(q) +
+					    template[i].tap[k] < PAGE_SIZE)
+						q[template[i].tap[k]] = 0;
+				}
 
 				temp += template[i].tap[k];
 			}
 
-			ablkcipher_request_set_crypt(req, sg, sg,
+			ablkcipher_request_set_crypt(req, sg,
+					(diff_dst) ? sgout : sg,
 					template[i].ilen, iv);
 
 			ret = enc ?
@@ -937,23 +1023,25 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 				}
 				/* fall through */
 			default:
-				printk(KERN_ERR "alg: skcipher: %s failed on "
-				       "chunk test %d for %s: ret=%d\n", e, j,
-				       algo, -ret);
+				pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
+				       d, e, j, algo, -ret);
 				goto out;
 			}
 
 			temp = 0;
 			ret = -EINVAL;
 			for (k = 0; k < template[i].np; k++) {
-				q = xbuf[IDX[k] >> PAGE_SHIFT] +
-				    offset_in_page(IDX[k]);
+				if (diff_dst)
+					q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
+				else
+					q = xbuf[IDX[k] >> PAGE_SHIFT] +
+					    offset_in_page(IDX[k]);
 
 				if (memcmp(q, template[i].result + temp,
 					   template[i].tap[k])) {
-					printk(KERN_ERR "alg: skcipher: Chunk "
-					       "test %d failed on %s at page "
-					       "%u for %s\n", j, e, k, algo);
+					pr_err("alg: skcipher%s: Chunk test %d failed on %s at page %u for %s\n",
+					       d, j, e, k, algo);
 					hexdump(q, template[i].tap[k]);
 					goto out;
 				}
@@ -962,11 +1050,8 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 				for (n = 0; offset_in_page(q + n) && q[n]; n++)
 					;
 				if (n) {
-					printk(KERN_ERR "alg: skcipher: "
-					       "Result buffer corruption in "
-					       "chunk test %d on %s at page "
-					       "%u for %s: %u bytes:\n", j, e,
-					       k, algo, n);
+					pr_err("alg: skcipher%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
+					       d, j, e, k, algo, n);
 					hexdump(q, n);
 					goto out;
 				}
@@ -979,11 +1064,28 @@ static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
 
 out:
 	ablkcipher_request_free(req);
+	if (diff_dst)
+		testmgr_free_buf(xoutbuf);
+out_nooutbuf:
 	testmgr_free_buf(xbuf);
 out_nobuf:
 	return ret;
 }
 
+static int test_skcipher(struct crypto_ablkcipher *tfm, int enc,
+			 struct cipher_testvec *template, unsigned int tcount)
+{
+	int ret;
+
+	/* test 'dst == src' case */
+	ret = __test_skcipher(tfm, enc, template, tcount, false);
+	if (ret)
+		return ret;
+
+	/* test 'dst != src' case */
+	return __test_skcipher(tfm, enc, template, tcount, true);
+}
+
 static int test_comp(struct crypto_comp *tfm, struct comp_testvec *ctemplate,
 		     struct comp_testvec *dtemplate, int ctcount, int dtcount)
 {
@@ -1534,6 +1636,36 @@ static int alg_test_null(const struct alg_test_desc *desc,
 /* Please keep this list sorted by algorithm name. */
 static const struct alg_test_desc alg_test_descs[] = {
 	{
+		.alg = "__cbc-cast5-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
+		.alg = "__cbc-cast6-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
 		.alg = "__cbc-serpent-avx",
 		.test = alg_test_null,
 		.suite = {
@@ -1595,6 +1727,36 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "__driver-cbc-cast5-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
+		.alg = "__driver-cbc-cast6-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
 		.alg = "__driver-cbc-serpent-avx",
 		.test = alg_test_null,
 		.suite = {
@@ -1656,6 +1818,36 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "__driver-ecb-cast5-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
+		.alg = "__driver-ecb-cast6-avx",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
 		.alg = "__driver-ecb-serpent-avx",
 		.test = alg_test_null,
 		.suite = {
@@ -1818,6 +2010,36 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "cbc(cast5)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast5_cbc_enc_tv_template,
+					.count = CAST5_CBC_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast5_cbc_dec_tv_template,
+					.count = CAST5_CBC_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
+		.alg = "cbc(cast6)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast6_cbc_enc_tv_template,
+					.count = CAST6_CBC_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast6_cbc_dec_tv_template,
+					.count = CAST6_CBC_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
 		.alg = "cbc(des)",
 		.test = alg_test_skcipher,
 		.suite = {
@@ -1937,6 +2159,36 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "cryptd(__driver-ecb-cast5-avx)",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
+		.alg = "cryptd(__driver-ecb-cast6-avx)",
+		.test = alg_test_null,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = NULL,
+					.count = 0
+				},
+				.dec = {
+					.vecs = NULL,
+					.count = 0
+				}
+			}
+		}
+	}, {
 		.alg = "cryptd(__driver-ecb-serpent-avx)",
 		.test = alg_test_null,
 		.suite = {
@@ -2054,6 +2306,36 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "ctr(cast5)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast5_ctr_enc_tv_template,
+					.count = CAST5_CTR_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast5_ctr_dec_tv_template,
+					.count = CAST5_CTR_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
+		.alg = "ctr(cast6)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast6_ctr_enc_tv_template,
+					.count = CAST6_CTR_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast6_ctr_dec_tv_template,
+					.count = CAST6_CTR_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
 		.alg = "ctr(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
@@ -2530,6 +2812,21 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "lrw(cast6)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast6_lrw_enc_tv_template,
+					.count = CAST6_LRW_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast6_lrw_dec_tv_template,
+					.count = CAST6_LRW_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
 		.alg = "lrw(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
@@ -2882,6 +3179,21 @@ static const struct alg_test_desc alg_test_descs[] = {
 			}
 		}
 	}, {
+		.alg = "xts(cast6)",
+		.test = alg_test_skcipher,
+		.suite = {
+			.cipher = {
+				.enc = {
+					.vecs = cast6_xts_enc_tv_template,
+					.count = CAST6_XTS_ENC_TEST_VECTORS
+				},
+				.dec = {
+					.vecs = cast6_xts_dec_tv_template,
+					.count = CAST6_XTS_DEC_TEST_VECTORS
+				}
+			}
+		}
+	}, {
 		.alg = "xts(serpent)",
 		.test = alg_test_skcipher,
 		.suite = {
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index f8179e0344e..76d7f6cc82f 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -53,6 +53,7 @@ struct cipher_testvec {
 	char *result;
 	unsigned short tap[MAX_TAP];
 	int np;
+	unsigned char also_non_np;
 	unsigned char fail;
 	unsigned char wk; /* weak key flag */
 	unsigned char klen;
@@ -2468,6 +2469,9 @@ static struct cipher_testvec bf_enc_tv_template[] = {
 			  "\xC2\xF4\x6D\xFF\xF6\xCD\x6B\x40"
 			  "\xE1\xB3\xBF\xD4\x38\x2B\xC8\x3B",
 		.rlen	= 40,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 40 - 8, 8 },
 	},
 };
 
@@ -2541,6 +2545,9 @@ static struct cipher_testvec bf_dec_tv_template[] = {
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
 			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
 		.rlen	= 40,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 40 - 8, 8 },
 	},
 };
 
@@ -2579,6 +2586,9 @@ static struct cipher_testvec bf_cbc_enc_tv_template[] = {
 			  "\x1B\xD9\x02\xB6\x48\xB0\x87\x25"
 			  "\x01\x9C\x93\x63\x51\x60\x82\xD2",
 		.rlen	= 40,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 40 - 8, 8 },
 	},
 };
 
@@ -2617,6 +2627,9 @@ static struct cipher_testvec bf_cbc_dec_tv_template[] = {
 			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
 			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
 		.rlen	= 40,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 40 - 8, 8 },
 	},
 };
 
@@ -2661,6 +2674,144 @@ static struct cipher_testvec bf_ctr_enc_tv_template[] = {
 			  "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC"
 			  "\x3D\xA7\xE9",
 		.rlen	= 43,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 43 - 8, 8 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
+		.ilen	= 504,
+		.result	= "\x5F\x58\x6E\x60\x51\x6E\xDC\x3D"
+			  "\xD1\xBB\xF7\xB7\xFD\x04\x44\x82"
+			  "\xDC\x9F\x4B\x02\xF1\xD2\x5A\x6F"
+			  "\x25\xF9\x27\x21\xF2\xD2\x9A\x01"
+			  "\xBD\xAD\x3D\x93\x87\xCA\x0D\xFE"
+			  "\xB7\x2C\x17\x1F\x42\x8C\x13\xB2"
+			  "\x62\x44\x72\xB9\x5D\xC0\xF8\x37"
+			  "\xDF\xEA\x78\x81\x8F\xA6\x34\xB2"
+			  "\x07\x09\x7C\xB9\x3A\xA0\x2B\x18"
+			  "\x34\x6A\x9D\x3D\xA5\xEB\xF4\x60"
+			  "\xF8\x98\xA2\x39\x81\x23\x6C\xA9"
+			  "\x70\xCA\xCC\x45\xD8\x1F\xDF\x44"
+			  "\x2A\x67\x7A\x88\x28\xDC\x36\x83"
+			  "\x18\xD7\x48\x43\x17\x2B\x1B\xE6"
+			  "\x0B\x82\x59\x14\x26\x67\x08\x09"
+			  "\x5B\x5D\x38\xD0\x81\xCE\x54\x2A"
+			  "\xCD\x22\x94\x42\xF5\xBA\x74\x7E"
+			  "\xD9\x00\x40\xA9\x0D\x0B\xBD\x8E"
+			  "\xC4\x8E\x5E\x17\x8F\x48\xE2\xB8"
+			  "\xF4\xCC\x19\x76\xAB\x48\x29\xAA"
+			  "\x81\xD5\xCE\xD5\x8A\x3B\xC9\x21"
+			  "\xEF\x50\x4F\x04\x02\xBF\xE1\x1F"
+			  "\x59\x28\x1A\xE4\x18\x16\xA0\x29"
+			  "\xBF\x34\xA9\x2D\x28\x83\xC0\x5E"
+			  "\xEA\x44\xC4\x6E\xAB\x24\x79\x9D"
+			  "\x2D\xA1\xE8\x55\xCA\x74\xFC\xBD"
+			  "\xFE\xDD\xDA\xA5\xFB\x34\x90\x31"
+			  "\x0E\x62\x28\x9B\xDC\xD7\xA1\xBB"
+			  "\xF0\x1A\xB3\xE2\xD0\xFA\xBD\xE8"
+			  "\x5C\x5A\x10\x67\xF6\x6A\x17\x3F"
+			  "\xC5\xE9\x09\x08\xDD\x22\x77\x42"
+			  "\x26\x6A\x6A\x7A\x3F\x87\x80\x0C"
+			  "\xF0\xFF\x15\x8E\x84\x86\xC0\x10"
+			  "\x0F\x8D\x33\x06\xB8\x72\xA4\x47"
+			  "\x6B\xED\x2E\x05\x94\x6C\x5C\x5B"
+			  "\x13\xF6\x77\xEE\x3B\x16\xDF\xC2"
+			  "\x63\x66\x07\x6D\x3F\x6C\x51\x7C"
+			  "\x1C\xAC\x80\xB6\x58\x48\xB7\x9D"
+			  "\xB4\x19\xD8\x19\x45\x66\x27\x02"
+			  "\xA1\xA9\x99\xF3\x1F\xE5\xA7\x1D"
+			  "\x31\xE7\x1B\x0D\xFF\xBB\xB5\xA1"
+			  "\xF5\x9C\x45\x1E\x18\x19\xA1\xE7"
+			  "\xC2\xF1\xBF\x68\xC3\xEC\xCF\x53"
+			  "\x67\xA6\x2B\x7D\x3C\x6D\x24\xC3"
+			  "\xE8\xE6\x07\x5A\x09\xE0\x32\xA8"
+			  "\x52\xF6\xE9\xED\x0E\xC6\x0A\x6A"
+			  "\xFC\x60\x2A\xE0\x93\xCE\xB8\x2E"
+			  "\xA2\xA8\x0E\x79\x9E\x34\x5D\x37"
+			  "\x6F\x12\xFE\x48\x7B\xE7\xB9\x22"
+			  "\x29\xE8\xD7\xBE\x5D\xD1\x8B\xD9"
+			  "\x91\x51\x4E\x71\xF2\x98\x85\x16"
+			  "\x25\x7A\x76\x8A\x51\x0E\x65\x14"
+			  "\x81\xB5\x3A\x37\xFD\xEC\xB5\x8A"
+			  "\xE1\xCF\x41\x72\x14\x29\x4C\xF0"
+			  "\x20\xD9\x9A\xC5\x66\xA4\x03\x76"
+			  "\x5B\xA4\x15\x4F\x0E\x64\x39\x40"
+			  "\x25\xF9\x20\x22\xF5\x88\xF5\xBA"
+			  "\xE4\xDF\x45\x61\xBF\x8D\x7A\x24"
+			  "\x4B\x92\x71\xD9\x2F\x77\xA7\x95"
+			  "\xA8\x7F\x61\xD5\xA4\x57\xB0\xFB"
+			  "\xB5\x77\xBA\x1C\xEE\x71\xFA\xB0"
+			  "\x16\x4C\x18\x6B\xF2\x69\xA0\x07"
+			  "\xEF\xBE\xEC\x69\xAC\xA8\x63\x9E",
+		.rlen	= 504,
 	},
 };
 
@@ -2705,6 +2856,144 @@ static struct cipher_testvec bf_ctr_dec_tv_template[] = {
 			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
 			  "\x6D\x04\x9B",
 		.rlen	= 43,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 43 - 8, 8 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x5F\x58\x6E\x60\x51\x6E\xDC\x3D"
+			  "\xD1\xBB\xF7\xB7\xFD\x04\x44\x82"
+			  "\xDC\x9F\x4B\x02\xF1\xD2\x5A\x6F"
+			  "\x25\xF9\x27\x21\xF2\xD2\x9A\x01"
+			  "\xBD\xAD\x3D\x93\x87\xCA\x0D\xFE"
+			  "\xB7\x2C\x17\x1F\x42\x8C\x13\xB2"
+			  "\x62\x44\x72\xB9\x5D\xC0\xF8\x37"
+			  "\xDF\xEA\x78\x81\x8F\xA6\x34\xB2"
+			  "\x07\x09\x7C\xB9\x3A\xA0\x2B\x18"
+			  "\x34\x6A\x9D\x3D\xA5\xEB\xF4\x60"
+			  "\xF8\x98\xA2\x39\x81\x23\x6C\xA9"
+			  "\x70\xCA\xCC\x45\xD8\x1F\xDF\x44"
+			  "\x2A\x67\x7A\x88\x28\xDC\x36\x83"
+			  "\x18\xD7\x48\x43\x17\x2B\x1B\xE6"
+			  "\x0B\x82\x59\x14\x26\x67\x08\x09"
+			  "\x5B\x5D\x38\xD0\x81\xCE\x54\x2A"
+			  "\xCD\x22\x94\x42\xF5\xBA\x74\x7E"
+			  "\xD9\x00\x40\xA9\x0D\x0B\xBD\x8E"
+			  "\xC4\x8E\x5E\x17\x8F\x48\xE2\xB8"
+			  "\xF4\xCC\x19\x76\xAB\x48\x29\xAA"
+			  "\x81\xD5\xCE\xD5\x8A\x3B\xC9\x21"
+			  "\xEF\x50\x4F\x04\x02\xBF\xE1\x1F"
+			  "\x59\x28\x1A\xE4\x18\x16\xA0\x29"
+			  "\xBF\x34\xA9\x2D\x28\x83\xC0\x5E"
+			  "\xEA\x44\xC4\x6E\xAB\x24\x79\x9D"
+			  "\x2D\xA1\xE8\x55\xCA\x74\xFC\xBD"
+			  "\xFE\xDD\xDA\xA5\xFB\x34\x90\x31"
+			  "\x0E\x62\x28\x9B\xDC\xD7\xA1\xBB"
+			  "\xF0\x1A\xB3\xE2\xD0\xFA\xBD\xE8"
+			  "\x5C\x5A\x10\x67\xF6\x6A\x17\x3F"
+			  "\xC5\xE9\x09\x08\xDD\x22\x77\x42"
+			  "\x26\x6A\x6A\x7A\x3F\x87\x80\x0C"
+			  "\xF0\xFF\x15\x8E\x84\x86\xC0\x10"
+			  "\x0F\x8D\x33\x06\xB8\x72\xA4\x47"
+			  "\x6B\xED\x2E\x05\x94\x6C\x5C\x5B"
+			  "\x13\xF6\x77\xEE\x3B\x16\xDF\xC2"
+			  "\x63\x66\x07\x6D\x3F\x6C\x51\x7C"
+			  "\x1C\xAC\x80\xB6\x58\x48\xB7\x9D"
+			  "\xB4\x19\xD8\x19\x45\x66\x27\x02"
+			  "\xA1\xA9\x99\xF3\x1F\xE5\xA7\x1D"
+			  "\x31\xE7\x1B\x0D\xFF\xBB\xB5\xA1"
+			  "\xF5\x9C\x45\x1E\x18\x19\xA1\xE7"
+			  "\xC2\xF1\xBF\x68\xC3\xEC\xCF\x53"
+			  "\x67\xA6\x2B\x7D\x3C\x6D\x24\xC3"
+			  "\xE8\xE6\x07\x5A\x09\xE0\x32\xA8"
+			  "\x52\xF6\xE9\xED\x0E\xC6\x0A\x6A"
+			  "\xFC\x60\x2A\xE0\x93\xCE\xB8\x2E"
+			  "\xA2\xA8\x0E\x79\x9E\x34\x5D\x37"
+			  "\x6F\x12\xFE\x48\x7B\xE7\xB9\x22"
+			  "\x29\xE8\xD7\xBE\x5D\xD1\x8B\xD9"
+			  "\x91\x51\x4E\x71\xF2\x98\x85\x16"
+			  "\x25\x7A\x76\x8A\x51\x0E\x65\x14"
+			  "\x81\xB5\x3A\x37\xFD\xEC\xB5\x8A"
+			  "\xE1\xCF\x41\x72\x14\x29\x4C\xF0"
+			  "\x20\xD9\x9A\xC5\x66\xA4\x03\x76"
+			  "\x5B\xA4\x15\x4F\x0E\x64\x39\x40"
+			  "\x25\xF9\x20\x22\xF5\x88\xF5\xBA"
+			  "\xE4\xDF\x45\x61\xBF\x8D\x7A\x24"
+			  "\x4B\x92\x71\xD9\x2F\x77\xA7\x95"
+			  "\xA8\x7F\x61\xD5\xA4\x57\xB0\xFB"
+			  "\xB5\x77\xBA\x1C\xEE\x71\xFA\xB0"
+			  "\x16\x4C\x18\x6B\xF2\x69\xA0\x07"
+			  "\xEF\xBE\xEC\x69\xAC\xA8\x63\x9E",
+		.ilen	= 504,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
+			  "\x2B\xC2\x59\xF0\x64\xFB\x92\x06",
+		.rlen	= 504,
 	},
 };
 
@@ -2884,6 +3173,9 @@ static struct cipher_testvec tf_enc_tv_template[] = {
 			  "\x58\x33\x9B\x78\xC7\x58\x48\x6B"
 			  "\x2C\x75\x64\xC4\xCA\xC1\x7E\xD5",
 		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -3049,6 +3341,9 @@ static struct cipher_testvec tf_dec_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
 		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -3229,6 +3524,9 @@ static struct cipher_testvec tf_cbc_enc_tv_template[] = {
 			  "\x30\x70\x56\xA4\x37\xDD\x7C\xC0"
 			  "\x0A\xA3\x30\x10\x26\x25\x41\x2C",
 		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -3409,6 +3707,9 @@ static struct cipher_testvec tf_cbc_dec_tv_template[] = {
 			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
 		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -3553,6 +3854,140 @@ static struct cipher_testvec tf_ctr_enc_tv_template[] = {
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\xEB\x44\xAF\x49\x27\xB8\xFB\x44"
+			  "\x4C\xA6\xC3\x0C\x8B\xD0\x01\x0C"
+			  "\x53\xC8\x16\x38\xDE\x40\x4F\x91"
+			  "\x25\x6D\x4C\xA0\x9A\x87\x1E\xDA"
+			  "\x88\x7E\x89\xE9\x67\x2B\x83\xA2"
+			  "\x5F\x2E\x23\x3E\x45\xB9\x77\x7B"
+			  "\xA6\x7E\x47\x36\x81\x9F\x9B\xF3"
+			  "\xE0\xF0\xD7\x47\xA9\xC8\xEF\x33"
+			  "\x0C\x43\xFE\x67\x50\x0A\x2C\x3E"
+			  "\xA0\xE1\x25\x8E\x80\x07\x4A\xC0"
+			  "\x64\x89\x9F\x6A\x27\x96\x07\xA6"
+			  "\x9B\xC8\x1B\x21\x60\xAE\x5D\x01"
+			  "\xE2\xCD\xC8\xAA\x6C\x9D\x1C\x34"
+			  "\x39\x18\x09\xA4\x82\x59\x78\xE7"
+			  "\xFC\x59\x65\xF2\x94\xFF\xFB\xE2"
+			  "\x3C\xDA\xB1\x90\x95\xBF\x91\xE3"
+			  "\xE6\x87\x31\x9E\x16\x85\xAD\xB1"
+			  "\x4C\xAE\x43\x4D\x19\x58\xB5\x5E"
+			  "\x2E\xF5\x09\xAA\x39\xF4\xC0\xB3"
+			  "\xD4\x4D\xDB\x73\x7A\xD4\xF1\xBF"
+			  "\x89\x16\x4D\x2D\xA2\x26\x33\x72"
+			  "\x18\x33\x7E\xD6\xD2\x16\xA4\x54"
+			  "\xF4\x8C\xB3\x52\xDF\x21\x9C\xEB"
+			  "\xBF\x49\xD3\xF9\x05\x06\xCB\xD2"
+			  "\xA9\xD2\x3B\x6E\x19\x8C\xBC\x19"
+			  "\xAB\x89\xD6\xD8\xCD\x56\x89\x5E"
+			  "\xAC\x00\xE3\x50\x63\x4A\x80\x9A"
+			  "\x05\xBC\x50\x39\xD3\x32\xD9\x0D"
+			  "\xE3\x20\x0D\x75\x54\xEC\xE6\x31"
+			  "\x14\xB9\x3A\x59\x00\x43\x37\x8E"
+			  "\x8C\x5A\x79\x62\x14\x76\x8A\xAE"
+			  "\x8F\xCC\xA1\x6C\x38\x78\xDD\x2D"
+			  "\x8B\x6D\xEA\xBD\x7B\x25\xFF\x60"
+			  "\xC9\x87\xB1\x79\x1E\xA5\x86\x68"
+			  "\x81\xB4\xE2\xC1\x05\x7D\x3A\x73"
+			  "\xD0\xDA\x75\x77\x9E\x05\x27\xF1"
+			  "\x08\xA9\x66\x64\x6C\xBC\x82\x17"
+			  "\x2C\x23\x5F\x62\x4D\x02\x1A\x58"
+			  "\xE7\xB7\x23\x6D\xE2\x20\xDA\xEF"
+			  "\xB4\xB3\x3F\xB2\x2B\x69\x98\x83"
+			  "\x95\x87\x13\x57\x60\xD7\xB5\xB1"
+			  "\xEE\x0A\x2F\x95\x36\x4C\x76\x5D"
+			  "\x5F\xD9\x19\xED\xB9\xA5\x48\xBF"
+			  "\xC8\xAB\x0F\x71\xCC\x61\x8E\x0A"
+			  "\xD0\x29\x44\xA8\xB9\xC1\xE8\xC8"
+			  "\xC9\xA8\x28\x81\xFB\x50\xF2\xF0"
+			  "\x26\xAE\x39\xB8\x91\xCD\xA8\xAC"
+			  "\xDE\x55\x1B\x50\x14\x53\x44\x17"
+			  "\x54\x46\xFC\xB1\xE4\x07\x6B\x9A"
+			  "\x01\x14\xF0\x2E\x2E\xDB\x46\x1B"
+			  "\x1A\x09\x97\xA9\xB6\x97\x79\x06"
+			  "\xFB\xCB\x85\xCF\xDD\xA1\x41\xB1"
+			  "\x00\xAA\xF7\xE0\x89\x73\xFB\xE5"
+			  "\xBF\x84\xDB\xC9\xCD\xC4\xA2\x0D"
+			  "\x3B\xAC\xF9\xDF\x96\xBF\x88\x23"
+			  "\x41\x67\xA1\x24\x99\x7E\xCC\x9B"
+			  "\x02\x8F\x6A\x49\xF6\x25\xBA\x7A"
+			  "\xF4\x78\xFD\x79\x62\x63\x4F\x14"
+			  "\xD6\x11\x11\x04\x05\x5F\x7E\xEA"
+			  "\x4C\xB6\xF8\xF4\x5F\x48\x52\x54"
+			  "\x94\x63\xA8\x4E\xCF\xD2\x1B\x1B"
+			  "\x22\x18\x6A\xAF\x6E\x3E\xE1\x0D",
+		.rlen	= 496,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
 		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
 			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
 		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
@@ -3683,6 +4118,9 @@ static struct cipher_testvec tf_ctr_enc_tv_template[] = {
 			  "\xC5\xC9\x7F\x9E\xCF\x33\x7A\xDF"
 			  "\x6C\x82\x9D",
 		.rlen	= 499,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 499 - 16, 16 },
 	},
 };
 
@@ -3827,6 +4265,140 @@ static struct cipher_testvec tf_ctr_dec_tv_template[] = {
 			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
 			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
 		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\xEB\x44\xAF\x49\x27\xB8\xFB\x44"
+			  "\x4C\xA6\xC3\x0C\x8B\xD0\x01\x0C"
+			  "\x53\xC8\x16\x38\xDE\x40\x4F\x91"
+			  "\x25\x6D\x4C\xA0\x9A\x87\x1E\xDA"
+			  "\x88\x7E\x89\xE9\x67\x2B\x83\xA2"
+			  "\x5F\x2E\x23\x3E\x45\xB9\x77\x7B"
+			  "\xA6\x7E\x47\x36\x81\x9F\x9B\xF3"
+			  "\xE0\xF0\xD7\x47\xA9\xC8\xEF\x33"
+			  "\x0C\x43\xFE\x67\x50\x0A\x2C\x3E"
+			  "\xA0\xE1\x25\x8E\x80\x07\x4A\xC0"
+			  "\x64\x89\x9F\x6A\x27\x96\x07\xA6"
+			  "\x9B\xC8\x1B\x21\x60\xAE\x5D\x01"
+			  "\xE2\xCD\xC8\xAA\x6C\x9D\x1C\x34"
+			  "\x39\x18\x09\xA4\x82\x59\x78\xE7"
+			  "\xFC\x59\x65\xF2\x94\xFF\xFB\xE2"
+			  "\x3C\xDA\xB1\x90\x95\xBF\x91\xE3"
+			  "\xE6\x87\x31\x9E\x16\x85\xAD\xB1"
+			  "\x4C\xAE\x43\x4D\x19\x58\xB5\x5E"
+			  "\x2E\xF5\x09\xAA\x39\xF4\xC0\xB3"
+			  "\xD4\x4D\xDB\x73\x7A\xD4\xF1\xBF"
+			  "\x89\x16\x4D\x2D\xA2\x26\x33\x72"
+			  "\x18\x33\x7E\xD6\xD2\x16\xA4\x54"
+			  "\xF4\x8C\xB3\x52\xDF\x21\x9C\xEB"
+			  "\xBF\x49\xD3\xF9\x05\x06\xCB\xD2"
+			  "\xA9\xD2\x3B\x6E\x19\x8C\xBC\x19"
+			  "\xAB\x89\xD6\xD8\xCD\x56\x89\x5E"
+			  "\xAC\x00\xE3\x50\x63\x4A\x80\x9A"
+			  "\x05\xBC\x50\x39\xD3\x32\xD9\x0D"
+			  "\xE3\x20\x0D\x75\x54\xEC\xE6\x31"
+			  "\x14\xB9\x3A\x59\x00\x43\x37\x8E"
+			  "\x8C\x5A\x79\x62\x14\x76\x8A\xAE"
+			  "\x8F\xCC\xA1\x6C\x38\x78\xDD\x2D"
+			  "\x8B\x6D\xEA\xBD\x7B\x25\xFF\x60"
+			  "\xC9\x87\xB1\x79\x1E\xA5\x86\x68"
+			  "\x81\xB4\xE2\xC1\x05\x7D\x3A\x73"
+			  "\xD0\xDA\x75\x77\x9E\x05\x27\xF1"
+			  "\x08\xA9\x66\x64\x6C\xBC\x82\x17"
+			  "\x2C\x23\x5F\x62\x4D\x02\x1A\x58"
+			  "\xE7\xB7\x23\x6D\xE2\x20\xDA\xEF"
+			  "\xB4\xB3\x3F\xB2\x2B\x69\x98\x83"
+			  "\x95\x87\x13\x57\x60\xD7\xB5\xB1"
+			  "\xEE\x0A\x2F\x95\x36\x4C\x76\x5D"
+			  "\x5F\xD9\x19\xED\xB9\xA5\x48\xBF"
+			  "\xC8\xAB\x0F\x71\xCC\x61\x8E\x0A"
+			  "\xD0\x29\x44\xA8\xB9\xC1\xE8\xC8"
+			  "\xC9\xA8\x28\x81\xFB\x50\xF2\xF0"
+			  "\x26\xAE\x39\xB8\x91\xCD\xA8\xAC"
+			  "\xDE\x55\x1B\x50\x14\x53\x44\x17"
+			  "\x54\x46\xFC\xB1\xE4\x07\x6B\x9A"
+			  "\x01\x14\xF0\x2E\x2E\xDB\x46\x1B"
+			  "\x1A\x09\x97\xA9\xB6\x97\x79\x06"
+			  "\xFB\xCB\x85\xCF\xDD\xA1\x41\xB1"
+			  "\x00\xAA\xF7\xE0\x89\x73\xFB\xE5"
+			  "\xBF\x84\xDB\xC9\xCD\xC4\xA2\x0D"
+			  "\x3B\xAC\xF9\xDF\x96\xBF\x88\x23"
+			  "\x41\x67\xA1\x24\x99\x7E\xCC\x9B"
+			  "\x02\x8F\x6A\x49\xF6\x25\xBA\x7A"
+			  "\xF4\x78\xFD\x79\x62\x63\x4F\x14"
+			  "\xD6\x11\x11\x04\x05\x5F\x7E\xEA"
+			  "\x4C\xB6\xF8\xF4\x5F\x48\x52\x54"
+			  "\x94\x63\xA8\x4E\xCF\xD2\x1B\x1B"
+			  "\x22\x18\x6A\xAF\x6E\x3E\xE1\x0D",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
 		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
 			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
 		.input	= "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
@@ -3957,6 +4529,9 @@ static struct cipher_testvec tf_ctr_dec_tv_template[] = {
 			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7"
 			  "\x2B\xC2\x59",
 		.rlen	= 499,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 499 - 16, 16 },
 	},
 };
 
@@ -4206,6 +4781,9 @@ static struct cipher_testvec tf_lrw_enc_tv_template[] = {
 			  "\x80\x18\xc4\x6c\x03\xd3\xb7\xba"
 			  "\x11\xd7\xb8\x6e\xea\xe1\x80\x30",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -4456,6 +5034,9 @@ static struct cipher_testvec tf_lrw_dec_tv_template[] = {
 			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
 			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -4795,6 +5376,9 @@ static struct cipher_testvec tf_xts_enc_tv_template[] = {
 			  "\xa4\x05\x0b\xb2\xb3\xa8\x30\x97"
 			  "\x37\x30\xe1\x91\x8d\xb3\x2a\xff",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -5135,6 +5719,9 @@ static struct cipher_testvec tf_xts_dec_tv_template[] = {
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -5242,6 +5829,9 @@ static struct cipher_testvec serpent_enc_tv_template[] = {
 			  "\x30\xD4\x2C\xF2\x8E\x06\x4B\x39"
 			  "\xB3\x12\x1D\xB3\x17\x46\xE6\xD6",
 		.rlen	= 144,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 144 - 16, 16 },
 	},
 };
 
@@ -5377,6 +5967,9 @@ static struct cipher_testvec serpent_dec_tv_template[] = {
 			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
 			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A",
 		.rlen	= 144,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 144 - 16, 16 },
 	},
 };
 
@@ -5468,6 +6061,9 @@ static struct cipher_testvec serpent_cbc_enc_tv_template[] = {
 			  "\x27\xDF\x89\x1D\x86\x3E\xF7\x5A"
 			  "\xF6\xE3\x0F\xC7\x6B\x4C\x96\x7C",
 		.rlen	= 144,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 144 - 16, 16 },
 	},
 };
 
@@ -5518,6 +6114,9 @@ static struct cipher_testvec serpent_cbc_dec_tv_template[] = {
 			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
 			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A",
 		.rlen	= 144,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 144 - 16, 16 },
 	},
 };
 
@@ -5616,6 +6215,143 @@ static struct cipher_testvec serpent_ctr_enc_tv_template[] = {
 			  "\x5D\xE1\x4F\xA1\xEA\xB3\xCA\xB9"
 			  "\xE6\xD0\x97",
 		.rlen	= 147,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 147 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\x06\x9A\xF8\xB4\x53\x88\x62\xFC"
+			  "\x68\xB8\x2E\xDF\xC1\x05\x0F\x3D"
+			  "\xAF\x4D\x95\xAE\xC4\xE9\x1C\xDC"
+			  "\xF6\x2B\x8F\x90\x89\xF6\x7E\x1A"
+			  "\xA6\xB9\xE4\xF4\xFA\xCA\xE5\x7E"
+			  "\x71\x28\x06\x4F\xE8\x08\x39\xDA"
+			  "\xA5\x0E\xC8\xC0\xB8\x16\xE5\x69"
+			  "\xE5\xCA\xEC\x4F\x63\x2C\xC0\x9B"
+			  "\x9F\x3E\x39\x79\xF0\xCD\x64\x35"
+			  "\x4A\xD3\xC8\xA9\x31\xCD\x48\x5B"
+			  "\x92\x3D\x8F\x3F\x96\xBD\xB3\x18"
+			  "\x74\x2A\x5D\x29\x3F\x57\x8F\xE2"
+			  "\x67\x9A\xE0\xE5\xD4\x4A\xE2\x47"
+			  "\xBC\xF6\xEB\x14\xF3\x8C\x20\xC2"
+			  "\x7D\xE2\x43\x81\x86\x72\x2E\xB1"
+			  "\x39\xF6\x95\xE1\x1F\xCB\x76\x33"
+			  "\x5B\x7D\x23\x0F\x3A\x67\x2A\x2F"
+			  "\xB9\x37\x9D\xDD\x1F\x16\xA1\x3C"
+			  "\x70\xFE\x52\xAA\x93\x3C\xC4\x46"
+			  "\xB1\xE5\xFF\xDA\xAF\xE2\x84\xFE"
+			  "\x25\x92\xB2\x63\xBD\x49\x77\xB4"
+			  "\x22\xA4\x6A\xD5\x04\xE0\x45\x58"
+			  "\x1C\x34\x96\x7C\x03\x0C\x13\xA2"
+			  "\x05\x22\xE2\xCB\x5A\x35\x03\x09"
+			  "\x40\xD2\x82\x05\xCA\x58\x73\xF2"
+			  "\x29\x5E\x01\x47\x13\x32\x78\xBE"
+			  "\x06\xB0\x51\xDB\x6C\x31\xA0\x1C"
+			  "\x74\xBC\x8D\x25\xDF\xF8\x65\xD1"
+			  "\x38\x35\x11\x26\x4A\xB4\x06\x32"
+			  "\xFA\xD2\x07\x77\xB3\x74\x98\x80"
+			  "\x61\x59\xA8\x9F\xF3\x6F\x2A\xBF"
+			  "\xE6\xA5\x9A\xC4\x6B\xA6\x49\x6F"
+			  "\xBC\x47\xD9\xFB\xC6\xEF\x25\x65"
+			  "\x96\xAC\x9F\xE4\x81\x4B\xD8\xBA"
+			  "\xD6\x9B\xC9\x6D\x58\x40\x81\x02"
+			  "\x73\x44\x4E\x43\x6E\x37\xBB\x11"
+			  "\xE3\xF9\xB8\x2F\xEC\x76\x34\xEA"
+			  "\x90\xCD\xB7\x2E\x0E\x32\x71\xE8"
+			  "\xBB\x4E\x0B\x98\xA4\x17\x17\x5B"
+			  "\x07\xB5\x82\x3A\xC4\xE8\x42\x51"
+			  "\x5A\x4C\x4E\x7D\xBF\xC4\xC0\x4F"
+			  "\x68\xB8\xC6\x4A\x32\x6F\x0B\xD7"
+			  "\x85\xED\x6B\xFB\x72\xD2\xA5\x8F"
+			  "\xBF\xF9\xAC\x59\x50\xA8\x08\x70"
+			  "\xEC\xBD\x0A\xBF\xE5\x87\xA1\xC2"
+			  "\x92\x14\x78\xAF\xE8\xEA\x2E\xDD"
+			  "\xC1\x03\x9A\xAA\x89\x8B\x32\x46"
+			  "\x5B\x18\x27\xBA\x46\xAA\x64\xDE"
+			  "\xE3\xD5\xA3\xFC\x7B\x5B\x61\xDB"
+			  "\x7E\xDA\xEC\x30\x17\x19\xF8\x80"
+			  "\xB5\x5E\x27\xB5\x37\x3A\x1F\x28"
+			  "\x07\x73\xC3\x63\xCE\xFF\x8C\xFE"
+			  "\x81\x4E\xF8\x24\xF3\xB8\xC7\xE8"
+			  "\x16\x9A\xCC\x58\x2F\x88\x1C\x4B"
+			  "\xBB\x33\xA2\x73\xF0\x1C\x89\x0E"
+			  "\xDC\x34\x27\x89\x98\xCE\x1C\xA2"
+			  "\xD8\xB8\x90\xBE\xEC\x72\x28\x13"
+			  "\xAC\x7B\xF1\xD0\x7F\x7A\x28\x50"
+			  "\xB7\x99\x65\x8A\xC9\xC6\x21\x34"
+			  "\x7F\x67\x9D\xB7\x2C\xCC\xF5\x17"
+			  "\x2B\x89\xAC\xB0\xD7\x1E\x47\xB0"
+			  "\x61\xAF\xD4\x63\x6D\xB8\x2D\x20",
+		.rlen	= 496,
 	},
 };
 
@@ -5714,6 +6450,143 @@ static struct cipher_testvec serpent_ctr_dec_tv_template[] = {
 			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
 			  "\xF1\x65\xFC",
 		.rlen	= 147,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 147 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x06\x9A\xF8\xB4\x53\x88\x62\xFC"
+			  "\x68\xB8\x2E\xDF\xC1\x05\x0F\x3D"
+			  "\xAF\x4D\x95\xAE\xC4\xE9\x1C\xDC"
+			  "\xF6\x2B\x8F\x90\x89\xF6\x7E\x1A"
+			  "\xA6\xB9\xE4\xF4\xFA\xCA\xE5\x7E"
+			  "\x71\x28\x06\x4F\xE8\x08\x39\xDA"
+			  "\xA5\x0E\xC8\xC0\xB8\x16\xE5\x69"
+			  "\xE5\xCA\xEC\x4F\x63\x2C\xC0\x9B"
+			  "\x9F\x3E\x39\x79\xF0\xCD\x64\x35"
+			  "\x4A\xD3\xC8\xA9\x31\xCD\x48\x5B"
+			  "\x92\x3D\x8F\x3F\x96\xBD\xB3\x18"
+			  "\x74\x2A\x5D\x29\x3F\x57\x8F\xE2"
+			  "\x67\x9A\xE0\xE5\xD4\x4A\xE2\x47"
+			  "\xBC\xF6\xEB\x14\xF3\x8C\x20\xC2"
+			  "\x7D\xE2\x43\x81\x86\x72\x2E\xB1"
+			  "\x39\xF6\x95\xE1\x1F\xCB\x76\x33"
+			  "\x5B\x7D\x23\x0F\x3A\x67\x2A\x2F"
+			  "\xB9\x37\x9D\xDD\x1F\x16\xA1\x3C"
+			  "\x70\xFE\x52\xAA\x93\x3C\xC4\x46"
+			  "\xB1\xE5\xFF\xDA\xAF\xE2\x84\xFE"
+			  "\x25\x92\xB2\x63\xBD\x49\x77\xB4"
+			  "\x22\xA4\x6A\xD5\x04\xE0\x45\x58"
+			  "\x1C\x34\x96\x7C\x03\x0C\x13\xA2"
+			  "\x05\x22\xE2\xCB\x5A\x35\x03\x09"
+			  "\x40\xD2\x82\x05\xCA\x58\x73\xF2"
+			  "\x29\x5E\x01\x47\x13\x32\x78\xBE"
+			  "\x06\xB0\x51\xDB\x6C\x31\xA0\x1C"
+			  "\x74\xBC\x8D\x25\xDF\xF8\x65\xD1"
+			  "\x38\x35\x11\x26\x4A\xB4\x06\x32"
+			  "\xFA\xD2\x07\x77\xB3\x74\x98\x80"
+			  "\x61\x59\xA8\x9F\xF3\x6F\x2A\xBF"
+			  "\xE6\xA5\x9A\xC4\x6B\xA6\x49\x6F"
+			  "\xBC\x47\xD9\xFB\xC6\xEF\x25\x65"
+			  "\x96\xAC\x9F\xE4\x81\x4B\xD8\xBA"
+			  "\xD6\x9B\xC9\x6D\x58\x40\x81\x02"
+			  "\x73\x44\x4E\x43\x6E\x37\xBB\x11"
+			  "\xE3\xF9\xB8\x2F\xEC\x76\x34\xEA"
+			  "\x90\xCD\xB7\x2E\x0E\x32\x71\xE8"
+			  "\xBB\x4E\x0B\x98\xA4\x17\x17\x5B"
+			  "\x07\xB5\x82\x3A\xC4\xE8\x42\x51"
+			  "\x5A\x4C\x4E\x7D\xBF\xC4\xC0\x4F"
+			  "\x68\xB8\xC6\x4A\x32\x6F\x0B\xD7"
+			  "\x85\xED\x6B\xFB\x72\xD2\xA5\x8F"
+			  "\xBF\xF9\xAC\x59\x50\xA8\x08\x70"
+			  "\xEC\xBD\x0A\xBF\xE5\x87\xA1\xC2"
+			  "\x92\x14\x78\xAF\xE8\xEA\x2E\xDD"
+			  "\xC1\x03\x9A\xAA\x89\x8B\x32\x46"
+			  "\x5B\x18\x27\xBA\x46\xAA\x64\xDE"
+			  "\xE3\xD5\xA3\xFC\x7B\x5B\x61\xDB"
+			  "\x7E\xDA\xEC\x30\x17\x19\xF8\x80"
+			  "\xB5\x5E\x27\xB5\x37\x3A\x1F\x28"
+			  "\x07\x73\xC3\x63\xCE\xFF\x8C\xFE"
+			  "\x81\x4E\xF8\x24\xF3\xB8\xC7\xE8"
+			  "\x16\x9A\xCC\x58\x2F\x88\x1C\x4B"
+			  "\xBB\x33\xA2\x73\xF0\x1C\x89\x0E"
+			  "\xDC\x34\x27\x89\x98\xCE\x1C\xA2"
+			  "\xD8\xB8\x90\xBE\xEC\x72\x28\x13"
+			  "\xAC\x7B\xF1\xD0\x7F\x7A\x28\x50"
+			  "\xB7\x99\x65\x8A\xC9\xC6\x21\x34"
+			  "\x7F\x67\x9D\xB7\x2C\xCC\xF5\x17"
+			  "\x2B\x89\xAC\xB0\xD7\x1E\x47\xB0"
+			  "\x61\xAF\xD4\x63\x6D\xB8\x2D\x20",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
 	},
 };
 
@@ -5963,6 +6836,9 @@ static struct cipher_testvec serpent_lrw_enc_tv_template[] = {
 			  "\x5c\xc6\x84\xfe\x7c\xcb\x26\xfd"
 			  "\xd9\x51\x0f\xd7\x94\x2f\xc5\xa7",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -6213,6 +7089,9 @@ static struct cipher_testvec serpent_lrw_dec_tv_template[] = {
 			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
 			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -6552,6 +7431,9 @@ static struct cipher_testvec serpent_xts_enc_tv_template[] = {
 			  "\xaf\x43\x0b\xc5\x20\x41\x92\x20"
 			  "\xd4\xa0\x91\x98\x11\x5f\x4d\xb1",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -6892,12 +7774,23 @@ static struct cipher_testvec serpent_xts_dec_tv_template[] = {
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
 /* Cast6 test vectors from RFC 2612 */
-#define CAST6_ENC_TEST_VECTORS	3
-#define CAST6_DEC_TEST_VECTORS  3
+#define CAST6_ENC_TEST_VECTORS		4
+#define CAST6_DEC_TEST_VECTORS		4
+#define CAST6_CBC_ENC_TEST_VECTORS	1
+#define CAST6_CBC_DEC_TEST_VECTORS	1
+#define CAST6_CTR_ENC_TEST_VECTORS	2
+#define CAST6_CTR_DEC_TEST_VECTORS	2
+#define CAST6_LRW_ENC_TEST_VECTORS	1
+#define CAST6_LRW_DEC_TEST_VECTORS	1
+#define CAST6_XTS_ENC_TEST_VECTORS	1
+#define CAST6_XTS_DEC_TEST_VECTORS	1
 
 static struct cipher_testvec cast6_enc_tv_template[] = {
 	{
@@ -6930,6 +7823,143 @@ static struct cipher_testvec cast6_enc_tv_template[] = {
 		.result	= "\x4f\x6a\x20\x38\x28\x68\x97\xb9"
 			  "\xc9\x87\x01\x36\x55\x33\x17\xfa",
 		.rlen	= 16,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\xC3\x70\x22\x32\xF5\x80\xCB\x54"
+			  "\xFC\x30\xE0\xF6\xEB\x39\x57\xA6"
+			  "\xB6\xB9\xC5\xA4\x91\x55\x14\x97"
+			  "\xC1\x20\xFF\x6C\x5C\xF0\x67\xEA"
+			  "\x2F\xED\xD8\xC9\xFB\x38\x3F\xFE"
+			  "\x93\xBE\xDC\x00\xD3\x7F\xAD\x4C"
+			  "\x5A\x08\x92\xD1\x47\x0C\xFA\x6C"
+			  "\xD0\x6A\x99\x10\x72\xF8\x47\x62"
+			  "\x81\x42\xF8\xD8\xF5\xBB\x94\x08"
+			  "\xAA\x97\xA2\x8B\x69\xB3\xD2\x7E"
+			  "\xBC\xB5\x00\x0C\xE5\x44\x4B\x58"
+			  "\xE8\x63\xDC\xB3\xC4\xE5\x23\x12"
+			  "\x5A\x72\x85\x47\x8B\xEC\x9F\x26"
+			  "\x84\xB6\xED\x10\x33\x63\x9B\x5F"
+			  "\x4D\x53\xEE\x94\x45\x8B\x60\x58"
+			  "\x86\x20\xF9\x1E\x82\x08\x3E\x58"
+			  "\x60\x1B\x34\x19\x02\xBE\x4E\x09"
+			  "\xBB\x7C\x15\xCC\x60\x27\x55\x7A"
+			  "\x12\xB8\xD8\x08\x89\x3C\xA6\xF3"
+			  "\xF1\xDD\xA7\x07\xA3\x12\x85\x28"
+			  "\xE9\x57\xAC\x80\x0C\x5C\x0F\x3A"
+			  "\x5D\xC2\x91\xC7\x90\xE4\x8C\x43"
+			  "\x92\xE4\x7C\x26\x69\x4D\x83\x68"
+			  "\x14\x96\x42\x47\xBD\xA9\xE4\x8A"
+			  "\x33\x19\xEB\x54\x8E\x0D\x4B\x6E"
+			  "\x91\x51\xB5\x36\x08\xDE\x1C\x06"
+			  "\x03\xBD\xDE\x81\x26\xF7\x99\xC2"
+			  "\xBA\xF7\x6D\x87\x0D\xE4\xA6\xCF"
+			  "\xC1\xF5\x27\x05\xB8\x02\x57\x72"
+			  "\xE6\x42\x13\x0B\xC6\x47\x05\x74"
+			  "\x24\x15\xF7\x0D\xC2\x23\x9D\xB9"
+			  "\x3C\x77\x18\x93\xBA\xB4\xFC\x8C"
+			  "\x98\x82\x67\x67\xB4\xD7\xD3\x43"
+			  "\x23\x08\x02\xB7\x9B\x99\x05\xFB"
+			  "\xD3\xB5\x00\x0A\xA9\x9D\x66\xD6"
+			  "\x2E\x49\x58\xD0\xA8\x57\x29\x7F"
+			  "\x0A\x0E\x7D\xFC\x92\x83\xCC\x67"
+			  "\xA2\xB1\x70\x3A\x8F\x87\x4A\x8D"
+			  "\x17\xE2\x58\x2B\x88\x0D\x68\x62"
+			  "\xBF\x35\xD1\x6F\xC0\xF0\x18\x62"
+			  "\xB2\xC7\x2D\x58\xC7\x16\xDE\x08"
+			  "\xEB\x84\x1D\x25\xA7\x38\x94\x06"
+			  "\x93\x9D\xF8\xFE\x88\x71\xE7\x84"
+			  "\x2C\xA0\x38\xA3\x1D\x48\xCF\x29"
+			  "\x0B\xBC\xD8\x50\x99\x1A\x26\xFB"
+			  "\x8E\x75\x3D\x73\xEB\x6A\xED\x29"
+			  "\xE0\x8E\xED\xFC\xFE\x6F\xF6\xBA"
+			  "\x41\xE2\x10\x4C\x01\x8B\x69\x2B"
+			  "\x25\x3F\x4D\x70\x7B\x92\xD6\x3B"
+			  "\xAC\xF9\x77\x18\xD9\x6A\x30\xA6"
+			  "\x2E\xFA\x30\xFF\xC8\xD5\x1D\x06"
+			  "\x59\x28\x1D\x86\x43\x04\x5D\x3B"
+			  "\x99\x4C\x04\x5A\x21\x17\x8B\x76"
+			  "\x8F\x72\xCB\xA1\x9C\x29\x4C\xC3"
+			  "\x65\xA2\x58\x2A\xC5\x66\x24\xBF"
+			  "\xBA\xE6\x0C\xDD\x34\x24\x74\xC8"
+			  "\x84\x0A\x66\x2C\xBE\x8F\x32\xA9"
+			  "\xE7\xE4\xA1\xD7\xDA\xAB\x23\x1E"
+			  "\xEB\xEE\x6C\x94\x6F\x9C\x2E\xD1"
+			  "\x49\x2C\xF3\xD4\x90\xCC\x93\x4C"
+			  "\x84\x52\x6D\x68\xDE\xC6\x64\xB2"
+			  "\x11\x74\x93\x57\xB4\x7E\xC6\x00",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -6964,6 +7994,1331 @@ static struct cipher_testvec cast6_dec_tv_template[] = {
 		.ilen	= 16,
 		.result	= zeroed_string,
 		.rlen	= 16,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\xC3\x70\x22\x32\xF5\x80\xCB\x54"
+			  "\xFC\x30\xE0\xF6\xEB\x39\x57\xA6"
+			  "\xB6\xB9\xC5\xA4\x91\x55\x14\x97"
+			  "\xC1\x20\xFF\x6C\x5C\xF0\x67\xEA"
+			  "\x2F\xED\xD8\xC9\xFB\x38\x3F\xFE"
+			  "\x93\xBE\xDC\x00\xD3\x7F\xAD\x4C"
+			  "\x5A\x08\x92\xD1\x47\x0C\xFA\x6C"
+			  "\xD0\x6A\x99\x10\x72\xF8\x47\x62"
+			  "\x81\x42\xF8\xD8\xF5\xBB\x94\x08"
+			  "\xAA\x97\xA2\x8B\x69\xB3\xD2\x7E"
+			  "\xBC\xB5\x00\x0C\xE5\x44\x4B\x58"
+			  "\xE8\x63\xDC\xB3\xC4\xE5\x23\x12"
+			  "\x5A\x72\x85\x47\x8B\xEC\x9F\x26"
+			  "\x84\xB6\xED\x10\x33\x63\x9B\x5F"
+			  "\x4D\x53\xEE\x94\x45\x8B\x60\x58"
+			  "\x86\x20\xF9\x1E\x82\x08\x3E\x58"
+			  "\x60\x1B\x34\x19\x02\xBE\x4E\x09"
+			  "\xBB\x7C\x15\xCC\x60\x27\x55\x7A"
+			  "\x12\xB8\xD8\x08\x89\x3C\xA6\xF3"
+			  "\xF1\xDD\xA7\x07\xA3\x12\x85\x28"
+			  "\xE9\x57\xAC\x80\x0C\x5C\x0F\x3A"
+			  "\x5D\xC2\x91\xC7\x90\xE4\x8C\x43"
+			  "\x92\xE4\x7C\x26\x69\x4D\x83\x68"
+			  "\x14\x96\x42\x47\xBD\xA9\xE4\x8A"
+			  "\x33\x19\xEB\x54\x8E\x0D\x4B\x6E"
+			  "\x91\x51\xB5\x36\x08\xDE\x1C\x06"
+			  "\x03\xBD\xDE\x81\x26\xF7\x99\xC2"
+			  "\xBA\xF7\x6D\x87\x0D\xE4\xA6\xCF"
+			  "\xC1\xF5\x27\x05\xB8\x02\x57\x72"
+			  "\xE6\x42\x13\x0B\xC6\x47\x05\x74"
+			  "\x24\x15\xF7\x0D\xC2\x23\x9D\xB9"
+			  "\x3C\x77\x18\x93\xBA\xB4\xFC\x8C"
+			  "\x98\x82\x67\x67\xB4\xD7\xD3\x43"
+			  "\x23\x08\x02\xB7\x9B\x99\x05\xFB"
+			  "\xD3\xB5\x00\x0A\xA9\x9D\x66\xD6"
+			  "\x2E\x49\x58\xD0\xA8\x57\x29\x7F"
+			  "\x0A\x0E\x7D\xFC\x92\x83\xCC\x67"
+			  "\xA2\xB1\x70\x3A\x8F\x87\x4A\x8D"
+			  "\x17\xE2\x58\x2B\x88\x0D\x68\x62"
+			  "\xBF\x35\xD1\x6F\xC0\xF0\x18\x62"
+			  "\xB2\xC7\x2D\x58\xC7\x16\xDE\x08"
+			  "\xEB\x84\x1D\x25\xA7\x38\x94\x06"
+			  "\x93\x9D\xF8\xFE\x88\x71\xE7\x84"
+			  "\x2C\xA0\x38\xA3\x1D\x48\xCF\x29"
+			  "\x0B\xBC\xD8\x50\x99\x1A\x26\xFB"
+			  "\x8E\x75\x3D\x73\xEB\x6A\xED\x29"
+			  "\xE0\x8E\xED\xFC\xFE\x6F\xF6\xBA"
+			  "\x41\xE2\x10\x4C\x01\x8B\x69\x2B"
+			  "\x25\x3F\x4D\x70\x7B\x92\xD6\x3B"
+			  "\xAC\xF9\x77\x18\xD9\x6A\x30\xA6"
+			  "\x2E\xFA\x30\xFF\xC8\xD5\x1D\x06"
+			  "\x59\x28\x1D\x86\x43\x04\x5D\x3B"
+			  "\x99\x4C\x04\x5A\x21\x17\x8B\x76"
+			  "\x8F\x72\xCB\xA1\x9C\x29\x4C\xC3"
+			  "\x65\xA2\x58\x2A\xC5\x66\x24\xBF"
+			  "\xBA\xE6\x0C\xDD\x34\x24\x74\xC8"
+			  "\x84\x0A\x66\x2C\xBE\x8F\x32\xA9"
+			  "\xE7\xE4\xA1\xD7\xDA\xAB\x23\x1E"
+			  "\xEB\xEE\x6C\x94\x6F\x9C\x2E\xD1"
+			  "\x49\x2C\xF3\xD4\x90\xCC\x93\x4C"
+			  "\x84\x52\x6D\x68\xDE\xC6\x64\xB2"
+			  "\x11\x74\x93\x57\xB4\x7E\xC6\x00",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_cbc_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\xDF\x77\x68\x96\xC7\xBA\xF8\xE2"
+			  "\x0E\x24\x99\x1A\xAA\xF3\xC6\x9F"
+			  "\xA0\x73\xB3\x70\xC3\x68\x64\x70"
+			  "\xAD\x33\x02\xFB\x88\x74\xAA\x78"
+			  "\xC7\x47\x1A\x18\x61\x2D\xAC\x9F"
+			  "\x7E\x6F\xDF\x05\x13\x76\xA6\x72"
+			  "\xB7\x13\x09\x0F\x7D\x38\xDF\x25"
+			  "\x4E\xFD\x50\x45\xFA\x35\x6A\xC0"
+			  "\x57\x95\xE1\x21\x26\x10\x9A\x21"
+			  "\xA1\x8A\x51\x05\xD1\xB1\x78\x35"
+			  "\x98\xF5\xAE\xC0\xC1\x8B\x94\xFF"
+			  "\xD0\x69\x3F\x42\xC2\x01\xA7\x9B"
+			  "\x23\x16\x47\x72\x81\x13\x3A\x72"
+			  "\xEC\xD9\x40\x88\x00\x9C\xB0\xA8"
+			  "\x9C\xAC\xCE\x11\x73\x7B\x63\x3E"
+			  "\xA3\x63\x98\x7D\x35\xE4\xD9\x83"
+			  "\xE2\xD0\x52\x87\x0C\x1F\xB0\xB3"
+			  "\x41\x1A\x93\x8D\x76\x31\x9F\xF2"
+			  "\xFE\x09\xA3\x8F\x22\x6A\x3B\xB9"
+			  "\x6C\x9E\xE4\xA1\xA0\xC4\xE7\xA1"
+			  "\x21\x9C\x1A\xCA\x65\xDE\x44\x03"
+			  "\x99\xF2\xD2\x39\xE3\x3F\x0F\x37"
+			  "\x53\x50\x23\xA4\x81\x6E\xDA\xFB"
+			  "\xF8\x7B\x01\xD7\xB2\x32\x9C\xB8"
+			  "\xB1\x0E\x99\x17\xB5\x38\xF9\xD7"
+			  "\x86\x2D\x6E\x94\x5C\x99\x9D\xB3"
+			  "\xD3\x63\x4B\x2A\x7D\x44\x6A\xB2"
+			  "\xC1\x03\xE6\x5A\x37\xD8\x64\x18"
+			  "\xAA\x32\xCE\x29\xED\xC0\xA2\xCB"
+			  "\x8D\xAF\xCD\xBE\x8F\xB6\xEC\xB4"
+			  "\x89\x05\x81\x6E\x71\x4F\xC3\x28"
+			  "\x10\xC1\x62\xC4\x41\xE9\xD2\x39"
+			  "\xF3\x22\x39\x12\x2C\xC2\x95\x2D"
+			  "\xBF\x93\x58\x4B\x04\xD1\x8D\x57"
+			  "\xAE\xEB\x60\x03\x56\x35\xAD\x5A"
+			  "\xE9\xC3\xFF\x4E\x31\xE1\x37\xF8"
+			  "\x7D\xEE\x65\x8A\xB6\x88\x1A\x3E"
+			  "\x07\x09\x82\xBA\xF0\x80\x8A\xD0"
+			  "\xA0\x3F\x6A\xE9\x24\x87\x19\x65"
+			  "\x73\x3F\x12\x91\x47\x54\xBA\x39"
+			  "\x30\x5B\x1E\xE5\xC2\xF9\x3F\xEF"
+			  "\xD6\x75\xF9\xB8\x7C\x8B\x05\x76"
+			  "\xEE\xB7\x08\x25\x4B\xB6\x7B\x47"
+			  "\x72\xC0\x4C\xD4\xDA\xE0\x75\xF1"
+			  "\x7C\xE8\x94\x9E\x16\x6E\xB8\x12"
+			  "\xA1\xC1\x6E\x3B\x1C\x59\x41\x2D"
+			  "\x23\xFA\x7D\x77\xB8\x46\x75\xFE"
+			  "\x4F\x10\xD3\x09\x60\xA1\x36\x96"
+			  "\x5B\xC2\xDC\x6E\x84\x7D\x9B\x14"
+			  "\x80\x21\x83\x58\x3C\x76\xFD\x28"
+			  "\x1D\xF9\x93\x13\xD7\x0E\x62\x14"
+			  "\x5A\xC5\x4E\x08\xA5\x56\xA4\x3C"
+			  "\x68\x93\x44\x70\xDF\xCF\x4A\x51"
+			  "\x0B\x81\x29\x41\xE5\x62\x4D\x36"
+			  "\xB3\xEA\x94\xA6\xB9\xDD\x3F\x09"
+			  "\x62\x34\xA0\x6A\x7E\x7D\xF5\xF6"
+			  "\x01\x91\xB4\x27\xDA\x59\xD6\x17"
+			  "\x56\x4D\x82\x62\x37\xA3\x48\x01"
+			  "\x99\x91\x77\xB2\x08\x6B\x2C\x37"
+			  "\xC5\x5C\xAD\xB6\x07\xB6\x84\xF3"
+			  "\x4D\x59\x7D\xC5\x28\x69\xFA\x92"
+			  "\x22\x46\x89\x2D\x0F\x2B\x08\x24",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_cbc_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\xDF\x77\x68\x96\xC7\xBA\xF8\xE2"
+			  "\x0E\x24\x99\x1A\xAA\xF3\xC6\x9F"
+			  "\xA0\x73\xB3\x70\xC3\x68\x64\x70"
+			  "\xAD\x33\x02\xFB\x88\x74\xAA\x78"
+			  "\xC7\x47\x1A\x18\x61\x2D\xAC\x9F"
+			  "\x7E\x6F\xDF\x05\x13\x76\xA6\x72"
+			  "\xB7\x13\x09\x0F\x7D\x38\xDF\x25"
+			  "\x4E\xFD\x50\x45\xFA\x35\x6A\xC0"
+			  "\x57\x95\xE1\x21\x26\x10\x9A\x21"
+			  "\xA1\x8A\x51\x05\xD1\xB1\x78\x35"
+			  "\x98\xF5\xAE\xC0\xC1\x8B\x94\xFF"
+			  "\xD0\x69\x3F\x42\xC2\x01\xA7\x9B"
+			  "\x23\x16\x47\x72\x81\x13\x3A\x72"
+			  "\xEC\xD9\x40\x88\x00\x9C\xB0\xA8"
+			  "\x9C\xAC\xCE\x11\x73\x7B\x63\x3E"
+			  "\xA3\x63\x98\x7D\x35\xE4\xD9\x83"
+			  "\xE2\xD0\x52\x87\x0C\x1F\xB0\xB3"
+			  "\x41\x1A\x93\x8D\x76\x31\x9F\xF2"
+			  "\xFE\x09\xA3\x8F\x22\x6A\x3B\xB9"
+			  "\x6C\x9E\xE4\xA1\xA0\xC4\xE7\xA1"
+			  "\x21\x9C\x1A\xCA\x65\xDE\x44\x03"
+			  "\x99\xF2\xD2\x39\xE3\x3F\x0F\x37"
+			  "\x53\x50\x23\xA4\x81\x6E\xDA\xFB"
+			  "\xF8\x7B\x01\xD7\xB2\x32\x9C\xB8"
+			  "\xB1\x0E\x99\x17\xB5\x38\xF9\xD7"
+			  "\x86\x2D\x6E\x94\x5C\x99\x9D\xB3"
+			  "\xD3\x63\x4B\x2A\x7D\x44\x6A\xB2"
+			  "\xC1\x03\xE6\x5A\x37\xD8\x64\x18"
+			  "\xAA\x32\xCE\x29\xED\xC0\xA2\xCB"
+			  "\x8D\xAF\xCD\xBE\x8F\xB6\xEC\xB4"
+			  "\x89\x05\x81\x6E\x71\x4F\xC3\x28"
+			  "\x10\xC1\x62\xC4\x41\xE9\xD2\x39"
+			  "\xF3\x22\x39\x12\x2C\xC2\x95\x2D"
+			  "\xBF\x93\x58\x4B\x04\xD1\x8D\x57"
+			  "\xAE\xEB\x60\x03\x56\x35\xAD\x5A"
+			  "\xE9\xC3\xFF\x4E\x31\xE1\x37\xF8"
+			  "\x7D\xEE\x65\x8A\xB6\x88\x1A\x3E"
+			  "\x07\x09\x82\xBA\xF0\x80\x8A\xD0"
+			  "\xA0\x3F\x6A\xE9\x24\x87\x19\x65"
+			  "\x73\x3F\x12\x91\x47\x54\xBA\x39"
+			  "\x30\x5B\x1E\xE5\xC2\xF9\x3F\xEF"
+			  "\xD6\x75\xF9\xB8\x7C\x8B\x05\x76"
+			  "\xEE\xB7\x08\x25\x4B\xB6\x7B\x47"
+			  "\x72\xC0\x4C\xD4\xDA\xE0\x75\xF1"
+			  "\x7C\xE8\x94\x9E\x16\x6E\xB8\x12"
+			  "\xA1\xC1\x6E\x3B\x1C\x59\x41\x2D"
+			  "\x23\xFA\x7D\x77\xB8\x46\x75\xFE"
+			  "\x4F\x10\xD3\x09\x60\xA1\x36\x96"
+			  "\x5B\xC2\xDC\x6E\x84\x7D\x9B\x14"
+			  "\x80\x21\x83\x58\x3C\x76\xFD\x28"
+			  "\x1D\xF9\x93\x13\xD7\x0E\x62\x14"
+			  "\x5A\xC5\x4E\x08\xA5\x56\xA4\x3C"
+			  "\x68\x93\x44\x70\xDF\xCF\x4A\x51"
+			  "\x0B\x81\x29\x41\xE5\x62\x4D\x36"
+			  "\xB3\xEA\x94\xA6\xB9\xDD\x3F\x09"
+			  "\x62\x34\xA0\x6A\x7E\x7D\xF5\xF6"
+			  "\x01\x91\xB4\x27\xDA\x59\xD6\x17"
+			  "\x56\x4D\x82\x62\x37\xA3\x48\x01"
+			  "\x99\x91\x77\xB2\x08\x6B\x2C\x37"
+			  "\xC5\x5C\xAD\xB6\x07\xB6\x84\xF3"
+			  "\x4D\x59\x7D\xC5\x28\x69\xFA\x92"
+			  "\x22\x46\x89\x2D\x0F\x2B\x08\x24",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_ctr_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.ilen	= 17,
+		.result	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57",
+		.rlen	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57\xA3\xEF\x47\x2A\xE8\x88\xA7"
+			  "\x3C\xD0\xEC\xB9\x94\x50\x7D\x56"
+			  "\xBC\xE1\xC1\xF5\xE1\xEE\x12\xF8"
+			  "\x4F\x03\x82\x3A\x93\x6B\x4C\xD3"
+			  "\xE3\xF3\xFA\xC2\x23\x55\x98\x20"
+			  "\x49\x76\x9B\x6B\xC1\x23\xBF\xE5"
+			  "\xD4\xC4\x2F\x61\xE1\x67\x2A\x30"
+			  "\x6F\x29\xCA\x54\xF8\x1B\xA6\x7D"
+			  "\x66\x45\xEE\xC8\x19\xBE\x50\xF0"
+			  "\x5F\x65\xF8\x1E\x4D\x07\x87\xD9"
+			  "\xD3\xD9\x1B\x09\x89\xFD\x42\xC5"
+			  "\xDB\xEB\x86\xF1\x67\x04\x0F\x5C"
+			  "\x81\xDF\x82\x12\xC7\x4C\x1B\x07"
+			  "\xDE\xE6\xFA\x29\x86\xD1\xB0\xBA"
+			  "\x3D\x6A\x69\x76\xEC\x0F\xB4\xE6"
+			  "\xCD\xA7\xF8\xA8\xB8\xE0\x33\xF5"
+			  "\x49\x61\x22\x52\x64\x8C\x46\x41"
+			  "\x1F\x48\x5F\x4F\xA2\x89\x36\x17"
+			  "\x20\xF8\x2F\x8F\x4B\xFA\xF2\xC0"
+			  "\x1E\x18\xA2\xF8\xB7\x6D\x98\xE3"
+			  "\x00\x14\x15\x59\xC1\x30\x64\xAF"
+			  "\xA8\x01\x38\xAB\xD4\x8B\xEC\x7C"
+			  "\x44\x9A\xC6\x2C\x2E\x2B\x2B\xF4"
+			  "\x02\x37\xC4\x69\xEF\x36\xC1\xF3"
+			  "\xA0\xFB\xFE\x29\xAD\x39\xCF\xD0"
+			  "\x51\x73\xA3\x22\x42\x41\xAB\xD2"
+			  "\x0F\x50\x14\xB9\x54\xD3\xD4\xFA"
+			  "\xBF\xC9\xBB\xCE\xC4\x1D\x2D\xAF"
+			  "\xC9\x3F\x07\x87\x42\x4B\x3A\x54"
+			  "\x34\x8E\x37\xA3\x03\x6F\x65\x66"
+			  "\xDB\x44\xC3\xE8\xD7\xDD\x7D\xDD"
+			  "\x61\xB4\x2B\x80\xA3\x98\x13\xF5"
+			  "\x5A\xD3\x34\x58\xC3\x6E\xF6\xB8"
+			  "\x0A\xC6\x50\x01\x8E\xD5\x6C\x7D"
+			  "\xFE\x16\xB6\xCF\xFC\x51\x40\xAE"
+			  "\xB3\x15\xAC\x90\x6F\x0B\x28\x3A"
+			  "\x60\x40\x38\x90\x20\x46\xC7\xB3"
+			  "\x0B\x12\x6D\x3B\x15\x14\xF9\xF4"
+			  "\x11\x41\x76\x6B\xB3\x60\x82\x3C"
+			  "\x84\xFB\x08\x2E\x92\x25\xCB\x79"
+			  "\x6F\x58\xC5\x94\x00\x00\x47\xB6"
+			  "\x9E\xDC\x0F\x29\x70\x46\x20\x76"
+			  "\x65\x75\x66\x5C\x00\x96\xB3\xE1"
+			  "\x0B\xA7\x11\x8B\x2E\x61\x4E\x45"
+			  "\x73\xFC\x91\xAB\x79\x41\x23\x14"
+			  "\x13\xB6\x72\x6C\x46\xB3\x03\x11"
+			  "\xE4\xF1\xEE\xC9\x7A\xCF\x96\x32"
+			  "\xB6\xF0\x8B\x97\xB4\xCF\x82\xB7"
+			  "\x15\x48\x44\x99\x09\xF6\xE0\xD7"
+			  "\xBC\xF1\x5B\x91\x4F\x30\x22\xA2"
+			  "\x45\xC4\x68\x55\xC2\xBE\xA7\xD2"
+			  "\x12\x53\x35\x9C\xF9\xE7\x35\x5D"
+			  "\x81\xE4\x86\x42\xC3\x58\xFB\xF0"
+			  "\x38\x9B\x8E\x5A\xEF\x83\x33\x0F"
+			  "\x00\x4E\x3F\x9F\xF5\x84\x62\xC4"
+			  "\x19\x35\x88\x22\x45\x59\x0E\x8F"
+			  "\xEC\x27\xDD\x4A\xA4\x1F\xBC\x41"
+			  "\x9B\x66\x8D\x32\xBA\x81\x34\x87"
+			  "\x0E\x74\x33\x30\x62\xB9\x89\xDF"
+			  "\xF9\xC5\xDD\x27\xB3\x39\xCB\xCB",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_ctr_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57",
+		.ilen	= 17,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.rlen	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+			  "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+		.input	= "\x26\x0A\xF1\xE2\x3F\x8A\xEF\xA3"
+			  "\x53\x9A\x5E\x1B\x2A\x1A\xC6\x0A"
+			  "\x57\xA3\xEF\x47\x2A\xE8\x88\xA7"
+			  "\x3C\xD0\xEC\xB9\x94\x50\x7D\x56"
+			  "\xBC\xE1\xC1\xF5\xE1\xEE\x12\xF8"
+			  "\x4F\x03\x82\x3A\x93\x6B\x4C\xD3"
+			  "\xE3\xF3\xFA\xC2\x23\x55\x98\x20"
+			  "\x49\x76\x9B\x6B\xC1\x23\xBF\xE5"
+			  "\xD4\xC4\x2F\x61\xE1\x67\x2A\x30"
+			  "\x6F\x29\xCA\x54\xF8\x1B\xA6\x7D"
+			  "\x66\x45\xEE\xC8\x19\xBE\x50\xF0"
+			  "\x5F\x65\xF8\x1E\x4D\x07\x87\xD9"
+			  "\xD3\xD9\x1B\x09\x89\xFD\x42\xC5"
+			  "\xDB\xEB\x86\xF1\x67\x04\x0F\x5C"
+			  "\x81\xDF\x82\x12\xC7\x4C\x1B\x07"
+			  "\xDE\xE6\xFA\x29\x86\xD1\xB0\xBA"
+			  "\x3D\x6A\x69\x76\xEC\x0F\xB4\xE6"
+			  "\xCD\xA7\xF8\xA8\xB8\xE0\x33\xF5"
+			  "\x49\x61\x22\x52\x64\x8C\x46\x41"
+			  "\x1F\x48\x5F\x4F\xA2\x89\x36\x17"
+			  "\x20\xF8\x2F\x8F\x4B\xFA\xF2\xC0"
+			  "\x1E\x18\xA2\xF8\xB7\x6D\x98\xE3"
+			  "\x00\x14\x15\x59\xC1\x30\x64\xAF"
+			  "\xA8\x01\x38\xAB\xD4\x8B\xEC\x7C"
+			  "\x44\x9A\xC6\x2C\x2E\x2B\x2B\xF4"
+			  "\x02\x37\xC4\x69\xEF\x36\xC1\xF3"
+			  "\xA0\xFB\xFE\x29\xAD\x39\xCF\xD0"
+			  "\x51\x73\xA3\x22\x42\x41\xAB\xD2"
+			  "\x0F\x50\x14\xB9\x54\xD3\xD4\xFA"
+			  "\xBF\xC9\xBB\xCE\xC4\x1D\x2D\xAF"
+			  "\xC9\x3F\x07\x87\x42\x4B\x3A\x54"
+			  "\x34\x8E\x37\xA3\x03\x6F\x65\x66"
+			  "\xDB\x44\xC3\xE8\xD7\xDD\x7D\xDD"
+			  "\x61\xB4\x2B\x80\xA3\x98\x13\xF5"
+			  "\x5A\xD3\x34\x58\xC3\x6E\xF6\xB8"
+			  "\x0A\xC6\x50\x01\x8E\xD5\x6C\x7D"
+			  "\xFE\x16\xB6\xCF\xFC\x51\x40\xAE"
+			  "\xB3\x15\xAC\x90\x6F\x0B\x28\x3A"
+			  "\x60\x40\x38\x90\x20\x46\xC7\xB3"
+			  "\x0B\x12\x6D\x3B\x15\x14\xF9\xF4"
+			  "\x11\x41\x76\x6B\xB3\x60\x82\x3C"
+			  "\x84\xFB\x08\x2E\x92\x25\xCB\x79"
+			  "\x6F\x58\xC5\x94\x00\x00\x47\xB6"
+			  "\x9E\xDC\x0F\x29\x70\x46\x20\x76"
+			  "\x65\x75\x66\x5C\x00\x96\xB3\xE1"
+			  "\x0B\xA7\x11\x8B\x2E\x61\x4E\x45"
+			  "\x73\xFC\x91\xAB\x79\x41\x23\x14"
+			  "\x13\xB6\x72\x6C\x46\xB3\x03\x11"
+			  "\xE4\xF1\xEE\xC9\x7A\xCF\x96\x32"
+			  "\xB6\xF0\x8B\x97\xB4\xCF\x82\xB7"
+			  "\x15\x48\x44\x99\x09\xF6\xE0\xD7"
+			  "\xBC\xF1\x5B\x91\x4F\x30\x22\xA2"
+			  "\x45\xC4\x68\x55\xC2\xBE\xA7\xD2"
+			  "\x12\x53\x35\x9C\xF9\xE7\x35\x5D"
+			  "\x81\xE4\x86\x42\xC3\x58\xFB\xF0"
+			  "\x38\x9B\x8E\x5A\xEF\x83\x33\x0F"
+			  "\x00\x4E\x3F\x9F\xF5\x84\x62\xC4"
+			  "\x19\x35\x88\x22\x45\x59\x0E\x8F"
+			  "\xEC\x27\xDD\x4A\xA4\x1F\xBC\x41"
+			  "\x9B\x66\x8D\x32\xBA\x81\x34\x87"
+			  "\x0E\x74\x33\x30\x62\xB9\x89\xDF"
+			  "\xF9\xC5\xDD\x27\xB3\x39\xCB\xCB",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_lrw_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.ilen	= 512,
+		.result	= "\x55\x25\x09\x8B\xB5\xD5\xF8\xBF"
+			  "\x37\x4A\xFE\x3C\x47\xD8\xE6\xEB"
+			  "\xCA\xA4\x9B\xB0\xAB\x6D\x64\xCA"
+			  "\x58\xB6\x73\xF0\xD7\x52\x34\xEF"
+			  "\xFB\x3E\x96\x81\xB7\x71\x34\xA4"
+			  "\x55\x20\xBE\x39\x5A\x2B\xF9\xD1"
+			  "\x65\x0B\xDA\xD3\x7E\xB3\xA6\xF7"
+			  "\x2E\x0B\x5A\x52\xDB\x39\x8C\x9B"
+			  "\x61\x17\x5F\xAF\xB6\x5A\xC8\x08"
+			  "\xA7\xB7\x2A\x11\x7C\x97\x38\x9D"
+			  "\x59\x0E\x66\x59\x5E\xD8\x8B\xCE"
+			  "\x70\xE0\xC3\x42\xB0\x8C\x0F\xBA"
+			  "\xB2\x0D\x81\xB6\xBE\x61\x1C\x2D"
+			  "\x7E\xEA\x91\x25\xAC\xEC\xF8\x28"
+			  "\x80\x1D\xF0\x30\xBA\x62\x77\x7D"
+			  "\xDB\x15\x69\xDF\xFA\x2A\x81\x64"
+			  "\x95\x5B\xA4\x7F\x3E\x4F\xE3\x30"
+			  "\xB0\x5C\xC2\x05\xF8\xF0\x29\xE7"
+			  "\x0A\xA0\x66\xB2\x5D\x0F\x39\x2B"
+			  "\xB4\xB3\x00\xA9\xD0\xAB\x63\x61"
+			  "\x5E\xDB\xFC\x11\x74\x25\x96\x65"
+			  "\xE8\xE2\x34\x57\x77\x15\x5E\x70"
+			  "\xFF\x10\x90\xC3\x64\xF0\x11\x0A"
+			  "\x63\x3A\xD3\x55\x92\x15\x4B\x0C"
+			  "\xC7\x08\x89\x17\x3B\x99\xAD\x63"
+			  "\xE7\x06\xDF\x52\xBC\x15\x64\x45"
+			  "\x9D\x7A\xFB\x69\xBC\x2D\x6E\xA9"
+			  "\x35\xD9\xD8\xF5\x0C\xC4\xA2\x23"
+			  "\x9C\x18\x8B\xA8\x8C\xFE\xF8\x0E"
+			  "\xBD\xAB\x60\x1A\x51\x17\x54\x27"
+			  "\xB6\xE8\xBE\x0F\xA9\xA5\x82\x19"
+			  "\x2F\x6F\x20\xA7\x47\xED\x74\x6C"
+			  "\x4E\xC1\xF8\x8C\x14\xF3\xBB\x1F"
+			  "\xED\x4D\x8F\x7C\x37\xEF\x19\xA1"
+			  "\x07\x16\xDE\x76\xCC\x5E\x94\x02"
+			  "\xFB\xBF\xE4\x81\x50\xCE\xFC\x0F"
+			  "\x9E\xCF\x3D\xF6\x67\x00\xBF\xA7"
+			  "\x6E\x21\x58\x36\x06\xDE\xB3\xD4"
+			  "\xA2\xFA\xD8\x4E\xE0\xB9\x7F\x23"
+			  "\x51\x21\x2B\x32\x68\xAA\xF8\xA8"
+			  "\x93\x08\xB5\x6D\xE6\x43\x2C\xB7"
+			  "\x31\xB2\x0F\xD0\xA2\x51\xC0\x25"
+			  "\x30\xC7\x10\x3F\x97\x27\x01\x8E"
+			  "\xFA\xD8\x4F\x78\xD8\x2E\x1D\xEB"
+			  "\xA1\x37\x52\x0F\x7B\x5E\x87\xA8"
+			  "\x22\xE2\xE6\x92\xA7\x5F\x11\x32"
+			  "\xCC\x93\x34\xFC\xD1\x7E\xAE\x54"
+			  "\xBC\x6A\x1B\x91\xD1\x2E\x21\xEC"
+			  "\x5D\xF1\xC4\xF1\x55\x20\xBF\xE5"
+			  "\x96\x3D\x69\x91\x20\x4E\xF2\x61"
+			  "\xDA\x77\xFE\xEE\xC3\x74\x57\x2A"
+			  "\x78\x39\xB0\xE0\xCF\x12\x56\xD6"
+			  "\x05\xDC\xF9\x19\x66\x44\x1D\xF9"
+			  "\x82\x37\xD4\xC2\x60\xB6\x31\xDF"
+			  "\x0C\xAF\xBC\x8B\x55\x9A\xC8\x2D"
+			  "\xAB\xA7\x88\x7B\x41\xE8\x29\xC9"
+			  "\x9B\x8D\xA7\x00\x86\x25\xB6\x14"
+			  "\xF5\x13\x73\xD7\x4B\x6B\x83\xF3"
+			  "\xAF\x96\x00\xE4\xB7\x3C\x65\xA6"
+			  "\x15\xB7\x94\x7D\x4E\x70\x4C\x75"
+			  "\xF3\xB4\x02\xA9\x17\x1C\x7A\x0A"
+			  "\xC0\xD5\x33\x11\x56\xDE\xDC\xF5"
+			  "\x8D\xD9\xCD\x3B\x22\x67\x18\xC7"
+			  "\xC4\xF5\x99\x61\xBC\xBB\x5B\x46",
+		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_lrw_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\xf8\xd4\x76\xff\xd6\x46\xee\x6c"
+			  "\x23\x84\xcb\x1c\x77\xd6\x19\x5d"
+			  "\xfe\xf1\xa9\xf3\x7b\xbc\x8d\x21"
+			  "\xa7\x9c\x21\xf8\xcb\x90\x02\x89"
+			  "\xa8\x45\x34\x8e\xc8\xc5\xb5\xf1"
+			  "\x26\xf5\x0e\x76\xfe\xfd\x1b\x1e",
+		.klen	= 48,
+		.iv	= "\x00\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x01",
+		.input	= "\x55\x25\x09\x8B\xB5\xD5\xF8\xBF"
+			  "\x37\x4A\xFE\x3C\x47\xD8\xE6\xEB"
+			  "\xCA\xA4\x9B\xB0\xAB\x6D\x64\xCA"
+			  "\x58\xB6\x73\xF0\xD7\x52\x34\xEF"
+			  "\xFB\x3E\x96\x81\xB7\x71\x34\xA4"
+			  "\x55\x20\xBE\x39\x5A\x2B\xF9\xD1"
+			  "\x65\x0B\xDA\xD3\x7E\xB3\xA6\xF7"
+			  "\x2E\x0B\x5A\x52\xDB\x39\x8C\x9B"
+			  "\x61\x17\x5F\xAF\xB6\x5A\xC8\x08"
+			  "\xA7\xB7\x2A\x11\x7C\x97\x38\x9D"
+			  "\x59\x0E\x66\x59\x5E\xD8\x8B\xCE"
+			  "\x70\xE0\xC3\x42\xB0\x8C\x0F\xBA"
+			  "\xB2\x0D\x81\xB6\xBE\x61\x1C\x2D"
+			  "\x7E\xEA\x91\x25\xAC\xEC\xF8\x28"
+			  "\x80\x1D\xF0\x30\xBA\x62\x77\x7D"
+			  "\xDB\x15\x69\xDF\xFA\x2A\x81\x64"
+			  "\x95\x5B\xA4\x7F\x3E\x4F\xE3\x30"
+			  "\xB0\x5C\xC2\x05\xF8\xF0\x29\xE7"
+			  "\x0A\xA0\x66\xB2\x5D\x0F\x39\x2B"
+			  "\xB4\xB3\x00\xA9\xD0\xAB\x63\x61"
+			  "\x5E\xDB\xFC\x11\x74\x25\x96\x65"
+			  "\xE8\xE2\x34\x57\x77\x15\x5E\x70"
+			  "\xFF\x10\x90\xC3\x64\xF0\x11\x0A"
+			  "\x63\x3A\xD3\x55\x92\x15\x4B\x0C"
+			  "\xC7\x08\x89\x17\x3B\x99\xAD\x63"
+			  "\xE7\x06\xDF\x52\xBC\x15\x64\x45"
+			  "\x9D\x7A\xFB\x69\xBC\x2D\x6E\xA9"
+			  "\x35\xD9\xD8\xF5\x0C\xC4\xA2\x23"
+			  "\x9C\x18\x8B\xA8\x8C\xFE\xF8\x0E"
+			  "\xBD\xAB\x60\x1A\x51\x17\x54\x27"
+			  "\xB6\xE8\xBE\x0F\xA9\xA5\x82\x19"
+			  "\x2F\x6F\x20\xA7\x47\xED\x74\x6C"
+			  "\x4E\xC1\xF8\x8C\x14\xF3\xBB\x1F"
+			  "\xED\x4D\x8F\x7C\x37\xEF\x19\xA1"
+			  "\x07\x16\xDE\x76\xCC\x5E\x94\x02"
+			  "\xFB\xBF\xE4\x81\x50\xCE\xFC\x0F"
+			  "\x9E\xCF\x3D\xF6\x67\x00\xBF\xA7"
+			  "\x6E\x21\x58\x36\x06\xDE\xB3\xD4"
+			  "\xA2\xFA\xD8\x4E\xE0\xB9\x7F\x23"
+			  "\x51\x21\x2B\x32\x68\xAA\xF8\xA8"
+			  "\x93\x08\xB5\x6D\xE6\x43\x2C\xB7"
+			  "\x31\xB2\x0F\xD0\xA2\x51\xC0\x25"
+			  "\x30\xC7\x10\x3F\x97\x27\x01\x8E"
+			  "\xFA\xD8\x4F\x78\xD8\x2E\x1D\xEB"
+			  "\xA1\x37\x52\x0F\x7B\x5E\x87\xA8"
+			  "\x22\xE2\xE6\x92\xA7\x5F\x11\x32"
+			  "\xCC\x93\x34\xFC\xD1\x7E\xAE\x54"
+			  "\xBC\x6A\x1B\x91\xD1\x2E\x21\xEC"
+			  "\x5D\xF1\xC4\xF1\x55\x20\xBF\xE5"
+			  "\x96\x3D\x69\x91\x20\x4E\xF2\x61"
+			  "\xDA\x77\xFE\xEE\xC3\x74\x57\x2A"
+			  "\x78\x39\xB0\xE0\xCF\x12\x56\xD6"
+			  "\x05\xDC\xF9\x19\x66\x44\x1D\xF9"
+			  "\x82\x37\xD4\xC2\x60\xB6\x31\xDF"
+			  "\x0C\xAF\xBC\x8B\x55\x9A\xC8\x2D"
+			  "\xAB\xA7\x88\x7B\x41\xE8\x29\xC9"
+			  "\x9B\x8D\xA7\x00\x86\x25\xB6\x14"
+			  "\xF5\x13\x73\xD7\x4B\x6B\x83\xF3"
+			  "\xAF\x96\x00\xE4\xB7\x3C\x65\xA6"
+			  "\x15\xB7\x94\x7D\x4E\x70\x4C\x75"
+			  "\xF3\xB4\x02\xA9\x17\x1C\x7A\x0A"
+			  "\xC0\xD5\x33\x11\x56\xDE\xDC\xF5"
+			  "\x8D\xD9\xCD\x3B\x22\x67\x18\xC7"
+			  "\xC4\xF5\x99\x61\xBC\xBB\x5B\x46",
+		.ilen	= 512,
+		.result	= "\x05\x11\xb7\x18\xab\xc6\x2d\xac"
+			  "\x70\x5d\xf6\x22\x94\xcd\xe5\x6c"
+			  "\x17\x6b\xf6\x1c\xf0\xf3\x6e\xf8"
+			  "\x50\x38\x1f\x71\x49\xb6\x57\xd6"
+			  "\x8f\xcb\x8d\x6b\xe3\xa6\x29\x90"
+			  "\xfe\x2a\x62\x82\xae\x6d\x8b\xf6"
+			  "\xad\x1e\x9e\x20\x5f\x38\xbe\x04"
+			  "\xda\x10\x8e\xed\xa2\xa4\x87\xab"
+			  "\xda\x6b\xb4\x0c\x75\xba\xd3\x7c"
+			  "\xc9\xac\x42\x31\x95\x7c\xc9\x04"
+			  "\xeb\xd5\x6e\x32\x69\x8a\xdb\xa6"
+			  "\x15\xd7\x3f\x4f\x2f\x66\x69\x03"
+			  "\x9c\x1f\x54\x0f\xde\x1f\xf3\x65"
+			  "\x4c\x96\x12\xed\x7c\x92\x03\x01"
+			  "\x6f\xbc\x35\x93\xac\xf1\x27\xf1"
+			  "\xb4\x96\x82\x5a\x5f\xb0\xa0\x50"
+			  "\x89\xa4\x8e\x66\x44\x85\xcc\xfd"
+			  "\x33\x14\x70\xe3\x96\xb2\xc3\xd3"
+			  "\xbb\x54\x5a\x1a\xf9\x74\xa2\xc5"
+			  "\x2d\x64\x75\xdd\xb4\x54\xe6\x74"
+			  "\x8c\xd3\x9d\x9e\x86\xab\x51\x53"
+			  "\xb7\x93\x3e\x6f\xd0\x4e\x2c\x40"
+			  "\xf6\xa8\x2e\x3e\x9d\xf4\x66\xa5"
+			  "\x76\x12\x73\x44\x1a\x56\xd7\x72"
+			  "\x88\xcd\x21\x8c\x4c\x0f\xfe\xda"
+			  "\x95\xe0\x3a\xa6\xa5\x84\x46\xcd"
+			  "\xd5\x3e\x9d\x3a\xe2\x67\xe6\x60"
+			  "\x1a\xe2\x70\x85\x58\xc2\x1b\x09"
+			  "\xe1\xd7\x2c\xca\xad\xa8\x8f\xf9"
+			  "\xac\xb3\x0e\xdb\xca\x2e\xe2\xb8"
+			  "\x51\x71\xd9\x3c\x6c\xf1\x56\xf8"
+			  "\xea\x9c\xf1\xfb\x0c\xe6\xb7\x10"
+			  "\x1c\xf8\xa9\x7c\xe8\x53\x35\xc1"
+			  "\x90\x3e\x76\x4a\x74\xa4\x21\x2c"
+			  "\xf6\x2c\x4e\x0f\x94\x3a\x88\x2e"
+			  "\x41\x09\x6a\x33\x7d\xf6\xdd\x3f"
+			  "\x8d\x23\x31\x74\x84\xeb\x88\x6e"
+			  "\xcc\xb9\xbc\x22\x83\x19\x07\x22"
+			  "\xa5\x2d\xdf\xa5\xf3\x80\x85\x78"
+			  "\x84\x39\x6a\x6d\x6a\x99\x4f\xa5"
+			  "\x15\xfe\x46\xb0\xe4\x6c\xa5\x41"
+			  "\x3c\xce\x8f\x42\x60\x71\xa7\x75"
+			  "\x08\x40\x65\x8a\x82\xbf\xf5\x43"
+			  "\x71\x96\xa9\x4d\x44\x8a\x20\xbe"
+			  "\xfa\x4d\xbb\xc0\x7d\x31\x96\x65"
+			  "\xe7\x75\xe5\x3e\xfd\x92\x3b\xc9"
+			  "\x55\xbb\x16\x7e\xf7\xc2\x8c\xa4"
+			  "\x40\x1d\xe5\xef\x0e\xdf\xe4\x9a"
+			  "\x62\x73\x65\xfd\x46\x63\x25\x3d"
+			  "\x2b\xaf\xe5\x64\xfe\xa5\x5c\xcf"
+			  "\x24\xf3\xb4\xac\x64\xba\xdf\x4b"
+			  "\xc6\x96\x7d\x81\x2d\x8d\x97\xf7"
+			  "\xc5\x68\x77\x84\x32\x2b\xcc\x85"
+			  "\x74\x96\xf0\x12\x77\x61\xb9\xeb"
+			  "\x71\xaa\x82\xcb\x1c\xdb\x89\xc8"
+			  "\xc6\xb5\xe3\x5c\x7d\x39\x07\x24"
+			  "\xda\x39\x87\x45\xc0\x2b\xbb\x01"
+			  "\xac\xbc\x2a\x5c\x7f\xfc\xe8\xce"
+			  "\x6d\x9c\x6f\xed\xd3\xc1\xa1\xd6"
+			  "\xc5\x55\xa9\x66\x2f\xe1\xc8\x32"
+			  "\xa6\x5d\xa4\x3a\x98\x73\xe8\x45"
+			  "\xa4\xc7\xa8\xb4\xf6\x13\x03\xf6"
+			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
+			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
+		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_xts_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.input	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.ilen	= 512,
+		.result	= "\xDE\x6F\x22\xA5\xE8\x39\xE8\x78"
+			  "\x88\x5A\x4F\x8D\x82\x76\x52\x6D"
+			  "\xB2\x41\x16\xF4\x2B\xA6\xEB\xF6"
+			  "\xE2\xC5\x62\x8D\x61\xA1\x01\xED"
+			  "\xD9\x38\x01\xC1\x43\x63\x4E\x88"
+			  "\xC9\x4B\x5A\x88\x80\xB7\x5C\x71"
+			  "\x47\xEE\x11\xD8\xB7\x2D\x5D\x13"
+			  "\x1A\xB1\x68\x5B\x61\xA7\xA9\x81"
+			  "\x8B\x83\xA1\x6A\xAA\x36\xD6\xB6"
+			  "\x60\x54\x09\x32\xFE\x6A\x76\x2E"
+			  "\x28\xFF\xD5\xD6\xDD\x1D\x45\x7D"
+			  "\xF0\x8B\xF3\x32\x4E\x6C\x12\xCB"
+			  "\xB8\x25\x70\xF8\x40\xBC\x90\x1B"
+			  "\x11\xC3\x59\xAF\xF0\x2F\x92\xDD"
+			  "\xD3\x3B\xCF\x60\xA1\x78\x94\x57"
+			  "\xAF\x76\xC1\x67\xA6\x3C\xCD\x98"
+			  "\xB1\xF7\x27\xB9\xA3\xBD\x10\xEA"
+			  "\xCD\x8B\xC2\xF2\x14\xF2\xB2\x67"
+			  "\x05\xDD\x1D\x58\x6E\x2F\x95\x08"
+			  "\x3A\xF8\x78\x76\x82\x56\xA7\xEC"
+			  "\x51\x4B\x85\x77\xC2\x4C\x4A\x34"
+			  "\x71\x38\x17\x91\x44\xE8\xFC\x65"
+			  "\x99\x0D\x52\x91\xEE\xF8\xEF\x27"
+			  "\x2A\x9E\x6E\x78\xC4\x26\x87\xF4"
+			  "\x8A\xF0\x2D\x04\xE8\x14\x92\x5D"
+			  "\x59\x22\x9B\x29\x5C\x18\xF0\xC3"
+			  "\x47\xF3\x76\xD8\xE4\xF3\x1B\xD1"
+			  "\x70\xA3\x0D\xB5\x70\x02\x1D\xA3"
+			  "\x91\x3B\x49\x73\x18\xAB\xD4\xC9"
+			  "\xC3\x1E\xEF\x1F\xFE\xD5\x59\x8A"
+			  "\xD7\xF6\xC9\x71\x67\x79\xD7\x0E"
+			  "\xBE\x1F\x8E\xEC\x55\x7E\x4F\x24"
+			  "\xE6\x87\xEA\xFE\x96\x25\x67\x8E"
+			  "\x93\x03\xFA\xFF\xCE\xAF\xB2\x3C"
+			  "\x6F\xEB\x57\xFB\xD3\x28\x87\xA9"
+			  "\xCE\xC2\xF5\x9C\xC6\x67\xB5\x97"
+			  "\x49\xF7\x04\xCB\xEF\x84\x98\x33"
+			  "\xAF\x38\xD3\x04\x1C\x24\x71\x38"
+			  "\xC7\x71\xDD\x43\x0D\x12\x4A\x18"
+			  "\xBA\xC4\xAF\xBA\xB2\x5B\xEB\x95"
+			  "\x02\x43\x5D\xCE\x19\xCC\xCD\x66"
+			  "\x91\x0B\x8C\x7F\x51\xC4\xBF\x3C"
+			  "\x8B\xF1\xCC\xAA\x29\xD7\x87\xCB"
+			  "\x3E\xC5\xF3\xC9\x75\xE8\xA3\x5B"
+			  "\x30\x45\xA9\xB7\xAF\x80\x64\x6F"
+			  "\x75\x4A\xA7\xC0\x6D\x19\x6B\xDE"
+			  "\x17\xDE\x6D\xEA\x87\x9F\x95\xAE"
+			  "\xF5\x3C\xEE\x54\xB8\x27\x84\xF8"
+			  "\x97\xA3\xE1\x6F\x38\x24\x34\x88"
+			  "\xCE\xBD\x32\x52\xE0\x00\x6C\x94"
+			  "\xC9\xD7\x5D\x37\x81\x33\x2E\x7F"
+			  "\x4F\x7E\x2E\x0D\x94\xBD\xEA\x59"
+			  "\x34\x39\xA8\x35\x12\xB7\xBC\xAC"
+			  "\xEA\x52\x9C\x78\x02\x6D\x92\x36"
+			  "\xFB\x59\x2B\xA4\xEA\x7B\x1B\x83"
+			  "\xE1\x4D\x5E\x2A\x7E\x92\xB1\x64"
+			  "\xDE\xE0\x27\x4B\x0A\x6F\x4C\xE3"
+			  "\xB0\xEB\x31\xE4\x69\x95\xAB\x35"
+			  "\x8B\x2C\xF5\x6B\x7F\xF1\xA2\x82"
+			  "\xF8\xD9\x47\x82\xA9\x82\x03\x91"
+			  "\x69\x1F\xBE\x4C\xE7\xC7\x34\x2F"
+			  "\x45\x72\x80\x17\x81\xBD\x9D\x62"
+			  "\xA1\xAC\xE8\xCF\xC6\x74\xCF\xDC"
+			  "\x22\x60\x4E\xE8\xA4\x5D\x85\xB9",
+		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast6_xts_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x27\x18\x28\x18\x28\x45\x90\x45"
+			  "\x23\x53\x60\x28\x74\x71\x35\x26"
+			  "\x62\x49\x77\x57\x24\x70\x93\x69"
+			  "\x99\x59\x57\x49\x66\x96\x76\x27"
+			  "\x31\x41\x59\x26\x53\x58\x97\x93"
+			  "\x23\x84\x62\x64\x33\x83\x27\x95"
+			  "\x02\x88\x41\x97\x16\x93\x99\x37"
+			  "\x51\x05\x82\x09\x74\x94\x45\x92",
+		.klen	= 64,
+		.iv	= "\xff\x00\x00\x00\x00\x00\x00\x00"
+			  "\x00\x00\x00\x00\x00\x00\x00\x00",
+		.input	= "\xDE\x6F\x22\xA5\xE8\x39\xE8\x78"
+			  "\x88\x5A\x4F\x8D\x82\x76\x52\x6D"
+			  "\xB2\x41\x16\xF4\x2B\xA6\xEB\xF6"
+			  "\xE2\xC5\x62\x8D\x61\xA1\x01\xED"
+			  "\xD9\x38\x01\xC1\x43\x63\x4E\x88"
+			  "\xC9\x4B\x5A\x88\x80\xB7\x5C\x71"
+			  "\x47\xEE\x11\xD8\xB7\x2D\x5D\x13"
+			  "\x1A\xB1\x68\x5B\x61\xA7\xA9\x81"
+			  "\x8B\x83\xA1\x6A\xAA\x36\xD6\xB6"
+			  "\x60\x54\x09\x32\xFE\x6A\x76\x2E"
+			  "\x28\xFF\xD5\xD6\xDD\x1D\x45\x7D"
+			  "\xF0\x8B\xF3\x32\x4E\x6C\x12\xCB"
+			  "\xB8\x25\x70\xF8\x40\xBC\x90\x1B"
+			  "\x11\xC3\x59\xAF\xF0\x2F\x92\xDD"
+			  "\xD3\x3B\xCF\x60\xA1\x78\x94\x57"
+			  "\xAF\x76\xC1\x67\xA6\x3C\xCD\x98"
+			  "\xB1\xF7\x27\xB9\xA3\xBD\x10\xEA"
+			  "\xCD\x8B\xC2\xF2\x14\xF2\xB2\x67"
+			  "\x05\xDD\x1D\x58\x6E\x2F\x95\x08"
+			  "\x3A\xF8\x78\x76\x82\x56\xA7\xEC"
+			  "\x51\x4B\x85\x77\xC2\x4C\x4A\x34"
+			  "\x71\x38\x17\x91\x44\xE8\xFC\x65"
+			  "\x99\x0D\x52\x91\xEE\xF8\xEF\x27"
+			  "\x2A\x9E\x6E\x78\xC4\x26\x87\xF4"
+			  "\x8A\xF0\x2D\x04\xE8\x14\x92\x5D"
+			  "\x59\x22\x9B\x29\x5C\x18\xF0\xC3"
+			  "\x47\xF3\x76\xD8\xE4\xF3\x1B\xD1"
+			  "\x70\xA3\x0D\xB5\x70\x02\x1D\xA3"
+			  "\x91\x3B\x49\x73\x18\xAB\xD4\xC9"
+			  "\xC3\x1E\xEF\x1F\xFE\xD5\x59\x8A"
+			  "\xD7\xF6\xC9\x71\x67\x79\xD7\x0E"
+			  "\xBE\x1F\x8E\xEC\x55\x7E\x4F\x24"
+			  "\xE6\x87\xEA\xFE\x96\x25\x67\x8E"
+			  "\x93\x03\xFA\xFF\xCE\xAF\xB2\x3C"
+			  "\x6F\xEB\x57\xFB\xD3\x28\x87\xA9"
+			  "\xCE\xC2\xF5\x9C\xC6\x67\xB5\x97"
+			  "\x49\xF7\x04\xCB\xEF\x84\x98\x33"
+			  "\xAF\x38\xD3\x04\x1C\x24\x71\x38"
+			  "\xC7\x71\xDD\x43\x0D\x12\x4A\x18"
+			  "\xBA\xC4\xAF\xBA\xB2\x5B\xEB\x95"
+			  "\x02\x43\x5D\xCE\x19\xCC\xCD\x66"
+			  "\x91\x0B\x8C\x7F\x51\xC4\xBF\x3C"
+			  "\x8B\xF1\xCC\xAA\x29\xD7\x87\xCB"
+			  "\x3E\xC5\xF3\xC9\x75\xE8\xA3\x5B"
+			  "\x30\x45\xA9\xB7\xAF\x80\x64\x6F"
+			  "\x75\x4A\xA7\xC0\x6D\x19\x6B\xDE"
+			  "\x17\xDE\x6D\xEA\x87\x9F\x95\xAE"
+			  "\xF5\x3C\xEE\x54\xB8\x27\x84\xF8"
+			  "\x97\xA3\xE1\x6F\x38\x24\x34\x88"
+			  "\xCE\xBD\x32\x52\xE0\x00\x6C\x94"
+			  "\xC9\xD7\x5D\x37\x81\x33\x2E\x7F"
+			  "\x4F\x7E\x2E\x0D\x94\xBD\xEA\x59"
+			  "\x34\x39\xA8\x35\x12\xB7\xBC\xAC"
+			  "\xEA\x52\x9C\x78\x02\x6D\x92\x36"
+			  "\xFB\x59\x2B\xA4\xEA\x7B\x1B\x83"
+			  "\xE1\x4D\x5E\x2A\x7E\x92\xB1\x64"
+			  "\xDE\xE0\x27\x4B\x0A\x6F\x4C\xE3"
+			  "\xB0\xEB\x31\xE4\x69\x95\xAB\x35"
+			  "\x8B\x2C\xF5\x6B\x7F\xF1\xA2\x82"
+			  "\xF8\xD9\x47\x82\xA9\x82\x03\x91"
+			  "\x69\x1F\xBE\x4C\xE7\xC7\x34\x2F"
+			  "\x45\x72\x80\x17\x81\xBD\x9D\x62"
+			  "\xA1\xAC\xE8\xCF\xC6\x74\xCF\xDC"
+			  "\x22\x60\x4E\xE8\xA4\x5D\x85\xB9",
+		.ilen	= 512,
+		.result	= "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"
+			  "\x00\x01\x02\x03\x04\x05\x06\x07"
+			  "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+			  "\x10\x11\x12\x13\x14\x15\x16\x17"
+			  "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
+			  "\x20\x21\x22\x23\x24\x25\x26\x27"
+			  "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f"
+			  "\x30\x31\x32\x33\x34\x35\x36\x37"
+			  "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f"
+			  "\x40\x41\x42\x43\x44\x45\x46\x47"
+			  "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f"
+			  "\x50\x51\x52\x53\x54\x55\x56\x57"
+			  "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f"
+			  "\x60\x61\x62\x63\x64\x65\x66\x67"
+			  "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f"
+			  "\x70\x71\x72\x73\x74\x75\x76\x77"
+			  "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f"
+			  "\x80\x81\x82\x83\x84\x85\x86\x87"
+			  "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
+			  "\x90\x91\x92\x93\x94\x95\x96\x97"
+			  "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
+			  "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+			  "\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
+			  "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+			  "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf"
+			  "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
+			  "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"
+			  "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7"
+			  "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf"
+			  "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7"
+			  "\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
+		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -8313,6 +10668,9 @@ static struct cipher_testvec aes_lrw_enc_tv_template[] = {
 			  "\xcd\x7e\x2b\x5d\x43\xea\x42\xe7"
 			  "\x74\x3f\x7d\x58\x88\x75\xde\x3e",
 		.rlen   = 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	}
 };
 
@@ -8564,6 +10922,9 @@ static struct cipher_testvec aes_lrw_dec_tv_template[] = {
 			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
 			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
 		.rlen   = 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	}
 };
 
@@ -8905,6 +11266,9 @@ static struct cipher_testvec aes_xts_enc_tv_template[] = {
 			  "\xc4\xf3\x6f\xfd\xa9\xfc\xea\x70"
 			  "\xb9\xc6\xe6\x93\xe1\x48\xc1\x51",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	}
 };
 
@@ -9246,7 +11610,9 @@ static struct cipher_testvec aes_xts_dec_tv_template[] = {
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		.rlen	= 512,
-
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	}
 };
 
@@ -12125,8 +14491,12 @@ static struct cprng_testvec ansi_cprng_aes_tv_template[] = {
 };
 
 /* Cast5 test vectors from RFC 2144 */
-#define CAST5_ENC_TEST_VECTORS	3
-#define CAST5_DEC_TEST_VECTORS	3
+#define CAST5_ENC_TEST_VECTORS		4
+#define CAST5_DEC_TEST_VECTORS		4
+#define CAST5_CBC_ENC_TEST_VECTORS	1
+#define CAST5_CBC_DEC_TEST_VECTORS	1
+#define CAST5_CTR_ENC_TEST_VECTORS	2
+#define CAST5_CTR_DEC_TEST_VECTORS	2
 
 static struct cipher_testvec cast5_enc_tv_template[] = {
 	{
@@ -12152,6 +14522,140 @@ static struct cipher_testvec cast5_enc_tv_template[] = {
 		.ilen	= 8,
 		.result	= "\x7a\xc8\x16\xd1\x6e\x9b\x30\x2e",
 		.rlen	= 8,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\x8D\xFC\x81\x9C\xCB\xAA\x5A\x1C"
+			  "\x7E\x95\xCF\x40\xAB\x4D\x6F\xEA"
+			  "\xD3\xD9\xB0\x9A\xB7\xC7\xE0\x2E"
+			  "\xD1\x39\x34\x92\x8F\xFA\x14\xF1"
+			  "\xD5\xD2\x7B\x59\x1F\x35\x28\xC2"
+			  "\x20\xD9\x42\x06\xC9\x0B\x10\x04"
+			  "\xF8\x79\xCD\x32\x86\x75\x4C\xB6"
+			  "\x7B\x1C\x52\xB1\x91\x64\x22\x4B"
+			  "\x13\xC7\xAE\x98\x0E\xB5\xCF\x6F"
+			  "\x3F\xF4\x43\x96\x73\x0D\xA2\x05"
+			  "\xDB\xFD\x28\x90\x2C\x56\xB9\x37"
+			  "\x5B\x69\x0C\xAD\x84\x67\xFF\x15"
+			  "\x4A\xD4\xA7\xD3\xDD\x99\x47\x3A"
+			  "\xED\x34\x35\x78\x6B\x91\xC9\x32"
+			  "\xE1\xBF\xBC\xB4\x04\x85\x6A\x39"
+			  "\xC0\xBA\x51\xD0\x0F\x4E\xD1\xE2"
+			  "\x1C\xFD\x0E\x05\x07\xF4\x10\xED"
+			  "\xA2\x17\xFF\xF5\x64\xC6\x1A\x22"
+			  "\xAD\x78\xE7\xD7\x11\xE9\x99\xB9"
+			  "\xAA\xEC\x6F\xF8\x3B\xBF\xCE\x77"
+			  "\x93\xE8\xAD\x1D\x50\x6C\xAE\xBC"
+			  "\xBA\x5C\x80\xD1\x91\x65\x51\x1B"
+			  "\xE8\x0A\xCD\x99\x96\x71\x3D\xB6"
+			  "\x78\x75\x37\x55\xC1\xF5\x90\x40"
+			  "\x34\xF4\x7E\xC8\xCC\x3A\x5F\x6E"
+			  "\x36\xA1\xA1\xC2\x3A\x72\x42\x8E"
+			  "\x0E\x37\x88\xE8\xCE\x83\xCB\xAD"
+			  "\xE0\x69\x77\x50\xC7\x0C\x99\xCA"
+			  "\x19\x5B\x30\x25\x9A\xEF\x9B\x0C"
+			  "\xEF\x8F\x74\x4C\xCF\x49\x4E\xB9"
+			  "\xC5\xAE\x9E\x2E\x78\x9A\xB9\x48"
+			  "\xD5\x81\xE4\x37\x1D\xBF\x27\xD9"
+			  "\xC5\xD6\x65\x43\x45\x8C\xBB\xB6"
+			  "\x55\xF4\x06\xBB\x49\x53\x8B\x1B"
+			  "\x07\xA9\x96\x69\x5B\xCB\x0F\xBC"
+			  "\x93\x85\x90\x0F\x0A\x68\x40\x2A"
+			  "\x95\xED\x2D\x88\xBF\x71\xD0\xBB"
+			  "\xEC\xB0\x77\x6C\x79\xFC\x3C\x05"
+			  "\x49\x3F\xB8\x24\xEF\x8E\x09\xA2"
+			  "\x1D\xEF\x92\x02\x96\xD4\x7F\xC8"
+			  "\x03\xB2\xCA\xDB\x17\x5C\x52\xCF"
+			  "\xDD\x70\x37\x63\xAA\xA5\x83\x20"
+			  "\x52\x02\xF6\xB9\xE7\x6E\x0A\xB6"
+			  "\x79\x03\xA0\xDA\xA3\x79\x21\xBD"
+			  "\xE3\x37\x3A\xC0\xF7\x2C\x32\xBE"
+			  "\x8B\xE8\xA6\x00\xC7\x32\xD5\x06"
+			  "\xBB\xE3\xAB\x06\x21\x82\xB8\x32"
+			  "\x31\x34\x2A\xA7\x1F\x64\x99\xBF"
+			  "\xFA\xDA\x3D\x75\xF7\x48\xD5\x48"
+			  "\x4B\x52\x7E\xF6\x7C\xAB\x67\x59"
+			  "\xC5\xDC\xA8\xC6\x63\x85\x4A\xDF"
+			  "\xF0\x40\x5F\xCF\xE3\x58\x52\x67"
+			  "\x7A\x24\x32\xC5\xEC\x9E\xA9\x6F"
+			  "\x58\x56\xDD\x94\x1F\x71\x8D\xF4"
+			  "\x6E\xFF\x2C\xA7\xA5\xD8\xBA\xAF"
+			  "\x1D\x8B\xA2\x46\xB5\xC4\x9F\x57"
+			  "\x8D\xD8\xB3\x3C\x02\x0D\xBB\x84"
+			  "\xC7\xBD\xB4\x9A\x6E\xBB\xB1\x37"
+			  "\x95\x79\xC4\xA7\xEA\x1D\xDC\x33"
+			  "\x5D\x0B\x3F\x03\x8F\x30\xF9\xAE"
+			  "\x4F\xFE\x24\x9C\x9A\x02\xE5\x57"
+			  "\xF5\xBC\x25\xD6\x02\x56\x57\x1C",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -12179,6 +14683,718 @@ static struct cipher_testvec cast5_dec_tv_template[] = {
 		.ilen	= 8,
 		.result	= "\x01\x23\x45\x67\x89\xab\xcd\xef",
 		.rlen	= 8,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x8D\xFC\x81\x9C\xCB\xAA\x5A\x1C"
+			  "\x7E\x95\xCF\x40\xAB\x4D\x6F\xEA"
+			  "\xD3\xD9\xB0\x9A\xB7\xC7\xE0\x2E"
+			  "\xD1\x39\x34\x92\x8F\xFA\x14\xF1"
+			  "\xD5\xD2\x7B\x59\x1F\x35\x28\xC2"
+			  "\x20\xD9\x42\x06\xC9\x0B\x10\x04"
+			  "\xF8\x79\xCD\x32\x86\x75\x4C\xB6"
+			  "\x7B\x1C\x52\xB1\x91\x64\x22\x4B"
+			  "\x13\xC7\xAE\x98\x0E\xB5\xCF\x6F"
+			  "\x3F\xF4\x43\x96\x73\x0D\xA2\x05"
+			  "\xDB\xFD\x28\x90\x2C\x56\xB9\x37"
+			  "\x5B\x69\x0C\xAD\x84\x67\xFF\x15"
+			  "\x4A\xD4\xA7\xD3\xDD\x99\x47\x3A"
+			  "\xED\x34\x35\x78\x6B\x91\xC9\x32"
+			  "\xE1\xBF\xBC\xB4\x04\x85\x6A\x39"
+			  "\xC0\xBA\x51\xD0\x0F\x4E\xD1\xE2"
+			  "\x1C\xFD\x0E\x05\x07\xF4\x10\xED"
+			  "\xA2\x17\xFF\xF5\x64\xC6\x1A\x22"
+			  "\xAD\x78\xE7\xD7\x11\xE9\x99\xB9"
+			  "\xAA\xEC\x6F\xF8\x3B\xBF\xCE\x77"
+			  "\x93\xE8\xAD\x1D\x50\x6C\xAE\xBC"
+			  "\xBA\x5C\x80\xD1\x91\x65\x51\x1B"
+			  "\xE8\x0A\xCD\x99\x96\x71\x3D\xB6"
+			  "\x78\x75\x37\x55\xC1\xF5\x90\x40"
+			  "\x34\xF4\x7E\xC8\xCC\x3A\x5F\x6E"
+			  "\x36\xA1\xA1\xC2\x3A\x72\x42\x8E"
+			  "\x0E\x37\x88\xE8\xCE\x83\xCB\xAD"
+			  "\xE0\x69\x77\x50\xC7\x0C\x99\xCA"
+			  "\x19\x5B\x30\x25\x9A\xEF\x9B\x0C"
+			  "\xEF\x8F\x74\x4C\xCF\x49\x4E\xB9"
+			  "\xC5\xAE\x9E\x2E\x78\x9A\xB9\x48"
+			  "\xD5\x81\xE4\x37\x1D\xBF\x27\xD9"
+			  "\xC5\xD6\x65\x43\x45\x8C\xBB\xB6"
+			  "\x55\xF4\x06\xBB\x49\x53\x8B\x1B"
+			  "\x07\xA9\x96\x69\x5B\xCB\x0F\xBC"
+			  "\x93\x85\x90\x0F\x0A\x68\x40\x2A"
+			  "\x95\xED\x2D\x88\xBF\x71\xD0\xBB"
+			  "\xEC\xB0\x77\x6C\x79\xFC\x3C\x05"
+			  "\x49\x3F\xB8\x24\xEF\x8E\x09\xA2"
+			  "\x1D\xEF\x92\x02\x96\xD4\x7F\xC8"
+			  "\x03\xB2\xCA\xDB\x17\x5C\x52\xCF"
+			  "\xDD\x70\x37\x63\xAA\xA5\x83\x20"
+			  "\x52\x02\xF6\xB9\xE7\x6E\x0A\xB6"
+			  "\x79\x03\xA0\xDA\xA3\x79\x21\xBD"
+			  "\xE3\x37\x3A\xC0\xF7\x2C\x32\xBE"
+			  "\x8B\xE8\xA6\x00\xC7\x32\xD5\x06"
+			  "\xBB\xE3\xAB\x06\x21\x82\xB8\x32"
+			  "\x31\x34\x2A\xA7\x1F\x64\x99\xBF"
+			  "\xFA\xDA\x3D\x75\xF7\x48\xD5\x48"
+			  "\x4B\x52\x7E\xF6\x7C\xAB\x67\x59"
+			  "\xC5\xDC\xA8\xC6\x63\x85\x4A\xDF"
+			  "\xF0\x40\x5F\xCF\xE3\x58\x52\x67"
+			  "\x7A\x24\x32\xC5\xEC\x9E\xA9\x6F"
+			  "\x58\x56\xDD\x94\x1F\x71\x8D\xF4"
+			  "\x6E\xFF\x2C\xA7\xA5\xD8\xBA\xAF"
+			  "\x1D\x8B\xA2\x46\xB5\xC4\x9F\x57"
+			  "\x8D\xD8\xB3\x3C\x02\x0D\xBB\x84"
+			  "\xC7\xBD\xB4\x9A\x6E\xBB\xB1\x37"
+			  "\x95\x79\xC4\xA7\xEA\x1D\xDC\x33"
+			  "\x5D\x0B\x3F\x03\x8F\x30\xF9\xAE"
+			  "\x4F\xFE\x24\x9C\x9A\x02\xE5\x57"
+			  "\xF5\xBC\x25\xD6\x02\x56\x57\x1C",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast5_cbc_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\x05\x28\xCE\x61\x90\x80\xE1\x78"
+			  "\xB9\x2A\x97\x7C\xB0\x83\xD8\x1A"
+			  "\xDE\x58\x7F\xD7\xFD\x72\xB8\xFB"
+			  "\xDA\xF0\x6E\x77\x14\x47\x82\xBA"
+			  "\x29\x0E\x25\x6E\xB4\x39\xD9\x7F"
+			  "\x05\xA7\xA7\x3A\xC1\x5D\x9E\x39"
+			  "\xA7\xFB\x0D\x05\x00\xF3\x58\x67"
+			  "\x60\xEC\x73\x77\x46\x85\x9B\x6A"
+			  "\x08\x3E\xBE\x59\xFB\xE4\x96\x34"
+			  "\xB4\x05\x49\x1A\x97\x43\xAD\xA0"
+			  "\xA9\x1E\x6E\x74\xF1\x94\xEC\xA8"
+			  "\xB5\x8A\x20\xEA\x89\x6B\x19\xAA"
+			  "\xA7\xF1\x33\x67\x90\x23\x0D\xEE"
+			  "\x81\xD5\x78\x4F\xD3\x63\xEA\x46"
+			  "\xB5\xB2\x6E\xBB\xCA\x76\x06\x10"
+			  "\x96\x2A\x0A\xBA\xF9\x41\x5A\x1D"
+			  "\x36\x7C\x56\x14\x54\x83\xFA\xA1"
+			  "\x27\xDD\xBA\x8A\x90\x29\xD6\xA6"
+			  "\xFA\x48\x3E\x1E\x23\x6E\x98\xA8"
+			  "\xA7\xD9\x67\x92\x5C\x13\xB4\x71"
+			  "\xA8\xAA\x89\x4A\xA4\xB3\x49\x7C"
+			  "\x7D\x7F\xCE\x6F\x29\x2E\x7E\x37"
+			  "\xC8\x52\x60\xD9\xE7\xCA\x60\x98"
+			  "\xED\xCD\xE8\x60\x83\xAD\x34\x4D"
+			  "\x96\x4A\x99\x2B\xB7\x14\x75\x66"
+			  "\x6C\x2C\x1A\xBA\x4B\xBB\x49\x56"
+			  "\xE1\x86\xA2\x0E\xD0\xF0\x07\xD3"
+			  "\x18\x38\x09\x9C\x0E\x8B\x86\x07"
+			  "\x90\x12\x37\x49\x27\x98\x69\x18"
+			  "\xB0\xCC\xFB\xD3\xBD\x04\xA0\x85"
+			  "\x4B\x22\x97\x07\xB6\x97\xE9\x95"
+			  "\x0F\x88\x36\xA9\x44\x00\xC6\xE9"
+			  "\x27\x53\x5C\x5B\x1F\xD3\xE2\xEE"
+			  "\xD0\xCD\x63\x30\xA9\xC0\xDD\x49"
+			  "\xFE\x16\xA4\x07\x0D\xE2\x5D\x97"
+			  "\xDE\x89\xBA\x2E\xF3\xA9\x5E\xBE"
+			  "\x03\x55\x0E\x02\x41\x4A\x45\x06"
+			  "\xBE\xEA\x32\xF2\xDC\x91\x5C\x20"
+			  "\x94\x02\x30\xD2\xFC\x29\xFA\x8E"
+			  "\x34\xA0\x31\xB8\x34\xBA\xAE\x54"
+			  "\xB5\x88\x1F\xDC\x43\xDC\x22\x9F"
+			  "\xDC\xCE\xD3\xFA\xA4\xA8\xBC\x8A"
+			  "\xC7\x5A\x43\x21\xA5\xB1\xDB\xC3"
+			  "\x84\x3B\xB4\x9B\xB5\xA7\xF1\x0A"
+			  "\xB6\x37\x21\x19\x55\xC2\xBD\x99"
+			  "\x49\x24\xBB\x7C\xB3\x8E\xEF\xD2"
+			  "\x3A\xCF\xA0\x31\x28\x0E\x25\xA2"
+			  "\x11\xB4\x18\x17\x1A\x65\x92\x56"
+			  "\xE8\xE0\x52\x9C\x61\x18\x2A\xB1"
+			  "\x1A\x01\x22\x45\x17\x62\x52\x6C"
+			  "\x91\x44\xCF\x98\xC7\xC0\x79\x26"
+			  "\x32\x66\x6F\x23\x7F\x94\x36\x88"
+			  "\x3C\xC9\xD0\xB7\x45\x30\x31\x86"
+			  "\x3D\xC6\xA3\x98\x62\x84\x1A\x8B"
+			  "\x16\x88\xC7\xA3\xE9\x4F\xE0\x86"
+			  "\xA4\x93\xA8\x34\x5A\xCA\xDF\xCA"
+			  "\x46\x38\xD2\xF4\xE0\x2D\x1E\xC9"
+			  "\x7C\xEF\x53\xB7\x60\x72\x41\xBF"
+			  "\x29\x00\x87\x02\xAF\x44\x4C\xB7"
+			  "\x8C\xF5\x3F\x19\xF4\x80\x45\xA7"
+			  "\x15\x5F\xDB\xE9\xB1\x83\xD2\xE6"
+			  "\x1D\x18\x66\x44\x5B\x8F\x14\xEB",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast5_cbc_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x05\x28\xCE\x61\x90\x80\xE1\x78"
+			  "\xB9\x2A\x97\x7C\xB0\x83\xD8\x1A"
+			  "\xDE\x58\x7F\xD7\xFD\x72\xB8\xFB"
+			  "\xDA\xF0\x6E\x77\x14\x47\x82\xBA"
+			  "\x29\x0E\x25\x6E\xB4\x39\xD9\x7F"
+			  "\x05\xA7\xA7\x3A\xC1\x5D\x9E\x39"
+			  "\xA7\xFB\x0D\x05\x00\xF3\x58\x67"
+			  "\x60\xEC\x73\x77\x46\x85\x9B\x6A"
+			  "\x08\x3E\xBE\x59\xFB\xE4\x96\x34"
+			  "\xB4\x05\x49\x1A\x97\x43\xAD\xA0"
+			  "\xA9\x1E\x6E\x74\xF1\x94\xEC\xA8"
+			  "\xB5\x8A\x20\xEA\x89\x6B\x19\xAA"
+			  "\xA7\xF1\x33\x67\x90\x23\x0D\xEE"
+			  "\x81\xD5\x78\x4F\xD3\x63\xEA\x46"
+			  "\xB5\xB2\x6E\xBB\xCA\x76\x06\x10"
+			  "\x96\x2A\x0A\xBA\xF9\x41\x5A\x1D"
+			  "\x36\x7C\x56\x14\x54\x83\xFA\xA1"
+			  "\x27\xDD\xBA\x8A\x90\x29\xD6\xA6"
+			  "\xFA\x48\x3E\x1E\x23\x6E\x98\xA8"
+			  "\xA7\xD9\x67\x92\x5C\x13\xB4\x71"
+			  "\xA8\xAA\x89\x4A\xA4\xB3\x49\x7C"
+			  "\x7D\x7F\xCE\x6F\x29\x2E\x7E\x37"
+			  "\xC8\x52\x60\xD9\xE7\xCA\x60\x98"
+			  "\xED\xCD\xE8\x60\x83\xAD\x34\x4D"
+			  "\x96\x4A\x99\x2B\xB7\x14\x75\x66"
+			  "\x6C\x2C\x1A\xBA\x4B\xBB\x49\x56"
+			  "\xE1\x86\xA2\x0E\xD0\xF0\x07\xD3"
+			  "\x18\x38\x09\x9C\x0E\x8B\x86\x07"
+			  "\x90\x12\x37\x49\x27\x98\x69\x18"
+			  "\xB0\xCC\xFB\xD3\xBD\x04\xA0\x85"
+			  "\x4B\x22\x97\x07\xB6\x97\xE9\x95"
+			  "\x0F\x88\x36\xA9\x44\x00\xC6\xE9"
+			  "\x27\x53\x5C\x5B\x1F\xD3\xE2\xEE"
+			  "\xD0\xCD\x63\x30\xA9\xC0\xDD\x49"
+			  "\xFE\x16\xA4\x07\x0D\xE2\x5D\x97"
+			  "\xDE\x89\xBA\x2E\xF3\xA9\x5E\xBE"
+			  "\x03\x55\x0E\x02\x41\x4A\x45\x06"
+			  "\xBE\xEA\x32\xF2\xDC\x91\x5C\x20"
+			  "\x94\x02\x30\xD2\xFC\x29\xFA\x8E"
+			  "\x34\xA0\x31\xB8\x34\xBA\xAE\x54"
+			  "\xB5\x88\x1F\xDC\x43\xDC\x22\x9F"
+			  "\xDC\xCE\xD3\xFA\xA4\xA8\xBC\x8A"
+			  "\xC7\x5A\x43\x21\xA5\xB1\xDB\xC3"
+			  "\x84\x3B\xB4\x9B\xB5\xA7\xF1\x0A"
+			  "\xB6\x37\x21\x19\x55\xC2\xBD\x99"
+			  "\x49\x24\xBB\x7C\xB3\x8E\xEF\xD2"
+			  "\x3A\xCF\xA0\x31\x28\x0E\x25\xA2"
+			  "\x11\xB4\x18\x17\x1A\x65\x92\x56"
+			  "\xE8\xE0\x52\x9C\x61\x18\x2A\xB1"
+			  "\x1A\x01\x22\x45\x17\x62\x52\x6C"
+			  "\x91\x44\xCF\x98\xC7\xC0\x79\x26"
+			  "\x32\x66\x6F\x23\x7F\x94\x36\x88"
+			  "\x3C\xC9\xD0\xB7\x45\x30\x31\x86"
+			  "\x3D\xC6\xA3\x98\x62\x84\x1A\x8B"
+			  "\x16\x88\xC7\xA3\xE9\x4F\xE0\x86"
+			  "\xA4\x93\xA8\x34\x5A\xCA\xDF\xCA"
+			  "\x46\x38\xD2\xF4\xE0\x2D\x1E\xC9"
+			  "\x7C\xEF\x53\xB7\x60\x72\x41\xBF"
+			  "\x29\x00\x87\x02\xAF\x44\x4C\xB7"
+			  "\x8C\xF5\x3F\x19\xF4\x80\x45\xA7"
+			  "\x15\x5F\xDB\xE9\xB1\x83\xD2\xE6"
+			  "\x1D\x18\x66\x44\x5B\x8F\x14\xEB",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast5_ctr_enc_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.ilen	= 17,
+		.result	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C",
+		.rlen	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C\x63\xA5\x55\xE3\xF8\x1C\x7F"
+			  "\xDC\x59\xF9\xA0\x52\xAD\x83\xDF"
+			  "\xD5\x3B\x53\x4A\xAA\x1F\x49\x44"
+			  "\xE8\x20\xCC\xF8\x97\xE6\xE0\x3C"
+			  "\x5A\xD2\x83\xEC\xEE\x25\x3F\xCF"
+			  "\x0D\xC2\x79\x80\x99\x6E\xFF\x7B"
+			  "\x64\xB0\x7B\x86\x29\x1D\x9F\x17"
+			  "\x10\xA5\xA5\xEB\x16\x55\x9E\xE3"
+			  "\x88\x18\x52\x56\x48\x58\xD1\x6B"
+			  "\xE8\x74\x6E\x48\xB0\x2E\x69\x63"
+			  "\x32\xAA\xAC\x26\x55\x45\x94\xDE"
+			  "\x30\x26\x26\xE6\x08\x82\x2F\x5F"
+			  "\xA7\x15\x94\x07\x75\x2D\xC6\x3A"
+			  "\x1B\xA0\x39\xFB\xBA\xB9\x06\x56"
+			  "\xF6\x9F\xF1\x2F\x9B\xF3\x89\x8B"
+			  "\x08\xC8\x9D\x5E\x6B\x95\x09\xC7"
+			  "\x98\xB7\x62\xA4\x1D\x25\xFA\xC5"
+			  "\x62\xC8\x5D\x6B\xB4\x85\x88\x7F"
+			  "\x3B\x29\xF9\xB4\x32\x62\x69\xBF"
+			  "\x32\xB8\xEB\xFD\x0E\x26\xAA\xA3"
+			  "\x44\x67\x90\x20\xAC\x41\xDF\x43"
+			  "\xC6\xC7\x19\x9F\x2C\x28\x74\xEB"
+			  "\x3E\x7F\x7A\x80\x5B\xE4\x08\x60"
+			  "\xC7\xC9\x71\x34\x44\xCE\x05\xFD"
+			  "\xA8\x91\xA8\x44\x5E\xD3\x89\x2C"
+			  "\xAE\x59\x0F\x07\x88\x79\x53\x26"
+			  "\xAF\xAC\xCB\x1D\x6F\x08\x25\x62"
+			  "\xD0\x82\x65\x66\xE4\x2A\x29\x1C"
+			  "\x9C\x64\x5F\x49\x9D\xF8\x62\xF9"
+			  "\xED\xC4\x13\x52\x75\xDC\xE4\xF9"
+			  "\x68\x0F\x8A\xCD\xA6\x8D\x75\xAA"
+			  "\x49\xA1\x86\x86\x37\x5C\x6B\x3D"
+			  "\x56\xE5\x6F\xBE\x27\xC0\x10\xF8"
+			  "\x3C\x4D\x17\x35\x14\xDC\x1C\xA0"
+			  "\x6E\xAE\xD1\x10\xDD\x83\x06\xC2"
+			  "\x23\xD3\xC7\x27\x15\x04\x2C\x27"
+			  "\xDD\x1F\x2E\x97\x09\x9C\x33\x7D"
+			  "\xAC\x50\x1B\x2E\xC9\x52\x0C\x14"
+			  "\x4B\x78\xC4\xDE\x07\x6A\x12\x02"
+			  "\x6E\xD7\x4B\x91\xB9\x88\x4D\x02"
+			  "\xC3\xB5\x04\xBC\xE0\x67\xCA\x18"
+			  "\x22\xA1\xAE\x9A\x21\xEF\xB2\x06"
+			  "\x35\xCD\xEC\x37\x70\x2D\xFC\x1E"
+			  "\xA8\x31\xE7\xFC\xE5\x8E\x88\x66"
+			  "\x16\xB5\xC8\x45\x21\x37\xBD\x24"
+			  "\xA9\xD5\x36\x12\x9F\x6E\x67\x80"
+			  "\x87\x54\xD5\xAF\x97\xE1\x15\xA7"
+			  "\x11\xF0\x63\x7B\xE1\x44\x14\x1C"
+			  "\x06\x32\x05\x8C\x6C\xDB\x9B\x36"
+			  "\x6A\x6B\xAD\x3A\x27\x55\x20\x4C"
+			  "\x76\x36\x43\xE8\x16\x60\xB5\xF3"
+			  "\xDF\x5A\xC6\xA5\x69\x78\x59\x51"
+			  "\x54\x68\x65\x06\x84\xDE\x3D\xAE"
+			  "\x38\x91\xBD\xCC\xA2\x8A\xEC\xE6"
+			  "\x9E\x83\xAE\x1E\x8E\x34\x5D\xDE"
+			  "\x91\xCE\x8F\xED\x40\xF7\xC8\x8B"
+			  "\x9A\x13\x4C\xAD\x89\x97\x9E\xD1"
+			  "\x91\x01\xD7\x21\x23\x28\x1E\xCC"
+			  "\x8C\x98\xDB\xDE\xFC\x72\x94\xAA"
+			  "\xC0\x0D\x96\xAA\x23\xF8\xFE\x13",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
+	},
+};
+
+static struct cipher_testvec cast5_ctr_dec_tv_template[] = {
+	{ /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C",
+		.ilen	= 17,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A",
+		.rlen	= 17,
+	}, { /* Generated from TF test vectors */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A",
+		.klen	= 16,
+		.iv	= "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+		.input	= "\xFF\xC4\x2E\x82\x3D\xF8\xA8\x39"
+			  "\x7C\x52\xC4\xD3\xBB\x62\xC6\xA8"
+			  "\x0C\x63\xA5\x55\xE3\xF8\x1C\x7F"
+			  "\xDC\x59\xF9\xA0\x52\xAD\x83\xDF"
+			  "\xD5\x3B\x53\x4A\xAA\x1F\x49\x44"
+			  "\xE8\x20\xCC\xF8\x97\xE6\xE0\x3C"
+			  "\x5A\xD2\x83\xEC\xEE\x25\x3F\xCF"
+			  "\x0D\xC2\x79\x80\x99\x6E\xFF\x7B"
+			  "\x64\xB0\x7B\x86\x29\x1D\x9F\x17"
+			  "\x10\xA5\xA5\xEB\x16\x55\x9E\xE3"
+			  "\x88\x18\x52\x56\x48\x58\xD1\x6B"
+			  "\xE8\x74\x6E\x48\xB0\x2E\x69\x63"
+			  "\x32\xAA\xAC\x26\x55\x45\x94\xDE"
+			  "\x30\x26\x26\xE6\x08\x82\x2F\x5F"
+			  "\xA7\x15\x94\x07\x75\x2D\xC6\x3A"
+			  "\x1B\xA0\x39\xFB\xBA\xB9\x06\x56"
+			  "\xF6\x9F\xF1\x2F\x9B\xF3\x89\x8B"
+			  "\x08\xC8\x9D\x5E\x6B\x95\x09\xC7"
+			  "\x98\xB7\x62\xA4\x1D\x25\xFA\xC5"
+			  "\x62\xC8\x5D\x6B\xB4\x85\x88\x7F"
+			  "\x3B\x29\xF9\xB4\x32\x62\x69\xBF"
+			  "\x32\xB8\xEB\xFD\x0E\x26\xAA\xA3"
+			  "\x44\x67\x90\x20\xAC\x41\xDF\x43"
+			  "\xC6\xC7\x19\x9F\x2C\x28\x74\xEB"
+			  "\x3E\x7F\x7A\x80\x5B\xE4\x08\x60"
+			  "\xC7\xC9\x71\x34\x44\xCE\x05\xFD"
+			  "\xA8\x91\xA8\x44\x5E\xD3\x89\x2C"
+			  "\xAE\x59\x0F\x07\x88\x79\x53\x26"
+			  "\xAF\xAC\xCB\x1D\x6F\x08\x25\x62"
+			  "\xD0\x82\x65\x66\xE4\x2A\x29\x1C"
+			  "\x9C\x64\x5F\x49\x9D\xF8\x62\xF9"
+			  "\xED\xC4\x13\x52\x75\xDC\xE4\xF9"
+			  "\x68\x0F\x8A\xCD\xA6\x8D\x75\xAA"
+			  "\x49\xA1\x86\x86\x37\x5C\x6B\x3D"
+			  "\x56\xE5\x6F\xBE\x27\xC0\x10\xF8"
+			  "\x3C\x4D\x17\x35\x14\xDC\x1C\xA0"
+			  "\x6E\xAE\xD1\x10\xDD\x83\x06\xC2"
+			  "\x23\xD3\xC7\x27\x15\x04\x2C\x27"
+			  "\xDD\x1F\x2E\x97\x09\x9C\x33\x7D"
+			  "\xAC\x50\x1B\x2E\xC9\x52\x0C\x14"
+			  "\x4B\x78\xC4\xDE\x07\x6A\x12\x02"
+			  "\x6E\xD7\x4B\x91\xB9\x88\x4D\x02"
+			  "\xC3\xB5\x04\xBC\xE0\x67\xCA\x18"
+			  "\x22\xA1\xAE\x9A\x21\xEF\xB2\x06"
+			  "\x35\xCD\xEC\x37\x70\x2D\xFC\x1E"
+			  "\xA8\x31\xE7\xFC\xE5\x8E\x88\x66"
+			  "\x16\xB5\xC8\x45\x21\x37\xBD\x24"
+			  "\xA9\xD5\x36\x12\x9F\x6E\x67\x80"
+			  "\x87\x54\xD5\xAF\x97\xE1\x15\xA7"
+			  "\x11\xF0\x63\x7B\xE1\x44\x14\x1C"
+			  "\x06\x32\x05\x8C\x6C\xDB\x9B\x36"
+			  "\x6A\x6B\xAD\x3A\x27\x55\x20\x4C"
+			  "\x76\x36\x43\xE8\x16\x60\xB5\xF3"
+			  "\xDF\x5A\xC6\xA5\x69\x78\x59\x51"
+			  "\x54\x68\x65\x06\x84\xDE\x3D\xAE"
+			  "\x38\x91\xBD\xCC\xA2\x8A\xEC\xE6"
+			  "\x9E\x83\xAE\x1E\x8E\x34\x5D\xDE"
+			  "\x91\xCE\x8F\xED\x40\xF7\xC8\x8B"
+			  "\x9A\x13\x4C\xAD\x89\x97\x9E\xD1"
+			  "\x91\x01\xD7\x21\x23\x28\x1E\xCC"
+			  "\x8C\x98\xDB\xDE\xFC\x72\x94\xAA"
+			  "\xC0\x0D\x96\xAA\x23\xF8\xFE\x13",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 496 - 16, 16 },
 	},
 };
 
@@ -13096,6 +16312,9 @@ static struct cipher_testvec camellia_enc_tv_template[] = {
 			  "\x0D\xD0\xFD\xC4\x65\xA5\x69\xB9"
 			  "\xF1\xF6\xB1\xA5\xB2\x75\x4F\x8A",
 		.rlen	= 48,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 48 - 16, 16 },
 	},
 };
 
@@ -13154,6 +16373,9 @@ static struct cipher_testvec camellia_dec_tv_template[] = {
 			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
 			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48",
 		.rlen	= 48,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 48 - 16, 16 },
 	},
 };
 
@@ -13208,6 +16430,9 @@ static struct cipher_testvec camellia_cbc_enc_tv_template[] = {
 			  "\xB9\xF9\xC2\x27\x6A\xB6\x31\x27"
 			  "\xA6\xAD\xEF\xE5\x5D\xE4\x02\x01",
 		.rlen	= 48,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 48 - 16, 16 },
 	},
 };
 
@@ -13262,6 +16487,9 @@ static struct cipher_testvec camellia_cbc_dec_tv_template[] = {
 			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
 			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48",
 		.rlen	= 48,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 48 - 16, 16 },
 	},
 };
 
@@ -13313,6 +16541,143 @@ static struct cipher_testvec camellia_ctr_enc_tv_template[] = {
 			  "\x60\xFC\xE8\x94\xE8\xB5\x09\x2C"
 			  "\x1E\x43\xEF",
 		.rlen	= 51,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 51 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.ilen	= 496,
+		.result	= "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9"
+			  "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E"
+			  "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB"
+			  "\xF7\xC4\x27\x04\x51\x87\xD7\x6F"
+			  "\xB8\x4E\x07\xFB\xAC\x3B\x08\xB4"
+			  "\x4D\xCB\xE8\xE1\x71\x7D\x4F\x48"
+			  "\xCD\x81\x64\xA5\xC4\x07\x1A\x9A"
+			  "\x4B\x62\x90\x0E\xC8\xB3\x2B\x6B"
+			  "\x8F\x9C\x6E\x72\x4B\xBA\xEF\x07"
+			  "\x2C\x56\x07\x5E\x37\x30\x60\xA9"
+			  "\xE3\xEF\xD6\x69\xE1\xA1\x77\x64"
+			  "\x93\x75\x7A\xB7\x7A\x3B\xE9\x43"
+			  "\x23\x35\x95\x91\x80\x8A\xC7\xCF"
+			  "\xC3\xD5\xBF\xE7\xFE\x4C\x06\x6B"
+			  "\x05\x19\x48\xE2\x62\xBA\x4F\xF2"
+			  "\xFB\xEE\xE4\xCB\x79\x9D\xA3\x10"
+			  "\x1D\x29\x8C\x1D\x7A\x88\x5A\xDD"
+			  "\x4E\xB6\x18\xAA\xCD\xE6\x33\x96"
+			  "\xD9\x0F\x90\x5A\x78\x76\x4D\x77"
+			  "\x3C\x20\x89\x3B\xA3\xF9\x07\xFD"
+			  "\xE4\xE8\x20\x2D\x15\x0A\x63\x49"
+			  "\xF5\x4F\x89\xD8\xDE\xA1\x28\x78"
+			  "\x28\x07\x09\x1B\x03\x94\x1D\x4B"
+			  "\x82\x28\x1E\x1D\x95\xBA\xAC\x85"
+			  "\x71\x6E\x3C\x18\x4B\x77\x74\x79"
+			  "\xBF\x67\x0A\x53\x3C\x94\xD9\x60"
+			  "\xE9\x6D\x40\x34\xA0\x2A\x53\x5D"
+			  "\x27\xD5\x47\xF9\xC3\x4B\x27\x29"
+			  "\xE4\x76\x9C\x3F\xA7\x1C\x87\xFC"
+			  "\x6E\x0F\xCF\x9B\x60\xF0\xF0\x8B"
+			  "\x70\x1C\x84\x81\x72\x4D\xB4\x98"
+			  "\x23\x62\xE7\x6A\x2B\xFC\xA5\xB2"
+			  "\xFF\xF5\x71\x07\xCD\x90\x23\x13"
+			  "\x19\xD7\x79\x36\x6C\x9D\x55\x8B"
+			  "\x93\x78\x86\x05\x69\x46\xD0\xC5"
+			  "\x39\x09\xEB\x79\xEF\xFA\x9F\xAE"
+			  "\xF3\xD5\x44\xC3\xFD\x86\xD2\x7C"
+			  "\x83\x4B\xD8\x75\x9C\x18\x04\x7B"
+			  "\x73\xAD\x72\xA4\xF6\xAB\xCF\x4B"
+			  "\xCC\x01\x45\x90\xA6\x43\x05\x0C"
+			  "\x6C\x4F\x62\x77\x57\x97\x9F\xEE"
+			  "\x75\xA7\x3C\x38\xD1\x0F\x3D\x0E"
+			  "\x2C\x43\x98\xFB\x13\x65\x73\xE4"
+			  "\x3C\x1E\xD6\x90\x08\xF7\xE0\x99"
+			  "\x3B\xF1\x9D\x6C\x48\xA9\x0E\x32"
+			  "\x17\xC2\xCC\x20\xA1\x19\x26\xAA"
+			  "\xE0\x75\x2F\xFB\x54\x66\x0A\xDF"
+			  "\xB5\xF2\x1F\xC1\x34\x3C\x30\x56"
+			  "\xE8\xDC\xF7\x92\x6B\xBF\x17\x24"
+			  "\xEC\x94\xB5\x3B\xD6\xCE\xA2\x54"
+			  "\x10\x7F\x50\xDE\x69\x77\xD5\x37"
+			  "\xFE\x9C\x10\x83\xC5\xEB\xC9\x53"
+			  "\xB7\xF3\xC4\x20\xAF\x0A\x7E\x57"
+			  "\x3A\xE6\x75\xFE\x89\x00\x6E\x48"
+			  "\xFB\x99\x17\x2C\xF6\x64\x40\x95"
+			  "\x5E\xDC\x7A\xA6\x70\xC7\xF4\xDD"
+			  "\x52\x05\x24\x34\xF9\x0E\xC8\x64"
+			  "\x6D\xE2\xD8\x80\x53\x31\x4C\xFE"
+			  "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22"
+			  "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E"
+			  "\x12\xA8\x01\x64\x16\x0B\x26\x5A"
+			  "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C",
+		.rlen	= 496,
 	},
 };
 
@@ -13364,8 +16729,144 @@ static struct cipher_testvec camellia_ctr_dec_tv_template[] = {
 			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
 			  "\xDF\x76\x0D",
 		.rlen	= 51,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 51 - 16, 16 },
+	}, { /* Generated with Crypto++ */
+		.key	= "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+			  "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+			  "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+			  "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+		.klen	= 32,
+		.iv	= "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+			  "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFD",
+		.input	= "\x85\x79\x6C\x8B\x2B\x6D\x14\xF9"
+			  "\xA6\x83\xB6\x80\x5B\x3A\xF3\x7E"
+			  "\x30\x29\xEB\x1F\xDC\x19\x5F\xEB"
+			  "\xF7\xC4\x27\x04\x51\x87\xD7\x6F"
+			  "\xB8\x4E\x07\xFB\xAC\x3B\x08\xB4"
+			  "\x4D\xCB\xE8\xE1\x71\x7D\x4F\x48"
+			  "\xCD\x81\x64\xA5\xC4\x07\x1A\x9A"
+			  "\x4B\x62\x90\x0E\xC8\xB3\x2B\x6B"
+			  "\x8F\x9C\x6E\x72\x4B\xBA\xEF\x07"
+			  "\x2C\x56\x07\x5E\x37\x30\x60\xA9"
+			  "\xE3\xEF\xD6\x69\xE1\xA1\x77\x64"
+			  "\x93\x75\x7A\xB7\x7A\x3B\xE9\x43"
+			  "\x23\x35\x95\x91\x80\x8A\xC7\xCF"
+			  "\xC3\xD5\xBF\xE7\xFE\x4C\x06\x6B"
+			  "\x05\x19\x48\xE2\x62\xBA\x4F\xF2"
+			  "\xFB\xEE\xE4\xCB\x79\x9D\xA3\x10"
+			  "\x1D\x29\x8C\x1D\x7A\x88\x5A\xDD"
+			  "\x4E\xB6\x18\xAA\xCD\xE6\x33\x96"
+			  "\xD9\x0F\x90\x5A\x78\x76\x4D\x77"
+			  "\x3C\x20\x89\x3B\xA3\xF9\x07\xFD"
+			  "\xE4\xE8\x20\x2D\x15\x0A\x63\x49"
+			  "\xF5\x4F\x89\xD8\xDE\xA1\x28\x78"
+			  "\x28\x07\x09\x1B\x03\x94\x1D\x4B"
+			  "\x82\x28\x1E\x1D\x95\xBA\xAC\x85"
+			  "\x71\x6E\x3C\x18\x4B\x77\x74\x79"
+			  "\xBF\x67\x0A\x53\x3C\x94\xD9\x60"
+			  "\xE9\x6D\x40\x34\xA0\x2A\x53\x5D"
+			  "\x27\xD5\x47\xF9\xC3\x4B\x27\x29"
+			  "\xE4\x76\x9C\x3F\xA7\x1C\x87\xFC"
+			  "\x6E\x0F\xCF\x9B\x60\xF0\xF0\x8B"
+			  "\x70\x1C\x84\x81\x72\x4D\xB4\x98"
+			  "\x23\x62\xE7\x6A\x2B\xFC\xA5\xB2"
+			  "\xFF\xF5\x71\x07\xCD\x90\x23\x13"
+			  "\x19\xD7\x79\x36\x6C\x9D\x55\x8B"
+			  "\x93\x78\x86\x05\x69\x46\xD0\xC5"
+			  "\x39\x09\xEB\x79\xEF\xFA\x9F\xAE"
+			  "\xF3\xD5\x44\xC3\xFD\x86\xD2\x7C"
+			  "\x83\x4B\xD8\x75\x9C\x18\x04\x7B"
+			  "\x73\xAD\x72\xA4\xF6\xAB\xCF\x4B"
+			  "\xCC\x01\x45\x90\xA6\x43\x05\x0C"
+			  "\x6C\x4F\x62\x77\x57\x97\x9F\xEE"
+			  "\x75\xA7\x3C\x38\xD1\x0F\x3D\x0E"
+			  "\x2C\x43\x98\xFB\x13\x65\x73\xE4"
+			  "\x3C\x1E\xD6\x90\x08\xF7\xE0\x99"
+			  "\x3B\xF1\x9D\x6C\x48\xA9\x0E\x32"
+			  "\x17\xC2\xCC\x20\xA1\x19\x26\xAA"
+			  "\xE0\x75\x2F\xFB\x54\x66\x0A\xDF"
+			  "\xB5\xF2\x1F\xC1\x34\x3C\x30\x56"
+			  "\xE8\xDC\xF7\x92\x6B\xBF\x17\x24"
+			  "\xEC\x94\xB5\x3B\xD6\xCE\xA2\x54"
+			  "\x10\x7F\x50\xDE\x69\x77\xD5\x37"
+			  "\xFE\x9C\x10\x83\xC5\xEB\xC9\x53"
+			  "\xB7\xF3\xC4\x20\xAF\x0A\x7E\x57"
+			  "\x3A\xE6\x75\xFE\x89\x00\x6E\x48"
+			  "\xFB\x99\x17\x2C\xF6\x64\x40\x95"
+			  "\x5E\xDC\x7A\xA6\x70\xC7\xF4\xDD"
+			  "\x52\x05\x24\x34\xF9\x0E\xC8\x64"
+			  "\x6D\xE2\xD8\x80\x53\x31\x4C\xFE"
+			  "\xB4\x3A\x5F\x19\xCF\x42\x1B\x22"
+			  "\x0B\x2D\x7B\xF1\xC5\x43\xF7\x5E"
+			  "\x12\xA8\x01\x64\x16\x0B\x26\x5A"
+			  "\x0C\x95\x0F\x40\xC5\x5A\x06\x7C",
+		.ilen	= 496,
+		.result	= "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+			  "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+			  "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+			  "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+			  "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+			  "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+			  "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+			  "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+			  "\xC3\x37\xCE\x65\xFC\x70\x07\x9E"
+			  "\x12\xA9\x40\xD7\x4B\xE2\x79\x10"
+			  "\x84\x1B\xB2\x26\xBD\x54\xEB\x5F"
+			  "\xF6\x8D\x01\x98\x2F\xC6\x3A\xD1"
+			  "\x68\xFF\x73\x0A\xA1\x15\xAC\x43"
+			  "\xDA\x4E\xE5\x7C\x13\x87\x1E\xB5"
+			  "\x29\xC0\x57\xEE\x62\xF9\x90\x04"
+			  "\x9B\x32\xC9\x3D\xD4\x6B\x02\x76"
+			  "\x0D\xA4\x18\xAF\x46\xDD\x51\xE8"
+			  "\x7F\x16\x8A\x21\xB8\x2C\xC3\x5A"
+			  "\xF1\x65\xFC\x93\x07\x9E\x35\xCC"
+			  "\x40\xD7\x6E\x05\x79\x10\xA7\x1B"
+			  "\xB2\x49\xE0\x54\xEB\x82\x19\x8D"
+			  "\x24\xBB\x2F\xC6\x5D\xF4\x68\xFF"
+			  "\x96\x0A\xA1\x38\xCF\x43\xDA\x71"
+			  "\x08\x7C\x13\xAA\x1E\xB5\x4C\xE3"
+			  "\x57\xEE\x85\x1C\x90\x27\xBE\x32"
+			  "\xC9\x60\xF7\x6B\x02\x99\x0D\xA4"
+			  "\x3B\xD2\x46\xDD\x74\x0B\x7F\x16"
+			  "\xAD\x21\xB8\x4F\xE6\x5A\xF1\x88"
+			  "\x1F\x93\x2A\xC1\x35\xCC\x63\xFA"
+			  "\x6E\x05\x9C\x10\xA7\x3E\xD5\x49"
+			  "\xE0\x77\x0E\x82\x19\xB0\x24\xBB"
+			  "\x52\xE9\x5D\xF4\x8B\x22\x96\x2D"
+			  "\xC4\x38\xCF\x66\xFD\x71\x08\x9F"
+			  "\x13\xAA\x41\xD8\x4C\xE3\x7A\x11"
+			  "\x85\x1C\xB3\x27\xBE\x55\xEC\x60"
+			  "\xF7\x8E\x02\x99\x30\xC7\x3B\xD2"
+			  "\x69\x00\x74\x0B\xA2\x16\xAD\x44"
+			  "\xDB\x4F\xE6\x7D\x14\x88\x1F\xB6"
+			  "\x2A\xC1\x58\xEF\x63\xFA\x91\x05"
+			  "\x9C\x33\xCA\x3E\xD5\x6C\x03\x77"
+			  "\x0E\xA5\x19\xB0\x47\xDE\x52\xE9"
+			  "\x80\x17\x8B\x22\xB9\x2D\xC4\x5B"
+			  "\xF2\x66\xFD\x94\x08\x9F\x36\xCD"
+			  "\x41\xD8\x6F\x06\x7A\x11\xA8\x1C"
+			  "\xB3\x4A\xE1\x55\xEC\x83\x1A\x8E"
+			  "\x25\xBC\x30\xC7\x5E\xF5\x69\x00"
+			  "\x97\x0B\xA2\x39\xD0\x44\xDB\x72"
+			  "\x09\x7D\x14\xAB\x1F\xB6\x4D\xE4"
+			  "\x58\xEF\x86\x1D\x91\x28\xBF\x33"
+			  "\xCA\x61\xF8\x6C\x03\x9A\x0E\xA5"
+			  "\x3C\xD3\x47\xDE\x75\x0C\x80\x17"
+			  "\xAE\x22\xB9\x50\xE7\x5B\xF2\x89"
+			  "\x20\x94\x2B\xC2\x36\xCD\x64\xFB"
+			  "\x6F\x06\x9D\x11\xA8\x3F\xD6\x4A"
+			  "\xE1\x78\x0F\x83\x1A\xB1\x25\xBC"
+			  "\x53\xEA\x5E\xF5\x8C\x00\x97\x2E"
+			  "\xC5\x39\xD0\x67\xFE\x72\x09\xA0"
+			  "\x14\xAB\x42\xD9\x4D\xE4\x7B\x12"
+			  "\x86\x1D\xB4\x28\xBF\x56\xED\x61"
+			  "\xF8\x8F\x03\x9A\x31\xC8\x3C\xD3"
+			  "\x6A\x01\x75\x0C\xA3\x17\xAE\x45"
+			  "\xDC\x50\xE7\x7E\x15\x89\x20\xB7",
+		.rlen	= 496,
 	},
-
 };
 
 static struct cipher_testvec camellia_lrw_enc_tv_template[] = {
@@ -13614,6 +17115,9 @@ static struct cipher_testvec camellia_lrw_enc_tv_template[] = {
 			  "\xb2\x1a\xd8\x4c\xbd\x1d\x10\xe9"
 			  "\x5a\xa8\x92\x7f\xba\xe6\x0c\x95",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -13864,6 +17368,9 @@ static struct cipher_testvec camellia_lrw_dec_tv_template[] = {
 			  "\xe9\x2e\xc4\x29\x0f\x84\xdb\xc4"
 			  "\x21\xc4\xc2\x75\x67\x89\x37\x0a",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -14203,6 +17710,9 @@ static struct cipher_testvec camellia_xts_enc_tv_template[] = {
 			  "\xb7\x16\xd8\x12\x5c\xcd\x7d\x4e"
 			  "\xd5\xc6\x99\xcc\x4e\x6c\x94\x95",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
@@ -14543,6 +18053,9 @@ static struct cipher_testvec camellia_xts_dec_tv_template[] = {
 			  "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
 			  "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 		.rlen	= 512,
+		.also_non_np = 1,
+		.np	= 2,
+		.tap	= { 512 - 16, 16 },
 	},
 };
 
diff --git a/crypto/tgr192.c b/crypto/tgr192.c
index cbca4f208c9..87403556fd0 100644
--- a/crypto/tgr192.c
+++ b/crypto/tgr192.c
@@ -628,7 +628,7 @@ static int tgr128_final(struct shash_desc *desc, u8 * out)
 	return 0;
 }
 
-static struct shash_alg tgr192 = {
+static struct shash_alg tgr_algs[3] = { {
 	.digestsize	=	TGR192_DIGEST_SIZE,
 	.init		=	tgr192_init,
 	.update		=	tgr192_update,
@@ -640,9 +640,7 @@ static struct shash_alg tgr192 = {
 		.cra_blocksize	=	TGR192_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg tgr160 = {
+}, {
 	.digestsize	=	TGR160_DIGEST_SIZE,
 	.init		=	tgr192_init,
 	.update		=	tgr192_update,
@@ -654,9 +652,7 @@ static struct shash_alg tgr160 = {
 		.cra_blocksize	=	TGR192_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg tgr128 = {
+}, {
 	.digestsize	=	TGR128_DIGEST_SIZE,
 	.init		=	tgr192_init,
 	.update		=	tgr192_update,
@@ -668,38 +664,16 @@ static struct shash_alg tgr128 = {
 		.cra_blocksize	=	TGR192_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
+} };
 
 static int __init tgr192_mod_init(void)
 {
-	int ret = 0;
-
-	ret = crypto_register_shash(&tgr192);
-
-	if (ret < 0) {
-		goto out;
-	}
-
-	ret = crypto_register_shash(&tgr160);
-	if (ret < 0) {
-		crypto_unregister_shash(&tgr192);
-		goto out;
-	}
-
-	ret = crypto_register_shash(&tgr128);
-	if (ret < 0) {
-		crypto_unregister_shash(&tgr192);
-		crypto_unregister_shash(&tgr160);
-	}
-      out:
-	return ret;
+	return crypto_register_shashes(tgr_algs, ARRAY_SIZE(tgr_algs));
 }
 
 static void __exit tgr192_mod_fini(void)
 {
-	crypto_unregister_shash(&tgr192);
-	crypto_unregister_shash(&tgr160);
-	crypto_unregister_shash(&tgr128);
+	crypto_unregister_shashes(tgr_algs, ARRAY_SIZE(tgr_algs));
 }
 
 MODULE_ALIAS("tgr160");
diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c
index 1f07b843e07..2d5000552d0 100644
--- a/crypto/twofish_generic.c
+++ b/crypto/twofish_generic.c
@@ -188,7 +188,6 @@ static struct crypto_alg alg = {
 	.cra_ctxsize        =   sizeof(struct twofish_ctx),
 	.cra_alignmask      =	3,
 	.cra_module         =   THIS_MODULE,
-	.cra_list           =   LIST_HEAD_INIT(alg.cra_list),
 	.cra_u              =   { .cipher = {
 	.cia_min_keysize    =   TF_MIN_KEY_SIZE,
 	.cia_max_keysize    =   TF_MAX_KEY_SIZE,
diff --git a/crypto/vmac.c b/crypto/vmac.c
index 4243905ba13..f2338ca9836 100644
--- a/crypto/vmac.c
+++ b/crypto/vmac.c
@@ -38,11 +38,11 @@
  * Constants and masks
  */
 #define UINT64_C(x) x##ULL
-const u64 p64   = UINT64_C(0xfffffffffffffeff);  /* 2^64 - 257 prime  */
-const u64 m62   = UINT64_C(0x3fffffffffffffff);  /* 62-bit mask       */
-const u64 m63   = UINT64_C(0x7fffffffffffffff);  /* 63-bit mask       */
-const u64 m64   = UINT64_C(0xffffffffffffffff);  /* 64-bit mask       */
-const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */
+static const u64 p64   = UINT64_C(0xfffffffffffffeff);	/* 2^64 - 257 prime  */
+static const u64 m62   = UINT64_C(0x3fffffffffffffff);	/* 62-bit mask       */
+static const u64 m63   = UINT64_C(0x7fffffffffffffff);	/* 63-bit mask       */
+static const u64 m64   = UINT64_C(0xffffffffffffffff);	/* 64-bit mask       */
+static const u64 mpoly = UINT64_C(0x1fffffff1fffffff);	/* Poly key mask     */
 
 #define pe64_to_cpup le64_to_cpup		/* Prefer little endian */
 
diff --git a/crypto/wp512.c b/crypto/wp512.c
index 71719a2be25..180f1d6e03f 100644
--- a/crypto/wp512.c
+++ b/crypto/wp512.c
@@ -1119,7 +1119,7 @@ static int wp256_final(struct shash_desc *desc, u8 *out)
 	return 0;
 }
 
-static struct shash_alg wp512 = {
+static struct shash_alg wp_algs[3] = { {
 	.digestsize	=	WP512_DIGEST_SIZE,
 	.init		=	wp512_init,
 	.update		=	wp512_update,
@@ -1131,9 +1131,7 @@ static struct shash_alg wp512 = {
 		.cra_blocksize	=	WP512_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg wp384 = {
+}, {
 	.digestsize	=	WP384_DIGEST_SIZE,
 	.init		=	wp512_init,
 	.update		=	wp512_update,
@@ -1145,9 +1143,7 @@ static struct shash_alg wp384 = {
 		.cra_blocksize	=	WP512_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
-
-static struct shash_alg wp256 = {
+}, {
 	.digestsize	=	WP256_DIGEST_SIZE,
 	.init		=	wp512_init,
 	.update		=	wp512_update,
@@ -1159,39 +1155,16 @@ static struct shash_alg wp256 = {
 		.cra_blocksize	=	WP512_BLOCK_SIZE,
 		.cra_module	=	THIS_MODULE,
 	}
-};
+} };
 
 static int __init wp512_mod_init(void)
 {
-	int ret = 0;
-
-	ret = crypto_register_shash(&wp512);
-
-	if (ret < 0)
-		goto out;
-
-	ret = crypto_register_shash(&wp384);
-	if (ret < 0)
-	{
-		crypto_unregister_shash(&wp512);
-		goto out;
-	}
-
-	ret = crypto_register_shash(&wp256);
-	if (ret < 0)
-	{
-		crypto_unregister_shash(&wp512);
-		crypto_unregister_shash(&wp384);
-	}
-out:
-	return ret;
+	return crypto_register_shashes(wp_algs, ARRAY_SIZE(wp_algs));
 }
 
 static void __exit wp512_mod_fini(void)
 {
-	crypto_unregister_shash(&wp512);
-	crypto_unregister_shash(&wp384);
-	crypto_unregister_shash(&wp256);
+	crypto_unregister_shashes(wp_algs, ARRAY_SIZE(wp_algs));
 }
 
 MODULE_ALIAS("wp384");
diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c
index bfc31cb0dd3..e78c2a52ea4 100644
--- a/drivers/acpi/processor_driver.c
+++ b/drivers/acpi/processor_driver.c
@@ -475,7 +475,7 @@ static __ref int acpi_processor_start(struct acpi_processor *pr)
 	acpi_processor_get_limit_info(pr);
 
 	if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver)
-		acpi_processor_power_init(pr, device);
+		acpi_processor_power_init(pr);
 
 	pr->cdev = thermal_cooling_device_register("Processor", device,
 						   &processor_cooling_ops);
@@ -509,7 +509,7 @@ err_remove_sysfs_thermal:
 err_thermal_unregister:
 	thermal_cooling_device_unregister(pr->cdev);
 err_power_exit:
-	acpi_processor_power_exit(pr, device);
+	acpi_processor_power_exit(pr);
 
 	return result;
 }
@@ -620,7 +620,7 @@ static int acpi_processor_remove(struct acpi_device *device, int type)
 			return -EINVAL;
 	}
 
-	acpi_processor_power_exit(pr, device);
+	acpi_processor_power_exit(pr);
 
 	sysfs_remove_link(&device->dev.kobj, "sysdev");
 
@@ -905,8 +905,6 @@ static int __init acpi_processor_init(void)
 	if (acpi_disabled)
 		return 0;
 
-	memset(&errata, 0, sizeof(errata));
-
 	result = acpi_bus_register_driver(&acpi_processor_driver);
 	if (result < 0)
 		return result;
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index ad3730b4038..3655ab92381 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -79,6 +79,8 @@ module_param(bm_check_disable, uint, 0000);
 static unsigned int latency_factor __read_mostly = 2;
 module_param(latency_factor, uint, 0644);
 
+static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device);
+
 static int disabled_by_idle_boot_param(void)
 {
 	return boot_option_idle_override == IDLE_POLL ||
@@ -483,8 +485,6 @@ static int acpi_processor_get_power_info_cst(struct acpi_processor *pr)
 		if (obj->type != ACPI_TYPE_INTEGER)
 			continue;
 
-		cx.power = obj->integer.value;
-
 		current_count++;
 		memcpy(&(pr->power.states[current_count]), &cx, sizeof(cx));
 
@@ -1000,7 +1000,7 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr)
 	int i, count = CPUIDLE_DRIVER_STATE_START;
 	struct acpi_processor_cx *cx;
 	struct cpuidle_state_usage *state_usage;
-	struct cpuidle_device *dev = &pr->power.dev;
+	struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
 
 	if (!pr->flags.power_setup_done)
 		return -EINVAL;
@@ -1132,6 +1132,7 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr)
 int acpi_processor_hotplug(struct acpi_processor *pr)
 {
 	int ret = 0;
+	struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
 
 	if (disabled_by_idle_boot_param())
 		return 0;
@@ -1147,11 +1148,11 @@ int acpi_processor_hotplug(struct acpi_processor *pr)
 		return -ENODEV;
 
 	cpuidle_pause_and_lock();
-	cpuidle_disable_device(&pr->power.dev);
+	cpuidle_disable_device(dev);
 	acpi_processor_get_power_info(pr);
 	if (pr->flags.power) {
 		acpi_processor_setup_cpuidle_cx(pr);
-		ret = cpuidle_enable_device(&pr->power.dev);
+		ret = cpuidle_enable_device(dev);
 	}
 	cpuidle_resume_and_unlock();
 
@@ -1162,6 +1163,7 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr)
 {
 	int cpu;
 	struct acpi_processor *_pr;
+	struct cpuidle_device *dev;
 
 	if (disabled_by_idle_boot_param())
 		return 0;
@@ -1192,7 +1194,8 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr)
 			_pr = per_cpu(processors, cpu);
 			if (!_pr || !_pr->flags.power_setup_done)
 				continue;
-			cpuidle_disable_device(&_pr->power.dev);
+			dev = per_cpu(acpi_cpuidle_device, cpu);
+			cpuidle_disable_device(dev);
 		}
 
 		/* Populate Updated C-state information */
@@ -1206,7 +1209,8 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr)
 			acpi_processor_get_power_info(_pr);
 			if (_pr->flags.power) {
 				acpi_processor_setup_cpuidle_cx(_pr);
-				cpuidle_enable_device(&_pr->power.dev);
+				dev = per_cpu(acpi_cpuidle_device, cpu);
+				cpuidle_enable_device(dev);
 			}
 		}
 		put_online_cpus();
@@ -1218,11 +1222,11 @@ int acpi_processor_cst_has_changed(struct acpi_processor *pr)
 
 static int acpi_processor_registered;
 
-int __cpuinit acpi_processor_power_init(struct acpi_processor *pr,
-			      struct acpi_device *device)
+int __cpuinit acpi_processor_power_init(struct acpi_processor *pr)
 {
 	acpi_status status = 0;
 	int retval;
+	struct cpuidle_device *dev;
 	static int first_run;
 
 	if (disabled_by_idle_boot_param())
@@ -1268,11 +1272,18 @@ int __cpuinit acpi_processor_power_init(struct acpi_processor *pr,
 			printk(KERN_DEBUG "ACPI: %s registered with cpuidle\n",
 					acpi_idle_driver.name);
 		}
+
+		dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+		if (!dev)
+			return -ENOMEM;
+		per_cpu(acpi_cpuidle_device, pr->id) = dev;
+
+		acpi_processor_setup_cpuidle_cx(pr);
+
 		/* Register per-cpu cpuidle_device. Cpuidle driver
 		 * must already be registered before registering device
 		 */
-		acpi_processor_setup_cpuidle_cx(pr);
-		retval = cpuidle_register_device(&pr->power.dev);
+		retval = cpuidle_register_device(dev);
 		if (retval) {
 			if (acpi_processor_registered == 0)
 				cpuidle_unregister_driver(&acpi_idle_driver);
@@ -1283,14 +1294,15 @@ int __cpuinit acpi_processor_power_init(struct acpi_processor *pr,
 	return 0;
 }
 
-int acpi_processor_power_exit(struct acpi_processor *pr,
-			      struct acpi_device *device)
+int acpi_processor_power_exit(struct acpi_processor *pr)
 {
+	struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
+
 	if (disabled_by_idle_boot_param())
 		return 0;
 
 	if (pr->flags.power) {
-		cpuidle_unregister_device(&pr->power.dev);
+		cpuidle_unregister_device(dev);
 		acpi_processor_registered--;
 		if (acpi_processor_registered == 0)
 			cpuidle_unregister_driver(&acpi_idle_driver);
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index a093dc163a4..836bfe06904 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -324,6 +324,34 @@ static int acpi_processor_get_performance_control(struct acpi_processor *pr)
 	return result;
 }
 
+#ifdef CONFIG_X86
+/*
+ * Some AMDs have 50MHz frequency multiples, but only provide 100MHz rounding
+ * in their ACPI data. Calculate the real values and fix up the _PSS data.
+ */
+static void amd_fixup_frequency(struct acpi_processor_px *px, int i)
+{
+	u32 hi, lo, fid, did;
+	int index = px->control & 0x00000007;
+
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
+		return;
+
+	if ((boot_cpu_data.x86 == 0x10 && boot_cpu_data.x86_model < 10)
+	    || boot_cpu_data.x86 == 0x11) {
+		rdmsr(MSR_AMD_PSTATE_DEF_BASE + index, lo, hi);
+		fid = lo & 0x3f;
+		did = (lo >> 6) & 7;
+		if (boot_cpu_data.x86 == 0x10)
+			px->core_frequency = (100 * (fid + 0x10)) >> did;
+		else
+			px->core_frequency = (100 * (fid + 8)) >> did;
+	}
+}
+#else
+static void amd_fixup_frequency(struct acpi_processor_px *px, int i) {};
+#endif
+
 static int acpi_processor_get_performance_states(struct acpi_processor *pr)
 {
 	int result = 0;
@@ -379,6 +407,8 @@ static int acpi_processor_get_performance_states(struct acpi_processor *pr)
 			goto end;
 		}
 
+		amd_fixup_frequency(px, i);
+
 		ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 				  "State [%d]: core_frequency[%d] power[%d] transition_latency[%d] bus_master_latency[%d] control[0x%x] status[0x%x]\n",
 				  i,
diff --git a/drivers/acpi/video.c b/drivers/acpi/video.c
index 1e0a9e17c31..f94d4c818fc 100644
--- a/drivers/acpi/video.c
+++ b/drivers/acpi/video.c
@@ -1448,8 +1448,7 @@ static void acpi_video_bus_notify(struct acpi_device *device, u32 event)
 	case ACPI_VIDEO_NOTIFY_SWITCH:	/* User requested a switch,
 					 * most likely via hotkey. */
 		acpi_bus_generate_proc_event(device, event, 0);
-		if (!acpi_notifier_call_chain(device, event, 0))
-			keycode = KEY_SWITCHVIDEOMODE;
+		keycode = KEY_SWITCHVIDEOMODE;
 		break;
 
 	case ACPI_VIDEO_NOTIFY_PROBE:	/* User plugged in or removed a video
@@ -1479,8 +1478,9 @@ static void acpi_video_bus_notify(struct acpi_device *device, u32 event)
 		break;
 	}
 
-	if (event != ACPI_VIDEO_NOTIFY_SWITCH)
-		acpi_notifier_call_chain(device, event, 0);
+	if (acpi_notifier_call_chain(device, event, 0))
+		/* Something vetoed the keypress. */
+		keycode = 0;
 
 	if (keycode) {
 		input_report_key(input, keycode, 1);
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index 27cecd313e7..e08d322d01d 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -214,6 +214,14 @@ config SATA_DWC_VDEBUG
 	help
 	  This option enables the taskfile dumping and NCQ debugging.
 
+config SATA_HIGHBANK
+	tristate "Calxeda Highbank SATA support"
+	help
+	  This option enables support for the Calxeda Highbank SoC's
+	  onboard SATA.
+
+	  If unsure, say N.
+
 config SATA_MV
 	tristate "Marvell SATA support"
 	help
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index a454a139b1d..9329dafba91 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_SATA_FSL)		+= sata_fsl.o
 obj-$(CONFIG_SATA_INIC162X)	+= sata_inic162x.o
 obj-$(CONFIG_SATA_SIL24)	+= sata_sil24.o
 obj-$(CONFIG_SATA_DWC)		+= sata_dwc_460ex.o
+obj-$(CONFIG_SATA_HIGHBANK)	+= sata_highbank.o libahci.o
 
 # SFF w/ custom DMA
 obj-$(CONFIG_PDC_ADMA)		+= pdc_adma.o
diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h
index 57eb1c212a4..9be471200a0 100644
--- a/drivers/ata/ahci.h
+++ b/drivers/ata/ahci.h
@@ -35,6 +35,7 @@
 #ifndef _AHCI_H
 #define _AHCI_H
 
+#include <linux/clk.h>
 #include <linux/libata.h>
 
 /* Enclosure Management Control */
@@ -115,6 +116,9 @@ enum {
 	HOST_CAP2_BOH		= (1 << 0),  /* BIOS/OS handoff supported */
 	HOST_CAP2_NVMHCI	= (1 << 1),  /* NVMHCI supported */
 	HOST_CAP2_APST		= (1 << 2),  /* Automatic partial to slumber */
+	HOST_CAP2_SDS		= (1 << 3),  /* Support device sleep */
+	HOST_CAP2_SADM		= (1 << 4),  /* Support aggressive DevSlp */
+	HOST_CAP2_DESO		= (1 << 5),  /* DevSlp from slumber only */
 
 	/* registers for each SATA port */
 	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
@@ -133,6 +137,7 @@ enum {
 	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
 	PORT_SCR_NTF		= 0x3c, /* SATA phy register: SNotification */
 	PORT_FBS		= 0x40, /* FIS-based Switching */
+	PORT_DEVSLP		= 0x44, /* device sleep */
 
 	/* PORT_IRQ_{STAT,MASK} bits */
 	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
@@ -186,6 +191,7 @@ enum {
 	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
 	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
 
+	/* PORT_FBS bits */
 	PORT_FBS_DWE_OFFSET	= 16, /* FBS device with error offset */
 	PORT_FBS_ADO_OFFSET	= 12, /* FBS active dev optimization offset */
 	PORT_FBS_DEV_OFFSET	= 8,  /* FBS device to issue offset */
@@ -194,6 +200,15 @@ enum {
 	PORT_FBS_DEC		= (1 << 1), /* FBS device error clear */
 	PORT_FBS_EN		= (1 << 0), /* Enable FBS */
 
+	/* PORT_DEVSLP bits */
+	PORT_DEVSLP_DM_OFFSET	= 25,             /* DITO multiplier offset */
+	PORT_DEVSLP_DM_MASK	= (0xf << 25),    /* DITO multiplier mask */
+	PORT_DEVSLP_DITO_OFFSET	= 15,             /* DITO offset */
+	PORT_DEVSLP_MDAT_OFFSET	= 10,             /* Minimum assertion time */
+	PORT_DEVSLP_DETO_OFFSET	= 2,              /* DevSlp exit timeout */
+	PORT_DEVSLP_DSP		= (1 << 1),       /* DevSlp present */
+	PORT_DEVSLP_ADSE	= (1 << 0),       /* Aggressive DevSlp enable */
+
 	/* hpriv->flags bits */
 
 #define AHCI_HFLAGS(flags)		.private_data	= (void *)(flags)
@@ -302,6 +317,7 @@ struct ahci_host_priv {
 	u32 			em_loc; /* enclosure management location */
 	u32			em_buf_sz;	/* EM buffer size in byte */
 	u32			em_msg_type;	/* EM message type */
+	struct clk		*clk;		/* Only for platforms supporting clk */
 };
 
 extern int ahci_ignore_sss;
diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c
index 09728e09cb3..b1ae48054dc 100644
--- a/drivers/ata/ahci_platform.c
+++ b/drivers/ata/ahci_platform.c
@@ -12,6 +12,7 @@
  * any later version.
  */
 
+#include <linux/clk.h>
 #include <linux/kernel.h>
 #include <linux/gfp.h>
 #include <linux/module.h>
@@ -118,6 +119,17 @@ static int __init ahci_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	}
 
+	hpriv->clk = clk_get(dev, NULL);
+	if (IS_ERR(hpriv->clk)) {
+		dev_err(dev, "can't get clock\n");
+	} else {
+		rc = clk_prepare_enable(hpriv->clk);
+		if (rc) {
+			dev_err(dev, "clock prepare enable failed");
+			goto free_clk;
+		}
+	}
+
 	/*
 	 * Some platforms might need to prepare for mmio region access,
 	 * which could be done in the following init call. So, the mmio
@@ -127,7 +139,7 @@ static int __init ahci_probe(struct platform_device *pdev)
 	if (pdata && pdata->init) {
 		rc = pdata->init(dev, hpriv->mmio);
 		if (rc)
-			return rc;
+			goto disable_unprepare_clk;
 	}
 
 	ahci_save_initial_config(dev, hpriv,
@@ -153,7 +165,7 @@ static int __init ahci_probe(struct platform_device *pdev)
 	host = ata_host_alloc_pinfo(dev, ppi, n_ports);
 	if (!host) {
 		rc = -ENOMEM;
-		goto err0;
+		goto pdata_exit;
 	}
 
 	host->private_data = hpriv;
@@ -183,7 +195,7 @@ static int __init ahci_probe(struct platform_device *pdev)
 
 	rc = ahci_reset_controller(host);
 	if (rc)
-		goto err0;
+		goto pdata_exit;
 
 	ahci_init_controller(host);
 	ahci_print_info(host, "platform");
@@ -191,12 +203,18 @@ static int __init ahci_probe(struct platform_device *pdev)
 	rc = ata_host_activate(host, irq, ahci_interrupt, IRQF_SHARED,
 			       &ahci_platform_sht);
 	if (rc)
-		goto err0;
+		goto pdata_exit;
 
 	return 0;
-err0:
+pdata_exit:
 	if (pdata && pdata->exit)
 		pdata->exit(dev);
+disable_unprepare_clk:
+	if (!IS_ERR(hpriv->clk))
+		clk_disable_unprepare(hpriv->clk);
+free_clk:
+	if (!IS_ERR(hpriv->clk))
+		clk_put(hpriv->clk);
 	return rc;
 }
 
@@ -205,12 +223,18 @@ static int __devexit ahci_remove(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct ahci_platform_data *pdata = dev_get_platdata(dev);
 	struct ata_host *host = dev_get_drvdata(dev);
+	struct ahci_host_priv *hpriv = host->private_data;
 
 	ata_host_detach(host);
 
 	if (pdata && pdata->exit)
 		pdata->exit(dev);
 
+	if (!IS_ERR(hpriv->clk)) {
+		clk_disable_unprepare(hpriv->clk);
+		clk_put(hpriv->clk);
+	}
+
 	return 0;
 }
 
@@ -245,6 +269,10 @@ static int ahci_suspend(struct device *dev)
 
 	if (pdata && pdata->suspend)
 		return pdata->suspend(dev);
+
+	if (!IS_ERR(hpriv->clk))
+		clk_disable_unprepare(hpriv->clk);
+
 	return 0;
 }
 
@@ -252,18 +280,27 @@ static int ahci_resume(struct device *dev)
 {
 	struct ahci_platform_data *pdata = dev_get_platdata(dev);
 	struct ata_host *host = dev_get_drvdata(dev);
+	struct ahci_host_priv *hpriv = host->private_data;
 	int rc;
 
+	if (!IS_ERR(hpriv->clk)) {
+		rc = clk_prepare_enable(hpriv->clk);
+		if (rc) {
+			dev_err(dev, "clock prepare enable failed");
+			return rc;
+		}
+	}
+
 	if (pdata && pdata->resume) {
 		rc = pdata->resume(dev);
 		if (rc)
-			return rc;
+			goto disable_unprepare_clk;
 	}
 
 	if (dev->power.power_state.event == PM_EVENT_SUSPEND) {
 		rc = ahci_reset_controller(host);
 		if (rc)
-			return rc;
+			goto disable_unprepare_clk;
 
 		ahci_init_controller(host);
 	}
@@ -271,13 +308,18 @@ static int ahci_resume(struct device *dev)
 	ata_host_resume(host);
 
 	return 0;
+
+disable_unprepare_clk:
+	if (!IS_ERR(hpriv->clk))
+		clk_disable_unprepare(hpriv->clk);
+
+	return rc;
 }
 #endif
 
 SIMPLE_DEV_PM_OPS(ahci_pm_ops, ahci_suspend, ahci_resume);
 
 static const struct of_device_id ahci_of_match[] = {
-	{ .compatible = "calxeda,hb-ahci", },
 	{ .compatible = "snps,spear-ahci", },
 	{},
 };
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index 555c07afa05..4201e535a8c 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -45,6 +45,7 @@
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
 #include "ahci.h"
+#include "libata.h"
 
 static int ahci_skip_host_reset;
 int ahci_ignore_sss;
@@ -76,6 +77,7 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc);
 static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
 static void ahci_freeze(struct ata_port *ap);
 static void ahci_thaw(struct ata_port *ap);
+static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
 static void ahci_enable_fbs(struct ata_port *ap);
 static void ahci_disable_fbs(struct ata_port *ap);
 static void ahci_pmp_attach(struct ata_port *ap);
@@ -193,6 +195,10 @@ module_param(ahci_em_messages, int, 0444);
 MODULE_PARM_DESC(ahci_em_messages,
 	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
 
+int devslp_idle_timeout = 1000;	/* device sleep idle timeout in ms */
+module_param(devslp_idle_timeout, int, 0644);
+MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
+
 static void ahci_enable_ahci(void __iomem *mmio)
 {
 	int i;
@@ -702,6 +708,16 @@ static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 		}
 	}
 
+	/* set aggressive device sleep */
+	if ((hpriv->cap2 & HOST_CAP2_SDS) &&
+	    (hpriv->cap2 & HOST_CAP2_SADM) &&
+	    (link->device->flags & ATA_DFLAG_DEVSLP)) {
+		if (policy == ATA_LPM_MIN_POWER)
+			ahci_set_aggressive_devslp(ap, true);
+		else
+			ahci_set_aggressive_devslp(ap, false);
+	}
+
 	if (policy == ATA_LPM_MAX_POWER) {
 		sata_link_scr_lpm(link, policy, false);
 
@@ -1890,6 +1906,81 @@ static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
 		ahci_kick_engine(ap);
 }
 
+static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
+{
+	void __iomem *port_mmio = ahci_port_base(ap);
+	struct ata_device *dev = ap->link.device;
+	u32 devslp, dm, dito, mdat, deto;
+	int rc;
+	unsigned int err_mask;
+
+	devslp = readl(port_mmio + PORT_DEVSLP);
+	if (!(devslp & PORT_DEVSLP_DSP)) {
+		dev_err(ap->host->dev, "port does not support device sleep\n");
+		return;
+	}
+
+	/* disable device sleep */
+	if (!sleep) {
+		if (devslp & PORT_DEVSLP_ADSE) {
+			writel(devslp & ~PORT_DEVSLP_ADSE,
+			       port_mmio + PORT_DEVSLP);
+			err_mask = ata_dev_set_feature(dev,
+						       SETFEATURES_SATA_DISABLE,
+						       SATA_DEVSLP);
+			if (err_mask && err_mask != AC_ERR_DEV)
+				ata_dev_warn(dev, "failed to disable DEVSLP\n");
+		}
+		return;
+	}
+
+	/* device sleep was already enabled */
+	if (devslp & PORT_DEVSLP_ADSE)
+		return;
+
+	/* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
+	rc = ahci_stop_engine(ap);
+	if (rc)
+		return;
+
+	dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
+	dito = devslp_idle_timeout / (dm + 1);
+	if (dito > 0x3ff)
+		dito = 0x3ff;
+
+	/* Use the nominal value 10 ms if the read MDAT is zero,
+	 * the nominal value of DETO is 20 ms.
+	 */
+	if (dev->sata_settings[ATA_LOG_DEVSLP_VALID] &
+	    ATA_LOG_DEVSLP_VALID_MASK) {
+		mdat = dev->sata_settings[ATA_LOG_DEVSLP_MDAT] &
+		       ATA_LOG_DEVSLP_MDAT_MASK;
+		if (!mdat)
+			mdat = 10;
+		deto = dev->sata_settings[ATA_LOG_DEVSLP_DETO];
+		if (!deto)
+			deto = 20;
+	} else {
+		mdat = 10;
+		deto = 20;
+	}
+
+	devslp |= ((dito << PORT_DEVSLP_DITO_OFFSET) |
+		   (mdat << PORT_DEVSLP_MDAT_OFFSET) |
+		   (deto << PORT_DEVSLP_DETO_OFFSET) |
+		   PORT_DEVSLP_ADSE);
+	writel(devslp, port_mmio + PORT_DEVSLP);
+
+	ahci_start_engine(ap);
+
+	/* enable device sleep feature for the drive */
+	err_mask = ata_dev_set_feature(dev,
+				       SETFEATURES_SATA_ENABLE,
+				       SATA_DEVSLP);
+	if (err_mask && err_mask != AC_ERR_DEV)
+		ata_dev_warn(dev, "failed to enable DEVSLP\n");
+}
+
 static void ahci_enable_fbs(struct ata_port *ap)
 {
 	struct ahci_port_priv *pp = ap->private_data;
@@ -2164,7 +2255,8 @@ void ahci_print_info(struct ata_host *host, const char *scc_s)
 		"flags: "
 		"%s%s%s%s%s%s%s"
 		"%s%s%s%s%s%s%s"
-		"%s%s%s%s%s%s\n"
+		"%s%s%s%s%s%s%s"
+		"%s%s\n"
 		,
 
 		cap & HOST_CAP_64 ? "64bit " : "",
@@ -2184,6 +2276,9 @@ void ahci_print_info(struct ata_host *host, const char *scc_s)
 		cap & HOST_CAP_CCC ? "ccc " : "",
 		cap & HOST_CAP_EMS ? "ems " : "",
 		cap & HOST_CAP_SXS ? "sxs " : "",
+		cap2 & HOST_CAP2_DESO ? "deso " : "",
+		cap2 & HOST_CAP2_SADM ? "sadm " : "",
+		cap2 & HOST_CAP2_SDS ? "sds " : "",
 		cap2 & HOST_CAP2_APST ? "apst " : "",
 		cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
 		cap2 & HOST_CAP2_BOH ? "boh " : ""
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 8e1039c8e15..3cc7096cfda 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -774,7 +774,7 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev,
 		tf->lbam = (block >> 8) & 0xff;
 		tf->lbal = block & 0xff;
 
-		tf->device = 1 << 6;
+		tf->device = ATA_LBA;
 		if (tf->flags & ATA_TFLAG_FUA)
 			tf->device |= 1 << 7;
 	} else if (dev->flags & ATA_DFLAG_LBA) {
@@ -2155,6 +2155,7 @@ int ata_dev_configure(struct ata_device *dev)
 	int print_info = ehc->i.flags & ATA_EHI_PRINTINFO;
 	const u16 *id = dev->id;
 	unsigned long xfer_mask;
+	unsigned int err_mask;
 	char revbuf[7];		/* XYZ-99\0 */
 	char fwrevbuf[ATA_ID_FW_REV_LEN+1];
 	char modelbuf[ATA_ID_PROD_LEN+1];
@@ -2323,6 +2324,26 @@ int ata_dev_configure(struct ata_device *dev)
 			}
 		}
 
+		/* check and mark DevSlp capability */
+		if (ata_id_has_devslp(dev->id))
+			dev->flags |= ATA_DFLAG_DEVSLP;
+
+		/* Obtain SATA Settings page from Identify Device Data Log,
+		 * which contains DevSlp timing variables etc.
+		 * Exclude old devices with ata_id_has_ncq()
+		 */
+		if (ata_id_has_ncq(dev->id)) {
+			err_mask = ata_read_log_page(dev,
+						     ATA_LOG_SATA_ID_DEV_DATA,
+						     ATA_LOG_SATA_SETTINGS,
+						     dev->sata_settings,
+						     1);
+			if (err_mask)
+				ata_dev_dbg(dev,
+					    "failed to get Identify Device Data, Emask 0x%x\n",
+					    err_mask);
+		}
+
 		dev->cdb_len = 16;
 	}
 
@@ -2351,8 +2372,6 @@ int ata_dev_configure(struct ata_device *dev)
 		    (ap->flags & ATA_FLAG_AN) && ata_id_has_atapi_AN(id) &&
 		    (!sata_pmp_attached(ap) ||
 		     sata_scr_read(&ap->link, SCR_NOTIFICATION, &sntf) == 0)) {
-			unsigned int err_mask;
-
 			/* issue SET feature command to turn this on */
 			err_mask = ata_dev_set_feature(dev,
 					SETFEATURES_SATA_ENABLE, SATA_AN);
@@ -3598,7 +3617,7 @@ int sata_link_scr_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 	switch (policy) {
 	case ATA_LPM_MAX_POWER:
 		/* disable all LPM transitions */
-		scontrol |= (0x3 << 8);
+		scontrol |= (0x7 << 8);
 		/* initiate transition to active state */
 		if (spm_wakeup) {
 			scontrol |= (0x4 << 12);
@@ -3608,12 +3627,12 @@ int sata_link_scr_lpm(struct ata_link *link, enum ata_lpm_policy policy,
 	case ATA_LPM_MED_POWER:
 		/* allow LPM to PARTIAL */
 		scontrol &= ~(0x1 << 8);
-		scontrol |= (0x2 << 8);
+		scontrol |= (0x6 << 8);
 		break;
 	case ATA_LPM_MIN_POWER:
 		if (ata_link_nr_enabled(link) > 0)
 			/* no restrictions on LPM transitions */
-			scontrol &= ~(0x3 << 8);
+			scontrol &= ~(0x7 << 8);
 		else {
 			/* empty port, power off */
 			scontrol &= ~0xf;
@@ -4472,6 +4491,7 @@ unsigned int ata_dev_set_feature(struct ata_device *dev, u8 enable, u8 feature)
 	DPRINTK("EXIT, err_mask=%x\n", err_mask);
 	return err_mask;
 }
+EXPORT_SYMBOL_GPL(ata_dev_set_feature);
 
 /**
  *	ata_dev_init_params - Issue INIT DEV PARAMS command
@@ -5253,16 +5273,20 @@ bool ata_link_offline(struct ata_link *link)
 #ifdef CONFIG_PM
 static int ata_port_request_pm(struct ata_port *ap, pm_message_t mesg,
 			       unsigned int action, unsigned int ehi_flags,
-			       int wait)
+			       int *async)
 {
 	struct ata_link *link;
 	unsigned long flags;
-	int rc;
+	int rc = 0;
 
 	/* Previous resume operation might still be in
 	 * progress.  Wait for PM_PENDING to clear.
 	 */
 	if (ap->pflags & ATA_PFLAG_PM_PENDING) {
+		if (async) {
+			*async = -EAGAIN;
+			return 0;
+		}
 		ata_port_wait_eh(ap);
 		WARN_ON(ap->pflags & ATA_PFLAG_PM_PENDING);
 	}
@@ -5271,10 +5295,10 @@ static int ata_port_request_pm(struct ata_port *ap, pm_message_t mesg,
 	spin_lock_irqsave(ap->lock, flags);
 
 	ap->pm_mesg = mesg;
-	if (wait) {
-		rc = 0;
+	if (async)
+		ap->pm_result = async;
+	else
 		ap->pm_result = &rc;
-	}
 
 	ap->pflags |= ATA_PFLAG_PM_PENDING;
 	ata_for_each_link(link, ap, HOST_FIRST) {
@@ -5287,7 +5311,7 @@ static int ata_port_request_pm(struct ata_port *ap, pm_message_t mesg,
 	spin_unlock_irqrestore(ap->lock, flags);
 
 	/* wait and check result */
-	if (wait) {
+	if (!async) {
 		ata_port_wait_eh(ap);
 		WARN_ON(ap->pflags & ATA_PFLAG_PM_PENDING);
 	}
@@ -5295,9 +5319,8 @@ static int ata_port_request_pm(struct ata_port *ap, pm_message_t mesg,
 	return rc;
 }
 
-static int ata_port_suspend_common(struct device *dev, pm_message_t mesg)
+static int __ata_port_suspend_common(struct ata_port *ap, pm_message_t mesg, int *async)
 {
-	struct ata_port *ap = to_ata_port(dev);
 	unsigned int ehi_flags = ATA_EHI_QUIET;
 	int rc;
 
@@ -5312,10 +5335,17 @@ static int ata_port_suspend_common(struct device *dev, pm_message_t mesg)
 	if (mesg.event == PM_EVENT_SUSPEND)
 		ehi_flags |= ATA_EHI_NO_AUTOPSY | ATA_EHI_NO_RECOVERY;
 
-	rc = ata_port_request_pm(ap, mesg, 0, ehi_flags, 1);
+	rc = ata_port_request_pm(ap, mesg, 0, ehi_flags, async);
 	return rc;
 }
 
+static int ata_port_suspend_common(struct device *dev, pm_message_t mesg)
+{
+	struct ata_port *ap = to_ata_port(dev);
+
+	return __ata_port_suspend_common(ap, mesg, NULL);
+}
+
 static int ata_port_suspend(struct device *dev)
 {
 	if (pm_runtime_suspended(dev))
@@ -5340,16 +5370,22 @@ static int ata_port_poweroff(struct device *dev)
 	return ata_port_suspend_common(dev, PMSG_HIBERNATE);
 }
 
-static int ata_port_resume_common(struct device *dev)
+static int __ata_port_resume_common(struct ata_port *ap, int *async)
 {
-	struct ata_port *ap = to_ata_port(dev);
 	int rc;
 
 	rc = ata_port_request_pm(ap, PMSG_ON, ATA_EH_RESET,
-		ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET, 1);
+		ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET, async);
 	return rc;
 }
 
+static int ata_port_resume_common(struct device *dev)
+{
+	struct ata_port *ap = to_ata_port(dev);
+
+	return __ata_port_resume_common(ap, NULL);
+}
+
 static int ata_port_resume(struct device *dev)
 {
 	int rc;
@@ -5382,6 +5418,24 @@ static const struct dev_pm_ops ata_port_pm_ops = {
 	.runtime_idle = ata_port_runtime_idle,
 };
 
+/* sas ports don't participate in pm runtime management of ata_ports,
+ * and need to resume ata devices at the domain level, not the per-port
+ * level. sas suspend/resume is async to allow parallel port recovery
+ * since sas has multiple ata_port instances per Scsi_Host.
+ */
+int ata_sas_port_async_suspend(struct ata_port *ap, int *async)
+{
+	return __ata_port_suspend_common(ap, PMSG_SUSPEND, async);
+}
+EXPORT_SYMBOL_GPL(ata_sas_port_async_suspend);
+
+int ata_sas_port_async_resume(struct ata_port *ap, int *async)
+{
+	return __ata_port_resume_common(ap, async);
+}
+EXPORT_SYMBOL_GPL(ata_sas_port_async_resume);
+
+
 /**
  *	ata_host_suspend - suspend host
  *	@host: host to suspend
@@ -5927,24 +5981,18 @@ int ata_host_start(struct ata_host *host)
 }
 
 /**
- *	ata_sas_host_init - Initialize a host struct
+ *	ata_sas_host_init - Initialize a host struct for sas (ipr, libsas)
  *	@host:	host to initialize
  *	@dev:	device host is attached to
- *	@flags:	host flags
  *	@ops:	port_ops
  *
- *	LOCKING:
- *	PCI/etc. bus probe sem.
- *
  */
-/* KILLME - the only user left is ipr */
 void ata_host_init(struct ata_host *host, struct device *dev,
-		   unsigned long flags, struct ata_port_operations *ops)
+		   struct ata_port_operations *ops)
 {
 	spin_lock_init(&host->lock);
 	mutex_init(&host->eh_mutex);
 	host->dev = dev;
-	host->flags = flags;
 	host->ops = ops;
 }
 
@@ -6388,6 +6436,7 @@ static int __init ata_parse_force_one(char **cur,
 		{ "nohrst",	.lflags		= ATA_LFLAG_NO_HRST },
 		{ "nosrst",	.lflags		= ATA_LFLAG_NO_SRST },
 		{ "norst",	.lflags		= ATA_LFLAG_NO_HRST | ATA_LFLAG_NO_SRST },
+		{ "rstonce",	.lflags		= ATA_LFLAG_RST_ONCE },
 	};
 	char *start = *cur, *p = *cur;
 	char *id, *val, *endp;
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 7d4535e989b..e60437cd0d1 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -1487,6 +1487,7 @@ static const char *ata_err_string(unsigned int err_mask)
 /**
  *	ata_read_log_page - read a specific log page
  *	@dev: target device
+ *	@log: log to read
  *	@page: page to read
  *	@buf: buffer to store read page
  *	@sectors: number of sectors to read
@@ -1499,17 +1500,18 @@ static const char *ata_err_string(unsigned int err_mask)
  *	RETURNS:
  *	0 on success, AC_ERR_* mask otherwise.
  */
-static unsigned int ata_read_log_page(struct ata_device *dev,
-				      u8 page, void *buf, unsigned int sectors)
+unsigned int ata_read_log_page(struct ata_device *dev, u8 log,
+			       u8 page, void *buf, unsigned int sectors)
 {
 	struct ata_taskfile tf;
 	unsigned int err_mask;
 
-	DPRINTK("read log page - page %d\n", page);
+	DPRINTK("read log page - log 0x%x, page 0x%x\n", log, page);
 
 	ata_tf_init(dev, &tf);
 	tf.command = ATA_CMD_READ_LOG_EXT;
-	tf.lbal = page;
+	tf.lbal = log;
+	tf.lbam = page;
 	tf.nsect = sectors;
 	tf.hob_nsect = sectors >> 8;
 	tf.flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_LBA48 | ATA_TFLAG_DEVICE;
@@ -1545,7 +1547,7 @@ static int ata_eh_read_log_10h(struct ata_device *dev,
 	u8 csum;
 	int i;
 
-	err_mask = ata_read_log_page(dev, ATA_LOG_SATA_NCQ, buf, 1);
+	err_mask = ata_read_log_page(dev, ATA_LOG_SATA_NCQ, 0, buf, 1);
 	if (err_mask)
 		return -EIO;
 
@@ -2623,6 +2625,8 @@ int ata_eh_reset(struct ata_link *link, int classify,
 	 */
 	while (ata_eh_reset_timeouts[max_tries] != ULONG_MAX)
 		max_tries++;
+	if (link->flags & ATA_LFLAG_RST_ONCE)
+		max_tries = 1;
 	if (link->flags & ATA_LFLAG_NO_HRST)
 		hardreset = NULL;
 	if (link->flags & ATA_LFLAG_NO_SRST)
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 8ec81ca8f65..e3bda074fa1 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1655,7 +1655,7 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 		if (unlikely(scmd->cmd_len < 10))
 			goto invalid_fld;
 		scsi_10_lba_len(cdb, &block, &n_block);
-		if (unlikely(cdb[1] & (1 << 3)))
+		if (cdb[1] & (1 << 3))
 			tf_flags |= ATA_TFLAG_FUA;
 		break;
 	case READ_6:
@@ -1675,7 +1675,7 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc)
 		if (unlikely(scmd->cmd_len < 16))
 			goto invalid_fld;
 		scsi_16_lba_len(cdb, &block, &n_block);
-		if (unlikely(cdb[1] & (1 << 3)))
+		if (cdb[1] & (1 << 3))
 			tf_flags |= ATA_TFLAG_FUA;
 		break;
 	default:
@@ -2205,9 +2205,33 @@ static unsigned int ata_scsiop_noop(struct ata_scsi_args *args, u8 *rbuf)
 }
 
 /**
+ *	modecpy - Prepare response for MODE SENSE
+ *	@dest: output buffer
+ *	@src: data being copied
+ *	@n: length of mode page
+ *	@changeable: whether changeable parameters are requested
+ *
+ *	Generate a generic MODE SENSE page for either current or changeable
+ *	parameters.
+ *
+ *	LOCKING:
+ *	None.
+ */
+static void modecpy(u8 *dest, const u8 *src, int n, bool changeable)
+{
+	if (changeable) {
+		memcpy(dest, src, 2);
+		memset(dest + 2, 0, n - 2);
+	} else {
+		memcpy(dest, src, n);
+	}
+}
+
+/**
  *	ata_msense_caching - Simulate MODE SENSE caching info page
  *	@id: device IDENTIFY data
  *	@buf: output buffer
+ *	@changeable: whether changeable parameters are requested
  *
  *	Generate a caching info page, which conditionally indicates
  *	write caching to the SCSI layer, depending on device
@@ -2216,12 +2240,12 @@ static unsigned int ata_scsiop_noop(struct ata_scsi_args *args, u8 *rbuf)
  *	LOCKING:
  *	None.
  */
-static unsigned int ata_msense_caching(u16 *id, u8 *buf)
+static unsigned int ata_msense_caching(u16 *id, u8 *buf, bool changeable)
 {
-	memcpy(buf, def_cache_mpage, sizeof(def_cache_mpage));
-	if (ata_id_wcache_enabled(id))
+	modecpy(buf, def_cache_mpage, sizeof(def_cache_mpage), changeable);
+	if (changeable || ata_id_wcache_enabled(id))
 		buf[2] |= (1 << 2);	/* write cache enable */
-	if (!ata_id_rahead_enabled(id))
+	if (!changeable && !ata_id_rahead_enabled(id))
 		buf[12] |= (1 << 5);	/* disable read ahead */
 	return sizeof(def_cache_mpage);
 }
@@ -2229,30 +2253,33 @@ static unsigned int ata_msense_caching(u16 *id, u8 *buf)
 /**
  *	ata_msense_ctl_mode - Simulate MODE SENSE control mode page
  *	@buf: output buffer
+ *	@changeable: whether changeable parameters are requested
  *
  *	Generate a generic MODE SENSE control mode page.
  *
  *	LOCKING:
  *	None.
  */
-static unsigned int ata_msense_ctl_mode(u8 *buf)
+static unsigned int ata_msense_ctl_mode(u8 *buf, bool changeable)
 {
-	memcpy(buf, def_control_mpage, sizeof(def_control_mpage));
+	modecpy(buf, def_control_mpage, sizeof(def_control_mpage), changeable);
 	return sizeof(def_control_mpage);
 }
 
 /**
  *	ata_msense_rw_recovery - Simulate MODE SENSE r/w error recovery page
  *	@buf: output buffer
+ *	@changeable: whether changeable parameters are requested
  *
  *	Generate a generic MODE SENSE r/w error recovery page.
  *
  *	LOCKING:
  *	None.
  */
-static unsigned int ata_msense_rw_recovery(u8 *buf)
+static unsigned int ata_msense_rw_recovery(u8 *buf, bool changeable)
 {
-	memcpy(buf, def_rw_recovery_mpage, sizeof(def_rw_recovery_mpage));
+	modecpy(buf, def_rw_recovery_mpage, sizeof(def_rw_recovery_mpage),
+		changeable);
 	return sizeof(def_rw_recovery_mpage);
 }
 
@@ -2316,11 +2343,11 @@ static unsigned int ata_scsiop_mode_sense(struct ata_scsi_args *args, u8 *rbuf)
 	page_control = scsicmd[2] >> 6;
 	switch (page_control) {
 	case 0: /* current */
+	case 1: /* changeable */
+	case 2: /* defaults */
 		break;  /* supported */
 	case 3: /* saved */
 		goto saving_not_supp;
-	case 1: /* changeable */
-	case 2: /* defaults */
 	default:
 		goto invalid_fld;
 	}
@@ -2341,21 +2368,21 @@ static unsigned int ata_scsiop_mode_sense(struct ata_scsi_args *args, u8 *rbuf)
 
 	switch(pg) {
 	case RW_RECOVERY_MPAGE:
-		p += ata_msense_rw_recovery(p);
+		p += ata_msense_rw_recovery(p, page_control == 1);
 		break;
 
 	case CACHE_MPAGE:
-		p += ata_msense_caching(args->id, p);
+		p += ata_msense_caching(args->id, p, page_control == 1);
 		break;
 
 	case CONTROL_MPAGE:
-		p += ata_msense_ctl_mode(p);
+		p += ata_msense_ctl_mode(p, page_control == 1);
 		break;
 
 	case ALL_MPAGES:
-		p += ata_msense_rw_recovery(p);
-		p += ata_msense_caching(args->id, p);
-		p += ata_msense_ctl_mode(p);
+		p += ata_msense_rw_recovery(p, page_control == 1);
+		p += ata_msense_caching(args->id, p, page_control == 1);
+		p += ata_msense_ctl_mode(p, page_control == 1);
 		break;
 
 	default:		/* invalid page code */
@@ -3080,6 +3107,188 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc)
 }
 
 /**
+ *	ata_mselect_caching - Simulate MODE SELECT for caching info page
+ *	@qc: Storage for translated ATA taskfile
+ *	@buf: input buffer
+ *	@len: number of valid bytes in the input buffer
+ *
+ *	Prepare a taskfile to modify caching information for the device.
+ *
+ *	LOCKING:
+ *	None.
+ */
+static int ata_mselect_caching(struct ata_queued_cmd *qc,
+			       const u8 *buf, int len)
+{
+	struct ata_taskfile *tf = &qc->tf;
+	struct ata_device *dev = qc->dev;
+	char mpage[CACHE_MPAGE_LEN];
+	u8 wce;
+
+	/*
+	 * The first two bytes of def_cache_mpage are a header, so offsets
+	 * in mpage are off by 2 compared to buf.  Same for len.
+	 */
+
+	if (len != CACHE_MPAGE_LEN - 2)
+		return -EINVAL;
+
+	wce = buf[0] & (1 << 2);
+
+	/*
+	 * Check that read-only bits are not modified.
+	 */
+	ata_msense_caching(dev->id, mpage, false);
+	mpage[2] &= ~(1 << 2);
+	mpage[2] |= wce;
+	if (memcmp(mpage + 2, buf, CACHE_MPAGE_LEN - 2) != 0)
+		return -EINVAL;
+
+	tf->flags |= ATA_TFLAG_DEVICE | ATA_TFLAG_ISADDR;
+	tf->protocol = ATA_PROT_NODATA;
+	tf->nsect = 0;
+	tf->command = ATA_CMD_SET_FEATURES;
+	tf->feature = wce ? SETFEATURES_WC_ON : SETFEATURES_WC_OFF;
+	return 0;
+}
+
+/**
+ *	ata_scsiop_mode_select - Simulate MODE SELECT 6, 10 commands
+ *	@qc: Storage for translated ATA taskfile
+ *
+ *	Converts a MODE SELECT command to an ATA SET FEATURES taskfile.
+ *	Assume this is invoked for direct access devices (e.g. disks) only.
+ *	There should be no block descriptor for other device types.
+ *
+ *	LOCKING:
+ *	spin_lock_irqsave(host lock)
+ */
+static unsigned int ata_scsi_mode_select_xlat(struct ata_queued_cmd *qc)
+{
+	struct scsi_cmnd *scmd = qc->scsicmd;
+	const u8 *cdb = scmd->cmnd;
+	const u8 *p;
+	u8 pg, spg;
+	unsigned six_byte, pg_len, hdr_len, bd_len;
+	int len;
+
+	VPRINTK("ENTER\n");
+
+	six_byte = (cdb[0] == MODE_SELECT);
+	if (six_byte) {
+		if (scmd->cmd_len < 5)
+			goto invalid_fld;
+
+		len = cdb[4];
+		hdr_len = 4;
+	} else {
+		if (scmd->cmd_len < 9)
+			goto invalid_fld;
+
+		len = (cdb[7] << 8) + cdb[8];
+		hdr_len = 8;
+	}
+
+	/* We only support PF=1, SP=0.  */
+	if ((cdb[1] & 0x11) != 0x10)
+		goto invalid_fld;
+
+	/* Test early for possible overrun.  */
+	if (!scsi_sg_count(scmd) || scsi_sglist(scmd)->length < len)
+		goto invalid_param_len;
+
+	p = page_address(sg_page(scsi_sglist(scmd)));
+
+	/* Move past header and block descriptors.  */
+	if (len < hdr_len)
+		goto invalid_param_len;
+
+	if (six_byte)
+		bd_len = p[3];
+	else
+		bd_len = (p[6] << 8) + p[7];
+
+	len -= hdr_len;
+	p += hdr_len;
+	if (len < bd_len)
+		goto invalid_param_len;
+	if (bd_len != 0 && bd_len != 8)
+		goto invalid_param;
+
+	len -= bd_len;
+	p += bd_len;
+	if (len == 0)
+		goto skip;
+
+	/* Parse both possible formats for the mode page headers.  */
+	pg = p[0] & 0x3f;
+	if (p[0] & 0x40) {
+		if (len < 4)
+			goto invalid_param_len;
+
+		spg = p[1];
+		pg_len = (p[2] << 8) | p[3];
+		p += 4;
+		len -= 4;
+	} else {
+		if (len < 2)
+			goto invalid_param_len;
+
+		spg = 0;
+		pg_len = p[1];
+		p += 2;
+		len -= 2;
+	}
+
+	/*
+	 * No mode subpages supported (yet) but asking for _all_
+	 * subpages may be valid
+	 */
+	if (spg && (spg != ALL_SUB_MPAGES))
+		goto invalid_param;
+	if (pg_len > len)
+		goto invalid_param_len;
+
+	switch (pg) {
+	case CACHE_MPAGE:
+		if (ata_mselect_caching(qc, p, pg_len) < 0)
+			goto invalid_param;
+		break;
+
+	default:		/* invalid page code */
+		goto invalid_param;
+	}
+
+	/*
+	 * Only one page has changeable data, so we only support setting one
+	 * page at a time.
+	 */
+	if (len > pg_len)
+		goto invalid_param;
+
+	return 0;
+
+ invalid_fld:
+	/* "Invalid field in CDB" */
+	ata_scsi_set_sense(scmd, ILLEGAL_REQUEST, 0x24, 0x0);
+	return 1;
+
+ invalid_param:
+	/* "Invalid field in parameter list" */
+	ata_scsi_set_sense(scmd, ILLEGAL_REQUEST, 0x26, 0x0);
+	return 1;
+
+ invalid_param_len:
+	/* "Parameter list length error" */
+	ata_scsi_set_sense(scmd, ILLEGAL_REQUEST, 0x1a, 0x0);
+	return 1;
+
+ skip:
+	scmd->result = SAM_STAT_GOOD;
+	return 1;
+}
+
+/**
  *	ata_get_xlat_func - check if SCSI to ATA translation is possible
  *	@dev: ATA device
  *	@cmd: SCSI command opcode to consider
@@ -3119,6 +3328,11 @@ static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev, u8 cmd)
 	case ATA_16:
 		return ata_scsi_pass_thru;
 
+	case MODE_SELECT:
+	case MODE_SELECT_10:
+		return ata_scsi_mode_select_xlat;
+		break;
+
 	case START_STOP:
 		return ata_scsi_start_stop_xlat;
 	}
@@ -3311,11 +3525,6 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd)
 		ata_scsi_rbuf_fill(&args, ata_scsiop_mode_sense);
 		break;
 
-	case MODE_SELECT:	/* unconditionally return */
-	case MODE_SELECT_10:	/* bad-field-in-cdb */
-		ata_scsi_invalid_field(cmd);
-		break;
-
 	case READ_CAPACITY:
 		ata_scsi_rbuf_fill(&args, ata_scsiop_read_cap);
 		break;
diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h
index 50e4dff0604..7148a58020b 100644
--- a/drivers/ata/libata.h
+++ b/drivers/ata/libata.h
@@ -165,6 +165,8 @@ extern void ata_eh_about_to_do(struct ata_link *link, struct ata_device *dev,
 			       unsigned int action);
 extern void ata_eh_done(struct ata_link *link, struct ata_device *dev,
 			unsigned int action);
+extern unsigned int ata_read_log_page(struct ata_device *dev, u8 log,
+				      u8 page, void *buf, unsigned int sectors);
 extern void ata_eh_autopsy(struct ata_port *ap);
 const char *ata_get_cmd_descript(u8 command);
 extern void ata_eh_report(struct ata_port *ap);
diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c
index bfaa5cb1629..26201ebef3c 100644
--- a/drivers/ata/pata_arasan_cf.c
+++ b/drivers/ata/pata_arasan_cf.c
@@ -31,6 +31,7 @@
 #include <linux/kernel.h>
 #include <linux/libata.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/pata_arasan_cf_data.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
@@ -310,7 +311,7 @@ static int cf_init(struct arasan_cf_dev *acdev)
 	unsigned long flags;
 	int ret = 0;
 
-	ret = clk_enable(acdev->clk);
+	ret = clk_prepare_enable(acdev->clk);
 	if (ret) {
 		dev_dbg(acdev->host->dev, "clock enable failed");
 		return ret;
@@ -340,7 +341,7 @@ static void cf_exit(struct arasan_cf_dev *acdev)
 	writel(readl(acdev->vbase + OP_MODE) & ~CFHOST_ENB,
 			acdev->vbase + OP_MODE);
 	spin_unlock_irqrestore(&acdev->host->lock, flags);
-	clk_disable(acdev->clk);
+	clk_disable_unprepare(acdev->clk);
 }
 
 static void dma_callback(void *dev)
@@ -935,6 +936,14 @@ static int arasan_cf_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(arasan_cf_pm_ops, arasan_cf_suspend, arasan_cf_resume);
 
+#ifdef CONFIG_OF
+static const struct of_device_id arasan_cf_id_table[] = {
+	{ .compatible = "arasan,cf-spear1340" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, arasan_cf_id_table);
+#endif
+
 static struct platform_driver arasan_cf_driver = {
 	.probe		= arasan_cf_probe,
 	.remove		= __devexit_p(arasan_cf_remove),
@@ -942,6 +951,7 @@ static struct platform_driver arasan_cf_driver = {
 		.name	= DRIVER_NAME,
 		.owner	= THIS_MODULE,
 		.pm	= &arasan_cf_pm_ops,
+		.of_match_table = of_match_ptr(arasan_cf_id_table),
 	},
 };
 
diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c
index d6577b93bee..124b2c1d9c0 100644
--- a/drivers/ata/sata_fsl.c
+++ b/drivers/ata/sata_fsl.c
@@ -123,6 +123,7 @@ enum {
 	ONLINE = (1 << 31),
 	GOING_OFFLINE = (1 << 30),
 	BIST_ERR = (1 << 29),
+	CLEAR_ERROR = (1 << 27),
 
 	FATAL_ERR_HC_MASTER_ERR = (1 << 18),
 	FATAL_ERR_PARITY_ERR_TX = (1 << 17),
@@ -143,6 +144,7 @@ enum {
 	    FATAL_ERR_CRC_ERR_RX |
 	    FATAL_ERR_FIFO_OVRFL_TX | FATAL_ERR_FIFO_OVRFL_RX,
 
+	INT_ON_DATA_LENGTH_MISMATCH = (1 << 12),
 	INT_ON_FATAL_ERR = (1 << 5),
 	INT_ON_PHYRDY_CHG = (1 << 4),
 
@@ -1181,25 +1183,54 @@ static void sata_fsl_host_intr(struct ata_port *ap)
 	u32 hstatus, done_mask = 0;
 	struct ata_queued_cmd *qc;
 	u32 SError;
+	u32 tag;
+	u32 status_mask = INT_ON_ERROR;
 
 	hstatus = ioread32(hcr_base + HSTATUS);
 
 	sata_fsl_scr_read(&ap->link, SCR_ERROR, &SError);
 
+	/* Read command completed register */
+	done_mask = ioread32(hcr_base + CC);
+
+	/* Workaround for data length mismatch errata */
+	if (unlikely(hstatus & INT_ON_DATA_LENGTH_MISMATCH)) {
+		for (tag = 0; tag < ATA_MAX_QUEUE; tag++) {
+			qc = ata_qc_from_tag(ap, tag);
+			if (qc && ata_is_atapi(qc->tf.protocol)) {
+				u32 hcontrol;
+				/* Set HControl[27] to clear error registers */
+				hcontrol = ioread32(hcr_base + HCONTROL);
+				iowrite32(hcontrol | CLEAR_ERROR,
+						hcr_base + HCONTROL);
+
+				/* Clear HControl[27] */
+				iowrite32(hcontrol & ~CLEAR_ERROR,
+						hcr_base + HCONTROL);
+
+				/* Clear SError[E] bit */
+				sata_fsl_scr_write(&ap->link, SCR_ERROR,
+						SError);
+
+				/* Ignore fatal error and device error */
+				status_mask &= ~(INT_ON_SINGL_DEVICE_ERR
+						| INT_ON_FATAL_ERR);
+				break;
+			}
+		}
+	}
+
 	if (unlikely(SError & 0xFFFF0000)) {
 		DPRINTK("serror @host_intr : 0x%x\n", SError);
 		sata_fsl_error_intr(ap);
 	}
 
-	if (unlikely(hstatus & INT_ON_ERROR)) {
+	if (unlikely(hstatus & status_mask)) {
 		DPRINTK("error interrupt!!\n");
 		sata_fsl_error_intr(ap);
 		return;
 	}
 
-	/* Read command completed register */
-	done_mask = ioread32(hcr_base + CC);
-
 	VPRINTK("Status of all queues :\n");
 	VPRINTK("done_mask/CC = 0x%x, CA = 0x%x, CE=0x%x,CQ=0x%x,apqa=0x%x\n",
 		done_mask,
diff --git a/drivers/ata/sata_highbank.c b/drivers/ata/sata_highbank.c
new file mode 100644
index 00000000000..0d7c4c2cd26
--- /dev/null
+++ b/drivers/ata/sata_highbank.c
@@ -0,0 +1,450 @@
+/*
+ * Calxeda Highbank AHCI SATA platform driver
+ * Copyright 2012 Calxeda, Inc.
+ *
+ * based on the AHCI SATA platform driver by Jeff Garzik and Anton Vorontsov
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/gfp.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/libata.h>
+#include <linux/ahci_platform.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include "ahci.h"
+
+#define CPHY_MAP(dev, addr) ((((dev) & 0x1f) << 7) | (((addr) >> 9) & 0x7f))
+#define CPHY_ADDR(addr) (((addr) & 0x1ff) << 2)
+#define SERDES_CR_CTL			0x80a0
+#define SERDES_CR_ADDR			0x80a1
+#define SERDES_CR_DATA			0x80a2
+#define CR_BUSY				0x0001
+#define CR_START			0x0001
+#define CR_WR_RDN			0x0002
+#define CPHY_RX_INPUT_STS		0x2002
+#define CPHY_SATA_OVERRIDE	 	0x4000
+#define CPHY_OVERRIDE			0x2005
+#define SPHY_LANE			0x100
+#define SPHY_HALF_RATE			0x0001
+#define CPHY_SATA_DPLL_MODE		0x0700
+#define CPHY_SATA_DPLL_SHIFT		8
+#define CPHY_SATA_DPLL_RESET		(1 << 11)
+#define CPHY_PHY_COUNT			6
+#define CPHY_LANE_COUNT			4
+#define CPHY_PORT_COUNT			(CPHY_PHY_COUNT * CPHY_LANE_COUNT)
+
+static DEFINE_SPINLOCK(cphy_lock);
+/* Each of the 6 phys can have up to 4 sata ports attached to i. Map 0-based
+ * sata ports to their phys and then to their lanes within the phys
+ */
+struct phy_lane_info {
+	void __iomem *phy_base;
+	u8 lane_mapping;
+	u8 phy_devs;
+};
+static struct phy_lane_info port_data[CPHY_PORT_COUNT];
+
+static u32 __combo_phy_reg_read(u8 sata_port, u32 addr)
+{
+	u32 data;
+	u8 dev = port_data[sata_port].phy_devs;
+	spin_lock(&cphy_lock);
+	writel(CPHY_MAP(dev, addr), port_data[sata_port].phy_base + 0x800);
+	data = readl(port_data[sata_port].phy_base + CPHY_ADDR(addr));
+	spin_unlock(&cphy_lock);
+	return data;
+}
+
+static void __combo_phy_reg_write(u8 sata_port, u32 addr, u32 data)
+{
+	u8 dev = port_data[sata_port].phy_devs;
+	spin_lock(&cphy_lock);
+	writel(CPHY_MAP(dev, addr), port_data[sata_port].phy_base + 0x800);
+	writel(data, port_data[sata_port].phy_base + CPHY_ADDR(addr));
+	spin_unlock(&cphy_lock);
+}
+
+static void combo_phy_wait_for_ready(u8 sata_port)
+{
+	while (__combo_phy_reg_read(sata_port, SERDES_CR_CTL) & CR_BUSY)
+		udelay(5);
+}
+
+static u32 combo_phy_read(u8 sata_port, u32 addr)
+{
+	combo_phy_wait_for_ready(sata_port);
+	__combo_phy_reg_write(sata_port, SERDES_CR_ADDR, addr);
+	__combo_phy_reg_write(sata_port, SERDES_CR_CTL, CR_START);
+	combo_phy_wait_for_ready(sata_port);
+	return __combo_phy_reg_read(sata_port, SERDES_CR_DATA);
+}
+
+static void combo_phy_write(u8 sata_port, u32 addr, u32 data)
+{
+	combo_phy_wait_for_ready(sata_port);
+	__combo_phy_reg_write(sata_port, SERDES_CR_ADDR, addr);
+	__combo_phy_reg_write(sata_port, SERDES_CR_DATA, data);
+	__combo_phy_reg_write(sata_port, SERDES_CR_CTL, CR_WR_RDN | CR_START);
+}
+
+static void highbank_cphy_disable_overrides(u8 sata_port)
+{
+	u8 lane = port_data[sata_port].lane_mapping;
+	u32 tmp;
+	if (unlikely(port_data[sata_port].phy_base == NULL))
+		return;
+	tmp = combo_phy_read(sata_port, CPHY_RX_INPUT_STS + lane * SPHY_LANE);
+	tmp &= ~CPHY_SATA_OVERRIDE;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+}
+
+static void cphy_override_rx_mode(u8 sata_port, u32 val)
+{
+	u8 lane = port_data[sata_port].lane_mapping;
+	u32 tmp;
+	tmp = combo_phy_read(sata_port, CPHY_RX_INPUT_STS + lane * SPHY_LANE);
+	tmp &= ~CPHY_SATA_OVERRIDE;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+
+	tmp |= CPHY_SATA_OVERRIDE;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+
+	tmp &= ~CPHY_SATA_DPLL_MODE;
+	tmp |= val << CPHY_SATA_DPLL_SHIFT;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+
+	tmp |= CPHY_SATA_DPLL_RESET;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+
+	tmp &= ~CPHY_SATA_DPLL_RESET;
+	combo_phy_write(sata_port, CPHY_OVERRIDE + lane * SPHY_LANE, tmp);
+
+	msleep(15);
+}
+
+static void highbank_cphy_override_lane(u8 sata_port)
+{
+	u8 lane = port_data[sata_port].lane_mapping;
+	u32 tmp, k = 0;
+
+	if (unlikely(port_data[sata_port].phy_base == NULL))
+		return;
+	do {
+		tmp = combo_phy_read(sata_port, CPHY_RX_INPUT_STS +
+						lane * SPHY_LANE);
+	} while ((tmp & SPHY_HALF_RATE) && (k++ < 1000));
+	cphy_override_rx_mode(sata_port, 3);
+}
+
+static int highbank_initialize_phys(struct device *dev, void __iomem *addr)
+{
+	struct device_node *sata_node = dev->of_node;
+	int phy_count = 0, phy, port = 0;
+	void __iomem *cphy_base[CPHY_PHY_COUNT];
+	struct device_node *phy_nodes[CPHY_PHY_COUNT];
+	memset(port_data, 0, sizeof(struct phy_lane_info) * CPHY_PORT_COUNT);
+	memset(phy_nodes, 0, sizeof(struct device_node*) * CPHY_PHY_COUNT);
+
+	do {
+		u32 tmp;
+		struct of_phandle_args phy_data;
+		if (of_parse_phandle_with_args(sata_node,
+				"calxeda,port-phys", "#phy-cells",
+				port, &phy_data))
+			break;
+		for (phy = 0; phy < phy_count; phy++) {
+			if (phy_nodes[phy] == phy_data.np)
+				break;
+		}
+		if (phy_nodes[phy] == NULL) {
+			phy_nodes[phy] = phy_data.np;
+			cphy_base[phy] = of_iomap(phy_nodes[phy], 0);
+			if (cphy_base[phy] == NULL) {
+				return 0;
+			}
+			phy_count += 1;
+		}
+		port_data[port].lane_mapping = phy_data.args[0];
+		of_property_read_u32(phy_nodes[phy], "phydev", &tmp);
+		port_data[port].phy_devs = tmp;
+		port_data[port].phy_base = cphy_base[phy];
+		of_node_put(phy_data.np);
+		port += 1;
+	} while (port < CPHY_PORT_COUNT);
+	return 0;
+}
+
+static int ahci_highbank_hardreset(struct ata_link *link, unsigned int *class,
+				unsigned long deadline)
+{
+	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
+	struct ata_port *ap = link->ap;
+	struct ahci_port_priv *pp = ap->private_data;
+	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
+	struct ata_taskfile tf;
+	bool online;
+	u32 sstatus;
+	int rc;
+	int retry = 10;
+
+	ahci_stop_engine(ap);
+
+	/* clear D2H reception area to properly wait for D2H FIS */
+	ata_tf_init(link->device, &tf);
+	tf.command = 0x80;
+	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
+
+	do {
+		highbank_cphy_disable_overrides(link->ap->port_no);
+		rc = sata_link_hardreset(link, timing, deadline, &online, NULL);
+		highbank_cphy_override_lane(link->ap->port_no);
+
+		/* If the status is 1, we are connected, but the link did not
+		 * come up. So retry resetting the link again.
+		 */
+		if (sata_scr_read(link, SCR_STATUS, &sstatus))
+			break;
+		if (!(sstatus & 0x3))
+			break;
+	} while (!online && retry--);
+
+	ahci_start_engine(ap);
+
+	if (online)
+		*class = ahci_dev_classify(ap);
+
+	return rc;
+}
+
+static struct ata_port_operations ahci_highbank_ops = {
+	.inherits		= &ahci_ops,
+	.hardreset		= ahci_highbank_hardreset,
+};
+
+static const struct ata_port_info ahci_highbank_port_info = {
+	.flags          = AHCI_FLAG_COMMON,
+	.pio_mask       = ATA_PIO4,
+	.udma_mask      = ATA_UDMA6,
+	.port_ops       = &ahci_highbank_ops,
+};
+
+static struct scsi_host_template ahci_highbank_platform_sht = {
+	AHCI_SHT("highbank-ahci"),
+};
+
+static const struct of_device_id ahci_of_match[] = {
+	{ .compatible = "calxeda,hb-ahci" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ahci_of_match);
+
+static int __init ahci_highbank_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ahci_host_priv *hpriv;
+	struct ata_host *host;
+	struct resource *mem;
+	int irq;
+	int n_ports;
+	int i;
+	int rc;
+	struct ata_port_info pi = ahci_highbank_port_info;
+	const struct ata_port_info *ppi[] = { &pi, NULL };
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!mem) {
+		dev_err(dev, "no mmio space\n");
+		return -EINVAL;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0) {
+		dev_err(dev, "no irq\n");
+		return -EINVAL;
+	}
+
+	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
+	if (!hpriv) {
+		dev_err(dev, "can't alloc ahci_host_priv\n");
+		return -ENOMEM;
+	}
+
+	hpriv->flags |= (unsigned long)pi.private_data;
+
+	hpriv->mmio = devm_ioremap(dev, mem->start, resource_size(mem));
+	if (!hpriv->mmio) {
+		dev_err(dev, "can't map %pR\n", mem);
+		return -ENOMEM;
+	}
+
+	rc = highbank_initialize_phys(dev, hpriv->mmio);
+	if (rc)
+		return rc;
+
+
+	ahci_save_initial_config(dev, hpriv, 0, 0);
+
+	/* prepare host */
+	if (hpriv->cap & HOST_CAP_NCQ)
+		pi.flags |= ATA_FLAG_NCQ;
+
+	if (hpriv->cap & HOST_CAP_PMP)
+		pi.flags |= ATA_FLAG_PMP;
+
+	ahci_set_em_messages(hpriv, &pi);
+
+	/* CAP.NP sometimes indicate the index of the last enabled
+	 * port, at other times, that of the last possible port, so
+	 * determining the maximum port number requires looking at
+	 * both CAP.NP and port_map.
+	 */
+	n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
+
+	host = ata_host_alloc_pinfo(dev, ppi, n_ports);
+	if (!host) {
+		rc = -ENOMEM;
+		goto err0;
+	}
+
+	host->private_data = hpriv;
+
+	if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss)
+		host->flags |= ATA_HOST_PARALLEL_SCAN;
+
+	if (pi.flags & ATA_FLAG_EM)
+		ahci_reset_em(host);
+
+	for (i = 0; i < host->n_ports; i++) {
+		struct ata_port *ap = host->ports[i];
+
+		ata_port_desc(ap, "mmio %pR", mem);
+		ata_port_desc(ap, "port 0x%x", 0x100 + ap->port_no * 0x80);
+
+		/* set enclosure management message type */
+		if (ap->flags & ATA_FLAG_EM)
+			ap->em_message_type = hpriv->em_msg_type;
+
+		/* disabled/not-implemented port */
+		if (!(hpriv->port_map & (1 << i)))
+			ap->ops = &ata_dummy_port_ops;
+	}
+
+	rc = ahci_reset_controller(host);
+	if (rc)
+		goto err0;
+
+	ahci_init_controller(host);
+	ahci_print_info(host, "platform");
+
+	rc = ata_host_activate(host, irq, ahci_interrupt, 0,
+					&ahci_highbank_platform_sht);
+	if (rc)
+		goto err0;
+
+	return 0;
+err0:
+	return rc;
+}
+
+static int __devexit ahci_highbank_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ata_host *host = dev_get_drvdata(dev);
+
+	ata_host_detach(host);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int ahci_highbank_suspend(struct device *dev)
+{
+	struct ata_host *host = dev_get_drvdata(dev);
+	struct ahci_host_priv *hpriv = host->private_data;
+	void __iomem *mmio = hpriv->mmio;
+	u32 ctl;
+	int rc;
+
+	if (hpriv->flags & AHCI_HFLAG_NO_SUSPEND) {
+		dev_err(dev, "firmware update required for suspend/resume\n");
+		return -EIO;
+	}
+
+	/*
+	 * AHCI spec rev1.1 section 8.3.3:
+	 * Software must disable interrupts prior to requesting a
+	 * transition of the HBA to D3 state.
+	 */
+	ctl = readl(mmio + HOST_CTL);
+	ctl &= ~HOST_IRQ_EN;
+	writel(ctl, mmio + HOST_CTL);
+	readl(mmio + HOST_CTL); /* flush */
+
+	rc = ata_host_suspend(host, PMSG_SUSPEND);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
+static int ahci_highbank_resume(struct device *dev)
+{
+	struct ata_host *host = dev_get_drvdata(dev);
+	int rc;
+
+	if (dev->power.power_state.event == PM_EVENT_SUSPEND) {
+		rc = ahci_reset_controller(host);
+		if (rc)
+			return rc;
+
+		ahci_init_controller(host);
+	}
+
+	ata_host_resume(host);
+
+	return 0;
+}
+#endif
+
+SIMPLE_DEV_PM_OPS(ahci_highbank_pm_ops,
+		  ahci_highbank_suspend, ahci_highbank_resume);
+
+static struct platform_driver ahci_highbank_driver = {
+        .remove = __devexit_p(ahci_highbank_remove),
+        .driver = {
+                .name = "highbank-ahci",
+                .owner = THIS_MODULE,
+                .of_match_table = ahci_of_match,
+                .pm = &ahci_highbank_pm_ops,
+        },
+	.probe = ahci_highbank_probe,
+};
+
+module_platform_driver(ahci_highbank_driver);
+
+MODULE_DESCRIPTION("Calxeda Highbank AHCI SATA platform driver");
+MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@calxeda.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("sata:highbank");
diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c
index 311be18d3f0..68f4fb54d62 100644
--- a/drivers/ata/sata_mv.c
+++ b/drivers/ata/sata_mv.c
@@ -79,8 +79,8 @@
  * module options
  */
 
-static int msi;
 #ifdef CONFIG_PCI
+static int msi;
 module_param(msi, int, S_IRUGO);
 MODULE_PARM_DESC(msi, "Enable use of PCI MSI (0=off, 1=on)");
 #endif
@@ -652,12 +652,13 @@ static u8 mv_sff_check_status(struct ata_port *ap);
  * because we have to allow room for worst case splitting of
  * PRDs for 64K boundaries in mv_fill_sg().
  */
+#ifdef CONFIG_PCI
 static struct scsi_host_template mv5_sht = {
 	ATA_BASE_SHT(DRV_NAME),
 	.sg_tablesize		= MV_MAX_SG_CT / 2,
 	.dma_boundary		= MV_DMA_BOUNDARY,
 };
-
+#endif
 static struct scsi_host_template mv6_sht = {
 	ATA_NCQ_SHT(DRV_NAME),
 	.can_queue		= MV_MAX_Q_DEPTH - 1,
@@ -1252,7 +1253,7 @@ static void mv_dump_mem(void __iomem *start, unsigned bytes)
 	}
 }
 #endif
-
+#if defined(ATA_DEBUG) || defined(CONFIG_PCI)
 static void mv_dump_pci_cfg(struct pci_dev *pdev, unsigned bytes)
 {
 #ifdef ATA_DEBUG
@@ -1269,6 +1270,7 @@ static void mv_dump_pci_cfg(struct pci_dev *pdev, unsigned bytes)
 	}
 #endif
 }
+#endif
 static void mv_dump_all_regs(void __iomem *mmio_base, int port,
 			     struct pci_dev *pdev)
 {
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index 2059ee460b0..81e44f7b0ab 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -1567,7 +1567,7 @@ tx_complete++;
 /*--------------------------------- entries ---------------------------------*/
 
 
-static const char *media_name[] __devinitdata = {
+static char * const media_name[] __devinitconst = {
     "MMF", "SMF", "MMF", "03?", /*  0- 3 */
     "UTP", "05?", "06?", "07?", /*  4- 7 */
     "TAXI","09?", "10?", "11?", /*  8-11 */
diff --git a/drivers/base/dma-buf.c b/drivers/base/dma-buf.c
index c30f3e1d0ef..460e22dee36 100644
--- a/drivers/base/dma-buf.c
+++ b/drivers/base/dma-buf.c
@@ -460,8 +460,7 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
 	if (vma->vm_file)
 		fput(vma->vm_file);
 
-	vma->vm_file = dmabuf->file;
-	get_file(vma->vm_file);
+	vma->vm_file = get_file(dmabuf->file);
 
 	vma->vm_pgoff = pgoff;
 
diff --git a/drivers/base/dma-contiguous.c b/drivers/base/dma-contiguous.c
index 34d94c762a1..9a1469474f5 100644
--- a/drivers/base/dma-contiguous.c
+++ b/drivers/base/dma-contiguous.c
@@ -315,6 +315,7 @@ struct page *dma_alloc_from_contiguous(struct device *dev, int count,
 {
 	unsigned long mask, pfn, pageno, start = 0;
 	struct cma *cma = dev_get_cma_area(dev);
+	struct page *page = NULL;
 	int ret;
 
 	if (!cma || !cma->count)
@@ -336,18 +337,17 @@ struct page *dma_alloc_from_contiguous(struct device *dev, int count,
 	for (;;) {
 		pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,
 						    start, count, mask);
-		if (pageno >= cma->count) {
-			ret = -ENOMEM;
-			goto error;
-		}
+		if (pageno >= cma->count)
+			break;
 
 		pfn = cma->base_pfn + pageno;
 		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
 		if (ret == 0) {
 			bitmap_set(cma->bitmap, pageno, count);
+			page = pfn_to_page(pfn);
 			break;
 		} else if (ret != -EBUSY) {
-			goto error;
+			break;
 		}
 		pr_debug("%s(): memory range at %p is busy, retrying\n",
 			 __func__, pfn_to_page(pfn));
@@ -356,12 +356,8 @@ struct page *dma_alloc_from_contiguous(struct device *dev, int count,
 	}
 
 	mutex_unlock(&cma_mutex);
-
-	pr_debug("%s(): returned %p\n", __func__, pfn_to_page(pfn));
-	return pfn_to_page(pfn);
-error:
-	mutex_unlock(&cma_mutex);
-	return NULL;
+	pr_debug("%s(): returned %p\n", __func__, page);
+	return page;
 }
 
 /**
diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c
index 6e210802c37..81541452887 100644
--- a/drivers/base/firmware_class.c
+++ b/drivers/base/firmware_class.c
@@ -21,18 +21,83 @@
 #include <linux/firmware.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
+#include <linux/file.h>
 #include <linux/list.h>
 #include <linux/async.h>
 #include <linux/pm.h>
 #include <linux/suspend.h>
 #include <linux/syscore_ops.h>
 
+#include <generated/utsrelease.h>
+
 #include "base.h"
 
 MODULE_AUTHOR("Manuel Estrada Sainz");
 MODULE_DESCRIPTION("Multi purpose firmware loading support");
 MODULE_LICENSE("GPL");
 
+static const char *fw_path[] = {
+	"/lib/firmware/updates/" UTS_RELEASE,
+	"/lib/firmware/updates",
+	"/lib/firmware/" UTS_RELEASE,
+	"/lib/firmware"
+};
+
+/* Don't inline this: 'struct kstat' is biggish */
+static noinline long fw_file_size(struct file *file)
+{
+	struct kstat st;
+	if (vfs_getattr(file->f_path.mnt, file->f_path.dentry, &st))
+		return -1;
+	if (!S_ISREG(st.mode))
+		return -1;
+	if (st.size != (long)st.size)
+		return -1;
+	return st.size;
+}
+
+static bool fw_read_file_contents(struct file *file, struct firmware *fw)
+{
+	long size;
+	char *buf;
+
+	size = fw_file_size(file);
+	if (size < 0)
+		return false;
+	buf = vmalloc(size);
+	if (!buf)
+		return false;
+	if (kernel_read(file, 0, buf, size) != size) {
+		vfree(buf);
+		return false;
+	}
+	fw->data = buf;
+	fw->size = size;
+	return true;
+}
+
+static bool fw_get_filesystem_firmware(struct firmware *fw, const char *name)
+{
+	int i;
+	bool success = false;
+	char *path = __getname();
+
+	for (i = 0; i < ARRAY_SIZE(fw_path); i++) {
+		struct file *file;
+		snprintf(path, PATH_MAX, "%s/%s", fw_path[i], name);
+
+		file = filp_open(path, O_RDONLY, 0);
+		if (IS_ERR(file))
+			continue;
+		success = fw_read_file_contents(file, fw);
+		fput(file);
+		if (success)
+			break;
+	}
+	__putname(path);
+	return success;
+}
+
 /* Builtin firmware support */
 
 #ifdef CONFIG_FW_LOADER
@@ -346,7 +411,11 @@ static ssize_t firmware_loading_show(struct device *dev,
 /* firmware holds the ownership of pages */
 static void firmware_free_data(const struct firmware *fw)
 {
-	WARN_ON(!fw->priv);
+	/* Loaded directly? */
+	if (!fw->priv) {
+		vfree(fw->data);
+		return;
+	}
 	fw_free_buf(fw->priv);
 }
 
@@ -709,6 +778,11 @@ _request_firmware_prepare(const struct firmware **firmware_p, const char *name,
 		return NULL;
 	}
 
+	if (fw_get_filesystem_firmware(firmware, name)) {
+		dev_dbg(device, "firmware: direct-loading firmware %s\n", name);
+		return NULL;
+	}
+
 	ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf);
 	if (!ret)
 		fw_priv = fw_create_instance(firmware, name, device,
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index ddeca142293..8727e9c5eea 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -23,6 +23,7 @@
 #include <linux/idr.h>
 
 #include "base.h"
+#include "power/power.h"
 
 /* For automatically allocated device IDs */
 static DEFINE_IDA(platform_devid_ida);
@@ -983,6 +984,7 @@ void __init early_platform_add_devices(struct platform_device **devs, int num)
 		dev = &devs[i]->dev;
 
 		if (!dev->devres_head.next) {
+			pm_runtime_early_init(dev);
 			INIT_LIST_HEAD(&dev->devres_head);
 			list_add_tail(&dev->devres_head,
 				      &early_platform_device_list);
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index ba3487c9835..c22b869245d 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -53,6 +53,24 @@
 static LIST_HEAD(gpd_list);
 static DEFINE_MUTEX(gpd_list_lock);
 
+static struct generic_pm_domain *pm_genpd_lookup_name(const char *domain_name)
+{
+	struct generic_pm_domain *genpd = NULL, *gpd;
+
+	if (IS_ERR_OR_NULL(domain_name))
+		return NULL;
+
+	mutex_lock(&gpd_list_lock);
+	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
+		if (!strcmp(gpd->name, domain_name)) {
+			genpd = gpd;
+			break;
+		}
+	}
+	mutex_unlock(&gpd_list_lock);
+	return genpd;
+}
+
 #ifdef CONFIG_PM
 
 struct generic_pm_domain *dev_to_genpd(struct device *dev)
@@ -256,10 +274,28 @@ int pm_genpd_poweron(struct generic_pm_domain *genpd)
 	return ret;
 }
 
+/**
+ * pm_genpd_name_poweron - Restore power to a given PM domain and its masters.
+ * @domain_name: Name of the PM domain to power up.
+ */
+int pm_genpd_name_poweron(const char *domain_name)
+{
+	struct generic_pm_domain *genpd;
+
+	genpd = pm_genpd_lookup_name(domain_name);
+	return genpd ? pm_genpd_poweron(genpd) : -EINVAL;
+}
+
 #endif /* CONFIG_PM */
 
 #ifdef CONFIG_PM_RUNTIME
 
+static int genpd_start_dev_no_timing(struct generic_pm_domain *genpd,
+				     struct device *dev)
+{
+	return GENPD_DEV_CALLBACK(genpd, int, start, dev);
+}
+
 static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)
 {
 	return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev,
@@ -436,7 +472,7 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd)
 	not_suspended = 0;
 	list_for_each_entry(pdd, &genpd->dev_list, list_node)
 		if (pdd->dev->driver && (!pm_runtime_suspended(pdd->dev)
-		    || pdd->dev->power.irq_safe || to_gpd_data(pdd)->always_on))
+		    || pdd->dev->power.irq_safe))
 			not_suspended++;
 
 	if (not_suspended > genpd->in_progress)
@@ -578,9 +614,6 @@ static int pm_genpd_runtime_suspend(struct device *dev)
 
 	might_sleep_if(!genpd->dev_irq_safe);
 
-	if (dev_gpd_data(dev)->always_on)
-		return -EBUSY;
-
 	stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL;
 	if (stop_ok && !stop_ok(dev))
 		return -EBUSY;
@@ -629,7 +662,7 @@ static int pm_genpd_runtime_resume(struct device *dev)
 
 	/* If power.irq_safe, the PM domain is never powered off. */
 	if (dev->power.irq_safe)
-		return genpd_start_dev(genpd, dev);
+		return genpd_start_dev_no_timing(genpd, dev);
 
 	mutex_lock(&genpd->lock);
 	ret = __pm_genpd_poweron(genpd);
@@ -697,6 +730,24 @@ static inline void genpd_power_off_work_fn(struct work_struct *work) {}
 
 #ifdef CONFIG_PM_SLEEP
 
+/**
+ * pm_genpd_present - Check if the given PM domain has been initialized.
+ * @genpd: PM domain to check.
+ */
+static bool pm_genpd_present(struct generic_pm_domain *genpd)
+{
+	struct generic_pm_domain *gpd;
+
+	if (IS_ERR_OR_NULL(genpd))
+		return false;
+
+	list_for_each_entry(gpd, &gpd_list, gpd_list_node)
+		if (gpd == genpd)
+			return true;
+
+	return false;
+}
+
 static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
 				    struct device *dev)
 {
@@ -750,9 +801,10 @@ static int genpd_thaw_dev(struct generic_pm_domain *genpd, struct device *dev)
  * Check if the given PM domain can be powered off (during system suspend or
  * hibernation) and do that if so.  Also, in that case propagate to its masters.
  *
- * This function is only called in "noirq" stages of system power transitions,
- * so it need not acquire locks (all of the "noirq" callbacks are executed
- * sequentially, so it is guaranteed that it will never run twice in parallel).
+ * This function is only called in "noirq" and "syscore" stages of system power
+ * transitions, so it need not acquire locks (all of the "noirq" callbacks are
+ * executed sequentially, so it is guaranteed that it will never run twice in
+ * parallel).
  */
 static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd)
 {
@@ -777,6 +829,33 @@ static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd)
 }
 
 /**
+ * pm_genpd_sync_poweron - Synchronously power on a PM domain and its masters.
+ * @genpd: PM domain to power on.
+ *
+ * This function is only called in "noirq" and "syscore" stages of system power
+ * transitions, so it need not acquire locks (all of the "noirq" callbacks are
+ * executed sequentially, so it is guaranteed that it will never run twice in
+ * parallel).
+ */
+static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd)
+{
+	struct gpd_link *link;
+
+	if (genpd->status != GPD_STATE_POWER_OFF)
+		return;
+
+	list_for_each_entry(link, &genpd->slave_links, slave_node) {
+		pm_genpd_sync_poweron(link->master);
+		genpd_sd_counter_inc(link->master);
+	}
+
+	if (genpd->power_on)
+		genpd->power_on(genpd);
+
+	genpd->status = GPD_STATE_ACTIVE;
+}
+
+/**
  * resume_needed - Check whether to resume a device before system suspend.
  * @dev: Device to check.
  * @genpd: PM domain the device belongs to.
@@ -937,7 +1016,7 @@ static int pm_genpd_suspend_noirq(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	if (genpd->suspend_power_off || dev_gpd_data(dev)->always_on
+	if (genpd->suspend_power_off
 	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
 		return 0;
 
@@ -970,7 +1049,7 @@ static int pm_genpd_resume_noirq(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	if (genpd->suspend_power_off || dev_gpd_data(dev)->always_on
+	if (genpd->suspend_power_off
 	    || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
 		return 0;
 
@@ -979,7 +1058,7 @@ static int pm_genpd_resume_noirq(struct device *dev)
 	 * guaranteed that this function will never run twice in parallel for
 	 * the same PM domain, so it is not necessary to use locking here.
 	 */
-	pm_genpd_poweron(genpd);
+	pm_genpd_sync_poweron(genpd);
 	genpd->suspended_count--;
 
 	return genpd_start_dev(genpd, dev);
@@ -1090,8 +1169,7 @@ static int pm_genpd_freeze_noirq(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	return genpd->suspend_power_off || dev_gpd_data(dev)->always_on ?
-		0 : genpd_stop_dev(genpd, dev);
+	return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev);
 }
 
 /**
@@ -1111,8 +1189,7 @@ static int pm_genpd_thaw_noirq(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	return genpd->suspend_power_off || dev_gpd_data(dev)->always_on ?
-		0 : genpd_start_dev(genpd, dev);
+	return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev);
 }
 
 /**
@@ -1186,8 +1263,8 @@ static int pm_genpd_restore_noirq(struct device *dev)
 	if (genpd->suspended_count++ == 0) {
 		/*
 		 * The boot kernel might put the domain into arbitrary state,
-		 * so make it appear as powered off to pm_genpd_poweron(), so
-		 * that it tries to power it on in case it was really off.
+		 * so make it appear as powered off to pm_genpd_sync_poweron(),
+		 * so that it tries to power it on in case it was really off.
 		 */
 		genpd->status = GPD_STATE_POWER_OFF;
 		if (genpd->suspend_power_off) {
@@ -1205,9 +1282,9 @@ static int pm_genpd_restore_noirq(struct device *dev)
 	if (genpd->suspend_power_off)
 		return 0;
 
-	pm_genpd_poweron(genpd);
+	pm_genpd_sync_poweron(genpd);
 
-	return dev_gpd_data(dev)->always_on ? 0 : genpd_start_dev(genpd, dev);
+	return genpd_start_dev(genpd, dev);
 }
 
 /**
@@ -1246,6 +1323,31 @@ static void pm_genpd_complete(struct device *dev)
 	}
 }
 
+/**
+ * pm_genpd_syscore_switch - Switch power during system core suspend or resume.
+ * @dev: Device that normally is marked as "always on" to switch power for.
+ *
+ * This routine may only be called during the system core (syscore) suspend or
+ * resume phase for devices whose "always on" flags are set.
+ */
+void pm_genpd_syscore_switch(struct device *dev, bool suspend)
+{
+	struct generic_pm_domain *genpd;
+
+	genpd = dev_to_genpd(dev);
+	if (!pm_genpd_present(genpd))
+		return;
+
+	if (suspend) {
+		genpd->suspended_count++;
+		pm_genpd_sync_poweroff(genpd);
+	} else {
+		pm_genpd_sync_poweron(genpd);
+		genpd->suspended_count--;
+	}
+}
+EXPORT_SYMBOL_GPL(pm_genpd_syscore_switch);
+
 #else
 
 #define pm_genpd_prepare		NULL
@@ -1393,6 +1495,19 @@ int __pm_genpd_of_add_device(struct device_node *genpd_node, struct device *dev,
 	return __pm_genpd_add_device(genpd, dev, td);
 }
 
+
+/**
+ * __pm_genpd_name_add_device - Find I/O PM domain and add a device to it.
+ * @domain_name: Name of the PM domain to add the device to.
+ * @dev: Device to be added.
+ * @td: Set of PM QoS timing parameters to attach to the device.
+ */
+int __pm_genpd_name_add_device(const char *domain_name, struct device *dev,
+			       struct gpd_timing_data *td)
+{
+	return __pm_genpd_add_device(pm_genpd_lookup_name(domain_name), dev, td);
+}
+
 /**
  * pm_genpd_remove_device - Remove a device from an I/O PM domain.
  * @genpd: PM domain to remove the device from.
@@ -1455,26 +1570,6 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd,
 }
 
 /**
- * pm_genpd_dev_always_on - Set/unset the "always on" flag for a given device.
- * @dev: Device to set/unset the flag for.
- * @val: The new value of the device's "always on" flag.
- */
-void pm_genpd_dev_always_on(struct device *dev, bool val)
-{
-	struct pm_subsys_data *psd;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev->power.lock, flags);
-
-	psd = dev_to_psd(dev);
-	if (psd && psd->domain_data)
-		to_gpd_data(psd->domain_data)->always_on = val;
-
-	spin_unlock_irqrestore(&dev->power.lock, flags);
-}
-EXPORT_SYMBOL_GPL(pm_genpd_dev_always_on);
-
-/**
  * pm_genpd_dev_need_restore - Set/unset the device's "need restore" flag.
  * @dev: Device to set/unset the flag for.
  * @val: The new value of the device's "need restore" flag.
@@ -1505,7 +1600,8 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 	struct gpd_link *link;
 	int ret = 0;
 
-	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
+	if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)
+	    || genpd == subdomain)
 		return -EINVAL;
 
  start:
@@ -1552,6 +1648,35 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 }
 
 /**
+ * pm_genpd_add_subdomain_names - Add a subdomain to an I/O PM domain.
+ * @master_name: Name of the master PM domain to add the subdomain to.
+ * @subdomain_name: Name of the subdomain to be added.
+ */
+int pm_genpd_add_subdomain_names(const char *master_name,
+				 const char *subdomain_name)
+{
+	struct generic_pm_domain *master = NULL, *subdomain = NULL, *gpd;
+
+	if (IS_ERR_OR_NULL(master_name) || IS_ERR_OR_NULL(subdomain_name))
+		return -EINVAL;
+
+	mutex_lock(&gpd_list_lock);
+	list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
+		if (!master && !strcmp(gpd->name, master_name))
+			master = gpd;
+
+		if (!subdomain && !strcmp(gpd->name, subdomain_name))
+			subdomain = gpd;
+
+		if (master && subdomain)
+			break;
+	}
+	mutex_unlock(&gpd_list_lock);
+
+	return pm_genpd_add_subdomain(master, subdomain);
+}
+
+/**
  * pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain.
  * @genpd: Master PM domain to remove the subdomain from.
  * @subdomain: Subdomain to be removed.
@@ -1704,7 +1829,16 @@ int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td)
 }
 EXPORT_SYMBOL_GPL(__pm_genpd_remove_callbacks);
 
-int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state)
+/**
+ * pm_genpd_attach_cpuidle - Connect the given PM domain with cpuidle.
+ * @genpd: PM domain to be connected with cpuidle.
+ * @state: cpuidle state this domain can disable/enable.
+ *
+ * Make a PM domain behave as though it contained a CPU core, that is, instead
+ * of calling its power down routine it will enable the given cpuidle state so
+ * that the cpuidle subsystem can power it down (if possible and desirable).
+ */
+int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state)
 {
 	struct cpuidle_driver *cpuidle_drv;
 	struct gpd_cpu_data *cpu_data;
@@ -1753,7 +1887,24 @@ int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state)
 	goto out;
 }
 
-int genpd_detach_cpuidle(struct generic_pm_domain *genpd)
+/**
+ * pm_genpd_name_attach_cpuidle - Find PM domain and connect cpuidle to it.
+ * @name: Name of the domain to connect to cpuidle.
+ * @state: cpuidle state this domain can manipulate.
+ */
+int pm_genpd_name_attach_cpuidle(const char *name, int state)
+{
+	return pm_genpd_attach_cpuidle(pm_genpd_lookup_name(name), state);
+}
+
+/**
+ * pm_genpd_detach_cpuidle - Remove the cpuidle connection from a PM domain.
+ * @genpd: PM domain to remove the cpuidle connection from.
+ *
+ * Remove the cpuidle connection set up by pm_genpd_attach_cpuidle() from the
+ * given PM domain.
+ */
+int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd)
 {
 	struct gpd_cpu_data *cpu_data;
 	struct cpuidle_state *idle_state;
@@ -1784,6 +1935,15 @@ int genpd_detach_cpuidle(struct generic_pm_domain *genpd)
 	return ret;
 }
 
+/**
+ * pm_genpd_name_detach_cpuidle - Find PM domain and disconnect cpuidle from it.
+ * @name: Name of the domain to disconnect cpuidle from.
+ */
+int pm_genpd_name_detach_cpuidle(const char *name)
+{
+	return pm_genpd_detach_cpuidle(pm_genpd_lookup_name(name));
+}
+
 /* Default device callbacks for generic PM domains. */
 
 /**
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index b0b072a88f5..a3c1404c793 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -57,20 +57,17 @@ static pm_message_t pm_transition;
 static int async_error;
 
 /**
- * device_pm_init - Initialize the PM-related part of a device object.
+ * device_pm_sleep_init - Initialize system suspend-related device fields.
  * @dev: Device object being initialized.
  */
-void device_pm_init(struct device *dev)
+void device_pm_sleep_init(struct device *dev)
 {
 	dev->power.is_prepared = false;
 	dev->power.is_suspended = false;
 	init_completion(&dev->power.completion);
 	complete_all(&dev->power.completion);
 	dev->power.wakeup = NULL;
-	spin_lock_init(&dev->power.lock);
-	pm_runtime_init(dev);
 	INIT_LIST_HEAD(&dev->power.entry);
-	dev->power.power_state = PMSG_INVALID;
 }
 
 /**
@@ -408,6 +405,9 @@ static int device_resume_noirq(struct device *dev, pm_message_t state)
 	TRACE_DEVICE(dev);
 	TRACE_RESUME(0);
 
+	if (dev->power.syscore)
+		goto Out;
+
 	if (dev->pm_domain) {
 		info = "noirq power domain ";
 		callback = pm_noirq_op(&dev->pm_domain->ops, state);
@@ -429,6 +429,7 @@ static int device_resume_noirq(struct device *dev, pm_message_t state)
 
 	error = dpm_run_callback(callback, dev, state, info);
 
+ Out:
 	TRACE_RESUME(error);
 	return error;
 }
@@ -486,6 +487,9 @@ static int device_resume_early(struct device *dev, pm_message_t state)
 	TRACE_DEVICE(dev);
 	TRACE_RESUME(0);
 
+	if (dev->power.syscore)
+		goto Out;
+
 	if (dev->pm_domain) {
 		info = "early power domain ";
 		callback = pm_late_early_op(&dev->pm_domain->ops, state);
@@ -507,6 +511,7 @@ static int device_resume_early(struct device *dev, pm_message_t state)
 
 	error = dpm_run_callback(callback, dev, state, info);
 
+ Out:
 	TRACE_RESUME(error);
 	return error;
 }
@@ -565,11 +570,13 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 	pm_callback_t callback = NULL;
 	char *info = NULL;
 	int error = 0;
-	bool put = false;
 
 	TRACE_DEVICE(dev);
 	TRACE_RESUME(0);
 
+	if (dev->power.syscore)
+		goto Complete;
+
 	dpm_wait(dev->parent, async);
 	device_lock(dev);
 
@@ -583,7 +590,6 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 		goto Unlock;
 
 	pm_runtime_enable(dev);
-	put = true;
 
 	if (dev->pm_domain) {
 		info = "power domain ";
@@ -632,13 +638,12 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 
  Unlock:
 	device_unlock(dev);
+
+ Complete:
 	complete_all(&dev->power.completion);
 
 	TRACE_RESUME(error);
 
-	if (put)
-		pm_runtime_put_sync(dev);
-
 	return error;
 }
 
@@ -722,6 +727,9 @@ static void device_complete(struct device *dev, pm_message_t state)
 	void (*callback)(struct device *) = NULL;
 	char *info = NULL;
 
+	if (dev->power.syscore)
+		return;
+
 	device_lock(dev);
 
 	if (dev->pm_domain) {
@@ -749,6 +757,8 @@ static void device_complete(struct device *dev, pm_message_t state)
 	}
 
 	device_unlock(dev);
+
+	pm_runtime_put_sync(dev);
 }
 
 /**
@@ -834,6 +844,9 @@ static int device_suspend_noirq(struct device *dev, pm_message_t state)
 	pm_callback_t callback = NULL;
 	char *info = NULL;
 
+	if (dev->power.syscore)
+		return 0;
+
 	if (dev->pm_domain) {
 		info = "noirq power domain ";
 		callback = pm_noirq_op(&dev->pm_domain->ops, state);
@@ -917,6 +930,9 @@ static int device_suspend_late(struct device *dev, pm_message_t state)
 	pm_callback_t callback = NULL;
 	char *info = NULL;
 
+	if (dev->power.syscore)
+		return 0;
+
 	if (dev->pm_domain) {
 		info = "late power domain ";
 		callback = pm_late_early_op(&dev->pm_domain->ops, state);
@@ -996,7 +1012,7 @@ int dpm_suspend_end(pm_message_t state)
 
 	error = dpm_suspend_noirq(state);
 	if (error) {
-		dpm_resume_early(state);
+		dpm_resume_early(resume_event(state));
 		return error;
 	}
 
@@ -1043,16 +1059,23 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	if (async_error)
 		goto Complete;
 
-	pm_runtime_get_noresume(dev);
+	/*
+	 * If a device configured to wake up the system from sleep states
+	 * has been suspended at run time and there's a resume request pending
+	 * for it, this is equivalent to the device signaling wakeup, so the
+	 * system suspend operation should be aborted.
+	 */
 	if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
 		pm_wakeup_event(dev, 0);
 
 	if (pm_wakeup_pending()) {
-		pm_runtime_put_sync(dev);
 		async_error = -EBUSY;
 		goto Complete;
 	}
 
+	if (dev->power.syscore)
+		goto Complete;
+
 	device_lock(dev);
 
 	if (dev->pm_domain) {
@@ -1111,12 +1134,10 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
  Complete:
 	complete_all(&dev->power.completion);
 
-	if (error) {
-		pm_runtime_put_sync(dev);
+	if (error)
 		async_error = error;
-	} else if (dev->power.is_suspended) {
+	else if (dev->power.is_suspended)
 		__pm_runtime_disable(dev, false);
-	}
 
 	return error;
 }
@@ -1209,6 +1230,17 @@ static int device_prepare(struct device *dev, pm_message_t state)
 	char *info = NULL;
 	int error = 0;
 
+	if (dev->power.syscore)
+		return 0;
+
+	/*
+	 * If a device's parent goes into runtime suspend at the wrong time,
+	 * it won't be possible to resume the device.  To prevent this we
+	 * block runtime suspend here, during the prepare phase, and allow
+	 * it again during the complete phase.
+	 */
+	pm_runtime_get_noresume(dev);
+
 	device_lock(dev);
 
 	dev->power.wakeup_path = device_may_wakeup(dev);
diff --git a/drivers/base/power/opp.c b/drivers/base/power/opp.c
index ac993eafec8..d9468642fc4 100644
--- a/drivers/base/power/opp.c
+++ b/drivers/base/power/opp.c
@@ -22,6 +22,7 @@
 #include <linux/rculist.h>
 #include <linux/rcupdate.h>
 #include <linux/opp.h>
+#include <linux/of.h>
 
 /*
  * Internal data structure organization with the OPP layer library is as
@@ -674,3 +675,49 @@ struct srcu_notifier_head *opp_get_notifier(struct device *dev)
 
 	return &dev_opp->head;
 }
+
+#ifdef CONFIG_OF
+/**
+ * of_init_opp_table() - Initialize opp table from device tree
+ * @dev:	device pointer used to lookup device OPPs.
+ *
+ * Register the initial OPP table with the OPP library for given device.
+ */
+int of_init_opp_table(struct device *dev)
+{
+	const struct property *prop;
+	const __be32 *val;
+	int nr;
+
+	prop = of_find_property(dev->of_node, "operating-points", NULL);
+	if (!prop)
+		return -ENODEV;
+	if (!prop->value)
+		return -ENODATA;
+
+	/*
+	 * Each OPP is a set of tuples consisting of frequency and
+	 * voltage like <freq-kHz vol-uV>.
+	 */
+	nr = prop->length / sizeof(u32);
+	if (nr % 2) {
+		dev_err(dev, "%s: Invalid OPP list\n", __func__);
+		return -EINVAL;
+	}
+
+	val = prop->value;
+	while (nr) {
+		unsigned long freq = be32_to_cpup(val++) * 1000;
+		unsigned long volt = be32_to_cpup(val++);
+
+		if (opp_add(dev, freq, volt)) {
+			dev_warn(dev, "%s: Failed to add OPP %ld\n",
+				 __func__, freq);
+			continue;
+		}
+		nr -= 2;
+	}
+
+	return 0;
+}
+#endif
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index eeb4bff9505..0dbfdf4419a 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -1,12 +1,32 @@
 #include <linux/pm_qos.h>
 
+static inline void device_pm_init_common(struct device *dev)
+{
+	if (!dev->power.early_init) {
+		spin_lock_init(&dev->power.lock);
+		dev->power.power_state = PMSG_INVALID;
+		dev->power.early_init = true;
+	}
+}
+
 #ifdef CONFIG_PM_RUNTIME
 
+static inline void pm_runtime_early_init(struct device *dev)
+{
+	dev->power.disable_depth = 1;
+	device_pm_init_common(dev);
+}
+
 extern void pm_runtime_init(struct device *dev);
 extern void pm_runtime_remove(struct device *dev);
 
 #else /* !CONFIG_PM_RUNTIME */
 
+static inline void pm_runtime_early_init(struct device *dev)
+{
+	device_pm_init_common(dev);
+}
+
 static inline void pm_runtime_init(struct device *dev) {}
 static inline void pm_runtime_remove(struct device *dev) {}
 
@@ -25,7 +45,7 @@ static inline struct device *to_device(struct list_head *entry)
 	return container_of(entry, struct device, power.entry);
 }
 
-extern void device_pm_init(struct device *dev);
+extern void device_pm_sleep_init(struct device *dev);
 extern void device_pm_add(struct device *);
 extern void device_pm_remove(struct device *);
 extern void device_pm_move_before(struct device *, struct device *);
@@ -34,12 +54,7 @@ extern void device_pm_move_last(struct device *);
 
 #else /* !CONFIG_PM_SLEEP */
 
-static inline void device_pm_init(struct device *dev)
-{
-	spin_lock_init(&dev->power.lock);
-	dev->power.power_state = PMSG_INVALID;
-	pm_runtime_init(dev);
-}
+static inline void device_pm_sleep_init(struct device *dev) {}
 
 static inline void device_pm_add(struct device *dev)
 {
@@ -60,6 +75,13 @@ static inline void device_pm_move_last(struct device *dev) {}
 
 #endif /* !CONFIG_PM_SLEEP */
 
+static inline void device_pm_init(struct device *dev)
+{
+	device_pm_init_common(dev);
+	device_pm_sleep_init(dev);
+	pm_runtime_init(dev);
+}
+
 #ifdef CONFIG_PM
 
 /*
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 7d9c1cb1c39..3148b10dc2e 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -509,6 +509,9 @@ static int rpm_resume(struct device *dev, int rpmflags)
  repeat:
 	if (dev->power.runtime_error)
 		retval = -EINVAL;
+	else if (dev->power.disable_depth == 1 && dev->power.is_suspended
+	    && dev->power.runtime_status == RPM_ACTIVE)
+		retval = 1;
 	else if (dev->power.disable_depth > 0)
 		retval = -EACCES;
 	if (retval)
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index cbb463b3a75..e6ee5e80e54 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -127,6 +127,8 @@ EXPORT_SYMBOL_GPL(wakeup_source_destroy);
  */
 void wakeup_source_add(struct wakeup_source *ws)
 {
+	unsigned long flags;
+
 	if (WARN_ON(!ws))
 		return;
 
@@ -135,9 +137,9 @@ void wakeup_source_add(struct wakeup_source *ws)
 	ws->active = false;
 	ws->last_time = ktime_get();
 
-	spin_lock_irq(&events_lock);
+	spin_lock_irqsave(&events_lock, flags);
 	list_add_rcu(&ws->entry, &wakeup_sources);
-	spin_unlock_irq(&events_lock);
+	spin_unlock_irqrestore(&events_lock, flags);
 }
 EXPORT_SYMBOL_GPL(wakeup_source_add);
 
@@ -147,12 +149,14 @@ EXPORT_SYMBOL_GPL(wakeup_source_add);
  */
 void wakeup_source_remove(struct wakeup_source *ws)
 {
+	unsigned long flags;
+
 	if (WARN_ON(!ws))
 		return;
 
-	spin_lock_irq(&events_lock);
+	spin_lock_irqsave(&events_lock, flags);
 	list_del_rcu(&ws->entry);
-	spin_unlock_irq(&events_lock);
+	spin_unlock_irqrestore(&events_lock, flags);
 	synchronize_rcu();
 }
 EXPORT_SYMBOL_GPL(wakeup_source_remove);
@@ -649,6 +653,31 @@ void pm_wakeup_event(struct device *dev, unsigned int msec)
 }
 EXPORT_SYMBOL_GPL(pm_wakeup_event);
 
+static void print_active_wakeup_sources(void)
+{
+	struct wakeup_source *ws;
+	int active = 0;
+	struct wakeup_source *last_activity_ws = NULL;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(ws, &wakeup_sources, entry) {
+		if (ws->active) {
+			pr_info("active wakeup source: %s\n", ws->name);
+			active = 1;
+		} else if (!active &&
+			   (!last_activity_ws ||
+			    ktime_to_ns(ws->last_time) >
+			    ktime_to_ns(last_activity_ws->last_time))) {
+			last_activity_ws = ws;
+		}
+	}
+
+	if (!active && last_activity_ws)
+		pr_info("last active wakeup source: %s\n",
+			last_activity_ws->name);
+	rcu_read_unlock();
+}
+
 /**
  * pm_wakeup_pending - Check if power transition in progress should be aborted.
  *
@@ -671,6 +700,10 @@ bool pm_wakeup_pending(void)
 		events_check_enabled = !ret;
 	}
 	spin_unlock_irqrestore(&events_lock, flags);
+
+	if (ret)
+		print_active_wakeup_sources();
+
 	return ret;
 }
 
@@ -723,15 +756,16 @@ bool pm_get_wakeup_count(unsigned int *count, bool block)
 bool pm_save_wakeup_count(unsigned int count)
 {
 	unsigned int cnt, inpr;
+	unsigned long flags;
 
 	events_check_enabled = false;
-	spin_lock_irq(&events_lock);
+	spin_lock_irqsave(&events_lock, flags);
 	split_counters(&cnt, &inpr);
 	if (cnt == count && inpr == 0) {
 		saved_count = count;
 		events_check_enabled = true;
 	}
-	spin_unlock_irq(&events_lock);
+	spin_unlock_irqrestore(&events_lock, flags);
 	return events_check_enabled;
 }
 
diff --git a/drivers/block/aoe/aoe.h b/drivers/block/aoe/aoe.h
index db195abad69..d2ed7f18d1a 100644
--- a/drivers/block/aoe/aoe.h
+++ b/drivers/block/aoe/aoe.h
@@ -1,5 +1,5 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
-#define VERSION "47"
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
+#define VERSION "50"
 #define AOE_MAJOR 152
 #define DEVICE_NAME "aoe"
 
@@ -10,9 +10,6 @@
 #define AOE_PARTITIONS (16)
 #endif
 
-#define SYSMINOR(aoemajor, aoeminor) ((aoemajor) * NPERSHELF + (aoeminor))
-#define AOEMAJOR(sysminor) ((sysminor) / NPERSHELF)
-#define AOEMINOR(sysminor) ((sysminor) % NPERSHELF)
 #define WHITESPACE " \t\v\f\n"
 
 enum {
@@ -75,72 +72,67 @@ enum {
 	DEVFL_UP = 1,	/* device is installed in system and ready for AoE->ATA commands */
 	DEVFL_TKILL = (1<<1),	/* flag for timer to know when to kill self */
 	DEVFL_EXT = (1<<2),	/* device accepts lba48 commands */
-	DEVFL_CLOSEWAIT = (1<<3), /* device is waiting for all closes to revalidate */
-	DEVFL_GDALLOC = (1<<4),	/* need to alloc gendisk */
-	DEVFL_KICKME = (1<<5),	/* slow polling network card catch */
-	DEVFL_NEWSIZE = (1<<6),	/* need to update dev size in block layer */
-
-	BUFFL_FAIL = 1,
+	DEVFL_GDALLOC = (1<<3),	/* need to alloc gendisk */
+	DEVFL_KICKME = (1<<4),	/* slow polling network card catch */
+	DEVFL_NEWSIZE = (1<<5),	/* need to update dev size in block layer */
 };
 
 enum {
 	DEFAULTBCNT = 2 * 512,	/* 2 sectors */
-	NPERSHELF = 16,		/* number of slots per shelf address */
-	FREETAG = -1,
 	MIN_BUFS = 16,
 	NTARGETS = 8,
 	NAOEIFS = 8,
-	NSKBPOOLMAX = 128,
+	NSKBPOOLMAX = 256,
+	NFACTIVE = 61,
 
 	TIMERTICK = HZ / 10,
 	MINTIMER = HZ >> 2,
 	MAXTIMER = HZ << 1,
-	HELPWAIT = 20,
 };
 
 struct buf {
-	struct list_head bufs;
-	ulong stime;	/* for disk stats */
-	ulong flags;
 	ulong nframesout;
 	ulong resid;
 	ulong bv_resid;
-	ulong bv_off;
 	sector_t sector;
 	struct bio *bio;
 	struct bio_vec *bv;
+	struct request *rq;
 };
 
 struct frame {
-	int tag;
+	struct list_head head;
+	u32 tag;
 	ulong waited;
+	struct aoetgt *t;		/* parent target I belong to */
+	sector_t lba;
+	struct sk_buff *skb;		/* command skb freed on module exit */
+	struct sk_buff *r_skb;		/* response skb for async processing */
 	struct buf *buf;
-	char *bufaddr;
+	struct bio_vec *bv;
 	ulong bcnt;
-	sector_t lba;
-	struct sk_buff *skb;
+	ulong bv_off;
 };
 
 struct aoeif {
 	struct net_device *nd;
-	unsigned char lost;
-	unsigned char lostjumbo;
-	ushort maxbcnt;
+	ulong lost;
+	int bcnt;
 };
 
 struct aoetgt {
 	unsigned char addr[6];
 	ushort nframes;
-	struct frame *frames;
+	struct aoedev *d;			/* parent device I belong to */
+	struct list_head ffree;			/* list of free frames */
 	struct aoeif ifs[NAOEIFS];
 	struct aoeif *ifp;	/* current aoeif in use */
 	ushort nout;
 	ushort maxout;
-	u16 lasttag;		/* last tag sent */
-	u16 useme;
+	ulong falloc;
 	ulong lastwadj;		/* last window adjustment */
+	int minbcnt;
 	int wpkts, rpkts;
-	int dataref;
 };
 
 struct aoedev {
@@ -153,6 +145,9 @@ struct aoedev {
 	u16 rttavg;		/* round trip average of requests/responses */
 	u16 mintimer;
 	u16 fw_ver;		/* version of blade's firmware */
+	u16 lasttag;		/* last tag sent */
+	u16 useme;
+	ulong ref;
 	struct work_struct work;/* disk create work struct */
 	struct gendisk *gd;
 	struct request_queue *blkq;
@@ -160,16 +155,31 @@ struct aoedev {
 	sector_t ssize;
 	struct timer_list timer;
 	spinlock_t lock;
-	struct sk_buff_head sendq;
 	struct sk_buff_head skbpool;
 	mempool_t *bufpool;	/* for deadlock-free Buf allocation */
-	struct list_head bufq;	/* queue of bios to work on */
-	struct buf *inprocess;	/* the one we're currently working on */
+	struct {		/* pointers to work in progress */
+		struct buf *buf;
+		struct bio *nxbio;
+		struct request *rq;
+	} ip;
+	ulong maxbcnt;
+	struct list_head factive[NFACTIVE];	/* hash of active frames */
 	struct aoetgt *targets[NTARGETS];
 	struct aoetgt **tgt;	/* target in use when working */
-	struct aoetgt **htgt;	/* target needing rexmit assistance */
+	struct aoetgt *htgt;	/* target needing rexmit assistance */
+	ulong ntargets;
+	ulong kicked;
 };
 
+/* kthread tracking */
+struct ktstate {
+	struct completion rendez;
+	struct task_struct *task;
+	wait_queue_head_t *waitq;
+	int (*fn) (void);
+	char *name;
+	spinlock_t *lock;
+};
 
 int aoeblk_init(void);
 void aoeblk_exit(void);
@@ -182,22 +192,29 @@ void aoechr_error(char *);
 
 void aoecmd_work(struct aoedev *d);
 void aoecmd_cfg(ushort aoemajor, unsigned char aoeminor);
-void aoecmd_ata_rsp(struct sk_buff *);
+struct sk_buff *aoecmd_ata_rsp(struct sk_buff *);
 void aoecmd_cfg_rsp(struct sk_buff *);
 void aoecmd_sleepwork(struct work_struct *);
 void aoecmd_cleanslate(struct aoedev *);
+void aoecmd_exit(void);
+int aoecmd_init(void);
 struct sk_buff *aoecmd_ata_id(struct aoedev *);
+void aoe_freetframe(struct frame *);
+void aoe_flush_iocq(void);
+void aoe_end_request(struct aoedev *, struct request *, int);
+int aoe_ktstart(struct ktstate *k);
+void aoe_ktstop(struct ktstate *k);
 
 int aoedev_init(void);
 void aoedev_exit(void);
-struct aoedev *aoedev_by_aoeaddr(int maj, int min);
-struct aoedev *aoedev_by_sysminor_m(ulong sysminor);
+struct aoedev *aoedev_by_aoeaddr(ulong maj, int min, int do_alloc);
 void aoedev_downdev(struct aoedev *d);
 int aoedev_flush(const char __user *str, size_t size);
+void aoe_failbuf(struct aoedev *, struct buf *);
+void aoedev_put(struct aoedev *);
 
 int aoenet_init(void);
 void aoenet_exit(void);
 void aoenet_xmit(struct sk_buff_head *);
 int is_aoe_netif(struct net_device *ifp);
 int set_aoe_iflist(const char __user *str, size_t size);
-
diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c
index 321de7b6c44..00dfc5008ad 100644
--- a/drivers/block/aoe/aoeblk.c
+++ b/drivers/block/aoe/aoeblk.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoeblk.c
  * block device routines
@@ -161,68 +161,22 @@ aoeblk_release(struct gendisk *disk, fmode_t mode)
 }
 
 static void
-aoeblk_make_request(struct request_queue *q, struct bio *bio)
+aoeblk_request(struct request_queue *q)
 {
-	struct sk_buff_head queue;
 	struct aoedev *d;
-	struct buf *buf;
-	ulong flags;
-
-	blk_queue_bounce(q, &bio);
-
-	if (bio == NULL) {
-		printk(KERN_ERR "aoe: bio is NULL\n");
-		BUG();
-		return;
-	}
-	d = bio->bi_bdev->bd_disk->private_data;
-	if (d == NULL) {
-		printk(KERN_ERR "aoe: bd_disk->private_data is NULL\n");
-		BUG();
-		bio_endio(bio, -ENXIO);
-		return;
-	} else if (bio->bi_io_vec == NULL) {
-		printk(KERN_ERR "aoe: bi_io_vec is NULL\n");
-		BUG();
-		bio_endio(bio, -ENXIO);
-		return;
-	}
-	buf = mempool_alloc(d->bufpool, GFP_NOIO);
-	if (buf == NULL) {
-		printk(KERN_INFO "aoe: buf allocation failure\n");
-		bio_endio(bio, -ENOMEM);
-		return;
-	}
-	memset(buf, 0, sizeof(*buf));
-	INIT_LIST_HEAD(&buf->bufs);
-	buf->stime = jiffies;
-	buf->bio = bio;
-	buf->resid = bio->bi_size;
-	buf->sector = bio->bi_sector;
-	buf->bv = &bio->bi_io_vec[bio->bi_idx];
-	buf->bv_resid = buf->bv->bv_len;
-	WARN_ON(buf->bv_resid == 0);
-	buf->bv_off = buf->bv->bv_offset;
-
-	spin_lock_irqsave(&d->lock, flags);
+	struct request *rq;
 
+	d = q->queuedata;
 	if ((d->flags & DEVFL_UP) == 0) {
 		pr_info_ratelimited("aoe: device %ld.%d is not up\n",
 			d->aoemajor, d->aoeminor);
-		spin_unlock_irqrestore(&d->lock, flags);
-		mempool_free(buf, d->bufpool);
-		bio_endio(bio, -ENXIO);
+		while ((rq = blk_peek_request(q))) {
+			blk_start_request(rq);
+			aoe_end_request(d, rq, 1);
+		}
 		return;
 	}
-
-	list_add_tail(&buf->bufs, &d->bufq);
-
 	aoecmd_work(d);
-	__skb_queue_head_init(&queue);
-	skb_queue_splice_init(&d->sendq, &queue);
-
-	spin_unlock_irqrestore(&d->lock, flags);
-	aoenet_xmit(&queue);
 }
 
 static int
@@ -254,41 +208,54 @@ aoeblk_gdalloc(void *vp)
 {
 	struct aoedev *d = vp;
 	struct gendisk *gd;
+	mempool_t *mp;
+	struct request_queue *q;
+	enum { KB = 1024, MB = KB * KB, READ_AHEAD = 2 * MB, };
 	ulong flags;
 
 	gd = alloc_disk(AOE_PARTITIONS);
 	if (gd == NULL) {
-		printk(KERN_ERR
-			"aoe: cannot allocate disk structure for %ld.%d\n",
+		pr_err("aoe: cannot allocate disk structure for %ld.%d\n",
 			d->aoemajor, d->aoeminor);
 		goto err;
 	}
 
-	d->bufpool = mempool_create_slab_pool(MIN_BUFS, buf_pool_cache);
-	if (d->bufpool == NULL) {
+	mp = mempool_create(MIN_BUFS, mempool_alloc_slab, mempool_free_slab,
+		buf_pool_cache);
+	if (mp == NULL) {
 		printk(KERN_ERR "aoe: cannot allocate bufpool for %ld.%d\n",
 			d->aoemajor, d->aoeminor);
 		goto err_disk;
 	}
+	q = blk_init_queue(aoeblk_request, &d->lock);
+	if (q == NULL) {
+		pr_err("aoe: cannot allocate block queue for %ld.%d\n",
+			d->aoemajor, d->aoeminor);
+		mempool_destroy(mp);
+		goto err_disk;
+	}
 
 	d->blkq = blk_alloc_queue(GFP_KERNEL);
 	if (!d->blkq)
 		goto err_mempool;
-	blk_queue_make_request(d->blkq, aoeblk_make_request);
 	d->blkq->backing_dev_info.name = "aoe";
 	if (bdi_init(&d->blkq->backing_dev_info))
 		goto err_blkq;
 	spin_lock_irqsave(&d->lock, flags);
+	blk_queue_max_hw_sectors(d->blkq, BLK_DEF_MAX_SECTORS);
+	q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE;
+	d->bufpool = mp;
+	d->blkq = gd->queue = q;
+	q->queuedata = d;
+	d->gd = gd;
 	gd->major = AOE_MAJOR;
-	gd->first_minor = d->sysminor * AOE_PARTITIONS;
+	gd->first_minor = d->sysminor;
 	gd->fops = &aoe_bdops;
 	gd->private_data = d;
 	set_capacity(gd, d->ssize);
 	snprintf(gd->disk_name, sizeof gd->disk_name, "etherd/e%ld.%d",
 		d->aoemajor, d->aoeminor);
 
-	gd->queue = d->blkq;
-	d->gd = gd;
 	d->flags &= ~DEVFL_GDALLOC;
 	d->flags |= DEVFL_UP;
 
diff --git a/drivers/block/aoe/aoechr.c b/drivers/block/aoe/aoechr.c
index e86d2062a16..ed57a890c64 100644
--- a/drivers/block/aoe/aoechr.c
+++ b/drivers/block/aoe/aoechr.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoechr.c
  * AoE character device driver
@@ -86,34 +86,34 @@ revalidate(const char __user *str, size_t size)
 	if (copy_from_user(buf, str, size))
 		return -EFAULT;
 
-	/* should be e%d.%d format */
 	n = sscanf(buf, "e%d.%d", &major, &minor);
 	if (n != 2) {
-		printk(KERN_ERR "aoe: invalid device specification\n");
+		pr_err("aoe: invalid device specification %s\n", buf);
 		return -EINVAL;
 	}
-	d = aoedev_by_aoeaddr(major, minor);
+	d = aoedev_by_aoeaddr(major, minor, 0);
 	if (!d)
 		return -EINVAL;
 	spin_lock_irqsave(&d->lock, flags);
 	aoecmd_cleanslate(d);
+	aoecmd_cfg(major, minor);
 loop:
 	skb = aoecmd_ata_id(d);
 	spin_unlock_irqrestore(&d->lock, flags);
 	/* try again if we are able to sleep a bit,
 	 * otherwise give up this revalidation
 	 */
-	if (!skb && !msleep_interruptible(200)) {
+	if (!skb && !msleep_interruptible(250)) {
 		spin_lock_irqsave(&d->lock, flags);
 		goto loop;
 	}
+	aoedev_put(d);
 	if (skb) {
 		struct sk_buff_head queue;
 		__skb_queue_head_init(&queue);
 		__skb_queue_tail(&queue, skb);
 		aoenet_xmit(&queue);
 	}
-	aoecmd_cfg(major, minor);
 	return 0;
 }
 
@@ -174,6 +174,7 @@ aoechr_write(struct file *filp, const char __user *buf, size_t cnt, loff_t *offp
 		break;
 	case MINOR_FLUSH:
 		ret = aoedev_flush(buf, cnt);
+		break;
 	}
 	if (ret == 0)
 		ret = cnt;
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 887f68f6d79..3804a0af3ef 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoecmd.c
  * Filesystem request handling methods
@@ -12,10 +12,19 @@
 #include <linux/netdevice.h>
 #include <linux/genhd.h>
 #include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
 #include <net/net_namespace.h>
 #include <asm/unaligned.h>
+#include <linux/uio.h>
 #include "aoe.h"
 
+#define MAXIOC (8192)	/* default meant to avoid most soft lockups */
+
+static void ktcomplete(struct frame *, struct sk_buff *);
+
+static struct buf *nextbuf(struct aoedev *);
+
 static int aoe_deadsecs = 60 * 3;
 module_param(aoe_deadsecs, int, 0644);
 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
@@ -25,6 +34,15 @@ module_param(aoe_maxout, int, 0644);
 MODULE_PARM_DESC(aoe_maxout,
 	"Only aoe_maxout outstanding packets for every MAC on eX.Y.");
 
+static wait_queue_head_t ktiowq;
+static struct ktstate kts;
+
+/* io completion queue */
+static struct {
+	struct list_head head;
+	spinlock_t lock;
+} iocq;
+
 static struct sk_buff *
 new_skb(ulong len)
 {
@@ -41,15 +59,21 @@ new_skb(ulong len)
 }
 
 static struct frame *
-getframe(struct aoetgt *t, int tag)
+getframe(struct aoedev *d, u32 tag)
 {
-	struct frame *f, *e;
+	struct frame *f;
+	struct list_head *head, *pos, *nx;
+	u32 n;
 
-	f = t->frames;
-	e = f + t->nframes;
-	for (; f<e; f++)
-		if (f->tag == tag)
+	n = tag % NFACTIVE;
+	head = &d->factive[n];
+	list_for_each_safe(pos, nx, head) {
+		f = list_entry(pos, struct frame, head);
+		if (f->tag == tag) {
+			list_del(pos);
 			return f;
+		}
+	}
 	return NULL;
 }
 
@@ -59,18 +83,18 @@ getframe(struct aoetgt *t, int tag)
  * This driver reserves tag -1 to mean "unused frame."
  */
 static int
-newtag(struct aoetgt *t)
+newtag(struct aoedev *d)
 {
 	register ulong n;
 
 	n = jiffies & 0xffff;
-	return n |= (++t->lasttag & 0x7fff) << 16;
+	return n |= (++d->lasttag & 0x7fff) << 16;
 }
 
-static int
+static u32
 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
 {
-	u32 host_tag = newtag(t);
+	u32 host_tag = newtag(d);
 
 	memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 	memcpy(h->dst, t->addr, sizeof h->dst);
@@ -95,16 +119,18 @@ put_lba(struct aoe_atahdr *ah, sector_t lba)
 	ah->lba5 = lba >>= 8;
 }
 
-static void
+static struct aoeif *
 ifrotate(struct aoetgt *t)
 {
-	t->ifp++;
-	if (t->ifp >= &t->ifs[NAOEIFS] || t->ifp->nd == NULL)
-		t->ifp = t->ifs;
-	if (t->ifp->nd == NULL) {
-		printk(KERN_INFO "aoe: no interface to rotate to\n");
-		BUG();
-	}
+	struct aoeif *ifp;
+
+	ifp = t->ifp;
+	ifp++;
+	if (ifp >= &t->ifs[NAOEIFS] || ifp->nd == NULL)
+		ifp = t->ifs;
+	if (ifp->nd == NULL)
+		return NULL;
+	return t->ifp = ifp;
 }
 
 static void
@@ -129,78 +155,128 @@ skb_pool_get(struct aoedev *d)
 	return NULL;
 }
 
-/* freeframe is where we do our load balancing so it's a little hairy. */
+void
+aoe_freetframe(struct frame *f)
+{
+	struct aoetgt *t;
+
+	t = f->t;
+	f->buf = NULL;
+	f->bv = NULL;
+	f->r_skb = NULL;
+	list_add(&f->head, &t->ffree);
+}
+
 static struct frame *
-freeframe(struct aoedev *d)
+newtframe(struct aoedev *d, struct aoetgt *t)
 {
-	struct frame *f, *e, *rf;
-	struct aoetgt **t;
+	struct frame *f;
 	struct sk_buff *skb;
+	struct list_head *pos;
+
+	if (list_empty(&t->ffree)) {
+		if (t->falloc >= NSKBPOOLMAX*2)
+			return NULL;
+		f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
+		if (f == NULL)
+			return NULL;
+		t->falloc++;
+		f->t = t;
+	} else {
+		pos = t->ffree.next;
+		list_del(pos);
+		f = list_entry(pos, struct frame, head);
+	}
+
+	skb = f->skb;
+	if (skb == NULL) {
+		f->skb = skb = new_skb(ETH_ZLEN);
+		if (!skb) {
+bail:			aoe_freetframe(f);
+			return NULL;
+		}
+	}
+
+	if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
+		skb = skb_pool_get(d);
+		if (skb == NULL)
+			goto bail;
+		skb_pool_put(d, f->skb);
+		f->skb = skb;
+	}
+
+	skb->truesize -= skb->data_len;
+	skb_shinfo(skb)->nr_frags = skb->data_len = 0;
+	skb_trim(skb, 0);
+	return f;
+}
+
+static struct frame *
+newframe(struct aoedev *d)
+{
+	struct frame *f;
+	struct aoetgt *t, **tt;
+	int totout = 0;
 
 	if (d->targets[0] == NULL) {	/* shouldn't happen, but I'm paranoid */
 		printk(KERN_ERR "aoe: NULL TARGETS!\n");
 		return NULL;
 	}
-	t = d->tgt;
-	t++;
-	if (t >= &d->targets[NTARGETS] || !*t)
-		t = d->targets;
+	tt = d->tgt;	/* last used target */
 	for (;;) {
-		if ((*t)->nout < (*t)->maxout
+		tt++;
+		if (tt >= &d->targets[NTARGETS] || !*tt)
+			tt = d->targets;
+		t = *tt;
+		totout += t->nout;
+		if (t->nout < t->maxout
 		&& t != d->htgt
-		&& (*t)->ifp->nd) {
-			rf = NULL;
-			f = (*t)->frames;
-			e = f + (*t)->nframes;
-			for (; f < e; f++) {
-				if (f->tag != FREETAG)
-					continue;
-				skb = f->skb;
-				if (!skb
-				&& !(f->skb = skb = new_skb(ETH_ZLEN)))
-					continue;
-				if (atomic_read(&skb_shinfo(skb)->dataref)
-					!= 1) {
-					if (!rf)
-						rf = f;
-					continue;
-				}
-gotone:				skb_shinfo(skb)->nr_frags = skb->data_len = 0;
-				skb_trim(skb, 0);
-				d->tgt = t;
-				ifrotate(*t);
+		&& t->ifp->nd) {
+			f = newtframe(d, t);
+			if (f) {
+				ifrotate(t);
+				d->tgt = tt;
 				return f;
 			}
-			/* Work can be done, but the network layer is
-			   holding our precious packets.  Try to grab
-			   one from the pool. */
-			f = rf;
-			if (f == NULL) {	/* more paranoia */
-				printk(KERN_ERR
-					"aoe: freeframe: %s.\n",
-					"unexpected null rf");
-				d->flags |= DEVFL_KICKME;
-				return NULL;
-			}
-			skb = skb_pool_get(d);
-			if (skb) {
-				skb_pool_put(d, f->skb);
-				f->skb = skb;
-				goto gotone;
-			}
-			(*t)->dataref++;
-			if ((*t)->nout == 0)
-				d->flags |= DEVFL_KICKME;
 		}
-		if (t == d->tgt)	/* we've looped and found nada */
+		if (tt == d->tgt)	/* we've looped and found nada */
 			break;
-		t++;
-		if (t >= &d->targets[NTARGETS] || !*t)
-			t = d->targets;
+	}
+	if (totout == 0) {
+		d->kicked++;
+		d->flags |= DEVFL_KICKME;
 	}
 	return NULL;
 }
 
+static void
+skb_fillup(struct sk_buff *skb, struct bio_vec *bv, ulong off, ulong cnt)
+{
+	int frag = 0;
+	ulong fcnt;
+loop:
+	fcnt = bv->bv_len - (off - bv->bv_offset);
+	if (fcnt > cnt)
+		fcnt = cnt;
+	skb_fill_page_desc(skb, frag++, bv->bv_page, off, fcnt);
+	cnt -= fcnt;
+	if (cnt <= 0)
+		return;
+	bv++;
+	off = bv->bv_offset;
+	goto loop;
+}
+
+static void
+fhash(struct frame *f)
+{
+	struct aoedev *d = f->t->d;
+	u32 n;
+
+	n = f->tag % NFACTIVE;
+	list_add_tail(&f->head, &d->factive[n]);
+}
+
 static int
 aoecmd_ata_rw(struct aoedev *d)
 {
@@ -208,26 +284,47 @@ aoecmd_ata_rw(struct aoedev *d)
 	struct aoe_hdr *h;
 	struct aoe_atahdr *ah;
 	struct buf *buf;
-	struct bio_vec *bv;
 	struct aoetgt *t;
 	struct sk_buff *skb;
-	ulong bcnt;
+	struct sk_buff_head queue;
+	ulong bcnt, fbcnt;
 	char writebit, extbit;
 
 	writebit = 0x10;
 	extbit = 0x4;
 
-	f = freeframe(d);
+	buf = nextbuf(d);
+	if (buf == NULL)
+		return 0;
+	f = newframe(d);
 	if (f == NULL)
 		return 0;
 	t = *d->tgt;
-	buf = d->inprocess;
-	bv = buf->bv;
-	bcnt = t->ifp->maxbcnt;
+	bcnt = d->maxbcnt;
 	if (bcnt == 0)
 		bcnt = DEFAULTBCNT;
-	if (bcnt > buf->bv_resid)
-		bcnt = buf->bv_resid;
+	if (bcnt > buf->resid)
+		bcnt = buf->resid;
+	fbcnt = bcnt;
+	f->bv = buf->bv;
+	f->bv_off = f->bv->bv_offset + (f->bv->bv_len - buf->bv_resid);
+	do {
+		if (fbcnt < buf->bv_resid) {
+			buf->bv_resid -= fbcnt;
+			buf->resid -= fbcnt;
+			break;
+		}
+		fbcnt -= buf->bv_resid;
+		buf->resid -= buf->bv_resid;
+		if (buf->resid == 0) {
+			d->ip.buf = NULL;
+			break;
+		}
+		buf->bv++;
+		buf->bv_resid = buf->bv->bv_len;
+		WARN_ON(buf->bv_resid == 0);
+	} while (fbcnt);
+
 	/* initialize the headers & frame */
 	skb = f->skb;
 	h = (struct aoe_hdr *) skb_mac_header(skb);
@@ -235,10 +332,10 @@ aoecmd_ata_rw(struct aoedev *d)
 	skb_put(skb, sizeof *h + sizeof *ah);
 	memset(h, 0, skb->len);
 	f->tag = aoehdr_atainit(d, t, h);
+	fhash(f);
 	t->nout++;
 	f->waited = 0;
 	f->buf = buf;
-	f->bufaddr = page_address(bv->bv_page) + buf->bv_off;
 	f->bcnt = bcnt;
 	f->lba = buf->sector;
 
@@ -253,10 +350,11 @@ aoecmd_ata_rw(struct aoedev *d)
 		ah->lba3 |= 0xe0;	/* LBA bit + obsolete 0xa0 */
 	}
 	if (bio_data_dir(buf->bio) == WRITE) {
-		skb_fill_page_desc(skb, 0, bv->bv_page, buf->bv_off, bcnt);
+		skb_fillup(skb, f->bv, f->bv_off, bcnt);
 		ah->aflags |= AOEAFL_WRITE;
 		skb->len += bcnt;
 		skb->data_len = bcnt;
+		skb->truesize += bcnt;
 		t->wpkts++;
 	} else {
 		t->rpkts++;
@@ -267,23 +365,15 @@ aoecmd_ata_rw(struct aoedev *d)
 
 	/* mark all tracking fields and load out */
 	buf->nframesout += 1;
-	buf->bv_off += bcnt;
-	buf->bv_resid -= bcnt;
-	buf->resid -= bcnt;
 	buf->sector += bcnt >> 9;
-	if (buf->resid == 0) {
-		d->inprocess = NULL;
-	} else if (buf->bv_resid == 0) {
-		buf->bv = ++bv;
-		buf->bv_resid = bv->bv_len;
-		WARN_ON(buf->bv_resid == 0);
-		buf->bv_off = bv->bv_offset;
-	}
 
 	skb->dev = t->ifp->nd;
 	skb = skb_clone(skb, GFP_ATOMIC);
-	if (skb)
-		__skb_queue_tail(&d->sendq, skb);
+	if (skb) {
+		__skb_queue_head_init(&queue);
+		__skb_queue_tail(&queue, skb);
+		aoenet_xmit(&queue);
+	}
 	return 1;
 }
 
@@ -330,17 +420,25 @@ cont:
 }
 
 static void
-resend(struct aoedev *d, struct aoetgt *t, struct frame *f)
+resend(struct aoedev *d, struct frame *f)
 {
 	struct sk_buff *skb;
+	struct sk_buff_head queue;
 	struct aoe_hdr *h;
 	struct aoe_atahdr *ah;
+	struct aoetgt *t;
 	char buf[128];
 	u32 n;
 
-	ifrotate(t);
-	n = newtag(t);
+	t = f->t;
+	n = newtag(d);
 	skb = f->skb;
+	if (ifrotate(t) == NULL) {
+		/* probably can't happen, but set it up to fail anyway */
+		pr_info("aoe: resend: no interfaces to rotate to.\n");
+		ktcomplete(f, NULL);
+		return;
+	}
 	h = (struct aoe_hdr *) skb_mac_header(skb);
 	ah = (struct aoe_atahdr *) (h+1);
 
@@ -351,39 +449,22 @@ resend(struct aoedev *d, struct aoetgt *t, struct frame *f)
 	aoechr_error(buf);
 
 	f->tag = n;
+	fhash(f);
 	h->tag = cpu_to_be32(n);
 	memcpy(h->dst, t->addr, sizeof h->dst);
 	memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 
-	switch (ah->cmdstat) {
-	default:
-		break;
-	case ATA_CMD_PIO_READ:
-	case ATA_CMD_PIO_READ_EXT:
-	case ATA_CMD_PIO_WRITE:
-	case ATA_CMD_PIO_WRITE_EXT:
-		put_lba(ah, f->lba);
-
-		n = f->bcnt;
-		if (n > DEFAULTBCNT)
-			n = DEFAULTBCNT;
-		ah->scnt = n >> 9;
-		if (ah->aflags & AOEAFL_WRITE) {
-			skb_fill_page_desc(skb, 0, virt_to_page(f->bufaddr),
-				offset_in_page(f->bufaddr), n);
-			skb->len = sizeof *h + sizeof *ah + n;
-			skb->data_len = n;
-		}
-	}
 	skb->dev = t->ifp->nd;
 	skb = skb_clone(skb, GFP_ATOMIC);
 	if (skb == NULL)
 		return;
-	__skb_queue_tail(&d->sendq, skb);
+	__skb_queue_head_init(&queue);
+	__skb_queue_tail(&queue, skb);
+	aoenet_xmit(&queue);
 }
 
 static int
-tsince(int tag)
+tsince(u32 tag)
 {
 	int n;
 
@@ -407,58 +488,65 @@ getif(struct aoetgt *t, struct net_device *nd)
 	return NULL;
 }
 
-static struct aoeif *
-addif(struct aoetgt *t, struct net_device *nd)
-{
-	struct aoeif *p;
-
-	p = getif(t, NULL);
-	if (!p)
-		return NULL;
-	p->nd = nd;
-	p->maxbcnt = DEFAULTBCNT;
-	p->lost = 0;
-	p->lostjumbo = 0;
-	return p;
-}
-
 static void
 ejectif(struct aoetgt *t, struct aoeif *ifp)
 {
 	struct aoeif *e;
+	struct net_device *nd;
 	ulong n;
 
+	nd = ifp->nd;
 	e = t->ifs + NAOEIFS - 1;
 	n = (e - ifp) * sizeof *ifp;
 	memmove(ifp, ifp+1, n);
 	e->nd = NULL;
+	dev_put(nd);
 }
 
 static int
 sthtith(struct aoedev *d)
 {
-	struct frame *f, *e, *nf;
+	struct frame *f, *nf;
+	struct list_head *nx, *pos, *head;
 	struct sk_buff *skb;
-	struct aoetgt *ht = *d->htgt;
-
-	f = ht->frames;
-	e = f + ht->nframes;
-	for (; f < e; f++) {
-		if (f->tag == FREETAG)
-			continue;
-		nf = freeframe(d);
-		if (!nf)
-			return 0;
-		skb = nf->skb;
-		*nf = *f;
-		f->skb = skb;
-		f->tag = FREETAG;
-		nf->waited = 0;
-		ht->nout--;
-		(*d->tgt)->nout++;
-		resend(d, *d->tgt, nf);
+	struct aoetgt *ht = d->htgt;
+	int i;
+
+	for (i = 0; i < NFACTIVE; i++) {
+		head = &d->factive[i];
+		list_for_each_safe(pos, nx, head) {
+			f = list_entry(pos, struct frame, head);
+			if (f->t != ht)
+				continue;
+
+			nf = newframe(d);
+			if (!nf)
+				return 0;
+
+			/* remove frame from active list */
+			list_del(pos);
+
+			/* reassign all pertinent bits to new outbound frame */
+			skb = nf->skb;
+			nf->skb = f->skb;
+			nf->buf = f->buf;
+			nf->bcnt = f->bcnt;
+			nf->lba = f->lba;
+			nf->bv = f->bv;
+			nf->bv_off = f->bv_off;
+			nf->waited = 0;
+			f->skb = skb;
+			aoe_freetframe(f);
+			ht->nout--;
+			nf->t->nout++;
+			resend(d, nf);
+		}
 	}
-	/* he's clean, he's useless.  take away his interfaces */
+	/* We've cleaned up the outstanding so take away his
+	 * interfaces so he won't be used.  We should remove him from
+	 * the target array here, but cleaning up a target is
+	 * involved.  PUNT!
+	 */
 	memset(ht->ifs, 0, sizeof ht->ifs);
 	d->htgt = NULL;
 	return 1;
@@ -477,13 +565,15 @@ ata_scnt(unsigned char *packet) {
 static void
 rexmit_timer(ulong vp)
 {
-	struct sk_buff_head queue;
 	struct aoedev *d;
 	struct aoetgt *t, **tt, **te;
 	struct aoeif *ifp;
-	struct frame *f, *e;
+	struct frame *f;
+	struct list_head *head, *pos, *nx;
+	LIST_HEAD(flist);
 	register long timeout;
 	ulong flags, n;
+	int i;
 
 	d = (struct aoedev *) vp;
 
@@ -497,58 +587,22 @@ rexmit_timer(ulong vp)
 		spin_unlock_irqrestore(&d->lock, flags);
 		return;
 	}
-	tt = d->targets;
-	te = tt + NTARGETS;
-	for (; tt < te && *tt; tt++) {
-		t = *tt;
-		f = t->frames;
-		e = f + t->nframes;
-		for (; f < e; f++) {
-			if (f->tag == FREETAG
-			|| tsince(f->tag) < timeout)
-				continue;
-			n = f->waited += timeout;
-			n /= HZ;
-			if (n > aoe_deadsecs) {
-				/* waited too long.  device failure. */
-				aoedev_downdev(d);
-				break;
-			}
-
-			if (n > HELPWAIT /* see if another target can help */
-			&& (tt != d->targets || d->targets[1]))
-				d->htgt = tt;
-
-			if (t->nout == t->maxout) {
-				if (t->maxout > 1)
-					t->maxout--;
-				t->lastwadj = jiffies;
-			}
-
-			ifp = getif(t, f->skb->dev);
-			if (ifp && ++ifp->lost > (t->nframes << 1)
-			&& (ifp != t->ifs || t->ifs[1].nd)) {
-				ejectif(t, ifp);
-				ifp = NULL;
-			}
 
-			if (ata_scnt(skb_mac_header(f->skb)) > DEFAULTBCNT / 512
-			&& ifp && ++ifp->lostjumbo > (t->nframes << 1)
-			&& ifp->maxbcnt != DEFAULTBCNT) {
-				printk(KERN_INFO
-					"aoe: e%ld.%d: "
-					"too many lost jumbo on "
-					"%s:%pm - "
-					"falling back to %d frames.\n",
-					d->aoemajor, d->aoeminor,
-					ifp->nd->name, t->addr,
-					DEFAULTBCNT);
-				ifp->maxbcnt = 0;
-			}
-			resend(d, t, f);
+	/* collect all frames to rexmit into flist */
+	for (i = 0; i < NFACTIVE; i++) {
+		head = &d->factive[i];
+		list_for_each_safe(pos, nx, head) {
+			f = list_entry(pos, struct frame, head);
+			if (tsince(f->tag) < timeout)
+				break;	/* end of expired frames */
+			/* move to flist for later processing */
+			list_move_tail(pos, &flist);
 		}
-
-		/* window check */
+	}
+	/* window check */
+	tt = d->targets;
+	te = tt + d->ntargets;
+	for (; tt < te && (t = *tt); tt++) {
 		if (t->nout == t->maxout
 		&& t->maxout < t->nframes
 		&& (jiffies - t->lastwadj)/HZ > 10) {
@@ -557,45 +611,173 @@ rexmit_timer(ulong vp)
 		}
 	}
 
-	if (!skb_queue_empty(&d->sendq)) {
+	if (!list_empty(&flist)) {	/* retransmissions necessary */
 		n = d->rttavg <<= 1;
 		if (n > MAXTIMER)
 			d->rttavg = MAXTIMER;
 	}
 
-	if (d->flags & DEVFL_KICKME || d->htgt) {
-		d->flags &= ~DEVFL_KICKME;
-		aoecmd_work(d);
+	/* process expired frames */
+	while (!list_empty(&flist)) {
+		pos = flist.next;
+		f = list_entry(pos, struct frame, head);
+		n = f->waited += timeout;
+		n /= HZ;
+		if (n > aoe_deadsecs) {
+			/* Waited too long.  Device failure.
+			 * Hang all frames on first hash bucket for downdev
+			 * to clean up.
+			 */
+			list_splice(&flist, &d->factive[0]);
+			aoedev_downdev(d);
+			break;
+		}
+		list_del(pos);
+
+		t = f->t;
+		if (n > aoe_deadsecs/2)
+			d->htgt = t; /* see if another target can help */
+
+		if (t->nout == t->maxout) {
+			if (t->maxout > 1)
+				t->maxout--;
+			t->lastwadj = jiffies;
+		}
+
+		ifp = getif(t, f->skb->dev);
+		if (ifp && ++ifp->lost > (t->nframes << 1)
+		&& (ifp != t->ifs || t->ifs[1].nd)) {
+			ejectif(t, ifp);
+			ifp = NULL;
+		}
+		resend(d, f);
 	}
 
-	__skb_queue_head_init(&queue);
-	skb_queue_splice_init(&d->sendq, &queue);
+	if ((d->flags & DEVFL_KICKME || d->htgt) && d->blkq) {
+		d->flags &= ~DEVFL_KICKME;
+		d->blkq->request_fn(d->blkq);
+	}
 
 	d->timer.expires = jiffies + TIMERTICK;
 	add_timer(&d->timer);
 
 	spin_unlock_irqrestore(&d->lock, flags);
+}
 
-	aoenet_xmit(&queue);
+static unsigned long
+rqbiocnt(struct request *r)
+{
+	struct bio *bio;
+	unsigned long n = 0;
+
+	__rq_for_each_bio(bio, r)
+		n++;
+	return n;
+}
+
+/* This can be removed if we are certain that no users of the block
+ * layer will ever use zero-count pages in bios.  Otherwise we have to
+ * protect against the put_page sometimes done by the network layer.
+ *
+ * See http://oss.sgi.com/archives/xfs/2007-01/msg00594.html for
+ * discussion.
+ *
+ * We cannot use get_page in the workaround, because it insists on a
+ * positive page count as a precondition.  So we use _count directly.
+ */
+static void
+bio_pageinc(struct bio *bio)
+{
+	struct bio_vec *bv;
+	struct page *page;
+	int i;
+
+	bio_for_each_segment(bv, bio, i) {
+		page = bv->bv_page;
+		/* Non-zero page count for non-head members of
+		 * compound pages is no longer allowed by the kernel,
+		 * but this has never been seen here.
+		 */
+		if (unlikely(PageCompound(page)))
+			if (compound_trans_head(page) != page) {
+				pr_crit("page tail used for block I/O\n");
+				BUG();
+			}
+		atomic_inc(&page->_count);
+	}
+}
+
+static void
+bio_pagedec(struct bio *bio)
+{
+	struct bio_vec *bv;
+	int i;
+
+	bio_for_each_segment(bv, bio, i)
+		atomic_dec(&bv->bv_page->_count);
+}
+
+static void
+bufinit(struct buf *buf, struct request *rq, struct bio *bio)
+{
+	struct bio_vec *bv;
+
+	memset(buf, 0, sizeof(*buf));
+	buf->rq = rq;
+	buf->bio = bio;
+	buf->resid = bio->bi_size;
+	buf->sector = bio->bi_sector;
+	bio_pageinc(bio);
+	buf->bv = bv = &bio->bi_io_vec[bio->bi_idx];
+	buf->bv_resid = bv->bv_len;
+	WARN_ON(buf->bv_resid == 0);
+}
+
+static struct buf *
+nextbuf(struct aoedev *d)
+{
+	struct request *rq;
+	struct request_queue *q;
+	struct buf *buf;
+	struct bio *bio;
+
+	q = d->blkq;
+	if (q == NULL)
+		return NULL;	/* initializing */
+	if (d->ip.buf)
+		return d->ip.buf;
+	rq = d->ip.rq;
+	if (rq == NULL) {
+		rq = blk_peek_request(q);
+		if (rq == NULL)
+			return NULL;
+		blk_start_request(rq);
+		d->ip.rq = rq;
+		d->ip.nxbio = rq->bio;
+		rq->special = (void *) rqbiocnt(rq);
+	}
+	buf = mempool_alloc(d->bufpool, GFP_ATOMIC);
+	if (buf == NULL) {
+		pr_err("aoe: nextbuf: unable to mempool_alloc!\n");
+		return NULL;
+	}
+	bio = d->ip.nxbio;
+	bufinit(buf, rq, bio);
+	bio = bio->bi_next;
+	d->ip.nxbio = bio;
+	if (bio == NULL)
+		d->ip.rq = NULL;
+	return d->ip.buf = buf;
 }
 
 /* enters with d->lock held */
 void
 aoecmd_work(struct aoedev *d)
 {
-	struct buf *buf;
-loop:
 	if (d->htgt && !sthtith(d))
 		return;
-	if (d->inprocess == NULL) {
-		if (list_empty(&d->bufq))
-			return;
-		buf = container_of(d->bufq.next, struct buf, bufs);
-		list_del(d->bufq.next);
-		d->inprocess = buf;
-	}
-	if (aoecmd_ata_rw(d))
-		goto loop;
+	while (aoecmd_ata_rw(d))
+		;
 }
 
 /* this function performs work that has been deferred until sleeping is OK
@@ -604,28 +786,25 @@ void
 aoecmd_sleepwork(struct work_struct *work)
 {
 	struct aoedev *d = container_of(work, struct aoedev, work);
+	struct block_device *bd;
+	u64 ssize;
 
 	if (d->flags & DEVFL_GDALLOC)
 		aoeblk_gdalloc(d);
 
 	if (d->flags & DEVFL_NEWSIZE) {
-		struct block_device *bd;
-		unsigned long flags;
-		u64 ssize;
-
 		ssize = get_capacity(d->gd);
 		bd = bdget_disk(d->gd, 0);
-
 		if (bd) {
 			mutex_lock(&bd->bd_inode->i_mutex);
 			i_size_write(bd->bd_inode, (loff_t)ssize<<9);
 			mutex_unlock(&bd->bd_inode->i_mutex);
 			bdput(bd);
 		}
-		spin_lock_irqsave(&d->lock, flags);
+		spin_lock_irq(&d->lock);
 		d->flags |= DEVFL_UP;
 		d->flags &= ~DEVFL_NEWSIZE;
-		spin_unlock_irqrestore(&d->lock, flags);
+		spin_unlock_irq(&d->lock);
 	}
 }
 
@@ -718,163 +897,299 @@ gettgt(struct aoedev *d, char *addr)
 	return NULL;
 }
 
-static inline void
-diskstats(struct gendisk *disk, struct bio *bio, ulong duration, sector_t sector)
+static void
+bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt)
+{
+	ulong fcnt;
+	char *p;
+	int soff = 0;
+loop:
+	fcnt = bv->bv_len - (off - bv->bv_offset);
+	if (fcnt > cnt)
+		fcnt = cnt;
+	p = page_address(bv->bv_page) + off;
+	skb_copy_bits(skb, soff, p, fcnt);
+	soff += fcnt;
+	cnt -= fcnt;
+	if (cnt <= 0)
+		return;
+	bv++;
+	off = bv->bv_offset;
+	goto loop;
+}
+
+void
+aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
+{
+	struct bio *bio;
+	int bok;
+	struct request_queue *q;
+
+	q = d->blkq;
+	if (rq == d->ip.rq)
+		d->ip.rq = NULL;
+	do {
+		bio = rq->bio;
+		bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags);
+	} while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_size));
+
+	/* cf. http://lkml.org/lkml/2006/10/31/28 */
+	if (!fastfail)
+		q->request_fn(q);
+}
+
+static void
+aoe_end_buf(struct aoedev *d, struct buf *buf)
+{
+	struct request *rq;
+	unsigned long n;
+
+	if (buf == d->ip.buf)
+		d->ip.buf = NULL;
+	rq = buf->rq;
+	bio_pagedec(buf->bio);
+	mempool_free(buf, d->bufpool);
+	n = (unsigned long) rq->special;
+	rq->special = (void *) --n;
+	if (n == 0)
+		aoe_end_request(d, rq, 0);
+}
+
+static void
+ktiocomplete(struct frame *f)
 {
-	unsigned long n_sect = bio->bi_size >> 9;
-	const int rw = bio_data_dir(bio);
-	struct hd_struct *part;
-	int cpu;
+	struct aoe_hdr *hin, *hout;
+	struct aoe_atahdr *ahin, *ahout;
+	struct buf *buf;
+	struct sk_buff *skb;
+	struct aoetgt *t;
+	struct aoeif *ifp;
+	struct aoedev *d;
+	long n;
+
+	if (f == NULL)
+		return;
+
+	t = f->t;
+	d = t->d;
+
+	hout = (struct aoe_hdr *) skb_mac_header(f->skb);
+	ahout = (struct aoe_atahdr *) (hout+1);
+	buf = f->buf;
+	skb = f->r_skb;
+	if (skb == NULL)
+		goto noskb;	/* just fail the buf. */
+
+	hin = (struct aoe_hdr *) skb->data;
+	skb_pull(skb, sizeof(*hin));
+	ahin = (struct aoe_atahdr *) skb->data;
+	skb_pull(skb, sizeof(*ahin));
+	if (ahin->cmdstat & 0xa9) {	/* these bits cleared on success */
+		pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
+			ahout->cmdstat, ahin->cmdstat,
+			d->aoemajor, d->aoeminor);
+noskb:	if (buf)
+			clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
+		goto badrsp;
+	}
 
-	cpu = part_stat_lock();
-	part = disk_map_sector_rcu(disk, sector);
+	n = ahout->scnt << 9;
+	switch (ahout->cmdstat) {
+	case ATA_CMD_PIO_READ:
+	case ATA_CMD_PIO_READ_EXT:
+		if (skb->len < n) {
+			pr_err("aoe: runt data size in read.  skb->len=%d need=%ld\n",
+				skb->len, n);
+			clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
+			break;
+		}
+		bvcpy(f->bv, f->bv_off, skb, n);
+	case ATA_CMD_PIO_WRITE:
+	case ATA_CMD_PIO_WRITE_EXT:
+		spin_lock_irq(&d->lock);
+		ifp = getif(t, skb->dev);
+		if (ifp)
+			ifp->lost = 0;
+		if (d->htgt == t) /* I'll help myself, thank you. */
+			d->htgt = NULL;
+		spin_unlock_irq(&d->lock);
+		break;
+	case ATA_CMD_ID_ATA:
+		if (skb->len < 512) {
+			pr_info("aoe: runt data size in ataid.  skb->len=%d\n",
+				skb->len);
+			break;
+		}
+		if (skb_linearize(skb))
+			break;
+		spin_lock_irq(&d->lock);
+		ataid_complete(d, t, skb->data);
+		spin_unlock_irq(&d->lock);
+		break;
+	default:
+		pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
+			ahout->cmdstat,
+			be16_to_cpu(get_unaligned(&hin->major)),
+			hin->minor);
+	}
+badrsp:
+	spin_lock_irq(&d->lock);
+
+	aoe_freetframe(f);
+
+	if (buf && --buf->nframesout == 0 && buf->resid == 0)
+		aoe_end_buf(d, buf);
+
+	aoecmd_work(d);
+
+	spin_unlock_irq(&d->lock);
+	aoedev_put(d);
+	dev_kfree_skb(skb);
+}
+
+/* Enters with iocq.lock held.
+ * Returns true iff responses needing processing remain.
+ */
+static int
+ktio(void)
+{
+	struct frame *f;
+	struct list_head *pos;
+	int i;
 
-	part_stat_inc(cpu, part, ios[rw]);
-	part_stat_add(cpu, part, ticks[rw], duration);
-	part_stat_add(cpu, part, sectors[rw], n_sect);
-	part_stat_add(cpu, part, io_ticks, duration);
+	for (i = 0; ; ++i) {
+		if (i == MAXIOC)
+			return 1;
+		if (list_empty(&iocq.head))
+			return 0;
+		pos = iocq.head.next;
+		list_del(pos);
+		spin_unlock_irq(&iocq.lock);
+		f = list_entry(pos, struct frame, head);
+		ktiocomplete(f);
+		spin_lock_irq(&iocq.lock);
+	}
+}
 
-	part_stat_unlock();
+static int
+kthread(void *vp)
+{
+	struct ktstate *k;
+	DECLARE_WAITQUEUE(wait, current);
+	int more;
+
+	k = vp;
+	current->flags |= PF_NOFREEZE;
+	set_user_nice(current, -10);
+	complete(&k->rendez);	/* tell spawner we're running */
+	do {
+		spin_lock_irq(k->lock);
+		more = k->fn();
+		if (!more) {
+			add_wait_queue(k->waitq, &wait);
+			__set_current_state(TASK_INTERRUPTIBLE);
+		}
+		spin_unlock_irq(k->lock);
+		if (!more) {
+			schedule();
+			remove_wait_queue(k->waitq, &wait);
+		} else
+			cond_resched();
+	} while (!kthread_should_stop());
+	complete(&k->rendez);	/* tell spawner we're stopping */
+	return 0;
 }
 
 void
+aoe_ktstop(struct ktstate *k)
+{
+	kthread_stop(k->task);
+	wait_for_completion(&k->rendez);
+}
+
+int
+aoe_ktstart(struct ktstate *k)
+{
+	struct task_struct *task;
+
+	init_completion(&k->rendez);
+	task = kthread_run(kthread, k, k->name);
+	if (task == NULL || IS_ERR(task))
+		return -ENOMEM;
+	k->task = task;
+	wait_for_completion(&k->rendez); /* allow kthread to start */
+	init_completion(&k->rendez);	/* for waiting for exit later */
+	return 0;
+}
+
+/* pass it off to kthreads for processing */
+static void
+ktcomplete(struct frame *f, struct sk_buff *skb)
+{
+	ulong flags;
+
+	f->r_skb = skb;
+	spin_lock_irqsave(&iocq.lock, flags);
+	list_add_tail(&f->head, &iocq.head);
+	spin_unlock_irqrestore(&iocq.lock, flags);
+	wake_up(&ktiowq);
+}
+
+struct sk_buff *
 aoecmd_ata_rsp(struct sk_buff *skb)
 {
-	struct sk_buff_head queue;
 	struct aoedev *d;
-	struct aoe_hdr *hin, *hout;
-	struct aoe_atahdr *ahin, *ahout;
+	struct aoe_hdr *h;
 	struct frame *f;
-	struct buf *buf;
 	struct aoetgt *t;
-	struct aoeif *ifp;
-	register long n;
+	u32 n;
 	ulong flags;
 	char ebuf[128];
 	u16 aoemajor;
 
-	hin = (struct aoe_hdr *) skb_mac_header(skb);
-	aoemajor = get_unaligned_be16(&hin->major);
-	d = aoedev_by_aoeaddr(aoemajor, hin->minor);
+	h = (struct aoe_hdr *) skb->data;
+	aoemajor = be16_to_cpu(get_unaligned(&h->major));
+	d = aoedev_by_aoeaddr(aoemajor, h->minor, 0);
 	if (d == NULL) {
 		snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
 			"for unknown device %d.%d\n",
-			 aoemajor, hin->minor);
+			aoemajor, h->minor);
 		aoechr_error(ebuf);
-		return;
+		return skb;
 	}
 
 	spin_lock_irqsave(&d->lock, flags);
 
-	n = get_unaligned_be32(&hin->tag);
-	t = gettgt(d, hin->src);
-	if (t == NULL) {
-		printk(KERN_INFO "aoe: can't find target e%ld.%d:%pm\n",
-			d->aoemajor, d->aoeminor, hin->src);
-		spin_unlock_irqrestore(&d->lock, flags);
-		return;
-	}
-	f = getframe(t, n);
+	n = be32_to_cpu(get_unaligned(&h->tag));
+	f = getframe(d, n);
 	if (f == NULL) {
 		calc_rttavg(d, -tsince(n));
 		spin_unlock_irqrestore(&d->lock, flags);
+		aoedev_put(d);
 		snprintf(ebuf, sizeof ebuf,
 			"%15s e%d.%d    tag=%08x@%08lx\n",
 			"unexpected rsp",
-			get_unaligned_be16(&hin->major),
-			hin->minor,
-			get_unaligned_be32(&hin->tag),
+			get_unaligned_be16(&h->major),
+			h->minor,
+			get_unaligned_be32(&h->tag),
 			jiffies);
 		aoechr_error(ebuf);
-		return;
+		return skb;
 	}
-
+	t = f->t;
 	calc_rttavg(d, tsince(f->tag));
-
-	ahin = (struct aoe_atahdr *) (hin+1);
-	hout = (struct aoe_hdr *) skb_mac_header(f->skb);
-	ahout = (struct aoe_atahdr *) (hout+1);
-	buf = f->buf;
-
-	if (ahin->cmdstat & 0xa9) {	/* these bits cleared on success */
-		printk(KERN_ERR
-			"aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
-			ahout->cmdstat, ahin->cmdstat,
-			d->aoemajor, d->aoeminor);
-		if (buf)
-			buf->flags |= BUFFL_FAIL;
-	} else {
-		if (d->htgt && t == *d->htgt) /* I'll help myself, thank you. */
-			d->htgt = NULL;
-		n = ahout->scnt << 9;
-		switch (ahout->cmdstat) {
-		case ATA_CMD_PIO_READ:
-		case ATA_CMD_PIO_READ_EXT:
-			if (skb->len - sizeof *hin - sizeof *ahin < n) {
-				printk(KERN_ERR
-					"aoe: %s.  skb->len=%d need=%ld\n",
-					"runt data size in read", skb->len, n);
-				/* fail frame f?  just returning will rexmit. */
-				spin_unlock_irqrestore(&d->lock, flags);
-				return;
-			}
-			memcpy(f->bufaddr, ahin+1, n);
-		case ATA_CMD_PIO_WRITE:
-		case ATA_CMD_PIO_WRITE_EXT:
-			ifp = getif(t, skb->dev);
-			if (ifp) {
-				ifp->lost = 0;
-				if (n > DEFAULTBCNT)
-					ifp->lostjumbo = 0;
-			}
-			if (f->bcnt -= n) {
-				f->lba += n >> 9;
-				f->bufaddr += n;
-				resend(d, t, f);
-				goto xmit;
-			}
-			break;
-		case ATA_CMD_ID_ATA:
-			if (skb->len - sizeof *hin - sizeof *ahin < 512) {
-				printk(KERN_INFO
-					"aoe: runt data size in ataid.  skb->len=%d\n",
-					skb->len);
-				spin_unlock_irqrestore(&d->lock, flags);
-				return;
-			}
-			ataid_complete(d, t, (char *) (ahin+1));
-			break;
-		default:
-			printk(KERN_INFO
-				"aoe: unrecognized ata command %2.2Xh for %d.%d\n",
-				ahout->cmdstat,
-				get_unaligned_be16(&hin->major),
-				hin->minor);
-		}
-	}
-
-	if (buf && --buf->nframesout == 0 && buf->resid == 0) {
-		diskstats(d->gd, buf->bio, jiffies - buf->stime, buf->sector);
-		if (buf->flags & BUFFL_FAIL)
-			bio_endio(buf->bio, -EIO);
-		else {
-			bio_flush_dcache_pages(buf->bio);
-			bio_endio(buf->bio, 0);
-		}
-		mempool_free(buf, d->bufpool);
-	}
-
-	f->buf = NULL;
-	f->tag = FREETAG;
 	t->nout--;
-
 	aoecmd_work(d);
-xmit:
-	__skb_queue_head_init(&queue);
-	skb_queue_splice_init(&d->sendq, &queue);
 
 	spin_unlock_irqrestore(&d->lock, flags);
-	aoenet_xmit(&queue);
+
+	ktcomplete(f, skb);
+
+	/*
+	 * Note here that we do not perform an aoedev_put, as we are
+	 * leaving this reference for the ktio to release.
+	 */
+	return NULL;
 }
 
 void
@@ -896,7 +1211,7 @@ aoecmd_ata_id(struct aoedev *d)
 	struct sk_buff *skb;
 	struct aoetgt *t;
 
-	f = freeframe(d);
+	f = newframe(d);
 	if (f == NULL)
 		return NULL;
 
@@ -909,6 +1224,7 @@ aoecmd_ata_id(struct aoedev *d)
 	skb_put(skb, sizeof *h + sizeof *ah);
 	memset(h, 0, skb->len);
 	f->tag = aoehdr_atainit(d, t, h);
+	fhash(f);
 	t->nout++;
 	f->waited = 0;
 
@@ -929,7 +1245,6 @@ static struct aoetgt *
 addtgt(struct aoedev *d, char *addr, ulong nframes)
 {
 	struct aoetgt *t, **tt, **te;
-	struct frame *f, *e;
 
 	tt = d->targets;
 	te = tt + NTARGETS;
@@ -941,26 +1256,73 @@ addtgt(struct aoedev *d, char *addr, ulong nframes)
 			"aoe: device addtgt failure; too many targets\n");
 		return NULL;
 	}
-	t = kcalloc(1, sizeof *t, GFP_ATOMIC);
-	f = kcalloc(nframes, sizeof *f, GFP_ATOMIC);
-	if (!t || !f) {
-		kfree(f);
-		kfree(t);
+	t = kzalloc(sizeof(*t), GFP_ATOMIC);
+	if (!t) {
 		printk(KERN_INFO "aoe: cannot allocate memory to add target\n");
 		return NULL;
 	}
 
+	d->ntargets++;
 	t->nframes = nframes;
-	t->frames = f;
-	e = f + nframes;
-	for (; f < e; f++)
-		f->tag = FREETAG;
+	t->d = d;
 	memcpy(t->addr, addr, sizeof t->addr);
 	t->ifp = t->ifs;
 	t->maxout = t->nframes;
+	INIT_LIST_HEAD(&t->ffree);
 	return *tt = t;
 }
 
+static void
+setdbcnt(struct aoedev *d)
+{
+	struct aoetgt **t, **e;
+	int bcnt = 0;
+
+	t = d->targets;
+	e = t + NTARGETS;
+	for (; t < e && *t; t++)
+		if (bcnt == 0 || bcnt > (*t)->minbcnt)
+			bcnt = (*t)->minbcnt;
+	if (bcnt != d->maxbcnt) {
+		d->maxbcnt = bcnt;
+		pr_info("aoe: e%ld.%d: setting %d byte data frames\n",
+			d->aoemajor, d->aoeminor, bcnt);
+	}
+}
+
+static void
+setifbcnt(struct aoetgt *t, struct net_device *nd, int bcnt)
+{
+	struct aoedev *d;
+	struct aoeif *p, *e;
+	int minbcnt;
+
+	d = t->d;
+	minbcnt = bcnt;
+	p = t->ifs;
+	e = p + NAOEIFS;
+	for (; p < e; p++) {
+		if (p->nd == NULL)
+			break;		/* end of the valid interfaces */
+		if (p->nd == nd) {
+			p->bcnt = bcnt;	/* we're updating */
+			nd = NULL;
+		} else if (minbcnt > p->bcnt)
+			minbcnt = p->bcnt; /* find the min interface */
+	}
+	if (nd) {
+		if (p == e) {
+			pr_err("aoe: device setifbcnt failure; too many interfaces.\n");
+			return;
+		}
+		dev_hold(nd);
+		p->nd = nd;
+		p->bcnt = bcnt;
+	}
+	t->minbcnt = minbcnt;
+	setdbcnt(d);
+}
+
 void
 aoecmd_cfg_rsp(struct sk_buff *skb)
 {
@@ -968,11 +1330,12 @@ aoecmd_cfg_rsp(struct sk_buff *skb)
 	struct aoe_hdr *h;
 	struct aoe_cfghdr *ch;
 	struct aoetgt *t;
-	struct aoeif *ifp;
-	ulong flags, sysminor, aoemajor;
+	ulong flags, aoemajor;
 	struct sk_buff *sl;
+	struct sk_buff_head queue;
 	u16 n;
 
+	sl = NULL;
 	h = (struct aoe_hdr *) skb_mac_header(skb);
 	ch = (struct aoe_cfghdr *) (h+1);
 
@@ -986,10 +1349,13 @@ aoecmd_cfg_rsp(struct sk_buff *skb)
 			"Check shelf dip switches.\n");
 		return;
 	}
-
-	sysminor = SYSMINOR(aoemajor, h->minor);
-	if (sysminor * AOE_PARTITIONS + AOE_PARTITIONS > MINORMASK) {
-		printk(KERN_INFO "aoe: e%ld.%d: minor number too large\n",
+	if (aoemajor == 0xffff) {
+		pr_info("aoe: e%ld.%d: broadcast shelf number invalid\n",
+			aoemajor, (int) h->minor);
+		return;
+	}
+	if (h->minor == 0xff) {
+		pr_info("aoe: e%ld.%d: broadcast slot number invalid\n",
 			aoemajor, (int) h->minor);
 		return;
 	}
@@ -998,9 +1364,9 @@ aoecmd_cfg_rsp(struct sk_buff *skb)
 	if (n > aoe_maxout)	/* keep it reasonable */
 		n = aoe_maxout;
 
-	d = aoedev_by_sysminor_m(sysminor);
+	d = aoedev_by_aoeaddr(aoemajor, h->minor, 1);
 	if (d == NULL) {
-		printk(KERN_INFO "aoe: device sysminor_m failure\n");
+		pr_info("aoe: device allocation failure\n");
 		return;
 	}
 
@@ -1009,52 +1375,26 @@ aoecmd_cfg_rsp(struct sk_buff *skb)
 	t = gettgt(d, h->src);
 	if (!t) {
 		t = addtgt(d, h->src, n);
-		if (!t) {
-			spin_unlock_irqrestore(&d->lock, flags);
-			return;
-		}
-	}
-	ifp = getif(t, skb->dev);
-	if (!ifp) {
-		ifp = addif(t, skb->dev);
-		if (!ifp) {
-			printk(KERN_INFO
-				"aoe: device addif failure; "
-				"too many interfaces?\n");
-			spin_unlock_irqrestore(&d->lock, flags);
-			return;
-		}
-	}
-	if (ifp->maxbcnt) {
-		n = ifp->nd->mtu;
-		n -= sizeof (struct aoe_hdr) + sizeof (struct aoe_atahdr);
-		n /= 512;
-		if (n > ch->scnt)
-			n = ch->scnt;
-		n = n ? n * 512 : DEFAULTBCNT;
-		if (n != ifp->maxbcnt) {
-			printk(KERN_INFO
-				"aoe: e%ld.%d: setting %d%s%s:%pm\n",
-				d->aoemajor, d->aoeminor, n,
-				" byte data frames on ", ifp->nd->name,
-				t->addr);
-			ifp->maxbcnt = n;
-		}
+		if (!t)
+			goto bail;
 	}
+	n = skb->dev->mtu;
+	n -= sizeof(struct aoe_hdr) + sizeof(struct aoe_atahdr);
+	n /= 512;
+	if (n > ch->scnt)
+		n = ch->scnt;
+	n = n ? n * 512 : DEFAULTBCNT;
+	setifbcnt(t, skb->dev, n);
 
 	/* don't change users' perspective */
-	if (d->nopen) {
-		spin_unlock_irqrestore(&d->lock, flags);
-		return;
+	if (d->nopen == 0) {
+		d->fw_ver = be16_to_cpu(ch->fwver);
+		sl = aoecmd_ata_id(d);
 	}
-	d->fw_ver = be16_to_cpu(ch->fwver);
-
-	sl = aoecmd_ata_id(d);
-
+bail:
 	spin_unlock_irqrestore(&d->lock, flags);
-
+	aoedev_put(d);
 	if (sl) {
-		struct sk_buff_head queue;
 		__skb_queue_head_init(&queue);
 		__skb_queue_tail(&queue, sl);
 		aoenet_xmit(&queue);
@@ -1065,20 +1405,74 @@ void
 aoecmd_cleanslate(struct aoedev *d)
 {
 	struct aoetgt **t, **te;
-	struct aoeif *p, *e;
 
 	d->mintimer = MINTIMER;
+	d->maxbcnt = 0;
 
 	t = d->targets;
 	te = t + NTARGETS;
-	for (; t < te && *t; t++) {
+	for (; t < te && *t; t++)
 		(*t)->maxout = (*t)->nframes;
-		p = (*t)->ifs;
-		e = p + NAOEIFS;
-		for (; p < e; p++) {
-			p->lostjumbo = 0;
-			p->lost = 0;
-			p->maxbcnt = DEFAULTBCNT;
+}
+
+void
+aoe_failbuf(struct aoedev *d, struct buf *buf)
+{
+	if (buf == NULL)
+		return;
+	buf->resid = 0;
+	clear_bit(BIO_UPTODATE, &buf->bio->bi_flags);
+	if (buf->nframesout == 0)
+		aoe_end_buf(d, buf);
+}
+
+void
+aoe_flush_iocq(void)
+{
+	struct frame *f;
+	struct aoedev *d;
+	LIST_HEAD(flist);
+	struct list_head *pos;
+	struct sk_buff *skb;
+	ulong flags;
+
+	spin_lock_irqsave(&iocq.lock, flags);
+	list_splice_init(&iocq.head, &flist);
+	spin_unlock_irqrestore(&iocq.lock, flags);
+	while (!list_empty(&flist)) {
+		pos = flist.next;
+		list_del(pos);
+		f = list_entry(pos, struct frame, head);
+		d = f->t->d;
+		skb = f->r_skb;
+		spin_lock_irqsave(&d->lock, flags);
+		if (f->buf) {
+			f->buf->nframesout--;
+			aoe_failbuf(d, f->buf);
 		}
+		aoe_freetframe(f);
+		spin_unlock_irqrestore(&d->lock, flags);
+		dev_kfree_skb(skb);
+		aoedev_put(d);
 	}
 }
+
+int __init
+aoecmd_init(void)
+{
+	INIT_LIST_HEAD(&iocq.head);
+	spin_lock_init(&iocq.lock);
+	init_waitqueue_head(&ktiowq);
+	kts.name = "aoe_ktio";
+	kts.fn = ktio;
+	kts.waitq = &ktiowq;
+	kts.lock = &iocq.lock;
+	return aoe_ktstart(&kts);
+}
+
+void
+aoecmd_exit(void)
+{
+	aoe_ktstop(&kts);
+	aoe_flush_iocq();
+}
diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c
index 6b5110a4745..90e5b537f94 100644
--- a/drivers/block/aoe/aoedev.c
+++ b/drivers/block/aoe/aoedev.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoedev.c
  * AoE device utility functions; maintains device list.
@@ -9,6 +9,9 @@
 #include <linux/netdevice.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
+#include <linux/bitmap.h>
+#include <linux/kdev_t.h>
+#include <linux/moduleparam.h>
 #include "aoe.h"
 
 static void dummy_timer(ulong);
@@ -16,23 +19,121 @@ static void aoedev_freedev(struct aoedev *);
 static void freetgt(struct aoedev *d, struct aoetgt *t);
 static void skbpoolfree(struct aoedev *d);
 
+static int aoe_dyndevs = 1;
+module_param(aoe_dyndevs, int, 0644);
+MODULE_PARM_DESC(aoe_dyndevs, "Use dynamic minor numbers for devices.");
+
 static struct aoedev *devlist;
 static DEFINE_SPINLOCK(devlist_lock);
 
-struct aoedev *
-aoedev_by_aoeaddr(int maj, int min)
+/* Because some systems will have one, many, or no
+ *   - partitions,
+ *   - slots per shelf,
+ *   - or shelves,
+ * we need some flexibility in the way the minor numbers
+ * are allocated.  So they are dynamic.
+ */
+#define N_DEVS ((1U<<MINORBITS)/AOE_PARTITIONS)
+
+static DEFINE_SPINLOCK(used_minors_lock);
+static DECLARE_BITMAP(used_minors, N_DEVS);
+
+static int
+minor_get_dyn(ulong *sysminor)
 {
-	struct aoedev *d;
 	ulong flags;
+	ulong n;
+	int error = 0;
+
+	spin_lock_irqsave(&used_minors_lock, flags);
+	n = find_first_zero_bit(used_minors, N_DEVS);
+	if (n < N_DEVS)
+		set_bit(n, used_minors);
+	else
+		error = -1;
+	spin_unlock_irqrestore(&used_minors_lock, flags);
+
+	*sysminor = n * AOE_PARTITIONS;
+	return error;
+}
 
-	spin_lock_irqsave(&devlist_lock, flags);
+static int
+minor_get_static(ulong *sysminor, ulong aoemaj, int aoemin)
+{
+	ulong flags;
+	ulong n;
+	int error = 0;
+	enum {
+		/* for backwards compatibility when !aoe_dyndevs,
+		 * a static number of supported slots per shelf */
+		NPERSHELF = 16,
+	};
+
+	n = aoemaj * NPERSHELF + aoemin;
+	if (aoemin >= NPERSHELF || n >= N_DEVS) {
+		pr_err("aoe: %s with e%ld.%d\n",
+			"cannot use static minor device numbers",
+			aoemaj, aoemin);
+		error = -1;
+	} else {
+		spin_lock_irqsave(&used_minors_lock, flags);
+		if (test_bit(n, used_minors)) {
+			pr_err("aoe: %s %lu\n",
+				"existing device already has static minor number",
+				n);
+			error = -1;
+		} else
+			set_bit(n, used_minors);
+		spin_unlock_irqrestore(&used_minors_lock, flags);
+	}
 
-	for (d=devlist; d; d=d->next)
-		if (d->aoemajor == maj && d->aoeminor == min)
-			break;
+	*sysminor = n;
+	return error;
+}
+
+static int
+minor_get(ulong *sysminor, ulong aoemaj, int aoemin)
+{
+	if (aoe_dyndevs)
+		return minor_get_dyn(sysminor);
+	else
+		return minor_get_static(sysminor, aoemaj, aoemin);
+}
+
+static void
+minor_free(ulong minor)
+{
+	ulong flags;
+
+	minor /= AOE_PARTITIONS;
+	BUG_ON(minor >= N_DEVS);
+
+	spin_lock_irqsave(&used_minors_lock, flags);
+	BUG_ON(!test_bit(minor, used_minors));
+	clear_bit(minor, used_minors);
+	spin_unlock_irqrestore(&used_minors_lock, flags);
+}
+
+/*
+ * Users who grab a pointer to the device with aoedev_by_aoeaddr
+ * automatically get a reference count and must be responsible
+ * for performing a aoedev_put.  With the addition of async
+ * kthread processing I'm no longer confident that we can
+ * guarantee consistency in the face of device flushes.
+ *
+ * For the time being, we only bother to add extra references for
+ * frames sitting on the iocq.  When the kthreads finish processing
+ * these frames, they will aoedev_put the device.
+ */
+
+void
+aoedev_put(struct aoedev *d)
+{
+	ulong flags;
 
+	spin_lock_irqsave(&devlist_lock, flags);
+	d->ref--;
 	spin_unlock_irqrestore(&devlist_lock, flags);
-	return d;
 }
 
 static void
@@ -47,54 +148,74 @@ dummy_timer(ulong vp)
 	add_timer(&d->timer);
 }
 
+static void
+aoe_failip(struct aoedev *d)
+{
+	struct request *rq;
+	struct bio *bio;
+	unsigned long n;
+
+	aoe_failbuf(d, d->ip.buf);
+
+	rq = d->ip.rq;
+	if (rq == NULL)
+		return;
+	while ((bio = d->ip.nxbio)) {
+		clear_bit(BIO_UPTODATE, &bio->bi_flags);
+		d->ip.nxbio = bio->bi_next;
+		n = (unsigned long) rq->special;
+		rq->special = (void *) --n;
+	}
+	if ((unsigned long) rq->special == 0)
+		aoe_end_request(d, rq, 0);
+}
+
 void
 aoedev_downdev(struct aoedev *d)
 {
-	struct aoetgt **t, **te;
-	struct frame *f, *e;
-	struct buf *buf;
-	struct bio *bio;
+	struct aoetgt *t, **tt, **te;
+	struct frame *f;
+	struct list_head *head, *pos, *nx;
+	struct request *rq;
+	int i;
 
-	t = d->targets;
-	te = t + NTARGETS;
-	for (; t < te && *t; t++) {
-		f = (*t)->frames;
-		e = f + (*t)->nframes;
-		for (; f < e; f->tag = FREETAG, f->buf = NULL, f++) {
-			if (f->tag == FREETAG || f->buf == NULL)
-				continue;
-			buf = f->buf;
-			bio = buf->bio;
-			if (--buf->nframesout == 0
-			&& buf != d->inprocess) {
-				mempool_free(buf, d->bufpool);
-				bio_endio(bio, -EIO);
+	d->flags &= ~DEVFL_UP;
+
+	/* clean out active buffers */
+	for (i = 0; i < NFACTIVE; i++) {
+		head = &d->factive[i];
+		list_for_each_safe(pos, nx, head) {
+			f = list_entry(pos, struct frame, head);
+			list_del(pos);
+			if (f->buf) {
+				f->buf->nframesout--;
+				aoe_failbuf(d, f->buf);
 			}
+			aoe_freetframe(f);
 		}
-		(*t)->maxout = (*t)->nframes;
-		(*t)->nout = 0;
 	}
-	buf = d->inprocess;
-	if (buf) {
-		bio = buf->bio;
-		mempool_free(buf, d->bufpool);
-		bio_endio(bio, -EIO);
+	/* reset window dressings */
+	tt = d->targets;
+	te = tt + NTARGETS;
+	for (; tt < te && (t = *tt); tt++) {
+		t->maxout = t->nframes;
+		t->nout = 0;
 	}
-	d->inprocess = NULL;
+
+	/* clean out the in-process request (if any) */
+	aoe_failip(d);
 	d->htgt = NULL;
 
-	while (!list_empty(&d->bufq)) {
-		buf = container_of(d->bufq.next, struct buf, bufs);
-		list_del(d->bufq.next);
-		bio = buf->bio;
-		mempool_free(buf, d->bufpool);
-		bio_endio(bio, -EIO);
+	/* fast fail all pending I/O */
+	if (d->blkq) {
+		while ((rq = blk_peek_request(d->blkq))) {
+			blk_start_request(rq);
+			aoe_end_request(d, rq, 1);
+		}
 	}
 
 	if (d->gd)
 		set_capacity(d->gd, 0);
-
-	d->flags &= ~DEVFL_UP;
 }
 
 static void
@@ -107,6 +228,7 @@ aoedev_freedev(struct aoedev *d)
 		aoedisk_rm_sysfs(d);
 		del_gendisk(d->gd);
 		put_disk(d->gd);
+		blk_cleanup_queue(d->blkq);
 	}
 	t = d->targets;
 	e = t + NTARGETS;
@@ -115,7 +237,7 @@ aoedev_freedev(struct aoedev *d)
 	if (d->bufpool)
 		mempool_destroy(d->bufpool);
 	skbpoolfree(d);
-	blk_cleanup_queue(d->blkq);
+	minor_free(d->sysminor);
 	kfree(d);
 }
 
@@ -142,7 +264,8 @@ aoedev_flush(const char __user *str, size_t cnt)
 		spin_lock(&d->lock);
 		if ((!all && (d->flags & DEVFL_UP))
 		|| (d->flags & (DEVFL_GDALLOC|DEVFL_NEWSIZE))
-		|| d->nopen) {
+		|| d->nopen
+		|| d->ref) {
 			spin_unlock(&d->lock);
 			dd = &d->next;
 			continue;
@@ -163,12 +286,15 @@ aoedev_flush(const char __user *str, size_t cnt)
 	return 0;
 }
 
-/* I'm not really sure that this is a realistic problem, but if the
-network driver goes gonzo let's just leak memory after complaining. */
+/* This has been confirmed to occur once with Tms=3*1000 due to the
+ * driver changing link and not processing its transmit ring.  The
+ * problem is hard enough to solve by returning an error that I'm
+ * still punting on "solving" this.
+ */
 static void
 skbfree(struct sk_buff *skb)
 {
-	enum { Sms = 100, Tms = 3*1000};
+	enum { Sms = 250, Tms = 30 * 1000};
 	int i = Tms / Sms;
 
 	if (skb == NULL)
@@ -182,6 +308,7 @@ skbfree(struct sk_buff *skb)
 			"cannot free skb -- memory leaked.");
 		return;
 	}
+	skb->truesize -= skb->data_len;
 	skb_shinfo(skb)->nr_frags = skb->data_len = 0;
 	skb_trim(skb, 0);
 	dev_kfree_skb(skb);
@@ -198,26 +325,29 @@ skbpoolfree(struct aoedev *d)
 	__skb_queue_head_init(&d->skbpool);
 }
 
-/* find it or malloc it */
+/* find it or allocate it */
 struct aoedev *
-aoedev_by_sysminor_m(ulong sysminor)
+aoedev_by_aoeaddr(ulong maj, int min, int do_alloc)
 {
 	struct aoedev *d;
+	int i;
 	ulong flags;
+	ulong sysminor;
 
 	spin_lock_irqsave(&devlist_lock, flags);
 
 	for (d=devlist; d; d=d->next)
-		if (d->sysminor == sysminor)
+		if (d->aoemajor == maj && d->aoeminor == min) {
+			d->ref++;
 			break;
-	if (d)
+		}
+	if (d || !do_alloc || minor_get(&sysminor, maj, min) < 0)
 		goto out;
 	d = kcalloc(1, sizeof *d, GFP_ATOMIC);
 	if (!d)
 		goto out;
 	INIT_WORK(&d->work, aoecmd_sleepwork);
 	spin_lock_init(&d->lock);
-	skb_queue_head_init(&d->sendq);
 	skb_queue_head_init(&d->skbpool);
 	init_timer(&d->timer);
 	d->timer.data = (ulong) d;
@@ -226,10 +356,12 @@ aoedev_by_sysminor_m(ulong sysminor)
 	add_timer(&d->timer);
 	d->bufpool = NULL;	/* defer to aoeblk_gdalloc */
 	d->tgt = d->targets;
-	INIT_LIST_HEAD(&d->bufq);
+	d->ref = 1;
+	for (i = 0; i < NFACTIVE; i++)
+		INIT_LIST_HEAD(&d->factive[i]);
 	d->sysminor = sysminor;
-	d->aoemajor = AOEMAJOR(sysminor);
-	d->aoeminor = AOEMINOR(sysminor);
+	d->aoemajor = maj;
+	d->aoeminor = min;
 	d->mintimer = MINTIMER;
 	d->next = devlist;
 	devlist = d;
@@ -241,13 +373,23 @@ aoedev_by_sysminor_m(ulong sysminor)
 static void
 freetgt(struct aoedev *d, struct aoetgt *t)
 {
-	struct frame *f, *e;
+	struct frame *f;
+	struct list_head *pos, *nx, *head;
+	struct aoeif *ifp;
 
-	f = t->frames;
-	e = f + t->nframes;
-	for (; f < e; f++)
+	for (ifp = t->ifs; ifp < &t->ifs[NAOEIFS]; ++ifp) {
+		if (!ifp->nd)
+			break;
+		dev_put(ifp->nd);
+	}
+
+	head = &t->ffree;
+	list_for_each_safe(pos, nx, head) {
+		list_del(pos);
+		f = list_entry(pos, struct frame, head);
 		skbfree(f->skb);
-	kfree(t->frames);
+		kfree(f);
+	}
 	kfree(t);
 }
 
@@ -257,6 +399,7 @@ aoedev_exit(void)
 	struct aoedev *d;
 	ulong flags;
 
+	aoe_flush_iocq();
 	while ((d = devlist)) {
 		devlist = d->next;
 
diff --git a/drivers/block/aoe/aoemain.c b/drivers/block/aoe/aoemain.c
index 7f83ad90e76..04793c2c701 100644
--- a/drivers/block/aoe/aoemain.c
+++ b/drivers/block/aoe/aoemain.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoemain.c
  * Module initialization routines, discover timer
@@ -61,6 +61,7 @@ aoe_exit(void)
 
 	aoenet_exit();
 	unregister_blkdev(AOE_MAJOR, DEVICE_NAME);
+	aoecmd_exit();
 	aoechr_exit();
 	aoedev_exit();
 	aoeblk_exit();		/* free cache after de-allocating bufs */
@@ -83,17 +84,20 @@ aoe_init(void)
 	ret = aoenet_init();
 	if (ret)
 		goto net_fail;
+	ret = aoecmd_init();
+	if (ret)
+		goto cmd_fail;
 	ret = register_blkdev(AOE_MAJOR, DEVICE_NAME);
 	if (ret < 0) {
 		printk(KERN_ERR "aoe: can't register major\n");
 		goto blkreg_fail;
 	}
-
 	printk(KERN_INFO "aoe: AoE v%s initialised.\n", VERSION);
 	discover_timer(TINIT);
 	return 0;
-
  blkreg_fail:
+	aoecmd_exit();
+ cmd_fail:
 	aoenet_exit();
  net_fail:
 	aoeblk_exit();
diff --git a/drivers/block/aoe/aoenet.c b/drivers/block/aoe/aoenet.c
index 4d3bc0d49df..162c6471275 100644
--- a/drivers/block/aoe/aoenet.c
+++ b/drivers/block/aoe/aoenet.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007 Coraid, Inc.  See COPYING for GPL terms. */
+/* Copyright (c) 2012 Coraid, Inc.  See COPYING for GPL terms. */
 /*
  * aoenet.c
  * Ethernet portion of AoE driver
@@ -33,6 +33,9 @@ static char aoe_iflist[IFLISTSZ];
 module_param_string(aoe_iflist, aoe_iflist, IFLISTSZ, 0600);
 MODULE_PARM_DESC(aoe_iflist, "aoe_iflist=\"dev1 [dev2 ...]\"");
 
+static wait_queue_head_t txwq;
+static struct ktstate kts;
+
 #ifndef MODULE
 static int __init aoe_iflist_setup(char *str)
 {
@@ -44,6 +47,23 @@ static int __init aoe_iflist_setup(char *str)
 __setup("aoe_iflist=", aoe_iflist_setup);
 #endif
 
+static spinlock_t txlock;
+static struct sk_buff_head skbtxq;
+
+/* enters with txlock held */
+static int
+tx(void)
+{
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&skbtxq))) {
+		spin_unlock_irq(&txlock);
+		dev_queue_xmit(skb);
+		spin_lock_irq(&txlock);
+	}
+	return 0;
+}
+
 int
 is_aoe_netif(struct net_device *ifp)
 {
@@ -88,10 +108,14 @@ void
 aoenet_xmit(struct sk_buff_head *queue)
 {
 	struct sk_buff *skb, *tmp;
+	ulong flags;
 
 	skb_queue_walk_safe(queue, skb, tmp) {
 		__skb_unlink(skb, queue);
-		dev_queue_xmit(skb);
+		spin_lock_irqsave(&txlock, flags);
+		skb_queue_tail(&skbtxq, skb);
+		spin_unlock_irqrestore(&txlock, flags);
+		wake_up(&txwq);
 	}
 }
 
@@ -102,7 +126,9 @@ static int
 aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt, struct net_device *orig_dev)
 {
 	struct aoe_hdr *h;
+	struct aoe_atahdr *ah;
 	u32 n;
+	int sn;
 
 	if (dev_net(ifp) != &init_net)
 		goto exit;
@@ -110,13 +136,16 @@ aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt,
 	skb = skb_share_check(skb, GFP_ATOMIC);
 	if (skb == NULL)
 		return 0;
-	if (skb_linearize(skb))
-		goto exit;
 	if (!is_aoe_netif(ifp))
 		goto exit;
 	skb_push(skb, ETH_HLEN);	/* (1) */
-
-	h = (struct aoe_hdr *) skb_mac_header(skb);
+	sn = sizeof(*h) + sizeof(*ah);
+	if (skb->len >= sn) {
+		sn -= skb_headlen(skb);
+		if (sn > 0 && !__pskb_pull_tail(skb, sn))
+			goto exit;
+	}
+	h = (struct aoe_hdr *) skb->data;
 	n = get_unaligned_be32(&h->tag);
 	if ((h->verfl & AOEFL_RSP) == 0 || (n & 1<<31))
 		goto exit;
@@ -137,7 +166,8 @@ aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt,
 
 	switch (h->cmd) {
 	case AOECMD_ATA:
-		aoecmd_ata_rsp(skb);
+		/* ata_rsp may keep skb for later processing or give it back */
+		skb = aoecmd_ata_rsp(skb);
 		break;
 	case AOECMD_CFG:
 		aoecmd_cfg_rsp(skb);
@@ -145,8 +175,12 @@ aoenet_rcv(struct sk_buff *skb, struct net_device *ifp, struct packet_type *pt,
 	default:
 		if (h->cmd >= AOECMD_VEND_MIN)
 			break;	/* don't complain about vendor commands */
-		printk(KERN_INFO "aoe: unknown cmd %d\n", h->cmd);
+		pr_info("aoe: unknown AoE command type 0x%02x\n", h->cmd);
+		break;
 	}
+
+	if (!skb)
+		return 0;
 exit:
 	dev_kfree_skb(skb);
 	return 0;
@@ -160,6 +194,15 @@ static struct packet_type aoe_pt __read_mostly = {
 int __init
 aoenet_init(void)
 {
+	skb_queue_head_init(&skbtxq);
+	init_waitqueue_head(&txwq);
+	spin_lock_init(&txlock);
+	kts.lock = &txlock;
+	kts.fn = tx;
+	kts.waitq = &txwq;
+	kts.name = "aoe_tx";
+	if (aoe_ktstart(&kts))
+		return -EAGAIN;
 	dev_add_pack(&aoe_pt);
 	return 0;
 }
@@ -167,6 +210,8 @@ aoenet_init(void)
 void
 aoenet_exit(void)
 {
+	aoe_ktstop(&kts);
+	skb_queue_purge(&skbtxq);
 	dev_remove_pack(&aoe_pt);
 }
 
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 0c03411c59e..043ddcca4ab 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -78,6 +78,8 @@ static const char *ioctl_cmd_to_ascii(int cmd)
 	case NBD_SET_SOCK: return "set-sock";
 	case NBD_SET_BLKSIZE: return "set-blksize";
 	case NBD_SET_SIZE: return "set-size";
+	case NBD_SET_TIMEOUT: return "set-timeout";
+	case NBD_SET_FLAGS: return "set-flags";
 	case NBD_DO_IT: return "do-it";
 	case NBD_CLEAR_SOCK: return "clear-sock";
 	case NBD_CLEAR_QUE: return "clear-que";
@@ -96,6 +98,7 @@ static const char *nbdcmd_to_ascii(int cmd)
 	case  NBD_CMD_READ: return "read";
 	case NBD_CMD_WRITE: return "write";
 	case  NBD_CMD_DISC: return "disconnect";
+	case  NBD_CMD_TRIM: return "trim/discard";
 	}
 	return "invalid";
 }
@@ -467,8 +470,12 @@ static void nbd_handle_req(struct nbd_device *nbd, struct request *req)
 
 	nbd_cmd(req) = NBD_CMD_READ;
 	if (rq_data_dir(req) == WRITE) {
-		nbd_cmd(req) = NBD_CMD_WRITE;
-		if (nbd->flags & NBD_READ_ONLY) {
+		if ((req->cmd_flags & REQ_DISCARD)) {
+			WARN_ON(!(nbd->flags & NBD_FLAG_SEND_TRIM));
+			nbd_cmd(req) = NBD_CMD_TRIM;
+		} else
+			nbd_cmd(req) = NBD_CMD_WRITE;
+		if (nbd->flags & NBD_FLAG_READ_ONLY) {
 			dev_err(disk_to_dev(nbd->disk),
 				"Write on read-only\n");
 			goto error_out;
@@ -651,6 +658,10 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		nbd->xmit_timeout = arg * HZ;
 		return 0;
 
+	case NBD_SET_FLAGS:
+		nbd->flags = arg;
+		return 0;
+
 	case NBD_SET_SIZE_BLOCKS:
 		nbd->bytesize = ((u64) arg) * nbd->blksize;
 		bdev->bd_inode->i_size = nbd->bytesize;
@@ -670,6 +681,10 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 
 		mutex_unlock(&nbd->tx_lock);
 
+		if (nbd->flags & NBD_FLAG_SEND_TRIM)
+			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
+				nbd->disk->queue);
+
 		thread = kthread_create(nbd_thread, nbd, nbd->disk->disk_name);
 		if (IS_ERR(thread)) {
 			mutex_lock(&nbd->tx_lock);
@@ -687,6 +702,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd,
 		nbd->file = NULL;
 		nbd_clear_que(nbd);
 		dev_warn(disk_to_dev(nbd->disk), "queue cleared\n");
+		queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue);
 		if (file)
 			fput(file);
 		nbd->bytesize = 0;
@@ -805,6 +821,9 @@ static int __init nbd_init(void)
 		 * Tell the block layer that we are not a rotational device
 		 */
 		queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
+		disk->queue->limits.discard_granularity = 512;
+		disk->queue->limits.max_discard_sectors = UINT_MAX;
+		disk->queue->limits.discard_zeroes_data = 0;
 	}
 
 	if (register_blkdev(NBD_MAJOR, "nbd")) {
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 58e32f7c322..e01f5eaaec8 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -84,40 +84,33 @@ static struct _intel_private {
 #define IS_IRONLAKE	intel_private.driver->is_ironlake
 #define HAS_PGTBL_EN	intel_private.driver->has_pgtbl_enable
 
-int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
-			 struct scatterlist **sg_list, int *num_sg)
+static int intel_gtt_map_memory(struct page **pages,
+				unsigned int num_entries,
+				struct sg_table *st)
 {
-	struct sg_table st;
 	struct scatterlist *sg;
 	int i;
 
-	if (*sg_list)
-		return 0; /* already mapped (for e.g. resume */
-
 	DBG("try mapping %lu pages\n", (unsigned long)num_entries);
 
-	if (sg_alloc_table(&st, num_entries, GFP_KERNEL))
+	if (sg_alloc_table(st, num_entries, GFP_KERNEL))
 		goto err;
 
-	*sg_list = sg = st.sgl;
-
-	for (i = 0 ; i < num_entries; i++, sg = sg_next(sg))
+	for_each_sg(st->sgl, sg, num_entries, i)
 		sg_set_page(sg, pages[i], PAGE_SIZE, 0);
 
-	*num_sg = pci_map_sg(intel_private.pcidev, *sg_list,
-				 num_entries, PCI_DMA_BIDIRECTIONAL);
-	if (unlikely(!*num_sg))
+	if (!pci_map_sg(intel_private.pcidev,
+			st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL))
 		goto err;
 
 	return 0;
 
 err:
-	sg_free_table(&st);
+	sg_free_table(st);
 	return -ENOMEM;
 }
-EXPORT_SYMBOL(intel_gtt_map_memory);
 
-void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
+static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
 {
 	struct sg_table st;
 	DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
@@ -130,7 +123,6 @@ void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
 
 	sg_free_table(&st);
 }
-EXPORT_SYMBOL(intel_gtt_unmap_memory);
 
 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 {
@@ -674,9 +666,14 @@ static int intel_gtt_init(void)
 
 	gtt_map_size = intel_private.base.gtt_total_entries * 4;
 
-	intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
-				    gtt_map_size);
-	if (!intel_private.gtt) {
+	intel_private.gtt = NULL;
+	if (INTEL_GTT_GEN < 6)
+		intel_private.gtt = ioremap_wc(intel_private.gtt_bus_addr,
+					       gtt_map_size);
+	if (intel_private.gtt == NULL)
+		intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
+					    gtt_map_size);
+	if (intel_private.gtt == NULL) {
 		intel_private.driver->cleanup();
 		iounmap(intel_private.registers);
 		return -ENOMEM;
@@ -879,8 +876,7 @@ static bool i830_check_flags(unsigned int flags)
 	return false;
 }
 
-void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
-				 unsigned int sg_len,
+void intel_gtt_insert_sg_entries(struct sg_table *st,
 				 unsigned int pg_start,
 				 unsigned int flags)
 {
@@ -892,12 +888,11 @@ void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 
 	/* sg may merge pages, but we have to separate
 	 * per-page addr for GTT */
-	for_each_sg(sg_list, sg, sg_len, i) {
+	for_each_sg(st->sgl, sg, st->nents, i) {
 		len = sg_dma_len(sg) >> PAGE_SHIFT;
 		for (m = 0; m < len; m++) {
 			dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
-			intel_private.driver->write_entry(addr,
-							  j, flags);
+			intel_private.driver->write_entry(addr, j, flags);
 			j++;
 		}
 	}
@@ -905,8 +900,10 @@ void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 }
 EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
 
-void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
-			    struct page **pages, unsigned int flags)
+static void intel_gtt_insert_pages(unsigned int first_entry,
+				   unsigned int num_entries,
+				   struct page **pages,
+				   unsigned int flags)
 {
 	int i, j;
 
@@ -917,7 +914,6 @@ void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
 	}
 	readl(intel_private.gtt+j-1);
 }
-EXPORT_SYMBOL(intel_gtt_insert_pages);
 
 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 					 off_t pg_start, int type)
@@ -953,13 +949,15 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 		global_cache_flush();
 
 	if (intel_private.base.needs_dmar) {
-		ret = intel_gtt_map_memory(mem->pages, mem->page_count,
-					   &mem->sg_list, &mem->num_sg);
+		struct sg_table st;
+
+		ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st);
 		if (ret != 0)
 			return ret;
 
-		intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
-					    pg_start, type);
+		intel_gtt_insert_sg_entries(&st, pg_start, type);
+		mem->sg_list = st.sgl;
+		mem->num_sg = st.nents;
 	} else
 		intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
 				       type);
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 7c0d391996b..fbd9b2b850e 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -289,3 +289,16 @@ config HW_RANDOM_EXYNOS
 	  module will be called exynos-rng.
 
 	  If unsure, say Y.
+
+config HW_RANDOM_TPM
+	tristate "TPM HW Random Number Generator support"
+	depends on HW_RANDOM && TCG_TPM
+	default HW_RANDOM
+	---help---
+	  This driver provides kernel-side support for the Random Number
+	  Generator in the Trusted Platform Module
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called tpm-rng.
+
+	  If unsure, say Y.
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 39a757ca15b..1fd7eec9fbf 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -25,3 +25,4 @@ obj-$(CONFIG_HW_RANDOM_PICOXCELL) += picoxcell-rng.o
 obj-$(CONFIG_HW_RANDOM_PPC4XX) += ppc4xx-rng.o
 obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
 obj-$(CONFIG_HW_RANDOM_EXYNOS)	+= exynos-rng.o
+obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o
diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c
index 85074de5042..f05d85713fd 100644
--- a/drivers/char/hw_random/mxc-rnga.c
+++ b/drivers/char/hw_random/mxc-rnga.c
@@ -59,16 +59,21 @@
 #define RNGA_STATUS_LAST_READ_STATUS	0x00000002
 #define RNGA_STATUS_SECURITY_VIOLATION	0x00000001
 
-static struct platform_device *rng_dev;
+struct mxc_rng {
+	struct device *dev;
+	struct hwrng rng;
+	void __iomem *mem;
+	struct clk *clk;
+};
 
 static int mxc_rnga_data_present(struct hwrng *rng, int wait)
 {
-	void __iomem *rng_base = (void __iomem *)rng->priv;
 	int i;
+	struct mxc_rng *mxc_rng = container_of(rng, struct mxc_rng, rng);
 
 	for (i = 0; i < 20; i++) {
 		/* how many random numbers are in FIFO? [0-16] */
-		int level = (__raw_readl(rng_base + RNGA_STATUS) &
+		int level = (__raw_readl(mxc_rng->mem + RNGA_STATUS) &
 				RNGA_STATUS_LEVEL_MASK) >> 8;
 		if (level || !wait)
 			return !!level;
@@ -81,20 +86,20 @@ static int mxc_rnga_data_read(struct hwrng *rng, u32 * data)
 {
 	int err;
 	u32 ctrl;
-	void __iomem *rng_base = (void __iomem *)rng->priv;
+	struct mxc_rng *mxc_rng = container_of(rng, struct mxc_rng, rng);
 
 	/* retrieve a random number from FIFO */
-	*data = __raw_readl(rng_base + RNGA_OUTPUT_FIFO);
+	*data = __raw_readl(mxc_rng->mem + RNGA_OUTPUT_FIFO);
 
 	/* some error while reading this random number? */
-	err = __raw_readl(rng_base + RNGA_STATUS) & RNGA_STATUS_ERROR_INT;
+	err = __raw_readl(mxc_rng->mem + RNGA_STATUS) & RNGA_STATUS_ERROR_INT;
 
 	/* if error: clear error interrupt, but doesn't return random number */
 	if (err) {
-		dev_dbg(&rng_dev->dev, "Error while reading random number!\n");
-		ctrl = __raw_readl(rng_base + RNGA_CONTROL);
+		dev_dbg(mxc_rng->dev, "Error while reading random number!\n");
+		ctrl = __raw_readl(mxc_rng->mem + RNGA_CONTROL);
 		__raw_writel(ctrl | RNGA_CONTROL_CLEAR_INT,
-					rng_base + RNGA_CONTROL);
+					mxc_rng->mem + RNGA_CONTROL);
 		return 0;
 	} else
 		return 4;
@@ -103,22 +108,22 @@ static int mxc_rnga_data_read(struct hwrng *rng, u32 * data)
 static int mxc_rnga_init(struct hwrng *rng)
 {
 	u32 ctrl, osc;
-	void __iomem *rng_base = (void __iomem *)rng->priv;
+	struct mxc_rng *mxc_rng = container_of(rng, struct mxc_rng, rng);
 
 	/* wake up */
-	ctrl = __raw_readl(rng_base + RNGA_CONTROL);
-	__raw_writel(ctrl & ~RNGA_CONTROL_SLEEP, rng_base + RNGA_CONTROL);
+	ctrl = __raw_readl(mxc_rng->mem + RNGA_CONTROL);
+	__raw_writel(ctrl & ~RNGA_CONTROL_SLEEP, mxc_rng->mem + RNGA_CONTROL);
 
 	/* verify if oscillator is working */
-	osc = __raw_readl(rng_base + RNGA_STATUS);
+	osc = __raw_readl(mxc_rng->mem + RNGA_STATUS);
 	if (osc & RNGA_STATUS_OSC_DEAD) {
-		dev_err(&rng_dev->dev, "RNGA Oscillator is dead!\n");
+		dev_err(mxc_rng->dev, "RNGA Oscillator is dead!\n");
 		return -ENODEV;
 	}
 
 	/* go running */
-	ctrl = __raw_readl(rng_base + RNGA_CONTROL);
-	__raw_writel(ctrl | RNGA_CONTROL_GO, rng_base + RNGA_CONTROL);
+	ctrl = __raw_readl(mxc_rng->mem + RNGA_CONTROL);
+	__raw_writel(ctrl | RNGA_CONTROL_GO, mxc_rng->mem + RNGA_CONTROL);
 
 	return 0;
 }
@@ -126,40 +131,40 @@ static int mxc_rnga_init(struct hwrng *rng)
 static void mxc_rnga_cleanup(struct hwrng *rng)
 {
 	u32 ctrl;
-	void __iomem *rng_base = (void __iomem *)rng->priv;
+	struct mxc_rng *mxc_rng = container_of(rng, struct mxc_rng, rng);
 
-	ctrl = __raw_readl(rng_base + RNGA_CONTROL);
+	ctrl = __raw_readl(mxc_rng->mem + RNGA_CONTROL);
 
 	/* stop rnga */
-	__raw_writel(ctrl & ~RNGA_CONTROL_GO, rng_base + RNGA_CONTROL);
+	__raw_writel(ctrl & ~RNGA_CONTROL_GO, mxc_rng->mem + RNGA_CONTROL);
 }
 
-static struct hwrng mxc_rnga = {
-	.name = "mxc-rnga",
-	.init = mxc_rnga_init,
-	.cleanup = mxc_rnga_cleanup,
-	.data_present = mxc_rnga_data_present,
-	.data_read = mxc_rnga_data_read
-};
-
 static int __init mxc_rnga_probe(struct platform_device *pdev)
 {
 	int err = -ENODEV;
-	struct clk *clk;
 	struct resource *res, *mem;
-	void __iomem *rng_base = NULL;
-
-	if (rng_dev)
-		return -EBUSY;
-
-	clk = clk_get(&pdev->dev, "rng");
-	if (IS_ERR(clk)) {
+	struct mxc_rng *mxc_rng;
+
+	mxc_rng = devm_kzalloc(&pdev->dev, sizeof(struct mxc_rng),
+					GFP_KERNEL);
+	if (!mxc_rng)
+		return -ENOMEM;
+
+	mxc_rng->dev = &pdev->dev;
+	mxc_rng->rng.name = "mxc-rnga";
+	mxc_rng->rng.init = mxc_rnga_init;
+	mxc_rng->rng.cleanup = mxc_rnga_cleanup,
+	mxc_rng->rng.data_present = mxc_rnga_data_present,
+	mxc_rng->rng.data_read = mxc_rnga_data_read,
+
+	mxc_rng->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(mxc_rng->clk)) {
 		dev_err(&pdev->dev, "Could not get rng_clk!\n");
-		err = PTR_ERR(clk);
+		err = PTR_ERR(mxc_rng->clk);
 		goto out;
 	}
 
-	clk_enable(clk);
+	clk_prepare_enable(mxc_rng->clk);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
@@ -173,36 +178,27 @@ static int __init mxc_rnga_probe(struct platform_device *pdev)
 		goto err_region;
 	}
 
-	rng_base = ioremap(res->start, resource_size(res));
-	if (!rng_base) {
+	mxc_rng->mem = ioremap(res->start, resource_size(res));
+	if (!mxc_rng->mem) {
 		err = -ENOMEM;
 		goto err_ioremap;
 	}
 
-	mxc_rnga.priv = (unsigned long)rng_base;
-
-	err = hwrng_register(&mxc_rnga);
+	err = hwrng_register(&mxc_rng->rng);
 	if (err) {
 		dev_err(&pdev->dev, "MXC RNGA registering failed (%d)\n", err);
-		goto err_register;
+		goto err_ioremap;
 	}
 
-	rng_dev = pdev;
-
 	dev_info(&pdev->dev, "MXC RNGA Registered.\n");
 
 	return 0;
 
-err_register:
-	iounmap(rng_base);
-	rng_base = NULL;
-
 err_ioremap:
 	release_mem_region(res->start, resource_size(res));
 
 err_region:
-	clk_disable(clk);
-	clk_put(clk);
+	clk_disable_unprepare(mxc_rng->clk);
 
 out:
 	return err;
@@ -211,17 +207,15 @@ out:
 static int __exit mxc_rnga_remove(struct platform_device *pdev)
 {
 	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	void __iomem *rng_base = (void __iomem *)mxc_rnga.priv;
-	struct clk *clk = clk_get(&pdev->dev, "rng");
+	struct mxc_rng *mxc_rng = platform_get_drvdata(pdev);
 
-	hwrng_unregister(&mxc_rnga);
+	hwrng_unregister(&mxc_rng->rng);
 
-	iounmap(rng_base);
+	iounmap(mxc_rng->mem);
 
 	release_mem_region(res->start, resource_size(res));
 
-	clk_disable(clk);
-	clk_put(clk);
+	clk_disable_unprepare(mxc_rng->clk);
 
 	return 0;
 }
diff --git a/drivers/char/hw_random/octeon-rng.c b/drivers/char/hw_random/octeon-rng.c
index 0943edc782a..5c34c092af7 100644
--- a/drivers/char/hw_random/octeon-rng.c
+++ b/drivers/char/hw_random/octeon-rng.c
@@ -75,42 +75,35 @@ static int __devinit octeon_rng_probe(struct platform_device *pdev)
 
 	res_ports = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res_ports)
-		goto err_ports;
+		return -ENOENT;
 
 	res_result = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	if (!res_result)
-		goto err_ports;
+		return -ENOENT;
 
 
 	rng->control_status = devm_ioremap_nocache(&pdev->dev,
 						   res_ports->start,
 						   sizeof(u64));
 	if (!rng->control_status)
-		goto err_ports;
+		return -ENOENT;
 
 	rng->result = devm_ioremap_nocache(&pdev->dev,
 					   res_result->start,
 					   sizeof(u64));
 	if (!rng->result)
-		goto err_r;
+		return -ENOENT;
 
 	rng->ops = ops;
 
 	dev_set_drvdata(&pdev->dev, &rng->ops);
 	ret = hwrng_register(&rng->ops);
 	if (ret)
-		goto err;
+		return -ENOENT;
 
 	dev_info(&pdev->dev, "Octeon Random Number Generator\n");
 
 	return 0;
-err:
-	devm_iounmap(&pdev->dev, rng->control_status);
-err_r:
-	devm_iounmap(&pdev->dev, rng->result);
-err_ports:
-	devm_kfree(&pdev->dev, rng);
-	return -ENOENT;
 }
 
 static int __exit octeon_rng_remove(struct platform_device *pdev)
diff --git a/drivers/char/hw_random/tpm-rng.c b/drivers/char/hw_random/tpm-rng.c
new file mode 100644
index 00000000000..d6d448266f0
--- /dev/null
+++ b/drivers/char/hw_random/tpm-rng.c
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2012 Kent Yoder IBM Corporation
+ *
+ * HWRNG interfaces to pull RNG data from a TPM
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+#include <linux/module.h>
+#include <linux/hw_random.h>
+#include <linux/tpm.h>
+
+#define MODULE_NAME "tpm-rng"
+
+static int tpm_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	return tpm_get_random(TPM_ANY_NUM, data, max);
+}
+
+static struct hwrng tpm_rng = {
+	.name = MODULE_NAME,
+	.read = tpm_rng_read,
+};
+
+static int __init rng_init(void)
+{
+	return hwrng_register(&tpm_rng);
+}
+module_init(rng_init);
+
+static void __exit rng_exit(void)
+{
+	hwrng_unregister(&tpm_rng);
+}
+module_exit(rng_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Kent Yoder <key@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("RNG driver for TPM devices");
diff --git a/drivers/char/mbcs.c b/drivers/char/mbcs.c
index 47ff7e470d8..0c7d340b9ab 100644
--- a/drivers/char/mbcs.c
+++ b/drivers/char/mbcs.c
@@ -799,7 +799,7 @@ static int mbcs_remove(struct cx_dev *dev)
 	return 0;
 }
 
-static const struct cx_device_id __devinitdata mbcs_id_table[] = {
+static const struct cx_device_id __devinitconst mbcs_id_table[] = {
 	{
 	 .part_num = MBCS_PART_NUM,
 	 .mfg_num = MBCS_MFG_NUM,
diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig
index a048199ce86..915875e431d 100644
--- a/drivers/char/tpm/Kconfig
+++ b/drivers/char/tpm/Kconfig
@@ -33,6 +33,17 @@ config TCG_TIS
 	  from within Linux.  To compile this driver as a module, choose
 	  M here; the module will be called tpm_tis.
 
+config TCG_TIS_I2C_INFINEON
+	tristate "TPM Interface Specification 1.2 Interface (I2C - Infineon)"
+	depends on I2C
+	---help---
+	  If you have a TPM security chip that is compliant with the
+	  TCG TIS 1.2 TPM specification and Infineon's I2C Protocol Stack
+	  Specification 0.20 say Yes and it will be accessible from within
+	  Linux.
+	  To compile this driver as a module, choose M here; the module
+	  will be called tpm_tis_i2c_infineon.
+
 config TCG_NSC
 	tristate "National Semiconductor TPM Interface"
 	depends on X86
@@ -62,4 +73,12 @@ config TCG_INFINEON
 	  Further information on this driver and the supported hardware
 	  can be found at http://www.trust.rub.de/projects/linux-device-driver-infineon-tpm/ 
 
+config TCG_IBMVTPM
+	tristate "IBM VTPM Interface"
+	depends on PPC64
+	---help---
+	  If you have IBM virtual TPM (VTPM) support say Yes and it
+	  will be accessible from within Linux.  To compile this driver
+	  as a module, choose M here; the module will be called tpm_ibmvtpm.
+
 endif # TCG_TPM
diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile
index ea3a1e02a82..5b3fc8bc6c1 100644
--- a/drivers/char/tpm/Makefile
+++ b/drivers/char/tpm/Makefile
@@ -4,8 +4,16 @@
 obj-$(CONFIG_TCG_TPM) += tpm.o
 ifdef CONFIG_ACPI
 	obj-$(CONFIG_TCG_TPM) += tpm_bios.o
+	tpm_bios-objs += tpm_eventlog.o tpm_acpi.o tpm_ppi.o
+else
+ifdef CONFIG_TCG_IBMVTPM
+	obj-$(CONFIG_TCG_TPM) += tpm_bios.o
+	tpm_bios-objs += tpm_eventlog.o tpm_of.o
+endif
 endif
 obj-$(CONFIG_TCG_TIS) += tpm_tis.o
+obj-$(CONFIG_TCG_TIS_I2C_INFINEON) += tpm_i2c_infineon.o
 obj-$(CONFIG_TCG_NSC) += tpm_nsc.o
 obj-$(CONFIG_TCG_ATMEL) += tpm_atmel.o
 obj-$(CONFIG_TCG_INFINEON) += tpm_infineon.o
+obj-$(CONFIG_TCG_IBMVTPM) += tpm_ibmvtpm.o
diff --git a/drivers/char/tpm/tpm.c b/drivers/char/tpm/tpm.c
index 3af9f4d1a23..f26afdb1a70 100644
--- a/drivers/char/tpm/tpm.c
+++ b/drivers/char/tpm/tpm.c
@@ -30,12 +30,7 @@
 #include <linux/freezer.h>
 
 #include "tpm.h"
-
-enum tpm_const {
-	TPM_MINOR = 224,	/* officially assigned */
-	TPM_BUFSIZE = 4096,
-	TPM_NUM_DEVICES = 256,
-};
+#include "tpm_eventlog.h"
 
 enum tpm_duration {
 	TPM_SHORT = 0,
@@ -482,6 +477,7 @@ static ssize_t transmit_cmd(struct tpm_chip *chip, struct tpm_cmd_t *cmd,
 #define TPM_INTERNAL_RESULT_SIZE 200
 #define TPM_TAG_RQU_COMMAND cpu_to_be16(193)
 #define TPM_ORD_GET_CAP cpu_to_be32(101)
+#define TPM_ORD_GET_RANDOM cpu_to_be32(70)
 
 static const struct tpm_input_header tpm_getcap_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
@@ -919,7 +915,7 @@ EXPORT_SYMBOL_GPL(tpm_show_pcrs);
 
 #define  READ_PUBEK_RESULT_SIZE 314
 #define TPM_ORD_READPUBEK cpu_to_be32(124)
-struct tpm_input_header tpm_readpubek_header = {
+static struct tpm_input_header tpm_readpubek_header = {
 	.tag = TPM_TAG_RQU_COMMAND,
 	.length = cpu_to_be32(30),
 	.ordinal = TPM_ORD_READPUBEK
@@ -1175,7 +1171,7 @@ int tpm_release(struct inode *inode, struct file *file)
 	flush_work(&chip->work);
 	file->private_data = NULL;
 	atomic_set(&chip->data_pending, 0);
-	kfree(chip->data_buffer);
+	kzfree(chip->data_buffer);
 	clear_bit(0, &chip->is_open);
 	put_device(chip->dev);
 	return 0;
@@ -1227,7 +1223,6 @@ ssize_t tpm_read(struct file *file, char __user *buf,
 	del_singleshot_timer_sync(&chip->user_read_timer);
 	flush_work(&chip->work);
 	ret_size = atomic_read(&chip->data_pending);
-	atomic_set(&chip->data_pending, 0);
 	if (ret_size > 0) {	/* relay data */
 		ssize_t orig_ret_size = ret_size;
 		if (size < ret_size)
@@ -1242,6 +1237,8 @@ ssize_t tpm_read(struct file *file, char __user *buf,
 		mutex_unlock(&chip->buffer_mutex);
 	}
 
+	atomic_set(&chip->data_pending, 0);
+
 	return ret_size;
 }
 EXPORT_SYMBOL_GPL(tpm_read);
@@ -1326,6 +1323,58 @@ int tpm_pm_resume(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(tpm_pm_resume);
 
+#define TPM_GETRANDOM_RESULT_SIZE	18
+static struct tpm_input_header tpm_getrandom_header = {
+	.tag = TPM_TAG_RQU_COMMAND,
+	.length = cpu_to_be32(14),
+	.ordinal = TPM_ORD_GET_RANDOM
+};
+
+/**
+ * tpm_get_random() - Get random bytes from the tpm's RNG
+ * @chip_num: A specific chip number for the request or TPM_ANY_NUM
+ * @out: destination buffer for the random bytes
+ * @max: the max number of bytes to write to @out
+ *
+ * Returns < 0 on error and the number of bytes read on success
+ */
+int tpm_get_random(u32 chip_num, u8 *out, size_t max)
+{
+	struct tpm_chip *chip;
+	struct tpm_cmd_t tpm_cmd;
+	u32 recd, num_bytes = min_t(u32, max, TPM_MAX_RNG_DATA);
+	int err, total = 0, retries = 5;
+	u8 *dest = out;
+
+	chip = tpm_chip_find_get(chip_num);
+	if (chip == NULL)
+		return -ENODEV;
+
+	if (!out || !num_bytes || max > TPM_MAX_RNG_DATA)
+		return -EINVAL;
+
+	do {
+		tpm_cmd.header.in = tpm_getrandom_header;
+		tpm_cmd.params.getrandom_in.num_bytes = cpu_to_be32(num_bytes);
+
+		err = transmit_cmd(chip, &tpm_cmd,
+				   TPM_GETRANDOM_RESULT_SIZE + num_bytes,
+				   "attempting get random");
+		if (err)
+			break;
+
+		recd = be32_to_cpu(tpm_cmd.params.getrandom_out.rng_data_len);
+		memcpy(dest, tpm_cmd.params.getrandom_out.rng_data, recd);
+
+		dest += recd;
+		total += recd;
+		num_bytes -= recd;
+	} while (retries-- && total < max);
+
+	return total ? total : -EIO;
+}
+EXPORT_SYMBOL_GPL(tpm_get_random);
+
 /* In case vendor provided release function, call it too.*/
 
 void tpm_dev_vendor_release(struct tpm_chip *chip)
@@ -1346,7 +1395,7 @@ EXPORT_SYMBOL_GPL(tpm_dev_vendor_release);
  * Once all references to platform device are down to 0,
  * release all allocated structures.
  */
-void tpm_dev_release(struct device *dev)
+static void tpm_dev_release(struct device *dev)
 {
 	struct tpm_chip *chip = dev_get_drvdata(dev);
 
@@ -1427,6 +1476,11 @@ struct tpm_chip *tpm_register_hardware(struct device *dev,
 		goto put_device;
 	}
 
+	if (sys_add_ppi(&dev->kobj)) {
+		misc_deregister(&chip->vendor.miscdev);
+		goto put_device;
+	}
+
 	chip->bios_dir = tpm_bios_log_setup(devname);
 
 	/* Make chip available */
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 917f727e674..02c266aa2bf 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -28,6 +28,12 @@
 #include <linux/io.h>
 #include <linux/tpm.h>
 
+enum tpm_const {
+	TPM_MINOR = 224,	/* officially assigned */
+	TPM_BUFSIZE = 4096,
+	TPM_NUM_DEVICES = 256,
+};
+
 enum tpm_timeout {
 	TPM_TIMEOUT = 5,	/* msecs */
 };
@@ -94,6 +100,7 @@ struct tpm_vendor_specific {
 	bool timeout_adjusted;
 	unsigned long duration[3]; /* jiffies */
 	bool duration_adjusted;
+	void *data;
 
 	wait_queue_head_t read_queue;
 	wait_queue_head_t int_queue;
@@ -269,6 +276,21 @@ struct tpm_pcrextend_in {
 	u8	hash[TPM_DIGEST_SIZE];
 }__attribute__((packed));
 
+/* 128 bytes is an arbitrary cap. This could be as large as TPM_BUFSIZE - 18
+ * bytes, but 128 is still a relatively large number of random bytes and
+ * anything much bigger causes users of struct tpm_cmd_t to start getting
+ * compiler warnings about stack frame size. */
+#define TPM_MAX_RNG_DATA	128
+
+struct tpm_getrandom_out {
+	__be32 rng_data_len;
+	u8     rng_data[TPM_MAX_RNG_DATA];
+}__attribute__((packed));
+
+struct tpm_getrandom_in {
+	__be32 num_bytes;
+}__attribute__((packed));
+
 typedef union {
 	struct	tpm_getcap_params_out getcap_out;
 	struct	tpm_readpubek_params_out readpubek_out;
@@ -277,6 +299,8 @@ typedef union {
 	struct	tpm_pcrread_in	pcrread_in;
 	struct	tpm_pcrread_out	pcrread_out;
 	struct	tpm_pcrextend_in pcrextend_in;
+	struct	tpm_getrandom_in getrandom_in;
+	struct	tpm_getrandom_out getrandom_out;
 } tpm_cmd_params;
 
 struct tpm_cmd_t {
@@ -303,15 +327,12 @@ extern int tpm_pm_suspend(struct device *);
 extern int tpm_pm_resume(struct device *);
 extern int wait_for_tpm_stat(struct tpm_chip *, u8, unsigned long,
 			     wait_queue_head_t *);
+
 #ifdef CONFIG_ACPI
-extern struct dentry ** tpm_bios_log_setup(char *);
-extern void tpm_bios_log_teardown(struct dentry **);
+extern ssize_t sys_add_ppi(struct kobject *parent);
 #else
-static inline struct dentry ** tpm_bios_log_setup(char *name)
-{
-	return NULL;
-}
-static inline void tpm_bios_log_teardown(struct dentry **dir)
+static inline ssize_t sys_add_ppi(struct kobject *parent)
 {
+	return 0;
 }
 #endif
diff --git a/drivers/char/tpm/tpm_acpi.c b/drivers/char/tpm/tpm_acpi.c
new file mode 100644
index 00000000000..56051d0c97a
--- /dev/null
+++ b/drivers/char/tpm/tpm_acpi.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2005 IBM Corporation
+ *
+ * Authors:
+ *	Seiji Munetoh <munetoh@jp.ibm.com>
+ *	Stefan Berger <stefanb@us.ibm.com>
+ *	Reiner Sailer <sailer@watson.ibm.com>
+ *	Kylene Hall <kjhall@us.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Access to the eventlog extended by the TCG BIOS of PC platform
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/security.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <acpi/acpi.h>
+
+#include "tpm.h"
+#include "tpm_eventlog.h"
+
+struct acpi_tcpa {
+	struct acpi_table_header hdr;
+	u16 platform_class;
+	union {
+		struct client_hdr {
+			u32 log_max_len __attribute__ ((packed));
+			u64 log_start_addr __attribute__ ((packed));
+		} client;
+		struct server_hdr {
+			u16 reserved;
+			u64 log_max_len __attribute__ ((packed));
+			u64 log_start_addr __attribute__ ((packed));
+		} server;
+	};
+};
+
+/* read binary bios log */
+int read_log(struct tpm_bios_log *log)
+{
+	struct acpi_tcpa *buff;
+	acpi_status status;
+	void __iomem *virt;
+	u64 len, start;
+
+	if (log->bios_event_log != NULL) {
+		printk(KERN_ERR
+		       "%s: ERROR - Eventlog already initialized\n",
+		       __func__);
+		return -EFAULT;
+	}
+
+	/* Find TCPA entry in RSDT (ACPI_LOGICAL_ADDRESSING) */
+	status = acpi_get_table(ACPI_SIG_TCPA, 1,
+				(struct acpi_table_header **)&buff);
+
+	if (ACPI_FAILURE(status)) {
+		printk(KERN_ERR "%s: ERROR - Could not get TCPA table\n",
+		       __func__);
+		return -EIO;
+	}
+
+	switch(buff->platform_class) {
+	case BIOS_SERVER:
+		len = buff->server.log_max_len;
+		start = buff->server.log_start_addr;
+		break;
+	case BIOS_CLIENT:
+	default:
+		len = buff->client.log_max_len;
+		start = buff->client.log_start_addr;
+		break;
+	}
+	if (!len) {
+		printk(KERN_ERR "%s: ERROR - TCPA log area empty\n", __func__);
+		return -EIO;
+	}
+
+	/* malloc EventLog space */
+	log->bios_event_log = kmalloc(len, GFP_KERNEL);
+	if (!log->bios_event_log) {
+		printk("%s: ERROR - Not enough  Memory for BIOS measurements\n",
+			__func__);
+		return -ENOMEM;
+	}
+
+	log->bios_event_log_end = log->bios_event_log + len;
+
+	virt = acpi_os_map_memory(start, len);
+	if (!virt) {
+		kfree(log->bios_event_log);
+		printk("%s: ERROR - Unable to map memory\n", __func__);
+		return -EIO;
+	}
+
+	memcpy_fromio(log->bios_event_log, virt, len);
+
+	acpi_os_unmap_memory(virt, len);
+	return 0;
+}
diff --git a/drivers/char/tpm/tpm_bios.c b/drivers/char/tpm/tpm_eventlog.c
index 0636520fa9b..84ddc557b8f 100644
--- a/drivers/char/tpm/tpm_bios.c
+++ b/drivers/char/tpm/tpm_eventlog.c
@@ -1,7 +1,8 @@
 /*
- * Copyright (C) 2005 IBM Corporation
+ * Copyright (C) 2005, 2012 IBM Corporation
  *
  * Authors:
+ *	Kent Yoder <key@linux.vnet.ibm.com>
  *	Seiji Munetoh <munetoh@jp.ibm.com>
  *	Stefan Berger <stefanb@us.ibm.com>
  *	Reiner Sailer <sailer@watson.ibm.com>
@@ -9,7 +10,7 @@
  *
  * Maintained by: <tpmdd-devel@lists.sourceforge.net>
  *
- * Access to the eventlog extended by the TCG BIOS of PC platform
+ * Access to the eventlog created by a system's firmware / BIOS
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -23,67 +24,10 @@
 #include <linux/security.h>
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <acpi/acpi.h>
-#include "tpm.h"
-
-#define TCG_EVENT_NAME_LEN_MAX	255
-#define MAX_TEXT_EVENT		1000	/* Max event string length */
-#define ACPI_TCPA_SIG		"TCPA"	/* 0x41504354 /'TCPA' */
-
-enum bios_platform_class {
-	BIOS_CLIENT = 0x00,
-	BIOS_SERVER = 0x01,
-};
-
-struct tpm_bios_log {
-	void *bios_event_log;
-	void *bios_event_log_end;
-};
-
-struct acpi_tcpa {
-	struct acpi_table_header hdr;
-	u16 platform_class;
-	union {
-		struct client_hdr {
-			u32 log_max_len __attribute__ ((packed));
-			u64 log_start_addr __attribute__ ((packed));
-		} client;
-		struct server_hdr {
-			u16 reserved;
-			u64 log_max_len __attribute__ ((packed));
-			u64 log_start_addr __attribute__ ((packed));
-		} server;
-	};
-};
 
-struct tcpa_event {
-	u32 pcr_index;
-	u32 event_type;
-	u8 pcr_value[20];	/* SHA1 */
-	u32 event_size;
-	u8 event_data[0];
-};
+#include "tpm.h"
+#include "tpm_eventlog.h"
 
-enum tcpa_event_types {
-	PREBOOT = 0,
-	POST_CODE,
-	UNUSED,
-	NO_ACTION,
-	SEPARATOR,
-	ACTION,
-	EVENT_TAG,
-	SCRTM_CONTENTS,
-	SCRTM_VERSION,
-	CPU_MICROCODE,
-	PLATFORM_CONFIG_FLAGS,
-	TABLE_OF_DEVICES,
-	COMPACT_HASH,
-	IPL,
-	IPL_PARTITION_DATA,
-	NONHOST_CODE,
-	NONHOST_CONFIG,
-	NONHOST_INFO,
-};
 
 static const char* tcpa_event_type_strings[] = {
 	"PREBOOT",
@@ -106,28 +50,6 @@ static const char* tcpa_event_type_strings[] = {
 	"Non-Host Info"
 };
 
-struct tcpa_pc_event {
-	u32 event_id;
-	u32 event_size;
-	u8 event_data[0];
-};
-
-enum tcpa_pc_event_ids {
-	SMBIOS = 1,
-	BIS_CERT,
-	POST_BIOS_ROM,
-	ESCD,
-	CMOS,
-	NVRAM,
-	OPTION_ROM_EXEC,
-	OPTION_ROM_CONFIG,
-	OPTION_ROM_MICROCODE = 10,
-	S_CRTM_VERSION,
-	S_CRTM_CONTENTS,
-	POST_CONTENTS,
-	HOST_TABLE_OF_DEVICES,
-};
-
 static const char* tcpa_pc_event_id_strings[] = {
 	"",
 	"SMBIOS",
@@ -358,65 +280,6 @@ static const struct seq_operations tpm_binary_b_measurments_seqops = {
 	.show = tpm_binary_bios_measurements_show,
 };
 
-/* read binary bios log */
-static int read_log(struct tpm_bios_log *log)
-{
-	struct acpi_tcpa *buff;
-	acpi_status status;
-	struct acpi_table_header *virt;
-	u64 len, start;
-
-	if (log->bios_event_log != NULL) {
-		printk(KERN_ERR
-		       "%s: ERROR - Eventlog already initialized\n",
-		       __func__);
-		return -EFAULT;
-	}
-
-	/* Find TCPA entry in RSDT (ACPI_LOGICAL_ADDRESSING) */
-	status = acpi_get_table(ACPI_SIG_TCPA, 1,
-				(struct acpi_table_header **)&buff);
-
-	if (ACPI_FAILURE(status)) {
-		printk(KERN_ERR "%s: ERROR - Could not get TCPA table\n",
-		       __func__);
-		return -EIO;
-	}
-
-	switch(buff->platform_class) {
-	case BIOS_SERVER:
-		len = buff->server.log_max_len;
-		start = buff->server.log_start_addr;
-		break;
-	case BIOS_CLIENT:
-	default:
-		len = buff->client.log_max_len;
-		start = buff->client.log_start_addr;
-		break;
-	}
-	if (!len) {
-		printk(KERN_ERR "%s: ERROR - TCPA log area empty\n", __func__);
-		return -EIO;
-	}
-
-	/* malloc EventLog space */
-	log->bios_event_log = kmalloc(len, GFP_KERNEL);
-	if (!log->bios_event_log) {
-		printk("%s: ERROR - Not enough  Memory for BIOS measurements\n",
-			__func__);
-		return -ENOMEM;
-	}
-
-	log->bios_event_log_end = log->bios_event_log + len;
-
-	virt = acpi_os_map_memory(start, len);
-
-	memcpy(log->bios_event_log, virt, len);
-
-	acpi_os_unmap_memory(virt, len);
-	return 0;
-}
-
 static int tpm_ascii_bios_measurements_open(struct inode *inode,
 					    struct file *file)
 {
diff --git a/drivers/char/tpm/tpm_eventlog.h b/drivers/char/tpm/tpm_eventlog.h
new file mode 100644
index 00000000000..e7da086d692
--- /dev/null
+++ b/drivers/char/tpm/tpm_eventlog.h
@@ -0,0 +1,86 @@
+
+#ifndef __TPM_EVENTLOG_H__
+#define __TPM_EVENTLOG_H__
+
+#define TCG_EVENT_NAME_LEN_MAX	255
+#define MAX_TEXT_EVENT		1000	/* Max event string length */
+#define ACPI_TCPA_SIG		"TCPA"	/* 0x41504354 /'TCPA' */
+
+enum bios_platform_class {
+	BIOS_CLIENT = 0x00,
+	BIOS_SERVER = 0x01,
+};
+
+struct tpm_bios_log {
+	void *bios_event_log;
+	void *bios_event_log_end;
+};
+
+struct tcpa_event {
+	u32 pcr_index;
+	u32 event_type;
+	u8 pcr_value[20];	/* SHA1 */
+	u32 event_size;
+	u8 event_data[0];
+};
+
+enum tcpa_event_types {
+	PREBOOT = 0,
+	POST_CODE,
+	UNUSED,
+	NO_ACTION,
+	SEPARATOR,
+	ACTION,
+	EVENT_TAG,
+	SCRTM_CONTENTS,
+	SCRTM_VERSION,
+	CPU_MICROCODE,
+	PLATFORM_CONFIG_FLAGS,
+	TABLE_OF_DEVICES,
+	COMPACT_HASH,
+	IPL,
+	IPL_PARTITION_DATA,
+	NONHOST_CODE,
+	NONHOST_CONFIG,
+	NONHOST_INFO,
+};
+
+struct tcpa_pc_event {
+	u32 event_id;
+	u32 event_size;
+	u8 event_data[0];
+};
+
+enum tcpa_pc_event_ids {
+	SMBIOS = 1,
+	BIS_CERT,
+	POST_BIOS_ROM,
+	ESCD,
+	CMOS,
+	NVRAM,
+	OPTION_ROM_EXEC,
+	OPTION_ROM_CONFIG,
+	OPTION_ROM_MICROCODE = 10,
+	S_CRTM_VERSION,
+	S_CRTM_CONTENTS,
+	POST_CONTENTS,
+	HOST_TABLE_OF_DEVICES,
+};
+
+int read_log(struct tpm_bios_log *log);
+
+#if defined(CONFIG_TCG_IBMVTPM) || defined(CONFIG_TCG_IBMVTPM_MODULE) || \
+	defined(CONFIG_ACPI)
+extern struct dentry **tpm_bios_log_setup(char *);
+extern void tpm_bios_log_teardown(struct dentry **);
+#else
+static inline struct dentry **tpm_bios_log_setup(char *name)
+{
+	return NULL;
+}
+static inline void tpm_bios_log_teardown(struct dentry **dir)
+{
+}
+#endif
+
+#endif
diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c
new file mode 100644
index 00000000000..5a831aec9d4
--- /dev/null
+++ b/drivers/char/tpm/tpm_i2c_infineon.c
@@ -0,0 +1,695 @@
+/*
+ * Copyright (C) 2012 Infineon Technologies
+ *
+ * Authors:
+ * Peter Huewe <peter.huewe@infineon.com>
+ *
+ * Device driver for TCG/TCPA TPM (trusted platform module).
+ * Specifications at www.trustedcomputinggroup.org
+ *
+ * This device driver implements the TPM interface as defined in
+ * the TCG TPM Interface Spec version 1.2, revision 1.0 and the
+ * Infineon I2C Protocol Stack Specification v0.20.
+ *
+ * It is based on the original tpm_tis device driver from Leendert van
+ * Dorn and Kyleen Hall.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2 of the
+ * License.
+ *
+ *
+ */
+#include <linux/init.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include "tpm.h"
+
+/* max. buffer size supported by our TPM */
+#define TPM_BUFSIZE 1260
+
+/* max. number of iterations after I2C NAK */
+#define MAX_COUNT 3
+
+#define SLEEP_DURATION_LOW 55
+#define SLEEP_DURATION_HI 65
+
+/* max. number of iterations after I2C NAK for 'long' commands
+ * we need this especially for sending TPM_READY, since the cleanup after the
+ * transtion to the ready state may take some time, but it is unpredictable
+ * how long it will take.
+ */
+#define MAX_COUNT_LONG 50
+
+#define SLEEP_DURATION_LONG_LOW 200
+#define SLEEP_DURATION_LONG_HI 220
+
+/* After sending TPM_READY to 'reset' the TPM we have to sleep even longer */
+#define SLEEP_DURATION_RESET_LOW 2400
+#define SLEEP_DURATION_RESET_HI 2600
+
+/* we want to use usleep_range instead of msleep for the 5ms TPM_TIMEOUT */
+#define TPM_TIMEOUT_US_LOW (TPM_TIMEOUT * 1000)
+#define TPM_TIMEOUT_US_HI  (TPM_TIMEOUT_US_LOW + 2000)
+
+/* expected value for DIDVID register */
+#define TPM_TIS_I2C_DID_VID 0x000b15d1L
+
+/* Structure to store I2C TPM specific stuff */
+struct tpm_inf_dev {
+	struct i2c_client *client;
+	u8 buf[TPM_BUFSIZE + sizeof(u8)]; /* max. buffer size + addr */
+	struct tpm_chip *chip;
+};
+
+static struct tpm_inf_dev tpm_dev;
+static struct i2c_driver tpm_tis_i2c_driver;
+
+/*
+ * iic_tpm_read() - read from TPM register
+ * @addr: register address to read from
+ * @buffer: provided by caller
+ * @len: number of bytes to read
+ *
+ * Read len bytes from TPM register and put them into
+ * buffer (little-endian format, i.e. first byte is put into buffer[0]).
+ *
+ * NOTE: TPM is big-endian for multi-byte values. Multi-byte
+ * values have to be swapped.
+ *
+ * NOTE: We can't unfortunately use the combined read/write functions
+ * provided by the i2c core as the TPM currently does not support the
+ * repeated start condition and due to it's special requirements.
+ * The i2c_smbus* functions do not work for this chip.
+ *
+ * Return -EIO on error, 0 on success.
+ */
+static int iic_tpm_read(u8 addr, u8 *buffer, size_t len)
+{
+
+	struct i2c_msg msg1 = { tpm_dev.client->addr, 0, 1, &addr };
+	struct i2c_msg msg2 = { tpm_dev.client->addr, I2C_M_RD, len, buffer };
+
+	int rc;
+	int count;
+
+	/* Lock the adapter for the duration of the whole sequence. */
+	if (!tpm_dev.client->adapter->algo->master_xfer)
+		return -EOPNOTSUPP;
+	i2c_lock_adapter(tpm_dev.client->adapter);
+
+	for (count = 0; count < MAX_COUNT; count++) {
+		rc = __i2c_transfer(tpm_dev.client->adapter, &msg1, 1);
+		if (rc > 0)
+			break;	/* break here to skip sleep */
+
+		usleep_range(SLEEP_DURATION_LOW, SLEEP_DURATION_HI);
+	}
+
+	if (rc <= 0)
+		goto out;
+
+	/* After the TPM has successfully received the register address it needs
+	 * some time, thus we're sleeping here again, before retrieving the data
+	 */
+	for (count = 0; count < MAX_COUNT; count++) {
+		usleep_range(SLEEP_DURATION_LOW, SLEEP_DURATION_HI);
+		rc = __i2c_transfer(tpm_dev.client->adapter, &msg2, 1);
+		if (rc > 0)
+			break;
+
+	}
+
+out:
+	i2c_unlock_adapter(tpm_dev.client->adapter);
+	if (rc <= 0)
+		return -EIO;
+
+	return 0;
+}
+
+static int iic_tpm_write_generic(u8 addr, u8 *buffer, size_t len,
+				 unsigned int sleep_low,
+				 unsigned int sleep_hi, u8 max_count)
+{
+	int rc = -EIO;
+	int count;
+
+	struct i2c_msg msg1 = { tpm_dev.client->addr, 0, len + 1, tpm_dev.buf };
+
+	if (len > TPM_BUFSIZE)
+		return -EINVAL;
+
+	if (!tpm_dev.client->adapter->algo->master_xfer)
+		return -EOPNOTSUPP;
+	i2c_lock_adapter(tpm_dev.client->adapter);
+
+	/* prepend the 'register address' to the buffer */
+	tpm_dev.buf[0] = addr;
+	memcpy(&(tpm_dev.buf[1]), buffer, len);
+
+	/*
+	 * NOTE: We have to use these special mechanisms here and unfortunately
+	 * cannot rely on the standard behavior of i2c_transfer.
+	 */
+	for (count = 0; count < max_count; count++) {
+		rc = __i2c_transfer(tpm_dev.client->adapter, &msg1, 1);
+		if (rc > 0)
+			break;
+
+		usleep_range(sleep_low, sleep_hi);
+	}
+
+	i2c_unlock_adapter(tpm_dev.client->adapter);
+	if (rc <= 0)
+		return -EIO;
+
+	return 0;
+}
+
+/*
+ * iic_tpm_write() - write to TPM register
+ * @addr: register address to write to
+ * @buffer: containing data to be written
+ * @len: number of bytes to write
+ *
+ * Write len bytes from provided buffer to TPM register (little
+ * endian format, i.e. buffer[0] is written as first byte).
+ *
+ * NOTE: TPM is big-endian for multi-byte values. Multi-byte
+ * values have to be swapped.
+ *
+ * NOTE: use this function instead of the iic_tpm_write_generic function.
+ *
+ * Return -EIO on error, 0 on success
+ */
+static int iic_tpm_write(u8 addr, u8 *buffer, size_t len)
+{
+	return iic_tpm_write_generic(addr, buffer, len, SLEEP_DURATION_LOW,
+				     SLEEP_DURATION_HI, MAX_COUNT);
+}
+
+/*
+ * This function is needed especially for the cleanup situation after
+ * sending TPM_READY
+ * */
+static int iic_tpm_write_long(u8 addr, u8 *buffer, size_t len)
+{
+	return iic_tpm_write_generic(addr, buffer, len, SLEEP_DURATION_LONG_LOW,
+				     SLEEP_DURATION_LONG_HI, MAX_COUNT_LONG);
+}
+
+enum tis_access {
+	TPM_ACCESS_VALID = 0x80,
+	TPM_ACCESS_ACTIVE_LOCALITY = 0x20,
+	TPM_ACCESS_REQUEST_PENDING = 0x04,
+	TPM_ACCESS_REQUEST_USE = 0x02,
+};
+
+enum tis_status {
+	TPM_STS_VALID = 0x80,
+	TPM_STS_COMMAND_READY = 0x40,
+	TPM_STS_GO = 0x20,
+	TPM_STS_DATA_AVAIL = 0x10,
+	TPM_STS_DATA_EXPECT = 0x08,
+};
+
+enum tis_defaults {
+	TIS_SHORT_TIMEOUT = 750,	/* ms */
+	TIS_LONG_TIMEOUT = 2000,	/* 2 sec */
+};
+
+#define	TPM_ACCESS(l)			(0x0000 | ((l) << 4))
+#define	TPM_STS(l)			(0x0001 | ((l) << 4))
+#define	TPM_DATA_FIFO(l)		(0x0005 | ((l) << 4))
+#define	TPM_DID_VID(l)			(0x0006 | ((l) << 4))
+
+static int check_locality(struct tpm_chip *chip, int loc)
+{
+	u8 buf;
+	int rc;
+
+	rc = iic_tpm_read(TPM_ACCESS(loc), &buf, 1);
+	if (rc < 0)
+		return rc;
+
+	if ((buf & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) ==
+	    (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) {
+		chip->vendor.locality = loc;
+		return loc;
+	}
+
+	return -EIO;
+}
+
+/* implementation similar to tpm_tis */
+static void release_locality(struct tpm_chip *chip, int loc, int force)
+{
+	u8 buf;
+	if (iic_tpm_read(TPM_ACCESS(loc), &buf, 1) < 0)
+		return;
+
+	if (force || (buf & (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) ==
+	    (TPM_ACCESS_REQUEST_PENDING | TPM_ACCESS_VALID)) {
+		buf = TPM_ACCESS_ACTIVE_LOCALITY;
+		iic_tpm_write(TPM_ACCESS(loc), &buf, 1);
+	}
+}
+
+static int request_locality(struct tpm_chip *chip, int loc)
+{
+	unsigned long stop;
+	u8 buf = TPM_ACCESS_REQUEST_USE;
+
+	if (check_locality(chip, loc) >= 0)
+		return loc;
+
+	iic_tpm_write(TPM_ACCESS(loc), &buf, 1);
+
+	/* wait for burstcount */
+	stop = jiffies + chip->vendor.timeout_a;
+	do {
+		if (check_locality(chip, loc) >= 0)
+			return loc;
+		usleep_range(TPM_TIMEOUT_US_LOW, TPM_TIMEOUT_US_HI);
+	} while (time_before(jiffies, stop));
+
+	return -ETIME;
+}
+
+static u8 tpm_tis_i2c_status(struct tpm_chip *chip)
+{
+	/* NOTE: since I2C read may fail, return 0 in this case --> time-out */
+	u8 buf;
+	if (iic_tpm_read(TPM_STS(chip->vendor.locality), &buf, 1) < 0)
+		return 0;
+	else
+		return buf;
+}
+
+static void tpm_tis_i2c_ready(struct tpm_chip *chip)
+{
+	/* this causes the current command to be aborted */
+	u8 buf = TPM_STS_COMMAND_READY;
+	iic_tpm_write_long(TPM_STS(chip->vendor.locality), &buf, 1);
+}
+
+static ssize_t get_burstcount(struct tpm_chip *chip)
+{
+	unsigned long stop;
+	ssize_t burstcnt;
+	u8 buf[3];
+
+	/* wait for burstcount */
+	/* which timeout value, spec has 2 answers (c & d) */
+	stop = jiffies + chip->vendor.timeout_d;
+	do {
+		/* Note: STS is little endian */
+		if (iic_tpm_read(TPM_STS(chip->vendor.locality)+1, buf, 3) < 0)
+			burstcnt = 0;
+		else
+			burstcnt = (buf[2] << 16) + (buf[1] << 8) + buf[0];
+
+		if (burstcnt)
+			return burstcnt;
+
+		usleep_range(TPM_TIMEOUT_US_LOW, TPM_TIMEOUT_US_HI);
+	} while (time_before(jiffies, stop));
+	return -EBUSY;
+}
+
+static int wait_for_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout,
+			 int *status)
+{
+	unsigned long stop;
+
+	/* check current status */
+	*status = tpm_tis_i2c_status(chip);
+	if ((*status & mask) == mask)
+		return 0;
+
+	stop = jiffies + timeout;
+	do {
+		/* since we just checked the status, give the TPM some time */
+		usleep_range(TPM_TIMEOUT_US_LOW, TPM_TIMEOUT_US_HI);
+		*status = tpm_tis_i2c_status(chip);
+		if ((*status & mask) == mask)
+			return 0;
+
+	} while (time_before(jiffies, stop));
+
+	return -ETIME;
+}
+
+static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count)
+{
+	size_t size = 0;
+	ssize_t burstcnt;
+	u8 retries = 0;
+	int rc;
+
+	while (size < count) {
+		burstcnt = get_burstcount(chip);
+
+		/* burstcnt < 0 = TPM is busy */
+		if (burstcnt < 0)
+			return burstcnt;
+
+		/* limit received data to max. left */
+		if (burstcnt > (count - size))
+			burstcnt = count - size;
+
+		rc = iic_tpm_read(TPM_DATA_FIFO(chip->vendor.locality),
+				  &(buf[size]), burstcnt);
+		if (rc == 0)
+			size += burstcnt;
+		else if (rc < 0)
+			retries++;
+
+		/* avoid endless loop in case of broken HW */
+		if (retries > MAX_COUNT_LONG)
+			return -EIO;
+
+	}
+	return size;
+}
+
+static int tpm_tis_i2c_recv(struct tpm_chip *chip, u8 *buf, size_t count)
+{
+	int size = 0;
+	int expected, status;
+
+	if (count < TPM_HEADER_SIZE) {
+		size = -EIO;
+		goto out;
+	}
+
+	/* read first 10 bytes, including tag, paramsize, and result */
+	size = recv_data(chip, buf, TPM_HEADER_SIZE);
+	if (size < TPM_HEADER_SIZE) {
+		dev_err(chip->dev, "Unable to read header\n");
+		goto out;
+	}
+
+	expected = be32_to_cpu(*(__be32 *)(buf + 2));
+	if ((size_t) expected > count) {
+		size = -EIO;
+		goto out;
+	}
+
+	size += recv_data(chip, &buf[TPM_HEADER_SIZE],
+			  expected - TPM_HEADER_SIZE);
+	if (size < expected) {
+		dev_err(chip->dev, "Unable to read remainder of result\n");
+		size = -ETIME;
+		goto out;
+	}
+
+	wait_for_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, &status);
+	if (status & TPM_STS_DATA_AVAIL) {	/* retry? */
+		dev_err(chip->dev, "Error left over data\n");
+		size = -EIO;
+		goto out;
+	}
+
+out:
+	tpm_tis_i2c_ready(chip);
+	/* The TPM needs some time to clean up here,
+	 * so we sleep rather than keeping the bus busy
+	 */
+	usleep_range(SLEEP_DURATION_RESET_LOW, SLEEP_DURATION_RESET_HI);
+	release_locality(chip, chip->vendor.locality, 0);
+	return size;
+}
+
+static int tpm_tis_i2c_send(struct tpm_chip *chip, u8 *buf, size_t len)
+{
+	int rc, status;
+	ssize_t burstcnt;
+	size_t count = 0;
+	u8 retries = 0;
+	u8 sts = TPM_STS_GO;
+
+	if (len > TPM_BUFSIZE)
+		return -E2BIG;	/* command is too long for our tpm, sorry */
+
+	if (request_locality(chip, 0) < 0)
+		return -EBUSY;
+
+	status = tpm_tis_i2c_status(chip);
+	if ((status & TPM_STS_COMMAND_READY) == 0) {
+		tpm_tis_i2c_ready(chip);
+		if (wait_for_stat
+		    (chip, TPM_STS_COMMAND_READY,
+		     chip->vendor.timeout_b, &status) < 0) {
+			rc = -ETIME;
+			goto out_err;
+		}
+	}
+
+	while (count < len - 1) {
+		burstcnt = get_burstcount(chip);
+
+		/* burstcnt < 0 = TPM is busy */
+		if (burstcnt < 0)
+			return burstcnt;
+
+		if (burstcnt > (len - 1 - count))
+			burstcnt = len - 1 - count;
+
+		rc = iic_tpm_write(TPM_DATA_FIFO(chip->vendor.locality),
+				   &(buf[count]), burstcnt);
+		if (rc == 0)
+			count += burstcnt;
+		else if (rc < 0)
+			retries++;
+
+		/* avoid endless loop in case of broken HW */
+		if (retries > MAX_COUNT_LONG) {
+			rc = -EIO;
+			goto out_err;
+		}
+
+		wait_for_stat(chip, TPM_STS_VALID,
+			      chip->vendor.timeout_c, &status);
+
+		if ((status & TPM_STS_DATA_EXPECT) == 0) {
+			rc = -EIO;
+			goto out_err;
+		}
+
+	}
+
+	/* write last byte */
+	iic_tpm_write(TPM_DATA_FIFO(chip->vendor.locality), &(buf[count]), 1);
+	wait_for_stat(chip, TPM_STS_VALID, chip->vendor.timeout_c, &status);
+	if ((status & TPM_STS_DATA_EXPECT) != 0) {
+		rc = -EIO;
+		goto out_err;
+	}
+
+	/* go and do it */
+	iic_tpm_write(TPM_STS(chip->vendor.locality), &sts, 1);
+
+	return len;
+out_err:
+	tpm_tis_i2c_ready(chip);
+	/* The TPM needs some time to clean up here,
+	 * so we sleep rather than keeping the bus busy
+	 */
+	usleep_range(SLEEP_DURATION_RESET_LOW, SLEEP_DURATION_RESET_HI);
+	release_locality(chip, chip->vendor.locality, 0);
+	return rc;
+}
+
+static const struct file_operations tis_ops = {
+	.owner = THIS_MODULE,
+	.llseek = no_llseek,
+	.open = tpm_open,
+	.read = tpm_read,
+	.write = tpm_write,
+	.release = tpm_release,
+};
+
+static DEVICE_ATTR(pubek, S_IRUGO, tpm_show_pubek, NULL);
+static DEVICE_ATTR(pcrs, S_IRUGO, tpm_show_pcrs, NULL);
+static DEVICE_ATTR(enabled, S_IRUGO, tpm_show_enabled, NULL);
+static DEVICE_ATTR(active, S_IRUGO, tpm_show_active, NULL);
+static DEVICE_ATTR(owned, S_IRUGO, tpm_show_owned, NULL);
+static DEVICE_ATTR(temp_deactivated, S_IRUGO, tpm_show_temp_deactivated, NULL);
+static DEVICE_ATTR(caps, S_IRUGO, tpm_show_caps_1_2, NULL);
+static DEVICE_ATTR(cancel, S_IWUSR | S_IWGRP, NULL, tpm_store_cancel);
+static DEVICE_ATTR(durations, S_IRUGO, tpm_show_durations, NULL);
+static DEVICE_ATTR(timeouts, S_IRUGO, tpm_show_timeouts, NULL);
+
+static struct attribute *tis_attrs[] = {
+	&dev_attr_pubek.attr,
+	&dev_attr_pcrs.attr,
+	&dev_attr_enabled.attr,
+	&dev_attr_active.attr,
+	&dev_attr_owned.attr,
+	&dev_attr_temp_deactivated.attr,
+	&dev_attr_caps.attr,
+	&dev_attr_cancel.attr,
+	&dev_attr_durations.attr,
+	&dev_attr_timeouts.attr,
+	NULL,
+};
+
+static struct attribute_group tis_attr_grp = {
+	.attrs = tis_attrs
+};
+
+static struct tpm_vendor_specific tpm_tis_i2c = {
+	.status = tpm_tis_i2c_status,
+	.recv = tpm_tis_i2c_recv,
+	.send = tpm_tis_i2c_send,
+	.cancel = tpm_tis_i2c_ready,
+	.req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID,
+	.req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID,
+	.req_canceled = TPM_STS_COMMAND_READY,
+	.attr_group = &tis_attr_grp,
+	.miscdev.fops = &tis_ops,
+};
+
+static int __devinit tpm_tis_i2c_init(struct device *dev)
+{
+	u32 vendor;
+	int rc = 0;
+	struct tpm_chip *chip;
+
+	chip = tpm_register_hardware(dev, &tpm_tis_i2c);
+	if (!chip) {
+		rc = -ENODEV;
+		goto out_err;
+	}
+
+	/* Disable interrupts */
+	chip->vendor.irq = 0;
+
+	/* Default timeouts */
+	chip->vendor.timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+	chip->vendor.timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT);
+	chip->vendor.timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+	chip->vendor.timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+
+	if (request_locality(chip, 0) != 0) {
+		rc = -ENODEV;
+		goto out_vendor;
+	}
+
+	/* read four bytes from DID_VID register */
+	if (iic_tpm_read(TPM_DID_VID(0), (u8 *)&vendor, 4) < 0) {
+		rc = -EIO;
+		goto out_release;
+	}
+
+	/* create DID_VID register value, after swapping to little-endian */
+	vendor = be32_to_cpu((__be32) vendor);
+
+	if (vendor != TPM_TIS_I2C_DID_VID) {
+		rc = -ENODEV;
+		goto out_release;
+	}
+
+	dev_info(dev, "1.2 TPM (device-id 0x%X)\n", vendor >> 16);
+
+	INIT_LIST_HEAD(&chip->vendor.list);
+	tpm_dev.chip = chip;
+
+	tpm_get_timeouts(chip);
+	tpm_do_selftest(chip);
+
+	return 0;
+
+out_release:
+	release_locality(chip, chip->vendor.locality, 1);
+
+out_vendor:
+	/* close file handles */
+	tpm_dev_vendor_release(chip);
+
+	/* remove hardware */
+	tpm_remove_hardware(chip->dev);
+
+	/* reset these pointers, otherwise we oops */
+	chip->dev->release = NULL;
+	chip->release = NULL;
+	tpm_dev.client = NULL;
+	dev_set_drvdata(chip->dev, chip);
+out_err:
+	return rc;
+}
+
+static const struct i2c_device_id tpm_tis_i2c_table[] = {
+	{"tpm_i2c_infineon", 0},
+	{},
+};
+
+MODULE_DEVICE_TABLE(i2c, tpm_tis_i2c_table);
+static SIMPLE_DEV_PM_OPS(tpm_tis_i2c_ops, tpm_pm_suspend, tpm_pm_resume);
+
+static int __devinit tpm_tis_i2c_probe(struct i2c_client *client,
+			     const struct i2c_device_id *id)
+{
+	int rc;
+	if (tpm_dev.client != NULL)
+		return -EBUSY;	/* We only support one client */
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev,
+			"no algorithms associated to the i2c bus\n");
+		return -ENODEV;
+	}
+
+	client->driver = &tpm_tis_i2c_driver;
+	tpm_dev.client = client;
+	rc = tpm_tis_i2c_init(&client->dev);
+	if (rc != 0) {
+		client->driver = NULL;
+		tpm_dev.client = NULL;
+		rc = -ENODEV;
+	}
+	return rc;
+}
+
+static int __devexit tpm_tis_i2c_remove(struct i2c_client *client)
+{
+	struct tpm_chip *chip = tpm_dev.chip;
+	release_locality(chip, chip->vendor.locality, 1);
+
+	/* close file handles */
+	tpm_dev_vendor_release(chip);
+
+	/* remove hardware */
+	tpm_remove_hardware(chip->dev);
+
+	/* reset these pointers, otherwise we oops */
+	chip->dev->release = NULL;
+	chip->release = NULL;
+	tpm_dev.client = NULL;
+	dev_set_drvdata(chip->dev, chip);
+
+	return 0;
+}
+
+static struct i2c_driver tpm_tis_i2c_driver = {
+
+	.id_table = tpm_tis_i2c_table,
+	.probe = tpm_tis_i2c_probe,
+	.remove = tpm_tis_i2c_remove,
+	.driver = {
+		   .name = "tpm_i2c_infineon",
+		   .owner = THIS_MODULE,
+		   .pm = &tpm_tis_i2c_ops,
+		   },
+};
+
+module_i2c_driver(tpm_tis_i2c_driver);
+MODULE_AUTHOR("Peter Huewe <peter.huewe@infineon.com>");
+MODULE_DESCRIPTION("TPM TIS I2C Infineon Driver");
+MODULE_VERSION("2.1.5");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/tpm/tpm_ibmvtpm.c b/drivers/char/tpm/tpm_ibmvtpm.c
new file mode 100644
index 00000000000..efc4ab36a9d
--- /dev/null
+++ b/drivers/char/tpm/tpm_ibmvtpm.c
@@ -0,0 +1,749 @@
+/*
+ * Copyright (C) 2012 IBM Corporation
+ *
+ * Author: Ashley Lai <adlai@us.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Device driver for TCG/TCPA TPM (trusted platform module).
+ * Specifications at www.trustedcomputinggroup.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2 of the
+ * License.
+ *
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/slab.h>
+#include <asm/vio.h>
+#include <asm/irq.h>
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/interrupt.h>
+#include <linux/wait.h>
+#include <asm/prom.h>
+
+#include "tpm.h"
+#include "tpm_ibmvtpm.h"
+
+static const char tpm_ibmvtpm_driver_name[] = "tpm_ibmvtpm";
+
+static struct vio_device_id tpm_ibmvtpm_device_table[] __devinitdata = {
+	{ "IBM,vtpm", "IBM,vtpm"},
+	{ "", "" }
+};
+MODULE_DEVICE_TABLE(vio, tpm_ibmvtpm_device_table);
+
+DECLARE_WAIT_QUEUE_HEAD(wq);
+
+/**
+ * ibmvtpm_send_crq - Send a CRQ request
+ * @vdev:	vio device struct
+ * @w1:		first word
+ * @w2:		second word
+ *
+ * Return value:
+ *	0 -Sucess
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_send_crq(struct vio_dev *vdev, u64 w1, u64 w2)
+{
+	return plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, w1, w2);
+}
+
+/**
+ * ibmvtpm_get_data - Retrieve ibm vtpm data
+ * @dev:	device struct
+ *
+ * Return value:
+ *	vtpm device struct
+ */
+static struct ibmvtpm_dev *ibmvtpm_get_data(const struct device *dev)
+{
+	struct tpm_chip *chip = dev_get_drvdata(dev);
+	if (chip)
+		return (struct ibmvtpm_dev *)chip->vendor.data;
+	return NULL;
+}
+
+/**
+ * tpm_ibmvtpm_recv - Receive data after send
+ * @chip:	tpm chip struct
+ * @buf:	buffer to read
+ * count:	size of buffer
+ *
+ * Return value:
+ *	Number of bytes read
+ */
+static int tpm_ibmvtpm_recv(struct tpm_chip *chip, u8 *buf, size_t count)
+{
+	struct ibmvtpm_dev *ibmvtpm;
+	u16 len;
+
+	ibmvtpm = (struct ibmvtpm_dev *)chip->vendor.data;
+
+	if (!ibmvtpm->rtce_buf) {
+		dev_err(ibmvtpm->dev, "ibmvtpm device is not ready\n");
+		return 0;
+	}
+
+	wait_event_interruptible(wq, ibmvtpm->crq_res.len != 0);
+
+	if (count < ibmvtpm->crq_res.len) {
+		dev_err(ibmvtpm->dev,
+			"Invalid size in recv: count=%ld, crq_size=%d\n",
+			count, ibmvtpm->crq_res.len);
+		return -EIO;
+	}
+
+	spin_lock(&ibmvtpm->rtce_lock);
+	memcpy((void *)buf, (void *)ibmvtpm->rtce_buf, ibmvtpm->crq_res.len);
+	memset(ibmvtpm->rtce_buf, 0, ibmvtpm->crq_res.len);
+	ibmvtpm->crq_res.valid = 0;
+	ibmvtpm->crq_res.msg = 0;
+	len = ibmvtpm->crq_res.len;
+	ibmvtpm->crq_res.len = 0;
+	spin_unlock(&ibmvtpm->rtce_lock);
+	return len;
+}
+
+/**
+ * tpm_ibmvtpm_send - Send tpm request
+ * @chip:	tpm chip struct
+ * @buf:	buffer contains data to send
+ * count:	size of buffer
+ *
+ * Return value:
+ *	Number of bytes sent
+ */
+static int tpm_ibmvtpm_send(struct tpm_chip *chip, u8 *buf, size_t count)
+{
+	struct ibmvtpm_dev *ibmvtpm;
+	struct ibmvtpm_crq crq;
+	u64 *word = (u64 *) &crq;
+	int rc;
+
+	ibmvtpm = (struct ibmvtpm_dev *)chip->vendor.data;
+
+	if (!ibmvtpm->rtce_buf) {
+		dev_err(ibmvtpm->dev, "ibmvtpm device is not ready\n");
+		return 0;
+	}
+
+	if (count > ibmvtpm->rtce_size) {
+		dev_err(ibmvtpm->dev,
+			"Invalid size in send: count=%ld, rtce_size=%d\n",
+			count, ibmvtpm->rtce_size);
+		return -EIO;
+	}
+
+	spin_lock(&ibmvtpm->rtce_lock);
+	memcpy((void *)ibmvtpm->rtce_buf, (void *)buf, count);
+	crq.valid = (u8)IBMVTPM_VALID_CMD;
+	crq.msg = (u8)VTPM_TPM_COMMAND;
+	crq.len = (u16)count;
+	crq.data = ibmvtpm->rtce_dma_handle;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, word[0], word[1]);
+	if (rc != H_SUCCESS) {
+		dev_err(ibmvtpm->dev, "tpm_ibmvtpm_send failed rc=%d\n", rc);
+		rc = 0;
+	} else
+		rc = count;
+
+	spin_unlock(&ibmvtpm->rtce_lock);
+	return rc;
+}
+
+static void tpm_ibmvtpm_cancel(struct tpm_chip *chip)
+{
+	return;
+}
+
+static u8 tpm_ibmvtpm_status(struct tpm_chip *chip)
+{
+	return 0;
+}
+
+/**
+ * ibmvtpm_crq_get_rtce_size - Send a CRQ request to get rtce size
+ * @ibmvtpm:	vtpm device struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_crq_get_rtce_size(struct ibmvtpm_dev *ibmvtpm)
+{
+	struct ibmvtpm_crq crq;
+	u64 *buf = (u64 *) &crq;
+	int rc;
+
+	crq.valid = (u8)IBMVTPM_VALID_CMD;
+	crq.msg = (u8)VTPM_GET_RTCE_BUFFER_SIZE;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, buf[0], buf[1]);
+	if (rc != H_SUCCESS)
+		dev_err(ibmvtpm->dev,
+			"ibmvtpm_crq_get_rtce_size failed rc=%d\n", rc);
+
+	return rc;
+}
+
+/**
+ * ibmvtpm_crq_get_version - Send a CRQ request to get vtpm version
+ *			   - Note that this is vtpm version and not tpm version
+ * @ibmvtpm:	vtpm device struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_crq_get_version(struct ibmvtpm_dev *ibmvtpm)
+{
+	struct ibmvtpm_crq crq;
+	u64 *buf = (u64 *) &crq;
+	int rc;
+
+	crq.valid = (u8)IBMVTPM_VALID_CMD;
+	crq.msg = (u8)VTPM_GET_VERSION;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, buf[0], buf[1]);
+	if (rc != H_SUCCESS)
+		dev_err(ibmvtpm->dev,
+			"ibmvtpm_crq_get_version failed rc=%d\n", rc);
+
+	return rc;
+}
+
+/**
+ * ibmvtpm_crq_send_init_complete - Send a CRQ initialize complete message
+ * @ibmvtpm:	vtpm device struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_crq_send_init_complete(struct ibmvtpm_dev *ibmvtpm)
+{
+	int rc;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, INIT_CRQ_COMP_CMD, 0);
+	if (rc != H_SUCCESS)
+		dev_err(ibmvtpm->dev,
+			"ibmvtpm_crq_send_init_complete failed rc=%d\n", rc);
+
+	return rc;
+}
+
+/**
+ * ibmvtpm_crq_send_init - Send a CRQ initialize message
+ * @ibmvtpm:	vtpm device struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_crq_send_init(struct ibmvtpm_dev *ibmvtpm)
+{
+	int rc;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, INIT_CRQ_CMD, 0);
+	if (rc != H_SUCCESS)
+		dev_err(ibmvtpm->dev,
+			"ibmvtpm_crq_send_init failed rc=%d\n", rc);
+
+	return rc;
+}
+
+/**
+ * tpm_ibmvtpm_remove - ibm vtpm remove entry point
+ * @vdev:	vio device struct
+ *
+ * Return value:
+ *	0
+ */
+static int __devexit tpm_ibmvtpm_remove(struct vio_dev *vdev)
+{
+	struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(&vdev->dev);
+	int rc = 0;
+
+	free_irq(vdev->irq, ibmvtpm);
+	tasklet_kill(&ibmvtpm->tasklet);
+
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
+	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
+
+	dma_unmap_single(ibmvtpm->dev, ibmvtpm->crq_dma_handle,
+			 CRQ_RES_BUF_SIZE, DMA_BIDIRECTIONAL);
+	free_page((unsigned long)ibmvtpm->crq_queue.crq_addr);
+
+	if (ibmvtpm->rtce_buf) {
+		dma_unmap_single(ibmvtpm->dev, ibmvtpm->rtce_dma_handle,
+				 ibmvtpm->rtce_size, DMA_BIDIRECTIONAL);
+		kfree(ibmvtpm->rtce_buf);
+	}
+
+	tpm_remove_hardware(ibmvtpm->dev);
+
+	kfree(ibmvtpm);
+
+	return 0;
+}
+
+/**
+ * tpm_ibmvtpm_get_desired_dma - Get DMA size needed by this driver
+ * @vdev:	vio device struct
+ *
+ * Return value:
+ *	Number of bytes the driver needs to DMA map
+ */
+static unsigned long tpm_ibmvtpm_get_desired_dma(struct vio_dev *vdev)
+{
+	struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(&vdev->dev);
+	return CRQ_RES_BUF_SIZE + ibmvtpm->rtce_size;
+}
+
+/**
+ * tpm_ibmvtpm_suspend - Suspend
+ * @dev:	device struct
+ *
+ * Return value:
+ *	0
+ */
+static int tpm_ibmvtpm_suspend(struct device *dev)
+{
+	struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(dev);
+	struct ibmvtpm_crq crq;
+	u64 *buf = (u64 *) &crq;
+	int rc = 0;
+
+	crq.valid = (u8)IBMVTPM_VALID_CMD;
+	crq.msg = (u8)VTPM_PREPARE_TO_SUSPEND;
+
+	rc = ibmvtpm_send_crq(ibmvtpm->vdev, buf[0], buf[1]);
+	if (rc != H_SUCCESS)
+		dev_err(ibmvtpm->dev,
+			"tpm_ibmvtpm_suspend failed rc=%d\n", rc);
+
+	return rc;
+}
+
+/**
+ * ibmvtpm_reset_crq - Reset CRQ
+ * @ibmvtpm:	ibm vtpm struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int ibmvtpm_reset_crq(struct ibmvtpm_dev *ibmvtpm)
+{
+	int rc = 0;
+
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_FREE_CRQ,
+					ibmvtpm->vdev->unit_address);
+	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));
+
+	memset(ibmvtpm->crq_queue.crq_addr, 0, CRQ_RES_BUF_SIZE);
+	ibmvtpm->crq_queue.index = 0;
+
+	return plpar_hcall_norets(H_REG_CRQ, ibmvtpm->vdev->unit_address,
+				  ibmvtpm->crq_dma_handle, CRQ_RES_BUF_SIZE);
+}
+
+/**
+ * tpm_ibmvtpm_resume - Resume from suspend
+ * @dev:	device struct
+ *
+ * Return value:
+ *	0
+ */
+static int tpm_ibmvtpm_resume(struct device *dev)
+{
+	struct ibmvtpm_dev *ibmvtpm = ibmvtpm_get_data(dev);
+	unsigned long flags;
+	int rc = 0;
+
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_ENABLE_CRQ,
+					ibmvtpm->vdev->unit_address);
+	} while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));
+
+	if (rc) {
+		dev_err(dev, "Error enabling ibmvtpm rc=%d\n", rc);
+		return rc;
+	}
+
+	spin_lock_irqsave(&ibmvtpm->lock, flags);
+	vio_disable_interrupts(ibmvtpm->vdev);
+	tasklet_schedule(&ibmvtpm->tasklet);
+	spin_unlock_irqrestore(&ibmvtpm->lock, flags);
+
+	rc = ibmvtpm_crq_send_init(ibmvtpm);
+	if (rc)
+		dev_err(dev, "Error send_init rc=%d\n", rc);
+
+	return rc;
+}
+
+static const struct file_operations ibmvtpm_ops = {
+	.owner = THIS_MODULE,
+	.llseek = no_llseek,
+	.open = tpm_open,
+	.read = tpm_read,
+	.write = tpm_write,
+	.release = tpm_release,
+};
+
+static DEVICE_ATTR(pubek, S_IRUGO, tpm_show_pubek, NULL);
+static DEVICE_ATTR(pcrs, S_IRUGO, tpm_show_pcrs, NULL);
+static DEVICE_ATTR(enabled, S_IRUGO, tpm_show_enabled, NULL);
+static DEVICE_ATTR(active, S_IRUGO, tpm_show_active, NULL);
+static DEVICE_ATTR(owned, S_IRUGO, tpm_show_owned, NULL);
+static DEVICE_ATTR(temp_deactivated, S_IRUGO, tpm_show_temp_deactivated,
+		   NULL);
+static DEVICE_ATTR(caps, S_IRUGO, tpm_show_caps_1_2, NULL);
+static DEVICE_ATTR(cancel, S_IWUSR | S_IWGRP, NULL, tpm_store_cancel);
+static DEVICE_ATTR(durations, S_IRUGO, tpm_show_durations, NULL);
+static DEVICE_ATTR(timeouts, S_IRUGO, tpm_show_timeouts, NULL);
+
+static struct attribute *ibmvtpm_attrs[] = {
+	&dev_attr_pubek.attr,
+	&dev_attr_pcrs.attr,
+	&dev_attr_enabled.attr,
+	&dev_attr_active.attr,
+	&dev_attr_owned.attr,
+	&dev_attr_temp_deactivated.attr,
+	&dev_attr_caps.attr,
+	&dev_attr_cancel.attr,
+	&dev_attr_durations.attr,
+	&dev_attr_timeouts.attr, NULL,
+};
+
+static struct attribute_group ibmvtpm_attr_grp = { .attrs = ibmvtpm_attrs };
+
+static const struct tpm_vendor_specific tpm_ibmvtpm = {
+	.recv = tpm_ibmvtpm_recv,
+	.send = tpm_ibmvtpm_send,
+	.cancel = tpm_ibmvtpm_cancel,
+	.status = tpm_ibmvtpm_status,
+	.req_complete_mask = 0,
+	.req_complete_val = 0,
+	.req_canceled = 0,
+	.attr_group = &ibmvtpm_attr_grp,
+	.miscdev = { .fops = &ibmvtpm_ops, },
+};
+
+static const struct dev_pm_ops tpm_ibmvtpm_pm_ops = {
+	.suspend = tpm_ibmvtpm_suspend,
+	.resume = tpm_ibmvtpm_resume,
+};
+
+/**
+ * ibmvtpm_crq_get_next - Get next responded crq
+ * @ibmvtpm	vtpm device struct
+ *
+ * Return value:
+ *	vtpm crq pointer
+ */
+static struct ibmvtpm_crq *ibmvtpm_crq_get_next(struct ibmvtpm_dev *ibmvtpm)
+{
+	struct ibmvtpm_crq_queue *crq_q = &ibmvtpm->crq_queue;
+	struct ibmvtpm_crq *crq = &crq_q->crq_addr[crq_q->index];
+
+	if (crq->valid & VTPM_MSG_RES) {
+		if (++crq_q->index == crq_q->num_entry)
+			crq_q->index = 0;
+		rmb();
+	} else
+		crq = NULL;
+	return crq;
+}
+
+/**
+ * ibmvtpm_crq_process - Process responded crq
+ * @crq		crq to be processed
+ * @ibmvtpm	vtpm device struct
+ *
+ * Return value:
+ *	Nothing
+ */
+static void ibmvtpm_crq_process(struct ibmvtpm_crq *crq,
+				struct ibmvtpm_dev *ibmvtpm)
+{
+	int rc = 0;
+
+	switch (crq->valid) {
+	case VALID_INIT_CRQ:
+		switch (crq->msg) {
+		case INIT_CRQ_RES:
+			dev_info(ibmvtpm->dev, "CRQ initialized\n");
+			rc = ibmvtpm_crq_send_init_complete(ibmvtpm);
+			if (rc)
+				dev_err(ibmvtpm->dev, "Unable to send CRQ init complete rc=%d\n", rc);
+			return;
+		case INIT_CRQ_COMP_RES:
+			dev_info(ibmvtpm->dev,
+				 "CRQ initialization completed\n");
+			return;
+		default:
+			dev_err(ibmvtpm->dev, "Unknown crq message type: %d\n", crq->msg);
+			return;
+		}
+		return;
+	case IBMVTPM_VALID_CMD:
+		switch (crq->msg) {
+		case VTPM_GET_RTCE_BUFFER_SIZE_RES:
+			if (crq->len <= 0) {
+				dev_err(ibmvtpm->dev, "Invalid rtce size\n");
+				return;
+			}
+			ibmvtpm->rtce_size = crq->len;
+			ibmvtpm->rtce_buf = kmalloc(ibmvtpm->rtce_size,
+						    GFP_KERNEL);
+			if (!ibmvtpm->rtce_buf) {
+				dev_err(ibmvtpm->dev, "Failed to allocate memory for rtce buffer\n");
+				return;
+			}
+
+			ibmvtpm->rtce_dma_handle = dma_map_single(ibmvtpm->dev,
+				ibmvtpm->rtce_buf, ibmvtpm->rtce_size,
+				DMA_BIDIRECTIONAL);
+
+			if (dma_mapping_error(ibmvtpm->dev,
+					      ibmvtpm->rtce_dma_handle)) {
+				kfree(ibmvtpm->rtce_buf);
+				ibmvtpm->rtce_buf = NULL;
+				dev_err(ibmvtpm->dev, "Failed to dma map rtce buffer\n");
+			}
+
+			return;
+		case VTPM_GET_VERSION_RES:
+			ibmvtpm->vtpm_version = crq->data;
+			return;
+		case VTPM_TPM_COMMAND_RES:
+			ibmvtpm->crq_res.valid = crq->valid;
+			ibmvtpm->crq_res.msg = crq->msg;
+			ibmvtpm->crq_res.len = crq->len;
+			ibmvtpm->crq_res.data = crq->data;
+			wake_up_interruptible(&wq);
+			return;
+		default:
+			return;
+		}
+	}
+	return;
+}
+
+/**
+ * ibmvtpm_interrupt -	Interrupt handler
+ * @irq:		irq number to handle
+ * @vtpm_instance:	vtpm that received interrupt
+ *
+ * Returns:
+ *	IRQ_HANDLED
+ **/
+static irqreturn_t ibmvtpm_interrupt(int irq, void *vtpm_instance)
+{
+	struct ibmvtpm_dev *ibmvtpm = (struct ibmvtpm_dev *) vtpm_instance;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ibmvtpm->lock, flags);
+	vio_disable_interrupts(ibmvtpm->vdev);
+	tasklet_schedule(&ibmvtpm->tasklet);
+	spin_unlock_irqrestore(&ibmvtpm->lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * ibmvtpm_tasklet - Interrupt handler tasklet
+ * @data:	ibm vtpm device struct
+ *
+ * Returns:
+ *	Nothing
+ **/
+static void ibmvtpm_tasklet(void *data)
+{
+	struct ibmvtpm_dev *ibmvtpm = data;
+	struct ibmvtpm_crq *crq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ibmvtpm->lock, flags);
+	while ((crq = ibmvtpm_crq_get_next(ibmvtpm)) != NULL) {
+		ibmvtpm_crq_process(crq, ibmvtpm);
+		crq->valid = 0;
+		wmb();
+	}
+
+	vio_enable_interrupts(ibmvtpm->vdev);
+	spin_unlock_irqrestore(&ibmvtpm->lock, flags);
+}
+
+/**
+ * tpm_ibmvtpm_probe - ibm vtpm initialize entry point
+ * @vio_dev:	vio device struct
+ * @id:		vio device id struct
+ *
+ * Return value:
+ *	0 - Success
+ *	Non-zero - Failure
+ */
+static int __devinit tpm_ibmvtpm_probe(struct vio_dev *vio_dev,
+				   const struct vio_device_id *id)
+{
+	struct ibmvtpm_dev *ibmvtpm;
+	struct device *dev = &vio_dev->dev;
+	struct ibmvtpm_crq_queue *crq_q;
+	struct tpm_chip *chip;
+	int rc = -ENOMEM, rc1;
+
+	chip = tpm_register_hardware(dev, &tpm_ibmvtpm);
+	if (!chip) {
+		dev_err(dev, "tpm_register_hardware failed\n");
+		return -ENODEV;
+	}
+
+	ibmvtpm = kzalloc(sizeof(struct ibmvtpm_dev), GFP_KERNEL);
+	if (!ibmvtpm) {
+		dev_err(dev, "kzalloc for ibmvtpm failed\n");
+		goto cleanup;
+	}
+
+	crq_q = &ibmvtpm->crq_queue;
+	crq_q->crq_addr = (struct ibmvtpm_crq *)get_zeroed_page(GFP_KERNEL);
+	if (!crq_q->crq_addr) {
+		dev_err(dev, "Unable to allocate memory for crq_addr\n");
+		goto cleanup;
+	}
+
+	crq_q->num_entry = CRQ_RES_BUF_SIZE / sizeof(*crq_q->crq_addr);
+	ibmvtpm->crq_dma_handle = dma_map_single(dev, crq_q->crq_addr,
+						 CRQ_RES_BUF_SIZE,
+						 DMA_BIDIRECTIONAL);
+
+	if (dma_mapping_error(dev, ibmvtpm->crq_dma_handle)) {
+		dev_err(dev, "dma mapping failed\n");
+		goto cleanup;
+	}
+
+	rc = plpar_hcall_norets(H_REG_CRQ, vio_dev->unit_address,
+				ibmvtpm->crq_dma_handle, CRQ_RES_BUF_SIZE);
+	if (rc == H_RESOURCE)
+		rc = ibmvtpm_reset_crq(ibmvtpm);
+
+	if (rc) {
+		dev_err(dev, "Unable to register CRQ rc=%d\n", rc);
+		goto reg_crq_cleanup;
+	}
+
+	tasklet_init(&ibmvtpm->tasklet, (void *)ibmvtpm_tasklet,
+		     (unsigned long)ibmvtpm);
+
+	rc = request_irq(vio_dev->irq, ibmvtpm_interrupt, 0,
+			 tpm_ibmvtpm_driver_name, ibmvtpm);
+	if (rc) {
+		dev_err(dev, "Error %d register irq 0x%x\n", rc, vio_dev->irq);
+		goto init_irq_cleanup;
+	}
+
+	rc = vio_enable_interrupts(vio_dev);
+	if (rc) {
+		dev_err(dev, "Error %d enabling interrupts\n", rc);
+		goto init_irq_cleanup;
+	}
+
+	crq_q->index = 0;
+
+	ibmvtpm->dev = dev;
+	ibmvtpm->vdev = vio_dev;
+	chip->vendor.data = (void *)ibmvtpm;
+
+	spin_lock_init(&ibmvtpm->lock);
+	spin_lock_init(&ibmvtpm->rtce_lock);
+
+	rc = ibmvtpm_crq_send_init(ibmvtpm);
+	if (rc)
+		goto init_irq_cleanup;
+
+	rc = ibmvtpm_crq_get_version(ibmvtpm);
+	if (rc)
+		goto init_irq_cleanup;
+
+	rc = ibmvtpm_crq_get_rtce_size(ibmvtpm);
+	if (rc)
+		goto init_irq_cleanup;
+
+	return rc;
+init_irq_cleanup:
+	tasklet_kill(&ibmvtpm->tasklet);
+	do {
+		rc1 = plpar_hcall_norets(H_FREE_CRQ, vio_dev->unit_address);
+	} while (rc1 == H_BUSY || H_IS_LONG_BUSY(rc1));
+reg_crq_cleanup:
+	dma_unmap_single(dev, ibmvtpm->crq_dma_handle, CRQ_RES_BUF_SIZE,
+			 DMA_BIDIRECTIONAL);
+cleanup:
+	if (ibmvtpm) {
+		if (crq_q->crq_addr)
+			free_page((unsigned long)crq_q->crq_addr);
+		kfree(ibmvtpm);
+	}
+
+	tpm_remove_hardware(dev);
+
+	return rc;
+}
+
+static struct vio_driver ibmvtpm_driver = {
+	.id_table	 = tpm_ibmvtpm_device_table,
+	.probe		 = tpm_ibmvtpm_probe,
+	.remove		 = tpm_ibmvtpm_remove,
+	.get_desired_dma = tpm_ibmvtpm_get_desired_dma,
+	.name		 = tpm_ibmvtpm_driver_name,
+	.pm		 = &tpm_ibmvtpm_pm_ops,
+};
+
+/**
+ * ibmvtpm_module_init - Initialize ibm vtpm module
+ *
+ * Return value:
+ *	0 -Success
+ *	Non-zero - Failure
+ */
+static int __init ibmvtpm_module_init(void)
+{
+	return vio_register_driver(&ibmvtpm_driver);
+}
+
+/**
+ * ibmvtpm_module_exit - Teardown ibm vtpm module
+ *
+ * Return value:
+ *	Nothing
+ */
+static void __exit ibmvtpm_module_exit(void)
+{
+	vio_unregister_driver(&ibmvtpm_driver);
+}
+
+module_init(ibmvtpm_module_init);
+module_exit(ibmvtpm_module_exit);
+
+MODULE_AUTHOR("adlai@us.ibm.com");
+MODULE_DESCRIPTION("IBM vTPM Driver");
+MODULE_VERSION("1.0");
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/tpm/tpm_ibmvtpm.h b/drivers/char/tpm/tpm_ibmvtpm.h
new file mode 100644
index 00000000000..4296eb4b4d8
--- /dev/null
+++ b/drivers/char/tpm/tpm_ibmvtpm.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2012 IBM Corporation
+ *
+ * Author: Ashley Lai <adlai@us.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Device driver for TCG/TCPA TPM (trusted platform module).
+ * Specifications at www.trustedcomputinggroup.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2 of the
+ * License.
+ *
+ */
+
+#ifndef __TPM_IBMVTPM_H__
+#define __TPM_IBMVTPM_H__
+
+/* vTPM Message Format 1 */
+struct ibmvtpm_crq {
+	u8 valid;
+	u8 msg;
+	u16 len;
+	u32 data;
+	u64 reserved;
+} __attribute__((packed, aligned(8)));
+
+struct ibmvtpm_crq_queue {
+	struct ibmvtpm_crq *crq_addr;
+	u32 index;
+	u32 num_entry;
+};
+
+struct ibmvtpm_dev {
+	struct device *dev;
+	struct vio_dev *vdev;
+	struct ibmvtpm_crq_queue crq_queue;
+	dma_addr_t crq_dma_handle;
+	spinlock_t lock;
+	struct tasklet_struct tasklet;
+	u32 rtce_size;
+	void __iomem *rtce_buf;
+	dma_addr_t rtce_dma_handle;
+	spinlock_t rtce_lock;
+	struct ibmvtpm_crq crq_res;
+	u32 vtpm_version;
+};
+
+#define CRQ_RES_BUF_SIZE	PAGE_SIZE
+
+/* Initialize CRQ */
+#define INIT_CRQ_CMD		0xC001000000000000LL /* Init cmd */
+#define INIT_CRQ_COMP_CMD	0xC002000000000000LL /* Init complete cmd */
+#define INIT_CRQ_RES		0x01	/* Init respond */
+#define INIT_CRQ_COMP_RES	0x02	/* Init complete respond */
+#define VALID_INIT_CRQ		0xC0	/* Valid command for init crq */
+
+/* vTPM CRQ response is the message type | 0x80 */
+#define VTPM_MSG_RES		0x80
+#define IBMVTPM_VALID_CMD	0x80
+
+/* vTPM CRQ message types */
+#define VTPM_GET_VERSION			0x01
+#define VTPM_GET_VERSION_RES			(0x01 | VTPM_MSG_RES)
+
+#define VTPM_TPM_COMMAND			0x02
+#define VTPM_TPM_COMMAND_RES			(0x02 | VTPM_MSG_RES)
+
+#define VTPM_GET_RTCE_BUFFER_SIZE		0x03
+#define VTPM_GET_RTCE_BUFFER_SIZE_RES		(0x03 | VTPM_MSG_RES)
+
+#define VTPM_PREPARE_TO_SUSPEND			0x04
+#define VTPM_PREPARE_TO_SUSPEND_RES		(0x04 | VTPM_MSG_RES)
+
+#endif
diff --git a/drivers/char/tpm/tpm_of.c b/drivers/char/tpm/tpm_of.c
new file mode 100644
index 00000000000..98ba2bd1a35
--- /dev/null
+++ b/drivers/char/tpm/tpm_of.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2012 IBM Corporation
+ *
+ * Author: Ashley Lai <adlai@us.ibm.com>
+ *
+ * Maintained by: <tpmdd-devel@lists.sourceforge.net>
+ *
+ * Read the event log created by the firmware on PPC64
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/slab.h>
+#include <linux/of.h>
+
+#include "tpm.h"
+#include "tpm_eventlog.h"
+
+int read_log(struct tpm_bios_log *log)
+{
+	struct device_node *np;
+	const u32 *sizep;
+	const __be64 *basep;
+
+	if (log->bios_event_log != NULL) {
+		pr_err("%s: ERROR - Eventlog already initialized\n", __func__);
+		return -EFAULT;
+	}
+
+	np = of_find_node_by_name(NULL, "ibm,vtpm");
+	if (!np) {
+		pr_err("%s: ERROR - IBMVTPM not supported\n", __func__);
+		return -ENODEV;
+	}
+
+	sizep = of_get_property(np, "linux,sml-size", NULL);
+	if (sizep == NULL) {
+		pr_err("%s: ERROR - SML size not found\n", __func__);
+		goto cleanup_eio;
+	}
+	if (*sizep == 0) {
+		pr_err("%s: ERROR - event log area empty\n", __func__);
+		goto cleanup_eio;
+	}
+
+	basep = of_get_property(np, "linux,sml-base", NULL);
+	if (basep == NULL) {
+		pr_err(KERN_ERR "%s: ERROR - SML not found\n", __func__);
+		goto cleanup_eio;
+	}
+
+	of_node_put(np);
+	log->bios_event_log = kmalloc(*sizep, GFP_KERNEL);
+	if (!log->bios_event_log) {
+		pr_err("%s: ERROR - Not enough memory for BIOS measurements\n",
+		       __func__);
+		return -ENOMEM;
+	}
+
+	log->bios_event_log_end = log->bios_event_log + *sizep;
+
+	memcpy(log->bios_event_log, __va(be64_to_cpup(basep)), *sizep);
+
+	return 0;
+
+cleanup_eio:
+	of_node_put(np);
+	return -EIO;
+}
diff --git a/drivers/char/tpm/tpm_ppi.c b/drivers/char/tpm/tpm_ppi.c
new file mode 100644
index 00000000000..f27b58cfae9
--- /dev/null
+++ b/drivers/char/tpm/tpm_ppi.c
@@ -0,0 +1,461 @@
+#include <linux/acpi.h>
+#include <acpi/acpi_drivers.h>
+#include "tpm.h"
+
+static const u8 tpm_ppi_uuid[] = {
+	0xA6, 0xFA, 0xDD, 0x3D,
+	0x1B, 0x36,
+	0xB4, 0x4E,
+	0xA4, 0x24,
+	0x8D, 0x10, 0x08, 0x9D, 0x16, 0x53
+};
+static char *tpm_device_name = "TPM";
+
+#define TPM_PPI_REVISION_ID	1
+#define TPM_PPI_FN_VERSION	1
+#define TPM_PPI_FN_SUBREQ	2
+#define TPM_PPI_FN_GETREQ	3
+#define TPM_PPI_FN_GETACT	4
+#define TPM_PPI_FN_GETRSP	5
+#define TPM_PPI_FN_SUBREQ2	7
+#define TPM_PPI_FN_GETOPR	8
+#define PPI_TPM_REQ_MAX		22
+#define PPI_VS_REQ_START	128
+#define PPI_VS_REQ_END		255
+#define PPI_VERSION_LEN		3
+
+static acpi_status ppi_callback(acpi_handle handle, u32 level, void *context,
+				void **return_value)
+{
+	acpi_status status;
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	status = acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer);
+	if (strstr(buffer.pointer, context) != NULL) {
+		*return_value = handle;
+		kfree(buffer.pointer);
+		return AE_CTRL_TERMINATE;
+	}
+	return AE_OK;
+}
+
+static inline void ppi_assign_params(union acpi_object params[4],
+				     u64 function_num)
+{
+	params[0].type = ACPI_TYPE_BUFFER;
+	params[0].buffer.length = sizeof(tpm_ppi_uuid);
+	params[0].buffer.pointer = (char *)tpm_ppi_uuid;
+	params[1].type = ACPI_TYPE_INTEGER;
+	params[1].integer.value = TPM_PPI_REVISION_ID;
+	params[2].type = ACPI_TYPE_INTEGER;
+	params[2].integer.value = function_num;
+	params[3].type = ACPI_TYPE_PACKAGE;
+	params[3].package.count = 0;
+	params[3].package.elements = NULL;
+}
+
+static ssize_t tpm_show_ppi_version(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	union acpi_object *obj;
+
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_VERSION);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					 ACPI_TYPE_STRING);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	obj = (union acpi_object *)output.pointer;
+	status = scnprintf(buf, PAGE_SIZE, "%s\n", obj->string.pointer);
+	kfree(output.pointer);
+	return status;
+}
+
+static ssize_t tpm_show_ppi_request(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	union acpi_object *ret_obj;
+
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_GETREQ);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_PACKAGE);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	/*
+	 * output.pointer should be of package type, including two integers.
+	 * The first is function return code, 0 means success and 1 means
+	 * error. The second is pending TPM operation requested by the OS, 0
+	 * means none and >0 means operation value.
+	 */
+	ret_obj = ((union acpi_object *)output.pointer)->package.elements;
+	if (ret_obj->type == ACPI_TYPE_INTEGER) {
+		if (ret_obj->integer.value) {
+			status = -EFAULT;
+			goto cleanup;
+		}
+		ret_obj++;
+		if (ret_obj->type == ACPI_TYPE_INTEGER)
+			status = scnprintf(buf, PAGE_SIZE, "%llu\n",
+					   ret_obj->integer.value);
+		else
+			status = -EINVAL;
+	} else {
+		status = -EINVAL;
+	}
+cleanup:
+	kfree(output.pointer);
+	return status;
+}
+
+static ssize_t tpm_store_ppi_request(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	char version[PPI_VERSION_LEN + 1];
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	union acpi_object obj;
+	u32 req;
+	u64 ret;
+
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_VERSION);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_STRING);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	strncpy(version,
+		((union acpi_object *)output.pointer)->string.pointer,
+		PPI_VERSION_LEN);
+	kfree(output.pointer);
+	output.length = ACPI_ALLOCATE_BUFFER;
+	output.pointer = NULL;
+	/*
+	 * the function to submit TPM operation request to pre-os environment
+	 * is updated with function index from SUBREQ to SUBREQ2 since PPI
+	 * version 1.1
+	 */
+	if (strcmp(version, "1.1") == -1)
+		params[2].integer.value = TPM_PPI_FN_SUBREQ;
+	else
+		params[2].integer.value = TPM_PPI_FN_SUBREQ2;
+	/*
+	 * PPI spec defines params[3].type as ACPI_TYPE_PACKAGE. Some BIOS
+	 * accept buffer/string/integer type, but some BIOS accept buffer/
+	 * string/package type. For PPI version 1.0 and 1.1, use buffer type
+	 * for compatibility, and use package type since 1.2 according to spec.
+	 */
+	if (strcmp(version, "1.2") == -1) {
+		params[3].type = ACPI_TYPE_BUFFER;
+		params[3].buffer.length = sizeof(req);
+		sscanf(buf, "%d", &req);
+		params[3].buffer.pointer = (char *)&req;
+	} else {
+		params[3].package.count = 1;
+		obj.type = ACPI_TYPE_INTEGER;
+		sscanf(buf, "%llu", &obj.integer.value);
+		params[3].package.elements = &obj;
+	}
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_INTEGER);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	ret = ((union acpi_object *)output.pointer)->integer.value;
+	if (ret == 0)
+		status = (acpi_status)count;
+	else if (ret == 1)
+		status = -EPERM;
+	else
+		status = -EFAULT;
+	kfree(output.pointer);
+	return status;
+}
+
+static ssize_t tpm_show_ppi_transition_action(struct device *dev,
+					      struct device_attribute *attr,
+					      char *buf)
+{
+	char version[PPI_VERSION_LEN + 1];
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	u32 ret;
+	char *info[] = {
+		"None",
+		"Shutdown",
+		"Reboot",
+		"OS Vendor-specific",
+		"Error",
+	};
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_VERSION);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_STRING);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	strncpy(version,
+		((union acpi_object *)output.pointer)->string.pointer,
+		PPI_VERSION_LEN);
+	/*
+	 * PPI spec defines params[3].type as empty package, but some platforms
+	 * (e.g. Capella with PPI 1.0) need integer/string/buffer type, so for
+	 * compatibility, define params[3].type as buffer, if PPI version < 1.2
+	 */
+	if (strcmp(version, "1.2") == -1) {
+		params[3].type = ACPI_TYPE_BUFFER;
+		params[3].buffer.length =  0;
+		params[3].buffer.pointer = NULL;
+	}
+	params[2].integer.value = TPM_PPI_FN_GETACT;
+	kfree(output.pointer);
+	output.length = ACPI_ALLOCATE_BUFFER;
+	output.pointer = NULL;
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_INTEGER);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	ret = ((union acpi_object *)output.pointer)->integer.value;
+	if (ret < ARRAY_SIZE(info) - 1)
+		status = scnprintf(buf, PAGE_SIZE, "%d: %s\n", ret, info[ret]);
+	else
+		status = scnprintf(buf, PAGE_SIZE, "%d: %s\n", ret,
+				   info[ARRAY_SIZE(info)-1]);
+	kfree(output.pointer);
+	return status;
+}
+
+static ssize_t tpm_show_ppi_response(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	union acpi_object *ret_obj;
+	u64 req;
+
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_GETRSP);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					    ACPI_TYPE_PACKAGE);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+	/*
+	 * parameter output.pointer should be of package type, including
+	 * 3 integers. The first means function return code, the second means
+	 * most recent TPM operation request, and the last means response to
+	 * the most recent TPM operation request. Only if the first is 0, and
+	 * the second integer is not 0, the response makes sense.
+	 */
+	ret_obj = ((union acpi_object *)output.pointer)->package.elements;
+	if (ret_obj->type != ACPI_TYPE_INTEGER) {
+		status = -EINVAL;
+		goto cleanup;
+	}
+	if (ret_obj->integer.value) {
+		status = -EFAULT;
+		goto cleanup;
+	}
+	ret_obj++;
+	if (ret_obj->type != ACPI_TYPE_INTEGER) {
+		status = -EINVAL;
+		goto cleanup;
+	}
+	if (ret_obj->integer.value) {
+		req = ret_obj->integer.value;
+		ret_obj++;
+		if (ret_obj->type != ACPI_TYPE_INTEGER) {
+			status = -EINVAL;
+			goto cleanup;
+		}
+		if (ret_obj->integer.value == 0)
+			status = scnprintf(buf, PAGE_SIZE, "%llu %s\n", req,
+					   "0: Success");
+		else if (ret_obj->integer.value == 0xFFFFFFF0)
+			status = scnprintf(buf, PAGE_SIZE, "%llu %s\n", req,
+					   "0xFFFFFFF0: User Abort");
+		else if (ret_obj->integer.value == 0xFFFFFFF1)
+			status = scnprintf(buf, PAGE_SIZE, "%llu %s\n", req,
+					   "0xFFFFFFF1: BIOS Failure");
+		else if (ret_obj->integer.value >= 1 &&
+			 ret_obj->integer.value <= 0x00000FFF)
+			status = scnprintf(buf, PAGE_SIZE, "%llu %llu: %s\n",
+					   req, ret_obj->integer.value,
+					   "Corresponding TPM error");
+		else
+			status = scnprintf(buf, PAGE_SIZE, "%llu %llu: %s\n",
+					   req, ret_obj->integer.value,
+					   "Error");
+	} else {
+		status = scnprintf(buf, PAGE_SIZE, "%llu: %s\n",
+				   ret_obj->integer.value, "No Recent Request");
+	}
+cleanup:
+	kfree(output.pointer);
+	return status;
+}
+
+static ssize_t show_ppi_operations(char *buf, u32 start, u32 end)
+{
+	char *str = buf;
+	char version[PPI_VERSION_LEN];
+	acpi_handle handle;
+	acpi_status status;
+	struct acpi_object_list input;
+	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object params[4];
+	union acpi_object obj;
+	int i;
+	u32 ret;
+	char *info[] = {
+		"Not implemented",
+		"BIOS only",
+		"Blocked for OS by BIOS",
+		"User required",
+		"User not required",
+	};
+	input.count = 4;
+	ppi_assign_params(params, TPM_PPI_FN_VERSION);
+	input.pointer = params;
+	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
+				     ACPI_UINT32_MAX, ppi_callback, NULL,
+				     tpm_device_name, &handle);
+	if (ACPI_FAILURE(status))
+		return -ENXIO;
+
+	status = acpi_evaluate_object_typed(handle, "_DSM", &input, &output,
+					 ACPI_TYPE_STRING);
+	if (ACPI_FAILURE(status))
+		return -ENOMEM;
+
+	strncpy(version,
+		((union acpi_object *)output.pointer)->string.pointer,
+		PPI_VERSION_LEN);
+	kfree(output.pointer);
+	output.length = ACPI_ALLOCATE_BUFFER;
+	output.pointer = NULL;
+	if (strcmp(version, "1.2") == -1)
+		return -EPERM;
+
+	params[2].integer.value = TPM_PPI_FN_GETOPR;
+	params[3].package.count = 1;
+	obj.type = ACPI_TYPE_INTEGER;
+	params[3].package.elements = &obj;
+	for (i = start; i <= end; i++) {
+		obj.integer.value = i;
+		status = acpi_evaluate_object_typed(handle, "_DSM",
+			 &input, &output, ACPI_TYPE_INTEGER);
+		if (ACPI_FAILURE(status))
+			return -ENOMEM;
+
+		ret = ((union acpi_object *)output.pointer)->integer.value;
+		if (ret > 0 && ret < ARRAY_SIZE(info))
+			str += scnprintf(str, PAGE_SIZE, "%d %d: %s\n",
+					 i, ret, info[ret]);
+		kfree(output.pointer);
+		output.length = ACPI_ALLOCATE_BUFFER;
+		output.pointer = NULL;
+	}
+	return str - buf;
+}
+
+static ssize_t tpm_show_ppi_tcg_operations(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	return show_ppi_operations(buf, 0, PPI_TPM_REQ_MAX);
+}
+
+static ssize_t tpm_show_ppi_vs_operations(struct device *dev,
+					  struct device_attribute *attr,
+					  char *buf)
+{
+	return show_ppi_operations(buf, PPI_VS_REQ_START, PPI_VS_REQ_END);
+}
+
+static DEVICE_ATTR(version, S_IRUGO, tpm_show_ppi_version, NULL);
+static DEVICE_ATTR(request, S_IRUGO | S_IWUSR | S_IWGRP,
+		   tpm_show_ppi_request, tpm_store_ppi_request);
+static DEVICE_ATTR(transition_action, S_IRUGO,
+		   tpm_show_ppi_transition_action, NULL);
+static DEVICE_ATTR(response, S_IRUGO, tpm_show_ppi_response, NULL);
+static DEVICE_ATTR(tcg_operations, S_IRUGO, tpm_show_ppi_tcg_operations, NULL);
+static DEVICE_ATTR(vs_operations, S_IRUGO, tpm_show_ppi_vs_operations, NULL);
+
+static struct attribute *ppi_attrs[] = {
+	&dev_attr_version.attr,
+	&dev_attr_request.attr,
+	&dev_attr_transition_action.attr,
+	&dev_attr_response.attr,
+	&dev_attr_tcg_operations.attr,
+	&dev_attr_vs_operations.attr, NULL,
+};
+static struct attribute_group ppi_attr_grp = {
+	.attrs = ppi_attrs
+};
+
+ssize_t sys_add_ppi(struct kobject *parent)
+{
+	struct kobject *ppi;
+	ppi = kobject_create_and_add("ppi", parent);
+	if (sysfs_create_group(ppi, &ppi_attr_grp))
+		return -EFAULT;
+	else
+		return 0;
+}
+EXPORT_SYMBOL_GPL(sys_add_ppi);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c
index c4be3519a58..6bdf2671254 100644
--- a/drivers/char/tpm/tpm_tis.c
+++ b/drivers/char/tpm/tpm_tis.c
@@ -705,6 +705,7 @@ out_err:
 	return rc;
 }
 
+#if defined(CONFIG_PNP) || defined(CONFIG_PM_SLEEP)
 static void tpm_tis_reenable_interrupts(struct tpm_chip *chip)
 {
 	u32 intmask;
@@ -725,7 +726,7 @@ static void tpm_tis_reenable_interrupts(struct tpm_chip *chip)
 	iowrite32(intmask,
 		  chip->vendor.iobase + TPM_INT_ENABLE(chip->vendor.locality));
 }
-
+#endif
 
 #ifdef CONFIG_PNP
 static int __devinit tpm_tis_pnp_init(struct pnp_dev *pnp_dev,
diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c
index 98b06baafcc..a5f7829f279 100644
--- a/drivers/clocksource/sh_cmt.c
+++ b/drivers/clocksource/sh_cmt.c
@@ -33,6 +33,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 
 struct sh_cmt_priv {
 	void __iomem *mapbase;
@@ -52,6 +53,7 @@ struct sh_cmt_priv {
 	struct clock_event_device ced;
 	struct clocksource cs;
 	unsigned long total_cycles;
+	bool cs_enabled;
 };
 
 static DEFINE_RAW_SPINLOCK(sh_cmt_lock);
@@ -155,6 +157,9 @@ static int sh_cmt_enable(struct sh_cmt_priv *p, unsigned long *rate)
 {
 	int k, ret;
 
+	pm_runtime_get_sync(&p->pdev->dev);
+	dev_pm_syscore_device(&p->pdev->dev, true);
+
 	/* enable clock */
 	ret = clk_enable(p->clk);
 	if (ret) {
@@ -221,6 +226,9 @@ static void sh_cmt_disable(struct sh_cmt_priv *p)
 
 	/* stop clock */
 	clk_disable(p->clk);
+
+	dev_pm_syscore_device(&p->pdev->dev, false);
+	pm_runtime_put(&p->pdev->dev);
 }
 
 /* private flags */
@@ -451,22 +459,42 @@ static int sh_cmt_clocksource_enable(struct clocksource *cs)
 	int ret;
 	struct sh_cmt_priv *p = cs_to_sh_cmt(cs);
 
+	WARN_ON(p->cs_enabled);
+
 	p->total_cycles = 0;
 
 	ret = sh_cmt_start(p, FLAG_CLOCKSOURCE);
-	if (!ret)
+	if (!ret) {
 		__clocksource_updatefreq_hz(cs, p->rate);
+		p->cs_enabled = true;
+	}
 	return ret;
 }
 
 static void sh_cmt_clocksource_disable(struct clocksource *cs)
 {
-	sh_cmt_stop(cs_to_sh_cmt(cs), FLAG_CLOCKSOURCE);
+	struct sh_cmt_priv *p = cs_to_sh_cmt(cs);
+
+	WARN_ON(!p->cs_enabled);
+
+	sh_cmt_stop(p, FLAG_CLOCKSOURCE);
+	p->cs_enabled = false;
+}
+
+static void sh_cmt_clocksource_suspend(struct clocksource *cs)
+{
+	struct sh_cmt_priv *p = cs_to_sh_cmt(cs);
+
+	sh_cmt_stop(p, FLAG_CLOCKSOURCE);
+	pm_genpd_syscore_poweroff(&p->pdev->dev);
 }
 
 static void sh_cmt_clocksource_resume(struct clocksource *cs)
 {
-	sh_cmt_start(cs_to_sh_cmt(cs), FLAG_CLOCKSOURCE);
+	struct sh_cmt_priv *p = cs_to_sh_cmt(cs);
+
+	pm_genpd_syscore_poweron(&p->pdev->dev);
+	sh_cmt_start(p, FLAG_CLOCKSOURCE);
 }
 
 static int sh_cmt_register_clocksource(struct sh_cmt_priv *p,
@@ -479,7 +507,7 @@ static int sh_cmt_register_clocksource(struct sh_cmt_priv *p,
 	cs->read = sh_cmt_clocksource_read;
 	cs->enable = sh_cmt_clocksource_enable;
 	cs->disable = sh_cmt_clocksource_disable;
-	cs->suspend = sh_cmt_clocksource_disable;
+	cs->suspend = sh_cmt_clocksource_suspend;
 	cs->resume = sh_cmt_clocksource_resume;
 	cs->mask = CLOCKSOURCE_MASK(sizeof(unsigned long) * 8);
 	cs->flags = CLOCK_SOURCE_IS_CONTINUOUS;
@@ -562,6 +590,16 @@ static int sh_cmt_clock_event_next(unsigned long delta,
 	return 0;
 }
 
+static void sh_cmt_clock_event_suspend(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweroff(&ced_to_sh_cmt(ced)->pdev->dev);
+}
+
+static void sh_cmt_clock_event_resume(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweron(&ced_to_sh_cmt(ced)->pdev->dev);
+}
+
 static void sh_cmt_register_clockevent(struct sh_cmt_priv *p,
 				       char *name, unsigned long rating)
 {
@@ -576,6 +614,8 @@ static void sh_cmt_register_clockevent(struct sh_cmt_priv *p,
 	ced->cpumask = cpumask_of(0);
 	ced->set_next_event = sh_cmt_clock_event_next;
 	ced->set_mode = sh_cmt_clock_event_mode;
+	ced->suspend = sh_cmt_clock_event_suspend;
+	ced->resume = sh_cmt_clock_event_resume;
 
 	dev_info(&p->pdev->dev, "used for clock events\n");
 	clockevents_register_device(ced);
@@ -670,6 +710,7 @@ static int sh_cmt_setup(struct sh_cmt_priv *p, struct platform_device *pdev)
 		dev_err(&p->pdev->dev, "registration failed\n");
 		goto err1;
 	}
+	p->cs_enabled = false;
 
 	ret = setup_irq(irq, &p->irqaction);
 	if (ret) {
@@ -688,14 +729,17 @@ err0:
 static int __devinit sh_cmt_probe(struct platform_device *pdev)
 {
 	struct sh_cmt_priv *p = platform_get_drvdata(pdev);
+	struct sh_timer_config *cfg = pdev->dev.platform_data;
 	int ret;
 
-	if (!is_early_platform_device(pdev))
-		pm_genpd_dev_always_on(&pdev->dev, true);
+	if (!is_early_platform_device(pdev)) {
+		pm_runtime_set_active(&pdev->dev);
+		pm_runtime_enable(&pdev->dev);
+	}
 
 	if (p) {
 		dev_info(&pdev->dev, "kept as earlytimer\n");
-		return 0;
+		goto out;
 	}
 
 	p = kmalloc(sizeof(*p), GFP_KERNEL);
@@ -708,8 +752,19 @@ static int __devinit sh_cmt_probe(struct platform_device *pdev)
 	if (ret) {
 		kfree(p);
 		platform_set_drvdata(pdev, NULL);
+		pm_runtime_idle(&pdev->dev);
+		return ret;
 	}
-	return ret;
+	if (is_early_platform_device(pdev))
+		return 0;
+
+ out:
+	if (cfg->clockevent_rating || cfg->clocksource_rating)
+		pm_runtime_irq_safe(&pdev->dev);
+	else
+		pm_runtime_idle(&pdev->dev);
+
+	return 0;
 }
 
 static int __devexit sh_cmt_remove(struct platform_device *pdev)
diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c
index d9b76ca64a6..c5eea858054 100644
--- a/drivers/clocksource/sh_mtu2.c
+++ b/drivers/clocksource/sh_mtu2.c
@@ -32,6 +32,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 
 struct sh_mtu2_priv {
 	void __iomem *mapbase;
@@ -123,6 +124,9 @@ static int sh_mtu2_enable(struct sh_mtu2_priv *p)
 {
 	int ret;
 
+	pm_runtime_get_sync(&p->pdev->dev);
+	dev_pm_syscore_device(&p->pdev->dev, true);
+
 	/* enable clock */
 	ret = clk_enable(p->clk);
 	if (ret) {
@@ -157,6 +161,9 @@ static void sh_mtu2_disable(struct sh_mtu2_priv *p)
 
 	/* stop clock */
 	clk_disable(p->clk);
+
+	dev_pm_syscore_device(&p->pdev->dev, false);
+	pm_runtime_put(&p->pdev->dev);
 }
 
 static irqreturn_t sh_mtu2_interrupt(int irq, void *dev_id)
@@ -208,6 +215,16 @@ static void sh_mtu2_clock_event_mode(enum clock_event_mode mode,
 	}
 }
 
+static void sh_mtu2_clock_event_suspend(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweroff(&ced_to_sh_mtu2(ced)->pdev->dev);
+}
+
+static void sh_mtu2_clock_event_resume(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweron(&ced_to_sh_mtu2(ced)->pdev->dev);
+}
+
 static void sh_mtu2_register_clockevent(struct sh_mtu2_priv *p,
 				       char *name, unsigned long rating)
 {
@@ -221,6 +238,8 @@ static void sh_mtu2_register_clockevent(struct sh_mtu2_priv *p,
 	ced->rating = rating;
 	ced->cpumask = cpumask_of(0);
 	ced->set_mode = sh_mtu2_clock_event_mode;
+	ced->suspend = sh_mtu2_clock_event_suspend;
+	ced->resume = sh_mtu2_clock_event_resume;
 
 	dev_info(&p->pdev->dev, "used for clock events\n");
 	clockevents_register_device(ced);
@@ -305,14 +324,17 @@ static int sh_mtu2_setup(struct sh_mtu2_priv *p, struct platform_device *pdev)
 static int __devinit sh_mtu2_probe(struct platform_device *pdev)
 {
 	struct sh_mtu2_priv *p = platform_get_drvdata(pdev);
+	struct sh_timer_config *cfg = pdev->dev.platform_data;
 	int ret;
 
-	if (!is_early_platform_device(pdev))
-		pm_genpd_dev_always_on(&pdev->dev, true);
+	if (!is_early_platform_device(pdev)) {
+		pm_runtime_set_active(&pdev->dev);
+		pm_runtime_enable(&pdev->dev);
+	}
 
 	if (p) {
 		dev_info(&pdev->dev, "kept as earlytimer\n");
-		return 0;
+		goto out;
 	}
 
 	p = kmalloc(sizeof(*p), GFP_KERNEL);
@@ -325,8 +347,19 @@ static int __devinit sh_mtu2_probe(struct platform_device *pdev)
 	if (ret) {
 		kfree(p);
 		platform_set_drvdata(pdev, NULL);
+		pm_runtime_idle(&pdev->dev);
+		return ret;
 	}
-	return ret;
+	if (is_early_platform_device(pdev))
+		return 0;
+
+ out:
+	if (cfg->clockevent_rating)
+		pm_runtime_irq_safe(&pdev->dev);
+	else
+		pm_runtime_idle(&pdev->dev);
+
+	return 0;
 }
 
 static int __devexit sh_mtu2_remove(struct platform_device *pdev)
diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c
index c1b51d49d10..0cc4add8827 100644
--- a/drivers/clocksource/sh_tmu.c
+++ b/drivers/clocksource/sh_tmu.c
@@ -33,6 +33,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
 
 struct sh_tmu_priv {
 	void __iomem *mapbase;
@@ -43,6 +44,8 @@ struct sh_tmu_priv {
 	unsigned long periodic;
 	struct clock_event_device ced;
 	struct clocksource cs;
+	bool cs_enabled;
+	unsigned int enable_count;
 };
 
 static DEFINE_RAW_SPINLOCK(sh_tmu_lock);
@@ -107,7 +110,7 @@ static void sh_tmu_start_stop_ch(struct sh_tmu_priv *p, int start)
 	raw_spin_unlock_irqrestore(&sh_tmu_lock, flags);
 }
 
-static int sh_tmu_enable(struct sh_tmu_priv *p)
+static int __sh_tmu_enable(struct sh_tmu_priv *p)
 {
 	int ret;
 
@@ -135,7 +138,18 @@ static int sh_tmu_enable(struct sh_tmu_priv *p)
 	return 0;
 }
 
-static void sh_tmu_disable(struct sh_tmu_priv *p)
+static int sh_tmu_enable(struct sh_tmu_priv *p)
+{
+	if (p->enable_count++ > 0)
+		return 0;
+
+	pm_runtime_get_sync(&p->pdev->dev);
+	dev_pm_syscore_device(&p->pdev->dev, true);
+
+	return __sh_tmu_enable(p);
+}
+
+static void __sh_tmu_disable(struct sh_tmu_priv *p)
 {
 	/* disable channel */
 	sh_tmu_start_stop_ch(p, 0);
@@ -147,6 +161,20 @@ static void sh_tmu_disable(struct sh_tmu_priv *p)
 	clk_disable(p->clk);
 }
 
+static void sh_tmu_disable(struct sh_tmu_priv *p)
+{
+	if (WARN_ON(p->enable_count == 0))
+		return;
+
+	if (--p->enable_count > 0)
+		return;
+
+	__sh_tmu_disable(p);
+
+	dev_pm_syscore_device(&p->pdev->dev, false);
+	pm_runtime_put(&p->pdev->dev);
+}
+
 static void sh_tmu_set_next(struct sh_tmu_priv *p, unsigned long delta,
 			    int periodic)
 {
@@ -203,15 +231,53 @@ static int sh_tmu_clocksource_enable(struct clocksource *cs)
 	struct sh_tmu_priv *p = cs_to_sh_tmu(cs);
 	int ret;
 
+	if (WARN_ON(p->cs_enabled))
+		return 0;
+
 	ret = sh_tmu_enable(p);
-	if (!ret)
+	if (!ret) {
 		__clocksource_updatefreq_hz(cs, p->rate);
+		p->cs_enabled = true;
+	}
+
 	return ret;
 }
 
 static void sh_tmu_clocksource_disable(struct clocksource *cs)
 {
-	sh_tmu_disable(cs_to_sh_tmu(cs));
+	struct sh_tmu_priv *p = cs_to_sh_tmu(cs);
+
+	if (WARN_ON(!p->cs_enabled))
+		return;
+
+	sh_tmu_disable(p);
+	p->cs_enabled = false;
+}
+
+static void sh_tmu_clocksource_suspend(struct clocksource *cs)
+{
+	struct sh_tmu_priv *p = cs_to_sh_tmu(cs);
+
+	if (!p->cs_enabled)
+		return;
+
+	if (--p->enable_count == 0) {
+		__sh_tmu_disable(p);
+		pm_genpd_syscore_poweroff(&p->pdev->dev);
+	}
+}
+
+static void sh_tmu_clocksource_resume(struct clocksource *cs)
+{
+	struct sh_tmu_priv *p = cs_to_sh_tmu(cs);
+
+	if (!p->cs_enabled)
+		return;
+
+	if (p->enable_count++ == 0) {
+		pm_genpd_syscore_poweron(&p->pdev->dev);
+		__sh_tmu_enable(p);
+	}
 }
 
 static int sh_tmu_register_clocksource(struct sh_tmu_priv *p,
@@ -224,6 +290,8 @@ static int sh_tmu_register_clocksource(struct sh_tmu_priv *p,
 	cs->read = sh_tmu_clocksource_read;
 	cs->enable = sh_tmu_clocksource_enable;
 	cs->disable = sh_tmu_clocksource_disable;
+	cs->suspend = sh_tmu_clocksource_suspend;
+	cs->resume = sh_tmu_clocksource_resume;
 	cs->mask = CLOCKSOURCE_MASK(32);
 	cs->flags = CLOCK_SOURCE_IS_CONTINUOUS;
 
@@ -301,6 +369,16 @@ static int sh_tmu_clock_event_next(unsigned long delta,
 	return 0;
 }
 
+static void sh_tmu_clock_event_suspend(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweroff(&ced_to_sh_tmu(ced)->pdev->dev);
+}
+
+static void sh_tmu_clock_event_resume(struct clock_event_device *ced)
+{
+	pm_genpd_syscore_poweron(&ced_to_sh_tmu(ced)->pdev->dev);
+}
+
 static void sh_tmu_register_clockevent(struct sh_tmu_priv *p,
 				       char *name, unsigned long rating)
 {
@@ -316,6 +394,8 @@ static void sh_tmu_register_clockevent(struct sh_tmu_priv *p,
 	ced->cpumask = cpumask_of(0);
 	ced->set_next_event = sh_tmu_clock_event_next;
 	ced->set_mode = sh_tmu_clock_event_mode;
+	ced->suspend = sh_tmu_clock_event_suspend;
+	ced->resume = sh_tmu_clock_event_resume;
 
 	dev_info(&p->pdev->dev, "used for clock events\n");
 
@@ -392,6 +472,8 @@ static int sh_tmu_setup(struct sh_tmu_priv *p, struct platform_device *pdev)
 		ret = PTR_ERR(p->clk);
 		goto err1;
 	}
+	p->cs_enabled = false;
+	p->enable_count = 0;
 
 	return sh_tmu_register(p, (char *)dev_name(&p->pdev->dev),
 			       cfg->clockevent_rating,
@@ -405,14 +487,17 @@ static int sh_tmu_setup(struct sh_tmu_priv *p, struct platform_device *pdev)
 static int __devinit sh_tmu_probe(struct platform_device *pdev)
 {
 	struct sh_tmu_priv *p = platform_get_drvdata(pdev);
+	struct sh_timer_config *cfg = pdev->dev.platform_data;
 	int ret;
 
-	if (!is_early_platform_device(pdev))
-		pm_genpd_dev_always_on(&pdev->dev, true);
+	if (!is_early_platform_device(pdev)) {
+		pm_runtime_set_active(&pdev->dev);
+		pm_runtime_enable(&pdev->dev);
+	}
 
 	if (p) {
 		dev_info(&pdev->dev, "kept as earlytimer\n");
-		return 0;
+		goto out;
 	}
 
 	p = kmalloc(sizeof(*p), GFP_KERNEL);
@@ -425,8 +510,19 @@ static int __devinit sh_tmu_probe(struct platform_device *pdev)
 	if (ret) {
 		kfree(p);
 		platform_set_drvdata(pdev, NULL);
+		pm_runtime_idle(&pdev->dev);
+		return ret;
 	}
-	return ret;
+	if (is_early_platform_device(pdev))
+		return 0;
+
+ out:
+	if (cfg->clockevent_rating || cfg->clocksource_rating)
+		pm_runtime_irq_safe(&pdev->dev);
+	else
+		pm_runtime_idle(&pdev->dev);
+
+	return 0;
 }
 
 static int __devexit sh_tmu_remove(struct platform_device *pdev)
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index e24a2a1b666..ea512f47b78 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -179,6 +179,17 @@ config CPU_FREQ_GOV_CONSERVATIVE
 
 	  If in doubt, say N.
 
+config GENERIC_CPUFREQ_CPU0
+	bool "Generic CPU0 cpufreq driver"
+	depends on HAVE_CLK && REGULATOR && PM_OPP && OF
+	select CPU_FREQ_TABLE
+	help
+	  This adds a generic cpufreq driver for CPU0 frequency management.
+	  It supports both uniprocessor (UP) and symmetric multiprocessor (SMP)
+	  systems which share clock and voltage across all CPUs.
+
+	  If in doubt, say N.
+
 menu "x86 CPU frequency scaling drivers"
 depends on X86
 source "drivers/cpufreq/Kconfig.x86"
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
index 78ff7ee4895..934854ae5eb 100644
--- a/drivers/cpufreq/Kconfig.x86
+++ b/drivers/cpufreq/Kconfig.x86
@@ -23,7 +23,8 @@ config X86_ACPI_CPUFREQ
 	help
 	  This driver adds a CPUFreq driver which utilizes the ACPI
 	  Processor Performance States.
-	  This driver also supports Intel Enhanced Speedstep.
+	  This driver also supports Intel Enhanced Speedstep and newer
+	  AMD CPUs.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called acpi-cpufreq.
@@ -32,6 +33,18 @@ config X86_ACPI_CPUFREQ
 
 	  If in doubt, say N.
 
+config X86_ACPI_CPUFREQ_CPB
+	default y
+	bool "Legacy cpb sysfs knob support for AMD CPUs"
+	depends on X86_ACPI_CPUFREQ && CPU_SUP_AMD
+	help
+	  The powernow-k8 driver used to provide a sysfs knob called "cpb"
+	  to disable the Core Performance Boosting feature of AMD CPUs. This
+	  file has now been superseeded by the more generic "boost" entry.
+
+	  By enabling this option the acpi_cpufreq driver provides the old
+	  entry in addition to the new boost ones, for compatibility reasons.
+
 config ELAN_CPUFREQ
 	tristate "AMD Elan SC400 and SC410"
 	select CPU_FREQ_TABLE
@@ -95,7 +108,8 @@ config X86_POWERNOW_K8
 	select CPU_FREQ_TABLE
 	depends on ACPI && ACPI_PROCESSOR
 	help
-	  This adds the CPUFreq driver for K8/K10 Opteron/Athlon64 processors.
+	  This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors.
+	  Support for K10 and newer processors is now in acpi-cpufreq.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called powernow-k8.
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 9531fc2eda2..1bc90e1306d 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -13,13 +13,15 @@ obj-$(CONFIG_CPU_FREQ_GOV_CONSERVATIVE)	+= cpufreq_conservative.o
 # CPUfreq cross-arch helpers
 obj-$(CONFIG_CPU_FREQ_TABLE)		+= freq_table.o
 
+obj-$(CONFIG_GENERIC_CPUFREQ_CPU0)	+= cpufreq-cpu0.o
+
 ##################################################################################
 # x86 drivers.
 # Link order matters. K8 is preferred to ACPI because of firmware bugs in early
 # K8 systems. ACPI is preferred to all other hardware-specific drivers.
 # speedstep-* is preferred over p4-clockmod.
 
-obj-$(CONFIG_X86_POWERNOW_K8)		+= powernow-k8.o mperf.o
+obj-$(CONFIG_X86_POWERNOW_K8)		+= powernow-k8.o
 obj-$(CONFIG_X86_ACPI_CPUFREQ)		+= acpi-cpufreq.o mperf.o
 obj-$(CONFIG_X86_PCC_CPUFREQ)		+= pcc-cpufreq.o
 obj-$(CONFIG_X86_POWERNOW_K6)		+= powernow-k6.o
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 56c6c6b4eb4..0d048f6a2b2 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -51,13 +51,19 @@ MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski");
 MODULE_DESCRIPTION("ACPI Processor P-States Driver");
 MODULE_LICENSE("GPL");
 
+#define PFX "acpi-cpufreq: "
+
 enum {
 	UNDEFINED_CAPABLE = 0,
 	SYSTEM_INTEL_MSR_CAPABLE,
+	SYSTEM_AMD_MSR_CAPABLE,
 	SYSTEM_IO_CAPABLE,
 };
 
 #define INTEL_MSR_RANGE		(0xffff)
+#define AMD_MSR_RANGE		(0x7)
+
+#define MSR_K7_HWCR_CPB_DIS	(1ULL << 25)
 
 struct acpi_cpufreq_data {
 	struct acpi_processor_performance *acpi_data;
@@ -74,6 +80,116 @@ static struct acpi_processor_performance __percpu *acpi_perf_data;
 static struct cpufreq_driver acpi_cpufreq_driver;
 
 static unsigned int acpi_pstate_strict;
+static bool boost_enabled, boost_supported;
+static struct msr __percpu *msrs;
+
+static bool boost_state(unsigned int cpu)
+{
+	u32 lo, hi;
+	u64 msr;
+
+	switch (boot_cpu_data.x86_vendor) {
+	case X86_VENDOR_INTEL:
+		rdmsr_on_cpu(cpu, MSR_IA32_MISC_ENABLE, &lo, &hi);
+		msr = lo | ((u64)hi << 32);
+		return !(msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE);
+	case X86_VENDOR_AMD:
+		rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
+		msr = lo | ((u64)hi << 32);
+		return !(msr & MSR_K7_HWCR_CPB_DIS);
+	}
+	return false;
+}
+
+static void boost_set_msrs(bool enable, const struct cpumask *cpumask)
+{
+	u32 cpu;
+	u32 msr_addr;
+	u64 msr_mask;
+
+	switch (boot_cpu_data.x86_vendor) {
+	case X86_VENDOR_INTEL:
+		msr_addr = MSR_IA32_MISC_ENABLE;
+		msr_mask = MSR_IA32_MISC_ENABLE_TURBO_DISABLE;
+		break;
+	case X86_VENDOR_AMD:
+		msr_addr = MSR_K7_HWCR;
+		msr_mask = MSR_K7_HWCR_CPB_DIS;
+		break;
+	default:
+		return;
+	}
+
+	rdmsr_on_cpus(cpumask, msr_addr, msrs);
+
+	for_each_cpu(cpu, cpumask) {
+		struct msr *reg = per_cpu_ptr(msrs, cpu);
+		if (enable)
+			reg->q &= ~msr_mask;
+		else
+			reg->q |= msr_mask;
+	}
+
+	wrmsr_on_cpus(cpumask, msr_addr, msrs);
+}
+
+static ssize_t _store_boost(const char *buf, size_t count)
+{
+	int ret;
+	unsigned long val = 0;
+
+	if (!boost_supported)
+		return -EINVAL;
+
+	ret = kstrtoul(buf, 10, &val);
+	if (ret || (val > 1))
+		return -EINVAL;
+
+	if ((val && boost_enabled) || (!val && !boost_enabled))
+		return count;
+
+	get_online_cpus();
+
+	boost_set_msrs(val, cpu_online_mask);
+
+	put_online_cpus();
+
+	boost_enabled = val;
+	pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis");
+
+	return count;
+}
+
+static ssize_t store_global_boost(struct kobject *kobj, struct attribute *attr,
+				  const char *buf, size_t count)
+{
+	return _store_boost(buf, count);
+}
+
+static ssize_t show_global_boost(struct kobject *kobj,
+				 struct attribute *attr, char *buf)
+{
+	return sprintf(buf, "%u\n", boost_enabled);
+}
+
+static struct global_attr global_boost = __ATTR(boost, 0644,
+						show_global_boost,
+						store_global_boost);
+
+#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
+static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
+			 size_t count)
+{
+	return _store_boost(buf, count);
+}
+
+static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)
+{
+	return sprintf(buf, "%u\n", boost_enabled);
+}
+
+static struct freq_attr cpb = __ATTR(cpb, 0644, show_cpb, store_cpb);
+#endif
 
 static int check_est_cpu(unsigned int cpuid)
 {
@@ -82,6 +198,13 @@ static int check_est_cpu(unsigned int cpuid)
 	return cpu_has(cpu, X86_FEATURE_EST);
 }
 
+static int check_amd_hwpstate_cpu(unsigned int cpuid)
+{
+	struct cpuinfo_x86 *cpu = &cpu_data(cpuid);
+
+	return cpu_has(cpu, X86_FEATURE_HW_PSTATE);
+}
+
 static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data)
 {
 	struct acpi_processor_performance *perf;
@@ -101,7 +224,11 @@ static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data)
 	int i;
 	struct acpi_processor_performance *perf;
 
-	msr &= INTEL_MSR_RANGE;
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+		msr &= AMD_MSR_RANGE;
+	else
+		msr &= INTEL_MSR_RANGE;
+
 	perf = data->acpi_data;
 
 	for (i = 0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
@@ -115,6 +242,7 @@ static unsigned extract_freq(u32 val, struct acpi_cpufreq_data *data)
 {
 	switch (data->cpu_feature) {
 	case SYSTEM_INTEL_MSR_CAPABLE:
+	case SYSTEM_AMD_MSR_CAPABLE:
 		return extract_msr(val, data);
 	case SYSTEM_IO_CAPABLE:
 		return extract_io(val, data);
@@ -150,6 +278,7 @@ static void do_drv_read(void *_cmd)
 
 	switch (cmd->type) {
 	case SYSTEM_INTEL_MSR_CAPABLE:
+	case SYSTEM_AMD_MSR_CAPABLE:
 		rdmsr(cmd->addr.msr.reg, cmd->val, h);
 		break;
 	case SYSTEM_IO_CAPABLE:
@@ -174,6 +303,9 @@ static void do_drv_write(void *_cmd)
 		lo = (lo & ~INTEL_MSR_RANGE) | (cmd->val & INTEL_MSR_RANGE);
 		wrmsr(cmd->addr.msr.reg, lo, hi);
 		break;
+	case SYSTEM_AMD_MSR_CAPABLE:
+		wrmsr(cmd->addr.msr.reg, cmd->val, 0);
+		break;
 	case SYSTEM_IO_CAPABLE:
 		acpi_os_write_port((acpi_io_address)cmd->addr.io.port,
 				cmd->val,
@@ -217,6 +349,10 @@ static u32 get_cur_val(const struct cpumask *mask)
 		cmd.type = SYSTEM_INTEL_MSR_CAPABLE;
 		cmd.addr.msr.reg = MSR_IA32_PERF_STATUS;
 		break;
+	case SYSTEM_AMD_MSR_CAPABLE:
+		cmd.type = SYSTEM_AMD_MSR_CAPABLE;
+		cmd.addr.msr.reg = MSR_AMD_PERF_STATUS;
+		break;
 	case SYSTEM_IO_CAPABLE:
 		cmd.type = SYSTEM_IO_CAPABLE;
 		perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data;
@@ -326,6 +462,11 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy,
 		cmd.addr.msr.reg = MSR_IA32_PERF_CTL;
 		cmd.val = (u32) perf->states[next_perf_state].control;
 		break;
+	case SYSTEM_AMD_MSR_CAPABLE:
+		cmd.type = SYSTEM_AMD_MSR_CAPABLE;
+		cmd.addr.msr.reg = MSR_AMD_PERF_CTL;
+		cmd.val = (u32) perf->states[next_perf_state].control;
+		break;
 	case SYSTEM_IO_CAPABLE:
 		cmd.type = SYSTEM_IO_CAPABLE;
 		cmd.addr.io.port = perf->control_register.address;
@@ -419,6 +560,44 @@ static void free_acpi_perf_data(void)
 	free_percpu(acpi_perf_data);
 }
 
+static int boost_notify(struct notifier_block *nb, unsigned long action,
+		      void *hcpu)
+{
+	unsigned cpu = (long)hcpu;
+	const struct cpumask *cpumask;
+
+	cpumask = get_cpu_mask(cpu);
+
+	/*
+	 * Clear the boost-disable bit on the CPU_DOWN path so that
+	 * this cpu cannot block the remaining ones from boosting. On
+	 * the CPU_UP path we simply keep the boost-disable flag in
+	 * sync with the current global state.
+	 */
+
+	switch (action) {
+	case CPU_UP_PREPARE:
+	case CPU_UP_PREPARE_FROZEN:
+		boost_set_msrs(boost_enabled, cpumask);
+		break;
+
+	case CPU_DOWN_PREPARE:
+	case CPU_DOWN_PREPARE_FROZEN:
+		boost_set_msrs(1, cpumask);
+		break;
+
+	default:
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+
+static struct notifier_block boost_nb = {
+	.notifier_call          = boost_notify,
+};
+
 /*
  * acpi_cpufreq_early_init - initialize ACPI P-States library
  *
@@ -559,6 +738,14 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;
 		cpumask_copy(policy->cpus, cpu_core_mask(cpu));
 	}
+
+	if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) {
+		cpumask_clear(policy->cpus);
+		cpumask_set_cpu(cpu, policy->cpus);
+		cpumask_copy(policy->related_cpus, cpu_sibling_mask(cpu));
+		policy->shared_type = CPUFREQ_SHARED_TYPE_HW;
+		pr_info_once(PFX "overriding BIOS provided _PSD data\n");
+	}
 #endif
 
 	/* capability check */
@@ -580,12 +767,16 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
 		break;
 	case ACPI_ADR_SPACE_FIXED_HARDWARE:
 		pr_debug("HARDWARE addr space\n");
-		if (!check_est_cpu(cpu)) {
-			result = -ENODEV;
-			goto err_unreg;
+		if (check_est_cpu(cpu)) {
+			data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
+			break;
 		}
-		data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
-		break;
+		if (check_amd_hwpstate_cpu(cpu)) {
+			data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE;
+			break;
+		}
+		result = -ENODEV;
+		goto err_unreg;
 	default:
 		pr_debug("Unknown addr space %d\n",
 			(u32) (perf->control_register.space_id));
@@ -718,6 +909,7 @@ static int acpi_cpufreq_resume(struct cpufreq_policy *policy)
 
 static struct freq_attr *acpi_cpufreq_attr[] = {
 	&cpufreq_freq_attr_scaling_available_freqs,
+	NULL,	/* this is a placeholder for cpb, do not remove */
 	NULL,
 };
 
@@ -733,6 +925,49 @@ static struct cpufreq_driver acpi_cpufreq_driver = {
 	.attr		= acpi_cpufreq_attr,
 };
 
+static void __init acpi_cpufreq_boost_init(void)
+{
+	if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) {
+		msrs = msrs_alloc();
+
+		if (!msrs)
+			return;
+
+		boost_supported = true;
+		boost_enabled = boost_state(0);
+
+		get_online_cpus();
+
+		/* Force all MSRs to the same value */
+		boost_set_msrs(boost_enabled, cpu_online_mask);
+
+		register_cpu_notifier(&boost_nb);
+
+		put_online_cpus();
+	} else
+		global_boost.attr.mode = 0444;
+
+	/* We create the boost file in any case, though for systems without
+	 * hardware support it will be read-only and hardwired to return 0.
+	 */
+	if (sysfs_create_file(cpufreq_global_kobject, &(global_boost.attr)))
+		pr_warn(PFX "could not register global boost sysfs file\n");
+	else
+		pr_debug("registered global boost sysfs file\n");
+}
+
+static void __exit acpi_cpufreq_boost_exit(void)
+{
+	sysfs_remove_file(cpufreq_global_kobject, &(global_boost.attr));
+
+	if (msrs) {
+		unregister_cpu_notifier(&boost_nb);
+
+		msrs_free(msrs);
+		msrs = NULL;
+	}
+}
+
 static int __init acpi_cpufreq_init(void)
 {
 	int ret;
@@ -746,9 +981,32 @@ static int __init acpi_cpufreq_init(void)
 	if (ret)
 		return ret;
 
+#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB
+	/* this is a sysfs file with a strange name and an even stranger
+	 * semantic - per CPU instantiation, but system global effect.
+	 * Lets enable it only on AMD CPUs for compatibility reasons and
+	 * only if configured. This is considered legacy code, which
+	 * will probably be removed at some point in the future.
+	 */
+	if (check_amd_hwpstate_cpu(0)) {
+		struct freq_attr **iter;
+
+		pr_debug("adding sysfs entry for cpb\n");
+
+		for (iter = acpi_cpufreq_attr; *iter != NULL; iter++)
+			;
+
+		/* make sure there is a terminator behind it */
+		if (iter[1] == NULL)
+			*iter = &cpb;
+	}
+#endif
+
 	ret = cpufreq_register_driver(&acpi_cpufreq_driver);
 	if (ret)
 		free_acpi_perf_data();
+	else
+		acpi_cpufreq_boost_init();
 
 	return ret;
 }
@@ -757,6 +1015,8 @@ static void __exit acpi_cpufreq_exit(void)
 {
 	pr_debug("acpi_cpufreq_exit\n");
 
+	acpi_cpufreq_boost_exit();
+
 	cpufreq_unregister_driver(&acpi_cpufreq_driver);
 
 	free_acpi_perf_data();
diff --git a/drivers/cpufreq/cpufreq-cpu0.c b/drivers/cpufreq/cpufreq-cpu0.c
new file mode 100644
index 00000000000..e9158278c71
--- /dev/null
+++ b/drivers/cpufreq/cpufreq-cpu0.c
@@ -0,0 +1,269 @@
+/*
+ * Copyright (C) 2012 Freescale Semiconductor, Inc.
+ *
+ * The OPP code in function cpu0_set_target() is reused from
+ * drivers/cpufreq/omap-cpufreq.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/opp.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+
+static unsigned int transition_latency;
+static unsigned int voltage_tolerance; /* in percentage */
+
+static struct device *cpu_dev;
+static struct clk *cpu_clk;
+static struct regulator *cpu_reg;
+static struct cpufreq_frequency_table *freq_table;
+
+static int cpu0_verify_speed(struct cpufreq_policy *policy)
+{
+	return cpufreq_frequency_table_verify(policy, freq_table);
+}
+
+static unsigned int cpu0_get_speed(unsigned int cpu)
+{
+	return clk_get_rate(cpu_clk) / 1000;
+}
+
+static int cpu0_set_target(struct cpufreq_policy *policy,
+			   unsigned int target_freq, unsigned int relation)
+{
+	struct cpufreq_freqs freqs;
+	struct opp *opp;
+	unsigned long freq_Hz, volt = 0, volt_old = 0, tol = 0;
+	unsigned int index, cpu;
+	int ret;
+
+	ret = cpufreq_frequency_table_target(policy, freq_table, target_freq,
+					     relation, &index);
+	if (ret) {
+		pr_err("failed to match target freqency %d: %d\n",
+		       target_freq, ret);
+		return ret;
+	}
+
+	freq_Hz = clk_round_rate(cpu_clk, freq_table[index].frequency * 1000);
+	if (freq_Hz < 0)
+		freq_Hz = freq_table[index].frequency * 1000;
+	freqs.new = freq_Hz / 1000;
+	freqs.old = clk_get_rate(cpu_clk) / 1000;
+
+	if (freqs.old == freqs.new)
+		return 0;
+
+	for_each_online_cpu(cpu) {
+		freqs.cpu = cpu;
+		cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
+	}
+
+	if (cpu_reg) {
+		opp = opp_find_freq_ceil(cpu_dev, &freq_Hz);
+		if (IS_ERR(opp)) {
+			pr_err("failed to find OPP for %ld\n", freq_Hz);
+			return PTR_ERR(opp);
+		}
+		volt = opp_get_voltage(opp);
+		tol = volt * voltage_tolerance / 100;
+		volt_old = regulator_get_voltage(cpu_reg);
+	}
+
+	pr_debug("%u MHz, %ld mV --> %u MHz, %ld mV\n",
+		 freqs.old / 1000, volt_old ? volt_old / 1000 : -1,
+		 freqs.new / 1000, volt ? volt / 1000 : -1);
+
+	/* scaling up?  scale voltage before frequency */
+	if (cpu_reg && freqs.new > freqs.old) {
+		ret = regulator_set_voltage_tol(cpu_reg, volt, tol);
+		if (ret) {
+			pr_err("failed to scale voltage up: %d\n", ret);
+			freqs.new = freqs.old;
+			return ret;
+		}
+	}
+
+	ret = clk_set_rate(cpu_clk, freqs.new * 1000);
+	if (ret) {
+		pr_err("failed to set clock rate: %d\n", ret);
+		if (cpu_reg)
+			regulator_set_voltage_tol(cpu_reg, volt_old, tol);
+		return ret;
+	}
+
+	/* scaling down?  scale voltage after frequency */
+	if (cpu_reg && freqs.new < freqs.old) {
+		ret = regulator_set_voltage_tol(cpu_reg, volt, tol);
+		if (ret) {
+			pr_err("failed to scale voltage down: %d\n", ret);
+			clk_set_rate(cpu_clk, freqs.old * 1000);
+			freqs.new = freqs.old;
+			return ret;
+		}
+	}
+
+	for_each_online_cpu(cpu) {
+		freqs.cpu = cpu;
+		cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
+	}
+
+	return 0;
+}
+
+static int cpu0_cpufreq_init(struct cpufreq_policy *policy)
+{
+	int ret;
+
+	if (policy->cpu != 0)
+		return -EINVAL;
+
+	ret = cpufreq_frequency_table_cpuinfo(policy, freq_table);
+	if (ret) {
+		pr_err("invalid frequency table: %d\n", ret);
+		return ret;
+	}
+
+	policy->cpuinfo.transition_latency = transition_latency;
+	policy->cur = clk_get_rate(cpu_clk) / 1000;
+
+	/*
+	 * The driver only supports the SMP configuartion where all processors
+	 * share the clock and voltage and clock.  Use cpufreq affected_cpus
+	 * interface to have all CPUs scaled together.
+	 */
+	policy->shared_type = CPUFREQ_SHARED_TYPE_ANY;
+	cpumask_setall(policy->cpus);
+
+	cpufreq_frequency_table_get_attr(freq_table, policy->cpu);
+
+	return 0;
+}
+
+static int cpu0_cpufreq_exit(struct cpufreq_policy *policy)
+{
+	cpufreq_frequency_table_put_attr(policy->cpu);
+
+	return 0;
+}
+
+static struct freq_attr *cpu0_cpufreq_attr[] = {
+	&cpufreq_freq_attr_scaling_available_freqs,
+	NULL,
+};
+
+static struct cpufreq_driver cpu0_cpufreq_driver = {
+	.flags = CPUFREQ_STICKY,
+	.verify = cpu0_verify_speed,
+	.target = cpu0_set_target,
+	.get = cpu0_get_speed,
+	.init = cpu0_cpufreq_init,
+	.exit = cpu0_cpufreq_exit,
+	.name = "generic_cpu0",
+	.attr = cpu0_cpufreq_attr,
+};
+
+static int __devinit cpu0_cpufreq_driver_init(void)
+{
+	struct device_node *np;
+	int ret;
+
+	np = of_find_node_by_path("/cpus/cpu@0");
+	if (!np) {
+		pr_err("failed to find cpu0 node\n");
+		return -ENOENT;
+	}
+
+	cpu_dev = get_cpu_device(0);
+	if (!cpu_dev) {
+		pr_err("failed to get cpu0 device\n");
+		ret = -ENODEV;
+		goto out_put_node;
+	}
+
+	cpu_dev->of_node = np;
+
+	cpu_clk = clk_get(cpu_dev, NULL);
+	if (IS_ERR(cpu_clk)) {
+		ret = PTR_ERR(cpu_clk);
+		pr_err("failed to get cpu0 clock: %d\n", ret);
+		goto out_put_node;
+	}
+
+	cpu_reg = regulator_get(cpu_dev, "cpu0");
+	if (IS_ERR(cpu_reg)) {
+		pr_warn("failed to get cpu0 regulator\n");
+		cpu_reg = NULL;
+	}
+
+	ret = of_init_opp_table(cpu_dev);
+	if (ret) {
+		pr_err("failed to init OPP table: %d\n", ret);
+		goto out_put_node;
+	}
+
+	ret = opp_init_cpufreq_table(cpu_dev, &freq_table);
+	if (ret) {
+		pr_err("failed to init cpufreq table: %d\n", ret);
+		goto out_put_node;
+	}
+
+	of_property_read_u32(np, "voltage-tolerance", &voltage_tolerance);
+
+	if (of_property_read_u32(np, "clock-latency", &transition_latency))
+		transition_latency = CPUFREQ_ETERNAL;
+
+	if (cpu_reg) {
+		struct opp *opp;
+		unsigned long min_uV, max_uV;
+		int i;
+
+		/*
+		 * OPP is maintained in order of increasing frequency, and
+		 * freq_table initialised from OPP is therefore sorted in the
+		 * same order.
+		 */
+		for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++)
+			;
+		opp = opp_find_freq_exact(cpu_dev,
+				freq_table[0].frequency * 1000, true);
+		min_uV = opp_get_voltage(opp);
+		opp = opp_find_freq_exact(cpu_dev,
+				freq_table[i-1].frequency * 1000, true);
+		max_uV = opp_get_voltage(opp);
+		ret = regulator_set_voltage_time(cpu_reg, min_uV, max_uV);
+		if (ret > 0)
+			transition_latency += ret * 1000;
+	}
+
+	ret = cpufreq_register_driver(&cpu0_cpufreq_driver);
+	if (ret) {
+		pr_err("failed register driver: %d\n", ret);
+		goto out_free_table;
+	}
+
+	of_node_put(np);
+	return 0;
+
+out_free_table:
+	opp_free_cpufreq_table(cpu_dev, &freq_table);
+out_put_node:
+	of_node_put(np);
+	return ret;
+}
+late_initcall(cpu0_cpufreq_driver_init);
+
+MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
+MODULE_DESCRIPTION("Generic CPU0 cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c
index 55f0354864e..a152af7e199 100644
--- a/drivers/cpufreq/cpufreq_conservative.c
+++ b/drivers/cpufreq/cpufreq_conservative.c
@@ -504,6 +504,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 				j_dbs_info->prev_cpu_nice =
 						kcpustat_cpu(j).cpustat[CPUTIME_NICE];
 		}
+		this_dbs_info->cpu = cpu;
 		this_dbs_info->down_skip = 0;
 		this_dbs_info->requested_freq = policy->cur;
 
@@ -583,6 +584,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 			__cpufreq_driver_target(
 					this_dbs_info->cur_policy,
 					policy->min, CPUFREQ_RELATION_L);
+		dbs_check_cpu(this_dbs_info);
 		mutex_unlock(&this_dbs_info->timer_mutex);
 
 		break;
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 14c1af5a264..396322f2a83 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -761,6 +761,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 		else if (policy->min > this_dbs_info->cur_policy->cur)
 			__cpufreq_driver_target(this_dbs_info->cur_policy,
 				policy->min, CPUFREQ_RELATION_L);
+		dbs_check_cpu(this_dbs_info);
 		mutex_unlock(&this_dbs_info->timer_mutex);
 		break;
 	}
diff --git a/drivers/cpufreq/longhaul.h b/drivers/cpufreq/longhaul.h
index cbf48fbca88..e2dc436099d 100644
--- a/drivers/cpufreq/longhaul.h
+++ b/drivers/cpufreq/longhaul.h
@@ -56,7 +56,7 @@ union msr_longhaul {
 /*
  * VIA C3 Samuel 1  & Samuel 2 (stepping 0)
  */
-static const int __cpuinitdata samuel1_mults[16] = {
+static const int __cpuinitconst samuel1_mults[16] = {
 	-1, /* 0000 -> RESERVED */
 	30, /* 0001 ->  3.0x */
 	40, /* 0010 ->  4.0x */
@@ -75,7 +75,7 @@ static const int __cpuinitdata samuel1_mults[16] = {
 	-1, /* 1111 -> RESERVED */
 };
 
-static const int __cpuinitdata samuel1_eblcr[16] = {
+static const int __cpuinitconst samuel1_eblcr[16] = {
 	50, /* 0000 -> RESERVED */
 	30, /* 0001 ->  3.0x */
 	40, /* 0010 ->  4.0x */
@@ -97,7 +97,7 @@ static const int __cpuinitdata samuel1_eblcr[16] = {
 /*
  * VIA C3 Samuel2 Stepping 1->15
  */
-static const int __cpuinitdata samuel2_eblcr[16] = {
+static const int __cpuinitconst samuel2_eblcr[16] = {
 	50,  /* 0000 ->  5.0x */
 	30,  /* 0001 ->  3.0x */
 	40,  /* 0010 ->  4.0x */
@@ -119,7 +119,7 @@ static const int __cpuinitdata samuel2_eblcr[16] = {
 /*
  * VIA C3 Ezra
  */
-static const int __cpuinitdata ezra_mults[16] = {
+static const int __cpuinitconst ezra_mults[16] = {
 	100, /* 0000 -> 10.0x */
 	30,  /* 0001 ->  3.0x */
 	40,  /* 0010 ->  4.0x */
@@ -138,7 +138,7 @@ static const int __cpuinitdata ezra_mults[16] = {
 	120, /* 1111 -> 12.0x */
 };
 
-static const int __cpuinitdata ezra_eblcr[16] = {
+static const int __cpuinitconst ezra_eblcr[16] = {
 	50,  /* 0000 ->  5.0x */
 	30,  /* 0001 ->  3.0x */
 	40,  /* 0010 ->  4.0x */
@@ -160,7 +160,7 @@ static const int __cpuinitdata ezra_eblcr[16] = {
 /*
  * VIA C3 (Ezra-T) [C5M].
  */
-static const int __cpuinitdata ezrat_mults[32] = {
+static const int __cpuinitconst ezrat_mults[32] = {
 	100, /* 0000 -> 10.0x */
 	30,  /* 0001 ->  3.0x */
 	40,  /* 0010 ->  4.0x */
@@ -196,7 +196,7 @@ static const int __cpuinitdata ezrat_mults[32] = {
 	-1,  /* 1111 -> RESERVED (12.0x) */
 };
 
-static const int __cpuinitdata ezrat_eblcr[32] = {
+static const int __cpuinitconst ezrat_eblcr[32] = {
 	50,  /* 0000 ->  5.0x */
 	30,  /* 0001 ->  3.0x */
 	40,  /* 0010 ->  4.0x */
@@ -235,7 +235,7 @@ static const int __cpuinitdata ezrat_eblcr[32] = {
 /*
  * VIA C3 Nehemiah */
 
-static const int __cpuinitdata nehemiah_mults[32] = {
+static const int __cpuinitconst nehemiah_mults[32] = {
 	100, /* 0000 -> 10.0x */
 	-1, /* 0001 -> 16.0x */
 	40,  /* 0010 ->  4.0x */
@@ -270,7 +270,7 @@ static const int __cpuinitdata nehemiah_mults[32] = {
 	-1, /* 1111 -> 12.0x */
 };
 
-static const int __cpuinitdata nehemiah_eblcr[32] = {
+static const int __cpuinitconst nehemiah_eblcr[32] = {
 	50,  /* 0000 ->  5.0x */
 	160, /* 0001 -> 16.0x */
 	40,  /* 0010 ->  4.0x */
@@ -315,7 +315,7 @@ struct mV_pos {
 	unsigned short pos;
 };
 
-static const struct mV_pos __cpuinitdata vrm85_mV[32] = {
+static const struct mV_pos __cpuinitconst vrm85_mV[32] = {
 	{1250, 8},	{1200, 6},	{1150, 4},	{1100, 2},
 	{1050, 0},	{1800, 30},	{1750, 28},	{1700, 26},
 	{1650, 24},	{1600, 22},	{1550, 20},	{1500, 18},
@@ -326,14 +326,14 @@ static const struct mV_pos __cpuinitdata vrm85_mV[32] = {
 	{1475, 17},	{1425, 15},	{1375, 13},	{1325, 11}
 };
 
-static const unsigned char __cpuinitdata mV_vrm85[32] = {
+static const unsigned char __cpuinitconst mV_vrm85[32] = {
 	0x04,	0x14,	0x03,	0x13,	0x02,	0x12,	0x01,	0x11,
 	0x00,	0x10,	0x0f,	0x1f,	0x0e,	0x1e,	0x0d,	0x1d,
 	0x0c,	0x1c,	0x0b,	0x1b,	0x0a,	0x1a,	0x09,	0x19,
 	0x08,	0x18,	0x07,	0x17,	0x06,	0x16,	0x05,	0x15
 };
 
-static const struct mV_pos __cpuinitdata mobilevrm_mV[32] = {
+static const struct mV_pos __cpuinitconst mobilevrm_mV[32] = {
 	{1750, 31},	{1700, 30},	{1650, 29},	{1600, 28},
 	{1550, 27},	{1500, 26},	{1450, 25},	{1400, 24},
 	{1350, 23},	{1300, 22},	{1250, 21},	{1200, 20},
@@ -344,7 +344,7 @@ static const struct mV_pos __cpuinitdata mobilevrm_mV[32] = {
 	{675, 3},	{650, 2},	{625, 1},	{600, 0}
 };
 
-static const unsigned char __cpuinitdata mV_mobilevrm[32] = {
+static const unsigned char __cpuinitconst mV_mobilevrm[32] = {
 	0x1f,	0x1e,	0x1d,	0x1c,	0x1b,	0x1a,	0x19,	0x18,
 	0x17,	0x16,	0x15,	0x14,	0x13,	0x12,	0x11,	0x10,
 	0x0f,	0x0e,	0x0d,	0x0c,	0x0b,	0x0a,	0x09,	0x08,
diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c
index b47034e650a..65f8e9a5497 100644
--- a/drivers/cpufreq/omap-cpufreq.c
+++ b/drivers/cpufreq/omap-cpufreq.c
@@ -40,16 +40,6 @@
 /* OPP tolerance in percentage */
 #define	OPP_TOLERANCE	4
 
-#ifdef CONFIG_SMP
-struct lpj_info {
-	unsigned long	ref;
-	unsigned int	freq;
-};
-
-static DEFINE_PER_CPU(struct lpj_info, lpj_ref);
-static struct lpj_info global_lpj_ref;
-#endif
-
 static struct cpufreq_frequency_table *freq_table;
 static atomic_t freq_table_users = ATOMIC_INIT(0);
 static struct clk *mpu_clk;
@@ -161,31 +151,6 @@ static int omap_target(struct cpufreq_policy *policy,
 	}
 
 	freqs.new = omap_getspeed(policy->cpu);
-#ifdef CONFIG_SMP
-	/*
-	 * Note that loops_per_jiffy is not updated on SMP systems in
-	 * cpufreq driver. So, update the per-CPU loops_per_jiffy value
-	 * on frequency transition. We need to update all dependent CPUs.
-	 */
-	for_each_cpu(i, policy->cpus) {
-		struct lpj_info *lpj = &per_cpu(lpj_ref, i);
-		if (!lpj->freq) {
-			lpj->ref = per_cpu(cpu_data, i).loops_per_jiffy;
-			lpj->freq = freqs.old;
-		}
-
-		per_cpu(cpu_data, i).loops_per_jiffy =
-			cpufreq_scale(lpj->ref, lpj->freq, freqs.new);
-	}
-
-	/* And don't forget to adjust the global one */
-	if (!global_lpj_ref.freq) {
-		global_lpj_ref.ref = loops_per_jiffy;
-		global_lpj_ref.freq = freqs.old;
-	}
-	loops_per_jiffy = cpufreq_scale(global_lpj_ref.ref, global_lpj_ref.freq,
-					freqs.new);
-#endif
 
 done:
 	/* notifiers */
@@ -301,9 +266,9 @@ static int __init omap_cpufreq_init(void)
 	}
 
 	mpu_dev = omap_device_get_by_hwmod_name("mpu");
-	if (!mpu_dev) {
+	if (IS_ERR(mpu_dev)) {
 		pr_warning("%s: unable to get the mpu device\n", __func__);
-		return -EINVAL;
+		return PTR_ERR(mpu_dev);
 	}
 
 	mpu_reg = regulator_get(mpu_dev, "vcc");
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index 1a40935c85f..129e80bfff2 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -48,22 +48,12 @@
 #define PFX "powernow-k8: "
 #define VERSION "version 2.20.00"
 #include "powernow-k8.h"
-#include "mperf.h"
 
 /* serialize freq changes  */
 static DEFINE_MUTEX(fidvid_mutex);
 
 static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data);
 
-static int cpu_family = CPU_OPTERON;
-
-/* array to map SW pstate number to acpi state */
-static u32 ps_to_as[8];
-
-/* core performance boost */
-static bool cpb_capable, cpb_enabled;
-static struct msr __percpu *msrs;
-
 static struct cpufreq_driver cpufreq_amd64_driver;
 
 #ifndef CONFIG_SMP
@@ -85,12 +75,6 @@ static u32 find_khz_freq_from_fid(u32 fid)
 	return 1000 * find_freq_from_fid(fid);
 }
 
-static u32 find_khz_freq_from_pstate(struct cpufreq_frequency_table *data,
-				     u32 pstate)
-{
-	return data[ps_to_as[pstate]].frequency;
-}
-
 /* Return the vco fid for an input fid
  *
  * Each "low" fid has corresponding "high" fid, and you can get to "low" fids
@@ -113,9 +97,6 @@ static int pending_bit_stuck(void)
 {
 	u32 lo, hi;
 
-	if (cpu_family == CPU_HW_PSTATE)
-		return 0;
-
 	rdmsr(MSR_FIDVID_STATUS, lo, hi);
 	return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
 }
@@ -129,20 +110,6 @@ static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
 	u32 lo, hi;
 	u32 i = 0;
 
-	if (cpu_family == CPU_HW_PSTATE) {
-		rdmsr(MSR_PSTATE_STATUS, lo, hi);
-		i = lo & HW_PSTATE_MASK;
-		data->currpstate = i;
-
-		/*
-		 * a workaround for family 11h erratum 311 might cause
-		 * an "out-of-range Pstate if the core is in Pstate-0
-		 */
-		if ((boot_cpu_data.x86 == 0x11) && (i >= data->numps))
-			data->currpstate = HW_PSTATE_0;
-
-		return 0;
-	}
 	do {
 		if (i++ > 10000) {
 			pr_debug("detected change pending stuck\n");
@@ -299,14 +266,6 @@ static int decrease_vid_code_by_step(struct powernow_k8_data *data,
 	return 0;
 }
 
-/* Change hardware pstate by single MSR write */
-static int transition_pstate(struct powernow_k8_data *data, u32 pstate)
-{
-	wrmsr(MSR_PSTATE_CTRL, pstate, 0);
-	data->currpstate = pstate;
-	return 0;
-}
-
 /* Change Opteron/Athlon64 fid and vid, by the 3 phases. */
 static int transition_fid_vid(struct powernow_k8_data *data,
 		u32 reqfid, u32 reqvid)
@@ -523,8 +482,6 @@ static int core_voltage_post_transition(struct powernow_k8_data *data,
 static const struct x86_cpu_id powernow_k8_ids[] = {
 	/* IO based frequency switching */
 	{ X86_VENDOR_AMD, 0xf },
-	/* MSR based frequency switching supported */
-	X86_FEATURE_MATCH(X86_FEATURE_HW_PSTATE),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, powernow_k8_ids);
@@ -560,15 +517,8 @@ static void check_supported_cpu(void *_rc)
 				"Power state transitions not supported\n");
 			return;
 		}
-	} else { /* must be a HW Pstate capable processor */
-		cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
-		if ((edx & USE_HW_PSTATE) == USE_HW_PSTATE)
-			cpu_family = CPU_HW_PSTATE;
-		else
-			return;
+		*rc = 0;
 	}
-
-	*rc = 0;
 }
 
 static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst,
@@ -632,18 +582,11 @@ static void print_basics(struct powernow_k8_data *data)
 	for (j = 0; j < data->numps; j++) {
 		if (data->powernow_table[j].frequency !=
 				CPUFREQ_ENTRY_INVALID) {
-			if (cpu_family == CPU_HW_PSTATE) {
-				printk(KERN_INFO PFX
-					"   %d : pstate %d (%d MHz)\n", j,
-					data->powernow_table[j].index,
-					data->powernow_table[j].frequency/1000);
-			} else {
 				printk(KERN_INFO PFX
 					"fid 0x%x (%d MHz), vid 0x%x\n",
 					data->powernow_table[j].index & 0xff,
 					data->powernow_table[j].frequency/1000,
 					data->powernow_table[j].index >> 8);
-			}
 		}
 	}
 	if (data->batps)
@@ -651,20 +594,6 @@ static void print_basics(struct powernow_k8_data *data)
 				data->batps);
 }
 
-static u32 freq_from_fid_did(u32 fid, u32 did)
-{
-	u32 mhz = 0;
-
-	if (boot_cpu_data.x86 == 0x10)
-		mhz = (100 * (fid + 0x10)) >> did;
-	else if (boot_cpu_data.x86 == 0x11)
-		mhz = (100 * (fid + 8)) >> did;
-	else
-		BUG();
-
-	return mhz * 1000;
-}
-
 static int fill_powernow_table(struct powernow_k8_data *data,
 		struct pst_s *pst, u8 maxvid)
 {
@@ -824,7 +753,7 @@ static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data,
 {
 	u64 control;
 
-	if (!data->acpi_data.state_count || (cpu_family == CPU_HW_PSTATE))
+	if (!data->acpi_data.state_count)
 		return;
 
 	control = data->acpi_data.states[index].control;
@@ -875,10 +804,7 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
 	data->numps = data->acpi_data.state_count;
 	powernow_k8_acpi_pst_values(data, 0);
 
-	if (cpu_family == CPU_HW_PSTATE)
-		ret_val = fill_powernow_table_pstate(data, powernow_table);
-	else
-		ret_val = fill_powernow_table_fidvid(data, powernow_table);
+	ret_val = fill_powernow_table_fidvid(data, powernow_table);
 	if (ret_val)
 		goto err_out_mem;
 
@@ -915,51 +841,6 @@ err_out:
 	return ret_val;
 }
 
-static int fill_powernow_table_pstate(struct powernow_k8_data *data,
-		struct cpufreq_frequency_table *powernow_table)
-{
-	int i;
-	u32 hi = 0, lo = 0;
-	rdmsr(MSR_PSTATE_CUR_LIMIT, lo, hi);
-	data->max_hw_pstate = (lo & HW_PSTATE_MAX_MASK) >> HW_PSTATE_MAX_SHIFT;
-
-	for (i = 0; i < data->acpi_data.state_count; i++) {
-		u32 index;
-
-		index = data->acpi_data.states[i].control & HW_PSTATE_MASK;
-		if (index > data->max_hw_pstate) {
-			printk(KERN_ERR PFX "invalid pstate %d - "
-					"bad value %d.\n", i, index);
-			printk(KERN_ERR PFX "Please report to BIOS "
-					"manufacturer\n");
-			invalidate_entry(powernow_table, i);
-			continue;
-		}
-
-		ps_to_as[index] = i;
-
-		/* Frequency may be rounded for these */
-		if ((boot_cpu_data.x86 == 0x10 && boot_cpu_data.x86_model < 10)
-				 || boot_cpu_data.x86 == 0x11) {
-
-			rdmsr(MSR_PSTATE_DEF_BASE + index, lo, hi);
-			if (!(hi & HW_PSTATE_VALID_MASK)) {
-				pr_debug("invalid pstate %d, ignoring\n", index);
-				invalidate_entry(powernow_table, i);
-				continue;
-			}
-
-			powernow_table[i].frequency =
-				freq_from_fid_did(lo & 0x3f, (lo >> 6) & 7);
-		} else
-			powernow_table[i].frequency =
-				data->acpi_data.states[i].core_frequency * 1000;
-
-		powernow_table[i].index = index;
-	}
-	return 0;
-}
-
 static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
 		struct cpufreq_frequency_table *powernow_table)
 {
@@ -1036,15 +917,7 @@ static int get_transition_latency(struct powernow_k8_data *data)
 			max_latency = cur_latency;
 	}
 	if (max_latency == 0) {
-		/*
-		 * Fam 11h and later may return 0 as transition latency. This
-		 * is intended and means "very fast". While cpufreq core and
-		 * governors currently can handle that gracefully, better set it
-		 * to 1 to avoid problems in the future.
-		 */
-		if (boot_cpu_data.x86 < 0x11)
-			printk(KERN_ERR FW_WARN PFX "Invalid zero transition "
-				"latency\n");
+		pr_err(FW_WARN PFX "Invalid zero transition latency\n");
 		max_latency = 1;
 	}
 	/* value in usecs, needs to be in nanoseconds */
@@ -1104,40 +977,6 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data,
 	return res;
 }
 
-/* Take a frequency, and issue the hardware pstate transition command */
-static int transition_frequency_pstate(struct powernow_k8_data *data,
-		unsigned int index)
-{
-	u32 pstate = 0;
-	int res, i;
-	struct cpufreq_freqs freqs;
-
-	pr_debug("cpu %d transition to index %u\n", smp_processor_id(), index);
-
-	/* get MSR index for hardware pstate transition */
-	pstate = index & HW_PSTATE_MASK;
-	if (pstate > data->max_hw_pstate)
-		return -EINVAL;
-
-	freqs.old = find_khz_freq_from_pstate(data->powernow_table,
-			data->currpstate);
-	freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate);
-
-	for_each_cpu(i, data->available_cores) {
-		freqs.cpu = i;
-		cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
-	}
-
-	res = transition_pstate(data, pstate);
-	freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate);
-
-	for_each_cpu(i, data->available_cores) {
-		freqs.cpu = i;
-		cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
-	}
-	return res;
-}
-
 struct powernowk8_target_arg {
 	struct cpufreq_policy		*pol;
 	unsigned			targfreq;
@@ -1173,18 +1012,15 @@ static long powernowk8_target_fn(void *arg)
 	if (query_current_values_with_pending_wait(data))
 		return -EIO;
 
-	if (cpu_family != CPU_HW_PSTATE) {
-		pr_debug("targ: curr fid 0x%x, vid 0x%x\n",
-		data->currfid, data->currvid);
+	pr_debug("targ: curr fid 0x%x, vid 0x%x\n",
+		 data->currfid, data->currvid);
 
-		if ((checkvid != data->currvid) ||
-		    (checkfid != data->currfid)) {
-			printk(KERN_INFO PFX
-				"error - out of sync, fix 0x%x 0x%x, "
-				"vid 0x%x 0x%x\n",
-				checkfid, data->currfid,
-				checkvid, data->currvid);
-		}
+	if ((checkvid != data->currvid) ||
+	    (checkfid != data->currfid)) {
+		pr_info(PFX
+		       "error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n",
+		       checkfid, data->currfid,
+		       checkvid, data->currvid);
 	}
 
 	if (cpufreq_frequency_table_target(pol, data->powernow_table,
@@ -1195,11 +1031,8 @@ static long powernowk8_target_fn(void *arg)
 
 	powernow_k8_acpi_pst_values(data, newstate);
 
-	if (cpu_family == CPU_HW_PSTATE)
-		ret = transition_frequency_pstate(data,
-			data->powernow_table[newstate].index);
-	else
-		ret = transition_frequency_fidvid(data, newstate);
+	ret = transition_frequency_fidvid(data, newstate);
+
 	if (ret) {
 		printk(KERN_ERR PFX "transition frequency failed\n");
 		mutex_unlock(&fidvid_mutex);
@@ -1207,11 +1040,7 @@ static long powernowk8_target_fn(void *arg)
 	}
 	mutex_unlock(&fidvid_mutex);
 
-	if (cpu_family == CPU_HW_PSTATE)
-		pol->cur = find_khz_freq_from_pstate(data->powernow_table,
-				data->powernow_table[newstate].index);
-	else
-		pol->cur = find_khz_freq_from_fid(data->currfid);
+	pol->cur = find_khz_freq_from_fid(data->currfid);
 
 	return 0;
 }
@@ -1264,22 +1093,23 @@ static void __cpuinit powernowk8_cpu_init_on_cpu(void *_init_on_cpu)
 		return;
 	}
 
-	if (cpu_family == CPU_OPTERON)
-		fidvid_msr_init();
+	fidvid_msr_init();
 
 	init_on_cpu->rc = 0;
 }
 
+static const char missing_pss_msg[] =
+	KERN_ERR
+	FW_BUG PFX "No compatible ACPI _PSS objects found.\n"
+	FW_BUG PFX "First, make sure Cool'N'Quiet is enabled in the BIOS.\n"
+	FW_BUG PFX "If that doesn't help, try upgrading your BIOS.\n";
+
 /* per CPU init entry point to the driver */
 static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 {
-	static const char ACPI_PSS_BIOS_BUG_MSG[] =
-		KERN_ERR FW_BUG PFX "No compatible ACPI _PSS objects found.\n"
-		FW_BUG PFX "Try again with latest BIOS.\n";
 	struct powernow_k8_data *data;
 	struct init_on_cpu init_on_cpu;
 	int rc;
-	struct cpuinfo_x86 *c = &cpu_data(pol->cpu);
 
 	if (!cpu_online(pol->cpu))
 		return -ENODEV;
@@ -1295,7 +1125,6 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 	}
 
 	data->cpu = pol->cpu;
-	data->currpstate = HW_PSTATE_INVALID;
 
 	if (powernow_k8_cpu_init_acpi(data)) {
 		/*
@@ -1303,7 +1132,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 		 * an UP version, and is deprecated by AMD.
 		 */
 		if (num_online_cpus() != 1) {
-			printk_once(ACPI_PSS_BIOS_BUG_MSG);
+			printk_once(missing_pss_msg);
 			goto err_out;
 		}
 		if (pol->cpu != 0) {
@@ -1332,17 +1161,10 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 	if (rc != 0)
 		goto err_out_exit_acpi;
 
-	if (cpu_family == CPU_HW_PSTATE)
-		cpumask_copy(pol->cpus, cpumask_of(pol->cpu));
-	else
-		cpumask_copy(pol->cpus, cpu_core_mask(pol->cpu));
+	cpumask_copy(pol->cpus, cpu_core_mask(pol->cpu));
 	data->available_cores = pol->cpus;
 
-	if (cpu_family == CPU_HW_PSTATE)
-		pol->cur = find_khz_freq_from_pstate(data->powernow_table,
-				data->currpstate);
-	else
-		pol->cur = find_khz_freq_from_fid(data->currfid);
+	pol->cur = find_khz_freq_from_fid(data->currfid);
 	pr_debug("policy current frequency %d kHz\n", pol->cur);
 
 	/* min/max the cpu is capable of */
@@ -1354,18 +1176,10 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol)
 		return -EINVAL;
 	}
 
-	/* Check for APERF/MPERF support in hardware */
-	if (cpu_has(c, X86_FEATURE_APERFMPERF))
-		cpufreq_amd64_driver.getavg = cpufreq_get_measured_perf;
-
 	cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu);
 
-	if (cpu_family == CPU_HW_PSTATE)
-		pr_debug("cpu_init done, current pstate 0x%x\n",
-				data->currpstate);
-	else
-		pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n",
-			data->currfid, data->currvid);
+	pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n",
+		 data->currfid, data->currvid);
 
 	per_cpu(powernow_data, pol->cpu) = data;
 
@@ -1418,88 +1232,15 @@ static unsigned int powernowk8_get(unsigned int cpu)
 	if (err)
 		goto out;
 
-	if (cpu_family == CPU_HW_PSTATE)
-		khz = find_khz_freq_from_pstate(data->powernow_table,
-						data->currpstate);
-	else
-		khz = find_khz_freq_from_fid(data->currfid);
+	khz = find_khz_freq_from_fid(data->currfid);
 
 
 out:
 	return khz;
 }
 
-static void _cpb_toggle_msrs(bool t)
-{
-	int cpu;
-
-	get_online_cpus();
-
-	rdmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
-
-	for_each_cpu(cpu, cpu_online_mask) {
-		struct msr *reg = per_cpu_ptr(msrs, cpu);
-		if (t)
-			reg->l &= ~BIT(25);
-		else
-			reg->l |= BIT(25);
-	}
-	wrmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
-
-	put_online_cpus();
-}
-
-/*
- * Switch on/off core performance boosting.
- *
- * 0=disable
- * 1=enable.
- */
-static void cpb_toggle(bool t)
-{
-	if (!cpb_capable)
-		return;
-
-	if (t && !cpb_enabled) {
-		cpb_enabled = true;
-		_cpb_toggle_msrs(t);
-		printk(KERN_INFO PFX "Core Boosting enabled.\n");
-	} else if (!t && cpb_enabled) {
-		cpb_enabled = false;
-		_cpb_toggle_msrs(t);
-		printk(KERN_INFO PFX "Core Boosting disabled.\n");
-	}
-}
-
-static ssize_t store_cpb(struct cpufreq_policy *policy, const char *buf,
-				 size_t count)
-{
-	int ret = -EINVAL;
-	unsigned long val = 0;
-
-	ret = strict_strtoul(buf, 10, &val);
-	if (!ret && (val == 0 || val == 1) && cpb_capable)
-		cpb_toggle(val);
-	else
-		return -EINVAL;
-
-	return count;
-}
-
-static ssize_t show_cpb(struct cpufreq_policy *policy, char *buf)
-{
-	return sprintf(buf, "%u\n", cpb_enabled);
-}
-
-#define define_one_rw(_name) \
-static struct freq_attr _name = \
-__ATTR(_name, 0644, show_##_name, store_##_name)
-
-define_one_rw(cpb);
-
 static struct freq_attr *powernow_k8_attr[] = {
 	&cpufreq_freq_attr_scaling_available_freqs,
-	&cpb,
 	NULL,
 };
 
@@ -1515,53 +1256,18 @@ static struct cpufreq_driver cpufreq_amd64_driver = {
 	.attr		= powernow_k8_attr,
 };
 
-/*
- * Clear the boost-disable flag on the CPU_DOWN path so that this cpu
- * cannot block the remaining ones from boosting. On the CPU_UP path we
- * simply keep the boost-disable flag in sync with the current global
- * state.
- */
-static int cpb_notify(struct notifier_block *nb, unsigned long action,
-		      void *hcpu)
-{
-	unsigned cpu = (long)hcpu;
-	u32 lo, hi;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-
-		if (!cpb_enabled) {
-			rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
-			lo |= BIT(25);
-			wrmsr_on_cpu(cpu, MSR_K7_HWCR, lo, hi);
-		}
-		break;
-
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		rdmsr_on_cpu(cpu, MSR_K7_HWCR, &lo, &hi);
-		lo &= ~BIT(25);
-		wrmsr_on_cpu(cpu, MSR_K7_HWCR, lo, hi);
-		break;
-
-	default:
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block cpb_nb = {
-	.notifier_call		= cpb_notify,
-};
-
 /* driver entry point for init */
 static int __cpuinit powernowk8_init(void)
 {
-	unsigned int i, supported_cpus = 0, cpu;
+	unsigned int i, supported_cpus = 0;
 	int rv;
 
+	if (static_cpu_has(X86_FEATURE_HW_PSTATE)) {
+		pr_warn(PFX "this CPU is not supported anymore, using acpi-cpufreq instead.\n");
+		request_module("acpi-cpufreq");
+		return -ENODEV;
+	}
+
 	if (!x86_match_cpu(powernow_k8_ids))
 		return -ENODEV;
 
@@ -1575,38 +1281,13 @@ static int __cpuinit powernowk8_init(void)
 	if (supported_cpus != num_online_cpus())
 		return -ENODEV;
 
-	printk(KERN_INFO PFX "Found %d %s (%d cpu cores) (" VERSION ")\n",
-		num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus);
-
-	if (boot_cpu_has(X86_FEATURE_CPB)) {
-
-		cpb_capable = true;
-
-		msrs = msrs_alloc();
-		if (!msrs) {
-			printk(KERN_ERR "%s: Error allocating msrs!\n", __func__);
-			return -ENOMEM;
-		}
-
-		register_cpu_notifier(&cpb_nb);
-
-		rdmsr_on_cpus(cpu_online_mask, MSR_K7_HWCR, msrs);
+	rv = cpufreq_register_driver(&cpufreq_amd64_driver);
 
-		for_each_cpu(cpu, cpu_online_mask) {
-			struct msr *reg = per_cpu_ptr(msrs, cpu);
-			cpb_enabled |= !(!!(reg->l & BIT(25)));
-		}
+	if (!rv)
+		pr_info(PFX "Found %d %s (%d cpu cores) (" VERSION ")\n",
+			num_online_nodes(), boot_cpu_data.x86_model_id,
+			supported_cpus);
 
-		printk(KERN_INFO PFX "Core Performance Boosting: %s.\n",
-			(cpb_enabled ? "on" : "off"));
-	}
-
-	rv = cpufreq_register_driver(&cpufreq_amd64_driver);
-	if (rv < 0 && boot_cpu_has(X86_FEATURE_CPB)) {
-		unregister_cpu_notifier(&cpb_nb);
-		msrs_free(msrs);
-		msrs = NULL;
-	}
 	return rv;
 }
 
@@ -1615,13 +1296,6 @@ static void __exit powernowk8_exit(void)
 {
 	pr_debug("exit\n");
 
-	if (boot_cpu_has(X86_FEATURE_CPB)) {
-		msrs_free(msrs);
-		msrs = NULL;
-
-		unregister_cpu_notifier(&cpb_nb);
-	}
-
 	cpufreq_unregister_driver(&cpufreq_amd64_driver);
 }
 
diff --git a/drivers/cpufreq/powernow-k8.h b/drivers/cpufreq/powernow-k8.h
index 3744d26cdc2..79329d4d5ab 100644
--- a/drivers/cpufreq/powernow-k8.h
+++ b/drivers/cpufreq/powernow-k8.h
@@ -5,24 +5,11 @@
  *  http://www.gnu.org/licenses/gpl.html
  */
 
-enum pstate {
-	HW_PSTATE_INVALID = 0xff,
-	HW_PSTATE_0 = 0,
-	HW_PSTATE_1 = 1,
-	HW_PSTATE_2 = 2,
-	HW_PSTATE_3 = 3,
-	HW_PSTATE_4 = 4,
-	HW_PSTATE_5 = 5,
-	HW_PSTATE_6 = 6,
-	HW_PSTATE_7 = 7,
-};
-
 struct powernow_k8_data {
 	unsigned int cpu;
 
 	u32 numps;  /* number of p-states */
 	u32 batps;  /* number of p-states supported on battery */
-	u32 max_hw_pstate; /* maximum legal hardware pstate */
 
 	/* these values are constant when the PSB is used to determine
 	 * vid/fid pairings, but are modified during the ->target() call
@@ -37,7 +24,6 @@ struct powernow_k8_data {
 	/* keep track of the current fid / vid or pstate */
 	u32 currvid;
 	u32 currfid;
-	enum pstate currpstate;
 
 	/* the powernow_table includes all frequency and vid/fid pairings:
 	 * fid are the lower 8 bits of the index, vid are the upper 8 bits.
@@ -97,23 +83,6 @@ struct powernow_k8_data {
 #define MSR_S_HI_CURRENT_VID      0x0000003f
 #define MSR_C_HI_STP_GNT_BENIGN	  0x00000001
 
-
-/* Hardware Pstate _PSS and MSR definitions */
-#define USE_HW_PSTATE		0x00000080
-#define HW_PSTATE_MASK 		0x00000007
-#define HW_PSTATE_VALID_MASK 	0x80000000
-#define HW_PSTATE_MAX_MASK	0x000000f0
-#define HW_PSTATE_MAX_SHIFT	4
-#define MSR_PSTATE_DEF_BASE 	0xc0010064 /* base of Pstate MSRs */
-#define MSR_PSTATE_STATUS 	0xc0010063 /* Pstate Status MSR */
-#define MSR_PSTATE_CTRL 	0xc0010062 /* Pstate control MSR */
-#define MSR_PSTATE_CUR_LIMIT	0xc0010061 /* pstate current limit MSR */
-
-/* define the two driver architectures */
-#define CPU_OPTERON 0
-#define CPU_HW_PSTATE 1
-
-
 /*
  * There are restrictions frequencies have to follow:
  * - only 1 entry in the low fid table ( <=1.4GHz )
@@ -218,5 +187,4 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid);
 
 static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index);
 
-static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table);
 static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table);
diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c
index 58bf3b1ac9c..87db3877fea 100644
--- a/drivers/cpuidle/driver.c
+++ b/drivers/cpuidle/driver.c
@@ -18,9 +18,10 @@ static struct cpuidle_driver *cpuidle_curr_driver;
 DEFINE_SPINLOCK(cpuidle_driver_lock);
 int cpuidle_driver_refcount;
 
-static void __cpuidle_register_driver(struct cpuidle_driver *drv)
+static void set_power_states(struct cpuidle_driver *drv)
 {
 	int i;
+
 	/*
 	 * cpuidle driver should set the drv->power_specified bit
 	 * before registering if the driver provides
@@ -35,13 +36,10 @@ static void __cpuidle_register_driver(struct cpuidle_driver *drv)
 	 * an power value of -1.  So we use -2, -3, etc, for other
 	 * c-states.
 	 */
-	if (!drv->power_specified) {
-		for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++)
-			drv->states[i].power_usage = -1 - i;
-	}
+	for (i = CPUIDLE_DRIVER_STATE_START; i < drv->state_count; i++)
+		drv->states[i].power_usage = -1 - i;
 }
 
-
 /**
  * cpuidle_register_driver - registers a driver
  * @drv: the driver
@@ -59,13 +57,16 @@ int cpuidle_register_driver(struct cpuidle_driver *drv)
 		spin_unlock(&cpuidle_driver_lock);
 		return -EBUSY;
 	}
-	__cpuidle_register_driver(drv);
+
+	if (!drv->power_specified)
+		set_power_states(drv);
+
 	cpuidle_curr_driver = drv;
+
 	spin_unlock(&cpuidle_driver_lock);
 
 	return 0;
 }
-
 EXPORT_SYMBOL_GPL(cpuidle_register_driver);
 
 /**
@@ -96,7 +97,6 @@ void cpuidle_unregister_driver(struct cpuidle_driver *drv)
 
 	spin_unlock(&cpuidle_driver_lock);
 }
-
 EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
 
 struct cpuidle_driver *cpuidle_driver_ref(void)
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index b6a09ea859b..9b784051ec1 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -88,6 +88,8 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 
 	/* consider promotion */
 	if (last_idx < drv->state_count - 1 &&
+	    !drv->states[last_idx + 1].disabled &&
+	    !dev->states_usage[last_idx + 1].disable &&
 	    last_residency > last_state->threshold.promotion_time &&
 	    drv->states[last_idx + 1].exit_latency <= latency_req) {
 		last_state->stats.promotion_count++;
@@ -100,7 +102,9 @@ static int ladder_select_state(struct cpuidle_driver *drv,
 
 	/* consider demotion */
 	if (last_idx > CPUIDLE_DRIVER_STATE_START &&
-	    drv->states[last_idx].exit_latency > latency_req) {
+	    (drv->states[last_idx].disabled ||
+	    dev->states_usage[last_idx].disable ||
+	    drv->states[last_idx].exit_latency > latency_req)) {
 		int i;
 
 		for (i = last_idx - 1; i > CPUIDLE_DRIVER_STATE_START; i--) {
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 7d74d092aa8..308c7fb92a6 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -298,21 +298,15 @@ config CRYPTO_DEV_TEGRA_AES
 	  will be called tegra-aes.
 
 config CRYPTO_DEV_NX
-	tristate "Support for Power7+ in-Nest cryptographic acceleration"
+	bool "Support for IBM Power7+ in-Nest cryptographic acceleration"
 	depends on PPC64 && IBMVIO
-	select CRYPTO_AES
-	select CRYPTO_CBC
-	select CRYPTO_ECB
-	select CRYPTO_CCM
-	select CRYPTO_GCM
-	select CRYPTO_AUTHENC
-	select CRYPTO_XCBC
-	select CRYPTO_SHA256
-	select CRYPTO_SHA512
+	default n
 	help
-	  Support for Power7+ in-Nest cryptographic acceleration. This
-	  module supports acceleration for AES and SHA2 algorithms. If you
-	  choose 'M' here, this module will be called nx_crypto.
+	  Support for Power7+ in-Nest cryptographic acceleration.
+
+if CRYPTO_DEV_NX
+	source "drivers/crypto/nx/Kconfig"
+endif
 
 config CRYPTO_DEV_UX500
 	tristate "Driver for ST-Ericsson UX500 crypto hardware acceleration"
@@ -340,7 +334,7 @@ config CRYPTO_DEV_ATMEL_AES
 	select CRYPTO_AES
 	select CRYPTO_ALGAPI
 	select CRYPTO_BLKCIPHER
-	select CONFIG_AT_HDMAC
+	select AT_HDMAC
 	help
 	  Some Atmel processors have AES hw accelerator.
 	  Select this if you want to use the Atmel module for
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 802e85102c3..f88e3d8f6b6 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1226,6 +1226,7 @@ static int __init crypto4xx_probe(struct platform_device *ofdev)
 	core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0);
 	if (!core_dev->dev->ce_base) {
 		dev_err(dev, "failed to of_iomap\n");
+		rc = -ENOMEM;
 		goto err_iomap;
 	}
 
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 6bb20fffbf4..8061336e07e 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -24,15 +24,10 @@
 #include <linux/platform_device.h>
 
 #include <linux/device.h>
-#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/clk.h>
 #include <linux/irq.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <linux/dma-mapping.h>
 #include <linux/delay.h>
@@ -1017,7 +1012,6 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
 	int err, i, j;
 
 	for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
-		INIT_LIST_HEAD(&aes_algs[i].cra_list);
 		err = crypto_register_alg(&aes_algs[i]);
 		if (err)
 			goto err_aes_algs;
@@ -1026,7 +1020,6 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
 	atmel_aes_hw_version_init(dd);
 
 	if (dd->hw_version >= 0x130) {
-		INIT_LIST_HEAD(&aes_cfb64_alg[0].cra_list);
 		err = crypto_register_alg(&aes_cfb64_alg[0]);
 		if (err)
 			goto err_aes_cfb64_alg;
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index f938b9d79b6..bcdf55fdc62 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -24,15 +24,10 @@
 #include <linux/platform_device.h>
 
 #include <linux/device.h>
-#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/clk.h>
 #include <linux/irq.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <linux/dma-mapping.h>
 #include <linux/delay.h>
diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c
index eb2b61e57e2..7495f98c722 100644
--- a/drivers/crypto/atmel-tdes.c
+++ b/drivers/crypto/atmel-tdes.c
@@ -24,15 +24,10 @@
 #include <linux/platform_device.h>
 
 #include <linux/device.h>
-#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/clk.h>
 #include <linux/irq.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <linux/dma-mapping.h>
 #include <linux/delay.h>
@@ -1044,7 +1039,6 @@ static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)
 	int err, i, j;
 
 	for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) {
-		INIT_LIST_HEAD(&tdes_algs[i].cra_list);
 		err = crypto_register_alg(&tdes_algs[i]);
 		if (err)
 			goto err_tdes_algs;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 0c1ea8492ef..b2a0a0726a5 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -205,7 +205,7 @@ static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 {
 	u32 *key_jump_cmd;
 
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
@@ -224,7 +224,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	struct aead_tfm *tfm = &aead->base.crt_aead;
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
-	bool keys_fit_inline = 0;
+	bool keys_fit_inline = false;
 	u32 *key_jump_cmd, *jump_cmd;
 	u32 geniv, moveiv;
 	u32 *desc;
@@ -239,7 +239,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
 	    ctx->split_key_pad_len + ctx->enckeylen <=
 	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = 1;
+		keys_fit_inline = true;
 
 	/* aead_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
@@ -297,12 +297,12 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
 	    ctx->split_key_pad_len + ctx->enckeylen <=
 	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = 1;
+		keys_fit_inline = true;
 
 	desc = ctx->sh_desc_dec;
 
 	/* aead_decrypt shared descriptor */
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
@@ -365,7 +365,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
 	    ctx->split_key_pad_len + ctx->enckeylen <=
 	    CAAM_DESC_BYTES_MAX)
-		keys_fit_inline = 1;
+		keys_fit_inline = true;
 
 	/* aead_givencrypt shared descriptor */
 	desc = ctx->sh_desc_givenc;
@@ -564,7 +564,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 
 	/* ablkcipher_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 				   JUMP_COND_SHRD);
@@ -605,7 +605,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 	/* ablkcipher_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
 
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 				   JUMP_COND_SHRD);
@@ -1354,10 +1354,10 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
 		contig &= ~GIV_SRC_CONTIG;
 	if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
 		contig &= ~GIV_DST_CONTIG;
-		if (unlikely(req->src != req->dst)) {
-			dst_nents = dst_nents ? : 1;
-			sec4_sg_len += 1;
-		}
+	if (unlikely(req->src != req->dst)) {
+		dst_nents = dst_nents ? : 1;
+		sec4_sg_len += 1;
+	}
 	if (!(contig & GIV_SRC_CONTIG)) {
 		assoc_nents = assoc_nents ? : 1;
 		src_nents = src_nents ? : 1;
@@ -1650,7 +1650,11 @@ struct caam_alg_template {
 };
 
 static struct caam_alg_template driver_algs[] = {
-	/* single-pass ipsec_esp descriptor */
+	/*
+	 * single-pass ipsec_esp descriptor
+	 * authencesn(*,*) is also registered, although not present
+	 * explicitly here.
+	 */
 	{
 		.name = "authenc(hmac(md5),cbc(aes))",
 		.driver_name = "authenc-hmac-md5-cbc-aes-caam",
@@ -2213,7 +2217,9 @@ static int __init caam_algapi_init(void)
 	for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
 		/* TODO: check if h/w supports alg */
 		struct caam_crypto_alg *t_alg;
+		bool done = false;
 
+authencesn:
 		t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]);
 		if (IS_ERR(t_alg)) {
 			err = PTR_ERR(t_alg);
@@ -2227,8 +2233,25 @@ static int __init caam_algapi_init(void)
 			dev_warn(ctrldev, "%s alg registration failed\n",
 				t_alg->crypto_alg.cra_driver_name);
 			kfree(t_alg);
-		} else
+		} else {
 			list_add_tail(&t_alg->entry, &priv->alg_list);
+			if (driver_algs[i].type == CRYPTO_ALG_TYPE_AEAD &&
+			    !memcmp(driver_algs[i].name, "authenc", 7) &&
+			    !done) {
+				char *name;
+
+				name = driver_algs[i].name;
+				memmove(name + 10, name + 7, strlen(name) - 7);
+				memcpy(name + 7, "esn", 3);
+
+				name = driver_algs[i].driver_name;
+				memmove(name + 10, name + 7, strlen(name) - 7);
+				memcpy(name + 7, "esn", 3);
+
+				done = true;
+				goto authencesn;
+			}
+		}
 	}
 	if (!list_empty(&priv->alg_list))
 		dev_info(ctrldev, "%s algorithms registered in /proc/crypto\n",
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 895aaf2bca9..32aba7a6150 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -225,7 +225,7 @@ static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx)
 {
 	u32 *key_jump_cmd;
 
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
 	if (ctx->split_key_len) {
 		/* Skip if already shared */
@@ -311,7 +311,7 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 	/* ahash_update shared descriptor */
 	desc = ctx->sh_desc_update;
 
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
 	/* Import context from software */
 	append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
@@ -430,6 +430,10 @@ static u32 hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
 	int ret = 0;
 
 	desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+	if (!desc) {
+		dev_err(jrdev, "unable to allocate key input memory\n");
+		return -ENOMEM;
+	}
 
 	init_job_desc(desc, 0);
 
@@ -1736,8 +1740,11 @@ static void __exit caam_algapi_hash_exit(void)
 	struct caam_hash_alg *t_alg, *n;
 
 	dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-	if (!dev_node)
-		return;
+	if (!dev_node) {
+		dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
+		if (!dev_node)
+			return;
+	}
 
 	pdev = of_find_device_by_node(dev_node);
 	if (!pdev)
@@ -1812,8 +1819,11 @@ static int __init caam_algapi_hash_init(void)
 	int i = 0, err = 0;
 
 	dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-	if (!dev_node)
-		return -ENODEV;
+	if (!dev_node) {
+		dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
+		if (!dev_node)
+			return -ENODEV;
+	}
 
 	pdev = of_find_device_by_node(dev_node);
 	if (!pdev)
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index e2bfe161dec..d1939a9539c 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -193,7 +193,7 @@ static inline void rng_create_sh_desc(struct caam_rng_ctx *ctx)
 	struct device *jrdev = ctx->jrdev;
 	u32 *desc = ctx->sh_desc;
 
-	init_sh_desc(desc, HDR_SHARE_WAIT);
+	init_sh_desc(desc, HDR_SHARE_SERIAL);
 
 	/* Propagate errors from shared to job descriptor */
 	append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
@@ -284,8 +284,11 @@ static int __init caam_rng_init(void)
 	struct caam_drv_private *priv;
 
 	dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-	if (!dev_node)
-		return -ENODEV;
+	if (!dev_node) {
+		dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
+		if (!dev_node)
+			return -ENODEV;
+	}
 
 	pdev = of_find_device_by_node(dev_node);
 	if (!pdev)
diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h
index 762aeff626a..cf15e781380 100644
--- a/drivers/crypto/caam/compat.h
+++ b/drivers/crypto/caam/compat.h
@@ -23,6 +23,7 @@
 #include <linux/types.h>
 #include <linux/debugfs.h>
 #include <linux/circ_buf.h>
+#include <linux/string.h>
 #include <net/xfrm.h>
 
 #include <crypto/algapi.h>
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 414ba20c05a..bf20dd89170 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -129,7 +129,7 @@ static int instantiate_rng(struct device *jrdev)
 
 /*
  * By default, the TRNG runs for 200 clocks per sample;
- * 800 clocks per sample generates better entropy.
+ * 1600 clocks per sample generates better entropy.
  */
 static void kick_trng(struct platform_device *pdev)
 {
@@ -144,9 +144,9 @@ static void kick_trng(struct platform_device *pdev)
 
 	/* put RNG4 into program mode */
 	setbits32(&r4tst->rtmctl, RTMCTL_PRGM);
-	/* 800 clocks per sample */
+	/* 1600 clocks per sample */
 	val = rd_reg32(&r4tst->rtsdctl);
-	val = (val & ~RTSDCTL_ENT_DLY_MASK) | (800 << RTSDCTL_ENT_DLY_SHIFT);
+	val = (val & ~RTSDCTL_ENT_DLY_MASK) | (1600 << RTSDCTL_ENT_DLY_SHIFT);
 	wr_reg32(&r4tst->rtsdctl, val);
 	/* min. freq. count */
 	wr_reg32(&r4tst->rtfrqmin, 400);
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 9955ed9643e..30b8f74833d 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -77,10 +77,8 @@ static void report_ccb_status(u32 status, char *outstr)
 		"Not instantiated",
 		"Test instantiate",
 		"Prediction resistance",
-		"",
 		"Prediction resistance and test request",
 		"Uninstantiate",
-		"",
 		"Secure key generation",
 	};
 	u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >>
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index d216cd3cc56..f6dba10246c 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -54,6 +54,10 @@ u32 gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len,
 	int ret = 0;
 
 	desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+	if (!desc) {
+		dev_err(jrdev, "unable to allocate key input memory\n");
+		return -ENOMEM;
+	}
 
 	init_job_desc(desc, 0);
 
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
index f3e36c86b6c..51f196d77f2 100644
--- a/drivers/crypto/geode-aes.c
+++ b/drivers/crypto/geode-aes.c
@@ -289,7 +289,6 @@ static struct crypto_alg geode_alg = {
 	.cra_blocksize		=	AES_MIN_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct geode_aes_op),
 	.cra_module			=	THIS_MODULE,
-	.cra_list			=	LIST_HEAD_INIT(geode_alg.cra_list),
 	.cra_u				=	{
 		.cipher	=	{
 			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
@@ -402,7 +401,6 @@ static struct crypto_alg geode_cbc_alg = {
 	.cra_alignmask		=	15,
 	.cra_type			=	&crypto_blkcipher_type,
 	.cra_module			=	THIS_MODULE,
-	.cra_list			=	LIST_HEAD_INIT(geode_cbc_alg.cra_list),
 	.cra_u				=	{
 		.blkcipher	=	{
 			.min_keysize	=	AES_MIN_KEY_SIZE,
@@ -489,7 +487,6 @@ static struct crypto_alg geode_ecb_alg = {
 	.cra_alignmask		=	15,
 	.cra_type			=	&crypto_blkcipher_type,
 	.cra_module			=	THIS_MODULE,
-	.cra_list			=	LIST_HEAD_INIT(geode_ecb_alg.cra_list),
 	.cra_u				=	{
 		.blkcipher	=	{
 			.min_keysize	=	AES_MIN_KEY_SIZE,
@@ -588,21 +585,8 @@ static struct pci_driver geode_aes_driver = {
 	.remove = __devexit_p(geode_aes_remove)
 };
 
-static int __init
-geode_aes_init(void)
-{
-	return pci_register_driver(&geode_aes_driver);
-}
-
-static void __exit
-geode_aes_exit(void)
-{
-	pci_unregister_driver(&geode_aes_driver);
-}
+module_pci_driver(geode_aes_driver);
 
 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
 MODULE_LICENSE("GPL");
-
-module_init(geode_aes_init);
-module_exit(geode_aes_exit);
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index df14358d7fa..fda32968a66 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -2611,14 +2611,17 @@ static int __devinit hifn_probe(struct pci_dev *pdev, const struct pci_device_id
 		size = pci_resource_len(pdev, i);
 
 		dev->bar[i] = ioremap_nocache(addr, size);
-		if (!dev->bar[i])
+		if (!dev->bar[i]) {
+			err = -ENOMEM;
 			goto err_out_unmap_bars;
+		}
 	}
 
 	dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
 			&dev->desc_dma);
 	if (!dev->desc_virt) {
 		dprintk("Failed to allocate descriptor rings.\n");
+		err = -ENOMEM;
 		goto err_out_unmap_bars;
 	}
 	memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig
new file mode 100644
index 00000000000..f82616621ae
--- /dev/null
+++ b/drivers/crypto/nx/Kconfig
@@ -0,0 +1,26 @@
+config CRYPTO_DEV_NX_ENCRYPT
+	tristate "Encryption acceleration support"
+	depends on PPC64 && IBMVIO
+	default y
+	select CRYPTO_AES
+	select CRYPTO_CBC
+	select CRYPTO_ECB
+	select CRYPTO_CCM
+	select CRYPTO_GCM
+	select CRYPTO_AUTHENC
+	select CRYPTO_XCBC
+	select CRYPTO_SHA256
+	select CRYPTO_SHA512
+	help
+	  Support for Power7+ in-Nest encryption acceleration. This
+	  module supports acceleration for AES and SHA2 algorithms. If you
+	  choose 'M' here, this module will be called nx_crypto.
+
+config CRYPTO_DEV_NX_COMPRESS
+	tristate "Compression acceleration support"
+	depends on PPC64 && IBMVIO
+	default y
+	help
+	  Support for Power7+ in-Nest compression acceleration. This
+	  module supports acceleration for AES and SHA2 algorithms. If you
+	  choose 'M' here, this module will be called nx_compress.
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
index 411ce59c80d..bb770ea45ce 100644
--- a/drivers/crypto/nx/Makefile
+++ b/drivers/crypto/nx/Makefile
@@ -1,4 +1,4 @@
-obj-$(CONFIG_CRYPTO_DEV_NX) += nx-crypto.o
+obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o
 nx-crypto-objs := nx.o \
 		  nx_debugfs.o \
 		  nx-aes-cbc.o \
@@ -9,3 +9,6 @@ nx-crypto-objs := nx.o \
 		  nx-aes-xcbc.o \
 		  nx-sha256.o \
 		  nx-sha512.o
+
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o
+nx-compress-objs := nx-842.o
diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c
new file mode 100644
index 00000000000..0ce62573867
--- /dev/null
+++ b/drivers/crypto/nx/nx-842.c
@@ -0,0 +1,1617 @@
+/*
+ * Driver for IBM Power 842 compression accelerator
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ *          Seth Jennings <sjenning@linux.vnet.ibm.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nx842.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+
+#include <asm/page.h>
+#include <asm/pSeries_reconfig.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h" /* struct nx_csbcpb */
+
+#define MODULE_NAME "nx-compress"
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
+
+#define SHIFT_4K 12
+#define SHIFT_64K 16
+#define SIZE_4K (1UL << SHIFT_4K)
+#define SIZE_64K (1UL << SHIFT_64K)
+
+/* IO buffer must be 128 byte aligned */
+#define IO_BUFFER_ALIGN 128
+
+struct nx842_header {
+	int blocks_nr; /* number of compressed blocks */
+	int offset; /* offset of the first block (from beginning of header) */
+	int sizes[0]; /* size of compressed blocks */
+};
+
+static inline int nx842_header_size(const struct nx842_header *hdr)
+{
+	return sizeof(struct nx842_header) +
+			hdr->blocks_nr * sizeof(hdr->sizes[0]);
+}
+
+/* Macros for fields within nx_csbcpb */
+/* Check the valid bit within the csbcpb valid field */
+#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
+
+/* CE macros operate on the completion_extension field bits in the csbcpb.
+ * CE0 0=full completion, 1=partial completion
+ * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
+ * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
+#define NX842_CSBCPB_CE0(x)	(x & BIT_MASK(7))
+#define NX842_CSBCPB_CE1(x)	(x & BIT_MASK(6))
+#define NX842_CSBCPB_CE2(x)	(x & BIT_MASK(5))
+
+/* The NX unit accepts data only on 4K page boundaries */
+#define NX842_HW_PAGE_SHIFT	SHIFT_4K
+#define NX842_HW_PAGE_SIZE	(ASM_CONST(1) << NX842_HW_PAGE_SHIFT)
+#define NX842_HW_PAGE_MASK	(~(NX842_HW_PAGE_SIZE-1))
+
+enum nx842_status {
+	UNAVAILABLE,
+	AVAILABLE
+};
+
+struct ibm_nx842_counters {
+	atomic64_t comp_complete;
+	atomic64_t comp_failed;
+	atomic64_t decomp_complete;
+	atomic64_t decomp_failed;
+	atomic64_t swdecomp;
+	atomic64_t comp_times[32];
+	atomic64_t decomp_times[32];
+};
+
+static struct nx842_devdata {
+	struct vio_dev *vdev;
+	struct device *dev;
+	struct ibm_nx842_counters *counters;
+	unsigned int max_sg_len;
+	unsigned int max_sync_size;
+	unsigned int max_sync_sg;
+	enum nx842_status status;
+} __rcu *devdata;
+static DEFINE_SPINLOCK(devdata_mutex);
+
+#define NX842_COUNTER_INC(_x) \
+static inline void nx842_inc_##_x( \
+	const struct nx842_devdata *dev) { \
+	if (dev) \
+		atomic64_inc(&dev->counters->_x); \
+}
+NX842_COUNTER_INC(comp_complete);
+NX842_COUNTER_INC(comp_failed);
+NX842_COUNTER_INC(decomp_complete);
+NX842_COUNTER_INC(decomp_failed);
+NX842_COUNTER_INC(swdecomp);
+
+#define NX842_HIST_SLOTS 16
+
+static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
+{
+	int bucket = fls(time);
+
+	if (bucket)
+		bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
+
+	atomic64_inc(&times[bucket]);
+}
+
+/* NX unit operation flags */
+#define NX842_OP_COMPRESS	0x0
+#define NX842_OP_CRC		0x1
+#define NX842_OP_DECOMPRESS	0x2
+#define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
+#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
+#define NX842_OP_ASYNC		(1<<23)
+#define NX842_OP_NOTIFY		(1<<22)
+#define NX842_OP_NOTIFY_INT(x)	((x & 0xff)<<8)
+
+static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
+{
+	/* No use of DMA mappings within the driver. */
+	return 0;
+}
+
+struct nx842_slentry {
+	unsigned long ptr; /* Real address (use __pa()) */
+	unsigned long len;
+};
+
+/* pHyp scatterlist entry */
+struct nx842_scatterlist {
+	int entry_nr; /* number of slentries */
+	struct nx842_slentry *entries; /* ptr to array of slentries */
+};
+
+/* Does not include sizeof(entry_nr) in the size */
+static inline unsigned long nx842_get_scatterlist_size(
+				struct nx842_scatterlist *sl)
+{
+	return sl->entry_nr * sizeof(struct nx842_slentry);
+}
+
+static int nx842_build_scatterlist(unsigned long buf, int len,
+			struct nx842_scatterlist *sl)
+{
+	unsigned long nextpage;
+	struct nx842_slentry *entry;
+
+	sl->entry_nr = 0;
+
+	entry = sl->entries;
+	while (len) {
+		entry->ptr = __pa(buf);
+		nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE);
+		if (nextpage < buf + len) {
+			/* we aren't at the end yet */
+			if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE))
+				/* we are in the middle (or beginning) */
+				entry->len = NX842_HW_PAGE_SIZE;
+			else
+				/* we are at the beginning */
+				entry->len = nextpage - buf;
+		} else {
+			/* at the end */
+			entry->len = len;
+		}
+
+		len -= entry->len;
+		buf += entry->len;
+		sl->entry_nr++;
+		entry++;
+	}
+
+	return 0;
+}
+
+/*
+ * Working memory for software decompression
+ */
+struct sw842_fifo {
+	union {
+		char f8[256][8];
+		char f4[512][4];
+	};
+	char f2[256][2];
+	unsigned char f84_full;
+	unsigned char f2_full;
+	unsigned char f8_count;
+	unsigned char f2_count;
+	unsigned int f4_count;
+};
+
+/*
+ * Working memory for crypto API
+ */
+struct nx842_workmem {
+	char bounce[PAGE_SIZE]; /* bounce buffer for decompression input */
+	union {
+		/* hardware working memory */
+		struct {
+			/* scatterlist */
+			char slin[SIZE_4K];
+			char slout[SIZE_4K];
+			/* coprocessor status/parameter block */
+			struct nx_csbcpb csbcpb;
+		};
+		/* software working memory */
+		struct sw842_fifo swfifo; /* software decompression fifo */
+	};
+};
+
+int nx842_get_workmem_size(void)
+{
+	return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE;
+}
+EXPORT_SYMBOL_GPL(nx842_get_workmem_size);
+
+int nx842_get_workmem_size_aligned(void)
+{
+	return sizeof(struct nx842_workmem);
+}
+EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned);
+
+static int nx842_validate_result(struct device *dev,
+	struct cop_status_block *csb)
+{
+	/* The csb must be valid after returning from vio_h_cop_sync */
+	if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
+		dev_err(dev, "%s: cspcbp not valid upon completion.\n",
+				__func__);
+		dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
+				csb->valid,
+				csb->crb_seq_number,
+				csb->completion_code,
+				csb->completion_extension);
+		dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
+				csb->processed_byte_count,
+				(unsigned long)csb->address);
+		return -EIO;
+	}
+
+	/* Check return values from the hardware in the CSB */
+	switch (csb->completion_code) {
+	case 0:	/* Completed without error */
+		break;
+	case 64: /* Target bytes > Source bytes during compression */
+	case 13: /* Output buffer too small */
+		dev_dbg(dev, "%s: Compression output larger than input\n",
+					__func__);
+		return -ENOSPC;
+	case 66: /* Input data contains an illegal template field */
+	case 67: /* Template indicates data past the end of the input stream */
+		dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
+					__func__, csb->completion_code);
+		return -EINVAL;
+	default:
+		dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
+					__func__, csb->completion_code);
+		return -EIO;
+	}
+
+	/* Hardware sanity check */
+	if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
+		dev_err(dev, "%s: No error returned by hardware, but "
+				"data returned is unusable, contact support.\n"
+				"(Additional info: csbcbp->processed bytes "
+				"does not specify processed bytes for the "
+				"target buffer.)\n", __func__);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * nx842_compress - Compress data using the 842 algorithm
+ *
+ * Compression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data.  If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer, must be page aligned
+ * @inlen: Length of input buffer, must be PAGE_SIZE
+ * @out: Pointer to output buffer
+ * @outlen: Length of output buffer
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ *          nx842_get_workmem_size()
+ *
+ * Returns:
+ *   0		Success, output of length @outlen stored in the buffer at @out
+ *   -ENOMEM	Unable to allocate internal buffers
+ *   -ENOSPC	Output buffer is to small
+ *   -EMSGSIZE	XXX Difficult to describe this limitation
+ *   -EIO	Internal error
+ *   -ENODEV	Hardware unavailable
+ */
+int nx842_compress(const unsigned char *in, unsigned int inlen,
+		       unsigned char *out, unsigned int *outlen, void *wmem)
+{
+	struct nx842_header *hdr;
+	struct nx842_devdata *local_devdata;
+	struct device *dev = NULL;
+	struct nx842_workmem *workmem;
+	struct nx842_scatterlist slin, slout;
+	struct nx_csbcpb *csbcpb;
+	int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
+	unsigned long inbuf, outbuf, padding;
+	struct vio_pfo_op op = {
+		.done = NULL,
+		.handle = 0,
+		.timeout = 0,
+	};
+	unsigned long start_time = get_tb();
+
+	/*
+	 * Make sure input buffer is 64k page aligned.  This is assumed since
+	 * this driver is designed for page compression only (for now).  This
+	 * is very nice since we can now use direct DDE(s) for the input and
+	 * the alignment is guaranteed.
+	*/
+	inbuf = (unsigned long)in;
+	if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
+		return -EINVAL;
+
+	rcu_read_lock();
+	local_devdata = rcu_dereference(devdata);
+	if (!local_devdata || !local_devdata->dev) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+	max_sync_size = local_devdata->max_sync_size;
+	dev = local_devdata->dev;
+
+	/* Create the header */
+	hdr = (struct nx842_header *)out;
+	hdr->blocks_nr = PAGE_SIZE / max_sync_size;
+	hdrsize = nx842_header_size(hdr);
+	outbuf = (unsigned long)out + hdrsize;
+	bytesleft = *outlen - hdrsize;
+
+	/* Init scatterlist */
+	workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
+		NX842_HW_PAGE_SIZE);
+	slin.entries = (struct nx842_slentry *)workmem->slin;
+	slout.entries = (struct nx842_slentry *)workmem->slout;
+
+	/* Init operation */
+	op.flags = NX842_OP_COMPRESS;
+	csbcpb = &workmem->csbcpb;
+	memset(csbcpb, 0, sizeof(*csbcpb));
+	op.csbcpb = __pa(csbcpb);
+	op.out = __pa(slout.entries);
+
+	for (i = 0; i < hdr->blocks_nr; i++) {
+		/*
+		 * Aligning the output blocks to 128 bytes does waste space,
+		 * but it prevents the need for bounce buffers and memory
+		 * copies.  It also simplifies the code a lot.  In the worst
+		 * case (64k page, 4k max_sync_size), you lose up to
+		 * (128*16)/64k = ~3% the compression factor. For 64k
+		 * max_sync_size, the loss would be at most 128/64k = ~0.2%.
+		 */
+		padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
+		outbuf += padding;
+		bytesleft -= padding;
+		if (i == 0)
+			/* save offset into first block in header */
+			hdr->offset = padding + hdrsize;
+
+		if (bytesleft <= 0) {
+			ret = -ENOSPC;
+			goto unlock;
+		}
+
+		/*
+		 * NOTE: If the default max_sync_size is changed from 4k
+		 * to 64k, remove the "likely" case below, since a
+		 * scatterlist will always be needed.
+		 */
+		if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
+			/* Create direct DDE */
+			op.in = __pa(inbuf);
+			op.inlen = max_sync_size;
+
+		} else {
+			/* Create indirect DDE (scatterlist) */
+			nx842_build_scatterlist(inbuf, max_sync_size, &slin);
+			op.in = __pa(slin.entries);
+			op.inlen = -nx842_get_scatterlist_size(&slin);
+		}
+
+		/*
+		 * If max_sync_size != NX842_HW_PAGE_SIZE, an indirect
+		 * DDE is required for the outbuf.
+		 * If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must
+		 * also be page aligned (1 in 128/4k=32 chance) in order
+		 * to use a direct DDE.
+		 * This is unlikely, just use an indirect DDE always.
+		 */
+		nx842_build_scatterlist(outbuf,
+			min(bytesleft, max_sync_size), &slout);
+		/* op.out set before loop */
+		op.outlen = -nx842_get_scatterlist_size(&slout);
+
+		/* Send request to pHyp */
+		ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+		/* Check for pHyp error */
+		if (ret) {
+			dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+				__func__, ret, op.hcall_err);
+			ret = -EIO;
+			goto unlock;
+		}
+
+		/* Check for hardware error */
+		ret = nx842_validate_result(dev, &csbcpb->csb);
+		if (ret && ret != -ENOSPC)
+			goto unlock;
+
+		/* Handle incompressible data */
+		if (unlikely(ret == -ENOSPC)) {
+			if (bytesleft < max_sync_size) {
+				/*
+				 * Not enough space left in the output buffer
+				 * to store uncompressed block
+				 */
+				goto unlock;
+			} else {
+				/* Store incompressible block */
+				memcpy((void *)outbuf, (void *)inbuf,
+					max_sync_size);
+				hdr->sizes[i] = -max_sync_size;
+				outbuf += max_sync_size;
+				bytesleft -= max_sync_size;
+				/* Reset ret, incompressible data handled */
+				ret = 0;
+			}
+		} else {
+			/* Normal case, compression was successful */
+			size = csbcpb->csb.processed_byte_count;
+			dev_dbg(dev, "%s: processed_bytes=%d\n",
+				__func__, size);
+			hdr->sizes[i] = size;
+			outbuf += size;
+			bytesleft -= size;
+		}
+
+		inbuf += max_sync_size;
+	}
+
+	*outlen = (unsigned int)(outbuf - (unsigned long)out);
+
+unlock:
+	if (ret)
+		nx842_inc_comp_failed(local_devdata);
+	else {
+		nx842_inc_comp_complete(local_devdata);
+		ibm_nx842_incr_hist(local_devdata->counters->comp_times,
+			(get_tb() - start_time) / tb_ticks_per_usec);
+	}
+	rcu_read_unlock();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_compress);
+
+static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
+			const void *);
+
+/**
+ * nx842_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer.  The size allocated to the output buffer is
+ * provided by the caller of this function in @outlen.  Upon return from
+ * this function @outlen contains the length of the decompressed data.
+ * If there is an error then @outlen will be 0 and an error will be
+ * specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer, will use bounce buffer if not 128 byte
+ *      aligned
+ * @inlen: Length of input buffer
+ * @out: Pointer to output buffer, must be page aligned
+ * @outlen: Length of output buffer, must be PAGE_SIZE
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ *          nx842_get_workmem_size()
+ *
+ * Returns:
+ *   0		Success, output of length @outlen stored in the buffer at @out
+ *   -ENODEV	Hardware decompression device is unavailable
+ *   -ENOMEM	Unable to allocate internal buffers
+ *   -ENOSPC	Output buffer is to small
+ *   -EINVAL	Bad input data encountered when attempting decompress
+ *   -EIO	Internal error
+ */
+int nx842_decompress(const unsigned char *in, unsigned int inlen,
+			 unsigned char *out, unsigned int *outlen, void *wmem)
+{
+	struct nx842_header *hdr;
+	struct nx842_devdata *local_devdata;
+	struct device *dev = NULL;
+	struct nx842_workmem *workmem;
+	struct nx842_scatterlist slin, slout;
+	struct nx_csbcpb *csbcpb;
+	int ret = 0, i, size, max_sync_size;
+	unsigned long inbuf, outbuf;
+	struct vio_pfo_op op = {
+		.done = NULL,
+		.handle = 0,
+		.timeout = 0,
+	};
+	unsigned long start_time = get_tb();
+
+	/* Ensure page alignment and size */
+	outbuf = (unsigned long)out;
+	if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
+		return -EINVAL;
+
+	rcu_read_lock();
+	local_devdata = rcu_dereference(devdata);
+	if (local_devdata)
+		dev = local_devdata->dev;
+
+	/* Get header */
+	hdr = (struct nx842_header *)in;
+
+	workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
+		NX842_HW_PAGE_SIZE);
+
+	inbuf = (unsigned long)in + hdr->offset;
+	if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
+		/* Copy block(s) into bounce buffer for alignment */
+		memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
+		inbuf = (unsigned long)workmem->bounce;
+	}
+
+	/* Init scatterlist */
+	slin.entries = (struct nx842_slentry *)workmem->slin;
+	slout.entries = (struct nx842_slentry *)workmem->slout;
+
+	/* Init operation */
+	op.flags = NX842_OP_DECOMPRESS;
+	csbcpb = &workmem->csbcpb;
+	memset(csbcpb, 0, sizeof(*csbcpb));
+	op.csbcpb = __pa(csbcpb);
+
+	/*
+	 * max_sync_size may have changed since compression,
+	 * so we can't read it from the device info. We need
+	 * to derive it from hdr->blocks_nr.
+	 */
+	max_sync_size = PAGE_SIZE / hdr->blocks_nr;
+
+	for (i = 0; i < hdr->blocks_nr; i++) {
+		/* Skip padding */
+		inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
+
+		if (hdr->sizes[i] < 0) {
+			/* Negative sizes indicate uncompressed data blocks */
+			size = abs(hdr->sizes[i]);
+			memcpy((void *)outbuf, (void *)inbuf, size);
+			outbuf += size;
+			inbuf += size;
+			continue;
+		}
+
+		if (!dev)
+			goto sw;
+
+		/*
+		 * The better the compression, the more likely the "likely"
+		 * case becomes.
+		 */
+		if (likely((inbuf & NX842_HW_PAGE_MASK) ==
+			((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
+			/* Create direct DDE */
+			op.in = __pa(inbuf);
+			op.inlen = hdr->sizes[i];
+		} else {
+			/* Create indirect DDE (scatterlist) */
+			nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
+			op.in = __pa(slin.entries);
+			op.inlen = -nx842_get_scatterlist_size(&slin);
+		}
+
+		/*
+		 * NOTE: If the default max_sync_size is changed from 4k
+		 * to 64k, remove the "likely" case below, since a
+		 * scatterlist will always be needed.
+		 */
+		if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
+			/* Create direct DDE */
+			op.out = __pa(outbuf);
+			op.outlen = max_sync_size;
+		} else {
+			/* Create indirect DDE (scatterlist) */
+			nx842_build_scatterlist(outbuf, max_sync_size, &slout);
+			op.out = __pa(slout.entries);
+			op.outlen = -nx842_get_scatterlist_size(&slout);
+		}
+
+		/* Send request to pHyp */
+		ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+		/* Check for pHyp error */
+		if (ret) {
+			dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+				__func__, ret, op.hcall_err);
+			dev = NULL;
+			goto sw;
+		}
+
+		/* Check for hardware error */
+		ret = nx842_validate_result(dev, &csbcpb->csb);
+		if (ret) {
+			dev = NULL;
+			goto sw;
+		}
+
+		/* HW decompression success */
+		inbuf += hdr->sizes[i];
+		outbuf += csbcpb->csb.processed_byte_count;
+		continue;
+
+sw:
+		/* software decompression */
+		size = max_sync_size;
+		ret = sw842_decompress(
+			(unsigned char *)inbuf, hdr->sizes[i],
+			(unsigned char *)outbuf, &size, wmem);
+		if (ret)
+			pr_debug("%s: sw842_decompress failed with %d\n",
+				__func__, ret);
+
+		if (ret) {
+			if (ret != -ENOSPC && ret != -EINVAL &&
+					ret != -EMSGSIZE)
+				ret = -EIO;
+			goto unlock;
+		}
+
+		/* SW decompression success */
+		inbuf += hdr->sizes[i];
+		outbuf += size;
+	}
+
+	*outlen = (unsigned int)(outbuf - (unsigned long)out);
+
+unlock:
+	if (ret)
+		/* decompress fail */
+		nx842_inc_decomp_failed(local_devdata);
+	else {
+		if (!dev)
+			/* software decompress */
+			nx842_inc_swdecomp(local_devdata);
+		nx842_inc_decomp_complete(local_devdata);
+		ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
+			(get_tb() - start_time) / tb_ticks_per_usec);
+	}
+
+	rcu_read_unlock();
+	return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_decompress);
+
+/**
+ * nx842_OF_set_defaults -- Set default (disabled) values for devdata
+ *
+ * @devdata - struct nx842_devdata to update
+ *
+ * Returns:
+ *  0 on success
+ *  -ENOENT if @devdata ptr is NULL
+ */
+static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
+{
+	if (devdata) {
+		devdata->max_sync_size = 0;
+		devdata->max_sync_sg = 0;
+		devdata->max_sg_len = 0;
+		devdata->status = UNAVAILABLE;
+		return 0;
+	} else
+		return -ENOENT;
+}
+
+/**
+ * nx842_OF_upd_status -- Update the device info from OF status prop
+ *
+ * The status property indicates if the accelerator is enabled.  If the
+ * device is in the OF tree it indicates that the hardware is present.
+ * The status field indicates if the device is enabled when the status
+ * is 'okay'.  Otherwise the device driver will be disabled.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ *  0 - Device is available
+ *  -EINVAL - Device is not available
+ */
+static int nx842_OF_upd_status(struct nx842_devdata *devdata,
+					struct property *prop) {
+	int ret = 0;
+	const char *status = (const char *)prop->value;
+
+	if (!strncmp(status, "okay", (size_t)prop->length)) {
+		devdata->status = AVAILABLE;
+	} else {
+		dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
+				__func__, status);
+		devdata->status = UNAVAILABLE;
+	}
+
+	return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
+ *
+ * Definition of the 'ibm,max-sg-len' OF property:
+ *  This field indicates the maximum byte length of a scatter list
+ *  for the platform facility. It is a single cell encoded as with encode-int.
+ *
+ * Example:
+ *  # od -x ibm,max-sg-len
+ *  0000000 0000 0ff0
+ *
+ *  In this example, the maximum byte length of a scatter list is
+ *  0x0ff0 (4,080).
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ *  0 on success
+ *  -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
+					struct property *prop) {
+	int ret = 0;
+	const int *maxsglen = prop->value;
+
+	if (prop->length != sizeof(*maxsglen)) {
+		dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
+		dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
+				prop->length, sizeof(*maxsglen));
+		ret = -EINVAL;
+	} else {
+		devdata->max_sg_len = (unsigned int)min(*maxsglen,
+				(int)NX842_HW_PAGE_SIZE);
+	}
+
+	return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
+ *
+ * Definition of the 'ibm,max-sync-cop' OF property:
+ *  Two series of cells.  The first series of cells represents the maximums
+ *  that can be synchronously compressed. The second series of cells
+ *  represents the maximums that can be synchronously decompressed.
+ *  1. The first cell in each series contains the count of the number of
+ *     data length, scatter list elements pairs that follow – each being
+ *     of the form
+ *    a. One cell data byte length
+ *    b. One cell total number of scatter list elements
+ *
+ * Example:
+ *  # od -x ibm,max-sync-cop
+ *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
+ *  0000020 0000 1000 0000 01fe
+ *
+ *  In this example, compression supports 0x1000 (4,096) data byte length
+ *  and 0x1fe (510) total scatter list elements.  Decompression supports
+ *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
+ *  elements.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ *  0 on success
+ *  -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
+					struct property *prop) {
+	int ret = 0;
+	const struct maxsynccop_t {
+		int comp_elements;
+		int comp_data_limit;
+		int comp_sg_limit;
+		int decomp_elements;
+		int decomp_data_limit;
+		int decomp_sg_limit;
+	} *maxsynccop;
+
+	if (prop->length != sizeof(*maxsynccop)) {
+		dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
+		dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
+				sizeof(*maxsynccop));
+		ret = -EINVAL;
+		goto out;
+	}
+
+	maxsynccop = (const struct maxsynccop_t *)prop->value;
+
+	/* Use one limit rather than separate limits for compression and
+	 * decompression. Set a maximum for this so as not to exceed the
+	 * size that the header can support and round the value down to
+	 * the hardware page size (4K) */
+	devdata->max_sync_size =
+			(unsigned int)min(maxsynccop->comp_data_limit,
+					maxsynccop->decomp_data_limit);
+
+	devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
+					SIZE_64K);
+
+	if (devdata->max_sync_size < SIZE_4K) {
+		dev_err(devdata->dev, "%s: hardware max data size (%u) is "
+				"less than the driver minimum, unable to use "
+				"the hardware device\n",
+				__func__, devdata->max_sync_size);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit,
+						maxsynccop->decomp_sg_limit);
+	if (devdata->max_sync_sg < 1) {
+		dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
+				"less than the driver minimum, unable to use "
+				"the hardware device\n",
+				__func__, devdata->max_sync_sg);
+		ret = -EINVAL;
+		goto out;
+	}
+
+out:
+	return ret;
+}
+
+/**
+ *
+ * nx842_OF_upd -- Handle OF properties updates for the device.
+ *
+ * Set all properties from the OF tree.  Optionally, a new property
+ * can be provided by the @new_prop pointer to overwrite an existing value.
+ * The device will remain disabled until all values are valid, this function
+ * will return an error for updates unless all values are valid.
+ *
+ * @new_prop: If not NULL, this property is being updated.  If NULL, update
+ *  all properties from the current values in the OF tree.
+ *
+ * Returns:
+ *  0 - Success
+ *  -ENOMEM - Could not allocate memory for new devdata structure
+ *  -EINVAL - property value not found, new_prop is not a recognized
+ *	property for the device or property value is not valid.
+ *  -ENODEV - Device is not available
+ */
+static int nx842_OF_upd(struct property *new_prop)
+{
+	struct nx842_devdata *old_devdata = NULL;
+	struct nx842_devdata *new_devdata = NULL;
+	struct device_node *of_node = NULL;
+	struct property *status = NULL;
+	struct property *maxsglen = NULL;
+	struct property *maxsyncop = NULL;
+	int ret = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&devdata_mutex, flags);
+	old_devdata = rcu_dereference_check(devdata,
+			lockdep_is_held(&devdata_mutex));
+	if (old_devdata)
+		of_node = old_devdata->dev->of_node;
+
+	if (!old_devdata || !of_node) {
+		pr_err("%s: device is not available\n", __func__);
+		spin_unlock_irqrestore(&devdata_mutex, flags);
+		return -ENODEV;
+	}
+
+	new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+	if (!new_devdata) {
+		dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
+		ret = -ENOMEM;
+		goto error_out;
+	}
+
+	memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
+	new_devdata->counters = old_devdata->counters;
+
+	/* Set ptrs for existing properties */
+	status = of_find_property(of_node, "status", NULL);
+	maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
+	maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
+	if (!status || !maxsglen || !maxsyncop) {
+		dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
+		ret = -EINVAL;
+		goto error_out;
+	}
+
+	/* Set ptr to new property if provided */
+	if (new_prop) {
+		/* Single property */
+		if (!strncmp(new_prop->name, "status", new_prop->length)) {
+			status = new_prop;
+
+		} else if (!strncmp(new_prop->name, "ibm,max-sg-len",
+					new_prop->length)) {
+			maxsglen = new_prop;
+
+		} else if (!strncmp(new_prop->name, "ibm,max-sync-cop",
+					new_prop->length)) {
+			maxsyncop = new_prop;
+
+		} else {
+			/*
+			 * Skip the update, the property being updated
+			 * has no impact.
+			 */
+			goto out;
+		}
+	}
+
+	/* Perform property updates */
+	ret = nx842_OF_upd_status(new_devdata, status);
+	if (ret)
+		goto error_out;
+
+	ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
+	if (ret)
+		goto error_out;
+
+	ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
+	if (ret)
+		goto error_out;
+
+out:
+	dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
+			__func__, new_devdata->max_sync_size,
+			old_devdata->max_sync_size);
+	dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
+			__func__, new_devdata->max_sync_sg,
+			old_devdata->max_sync_sg);
+	dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
+			__func__, new_devdata->max_sg_len,
+			old_devdata->max_sg_len);
+
+	rcu_assign_pointer(devdata, new_devdata);
+	spin_unlock_irqrestore(&devdata_mutex, flags);
+	synchronize_rcu();
+	dev_set_drvdata(new_devdata->dev, new_devdata);
+	kfree(old_devdata);
+	return 0;
+
+error_out:
+	if (new_devdata) {
+		dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
+		nx842_OF_set_defaults(new_devdata);
+		rcu_assign_pointer(devdata, new_devdata);
+		spin_unlock_irqrestore(&devdata_mutex, flags);
+		synchronize_rcu();
+		dev_set_drvdata(new_devdata->dev, new_devdata);
+		kfree(old_devdata);
+	} else {
+		dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
+		spin_unlock_irqrestore(&devdata_mutex, flags);
+	}
+
+	if (!ret)
+		ret = -EINVAL;
+	return ret;
+}
+
+/**
+ * nx842_OF_notifier - Process updates to OF properties for the device
+ *
+ * @np: notifier block
+ * @action: notifier action
+ * @update: struct pSeries_reconfig_prop_update pointer if action is
+ *	PSERIES_UPDATE_PROPERTY
+ *
+ * Returns:
+ *	NOTIFY_OK on success
+ *	NOTIFY_BAD encoded with error number on failure, use
+ *		notifier_to_errno() to decode this value
+ */
+static int nx842_OF_notifier(struct notifier_block *np,
+					unsigned long action,
+					void *update)
+{
+	struct pSeries_reconfig_prop_update *upd;
+	struct nx842_devdata *local_devdata;
+	struct device_node *node = NULL;
+
+	upd = (struct pSeries_reconfig_prop_update *)update;
+
+	rcu_read_lock();
+	local_devdata = rcu_dereference(devdata);
+	if (local_devdata)
+		node = local_devdata->dev->of_node;
+
+	if (local_devdata &&
+			action == PSERIES_UPDATE_PROPERTY &&
+			!strcmp(upd->node->name, node->name)) {
+		rcu_read_unlock();
+		nx842_OF_upd(upd->property);
+	} else
+		rcu_read_unlock();
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block nx842_of_nb = {
+	.notifier_call = nx842_OF_notifier,
+};
+
+#define nx842_counter_read(_name)					\
+static ssize_t nx842_##_name##_show(struct device *dev,		\
+		struct device_attribute *attr,				\
+		char *buf) {						\
+	struct nx842_devdata *local_devdata;			\
+	int p = 0;							\
+	rcu_read_lock();						\
+	local_devdata = rcu_dereference(devdata);			\
+	if (local_devdata)						\
+		p = snprintf(buf, PAGE_SIZE, "%ld\n",			\
+		       atomic64_read(&local_devdata->counters->_name));	\
+	rcu_read_unlock();						\
+	return p;							\
+}
+
+#define NX842DEV_COUNTER_ATTR_RO(_name)					\
+	nx842_counter_read(_name);					\
+	static struct device_attribute dev_attr_##_name = __ATTR(_name,	\
+						0444,			\
+						nx842_##_name##_show,\
+						NULL);
+
+NX842DEV_COUNTER_ATTR_RO(comp_complete);
+NX842DEV_COUNTER_ATTR_RO(comp_failed);
+NX842DEV_COUNTER_ATTR_RO(decomp_complete);
+NX842DEV_COUNTER_ATTR_RO(decomp_failed);
+NX842DEV_COUNTER_ATTR_RO(swdecomp);
+
+static ssize_t nx842_timehist_show(struct device *,
+		struct device_attribute *, char *);
+
+static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
+		nx842_timehist_show, NULL);
+static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
+		0444, nx842_timehist_show, NULL);
+
+static ssize_t nx842_timehist_show(struct device *dev,
+		struct device_attribute *attr, char *buf) {
+	char *p = buf;
+	struct nx842_devdata *local_devdata;
+	atomic64_t *times;
+	int bytes_remain = PAGE_SIZE;
+	int bytes;
+	int i;
+
+	rcu_read_lock();
+	local_devdata = rcu_dereference(devdata);
+	if (!local_devdata) {
+		rcu_read_unlock();
+		return 0;
+	}
+
+	if (attr == &dev_attr_comp_times)
+		times = local_devdata->counters->comp_times;
+	else if (attr == &dev_attr_decomp_times)
+		times = local_devdata->counters->decomp_times;
+	else {
+		rcu_read_unlock();
+		return 0;
+	}
+
+	for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
+		bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
+			       i ? (2<<(i-1)) : 0, (2<<i)-1,
+			       atomic64_read(&times[i]));
+		bytes_remain -= bytes;
+		p += bytes;
+	}
+	/* The last bucket holds everything over
+	 * 2<<(NX842_HIST_SLOTS - 2) us */
+	bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
+			2<<(NX842_HIST_SLOTS - 2),
+			atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
+	p += bytes;
+
+	rcu_read_unlock();
+	return p - buf;
+}
+
+static struct attribute *nx842_sysfs_entries[] = {
+	&dev_attr_comp_complete.attr,
+	&dev_attr_comp_failed.attr,
+	&dev_attr_decomp_complete.attr,
+	&dev_attr_decomp_failed.attr,
+	&dev_attr_swdecomp.attr,
+	&dev_attr_comp_times.attr,
+	&dev_attr_decomp_times.attr,
+	NULL,
+};
+
+static struct attribute_group nx842_attribute_group = {
+	.name = NULL,		/* put in device directory */
+	.attrs = nx842_sysfs_entries,
+};
+
+static int __init nx842_probe(struct vio_dev *viodev,
+				  const struct vio_device_id *id)
+{
+	struct nx842_devdata *old_devdata, *new_devdata = NULL;
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&devdata_mutex, flags);
+	old_devdata = rcu_dereference_check(devdata,
+			lockdep_is_held(&devdata_mutex));
+
+	if (old_devdata && old_devdata->vdev != NULL) {
+		dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
+		ret = -1;
+		goto error_unlock;
+	}
+
+	dev_set_drvdata(&viodev->dev, NULL);
+
+	new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+	if (!new_devdata) {
+		dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
+		ret = -ENOMEM;
+		goto error_unlock;
+	}
+
+	new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
+			GFP_NOFS);
+	if (!new_devdata->counters) {
+		dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
+		ret = -ENOMEM;
+		goto error_unlock;
+	}
+
+	new_devdata->vdev = viodev;
+	new_devdata->dev = &viodev->dev;
+	nx842_OF_set_defaults(new_devdata);
+
+	rcu_assign_pointer(devdata, new_devdata);
+	spin_unlock_irqrestore(&devdata_mutex, flags);
+	synchronize_rcu();
+	kfree(old_devdata);
+
+	pSeries_reconfig_notifier_register(&nx842_of_nb);
+
+	ret = nx842_OF_upd(NULL);
+	if (ret && ret != -ENODEV) {
+		dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
+		ret = -1;
+		goto error;
+	}
+
+	rcu_read_lock();
+	if (dev_set_drvdata(&viodev->dev, rcu_dereference(devdata))) {
+		rcu_read_unlock();
+		dev_err(&viodev->dev, "failed to set driver data for device\n");
+		ret = -1;
+		goto error;
+	}
+	rcu_read_unlock();
+
+	if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
+		dev_err(&viodev->dev, "could not create sysfs device attributes\n");
+		ret = -1;
+		goto error;
+	}
+
+	return 0;
+
+error_unlock:
+	spin_unlock_irqrestore(&devdata_mutex, flags);
+	if (new_devdata)
+		kfree(new_devdata->counters);
+	kfree(new_devdata);
+error:
+	return ret;
+}
+
+static int __exit nx842_remove(struct vio_dev *viodev)
+{
+	struct nx842_devdata *old_devdata;
+	unsigned long flags;
+
+	pr_info("Removing IBM Power 842 compression device\n");
+	sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
+
+	spin_lock_irqsave(&devdata_mutex, flags);
+	old_devdata = rcu_dereference_check(devdata,
+			lockdep_is_held(&devdata_mutex));
+	pSeries_reconfig_notifier_unregister(&nx842_of_nb);
+	rcu_assign_pointer(devdata, NULL);
+	spin_unlock_irqrestore(&devdata_mutex, flags);
+	synchronize_rcu();
+	dev_set_drvdata(&viodev->dev, NULL);
+	if (old_devdata)
+		kfree(old_devdata->counters);
+	kfree(old_devdata);
+	return 0;
+}
+
+static struct vio_device_id nx842_driver_ids[] = {
+	{"ibm,compression-v1", "ibm,compression"},
+	{"", ""},
+};
+
+static struct vio_driver nx842_driver = {
+	.name = MODULE_NAME,
+	.probe = nx842_probe,
+	.remove = nx842_remove,
+	.get_desired_dma = nx842_get_desired_dma,
+	.id_table = nx842_driver_ids,
+};
+
+static int __init nx842_init(void)
+{
+	struct nx842_devdata *new_devdata;
+	pr_info("Registering IBM Power 842 compression driver\n");
+
+	RCU_INIT_POINTER(devdata, NULL);
+	new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
+	if (!new_devdata) {
+		pr_err("Could not allocate memory for device data\n");
+		return -ENOMEM;
+	}
+	new_devdata->status = UNAVAILABLE;
+	RCU_INIT_POINTER(devdata, new_devdata);
+
+	return vio_register_driver(&nx842_driver);
+}
+
+module_init(nx842_init);
+
+static void __exit nx842_exit(void)
+{
+	struct nx842_devdata *old_devdata;
+	unsigned long flags;
+
+	pr_info("Exiting IBM Power 842 compression driver\n");
+	spin_lock_irqsave(&devdata_mutex, flags);
+	old_devdata = rcu_dereference_check(devdata,
+			lockdep_is_held(&devdata_mutex));
+	rcu_assign_pointer(devdata, NULL);
+	spin_unlock_irqrestore(&devdata_mutex, flags);
+	synchronize_rcu();
+	if (old_devdata)
+		dev_set_drvdata(old_devdata->dev, NULL);
+	kfree(old_devdata);
+	vio_unregister_driver(&nx842_driver);
+}
+
+module_exit(nx842_exit);
+
+/*********************************
+ * 842 software decompressor
+*********************************/
+typedef int (*sw842_template_op)(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+
+static int sw842_data8(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+static int sw842_data4(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+static int sw842_data2(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+static int sw842_ptr8(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+static int sw842_ptr4(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+static int sw842_ptr2(const char **, int *, unsigned char **,
+						struct sw842_fifo *);
+
+/* special templates */
+#define SW842_TMPL_REPEAT 0x1B
+#define SW842_TMPL_ZEROS 0x1C
+#define SW842_TMPL_EOF 0x1E
+
+static sw842_template_op sw842_tmpl_ops[26][4] = {
+	{ sw842_data8, NULL}, /* 0 (00000) */
+	{ sw842_data4, sw842_data2, sw842_ptr2,  NULL},
+	{ sw842_data4, sw842_ptr2,  sw842_data2, NULL},
+	{ sw842_data4, sw842_ptr2,  sw842_ptr2,  NULL},
+	{ sw842_data4, sw842_ptr4,  NULL},
+	{ sw842_data2, sw842_ptr2,  sw842_data4, NULL},
+	{ sw842_data2, sw842_ptr2,  sw842_data2, sw842_ptr2},
+	{ sw842_data2, sw842_ptr2,  sw842_ptr2,  sw842_data2},
+	{ sw842_data2, sw842_ptr2,  sw842_ptr2,  sw842_ptr2,},
+	{ sw842_data2, sw842_ptr2,  sw842_ptr4,  NULL},
+	{ sw842_ptr2,  sw842_data2, sw842_data4, NULL}, /* 10 (01010) */
+	{ sw842_ptr2,  sw842_data4, sw842_ptr2,  NULL},
+	{ sw842_ptr2,  sw842_data2, sw842_ptr2,  sw842_data2},
+	{ sw842_ptr2,  sw842_data2, sw842_ptr2,  sw842_ptr2},
+	{ sw842_ptr2,  sw842_data2, sw842_ptr4,  NULL},
+	{ sw842_ptr2,  sw842_ptr2,  sw842_data4, NULL},
+	{ sw842_ptr2,  sw842_ptr2,  sw842_data2, sw842_ptr2},
+	{ sw842_ptr2,  sw842_ptr2,  sw842_ptr2,  sw842_data2},
+	{ sw842_ptr2,  sw842_ptr2,  sw842_ptr2,  sw842_ptr2},
+	{ sw842_ptr2,  sw842_ptr2,  sw842_ptr4,  NULL},
+	{ sw842_ptr4,  sw842_data4, NULL}, /* 20 (10100) */
+	{ sw842_ptr4,  sw842_data2, sw842_ptr2,  NULL},
+	{ sw842_ptr4,  sw842_ptr2,  sw842_data2, NULL},
+	{ sw842_ptr4,  sw842_ptr2,  sw842_ptr2,  NULL},
+	{ sw842_ptr4,  sw842_ptr4,  NULL},
+	{ sw842_ptr8,  NULL}
+};
+
+/* Software decompress helpers */
+
+static uint8_t sw842_get_byte(const char *buf, int bit)
+{
+	uint8_t tmpl;
+	uint16_t tmp;
+	tmp = htons(*(uint16_t *)(buf));
+	tmp = (uint16_t)(tmp << bit);
+	tmp = ntohs(tmp);
+	memcpy(&tmpl, &tmp, 1);
+	return tmpl;
+}
+
+static uint8_t sw842_get_template(const char **buf, int *bit)
+{
+	uint8_t byte;
+	byte = sw842_get_byte(*buf, *bit);
+	byte = byte >> 3;
+	byte &= 0x1F;
+	*buf += (*bit + 5) / 8;
+	*bit = (*bit + 5) % 8;
+	return byte;
+}
+
+/* repeat_count happens to be 5-bit too (like the template) */
+static uint8_t sw842_get_repeat_count(const char **buf, int *bit)
+{
+	uint8_t byte;
+	byte = sw842_get_byte(*buf, *bit);
+	byte = byte >> 2;
+	byte &= 0x3F;
+	*buf += (*bit + 6) / 8;
+	*bit = (*bit + 6) % 8;
+	return byte;
+}
+
+static uint8_t sw842_get_ptr2(const char **buf, int *bit)
+{
+	uint8_t ptr;
+	ptr = sw842_get_byte(*buf, *bit);
+	(*buf)++;
+	return ptr;
+}
+
+static uint16_t sw842_get_ptr4(const char **buf, int *bit,
+		struct sw842_fifo *fifo)
+{
+	uint16_t ptr;
+	ptr = htons(*(uint16_t *)(*buf));
+	ptr = (uint16_t)(ptr << *bit);
+	ptr = ptr >> 7;
+	ptr &= 0x01FF;
+	*buf += (*bit + 9) / 8;
+	*bit = (*bit + 9) % 8;
+	return ptr;
+}
+
+static uint8_t sw842_get_ptr8(const char **buf, int *bit,
+		struct sw842_fifo *fifo)
+{
+	return sw842_get_ptr2(buf, bit);
+}
+
+/* Software decompress template ops */
+
+static int sw842_data8(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	int ret;
+
+	ret = sw842_data4(inbuf, inbit, outbuf, fifo);
+	if (ret)
+		return ret;
+	ret = sw842_data4(inbuf, inbit, outbuf, fifo);
+	return ret;
+}
+
+static int sw842_data4(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	int ret;
+
+	ret = sw842_data2(inbuf, inbit, outbuf, fifo);
+	if (ret)
+		return ret;
+	ret = sw842_data2(inbuf, inbit, outbuf, fifo);
+	return ret;
+}
+
+static int sw842_data2(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	**outbuf = sw842_get_byte(*inbuf, *inbit);
+	(*inbuf)++;
+	(*outbuf)++;
+	**outbuf = sw842_get_byte(*inbuf, *inbit);
+	(*inbuf)++;
+	(*outbuf)++;
+	return 0;
+}
+
+static int sw842_ptr8(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	uint8_t ptr;
+	ptr = sw842_get_ptr8(inbuf, inbit, fifo);
+	if (!fifo->f84_full && (ptr >= fifo->f8_count))
+		return 1;
+	memcpy(*outbuf, fifo->f8[ptr], 8);
+	*outbuf += 8;
+	return 0;
+}
+
+static int sw842_ptr4(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	uint16_t ptr;
+	ptr = sw842_get_ptr4(inbuf, inbit, fifo);
+	if (!fifo->f84_full && (ptr >= fifo->f4_count))
+		return 1;
+	memcpy(*outbuf, fifo->f4[ptr], 4);
+	*outbuf += 4;
+	return 0;
+}
+
+static int sw842_ptr2(const char **inbuf, int *inbit,
+		unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+	uint8_t ptr;
+	ptr = sw842_get_ptr2(inbuf, inbit);
+	if (!fifo->f2_full && (ptr >= fifo->f2_count))
+		return 1;
+	memcpy(*outbuf, fifo->f2[ptr], 2);
+	*outbuf += 2;
+	return 0;
+}
+
+static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo)
+{
+	unsigned char initial_f2count = fifo->f2_count;
+
+	memcpy(fifo->f8[fifo->f8_count], buf, 8);
+	fifo->f4_count += 2;
+	fifo->f8_count += 1;
+
+	if (!fifo->f84_full && fifo->f4_count >= 512) {
+		fifo->f84_full = 1;
+		fifo->f4_count /= 512;
+	}
+
+	memcpy(fifo->f2[fifo->f2_count++], buf, 2);
+	memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2);
+	memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2);
+	memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2);
+	if (fifo->f2_count < initial_f2count)
+		fifo->f2_full = 1;
+}
+
+static int sw842_decompress(const unsigned char *src, int srclen,
+			unsigned char *dst, int *destlen,
+			const void *wrkmem)
+{
+	uint8_t tmpl;
+	const char *inbuf;
+	int inbit = 0;
+	unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf;
+	const char *inbuf_end;
+	sw842_template_op op;
+	int opindex;
+	int i, repeat_count;
+	struct sw842_fifo *fifo;
+	int ret = 0;
+
+	fifo = &((struct nx842_workmem *)(wrkmem))->swfifo;
+	memset(fifo, 0, sizeof(*fifo));
+
+	origbuf = NULL;
+	inbuf = src;
+	inbuf_end = src + srclen;
+	outbuf = dst;
+	outbuf_end = dst + *destlen;
+
+	while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) {
+		if (inbuf >= inbuf_end) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		opindex = 0;
+		prevbuf = origbuf;
+		origbuf = outbuf;
+		switch (tmpl) {
+		case SW842_TMPL_REPEAT:
+			if (prevbuf == NULL) {
+				ret = -EINVAL;
+				goto out;
+			}
+
+			repeat_count = sw842_get_repeat_count(&inbuf,
+								&inbit) + 1;
+
+			/* Did the repeat count advance past the end of input */
+			if (inbuf > inbuf_end) {
+				ret = -EINVAL;
+				goto out;
+			}
+
+			for (i = 0; i < repeat_count; i++) {
+				/* Would this overflow the output buffer */
+				if ((outbuf + 8) > outbuf_end) {
+					ret = -ENOSPC;
+					goto out;
+				}
+
+				memcpy(outbuf, prevbuf, 8);
+				sw842_copy_to_fifo(outbuf, fifo);
+				outbuf += 8;
+			}
+			break;
+
+		case SW842_TMPL_ZEROS:
+			/* Would this overflow the output buffer */
+			if ((outbuf + 8) > outbuf_end) {
+				ret = -ENOSPC;
+				goto out;
+			}
+
+			memset(outbuf, 0, 8);
+			sw842_copy_to_fifo(outbuf, fifo);
+			outbuf += 8;
+			break;
+
+		default:
+			if (tmpl > 25) {
+				ret = -EINVAL;
+				goto out;
+			}
+
+			/* Does this go past the end of the input buffer */
+			if ((inbuf + 2) > inbuf_end) {
+				ret = -EINVAL;
+				goto out;
+			}
+
+			/* Would this overflow the output buffer */
+			if ((outbuf + 8) > outbuf_end) {
+				ret = -ENOSPC;
+				goto out;
+			}
+
+			while (opindex < 4 &&
+				(op = sw842_tmpl_ops[tmpl][opindex++])
+					!= NULL) {
+				ret = (*op)(&inbuf, &inbit, &outbuf, fifo);
+				if (ret) {
+					ret = -EINVAL;
+					goto out;
+				}
+				sw842_copy_to_fifo(origbuf, fifo);
+			}
+		}
+	}
+
+out:
+	if (!ret)
+		*destlen = (unsigned int)(outbuf - dst);
+	else
+		*destlen = 0;
+
+	return ret;
+}
diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c
index 69ed796ee32..a76d4c4f29f 100644
--- a/drivers/crypto/nx/nx-aes-cbc.c
+++ b/drivers/crypto/nx/nx-aes-cbc.c
@@ -127,7 +127,6 @@ struct crypto_alg nx_cbc_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_blkcipher_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_cbc_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_cbc_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_blkcipher = {
diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c
index 7aeac678b9c..ef5eae6d140 100644
--- a/drivers/crypto/nx/nx-aes-ccm.c
+++ b/drivers/crypto/nx/nx-aes-ccm.c
@@ -430,7 +430,6 @@ struct crypto_alg nx_ccm_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_aead_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_ccm_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_ccm_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_aead = {
@@ -453,7 +452,6 @@ struct crypto_alg nx_ccm4309_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_nivaead_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_ccm4309_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_ccm_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_aead = {
diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c
index 52d4eb05e8f..b6286f14680 100644
--- a/drivers/crypto/nx/nx-aes-ctr.c
+++ b/drivers/crypto/nx/nx-aes-ctr.c
@@ -141,7 +141,6 @@ struct crypto_alg nx_ctr_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_blkcipher_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_ctr_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_ctr_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_blkcipher = {
@@ -163,7 +162,6 @@ struct crypto_alg nx_ctr3686_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_blkcipher_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_ctr3686_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_ctr_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_blkcipher = {
diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c
index 7b77bc2d1df..ba5f1611336 100644
--- a/drivers/crypto/nx/nx-aes-ecb.c
+++ b/drivers/crypto/nx/nx-aes-ecb.c
@@ -126,7 +126,6 @@ struct crypto_alg nx_ecb_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_blkcipher_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_ecb_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_ecb_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_blkcipher = {
diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c
index 9ab1c7341da..c8109edc5cf 100644
--- a/drivers/crypto/nx/nx-aes-gcm.c
+++ b/drivers/crypto/nx/nx-aes-gcm.c
@@ -316,7 +316,6 @@ struct crypto_alg nx_gcm_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_aead_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_gcm_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_gcm_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_aead = {
@@ -338,7 +337,6 @@ struct crypto_alg nx_gcm4106_aes_alg = {
 	.cra_ctxsize     = sizeof(struct nx_crypto_ctx),
 	.cra_type        = &crypto_nivaead_type,
 	.cra_module      = THIS_MODULE,
-	.cra_list        = LIST_HEAD_INIT(nx_gcm4106_aes_alg.cra_list),
 	.cra_init        = nx_crypto_ctx_aes_gcm_init,
 	.cra_exit        = nx_crypto_ctx_exit,
 	.cra_aead = {
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 63e57b57a12..093a8af59cb 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -876,7 +876,6 @@ static int omap_aes_probe(struct platform_device *pdev)
 
 	for (i = 0; i < ARRAY_SIZE(algs); i++) {
 		pr_debug("i: %d\n", i);
-		INIT_LIST_HEAD(&algs[i].cra_list);
 		err = crypto_register_alg(&algs[i]);
 		if (err)
 			goto err_algs;
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index 37b2e9406af..633ba945e15 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -328,7 +328,6 @@ static struct crypto_alg aes_alg = {
 	.cra_ctxsize		=	sizeof(struct aes_ctx),
 	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
 	.cra_u			=	{
 		.cipher = {
 			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
@@ -408,7 +407,6 @@ static struct crypto_alg ecb_aes_alg = {
 	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	AES_MIN_KEY_SIZE,
@@ -491,7 +489,6 @@ static struct crypto_alg cbc_aes_alg = {
 	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
 	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
 	.cra_u			=	{
 		.blkcipher = {
 			.min_keysize		=	AES_MIN_KEY_SIZE,
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index bc986f80608..a22714412cd 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -626,7 +626,6 @@ static int s5p_aes_probe(struct platform_device *pdev)
 	crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
 
 	for (i = 0; i < ARRAY_SIZE(algs); i++) {
-		INIT_LIST_HEAD(&algs[i].cra_list);
 		err = crypto_register_alg(&algs[i]);
 		if (err)
 			goto err_algs;
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index efff788d2f1..da1112765a4 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -38,6 +38,7 @@
 #include <linux/spinlock.h>
 #include <linux/rtnetlink.h>
 #include <linux/slab.h>
+#include <linux/string.h>
 
 #include <crypto/algapi.h>
 #include <crypto/aes.h>
@@ -714,8 +715,13 @@ badkey:
 
 /*
  * talitos_edesc - s/w-extended descriptor
+ * @assoc_nents: number of segments in associated data scatterlist
  * @src_nents: number of segments in input scatterlist
  * @dst_nents: number of segments in output scatterlist
+ * @assoc_chained: whether assoc is chained or not
+ * @src_chained: whether src is chained or not
+ * @dst_chained: whether dst is chained or not
+ * @iv_dma: dma address of iv for checking continuity and link table
  * @dma_len: length of dma mapped link_tbl space
  * @dma_link_tbl: bus physical address of link_tbl
  * @desc: h/w descriptor
@@ -726,10 +732,13 @@ badkey:
  * of link_tbl data
  */
 struct talitos_edesc {
+	int assoc_nents;
 	int src_nents;
 	int dst_nents;
-	int src_is_chained;
-	int dst_is_chained;
+	bool assoc_chained;
+	bool src_chained;
+	bool dst_chained;
+	dma_addr_t iv_dma;
 	int dma_len;
 	dma_addr_t dma_link_tbl;
 	struct talitos_desc desc;
@@ -738,7 +747,7 @@ struct talitos_edesc {
 
 static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
 			  unsigned int nents, enum dma_data_direction dir,
-			  int chained)
+			  bool chained)
 {
 	if (unlikely(chained))
 		while (sg) {
@@ -768,13 +777,13 @@ static void talitos_sg_unmap(struct device *dev,
 	unsigned int dst_nents = edesc->dst_nents ? : 1;
 
 	if (src != dst) {
-		if (edesc->src_is_chained)
+		if (edesc->src_chained)
 			talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
 		else
 			dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 
 		if (dst) {
-			if (edesc->dst_is_chained)
+			if (edesc->dst_chained)
 				talitos_unmap_sg_chain(dev, dst,
 						       DMA_FROM_DEVICE);
 			else
@@ -782,7 +791,7 @@ static void talitos_sg_unmap(struct device *dev,
 					     DMA_FROM_DEVICE);
 		}
 	} else
-		if (edesc->src_is_chained)
+		if (edesc->src_chained)
 			talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
 		else
 			dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
@@ -797,7 +806,13 @@ static void ipsec_esp_unmap(struct device *dev,
 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 	unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 
-	dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
+	if (edesc->assoc_chained)
+		talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE);
+	else
+		/* assoc_nents counts also for IV in non-contiguous cases */
+		dma_unmap_sg(dev, areq->assoc,
+			     edesc->assoc_nents ? edesc->assoc_nents - 1 : 1,
+			     DMA_TO_DEVICE);
 
 	talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 
@@ -825,9 +840,10 @@ static void ipsec_esp_encrypt_done(struct device *dev,
 	ipsec_esp_unmap(dev, edesc, areq);
 
 	/* copy the generated ICV to dst */
-	if (edesc->dma_len) {
+	if (edesc->dst_nents) {
 		icvdata = &edesc->link_tbl[edesc->src_nents +
-					   edesc->dst_nents + 2];
+					   edesc->dst_nents + 2 +
+					   edesc->assoc_nents];
 		sg = sg_last(areq->dst, edesc->dst_nents);
 		memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 		       icvdata, ctx->authsize);
@@ -857,7 +873,8 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 		/* auth check */
 		if (edesc->dma_len)
 			icvdata = &edesc->link_tbl[edesc->src_nents +
-						   edesc->dst_nents + 2];
+						   edesc->dst_nents + 2 +
+						   edesc->assoc_nents];
 		else
 			icvdata = &edesc->link_tbl[0];
 
@@ -932,10 +949,9 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
  * fill in and submit ipsec_esp descriptor
  */
 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
-		     u8 *giv, u64 seq,
-		     void (*callback) (struct device *dev,
-				       struct talitos_desc *desc,
-				       void *context, int error))
+		     u64 seq, void (*callback) (struct device *dev,
+						struct talitos_desc *desc,
+						void *context, int error))
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_aead_ctx(aead);
@@ -950,12 +966,42 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 	/* hmac key */
 	map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
 			       0, DMA_TO_DEVICE);
+
 	/* hmac data */
-	map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
-			       sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
+	desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize);
+	if (edesc->assoc_nents) {
+		int tbl_off = edesc->src_nents + edesc->dst_nents + 2;
+		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
+
+		to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
+			       sizeof(struct talitos_ptr));
+		desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
+
+		/* assoc_nents - 1 entries for assoc, 1 for IV */
+		sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1,
+					  areq->assoclen, tbl_ptr);
+
+		/* add IV to link table */
+		tbl_ptr += sg_count - 1;
+		tbl_ptr->j_extent = 0;
+		tbl_ptr++;
+		to_talitos_ptr(tbl_ptr, edesc->iv_dma);
+		tbl_ptr->len = cpu_to_be16(ivsize);
+		tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
+
+		dma_sync_single_for_device(dev, edesc->dma_link_tbl,
+					   edesc->dma_len, DMA_BIDIRECTIONAL);
+	} else {
+		to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->assoc));
+		desc->ptr[1].j_extent = 0;
+	}
+
 	/* cipher iv */
-	map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
-			       DMA_TO_DEVICE);
+	to_talitos_ptr(&desc->ptr[2], edesc->iv_dma);
+	desc->ptr[2].len = cpu_to_be16(ivsize);
+	desc->ptr[2].j_extent = 0;
+	/* Sync needed for the aead_givencrypt case */
+	dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
 
 	/* cipher key */
 	map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
@@ -974,7 +1020,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
 				  (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
 							   : DMA_TO_DEVICE,
-				  edesc->src_is_chained);
+				  edesc->src_chained);
 
 	if (sg_count == 1) {
 		to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));
@@ -1006,32 +1052,30 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 	if (areq->src != areq->dst)
 		sg_count = talitos_map_sg(dev, areq->dst,
 					  edesc->dst_nents ? : 1,
-					  DMA_FROM_DEVICE,
-					  edesc->dst_is_chained);
+					  DMA_FROM_DEVICE, edesc->dst_chained);
 
 	if (sg_count == 1) {
 		to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));
 	} else {
-		struct talitos_ptr *link_tbl_ptr =
-			&edesc->link_tbl[edesc->src_nents + 1];
+		int tbl_off = edesc->src_nents + 1;
+		struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
 
 		to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
-			       (edesc->src_nents + 1) *
-			       sizeof(struct talitos_ptr));
+			       tbl_off * sizeof(struct talitos_ptr));
 		sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
-					  link_tbl_ptr);
+					  tbl_ptr);
 
 		/* Add an entry to the link table for ICV data */
-		link_tbl_ptr += sg_count - 1;
-		link_tbl_ptr->j_extent = 0;
-		sg_count++;
-		link_tbl_ptr++;
-		link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
-		link_tbl_ptr->len = cpu_to_be16(authsize);
+		tbl_ptr += sg_count - 1;
+		tbl_ptr->j_extent = 0;
+		tbl_ptr++;
+		tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
+		tbl_ptr->len = cpu_to_be16(authsize);
 
 		/* icv data follows link tables */
-		to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl +
-			       (edesc->src_nents + edesc->dst_nents + 2) *
+		to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
+			       (tbl_off + edesc->dst_nents + 1 +
+				edesc->assoc_nents) *
 			       sizeof(struct talitos_ptr));
 		desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
 		dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
@@ -1053,17 +1097,17 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 /*
  * derive number of elements in scatterlist
  */
-static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
+static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained)
 {
 	struct scatterlist *sg = sg_list;
 	int sg_nents = 0;
 
-	*chained = 0;
+	*chained = false;
 	while (nbytes > 0) {
 		sg_nents++;
 		nbytes -= sg->length;
 		if (!sg_is_last(sg) && (sg + 1)->length == 0)
-			*chained = 1;
+			*chained = true;
 		sg = scatterwalk_sg_next(sg);
 	}
 
@@ -1132,17 +1176,21 @@ static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
  * allocate and map the extended descriptor
  */
 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
+						 struct scatterlist *assoc,
 						 struct scatterlist *src,
 						 struct scatterlist *dst,
-						 int hash_result,
+						 u8 *iv,
+						 unsigned int assoclen,
 						 unsigned int cryptlen,
 						 unsigned int authsize,
+						 unsigned int ivsize,
 						 int icv_stashing,
 						 u32 cryptoflags)
 {
 	struct talitos_edesc *edesc;
-	int src_nents, dst_nents, alloc_len, dma_len;
-	int src_chained, dst_chained = 0;
+	int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len;
+	bool assoc_chained = false, src_chained = false, dst_chained = false;
+	dma_addr_t iv_dma = 0;
 	gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
 		      GFP_ATOMIC;
 
@@ -1151,10 +1199,29 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 		return ERR_PTR(-EINVAL);
 	}
 
+	if (iv)
+		iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
+
+	if (assoc) {
+		/*
+		 * Currently it is assumed that iv is provided whenever assoc
+		 * is.
+		 */
+		BUG_ON(!iv);
+
+		assoc_nents = sg_count(assoc, assoclen, &assoc_chained);
+		talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE,
+			       assoc_chained);
+		assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents;
+
+		if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma)
+			assoc_nents = assoc_nents ? assoc_nents + 1 : 2;
+	}
+
 	src_nents = sg_count(src, cryptlen + authsize, &src_chained);
 	src_nents = (src_nents == 1) ? 0 : src_nents;
 
-	if (hash_result) {
+	if (!dst) {
 		dst_nents = 0;
 	} else {
 		if (dst == src) {
@@ -1172,9 +1239,9 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 	 * and the ICV data itself
 	 */
 	alloc_len = sizeof(struct talitos_edesc);
-	if (src_nents || dst_nents) {
-		dma_len = (src_nents + dst_nents + 2) *
-				 sizeof(struct talitos_ptr) + authsize;
+	if (assoc_nents || src_nents || dst_nents) {
+		dma_len = (src_nents + dst_nents + 2 + assoc_nents) *
+			  sizeof(struct talitos_ptr) + authsize;
 		alloc_len += dma_len;
 	} else {
 		dma_len = 0;
@@ -1183,14 +1250,20 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 
 	edesc = kmalloc(alloc_len, GFP_DMA | flags);
 	if (!edesc) {
+		talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE);
+		if (iv_dma)
+			dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 		dev_err(dev, "could not allocate edescriptor\n");
 		return ERR_PTR(-ENOMEM);
 	}
 
+	edesc->assoc_nents = assoc_nents;
 	edesc->src_nents = src_nents;
 	edesc->dst_nents = dst_nents;
-	edesc->src_is_chained = src_chained;
-	edesc->dst_is_chained = dst_chained;
+	edesc->assoc_chained = assoc_chained;
+	edesc->src_chained = src_chained;
+	edesc->dst_chained = dst_chained;
+	edesc->iv_dma = iv_dma;
 	edesc->dma_len = dma_len;
 	if (dma_len)
 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
@@ -1200,14 +1273,16 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 	return edesc;
 }
 
-static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
+static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
 					      int icv_stashing)
 {
 	struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+	unsigned int ivsize = crypto_aead_ivsize(authenc);
 
-	return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
-				   areq->cryptlen, ctx->authsize, icv_stashing,
+	return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst,
+				   iv, areq->assoclen, areq->cryptlen,
+				   ctx->authsize, ivsize, icv_stashing,
 				   areq->base.flags);
 }
 
@@ -1218,14 +1293,14 @@ static int aead_encrypt(struct aead_request *req)
 	struct talitos_edesc *edesc;
 
 	/* allocate extended descriptor */
-	edesc = aead_edesc_alloc(req, 0);
+	edesc = aead_edesc_alloc(req, req->iv, 0);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
 	/* set encrypt */
 	edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
 
-	return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
+	return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done);
 }
 
 static int aead_decrypt(struct aead_request *req)
@@ -1241,7 +1316,7 @@ static int aead_decrypt(struct aead_request *req)
 	req->cryptlen -= authsize;
 
 	/* allocate extended descriptor */
-	edesc = aead_edesc_alloc(req, 1);
+	edesc = aead_edesc_alloc(req, req->iv, 1);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
@@ -1257,9 +1332,7 @@ static int aead_decrypt(struct aead_request *req)
 		/* reset integrity check result bits */
 		edesc->desc.hdr_lo = 0;
 
-		return ipsec_esp(edesc, req, NULL, 0,
-				 ipsec_esp_decrypt_hwauth_done);
-
+		return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done);
 	}
 
 	/* Have to check the ICV with software */
@@ -1268,7 +1341,8 @@ static int aead_decrypt(struct aead_request *req)
 	/* stash incoming ICV for later cmp with ICV generated by the h/w */
 	if (edesc->dma_len)
 		icvdata = &edesc->link_tbl[edesc->src_nents +
-					   edesc->dst_nents + 2];
+					   edesc->dst_nents + 2 +
+					   edesc->assoc_nents];
 	else
 		icvdata = &edesc->link_tbl[0];
 
@@ -1277,7 +1351,7 @@ static int aead_decrypt(struct aead_request *req)
 	memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
 	       ctx->authsize);
 
-	return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
+	return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done);
 }
 
 static int aead_givencrypt(struct aead_givcrypt_request *req)
@@ -1288,7 +1362,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *req)
 	struct talitos_edesc *edesc;
 
 	/* allocate extended descriptor */
-	edesc = aead_edesc_alloc(areq, 0);
+	edesc = aead_edesc_alloc(areq, req->giv, 0);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
@@ -1299,8 +1373,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *req)
 	/* avoid consecutive packets going out with same IV */
 	*(__be64 *)req->giv ^= cpu_to_be64(req->seq);
 
-	return ipsec_esp(edesc, areq, req->giv, req->seq,
-			 ipsec_esp_encrypt_done);
+	return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done);
 }
 
 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
@@ -1356,7 +1429,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
 	struct device *dev = ctx->dev;
 	struct talitos_desc *desc = &edesc->desc;
 	unsigned int cryptlen = areq->nbytes;
-	unsigned int ivsize;
+	unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
 	int sg_count, ret;
 
 	/* first DWORD empty */
@@ -1365,9 +1438,9 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
 	desc->ptr[0].j_extent = 0;
 
 	/* cipher iv */
-	ivsize = crypto_ablkcipher_ivsize(cipher);
-	map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
-			       DMA_TO_DEVICE);
+	to_talitos_ptr(&desc->ptr[1], edesc->iv_dma);
+	desc->ptr[1].len = cpu_to_be16(ivsize);
+	desc->ptr[1].j_extent = 0;
 
 	/* cipher key */
 	map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
@@ -1382,7 +1455,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
 	sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
 				  (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
 							   : DMA_TO_DEVICE,
-				  edesc->src_is_chained);
+				  edesc->src_chained);
 
 	if (sg_count == 1) {
 		to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));
@@ -1409,8 +1482,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
 	if (areq->src != areq->dst)
 		sg_count = talitos_map_sg(dev, areq->dst,
 					  edesc->dst_nents ? : 1,
-					  DMA_FROM_DEVICE,
-					  edesc->dst_is_chained);
+					  DMA_FROM_DEVICE, edesc->dst_chained);
 
 	if (sg_count == 1) {
 		to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));
@@ -1450,9 +1522,11 @@ static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
 {
 	struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+	unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
 
-	return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
-				   areq->nbytes, 0, 0, areq->base.flags);
+	return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst,
+				   areq->info, 0, areq->nbytes, 0, ivsize, 0,
+				   areq->base.flags);
 }
 
 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
@@ -1578,8 +1652,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
 
 	sg_count = talitos_map_sg(dev, req_ctx->psrc,
 				  edesc->src_nents ? : 1,
-				  DMA_TO_DEVICE,
-				  edesc->src_is_chained);
+				  DMA_TO_DEVICE, edesc->src_chained);
 
 	if (sg_count == 1) {
 		to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
@@ -1631,8 +1704,8 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 
-	return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
-				   nbytes, 0, 0, areq->base.flags);
+	return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0,
+				   nbytes, 0, 0, 0, areq->base.flags);
 }
 
 static int ahash_init(struct ahash_request *areq)
@@ -1690,7 +1763,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
 	unsigned int nbytes_to_hash;
 	unsigned int to_hash_later;
 	unsigned int nsg;
-	int chained;
+	bool chained;
 
 	if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
 		/* Buffer up to one whole block */
@@ -1902,21 +1975,18 @@ struct talitos_alg_template {
 };
 
 static struct talitos_alg_template driver_algs[] = {
-	/* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
+	/*
+	 * AEAD algorithms. These use a single-pass ipsec_esp descriptor.
+	 * authencesn(*,*) is also registered, although not present
+	 * explicitly here.
+	 */
 	{	.type = CRYPTO_ALG_TYPE_AEAD,
 		.alg.crypto = {
 			.cra_name = "authenc(hmac(sha1),cbc(aes))",
 			.cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = SHA1_DIGEST_SIZE,
 			}
@@ -1935,14 +2005,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = SHA1_DIGEST_SIZE,
 			}
@@ -1962,14 +2025,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = SHA224_DIGEST_SIZE,
 			}
@@ -1988,14 +2044,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = SHA224_DIGEST_SIZE,
 			}
@@ -2015,14 +2064,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = SHA256_DIGEST_SIZE,
 			}
@@ -2041,14 +2083,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = SHA256_DIGEST_SIZE,
 			}
@@ -2068,14 +2103,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = SHA384_DIGEST_SIZE,
 			}
@@ -2094,14 +2122,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = SHA384_DIGEST_SIZE,
 			}
@@ -2121,14 +2142,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = SHA512_DIGEST_SIZE,
 			}
@@ -2147,14 +2161,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = SHA512_DIGEST_SIZE,
 			}
@@ -2174,14 +2181,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = AES_BLOCK_SIZE,
 				.maxauthsize = MD5_DIGEST_SIZE,
 			}
@@ -2200,14 +2200,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_aead_type,
 			.cra_aead = {
-				.setkey = aead_setkey,
-				.setauthsize = aead_setauthsize,
-				.encrypt = aead_encrypt,
-				.decrypt = aead_decrypt,
-				.givencrypt = aead_givencrypt,
-				.geniv = "<built-in>",
 				.ivsize = DES3_EDE_BLOCK_SIZE,
 				.maxauthsize = MD5_DIGEST_SIZE,
 			}
@@ -2229,12 +2222,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_blocksize = AES_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
                                      CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_ablkcipher_type,
 			.cra_ablkcipher = {
-				.setkey = ablkcipher_setkey,
-				.encrypt = ablkcipher_encrypt,
-				.decrypt = ablkcipher_decrypt,
-				.geniv = "eseqiv",
 				.min_keysize = AES_MIN_KEY_SIZE,
 				.max_keysize = AES_MAX_KEY_SIZE,
 				.ivsize = AES_BLOCK_SIZE,
@@ -2251,12 +2239,7 @@ static struct talitos_alg_template driver_algs[] = {
 			.cra_blocksize = DES3_EDE_BLOCK_SIZE,
 			.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
                                      CRYPTO_ALG_ASYNC,
-			.cra_type = &crypto_ablkcipher_type,
 			.cra_ablkcipher = {
-				.setkey = ablkcipher_setkey,
-				.encrypt = ablkcipher_encrypt,
-				.decrypt = ablkcipher_decrypt,
-				.geniv = "eseqiv",
 				.min_keysize = DES3_EDE_KEY_SIZE,
 				.max_keysize = DES3_EDE_KEY_SIZE,
 				.ivsize = DES3_EDE_BLOCK_SIZE,
@@ -2270,11 +2253,6 @@ static struct talitos_alg_template driver_algs[] = {
 	/* AHASH algorithms. */
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = MD5_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "md5",
@@ -2282,7 +2260,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = MD5_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2291,11 +2268,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = SHA1_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "sha1",
@@ -2303,7 +2275,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA1_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2312,11 +2283,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = SHA224_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "sha224",
@@ -2324,7 +2290,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA224_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2333,11 +2298,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = SHA256_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "sha256",
@@ -2345,7 +2305,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA256_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2354,11 +2313,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = SHA384_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "sha384",
@@ -2366,7 +2320,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA384_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2375,11 +2328,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
 			.halg.digestsize = SHA512_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "sha512",
@@ -2387,7 +2335,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA512_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2396,12 +2343,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = MD5_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(md5)",
@@ -2409,7 +2350,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = MD5_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2418,12 +2358,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = SHA1_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha1)",
@@ -2431,7 +2365,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA1_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2440,12 +2373,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = SHA224_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha224)",
@@ -2453,7 +2380,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA224_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2462,12 +2388,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = SHA256_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha256)",
@@ -2475,7 +2395,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA256_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2484,12 +2403,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = SHA384_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha384)",
@@ -2497,7 +2410,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA384_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2506,12 +2418,6 @@ static struct talitos_alg_template driver_algs[] = {
 	},
 	{	.type = CRYPTO_ALG_TYPE_AHASH,
 		.alg.hash = {
-			.init = ahash_init,
-			.update = ahash_update,
-			.final = ahash_final,
-			.finup = ahash_finup,
-			.digest = ahash_digest,
-			.setkey = ahash_setkey,
 			.halg.digestsize = SHA512_DIGEST_SIZE,
 			.halg.base = {
 				.cra_name = "hmac(sha512)",
@@ -2519,7 +2425,6 @@ static struct talitos_alg_template driver_algs[] = {
 				.cra_blocksize = SHA512_BLOCK_SIZE,
 				.cra_flags = CRYPTO_ALG_TYPE_AHASH |
 					     CRYPTO_ALG_ASYNC,
-				.cra_type = &crypto_ahash_type
 			}
 		},
 		.desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2677,14 +2582,34 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
 	case CRYPTO_ALG_TYPE_ABLKCIPHER:
 		alg = &t_alg->algt.alg.crypto;
 		alg->cra_init = talitos_cra_init;
+		alg->cra_type = &crypto_ablkcipher_type;
+		alg->cra_ablkcipher.setkey = ablkcipher_setkey;
+		alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
+		alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
+		alg->cra_ablkcipher.geniv = "eseqiv";
 		break;
 	case CRYPTO_ALG_TYPE_AEAD:
 		alg = &t_alg->algt.alg.crypto;
 		alg->cra_init = talitos_cra_init_aead;
+		alg->cra_type = &crypto_aead_type;
+		alg->cra_aead.setkey = aead_setkey;
+		alg->cra_aead.setauthsize = aead_setauthsize;
+		alg->cra_aead.encrypt = aead_encrypt;
+		alg->cra_aead.decrypt = aead_decrypt;
+		alg->cra_aead.givencrypt = aead_givencrypt;
+		alg->cra_aead.geniv = "<built-in>";
 		break;
 	case CRYPTO_ALG_TYPE_AHASH:
 		alg = &t_alg->algt.alg.hash.halg.base;
 		alg->cra_init = talitos_cra_init_ahash;
+		alg->cra_type = &crypto_ahash_type;
+		t_alg->algt.alg.hash.init = ahash_init;
+		t_alg->algt.alg.hash.update = ahash_update;
+		t_alg->algt.alg.hash.final = ahash_final;
+		t_alg->algt.alg.hash.finup = ahash_finup;
+		t_alg->algt.alg.hash.digest = ahash_digest;
+		t_alg->algt.alg.hash.setkey = ahash_setkey;
+
 		if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
 		    !strncmp(alg->cra_name, "hmac", 4)) {
 			kfree(t_alg);
@@ -2896,7 +2821,9 @@ static int talitos_probe(struct platform_device *ofdev)
 		if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
 			struct talitos_crypto_alg *t_alg;
 			char *name = NULL;
+			bool authenc = false;
 
+authencesn:
 			t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
 			if (IS_ERR(t_alg)) {
 				err = PTR_ERR(t_alg);
@@ -2911,6 +2838,8 @@ static int talitos_probe(struct platform_device *ofdev)
 				err = crypto_register_alg(
 						&t_alg->algt.alg.crypto);
 				name = t_alg->algt.alg.crypto.cra_driver_name;
+				authenc = authenc ? !authenc :
+					  !(bool)memcmp(name, "authenc", 7);
 				break;
 			case CRYPTO_ALG_TYPE_AHASH:
 				err = crypto_register_ahash(
@@ -2923,8 +2852,25 @@ static int talitos_probe(struct platform_device *ofdev)
 				dev_err(dev, "%s alg registration failed\n",
 					name);
 				kfree(t_alg);
-			} else
+			} else {
 				list_add_tail(&t_alg->entry, &priv->alg_list);
+				if (authenc) {
+					struct crypto_alg *alg =
+						&driver_algs[i].alg.crypto;
+
+					name = alg->cra_name;
+					memmove(name + 10, name + 7,
+						strlen(name) - 7);
+					memcpy(name + 7, "esn", 3);
+
+					name = alg->cra_driver_name;
+					memmove(name + 10, name + 7,
+						strlen(name) - 7);
+					memcpy(name + 7, "esn", 3);
+
+					goto authencesn;
+				}
+			}
 		}
 	}
 	if (!list_empty(&priv->alg_list))
diff --git a/drivers/crypto/tegra-aes.c b/drivers/crypto/tegra-aes.c
index ac236f6724f..37185e6630c 100644
--- a/drivers/crypto/tegra-aes.c
+++ b/drivers/crypto/tegra-aes.c
@@ -969,6 +969,7 @@ static int tegra_aes_probe(struct platform_device *pdev)
 	aes_wq = alloc_workqueue("tegra_aes_wq", WQ_HIGHPRI | WQ_UNBOUND, 1);
 	if (!aes_wq) {
 		dev_err(dev, "alloc_workqueue failed\n");
+		err = -ENOMEM;
 		goto out;
 	}
 
@@ -1004,8 +1005,6 @@ static int tegra_aes_probe(struct platform_device *pdev)
 
 	aes_dev = dd;
 	for (i = 0; i < ARRAY_SIZE(algs); i++) {
-		INIT_LIST_HEAD(&algs[i].cra_list);
-
 		algs[i].cra_priority = 300;
 		algs[i].cra_ctxsize = sizeof(struct tegra_aes_ctx);
 		algs[i].cra_module = THIS_MODULE;
diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c
index ef17e3871c7..bc615cc5626 100644
--- a/drivers/crypto/ux500/cryp/cryp_core.c
+++ b/drivers/crypto/ux500/cryp/cryp_core.c
@@ -1486,6 +1486,7 @@ static int ux500_cryp_probe(struct platform_device *pdev)
 	if (!res_irq) {
 		dev_err(dev, "[%s]: IORESOURCE_IRQ unavailable",
 			__func__);
+		ret = -ENODEV;
 		goto out_power;
 	}
 
diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c
index 08765072a2b..632c3339895 100644
--- a/drivers/crypto/ux500/hash/hash_core.c
+++ b/drivers/crypto/ux500/hash/hash_core.c
@@ -1991,7 +1991,6 @@ static int __init ux500_hash_mod_init(void)
 static void __exit ux500_hash_mod_fini(void)
 {
 	platform_driver_unregister(&hash_driver);
-	return;
 }
 
 module_init(ux500_hash_mod_init);
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 3491654cdf7..a815d44c70a 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -582,7 +582,7 @@ void dmaengine_get(void)
 				list_del_rcu(&device->global_node);
 				break;
 			} else if (err)
-				pr_err("%s: failed to get %s: (%d)\n",
+				pr_debug("%s: failed to get %s: (%d)\n",
 				       __func__, dma_chan_name(chan), err);
 		}
 	}
diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c
index 7a05fd24d68..3873d535b28 100644
--- a/drivers/firewire/core-device.c
+++ b/drivers/firewire/core-device.c
@@ -32,6 +32,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/random.h>
 #include <linux/rwsem.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -1066,6 +1067,8 @@ static void fw_device_init(struct work_struct *work)
 		device->config_rom_retries = 0;
 
 		set_broadcast_channel(device, device->generation);
+
+		add_device_randomness(&device->config_rom[3], 8);
 	}
 
 	/*
diff --git a/drivers/firewire/core-transaction.c b/drivers/firewire/core-transaction.c
index 87d6f2d2f02..28a94c7ec6e 100644
--- a/drivers/firewire/core-transaction.c
+++ b/drivers/firewire/core-transaction.c
@@ -31,6 +31,7 @@
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/rculist.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
@@ -489,7 +490,7 @@ static struct fw_address_handler *lookup_overlapping_address_handler(
 {
 	struct fw_address_handler *handler;
 
-	list_for_each_entry(handler, list, link) {
+	list_for_each_entry_rcu(handler, list, link) {
 		if (handler->offset < offset + length &&
 		    offset < handler->offset + handler->length)
 			return handler;
@@ -510,7 +511,7 @@ static struct fw_address_handler *lookup_enclosing_address_handler(
 {
 	struct fw_address_handler *handler;
 
-	list_for_each_entry(handler, list, link) {
+	list_for_each_entry_rcu(handler, list, link) {
 		if (is_enclosing_handler(handler, offset, length))
 			return handler;
 	}
@@ -518,7 +519,7 @@ static struct fw_address_handler *lookup_enclosing_address_handler(
 	return NULL;
 }
 
-static DEFINE_SPINLOCK(address_handler_lock);
+static DEFINE_SPINLOCK(address_handler_list_lock);
 static LIST_HEAD(address_handler_list);
 
 const struct fw_address_region fw_high_memory_region =
@@ -555,6 +556,7 @@ static bool is_in_fcp_region(u64 offset, size_t length)
  * the specified callback is invoked.  The parameters passed to the callback
  * give the details of the particular request.
  *
+ * To be called in process context.
  * Return value:  0 on success, non-zero otherwise.
  *
  * The start offset of the handler's address region is determined by
@@ -575,7 +577,7 @@ int fw_core_add_address_handler(struct fw_address_handler *handler,
 	    handler->length == 0)
 		return -EINVAL;
 
-	spin_lock_bh(&address_handler_lock);
+	spin_lock(&address_handler_list_lock);
 
 	handler->offset = region->start;
 	while (handler->offset + handler->length <= region->end) {
@@ -588,13 +590,13 @@ int fw_core_add_address_handler(struct fw_address_handler *handler,
 		if (other != NULL) {
 			handler->offset += other->length;
 		} else {
-			list_add_tail(&handler->link, &address_handler_list);
+			list_add_tail_rcu(&handler->link, &address_handler_list);
 			ret = 0;
 			break;
 		}
 	}
 
-	spin_unlock_bh(&address_handler_lock);
+	spin_unlock(&address_handler_list_lock);
 
 	return ret;
 }
@@ -603,14 +605,17 @@ EXPORT_SYMBOL(fw_core_add_address_handler);
 /**
  * fw_core_remove_address_handler() - unregister an address handler
  *
+ * To be called in process context.
+ *
  * When fw_core_remove_address_handler() returns, @handler->callback() is
  * guaranteed to not run on any CPU anymore.
  */
 void fw_core_remove_address_handler(struct fw_address_handler *handler)
 {
-	spin_lock_bh(&address_handler_lock);
-	list_del(&handler->link);
-	spin_unlock_bh(&address_handler_lock);
+	spin_lock(&address_handler_list_lock);
+	list_del_rcu(&handler->link);
+	spin_unlock(&address_handler_list_lock);
+	synchronize_rcu();
 }
 EXPORT_SYMBOL(fw_core_remove_address_handler);
 
@@ -844,7 +849,7 @@ static void handle_exclusive_region_request(struct fw_card *card,
 	if (tcode == TCODE_LOCK_REQUEST)
 		tcode = 0x10 + HEADER_GET_EXTENDED_TCODE(p->header[3]);
 
-	spin_lock_bh(&address_handler_lock);
+	rcu_read_lock();
 	handler = lookup_enclosing_address_handler(&address_handler_list,
 						   offset, request->length);
 	if (handler)
@@ -853,7 +858,7 @@ static void handle_exclusive_region_request(struct fw_card *card,
 					  p->generation, offset,
 					  request->data, request->length,
 					  handler->callback_data);
-	spin_unlock_bh(&address_handler_lock);
+	rcu_read_unlock();
 
 	if (!handler)
 		fw_send_response(card, request, RCODE_ADDRESS_ERROR);
@@ -886,8 +891,8 @@ static void handle_fcp_region_request(struct fw_card *card,
 		return;
 	}
 
-	spin_lock_bh(&address_handler_lock);
-	list_for_each_entry(handler, &address_handler_list, link) {
+	rcu_read_lock();
+	list_for_each_entry_rcu(handler, &address_handler_list, link) {
 		if (is_enclosing_handler(handler, offset, request->length))
 			handler->address_callback(card, NULL, tcode,
 						  destination, source,
@@ -896,7 +901,7 @@ static void handle_fcp_region_request(struct fw_card *card,
 						  request->length,
 						  handler->callback_data);
 	}
-	spin_unlock_bh(&address_handler_lock);
+	rcu_read_unlock();
 
 	fw_send_response(card, request, RCODE_COMPLETE);
 }
diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c
index c788dbdaf3b..834e71d2324 100644
--- a/drivers/firewire/ohci.c
+++ b/drivers/firewire/ohci.c
@@ -1777,11 +1777,35 @@ static int get_self_id_pos(struct fw_ohci *ohci, u32 self_id,
 	return i;
 }
 
+static int initiated_reset(struct fw_ohci *ohci)
+{
+	int reg;
+	int ret = 0;
+
+	mutex_lock(&ohci->phy_reg_mutex);
+	reg = write_phy_reg(ohci, 7, 0xe0); /* Select page 7 */
+	if (reg >= 0) {
+		reg = read_phy_reg(ohci, 8);
+		reg |= 0x40;
+		reg = write_phy_reg(ohci, 8, reg); /* set PMODE bit */
+		if (reg >= 0) {
+			reg = read_phy_reg(ohci, 12); /* read register 12 */
+			if (reg >= 0) {
+				if ((reg & 0x08) == 0x08) {
+					/* bit 3 indicates "initiated reset" */
+					ret = 0x2;
+				}
+			}
+		}
+	}
+	mutex_unlock(&ohci->phy_reg_mutex);
+	return ret;
+}
+
 /*
  * TI TSB82AA2B and TSB12LV26 do not receive the selfID of a locally
  * attached TSB41BA3D phy; see http://www.ti.com/litv/pdf/sllz059.
  * Construct the selfID from phy register contents.
- * FIXME:  How to determine the selfID.i flag?
  */
 static int find_and_insert_self_id(struct fw_ohci *ohci, int self_id_count)
 {
@@ -1814,6 +1838,8 @@ static int find_and_insert_self_id(struct fw_ohci *ohci, int self_id_count)
 		self_id |= ((status & 0x3) << (6 - (i * 2)));
 	}
 
+	self_id |= initiated_reset(ohci);
+
 	pos = get_self_id_pos(ohci, self_id, self_id_count);
 	if (pos >= 0) {
 		memmove(&(ohci->self_id_buffer[pos+1]),
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 8382dc83292..aa73ef3233b 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -409,6 +409,13 @@ config GPIO_TWL4030
 	  Say yes here to access the GPIO signals of various multi-function
 	  power management chips from Texas Instruments.
 
+config GPIO_TWL6040
+	tristate "TWL6040 GPO"
+	depends on TWL6040_CORE
+	help
+	  Say yes here to access the GPO signals of twl6040
+	  audio chip from Texas Instruments.
+
 config GPIO_WM831X
 	tristate "WM831x GPIOs"
 	depends on MFD_WM831X
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 0ffaa8423e8..b2c109d1303 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -68,6 +68,7 @@ obj-$(CONFIG_GPIO_TPS6586X)	+= gpio-tps6586x.o
 obj-$(CONFIG_GPIO_TPS65910)	+= gpio-tps65910.o
 obj-$(CONFIG_GPIO_TPS65912)	+= gpio-tps65912.o
 obj-$(CONFIG_GPIO_TWL4030)	+= gpio-twl4030.o
+obj-$(CONFIG_GPIO_TWL6040)	+= gpio-twl6040.o
 obj-$(CONFIG_GPIO_UCB1400)	+= gpio-ucb1400.o
 obj-$(CONFIG_GPIO_VR41XX)	+= gpio-vr41xx.o
 obj-$(CONFIG_GPIO_VT8500)	+= gpio-vt8500.o
diff --git a/drivers/gpio/gpio-ich.c b/drivers/gpio/gpio-ich.c
index b7c06517403..d4d61796669 100644
--- a/drivers/gpio/gpio-ich.c
+++ b/drivers/gpio/gpio-ich.c
@@ -49,6 +49,10 @@ static const u8 ichx_regs[3][3] = {
 	{0x0c, 0x38, 0x48},	/* LVL[1-3] offsets */
 };
 
+static const u8 ichx_reglen[3] = {
+	0x30, 0x10, 0x10,
+};
+
 #define ICHX_WRITE(val, reg, base_res)	outl(val, (reg) + (base_res)->start)
 #define ICHX_READ(reg, base_res)	inl((reg) + (base_res)->start)
 
@@ -75,6 +79,7 @@ static struct {
 	struct resource *pm_base;	/* Power Mangagment IO base */
 	struct ichx_desc *desc;	/* Pointer to chipset-specific description */
 	u32 orig_gpio_ctrl;	/* Orig CTRL value, used to restore on exit */
+	u8 use_gpio;		/* Which GPIO groups are usable */
 } ichx_priv;
 
 static int modparam_gpiobase = -1;	/* dynamic */
@@ -123,8 +128,16 @@ static int ichx_read_bit(int reg, unsigned nr)
 	return data & (1 << bit) ? 1 : 0;
 }
 
+static int ichx_gpio_check_available(struct gpio_chip *gpio, unsigned nr)
+{
+	return (ichx_priv.use_gpio & (1 << (nr / 32))) ? 0 : -ENXIO;
+}
+
 static int ichx_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 {
+	if (!ichx_gpio_check_available(gpio, nr))
+		return -ENXIO;
+
 	/*
 	 * Try setting pin as an input and verify it worked since many pins
 	 * are output-only.
@@ -138,6 +151,9 @@ static int ichx_gpio_direction_input(struct gpio_chip *gpio, unsigned nr)
 static int ichx_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 					int val)
 {
+	if (!ichx_gpio_check_available(gpio, nr))
+		return -ENXIO;
+
 	/* Set GPIO output value. */
 	ichx_write_bit(GPIO_LVL, nr, val, 0);
 
@@ -153,6 +169,9 @@ static int ichx_gpio_direction_output(struct gpio_chip *gpio, unsigned nr,
 
 static int ichx_gpio_get(struct gpio_chip *chip, unsigned nr)
 {
+	if (!ichx_gpio_check_available(chip, nr))
+		return -ENXIO;
+
 	return ichx_read_bit(GPIO_LVL, nr);
 }
 
@@ -161,6 +180,9 @@ static int ich6_gpio_get(struct gpio_chip *chip, unsigned nr)
 	unsigned long flags;
 	u32 data;
 
+	if (!ichx_gpio_check_available(chip, nr))
+		return -ENXIO;
+
 	/*
 	 * GPI 0 - 15 need to be read from the power management registers on
 	 * a ICH6/3100 bridge.
@@ -291,6 +313,46 @@ static struct ichx_desc intel5_desc = {
 	.ngpio = 76,
 };
 
+static int __devinit ichx_gpio_request_regions(struct resource *res_base,
+						const char *name, u8 use_gpio)
+{
+	int i;
+
+	if (!res_base || !res_base->start || !res_base->end)
+		return -ENODEV;
+
+	for (i = 0; i < ARRAY_SIZE(ichx_regs[0]); i++) {
+		if (!(use_gpio & (1 << i)))
+			continue;
+		if (!request_region(res_base->start + ichx_regs[0][i],
+				    ichx_reglen[i], name))
+			goto request_err;
+	}
+	return 0;
+
+request_err:
+	/* Clean up: release already requested regions, if any */
+	for (i--; i >= 0; i--) {
+		if (!(use_gpio & (1 << i)))
+			continue;
+		release_region(res_base->start + ichx_regs[0][i],
+			       ichx_reglen[i]);
+	}
+	return -EBUSY;
+}
+
+static void ichx_gpio_release_regions(struct resource *res_base, u8 use_gpio)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ichx_regs[0]); i++) {
+		if (!(use_gpio & (1 << i)))
+			continue;
+		release_region(res_base->start + ichx_regs[0][i],
+			       ichx_reglen[i]);
+	}
+}
+
 static int __devinit ichx_gpio_probe(struct platform_device *pdev)
 {
 	struct resource *res_base, *res_pm;
@@ -329,12 +391,11 @@ static int __devinit ichx_gpio_probe(struct platform_device *pdev)
 	}
 
 	res_base = platform_get_resource(pdev, IORESOURCE_IO, ICH_RES_GPIO);
-	if (!res_base || !res_base->start || !res_base->end)
-		return -ENODEV;
-
-	if (!request_region(res_base->start, resource_size(res_base),
-				pdev->name))
-		return -EBUSY;
+	ichx_priv.use_gpio = ich_info->use_gpio;
+	err = ichx_gpio_request_regions(res_base, pdev->name,
+					ichx_priv.use_gpio);
+	if (err)
+		return err;
 
 	ichx_priv.gpio_base = res_base;
 
@@ -374,8 +435,7 @@ init:
 	return 0;
 
 add_err:
-	release_region(ichx_priv.gpio_base->start,
-			resource_size(ichx_priv.gpio_base));
+	ichx_gpio_release_regions(ichx_priv.gpio_base, ichx_priv.use_gpio);
 	if (ichx_priv.pm_base)
 		release_region(ichx_priv.pm_base->start,
 				resource_size(ichx_priv.pm_base));
@@ -393,8 +453,7 @@ static int __devexit ichx_gpio_remove(struct platform_device *pdev)
 		return err;
 	}
 
-	release_region(ichx_priv.gpio_base->start,
-				resource_size(ichx_priv.gpio_base));
+	ichx_gpio_release_regions(ichx_priv.gpio_base, ichx_priv.use_gpio);
 	if (ichx_priv.pm_base)
 		release_region(ichx_priv.pm_base->start,
 				resource_size(ichx_priv.pm_base));
diff --git a/drivers/gpio/gpio-twl6040.c b/drivers/gpio/gpio-twl6040.c
new file mode 100644
index 00000000000..dd58e8b2504
--- /dev/null
+++ b/drivers/gpio/gpio-twl6040.c
@@ -0,0 +1,137 @@
+/*
+ * Access to GPOs on TWL6040 chip
+ *
+ * Copyright (C) 2012 Texas Instruments, Inc.
+ *
+ * Authors:
+ *	Sergio Aguirre <saaguirre@ti.com>
+ *	Peter Ujfalusi <peter.ujfalusi@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kthread.h>
+#include <linux/irq.h>
+#include <linux/gpio.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+
+#include <linux/mfd/twl6040.h>
+
+static struct gpio_chip twl6040gpo_chip;
+
+static int twl6040gpo_get(struct gpio_chip *chip, unsigned offset)
+{
+	struct twl6040 *twl6040 = dev_get_drvdata(chip->dev->parent);
+	int ret = 0;
+
+	ret = twl6040_reg_read(twl6040, TWL6040_REG_GPOCTL);
+	if (ret < 0)
+		return ret;
+
+	return (ret >> offset) & 1;
+}
+
+static int twl6040gpo_direction_out(struct gpio_chip *chip, unsigned offset,
+				    int value)
+{
+	/* This only drives GPOs, and can't change direction */
+	return 0;
+}
+
+static void twl6040gpo_set(struct gpio_chip *chip, unsigned offset, int value)
+{
+	struct twl6040 *twl6040 = dev_get_drvdata(chip->dev->parent);
+	int ret;
+	u8 gpoctl;
+
+	ret = twl6040_reg_read(twl6040, TWL6040_REG_GPOCTL);
+	if (ret < 0)
+		return;
+
+	if (value)
+		gpoctl = ret | (1 << offset);
+	else
+		gpoctl = ret & ~(1 << offset);
+
+	twl6040_reg_write(twl6040, TWL6040_REG_GPOCTL, gpoctl);
+}
+
+static struct gpio_chip twl6040gpo_chip = {
+	.label			= "twl6040",
+	.owner			= THIS_MODULE,
+	.get			= twl6040gpo_get,
+	.direction_output	= twl6040gpo_direction_out,
+	.set			= twl6040gpo_set,
+	.can_sleep		= 1,
+};
+
+/*----------------------------------------------------------------------*/
+
+static int __devinit gpo_twl6040_probe(struct platform_device *pdev)
+{
+	struct twl6040_gpo_data *pdata = pdev->dev.platform_data;
+	struct device *twl6040_core_dev = pdev->dev.parent;
+	struct twl6040 *twl6040 = dev_get_drvdata(twl6040_core_dev);
+	int ret;
+
+	if (pdata)
+		twl6040gpo_chip.base = pdata->gpio_base;
+	else
+		twl6040gpo_chip.base = -1;
+
+	if (twl6040_get_revid(twl6040) < TWL6041_REV_ES2_0)
+		twl6040gpo_chip.ngpio = 3; /* twl6040 have 3 GPO */
+	else
+		twl6040gpo_chip.ngpio = 1; /* twl6041 have 1 GPO */
+
+	twl6040gpo_chip.dev = &pdev->dev;
+#ifdef CONFIG_OF_GPIO
+	twl6040gpo_chip.of_node = twl6040_core_dev->of_node;
+#endif
+
+	ret = gpiochip_add(&twl6040gpo_chip);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "could not register gpiochip, %d\n", ret);
+		twl6040gpo_chip.ngpio = 0;
+	}
+
+	return ret;
+}
+
+static int __devexit gpo_twl6040_remove(struct platform_device *pdev)
+{
+	return gpiochip_remove(&twl6040gpo_chip);
+}
+
+/* Note:  this hardware lives inside an I2C-based multi-function device. */
+MODULE_ALIAS("platform:twl6040-gpo");
+
+static struct platform_driver gpo_twl6040_driver = {
+	.driver = {
+		.name	= "twl6040-gpo",
+		.owner	= THIS_MODULE,
+	},
+	.probe		= gpo_twl6040_probe,
+	.remove		= gpo_twl6040_remove,
+};
+
+module_platform_driver(gpo_twl6040_driver);
+
+MODULE_AUTHOR("Texas Instruments, Inc.");
+MODULE_DESCRIPTION("GPO interface for TWL6040");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 90e28081712..18321b68b88 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -22,7 +22,7 @@ menuconfig DRM
 config DRM_USB
 	tristate
 	depends on DRM
-	depends on USB_ARCH_HAS_HCD
+	depends on USB_SUPPORT && USB_ARCH_HAS_HCD
 	select USB
 
 config DRM_KMS_HELPER
@@ -54,6 +54,21 @@ config DRM_TTM
 	  GPU memory types. Will be enabled automatically if a device driver
 	  uses it.
 
+config DRM_GEM_CMA_HELPER
+	bool
+	depends on DRM
+	help
+	  Choose this if you need the GEM CMA helper functions
+
+config DRM_KMS_CMA_HELPER
+	bool
+	select DRM_GEM_CMA_HELPER
+	select FB_SYS_FILLRECT
+	select FB_SYS_COPYAREA
+	select FB_SYS_IMAGEBLIT
+	help
+	  Choose this if you need the KMS CMA helper functions
+
 config DRM_TDFX
 	tristate "3dfx Banshee/Voodoo3+"
 	depends on DRM && PCI
@@ -193,3 +208,5 @@ source "drivers/gpu/drm/ast/Kconfig"
 source "drivers/gpu/drm/mgag200/Kconfig"
 
 source "drivers/gpu/drm/cirrus/Kconfig"
+
+source "drivers/gpu/drm/shmobile/Kconfig"
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index f65f65ed0dd..2ff5cefe9ea 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -15,11 +15,13 @@ drm-y       :=	drm_auth.o drm_buffer.o drm_bufs.o drm_cache.o \
 		drm_trace_points.o drm_global.o drm_prime.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
+drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
 
 drm-usb-y   := drm_usb.o
 
 drm_kms_helper-y := drm_fb_helper.o drm_crtc_helper.o drm_dp_i2c_helper.o
 drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
+drm_kms_helper-$(CONFIG_DRM_KMS_CMA_HELPER) += drm_fb_cma_helper.o
 
 obj-$(CONFIG_DRM_KMS_HELPER) += drm_kms_helper.o
 
@@ -45,4 +47,5 @@ obj-$(CONFIG_DRM_EXYNOS) +=exynos/
 obj-$(CONFIG_DRM_GMA500) += gma500/
 obj-$(CONFIG_DRM_UDL) += udl/
 obj-$(CONFIG_DRM_AST) += ast/
+obj-$(CONFIG_DRM_SHMOBILE) +=shmobile/
 obj-y			+= i2c/
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index 36164806b9d..31123b6a0be 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -28,9 +28,8 @@
 #include <linux/module.h>
 #include <linux/console.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "ast_drv.h"
 
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index d4af9edcbb9..5ccf984f063 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -28,13 +28,13 @@
 #ifndef __AST_DRV_H__
 #define __AST_DRV_H__
 
-#include "drm_fb_helper.h"
+#include <drm/drm_fb_helper.h>
 
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
+#include <drm/ttm/ttm_bo_api.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_memory.h>
+#include <drm/ttm/ttm_module.h>
 
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
@@ -94,7 +94,6 @@ struct ast_private {
 		struct drm_global_reference mem_global_ref;
 		struct ttm_bo_global_ref bo_global_ref;
 		struct ttm_bo_device bdev;
-		atomic_t validate_sequence;
 	} ttm;
 
 	struct drm_gem_object *cursor_cache;
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index 2fc8e9e860b..d9ec77959df 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -37,10 +37,9 @@
 #include <linux/init.h>
 
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fb_helper.h>
 #include "ast_drv.h"
 
 static void ast_dirty_update(struct ast_fbdev *afbdev,
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index 95ae55b8214..f668e6cc0f7 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -25,12 +25,12 @@
 /*
  * Authors: Dave Airlie <airlied@redhat.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "ast_drv.h"
 
 
-#include "drm_fb_helper.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "ast_dram_tables.h"
 
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index a712cafcfa1..7fc9f7272b5 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -28,9 +28,9 @@
  * Authors: Dave Airlie <airlied@redhat.com>
  */
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
 #include "ast_drv.h"
 
 #include "ast_tables.h"
@@ -582,7 +582,6 @@ static const struct drm_crtc_helper_funcs ast_crtc_helper_funcs = {
 	.mode_set_base = ast_crtc_mode_set_base,
 	.disable = ast_crtc_disable,
 	.load_lut = ast_crtc_load_lut,
-	.disable = ast_crtc_disable,
 	.prepare = ast_crtc_prepare,
 	.commit = ast_crtc_commit,
 
@@ -737,6 +736,7 @@ static int ast_get_modes(struct drm_connector *connector)
 	if (edid) {
 		drm_mode_connector_update_edid_property(&ast_connector->base, edid);
 		ret = drm_add_edid_modes(connector, edid);
+		kfree(edid);
 		return ret;
 	} else
 		drm_mode_connector_update_edid_property(&ast_connector->base, NULL);
diff --git a/drivers/gpu/drm/ast/ast_post.c b/drivers/gpu/drm/ast/ast_post.c
index 6edbee63b0c..977cfb35837 100644
--- a/drivers/gpu/drm/ast/ast_post.c
+++ b/drivers/gpu/drm/ast/ast_post.c
@@ -26,7 +26,7 @@
  * Authors: Dave Airlie <airlied@redhat.com>
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "ast_drv.h"
 
 #include "ast_dram_tables.h"
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 6cf2adea66b..1a026ac2dfb 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -25,7 +25,7 @@
 /*
  * Authors: Dave Airlie <airlied@redhat.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "ast_drv.h"
 #include <ttm/ttm_page_alloc.h>
 
diff --git a/drivers/gpu/drm/ati_pcigart.c b/drivers/gpu/drm/ati_pcigart.c
index 9afe495c12c..c399dea27a3 100644
--- a/drivers/gpu/drm/ati_pcigart.c
+++ b/drivers/gpu/drm/ati_pcigart.c
@@ -32,7 +32,7 @@
  */
 
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 # define ATI_PCIGART_PAGE_SIZE		4096	/**< PCI GART page size */
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
index b83a2d7ddd1..101e423c899 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.c
@@ -10,8 +10,7 @@
  */
 #include <linux/module.h>
 #include <linux/console.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include "cirrus_drv.h"
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.h b/drivers/gpu/drm/cirrus/cirrus_drv.h
index 64ea597cb6d..6e0cc724e5a 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.h
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.h
@@ -15,11 +15,11 @@
 
 #include <drm/drm_fb_helper.h>
 
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
+#include <drm/ttm/ttm_bo_api.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_memory.h>
+#include <drm/ttm/ttm_module.h>
 
 #define DRIVER_AUTHOR		"Matthew Garrett"
 
@@ -143,7 +143,6 @@ struct cirrus_device {
 		struct drm_global_reference mem_global_ref;
 		struct ttm_bo_global_ref bo_global_ref;
 		struct ttm_bo_device bdev;
-		atomic_t validate_sequence;
 	} ttm;
 	bool mm_inited;
 };
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
index 9a276a53699..6c6b4c87d30 100644
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
@@ -9,9 +9,8 @@
  *          Dave Airlie
  */
 #include <linux/module.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_fb_helper.h>
 
 #include <linux/fb.h>
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index e3c12257841..6a9b12e88d4 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -8,9 +8,8 @@
  * Authors: Matthew Garrett
  *          Dave Airlie
  */
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "cirrus_drv.h"
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index a44d31aa4e3..60685b21cc3 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -14,9 +14,8 @@
  *
  * Copyright 1999-2001 Jeff Garzik <jgarzik@pobox.com>
  */
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include <video/cirrus.h>
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 50e170f879d..bc83f835c83 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -25,7 +25,7 @@
 /*
  * Authors: Dave Airlie <airlied@redhat.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "cirrus_drv.h"
 #include <ttm/ttm_page_alloc.h>
 
diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c
index 0cb2ba50af5..3d8fed17979 100644
--- a/drivers/gpu/drm/drm_agpsupport.c
+++ b/drivers/gpu/drm/drm_agpsupport.c
@@ -31,7 +31,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index ba23790450e..3cedae12b3c 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -33,7 +33,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**
  * Find the file with the given magic number.
diff --git a/drivers/gpu/drm/drm_buffer.c b/drivers/gpu/drm/drm_buffer.c
index 08ccefedb32..39a71834031 100644
--- a/drivers/gpu/drm/drm_buffer.c
+++ b/drivers/gpu/drm/drm_buffer.c
@@ -33,7 +33,7 @@
  */
 
 #include <linux/export.h>
-#include "drm_buffer.h"
+#include <drm/drm_buffer.h>
 
 /**
  * Allocate the drm buffer object.
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index b356c719f2f..0128147265f 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -38,7 +38,7 @@
 #include <linux/log2.h>
 #include <linux/export.h>
 #include <asm/shmparam.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 static struct drm_map_list *drm_find_matching_map(struct drm_device *dev,
 						  struct drm_local_map *map)
diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c
index 08758e06147..a575cb2e6bd 100644
--- a/drivers/gpu/drm/drm_cache.c
+++ b/drivers/gpu/drm/drm_cache.c
@@ -29,7 +29,7 @@
  */
 
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #if defined(CONFIG_X86)
 static void
@@ -37,12 +37,13 @@ drm_clflush_page(struct page *page)
 {
 	uint8_t *page_virtual;
 	unsigned int i;
+	const int size = boot_cpu_data.x86_clflush_size;
 
 	if (unlikely(page == NULL))
 		return;
 
 	page_virtual = kmap_atomic(page);
-	for (i = 0; i < PAGE_SIZE; i += boot_cpu_data.x86_clflush_size)
+	for (i = 0; i < PAGE_SIZE; i += size)
 		clflush(page_virtual + i);
 	kunmap_atomic(page_virtual);
 }
@@ -100,6 +101,31 @@ drm_clflush_pages(struct page *pages[], unsigned long num_pages)
 EXPORT_SYMBOL(drm_clflush_pages);
 
 void
+drm_clflush_sg(struct sg_table *st)
+{
+#if defined(CONFIG_X86)
+	if (cpu_has_clflush) {
+		struct scatterlist *sg;
+		int i;
+
+		mb();
+		for_each_sg(st->sgl, sg, st->nents, i)
+			drm_clflush_page(sg_page(sg));
+		mb();
+
+		return;
+	}
+
+	if (on_each_cpu(drm_clflush_ipi_handler, NULL, 1) != 0)
+		printk(KERN_ERR "Timed out waiting for cache flush.\n");
+#else
+	printk(KERN_ERR "Architecture has no drm_cache.c support\n");
+	WARN_ON_ONCE(1);
+#endif
+}
+EXPORT_SYMBOL(drm_clflush_sg);
+
+void
 drm_clflush_virt_range(char *addr, unsigned long length)
 {
 #if defined(CONFIG_X86)
diff --git a/drivers/gpu/drm/drm_context.c b/drivers/gpu/drm/drm_context.c
index affa629589a..45adf97e678 100644
--- a/drivers/gpu/drm/drm_context.c
+++ b/drivers/gpu/drm/drm_context.c
@@ -40,7 +40,7 @@
  *		needed by SiS driver's memory management.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /******************************************************************/
 /** \name Context bitmap support */
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 6fbfc244748..ef1b22144d3 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -32,11 +32,10 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drm.h"
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
-#include "drm_fourcc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_fourcc.h>
 
 /* Avoid boilerplate.  I'm tired of typing. */
 #define DRM_ENUM_NAME_FN(fnname, list)				\
@@ -294,6 +293,8 @@ int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
 {
 	int ret;
 
+	kref_init(&fb->refcount);
+
 	ret = drm_mode_object_get(dev, &fb->base, DRM_MODE_OBJECT_FB);
 	if (ret)
 		return ret;
@@ -307,6 +308,38 @@ int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
 }
 EXPORT_SYMBOL(drm_framebuffer_init);
 
+static void drm_framebuffer_free(struct kref *kref)
+{
+	struct drm_framebuffer *fb =
+			container_of(kref, struct drm_framebuffer, refcount);
+	fb->funcs->destroy(fb);
+}
+
+/**
+ * drm_framebuffer_unreference - unref a framebuffer
+ *
+ * LOCKING:
+ * Caller must hold mode config lock.
+ */
+void drm_framebuffer_unreference(struct drm_framebuffer *fb)
+{
+	struct drm_device *dev = fb->dev;
+	DRM_DEBUG("FB ID: %d\n", fb->base.id);
+	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
+	kref_put(&fb->refcount, drm_framebuffer_free);
+}
+EXPORT_SYMBOL(drm_framebuffer_unreference);
+
+/**
+ * drm_framebuffer_reference - incr the fb refcnt
+ */
+void drm_framebuffer_reference(struct drm_framebuffer *fb)
+{
+	DRM_DEBUG("FB ID: %d\n", fb->base.id);
+	kref_get(&fb->refcount);
+}
+EXPORT_SYMBOL(drm_framebuffer_reference);
+
 /**
  * drm_framebuffer_cleanup - remove a framebuffer object
  * @fb: framebuffer to remove
@@ -320,6 +353,32 @@ EXPORT_SYMBOL(drm_framebuffer_init);
 void drm_framebuffer_cleanup(struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = fb->dev;
+	/*
+	 * This could be moved to drm_framebuffer_remove(), but for
+	 * debugging is nice to keep around the list of fb's that are
+	 * no longer associated w/ a drm_file but are not unreferenced
+	 * yet.  (i915 and omapdrm have debugfs files which will show
+	 * this.)
+	 */
+	drm_mode_object_put(dev, &fb->base);
+	list_del(&fb->head);
+	dev->mode_config.num_fb--;
+}
+EXPORT_SYMBOL(drm_framebuffer_cleanup);
+
+/**
+ * drm_framebuffer_remove - remove and unreference a framebuffer object
+ * @fb: framebuffer to remove
+ *
+ * LOCKING:
+ * Caller must hold mode config lock.
+ *
+ * Scans all the CRTCs and planes in @dev's mode_config.  If they're
+ * using @fb, removes it, setting it to NULL.
+ */
+void drm_framebuffer_remove(struct drm_framebuffer *fb)
+{
+	struct drm_device *dev = fb->dev;
 	struct drm_crtc *crtc;
 	struct drm_plane *plane;
 	struct drm_mode_set set;
@@ -350,11 +409,11 @@ void drm_framebuffer_cleanup(struct drm_framebuffer *fb)
 		}
 	}
 
-	drm_mode_object_put(dev, &fb->base);
-	list_del(&fb->head);
-	dev->mode_config.num_fb--;
+	list_del(&fb->filp_head);
+
+	drm_framebuffer_unreference(fb);
 }
-EXPORT_SYMBOL(drm_framebuffer_cleanup);
+EXPORT_SYMBOL(drm_framebuffer_remove);
 
 /**
  * drm_crtc_init - Initialise a new CRTC object
@@ -377,6 +436,7 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 
 	crtc->dev = dev;
 	crtc->funcs = funcs;
+	crtc->invert_dimensions = false;
 
 	mutex_lock(&dev->mode_config.mutex);
 
@@ -1031,11 +1091,7 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 	}
 
 	list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
-		fb->funcs->destroy(fb);
-	}
-
-	list_for_each_entry_safe(crtc, ct, &dev->mode_config.crtc_list, head) {
-		crtc->funcs->destroy(crtc);
+		drm_framebuffer_remove(fb);
 	}
 
 	list_for_each_entry_safe(plane, plt, &dev->mode_config.plane_list,
@@ -1043,6 +1099,10 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 		plane->funcs->destroy(plane);
 	}
 
+	list_for_each_entry_safe(crtc, ct, &dev->mode_config.crtc_list, head) {
+		crtc->funcs->destroy(crtc);
+	}
+
 	idr_remove_all(&dev->mode_config.crtc_idr);
 	idr_destroy(&dev->mode_config.crtc_idr);
 }
@@ -1852,6 +1912,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 	DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
 
 	if (crtc_req->mode_valid) {
+		int hdisplay, vdisplay;
 		/* If we have a mode we need a framebuffer. */
 		/* If we pass -1, set the mode with the currently bound fb */
 		if (crtc_req->fb_id == -1) {
@@ -1887,14 +1948,20 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 
 		drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
 
-		if (mode->hdisplay > fb->width ||
-		    mode->vdisplay > fb->height ||
-		    crtc_req->x > fb->width - mode->hdisplay ||
-		    crtc_req->y > fb->height - mode->vdisplay) {
-			DRM_DEBUG_KMS("Invalid CRTC viewport %ux%u+%u+%u for fb size %ux%u.\n",
-				      mode->hdisplay, mode->vdisplay,
-				      crtc_req->x, crtc_req->y,
-				      fb->width, fb->height);
+		hdisplay = mode->hdisplay;
+		vdisplay = mode->vdisplay;
+
+		if (crtc->invert_dimensions)
+			swap(hdisplay, vdisplay);
+
+		if (hdisplay > fb->width ||
+		    vdisplay > fb->height ||
+		    crtc_req->x > fb->width - hdisplay ||
+		    crtc_req->y > fb->height - vdisplay) {
+			DRM_DEBUG_KMS("Invalid fb size %ux%u for CRTC viewport %ux%u+%d+%d%s.\n",
+				      fb->width, fb->height,
+				      hdisplay, vdisplay, crtc_req->x, crtc_req->y,
+				      crtc->invert_dimensions ? " (inverted)" : "");
 			ret = -ENOSPC;
 			goto out;
 		}
@@ -2169,6 +2236,8 @@ static int format_check(const struct drm_mode_fb_cmd2 *r)
 	case DRM_FORMAT_NV21:
 	case DRM_FORMAT_NV16:
 	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV24:
+	case DRM_FORMAT_NV42:
 	case DRM_FORMAT_YUV410:
 	case DRM_FORMAT_YVU410:
 	case DRM_FORMAT_YUV411:
@@ -2335,11 +2404,7 @@ int drm_mode_rmfb(struct drm_device *dev,
 		goto out;
 	}
 
-	/* TODO release all crtc connected to the framebuffer */
-	/* TODO unhock the destructor from the buffer object */
-
-	list_del(&fb->filp_head);
-	fb->funcs->destroy(fb);
+	drm_framebuffer_remove(fb);
 
 out:
 	mutex_unlock(&dev->mode_config.mutex);
@@ -2489,8 +2554,7 @@ void drm_fb_release(struct drm_file *priv)
 
 	mutex_lock(&dev->mode_config.mutex);
 	list_for_each_entry_safe(fb, tfb, &priv->fbs, filp_head) {
-		list_del(&fb->filp_head);
-		fb->funcs->destroy(fb);
+		drm_framebuffer_remove(fb);
 	}
 	mutex_unlock(&dev->mode_config.mutex);
 }
@@ -3489,6 +3553,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 	struct drm_framebuffer *fb;
 	struct drm_pending_vblank_event *e = NULL;
 	unsigned long flags;
+	int hdisplay, vdisplay;
 	int ret = -EINVAL;
 
 	if (page_flip->flags & ~DRM_MODE_PAGE_FLIP_FLAGS ||
@@ -3518,14 +3583,19 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 		goto out;
 	fb = obj_to_fb(obj);
 
-	if (crtc->mode.hdisplay > fb->width ||
-	    crtc->mode.vdisplay > fb->height ||
-	    crtc->x > fb->width - crtc->mode.hdisplay ||
-	    crtc->y > fb->height - crtc->mode.vdisplay) {
-		DRM_DEBUG_KMS("Invalid fb size %ux%u for CRTC viewport %ux%u+%d+%d.\n",
-			      fb->width, fb->height,
-			      crtc->mode.hdisplay, crtc->mode.vdisplay,
-			      crtc->x, crtc->y);
+	hdisplay = crtc->mode.hdisplay;
+	vdisplay = crtc->mode.vdisplay;
+
+	if (crtc->invert_dimensions)
+		swap(hdisplay, vdisplay);
+
+	if (hdisplay > fb->width ||
+	    vdisplay > fb->height ||
+	    crtc->x > fb->width - hdisplay ||
+	    crtc->y > fb->height - vdisplay) {
+		DRM_DEBUG_KMS("Invalid fb size %ux%u for CRTC viewport %ux%u+%d+%d%s.\n",
+			      fb->width, fb->height, hdisplay, vdisplay, crtc->x, crtc->y,
+			      crtc->invert_dimensions ? " (inverted)" : "");
 		ret = -ENOSPC;
 		goto out;
 	}
@@ -3718,6 +3788,8 @@ int drm_format_num_planes(uint32_t format)
 	case DRM_FORMAT_NV21:
 	case DRM_FORMAT_NV16:
 	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV24:
+	case DRM_FORMAT_NV42:
 		return 2;
 	default:
 		return 1;
@@ -3751,6 +3823,8 @@ int drm_format_plane_cpp(uint32_t format, int plane)
 	case DRM_FORMAT_NV21:
 	case DRM_FORMAT_NV16:
 	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV24:
+	case DRM_FORMAT_NV42:
 		return plane ? 2 : 1;
 	case DRM_FORMAT_YUV410:
 	case DRM_FORMAT_YVU410:
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 8fa9d52820d..1227adf74db 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -32,12 +32,12 @@
 #include <linux/export.h>
 #include <linux/moduleparam.h>
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_fourcc.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_edid.h>
 
 static bool drm_kms_helper_poll = true;
 module_param_named(poll, drm_kms_helper_poll, bool, 0600);
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 70b13fc1939..a05087cf846 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -34,7 +34,7 @@
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #if defined(CONFIG_DEBUG_FS)
 
diff --git a/drivers/gpu/drm/drm_dma.c b/drivers/gpu/drm/drm_dma.c
index 08f5e5309b2..495b5fd2787 100644
--- a/drivers/gpu/drm/drm_dma.c
+++ b/drivers/gpu/drm/drm_dma.c
@@ -34,7 +34,7 @@
  */
 
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**
  * Initialize the DMA data.
diff --git a/drivers/gpu/drm/drm_dp_i2c_helper.c b/drivers/gpu/drm/drm_dp_i2c_helper.c
index f7eba0a0973..7f246f21245 100644
--- a/drivers/gpu/drm/drm_dp_i2c_helper.c
+++ b/drivers/gpu/drm/drm_dp_i2c_helper.c
@@ -27,8 +27,8 @@
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/i2c.h>
-#include "drm_dp_helper.h"
-#include "drmP.h"
+#include <drm/drm_dp_helper.h>
+#include <drm/drmP.h>
 
 /* Run a single AUX_CH I2C transaction, writing/reading data as necessary */
 static int
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 9238de4009f..be174cab105 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -49,8 +49,8 @@
 #include <linux/debugfs.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm_core.h"
+#include <drm/drmP.h>
+#include <drm/drm_core.h>
 
 
 static int drm_version(struct drm_device *dev, void *data,
@@ -140,10 +140,10 @@ static struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_PRIME_HANDLE_TO_FD, drm_prime_handle_to_fd_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_PRIME_FD_TO_HANDLE, drm_prime_fd_to_handle_ioctl, DRM_AUTH|DRM_UNLOCKED),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANERESOURCES, drm_mode_getplane_res, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANERESOURCES, drm_mode_getplane_res, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCRTC, drm_mode_getcrtc, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETCRTC, drm_mode_setcrtc, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANE, drm_mode_getplane, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPLANE, drm_mode_getplane, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPLANE, drm_mode_setplane, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CURSOR, drm_mode_cursor_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETGAMMA, drm_mode_gamma_get_ioctl, DRM_UNLOCKED),
@@ -152,19 +152,19 @@ static struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETCONNECTOR, drm_mode_getconnector, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ATTACHMODE, drm_mode_attachmode_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DETACHMODE, drm_mode_detachmode_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPERTY, drm_mode_getproperty_ioctl, DRM_MASTER | DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPERTY, drm_mode_getproperty_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_SETPROPERTY, drm_mode_connector_property_set_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPBLOB, drm_mode_getblob_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETPROPBLOB, drm_mode_getblob_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_GETFB, drm_mode_getfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
-	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 };
 
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index b7ee230572b..5dda07cf709 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -31,8 +31,8 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
-#include "drmP.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
 #include "drm_edid_modes.h"
 
 #define version_greater(edid, maj, min) \
@@ -161,7 +161,7 @@ MODULE_PARM_DESC(edid_fixup,
  * Sanity check the EDID block (base or extension).  Return 0 if the block
  * doesn't check out, or 1 if it's valid.
  */
-bool drm_edid_block_valid(u8 *raw_edid, int block)
+bool drm_edid_block_valid(u8 *raw_edid, int block, bool print_bad_edid)
 {
 	int i;
 	u8 csum = 0;
@@ -184,7 +184,9 @@ bool drm_edid_block_valid(u8 *raw_edid, int block)
 	for (i = 0; i < EDID_LENGTH; i++)
 		csum += raw_edid[i];
 	if (csum) {
-		DRM_ERROR("EDID checksum is invalid, remainder is %d\n", csum);
+		if (print_bad_edid) {
+			DRM_ERROR("EDID checksum is invalid, remainder is %d\n", csum);
+		}
 
 		/* allow CEA to slide through, switches mangle this */
 		if (raw_edid[0] != 0x02)
@@ -210,7 +212,7 @@ bool drm_edid_block_valid(u8 *raw_edid, int block)
 	return 1;
 
 bad:
-	if (raw_edid) {
+	if (raw_edid && print_bad_edid) {
 		printk(KERN_ERR "Raw EDID:\n");
 		print_hex_dump(KERN_ERR, " \t", DUMP_PREFIX_NONE, 16, 1,
 			       raw_edid, EDID_LENGTH, false);
@@ -234,7 +236,7 @@ bool drm_edid_is_valid(struct edid *edid)
 		return false;
 
 	for (i = 0; i <= edid->extensions; i++)
-		if (!drm_edid_block_valid(raw + i * EDID_LENGTH, i))
+		if (!drm_edid_block_valid(raw + i * EDID_LENGTH, i, true))
 			return false;
 
 	return true;
@@ -257,6 +259,8 @@ drm_do_probe_ddc_edid(struct i2c_adapter *adapter, unsigned char *buf,
 		      int block, int len)
 {
 	unsigned char start = block * EDID_LENGTH;
+	unsigned char segment = block >> 1;
+	unsigned char xfers = segment ? 3 : 2;
 	int ret, retries = 5;
 
 	/* The core i2c driver will automatically retry the transfer if the
@@ -268,6 +272,11 @@ drm_do_probe_ddc_edid(struct i2c_adapter *adapter, unsigned char *buf,
 	do {
 		struct i2c_msg msgs[] = {
 			{
+				.addr	= DDC_SEGMENT_ADDR,
+				.flags	= 0,
+				.len	= 1,
+				.buf	= &segment,
+			}, {
 				.addr	= DDC_ADDR,
 				.flags	= 0,
 				.len	= 1,
@@ -279,15 +288,21 @@ drm_do_probe_ddc_edid(struct i2c_adapter *adapter, unsigned char *buf,
 				.buf	= buf,
 			}
 		};
-		ret = i2c_transfer(adapter, msgs, 2);
+
+	/*
+	 * Avoid sending the segment addr to not upset non-compliant ddc
+	 * monitors.
+	 */
+		ret = i2c_transfer(adapter, &msgs[3 - xfers], xfers);
+
 		if (ret == -ENXIO) {
 			DRM_DEBUG_KMS("drm: skipping non-existent adapter %s\n",
 					adapter->name);
 			break;
 		}
-	} while (ret != 2 && --retries);
+	} while (ret != xfers && --retries);
 
-	return ret == 2 ? 0 : -1;
+	return ret == xfers ? 0 : -1;
 }
 
 static bool drm_edid_is_zero(u8 *in_edid, int length)
@@ -306,6 +321,7 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 {
 	int i, j = 0, valid_extensions = 0;
 	u8 *block, *new;
+	bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS);
 
 	if ((block = kmalloc(EDID_LENGTH, GFP_KERNEL)) == NULL)
 		return NULL;
@@ -314,7 +330,7 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 	for (i = 0; i < 4; i++) {
 		if (drm_do_probe_ddc_edid(adapter, block, 0, EDID_LENGTH))
 			goto out;
-		if (drm_edid_block_valid(block, 0))
+		if (drm_edid_block_valid(block, 0, print_bad_edid))
 			break;
 		if (i == 0 && drm_edid_is_zero(block, EDID_LENGTH)) {
 			connector->null_edid_counter++;
@@ -339,7 +355,7 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 				  block + (valid_extensions + 1) * EDID_LENGTH,
 				  j, EDID_LENGTH))
 				goto out;
-			if (drm_edid_block_valid(block + (valid_extensions + 1) * EDID_LENGTH, j)) {
+			if (drm_edid_block_valid(block + (valid_extensions + 1) * EDID_LENGTH, j, print_bad_edid)) {
 				valid_extensions++;
 				break;
 			}
@@ -362,8 +378,11 @@ drm_do_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
 	return block;
 
 carp:
-	dev_warn(connector->dev->dev, "%s: EDID block %d invalid.\n",
-		 drm_get_connector_name(connector), j);
+	if (print_bad_edid) {
+		dev_warn(connector->dev->dev, "%s: EDID block %d invalid.\n",
+			 drm_get_connector_name(connector), j);
+	}
+	connector->bad_edid_counter++;
 
 out:
 	kfree(block);
@@ -402,10 +421,7 @@ struct edid *drm_get_edid(struct drm_connector *connector,
 	if (drm_probe_ddc(adapter))
 		edid = (struct edid *)drm_do_get_edid(connector, adapter);
 
-	connector->display_info.raw_edid = (char *)edid;
-
 	return edid;
-
 }
 EXPORT_SYMBOL(drm_get_edid);
 
@@ -1523,16 +1539,57 @@ do_cea_modes (struct drm_connector *connector, u8 *db, u8 len)
 }
 
 static int
+cea_db_payload_len(const u8 *db)
+{
+	return db[0] & 0x1f;
+}
+
+static int
+cea_db_tag(const u8 *db)
+{
+	return db[0] >> 5;
+}
+
+static int
+cea_revision(const u8 *cea)
+{
+	return cea[1];
+}
+
+static int
+cea_db_offsets(const u8 *cea, int *start, int *end)
+{
+	/* Data block offset in CEA extension block */
+	*start = 4;
+	*end = cea[2];
+	if (*end == 0)
+		*end = 127;
+	if (*end < 4 || *end > 127)
+		return -ERANGE;
+	return 0;
+}
+
+#define for_each_cea_db(cea, i, start, end) \
+	for ((i) = (start); (i) < (end) && (i) + cea_db_payload_len(&(cea)[(i)]) < (end); (i) += cea_db_payload_len(&(cea)[(i)]) + 1)
+
+static int
 add_cea_modes(struct drm_connector *connector, struct edid *edid)
 {
 	u8 * cea = drm_find_cea_extension(edid);
 	u8 * db, dbl;
 	int modes = 0;
 
-	if (cea && cea[1] >= 3) {
-		for (db = cea + 4; db < cea + cea[2]; db += dbl + 1) {
-			dbl = db[0] & 0x1f;
-			if (((db[0] & 0xe0) >> 5) == VIDEO_BLOCK)
+	if (cea && cea_revision(cea) >= 3) {
+		int i, start, end;
+
+		if (cea_db_offsets(cea, &start, &end))
+			return 0;
+
+		for_each_cea_db(cea, i, start, end) {
+			db = &cea[i];
+			dbl = cea_db_payload_len(db);
+
+			if (cea_db_tag(db) == VIDEO_BLOCK)
 				modes += do_cea_modes (connector, db+1, dbl);
 		}
 	}
@@ -1541,19 +1598,28 @@ add_cea_modes(struct drm_connector *connector, struct edid *edid)
 }
 
 static void
-parse_hdmi_vsdb(struct drm_connector *connector, uint8_t *db)
+parse_hdmi_vsdb(struct drm_connector *connector, const u8 *db)
 {
-	connector->eld[5] |= (db[6] >> 7) << 1;  /* Supports_AI */
-
-	connector->dvi_dual = db[6] & 1;
-	connector->max_tmds_clock = db[7] * 5;
+	u8 len = cea_db_payload_len(db);
 
-	connector->latency_present[0] = db[8] >> 7;
-	connector->latency_present[1] = (db[8] >> 6) & 1;
-	connector->video_latency[0] = db[9];
-	connector->audio_latency[0] = db[10];
-	connector->video_latency[1] = db[11];
-	connector->audio_latency[1] = db[12];
+	if (len >= 6) {
+		connector->eld[5] |= (db[6] >> 7) << 1;  /* Supports_AI */
+		connector->dvi_dual = db[6] & 1;
+	}
+	if (len >= 7)
+		connector->max_tmds_clock = db[7] * 5;
+	if (len >= 8) {
+		connector->latency_present[0] = db[8] >> 7;
+		connector->latency_present[1] = (db[8] >> 6) & 1;
+	}
+	if (len >= 9)
+		connector->video_latency[0] = db[9];
+	if (len >= 10)
+		connector->audio_latency[0] = db[10];
+	if (len >= 11)
+		connector->video_latency[1] = db[11];
+	if (len >= 12)
+		connector->audio_latency[1] = db[12];
 
 	DRM_LOG_KMS("HDMI: DVI dual %d, "
 		    "max TMDS clock %d, "
@@ -1577,6 +1643,21 @@ monitor_name(struct detailed_timing *t, void *data)
 		*(u8 **)data = t->data.other_data.data.str.str;
 }
 
+static bool cea_db_is_hdmi_vsdb(const u8 *db)
+{
+	int hdmi_id;
+
+	if (cea_db_tag(db) != VENDOR_BLOCK)
+		return false;
+
+	if (cea_db_payload_len(db) < 5)
+		return false;
+
+	hdmi_id = db[1] | (db[2] << 8) | (db[3] << 16);
+
+	return hdmi_id == HDMI_IDENTIFIER;
+}
+
 /**
  * drm_edid_to_eld - build ELD from EDID
  * @connector: connector corresponding to the HDMI/DP sink
@@ -1623,29 +1704,40 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 	eld[18] = edid->prod_code[0];
 	eld[19] = edid->prod_code[1];
 
-	if (cea[1] >= 3)
-		for (db = cea + 4; db < cea + cea[2]; db += dbl + 1) {
-			dbl = db[0] & 0x1f;
-			
-			switch ((db[0] & 0xe0) >> 5) {
+	if (cea_revision(cea) >= 3) {
+		int i, start, end;
+
+		if (cea_db_offsets(cea, &start, &end)) {
+			start = 0;
+			end = 0;
+		}
+
+		for_each_cea_db(cea, i, start, end) {
+			db = &cea[i];
+			dbl = cea_db_payload_len(db);
+
+			switch (cea_db_tag(db)) {
 			case AUDIO_BLOCK:
 				/* Audio Data Block, contains SADs */
 				sad_count = dbl / 3;
-				memcpy(eld + 20 + mnl, &db[1], dbl);
+				if (dbl >= 1)
+					memcpy(eld + 20 + mnl, &db[1], dbl);
 				break;
 			case SPEAKER_BLOCK:
-                                /* Speaker Allocation Data Block */
-				eld[7] = db[1];
+				/* Speaker Allocation Data Block */
+				if (dbl >= 1)
+					eld[7] = db[1];
 				break;
 			case VENDOR_BLOCK:
 				/* HDMI Vendor-Specific Data Block */
-				if (db[1] == 0x03 && db[2] == 0x0c && db[3] == 0)
+				if (cea_db_is_hdmi_vsdb(db))
 					parse_hdmi_vsdb(connector, db);
 				break;
 			default:
 				break;
 			}
 		}
+	}
 	eld[5] |= sad_count << 4;
 	eld[2] = (20 + mnl + sad_count * 3 + 3) / 4;
 
@@ -1723,38 +1815,26 @@ EXPORT_SYMBOL(drm_select_eld);
 bool drm_detect_hdmi_monitor(struct edid *edid)
 {
 	u8 *edid_ext;
-	int i, hdmi_id;
+	int i;
 	int start_offset, end_offset;
-	bool is_hdmi = false;
 
 	edid_ext = drm_find_cea_extension(edid);
 	if (!edid_ext)
-		goto end;
+		return false;
 
-	/* Data block offset in CEA extension block */
-	start_offset = 4;
-	end_offset = edid_ext[2];
+	if (cea_db_offsets(edid_ext, &start_offset, &end_offset))
+		return false;
 
 	/*
 	 * Because HDMI identifier is in Vendor Specific Block,
 	 * search it from all data blocks of CEA extension.
 	 */
-	for (i = start_offset; i < end_offset;
-		/* Increased by data block len */
-		i += ((edid_ext[i] & 0x1f) + 1)) {
-		/* Find vendor specific block */
-		if ((edid_ext[i] >> 5) == VENDOR_BLOCK) {
-			hdmi_id = edid_ext[i + 1] | (edid_ext[i + 2] << 8) |
-				  edid_ext[i + 3] << 16;
-			/* Find HDMI identifier */
-			if (hdmi_id == HDMI_IDENTIFIER)
-				is_hdmi = true;
-			break;
-		}
+	for_each_cea_db(edid_ext, i, start_offset, end_offset) {
+		if (cea_db_is_hdmi_vsdb(&edid_ext[i]))
+			return true;
 	}
 
-end:
-	return is_hdmi;
+	return false;
 }
 EXPORT_SYMBOL(drm_detect_hdmi_monitor);
 
@@ -1786,15 +1866,13 @@ bool drm_detect_monitor_audio(struct edid *edid)
 		goto end;
 	}
 
-	/* Data block offset in CEA extension block */
-	start_offset = 4;
-	end_offset = edid_ext[2];
+	if (cea_db_offsets(edid_ext, &start_offset, &end_offset))
+		goto end;
 
-	for (i = start_offset; i < end_offset;
-			i += ((edid_ext[i] & 0x1f) + 1)) {
-		if ((edid_ext[i] >> 5) == AUDIO_BLOCK) {
+	for_each_cea_db(edid_ext, i, start_offset, end_offset) {
+		if (cea_db_tag(&edid_ext[i]) == AUDIO_BLOCK) {
 			has_audio = true;
-			for (j = 1; j < (edid_ext[i] & 0x1f); j += 3)
+			for (j = 1; j < cea_db_payload_len(&edid_ext[i]) + 1; j += 3)
 				DRM_DEBUG_KMS("CEA audio format %d\n",
 					      (edid_ext[i + j] >> 3) & 0xf);
 			goto end;
diff --git a/drivers/gpu/drm/drm_edid_load.c b/drivers/gpu/drm/drm_edid_load.c
index 0303935d10e..38d3943f72d 100644
--- a/drivers/gpu/drm/drm_edid_load.c
+++ b/drivers/gpu/drm/drm_edid_load.c
@@ -21,10 +21,10 @@
 
 #include <linux/module.h>
 #include <linux/firmware.h>
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
 
 static char edid_firmware[PATH_MAX];
 module_param_string(edid_firmware, edid_firmware, sizeof(edid_firmware), 0644);
@@ -114,8 +114,8 @@ static u8 generic_edid[GENERIC_EDIDS][128] = {
 	},
 };
 
-static int edid_load(struct drm_connector *connector, char *name,
-		     char *connector_name)
+static u8 *edid_load(struct drm_connector *connector, char *name,
+			char *connector_name)
 {
 	const struct firmware *fw;
 	struct platform_device *pdev;
@@ -123,6 +123,7 @@ static int edid_load(struct drm_connector *connector, char *name,
 	int fwsize, expected;
 	int builtin = 0, err = 0;
 	int i, valid_extensions = 0;
+	bool print_bad_edid = !connector->bad_edid_counter || (drm_debug & DRM_UT_KMS);
 
 	pdev = platform_device_register_simple(connector_name, -1, NULL, 0);
 	if (IS_ERR(pdev)) {
@@ -173,7 +174,8 @@ static int edid_load(struct drm_connector *connector, char *name,
 	}
 	memcpy(edid, fwdata, fwsize);
 
-	if (!drm_edid_block_valid(edid, 0)) {
+	if (!drm_edid_block_valid(edid, 0, print_bad_edid)) {
+		connector->bad_edid_counter++;
 		DRM_ERROR("Base block of EDID firmware \"%s\" is invalid ",
 		    name);
 		kfree(edid);
@@ -185,7 +187,7 @@ static int edid_load(struct drm_connector *connector, char *name,
 		if (i != valid_extensions + 1)
 			memcpy(edid + (valid_extensions + 1) * EDID_LENGTH,
 			    edid + i * EDID_LENGTH, EDID_LENGTH);
-		if (drm_edid_block_valid(edid + i * EDID_LENGTH, i))
+		if (drm_edid_block_valid(edid + i * EDID_LENGTH, i, print_bad_edid))
 			valid_extensions++;
 	}
 
@@ -205,7 +207,6 @@ static int edid_load(struct drm_connector *connector, char *name,
 		edid = new_edid;
 	}
 
-	connector->display_info.raw_edid = edid;
 	DRM_INFO("Got %s EDID base block and %d extension%s from "
 	    "\"%s\" for connector \"%s\"\n", builtin ? "built-in" :
 	    "external", valid_extensions, valid_extensions == 1 ? "" : "s",
@@ -215,7 +216,10 @@ relfw_out:
 	release_firmware(fw);
 
 out:
-	return err;
+	if (err)
+		return ERR_PTR(err);
+
+	return edid;
 }
 
 int drm_load_edid_firmware(struct drm_connector *connector)
@@ -223,6 +227,7 @@ int drm_load_edid_firmware(struct drm_connector *connector)
 	char *connector_name = drm_get_connector_name(connector);
 	char *edidname = edid_firmware, *last, *colon;
 	int ret;
+	struct edid *edid;
 
 	if (*edidname == '\0')
 		return 0;
@@ -240,13 +245,13 @@ int drm_load_edid_firmware(struct drm_connector *connector)
 	if (*last == '\n')
 		*last = '\0';
 
-	ret = edid_load(connector, edidname, connector_name);
-	if (ret)
+	edid = (struct edid *) edid_load(connector, edidname, connector_name);
+	if (IS_ERR_OR_NULL(edid))
 		return 0;
 
-	drm_mode_connector_update_edid_property(connector,
-	    (struct edid *) connector->display_info.raw_edid);
+	drm_mode_connector_update_edid_property(connector, edid);
+	ret = drm_add_edid_modes(connector, edid);
+	kfree(edid);
 
-	return drm_add_edid_modes(connector, (struct edid *)
-	    connector->display_info.raw_edid);
+	return ret;
 }
diff --git a/drivers/gpu/drm/drm_edid_modes.h b/drivers/gpu/drm/drm_edid_modes.h
index ff98a7eb38d..5dbf7d2557b 100644
--- a/drivers/gpu/drm/drm_edid_modes.h
+++ b/drivers/gpu/drm/drm_edid_modes.h
@@ -24,8 +24,8 @@
  */
 
 #include <linux/kernel.h>
-#include "drmP.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
 
 /*
  * Autogenerated from the DMT spec.
@@ -89,7 +89,7 @@ static const struct drm_display_mode drm_dmt_modes[] = {
 		   976, 1088, 0, 480, 486, 494, 517, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
 	/* 1024x768@43Hz, interlace */
-	{ DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER, 44900, 1024, 1032,
+	{ DRM_MODE("1024x768i", DRM_MODE_TYPE_DRIVER, 44900, 1024, 1032,
 		   1208, 1264, 0, 768, 768, 772, 817, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
@@ -395,7 +395,7 @@ static const struct drm_display_mode edid_est_modes[] = {
 	{ DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER, 65000, 1024, 1048,
 		   1184, 1344, 0,  768, 771, 777, 806, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) }, /* 1024x768@60Hz */
-	{ DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER,44900, 1024, 1032,
+	{ DRM_MODE("1024x768i", DRM_MODE_TYPE_DRIVER,44900, 1024, 1032,
 		   1208, 1264, 0, 768, 768, 776, 817, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_INTERLACE) }, /* 1024x768@43Hz */
 	{ DRM_MODE("832x624", DRM_MODE_TYPE_DRIVER, 57284, 832, 864,
@@ -506,17 +506,17 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   1430, 1650, 0, 720, 725, 730, 750, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
 	/* 5 - 1920x1080i@60Hz */
-	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+	{ DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
 		   2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
 	/* 6 - 1440x480i@60Hz */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
 	/* 7 - 1440x480i@60Hz */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
@@ -531,12 +531,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_DBLCLK) },
 	/* 10 - 2880x480i@60Hz */
-	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+	{ DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
 		   3204, 3432, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
 	/* 11 - 2880x480i@60Hz */
-	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+	{ DRM_MODE("2880x480i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
 		   3204, 3432, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
@@ -573,17 +573,17 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   1760, 1980, 0, 720, 725, 730, 750, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
 	/* 20 - 1920x1080i@50Hz */
-	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+	{ DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
 		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
 	/* 21 - 1440x576i@50Hz */
-	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+	{ DRM_MODE("1440x576i", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
 		   1590, 1728, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
 	/* 22 - 1440x576i@50Hz */
-	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+	{ DRM_MODE("1440x576i", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
 		   1590, 1728, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
@@ -598,12 +598,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_DBLCLK) },
 	/* 25 - 2880x576i@50Hz */
-	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+	{ DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
 		   3180, 3456, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
 	/* 26 - 2880x576i@50Hz */
-	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+	{ DRM_MODE("2880x576i", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
 		   3180, 3456, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
@@ -656,12 +656,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   3184, 3456, 0, 576, 581, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 39 - 1920x1080i@50Hz */
-	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 72000, 1920, 1952,
+	{ DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 72000, 1920, 1952,
 		   2120, 2304, 0, 1080, 1126, 1136, 1250, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
 	/* 40 - 1920x1080i@100Hz */
-	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+	{ DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
 		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
@@ -688,7 +688,7 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_DBLCLK) },
 	/* 46 - 1920x1080i@120Hz */
-	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+	{ DRM_MODE("1920x1080i", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
 		   2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
 			DRM_MODE_FLAG_INTERLACE) },
@@ -705,12 +705,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   798, 858, 0, 480, 489, 495, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 50 - 1440x480i@120Hz */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
 	/* 51 - 1440x480i@120Hz */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
@@ -723,12 +723,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   796, 864, 0, 576, 581, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 54 - 1440x576i@200Hz */
-	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
+	{ DRM_MODE("1440x576i", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
 		   1590, 1728, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
 	/* 55 - 1440x576i@200Hz */
-	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
+	{ DRM_MODE("1440x576i", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
 		   1590, 1728, 0, 576, 580, 586, 625, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
@@ -741,12 +741,12 @@ static const struct drm_display_mode edid_cea_modes[] = {
 		   798, 858, 0, 480, 489, 495, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 58 - 1440x480i@240 */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
 	/* 59 - 1440x480i@240 */
-	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
+	{ DRM_MODE("1440x480i", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
 		   1602, 1716, 0, 480, 488, 494, 525, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
 			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
diff --git a/drivers/gpu/drm/drm_encoder_slave.c b/drivers/gpu/drm/drm_encoder_slave.c
index fb943551060..63e733408b6 100644
--- a/drivers/gpu/drm/drm_encoder_slave.c
+++ b/drivers/gpu/drm/drm_encoder_slave.c
@@ -26,7 +26,7 @@
 
 #include <linux/module.h>
 
-#include "drm_encoder_slave.h"
+#include <drm/drm_encoder_slave.h>
 
 /**
  * drm_i2c_encoder_init - Initialize an I2C slave encoder
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
new file mode 100644
index 00000000000..09e11a5d921
--- /dev/null
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -0,0 +1,406 @@
+/*
+ * drm kms/fb cma (contiguous memory allocator) helper functions
+ *
+ * Copyright (C) 2012 Analog Device Inc.
+ *   Author: Lars-Peter Clausen <lars@metafoo.de>
+ *
+ * Based on udl_fbdev.c
+ *  Copyright (C) 2012 Red Hat
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <linux/module.h>
+
+struct drm_fb_cma {
+	struct drm_framebuffer		fb;
+	struct drm_gem_cma_object	*obj[4];
+};
+
+struct drm_fbdev_cma {
+	struct drm_fb_helper	fb_helper;
+	struct drm_fb_cma	*fb;
+};
+
+static inline struct drm_fbdev_cma *to_fbdev_cma(struct drm_fb_helper *helper)
+{
+	return container_of(helper, struct drm_fbdev_cma, fb_helper);
+}
+
+static inline struct drm_fb_cma *to_fb_cma(struct drm_framebuffer *fb)
+{
+	return container_of(fb, struct drm_fb_cma, fb);
+}
+
+static void drm_fb_cma_destroy(struct drm_framebuffer *fb)
+{
+	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		if (fb_cma->obj[i])
+			drm_gem_object_unreference_unlocked(&fb_cma->obj[i]->base);
+	}
+
+	drm_framebuffer_cleanup(fb);
+	kfree(fb_cma);
+}
+
+static int drm_fb_cma_create_handle(struct drm_framebuffer *fb,
+	struct drm_file *file_priv, unsigned int *handle)
+{
+	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
+
+	return drm_gem_handle_create(file_priv,
+			&fb_cma->obj[0]->base, handle);
+}
+
+static struct drm_framebuffer_funcs drm_fb_cma_funcs = {
+	.destroy	= drm_fb_cma_destroy,
+	.create_handle	= drm_fb_cma_create_handle,
+};
+
+static struct drm_fb_cma *drm_fb_cma_alloc(struct drm_device *dev,
+	struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_cma_object **obj,
+	unsigned int num_planes)
+{
+	struct drm_fb_cma *fb_cma;
+	int ret;
+	int i;
+
+	fb_cma = kzalloc(sizeof(*fb_cma), GFP_KERNEL);
+	if (!fb_cma)
+		return ERR_PTR(-ENOMEM);
+
+	ret = drm_framebuffer_init(dev, &fb_cma->fb, &drm_fb_cma_funcs);
+	if (ret) {
+		dev_err(dev->dev, "Failed to initalize framebuffer: %d\n", ret);
+		kfree(fb_cma);
+		return ERR_PTR(ret);
+	}
+
+	drm_helper_mode_fill_fb_struct(&fb_cma->fb, mode_cmd);
+
+	for (i = 0; i < num_planes; i++)
+		fb_cma->obj[i] = obj[i];
+
+	return fb_cma;
+}
+
+/**
+ * drm_fb_cma_create() - (struct drm_mode_config_funcs *)->fb_create callback function
+ *
+ * If your hardware has special alignment or pitch requirements these should be
+ * checked before calling this function.
+ */
+struct drm_framebuffer *drm_fb_cma_create(struct drm_device *dev,
+	struct drm_file *file_priv, struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	struct drm_fb_cma *fb_cma;
+	struct drm_gem_cma_object *objs[4];
+	struct drm_gem_object *obj;
+	unsigned int hsub;
+	unsigned int vsub;
+	int ret;
+	int i;
+
+	hsub = drm_format_horz_chroma_subsampling(mode_cmd->pixel_format);
+	vsub = drm_format_vert_chroma_subsampling(mode_cmd->pixel_format);
+
+	for (i = 0; i < drm_format_num_planes(mode_cmd->pixel_format); i++) {
+		unsigned int width = mode_cmd->width / (i ? hsub : 1);
+		unsigned int height = mode_cmd->height / (i ? vsub : 1);
+		unsigned int min_size;
+
+		obj = drm_gem_object_lookup(dev, file_priv, mode_cmd->handles[i]);
+		if (!obj) {
+			dev_err(dev->dev, "Failed to lookup GEM object\n");
+			ret = -ENXIO;
+			goto err_gem_object_unreference;
+		}
+
+		min_size = (height - 1) * mode_cmd->pitches[i]
+			 + width * drm_format_plane_cpp(mode_cmd->pixel_format, i)
+			 + mode_cmd->offsets[i];
+
+		if (obj->size < min_size) {
+			drm_gem_object_unreference_unlocked(obj);
+			ret = -EINVAL;
+			goto err_gem_object_unreference;
+		}
+		objs[i] = to_drm_gem_cma_obj(obj);
+	}
+
+	fb_cma = drm_fb_cma_alloc(dev, mode_cmd, objs, i);
+	if (IS_ERR(fb_cma)) {
+		ret = PTR_ERR(fb_cma);
+		goto err_gem_object_unreference;
+	}
+
+	return &fb_cma->fb;
+
+err_gem_object_unreference:
+	for (i--; i >= 0; i--)
+		drm_gem_object_unreference_unlocked(&objs[i]->base);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_create);
+
+/**
+ * drm_fb_cma_get_gem_obj() - Get CMA GEM object for framebuffer
+ * @fb: The framebuffer
+ * @plane: Which plane
+ *
+ * Return the CMA GEM object for given framebuffer.
+ *
+ * This function will usually be called from the CRTC callback functions.
+ */
+struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
+	unsigned int plane)
+{
+	struct drm_fb_cma *fb_cma = to_fb_cma(fb);
+
+	if (plane >= 4)
+		return NULL;
+
+	return fb_cma->obj[plane];
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_obj);
+
+static struct fb_ops drm_fbdev_cma_ops = {
+	.owner		= THIS_MODULE,
+	.fb_fillrect	= sys_fillrect,
+	.fb_copyarea	= sys_copyarea,
+	.fb_imageblit	= sys_imageblit,
+	.fb_check_var	= drm_fb_helper_check_var,
+	.fb_set_par	= drm_fb_helper_set_par,
+	.fb_blank	= drm_fb_helper_blank,
+	.fb_pan_display	= drm_fb_helper_pan_display,
+	.fb_setcmap	= drm_fb_helper_setcmap,
+};
+
+static int drm_fbdev_cma_create(struct drm_fb_helper *helper,
+	struct drm_fb_helper_surface_size *sizes)
+{
+	struct drm_fbdev_cma *fbdev_cma = to_fbdev_cma(helper);
+	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
+	struct drm_device *dev = helper->dev;
+	struct drm_gem_cma_object *obj;
+	struct drm_framebuffer *fb;
+	unsigned int bytes_per_pixel;
+	unsigned long offset;
+	struct fb_info *fbi;
+	size_t size;
+	int ret;
+
+	DRM_DEBUG_KMS("surface width(%d), height(%d) and bpp(%d\n",
+			sizes->surface_width, sizes->surface_height,
+			sizes->surface_bpp);
+
+	bytes_per_pixel = DIV_ROUND_UP(sizes->surface_bpp, 8);
+
+	mode_cmd.width = sizes->surface_width;
+	mode_cmd.height = sizes->surface_height;
+	mode_cmd.pitches[0] = sizes->surface_width * bytes_per_pixel;
+	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+		sizes->surface_depth);
+
+	size = mode_cmd.pitches[0] * mode_cmd.height;
+	obj = drm_gem_cma_create(dev, size);
+	if (!obj)
+		return -ENOMEM;
+
+	fbi = framebuffer_alloc(0, dev->dev);
+	if (!fbi) {
+		dev_err(dev->dev, "Failed to allocate framebuffer info.\n");
+		ret = -ENOMEM;
+		goto err_drm_gem_cma_free_object;
+	}
+
+	fbdev_cma->fb = drm_fb_cma_alloc(dev, &mode_cmd, &obj, 1);
+	if (IS_ERR(fbdev_cma->fb)) {
+		dev_err(dev->dev, "Failed to allocate DRM framebuffer.\n");
+		ret = PTR_ERR(fbdev_cma->fb);
+		goto err_framebuffer_release;
+	}
+
+	fb = &fbdev_cma->fb->fb;
+	helper->fb = fb;
+	helper->fbdev = fbi;
+
+	fbi->par = helper;
+	fbi->flags = FBINFO_FLAG_DEFAULT;
+	fbi->fbops = &drm_fbdev_cma_ops;
+
+	ret = fb_alloc_cmap(&fbi->cmap, 256, 0);
+	if (ret) {
+		dev_err(dev->dev, "Failed to allocate color map.\n");
+		goto err_drm_fb_cma_destroy;
+	}
+
+	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->depth);
+	drm_fb_helper_fill_var(fbi, helper, fb->width, fb->height);
+
+	offset = fbi->var.xoffset * bytes_per_pixel;
+	offset += fbi->var.yoffset * fb->pitches[0];
+
+	dev->mode_config.fb_base = (resource_size_t)obj->paddr;
+	fbi->screen_base = obj->vaddr + offset;
+	fbi->fix.smem_start = (unsigned long)(obj->paddr + offset);
+	fbi->screen_size = size;
+	fbi->fix.smem_len = size;
+
+	return 0;
+
+err_drm_fb_cma_destroy:
+	drm_fb_cma_destroy(fb);
+err_framebuffer_release:
+	framebuffer_release(fbi);
+err_drm_gem_cma_free_object:
+	drm_gem_cma_free_object(&obj->base);
+	return ret;
+}
+
+static int drm_fbdev_cma_probe(struct drm_fb_helper *helper,
+	struct drm_fb_helper_surface_size *sizes)
+{
+	int ret = 0;
+
+	if (!helper->fb) {
+		ret = drm_fbdev_cma_create(helper, sizes);
+		if (ret < 0)
+			return ret;
+		ret = 1;
+	}
+
+	return ret;
+}
+
+static struct drm_fb_helper_funcs drm_fb_cma_helper_funcs = {
+	.fb_probe = drm_fbdev_cma_probe,
+};
+
+/**
+ * drm_fbdev_cma_init() - Allocate and initializes a drm_fbdev_cma struct
+ * @dev: DRM device
+ * @preferred_bpp: Preferred bits per pixel for the device
+ * @num_crtc: Number of CRTCs
+ * @max_conn_count: Maximum number of connectors
+ *
+ * Returns a newly allocated drm_fbdev_cma struct or a ERR_PTR.
+ */
+struct drm_fbdev_cma *drm_fbdev_cma_init(struct drm_device *dev,
+	unsigned int preferred_bpp, unsigned int num_crtc,
+	unsigned int max_conn_count)
+{
+	struct drm_fbdev_cma *fbdev_cma;
+	struct drm_fb_helper *helper;
+	int ret;
+
+	fbdev_cma = kzalloc(sizeof(*fbdev_cma), GFP_KERNEL);
+	if (!fbdev_cma) {
+		dev_err(dev->dev, "Failed to allocate drm fbdev.\n");
+		return ERR_PTR(-ENOMEM);
+	}
+
+	fbdev_cma->fb_helper.funcs = &drm_fb_cma_helper_funcs;
+	helper = &fbdev_cma->fb_helper;
+
+	ret = drm_fb_helper_init(dev, helper, num_crtc, max_conn_count);
+	if (ret < 0) {
+		dev_err(dev->dev, "Failed to initialize drm fb helper.\n");
+		goto err_free;
+	}
+
+	ret = drm_fb_helper_single_add_all_connectors(helper);
+	if (ret < 0) {
+		dev_err(dev->dev, "Failed to add connectors.\n");
+		goto err_drm_fb_helper_fini;
+
+	}
+
+	ret = drm_fb_helper_initial_config(helper, preferred_bpp);
+	if (ret < 0) {
+		dev_err(dev->dev, "Failed to set inital hw configuration.\n");
+		goto err_drm_fb_helper_fini;
+	}
+
+	return fbdev_cma;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(helper);
+err_free:
+	kfree(fbdev_cma);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_fbdev_cma_init);
+
+/**
+ * drm_fbdev_cma_fini() - Free drm_fbdev_cma struct
+ * @fbdev_cma: The drm_fbdev_cma struct
+ */
+void drm_fbdev_cma_fini(struct drm_fbdev_cma *fbdev_cma)
+{
+	if (fbdev_cma->fb_helper.fbdev) {
+		struct fb_info *info;
+		int ret;
+
+		info = fbdev_cma->fb_helper.fbdev;
+		ret = unregister_framebuffer(info);
+		if (ret < 0)
+			DRM_DEBUG_KMS("failed unregister_framebuffer()\n");
+
+		if (info->cmap.len)
+			fb_dealloc_cmap(&info->cmap);
+
+		framebuffer_release(info);
+	}
+
+	if (fbdev_cma->fb)
+		drm_fb_cma_destroy(&fbdev_cma->fb->fb);
+
+	drm_fb_helper_fini(&fbdev_cma->fb_helper);
+	kfree(fbdev_cma);
+}
+EXPORT_SYMBOL_GPL(drm_fbdev_cma_fini);
+
+/**
+ * drm_fbdev_cma_restore_mode() - Restores initial framebuffer mode
+ * @fbdev_cma: The drm_fbdev_cma struct, may be NULL
+ *
+ * This function is usually called from the DRM drivers lastclose callback.
+ */
+void drm_fbdev_cma_restore_mode(struct drm_fbdev_cma *fbdev_cma)
+{
+	if (fbdev_cma)
+		drm_fb_helper_restore_fbdev_mode(&fbdev_cma->fb_helper);
+}
+EXPORT_SYMBOL_GPL(drm_fbdev_cma_restore_mode);
+
+/**
+ * drm_fbdev_cma_hotplug_event() - Poll for hotpulug events
+ * @fbdev_cma: The drm_fbdev_cma struct, may be NULL
+ *
+ * This function is usually called from the DRM drivers output_poll_changed
+ * callback.
+ */
+void drm_fbdev_cma_hotplug_event(struct drm_fbdev_cma *fbdev_cma)
+{
+	if (fbdev_cma)
+		drm_fb_helper_hotplug_event(&fbdev_cma->fb_helper);
+}
+EXPORT_SYMBOL_GPL(drm_fbdev_cma_hotplug_event);
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index f546d1e8af8..4d58d7e6af3 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -32,10 +32,10 @@
 #include <linux/slab.h>
 #include <linux/fb.h>
 #include <linux/module.h>
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_fb_helper.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
 
 MODULE_AUTHOR("David Airlie, Jesse Barnes");
 MODULE_DESCRIPTION("DRM KMS helper");
@@ -236,7 +236,7 @@ bool drm_fb_helper_restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 }
 EXPORT_SYMBOL(drm_fb_helper_restore_fbdev_mode);
 
-bool drm_fb_helper_force_kernel_mode(void)
+static bool drm_fb_helper_force_kernel_mode(void)
 {
 	bool ret, error = false;
 	struct drm_fb_helper *helper;
@@ -330,7 +330,7 @@ static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode)
 		/* Walk the connectors & encoders on this fb turning them on/off */
 		for (j = 0; j < fb_helper->connector_count; j++) {
 			connector = fb_helper->connector_info[j]->connector;
-			drm_helper_connector_dpms(connector, dpms_mode);
+			connector->funcs->dpms(connector, dpms_mode);
 			drm_connector_property_set_value(connector,
 				dev->mode_config.dpms_property, dpms_mode);
 		}
@@ -1230,7 +1230,6 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper)
 	struct drm_device *dev = fb_helper->dev;
 	struct drm_fb_helper_crtc **crtcs;
 	struct drm_display_mode **modes;
-	struct drm_encoder *encoder;
 	struct drm_mode_set *modeset;
 	bool *enabled;
 	int width, height;
@@ -1241,11 +1240,6 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper)
 	width = dev->mode_config.max_width;
 	height = dev->mode_config.max_height;
 
-	/* clean out all the encoder/crtc combos */
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		encoder->crtc = NULL;
-	}
-
 	crtcs = kcalloc(dev->mode_config.num_connector,
 			sizeof(struct drm_fb_helper_crtc *), GFP_KERNEL);
 	modes = kcalloc(dev->mode_config.num_connector,
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 433d2fad1fe..7ef1b673e1b 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -34,7 +34,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include <linux/poll.h>
 #include <linux/slab.h>
 #include <linux/module.h>
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index fbe0842038b..92177d5aede 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -36,7 +36,7 @@
 #include <linux/pagemap.h>
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /** @file drm_gem.c
  *
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
new file mode 100644
index 00000000000..1aa8fee1e86
--- /dev/null
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -0,0 +1,251 @@
+/*
+ * drm gem CMA (contiguous memory allocator) helper functions
+ *
+ * Copyright (C) 2012 Sascha Hauer, Pengutronix
+ *
+ * Based on Samsung Exynos code
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/export.h>
+#include <linux/dma-mapping.h>
+
+#include <drm/drmP.h>
+#include <drm/drm.h>
+#include <drm/drm_gem_cma_helper.h>
+
+static unsigned int get_gem_mmap_offset(struct drm_gem_object *obj)
+{
+	return (unsigned int)obj->map_list.hash.key << PAGE_SHIFT;
+}
+
+static void drm_gem_cma_buf_destroy(struct drm_device *drm,
+		struct drm_gem_cma_object *cma_obj)
+{
+	dma_free_writecombine(drm->dev, cma_obj->base.size, cma_obj->vaddr,
+			cma_obj->paddr);
+}
+
+/*
+ * drm_gem_cma_create - allocate an object with the given size
+ *
+ * returns a struct drm_gem_cma_object* on success or ERR_PTR values
+ * on failure.
+ */
+struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
+		unsigned int size)
+{
+	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	size = round_up(size, PAGE_SIZE);
+
+	cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
+	if (!cma_obj)
+		return ERR_PTR(-ENOMEM);
+
+	cma_obj->vaddr = dma_alloc_writecombine(drm->dev, size,
+			&cma_obj->paddr, GFP_KERNEL | __GFP_NOWARN);
+	if (!cma_obj->vaddr) {
+		dev_err(drm->dev, "failed to allocate buffer with size %d\n", size);
+		ret = -ENOMEM;
+		goto err_dma_alloc;
+	}
+
+	gem_obj = &cma_obj->base;
+
+	ret = drm_gem_object_init(drm, gem_obj, size);
+	if (ret)
+		goto err_obj_init;
+
+	ret = drm_gem_create_mmap_offset(gem_obj);
+	if (ret)
+		goto err_create_mmap_offset;
+
+	return cma_obj;
+
+err_create_mmap_offset:
+	drm_gem_object_release(gem_obj);
+
+err_obj_init:
+	drm_gem_cma_buf_destroy(drm, cma_obj);
+
+err_dma_alloc:
+	kfree(cma_obj);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_create);
+
+/*
+ * drm_gem_cma_create_with_handle - allocate an object with the given
+ * size and create a gem handle on it
+ *
+ * returns a struct drm_gem_cma_object* on success or ERR_PTR values
+ * on failure.
+ */
+static struct drm_gem_cma_object *drm_gem_cma_create_with_handle(
+		struct drm_file *file_priv,
+		struct drm_device *drm, unsigned int size,
+		unsigned int *handle)
+{
+	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	cma_obj = drm_gem_cma_create(drm, size);
+	if (IS_ERR(cma_obj))
+		return cma_obj;
+
+	gem_obj = &cma_obj->base;
+
+	/*
+	 * allocate a id of idr table where the obj is registered
+	 * and handle has the id what user can see.
+	 */
+	ret = drm_gem_handle_create(file_priv, gem_obj, handle);
+	if (ret)
+		goto err_handle_create;
+
+	/* drop reference from allocate - handle holds it now. */
+	drm_gem_object_unreference_unlocked(gem_obj);
+
+	return cma_obj;
+
+err_handle_create:
+	drm_gem_cma_free_object(gem_obj);
+
+	return ERR_PTR(ret);
+}
+
+/*
+ * drm_gem_cma_free_object - (struct drm_driver)->gem_free_object callback
+ * function
+ */
+void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
+{
+	struct drm_gem_cma_object *cma_obj;
+
+	if (gem_obj->map_list.map)
+		drm_gem_free_mmap_offset(gem_obj);
+
+	drm_gem_object_release(gem_obj);
+
+	cma_obj = to_drm_gem_cma_obj(gem_obj);
+
+	drm_gem_cma_buf_destroy(gem_obj->dev, cma_obj);
+
+	kfree(cma_obj);
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_free_object);
+
+/*
+ * drm_gem_cma_dumb_create - (struct drm_driver)->dumb_create callback
+ * function
+ *
+ * This aligns the pitch and size arguments to the minimum required. wrap
+ * this into your own function if you need bigger alignment.
+ */
+int drm_gem_cma_dumb_create(struct drm_file *file_priv,
+		struct drm_device *dev, struct drm_mode_create_dumb *args)
+{
+	struct drm_gem_cma_object *cma_obj;
+	int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+
+	if (args->pitch < min_pitch)
+		args->pitch = min_pitch;
+
+	if (args->size < args->pitch * args->height)
+		args->size = args->pitch * args->height;
+
+	cma_obj = drm_gem_cma_create_with_handle(file_priv, dev,
+			args->size, &args->handle);
+	if (IS_ERR(cma_obj))
+		return PTR_ERR(cma_obj);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);
+
+/*
+ * drm_gem_cma_dumb_map_offset - (struct drm_driver)->dumb_map_offset callback
+ * function
+ */
+int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv,
+		struct drm_device *drm, uint32_t handle, uint64_t *offset)
+{
+	struct drm_gem_object *gem_obj;
+
+	mutex_lock(&drm->struct_mutex);
+
+	gem_obj = drm_gem_object_lookup(drm, file_priv, handle);
+	if (!gem_obj) {
+		dev_err(drm->dev, "failed to lookup gem object\n");
+		mutex_unlock(&drm->struct_mutex);
+		return -EINVAL;
+	}
+
+	*offset = get_gem_mmap_offset(gem_obj);
+
+	drm_gem_object_unreference(gem_obj);
+
+	mutex_unlock(&drm->struct_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_map_offset);
+
+const struct vm_operations_struct drm_gem_cma_vm_ops = {
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);
+
+/*
+ * drm_gem_cma_mmap - (struct file_operation)->mmap callback function
+ */
+int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *gem_obj;
+	struct drm_gem_cma_object *cma_obj;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	gem_obj = vma->vm_private_data;
+	cma_obj = to_drm_gem_cma_obj(gem_obj);
+
+	ret = remap_pfn_range(vma, vma->vm_start, cma_obj->paddr >> PAGE_SHIFT,
+			vma->vm_end - vma->vm_start, vma->vm_page_prot);
+	if (ret)
+		drm_gem_vm_close(vma);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);
+
+/*
+ * drm_gem_cma_dumb_destroy - (struct drm_driver)->dumb_destroy callback function
+ */
+int drm_gem_cma_dumb_destroy(struct drm_file *file_priv,
+		struct drm_device *drm, unsigned int handle)
+{
+	return drm_gem_handle_delete(file_priv, handle);
+}
+EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_destroy);
diff --git a/drivers/gpu/drm/drm_global.c b/drivers/gpu/drm/drm_global.c
index c87dc96444d..f7311162a61 100644
--- a/drivers/gpu/drm/drm_global.c
+++ b/drivers/gpu/drm/drm_global.c
@@ -31,7 +31,7 @@
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include "drm_global.h"
+#include <drm/drm_global.h>
 
 struct drm_global_item {
 	struct mutex mutex;
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c
index 68dc8744b63..c3745c4d46d 100644
--- a/drivers/gpu/drm/drm_hashtab.c
+++ b/drivers/gpu/drm/drm_hashtab.c
@@ -32,8 +32,8 @@
  * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  */
 
-#include "drmP.h"
-#include "drm_hashtab.h"
+#include <drm/drmP.h>
+#include <drm/drm_hashtab.h>
 #include <linux/hash.h>
 #include <linux/slab.h>
 #include <linux/export.h>
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index eb0af393e6e..cdf8b1e7602 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -34,7 +34,7 @@
  */
 
 #include <linux/seq_file.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**
  * Called when "/proc/dri/.../name" is read.
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index 637fcc3766c..2f4c4343dfa 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -31,8 +31,8 @@
 #include <linux/ratelimit.h>
 #include <linux/export.h>
 
-#include "drmP.h"
-#include "drm_core.h"
+#include <drm/drmP.h>
+#include <drm/drm_core.h>
 
 #define DRM_IOCTL_VERSION32		DRM_IOWR(0x00, drm_version32_t)
 #define DRM_IOCTL_GET_UNIQUE32		DRM_IOWR(0x01, drm_unique32_t)
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 39a43834cef..23dd97506f2 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -33,11 +33,11 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_core.h"
+#include <drm/drmP.h>
+#include <drm/drm_core.h>
 
-#include "linux/pci.h"
-#include "linux/export.h"
+#include <linux/pci.h>
+#include <linux/export.h>
 
 /**
  * Get the bus id.
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 03f16f352fe..3a3d0ce891b 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -33,7 +33,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "drm_trace.h"
 
 #include <linux/interrupt.h>	/* For task queue support */
@@ -1236,7 +1236,7 @@ done:
 	return ret;
 }
 
-void drm_handle_vblank_events(struct drm_device *dev, int crtc)
+static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
 {
 	struct drm_pending_vblank_event *e, *t;
 	struct timeval now;
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index 32039553e17..d752c96d609 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -34,7 +34,7 @@
  */
 
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 static int drm_notifier(void *priv);
 
diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c
index c86a0f1a435..126d50ea181 100644
--- a/drivers/gpu/drm/drm_memory.c
+++ b/drivers/gpu/drm/drm_memory.c
@@ -35,7 +35,7 @@
 
 #include <linux/highmem.h>
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #if __OS_HAS_AGP
 static void *agp_remap(unsigned long offset, unsigned long size,
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index 9bb82f7f006..0761a03cdbb 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -41,8 +41,8 @@
  * Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  */
 
-#include "drmP.h"
-#include "drm_mm.h"
+#include <drm/drmP.h>
+#include <drm/drm_mm.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 #include <linux/export.h>
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 28637c181b1..59450f39bf9 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -33,9 +33,8 @@
 #include <linux/list.h>
 #include <linux/list_sort.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
 
 /**
  * drm_mode_debug_printmodeline - debug print a mode
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index 5320364582c..ba33144257e 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -40,7 +40,7 @@
 #include <linux/slab.h>
 #include <linux/dma-mapping.h>
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**********************************************************************/
 /** \name PCI memory */
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 82431dcae37..aaeb6f8d69c 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -26,7 +26,7 @@
  */
 
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**
  * Register.
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index f546ff98a11..7f125738f44 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -28,7 +28,7 @@
 
 #include <linux/export.h>
 #include <linux/dma-buf.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /*
  * DMA-BUF/GEM Object references and lifetime overview:
diff --git a/drivers/gpu/drm/drm_proc.c b/drivers/gpu/drm/drm_proc.c
index da457b18eaa..ff5456b7df7 100644
--- a/drivers/gpu/drm/drm_proc.c
+++ b/drivers/gpu/drm/drm_proc.c
@@ -40,7 +40,7 @@
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /***************************************************
  * Initialization, etc.
diff --git a/drivers/gpu/drm/drm_scatter.c b/drivers/gpu/drm/drm_scatter.c
index 7525e0311e5..d87f60bbc33 100644
--- a/drivers/gpu/drm/drm_scatter.c
+++ b/drivers/gpu/drm/drm_scatter.c
@@ -33,7 +33,7 @@
 
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #define DEBUG_SCATTER 0
 
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index 21bcd4a555d..c236fd27eba 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -34,8 +34,8 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm_core.h"
+#include <drm/drmP.h>
+#include <drm/drm_core.h>
 
 unsigned int drm_debug = 0;	/* 1 to enable debug output */
 EXPORT_SYMBOL(drm_debug);
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 45ac8d6c92b..05cd8fe062a 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -18,9 +18,9 @@
 #include <linux/err.h>
 #include <linux/export.h>
 
-#include "drm_sysfs.h"
-#include "drm_core.h"
-#include "drmP.h"
+#include <drm/drm_sysfs.h>
+#include <drm/drm_core.h>
+#include <drm/drmP.h>
 
 #define to_drm_minor(d) container_of(d, struct drm_minor, kdev)
 #define to_drm_connector(d) container_of(d, struct drm_connector, kdev)
diff --git a/drivers/gpu/drm/drm_trace_points.c b/drivers/gpu/drm/drm_trace_points.c
index 0d0eb90864a..3bbc4deb4db 100644
--- a/drivers/gpu/drm/drm_trace_points.c
+++ b/drivers/gpu/drm/drm_trace_points.c
@@ -1,4 +1,4 @@
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #define CREATE_TRACE_POINTS
 #include "drm_trace.h"
diff --git a/drivers/gpu/drm/drm_usb.c b/drivers/gpu/drm/drm_usb.c
index 37c9a523dd1..3cec3061141 100644
--- a/drivers/gpu/drm/drm_usb.c
+++ b/drivers/gpu/drm/drm_usb.c
@@ -1,4 +1,4 @@
-#include "drmP.h"
+#include <drm/drmP.h>
 #include <linux/usb.h>
 #include <linux/module.h>
 
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index 961ee08927f..23a824e6a22 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -33,7 +33,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include <linux/export.h>
 #if defined(__ia64__)
 #include <linux/efi.h>
@@ -62,7 +62,7 @@ static pgprot_t drm_io_prot(uint32_t map_type, struct vm_area_struct *vma)
 		tmp = pgprot_writecombine(tmp);
 	else
 		tmp = pgprot_noncached(tmp);
-#elif defined(__sparc__) || defined(__arm__)
+#elif defined(__sparc__) || defined(__arm__) || defined(__mips__)
 	tmp = pgprot_noncached(tmp);
 #endif
 	return tmp;
@@ -619,20 +619,11 @@ int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
 		offset = drm_core_get_reg_ofs(dev);
 		vma->vm_flags |= VM_IO;	/* not in core dump */
 		vma->vm_page_prot = drm_io_prot(map->type, vma);
-#if !defined(__arm__)
 		if (io_remap_pfn_range(vma, vma->vm_start,
 				       (map->offset + offset) >> PAGE_SHIFT,
 				       vma->vm_end - vma->vm_start,
 				       vma->vm_page_prot))
 			return -EAGAIN;
-#else
-		if (remap_pfn_range(vma, vma->vm_start,
-					(map->offset + offset) >> PAGE_SHIFT,
-					vma->vm_end - vma->vm_start,
-					vma->vm_page_prot))
-			return -EAGAIN;
-#endif
-
 		DRM_DEBUG("   Type = %d; start = 0x%lx, end = 0x%lx,"
 			  " offset = 0x%llx\n",
 			  map->type,
diff --git a/drivers/gpu/drm/exynos/exynos_ddc.c b/drivers/gpu/drm/exynos/exynos_ddc.c
index 7e1051d07f1..961a1806a24 100644
--- a/drivers/gpu/drm/exynos/exynos_ddc.c
+++ b/drivers/gpu/drm/exynos/exynos_ddc.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include <linux/kernel.h>
 #include <linux/i2c.h>
diff --git a/drivers/gpu/drm/exynos/exynos_drm_buf.c b/drivers/gpu/drm/exynos/exynos_drm_buf.c
index b3cb0a69fbf..118c117b322 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_buf.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_buf.c
@@ -23,9 +23,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "exynos_drm.h"
+#include <drm/drmP.h>
+#include <drm/exynos_drm.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_gem.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_connector.c b/drivers/gpu/drm/exynos/exynos_drm_connector.c
index d9568198c30..c2b1b1441ed 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_connector.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_connector.c
@@ -25,8 +25,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include <drm/exynos_drm.h>
 #include "exynos_drm_drv.h"
@@ -147,9 +147,7 @@ static int exynos_drm_connector_get_modes(struct drm_connector *connector)
 
 		drm_mode_connector_update_edid_property(connector, edid);
 		count = drm_add_edid_modes(connector, edid);
-
-		kfree(connector->display_info.raw_edid);
-		connector->display_info.raw_edid = edid;
+		kfree(edid);
 	} else {
 		struct drm_display_mode *mode = drm_mode_create(connector->dev);
 		struct exynos_drm_panel_info *panel;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c
index 84dd099eae3..19bdf0a194e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_core.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_core.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "exynos_drm_drv.h"
 #include "exynos_drm_encoder.h"
 #include "exynos_drm_connector.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index abb1e2f8227..df1e34f0f09 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -26,8 +26,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_encoder.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
index ae13febe0ea..fae1f2ec886 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c
@@ -23,9 +23,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "exynos_drm.h"
+#include <drm/drmP.h>
+#include <drm/exynos_drm.h>
 #include "exynos_drm_drv.h"
 #include "exynos_drm_gem.h"
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index d0707193745..1de7baafddd 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -25,9 +25,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include <drm/exynos_drm.h>
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index e22704b249d..a4ab98b52dd 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -30,7 +30,6 @@
 #define _EXYNOS_DRM_DRV_H_
 
 #include <linux/module.h>
-#include "drm.h"
 
 #define MAX_CRTC	3
 #define MAX_PLANE	5
diff --git a/drivers/gpu/drm/exynos/exynos_drm_encoder.c b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
index 2c037cd7d2d..39bd8abff3f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_encoder.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
@@ -26,8 +26,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_encoder.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 4ccfe4328fa..53afcc5f094 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -26,10 +26,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_fb.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index d5586cc7516..bd4ff634823 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -26,10 +26,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_fb_helper.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_fb.h"
@@ -266,8 +266,8 @@ static void exynos_drm_fbdev_destroy(struct drm_device *dev,
 	/* release drm framebuffer and real buffer */
 	if (fb_helper->fb && fb_helper->fb->funcs) {
 		fb = fb_helper->fb;
-		if (fb && fb->funcs->destroy)
-			fb->funcs->destroy(fb);
+		if (fb)
+			drm_framebuffer_remove(fb);
 	}
 
 	/* release linux framebuffer */
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index b19cd93e704..58d50e368a5 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -11,7 +11,7 @@
  * option) any later version.
  *
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index 2526e82bea3..bc2a2e9be8e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -18,8 +18,8 @@
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 
-#include "drmP.h"
-#include "exynos_drm.h"
+#include <drm/drmP.h>
+#include <drm/exynos_drm.h>
 #include "exynos_drm_drv.h"
 #include "exynos_drm_gem.h"
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index a38051c95ec..fcdbe46914f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -23,8 +23,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include <linux/shmem_fs.h>
 #include <drm/exynos_drm.h>
diff --git a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c b/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
index 3fdf0b65f47..c3d3a5e4f10 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include <linux/kernel.h>
 #include <linux/wait.h>
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index e1f94b746bd..03b472b4301 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -9,9 +9,9 @@
  *
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
-#include "exynos_drm.h"
+#include <drm/exynos_drm.h>
 #include "exynos_drm_drv.h"
 #include "exynos_drm_encoder.h"
 #include "exynos_drm_fb.h"
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 537027a74fd..8fe431ae537 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -10,7 +10,7 @@
  * option) any later version.
  *
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -18,8 +18,8 @@
 
 #include <drm/exynos_drm.h>
 
-#include "drm_edid.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
@@ -102,7 +102,6 @@ static int vidi_get_edid(struct device *dev, struct drm_connector *connector,
 				u8 *edid, int len)
 {
 	struct vidi_context *ctx = get_vidi_context(dev);
-	struct edid *raw_edid;
 
 	DRM_DEBUG_KMS("%s\n", __FILE__);
 
@@ -115,18 +114,6 @@ static int vidi_get_edid(struct device *dev, struct drm_connector *connector,
 		return -EFAULT;
 	}
 
-	raw_edid = kzalloc(len, GFP_KERNEL);
-	if (!raw_edid) {
-		DRM_DEBUG_KMS("failed to allocate raw_edid.\n");
-		return -ENOMEM;
-	}
-
-	memcpy(raw_edid, ctx->raw_edid, min((1 + ctx->raw_edid->extensions)
-						* EDID_LENGTH, len));
-
-	/* attach the edid data to connector. */
-	connector->display_info.raw_edid = (char *)raw_edid;
-
 	memcpy(edid, ctx->raw_edid, min((1 + ctx->raw_edid->extensions)
 					* EDID_LENGTH, len));
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index a6aea6f3ea1..e1c53956aa2 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -14,9 +14,9 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_edid.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "regs-hdmi.h"
 
diff --git a/drivers/gpu/drm/exynos/exynos_hdmiphy.c b/drivers/gpu/drm/exynos/exynos_hdmiphy.c
index 9fe2995ab9f..0a8162b7de3 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmiphy.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmiphy.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include <linux/kernel.h>
 #include <linux/i2c.h>
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 25b97d5e5fc..e6098f247a5 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -14,7 +14,7 @@
  *
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #include "regs-mixer.h"
 #include "regs-vp.h"
diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile
index abfa2a93f0d..7a2d40a5c1e 100644
--- a/drivers/gpu/drm/gma500/Makefile
+++ b/drivers/gpu/drm/gma500/Makefile
@@ -3,7 +3,7 @@
 #
 ccflags-y += -I$(srctree)/include/drm
 
-gma500_gfx-y += gem_glue.o \
+gma500_gfx-y += \
 	  accel_2d.o \
 	  backlight.o \
 	  framebuffer.o \
@@ -30,7 +30,8 @@ gma500_gfx-$(CONFIG_DRM_GMA3600) +=  cdv_device.o \
 	  cdv_intel_crt.o \
 	  cdv_intel_display.o \
 	  cdv_intel_hdmi.o \
-	  cdv_intel_lvds.o
+	  cdv_intel_lvds.o \
+	  cdv_intel_dp.o
 
 gma500_gfx-$(CONFIG_DRM_GMA600) += oaktrail_device.o \
 	  oaktrail_crtc.o \
diff --git a/drivers/gpu/drm/gma500/backlight.c b/drivers/gpu/drm/gma500/backlight.c
index 20793951fca..143eba3309c 100644
--- a/drivers/gpu/drm/gma500/backlight.c
+++ b/drivers/gpu/drm/gma500/backlight.c
@@ -26,10 +26,55 @@
 #include "intel_bios.h"
 #include "power.h"
 
+static void do_gma_backlight_set(struct drm_device *dev)
+{
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	backlight_update_status(dev_priv->backlight_device);
+#endif	
+}
+
+void gma_backlight_enable(struct drm_device *dev)
+{
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	dev_priv->backlight_enabled = true;
+	if (dev_priv->backlight_device) {
+		dev_priv->backlight_device->props.brightness = dev_priv->backlight_level;
+		do_gma_backlight_set(dev);
+	}
+#endif	
+}
+
+void gma_backlight_disable(struct drm_device *dev)
+{
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	dev_priv->backlight_enabled = false;
+	if (dev_priv->backlight_device) {
+		dev_priv->backlight_device->props.brightness = 0;
+		do_gma_backlight_set(dev);
+	}
+#endif	
+}
+
+void gma_backlight_set(struct drm_device *dev, int v)
+{
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	dev_priv->backlight_level = v;
+	if (dev_priv->backlight_device && dev_priv->backlight_enabled) {
+		dev_priv->backlight_device->props.brightness = v;
+		do_gma_backlight_set(dev);
+	}
+#endif	
+}
+
 int gma_backlight_init(struct drm_device *dev)
 {
 #ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = dev->dev_private;
+	dev_priv->backlight_enabled = true;
 	return dev_priv->ops->backlight_init(dev);
 #else
 	return 0;
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index b7e7b49d8f6..1ceca3d13b6 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -20,7 +20,7 @@
 #include <linux/backlight.h>
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_reg.h"
 #include "psb_intel_reg.h"
@@ -58,10 +58,17 @@ static int cdv_output_init(struct drm_device *dev)
 	cdv_intel_lvds_init(dev, &dev_priv->mode_dev);
 
 	/* These bits indicate HDMI not SDVO on CDV */
-	if (REG_READ(SDVOB) & SDVO_DETECTED)
+	if (REG_READ(SDVOB) & SDVO_DETECTED) {
 		cdv_hdmi_init(dev, &dev_priv->mode_dev, SDVOB);
-	if (REG_READ(SDVOC) & SDVO_DETECTED)
+		if (REG_READ(DP_B) & DP_DETECTED)
+			cdv_intel_dp_init(dev, &dev_priv->mode_dev, DP_B);
+	}
+
+	if (REG_READ(SDVOC) & SDVO_DETECTED) {
 		cdv_hdmi_init(dev, &dev_priv->mode_dev, SDVOC);
+		if (REG_READ(DP_C) & DP_DETECTED)
+			cdv_intel_dp_init(dev, &dev_priv->mode_dev, DP_C);
+	}
 	return 0;
 }
 
@@ -163,6 +170,7 @@ static int cdv_backlight_init(struct drm_device *dev)
 			cdv_get_brightness(cdv_backlight_device);
 	backlight_update_status(cdv_backlight_device);
 	dev_priv->backlight_device = cdv_backlight_device;
+	dev_priv->backlight_enabled = true;
 	return 0;
 }
 
@@ -449,6 +457,7 @@ static void cdv_get_core_freq(struct drm_device *dev)
 	case 6:
 	case 7:
 		dev_priv->core_freq = 266;
+		break;
 	default:
 		dev_priv->core_freq = 0;
 	}
@@ -488,6 +497,65 @@ static void cdv_hotplug_enable(struct drm_device *dev, bool on)
 	}	
 }
 
+static const char *force_audio_names[] = {
+	"off",
+	"auto",
+	"on",
+};
+
+void cdv_intel_attach_force_audio_property(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	struct drm_property *prop;
+	int i;
+
+	prop = dev_priv->force_audio_property;
+	if (prop == NULL) {
+		prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
+					   "audio",
+					   ARRAY_SIZE(force_audio_names));
+		if (prop == NULL)
+			return;
+
+		for (i = 0; i < ARRAY_SIZE(force_audio_names); i++)
+			drm_property_add_enum(prop, i, i-1, force_audio_names[i]);
+
+		dev_priv->force_audio_property = prop;
+	}
+	drm_connector_attach_property(connector, prop, 0);
+}
+
+
+static const char *broadcast_rgb_names[] = {
+	"Full",
+	"Limited 16:235",
+};
+
+void cdv_intel_attach_broadcast_rgb_property(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	struct drm_property *prop;
+	int i;
+
+	prop = dev_priv->broadcast_rgb_property;
+	if (prop == NULL) {
+		prop = drm_property_create(dev, DRM_MODE_PROP_ENUM,
+					   "Broadcast RGB",
+					   ARRAY_SIZE(broadcast_rgb_names));
+		if (prop == NULL)
+			return;
+
+		for (i = 0; i < ARRAY_SIZE(broadcast_rgb_names); i++)
+			drm_property_add_enum(prop, i, i, broadcast_rgb_names[i]);
+
+		dev_priv->broadcast_rgb_property = prop;
+	}
+
+	drm_connector_attach_property(connector, prop, 0);
+}
+
 /* Cedarview */
 static const struct psb_offset cdv_regmap[2] = {
 	{
diff --git a/drivers/gpu/drm/gma500/cdv_intel_display.c b/drivers/gpu/drm/gma500/cdv_intel_display.c
index a68509ba22a..3cfd0931fbf 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_display.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_display.c
@@ -57,15 +57,26 @@ struct cdv_intel_clock_t {
 struct cdv_intel_limit_t {
 	struct cdv_intel_range_t dot, vco, n, m, m1, m2, p, p1;
 	struct cdv_intel_p2_t p2;
+	bool (*find_pll)(const struct cdv_intel_limit_t *, struct drm_crtc *,
+			int, int, struct cdv_intel_clock_t *);
 };
 
+static bool cdv_intel_find_best_PLL(const struct cdv_intel_limit_t *limit,
+	struct drm_crtc *crtc, int target, int refclk,
+	struct cdv_intel_clock_t *best_clock);
+static bool cdv_intel_find_dp_pll(const struct cdv_intel_limit_t *limit, struct drm_crtc *crtc, int target,
+				int refclk,
+				struct cdv_intel_clock_t *best_clock);
+
 #define CDV_LIMIT_SINGLE_LVDS_96	0
 #define CDV_LIMIT_SINGLE_LVDS_100	1
 #define CDV_LIMIT_DAC_HDMI_27		2
 #define CDV_LIMIT_DAC_HDMI_96		3
+#define CDV_LIMIT_DP_27			4
+#define CDV_LIMIT_DP_100		5
 
 static const struct cdv_intel_limit_t cdv_intel_limits[] = {
-	{			/* CDV_SIGNLE_LVDS_96MHz */
+	{			/* CDV_SINGLE_LVDS_96MHz */
 	 .dot = {.min = 20000, .max = 115500},
 	 .vco = {.min = 1800000, .max = 3600000},
 	 .n = {.min = 2, .max = 6},
@@ -76,6 +87,7 @@ static const struct cdv_intel_limit_t cdv_intel_limits[] = {
 	 .p1 = {.min = 2, .max = 10},
 	 .p2 = {.dot_limit = 200000,
 		.p2_slow = 14, .p2_fast = 14},
+		.find_pll = cdv_intel_find_best_PLL,
 	 },
 	{			/* CDV_SINGLE_LVDS_100MHz */
 	 .dot = {.min = 20000, .max = 115500},
@@ -90,6 +102,7 @@ static const struct cdv_intel_limit_t cdv_intel_limits[] = {
 	  * is 80-224Mhz.  Prefer single channel as much as possible.
 	  */
 	 .p2 = {.dot_limit = 200000, .p2_slow = 14, .p2_fast = 14},
+	.find_pll = cdv_intel_find_best_PLL,
 	 },
 	{			/* CDV_DAC_HDMI_27MHz */
 	 .dot = {.min = 20000, .max = 400000},
@@ -101,6 +114,7 @@ static const struct cdv_intel_limit_t cdv_intel_limits[] = {
 	 .p = {.min = 5, .max = 90},
 	 .p1 = {.min = 1, .max = 9},
 	 .p2 = {.dot_limit = 225000, .p2_slow = 10, .p2_fast = 5},
+	.find_pll = cdv_intel_find_best_PLL,
 	 },
 	{			/* CDV_DAC_HDMI_96MHz */
 	 .dot = {.min = 20000, .max = 400000},
@@ -112,7 +126,32 @@ static const struct cdv_intel_limit_t cdv_intel_limits[] = {
 	 .p = {.min = 5, .max = 100},
 	 .p1 = {.min = 1, .max = 10},
 	 .p2 = {.dot_limit = 225000, .p2_slow = 10, .p2_fast = 5},
+	.find_pll = cdv_intel_find_best_PLL,
+	 },
+	{			/* CDV_DP_27MHz */
+	 .dot = {.min = 160000, .max = 272000},
+	 .vco = {.min = 1809000, .max = 3564000},
+	 .n = {.min = 1, .max = 1},
+	 .m = {.min = 67, .max = 132},
+	 .m1 = {.min = 0, .max = 0},
+	 .m2 = {.min = 65, .max = 130},
+	 .p = {.min = 5, .max = 90},
+	 .p1 = {.min = 1, .max = 9},
+	 .p2 = {.dot_limit = 225000, .p2_slow = 10, .p2_fast = 10},
+	 .find_pll = cdv_intel_find_dp_pll,
 	 },
+	{			/* CDV_DP_100MHz */
+	 .dot = {.min = 160000, .max = 272000},
+	 .vco = {.min = 1800000, .max = 3600000},
+	 .n = {.min = 2, .max = 6},
+	 .m = {.min = 60, .max = 164},
+	 .m1 = {.min = 0, .max = 0},
+	 .m2 = {.min = 58, .max = 162},
+	 .p = {.min = 5, .max = 100},
+	 .p1 = {.min = 1, .max = 10},
+	 .p2 = {.dot_limit = 225000, .p2_slow = 10, .p2_fast = 10},
+	 .find_pll = cdv_intel_find_dp_pll,
+	 }	
 };
 
 #define _wait_for(COND, MS, W) ({ \
@@ -132,7 +171,7 @@ static const struct cdv_intel_limit_t cdv_intel_limits[] = {
 #define wait_for(COND, MS) _wait_for(COND, MS, 1)
 
 
-static int cdv_sb_read(struct drm_device *dev, u32 reg, u32 *val)
+int cdv_sb_read(struct drm_device *dev, u32 reg, u32 *val)
 {
 	int ret;
 
@@ -159,7 +198,7 @@ static int cdv_sb_read(struct drm_device *dev, u32 reg, u32 *val)
 	return 0;
 }
 
-static int cdv_sb_write(struct drm_device *dev, u32 reg, u32 val)
+int cdv_sb_write(struct drm_device *dev, u32 reg, u32 val)
 {
 	int ret;
 	static bool dpio_debug = true;
@@ -201,7 +240,7 @@ static int cdv_sb_write(struct drm_device *dev, u32 reg, u32 val)
 /* Reset the DPIO configuration register.  The BIOS does this at every
  * mode set.
  */
-static void cdv_sb_reset(struct drm_device *dev)
+void cdv_sb_reset(struct drm_device *dev)
 {
 
 	REG_WRITE(DPIO_CFG, 0);
@@ -216,7 +255,7 @@ static void cdv_sb_reset(struct drm_device *dev)
  */
 static int
 cdv_dpll_set_clock_cdv(struct drm_device *dev, struct drm_crtc *crtc,
-			       struct cdv_intel_clock_t *clock, bool is_lvds)
+			       struct cdv_intel_clock_t *clock, bool is_lvds, u32 ddi_select)
 {
 	struct psb_intel_crtc *psb_crtc = to_psb_intel_crtc(crtc);
 	int pipe = psb_crtc->pipe;
@@ -259,7 +298,7 @@ cdv_dpll_set_clock_cdv(struct drm_device *dev, struct drm_crtc *crtc,
 	ref_value &= ~(REF_CLK_MASK);
 
 	/* use DPLL_A for pipeB on CRT/HDMI */
-	if (pipe == 1 && !is_lvds) {
+	if (pipe == 1 && !is_lvds && !(ddi_select & DP_MASK)) {
 		DRM_DEBUG_KMS("use DPLLA for pipe B\n");
 		ref_value |= REF_CLK_DPLLA;
 	} else {
@@ -336,30 +375,33 @@ cdv_dpll_set_clock_cdv(struct drm_device *dev, struct drm_crtc *crtc,
 	if (ret)
 		return ret;
 
-	lane_reg = PSB_LANE0;
-	cdv_sb_read(dev, lane_reg, &lane_value);
-	lane_value &= ~(LANE_PLL_MASK);
-	lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
-	cdv_sb_write(dev, lane_reg, lane_value);
-
-	lane_reg = PSB_LANE1;
-	cdv_sb_read(dev, lane_reg, &lane_value);
-	lane_value &= ~(LANE_PLL_MASK);
-	lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
-	cdv_sb_write(dev, lane_reg, lane_value);
-
-	lane_reg = PSB_LANE2;
-	cdv_sb_read(dev, lane_reg, &lane_value);
-	lane_value &= ~(LANE_PLL_MASK);
-	lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
-	cdv_sb_write(dev, lane_reg, lane_value);
-
-	lane_reg = PSB_LANE3;
-	cdv_sb_read(dev, lane_reg, &lane_value);
-	lane_value &= ~(LANE_PLL_MASK);
-	lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
-	cdv_sb_write(dev, lane_reg, lane_value);
-
+	if (ddi_select) {
+		if ((ddi_select & DDI_MASK) == DDI0_SELECT) {
+			lane_reg = PSB_LANE0;
+			cdv_sb_read(dev, lane_reg, &lane_value);
+			lane_value &= ~(LANE_PLL_MASK);
+			lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
+			cdv_sb_write(dev, lane_reg, lane_value);
+
+			lane_reg = PSB_LANE1;
+			cdv_sb_read(dev, lane_reg, &lane_value);
+			lane_value &= ~(LANE_PLL_MASK);
+			lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
+			cdv_sb_write(dev, lane_reg, lane_value);
+		} else {
+			lane_reg = PSB_LANE2;
+			cdv_sb_read(dev, lane_reg, &lane_value);
+			lane_value &= ~(LANE_PLL_MASK);
+			lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
+			cdv_sb_write(dev, lane_reg, lane_value);
+
+			lane_reg = PSB_LANE3;
+			cdv_sb_read(dev, lane_reg, &lane_value);
+			lane_value &= ~(LANE_PLL_MASK);
+			lane_value |= LANE_PLL_ENABLE | LANE_PLL_PIPE(pipe);
+			cdv_sb_write(dev, lane_reg, lane_value);
+		}
+	}
 	return 0;
 }
 
@@ -396,6 +438,12 @@ static const struct cdv_intel_limit_t *cdv_intel_limit(struct drm_crtc *crtc,
 			limit = &cdv_intel_limits[CDV_LIMIT_SINGLE_LVDS_96];
 		else
 			limit = &cdv_intel_limits[CDV_LIMIT_SINGLE_LVDS_100];
+	} else if (psb_intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
+			psb_intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP)) {
+		if (refclk == 27000)
+			limit = &cdv_intel_limits[CDV_LIMIT_DP_27];
+		else
+			limit = &cdv_intel_limits[CDV_LIMIT_DP_100];
 	} else {
 		if (refclk == 27000)
 			limit = &cdv_intel_limits[CDV_LIMIT_DAC_HDMI_27];
@@ -438,13 +486,12 @@ static bool cdv_intel_PLL_is_valid(struct drm_crtc *crtc,
 	return true;
 }
 
-static bool cdv_intel_find_best_PLL(struct drm_crtc *crtc, int target,
-				int refclk,
-				struct cdv_intel_clock_t *best_clock)
+static bool cdv_intel_find_best_PLL(const struct cdv_intel_limit_t *limit,
+	struct drm_crtc *crtc, int target, int refclk,
+	struct cdv_intel_clock_t *best_clock)
 {
 	struct drm_device *dev = crtc->dev;
 	struct cdv_intel_clock_t clock;
-	const struct cdv_intel_limit_t *limit = cdv_intel_limit(crtc, refclk);
 	int err = target;
 
 
@@ -498,6 +545,49 @@ static bool cdv_intel_find_best_PLL(struct drm_crtc *crtc, int target,
 	return err != target;
 }
 
+static bool cdv_intel_find_dp_pll(const struct cdv_intel_limit_t *limit, struct drm_crtc *crtc, int target,
+				int refclk,
+				struct cdv_intel_clock_t *best_clock)
+{
+	struct cdv_intel_clock_t clock;
+	if (refclk == 27000) {
+		if (target < 200000) {
+			clock.p1 = 2;
+			clock.p2 = 10;
+			clock.n = 1;
+			clock.m1 = 0;
+			clock.m2 = 118;
+		} else {
+			clock.p1 = 1;
+			clock.p2 = 10;
+			clock.n = 1;
+			clock.m1 = 0;
+			clock.m2 = 98;
+		}
+	} else if (refclk == 100000) {
+		if (target < 200000) {
+			clock.p1 = 2;
+			clock.p2 = 10;
+			clock.n = 5;
+			clock.m1 = 0;
+			clock.m2 = 160;
+		} else {
+			clock.p1 = 1;
+			clock.p2 = 10;
+			clock.n = 5;
+			clock.m1 = 0;
+			clock.m2 = 133;
+		}
+	} else
+		return false;
+	clock.m = clock.m2 + 2;
+	clock.p = clock.p1 * clock.p2;
+	clock.vco = (refclk * clock.m) / clock.n;
+	clock.dot = clock.vco / clock.p;
+	memcpy(best_clock, &clock, sizeof(struct cdv_intel_clock_t));
+	return true;
+}
+
 static int cdv_intel_pipe_set_base(struct drm_crtc *crtc,
 			    int x, int y, struct drm_framebuffer *old_fb)
 {
@@ -791,7 +881,7 @@ static void cdv_intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 	case DRM_MODE_DPMS_STANDBY:
 	case DRM_MODE_DPMS_SUSPEND:
 		if (psb_intel_crtc->active)
-			return;
+			break;
 
 		psb_intel_crtc->active = true;
 
@@ -835,17 +925,15 @@ static void cdv_intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 		REG_WRITE(map->status, temp);
 		REG_READ(map->status);
 
-		cdv_intel_update_watermark(dev, crtc);
 		cdv_intel_crtc_load_lut(crtc);
 
 		/* Give the overlay scaler a chance to enable
 		 * if it's on this pipe */
 		/* psb_intel_crtc_dpms_video(crtc, true); TODO */
-		psb_intel_crtc->crtc_enable = true;
 		break;
 	case DRM_MODE_DPMS_OFF:
 		if (!psb_intel_crtc->active)
-			return;
+			break;
 
 		psb_intel_crtc->active = false;
 
@@ -892,10 +980,9 @@ static void cdv_intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 
 		/* Wait for the clocks to turn off. */
 		udelay(150);
-		cdv_intel_update_watermark(dev, crtc);
-		psb_intel_crtc->crtc_enable = false;
 		break;
 	}
+	cdv_intel_update_watermark(dev, crtc);
 	/*Set FIFO Watermarks*/
 	REG_WRITE(DSPARB, 0x3F3E);
 }
@@ -952,9 +1039,12 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	u32 dpll = 0, dspcntr, pipeconf;
 	bool ok;
 	bool is_crt = false, is_lvds = false, is_tv = false;
-	bool is_hdmi = false;
+	bool is_hdmi = false, is_dp = false;
 	struct drm_mode_config *mode_config = &dev->mode_config;
 	struct drm_connector *connector;
+	const struct cdv_intel_limit_t *limit;
+	u32 ddi_select = 0;
+	bool is_edp = false;
 
 	list_for_each_entry(connector, &mode_config->connector_list, head) {
 		struct psb_intel_encoder *psb_intel_encoder =
@@ -964,6 +1054,7 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 		    || connector->encoder->crtc != crtc)
 			continue;
 
+		ddi_select = psb_intel_encoder->ddi_select;
 		switch (psb_intel_encoder->type) {
 		case INTEL_OUTPUT_LVDS:
 			is_lvds = true;
@@ -977,6 +1068,15 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 		case INTEL_OUTPUT_HDMI:
 			is_hdmi = true;
 			break;
+		case INTEL_OUTPUT_DISPLAYPORT:
+			is_dp = true;
+			break;
+		case INTEL_OUTPUT_EDP:
+			is_edp = true;
+			break;
+		default:
+			DRM_ERROR("invalid output type.\n");
+			return 0;
 		}
 	}
 
@@ -986,6 +1086,20 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	else
 		/* high-end sku, 27/100 mhz */
 		refclk = 27000;
+	if (is_dp || is_edp) {
+		/*
+		 * Based on the spec the low-end SKU has only CRT/LVDS. So it is
+		 * unnecessary to consider it for DP/eDP.
+		 * On the high-end SKU, it will use the 27/100M reference clk
+		 * for DP/eDP. When using SSC clock, the ref clk is 100MHz.Otherwise
+		 * it will be 27MHz. From the VBIOS code it seems that the pipe A choose
+		 * 27MHz for DP/eDP while the Pipe B chooses the 100MHz.
+		 */ 
+		if (pipe == 0)
+			refclk = 27000;
+		else
+			refclk = 100000;
+	}
 
 	if (is_lvds && dev_priv->lvds_use_ssc) {
 		refclk = dev_priv->lvds_ssc_freq * 1000;
@@ -993,8 +1107,10 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	}
 
 	drm_mode_debug_printmodeline(adjusted_mode);
+	
+	limit = cdv_intel_limit(crtc, refclk);
 
-	ok = cdv_intel_find_best_PLL(crtc, adjusted_mode->clock, refclk,
+	ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk,
 				 &clock);
 	if (!ok) {
 		dev_err(dev->dev, "Couldn't find PLL settings for mode!\n");
@@ -1009,6 +1125,15 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	}
 /*		dpll |= PLL_REF_INPUT_DREFCLK; */
 
+	if (is_dp || is_edp) {
+		cdv_intel_dp_set_m_n(crtc, mode, adjusted_mode);
+	} else {
+		REG_WRITE(PIPE_GMCH_DATA_M(pipe), 0);
+		REG_WRITE(PIPE_GMCH_DATA_N(pipe), 0);
+		REG_WRITE(PIPE_DP_LINK_M(pipe), 0);
+		REG_WRITE(PIPE_DP_LINK_N(pipe), 0);
+	}
+
 	dpll |= DPLL_SYNCLOCK_ENABLE;
 /*	if (is_lvds)
 		dpll |= DPLLB_MODE_LVDS;
@@ -1019,6 +1144,31 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	/* setup pipeconf */
 	pipeconf = REG_READ(map->conf);
 
+	pipeconf &= ~(PIPE_BPC_MASK);
+	if (is_edp) {
+		switch (dev_priv->edp.bpp) {
+		case 24:
+			pipeconf |= PIPE_8BPC;
+			break;
+		case 18:
+			pipeconf |= PIPE_6BPC;
+			break;
+		case 30:
+			pipeconf |= PIPE_10BPC;
+			break;
+		default:
+			pipeconf |= PIPE_8BPC;
+			break;
+		}
+	} else if (is_lvds) {
+		/* the BPC will be 6 if it is 18-bit LVDS panel */
+		if ((REG_READ(LVDS) & LVDS_A3_POWER_MASK) == LVDS_A3_POWER_UP)
+			pipeconf |= PIPE_8BPC;
+		else
+			pipeconf |= PIPE_6BPC;
+	} else
+		pipeconf |= PIPE_8BPC;
+			
 	/* Set up the display plane register */
 	dspcntr = DISPPLANE_GAMMA_ENABLE;
 
@@ -1033,7 +1183,7 @@ static int cdv_intel_crtc_mode_set(struct drm_crtc *crtc,
 	REG_WRITE(map->dpll, dpll | DPLL_VGA_MODE_DIS | DPLL_SYNCLOCK_ENABLE);
 	REG_READ(map->dpll);
 
-	cdv_dpll_set_clock_cdv(dev, crtc, &clock, is_lvds);
+	cdv_dpll_set_clock_cdv(dev, crtc, &clock, is_lvds, ddi_select);
 
 	udelay(150);
 
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
new file mode 100644
index 00000000000..e3a3978cf32
--- /dev/null
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -0,0 +1,1950 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Keith Packard <keithp@keithp.com>
+ *
+ */
+
+#include <linux/i2c.h>
+#include <linux/slab.h>
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include "psb_drv.h"
+#include "psb_intel_drv.h"
+#include "psb_intel_reg.h"
+#include <drm/drm_dp_helper.h>
+
+#define _wait_for(COND, MS, W) ({ \
+        unsigned long timeout__ = jiffies + msecs_to_jiffies(MS);       \
+        int ret__ = 0;                                                  \
+        while (! (COND)) {                                              \
+                if (time_after(jiffies, timeout__)) {                   \
+                        ret__ = -ETIMEDOUT;                             \
+                        break;                                          \
+                }                                                       \
+                if (W && !in_dbg_master()) msleep(W);                   \
+        }                                                               \
+        ret__;                                                          \
+})      
+
+#define wait_for(COND, MS) _wait_for(COND, MS, 1)
+
+#define DP_LINK_STATUS_SIZE	6
+#define DP_LINK_CHECK_TIMEOUT	(10 * 1000)
+
+#define DP_LINK_CONFIGURATION_SIZE	9
+
+#define CDV_FAST_LINK_TRAIN	1
+
+struct cdv_intel_dp {
+	uint32_t output_reg;
+	uint32_t DP;
+	uint8_t  link_configuration[DP_LINK_CONFIGURATION_SIZE];
+	bool has_audio;
+	int force_audio;
+	uint32_t color_range;
+	uint8_t link_bw;
+	uint8_t lane_count;
+	uint8_t dpcd[4];
+	struct psb_intel_encoder *encoder;
+	struct i2c_adapter adapter;
+	struct i2c_algo_dp_aux_data algo;
+	uint8_t	train_set[4];
+	uint8_t link_status[DP_LINK_STATUS_SIZE];
+	int panel_power_up_delay;
+	int panel_power_down_delay;
+	int panel_power_cycle_delay;
+	int backlight_on_delay;
+	int backlight_off_delay;
+	struct drm_display_mode *panel_fixed_mode;  /* for eDP */
+	bool panel_on;
+};
+
+struct ddi_regoff {
+	uint32_t	PreEmph1;
+	uint32_t	PreEmph2;
+	uint32_t	VSwing1;
+	uint32_t	VSwing2;
+	uint32_t	VSwing3;
+	uint32_t	VSwing4;
+	uint32_t	VSwing5;
+};
+
+static struct ddi_regoff ddi_DP_train_table[] = {
+	{.PreEmph1 = 0x812c, .PreEmph2 = 0x8124, .VSwing1 = 0x8154,
+	.VSwing2 = 0x8148, .VSwing3 = 0x814C, .VSwing4 = 0x8150,
+	.VSwing5 = 0x8158,},
+	{.PreEmph1 = 0x822c, .PreEmph2 = 0x8224, .VSwing1 = 0x8254,
+	.VSwing2 = 0x8248, .VSwing3 = 0x824C, .VSwing4 = 0x8250,
+	.VSwing5 = 0x8258,},
+};
+
+static uint32_t dp_vswing_premph_table[] = {
+        0x55338954,	0x4000,
+        0x554d8954,	0x2000,
+        0x55668954,	0,
+        0x559ac0d4,	0x6000,
+};
+/**
+ * is_edp - is the given port attached to an eDP panel (either CPU or PCH)
+ * @intel_dp: DP struct
+ *
+ * If a CPU or PCH DP output is attached to an eDP panel, this function
+ * will return true, and false otherwise.
+ */
+static bool is_edp(struct psb_intel_encoder *encoder)
+{
+	return encoder->type == INTEL_OUTPUT_EDP;
+}
+
+
+static void cdv_intel_dp_start_link_train(struct psb_intel_encoder *encoder);
+static void cdv_intel_dp_complete_link_train(struct psb_intel_encoder *encoder);
+static void cdv_intel_dp_link_down(struct psb_intel_encoder *encoder);
+
+static int
+cdv_intel_dp_max_lane_count(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int max_lane_count = 4;
+
+	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) {
+		max_lane_count = intel_dp->dpcd[DP_MAX_LANE_COUNT] & 0x1f;
+		switch (max_lane_count) {
+		case 1: case 2: case 4:
+			break;
+		default:
+			max_lane_count = 4;
+		}
+	}
+	return max_lane_count;
+}
+
+static int
+cdv_intel_dp_max_link_bw(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE];
+
+	switch (max_link_bw) {
+	case DP_LINK_BW_1_62:
+	case DP_LINK_BW_2_7:
+		break;
+	default:
+		max_link_bw = DP_LINK_BW_1_62;
+		break;
+	}
+	return max_link_bw;
+}
+
+static int
+cdv_intel_dp_link_clock(uint8_t link_bw)
+{
+	if (link_bw == DP_LINK_BW_2_7)
+		return 270000;
+	else
+		return 162000;
+}
+
+static int
+cdv_intel_dp_link_required(int pixel_clock, int bpp)
+{
+	return (pixel_clock * bpp + 7) / 8;
+}
+
+static int
+cdv_intel_dp_max_data_rate(int max_link_clock, int max_lanes)
+{
+	return (max_link_clock * max_lanes * 19) / 20;
+}
+
+static void cdv_intel_edp_panel_vdd_on(struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	u32 pp;
+
+	if (intel_dp->panel_on) {
+		DRM_DEBUG_KMS("Skip VDD on because of panel on\n");
+		return;
+	}	
+	DRM_DEBUG_KMS("\n");
+
+	pp = REG_READ(PP_CONTROL);
+
+	pp |= EDP_FORCE_VDD;
+	REG_WRITE(PP_CONTROL, pp);
+	REG_READ(PP_CONTROL);
+	msleep(intel_dp->panel_power_up_delay);
+}
+
+static void cdv_intel_edp_panel_vdd_off(struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	u32 pp;
+
+	DRM_DEBUG_KMS("\n");
+	pp = REG_READ(PP_CONTROL);
+
+	pp &= ~EDP_FORCE_VDD;
+	REG_WRITE(PP_CONTROL, pp);
+	REG_READ(PP_CONTROL);
+
+}
+
+/* Returns true if the panel was already on when called */
+static bool cdv_intel_edp_panel_on(struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	u32 pp, idle_on_mask = PP_ON | PP_SEQUENCE_NONE;
+
+	if (intel_dp->panel_on)
+		return true;
+
+	DRM_DEBUG_KMS("\n");
+	pp = REG_READ(PP_CONTROL);
+	pp &= ~PANEL_UNLOCK_MASK;
+
+	pp |= (PANEL_UNLOCK_REGS | POWER_TARGET_ON);
+	REG_WRITE(PP_CONTROL, pp);
+	REG_READ(PP_CONTROL);
+
+	if (wait_for(((REG_READ(PP_STATUS) & idle_on_mask) == idle_on_mask), 1000)) {
+		DRM_DEBUG_KMS("Error in Powering up eDP panel, status %x\n", REG_READ(PP_STATUS));
+		intel_dp->panel_on = false;
+	} else
+		intel_dp->panel_on = true;	
+	msleep(intel_dp->panel_power_up_delay);
+
+	return false;
+}
+
+static void cdv_intel_edp_panel_off (struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	u32 pp, idle_off_mask = PP_ON ;
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+
+	DRM_DEBUG_KMS("\n");
+
+	pp = REG_READ(PP_CONTROL);
+
+	if ((pp & POWER_TARGET_ON) == 0) 
+		return;
+
+	intel_dp->panel_on = false;
+	pp &= ~PANEL_UNLOCK_MASK;
+	/* ILK workaround: disable reset around power sequence */
+
+	pp &= ~POWER_TARGET_ON;
+	pp &= ~EDP_FORCE_VDD;
+	pp &= ~EDP_BLC_ENABLE;
+	REG_WRITE(PP_CONTROL, pp);
+	REG_READ(PP_CONTROL);
+	DRM_DEBUG_KMS("PP_STATUS %x\n", REG_READ(PP_STATUS));
+
+	if (wait_for((REG_READ(PP_STATUS) & idle_off_mask) == 0, 1000)) {
+		DRM_DEBUG_KMS("Error in turning off Panel\n");	
+	}
+
+	msleep(intel_dp->panel_power_cycle_delay);
+	DRM_DEBUG_KMS("Over\n");
+}
+
+static void cdv_intel_edp_backlight_on (struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	u32 pp;
+
+	DRM_DEBUG_KMS("\n");
+	/*
+	 * If we enable the backlight right away following a panel power
+	 * on, we may see slight flicker as the panel syncs with the eDP
+	 * link.  So delay a bit to make sure the image is solid before
+	 * allowing it to appear.
+	 */
+	msleep(300);
+	pp = REG_READ(PP_CONTROL);
+
+	pp |= EDP_BLC_ENABLE;
+	REG_WRITE(PP_CONTROL, pp);
+	gma_backlight_enable(dev);
+}
+
+static void cdv_intel_edp_backlight_off (struct psb_intel_encoder *intel_encoder)
+{
+	struct drm_device *dev = intel_encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	u32 pp;
+
+	DRM_DEBUG_KMS("\n");
+	gma_backlight_disable(dev);
+	msleep(10);
+	pp = REG_READ(PP_CONTROL);
+
+	pp &= ~EDP_BLC_ENABLE;
+	REG_WRITE(PP_CONTROL, pp);
+	msleep(intel_dp->backlight_off_delay);
+}
+
+static int
+cdv_intel_dp_mode_valid(struct drm_connector *connector,
+		    struct drm_display_mode *mode)
+{
+	struct psb_intel_encoder *encoder = psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int max_link_clock = cdv_intel_dp_link_clock(cdv_intel_dp_max_link_bw(encoder));
+	int max_lanes = cdv_intel_dp_max_lane_count(encoder);
+	struct drm_psb_private *dev_priv = connector->dev->dev_private;
+
+	if (is_edp(encoder) && intel_dp->panel_fixed_mode) {
+		if (mode->hdisplay > intel_dp->panel_fixed_mode->hdisplay)
+			return MODE_PANEL;
+		if (mode->vdisplay > intel_dp->panel_fixed_mode->vdisplay)
+			return MODE_PANEL;
+	}
+
+	/* only refuse the mode on non eDP since we have seen some weird eDP panels
+	   which are outside spec tolerances but somehow work by magic */
+	if (!is_edp(encoder) &&
+	    (cdv_intel_dp_link_required(mode->clock, dev_priv->edp.bpp)
+	     > cdv_intel_dp_max_data_rate(max_link_clock, max_lanes)))
+		return MODE_CLOCK_HIGH;
+
+	if (is_edp(encoder)) {
+	    if (cdv_intel_dp_link_required(mode->clock, 24)
+	     	> cdv_intel_dp_max_data_rate(max_link_clock, max_lanes))
+		return MODE_CLOCK_HIGH;
+		
+	}
+	if (mode->clock < 10000)
+		return MODE_CLOCK_LOW;
+
+	return MODE_OK;
+}
+
+static uint32_t
+pack_aux(uint8_t *src, int src_bytes)
+{
+	int	i;
+	uint32_t v = 0;
+
+	if (src_bytes > 4)
+		src_bytes = 4;
+	for (i = 0; i < src_bytes; i++)
+		v |= ((uint32_t) src[i]) << ((3-i) * 8);
+	return v;
+}
+
+static void
+unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes)
+{
+	int i;
+	if (dst_bytes > 4)
+		dst_bytes = 4;
+	for (i = 0; i < dst_bytes; i++)
+		dst[i] = src >> ((3-i) * 8);
+}
+
+static int
+cdv_intel_dp_aux_ch(struct psb_intel_encoder *encoder,
+		uint8_t *send, int send_bytes,
+		uint8_t *recv, int recv_size)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	uint32_t output_reg = intel_dp->output_reg;
+	struct drm_device *dev = encoder->base.dev;
+	uint32_t ch_ctl = output_reg + 0x10;
+	uint32_t ch_data = ch_ctl + 4;
+	int i;
+	int recv_bytes;
+	uint32_t status;
+	uint32_t aux_clock_divider;
+	int try, precharge;
+
+	/* The clock divider is based off the hrawclk,
+	 * and would like to run at 2MHz. So, take the
+	 * hrawclk value and divide by 2 and use that
+	 * On CDV platform it uses 200MHz as hrawclk.
+	 *
+	 */
+	aux_clock_divider = 200 / 2;
+
+	precharge = 4;
+	if (is_edp(encoder))
+		precharge = 10;
+
+	if (REG_READ(ch_ctl) & DP_AUX_CH_CTL_SEND_BUSY) {
+		DRM_ERROR("dp_aux_ch not started status 0x%08x\n",
+			  REG_READ(ch_ctl));
+		return -EBUSY;
+	}
+
+	/* Must try at least 3 times according to DP spec */
+	for (try = 0; try < 5; try++) {
+		/* Load the send data into the aux channel data registers */
+		for (i = 0; i < send_bytes; i += 4)
+			REG_WRITE(ch_data + i,
+				   pack_aux(send + i, send_bytes - i));
+	
+		/* Send the command and wait for it to complete */
+		REG_WRITE(ch_ctl,
+			   DP_AUX_CH_CTL_SEND_BUSY |
+			   DP_AUX_CH_CTL_TIME_OUT_400us |
+			   (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
+			   (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
+			   (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT) |
+			   DP_AUX_CH_CTL_DONE |
+			   DP_AUX_CH_CTL_TIME_OUT_ERROR |
+			   DP_AUX_CH_CTL_RECEIVE_ERROR);
+		for (;;) {
+			status = REG_READ(ch_ctl);
+			if ((status & DP_AUX_CH_CTL_SEND_BUSY) == 0)
+				break;
+			udelay(100);
+		}
+	
+		/* Clear done status and any errors */
+		REG_WRITE(ch_ctl,
+			   status |
+			   DP_AUX_CH_CTL_DONE |
+			   DP_AUX_CH_CTL_TIME_OUT_ERROR |
+			   DP_AUX_CH_CTL_RECEIVE_ERROR);
+		if (status & DP_AUX_CH_CTL_DONE)
+			break;
+	}
+
+	if ((status & DP_AUX_CH_CTL_DONE) == 0) {
+		DRM_ERROR("dp_aux_ch not done status 0x%08x\n", status);
+		return -EBUSY;
+	}
+
+	/* Check for timeout or receive error.
+	 * Timeouts occur when the sink is not connected
+	 */
+	if (status & DP_AUX_CH_CTL_RECEIVE_ERROR) {
+		DRM_ERROR("dp_aux_ch receive error status 0x%08x\n", status);
+		return -EIO;
+	}
+
+	/* Timeouts occur when the device isn't connected, so they're
+	 * "normal" -- don't fill the kernel log with these */
+	if (status & DP_AUX_CH_CTL_TIME_OUT_ERROR) {
+		DRM_DEBUG_KMS("dp_aux_ch timeout status 0x%08x\n", status);
+		return -ETIMEDOUT;
+	}
+
+	/* Unload any bytes sent back from the other side */
+	recv_bytes = ((status & DP_AUX_CH_CTL_MESSAGE_SIZE_MASK) >>
+		      DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT);
+	if (recv_bytes > recv_size)
+		recv_bytes = recv_size;
+	
+	for (i = 0; i < recv_bytes; i += 4)
+		unpack_aux(REG_READ(ch_data + i),
+			   recv + i, recv_bytes - i);
+
+	return recv_bytes;
+}
+
+/* Write data to the aux channel in native mode */
+static int
+cdv_intel_dp_aux_native_write(struct psb_intel_encoder *encoder,
+			  uint16_t address, uint8_t *send, int send_bytes)
+{
+	int ret;
+	uint8_t	msg[20];
+	int msg_bytes;
+	uint8_t	ack;
+
+	if (send_bytes > 16)
+		return -1;
+	msg[0] = AUX_NATIVE_WRITE << 4;
+	msg[1] = address >> 8;
+	msg[2] = address & 0xff;
+	msg[3] = send_bytes - 1;
+	memcpy(&msg[4], send, send_bytes);
+	msg_bytes = send_bytes + 4;
+	for (;;) {
+		ret = cdv_intel_dp_aux_ch(encoder, msg, msg_bytes, &ack, 1);
+		if (ret < 0)
+			return ret;
+		if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK)
+			break;
+		else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
+			udelay(100);
+		else
+			return -EIO;
+	}
+	return send_bytes;
+}
+
+/* Write a single byte to the aux channel in native mode */
+static int
+cdv_intel_dp_aux_native_write_1(struct psb_intel_encoder *encoder,
+			    uint16_t address, uint8_t byte)
+{
+	return cdv_intel_dp_aux_native_write(encoder, address, &byte, 1);
+}
+
+/* read bytes from a native aux channel */
+static int
+cdv_intel_dp_aux_native_read(struct psb_intel_encoder *encoder,
+			 uint16_t address, uint8_t *recv, int recv_bytes)
+{
+	uint8_t msg[4];
+	int msg_bytes;
+	uint8_t reply[20];
+	int reply_bytes;
+	uint8_t ack;
+	int ret;
+
+	msg[0] = AUX_NATIVE_READ << 4;
+	msg[1] = address >> 8;
+	msg[2] = address & 0xff;
+	msg[3] = recv_bytes - 1;
+
+	msg_bytes = 4;
+	reply_bytes = recv_bytes + 1;
+
+	for (;;) {
+		ret = cdv_intel_dp_aux_ch(encoder, msg, msg_bytes,
+				      reply, reply_bytes);
+		if (ret == 0)
+			return -EPROTO;
+		if (ret < 0)
+			return ret;
+		ack = reply[0];
+		if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_ACK) {
+			memcpy(recv, reply + 1, ret - 1);
+			return ret - 1;
+		}
+		else if ((ack & AUX_NATIVE_REPLY_MASK) == AUX_NATIVE_REPLY_DEFER)
+			udelay(100);
+		else
+			return -EIO;
+	}
+}
+
+static int
+cdv_intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
+		    uint8_t write_byte, uint8_t *read_byte)
+{
+	struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data;
+	struct cdv_intel_dp *intel_dp = container_of(adapter,
+						struct cdv_intel_dp,
+						adapter);
+	struct psb_intel_encoder *encoder = intel_dp->encoder;
+	uint16_t address = algo_data->address;
+	uint8_t msg[5];
+	uint8_t reply[2];
+	unsigned retry;
+	int msg_bytes;
+	int reply_bytes;
+	int ret;
+
+	/* Set up the command byte */
+	if (mode & MODE_I2C_READ)
+		msg[0] = AUX_I2C_READ << 4;
+	else
+		msg[0] = AUX_I2C_WRITE << 4;
+
+	if (!(mode & MODE_I2C_STOP))
+		msg[0] |= AUX_I2C_MOT << 4;
+
+	msg[1] = address >> 8;
+	msg[2] = address;
+
+	switch (mode) {
+	case MODE_I2C_WRITE:
+		msg[3] = 0;
+		msg[4] = write_byte;
+		msg_bytes = 5;
+		reply_bytes = 1;
+		break;
+	case MODE_I2C_READ:
+		msg[3] = 0;
+		msg_bytes = 4;
+		reply_bytes = 2;
+		break;
+	default:
+		msg_bytes = 3;
+		reply_bytes = 1;
+		break;
+	}
+
+	for (retry = 0; retry < 5; retry++) {
+		ret = cdv_intel_dp_aux_ch(encoder,
+				      msg, msg_bytes,
+				      reply, reply_bytes);
+		if (ret < 0) {
+			DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
+			return ret;
+		}
+
+		switch (reply[0] & AUX_NATIVE_REPLY_MASK) {
+		case AUX_NATIVE_REPLY_ACK:
+			/* I2C-over-AUX Reply field is only valid
+			 * when paired with AUX ACK.
+			 */
+			break;
+		case AUX_NATIVE_REPLY_NACK:
+			DRM_DEBUG_KMS("aux_ch native nack\n");
+			return -EREMOTEIO;
+		case AUX_NATIVE_REPLY_DEFER:
+			udelay(100);
+			continue;
+		default:
+			DRM_ERROR("aux_ch invalid native reply 0x%02x\n",
+				  reply[0]);
+			return -EREMOTEIO;
+		}
+
+		switch (reply[0] & AUX_I2C_REPLY_MASK) {
+		case AUX_I2C_REPLY_ACK:
+			if (mode == MODE_I2C_READ) {
+				*read_byte = reply[1];
+			}
+			return reply_bytes - 1;
+		case AUX_I2C_REPLY_NACK:
+			DRM_DEBUG_KMS("aux_i2c nack\n");
+			return -EREMOTEIO;
+		case AUX_I2C_REPLY_DEFER:
+			DRM_DEBUG_KMS("aux_i2c defer\n");
+			udelay(100);
+			break;
+		default:
+			DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]);
+			return -EREMOTEIO;
+		}
+	}
+
+	DRM_ERROR("too many retries, giving up\n");
+	return -EREMOTEIO;
+}
+
+static int
+cdv_intel_dp_i2c_init(struct psb_intel_connector *connector, struct psb_intel_encoder *encoder, const char *name)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int ret;
+
+	DRM_DEBUG_KMS("i2c_init %s\n", name);
+
+	intel_dp->algo.running = false;
+	intel_dp->algo.address = 0;
+	intel_dp->algo.aux_ch = cdv_intel_dp_i2c_aux_ch;
+
+	memset(&intel_dp->adapter, '\0', sizeof (intel_dp->adapter));
+	intel_dp->adapter.owner = THIS_MODULE;
+	intel_dp->adapter.class = I2C_CLASS_DDC;
+	strncpy (intel_dp->adapter.name, name, sizeof(intel_dp->adapter.name) - 1);
+	intel_dp->adapter.name[sizeof(intel_dp->adapter.name) - 1] = '\0';
+	intel_dp->adapter.algo_data = &intel_dp->algo;
+	intel_dp->adapter.dev.parent = &connector->base.kdev;
+
+	if (is_edp(encoder))
+		cdv_intel_edp_panel_vdd_on(encoder);
+	ret = i2c_dp_aux_add_bus(&intel_dp->adapter);
+	if (is_edp(encoder))
+		cdv_intel_edp_panel_vdd_off(encoder);
+	
+	return ret;
+}
+
+void cdv_intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
+	struct drm_display_mode *adjusted_mode)
+{
+	adjusted_mode->hdisplay = fixed_mode->hdisplay;
+	adjusted_mode->hsync_start = fixed_mode->hsync_start;
+	adjusted_mode->hsync_end = fixed_mode->hsync_end;
+	adjusted_mode->htotal = fixed_mode->htotal;
+
+	adjusted_mode->vdisplay = fixed_mode->vdisplay;
+	adjusted_mode->vsync_start = fixed_mode->vsync_start;
+	adjusted_mode->vsync_end = fixed_mode->vsync_end;
+	adjusted_mode->vtotal = fixed_mode->vtotal;
+
+	adjusted_mode->clock = fixed_mode->clock;
+
+	drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V);
+}
+
+static bool
+cdv_intel_dp_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode,
+		    struct drm_display_mode *adjusted_mode)
+{
+	struct drm_psb_private *dev_priv = encoder->dev->dev_private;
+	struct psb_intel_encoder *intel_encoder = to_psb_intel_encoder(encoder);
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	int lane_count, clock;
+	int max_lane_count = cdv_intel_dp_max_lane_count(intel_encoder);
+	int max_clock = cdv_intel_dp_max_link_bw(intel_encoder) == DP_LINK_BW_2_7 ? 1 : 0;
+	static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 };
+	int refclock = mode->clock;
+	int bpp = 24;
+
+	if (is_edp(intel_encoder) && intel_dp->panel_fixed_mode) {
+		cdv_intel_fixed_panel_mode(intel_dp->panel_fixed_mode, adjusted_mode);
+		refclock = intel_dp->panel_fixed_mode->clock;
+		bpp = dev_priv->edp.bpp;
+	}
+
+	for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
+		for (clock = max_clock; clock >= 0; clock--) {
+			int link_avail = cdv_intel_dp_max_data_rate(cdv_intel_dp_link_clock(bws[clock]), lane_count);
+
+			if (cdv_intel_dp_link_required(refclock, bpp) <= link_avail) {
+				intel_dp->link_bw = bws[clock];
+				intel_dp->lane_count = lane_count;
+				adjusted_mode->clock = cdv_intel_dp_link_clock(intel_dp->link_bw);
+				DRM_DEBUG_KMS("Display port link bw %02x lane "
+						"count %d clock %d\n",
+				       intel_dp->link_bw, intel_dp->lane_count,
+				       adjusted_mode->clock);
+				return true;
+			}
+		}
+	}
+	if (is_edp(intel_encoder)) {
+		/* okay we failed just pick the highest */
+		intel_dp->lane_count = max_lane_count;
+		intel_dp->link_bw = bws[max_clock];
+		adjusted_mode->clock = cdv_intel_dp_link_clock(intel_dp->link_bw);
+		DRM_DEBUG_KMS("Force picking display port link bw %02x lane "
+			      "count %d clock %d\n",
+			      intel_dp->link_bw, intel_dp->lane_count,
+			      adjusted_mode->clock);
+
+		return true;
+	}
+	return false;
+}
+
+struct cdv_intel_dp_m_n {
+	uint32_t	tu;
+	uint32_t	gmch_m;
+	uint32_t	gmch_n;
+	uint32_t	link_m;
+	uint32_t	link_n;
+};
+
+static void
+cdv_intel_reduce_ratio(uint32_t *num, uint32_t *den)
+{
+	/*
+	while (*num > 0xffffff || *den > 0xffffff) {
+		*num >>= 1;
+		*den >>= 1;
+	}*/
+	uint64_t value, m;
+	m = *num;
+	value = m * (0x800000);
+	m = do_div(value, *den);
+	*num = value;
+	*den = 0x800000;
+}
+
+static void
+cdv_intel_dp_compute_m_n(int bpp,
+		     int nlanes,
+		     int pixel_clock,
+		     int link_clock,
+		     struct cdv_intel_dp_m_n *m_n)
+{
+	m_n->tu = 64;
+	m_n->gmch_m = (pixel_clock * bpp + 7) >> 3;
+	m_n->gmch_n = link_clock * nlanes;
+	cdv_intel_reduce_ratio(&m_n->gmch_m, &m_n->gmch_n);
+	m_n->link_m = pixel_clock;
+	m_n->link_n = link_clock;
+	cdv_intel_reduce_ratio(&m_n->link_m, &m_n->link_n);
+}
+
+void
+cdv_intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
+		 struct drm_display_mode *adjusted_mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	struct drm_mode_config *mode_config = &dev->mode_config;
+	struct drm_encoder *encoder;
+	struct psb_intel_crtc *intel_crtc = to_psb_intel_crtc(crtc);
+	int lane_count = 4, bpp = 24;
+	struct cdv_intel_dp_m_n m_n;
+	int pipe = intel_crtc->pipe;
+
+	/*
+	 * Find the lane count in the intel_encoder private
+	 */
+	list_for_each_entry(encoder, &mode_config->encoder_list, head) {
+		struct psb_intel_encoder *intel_encoder;
+		struct cdv_intel_dp *intel_dp;
+
+		if (encoder->crtc != crtc)
+			continue;
+
+		intel_encoder = to_psb_intel_encoder(encoder);
+		intel_dp = intel_encoder->dev_priv;
+		if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT) {
+			lane_count = intel_dp->lane_count;
+			break;
+		} else if (is_edp(intel_encoder)) {
+			lane_count = intel_dp->lane_count;
+			bpp = dev_priv->edp.bpp;
+			break;
+		}
+	}
+
+	/*
+	 * Compute the GMCH and Link ratios. The '3' here is
+	 * the number of bytes_per_pixel post-LUT, which we always
+	 * set up for 8-bits of R/G/B, or 3 bytes total.
+	 */
+	cdv_intel_dp_compute_m_n(bpp, lane_count,
+			     mode->clock, adjusted_mode->clock, &m_n);
+
+	{
+		REG_WRITE(PIPE_GMCH_DATA_M(pipe),
+			   ((m_n.tu - 1) << PIPE_GMCH_DATA_M_TU_SIZE_SHIFT) |
+			   m_n.gmch_m);
+		REG_WRITE(PIPE_GMCH_DATA_N(pipe), m_n.gmch_n);
+		REG_WRITE(PIPE_DP_LINK_M(pipe), m_n.link_m);
+		REG_WRITE(PIPE_DP_LINK_N(pipe), m_n.link_n);
+	}
+}
+
+static void
+cdv_intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
+		  struct drm_display_mode *adjusted_mode)
+{
+	struct psb_intel_encoder *intel_encoder = to_psb_intel_encoder(encoder);
+	struct drm_crtc *crtc = encoder->crtc;
+	struct psb_intel_crtc *intel_crtc = to_psb_intel_crtc(crtc);
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	struct drm_device *dev = encoder->dev;
+
+	intel_dp->DP = DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
+	intel_dp->DP |= intel_dp->color_range;
+
+	if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
+		intel_dp->DP |= DP_SYNC_HS_HIGH;
+	if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
+		intel_dp->DP |= DP_SYNC_VS_HIGH;
+
+	intel_dp->DP |= DP_LINK_TRAIN_OFF;
+
+	switch (intel_dp->lane_count) {
+	case 1:
+		intel_dp->DP |= DP_PORT_WIDTH_1;
+		break;
+	case 2:
+		intel_dp->DP |= DP_PORT_WIDTH_2;
+		break;
+	case 4:
+		intel_dp->DP |= DP_PORT_WIDTH_4;
+		break;
+	}
+	if (intel_dp->has_audio)
+		intel_dp->DP |= DP_AUDIO_OUTPUT_ENABLE;
+
+	memset(intel_dp->link_configuration, 0, DP_LINK_CONFIGURATION_SIZE);
+	intel_dp->link_configuration[0] = intel_dp->link_bw;
+	intel_dp->link_configuration[1] = intel_dp->lane_count;
+
+	/*
+	 * Check for DPCD version > 1.1 and enhanced framing support
+	 */
+	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
+	    (intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_ENHANCED_FRAME_CAP)) {
+		intel_dp->link_configuration[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+		intel_dp->DP |= DP_ENHANCED_FRAMING;
+	}
+
+	/* CPT DP's pipe select is decided in TRANS_DP_CTL */
+	if (intel_crtc->pipe == 1)
+		intel_dp->DP |= DP_PIPEB_SELECT;
+
+	REG_WRITE(intel_dp->output_reg, (intel_dp->DP | DP_PORT_EN));
+	DRM_DEBUG_KMS("DP expected reg is %x\n", intel_dp->DP);
+	if (is_edp(intel_encoder)) {
+		uint32_t pfit_control;
+		cdv_intel_edp_panel_on(intel_encoder);
+
+		if (mode->hdisplay != adjusted_mode->hdisplay ||
+			    mode->vdisplay != adjusted_mode->vdisplay)
+			pfit_control = PFIT_ENABLE;
+		else
+			pfit_control = 0;
+
+		pfit_control |= intel_crtc->pipe << PFIT_PIPE_SHIFT;
+
+		REG_WRITE(PFIT_CONTROL, pfit_control);
+	}
+}
+
+
+/* If the sink supports it, try to set the power state appropriately */
+static void cdv_intel_dp_sink_dpms(struct psb_intel_encoder *encoder, int mode)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int ret, i;
+
+	/* Should have a valid DPCD by this point */
+	if (intel_dp->dpcd[DP_DPCD_REV] < 0x11)
+		return;
+
+	if (mode != DRM_MODE_DPMS_ON) {
+		ret = cdv_intel_dp_aux_native_write_1(encoder, DP_SET_POWER,
+						  DP_SET_POWER_D3);
+		if (ret != 1)
+			DRM_DEBUG_DRIVER("failed to write sink power state\n");
+	} else {
+		/*
+		 * When turning on, we need to retry for 1ms to give the sink
+		 * time to wake up.
+		 */
+		for (i = 0; i < 3; i++) {
+			ret = cdv_intel_dp_aux_native_write_1(encoder,
+							  DP_SET_POWER,
+							  DP_SET_POWER_D0);
+			if (ret == 1)
+				break;
+			udelay(1000);
+		}
+	}
+}
+
+static void cdv_intel_dp_prepare(struct drm_encoder *encoder)
+{
+	struct psb_intel_encoder *intel_encoder = to_psb_intel_encoder(encoder);
+	int edp = is_edp(intel_encoder);
+
+	if (edp) {
+		cdv_intel_edp_backlight_off(intel_encoder);
+		cdv_intel_edp_panel_off(intel_encoder);
+		cdv_intel_edp_panel_vdd_on(intel_encoder);
+        }
+	/* Wake up the sink first */
+	cdv_intel_dp_sink_dpms(intel_encoder, DRM_MODE_DPMS_ON);
+	cdv_intel_dp_link_down(intel_encoder);
+	if (edp)
+		cdv_intel_edp_panel_vdd_off(intel_encoder);
+}
+
+static void cdv_intel_dp_commit(struct drm_encoder *encoder)
+{
+	struct psb_intel_encoder *intel_encoder = to_psb_intel_encoder(encoder);
+	int edp = is_edp(intel_encoder);
+
+	if (edp)
+		cdv_intel_edp_panel_on(intel_encoder);
+	cdv_intel_dp_start_link_train(intel_encoder);
+	cdv_intel_dp_complete_link_train(intel_encoder);
+	if (edp)
+		cdv_intel_edp_backlight_on(intel_encoder);
+}
+
+static void
+cdv_intel_dp_dpms(struct drm_encoder *encoder, int mode)
+{
+	struct psb_intel_encoder *intel_encoder = to_psb_intel_encoder(encoder);
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	struct drm_device *dev = encoder->dev;
+	uint32_t dp_reg = REG_READ(intel_dp->output_reg);
+	int edp = is_edp(intel_encoder);
+
+	if (mode != DRM_MODE_DPMS_ON) {
+		if (edp) {
+			cdv_intel_edp_backlight_off(intel_encoder);
+			cdv_intel_edp_panel_vdd_on(intel_encoder);
+		}
+		cdv_intel_dp_sink_dpms(intel_encoder, mode);
+		cdv_intel_dp_link_down(intel_encoder);
+		if (edp) {
+			cdv_intel_edp_panel_vdd_off(intel_encoder);
+			cdv_intel_edp_panel_off(intel_encoder);
+		}
+	} else {
+        	if (edp)
+			cdv_intel_edp_panel_on(intel_encoder);
+		cdv_intel_dp_sink_dpms(intel_encoder, mode);
+		if (!(dp_reg & DP_PORT_EN)) {
+			cdv_intel_dp_start_link_train(intel_encoder);
+			cdv_intel_dp_complete_link_train(intel_encoder);
+		}
+		if (edp)
+        		cdv_intel_edp_backlight_on(intel_encoder);
+	}
+}
+
+/*
+ * Native read with retry for link status and receiver capability reads for
+ * cases where the sink may still be asleep.
+ */
+static bool
+cdv_intel_dp_aux_native_read_retry(struct psb_intel_encoder *encoder, uint16_t address,
+			       uint8_t *recv, int recv_bytes)
+{
+	int ret, i;
+
+	/*
+	 * Sinks are *supposed* to come up within 1ms from an off state,
+	 * but we're also supposed to retry 3 times per the spec.
+	 */
+	for (i = 0; i < 3; i++) {
+		ret = cdv_intel_dp_aux_native_read(encoder, address, recv,
+					       recv_bytes);
+		if (ret == recv_bytes)
+			return true;
+		udelay(1000);
+	}
+
+	return false;
+}
+
+/*
+ * Fetch AUX CH registers 0x202 - 0x207 which contain
+ * link status information
+ */
+static bool
+cdv_intel_dp_get_link_status(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	return cdv_intel_dp_aux_native_read_retry(encoder,
+					      DP_LANE0_1_STATUS,
+					      intel_dp->link_status,
+					      DP_LINK_STATUS_SIZE);
+}
+
+static uint8_t
+cdv_intel_dp_link_status(uint8_t link_status[DP_LINK_STATUS_SIZE],
+		     int r)
+{
+	return link_status[r - DP_LANE0_1_STATUS];
+}
+
+static uint8_t
+cdv_intel_get_adjust_request_voltage(uint8_t link_status[DP_LINK_STATUS_SIZE],
+				 int lane)
+{
+	int	    i = DP_ADJUST_REQUEST_LANE0_1 + (lane >> 1);
+	int	    s = ((lane & 1) ?
+			 DP_ADJUST_VOLTAGE_SWING_LANE1_SHIFT :
+			 DP_ADJUST_VOLTAGE_SWING_LANE0_SHIFT);
+	uint8_t l = cdv_intel_dp_link_status(link_status, i);
+
+	return ((l >> s) & 3) << DP_TRAIN_VOLTAGE_SWING_SHIFT;
+}
+
+static uint8_t
+cdv_intel_get_adjust_request_pre_emphasis(uint8_t link_status[DP_LINK_STATUS_SIZE],
+				      int lane)
+{
+	int	    i = DP_ADJUST_REQUEST_LANE0_1 + (lane >> 1);
+	int	    s = ((lane & 1) ?
+			 DP_ADJUST_PRE_EMPHASIS_LANE1_SHIFT :
+			 DP_ADJUST_PRE_EMPHASIS_LANE0_SHIFT);
+	uint8_t l = cdv_intel_dp_link_status(link_status, i);
+
+	return ((l >> s) & 3) << DP_TRAIN_PRE_EMPHASIS_SHIFT;
+}
+
+
+#if 0
+static char	*voltage_names[] = {
+	"0.4V", "0.6V", "0.8V", "1.2V"
+};
+static char	*pre_emph_names[] = {
+	"0dB", "3.5dB", "6dB", "9.5dB"
+};
+static char	*link_train_names[] = {
+	"pattern 1", "pattern 2", "idle", "off"
+};
+#endif
+
+#define CDV_DP_VOLTAGE_MAX	    DP_TRAIN_VOLTAGE_SWING_1200
+/*
+static uint8_t
+cdv_intel_dp_pre_emphasis_max(uint8_t voltage_swing)
+{
+	switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
+	case DP_TRAIN_VOLTAGE_SWING_400:
+		return DP_TRAIN_PRE_EMPHASIS_6;
+	case DP_TRAIN_VOLTAGE_SWING_600:
+		return DP_TRAIN_PRE_EMPHASIS_6;
+	case DP_TRAIN_VOLTAGE_SWING_800:
+		return DP_TRAIN_PRE_EMPHASIS_3_5;
+	case DP_TRAIN_VOLTAGE_SWING_1200:
+	default:
+		return DP_TRAIN_PRE_EMPHASIS_0;
+	}
+}
+*/
+static void
+cdv_intel_get_adjust_train(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	uint8_t v = 0;
+	uint8_t p = 0;
+	int lane;
+
+	for (lane = 0; lane < intel_dp->lane_count; lane++) {
+		uint8_t this_v = cdv_intel_get_adjust_request_voltage(intel_dp->link_status, lane);
+		uint8_t this_p = cdv_intel_get_adjust_request_pre_emphasis(intel_dp->link_status, lane);
+
+		if (this_v > v)
+			v = this_v;
+		if (this_p > p)
+			p = this_p;
+	}
+	
+	if (v >= CDV_DP_VOLTAGE_MAX)
+		v = CDV_DP_VOLTAGE_MAX | DP_TRAIN_MAX_SWING_REACHED;
+
+	if (p == DP_TRAIN_PRE_EMPHASIS_MASK)
+		p |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+		
+	for (lane = 0; lane < 4; lane++)
+		intel_dp->train_set[lane] = v | p;
+}
+
+
+static uint8_t
+cdv_intel_get_lane_status(uint8_t link_status[DP_LINK_STATUS_SIZE],
+		      int lane)
+{
+	int i = DP_LANE0_1_STATUS + (lane >> 1);
+	int s = (lane & 1) * 4;
+	uint8_t l = cdv_intel_dp_link_status(link_status, i);
+
+	return (l >> s) & 0xf;
+}
+
+/* Check for clock recovery is done on all channels */
+static bool
+cdv_intel_clock_recovery_ok(uint8_t link_status[DP_LINK_STATUS_SIZE], int lane_count)
+{
+	int lane;
+	uint8_t lane_status;
+
+	for (lane = 0; lane < lane_count; lane++) {
+		lane_status = cdv_intel_get_lane_status(link_status, lane);
+		if ((lane_status & DP_LANE_CR_DONE) == 0)
+			return false;
+	}
+	return true;
+}
+
+/* Check to see if channel eq is done on all channels */
+#define CHANNEL_EQ_BITS (DP_LANE_CR_DONE|\
+			 DP_LANE_CHANNEL_EQ_DONE|\
+			 DP_LANE_SYMBOL_LOCKED)
+static bool
+cdv_intel_channel_eq_ok(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	uint8_t lane_align;
+	uint8_t lane_status;
+	int lane;
+
+	lane_align = cdv_intel_dp_link_status(intel_dp->link_status,
+					  DP_LANE_ALIGN_STATUS_UPDATED);
+	if ((lane_align & DP_INTERLANE_ALIGN_DONE) == 0)
+		return false;
+	for (lane = 0; lane < intel_dp->lane_count; lane++) {
+		lane_status = cdv_intel_get_lane_status(intel_dp->link_status, lane);
+		if ((lane_status & CHANNEL_EQ_BITS) != CHANNEL_EQ_BITS)
+			return false;
+	}
+	return true;
+}
+
+static bool
+cdv_intel_dp_set_link_train(struct psb_intel_encoder *encoder,
+			uint32_t dp_reg_value,
+			uint8_t dp_train_pat)
+{
+	
+	struct drm_device *dev = encoder->base.dev;
+	int ret;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+
+	REG_WRITE(intel_dp->output_reg, dp_reg_value);
+	REG_READ(intel_dp->output_reg);
+
+	ret = cdv_intel_dp_aux_native_write_1(encoder,
+				    DP_TRAINING_PATTERN_SET,
+				    dp_train_pat);
+
+	if (ret != 1) {
+		DRM_DEBUG_KMS("Failure in setting link pattern %x\n",
+				dp_train_pat);
+		return false;
+	}
+
+	return true;
+}
+
+
+static bool
+cdv_intel_dplink_set_level(struct psb_intel_encoder *encoder,
+			uint8_t dp_train_pat)
+{
+	
+	int ret;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+
+	ret = cdv_intel_dp_aux_native_write(encoder,
+					DP_TRAINING_LANE0_SET,
+					intel_dp->train_set,
+					intel_dp->lane_count);
+
+	if (ret != intel_dp->lane_count) {
+		DRM_DEBUG_KMS("Failure in setting level %d, lane_cnt= %d\n",
+				intel_dp->train_set[0], intel_dp->lane_count);
+		return false;
+	}
+	return true;
+}
+
+static void
+cdv_intel_dp_set_vswing_premph(struct psb_intel_encoder *encoder, uint8_t signal_level)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	struct ddi_regoff *ddi_reg;
+	int vswing, premph, index;
+
+	if (intel_dp->output_reg == DP_B)
+		ddi_reg = &ddi_DP_train_table[0];
+	else
+		ddi_reg = &ddi_DP_train_table[1];
+
+	vswing = (signal_level & DP_TRAIN_VOLTAGE_SWING_MASK);
+	premph = ((signal_level & DP_TRAIN_PRE_EMPHASIS_MASK)) >>
+				DP_TRAIN_PRE_EMPHASIS_SHIFT;
+
+	if (vswing + premph > 3)
+		return;
+#ifdef CDV_FAST_LINK_TRAIN
+	return;
+#endif
+	DRM_DEBUG_KMS("Test2\n");
+	//return ;
+	cdv_sb_reset(dev);
+	/* ;Swing voltage programming
+        ;gfx_dpio_set_reg(0xc058, 0x0505313A) */
+	cdv_sb_write(dev, ddi_reg->VSwing5, 0x0505313A);
+
+	/* ;gfx_dpio_set_reg(0x8154, 0x43406055) */
+	cdv_sb_write(dev, ddi_reg->VSwing1, 0x43406055);
+
+	/* ;gfx_dpio_set_reg(0x8148, 0x55338954)
+	 * The VSwing_PreEmph table is also considered based on the vswing/premp
+	 */
+	index = (vswing + premph) * 2;
+	if (premph == 1 && vswing == 1) {
+		cdv_sb_write(dev, ddi_reg->VSwing2, 0x055738954);
+	} else
+		cdv_sb_write(dev, ddi_reg->VSwing2, dp_vswing_premph_table[index]);
+
+	/* ;gfx_dpio_set_reg(0x814c, 0x40802040) */
+	if ((vswing + premph) == DP_TRAIN_VOLTAGE_SWING_1200)
+		cdv_sb_write(dev, ddi_reg->VSwing3, 0x70802040);
+	else
+		cdv_sb_write(dev, ddi_reg->VSwing3, 0x40802040);
+
+	/* ;gfx_dpio_set_reg(0x8150, 0x2b405555) */
+	/* cdv_sb_write(dev, ddi_reg->VSwing4, 0x2b405555); */
+
+	/* ;gfx_dpio_set_reg(0x8154, 0xc3406055) */
+	cdv_sb_write(dev, ddi_reg->VSwing1, 0xc3406055);
+
+	/* ;Pre emphasis programming
+	 * ;gfx_dpio_set_reg(0xc02c, 0x1f030040)
+	 */
+	cdv_sb_write(dev, ddi_reg->PreEmph1, 0x1f030040);
+
+	/* ;gfx_dpio_set_reg(0x8124, 0x00004000) */
+	index = 2 * premph + 1;
+	cdv_sb_write(dev, ddi_reg->PreEmph2, dp_vswing_premph_table[index]);
+	return;	
+}
+
+
+/* Enable corresponding port and start training pattern 1 */
+static void
+cdv_intel_dp_start_link_train(struct psb_intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int i;
+	uint8_t voltage;
+	bool clock_recovery = false;
+	int tries;
+	u32 reg;
+	uint32_t DP = intel_dp->DP;
+
+	DP |= DP_PORT_EN;
+	DP &= ~DP_LINK_TRAIN_MASK;
+		
+	reg = DP;	
+	reg |= DP_LINK_TRAIN_PAT_1;
+	/* Enable output, wait for it to become active */
+	REG_WRITE(intel_dp->output_reg, reg);
+	REG_READ(intel_dp->output_reg);
+	psb_intel_wait_for_vblank(dev);
+
+	DRM_DEBUG_KMS("Link config\n");
+	/* Write the link configuration data */
+	cdv_intel_dp_aux_native_write(encoder, DP_LINK_BW_SET,
+				  intel_dp->link_configuration,
+				  2);
+
+	memset(intel_dp->train_set, 0, 4);
+	voltage = 0;
+	tries = 0;
+	clock_recovery = false;
+
+	DRM_DEBUG_KMS("Start train\n");
+		reg = DP | DP_LINK_TRAIN_PAT_1;
+
+
+	for (;;) {
+		/* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */
+		DRM_DEBUG_KMS("DP Link Train Set %x, Link_config %x, %x\n",
+				intel_dp->train_set[0],
+				intel_dp->link_configuration[0],
+				intel_dp->link_configuration[1]);
+
+		if (!cdv_intel_dp_set_link_train(encoder, reg, DP_TRAINING_PATTERN_1)) {
+			DRM_DEBUG_KMS("Failure in aux-transfer setting pattern 1\n");
+		}
+		cdv_intel_dp_set_vswing_premph(encoder, intel_dp->train_set[0]);
+		/* Set training pattern 1 */
+
+		cdv_intel_dplink_set_level(encoder, DP_TRAINING_PATTERN_1);
+
+		udelay(200);
+		if (!cdv_intel_dp_get_link_status(encoder))
+			break;
+
+		DRM_DEBUG_KMS("DP Link status %x, %x, %x, %x, %x, %x\n",
+				intel_dp->link_status[0], intel_dp->link_status[1], intel_dp->link_status[2],
+				intel_dp->link_status[3], intel_dp->link_status[4], intel_dp->link_status[5]);
+
+		if (cdv_intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
+			DRM_DEBUG_KMS("PT1 train is done\n");
+			clock_recovery = true;
+			break;
+		}
+
+		/* Check to see if we've tried the max voltage */
+		for (i = 0; i < intel_dp->lane_count; i++)
+			if ((intel_dp->train_set[i] & DP_TRAIN_MAX_SWING_REACHED) == 0)
+				break;
+		if (i == intel_dp->lane_count)
+			break;
+
+		/* Check to see if we've tried the same voltage 5 times */
+		if ((intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK) == voltage) {
+			++tries;
+			if (tries == 5)
+				break;
+		} else
+			tries = 0;
+		voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK;
+
+		/* Compute new intel_dp->train_set as requested by target */
+		cdv_intel_get_adjust_train(encoder);
+
+	}
+
+	if (!clock_recovery) {
+		DRM_DEBUG_KMS("failure in DP patter 1 training, train set %x\n", intel_dp->train_set[0]);
+	}
+	
+	intel_dp->DP = DP;
+}
+
+static void
+cdv_intel_dp_complete_link_train(struct psb_intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	bool channel_eq = false;
+	int tries, cr_tries;
+	u32 reg;
+	uint32_t DP = intel_dp->DP;
+
+	/* channel equalization */
+	tries = 0;
+	cr_tries = 0;
+	channel_eq = false;
+
+	DRM_DEBUG_KMS("\n");
+		reg = DP | DP_LINK_TRAIN_PAT_2;
+
+	for (;;) {
+
+		DRM_DEBUG_KMS("DP Link Train Set %x, Link_config %x, %x\n",
+				intel_dp->train_set[0],
+				intel_dp->link_configuration[0],
+				intel_dp->link_configuration[1]);
+        	/* channel eq pattern */
+
+		if (!cdv_intel_dp_set_link_train(encoder, reg,
+					     DP_TRAINING_PATTERN_2)) {
+			DRM_DEBUG_KMS("Failure in aux-transfer setting pattern 2\n");
+		}
+		/* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */
+
+		if (cr_tries > 5) {
+			DRM_ERROR("failed to train DP, aborting\n");
+			cdv_intel_dp_link_down(encoder);
+			break;
+		}
+
+		cdv_intel_dp_set_vswing_premph(encoder, intel_dp->train_set[0]);
+
+		cdv_intel_dplink_set_level(encoder, DP_TRAINING_PATTERN_2);
+
+		udelay(1000);
+		if (!cdv_intel_dp_get_link_status(encoder))
+			break;
+
+		DRM_DEBUG_KMS("DP Link status %x, %x, %x, %x, %x, %x\n",
+				intel_dp->link_status[0], intel_dp->link_status[1], intel_dp->link_status[2],
+				intel_dp->link_status[3], intel_dp->link_status[4], intel_dp->link_status[5]);
+
+		/* Make sure clock is still ok */
+		if (!cdv_intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) {
+			cdv_intel_dp_start_link_train(encoder);
+			cr_tries++;
+			continue;
+		}
+
+		if (cdv_intel_channel_eq_ok(encoder)) {
+			DRM_DEBUG_KMS("PT2 train is done\n");
+			channel_eq = true;
+			break;
+		}
+
+		/* Try 5 times, then try clock recovery if that fails */
+		if (tries > 5) {
+			cdv_intel_dp_link_down(encoder);
+			cdv_intel_dp_start_link_train(encoder);
+			tries = 0;
+			cr_tries++;
+			continue;
+		}
+
+		/* Compute new intel_dp->train_set as requested by target */
+		cdv_intel_get_adjust_train(encoder);
+		++tries;
+
+	}
+
+	reg = DP | DP_LINK_TRAIN_OFF;
+
+	REG_WRITE(intel_dp->output_reg, reg);
+	REG_READ(intel_dp->output_reg);
+	cdv_intel_dp_aux_native_write_1(encoder,
+				    DP_TRAINING_PATTERN_SET, DP_TRAINING_PATTERN_DISABLE);
+}
+
+static void
+cdv_intel_dp_link_down(struct psb_intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	uint32_t DP = intel_dp->DP;
+
+	if ((REG_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)
+		return;
+
+	DRM_DEBUG_KMS("\n");
+
+
+	{
+		DP &= ~DP_LINK_TRAIN_MASK;
+		REG_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE);
+	}
+	REG_READ(intel_dp->output_reg);
+
+	msleep(17);
+
+	REG_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN);
+	REG_READ(intel_dp->output_reg);
+}
+
+static enum drm_connector_status
+cdv_dp_detect(struct psb_intel_encoder *encoder)
+{
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	enum drm_connector_status status;
+
+	status = connector_status_disconnected;
+	if (cdv_intel_dp_aux_native_read(encoder, 0x000, intel_dp->dpcd,
+				     sizeof (intel_dp->dpcd)) == sizeof (intel_dp->dpcd))
+	{
+		if (intel_dp->dpcd[DP_DPCD_REV] != 0)
+			status = connector_status_connected;
+	}
+	if (status == connector_status_connected)
+		DRM_DEBUG_KMS("DPCD: Rev=%x LN_Rate=%x LN_CNT=%x LN_DOWNSP=%x\n",
+			intel_dp->dpcd[0], intel_dp->dpcd[1],
+			intel_dp->dpcd[2], intel_dp->dpcd[3]);
+	return status;
+}
+
+/**
+ * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection.
+ *
+ * \return true if DP port is connected.
+ * \return false if DP port is disconnected.
+ */
+static enum drm_connector_status
+cdv_intel_dp_detect(struct drm_connector *connector, bool force)
+{
+	struct psb_intel_encoder *encoder = psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	enum drm_connector_status status;
+	struct edid *edid = NULL;
+	int edp = is_edp(encoder);
+
+	intel_dp->has_audio = false;
+
+	if (edp)
+		cdv_intel_edp_panel_vdd_on(encoder);
+	status = cdv_dp_detect(encoder);
+	if (status != connector_status_connected) {
+		if (edp)
+			cdv_intel_edp_panel_vdd_off(encoder);
+		return status;
+        }
+
+	if (intel_dp->force_audio) {
+		intel_dp->has_audio = intel_dp->force_audio > 0;
+	} else {
+		edid = drm_get_edid(connector, &intel_dp->adapter);
+		if (edid) {
+			intel_dp->has_audio = drm_detect_monitor_audio(edid);
+			kfree(edid);
+		}
+	}
+	if (edp)
+		cdv_intel_edp_panel_vdd_off(encoder);
+
+	return connector_status_connected;
+}
+
+static int cdv_intel_dp_get_modes(struct drm_connector *connector)
+{
+	struct psb_intel_encoder *intel_encoder = psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = intel_encoder->dev_priv;
+	struct edid *edid = NULL;
+	int ret = 0;
+	int edp = is_edp(intel_encoder);
+
+
+	edid = drm_get_edid(connector, &intel_dp->adapter);
+	if (edid) {
+		drm_mode_connector_update_edid_property(connector, edid);
+		ret = drm_add_edid_modes(connector, edid);
+		kfree(edid);
+	}
+
+	if (is_edp(intel_encoder)) {
+		struct drm_device *dev = connector->dev;
+		struct drm_psb_private *dev_priv = dev->dev_private;
+		
+		cdv_intel_edp_panel_vdd_off(intel_encoder);
+		if (ret) {
+			if (edp && !intel_dp->panel_fixed_mode) {
+				struct drm_display_mode *newmode;
+				list_for_each_entry(newmode, &connector->probed_modes,
+					    head) {
+					if (newmode->type & DRM_MODE_TYPE_PREFERRED) {
+						intel_dp->panel_fixed_mode =
+							drm_mode_duplicate(dev, newmode);
+						break;
+					}
+				}
+			}
+
+			return ret;
+		}
+		if (!intel_dp->panel_fixed_mode && dev_priv->lfp_lvds_vbt_mode) {
+			intel_dp->panel_fixed_mode =
+				drm_mode_duplicate(dev, dev_priv->lfp_lvds_vbt_mode);
+			if (intel_dp->panel_fixed_mode) {
+				intel_dp->panel_fixed_mode->type |=
+					DRM_MODE_TYPE_PREFERRED;
+			}
+		}
+		if (intel_dp->panel_fixed_mode != NULL) {
+			struct drm_display_mode *mode;
+			mode = drm_mode_duplicate(dev, intel_dp->panel_fixed_mode);
+			drm_mode_probed_add(connector, mode);
+			return 1;
+		}
+	}
+
+	return ret;
+}
+
+static bool
+cdv_intel_dp_detect_audio(struct drm_connector *connector)
+{
+	struct psb_intel_encoder *encoder = psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	struct edid *edid;
+	bool has_audio = false;
+	int edp = is_edp(encoder);
+
+	if (edp)
+		cdv_intel_edp_panel_vdd_on(encoder);
+
+	edid = drm_get_edid(connector, &intel_dp->adapter);
+	if (edid) {
+		has_audio = drm_detect_monitor_audio(edid);
+		kfree(edid);
+	}
+	if (edp)
+		cdv_intel_edp_panel_vdd_off(encoder);
+
+	return has_audio;
+}
+
+static int
+cdv_intel_dp_set_property(struct drm_connector *connector,
+		      struct drm_property *property,
+		      uint64_t val)
+{
+	struct drm_psb_private *dev_priv = connector->dev->dev_private;
+	struct psb_intel_encoder *encoder = psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = encoder->dev_priv;
+	int ret;
+
+	ret = drm_connector_property_set_value(connector, property, val);
+	if (ret)
+		return ret;
+
+	if (property == dev_priv->force_audio_property) {
+		int i = val;
+		bool has_audio;
+
+		if (i == intel_dp->force_audio)
+			return 0;
+
+		intel_dp->force_audio = i;
+
+		if (i == 0)
+			has_audio = cdv_intel_dp_detect_audio(connector);
+		else
+			has_audio = i > 0;
+
+		if (has_audio == intel_dp->has_audio)
+			return 0;
+
+		intel_dp->has_audio = has_audio;
+		goto done;
+	}
+
+	if (property == dev_priv->broadcast_rgb_property) {
+		if (val == !!intel_dp->color_range)
+			return 0;
+
+		intel_dp->color_range = val ? DP_COLOR_RANGE_16_235 : 0;
+		goto done;
+	}
+
+	return -EINVAL;
+
+done:
+	if (encoder->base.crtc) {
+		struct drm_crtc *crtc = encoder->base.crtc;
+		drm_crtc_helper_set_mode(crtc, &crtc->mode,
+					 crtc->x, crtc->y,
+					 crtc->fb);
+	}
+
+	return 0;
+}
+
+static void
+cdv_intel_dp_destroy(struct drm_connector *connector)
+{
+	struct psb_intel_encoder *psb_intel_encoder =
+					psb_intel_attached_encoder(connector);
+	struct cdv_intel_dp *intel_dp = psb_intel_encoder->dev_priv;
+
+	if (is_edp(psb_intel_encoder)) {
+	/*	cdv_intel_panel_destroy_backlight(connector->dev); */
+		if (intel_dp->panel_fixed_mode) {
+			kfree(intel_dp->panel_fixed_mode);
+			intel_dp->panel_fixed_mode = NULL;
+		}
+	}
+	i2c_del_adapter(&intel_dp->adapter);
+	drm_sysfs_connector_remove(connector);
+	drm_connector_cleanup(connector);
+	kfree(connector);
+}
+
+static void cdv_intel_dp_encoder_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+}
+
+static const struct drm_encoder_helper_funcs cdv_intel_dp_helper_funcs = {
+	.dpms = cdv_intel_dp_dpms,
+	.mode_fixup = cdv_intel_dp_mode_fixup,
+	.prepare = cdv_intel_dp_prepare,
+	.mode_set = cdv_intel_dp_mode_set,
+	.commit = cdv_intel_dp_commit,
+};
+
+static const struct drm_connector_funcs cdv_intel_dp_connector_funcs = {
+	.dpms = drm_helper_connector_dpms,
+	.detect = cdv_intel_dp_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.set_property = cdv_intel_dp_set_property,
+	.destroy = cdv_intel_dp_destroy,
+};
+
+static const struct drm_connector_helper_funcs cdv_intel_dp_connector_helper_funcs = {
+	.get_modes = cdv_intel_dp_get_modes,
+	.mode_valid = cdv_intel_dp_mode_valid,
+	.best_encoder = psb_intel_best_encoder,
+};
+
+static const struct drm_encoder_funcs cdv_intel_dp_enc_funcs = {
+	.destroy = cdv_intel_dp_encoder_destroy,
+};
+
+
+static void cdv_intel_dp_add_properties(struct drm_connector *connector)
+{
+	cdv_intel_attach_force_audio_property(connector);
+	cdv_intel_attach_broadcast_rgb_property(connector);
+}
+
+/* check the VBT to see whether the eDP is on DP-D port */
+static bool cdv_intel_dpc_is_edp(struct drm_device *dev)
+{
+	struct drm_psb_private *dev_priv = dev->dev_private;
+	struct child_device_config *p_child;
+	int i;
+
+	if (!dev_priv->child_dev_num)
+		return false;
+
+	for (i = 0; i < dev_priv->child_dev_num; i++) {
+		p_child = dev_priv->child_dev + i;
+
+		if (p_child->dvo_port == PORT_IDPC &&
+		    p_child->device_type == DEVICE_TYPE_eDP)
+			return true;
+	}
+	return false;
+}
+
+/* Cedarview display clock gating
+
+   We need this disable dot get correct behaviour while enabling
+   DP/eDP. TODO - investigate if we can turn it back to normality
+   after enabling */
+static void cdv_disable_intel_clock_gating(struct drm_device *dev)
+{
+	u32 reg_value;
+	reg_value = REG_READ(DSPCLK_GATE_D);
+
+	reg_value |= (DPUNIT_PIPEB_GATE_DISABLE |
+			DPUNIT_PIPEA_GATE_DISABLE |
+			DPCUNIT_CLOCK_GATE_DISABLE |
+			DPLSUNIT_CLOCK_GATE_DISABLE |
+			DPOUNIT_CLOCK_GATE_DISABLE |
+		 	DPIOUNIT_CLOCK_GATE_DISABLE);	
+
+	REG_WRITE(DSPCLK_GATE_D, reg_value);
+
+	udelay(500);		
+}
+
+void
+cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev, int output_reg)
+{
+	struct psb_intel_encoder *psb_intel_encoder;
+	struct psb_intel_connector *psb_intel_connector;
+	struct drm_connector *connector;
+	struct drm_encoder *encoder;
+	struct cdv_intel_dp *intel_dp;
+	const char *name = NULL;
+	int type = DRM_MODE_CONNECTOR_DisplayPort;
+
+	psb_intel_encoder = kzalloc(sizeof(struct psb_intel_encoder), GFP_KERNEL);
+	if (!psb_intel_encoder)
+		return;
+        psb_intel_connector = kzalloc(sizeof(struct psb_intel_connector), GFP_KERNEL);
+        if (!psb_intel_connector)
+                goto err_connector;
+	intel_dp = kzalloc(sizeof(struct cdv_intel_dp), GFP_KERNEL);
+	if (!intel_dp)
+	        goto err_priv;
+
+	if ((output_reg == DP_C) && cdv_intel_dpc_is_edp(dev))
+		type = DRM_MODE_CONNECTOR_eDP;
+
+	connector = &psb_intel_connector->base;
+	encoder = &psb_intel_encoder->base;
+
+	drm_connector_init(dev, connector, &cdv_intel_dp_connector_funcs, type);
+	drm_encoder_init(dev, encoder, &cdv_intel_dp_enc_funcs, DRM_MODE_ENCODER_TMDS);
+
+	psb_intel_connector_attach_encoder(psb_intel_connector, psb_intel_encoder);
+
+	if (type == DRM_MODE_CONNECTOR_DisplayPort)
+        	psb_intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+        else
+		psb_intel_encoder->type = INTEL_OUTPUT_EDP;
+
+
+	psb_intel_encoder->dev_priv=intel_dp;
+	intel_dp->encoder = psb_intel_encoder;
+	intel_dp->output_reg = output_reg;
+	
+	drm_encoder_helper_add(encoder, &cdv_intel_dp_helper_funcs);
+	drm_connector_helper_add(connector, &cdv_intel_dp_connector_helper_funcs);
+
+	connector->polled = DRM_CONNECTOR_POLL_HPD;
+	connector->interlace_allowed = false;
+	connector->doublescan_allowed = false;
+
+	drm_sysfs_connector_add(connector);
+
+	/* Set up the DDC bus. */
+	switch (output_reg) {
+		case DP_B:
+			name = "DPDDC-B";
+			psb_intel_encoder->ddi_select = (DP_MASK | DDI0_SELECT);
+			break;
+		case DP_C:
+			name = "DPDDC-C";
+			psb_intel_encoder->ddi_select = (DP_MASK | DDI1_SELECT);
+			break;
+	}
+
+	cdv_disable_intel_clock_gating(dev);
+
+	cdv_intel_dp_i2c_init(psb_intel_connector, psb_intel_encoder, name);
+        /* FIXME:fail check */
+	cdv_intel_dp_add_properties(connector);
+
+	if (is_edp(psb_intel_encoder)) {
+		int ret;
+		struct edp_power_seq cur;
+                u32 pp_on, pp_off, pp_div;
+		u32 pwm_ctrl;
+
+		pp_on = REG_READ(PP_CONTROL);
+		pp_on &= ~PANEL_UNLOCK_MASK;
+	        pp_on |= PANEL_UNLOCK_REGS;
+		
+		REG_WRITE(PP_CONTROL, pp_on);
+
+		pwm_ctrl = REG_READ(BLC_PWM_CTL2);
+		pwm_ctrl |= PWM_PIPE_B;
+		REG_WRITE(BLC_PWM_CTL2, pwm_ctrl);
+
+                pp_on = REG_READ(PP_ON_DELAYS);
+                pp_off = REG_READ(PP_OFF_DELAYS);
+                pp_div = REG_READ(PP_DIVISOR);
+	
+		/* Pull timing values out of registers */
+                cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
+                        PANEL_POWER_UP_DELAY_SHIFT;
+
+                cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
+                        PANEL_LIGHT_ON_DELAY_SHIFT;
+
+                cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
+                        PANEL_LIGHT_OFF_DELAY_SHIFT;
+
+                cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
+                        PANEL_POWER_DOWN_DELAY_SHIFT;
+
+                cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
+                               PANEL_POWER_CYCLE_DELAY_SHIFT);
+
+                DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
+                              cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12);
+
+
+		intel_dp->panel_power_up_delay = cur.t1_t3 / 10;
+                intel_dp->backlight_on_delay = cur.t8 / 10;
+                intel_dp->backlight_off_delay = cur.t9 / 10;
+                intel_dp->panel_power_down_delay = cur.t10 / 10;
+                intel_dp->panel_power_cycle_delay = (cur.t11_t12 - 1) * 100;
+
+                DRM_DEBUG_KMS("panel power up delay %d, power down delay %d, power cycle delay %d\n",
+                              intel_dp->panel_power_up_delay, intel_dp->panel_power_down_delay,
+                              intel_dp->panel_power_cycle_delay);
+
+                DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n",
+                              intel_dp->backlight_on_delay, intel_dp->backlight_off_delay);
+
+
+		cdv_intel_edp_panel_vdd_on(psb_intel_encoder);
+		ret = cdv_intel_dp_aux_native_read(psb_intel_encoder, DP_DPCD_REV,
+					       intel_dp->dpcd,
+					       sizeof(intel_dp->dpcd));
+		cdv_intel_edp_panel_vdd_off(psb_intel_encoder);
+		if (ret == 0) {
+			/* if this fails, presume the device is a ghost */
+			DRM_INFO("failed to retrieve link info, disabling eDP\n");
+			cdv_intel_dp_encoder_destroy(encoder);
+			cdv_intel_dp_destroy(connector);
+			goto err_priv;
+		} else {
+        		DRM_DEBUG_KMS("DPCD: Rev=%x LN_Rate=%x LN_CNT=%x LN_DOWNSP=%x\n",
+				intel_dp->dpcd[0], intel_dp->dpcd[1], 
+				intel_dp->dpcd[2], intel_dp->dpcd[3]);
+			
+		}
+		/* The CDV reference driver moves pnale backlight setup into the displays that
+		   have a backlight: this is a good idea and one we should probably adopt, however
+		   we need to migrate all the drivers before we can do that */
+                /*cdv_intel_panel_setup_backlight(dev); */
+	}
+	return;
+
+err_priv:
+	kfree(psb_intel_connector);
+err_connector:
+	kfree(psb_intel_encoder);
+}
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index a86f87b9ddd..7272a461edf 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -139,8 +139,6 @@ static enum drm_connector_status cdv_hdmi_detect(
 {
 	struct psb_intel_encoder *psb_intel_encoder =
 					psb_intel_attached_encoder(connector);
-	struct psb_intel_connector *psb_intel_connector =
-					to_psb_intel_connector(connector);
 	struct mid_intel_hdmi_priv *hdmi_priv = psb_intel_encoder->dev_priv;
 	struct edid *edid = NULL;
 	enum drm_connector_status status = connector_status_disconnected;
@@ -157,8 +155,6 @@ static enum drm_connector_status cdv_hdmi_detect(
 			hdmi_priv->has_hdmi_audio =
 						drm_detect_monitor_audio(edid);
 		}
-
-		psb_intel_connector->base.display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 	return status;
@@ -352,9 +348,11 @@ void cdv_hdmi_init(struct drm_device *dev,
 	switch (reg) {
 	case SDVOB:
 		ddc_bus = GPIOE;
+		psb_intel_encoder->ddi_select = DDI0_SELECT;
 		break;
 	case SDVOC:
 		ddc_bus = GPIOD;
+		psb_intel_encoder->ddi_select = DDI1_SELECT;
 		break;
 	default:
 		DRM_ERROR("unknown reg 0x%x for HDMI\n", reg);
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index c7f9468b74b..b362dd39bf5 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -506,16 +506,8 @@ static int cdv_intel_lvds_set_property(struct drm_connector *connector,
 							property,
 							value))
 			return -1;
-		else {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-			struct drm_psb_private *dev_priv =
-						encoder->dev->dev_private;
-			struct backlight_device *bd =
-						dev_priv->backlight_device;
-			bd->props.brightness = value;
-			backlight_update_status(bd);
-#endif
-		}
+		else
+                        gma_backlight_set(encoder->dev, value);
 	} else if (!strcmp(property->name, "DPMS") && encoder) {
 		struct drm_encoder_helper_funcs *helpers =
 					encoder->helper_private;
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 5732b5702e1..884ba73ac6c 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -764,6 +764,13 @@ static void psb_setup_outputs(struct drm_device *dev)
 		        crtc_mask = dev_priv->ops->hdmi_mask;
 			clone_mask = (1 << INTEL_OUTPUT_HDMI);
 			break;
+		case INTEL_OUTPUT_DISPLAYPORT:
+			crtc_mask = (1 << 0) | (1 << 1);
+			clone_mask = (1 << INTEL_OUTPUT_DISPLAYPORT);
+			break;
+		case INTEL_OUTPUT_EDP:
+			crtc_mask = (1 << 1);
+			clone_mask = (1 << INTEL_OUTPUT_EDP);
 		}
 		encoder->possible_crtcs = crtc_mask;
 		encoder->possible_clones =
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index fc7d144bc2d..eefd6cc5b80 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -25,7 +25,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 
 int psb_gem_init_object(struct drm_gem_object *obj)
@@ -36,7 +36,12 @@ int psb_gem_init_object(struct drm_gem_object *obj)
 void psb_gem_free_object(struct drm_gem_object *obj)
 {
 	struct gtt_range *gtt = container_of(obj, struct gtt_range, gem);
-	drm_gem_object_release_wrap(obj);
+
+	/* Remove the list map if one is present */
+	if (obj->map_list.map)
+		drm_gem_free_mmap_offset(obj);
+	drm_gem_object_release(obj);
+
 	/* This must occur last as it frees up the memory of the GEM object */
 	psb_gtt_free_range(obj->dev, gtt);
 }
@@ -77,7 +82,7 @@ int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
 
 	/* Make it mmapable */
 	if (!obj->map_list.map) {
-		ret = gem_create_mmap_offset(obj);
+		ret = drm_gem_create_mmap_offset(obj);
 		if (ret)
 			goto out;
 	}
diff --git a/drivers/gpu/drm/gma500/gem_glue.c b/drivers/gpu/drm/gma500/gem_glue.c
deleted file mode 100644
index 3c17634f606..00000000000
--- a/drivers/gpu/drm/gma500/gem_glue.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/**************************************************************************
- * Copyright (c) 2011, Intel Corporation.
- * All Rights Reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- **************************************************************************/
-
-#include <drm/drmP.h>
-#include <drm/drm.h>
-#include "gem_glue.h"
-
-void drm_gem_object_release_wrap(struct drm_gem_object *obj)
-{
-	/* Remove the list map if one is present */
-	if (obj->map_list.map) {
-		struct drm_gem_mm *mm = obj->dev->mm_private;
-		struct drm_map_list *list = &obj->map_list;
-		drm_ht_remove_item(&mm->offset_hash, &list->hash);
-		drm_mm_put_block(list->file_offset_node);
-		kfree(list->map);
-		list->map = NULL;
-	}
-	drm_gem_object_release(obj);
-}
-
-/**
- *	gem_create_mmap_offset		-	invent an mmap offset
- *	@obj: our object
- *
- *	Standard implementation of offset generation for mmap as is
- *	duplicated in several drivers. This belongs in GEM.
- */
-int gem_create_mmap_offset(struct drm_gem_object *obj)
-{
-	struct drm_device *dev = obj->dev;
-	struct drm_gem_mm *mm = dev->mm_private;
-	struct drm_map_list *list;
-	struct drm_local_map *map;
-	int ret;
-
-	list = &obj->map_list;
-	list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
-	if (list->map == NULL)
-		return -ENOMEM;
-	map = list->map;
-	map->type = _DRM_GEM;
-	map->size = obj->size;
-	map->handle = obj;
-
-	list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
-					obj->size / PAGE_SIZE, 0, 0);
-	if (!list->file_offset_node) {
-		dev_err(dev->dev, "failed to allocate offset for bo %d\n",
-								obj->name);
-		ret = -ENOSPC;
-		goto free_it;
-	}
-	list->file_offset_node = drm_mm_get_block(list->file_offset_node,
-					obj->size / PAGE_SIZE, 0);
-	if (!list->file_offset_node) {
-		ret = -ENOMEM;
-		goto free_it;
-	}
-	list->hash.key = list->file_offset_node->start;
-	ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
-	if (ret) {
-		dev_err(dev->dev, "failed to add to map hash\n");
-		goto free_mm;
-	}
-	return 0;
-
-free_mm:
-	drm_mm_put_block(list->file_offset_node);
-free_it:
-	kfree(list->map);
-	list->map = NULL;
-	return ret;
-}
diff --git a/drivers/gpu/drm/gma500/gem_glue.h b/drivers/gpu/drm/gma500/gem_glue.h
deleted file mode 100644
index ce5ce30f74d..00000000000
--- a/drivers/gpu/drm/gma500/gem_glue.h
+++ /dev/null
@@ -1,2 +0,0 @@
-extern void drm_gem_object_release_wrap(struct drm_gem_object *obj);
-extern int gem_create_mmap_offset(struct drm_gem_object *obj);
diff --git a/drivers/gpu/drm/gma500/intel_bios.c b/drivers/gpu/drm/gma500/intel_bios.c
index 8d7caf0f363..403fffb03ab 100644
--- a/drivers/gpu/drm/gma500/intel_bios.c
+++ b/drivers/gpu/drm/gma500/intel_bios.c
@@ -20,7 +20,7 @@
  */
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_intel_drv.h"
 #include "psb_intel_reg.h"
@@ -54,6 +54,98 @@ static void *find_section(struct bdb_header *bdb, int section_id)
 	return NULL;
 }
 
+static void
+parse_edp(struct drm_psb_private *dev_priv, struct bdb_header *bdb)
+{
+	struct bdb_edp *edp;
+	struct edp_power_seq *edp_pps;
+	struct edp_link_params *edp_link_params;
+	uint8_t	panel_type;
+
+	edp = find_section(bdb, BDB_EDP);
+	
+	dev_priv->edp.bpp = 18;
+	if (!edp) {
+		if (dev_priv->edp.support) {
+			DRM_DEBUG_KMS("No eDP BDB found but eDP panel supported, assume %dbpp panel color depth.\n",
+				      dev_priv->edp.bpp);
+		}
+		return;
+	}
+
+	panel_type = dev_priv->panel_type;
+	switch ((edp->color_depth >> (panel_type * 2)) & 3) {
+	case EDP_18BPP:
+		dev_priv->edp.bpp = 18;
+		break;
+	case EDP_24BPP:
+		dev_priv->edp.bpp = 24;
+		break;
+	case EDP_30BPP:
+		dev_priv->edp.bpp = 30;
+		break;
+	}
+
+	/* Get the eDP sequencing and link info */
+	edp_pps = &edp->power_seqs[panel_type];
+	edp_link_params = &edp->link_params[panel_type];
+
+	dev_priv->edp.pps = *edp_pps;
+
+	DRM_DEBUG_KMS("EDP timing in vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
+				dev_priv->edp.pps.t1_t3, dev_priv->edp.pps.t8, 
+				dev_priv->edp.pps.t9, dev_priv->edp.pps.t10,
+				dev_priv->edp.pps.t11_t12);
+
+	dev_priv->edp.rate = edp_link_params->rate ? DP_LINK_BW_2_7 :
+		DP_LINK_BW_1_62;
+	switch (edp_link_params->lanes) {
+	case 0:
+		dev_priv->edp.lanes = 1;
+		break;
+	case 1:
+		dev_priv->edp.lanes = 2;
+		break;
+	case 3:
+	default:
+		dev_priv->edp.lanes = 4;
+		break;
+	}
+	DRM_DEBUG_KMS("VBT reports EDP: Lane_count %d, Lane_rate %d, Bpp %d\n",
+			dev_priv->edp.lanes, dev_priv->edp.rate, dev_priv->edp.bpp);
+
+	switch (edp_link_params->preemphasis) {
+	case 0:
+		dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_0;
+		break;
+	case 1:
+		dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5;
+		break;
+	case 2:
+		dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_6;
+		break;
+	case 3:
+		dev_priv->edp.preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5;
+		break;
+	}
+	switch (edp_link_params->vswing) {
+	case 0:
+		dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_400;
+		break;
+	case 1:
+		dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_600;
+		break;
+	case 2:
+		dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_800;
+		break;
+	case 3:
+		dev_priv->edp.vswing = DP_TRAIN_VOLTAGE_SWING_1200;
+		break;
+	}
+	DRM_DEBUG_KMS("VBT reports EDP: VSwing  %d, Preemph %d\n",
+			dev_priv->edp.vswing, dev_priv->edp.preemphasis);
+}
+
 static u16
 get_blocksize(void *p)
 {
@@ -154,6 +246,8 @@ static void parse_lfp_panel_data(struct drm_psb_private *dev_priv,
 		return;
 
 	dev_priv->lvds_dither = lvds_options->pixel_dither;
+	dev_priv->panel_type = lvds_options->panel_type;
+
 	if (lvds_options->panel_type == 0xff)
 		return;
 
@@ -340,6 +434,9 @@ parse_driver_features(struct drm_psb_private *dev_priv,
 	if (!driver)
 		return;
 
+	if (driver->lvds_config == BDB_DRIVER_FEATURE_EDP)
+		dev_priv->edp.support = 1;
+
 	/* This bit means to use 96Mhz for DPLL_A or not */
 	if (driver->primary_lfp_id)
 		dev_priv->dplla_96mhz = true;
@@ -437,6 +534,9 @@ int psb_intel_init_bios(struct drm_device *dev)
 	size_t size;
 	int i;
 
+
+	dev_priv->panel_type = 0xff;
+
 	/* XXX Should this validation be moved to intel_opregion.c? */
 	if (dev_priv->opregion.vbt) {
 		struct vbt_header *vbt = dev_priv->opregion.vbt;
@@ -477,6 +577,7 @@ int psb_intel_init_bios(struct drm_device *dev)
 	parse_sdvo_device_mapping(dev_priv, bdb);
 	parse_device_mapping(dev_priv, bdb);
 	parse_backlight_data(dev_priv, bdb);
+	parse_edp(dev_priv, bdb);
 
 	if (bios)
 		pci_unmap_rom(pdev, bios);
diff --git a/drivers/gpu/drm/gma500/intel_bios.h b/drivers/gpu/drm/gma500/intel_bios.h
index 2e95523b84b..c6267c98c9e 100644
--- a/drivers/gpu/drm/gma500/intel_bios.h
+++ b/drivers/gpu/drm/gma500/intel_bios.h
@@ -23,6 +23,7 @@
 #define _I830_BIOS_H_
 
 #include <drm/drmP.h>
+#include <drm/drm_dp_helper.h>
 
 struct vbt_header {
 	u8 signature[20];		/**< Always starts with 'VBT$' */
@@ -93,6 +94,7 @@ struct vbios_data {
 #define BDB_SDVO_LVDS_PNP_IDS	 24
 #define BDB_SDVO_LVDS_POWER_SEQ	 25
 #define BDB_TV_OPTIONS		 26
+#define BDB_EDP			 27
 #define BDB_LVDS_OPTIONS	 40
 #define BDB_LVDS_LFP_DATA_PTRS	 41
 #define BDB_LVDS_LFP_DATA	 42
@@ -391,6 +393,11 @@ struct bdb_sdvo_lvds_options {
 	u8 panel_misc_bits_4;
 } __attribute__((packed));
 
+#define BDB_DRIVER_FEATURE_NO_LVDS		0
+#define BDB_DRIVER_FEATURE_INT_LVDS		1
+#define BDB_DRIVER_FEATURE_SDVO_LVDS		2
+#define BDB_DRIVER_FEATURE_EDP			3
+
 struct bdb_driver_features {
 	u8 boot_dev_algorithm:1;
 	u8 block_display_switch:1;
@@ -431,6 +438,45 @@ struct bdb_driver_features {
 	u8 custom_vbt_version;
 } __attribute__((packed));
 
+#define EDP_18BPP	0
+#define EDP_24BPP	1
+#define EDP_30BPP	2
+#define EDP_RATE_1_62	0
+#define EDP_RATE_2_7	1
+#define EDP_LANE_1	0
+#define EDP_LANE_2	1
+#define EDP_LANE_4	3
+#define EDP_PREEMPHASIS_NONE	0
+#define EDP_PREEMPHASIS_3_5dB	1
+#define EDP_PREEMPHASIS_6dB	2
+#define EDP_PREEMPHASIS_9_5dB	3
+#define EDP_VSWING_0_4V		0
+#define EDP_VSWING_0_6V		1
+#define EDP_VSWING_0_8V		2
+#define EDP_VSWING_1_2V		3
+
+struct edp_power_seq {
+	u16 t1_t3;
+	u16 t8;
+	u16 t9;
+	u16 t10;
+	u16 t11_t12;
+} __attribute__ ((packed));
+
+struct edp_link_params {
+	u8 rate:4;
+	u8 lanes:4;
+	u8 preemphasis:4;
+	u8 vswing:4;
+} __attribute__ ((packed));
+
+struct bdb_edp {
+	struct edp_power_seq power_seqs[16];
+	u32 color_depth;
+	u32 sdrrs_msa_timing_delay;
+	struct edp_link_params link_params[16];
+} __attribute__ ((packed));
+
 extern int psb_intel_init_bios(struct drm_device *dev);
 extern void psb_intel_destroy_bios(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/gma500/intel_gmbus.c b/drivers/gpu/drm/gma500/intel_gmbus.c
index 9db90527bf0..62cd42e88f2 100644
--- a/drivers/gpu/drm/gma500/intel_gmbus.c
+++ b/drivers/gpu/drm/gma500/intel_gmbus.c
@@ -29,10 +29,9 @@
 #include <linux/module.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "psb_intel_drv.h"
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_intel_reg.h"
 
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
index 5675d93b420..32dba2ab53e 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
@@ -299,17 +299,8 @@ static int mdfld_dsi_connector_set_property(struct drm_connector *connector,
 		if (drm_connector_property_set_value(connector, property,
 									value))
 			goto set_prop_error;
-		else {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-			struct backlight_device *psb_bd;
-
-			psb_bd = mdfld_get_backlight_device();
-			if (psb_bd) {
-				psb_bd->props.brightness = value;
-				mdfld_set_brightness(psb_bd);
-			}
-#endif
-		}
+		else
+			gma_backlight_set(encoder->dev, value);
 	}
 set_prop_done:
 	return 0;
diff --git a/drivers/gpu/drm/gma500/mid_bios.c b/drivers/gpu/drm/gma500/mid_bios.c
index b2a790bd989..a97e38e284f 100644
--- a/drivers/gpu/drm/gma500/mid_bios.c
+++ b/drivers/gpu/drm/gma500/mid_bios.c
@@ -25,7 +25,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "mid_bios.h"
 
@@ -118,20 +118,20 @@ static void mid_get_pci_revID(struct drm_psb_private *dev_priv)
 					dev_priv->platform_rev_id);
 }
 
-struct vbt_header {
+struct mid_vbt_header {
 	u32 signature;
 	u8 revision;
 } __packed;
 
 /* The same for r0 and r1 */
 struct vbt_r0 {
-	struct vbt_header vbt_header;
+	struct mid_vbt_header vbt_header;
 	u8 size;
 	u8 checksum;
 } __packed;
 
 struct vbt_r10 {
-	struct vbt_header vbt_header;
+	struct mid_vbt_header vbt_header;
 	u8 checksum;
 	u16 size;
 	u8 panel_count;
@@ -281,7 +281,7 @@ static void mid_get_vbt_data(struct drm_psb_private *dev_priv)
 	struct drm_device *dev = dev_priv->dev;
 	u32 addr;
 	u8 __iomem *vbt_virtual;
-	struct vbt_header vbt_header;
+	struct mid_vbt_header vbt_header;
 	struct pci_dev *pci_gfx_root = pci_get_bus_and_slot(0, PCI_DEVFN(2, 0));
 	int ret = -1;
 
diff --git a/drivers/gpu/drm/gma500/oaktrail_device.c b/drivers/gpu/drm/gma500/oaktrail_device.c
index cf49ba5a54b..010b820744a 100644
--- a/drivers/gpu/drm/gma500/oaktrail_device.c
+++ b/drivers/gpu/drm/gma500/oaktrail_device.c
@@ -22,7 +22,7 @@
 #include <linux/dmi.h>
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_reg.h"
 #include "psb_intel_reg.h"
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 2eb3dc4e9c9..69e51e903f3 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -252,7 +252,6 @@ static int oaktrail_hdmi_get_modes(struct drm_connector *connector)
 	if (edid) {
 		drm_mode_connector_update_edid_property(connector, edid);
 		ret = drm_add_edid_modes(connector, edid);
-		connector->display_info.raw_edid = NULL;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/gma500/opregion.c b/drivers/gpu/drm/gma500/opregion.c
index c430bd42468..ad0d6de938f 100644
--- a/drivers/gpu/drm/gma500/opregion.c
+++ b/drivers/gpu/drm/gma500/opregion.c
@@ -166,8 +166,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 
 	if (config_enabled(CONFIG_BACKLIGHT_CLASS_DEVICE)) {
 		int max = bd->props.max_brightness;
-		bd->props.brightness = bclp * max / 255;
-		backlight_update_status(bd);
+		gma_backlight_set(dev, bclp * max / 255);
 	}
 
 	asle->cblv = (bclp * 0x64) / 0xff | ASLE_CBLV_VALID;
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index 5971bc82b76..b58c4701c4e 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -20,7 +20,7 @@
 #include <linux/backlight.h>
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_reg.h"
 #include "psb_intel_reg.h"
@@ -290,6 +290,7 @@ static void psb_get_core_freq(struct drm_device *dev)
 	case 6:
 	case 7:
 		dev_priv->core_freq = 266;
+		break;
 	default:
 		dev_priv->core_freq = 0;
 	}
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 0c473743853..dd1fbfa7e46 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -21,7 +21,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "framebuffer.h"
 #include "psb_reg.h"
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 1bd115ecefe..a7fd6c48b79 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -23,11 +23,11 @@
 #include <linux/kref.h>
 
 #include <drm/drmP.h>
-#include "drm_global.h"
-#include "gem_glue.h"
-#include "gma_drm.h"
+#include <drm/drm_global.h>
+#include <drm/gma_drm.h>
 #include "psb_reg.h"
 #include "psb_intel_drv.h"
+#include "intel_bios.h"
 #include "gtt.h"
 #include "power.h"
 #include "opregion.h"
@@ -613,6 +613,8 @@ struct drm_psb_private {
 	 */
 	struct backlight_device *backlight_device;
 	struct drm_property *backlight_property;
+	bool backlight_enabled;
+	int backlight_level;
 	uint32_t blc_adj1;
 	uint32_t blc_adj2;
 
@@ -640,6 +642,19 @@ struct drm_psb_private {
 	int mdfld_panel_id;
 
 	bool dplla_96mhz;	/* DPLL data from the VBT */
+
+	struct {
+		int rate;
+		int lanes;
+		int preemphasis;
+		int vswing;
+
+		bool initialized;
+		bool support;
+		int bpp;
+		struct edp_power_seq pps;
+	} edp;
+	uint8_t panel_type;
 };
 
 
@@ -796,6 +811,9 @@ extern int psb_fbdev_init(struct drm_device *dev);
 /* backlight.c */
 int gma_backlight_init(struct drm_device *dev);
 void gma_backlight_exit(struct drm_device *dev);
+void gma_backlight_disable(struct drm_device *dev);
+void gma_backlight_enable(struct drm_device *dev);
+void gma_backlight_set(struct drm_device *dev, int v);
 
 /* oaktrail_crtc.c */
 extern const struct drm_crtc_helper_funcs oaktrail_helper_funcs;
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index ebe1a28f60e..90f2d11e686 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -29,10 +29,6 @@
  * Display related stuff
  */
 
-/* store information about an Ixxx DVO */
-/* The i830->i865 use multiple DVOs with multiple i2cs */
-/* the i915, i945 have a single sDVO i2c bus - which is different */
-#define MAX_OUTPUTS 6
 /* maximum connectors per crtcs in the mode set */
 #define INTELFB_CONN_LIMIT 4
 
@@ -69,6 +65,8 @@
 #define INTEL_OUTPUT_HDMI 6
 #define INTEL_OUTPUT_MIPI 7
 #define INTEL_OUTPUT_MIPI2 8
+#define INTEL_OUTPUT_DISPLAYPORT 9
+#define INTEL_OUTPUT_EDP 10
 
 #define INTEL_DVO_CHIP_NONE 0
 #define INTEL_DVO_CHIP_LVDS 1
@@ -133,6 +131,11 @@ struct psb_intel_encoder {
 	void (*hot_plug)(struct psb_intel_encoder *);
 	int crtc_mask;
 	int clone_mask;
+	u32 ddi_select;	/* Channel info */
+#define DDI0_SELECT	0x01
+#define DDI1_SELECT	0x02
+#define DP_MASK		0x8000
+#define DDI_MASK	0x03
 	void *dev_priv; /* For sdvo_priv, lvds_priv, etc... */
 
 	/* FIXME: Either make SDVO and LVDS store it's i2c here or give CDV it's
@@ -190,7 +193,6 @@ struct psb_intel_crtc {
 	u32 mode_flags;
 
 	bool active;
-	bool crtc_enable;
 
 	/* Saved Crtc HW states */
 	struct psb_intel_crtc_state *crtc_state;
@@ -285,4 +287,20 @@ extern void gma_intel_gmbus_set_speed(struct i2c_adapter *adapter, int speed);
 extern void gma_intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit);
 extern void gma_intel_teardown_gmbus(struct drm_device *dev);
 
+/* DP support */
+extern void cdv_intel_dp_init(struct drm_device *dev, struct psb_intel_mode_device *mode_dev, int output_reg);
+extern void cdv_intel_dp_set_m_n(struct drm_crtc *crtc,
+					struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode);
+
+extern void psb_intel_attach_force_audio_property(struct drm_connector *connector);
+extern void psb_intel_attach_broadcast_rgb_property(struct drm_connector *connector);
+
+extern int cdv_sb_read(struct drm_device *dev, u32 reg, u32 *val);
+extern int cdv_sb_write(struct drm_device *dev, u32 reg, u32 val);
+extern void cdv_sb_reset(struct drm_device *dev);
+
+extern void cdv_intel_attach_force_audio_property(struct drm_connector *connector);
+extern void cdv_intel_attach_broadcast_rgb_property(struct drm_connector *connector);
+
 #endif				/* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index 37adc9edf97..2a4c3a9e33e 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -630,17 +630,8 @@ int psb_intel_lvds_set_property(struct drm_connector *connector,
 							property,
 							value))
 			goto set_prop_error;
-		else {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-			struct drm_psb_private *devp =
-						encoder->dev->dev_private;
-			struct backlight_device *bd = devp->backlight_device;
-			if (bd) {
-				bd->props.brightness = value;
-				backlight_update_status(bd);
-			}
-#endif
-		}
+		else
+                        gma_backlight_set(encoder->dev, value);
 	} else if (!strcmp(property->name, "DPMS")) {
 		struct drm_encoder_helper_funcs *hfuncs
 						= encoder->helper_private;
diff --git a/drivers/gpu/drm/gma500/psb_intel_reg.h b/drivers/gpu/drm/gma500/psb_intel_reg.h
index 8e8c8efb0a8..d914719c4b6 100644
--- a/drivers/gpu/drm/gma500/psb_intel_reg.h
+++ b/drivers/gpu/drm/gma500/psb_intel_reg.h
@@ -173,15 +173,46 @@
 #define PP_SEQUENCE_ON			(1 << 28)
 #define PP_SEQUENCE_OFF			(2 << 28)
 #define PP_SEQUENCE_MASK		0x30000000
+#define	PP_CYCLE_DELAY_ACTIVE		(1 << 27)
+#define	PP_SEQUENCE_STATE_ON_IDLE	(1 << 3)
+#define	PP_SEQUENCE_STATE_MASK		0x0000000f
+
 #define PP_CONTROL		0x61204
 #define POWER_TARGET_ON			(1 << 0)
-
+#define	PANEL_UNLOCK_REGS		(0xabcd << 16)
+#define	PANEL_UNLOCK_MASK		(0xffff << 16)
+#define	EDP_FORCE_VDD			(1 << 3)
+#define	EDP_BLC_ENABLE			(1 << 2)
+#define	PANEL_POWER_RESET		(1 << 1)
+#define	PANEL_POWER_OFF			(0 << 0)
+#define	PANEL_POWER_ON			(1 << 0)
+
+/* Poulsbo/Oaktrail */
 #define LVDSPP_ON		0x61208
 #define LVDSPP_OFF		0x6120c
 #define PP_CYCLE		0x61210
 
+/* Cedartrail */
 #define PP_ON_DELAYS		0x61208		/* Cedartrail */
+#define PANEL_PORT_SELECT_MASK 		(3 << 30)
+#define PANEL_PORT_SELECT_LVDS 		(0 << 30)
+#define PANEL_PORT_SELECT_EDP		(1 << 30)
+#define PANEL_POWER_UP_DELAY_MASK	(0x1fff0000)
+#define PANEL_POWER_UP_DELAY_SHIFT	16
+#define PANEL_LIGHT_ON_DELAY_MASK	(0x1fff)
+#define PANEL_LIGHT_ON_DELAY_SHIFT	0
+
 #define PP_OFF_DELAYS		0x6120c		/* Cedartrail */
+#define PANEL_POWER_DOWN_DELAY_MASK	(0x1fff0000)
+#define PANEL_POWER_DOWN_DELAY_SHIFT	16
+#define PANEL_LIGHT_OFF_DELAY_MASK	(0x1fff)
+#define PANEL_LIGHT_OFF_DELAY_SHIFT	0
+
+#define PP_DIVISOR		0x61210		/* Cedartrail */
+#define  PP_REFERENCE_DIVIDER_MASK	(0xffffff00)
+#define  PP_REFERENCE_DIVIDER_SHIFT	8
+#define  PANEL_POWER_CYCLE_DELAY_MASK	(0x1f)
+#define  PANEL_POWER_CYCLE_DELAY_SHIFT	0
 
 #define PFIT_CONTROL		0x61230
 #define PFIT_ENABLE			(1 << 31)
@@ -1282,6 +1313,10 @@ No status bits are changed.
 # define VRHUNIT_CLOCK_GATE_DISABLE		(1 << 28) /* Fixed value on CDV */
 # define DPOUNIT_CLOCK_GATE_DISABLE		(1 << 11)
 # define DPIOUNIT_CLOCK_GATE_DISABLE		(1 << 6)
+# define DPUNIT_PIPEB_GATE_DISABLE		(1 << 30)
+# define DPUNIT_PIPEA_GATE_DISABLE		(1 << 25)
+# define DPCUNIT_CLOCK_GATE_DISABLE		(1 << 24)
+# define DPLSUNIT_CLOCK_GATE_DISABLE		(1 << 13)
 
 #define RAMCLK_GATE_D		0x6210
 
@@ -1347,5 +1382,165 @@ No status bits are changed.
 #define LANE_PLL_ENABLE		(0x3 << 20)
 #define LANE_PLL_PIPE(p)	(((p) == 0) ? (1 << 21) : (0 << 21))
 
+#define DP_B				0x64100
+#define DP_C				0x64200
+
+#define   DP_PORT_EN			(1 << 31)
+#define   DP_PIPEB_SELECT		(1 << 30)
+#define   DP_PIPE_MASK			(1 << 30)
+
+/* Link training mode - select a suitable mode for each stage */
+#define   DP_LINK_TRAIN_PAT_1		(0 << 28)
+#define   DP_LINK_TRAIN_PAT_2		(1 << 28)
+#define   DP_LINK_TRAIN_PAT_IDLE	(2 << 28)
+#define   DP_LINK_TRAIN_OFF		(3 << 28)
+#define   DP_LINK_TRAIN_MASK		(3 << 28)
+#define   DP_LINK_TRAIN_SHIFT		28
+
+/* Signal voltages. These are mostly controlled by the other end */
+#define   DP_VOLTAGE_0_4		(0 << 25)
+#define   DP_VOLTAGE_0_6		(1 << 25)
+#define   DP_VOLTAGE_0_8		(2 << 25)
+#define   DP_VOLTAGE_1_2		(3 << 25)
+#define   DP_VOLTAGE_MASK		(7 << 25)
+#define   DP_VOLTAGE_SHIFT		25
+
+/* Signal pre-emphasis levels, like voltages, the other end tells us what
+ * they want
+ */
+#define   DP_PRE_EMPHASIS_0		(0 << 22)
+#define   DP_PRE_EMPHASIS_3_5		(1 << 22)
+#define   DP_PRE_EMPHASIS_6		(2 << 22)
+#define   DP_PRE_EMPHASIS_9_5		(3 << 22)
+#define   DP_PRE_EMPHASIS_MASK		(7 << 22)
+#define   DP_PRE_EMPHASIS_SHIFT		22
+
+/* How many wires to use. I guess 3 was too hard */
+#define   DP_PORT_WIDTH_1		(0 << 19)
+#define   DP_PORT_WIDTH_2		(1 << 19)
+#define   DP_PORT_WIDTH_4		(3 << 19)
+#define   DP_PORT_WIDTH_MASK		(7 << 19)
+
+/* Mystic DPCD version 1.1 special mode */
+#define   DP_ENHANCED_FRAMING		(1 << 18)
+
+/** locked once port is enabled */
+#define   DP_PORT_REVERSAL		(1 << 15)
+
+/** sends the clock on lane 15 of the PEG for debug */
+#define   DP_CLOCK_OUTPUT_ENABLE	(1 << 13)
+
+#define   DP_SCRAMBLING_DISABLE		(1 << 12)
+#define   DP_SCRAMBLING_DISABLE_IRONLAKE	(1 << 7)
+
+/** limit RGB values to avoid confusing TVs */
+#define   DP_COLOR_RANGE_16_235		(1 << 8)
+
+/** Turn on the audio link */
+#define   DP_AUDIO_OUTPUT_ENABLE	(1 << 6)
+
+/** vs and hs sync polarity */
+#define   DP_SYNC_VS_HIGH		(1 << 4)
+#define   DP_SYNC_HS_HIGH		(1 << 3)
+
+/** A fantasy */
+#define   DP_DETECTED			(1 << 2)
+
+/** The aux channel provides a way to talk to the
+ * signal sink for DDC etc. Max packet size supported
+ * is 20 bytes in each direction, hence the 5 fixed
+ * data registers
+ */
+#define DPB_AUX_CH_CTL			0x64110
+#define DPB_AUX_CH_DATA1		0x64114
+#define DPB_AUX_CH_DATA2		0x64118
+#define DPB_AUX_CH_DATA3		0x6411c
+#define DPB_AUX_CH_DATA4		0x64120
+#define DPB_AUX_CH_DATA5		0x64124
+
+#define DPC_AUX_CH_CTL			0x64210
+#define DPC_AUX_CH_DATA1		0x64214
+#define DPC_AUX_CH_DATA2		0x64218
+#define DPC_AUX_CH_DATA3		0x6421c
+#define DPC_AUX_CH_DATA4		0x64220
+#define DPC_AUX_CH_DATA5		0x64224
+
+#define   DP_AUX_CH_CTL_SEND_BUSY	    (1 << 31)
+#define   DP_AUX_CH_CTL_DONE		    (1 << 30)
+#define   DP_AUX_CH_CTL_INTERRUPT	    (1 << 29)
+#define   DP_AUX_CH_CTL_TIME_OUT_ERROR	    (1 << 28)
+#define   DP_AUX_CH_CTL_TIME_OUT_400us	    (0 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_600us	    (1 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_800us	    (2 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_1600us	    (3 << 26)
+#define   DP_AUX_CH_CTL_TIME_OUT_MASK	    (3 << 26)
+#define   DP_AUX_CH_CTL_RECEIVE_ERROR	    (1 << 25)
+#define   DP_AUX_CH_CTL_MESSAGE_SIZE_MASK    (0x1f << 20)
+#define   DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT   20
+#define   DP_AUX_CH_CTL_PRECHARGE_2US_MASK   (0xf << 16)
+#define   DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT  16
+#define   DP_AUX_CH_CTL_AUX_AKSV_SELECT	    (1 << 15)
+#define   DP_AUX_CH_CTL_MANCHESTER_TEST	    (1 << 14)
+#define   DP_AUX_CH_CTL_SYNC_TEST	    (1 << 13)
+#define   DP_AUX_CH_CTL_DEGLITCH_TEST	    (1 << 12)
+#define   DP_AUX_CH_CTL_PRECHARGE_TEST	    (1 << 11)
+#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_MASK    (0x7ff)
+#define   DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT   0
+
+/*
+ * Computing GMCH M and N values for the Display Port link
+ *
+ * GMCH M/N = dot clock * bytes per pixel / ls_clk * # of lanes
+ *
+ * ls_clk (we assume) is the DP link clock (1.62 or 2.7 GHz)
+ *
+ * The GMCH value is used internally
+ *
+ * bytes_per_pixel is the number of bytes coming out of the plane,
+ * which is after the LUTs, so we want the bytes for our color format.
+ * For our current usage, this is always 3, one byte for R, G and B.
+ */
+
+#define _PIPEA_GMCH_DATA_M			0x70050
+#define _PIPEB_GMCH_DATA_M			0x71050
+
+/* Transfer unit size for display port - 1, default is 0x3f (for TU size 64) */
+#define   PIPE_GMCH_DATA_M_TU_SIZE_MASK		(0x3f << 25)
+#define   PIPE_GMCH_DATA_M_TU_SIZE_SHIFT	25
+
+#define   PIPE_GMCH_DATA_M_MASK			(0xffffff)
+
+#define _PIPEA_GMCH_DATA_N			0x70054
+#define _PIPEB_GMCH_DATA_N			0x71054
+#define   PIPE_GMCH_DATA_N_MASK			(0xffffff)
+
+/*
+ * Computing Link M and N values for the Display Port link
+ *
+ * Link M / N = pixel_clock / ls_clk
+ *
+ * (the DP spec calls pixel_clock the 'strm_clk')
+ *
+ * The Link value is transmitted in the Main Stream
+ * Attributes and VB-ID.
+ */
+
+#define _PIPEA_DP_LINK_M				0x70060
+#define _PIPEB_DP_LINK_M				0x71060
+#define   PIPEA_DP_LINK_M_MASK			(0xffffff)
+
+#define _PIPEA_DP_LINK_N				0x70064
+#define _PIPEB_DP_LINK_N				0x71064
+#define   PIPEA_DP_LINK_N_MASK			(0xffffff)
+
+#define PIPE_GMCH_DATA_M(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_M, _PIPEB_GMCH_DATA_M)
+#define PIPE_GMCH_DATA_N(pipe) _PIPE(pipe, _PIPEA_GMCH_DATA_N, _PIPEB_GMCH_DATA_N)
+#define PIPE_DP_LINK_M(pipe) _PIPE(pipe, _PIPEA_DP_LINK_M, _PIPEB_DP_LINK_M)
+#define PIPE_DP_LINK_N(pipe) _PIPE(pipe, _PIPEA_DP_LINK_N, _PIPEB_DP_LINK_N)
+
+#define   PIPE_BPC_MASK				(7 << 5)
+#define   PIPE_8BPC				(0 << 5)
+#define   PIPE_10BPC				(1 << 5)
+#define   PIPE_6BPC				(2 << 5)
 
 #endif
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index 0466c7b985f..fc9292705db 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -29,12 +29,11 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
 #include "psb_intel_drv.h"
-#include "gma_drm.h"
+#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "psb_intel_sdvo_regs.h"
 #include "psb_intel_reg.h"
@@ -1292,7 +1291,6 @@ psb_intel_sdvo_get_analog_edid(struct drm_connector *connector)
 
 	return drm_get_edid(connector,
 			    &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter);
-	return NULL;
 }
 
 static enum drm_connector_status
@@ -1343,7 +1341,6 @@ psb_intel_sdvo_hdmi_sink_detect(struct drm_connector *connector)
 			}
 		} else
 			status = connector_status_disconnected;
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 
@@ -1404,7 +1401,6 @@ psb_intel_sdvo_detect(struct drm_connector *connector, bool force)
 				ret = connector_status_disconnected;
 			else
 				ret = connector_status_connected;
-			connector->display_info.raw_edid = NULL;
 			kfree(edid);
 		} else
 			ret = connector_status_connected;
@@ -1453,7 +1449,6 @@ static void psb_intel_sdvo_get_ddc_modes(struct drm_connector *connector)
 			drm_add_edid_modes(connector, edid);
 		}
 
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 }
diff --git a/drivers/gpu/drm/i2c/ch7006_drv.c b/drivers/gpu/drm/i2c/ch7006_drv.c
index 36d952280c5..599099fe76e 100644
--- a/drivers/gpu/drm/i2c/ch7006_drv.c
+++ b/drivers/gpu/drm/i2c/ch7006_drv.c
@@ -427,15 +427,10 @@ static int ch7006_remove(struct i2c_client *client)
 	return 0;
 }
 
-static int ch7006_suspend(struct i2c_client *client, pm_message_t mesg)
+static int ch7006_resume(struct device *dev)
 {
-	ch7006_dbg(client, "\n");
-
-	return 0;
-}
+	struct i2c_client *client = to_i2c_client(dev);
 
-static int ch7006_resume(struct i2c_client *client)
-{
 	ch7006_dbg(client, "\n");
 
 	ch7006_write(client, 0x3d, 0x0);
@@ -499,15 +494,18 @@ static struct i2c_device_id ch7006_ids[] = {
 };
 MODULE_DEVICE_TABLE(i2c, ch7006_ids);
 
+static const struct dev_pm_ops ch7006_pm_ops = {
+	.resume = ch7006_resume,
+};
+
 static struct drm_i2c_encoder_driver ch7006_driver = {
 	.i2c_driver = {
 		.probe = ch7006_probe,
 		.remove = ch7006_remove,
-		.suspend = ch7006_suspend,
-		.resume = ch7006_resume,
 
 		.driver = {
 			.name = "ch7006",
+			.pm = &ch7006_pm_ops,
 		},
 
 		.id_table = ch7006_ids,
diff --git a/drivers/gpu/drm/i2c/ch7006_priv.h b/drivers/gpu/drm/i2c/ch7006_priv.h
index 09599f4c0c9..ce577841f93 100644
--- a/drivers/gpu/drm/i2c/ch7006_priv.h
+++ b/drivers/gpu/drm/i2c/ch7006_priv.h
@@ -27,10 +27,10 @@
 #ifndef __DRM_I2C_CH7006_PRIV_H__
 #define __DRM_I2C_CH7006_PRIV_H__
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "drm_encoder_slave.h"
-#include "i2c/ch7006.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder_slave.h>
+#include <drm/i2c/ch7006.h>
 
 typedef int64_t fixed;
 #define fixed1 (1LL << 32)
diff --git a/drivers/gpu/drm/i2c/sil164_drv.c b/drivers/gpu/drm/i2c/sil164_drv.c
index 30b8ae5e5c4..002ce787433 100644
--- a/drivers/gpu/drm/i2c/sil164_drv.c
+++ b/drivers/gpu/drm/i2c/sil164_drv.c
@@ -26,10 +26,10 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "drm_encoder_slave.h"
-#include "i2c/sil164.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder_slave.h>
+#include <drm/i2c/sil164.h>
 
 struct sil164_priv {
 	struct sil164_encoder_params config;
diff --git a/drivers/gpu/drm/i810/i810_dma.c b/drivers/gpu/drm/i810/i810_dma.c
index 463ec6871fe..004ecdfe1b5 100644
--- a/drivers/gpu/drm/i810/i810_dma.c
+++ b/drivers/gpu/drm/i810/i810_dma.c
@@ -30,9 +30,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i810_drm.h"
+#include <drm/drmP.h>
+#include <drm/i810_drm.h>
 #include "i810_drv.h"
 #include <linux/interrupt.h>	/* For task queue support */
 #include <linux/delay.h>
diff --git a/drivers/gpu/drm/i810/i810_drv.c b/drivers/gpu/drm/i810/i810_drv.c
index 48cfcca2b35..2e91fc3580b 100644
--- a/drivers/gpu/drm/i810/i810_drv.c
+++ b/drivers/gpu/drm/i810/i810_drv.c
@@ -32,12 +32,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "i810_drm.h"
+#include <drm/drmP.h>
+#include <drm/i810_drm.h>
 #include "i810_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static struct pci_device_id pciidlist[] = {
 	i810_PCI_IDS
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index b0bacdba6d7..0f2c5493242 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -40,6 +40,7 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o \
 	  dvo_ivch.o \
 	  dvo_tfp410.o \
 	  dvo_sil164.o \
+	  dvo_ns2501.o \
 	  i915_gem_dmabuf.o
 
 i915-$(CONFIG_COMPAT)   += i915_ioc32.o
diff --git a/drivers/gpu/drm/i915/dvo.h b/drivers/gpu/drm/i915/dvo.h
index 58914691a77..33a62ad8010 100644
--- a/drivers/gpu/drm/i915/dvo.h
+++ b/drivers/gpu/drm/i915/dvo.h
@@ -24,9 +24,8 @@
 #define _INTEL_DVO_H
 
 #include <linux/i2c.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
 #include "intel_drv.h"
 
 struct intel_dvo_device {
@@ -58,13 +57,12 @@ struct intel_dvo_dev_ops {
 	void (*create_resources)(struct intel_dvo_device *dvo);
 
 	/*
-	 * Turn on/off output or set intermediate power levels if available.
+	 * Turn on/off output.
 	 *
-	 * Unsupported intermediate modes drop to the lower power setting.
-	 * If the  mode is DPMSModeOff, the output must be disabled,
-	 * as the DPLL may be disabled afterwards.
+	 * Because none of our dvo drivers support an intermediate power levels,
+	 * we don't expose this in the interfac.
 	 */
-	void (*dpms)(struct intel_dvo_device *dvo, int mode);
+	void (*dpms)(struct intel_dvo_device *dvo, bool enable);
 
 	/*
 	 * Callback for testing a video mode for a given output.
@@ -115,6 +113,12 @@ struct intel_dvo_dev_ops {
 	 */
 	enum drm_connector_status (*detect)(struct intel_dvo_device *dvo);
 
+	/*
+	 * Probe the current hw status, returning true if the connected output
+	 * is active.
+	 */
+	bool (*get_hw_state)(struct intel_dvo_device *dev);
+
 	/**
 	 * Query the device for the modes it provides.
 	 *
@@ -140,5 +144,6 @@ extern struct intel_dvo_dev_ops ch7xxx_ops;
 extern struct intel_dvo_dev_ops ivch_ops;
 extern struct intel_dvo_dev_ops tfp410_ops;
 extern struct intel_dvo_dev_ops ch7017_ops;
+extern struct intel_dvo_dev_ops ns2501_ops;
 
 #endif /* _INTEL_DVO_H */
diff --git a/drivers/gpu/drm/i915/dvo_ch7017.c b/drivers/gpu/drm/i915/dvo_ch7017.c
index 1ca799a1e1f..86b27d1d90c 100644
--- a/drivers/gpu/drm/i915/dvo_ch7017.c
+++ b/drivers/gpu/drm/i915/dvo_ch7017.c
@@ -163,7 +163,7 @@ struct ch7017_priv {
 };
 
 static void ch7017_dump_regs(struct intel_dvo_device *dvo);
-static void ch7017_dpms(struct intel_dvo_device *dvo, int mode);
+static void ch7017_dpms(struct intel_dvo_device *dvo, bool enable);
 
 static bool ch7017_read(struct intel_dvo_device *dvo, u8 addr, u8 *val)
 {
@@ -309,7 +309,7 @@ static void ch7017_mode_set(struct intel_dvo_device *dvo,
 	lvds_power_down = CH7017_LVDS_POWER_DOWN_DEFAULT_RESERVED |
 			  (mode->hdisplay & 0x0700) >> 8;
 
-	ch7017_dpms(dvo, DRM_MODE_DPMS_OFF);
+	ch7017_dpms(dvo, false);
 	ch7017_write(dvo, CH7017_HORIZONTAL_ACTIVE_PIXEL_INPUT,
 			horizontal_active_pixel_input);
 	ch7017_write(dvo, CH7017_HORIZONTAL_ACTIVE_PIXEL_OUTPUT,
@@ -331,7 +331,7 @@ static void ch7017_mode_set(struct intel_dvo_device *dvo,
 }
 
 /* set the CH7017 power state */
-static void ch7017_dpms(struct intel_dvo_device *dvo, int mode)
+static void ch7017_dpms(struct intel_dvo_device *dvo, bool enable)
 {
 	uint8_t val;
 
@@ -345,7 +345,7 @@ static void ch7017_dpms(struct intel_dvo_device *dvo, int mode)
 			CH7017_DAC3_POWER_DOWN |
 			CH7017_TV_POWER_DOWN_EN);
 
-	if (mode == DRM_MODE_DPMS_ON) {
+	if (enable) {
 		/* Turn on the LVDS */
 		ch7017_write(dvo, CH7017_LVDS_POWER_DOWN,
 			     val & ~CH7017_LVDS_POWER_DOWN_EN);
@@ -359,6 +359,18 @@ static void ch7017_dpms(struct intel_dvo_device *dvo, int mode)
 	msleep(20);
 }
 
+static bool ch7017_get_hw_state(struct intel_dvo_device *dvo)
+{
+	uint8_t val;
+
+	ch7017_read(dvo, CH7017_LVDS_POWER_DOWN, &val);
+
+	if (val & CH7017_LVDS_POWER_DOWN_EN)
+		return false;
+	else
+		return true;
+}
+
 static void ch7017_dump_regs(struct intel_dvo_device *dvo)
 {
 	uint8_t val;
@@ -396,6 +408,7 @@ struct intel_dvo_dev_ops ch7017_ops = {
 	.mode_valid = ch7017_mode_valid,
 	.mode_set = ch7017_mode_set,
 	.dpms = ch7017_dpms,
+	.get_hw_state = ch7017_get_hw_state,
 	.dump_regs = ch7017_dump_regs,
 	.destroy = ch7017_destroy,
 };
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index 4a036600e80..38f3a6cb8c7 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -289,14 +289,26 @@ static void ch7xxx_mode_set(struct intel_dvo_device *dvo,
 }
 
 /* set the CH7xxx power state */
-static void ch7xxx_dpms(struct intel_dvo_device *dvo, int mode)
+static void ch7xxx_dpms(struct intel_dvo_device *dvo, bool enable)
 {
-	if (mode == DRM_MODE_DPMS_ON)
+	if (enable)
 		ch7xxx_writeb(dvo, CH7xxx_PM, CH7xxx_PM_DVIL | CH7xxx_PM_DVIP);
 	else
 		ch7xxx_writeb(dvo, CH7xxx_PM, CH7xxx_PM_FPD);
 }
 
+static bool ch7xxx_get_hw_state(struct intel_dvo_device *dvo)
+{
+	u8 val;
+
+	ch7xxx_readb(dvo, CH7xxx_PM, &val);
+
+	if (val & CH7xxx_PM_FPD)
+		return false;
+	else
+		return true;
+}
+
 static void ch7xxx_dump_regs(struct intel_dvo_device *dvo)
 {
 	int i;
@@ -326,6 +338,7 @@ struct intel_dvo_dev_ops ch7xxx_ops = {
 	.mode_valid = ch7xxx_mode_valid,
 	.mode_set = ch7xxx_mode_set,
 	.dpms = ch7xxx_dpms,
+	.get_hw_state = ch7xxx_get_hw_state,
 	.dump_regs = ch7xxx_dump_regs,
 	.destroy = ch7xxx_destroy,
 };
diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c
index 04f2893d5e3..baaf65bf0bd 100644
--- a/drivers/gpu/drm/i915/dvo_ivch.c
+++ b/drivers/gpu/drm/i915/dvo_ivch.c
@@ -288,7 +288,7 @@ static enum drm_mode_status ivch_mode_valid(struct intel_dvo_device *dvo,
 }
 
 /** Sets the power state of the panel connected to the ivch */
-static void ivch_dpms(struct intel_dvo_device *dvo, int mode)
+static void ivch_dpms(struct intel_dvo_device *dvo, bool enable)
 {
 	int i;
 	uint16_t vr01, vr30, backlight;
@@ -297,13 +297,13 @@ static void ivch_dpms(struct intel_dvo_device *dvo, int mode)
 	if (!ivch_read(dvo, VR01, &vr01))
 		return;
 
-	if (mode == DRM_MODE_DPMS_ON)
+	if (enable)
 		backlight = 1;
 	else
 		backlight = 0;
 	ivch_write(dvo, VR80, backlight);
 
-	if (mode == DRM_MODE_DPMS_ON)
+	if (enable)
 		vr01 |= VR01_LCD_ENABLE | VR01_DVO_ENABLE;
 	else
 		vr01 &= ~(VR01_LCD_ENABLE | VR01_DVO_ENABLE);
@@ -315,7 +315,7 @@ static void ivch_dpms(struct intel_dvo_device *dvo, int mode)
 		if (!ivch_read(dvo, VR30, &vr30))
 			break;
 
-		if (((vr30 & VR30_PANEL_ON) != 0) == (mode == DRM_MODE_DPMS_ON))
+		if (((vr30 & VR30_PANEL_ON) != 0) == enable)
 			break;
 		udelay(1000);
 	}
@@ -323,6 +323,20 @@ static void ivch_dpms(struct intel_dvo_device *dvo, int mode)
 	udelay(16 * 1000);
 }
 
+static bool ivch_get_hw_state(struct intel_dvo_device *dvo)
+{
+	uint16_t vr01;
+
+	/* Set the new power state of the panel. */
+	if (!ivch_read(dvo, VR01, &vr01))
+		return false;
+
+	if (vr01 & VR01_LCD_ENABLE)
+		return true;
+	else
+		return false;
+}
+
 static void ivch_mode_set(struct intel_dvo_device *dvo,
 			  struct drm_display_mode *mode,
 			  struct drm_display_mode *adjusted_mode)
@@ -413,6 +427,7 @@ static void ivch_destroy(struct intel_dvo_device *dvo)
 struct intel_dvo_dev_ops ivch_ops = {
 	.init = ivch_init,
 	.dpms = ivch_dpms,
+	.get_hw_state = ivch_get_hw_state,
 	.mode_valid = ivch_mode_valid,
 	.mode_set = ivch_mode_set,
 	.detect = ivch_detect,
diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c
new file mode 100644
index 00000000000..c4a255be697
--- /dev/null
+++ b/drivers/gpu/drm/i915/dvo_ns2501.c
@@ -0,0 +1,588 @@
+/*
+ *
+ * Copyright (c) 2012 Gilles Dartiguelongue, Thomas Richter
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "dvo.h"
+#include "i915_reg.h"
+#include "i915_drv.h"
+
+#define NS2501_VID 0x1305
+#define NS2501_DID 0x6726
+
+#define NS2501_VID_LO 0x00
+#define NS2501_VID_HI 0x01
+#define NS2501_DID_LO 0x02
+#define NS2501_DID_HI 0x03
+#define NS2501_REV 0x04
+#define NS2501_RSVD 0x05
+#define NS2501_FREQ_LO 0x06
+#define NS2501_FREQ_HI 0x07
+
+#define NS2501_REG8 0x08
+#define NS2501_8_VEN (1<<5)
+#define NS2501_8_HEN (1<<4)
+#define NS2501_8_DSEL (1<<3)
+#define NS2501_8_BPAS (1<<2)
+#define NS2501_8_RSVD (1<<1)
+#define NS2501_8_PD (1<<0)
+
+#define NS2501_REG9 0x09
+#define NS2501_9_VLOW (1<<7)
+#define NS2501_9_MSEL_MASK (0x7<<4)
+#define NS2501_9_TSEL (1<<3)
+#define NS2501_9_RSEN (1<<2)
+#define NS2501_9_RSVD (1<<1)
+#define NS2501_9_MDI (1<<0)
+
+#define NS2501_REGC 0x0c
+
+struct ns2501_priv {
+	//I2CDevRec d;
+	bool quiet;
+	int reg_8_shadow;
+	int reg_8_set;
+	// Shadow registers for i915
+	int dvoc;
+	int pll_a;
+	int srcdim;
+	int fw_blc;
+};
+
+#define NSPTR(d) ((NS2501Ptr)(d->DriverPrivate.ptr))
+
+/*
+ * For reasons unclear to me, the ns2501 at least on the Fujitsu/Siemens
+ * laptops does not react on the i2c bus unless
+ * both the PLL is running and the display is configured in its native
+ * resolution.
+ * This function forces the DVO on, and stores the registers it touches.
+ * Afterwards, registers are restored to regular values.
+ *
+ * This is pretty much a hack, though it works.
+ * Without that, ns2501_readb and ns2501_writeb fail
+ * when switching the resolution.
+ */
+
+static void enable_dvo(struct intel_dvo_device *dvo)
+{
+	struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
+	struct i2c_adapter *adapter = dvo->i2c_bus;
+	struct intel_gmbus *bus = container_of(adapter,
+					       struct intel_gmbus,
+					       adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+
+	DRM_DEBUG_KMS("%s: Trying to re-enable the DVO\n", __FUNCTION__);
+
+	ns->dvoc = I915_READ(DVO_C);
+	ns->pll_a = I915_READ(_DPLL_A);
+	ns->srcdim = I915_READ(DVOC_SRCDIM);
+	ns->fw_blc = I915_READ(FW_BLC);
+
+	I915_WRITE(DVOC, 0x10004084);
+	I915_WRITE(_DPLL_A, 0xd0820000);
+	I915_WRITE(DVOC_SRCDIM, 0x400300);	// 1024x768
+	I915_WRITE(FW_BLC, 0x1080304);
+
+	I915_WRITE(DVOC, 0x90004084);
+}
+
+/*
+ * Restore the I915 registers modified by the above
+ * trigger function.
+ */
+static void restore_dvo(struct intel_dvo_device *dvo)
+{
+	struct i2c_adapter *adapter = dvo->i2c_bus;
+	struct intel_gmbus *bus = container_of(adapter,
+					       struct intel_gmbus,
+					       adapter);
+	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
+
+	I915_WRITE(DVOC, ns->dvoc);
+	I915_WRITE(_DPLL_A, ns->pll_a);
+	I915_WRITE(DVOC_SRCDIM, ns->srcdim);
+	I915_WRITE(FW_BLC, ns->fw_blc);
+}
+
+/*
+** Read a register from the ns2501.
+** Returns true if successful, false otherwise.
+** If it returns false, it might be wise to enable the
+** DVO with the above function.
+*/
+static bool ns2501_readb(struct intel_dvo_device *dvo, int addr, uint8_t * ch)
+{
+	struct ns2501_priv *ns = dvo->dev_priv;
+	struct i2c_adapter *adapter = dvo->i2c_bus;
+	u8 out_buf[2];
+	u8 in_buf[2];
+
+	struct i2c_msg msgs[] = {
+		{
+		 .addr = dvo->slave_addr,
+		 .flags = 0,
+		 .len = 1,
+		 .buf = out_buf,
+		 },
+		{
+		 .addr = dvo->slave_addr,
+		 .flags = I2C_M_RD,
+		 .len = 1,
+		 .buf = in_buf,
+		 }
+	};
+
+	out_buf[0] = addr;
+	out_buf[1] = 0;
+
+	if (i2c_transfer(adapter, msgs, 2) == 2) {
+		*ch = in_buf[0];
+		return true;
+	};
+
+	if (!ns->quiet) {
+		DRM_DEBUG_KMS
+		    ("Unable to read register 0x%02x from %s:0x%02x.\n", addr,
+		     adapter->name, dvo->slave_addr);
+	}
+
+	return false;
+}
+
+/*
+** Write a register to the ns2501.
+** Returns true if successful, false otherwise.
+** If it returns false, it might be wise to enable the
+** DVO with the above function.
+*/
+static bool ns2501_writeb(struct intel_dvo_device *dvo, int addr, uint8_t ch)
+{
+	struct ns2501_priv *ns = dvo->dev_priv;
+	struct i2c_adapter *adapter = dvo->i2c_bus;
+	uint8_t out_buf[2];
+
+	struct i2c_msg msg = {
+		.addr = dvo->slave_addr,
+		.flags = 0,
+		.len = 2,
+		.buf = out_buf,
+	};
+
+	out_buf[0] = addr;
+	out_buf[1] = ch;
+
+	if (i2c_transfer(adapter, &msg, 1) == 1) {
+		return true;
+	}
+
+	if (!ns->quiet) {
+		DRM_DEBUG_KMS("Unable to write register 0x%02x to %s:%d\n",
+			      addr, adapter->name, dvo->slave_addr);
+	}
+
+	return false;
+}
+
+/* National Semiconductor 2501 driver for chip on i2c bus
+ * scan for the chip on the bus.
+ * Hope the VBIOS initialized the PLL correctly so we can
+ * talk to it. If not, it will not be seen and not detected.
+ * Bummer!
+ */
+static bool ns2501_init(struct intel_dvo_device *dvo,
+			struct i2c_adapter *adapter)
+{
+	/* this will detect the NS2501 chip on the specified i2c bus */
+	struct ns2501_priv *ns;
+	unsigned char ch;
+
+	ns = kzalloc(sizeof(struct ns2501_priv), GFP_KERNEL);
+	if (ns == NULL)
+		return false;
+
+	dvo->i2c_bus = adapter;
+	dvo->dev_priv = ns;
+	ns->quiet = true;
+
+	if (!ns2501_readb(dvo, NS2501_VID_LO, &ch))
+		goto out;
+
+	if (ch != (NS2501_VID & 0xff)) {
+		DRM_DEBUG_KMS("ns2501 not detected got %d: from %s Slave %d.\n",
+			      ch, adapter->name, dvo->slave_addr);
+		goto out;
+	}
+
+	if (!ns2501_readb(dvo, NS2501_DID_LO, &ch))
+		goto out;
+
+	if (ch != (NS2501_DID & 0xff)) {
+		DRM_DEBUG_KMS("ns2501 not detected got %d: from %s Slave %d.\n",
+			      ch, adapter->name, dvo->slave_addr);
+		goto out;
+	}
+	ns->quiet = false;
+	ns->reg_8_set = 0;
+	ns->reg_8_shadow =
+	    NS2501_8_PD | NS2501_8_BPAS | NS2501_8_VEN | NS2501_8_HEN;
+
+	DRM_DEBUG_KMS("init ns2501 dvo controller successfully!\n");
+	return true;
+
+out:
+	kfree(ns);
+	return false;
+}
+
+static enum drm_connector_status ns2501_detect(struct intel_dvo_device *dvo)
+{
+	/*
+	 * This is a Laptop display, it doesn't have hotplugging.
+	 * Even if not, the detection bit of the 2501 is unreliable as
+	 * it only works for some display types.
+	 * It is even more unreliable as the PLL must be active for
+	 * allowing reading from the chiop.
+	 */
+	return connector_status_connected;
+}
+
+static enum drm_mode_status ns2501_mode_valid(struct intel_dvo_device *dvo,
+					      struct drm_display_mode *mode)
+{
+	DRM_DEBUG_KMS
+	    ("%s: is mode valid (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d)\n",
+	     __FUNCTION__, mode->hdisplay, mode->htotal, mode->vdisplay,
+	     mode->vtotal);
+
+	/*
+	 * Currently, these are all the modes I have data from.
+	 * More might exist. Unclear how to find the native resolution
+	 * of the panel in here so we could always accept it
+	 * by disabling the scaler.
+	 */
+	if ((mode->hdisplay == 800 && mode->vdisplay == 600) ||
+	    (mode->hdisplay == 640 && mode->vdisplay == 480) ||
+	    (mode->hdisplay == 1024 && mode->vdisplay == 768)) {
+		return MODE_OK;
+	} else {
+		return MODE_ONE_SIZE;	/* Is this a reasonable error? */
+	}
+}
+
+static void ns2501_mode_set(struct intel_dvo_device *dvo,
+			    struct drm_display_mode *mode,
+			    struct drm_display_mode *adjusted_mode)
+{
+	bool ok;
+	bool restore = false;
+	struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
+
+	DRM_DEBUG_KMS
+	    ("%s: set mode (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d).\n",
+	     __FUNCTION__, mode->hdisplay, mode->htotal, mode->vdisplay,
+	     mode->vtotal);
+
+	/*
+	 * Where do I find the native resolution for which scaling is not required???
+	 *
+	 * First trigger the DVO on as otherwise the chip does not appear on the i2c
+	 * bus.
+	 */
+	do {
+		ok = true;
+
+		if (mode->hdisplay == 800 && mode->vdisplay == 600) {
+			/* mode 277 */
+			ns->reg_8_shadow &= ~NS2501_8_BPAS;
+			DRM_DEBUG_KMS("%s: switching to 800x600\n",
+				      __FUNCTION__);
+
+			/*
+			 * No, I do not know where this data comes from.
+			 * It is just what the video bios left in the DVO, so
+			 * I'm just copying it here over.
+			 * This also means that I cannot support any other modes
+			 * except the ones supported by the bios.
+			 */
+			ok &= ns2501_writeb(dvo, 0x11, 0xc8);	// 0xc7 also works.
+			ok &= ns2501_writeb(dvo, 0x1b, 0x19);
+			ok &= ns2501_writeb(dvo, 0x1c, 0x62);	// VBIOS left 0x64 here, but 0x62 works nicer
+			ok &= ns2501_writeb(dvo, 0x1d, 0x02);
+
+			ok &= ns2501_writeb(dvo, 0x34, 0x03);
+			ok &= ns2501_writeb(dvo, 0x35, 0xff);
+
+			ok &= ns2501_writeb(dvo, 0x80, 0x27);
+			ok &= ns2501_writeb(dvo, 0x81, 0x03);
+			ok &= ns2501_writeb(dvo, 0x82, 0x41);
+			ok &= ns2501_writeb(dvo, 0x83, 0x05);
+
+			ok &= ns2501_writeb(dvo, 0x8d, 0x02);
+			ok &= ns2501_writeb(dvo, 0x8e, 0x04);
+			ok &= ns2501_writeb(dvo, 0x8f, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0x90, 0xfe);	/* vertical. VBIOS left 0xff here, but 0xfe works better */
+			ok &= ns2501_writeb(dvo, 0x91, 0x07);
+			ok &= ns2501_writeb(dvo, 0x94, 0x00);
+			ok &= ns2501_writeb(dvo, 0x95, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0x96, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0x99, 0x00);
+			ok &= ns2501_writeb(dvo, 0x9a, 0x88);
+
+			ok &= ns2501_writeb(dvo, 0x9c, 0x23);	/* Looks like first and last line of the image. */
+			ok &= ns2501_writeb(dvo, 0x9d, 0x00);
+			ok &= ns2501_writeb(dvo, 0x9e, 0x25);
+			ok &= ns2501_writeb(dvo, 0x9f, 0x03);
+
+			ok &= ns2501_writeb(dvo, 0xa4, 0x80);
+
+			ok &= ns2501_writeb(dvo, 0xb6, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0xb9, 0xc8);	/* horizontal? */
+			ok &= ns2501_writeb(dvo, 0xba, 0x00);	/* horizontal? */
+
+			ok &= ns2501_writeb(dvo, 0xc0, 0x05);	/* horizontal? */
+			ok &= ns2501_writeb(dvo, 0xc1, 0xd7);
+
+			ok &= ns2501_writeb(dvo, 0xc2, 0x00);
+			ok &= ns2501_writeb(dvo, 0xc3, 0xf8);
+
+			ok &= ns2501_writeb(dvo, 0xc4, 0x03);
+			ok &= ns2501_writeb(dvo, 0xc5, 0x1a);
+
+			ok &= ns2501_writeb(dvo, 0xc6, 0x00);
+			ok &= ns2501_writeb(dvo, 0xc7, 0x73);
+			ok &= ns2501_writeb(dvo, 0xc8, 0x02);
+
+		} else if (mode->hdisplay == 640 && mode->vdisplay == 480) {
+			/* mode 274 */
+			DRM_DEBUG_KMS("%s: switching to 640x480\n",
+				      __FUNCTION__);
+			/*
+			 * No, I do not know where this data comes from.
+			 * It is just what the video bios left in the DVO, so
+			 * I'm just copying it here over.
+			 * This also means that I cannot support any other modes
+			 * except the ones supported by the bios.
+			 */
+			ns->reg_8_shadow &= ~NS2501_8_BPAS;
+
+			ok &= ns2501_writeb(dvo, 0x11, 0xa0);
+			ok &= ns2501_writeb(dvo, 0x1b, 0x11);
+			ok &= ns2501_writeb(dvo, 0x1c, 0x54);
+			ok &= ns2501_writeb(dvo, 0x1d, 0x03);
+
+			ok &= ns2501_writeb(dvo, 0x34, 0x03);
+			ok &= ns2501_writeb(dvo, 0x35, 0xff);
+
+			ok &= ns2501_writeb(dvo, 0x80, 0xff);
+			ok &= ns2501_writeb(dvo, 0x81, 0x07);
+			ok &= ns2501_writeb(dvo, 0x82, 0x3d);
+			ok &= ns2501_writeb(dvo, 0x83, 0x05);
+
+			ok &= ns2501_writeb(dvo, 0x8d, 0x02);
+			ok &= ns2501_writeb(dvo, 0x8e, 0x10);
+			ok &= ns2501_writeb(dvo, 0x8f, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0x90, 0xff);	/* vertical */
+			ok &= ns2501_writeb(dvo, 0x91, 0x07);
+			ok &= ns2501_writeb(dvo, 0x94, 0x00);
+			ok &= ns2501_writeb(dvo, 0x95, 0x00);
+
+			ok &= ns2501_writeb(dvo, 0x96, 0x05);
+
+			ok &= ns2501_writeb(dvo, 0x99, 0x00);
+			ok &= ns2501_writeb(dvo, 0x9a, 0x88);
+
+			ok &= ns2501_writeb(dvo, 0x9c, 0x24);
+			ok &= ns2501_writeb(dvo, 0x9d, 0x00);
+			ok &= ns2501_writeb(dvo, 0x9e, 0x25);
+			ok &= ns2501_writeb(dvo, 0x9f, 0x03);
+
+			ok &= ns2501_writeb(dvo, 0xa4, 0x84);
+
+			ok &= ns2501_writeb(dvo, 0xb6, 0x09);
+
+			ok &= ns2501_writeb(dvo, 0xb9, 0xa0);	/* horizontal? */
+			ok &= ns2501_writeb(dvo, 0xba, 0x00);	/* horizontal? */
+
+			ok &= ns2501_writeb(dvo, 0xc0, 0x05);	/* horizontal? */
+			ok &= ns2501_writeb(dvo, 0xc1, 0x90);
+
+			ok &= ns2501_writeb(dvo, 0xc2, 0x00);
+			ok &= ns2501_writeb(dvo, 0xc3, 0x0f);
+
+			ok &= ns2501_writeb(dvo, 0xc4, 0x03);
+			ok &= ns2501_writeb(dvo, 0xc5, 0x16);
+
+			ok &= ns2501_writeb(dvo, 0xc6, 0x00);
+			ok &= ns2501_writeb(dvo, 0xc7, 0x02);
+			ok &= ns2501_writeb(dvo, 0xc8, 0x02);
+
+		} else if (mode->hdisplay == 1024 && mode->vdisplay == 768) {
+			/* mode 280 */
+			DRM_DEBUG_KMS("%s: switching to 1024x768\n",
+				      __FUNCTION__);
+			/*
+			 * This might or might not work, actually. I'm silently
+			 * assuming here that the native panel resolution is
+			 * 1024x768. If not, then this leaves the scaler disabled
+			 * generating a picture that is likely not the expected.
+			 *
+			 * Problem is that I do not know where to take the panel
+			 * dimensions from.
+			 *
+			 * Enable the bypass, scaling not required.
+			 *
+			 * The scaler registers are irrelevant here....
+			 *
+			 */
+			ns->reg_8_shadow |= NS2501_8_BPAS;
+			ok &= ns2501_writeb(dvo, 0x37, 0x44);
+		} else {
+			/*
+			 * Data not known. Bummer!
+			 * Hopefully, the code should not go here
+			 * as mode_OK delivered no other modes.
+			 */
+			ns->reg_8_shadow |= NS2501_8_BPAS;
+		}
+		ok &= ns2501_writeb(dvo, NS2501_REG8, ns->reg_8_shadow);
+
+		if (!ok) {
+			if (restore)
+				restore_dvo(dvo);
+			enable_dvo(dvo);
+			restore = true;
+		}
+	} while (!ok);
+	/*
+	 * Restore the old i915 registers before
+	 * forcing the ns2501 on.
+	 */
+	if (restore)
+		restore_dvo(dvo);
+}
+
+/* set the NS2501 power state */
+static bool ns2501_get_hw_state(struct intel_dvo_device *dvo)
+{
+	unsigned char ch;
+
+	if (!ns2501_readb(dvo, NS2501_REG8, &ch))
+		return false;
+
+	if (ch & NS2501_8_PD)
+		return true;
+	else
+		return false;
+}
+
+/* set the NS2501 power state */
+static void ns2501_dpms(struct intel_dvo_device *dvo, bool enable)
+{
+	bool ok;
+	bool restore = false;
+	struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
+	unsigned char ch;
+
+	DRM_DEBUG_KMS("%s: Trying set the dpms of the DVO to %i\n",
+		      __FUNCTION__, enable);
+
+	ch = ns->reg_8_shadow;
+
+	if (enable)
+		ch |= NS2501_8_PD;
+	else
+		ch &= ~NS2501_8_PD;
+
+	if (ns->reg_8_set == 0 || ns->reg_8_shadow != ch) {
+		ns->reg_8_set = 1;
+		ns->reg_8_shadow = ch;
+
+		do {
+			ok = true;
+			ok &= ns2501_writeb(dvo, NS2501_REG8, ch);
+			ok &=
+			    ns2501_writeb(dvo, 0x34,
+					  enable ? 0x03 : 0x00);
+			ok &=
+			    ns2501_writeb(dvo, 0x35,
+					  enable ? 0xff : 0x00);
+			if (!ok) {
+				if (restore)
+					restore_dvo(dvo);
+				enable_dvo(dvo);
+				restore = true;
+			}
+		} while (!ok);
+
+		if (restore)
+			restore_dvo(dvo);
+	}
+}
+
+static void ns2501_dump_regs(struct intel_dvo_device *dvo)
+{
+	uint8_t val;
+
+	ns2501_readb(dvo, NS2501_FREQ_LO, &val);
+	DRM_LOG_KMS("NS2501_FREQ_LO: 0x%02x\n", val);
+	ns2501_readb(dvo, NS2501_FREQ_HI, &val);
+	DRM_LOG_KMS("NS2501_FREQ_HI: 0x%02x\n", val);
+	ns2501_readb(dvo, NS2501_REG8, &val);
+	DRM_LOG_KMS("NS2501_REG8: 0x%02x\n", val);
+	ns2501_readb(dvo, NS2501_REG9, &val);
+	DRM_LOG_KMS("NS2501_REG9: 0x%02x\n", val);
+	ns2501_readb(dvo, NS2501_REGC, &val);
+	DRM_LOG_KMS("NS2501_REGC: 0x%02x\n", val);
+}
+
+static void ns2501_destroy(struct intel_dvo_device *dvo)
+{
+	struct ns2501_priv *ns = dvo->dev_priv;
+
+	if (ns) {
+		kfree(ns);
+		dvo->dev_priv = NULL;
+	}
+}
+
+struct intel_dvo_dev_ops ns2501_ops = {
+	.init = ns2501_init,
+	.detect = ns2501_detect,
+	.mode_valid = ns2501_mode_valid,
+	.mode_set = ns2501_mode_set,
+	.dpms = ns2501_dpms,
+	.get_hw_state = ns2501_get_hw_state,
+	.dump_regs = ns2501_dump_regs,
+	.destroy = ns2501_destroy,
+};
diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c
index a0b13a6f619..4debd32e3e4 100644
--- a/drivers/gpu/drm/i915/dvo_sil164.c
+++ b/drivers/gpu/drm/i915/dvo_sil164.c
@@ -208,7 +208,7 @@ static void sil164_mode_set(struct intel_dvo_device *dvo,
 }
 
 /* set the SIL164 power state */
-static void sil164_dpms(struct intel_dvo_device *dvo, int mode)
+static void sil164_dpms(struct intel_dvo_device *dvo, bool enable)
 {
 	int ret;
 	unsigned char ch;
@@ -217,7 +217,7 @@ static void sil164_dpms(struct intel_dvo_device *dvo, int mode)
 	if (ret == false)
 		return;
 
-	if (mode == DRM_MODE_DPMS_ON)
+	if (enable)
 		ch |= SIL164_8_PD;
 	else
 		ch &= ~SIL164_8_PD;
@@ -226,6 +226,21 @@ static void sil164_dpms(struct intel_dvo_device *dvo, int mode)
 	return;
 }
 
+static bool sil164_get_hw_state(struct intel_dvo_device *dvo)
+{
+	int ret;
+	unsigned char ch;
+
+	ret = sil164_readb(dvo, SIL164_REG8, &ch);
+	if (ret == false)
+		return false;
+
+	if (ch & SIL164_8_PD)
+		return true;
+	else
+		return false;
+}
+
 static void sil164_dump_regs(struct intel_dvo_device *dvo)
 {
 	uint8_t val;
@@ -258,6 +273,7 @@ struct intel_dvo_dev_ops sil164_ops = {
 	.mode_valid = sil164_mode_valid,
 	.mode_set = sil164_mode_set,
 	.dpms = sil164_dpms,
+	.get_hw_state = sil164_get_hw_state,
 	.dump_regs = sil164_dump_regs,
 	.destroy = sil164_destroy,
 };
diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c
index aa2cd3ec54a..e17f1b07e91 100644
--- a/drivers/gpu/drm/i915/dvo_tfp410.c
+++ b/drivers/gpu/drm/i915/dvo_tfp410.c
@@ -234,14 +234,14 @@ static void tfp410_mode_set(struct intel_dvo_device *dvo,
 }
 
 /* set the tfp410 power state */
-static void tfp410_dpms(struct intel_dvo_device *dvo, int mode)
+static void tfp410_dpms(struct intel_dvo_device *dvo, bool enable)
 {
 	uint8_t ctl1;
 
 	if (!tfp410_readb(dvo, TFP410_CTL_1, &ctl1))
 		return;
 
-	if (mode == DRM_MODE_DPMS_ON)
+	if (enable)
 		ctl1 |= TFP410_CTL_1_PD;
 	else
 		ctl1 &= ~TFP410_CTL_1_PD;
@@ -249,6 +249,19 @@ static void tfp410_dpms(struct intel_dvo_device *dvo, int mode)
 	tfp410_writeb(dvo, TFP410_CTL_1, ctl1);
 }
 
+static bool tfp410_get_hw_state(struct intel_dvo_device *dvo)
+{
+	uint8_t ctl1;
+
+	if (!tfp410_readb(dvo, TFP410_CTL_1, &ctl1))
+		return false;
+
+	if (ctl1 & TFP410_CTL_1_PD)
+		return true;
+	else
+		return false;
+}
+
 static void tfp410_dump_regs(struct intel_dvo_device *dvo)
 {
 	uint8_t val, val2;
@@ -299,6 +312,7 @@ struct intel_dvo_dev_ops tfp410_ops = {
 	.mode_valid = tfp410_mode_valid,
 	.mode_set = tfp410_mode_set,
 	.dpms = tfp410_dpms,
+	.get_hw_state = tfp410_get_hw_state,
 	.dump_regs = tfp410_dump_regs,
 	.destroy = tfp410_destroy,
 };
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 359f6e8b9b0..dde8b505bf7 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -30,11 +30,10 @@
 #include <linux/debugfs.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "intel_drv.h"
 #include "intel_ringbuffer.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 #define DRM_I915_RING_DEBUG 1
@@ -44,7 +43,6 @@
 
 enum {
 	ACTIVE_LIST,
-	FLUSHING_LIST,
 	INACTIVE_LIST,
 	PINNED_LIST,
 };
@@ -62,28 +60,11 @@ static int i915_capabilities(struct seq_file *m, void *data)
 
 	seq_printf(m, "gen: %d\n", info->gen);
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev));
-#define B(x) seq_printf(m, #x ": %s\n", yesno(info->x))
-	B(is_mobile);
-	B(is_i85x);
-	B(is_i915g);
-	B(is_i945gm);
-	B(is_g33);
-	B(need_gfx_hws);
-	B(is_g4x);
-	B(is_pineview);
-	B(is_broadwater);
-	B(is_crestline);
-	B(has_fbc);
-	B(has_pipe_cxsr);
-	B(has_hotplug);
-	B(cursor_needs_physical);
-	B(has_overlay);
-	B(overlay_needs_physical);
-	B(supports_tv);
-	B(has_bsd_ring);
-	B(has_blt_ring);
-	B(has_llc);
-#undef B
+#define DEV_INFO_FLAG(x) seq_printf(m, #x ": %s\n", yesno(info->x))
+#define DEV_INFO_SEP ;
+	DEV_INFO_FLAGS;
+#undef DEV_INFO_FLAG
+#undef DEV_INFO_SEP
 
 	return 0;
 }
@@ -121,20 +102,23 @@ static const char *cache_level_str(int type)
 static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
-	seq_printf(m, "%p: %s%s %8zdKiB %04x %04x %d %d%s%s%s",
+	seq_printf(m, "%p: %s%s %8zdKiB %04x %04x %d %d %d%s%s%s",
 		   &obj->base,
 		   get_pin_flag(obj),
 		   get_tiling_flag(obj),
 		   obj->base.size / 1024,
 		   obj->base.read_domains,
 		   obj->base.write_domain,
-		   obj->last_rendering_seqno,
+		   obj->last_read_seqno,
+		   obj->last_write_seqno,
 		   obj->last_fenced_seqno,
 		   cache_level_str(obj->cache_level),
 		   obj->dirty ? " dirty" : "",
 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		seq_printf(m, " (name: %d)", obj->base.name);
+	if (obj->pin_count)
+		seq_printf(m, " (pinned x %d)", obj->pin_count);
 	if (obj->fence_reg != I915_FENCE_REG_NONE)
 		seq_printf(m, " (fence: %d)", obj->fence_reg);
 	if (obj->gtt_space != NULL)
@@ -177,10 +161,6 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data)
 		seq_printf(m, "Inactive:\n");
 		head = &dev_priv->mm.inactive_list;
 		break;
-	case FLUSHING_LIST:
-		seq_printf(m, "Flushing:\n");
-		head = &dev_priv->mm.flushing_list;
-		break;
 	default:
 		mutex_unlock(&dev->struct_mutex);
 		return -EINVAL;
@@ -218,8 +198,8 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 count, mappable_count;
-	size_t size, mappable_size;
+	u32 count, mappable_count, purgeable_count;
+	size_t size, mappable_size, purgeable_size;
 	struct drm_i915_gem_object *obj;
 	int ret;
 
@@ -232,13 +212,12 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 		   dev_priv->mm.object_memory);
 
 	size = count = mappable_size = mappable_count = 0;
-	count_objects(&dev_priv->mm.gtt_list, gtt_list);
+	count_objects(&dev_priv->mm.bound_list, gtt_list);
 	seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n",
 		   count, mappable_count, size, mappable_size);
 
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.active_list, mm_list);
-	count_objects(&dev_priv->mm.flushing_list, mm_list);
 	seq_printf(m, "  %u [%u] active objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 
@@ -247,8 +226,16 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 	seq_printf(m, "  %u [%u] inactive objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 
+	size = count = purgeable_size = purgeable_count = 0;
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list) {
+		size += obj->base.size, ++count;
+		if (obj->madv == I915_MADV_DONTNEED)
+			purgeable_size += obj->base.size, ++purgeable_count;
+	}
+	seq_printf(m, "%u unbound objects, %zu bytes\n", count, size);
+
 	size = count = mappable_size = mappable_count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 		if (obj->fault_mappable) {
 			size += obj->gtt_space->size;
 			++count;
@@ -257,7 +244,13 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 			mappable_size += obj->gtt_space->size;
 			++mappable_count;
 		}
+		if (obj->madv == I915_MADV_DONTNEED) {
+			purgeable_size += obj->base.size;
+			++purgeable_count;
+		}
 	}
+	seq_printf(m, "%u purgeable objects, %zu bytes\n",
+		   purgeable_count, purgeable_size);
 	seq_printf(m, "%u pinned mappable objects, %zu bytes\n",
 		   mappable_count, mappable_size);
 	seq_printf(m, "%u fault mappable objects, %zu bytes\n",
@@ -286,7 +279,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void* data)
 		return ret;
 
 	total_obj_size = total_gtt_size = count = 0;
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 		if (list == PINNED_LIST && obj->pin_count == 0)
 			continue;
 
@@ -359,40 +352,22 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring;
 	struct drm_i915_gem_request *gem_request;
-	int ret, count;
+	int ret, count, i;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 
 	count = 0;
-	if (!list_empty(&dev_priv->ring[RCS].request_list)) {
-		seq_printf(m, "Render requests:\n");
-		list_for_each_entry(gem_request,
-				    &dev_priv->ring[RCS].request_list,
-				    list) {
-			seq_printf(m, "    %d @ %d\n",
-				   gem_request->seqno,
-				   (int) (jiffies - gem_request->emitted_jiffies));
-		}
-		count++;
-	}
-	if (!list_empty(&dev_priv->ring[VCS].request_list)) {
-		seq_printf(m, "BSD requests:\n");
-		list_for_each_entry(gem_request,
-				    &dev_priv->ring[VCS].request_list,
-				    list) {
-			seq_printf(m, "    %d @ %d\n",
-				   gem_request->seqno,
-				   (int) (jiffies - gem_request->emitted_jiffies));
-		}
-		count++;
-	}
-	if (!list_empty(&dev_priv->ring[BCS].request_list)) {
-		seq_printf(m, "BLT requests:\n");
+	for_each_ring(ring, dev_priv, i) {
+		if (list_empty(&ring->request_list))
+			continue;
+
+		seq_printf(m, "%s requests:\n", ring->name);
 		list_for_each_entry(gem_request,
-				    &dev_priv->ring[BCS].request_list,
+				    &ring->request_list,
 				    list) {
 			seq_printf(m, "    %d @ %d\n",
 				   gem_request->seqno,
@@ -413,7 +388,7 @@ static void i915_ring_seqno_info(struct seq_file *m,
 {
 	if (ring->get_seqno) {
 		seq_printf(m, "Current sequence (%s): %d\n",
-			   ring->name, ring->get_seqno(ring));
+			   ring->name, ring->get_seqno(ring, false));
 	}
 }
 
@@ -422,14 +397,15 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring;
 	int ret, i;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)
 		return ret;
 
-	for (i = 0; i < I915_NUM_RINGS; i++)
-		i915_ring_seqno_info(m, &dev_priv->ring[i]);
+	for_each_ring(ring, dev_priv, i)
+		i915_ring_seqno_info(m, ring);
 
 	mutex_unlock(&dev->struct_mutex);
 
@@ -442,6 +418,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring;
 	int ret, i, pipe;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -519,13 +496,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 	}
 	seq_printf(m, "Interrupts received: %d\n",
 		   atomic_read(&dev_priv->irq_received));
-	for (i = 0; i < I915_NUM_RINGS; i++) {
+	for_each_ring(ring, dev_priv, i) {
 		if (IS_GEN6(dev) || IS_GEN7(dev)) {
-			seq_printf(m, "Graphics Interrupt mask (%s):	%08x\n",
-				   dev_priv->ring[i].name,
-				   I915_READ_IMR(&dev_priv->ring[i]));
+			seq_printf(m,
+				   "Graphics Interrupt mask (%s):	%08x\n",
+				   ring->name, I915_READ_IMR(ring));
 		}
-		i915_ring_seqno_info(m, &dev_priv->ring[i]);
+		i915_ring_seqno_info(m, ring);
 	}
 	mutex_unlock(&dev->struct_mutex);
 
@@ -548,7 +525,8 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj;
 
-		seq_printf(m, "Fenced object[%2d] = ", i);
+		seq_printf(m, "Fence %d, pin count = %d, object = ",
+			   i, dev_priv->fence_regs[i].pin_count);
 		if (obj == NULL)
 			seq_printf(m, "unused");
 		else
@@ -630,12 +608,12 @@ static void print_error_buffers(struct seq_file *m,
 	seq_printf(m, "%s [%d]:\n", name, count);
 
 	while (count--) {
-		seq_printf(m, "  %08x %8u %04x %04x %08x%s%s%s%s%s%s%s",
+		seq_printf(m, "  %08x %8u %04x %04x %x %x%s%s%s%s%s%s%s",
 			   err->gtt_offset,
 			   err->size,
 			   err->read_domains,
 			   err->write_domain,
-			   err->seqno,
+			   err->rseqno, err->wseqno,
 			   pin_flag(err->pinned),
 			   tiling_flag(err->tiling),
 			   dirty_flag(err->dirty),
@@ -667,10 +645,9 @@ static void i915_ring_error_state(struct seq_file *m,
 	seq_printf(m, "  IPEIR: 0x%08x\n", error->ipeir[ring]);
 	seq_printf(m, "  IPEHR: 0x%08x\n", error->ipehr[ring]);
 	seq_printf(m, "  INSTDONE: 0x%08x\n", error->instdone[ring]);
-	if (ring == RCS && INTEL_INFO(dev)->gen >= 4) {
-		seq_printf(m, "  INSTDONE1: 0x%08x\n", error->instdone1);
+	if (ring == RCS && INTEL_INFO(dev)->gen >= 4)
 		seq_printf(m, "  BBADDR: 0x%08llx\n", error->bbaddr);
-	}
+
 	if (INTEL_INFO(dev)->gen >= 4)
 		seq_printf(m, "  INSTPS: 0x%08x\n", error->instps[ring]);
 	seq_printf(m, "  INSTPM: 0x%08x\n", error->instpm[ring]);
@@ -719,11 +696,17 @@ static int i915_error_state(struct seq_file *m, void *unused)
 	for (i = 0; i < dev_priv->num_fence_regs; i++)
 		seq_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);
 
+	for (i = 0; i < ARRAY_SIZE(error->extra_instdone); i++)
+		seq_printf(m, "  INSTDONE_%d: 0x%08x\n", i, error->extra_instdone[i]);
+
 	if (INTEL_INFO(dev)->gen >= 6) {
 		seq_printf(m, "ERROR: 0x%08x\n", error->error);
 		seq_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
 	}
 
+	if (INTEL_INFO(dev)->gen == 7)
+		seq_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
+
 	for_each_ring(ring, dev_priv, i)
 		i915_ring_error_state(m, dev, error, i);
 
@@ -799,10 +782,14 @@ i915_error_state_write(struct file *filp,
 	struct seq_file *m = filp->private_data;
 	struct i915_error_state_file_priv *error_priv = m->private;
 	struct drm_device *dev = error_priv->dev;
+	int ret;
 
 	DRM_DEBUG_DRIVER("Resetting error state\n");
 
-	mutex_lock(&dev->struct_mutex);
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
 	i915_destroy_error_state(dev);
 	mutex_unlock(&dev->struct_mutex);
 
@@ -926,7 +913,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
 		seq_printf(m, "Render p-state limit: %d\n",
 			   rp_state_limits & 0xff);
 		seq_printf(m, "CAGF: %dMHz\n", ((rpstat & GEN6_CAGF_MASK) >>
-						GEN6_CAGF_SHIFT) * 50);
+						GEN6_CAGF_SHIFT) * GT_FREQUENCY_MULTIPLIER);
 		seq_printf(m, "RP CUR UP EI: %dus\n", rpupei &
 			   GEN6_CURICONT_MASK);
 		seq_printf(m, "RP CUR UP: %dus\n", rpcurup &
@@ -942,15 +929,15 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
 
 		max_freq = (rp_state_cap & 0xff0000) >> 16;
 		seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
-			   max_freq * 50);
+			   max_freq * GT_FREQUENCY_MULTIPLIER);
 
 		max_freq = (rp_state_cap & 0xff00) >> 8;
 		seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
-			   max_freq * 50);
+			   max_freq * GT_FREQUENCY_MULTIPLIER);
 
 		max_freq = rp_state_cap & 0xff;
 		seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
-			   max_freq * 50);
+			   max_freq * GT_FREQUENCY_MULTIPLIER);
 	} else {
 		seq_printf(m, "no P-state info available\n");
 	}
@@ -1292,7 +1279,8 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 
 	seq_printf(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\n");
 
-	for (gpu_freq = dev_priv->min_delay; gpu_freq <= dev_priv->max_delay;
+	for (gpu_freq = dev_priv->rps.min_delay;
+	     gpu_freq <= dev_priv->rps.max_delay;
 	     gpu_freq++) {
 		I915_WRITE(GEN6_PCODE_DATA, gpu_freq);
 		I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
@@ -1303,7 +1291,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 			continue;
 		}
 		ia_freq = I915_READ(GEN6_PCODE_DATA);
-		seq_printf(m, "%d\t\t%d\n", gpu_freq * 50, ia_freq * 100);
+		seq_printf(m, "%d\t\t%d\n", gpu_freq * GT_FREQUENCY_MULTIPLIER, ia_freq * 100);
 	}
 
 	mutex_unlock(&dev->struct_mutex);
@@ -1472,8 +1460,12 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
 
-	mutex_lock(&dev->struct_mutex);
 	seq_printf(m, "bit6 swizzle for X-tiling = %s\n",
 		   swizzle_string(dev_priv->mm.bit_6_swizzle_x));
 	seq_printf(m, "bit6 swizzle for Y-tiling = %s\n",
@@ -1520,9 +1512,7 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
 	if (INTEL_INFO(dev)->gen == 6)
 		seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
 
-	for (i = 0; i < I915_NUM_RINGS; i++) {
-		ring = &dev_priv->ring[i];
-
+	for_each_ring(ring, dev_priv, i) {
 		seq_printf(m, "%s\n", ring->name);
 		if (INTEL_INFO(dev)->gen == 7)
 			seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(ring)));
@@ -1674,7 +1664,7 @@ i915_ring_stop_write(struct file *filp,
 	struct drm_device *dev = filp->private_data;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	char buf[20];
-	int val = 0;
+	int val = 0, ret;
 
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
@@ -1689,7 +1679,10 @@ i915_ring_stop_write(struct file *filp,
 
 	DRM_DEBUG_DRIVER("Stopping rings 0x%08x\n", val);
 
-	mutex_lock(&dev->struct_mutex);
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
 	dev_priv->stop_rings = val;
 	mutex_unlock(&dev->struct_mutex);
 
@@ -1713,10 +1706,18 @@ i915_max_freq_read(struct file *filp,
 	struct drm_device *dev = filp->private_data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
-	int len;
+	int len, ret;
+
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
 
 	len = snprintf(buf, sizeof(buf),
-		       "max freq: %d\n", dev_priv->max_delay * 50);
+		       "max freq: %d\n", dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER);
+	mutex_unlock(&dev->struct_mutex);
 
 	if (len > sizeof(buf))
 		len = sizeof(buf);
@@ -1733,7 +1734,10 @@ i915_max_freq_write(struct file *filp,
 	struct drm_device *dev = filp->private_data;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	char buf[20];
-	int val = 1;
+	int val = 1, ret;
+
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
 
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
@@ -1748,12 +1752,17 @@ i915_max_freq_write(struct file *filp,
 
 	DRM_DEBUG_DRIVER("Manually setting max freq to %d\n", val);
 
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
 	/*
 	 * Turbo will still be enabled, but won't go above the set value.
 	 */
-	dev_priv->max_delay = val / 50;
+	dev_priv->rps.max_delay = val / GT_FREQUENCY_MULTIPLIER;
 
-	gen6_set_rps(dev, val / 50);
+	gen6_set_rps(dev, val / GT_FREQUENCY_MULTIPLIER);
+	mutex_unlock(&dev->struct_mutex);
 
 	return cnt;
 }
@@ -1773,10 +1782,18 @@ i915_min_freq_read(struct file *filp, char __user *ubuf, size_t max,
 	struct drm_device *dev = filp->private_data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
-	int len;
+	int len, ret;
+
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
 
 	len = snprintf(buf, sizeof(buf),
-		       "min freq: %d\n", dev_priv->min_delay * 50);
+		       "min freq: %d\n", dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER);
+	mutex_unlock(&dev->struct_mutex);
 
 	if (len > sizeof(buf))
 		len = sizeof(buf);
@@ -1791,7 +1808,10 @@ i915_min_freq_write(struct file *filp, const char __user *ubuf, size_t cnt,
 	struct drm_device *dev = filp->private_data;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	char buf[20];
-	int val = 1;
+	int val = 1, ret;
+
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
 
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
@@ -1806,12 +1826,17 @@ i915_min_freq_write(struct file *filp, const char __user *ubuf, size_t cnt,
 
 	DRM_DEBUG_DRIVER("Manually setting min freq to %d\n", val);
 
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
 	/*
 	 * Turbo will still be enabled, but won't go below the set value.
 	 */
-	dev_priv->min_delay = val / 50;
+	dev_priv->rps.min_delay = val / GT_FREQUENCY_MULTIPLIER;
 
-	gen6_set_rps(dev, val / 50);
+	gen6_set_rps(dev, val / GT_FREQUENCY_MULTIPLIER);
+	mutex_unlock(&dev->struct_mutex);
 
 	return cnt;
 }
@@ -1834,9 +1859,15 @@ i915_cache_sharing_read(struct file *filp,
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	char buf[80];
 	u32 snpcr;
-	int len;
+	int len, ret;
+
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
 
-	mutex_lock(&dev_priv->dev->struct_mutex);
 	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 	mutex_unlock(&dev_priv->dev->struct_mutex);
 
@@ -1862,6 +1893,9 @@ i915_cache_sharing_write(struct file *filp,
 	u32 snpcr;
 	int val = 1;
 
+	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+		return -ENODEV;
+
 	if (cnt > 0) {
 		if (cnt > sizeof(buf) - 1)
 			return -EINVAL;
@@ -1925,16 +1959,11 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
 {
 	struct drm_device *dev = inode->i_private;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int ret;
 
 	if (INTEL_INFO(dev)->gen < 6)
 		return 0;
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
 	gen6_gt_force_wake_get(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
@@ -1947,16 +1976,7 @@ static int i915_forcewake_release(struct inode *inode, struct file *file)
 	if (INTEL_INFO(dev)->gen < 6)
 		return 0;
 
-	/*
-	 * It's bad that we can potentially hang userspace if struct_mutex gets
-	 * forever stuck.  However, if we cannot acquire this lock it means that
-	 * almost certainly the driver has hung, is not unload-able. Therefore
-	 * hanging here is probably a minor inconvenience not to be seen my
-	 * almost every user.
-	 */
-	mutex_lock(&dev->struct_mutex);
 	gen6_gt_force_wake_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
 
 	return 0;
 }
@@ -2006,7 +2026,6 @@ static struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gem_gtt", i915_gem_gtt_info, 0},
 	{"i915_gem_pinned", i915_gem_gtt_info, 0, (void *) PINNED_LIST},
 	{"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST},
-	{"i915_gem_flushing", i915_gem_object_list_info, 0, (void *) FLUSHING_LIST},
 	{"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST},
 	{"i915_gem_pageflip", i915_gem_pageflip_info, 0},
 	{"i915_gem_request", i915_gem_request_info, 0},
@@ -2067,6 +2086,7 @@ int i915_debugfs_init(struct drm_minor *minor)
 				  &i915_cache_sharing_fops);
 	if (ret)
 		return ret;
+
 	ret = i915_debugfs_create(minor->debugfs_root, minor,
 				  "i915_ring_stop",
 				  &i915_ring_stop_fops);
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 914c0dfabe6..c9bfd83dde6 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -28,12 +28,11 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include <linux/pci.h>
@@ -235,10 +234,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 		}
 	}
 
-	dev_priv->cpp = init->cpp;
-	dev_priv->back_offset = init->back_offset;
-	dev_priv->front_offset = init->front_offset;
-	dev_priv->current_page = 0;
+	dev_priv->dri1.cpp = init->cpp;
+	dev_priv->dri1.back_offset = init->back_offset;
+	dev_priv->dri1.front_offset = init->front_offset;
+	dev_priv->dri1.current_page = 0;
 	if (master_priv->sarea_priv)
 		master_priv->sarea_priv->pf_current_page = 0;
 
@@ -575,7 +574,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
 
 	DRM_DEBUG_DRIVER("%s: page=%d pfCurrentPage=%d\n",
 			  __func__,
-			 dev_priv->current_page,
+			 dev_priv->dri1.current_page,
 			 master_priv->sarea_priv->pf_current_page);
 
 	i915_kernel_lost_context(dev);
@@ -589,12 +588,12 @@ static int i915_dispatch_flip(struct drm_device * dev)
 
 	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
 	OUT_RING(0);
-	if (dev_priv->current_page == 0) {
-		OUT_RING(dev_priv->back_offset);
-		dev_priv->current_page = 1;
+	if (dev_priv->dri1.current_page == 0) {
+		OUT_RING(dev_priv->dri1.back_offset);
+		dev_priv->dri1.current_page = 1;
 	} else {
-		OUT_RING(dev_priv->front_offset);
-		dev_priv->current_page = 0;
+		OUT_RING(dev_priv->dri1.front_offset);
+		dev_priv->dri1.current_page = 0;
 	}
 	OUT_RING(0);
 
@@ -613,7 +612,7 @@ static int i915_dispatch_flip(struct drm_device * dev)
 		ADVANCE_LP_RING();
 	}
 
-	master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
+	master_priv->sarea_priv->pf_current_page = dev_priv->dri1.current_page;
 	return 0;
 }
 
@@ -1009,6 +1008,12 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_WAIT_TIMEOUT:
 		value = 1;
 		break;
+	case I915_PARAM_HAS_SEMAPHORES:
+		value = i915_semaphore_is_enabled(dev);
+		break;
+	case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
+		value = 1;
+		break;
 	default:
 		DRM_DEBUG_DRIVER("Unknown parameter %d\n",
 				 param->param);
@@ -1425,6 +1430,21 @@ static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
 	kfree(ap);
 }
 
+static void i915_dump_device_info(struct drm_i915_private *dev_priv)
+{
+	const struct intel_device_info *info = dev_priv->info;
+
+#define DEV_INFO_FLAG(name) info->name ? #name "," : ""
+#define DEV_INFO_SEP ,
+	DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x flags="
+			 "%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s",
+			 info->gen,
+			 dev_priv->dev->pdev->device,
+			 DEV_INFO_FLAGS);
+#undef DEV_INFO_FLAG
+#undef DEV_INFO_SEP
+}
+
 /**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
@@ -1440,7 +1460,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
 	struct drm_i915_private *dev_priv;
 	struct intel_device_info *info;
-	int ret = 0, mmio_bar;
+	int ret = 0, mmio_bar, mmio_size;
 	uint32_t aperture_size;
 
 	info = (struct intel_device_info *) flags;
@@ -1449,7 +1469,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	if (info->gen >= 6 && !drm_core_check_feature(dev, DRIVER_MODESET))
 		return -ENODEV;
 
-
 	/* i915 has 4 more counters */
 	dev->counters += 4;
 	dev->types[6] = _DRM_STAT_IRQ;
@@ -1465,6 +1484,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	dev_priv->dev = dev;
 	dev_priv->info = info;
 
+	i915_dump_device_info(dev_priv);
+
 	if (i915_get_bridge_dev(dev)) {
 		ret = -EIO;
 		goto free_priv;
@@ -1504,7 +1525,19 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 		dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
 
 	mmio_bar = IS_GEN2(dev) ? 1 : 0;
-	dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0);
+	/* Before gen4, the registers and the GTT are behind different BARs.
+	 * However, from gen4 onwards, the registers and the GTT are shared
+	 * in the same BAR, so we want to restrict this ioremap from
+	 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
+	 * the register BAR remains the same size for all the earlier
+	 * generations up to Ironlake.
+	 */
+	if (info->gen < 5)
+		mmio_size = 512*1024;
+	else
+		mmio_size = 2*1024*1024;
+
+	dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
 	if (!dev_priv->regs) {
 		DRM_ERROR("failed to map registers\n");
 		ret = -EIO;
@@ -1536,11 +1569,9 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	 *
 	 * All tasks on the workqueue are expected to acquire the dev mutex
 	 * so there is no point in running more than one instance of the
-	 * workqueue at any time: max_active = 1 and NON_REENTRANT.
+	 * workqueue at any time.  Use an ordered one.
 	 */
-	dev_priv->wq = alloc_workqueue("i915",
-				       WQ_UNBOUND | WQ_NON_REENTRANT,
-				       1);
+	dev_priv->wq = alloc_ordered_workqueue("i915", 0);
 	if (dev_priv->wq == NULL) {
 		DRM_ERROR("Failed to create our workqueue.\n");
 		ret = -ENOMEM;
@@ -1586,7 +1617,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	spin_lock_init(&dev_priv->irq_lock);
 	spin_lock_init(&dev_priv->error_lock);
-	spin_lock_init(&dev_priv->rps_lock);
+	spin_lock_init(&dev_priv->rps.lock);
 	spin_lock_init(&dev_priv->dpio_lock);
 
 	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
@@ -1836,6 +1867,8 @@ struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED),
+	DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
@@ -1858,6 +1891,7 @@ struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED),
+	DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_UNLOCKED),
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index a24ffbe97c0..aac4e5e1a5b 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -28,16 +28,15 @@
  */
 
 #include <linux/device.h>
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 
 #include <linux/console.h>
 #include <linux/module.h>
-#include "drm_crtc_helper.h"
+#include <drm/drm_crtc_helper.h>
 
 static int i915_modeset __read_mostly = -1;
 module_param_named(modeset, i915_modeset, int, 0400);
@@ -470,6 +469,9 @@ static int i915_drm_freeze(struct drm_device *dev)
 				"GEM idle failed, resume might fail\n");
 			return error;
 		}
+
+		intel_modeset_disable(dev);
+
 		drm_irq_uninstall(dev);
 	}
 
@@ -543,13 +545,9 @@ static int i915_drm_thaw(struct drm_device *dev)
 		mutex_unlock(&dev->struct_mutex);
 
 		intel_modeset_init_hw(dev);
+		intel_modeset_setup_hw_state(dev);
 		drm_mode_config_reset(dev);
 		drm_irq_install(dev);
-
-		/* Resume the modeset for every activated CRTC */
-		mutex_lock(&dev->mode_config.mutex);
-		drm_helper_resume_force_mode(dev);
-		mutex_unlock(&dev->mode_config.mutex);
 	}
 
 	intel_opregion_init(dev);
@@ -1060,7 +1058,7 @@ static bool IS_DISPLAYREG(u32 reg)
 	 * This should make it easier to transition modules over to the
 	 * new register block scheme, since we can do it incrementally.
 	 */
-	if (reg >= 0x180000)
+	if (reg >= VLV_DISPLAY_BASE)
 		return false;
 
 	if (reg >= RENDER_RING_BASE &&
@@ -1174,9 +1172,59 @@ void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \
 	if (unlikely(__fifo_ret)) { \
 		gen6_gt_check_fifodbg(dev_priv); \
 	} \
+	if (IS_HASWELL(dev_priv->dev) && (I915_READ_NOTRACE(GEN7_ERR_INT) & ERR_INT_MMIO_UNCLAIMED)) { \
+		DRM_ERROR("Unclaimed write to %x\n", reg); \
+		writel(ERR_INT_MMIO_UNCLAIMED, dev_priv->regs + GEN7_ERR_INT);	\
+	} \
 }
 __i915_write(8, b)
 __i915_write(16, w)
 __i915_write(32, l)
 __i915_write(64, q)
 #undef __i915_write
+
+static const struct register_whitelist {
+	uint64_t offset;
+	uint32_t size;
+	uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
+} whitelist[] = {
+	{ RING_TIMESTAMP(RENDER_RING_BASE), 8, 0xF0 },
+};
+
+int i915_reg_read_ioctl(struct drm_device *dev,
+			void *data, struct drm_file *file)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_reg_read *reg = data;
+	struct register_whitelist const *entry = whitelist;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
+		if (entry->offset == reg->offset &&
+		    (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
+			break;
+	}
+
+	if (i == ARRAY_SIZE(whitelist))
+		return -EINVAL;
+
+	switch (entry->size) {
+	case 8:
+		reg->val = I915_READ64(reg->offset);
+		break;
+	case 4:
+		reg->val = I915_READ(reg->offset);
+		break;
+	case 2:
+		reg->val = I915_READ16(reg->offset);
+		break;
+	case 1:
+		reg->val = I915_READ8(reg->offset);
+		break;
+	default:
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 627fe35781b..4f2831aa5fe 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -109,6 +109,7 @@ struct intel_pch_pll {
 
 #define WATCH_COHERENCY	0
 #define WATCH_LISTS	0
+#define WATCH_GTT	0
 
 #define I915_GEM_PHYS_CURSOR_0 1
 #define I915_GEM_PHYS_CURSOR_1 2
@@ -195,9 +196,10 @@ struct drm_i915_error_state {
 	u32 cpu_ring_head[I915_NUM_RINGS];
 	u32 cpu_ring_tail[I915_NUM_RINGS];
 	u32 error; /* gen6+ */
+	u32 err_int; /* gen7 */
 	u32 instpm[I915_NUM_RINGS];
 	u32 instps[I915_NUM_RINGS];
-	u32 instdone1;
+	u32 extra_instdone[I915_NUM_INSTDONE_REG];
 	u32 seqno[I915_NUM_RINGS];
 	u64 bbaddr;
 	u32 fault_reg[I915_NUM_RINGS];
@@ -221,7 +223,7 @@ struct drm_i915_error_state {
 	struct drm_i915_error_buffer {
 		u32 size;
 		u32 name;
-		u32 seqno;
+		u32 rseqno, wseqno;
 		u32 gtt_offset;
 		u32 read_domains;
 		u32 write_domain;
@@ -239,7 +241,6 @@ struct drm_i915_error_state {
 };
 
 struct drm_i915_display_funcs {
-	void (*dpms)(struct drm_crtc *crtc, int mode);
 	bool (*fbc_enabled)(struct drm_device *dev);
 	void (*enable_fbc)(struct drm_crtc *crtc, unsigned long interval);
 	void (*disable_fbc)(struct drm_device *dev);
@@ -248,7 +249,6 @@ struct drm_i915_display_funcs {
 	void (*update_wm)(struct drm_device *dev);
 	void (*update_sprite_wm)(struct drm_device *dev, int pipe,
 				 uint32_t sprite_width, int pixel_size);
-	void (*sanitize_pm)(struct drm_device *dev);
 	void (*update_linetime_wm)(struct drm_device *dev, int pipe,
 				 struct drm_display_mode *mode);
 	int (*crtc_mode_set)(struct drm_crtc *crtc,
@@ -256,6 +256,8 @@ struct drm_i915_display_funcs {
 			     struct drm_display_mode *adjusted_mode,
 			     int x, int y,
 			     struct drm_framebuffer *old_fb);
+	void (*crtc_enable)(struct drm_crtc *crtc);
+	void (*crtc_disable)(struct drm_crtc *crtc);
 	void (*off)(struct drm_crtc *crtc);
 	void (*write_eld)(struct drm_connector *connector,
 			  struct drm_crtc *crtc);
@@ -279,6 +281,32 @@ struct drm_i915_gt_funcs {
 	void (*force_wake_put)(struct drm_i915_private *dev_priv);
 };
 
+#define DEV_INFO_FLAGS \
+	DEV_INFO_FLAG(is_mobile) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_i85x) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_i915g) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_i945gm) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_g33) DEV_INFO_SEP \
+	DEV_INFO_FLAG(need_gfx_hws) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_g4x) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_pineview) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_broadwater) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_crestline) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_ivybridge) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_valleyview) DEV_INFO_SEP \
+	DEV_INFO_FLAG(is_haswell) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_force_wake) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_fbc) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_pipe_cxsr) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_hotplug) DEV_INFO_SEP \
+	DEV_INFO_FLAG(cursor_needs_physical) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_overlay) DEV_INFO_SEP \
+	DEV_INFO_FLAG(overlay_needs_physical) DEV_INFO_SEP \
+	DEV_INFO_FLAG(supports_tv) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_bsd_ring) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_blt_ring) DEV_INFO_SEP \
+	DEV_INFO_FLAG(has_llc)
+
 struct intel_device_info {
 	u8 gen;
 	u8 is_mobile:1;
@@ -402,12 +430,6 @@ typedef struct drm_i915_private {
 
 	struct resource mch_res;
 
-	unsigned int cpp;
-	int back_offset;
-	int front_offset;
-	int current_page;
-	int page_flipping;
-
 	atomic_t irq_received;
 
 	/* protects the irq masks */
@@ -425,7 +447,6 @@ typedef struct drm_i915_private {
 	u32 hotplug_supported_mask;
 	struct work_struct hotplug_work;
 
-	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
 	int num_pipe;
 	int num_pch_pll;
 
@@ -434,8 +455,7 @@ typedef struct drm_i915_private {
 	struct timer_list hangcheck_timer;
 	int hangcheck_count;
 	uint32_t last_acthd[I915_NUM_RINGS];
-	uint32_t last_instdone;
-	uint32_t last_instdone1;
+	uint32_t prev_instdone[I915_NUM_INSTDONE_REG];
 
 	unsigned int stop_rings;
 
@@ -666,7 +686,13 @@ typedef struct drm_i915_private {
 		struct drm_mm gtt_space;
 		/** List of all objects in gtt_space. Used to restore gtt
 		 * mappings on resume */
-		struct list_head gtt_list;
+		struct list_head bound_list;
+		/**
+		 * List of objects which are not bound to the GTT (thus
+		 * are idle and not used by the GPU) but still have
+		 * (presumably uncached) pages still attached.
+		 */
+		struct list_head unbound_list;
 
 		/** Usable portion of the GTT for GEM */
 		unsigned long gtt_start;
@@ -696,17 +722,6 @@ typedef struct drm_i915_private {
 		struct list_head active_list;
 
 		/**
-		 * List of objects which are not in the ringbuffer but which
-		 * still have a write_domain which needs to be flushed before
-		 * unbinding.
-		 *
-		 * last_rendering_seqno is 0 while an object is in this list.
-		 *
-		 * A reference is held on the buffer while on this list.
-		 */
-		struct list_head flushing_list;
-
-		/**
 		 * LRU list of objects which are not in the ringbuffer and
 		 * are ready to unbind, but are still in the GTT.
 		 *
@@ -775,6 +790,12 @@ typedef struct drm_i915_private {
 	struct {
 		unsigned allow_batchbuffer : 1;
 		u32 __iomem *gfx_hws_cpu_addr;
+
+		unsigned int cpp;
+		int back_offset;
+		int front_offset;
+		int current_page;
+		int page_flipping;
 	} dri1;
 
 	/* Kernel Modesetting */
@@ -796,9 +817,6 @@ typedef struct drm_i915_private {
 	bool lvds_downclock_avail;
 	/* indicates the reduced downclock for LVDS*/
 	int lvds_downclock;
-	struct work_struct idle_work;
-	struct timer_list idle_timer;
-	bool busy;
 	u16 orig_clock;
 	int child_dev_num;
 	struct child_device_config *child_dev;
@@ -807,26 +825,41 @@ typedef struct drm_i915_private {
 
 	bool mchbar_need_disable;
 
-	struct work_struct rps_work;
-	spinlock_t rps_lock;
-	u32 pm_iir;
-
-	u8 cur_delay;
-	u8 min_delay;
-	u8 max_delay;
-	u8 fmax;
-	u8 fstart;
-
-	u64 last_count1;
-	unsigned long last_time1;
-	unsigned long chipset_power;
-	u64 last_count2;
-	struct timespec last_time2;
-	unsigned long gfx_power;
-	int c_m;
-	int r_t;
-	u8 corr;
-	spinlock_t *mchdev_lock;
+	/* gen6+ rps state */
+	struct {
+		struct work_struct work;
+		u32 pm_iir;
+		/* lock - irqsave spinlock that protectects the work_struct and
+		 * pm_iir. */
+		spinlock_t lock;
+
+		/* The below variables an all the rps hw state are protected by
+		 * dev->struct mutext. */
+		u8 cur_delay;
+		u8 min_delay;
+		u8 max_delay;
+	} rps;
+
+	/* ilk-only ips/rps state. Everything in here is protected by the global
+	 * mchdev_lock in intel_pm.c */
+	struct {
+		u8 cur_delay;
+		u8 min_delay;
+		u8 max_delay;
+		u8 fmax;
+		u8 fstart;
+
+		u64 last_count1;
+		unsigned long last_time1;
+		unsigned long chipset_power;
+		u64 last_count2;
+		struct timespec last_time2;
+		unsigned long gfx_power;
+		u8 corr;
+
+		int c_m;
+		int r_t;
+	} ips;
 
 	enum no_fbc_reason no_fbc_reason;
 
@@ -861,30 +894,48 @@ enum hdmi_force_audio {
 };
 
 enum i915_cache_level {
-	I915_CACHE_NONE,
+	I915_CACHE_NONE = 0,
 	I915_CACHE_LLC,
-	I915_CACHE_LLC_MLC, /* gen6+ */
+	I915_CACHE_LLC_MLC, /* gen6+, in docs at least! */
+};
+
+struct drm_i915_gem_object_ops {
+	/* Interface between the GEM object and its backing storage.
+	 * get_pages() is called once prior to the use of the associated set
+	 * of pages before to binding them into the GTT, and put_pages() is
+	 * called after we no longer need them. As we expect there to be
+	 * associated cost with migrating pages between the backing storage
+	 * and making them available for the GPU (e.g. clflush), we may hold
+	 * onto the pages after they are no longer referenced by the GPU
+	 * in case they may be used again shortly (for example migrating the
+	 * pages to a different memory domain within the GTT). put_pages()
+	 * will therefore most likely be called when the object itself is
+	 * being released or under memory pressure (where we attempt to
+	 * reap pages for the shrinker).
+	 */
+	int (*get_pages)(struct drm_i915_gem_object *);
+	void (*put_pages)(struct drm_i915_gem_object *);
 };
 
 struct drm_i915_gem_object {
 	struct drm_gem_object base;
 
+	const struct drm_i915_gem_object_ops *ops;
+
 	/** Current space allocated to this object in the GTT, if any. */
 	struct drm_mm_node *gtt_space;
 	struct list_head gtt_list;
 
-	/** This object's place on the active/flushing/inactive lists */
+	/** This object's place on the active/inactive lists */
 	struct list_head ring_list;
 	struct list_head mm_list;
-	/** This object's place on GPU write list */
-	struct list_head gpu_write_list;
 	/** This object's place in the batchbuffer or on the eviction list */
 	struct list_head exec_list;
 
 	/**
-	 * This is set if the object is on the active or flushing lists
-	 * (has pending rendering), and is not set if it's on inactive (ready
-	 * to be unbound).
+	 * This is set if the object is on the active lists (has pending
+	 * rendering and so a non-zero seqno), and is not set if it i s on
+	 * inactive (ready to be unbound) list.
 	 */
 	unsigned int active:1;
 
@@ -895,12 +946,6 @@ struct drm_i915_gem_object {
 	unsigned int dirty:1;
 
 	/**
-	 * This is set if the object has been written to since the last
-	 * GPU flush.
-	 */
-	unsigned int pending_gpu_write:1;
-
-	/**
 	 * Fence register bits (if any) for this object.  Will be set
 	 * as needed when mapped into the GTT.
 	 * Protected by dev->struct_mutex.
@@ -961,17 +1006,12 @@ struct drm_i915_gem_object {
 
 	unsigned int has_aliasing_ppgtt_mapping:1;
 	unsigned int has_global_gtt_mapping:1;
+	unsigned int has_dma_mapping:1;
 
-	struct page **pages;
-
-	/**
-	 * DMAR support
-	 */
-	struct scatterlist *sg_list;
-	int num_sg;
+	struct sg_table *pages;
+	int pages_pin_count;
 
 	/* prime dma-buf support */
-	struct sg_table *sg_table;
 	void *dma_buf_vmapping;
 	int vmapping_count;
 
@@ -992,7 +1032,8 @@ struct drm_i915_gem_object {
 	struct intel_ring_buffer *ring;
 
 	/** Breadcrumb of last rendering to the buffer. */
-	uint32_t last_rendering_seqno;
+	uint32_t last_read_seqno;
+	uint32_t last_write_seqno;
 	/** Breadcrumb of last fenced GPU access to the buffer. */
 	uint32_t last_fenced_seqno;
 
@@ -1135,6 +1176,10 @@ struct drm_i915_file_private {
 
 #define HAS_FORCE_WAKE(dev) (INTEL_INFO(dev)->has_force_wake)
 
+#define HAS_L3_GPU_CACHE(dev) (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
+
+#define GT_FREQUENCY_MULTIPLIER 50
+
 #include "i915_trace.h"
 
 /**
@@ -1256,6 +1301,10 @@ int i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
 int i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
+int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file);
+int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file);
 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
@@ -1274,24 +1323,42 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
 int i915_gem_init_object(struct drm_gem_object *obj);
-int __must_check i915_gem_flush_ring(struct intel_ring_buffer *ring,
-				     uint32_t invalidate_domains,
-				     uint32_t flush_domains);
+void i915_gem_object_init(struct drm_i915_gem_object *obj,
+			 const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
 						  size_t size);
 void i915_gem_free_object(struct drm_gem_object *obj);
 int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
 				     uint32_t alignment,
-				     bool map_and_fenceable);
+				     bool map_and_fenceable,
+				     bool nonblocking);
 void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 
-int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
-				  gfp_t gfpmask);
+int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+static inline struct page *i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
+{
+	struct scatterlist *sg = obj->pages->sgl;
+	while (n >= SG_MAX_SINGLE_ALLOC) {
+		sg = sg_chain_ptr(sg + SG_MAX_SINGLE_ALLOC - 1);
+		n -= SG_MAX_SINGLE_ALLOC - 1;
+	}
+	return sg_page(sg+n);
+}
+static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+	BUG_ON(obj->pages == NULL);
+	obj->pages_pin_count++;
+}
+static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+	BUG_ON(obj->pages_pin_count == 0);
+	obj->pages_pin_count--;
+}
+
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
-int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj);
 int i915_gem_object_sync(struct drm_i915_gem_object *obj,
 			 struct intel_ring_buffer *to);
 void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
@@ -1358,9 +1425,9 @@ void i915_gem_init_ppgtt(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
 int __must_check i915_gpu_idle(struct drm_device *dev);
 int __must_check i915_gem_idle(struct drm_device *dev);
-int __must_check i915_add_request(struct intel_ring_buffer *ring,
-				  struct drm_file *file,
-				  struct drm_i915_gem_request *request);
+int i915_add_request(struct intel_ring_buffer *ring,
+		     struct drm_file *file,
+		     struct drm_i915_gem_request *request);
 int __must_check i915_wait_seqno(struct intel_ring_buffer *ring,
 				 uint32_t seqno);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
@@ -1429,8 +1496,11 @@ void i915_gem_init_global_gtt(struct drm_device *dev,
 
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size,
-					  unsigned alignment, bool mappable);
-int i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only);
+					  unsigned alignment,
+					  unsigned cache_level,
+					  bool mappable,
+					  bool nonblock);
+int i915_gem_evict_everything(struct drm_device *dev);
 
 /* i915_gem_stolen.c */
 int i915_gem_init_stolen(struct drm_device *dev);
@@ -1519,6 +1589,7 @@ extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
+extern void intel_modeset_setup_hw_state(struct drm_device *dev);
 extern bool intel_fbc_enabled(struct drm_device *dev);
 extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
@@ -1529,6 +1600,8 @@ extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
 extern int intel_enable_rc6(const struct drm_device *dev);
 
 extern bool i915_semaphore_is_enabled(struct drm_device *dev);
+int i915_reg_read_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file);
 
 /* overlay */
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 274d25de521..e957f3740f6 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -25,9 +25,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
@@ -37,12 +36,12 @@
 #include <linux/pci.h>
 #include <linux/dma-buf.h>
 
-static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 						    unsigned alignment,
-						    bool map_and_fenceable);
+						    bool map_and_fenceable,
+						    bool nonblocking);
 static int i915_gem_phys_pwrite(struct drm_device *dev,
 				struct drm_i915_gem_object *obj,
 				struct drm_i915_gem_pwrite *args,
@@ -56,6 +55,8 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
 
 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
 				    struct shrink_control *sc);
+static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
+static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
 
 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
@@ -141,7 +142,7 @@ int i915_mutex_lock_interruptible(struct drm_device *dev)
 static inline bool
 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 {
-	return !obj->active;
+	return obj->gtt_space && !obj->active;
 }
 
 int
@@ -180,7 +181,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 
 	pinned = 0;
 	mutex_lock(&dev->struct_mutex);
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
 		if (obj->pin_count)
 			pinned += obj->gtt_space->size;
 	mutex_unlock(&dev->struct_mutex);
@@ -341,7 +342,7 @@ shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 				      page_length);
 	kunmap_atomic(vaddr);
 
-	return ret;
+	return ret ? -EFAULT : 0;
 }
 
 static void
@@ -392,7 +393,7 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 				     page_length);
 	kunmap(page);
 
-	return ret;
+	return ret ? - EFAULT : 0;
 }
 
 static int
@@ -401,7 +402,6 @@ i915_gem_shmem_pread(struct drm_device *dev,
 		     struct drm_i915_gem_pread *args,
 		     struct drm_file *file)
 {
-	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 	char __user *user_data;
 	ssize_t remain;
 	loff_t offset;
@@ -410,7 +410,8 @@ i915_gem_shmem_pread(struct drm_device *dev,
 	int hit_slowpath = 0;
 	int prefaulted = 0;
 	int needs_clflush = 0;
-	int release_page;
+	struct scatterlist *sg;
+	int i;
 
 	user_data = (char __user *) (uintptr_t) args->data_ptr;
 	remain = args->size;
@@ -424,16 +425,30 @@ i915_gem_shmem_pread(struct drm_device *dev,
 		 * anyway again before the next pread happens. */
 		if (obj->cache_level == I915_CACHE_NONE)
 			needs_clflush = 1;
-		ret = i915_gem_object_set_to_gtt_domain(obj, false);
-		if (ret)
-			return ret;
+		if (obj->gtt_space) {
+			ret = i915_gem_object_set_to_gtt_domain(obj, false);
+			if (ret)
+				return ret;
+		}
 	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ret;
+
+	i915_gem_object_pin_pages(obj);
+
 	offset = args->offset;
 
-	while (remain > 0) {
+	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
 		struct page *page;
 
+		if (i < offset >> PAGE_SHIFT)
+			continue;
+
+		if (remain <= 0)
+			break;
+
 		/* Operation in this page
 		 *
 		 * shmem_page_offset = offset within page in shmem file
@@ -444,18 +459,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
 			page_length = PAGE_SIZE - shmem_page_offset;
 
-		if (obj->pages) {
-			page = obj->pages[offset >> PAGE_SHIFT];
-			release_page = 0;
-		} else {
-			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
-			if (IS_ERR(page)) {
-				ret = PTR_ERR(page);
-				goto out;
-			}
-			release_page = 1;
-		}
-
+		page = sg_page(sg);
 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 			(page_to_phys(page) & (1 << 17)) != 0;
 
@@ -466,7 +470,6 @@ i915_gem_shmem_pread(struct drm_device *dev,
 			goto next_page;
 
 		hit_slowpath = 1;
-		page_cache_get(page);
 		mutex_unlock(&dev->struct_mutex);
 
 		if (!prefaulted) {
@@ -484,16 +487,12 @@ i915_gem_shmem_pread(struct drm_device *dev,
 				       needs_clflush);
 
 		mutex_lock(&dev->struct_mutex);
-		page_cache_release(page);
+
 next_page:
 		mark_page_accessed(page);
-		if (release_page)
-			page_cache_release(page);
 
-		if (ret) {
-			ret = -EFAULT;
+		if (ret)
 			goto out;
-		}
 
 		remain -= page_length;
 		user_data += page_length;
@@ -501,6 +500,8 @@ next_page:
 	}
 
 out:
+	i915_gem_object_unpin_pages(obj);
+
 	if (hit_slowpath) {
 		/* Fixup: Kill any reinstated backing storage pages */
 		if (obj->madv == __I915_MADV_PURGED)
@@ -606,7 +607,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 	char __user *user_data;
 	int page_offset, page_length, ret;
 
-	ret = i915_gem_object_pin(obj, 0, true);
+	ret = i915_gem_object_pin(obj, 0, true, true);
 	if (ret)
 		goto out;
 
@@ -686,7 +687,7 @@ shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 				       page_length);
 	kunmap_atomic(vaddr);
 
-	return ret;
+	return ret ? -EFAULT : 0;
 }
 
 /* Only difference to the fast-path function is that this can handle bit17
@@ -720,7 +721,7 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 					     page_do_bit17_swizzling);
 	kunmap(page);
 
-	return ret;
+	return ret ? -EFAULT : 0;
 }
 
 static int
@@ -729,7 +730,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 		      struct drm_i915_gem_pwrite *args,
 		      struct drm_file *file)
 {
-	struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 	ssize_t remain;
 	loff_t offset;
 	char __user *user_data;
@@ -738,7 +738,8 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 	int hit_slowpath = 0;
 	int needs_clflush_after = 0;
 	int needs_clflush_before = 0;
-	int release_page;
+	int i;
+	struct scatterlist *sg;
 
 	user_data = (char __user *) (uintptr_t) args->data_ptr;
 	remain = args->size;
@@ -752,9 +753,11 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 		 * right away and we therefore have to clflush anyway. */
 		if (obj->cache_level == I915_CACHE_NONE)
 			needs_clflush_after = 1;
-		ret = i915_gem_object_set_to_gtt_domain(obj, true);
-		if (ret)
-			return ret;
+		if (obj->gtt_space) {
+			ret = i915_gem_object_set_to_gtt_domain(obj, true);
+			if (ret)
+				return ret;
+		}
 	}
 	/* Same trick applies for invalidate partially written cachelines before
 	 * writing.  */
@@ -762,13 +765,25 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 	    && obj->cache_level == I915_CACHE_NONE)
 		needs_clflush_before = 1;
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ret;
+
+	i915_gem_object_pin_pages(obj);
+
 	offset = args->offset;
 	obj->dirty = 1;
 
-	while (remain > 0) {
+	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
 		struct page *page;
 		int partial_cacheline_write;
 
+		if (i < offset >> PAGE_SHIFT)
+			continue;
+
+		if (remain <= 0)
+			break;
+
 		/* Operation in this page
 		 *
 		 * shmem_page_offset = offset within page in shmem file
@@ -787,18 +802,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 			((shmem_page_offset | page_length)
 				& (boot_cpu_data.x86_clflush_size - 1));
 
-		if (obj->pages) {
-			page = obj->pages[offset >> PAGE_SHIFT];
-			release_page = 0;
-		} else {
-			page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
-			if (IS_ERR(page)) {
-				ret = PTR_ERR(page);
-				goto out;
-			}
-			release_page = 1;
-		}
-
+		page = sg_page(sg);
 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 			(page_to_phys(page) & (1 << 17)) != 0;
 
@@ -810,26 +814,20 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 			goto next_page;
 
 		hit_slowpath = 1;
-		page_cache_get(page);
 		mutex_unlock(&dev->struct_mutex);
-
 		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 					user_data, page_do_bit17_swizzling,
 					partial_cacheline_write,
 					needs_clflush_after);
 
 		mutex_lock(&dev->struct_mutex);
-		page_cache_release(page);
+
 next_page:
 		set_page_dirty(page);
 		mark_page_accessed(page);
-		if (release_page)
-			page_cache_release(page);
 
-		if (ret) {
-			ret = -EFAULT;
+		if (ret)
 			goto out;
-		}
 
 		remain -= page_length;
 		user_data += page_length;
@@ -837,6 +835,8 @@ next_page:
 	}
 
 out:
+	i915_gem_object_unpin_pages(obj);
+
 	if (hit_slowpath) {
 		/* Fixup: Kill any reinstated backing storage pages */
 		if (obj->madv == __I915_MADV_PURGED)
@@ -920,10 +920,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		goto out;
 	}
 
-	if (obj->gtt_space &&
-	    obj->cache_level == I915_CACHE_NONE &&
+	if (obj->cache_level == I915_CACHE_NONE &&
 	    obj->tiling_mode == I915_TILING_NONE &&
-	    obj->map_and_fenceable &&
 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 		/* Note that the gtt paths might fail with non-page-backed user
@@ -931,7 +929,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		 * textures). Fallback to the shmem path in that case. */
 	}
 
-	if (ret == -EFAULT)
+	if (ret == -EFAULT || ret == -ENOSPC)
 		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 
 out:
@@ -941,6 +939,240 @@ unlock:
 	return ret;
 }
 
+int
+i915_gem_check_wedge(struct drm_i915_private *dev_priv,
+		     bool interruptible)
+{
+	if (atomic_read(&dev_priv->mm.wedged)) {
+		struct completion *x = &dev_priv->error_completion;
+		bool recovery_complete;
+		unsigned long flags;
+
+		/* Give the error handler a chance to run. */
+		spin_lock_irqsave(&x->wait.lock, flags);
+		recovery_complete = x->done > 0;
+		spin_unlock_irqrestore(&x->wait.lock, flags);
+
+		/* Non-interruptible callers can't handle -EAGAIN, hence return
+		 * -EIO unconditionally for these. */
+		if (!interruptible)
+			return -EIO;
+
+		/* Recovery complete, but still wedged means reset failure. */
+		if (recovery_complete)
+			return -EIO;
+
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
+/*
+ * Compare seqno against outstanding lazy request. Emit a request if they are
+ * equal.
+ */
+static int
+i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
+{
+	int ret;
+
+	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
+
+	ret = 0;
+	if (seqno == ring->outstanding_lazy_request)
+		ret = i915_add_request(ring, NULL, NULL);
+
+	return ret;
+}
+
+/**
+ * __wait_seqno - wait until execution of seqno has finished
+ * @ring: the ring expected to report seqno
+ * @seqno: duh!
+ * @interruptible: do an interruptible wait (normally yes)
+ * @timeout: in - how long to wait (NULL forever); out - how much time remaining
+ *
+ * Returns 0 if the seqno was found within the alloted time. Else returns the
+ * errno with remaining time filled in timeout argument.
+ */
+static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
+			bool interruptible, struct timespec *timeout)
+{
+	drm_i915_private_t *dev_priv = ring->dev->dev_private;
+	struct timespec before, now, wait_time={1,0};
+	unsigned long timeout_jiffies;
+	long end;
+	bool wait_forever = true;
+	int ret;
+
+	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
+		return 0;
+
+	trace_i915_gem_request_wait_begin(ring, seqno);
+
+	if (timeout != NULL) {
+		wait_time = *timeout;
+		wait_forever = false;
+	}
+
+	timeout_jiffies = timespec_to_jiffies(&wait_time);
+
+	if (WARN_ON(!ring->irq_get(ring)))
+		return -ENODEV;
+
+	/* Record current time in case interrupted by signal, or wedged * */
+	getrawmonotonic(&before);
+
+#define EXIT_COND \
+	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
+	atomic_read(&dev_priv->mm.wedged))
+	do {
+		if (interruptible)
+			end = wait_event_interruptible_timeout(ring->irq_queue,
+							       EXIT_COND,
+							       timeout_jiffies);
+		else
+			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
+						 timeout_jiffies);
+
+		ret = i915_gem_check_wedge(dev_priv, interruptible);
+		if (ret)
+			end = ret;
+	} while (end == 0 && wait_forever);
+
+	getrawmonotonic(&now);
+
+	ring->irq_put(ring);
+	trace_i915_gem_request_wait_end(ring, seqno);
+#undef EXIT_COND
+
+	if (timeout) {
+		struct timespec sleep_time = timespec_sub(now, before);
+		*timeout = timespec_sub(*timeout, sleep_time);
+	}
+
+	switch (end) {
+	case -EIO:
+	case -EAGAIN: /* Wedged */
+	case -ERESTARTSYS: /* Signal */
+		return (int)end;
+	case 0: /* Timeout */
+		if (timeout)
+			set_normalized_timespec(timeout, 0, 0);
+		return -ETIME;
+	default: /* Completed */
+		WARN_ON(end < 0); /* We're not aware of other errors */
+		return 0;
+	}
+}
+
+/**
+ * Waits for a sequence number to be signaled, and cleans up the
+ * request and object lists appropriately for that event.
+ */
+int
+i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
+{
+	struct drm_device *dev = ring->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	bool interruptible = dev_priv->mm.interruptible;
+	int ret;
+
+	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+	BUG_ON(seqno == 0);
+
+	ret = i915_gem_check_wedge(dev_priv, interruptible);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_check_olr(ring, seqno);
+	if (ret)
+		return ret;
+
+	return __wait_seqno(ring, seqno, interruptible, NULL);
+}
+
+/**
+ * Ensures that all rendering to the object has completed and the object is
+ * safe to unbind from the GTT or access from the CPU.
+ */
+static __must_check int
+i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
+			       bool readonly)
+{
+	struct intel_ring_buffer *ring = obj->ring;
+	u32 seqno;
+	int ret;
+
+	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
+	if (seqno == 0)
+		return 0;
+
+	ret = i915_wait_seqno(ring, seqno);
+	if (ret)
+		return ret;
+
+	i915_gem_retire_requests_ring(ring);
+
+	/* Manually manage the write flush as we may have not yet
+	 * retired the buffer.
+	 */
+	if (obj->last_write_seqno &&
+	    i915_seqno_passed(seqno, obj->last_write_seqno)) {
+		obj->last_write_seqno = 0;
+		obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
+	}
+
+	return 0;
+}
+
+/* A nonblocking variant of the above wait. This is a highly dangerous routine
+ * as the object state may change during this call.
+ */
+static __must_check int
+i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
+					    bool readonly)
+{
+	struct drm_device *dev = obj->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = obj->ring;
+	u32 seqno;
+	int ret;
+
+	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+	BUG_ON(!dev_priv->mm.interruptible);
+
+	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
+	if (seqno == 0)
+		return 0;
+
+	ret = i915_gem_check_wedge(dev_priv, true);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_check_olr(ring, seqno);
+	if (ret)
+		return ret;
+
+	mutex_unlock(&dev->struct_mutex);
+	ret = __wait_seqno(ring, seqno, true, NULL);
+	mutex_lock(&dev->struct_mutex);
+
+	i915_gem_retire_requests_ring(ring);
+
+	/* Manually manage the write flush as we may have not yet
+	 * retired the buffer.
+	 */
+	if (obj->last_write_seqno &&
+	    i915_seqno_passed(seqno, obj->last_write_seqno)) {
+		obj->last_write_seqno = 0;
+		obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
+	}
+
+	return ret;
+}
+
 /**
  * Called when user space prepares to use an object with the CPU, either
  * through the mmap ioctl's mapping or a GTT mapping.
@@ -978,6 +1210,14 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 		goto unlock;
 	}
 
+	/* Try to flush the object off the GPU without holding the lock.
+	 * We will repeat the flush holding the lock in the normal manner
+	 * to catch cases where we are gazumped.
+	 */
+	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
+	if (ret)
+		goto unref;
+
 	if (read_domains & I915_GEM_DOMAIN_GTT) {
 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 
@@ -991,6 +1231,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 	}
 
+unref:
 	drm_gem_object_unreference(&obj->base);
 unlock:
 	mutex_unlock(&dev->struct_mutex);
@@ -1110,7 +1351,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 			goto unlock;
 	}
 	if (!obj->gtt_space) {
-		ret = i915_gem_object_bind_to_gtt(obj, 0, true);
+		ret = i915_gem_object_bind_to_gtt(obj, 0, true, false);
 		if (ret)
 			goto unlock;
 
@@ -1271,6 +1512,42 @@ i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
 	return i915_gem_get_gtt_size(dev, size, tiling_mode);
 }
 
+static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	int ret;
+
+	if (obj->base.map_list.map)
+		return 0;
+
+	ret = drm_gem_create_mmap_offset(&obj->base);
+	if (ret != -ENOSPC)
+		return ret;
+
+	/* Badly fragmented mmap space? The only way we can recover
+	 * space is by destroying unwanted objects. We can't randomly release
+	 * mmap_offsets as userspace expects them to be persistent for the
+	 * lifetime of the objects. The closest we can is to release the
+	 * offsets on purgeable objects by truncating it and marking it purged,
+	 * which prevents userspace from ever using that object again.
+	 */
+	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
+	ret = drm_gem_create_mmap_offset(&obj->base);
+	if (ret != -ENOSPC)
+		return ret;
+
+	i915_gem_shrink_all(dev_priv);
+	return drm_gem_create_mmap_offset(&obj->base);
+}
+
+static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
+{
+	if (!obj->base.map_list.map)
+		return;
+
+	drm_gem_free_mmap_offset(&obj->base);
+}
+
 int
 i915_gem_mmap_gtt(struct drm_file *file,
 		  struct drm_device *dev,
@@ -1302,11 +1579,9 @@ i915_gem_mmap_gtt(struct drm_file *file,
 		goto out;
 	}
 
-	if (!obj->base.map_list.map) {
-		ret = drm_gem_create_mmap_offset(&obj->base);
-		if (ret)
-			goto out;
-	}
+	ret = i915_gem_object_create_mmap_offset(obj);
+	if (ret)
+		goto out;
 
 	*offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
 
@@ -1341,83 +1616,245 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
 	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
 }
 
-int
-i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
-			      gfp_t gfpmask)
+/* Immediately discard the backing storage */
+static void
+i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 {
-	int page_count, i;
-	struct address_space *mapping;
 	struct inode *inode;
-	struct page *page;
 
-	if (obj->pages || obj->sg_table)
-		return 0;
+	i915_gem_object_free_mmap_offset(obj);
 
-	/* Get the list of pages out of our struct file.  They'll be pinned
-	 * at this point until we release them.
-	 */
-	page_count = obj->base.size / PAGE_SIZE;
-	BUG_ON(obj->pages != NULL);
-	obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
-	if (obj->pages == NULL)
-		return -ENOMEM;
+	if (obj->base.filp == NULL)
+		return;
 
+	/* Our goal here is to return as much of the memory as
+	 * is possible back to the system as we are called from OOM.
+	 * To do this we must instruct the shmfs to drop all of its
+	 * backing pages, *now*.
+	 */
 	inode = obj->base.filp->f_path.dentry->d_inode;
-	mapping = inode->i_mapping;
-	gfpmask |= mapping_gfp_mask(mapping);
-
-	for (i = 0; i < page_count; i++) {
-		page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
-		if (IS_ERR(page))
-			goto err_pages;
-
-		obj->pages[i] = page;
-	}
-
-	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_do_bit_17_swizzle(obj);
-
-	return 0;
+	shmem_truncate_range(inode, 0, (loff_t)-1);
 
-err_pages:
-	while (i--)
-		page_cache_release(obj->pages[i]);
+	obj->madv = __I915_MADV_PURGED;
+}
 
-	drm_free_large(obj->pages);
-	obj->pages = NULL;
-	return PTR_ERR(page);
+static inline int
+i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
+{
+	return obj->madv == I915_MADV_DONTNEED;
 }
 
 static void
 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 {
 	int page_count = obj->base.size / PAGE_SIZE;
-	int i;
-
-	if (!obj->pages)
-		return;
+	struct scatterlist *sg;
+	int ret, i;
 
 	BUG_ON(obj->madv == __I915_MADV_PURGED);
 
+	ret = i915_gem_object_set_to_cpu_domain(obj, true);
+	if (ret) {
+		/* In the event of a disaster, abandon all caches and
+		 * hope for the best.
+		 */
+		WARN_ON(ret != -EIO);
+		i915_gem_clflush_object(obj);
+		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	}
+
 	if (i915_gem_object_needs_bit17_swizzle(obj))
 		i915_gem_object_save_bit_17_swizzle(obj);
 
 	if (obj->madv == I915_MADV_DONTNEED)
 		obj->dirty = 0;
 
-	for (i = 0; i < page_count; i++) {
+	for_each_sg(obj->pages->sgl, sg, page_count, i) {
+		struct page *page = sg_page(sg);
+
 		if (obj->dirty)
-			set_page_dirty(obj->pages[i]);
+			set_page_dirty(page);
 
 		if (obj->madv == I915_MADV_WILLNEED)
-			mark_page_accessed(obj->pages[i]);
+			mark_page_accessed(page);
 
-		page_cache_release(obj->pages[i]);
+		page_cache_release(page);
 	}
 	obj->dirty = 0;
 
-	drm_free_large(obj->pages);
+	sg_free_table(obj->pages);
+	kfree(obj->pages);
+}
+
+static int
+i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+{
+	const struct drm_i915_gem_object_ops *ops = obj->ops;
+
+	if (obj->pages == NULL)
+		return 0;
+
+	BUG_ON(obj->gtt_space);
+
+	if (obj->pages_pin_count)
+		return -EBUSY;
+
+	ops->put_pages(obj);
 	obj->pages = NULL;
+
+	list_del(&obj->gtt_list);
+	if (i915_gem_object_is_purgeable(obj))
+		i915_gem_object_truncate(obj);
+
+	return 0;
+}
+
+static long
+i915_gem_purge(struct drm_i915_private *dev_priv, long target)
+{
+	struct drm_i915_gem_object *obj, *next;
+	long count = 0;
+
+	list_for_each_entry_safe(obj, next,
+				 &dev_priv->mm.unbound_list,
+				 gtt_list) {
+		if (i915_gem_object_is_purgeable(obj) &&
+		    i915_gem_object_put_pages(obj) == 0) {
+			count += obj->base.size >> PAGE_SHIFT;
+			if (count >= target)
+				return count;
+		}
+	}
+
+	list_for_each_entry_safe(obj, next,
+				 &dev_priv->mm.inactive_list,
+				 mm_list) {
+		if (i915_gem_object_is_purgeable(obj) &&
+		    i915_gem_object_unbind(obj) == 0 &&
+		    i915_gem_object_put_pages(obj) == 0) {
+			count += obj->base.size >> PAGE_SHIFT;
+			if (count >= target)
+				return count;
+		}
+	}
+
+	return count;
+}
+
+static void
+i915_gem_shrink_all(struct drm_i915_private *dev_priv)
+{
+	struct drm_i915_gem_object *obj, *next;
+
+	i915_gem_evict_everything(dev_priv->dev);
+
+	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
+		i915_gem_object_put_pages(obj);
+}
+
+static int
+i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	int page_count, i;
+	struct address_space *mapping;
+	struct sg_table *st;
+	struct scatterlist *sg;
+	struct page *page;
+	gfp_t gfp;
+
+	/* Assert that the object is not currently in any GPU domain. As it
+	 * wasn't in the GTT, there shouldn't be any way it could have been in
+	 * a GPU cache
+	 */
+	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
+	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
+
+	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	if (st == NULL)
+		return -ENOMEM;
+
+	page_count = obj->base.size / PAGE_SIZE;
+	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
+		sg_free_table(st);
+		kfree(st);
+		return -ENOMEM;
+	}
+
+	/* Get the list of pages out of our struct file.  They'll be pinned
+	 * at this point until we release them.
+	 *
+	 * Fail silently without starting the shrinker
+	 */
+	mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
+	gfp = mapping_gfp_mask(mapping);
+	gfp |= __GFP_NORETRY | __GFP_NOWARN;
+	gfp &= ~(__GFP_IO | __GFP_WAIT);
+	for_each_sg(st->sgl, sg, page_count, i) {
+		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
+		if (IS_ERR(page)) {
+			i915_gem_purge(dev_priv, page_count);
+			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
+		}
+		if (IS_ERR(page)) {
+			/* We've tried hard to allocate the memory by reaping
+			 * our own buffer, now let the real VM do its job and
+			 * go down in flames if truly OOM.
+			 */
+			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN);
+			gfp |= __GFP_IO | __GFP_WAIT;
+
+			i915_gem_shrink_all(dev_priv);
+			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
+			if (IS_ERR(page))
+				goto err_pages;
+
+			gfp |= __GFP_NORETRY | __GFP_NOWARN;
+			gfp &= ~(__GFP_IO | __GFP_WAIT);
+		}
+
+		sg_set_page(sg, page, PAGE_SIZE, 0);
+	}
+
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		i915_gem_object_do_bit_17_swizzle(obj);
+
+	obj->pages = st;
+	return 0;
+
+err_pages:
+	for_each_sg(st->sgl, sg, i, page_count)
+		page_cache_release(sg_page(sg));
+	sg_free_table(st);
+	kfree(st);
+	return PTR_ERR(page);
+}
+
+/* Ensure that the associated pages are gathered from the backing storage
+ * and pinned into our object. i915_gem_object_get_pages() may be called
+ * multiple times before they are released by a single call to
+ * i915_gem_object_put_pages() - once the pages are no longer referenced
+ * either as a result of memory pressure (reaping pages under the shrinker)
+ * or as the object is itself released.
+ */
+int
+i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	const struct drm_i915_gem_object_ops *ops = obj->ops;
+	int ret;
+
+	if (obj->pages)
+		return 0;
+
+	BUG_ON(obj->pages_pin_count);
+
+	ret = ops->get_pages(obj);
+	if (ret)
+		return ret;
+
+	list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
+	return 0;
 }
 
 void
@@ -1441,7 +1878,7 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
 	list_move_tail(&obj->ring_list, &ring->active_list);
 
-	obj->last_rendering_seqno = seqno;
+	obj->last_read_seqno = seqno;
 
 	if (obj->fenced_gpu_access) {
 		obj->last_fenced_seqno = seqno;
@@ -1458,97 +1895,35 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 }
 
 static void
-i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
-{
-	list_del_init(&obj->ring_list);
-	obj->last_rendering_seqno = 0;
-	obj->last_fenced_seqno = 0;
-}
-
-static void
-i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
+i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 {
 	struct drm_device *dev = obj->base.dev;
-	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_private *dev_priv = dev->dev_private;
 
+	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
 	BUG_ON(!obj->active);
-	list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
-
-	i915_gem_object_move_off_active(obj);
-}
 
-static void
-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
-{
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
+	if (obj->pin_count) /* are we a framebuffer? */
+		intel_mark_fb_idle(obj);
 
 	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
-	BUG_ON(!list_empty(&obj->gpu_write_list));
-	BUG_ON(!obj->active);
+	list_del_init(&obj->ring_list);
 	obj->ring = NULL;
 
-	i915_gem_object_move_off_active(obj);
+	obj->last_read_seqno = 0;
+	obj->last_write_seqno = 0;
+	obj->base.write_domain = 0;
+
+	obj->last_fenced_seqno = 0;
 	obj->fenced_gpu_access = false;
 
 	obj->active = 0;
-	obj->pending_gpu_write = false;
 	drm_gem_object_unreference(&obj->base);
 
 	WARN_ON(i915_verify_lists(dev));
 }
 
-/* Immediately discard the backing storage */
-static void
-i915_gem_object_truncate(struct drm_i915_gem_object *obj)
-{
-	struct inode *inode;
-
-	/* Our goal here is to return as much of the memory as
-	 * is possible back to the system as we are called from OOM.
-	 * To do this we must instruct the shmfs to drop all of its
-	 * backing pages, *now*.
-	 */
-	inode = obj->base.filp->f_path.dentry->d_inode;
-	shmem_truncate_range(inode, 0, (loff_t)-1);
-
-	if (obj->base.map_list.map)
-		drm_gem_free_mmap_offset(&obj->base);
-
-	obj->madv = __I915_MADV_PURGED;
-}
-
-static inline int
-i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
-{
-	return obj->madv == I915_MADV_DONTNEED;
-}
-
-static void
-i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
-			       uint32_t flush_domains)
-{
-	struct drm_i915_gem_object *obj, *next;
-
-	list_for_each_entry_safe(obj, next,
-				 &ring->gpu_write_list,
-				 gpu_write_list) {
-		if (obj->base.write_domain & flush_domains) {
-			uint32_t old_write_domain = obj->base.write_domain;
-
-			obj->base.write_domain = 0;
-			list_del_init(&obj->gpu_write_list);
-			i915_gem_object_move_to_active(obj, ring,
-						       i915_gem_next_request_seqno(ring));
-
-			trace_i915_gem_object_change_domain(obj,
-							    obj->base.read_domains,
-							    old_write_domain);
-		}
-	}
-}
-
 static u32
 i915_gem_get_seqno(struct drm_device *dev)
 {
@@ -1589,15 +1964,16 @@ i915_add_request(struct intel_ring_buffer *ring,
 	 * is that the flush _must_ happen before the next request, no matter
 	 * what.
 	 */
-	if (ring->gpu_caches_dirty) {
-		ret = i915_gem_flush_ring(ring, 0, I915_GEM_GPU_DOMAINS);
-		if (ret)
-			return ret;
+	ret = intel_ring_flush_all_caches(ring);
+	if (ret)
+		return ret;
 
-		ring->gpu_caches_dirty = false;
+	if (request == NULL) {
+		request = kmalloc(sizeof(*request), GFP_KERNEL);
+		if (request == NULL)
+			return -ENOMEM;
 	}
 
-	BUG_ON(request == NULL);
 	seqno = i915_gem_next_request_seqno(ring);
 
 	/* Record the position of the start of the request so that
@@ -1608,8 +1984,10 @@ i915_add_request(struct intel_ring_buffer *ring,
 	request_ring_position = intel_ring_get_tail(ring);
 
 	ret = ring->add_request(ring, &seqno);
-	if (ret)
-	    return ret;
+	if (ret) {
+		kfree(request);
+		return ret;
+	}
 
 	trace_i915_gem_request_add(ring, seqno);
 
@@ -1619,6 +1997,7 @@ i915_add_request(struct intel_ring_buffer *ring,
 	request->emitted_jiffies = jiffies;
 	was_empty = list_empty(&ring->request_list);
 	list_add_tail(&request->list, &ring->request_list);
+	request->file_priv = NULL;
 
 	if (file) {
 		struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -1638,13 +2017,13 @@ i915_add_request(struct intel_ring_buffer *ring,
 				  jiffies +
 				  msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
 		}
-		if (was_empty)
+		if (was_empty) {
 			queue_delayed_work(dev_priv->wq,
 					   &dev_priv->mm.retire_work, HZ);
+			intel_mark_busy(dev_priv->dev);
+		}
 	}
 
-	WARN_ON(!list_empty(&ring->gpu_write_list));
-
 	return 0;
 }
 
@@ -1686,8 +2065,6 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
 				       struct drm_i915_gem_object,
 				       ring_list);
 
-		obj->base.write_domain = 0;
-		list_del_init(&obj->gpu_write_list);
 		i915_gem_object_move_to_inactive(obj);
 	}
 }
@@ -1723,20 +2100,6 @@ void i915_gem_reset(struct drm_device *dev)
 	for_each_ring(ring, dev_priv, i)
 		i915_gem_reset_ring_lists(dev_priv, ring);
 
-	/* Remove anything from the flushing lists. The GPU cache is likely
-	 * to be lost on reset along with the data, so simply move the
-	 * lost bo to the inactive list.
-	 */
-	while (!list_empty(&dev_priv->mm.flushing_list)) {
-		obj = list_first_entry(&dev_priv->mm.flushing_list,
-				      struct drm_i915_gem_object,
-				      mm_list);
-
-		obj->base.write_domain = 0;
-		list_del_init(&obj->gpu_write_list);
-		i915_gem_object_move_to_inactive(obj);
-	}
-
 	/* Move everything out of the GPU domains to ensure we do any
 	 * necessary invalidation upon reuse.
 	 */
@@ -1765,7 +2128,7 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 
 	WARN_ON(i915_verify_lists(ring->dev));
 
-	seqno = ring->get_seqno(ring);
+	seqno = ring->get_seqno(ring, true);
 
 	for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
 		if (seqno >= ring->sync_seqno[i])
@@ -1804,13 +2167,10 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 				      struct drm_i915_gem_object,
 				      ring_list);
 
-		if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
+		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
 			break;
 
-		if (obj->base.write_domain != 0)
-			i915_gem_object_move_to_flushing(obj);
-		else
-			i915_gem_object_move_to_inactive(obj);
+		i915_gem_object_move_to_inactive(obj);
 	}
 
 	if (unlikely(ring->trace_irq_seqno &&
@@ -1859,216 +2219,20 @@ i915_gem_retire_work_handler(struct work_struct *work)
 	 */
 	idle = true;
 	for_each_ring(ring, dev_priv, i) {
-		if (ring->gpu_caches_dirty) {
-			struct drm_i915_gem_request *request;
-
-			request = kzalloc(sizeof(*request), GFP_KERNEL);
-			if (request == NULL ||
-			    i915_add_request(ring, NULL, request))
-			    kfree(request);
-		}
+		if (ring->gpu_caches_dirty)
+			i915_add_request(ring, NULL, NULL);
 
 		idle &= list_empty(&ring->request_list);
 	}
 
 	if (!dev_priv->mm.suspended && !idle)
 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+	if (idle)
+		intel_mark_idle(dev);
 
 	mutex_unlock(&dev->struct_mutex);
 }
 
-int
-i915_gem_check_wedge(struct drm_i915_private *dev_priv,
-		     bool interruptible)
-{
-	if (atomic_read(&dev_priv->mm.wedged)) {
-		struct completion *x = &dev_priv->error_completion;
-		bool recovery_complete;
-		unsigned long flags;
-
-		/* Give the error handler a chance to run. */
-		spin_lock_irqsave(&x->wait.lock, flags);
-		recovery_complete = x->done > 0;
-		spin_unlock_irqrestore(&x->wait.lock, flags);
-
-		/* Non-interruptible callers can't handle -EAGAIN, hence return
-		 * -EIO unconditionally for these. */
-		if (!interruptible)
-			return -EIO;
-
-		/* Recovery complete, but still wedged means reset failure. */
-		if (recovery_complete)
-			return -EIO;
-
-		return -EAGAIN;
-	}
-
-	return 0;
-}
-
-/*
- * Compare seqno against outstanding lazy request. Emit a request if they are
- * equal.
- */
-static int
-i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
-{
-	int ret = 0;
-
-	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
-
-	if (seqno == ring->outstanding_lazy_request) {
-		struct drm_i915_gem_request *request;
-
-		request = kzalloc(sizeof(*request), GFP_KERNEL);
-		if (request == NULL)
-			return -ENOMEM;
-
-		ret = i915_add_request(ring, NULL, request);
-		if (ret) {
-			kfree(request);
-			return ret;
-		}
-
-		BUG_ON(seqno != request->seqno);
-	}
-
-	return ret;
-}
-
-/**
- * __wait_seqno - wait until execution of seqno has finished
- * @ring: the ring expected to report seqno
- * @seqno: duh!
- * @interruptible: do an interruptible wait (normally yes)
- * @timeout: in - how long to wait (NULL forever); out - how much time remaining
- *
- * Returns 0 if the seqno was found within the alloted time. Else returns the
- * errno with remaining time filled in timeout argument.
- */
-static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
-			bool interruptible, struct timespec *timeout)
-{
-	drm_i915_private_t *dev_priv = ring->dev->dev_private;
-	struct timespec before, now, wait_time={1,0};
-	unsigned long timeout_jiffies;
-	long end;
-	bool wait_forever = true;
-	int ret;
-
-	if (i915_seqno_passed(ring->get_seqno(ring), seqno))
-		return 0;
-
-	trace_i915_gem_request_wait_begin(ring, seqno);
-
-	if (timeout != NULL) {
-		wait_time = *timeout;
-		wait_forever = false;
-	}
-
-	timeout_jiffies = timespec_to_jiffies(&wait_time);
-
-	if (WARN_ON(!ring->irq_get(ring)))
-		return -ENODEV;
-
-	/* Record current time in case interrupted by signal, or wedged * */
-	getrawmonotonic(&before);
-
-#define EXIT_COND \
-	(i915_seqno_passed(ring->get_seqno(ring), seqno) || \
-	atomic_read(&dev_priv->mm.wedged))
-	do {
-		if (interruptible)
-			end = wait_event_interruptible_timeout(ring->irq_queue,
-							       EXIT_COND,
-							       timeout_jiffies);
-		else
-			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
-						 timeout_jiffies);
-
-		ret = i915_gem_check_wedge(dev_priv, interruptible);
-		if (ret)
-			end = ret;
-	} while (end == 0 && wait_forever);
-
-	getrawmonotonic(&now);
-
-	ring->irq_put(ring);
-	trace_i915_gem_request_wait_end(ring, seqno);
-#undef EXIT_COND
-
-	if (timeout) {
-		struct timespec sleep_time = timespec_sub(now, before);
-		*timeout = timespec_sub(*timeout, sleep_time);
-	}
-
-	switch (end) {
-	case -EIO:
-	case -EAGAIN: /* Wedged */
-	case -ERESTARTSYS: /* Signal */
-		return (int)end;
-	case 0: /* Timeout */
-		if (timeout)
-			set_normalized_timespec(timeout, 0, 0);
-		return -ETIME;
-	default: /* Completed */
-		WARN_ON(end < 0); /* We're not aware of other errors */
-		return 0;
-	}
-}
-
-/**
- * Waits for a sequence number to be signaled, and cleans up the
- * request and object lists appropriately for that event.
- */
-int
-i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
-{
-	drm_i915_private_t *dev_priv = ring->dev->dev_private;
-	int ret = 0;
-
-	BUG_ON(seqno == 0);
-
-	ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
-	if (ret)
-		return ret;
-
-	ret = i915_gem_check_olr(ring, seqno);
-	if (ret)
-		return ret;
-
-	ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible, NULL);
-
-	return ret;
-}
-
-/**
- * Ensures that all rendering to the object has completed and the object is
- * safe to unbind from the GTT or access from the CPU.
- */
-int
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
-{
-	int ret;
-
-	/* This function only exists to support waiting for existing rendering,
-	 * not for emitting required flushes.
-	 */
-	BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
-
-	/* If there is rendering queued on the buffer being evicted, wait for
-	 * it.
-	 */
-	if (obj->active) {
-		ret = i915_wait_seqno(obj->ring, obj->last_rendering_seqno);
-		if (ret)
-			return ret;
-		i915_gem_retire_requests_ring(obj->ring);
-	}
-
-	return 0;
-}
-
 /**
  * Ensures that an object will eventually get non-busy by flushing any required
  * write domains, emitting any outstanding lazy request and retiring and
@@ -2080,14 +2244,10 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
 	int ret;
 
 	if (obj->active) {
-		ret = i915_gem_object_flush_gpu_write_domain(obj);
+		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
 		if (ret)
 			return ret;
 
-		ret = i915_gem_check_olr(obj->ring,
-					 obj->last_rendering_seqno);
-		if (ret)
-			return ret;
 		i915_gem_retire_requests_ring(obj->ring);
 	}
 
@@ -2147,7 +2307,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		goto out;
 
 	if (obj->active) {
-		seqno = obj->last_rendering_seqno;
+		seqno = obj->last_read_seqno;
 		ring = obj->ring;
 	}
 
@@ -2202,11 +2362,11 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj,
 		return 0;
 
 	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
-		return i915_gem_object_wait_rendering(obj);
+		return i915_gem_object_wait_rendering(obj, false);
 
 	idx = intel_ring_sync_index(from, to);
 
-	seqno = obj->last_rendering_seqno;
+	seqno = obj->last_read_seqno;
 	if (seqno <= from->sync_seqno[idx])
 		return 0;
 
@@ -2260,6 +2420,8 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	if (obj->pin_count)
 		return -EBUSY;
 
+	BUG_ON(obj->pages == NULL);
+
 	ret = i915_gem_object_finish_gpu(obj);
 	if (ret)
 		return ret;
@@ -2270,22 +2432,6 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 
 	i915_gem_object_finish_gtt(obj);
 
-	/* Move the object to the CPU domain to ensure that
-	 * any possible CPU writes while it's not in the GTT
-	 * are flushed when we go to remap it.
-	 */
-	if (ret == 0)
-		ret = i915_gem_object_set_to_cpu_domain(obj, 1);
-	if (ret == -ERESTARTSYS)
-		return ret;
-	if (ret) {
-		/* In the event of a disaster, abandon all caches and
-		 * hope for the best.
-		 */
-		i915_gem_clflush_object(obj);
-		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
-
 	/* release the fence reg _after_ flushing */
 	ret = i915_gem_object_put_fence(obj);
 	if (ret)
@@ -2301,10 +2447,8 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	}
 	i915_gem_gtt_finish_object(obj);
 
-	i915_gem_object_put_pages_gtt(obj);
-
-	list_del_init(&obj->gtt_list);
-	list_del_init(&obj->mm_list);
+	list_del(&obj->mm_list);
+	list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
 	/* Avoid an unnecessary call to unbind on rebind. */
 	obj->map_and_fenceable = true;
 
@@ -2312,48 +2456,14 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	obj->gtt_space = NULL;
 	obj->gtt_offset = 0;
 
-	if (i915_gem_object_is_purgeable(obj))
-		i915_gem_object_truncate(obj);
-
-	return ret;
-}
-
-int
-i915_gem_flush_ring(struct intel_ring_buffer *ring,
-		    uint32_t invalidate_domains,
-		    uint32_t flush_domains)
-{
-	int ret;
-
-	if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
-		return 0;
-
-	trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
-
-	ret = ring->flush(ring, invalidate_domains, flush_domains);
-	if (ret)
-		return ret;
-
-	if (flush_domains & I915_GEM_GPU_DOMAINS)
-		i915_gem_process_flushing_list(ring, flush_domains);
-
 	return 0;
 }
 
 static int i915_ring_idle(struct intel_ring_buffer *ring)
 {
-	int ret;
-
-	if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
+	if (list_empty(&ring->active_list))
 		return 0;
 
-	if (!list_empty(&ring->gpu_write_list)) {
-		ret = i915_gem_flush_ring(ring,
-				    I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
-		if (ret)
-			return ret;
-	}
-
 	return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
 }
 
@@ -2372,10 +2482,6 @@ int i915_gpu_idle(struct drm_device *dev)
 		ret = i915_ring_idle(ring);
 		if (ret)
 			return ret;
-
-		/* Is the device fubar? */
-		if (WARN_ON(!list_empty(&ring->gpu_write_list)))
-			return -EBUSY;
 	}
 
 	return 0;
@@ -2548,21 +2654,8 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
 static int
 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
 {
-	int ret;
-
-	if (obj->fenced_gpu_access) {
-		if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
-			ret = i915_gem_flush_ring(obj->ring,
-						  0, obj->base.write_domain);
-			if (ret)
-				return ret;
-		}
-
-		obj->fenced_gpu_access = false;
-	}
-
 	if (obj->last_fenced_seqno) {
-		ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
+		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
 		if (ret)
 			return ret;
 
@@ -2575,6 +2668,7 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
 	if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
 		mb();
 
+	obj->fenced_gpu_access = false;
 	return 0;
 }
 
@@ -2694,18 +2788,88 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
 	return 0;
 }
 
+static bool i915_gem_valid_gtt_space(struct drm_device *dev,
+				     struct drm_mm_node *gtt_space,
+				     unsigned long cache_level)
+{
+	struct drm_mm_node *other;
+
+	/* On non-LLC machines we have to be careful when putting differing
+	 * types of snoopable memory together to avoid the prefetcher
+	 * crossing memory domains and dieing.
+	 */
+	if (HAS_LLC(dev))
+		return true;
+
+	if (gtt_space == NULL)
+		return true;
+
+	if (list_empty(&gtt_space->node_list))
+		return true;
+
+	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
+	if (other->allocated && !other->hole_follows && other->color != cache_level)
+		return false;
+
+	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
+	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
+		return false;
+
+	return true;
+}
+
+static void i915_gem_verify_gtt(struct drm_device *dev)
+{
+#if WATCH_GTT
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_i915_gem_object *obj;
+	int err = 0;
+
+	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+		if (obj->gtt_space == NULL) {
+			printk(KERN_ERR "object found on GTT list with no space reserved\n");
+			err++;
+			continue;
+		}
+
+		if (obj->cache_level != obj->gtt_space->color) {
+			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
+			       obj->gtt_space->start,
+			       obj->gtt_space->start + obj->gtt_space->size,
+			       obj->cache_level,
+			       obj->gtt_space->color);
+			err++;
+			continue;
+		}
+
+		if (!i915_gem_valid_gtt_space(dev,
+					      obj->gtt_space,
+					      obj->cache_level)) {
+			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
+			       obj->gtt_space->start,
+			       obj->gtt_space->start + obj->gtt_space->size,
+			       obj->cache_level);
+			err++;
+			continue;
+		}
+	}
+
+	WARN_ON(err);
+#endif
+}
+
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
 static int
 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 			    unsigned alignment,
-			    bool map_and_fenceable)
+			    bool map_and_fenceable,
+			    bool nonblocking)
 {
 	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_mm_node *free_space;
-	gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
 	u32 size, fence_size, fence_alignment, unfenced_alignment;
 	bool mappable, fenceable;
 	int ret;
@@ -2745,89 +2909,67 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 		return -E2BIG;
 	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ret;
+
  search_free:
 	if (map_and_fenceable)
 		free_space =
-			drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
-						    size, alignment,
-						    0, dev_priv->mm.gtt_mappable_end,
-						    0);
+			drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space,
+							  size, alignment, obj->cache_level,
+							  0, dev_priv->mm.gtt_mappable_end,
+							  false);
 	else
-		free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
-						size, alignment, 0);
+		free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space,
+						      size, alignment, obj->cache_level,
+						      false);
 
 	if (free_space != NULL) {
 		if (map_and_fenceable)
 			obj->gtt_space =
 				drm_mm_get_block_range_generic(free_space,
-							       size, alignment, 0,
+							       size, alignment, obj->cache_level,
 							       0, dev_priv->mm.gtt_mappable_end,
-							       0);
+							       false);
 		else
 			obj->gtt_space =
-				drm_mm_get_block(free_space, size, alignment);
+				drm_mm_get_block_generic(free_space,
+							 size, alignment, obj->cache_level,
+							 false);
 	}
 	if (obj->gtt_space == NULL) {
-		/* If the gtt is empty and we're still having trouble
-		 * fitting our object in, we're out of memory.
-		 */
 		ret = i915_gem_evict_something(dev, size, alignment,
-					       map_and_fenceable);
+					       obj->cache_level,
+					       map_and_fenceable,
+					       nonblocking);
 		if (ret)
 			return ret;
 
 		goto search_free;
 	}
-
-	ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
-	if (ret) {
+	if (WARN_ON(!i915_gem_valid_gtt_space(dev,
+					      obj->gtt_space,
+					      obj->cache_level))) {
 		drm_mm_put_block(obj->gtt_space);
 		obj->gtt_space = NULL;
-
-		if (ret == -ENOMEM) {
-			/* first try to reclaim some memory by clearing the GTT */
-			ret = i915_gem_evict_everything(dev, false);
-			if (ret) {
-				/* now try to shrink everyone else */
-				if (gfpmask) {
-					gfpmask = 0;
-					goto search_free;
-				}
-
-				return -ENOMEM;
-			}
-
-			goto search_free;
-		}
-
-		return ret;
+		return -EINVAL;
 	}
 
+
 	ret = i915_gem_gtt_prepare_object(obj);
 	if (ret) {
-		i915_gem_object_put_pages_gtt(obj);
 		drm_mm_put_block(obj->gtt_space);
 		obj->gtt_space = NULL;
-
-		if (i915_gem_evict_everything(dev, false))
-			return ret;
-
-		goto search_free;
+		return ret;
 	}
 
 	if (!dev_priv->mm.aliasing_ppgtt)
 		i915_gem_gtt_bind_object(obj, obj->cache_level);
 
-	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
+	list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
 	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
-	/* Assert that the object is not currently in any GPU domain. As it
-	 * wasn't in the GTT, there shouldn't be any way it could have been in
-	 * a GPU cache
-	 */
-	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
-	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
-
 	obj->gtt_offset = obj->gtt_space->start;
 
 	fenceable =
@@ -2840,6 +2982,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 	obj->map_and_fenceable = mappable && fenceable;
 
 	trace_i915_gem_object_bind(obj, map_and_fenceable);
+	i915_gem_verify_gtt(dev);
 	return 0;
 }
 
@@ -2866,18 +3009,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj)
 
 	trace_i915_gem_object_clflush(obj);
 
-	drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
-}
-
-/** Flushes any GPU write domain for the object if it's dirty. */
-static int
-i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
-{
-	if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
-		return 0;
-
-	/* Queue the GPU write cache flushing we need. */
-	return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
+	drm_clflush_sg(obj->pages);
 }
 
 /** Flushes the GTT write domain for the object if it's dirty. */
@@ -2946,16 +3078,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
 		return 0;
 
-	ret = i915_gem_object_flush_gpu_write_domain(obj);
+	ret = i915_gem_object_wait_rendering(obj, !write);
 	if (ret)
 		return ret;
 
-	if (obj->pending_gpu_write || write) {
-		ret = i915_gem_object_wait_rendering(obj);
-		if (ret)
-			return ret;
-	}
-
 	i915_gem_object_flush_cpu_write_domain(obj);
 
 	old_write_domain = obj->base.write_domain;
@@ -2998,6 +3124,12 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		return -EBUSY;
 	}
 
+	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
+		ret = i915_gem_object_unbind(obj);
+		if (ret)
+			return ret;
+	}
+
 	if (obj->gtt_space) {
 		ret = i915_gem_object_finish_gpu(obj);
 		if (ret)
@@ -3009,7 +3141,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		 * registers with snooped memory, so relinquish any fences
 		 * currently pointing to our region in the aperture.
 		 */
-		if (INTEL_INFO(obj->base.dev)->gen < 6) {
+		if (INTEL_INFO(dev)->gen < 6) {
 			ret = i915_gem_object_put_fence(obj);
 			if (ret)
 				return ret;
@@ -3020,6 +3152,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		if (obj->has_aliasing_ppgtt_mapping)
 			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
 					       obj, cache_level);
+
+		obj->gtt_space->color = cache_level;
 	}
 
 	if (cache_level == I915_CACHE_NONE) {
@@ -3046,9 +3180,72 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 	}
 
 	obj->cache_level = cache_level;
+	i915_gem_verify_gtt(dev);
 	return 0;
 }
 
+int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file)
+{
+	struct drm_i915_gem_caching *args = data;
+	struct drm_i915_gem_object *obj;
+	int ret;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+	if (&obj->base == NULL) {
+		ret = -ENOENT;
+		goto unlock;
+	}
+
+	args->caching = obj->cache_level != I915_CACHE_NONE;
+
+	drm_gem_object_unreference(&obj->base);
+unlock:
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file)
+{
+	struct drm_i915_gem_caching *args = data;
+	struct drm_i915_gem_object *obj;
+	enum i915_cache_level level;
+	int ret;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	switch (args->caching) {
+	case I915_CACHING_NONE:
+		level = I915_CACHE_NONE;
+		break;
+	case I915_CACHING_CACHED:
+		level = I915_CACHE_LLC;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+	if (&obj->base == NULL) {
+		ret = -ENOENT;
+		goto unlock;
+	}
+
+	ret = i915_gem_object_set_cache_level(obj, level);
+
+	drm_gem_object_unreference(&obj->base);
+unlock:
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
 /*
  * Prepare buffer for display plane (scanout, cursors, etc).
  * Can be called from an uninterruptible phase (modesetting) and allows
@@ -3062,10 +3259,6 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	u32 old_read_domains, old_write_domain;
 	int ret;
 
-	ret = i915_gem_object_flush_gpu_write_domain(obj);
-	if (ret)
-		return ret;
-
 	if (pipelined != obj->ring) {
 		ret = i915_gem_object_sync(obj, pipelined);
 		if (ret)
@@ -3089,7 +3282,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	 * (e.g. libkms for the bootup splash), we have to ensure that we
 	 * always use map_and_fenceable for all scanout buffers.
 	 */
-	ret = i915_gem_object_pin(obj, alignment, true);
+	ret = i915_gem_object_pin(obj, alignment, true, false);
 	if (ret)
 		return ret;
 
@@ -3101,7 +3294,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+	obj->base.write_domain = 0;
 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 
 	trace_i915_gem_object_change_domain(obj,
@@ -3119,13 +3312,7 @@ i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
 	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
 		return 0;
 
-	if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
-		ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
-		if (ret)
-			return ret;
-	}
-
-	ret = i915_gem_object_wait_rendering(obj);
+	ret = i915_gem_object_wait_rendering(obj, false);
 	if (ret)
 		return ret;
 
@@ -3149,16 +3336,10 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
 		return 0;
 
-	ret = i915_gem_object_flush_gpu_write_domain(obj);
+	ret = i915_gem_object_wait_rendering(obj, !write);
 	if (ret)
 		return ret;
 
-	if (write || obj->pending_gpu_write) {
-		ret = i915_gem_object_wait_rendering(obj);
-		if (ret)
-			return ret;
-	}
-
 	i915_gem_object_flush_gtt_write_domain(obj);
 
 	old_write_domain = obj->base.write_domain;
@@ -3238,7 +3419,8 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 int
 i915_gem_object_pin(struct drm_i915_gem_object *obj,
 		    uint32_t alignment,
-		    bool map_and_fenceable)
+		    bool map_and_fenceable,
+		    bool nonblocking)
 {
 	int ret;
 
@@ -3263,7 +3445,8 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
 
 	if (obj->gtt_space == NULL) {
 		ret = i915_gem_object_bind_to_gtt(obj, alignment,
-						  map_and_fenceable);
+						  map_and_fenceable,
+						  nonblocking);
 		if (ret)
 			return ret;
 	}
@@ -3321,7 +3504,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 	obj->user_pin_count++;
 	obj->pin_filp = file;
 	if (obj->user_pin_count == 1) {
-		ret = i915_gem_object_pin(obj, args->alignment, true);
+		ret = i915_gem_object_pin(obj, args->alignment, true, false);
 		if (ret)
 			goto out;
 	}
@@ -3401,6 +3584,10 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 	ret = i915_gem_object_flush_active(obj);
 
 	args->busy = obj->active;
+	if (obj->ring) {
+		BUILD_BUG_ON(I915_NUM_RINGS > 16);
+		args->busy |= intel_ring_flag(obj->ring) << 16;
+	}
 
 	drm_gem_object_unreference(&obj->base);
 unlock:
@@ -3449,9 +3636,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	if (obj->madv != __I915_MADV_PURGED)
 		obj->madv = args->madv;
 
-	/* if the object is no longer bound, discard its backing storage */
-	if (i915_gem_object_is_purgeable(obj) &&
-	    obj->gtt_space == NULL)
+	/* if the object is no longer attached, discard its backing storage */
+	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
 		i915_gem_object_truncate(obj);
 
 	args->retained = obj->madv != __I915_MADV_PURGED;
@@ -3463,10 +3649,32 @@ unlock:
 	return ret;
 }
 
+void i915_gem_object_init(struct drm_i915_gem_object *obj,
+			  const struct drm_i915_gem_object_ops *ops)
+{
+	INIT_LIST_HEAD(&obj->mm_list);
+	INIT_LIST_HEAD(&obj->gtt_list);
+	INIT_LIST_HEAD(&obj->ring_list);
+	INIT_LIST_HEAD(&obj->exec_list);
+
+	obj->ops = ops;
+
+	obj->fence_reg = I915_FENCE_REG_NONE;
+	obj->madv = I915_MADV_WILLNEED;
+	/* Avoid an unnecessary call to unbind on the first bind. */
+	obj->map_and_fenceable = true;
+
+	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
+	.get_pages = i915_gem_object_get_pages_gtt,
+	.put_pages = i915_gem_object_put_pages_gtt,
+};
+
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
 						  size_t size)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	struct address_space *mapping;
 	u32 mask;
@@ -3490,7 +3698,7 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
 	mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 	mapping_set_gfp_mask(mapping, mask);
 
-	i915_gem_info_add_obj(dev_priv, size);
+	i915_gem_object_init(obj, &i915_gem_object_ops);
 
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
@@ -3512,17 +3720,6 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
 	} else
 		obj->cache_level = I915_CACHE_NONE;
 
-	obj->base.driver_private = NULL;
-	obj->fence_reg = I915_FENCE_REG_NONE;
-	INIT_LIST_HEAD(&obj->mm_list);
-	INIT_LIST_HEAD(&obj->gtt_list);
-	INIT_LIST_HEAD(&obj->ring_list);
-	INIT_LIST_HEAD(&obj->exec_list);
-	INIT_LIST_HEAD(&obj->gpu_write_list);
-	obj->madv = I915_MADV_WILLNEED;
-	/* Avoid an unnecessary call to unbind on the first bind. */
-	obj->map_and_fenceable = true;
-
 	return obj;
 }
 
@@ -3541,9 +3738,6 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 
 	trace_i915_gem_object_destroy(obj);
 
-	if (gem_obj->import_attach)
-		drm_prime_gem_destroy(gem_obj, obj->sg_table);
-
 	if (obj->phys_obj)
 		i915_gem_detach_phys_object(dev, obj);
 
@@ -3559,8 +3753,14 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 		dev_priv->mm.interruptible = was_interruptible;
 	}
 
-	if (obj->base.map_list.map)
-		drm_gem_free_mmap_offset(&obj->base);
+	obj->pages_pin_count = 0;
+	i915_gem_object_put_pages(obj);
+	i915_gem_object_free_mmap_offset(obj);
+
+	BUG_ON(obj->pages);
+
+	if (obj->base.import_attach)
+		drm_prime_gem_destroy(&obj->base, NULL);
 
 	drm_gem_object_release(&obj->base);
 	i915_gem_info_remove_obj(dev_priv, obj->base.size);
@@ -3591,7 +3791,7 @@ i915_gem_idle(struct drm_device *dev)
 
 	/* Under UMS, be paranoid and evict. */
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		i915_gem_evict_everything(dev, false);
+		i915_gem_evict_everything(dev);
 
 	i915_gem_reset_fences(dev);
 
@@ -3892,7 +4092,6 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 	}
 
 	BUG_ON(!list_empty(&dev_priv->mm.active_list));
-	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
 	BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
 	mutex_unlock(&dev->struct_mutex);
 
@@ -3940,7 +4139,6 @@ init_ring_lists(struct intel_ring_buffer *ring)
 {
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
-	INIT_LIST_HEAD(&ring->gpu_write_list);
 }
 
 void
@@ -3950,10 +4148,10 @@ i915_gem_load(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 
 	INIT_LIST_HEAD(&dev_priv->mm.active_list);
-	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
 	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
+	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
+	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
 	for (i = 0; i < I915_NUM_RINGS; i++)
 		init_ring_lists(&dev_priv->ring[i]);
 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
@@ -4198,18 +4396,6 @@ void i915_gem_release(struct drm_device *dev, struct drm_file *file)
 }
 
 static int
-i915_gpu_is_active(struct drm_device *dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	int lists_empty;
-
-	lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
-		      list_empty(&dev_priv->mm.active_list);
-
-	return !lists_empty;
-}
-
-static int
 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct drm_i915_private *dev_priv =
@@ -4217,60 +4403,27 @@ i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
 			     struct drm_i915_private,
 			     mm.inactive_shrinker);
 	struct drm_device *dev = dev_priv->dev;
-	struct drm_i915_gem_object *obj, *next;
+	struct drm_i915_gem_object *obj;
 	int nr_to_scan = sc->nr_to_scan;
 	int cnt;
 
 	if (!mutex_trylock(&dev->struct_mutex))
 		return 0;
 
-	/* "fast-path" to count number of available objects */
-	if (nr_to_scan == 0) {
-		cnt = 0;
-		list_for_each_entry(obj,
-				    &dev_priv->mm.inactive_list,
-				    mm_list)
-			cnt++;
-		mutex_unlock(&dev->struct_mutex);
-		return cnt / 100 * sysctl_vfs_cache_pressure;
-	}
-
-rescan:
-	/* first scan for clean buffers */
-	i915_gem_retire_requests(dev);
-
-	list_for_each_entry_safe(obj, next,
-				 &dev_priv->mm.inactive_list,
-				 mm_list) {
-		if (i915_gem_object_is_purgeable(obj)) {
-			if (i915_gem_object_unbind(obj) == 0 &&
-			    --nr_to_scan == 0)
-				break;
-		}
+	if (nr_to_scan) {
+		nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
+		if (nr_to_scan > 0)
+			i915_gem_shrink_all(dev_priv);
 	}
 
-	/* second pass, evict/count anything still on the inactive list */
 	cnt = 0;
-	list_for_each_entry_safe(obj, next,
-				 &dev_priv->mm.inactive_list,
-				 mm_list) {
-		if (nr_to_scan &&
-		    i915_gem_object_unbind(obj) == 0)
-			nr_to_scan--;
-		else
-			cnt++;
-	}
+	list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
+		if (obj->pages_pin_count == 0)
+			cnt += obj->base.size >> PAGE_SHIFT;
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
+		if (obj->pin_count == 0 && obj->pages_pin_count == 0)
+			cnt += obj->base.size >> PAGE_SHIFT;
 
-	if (nr_to_scan && i915_gpu_is_active(dev)) {
-		/*
-		 * We are desperate for pages, so as a last resort, wait
-		 * for the GPU to finish and discard whatever we can.
-		 * This has a dramatic impact to reduce the number of
-		 * OOM-killer events whilst running the GPU aggressively.
-		 */
-		if (i915_gpu_idle(dev) == 0)
-			goto rescan;
-	}
 	mutex_unlock(&dev->struct_mutex);
-	return cnt / 100 * sysctl_vfs_cache_pressure;
+	return cnt;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index a9d58d72bb4..1eb48faf741 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -85,8 +85,8 @@
  *
  */
 
-#include "drmP.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 /* This is a HW constraint. The value below is the largest known requirement
@@ -97,8 +97,7 @@
 
 static struct i915_hw_context *
 i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
-static int do_switch(struct drm_i915_gem_object *from_obj,
-		     struct i915_hw_context *to, u32 seqno);
+static int do_switch(struct i915_hw_context *to);
 
 static int get_context_size(struct drm_device *dev)
 {
@@ -113,7 +112,10 @@ static int get_context_size(struct drm_device *dev)
 		break;
 	case 7:
 		reg = I915_READ(GEN7_CXT_SIZE);
-		ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
+		if (IS_HASWELL(dev))
+			ret = HSW_CXT_TOTAL_SIZE(reg) * 64;
+		else
+			ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
 		break;
 	default:
 		BUG();
@@ -219,20 +221,21 @@ static int create_default_context(struct drm_i915_private *dev_priv)
 	 * default context.
 	 */
 	dev_priv->ring[RCS].default_context = ctx;
-	ret = i915_gem_object_pin(ctx->obj, CONTEXT_ALIGN, false);
-	if (ret) {
-		do_destroy(ctx);
-		return ret;
-	}
+	ret = i915_gem_object_pin(ctx->obj, CONTEXT_ALIGN, false, false);
+	if (ret)
+		goto err_destroy;
 
-	ret = do_switch(NULL, ctx, 0);
-	if (ret) {
-		i915_gem_object_unpin(ctx->obj);
-		do_destroy(ctx);
-	} else {
-		DRM_DEBUG_DRIVER("Default HW context loaded\n");
-	}
+	ret = do_switch(ctx);
+	if (ret)
+		goto err_unpin;
 
+	DRM_DEBUG_DRIVER("Default HW context loaded\n");
+	return 0;
+
+err_unpin:
+	i915_gem_object_unpin(ctx->obj);
+err_destroy:
+	do_destroy(ctx);
 	return ret;
 }
 
@@ -359,18 +362,19 @@ mi_set_context(struct intel_ring_buffer *ring,
 	return ret;
 }
 
-static int do_switch(struct drm_i915_gem_object *from_obj,
-		     struct i915_hw_context *to,
-		     u32 seqno)
+static int do_switch(struct i915_hw_context *to)
 {
-	struct intel_ring_buffer *ring = NULL;
+	struct intel_ring_buffer *ring = to->ring;
+	struct drm_i915_gem_object *from_obj = ring->last_context_obj;
 	u32 hw_flags = 0;
 	int ret;
 
-	BUG_ON(to == NULL);
 	BUG_ON(from_obj != NULL && from_obj->pin_count == 0);
 
-	ret = i915_gem_object_pin(to->obj, CONTEXT_ALIGN, false);
+	if (from_obj == to->obj)
+		return 0;
+
+	ret = i915_gem_object_pin(to->obj, CONTEXT_ALIGN, false, false);
 	if (ret)
 		return ret;
 
@@ -393,7 +397,6 @@ static int do_switch(struct drm_i915_gem_object *from_obj,
 	else if (WARN_ON_ONCE(from_obj == to->obj)) /* not yet expected */
 		hw_flags |= MI_FORCE_RESTORE;
 
-	ring = to->ring;
 	ret = mi_set_context(ring, to, hw_flags);
 	if (ret) {
 		i915_gem_object_unpin(to->obj);
@@ -407,6 +410,7 @@ static int do_switch(struct drm_i915_gem_object *from_obj,
 	 * MI_SET_CONTEXT instead of when the next seqno has completed.
 	 */
 	if (from_obj != NULL) {
+		u32 seqno = i915_gem_next_request_seqno(ring);
 		from_obj->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
 		i915_gem_object_move_to_active(from_obj, ring, seqno);
 		/* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
@@ -417,7 +421,7 @@ static int do_switch(struct drm_i915_gem_object *from_obj,
 		 * swapped, but there is no way to do that yet.
 		 */
 		from_obj->dirty = 1;
-		BUG_ON(from_obj->ring != to->ring);
+		BUG_ON(from_obj->ring != ring);
 		i915_gem_object_unpin(from_obj);
 
 		drm_gem_object_unreference(&from_obj->base);
@@ -448,9 +452,7 @@ int i915_switch_context(struct intel_ring_buffer *ring,
 			int to_id)
 {
 	struct drm_i915_private *dev_priv = ring->dev->dev_private;
-	struct drm_i915_file_private *file_priv = NULL;
 	struct i915_hw_context *to;
-	struct drm_i915_gem_object *from_obj = ring->last_context_obj;
 
 	if (dev_priv->hw_contexts_disabled)
 		return 0;
@@ -458,21 +460,18 @@ int i915_switch_context(struct intel_ring_buffer *ring,
 	if (ring != &dev_priv->ring[RCS])
 		return 0;
 
-	if (file)
-		file_priv = file->driver_priv;
-
 	if (to_id == DEFAULT_CONTEXT_ID) {
 		to = ring->default_context;
 	} else {
-		to = i915_gem_context_get(file_priv, to_id);
+		if (file == NULL)
+			return -EINVAL;
+
+		to = i915_gem_context_get(file->driver_priv, to_id);
 		if (to == NULL)
 			return -ENOENT;
 	}
 
-	if (from_obj == to->obj)
-		return 0;
-
-	return do_switch(from_obj, to, i915_gem_next_request_seqno(to->ring));
+	return do_switch(to);
 }
 
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index bddf7bed183..582e6a5f3da 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -25,9 +25,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 #if WATCH_LISTS
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index aa308e1337d..773ef77b6c2 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -23,40 +23,67 @@
  * Authors:
  *	Dave Airlie <airlied@redhat.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "i915_drv.h"
 #include <linux/dma-buf.h>
 
 static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment,
-				      enum dma_data_direction dir)
+					     enum dma_data_direction dir)
 {
 	struct drm_i915_gem_object *obj = attachment->dmabuf->priv;
-	struct drm_device *dev = obj->base.dev;
-	int npages = obj->base.size / PAGE_SIZE;
-	struct sg_table *sg = NULL;
-	int ret;
-	int nents;
+	struct sg_table *st;
+	struct scatterlist *src, *dst;
+	int ret, i;
 
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_mutex_lock_interruptible(obj->base.dev);
 	if (ret)
 		return ERR_PTR(ret);
 
-	if (!obj->pages) {
-		ret = i915_gem_object_get_pages_gtt(obj, __GFP_NORETRY | __GFP_NOWARN);
-		if (ret)
-			goto out;
+	ret = i915_gem_object_get_pages(obj);
+	if (ret) {
+		st = ERR_PTR(ret);
+		goto out;
+	}
+
+	/* Copy sg so that we make an independent mapping */
+	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+	if (st == NULL) {
+		st = ERR_PTR(-ENOMEM);
+		goto out;
 	}
 
-	/* link the pages into an SG then map the sg */
-	sg = drm_prime_pages_to_sg(obj->pages, npages);
-	nents = dma_map_sg(attachment->dev, sg->sgl, sg->nents, dir);
+	ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL);
+	if (ret) {
+		kfree(st);
+		st = ERR_PTR(ret);
+		goto out;
+	}
+
+	src = obj->pages->sgl;
+	dst = st->sgl;
+	for (i = 0; i < obj->pages->nents; i++) {
+		sg_set_page(dst, sg_page(src), PAGE_SIZE, 0);
+		dst = sg_next(dst);
+		src = sg_next(src);
+	}
+
+	if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {
+		sg_free_table(st);
+		kfree(st);
+		st = ERR_PTR(-ENOMEM);
+		goto out;
+	}
+
+	i915_gem_object_pin_pages(obj);
+
 out:
-	mutex_unlock(&dev->struct_mutex);
-	return sg;
+	mutex_unlock(&obj->base.dev->struct_mutex);
+	return st;
 }
 
 static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,
-			    struct sg_table *sg, enum dma_data_direction dir)
+				   struct sg_table *sg,
+				   enum dma_data_direction dir)
 {
 	dma_unmap_sg(attachment->dev, sg->sgl, sg->nents, dir);
 	sg_free_table(sg);
@@ -78,7 +105,9 @@ static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 {
 	struct drm_i915_gem_object *obj = dma_buf->priv;
 	struct drm_device *dev = obj->base.dev;
-	int ret;
+	struct scatterlist *sg;
+	struct page **pages;
+	int ret, i;
 
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
@@ -89,24 +118,34 @@ static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 		goto out_unlock;
 	}
 
-	if (!obj->pages) {
-		ret = i915_gem_object_get_pages_gtt(obj, __GFP_NORETRY | __GFP_NOWARN);
-		if (ret) {
-			mutex_unlock(&dev->struct_mutex);
-			return ERR_PTR(ret);
-		}
-	}
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		goto error;
 
-	obj->dma_buf_vmapping = vmap(obj->pages, obj->base.size / PAGE_SIZE, 0, PAGE_KERNEL);
-	if (!obj->dma_buf_vmapping) {
-		DRM_ERROR("failed to vmap object\n");
-		goto out_unlock;
-	}
+	ret = -ENOMEM;
+
+	pages = drm_malloc_ab(obj->pages->nents, sizeof(struct page *));
+	if (pages == NULL)
+		goto error;
+
+	for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i)
+		pages[i] = sg_page(sg);
+
+	obj->dma_buf_vmapping = vmap(pages, obj->pages->nents, 0, PAGE_KERNEL);
+	drm_free_large(pages);
+
+	if (!obj->dma_buf_vmapping)
+		goto error;
 
 	obj->vmapping_count = 1;
+	i915_gem_object_pin_pages(obj);
 out_unlock:
 	mutex_unlock(&dev->struct_mutex);
 	return obj->dma_buf_vmapping;
+
+error:
+	mutex_unlock(&dev->struct_mutex);
+	return ERR_PTR(ret);
 }
 
 static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
@@ -119,10 +158,11 @@ static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 	if (ret)
 		return;
 
-	--obj->vmapping_count;
-	if (obj->vmapping_count == 0) {
+	if (--obj->vmapping_count == 0) {
 		vunmap(obj->dma_buf_vmapping);
 		obj->dma_buf_vmapping = NULL;
+
+		i915_gem_object_unpin_pages(obj);
 	}
 	mutex_unlock(&dev->struct_mutex);
 }
@@ -151,6 +191,22 @@ static int i915_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *
 	return -EINVAL;
 }
 
+static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, size_t start, size_t length, enum dma_data_direction direction)
+{
+	struct drm_i915_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->base.dev;
+	int ret;
+	bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_object_set_to_cpu_domain(obj, write);
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
 static const struct dma_buf_ops i915_dmabuf_ops =  {
 	.map_dma_buf = i915_gem_map_dma_buf,
 	.unmap_dma_buf = i915_gem_unmap_dma_buf,
@@ -162,25 +218,47 @@ static const struct dma_buf_ops i915_dmabuf_ops =  {
 	.mmap = i915_gem_dmabuf_mmap,
 	.vmap = i915_gem_dmabuf_vmap,
 	.vunmap = i915_gem_dmabuf_vunmap,
+	.begin_cpu_access = i915_gem_begin_cpu_access,
 };
 
 struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
-				struct drm_gem_object *gem_obj, int flags)
+				      struct drm_gem_object *gem_obj, int flags)
 {
 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 
-	return dma_buf_export(obj, &i915_dmabuf_ops,
-						  obj->base.size, 0600);
+	return dma_buf_export(obj, &i915_dmabuf_ops, obj->base.size, 0600);
+}
+
+static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
+{
+	struct sg_table *sg;
+
+	sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sg))
+		return PTR_ERR(sg);
+
+	obj->pages = sg;
+	obj->has_dma_mapping = true;
+	return 0;
 }
 
+static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
+{
+	dma_buf_unmap_attachment(obj->base.import_attach,
+				 obj->pages, DMA_BIDIRECTIONAL);
+	obj->has_dma_mapping = false;
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
+	.get_pages = i915_gem_object_get_pages_dmabuf,
+	.put_pages = i915_gem_object_put_pages_dmabuf,
+};
+
 struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
-				struct dma_buf *dma_buf)
+					     struct dma_buf *dma_buf)
 {
 	struct dma_buf_attachment *attach;
-	struct sg_table *sg;
 	struct drm_i915_gem_object *obj;
-	int npages;
-	int size;
 	int ret;
 
 	/* is this one of own objects? */
@@ -198,34 +276,24 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 	if (IS_ERR(attach))
 		return ERR_CAST(attach);
 
-	sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sg)) {
-		ret = PTR_ERR(sg);
-		goto fail_detach;
-	}
-
-	size = dma_buf->size;
-	npages = size / PAGE_SIZE;
 
 	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 	if (obj == NULL) {
 		ret = -ENOMEM;
-		goto fail_unmap;
+		goto fail_detach;
 	}
 
-	ret = drm_gem_private_object_init(dev, &obj->base, size);
+	ret = drm_gem_private_object_init(dev, &obj->base, dma_buf->size);
 	if (ret) {
 		kfree(obj);
-		goto fail_unmap;
+		goto fail_detach;
 	}
 
-	obj->sg_table = sg;
+	i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops);
 	obj->base.import_attach = attach;
 
 	return &obj->base;
 
-fail_unmap:
-	dma_buf_unmap_attachment(attach, sg, DMA_BIDIRECTIONAL);
 fail_detach:
 	dma_buf_detach(dma_buf, attach);
 	return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index eba0308f10e..776a3225184 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -26,10 +26,9 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "i915_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_trace.h"
 
 static bool
@@ -44,7 +43,8 @@ mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
 
 int
 i915_gem_evict_something(struct drm_device *dev, int min_size,
-			 unsigned alignment, bool mappable)
+			 unsigned alignment, unsigned cache_level,
+			 bool mappable, bool nonblocking)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct list_head eviction_list, unwind_list;
@@ -79,11 +79,11 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
 	INIT_LIST_HEAD(&unwind_list);
 	if (mappable)
 		drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space,
-					    min_size, alignment, 0,
+					    min_size, alignment, cache_level,
 					    0, dev_priv->mm.gtt_mappable_end);
 	else
 		drm_mm_init_scan(&dev_priv->mm.gtt_space,
-				 min_size, alignment, 0);
+				 min_size, alignment, cache_level);
 
 	/* First see if there is a large enough contiguous idle region... */
 	list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) {
@@ -91,29 +91,16 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
 			goto found;
 	}
 
-	/* Now merge in the soon-to-be-expired objects... */
-	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
-		/* Does the object require an outstanding flush? */
-		if (obj->base.write_domain)
-			continue;
-
-		if (mark_free(obj, &unwind_list))
-			goto found;
-	}
+	if (nonblocking)
+		goto none;
 
-	/* Finally add anything with a pending flush (in order of retirement) */
-	list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
-		if (mark_free(obj, &unwind_list))
-			goto found;
-	}
+	/* Now merge in the soon-to-be-expired objects... */
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
-		if (!obj->base.write_domain)
-			continue;
-
 		if (mark_free(obj, &unwind_list))
 			goto found;
 	}
 
+none:
 	/* Nothing found, clean up and bail out! */
 	while (!list_empty(&unwind_list)) {
 		obj = list_first_entry(&unwind_list,
@@ -164,7 +151,7 @@ found:
 }
 
 int
-i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
+i915_gem_evict_everything(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj, *next;
@@ -172,12 +159,11 @@ i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 	int ret;
 
 	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
-		       list_empty(&dev_priv->mm.flushing_list) &&
 		       list_empty(&dev_priv->mm.active_list));
 	if (lists_empty)
 		return -ENOSPC;
 
-	trace_i915_gem_evict_everything(dev, purgeable_only);
+	trace_i915_gem_evict_everything(dev);
 
 	/* The gpu_idle will flush everything in the write domain to the
 	 * active list. Then we must move everything off the active list
@@ -189,16 +175,11 @@ i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 
 	i915_gem_retire_requests(dev);
 
-	BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
-
 	/* Having flushed everything, unbind() should never raise an error */
 	list_for_each_entry_safe(obj, next,
-				 &dev_priv->mm.inactive_list, mm_list) {
-		if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
-			if (obj->pin_count == 0)
-				WARN_ON(i915_gem_object_unbind(obj));
-		}
-	}
+				 &dev_priv->mm.inactive_list, mm_list)
+		if (obj->pin_count == 0)
+			WARN_ON(i915_gem_object_unbind(obj));
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index ff2819ea081..3eea143749f 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -26,188 +26,13 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 #include <linux/dma_remapping.h>
 
-struct change_domains {
-	uint32_t invalidate_domains;
-	uint32_t flush_domains;
-	uint32_t flush_rings;
-	uint32_t flips;
-};
-
-/*
- * Set the next domain for the specified object. This
- * may not actually perform the necessary flushing/invaliding though,
- * as that may want to be batched with other set_domain operations
- *
- * This is (we hope) the only really tricky part of gem. The goal
- * is fairly simple -- track which caches hold bits of the object
- * and make sure they remain coherent. A few concrete examples may
- * help to explain how it works. For shorthand, we use the notation
- * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
- * a pair of read and write domain masks.
- *
- * Case 1: the batch buffer
- *
- *	1. Allocated
- *	2. Written by CPU
- *	3. Mapped to GTT
- *	4. Read by GPU
- *	5. Unmapped from GTT
- *	6. Freed
- *
- *	Let's take these a step at a time
- *
- *	1. Allocated
- *		Pages allocated from the kernel may still have
- *		cache contents, so we set them to (CPU, CPU) always.
- *	2. Written by CPU (using pwrite)
- *		The pwrite function calls set_domain (CPU, CPU) and
- *		this function does nothing (as nothing changes)
- *	3. Mapped by GTT
- *		This function asserts that the object is not
- *		currently in any GPU-based read or write domains
- *	4. Read by GPU
- *		i915_gem_execbuffer calls set_domain (COMMAND, 0).
- *		As write_domain is zero, this function adds in the
- *		current read domains (CPU+COMMAND, 0).
- *		flush_domains is set to CPU.
- *		invalidate_domains is set to COMMAND
- *		clflush is run to get data out of the CPU caches
- *		then i915_dev_set_domain calls i915_gem_flush to
- *		emit an MI_FLUSH and drm_agp_chipset_flush
- *	5. Unmapped from GTT
- *		i915_gem_object_unbind calls set_domain (CPU, CPU)
- *		flush_domains and invalidate_domains end up both zero
- *		so no flushing/invalidating happens
- *	6. Freed
- *		yay, done
- *
- * Case 2: The shared render buffer
- *
- *	1. Allocated
- *	2. Mapped to GTT
- *	3. Read/written by GPU
- *	4. set_domain to (CPU,CPU)
- *	5. Read/written by CPU
- *	6. Read/written by GPU
- *
- *	1. Allocated
- *		Same as last example, (CPU, CPU)
- *	2. Mapped to GTT
- *		Nothing changes (assertions find that it is not in the GPU)
- *	3. Read/written by GPU
- *		execbuffer calls set_domain (RENDER, RENDER)
- *		flush_domains gets CPU
- *		invalidate_domains gets GPU
- *		clflush (obj)
- *		MI_FLUSH and drm_agp_chipset_flush
- *	4. set_domain (CPU, CPU)
- *		flush_domains gets GPU
- *		invalidate_domains gets CPU
- *		wait_rendering (obj) to make sure all drawing is complete.
- *		This will include an MI_FLUSH to get the data from GPU
- *		to memory
- *		clflush (obj) to invalidate the CPU cache
- *		Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
- *	5. Read/written by CPU
- *		cache lines are loaded and dirtied
- *	6. Read written by GPU
- *		Same as last GPU access
- *
- * Case 3: The constant buffer
- *
- *	1. Allocated
- *	2. Written by CPU
- *	3. Read by GPU
- *	4. Updated (written) by CPU again
- *	5. Read by GPU
- *
- *	1. Allocated
- *		(CPU, CPU)
- *	2. Written by CPU
- *		(CPU, CPU)
- *	3. Read by GPU
- *		(CPU+RENDER, 0)
- *		flush_domains = CPU
- *		invalidate_domains = RENDER
- *		clflush (obj)
- *		MI_FLUSH
- *		drm_agp_chipset_flush
- *	4. Updated (written) by CPU again
- *		(CPU, CPU)
- *		flush_domains = 0 (no previous write domain)
- *		invalidate_domains = 0 (no new read domains)
- *	5. Read by GPU
- *		(CPU+RENDER, 0)
- *		flush_domains = CPU
- *		invalidate_domains = RENDER
- *		clflush (obj)
- *		MI_FLUSH
- *		drm_agp_chipset_flush
- */
-static void
-i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
-				  struct intel_ring_buffer *ring,
-				  struct change_domains *cd)
-{
-	uint32_t invalidate_domains = 0, flush_domains = 0;
-
-	/*
-	 * If the object isn't moving to a new write domain,
-	 * let the object stay in multiple read domains
-	 */
-	if (obj->base.pending_write_domain == 0)
-		obj->base.pending_read_domains |= obj->base.read_domains;
-
-	/*
-	 * Flush the current write domain if
-	 * the new read domains don't match. Invalidate
-	 * any read domains which differ from the old
-	 * write domain
-	 */
-	if (obj->base.write_domain &&
-	    (((obj->base.write_domain != obj->base.pending_read_domains ||
-	       obj->ring != ring)) ||
-	     (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) {
-		flush_domains |= obj->base.write_domain;
-		invalidate_domains |=
-			obj->base.pending_read_domains & ~obj->base.write_domain;
-	}
-	/*
-	 * Invalidate any read caches which may have
-	 * stale data. That is, any new read domains.
-	 */
-	invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
-	if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
-		i915_gem_clflush_object(obj);
-
-	if (obj->base.pending_write_domain)
-		cd->flips |= atomic_read(&obj->pending_flip);
-
-	/* The actual obj->write_domain will be updated with
-	 * pending_write_domain after we emit the accumulated flush for all
-	 * of our domain changes in execbuffers (which clears objects'
-	 * write_domains).  So if we have a current write domain that we
-	 * aren't changing, set pending_write_domain to that.
-	 */
-	if (flush_domains == 0 && obj->base.pending_write_domain == 0)
-		obj->base.pending_write_domain = obj->base.write_domain;
-
-	cd->invalidate_domains |= invalidate_domains;
-	cd->flush_domains |= flush_domains;
-	if (flush_domains & I915_GEM_GPU_DOMAINS)
-		cd->flush_rings |= intel_ring_flag(obj->ring);
-	if (invalidate_domains & I915_GEM_GPU_DOMAINS)
-		cd->flush_rings |= intel_ring_flag(ring);
-}
-
 struct eb_objects {
 	int and;
 	struct hlist_head buckets[0];
@@ -218,6 +43,7 @@ eb_create(int size)
 {
 	struct eb_objects *eb;
 	int count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
+	BUILD_BUG_ON(!is_power_of_2(PAGE_SIZE / sizeof(struct hlist_head)));
 	while (count > size)
 		count >>= 1;
 	eb = kzalloc(count*sizeof(struct hlist_head) +
@@ -269,6 +95,7 @@ eb_destroy(struct eb_objects *eb)
 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 {
 	return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
+		!obj->map_and_fenceable ||
 		obj->cache_level != I915_CACHE_NONE);
 }
 
@@ -383,7 +210,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		if (ret)
 			return ret;
 
-		vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
+		vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+							     reloc->offset >> PAGE_SHIFT));
 		*(uint32_t *)(vaddr + page_offset) = reloc->delta;
 		kunmap_atomic(vaddr);
 	} else {
@@ -504,7 +332,8 @@ i915_gem_execbuffer_relocate(struct drm_device *dev,
 	return ret;
 }
 
-#define  __EXEC_OBJECT_HAS_FENCE (1<<31)
+#define  __EXEC_OBJECT_HAS_PIN (1<<31)
+#define  __EXEC_OBJECT_HAS_FENCE (1<<30)
 
 static int
 need_reloc_mappable(struct drm_i915_gem_object *obj)
@@ -514,9 +343,10 @@ need_reloc_mappable(struct drm_i915_gem_object *obj)
 }
 
 static int
-pin_and_fence_object(struct drm_i915_gem_object *obj,
-		     struct intel_ring_buffer *ring)
+i915_gem_execbuffer_reserve_object(struct drm_i915_gem_object *obj,
+				   struct intel_ring_buffer *ring)
 {
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 	struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
 	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 	bool need_fence, need_mappable;
@@ -528,15 +358,17 @@ pin_and_fence_object(struct drm_i915_gem_object *obj,
 		obj->tiling_mode != I915_TILING_NONE;
 	need_mappable = need_fence || need_reloc_mappable(obj);
 
-	ret = i915_gem_object_pin(obj, entry->alignment, need_mappable);
+	ret = i915_gem_object_pin(obj, entry->alignment, need_mappable, false);
 	if (ret)
 		return ret;
 
+	entry->flags |= __EXEC_OBJECT_HAS_PIN;
+
 	if (has_fenced_gpu_access) {
 		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
 			ret = i915_gem_object_get_fence(obj);
 			if (ret)
-				goto err_unpin;
+				return ret;
 
 			if (i915_gem_object_pin_fence(obj))
 				entry->flags |= __EXEC_OBJECT_HAS_FENCE;
@@ -545,12 +377,35 @@ pin_and_fence_object(struct drm_i915_gem_object *obj,
 		}
 	}
 
+	/* Ensure ppgtt mapping exists if needed */
+	if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
+		i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
+				       obj, obj->cache_level);
+
+		obj->has_aliasing_ppgtt_mapping = 1;
+	}
+
 	entry->offset = obj->gtt_offset;
 	return 0;
+}
 
-err_unpin:
-	i915_gem_object_unpin(obj);
-	return ret;
+static void
+i915_gem_execbuffer_unreserve_object(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_gem_exec_object2 *entry;
+
+	if (!obj->gtt_space)
+		return;
+
+	entry = obj->exec_entry;
+
+	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
+		i915_gem_object_unpin_fence(obj);
+
+	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
+		i915_gem_object_unpin(obj);
+
+	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 }
 
 static int
@@ -558,11 +413,10 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 			    struct drm_file *file,
 			    struct list_head *objects)
 {
-	drm_i915_private_t *dev_priv = ring->dev->dev_private;
 	struct drm_i915_gem_object *obj;
-	int ret, retry;
-	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 	struct list_head ordered_objects;
+	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
+	int retry;
 
 	INIT_LIST_HEAD(&ordered_objects);
 	while (!list_empty(objects)) {
@@ -587,6 +441,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 
 		obj->base.pending_read_domains = 0;
 		obj->base.pending_write_domain = 0;
+		obj->pending_fenced_gpu_access = false;
 	}
 	list_splice(&ordered_objects, objects);
 
@@ -599,12 +454,12 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 	 * 2.  Bind new objects.
 	 * 3.  Decrement pin count.
 	 *
-	 * This avoid unnecessary unbinding of later objects in order to makr
+	 * This avoid unnecessary unbinding of later objects in order to make
 	 * room for the earlier objects *unless* we need to defragment.
 	 */
 	retry = 0;
 	do {
-		ret = 0;
+		int ret = 0;
 
 		/* Unbind any ill-fitting objects or pin. */
 		list_for_each_entry(obj, objects, exec_list) {
@@ -624,7 +479,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 			    (need_mappable && !obj->map_and_fenceable))
 				ret = i915_gem_object_unbind(obj);
 			else
-				ret = pin_and_fence_object(obj, ring);
+				ret = i915_gem_execbuffer_reserve_object(obj, ring);
 			if (ret)
 				goto err;
 		}
@@ -634,77 +489,22 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 			if (obj->gtt_space)
 				continue;
 
-			ret = pin_and_fence_object(obj, ring);
-			if (ret) {
-				int ret_ignore;
-
-				/* This can potentially raise a harmless
-				 * -EINVAL if we failed to bind in the above
-				 * call. It cannot raise -EINTR since we know
-				 * that the bo is freshly bound and so will
-				 * not need to be flushed or waited upon.
-				 */
-				ret_ignore = i915_gem_object_unbind(obj);
-				(void)ret_ignore;
-				WARN_ON(obj->gtt_space);
-				break;
-			}
+			ret = i915_gem_execbuffer_reserve_object(obj, ring);
+			if (ret)
+				goto err;
 		}
 
-		/* Decrement pin count for bound objects */
-		list_for_each_entry(obj, objects, exec_list) {
-			struct drm_i915_gem_exec_object2 *entry;
-
-			if (!obj->gtt_space)
-				continue;
-
-			entry = obj->exec_entry;
-			if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
-				i915_gem_object_unpin_fence(obj);
-				entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
-			}
-
-			i915_gem_object_unpin(obj);
-
-			/* ... and ensure ppgtt mapping exist if needed. */
-			if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
-				i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
-						       obj, obj->cache_level);
+err:		/* Decrement pin count for bound objects */
+		list_for_each_entry(obj, objects, exec_list)
+			i915_gem_execbuffer_unreserve_object(obj);
 
-				obj->has_aliasing_ppgtt_mapping = 1;
-			}
-		}
-
-		if (ret != -ENOSPC || retry > 1)
+		if (ret != -ENOSPC || retry++)
 			return ret;
 
-		/* First attempt, just clear anything that is purgeable.
-		 * Second attempt, clear the entire GTT.
-		 */
-		ret = i915_gem_evict_everything(ring->dev, retry == 0);
+		ret = i915_gem_evict_everything(ring->dev);
 		if (ret)
 			return ret;
-
-		retry++;
 	} while (1);
-
-err:
-	list_for_each_entry_continue_reverse(obj, objects, exec_list) {
-		struct drm_i915_gem_exec_object2 *entry;
-
-		if (!obj->gtt_space)
-			continue;
-
-		entry = obj->exec_entry;
-		if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
-			i915_gem_object_unpin_fence(obj);
-			entry->flags &= ~__EXEC_OBJECT_HAS_FENCE;
-		}
-
-		i915_gem_object_unpin(obj);
-	}
-
-	return ret;
 }
 
 static int
@@ -810,18 +610,6 @@ err:
 	return ret;
 }
 
-static void
-i915_gem_execbuffer_flush(struct drm_device *dev,
-			  uint32_t invalidate_domains,
-			  uint32_t flush_domains)
-{
-	if (flush_domains & I915_GEM_DOMAIN_CPU)
-		intel_gtt_chipset_flush();
-
-	if (flush_domains & I915_GEM_DOMAIN_GTT)
-		wmb();
-}
-
 static int
 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 {
@@ -854,48 +642,45 @@ i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 	return 0;
 }
 
-
 static int
 i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 				struct list_head *objects)
 {
 	struct drm_i915_gem_object *obj;
-	struct change_domains cd;
+	uint32_t flush_domains = 0;
+	uint32_t flips = 0;
 	int ret;
 
-	memset(&cd, 0, sizeof(cd));
-	list_for_each_entry(obj, objects, exec_list)
-		i915_gem_object_set_to_gpu_domain(obj, ring, &cd);
-
-	if (cd.invalidate_domains | cd.flush_domains) {
-		i915_gem_execbuffer_flush(ring->dev,
-					  cd.invalidate_domains,
-					  cd.flush_domains);
-	}
-
-	if (cd.flips) {
-		ret = i915_gem_execbuffer_wait_for_flips(ring, cd.flips);
+	list_for_each_entry(obj, objects, exec_list) {
+		ret = i915_gem_object_sync(obj, ring);
 		if (ret)
 			return ret;
+
+		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
+			i915_gem_clflush_object(obj);
+
+		if (obj->base.pending_write_domain)
+			flips |= atomic_read(&obj->pending_flip);
+
+		flush_domains |= obj->base.write_domain;
 	}
 
-	list_for_each_entry(obj, objects, exec_list) {
-		ret = i915_gem_object_sync(obj, ring);
+	if (flips) {
+		ret = i915_gem_execbuffer_wait_for_flips(ring, flips);
 		if (ret)
 			return ret;
 	}
 
+	if (flush_domains & I915_GEM_DOMAIN_CPU)
+		intel_gtt_chipset_flush();
+
+	if (flush_domains & I915_GEM_DOMAIN_GTT)
+		wmb();
+
 	/* Unconditionally invalidate gpu caches and ensure that we do flush
 	 * any residual writes from the previous batch.
 	 */
-	ret = i915_gem_flush_ring(ring,
-				  I915_GEM_GPU_DOMAINS,
-				  ring->gpu_caches_dirty ? I915_GEM_GPU_DOMAINS : 0);
-	if (ret)
-		return ret;
-
-	ring->gpu_caches_dirty = false;
-	return 0;
+	return intel_ring_invalidate_all_caches(ring);
 }
 
 static bool
@@ -943,9 +728,8 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
 	struct drm_i915_gem_object *obj;
 
 	list_for_each_entry(obj, objects, exec_list) {
-		  u32 old_read = obj->base.read_domains;
-		  u32 old_write = obj->base.write_domain;
-
+		u32 old_read = obj->base.read_domains;
+		u32 old_write = obj->base.write_domain;
 
 		obj->base.read_domains = obj->base.pending_read_domains;
 		obj->base.write_domain = obj->base.pending_write_domain;
@@ -954,17 +738,13 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
 		i915_gem_object_move_to_active(obj, ring, seqno);
 		if (obj->base.write_domain) {
 			obj->dirty = 1;
-			obj->pending_gpu_write = true;
-			list_move_tail(&obj->gpu_write_list,
-				       &ring->gpu_write_list);
+			obj->last_write_seqno = seqno;
 			if (obj->pin_count) /* check for potential scanout */
-				intel_mark_busy(ring->dev, obj);
+				intel_mark_fb_busy(obj);
 		}
 
 		trace_i915_gem_object_change_domain(obj, old_read, old_write);
 	}
-
-	intel_mark_busy(ring->dev, NULL);
 }
 
 static void
@@ -972,16 +752,11 @@ i915_gem_execbuffer_retire_commands(struct drm_device *dev,
 				    struct drm_file *file,
 				    struct intel_ring_buffer *ring)
 {
-	struct drm_i915_gem_request *request;
-
 	/* Unconditionally force add_request to emit a full flush. */
 	ring->gpu_caches_dirty = true;
 
 	/* Add a breadcrumb for the completion of the batch buffer */
-	request = kzalloc(sizeof(*request), GFP_KERNEL);
-	if (request == NULL || i915_add_request(ring, file, request)) {
-		kfree(request);
-	}
+	(void)i915_add_request(ring, file, NULL);
 }
 
 static int
@@ -1327,8 +1102,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		return -ENOMEM;
 	}
 	ret = copy_from_user(exec_list,
-			     (struct drm_i915_relocation_entry __user *)
-			     (uintptr_t) args->buffers_ptr,
+			     (void __user *)(uintptr_t)args->buffers_ptr,
 			     sizeof(*exec_list) * args->buffer_count);
 	if (ret != 0) {
 		DRM_DEBUG("copy %d exec entries failed %d\n",
@@ -1367,8 +1141,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		for (i = 0; i < args->buffer_count; i++)
 			exec_list[i].offset = exec2_list[i].offset;
 		/* ... and back out to userspace */
-		ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-				   (uintptr_t) args->buffers_ptr,
+		ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
 				   exec_list,
 				   sizeof(*exec_list) * args->buffer_count);
 		if (ret) {
@@ -1422,8 +1195,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
 	if (!ret) {
 		/* Copy the new buffer offsets back to the user's exec list. */
-		ret = copy_to_user((struct drm_i915_relocation_entry __user *)
-				   (uintptr_t) args->buffers_ptr,
+		ret = copy_to_user((void __user *)(uintptr_t)args->buffers_ptr,
 				   exec2_list,
 				   sizeof(*exec2_list) * args->buffer_count);
 		if (ret) {
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 60815b861ec..df470b5e8d3 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -22,9 +22,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
@@ -167,8 +166,7 @@ void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 }
 
 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
-					 struct scatterlist *sg_list,
-					 unsigned sg_len,
+					 const struct sg_table *pages,
 					 unsigned first_entry,
 					 uint32_t pte_flags)
 {
@@ -180,12 +178,12 @@ static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 	struct scatterlist *sg;
 
 	/* init sg walking */
-	sg = sg_list;
+	sg = pages->sgl;
 	i = 0;
 	segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 	m = 0;
 
-	while (i < sg_len) {
+	while (i < pages->nents) {
 		pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 
 		for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
@@ -194,13 +192,11 @@ static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 			pt_vaddr[j] = pte | pte_flags;
 
 			/* grab the next page */
-			m++;
-			if (m == segment_len) {
-				sg = sg_next(sg);
-				i++;
-				if (i == sg_len)
+			if (++m == segment_len) {
+				if (++i == pages->nents)
 					break;
 
+				sg = sg_next(sg);
 				segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 				m = 0;
 			}
@@ -213,44 +209,10 @@ static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 	}
 }
 
-static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
-				    unsigned first_entry, unsigned num_entries,
-				    struct page **pages, uint32_t pte_flags)
-{
-	uint32_t *pt_vaddr, pte;
-	unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
-	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
-	unsigned last_pte, i;
-	dma_addr_t page_addr;
-
-	while (num_entries) {
-		last_pte = first_pte + num_entries;
-		last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
-
-		pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
-
-		for (i = first_pte; i < last_pte; i++) {
-			page_addr = page_to_phys(*pages);
-			pte = GEN6_PTE_ADDR_ENCODE(page_addr);
-			pt_vaddr[i] = pte | pte_flags;
-
-			pages++;
-		}
-
-		kunmap_atomic(pt_vaddr);
-
-		num_entries -= last_pte - first_pte;
-		first_pte = 0;
-		act_pd++;
-	}
-}
-
 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 			    struct drm_i915_gem_object *obj,
 			    enum i915_cache_level cache_level)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	uint32_t pte_flags = GEN6_PTE_VALID;
 
 	switch (cache_level) {
@@ -261,7 +223,7 @@ void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 		pte_flags |= GEN6_PTE_CACHE_LLC;
 		break;
 	case I915_CACHE_NONE:
-		if (IS_HASWELL(dev))
+		if (IS_HASWELL(obj->base.dev))
 			pte_flags |= HSW_PTE_UNCACHED;
 		else
 			pte_flags |= GEN6_PTE_UNCACHED;
@@ -270,26 +232,10 @@ void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 		BUG();
 	}
 
-	if (obj->sg_table) {
-		i915_ppgtt_insert_sg_entries(ppgtt,
-					     obj->sg_table->sgl,
-					     obj->sg_table->nents,
-					     obj->gtt_space->start >> PAGE_SHIFT,
-					     pte_flags);
-	} else if (dev_priv->mm.gtt->needs_dmar) {
-		BUG_ON(!obj->sg_list);
-
-		i915_ppgtt_insert_sg_entries(ppgtt,
-					     obj->sg_list,
-					     obj->num_sg,
-					     obj->gtt_space->start >> PAGE_SHIFT,
-					     pte_flags);
-	} else
-		i915_ppgtt_insert_pages(ppgtt,
-					obj->gtt_space->start >> PAGE_SHIFT,
-					obj->base.size >> PAGE_SHIFT,
-					obj->pages,
-					pte_flags);
+	i915_ppgtt_insert_sg_entries(ppgtt,
+				     obj->pages,
+				     obj->gtt_space->start >> PAGE_SHIFT,
+				     pte_flags);
 }
 
 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
@@ -351,7 +297,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 	intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 			      (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) {
 		i915_gem_clflush_object(obj);
 		i915_gem_gtt_bind_object(obj, obj->cache_level);
 	}
@@ -361,44 +307,26 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	/* don't map imported dma buf objects */
-	if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table)
-		return intel_gtt_map_memory(obj->pages,
-					    obj->base.size >> PAGE_SHIFT,
-					    &obj->sg_list,
-					    &obj->num_sg);
-	else
+	if (obj->has_dma_mapping)
 		return 0;
+
+	if (!dma_map_sg(&obj->base.dev->pdev->dev,
+			obj->pages->sgl, obj->pages->nents,
+			PCI_DMA_BIDIRECTIONAL))
+		return -ENOSPC;
+
+	return 0;
 }
 
 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
 			      enum i915_cache_level cache_level)
 {
 	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 
-	if (obj->sg_table) {
-		intel_gtt_insert_sg_entries(obj->sg_table->sgl,
-					    obj->sg_table->nents,
-					    obj->gtt_space->start >> PAGE_SHIFT,
-					    agp_type);
-	} else if (dev_priv->mm.gtt->needs_dmar) {
-		BUG_ON(!obj->sg_list);
-
-		intel_gtt_insert_sg_entries(obj->sg_list,
-					    obj->num_sg,
-					    obj->gtt_space->start >> PAGE_SHIFT,
-					    agp_type);
-	} else
-		intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
-				       obj->base.size >> PAGE_SHIFT,
-				       obj->pages,
-				       agp_type);
-
+	intel_gtt_insert_sg_entries(obj->pages,
+				    obj->gtt_space->start >> PAGE_SHIFT,
+				    agp_type);
 	obj->has_global_gtt_mapping = 1;
 }
 
@@ -418,14 +346,31 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 
 	interruptible = do_idling(dev_priv);
 
-	if (obj->sg_list) {
-		intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
-		obj->sg_list = NULL;
-	}
+	if (!obj->has_dma_mapping)
+		dma_unmap_sg(&dev->pdev->dev,
+			     obj->pages->sgl, obj->pages->nents,
+			     PCI_DMA_BIDIRECTIONAL);
 
 	undo_idling(dev_priv, interruptible);
 }
 
+static void i915_gtt_color_adjust(struct drm_mm_node *node,
+				  unsigned long color,
+				  unsigned long *start,
+				  unsigned long *end)
+{
+	if (node->color != color)
+		*start += 4096;
+
+	if (!list_empty(&node->node_list)) {
+		node = list_entry(node->node_list.next,
+				  struct drm_mm_node,
+				  node_list);
+		if (node->allocated && node->color != color)
+			*end -= 4096;
+	}
+}
+
 void i915_gem_init_global_gtt(struct drm_device *dev,
 			      unsigned long start,
 			      unsigned long mappable_end,
@@ -435,6 +380,8 @@ void i915_gem_init_global_gtt(struct drm_device *dev,
 
 	/* Substract the guard page ... */
 	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
+	if (!HAS_LLC(dev))
+		dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
 
 	dev_priv->mm.gtt_start = start;
 	dev_priv->mm.gtt_mappable_end = mappable_end;
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index ada2e90a2a6..8e91083b126 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -26,9 +26,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 /*
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index b964df51cec..3208650a235 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -25,11 +25,10 @@
  *
  */
 
-#include "linux/string.h"
-#include "linux/bitops.h"
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <linux/string.h>
+#include <linux/bitops.h>
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 /** @file i915_gem_tiling.c
@@ -470,18 +469,20 @@ i915_gem_swizzle_page(struct page *page)
 void
 i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
+	struct scatterlist *sg;
 	int page_count = obj->base.size >> PAGE_SHIFT;
 	int i;
 
 	if (obj->bit_17 == NULL)
 		return;
 
-	for (i = 0; i < page_count; i++) {
-		char new_bit_17 = page_to_phys(obj->pages[i]) >> 17;
+	for_each_sg(obj->pages->sgl, sg, page_count, i) {
+		struct page *page = sg_page(sg);
+		char new_bit_17 = page_to_phys(page) >> 17;
 		if ((new_bit_17 & 0x1) !=
 		    (test_bit(i, obj->bit_17) != 0)) {
-			i915_gem_swizzle_page(obj->pages[i]);
-			set_page_dirty(obj->pages[i]);
+			i915_gem_swizzle_page(page);
+			set_page_dirty(page);
 		}
 	}
 }
@@ -489,6 +490,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 void
 i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
+	struct scatterlist *sg;
 	int page_count = obj->base.size >> PAGE_SHIFT;
 	int i;
 
@@ -502,8 +504,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 		}
 	}
 
-	for (i = 0; i < page_count; i++) {
-		if (page_to_phys(obj->pages[i]) & (1 << 17))
+	for_each_sg(obj->pages->sgl, sg, page_count, i) {
+		struct page *page = sg_page(sg);
+		if (page_to_phys(page) & (1 << 17))
 			__set_bit(i, obj->bit_17);
 		else
 			__clear_bit(i, obj->bit_17);
diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c
index 0e72abb9f70..3c59584161c 100644
--- a/drivers/gpu/drm/i915/i915_ioc32.c
+++ b/drivers/gpu/drm/i915/i915_ioc32.c
@@ -31,9 +31,8 @@
  */
 #include <linux/compat.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 typedef struct _drm_i915_batchbuffer32 {
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 5249640cce1..4e9888388c0 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -30,9 +30,8 @@
 
 #include <linux/sysrq.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
@@ -296,11 +295,21 @@ static void i915_hotplug_work_func(struct work_struct *work)
 	drm_helper_hpd_irq_event(dev);
 }
 
-static void i915_handle_rps_change(struct drm_device *dev)
+/* defined intel_pm.c */
+extern spinlock_t mchdev_lock;
+
+static void ironlake_handle_rps_change(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	u32 busy_up, busy_down, max_avg, min_avg;
-	u8 new_delay = dev_priv->cur_delay;
+	u8 new_delay;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mchdev_lock, flags);
+
+	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
+
+	new_delay = dev_priv->ips.cur_delay;
 
 	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
 	busy_up = I915_READ(RCPREVBSYTUPAVG);
@@ -310,19 +319,21 @@ static void i915_handle_rps_change(struct drm_device *dev)
 
 	/* Handle RCS change request from hw */
 	if (busy_up > max_avg) {
-		if (dev_priv->cur_delay != dev_priv->max_delay)
-			new_delay = dev_priv->cur_delay - 1;
-		if (new_delay < dev_priv->max_delay)
-			new_delay = dev_priv->max_delay;
+		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
+			new_delay = dev_priv->ips.cur_delay - 1;
+		if (new_delay < dev_priv->ips.max_delay)
+			new_delay = dev_priv->ips.max_delay;
 	} else if (busy_down < min_avg) {
-		if (dev_priv->cur_delay != dev_priv->min_delay)
-			new_delay = dev_priv->cur_delay + 1;
-		if (new_delay > dev_priv->min_delay)
-			new_delay = dev_priv->min_delay;
+		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
+			new_delay = dev_priv->ips.cur_delay + 1;
+		if (new_delay > dev_priv->ips.min_delay)
+			new_delay = dev_priv->ips.min_delay;
 	}
 
 	if (ironlake_set_drps(dev, new_delay))
-		dev_priv->cur_delay = new_delay;
+		dev_priv->ips.cur_delay = new_delay;
+
+	spin_unlock_irqrestore(&mchdev_lock, flags);
 
 	return;
 }
@@ -335,7 +346,7 @@ static void notify_ring(struct drm_device *dev,
 	if (ring->obj == NULL)
 		return;
 
-	trace_i915_gem_request_complete(ring, ring->get_seqno(ring));
+	trace_i915_gem_request_complete(ring, ring->get_seqno(ring, false));
 
 	wake_up_all(&ring->irq_queue);
 	if (i915_enable_hangcheck) {
@@ -349,16 +360,16 @@ static void notify_ring(struct drm_device *dev,
 static void gen6_pm_rps_work(struct work_struct *work)
 {
 	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-						    rps_work);
+						    rps.work);
 	u32 pm_iir, pm_imr;
 	u8 new_delay;
 
-	spin_lock_irq(&dev_priv->rps_lock);
-	pm_iir = dev_priv->pm_iir;
-	dev_priv->pm_iir = 0;
+	spin_lock_irq(&dev_priv->rps.lock);
+	pm_iir = dev_priv->rps.pm_iir;
+	dev_priv->rps.pm_iir = 0;
 	pm_imr = I915_READ(GEN6_PMIMR);
 	I915_WRITE(GEN6_PMIMR, 0);
-	spin_unlock_irq(&dev_priv->rps_lock);
+	spin_unlock_irq(&dev_priv->rps.lock);
 
 	if ((pm_iir & GEN6_PM_DEFERRED_EVENTS) == 0)
 		return;
@@ -366,11 +377,17 @@ static void gen6_pm_rps_work(struct work_struct *work)
 	mutex_lock(&dev_priv->dev->struct_mutex);
 
 	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD)
-		new_delay = dev_priv->cur_delay + 1;
+		new_delay = dev_priv->rps.cur_delay + 1;
 	else
-		new_delay = dev_priv->cur_delay - 1;
+		new_delay = dev_priv->rps.cur_delay - 1;
 
-	gen6_set_rps(dev_priv->dev, new_delay);
+	/* sysfs frequency interfaces may have snuck in while servicing the
+	 * interrupt
+	 */
+	if (!(new_delay > dev_priv->rps.max_delay ||
+	      new_delay < dev_priv->rps.min_delay)) {
+		gen6_set_rps(dev_priv->dev, new_delay);
+	}
 
 	mutex_unlock(&dev_priv->dev->struct_mutex);
 }
@@ -444,7 +461,7 @@ static void ivybridge_handle_parity_error(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	unsigned long flags;
 
-	if (!IS_IVYBRIDGE(dev))
+	if (!HAS_L3_GPU_CACHE(dev))
 		return;
 
 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
@@ -488,19 +505,19 @@ static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
 	 * IIR bits should never already be set because IMR should
 	 * prevent an interrupt from being shown in IIR. The warning
 	 * displays a case where we've unsafely cleared
-	 * dev_priv->pm_iir. Although missing an interrupt of the same
+	 * dev_priv->rps.pm_iir. Although missing an interrupt of the same
 	 * type is not a problem, it displays a problem in the logic.
 	 *
-	 * The mask bit in IMR is cleared by rps_work.
+	 * The mask bit in IMR is cleared by dev_priv->rps.work.
 	 */
 
-	spin_lock_irqsave(&dev_priv->rps_lock, flags);
-	dev_priv->pm_iir |= pm_iir;
-	I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir);
+	spin_lock_irqsave(&dev_priv->rps.lock, flags);
+	dev_priv->rps.pm_iir |= pm_iir;
+	I915_WRITE(GEN6_PMIMR, dev_priv->rps.pm_iir);
 	POSTING_READ(GEN6_PMIMR);
-	spin_unlock_irqrestore(&dev_priv->rps_lock, flags);
+	spin_unlock_irqrestore(&dev_priv->rps.lock, flags);
 
-	queue_work(dev_priv->wq, &dev_priv->rps_work);
+	queue_work(dev_priv->wq, &dev_priv->rps.work);
 }
 
 static irqreturn_t valleyview_irq_handler(DRM_IRQ_ARGS)
@@ -793,10 +810,8 @@ static irqreturn_t ironlake_irq_handler(DRM_IRQ_ARGS)
 			ibx_irq_handler(dev, pch_iir);
 	}
 
-	if (de_iir & DE_PCU_EVENT) {
-		I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
-		i915_handle_rps_change(dev);
-	}
+	if (IS_GEN5(dev) &&  de_iir & DE_PCU_EVENT)
+		ironlake_handle_rps_change(dev);
 
 	if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
 		gen6_queue_rps_work(dev_priv, pm_iir);
@@ -843,26 +858,55 @@ static void i915_error_work_func(struct work_struct *work)
 	}
 }
 
+/* NB: please notice the memset */
+static void i915_get_extra_instdone(struct drm_device *dev,
+				    uint32_t *instdone)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
+
+	switch(INTEL_INFO(dev)->gen) {
+	case 2:
+	case 3:
+		instdone[0] = I915_READ(INSTDONE);
+		break;
+	case 4:
+	case 5:
+	case 6:
+		instdone[0] = I915_READ(INSTDONE_I965);
+		instdone[1] = I915_READ(INSTDONE1);
+		break;
+	default:
+		WARN_ONCE(1, "Unsupported platform\n");
+	case 7:
+		instdone[0] = I915_READ(GEN7_INSTDONE_1);
+		instdone[1] = I915_READ(GEN7_SC_INSTDONE);
+		instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
+		instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
+		break;
+	}
+}
+
 #ifdef CONFIG_DEBUG_FS
 static struct drm_i915_error_object *
 i915_error_object_create(struct drm_i915_private *dev_priv,
 			 struct drm_i915_gem_object *src)
 {
 	struct drm_i915_error_object *dst;
-	int page, page_count;
+	int i, count;
 	u32 reloc_offset;
 
 	if (src == NULL || src->pages == NULL)
 		return NULL;
 
-	page_count = src->base.size / PAGE_SIZE;
+	count = src->base.size / PAGE_SIZE;
 
-	dst = kmalloc(sizeof(*dst) + page_count * sizeof(u32 *), GFP_ATOMIC);
+	dst = kmalloc(sizeof(*dst) + count * sizeof(u32 *), GFP_ATOMIC);
 	if (dst == NULL)
 		return NULL;
 
 	reloc_offset = src->gtt_offset;
-	for (page = 0; page < page_count; page++) {
+	for (i = 0; i < count; i++) {
 		unsigned long flags;
 		void *d;
 
@@ -885,30 +929,33 @@ i915_error_object_create(struct drm_i915_private *dev_priv,
 			memcpy_fromio(d, s, PAGE_SIZE);
 			io_mapping_unmap_atomic(s);
 		} else {
+			struct page *page;
 			void *s;
 
-			drm_clflush_pages(&src->pages[page], 1);
+			page = i915_gem_object_get_page(src, i);
+
+			drm_clflush_pages(&page, 1);
 
-			s = kmap_atomic(src->pages[page]);
+			s = kmap_atomic(page);
 			memcpy(d, s, PAGE_SIZE);
 			kunmap_atomic(s);
 
-			drm_clflush_pages(&src->pages[page], 1);
+			drm_clflush_pages(&page, 1);
 		}
 		local_irq_restore(flags);
 
-		dst->pages[page] = d;
+		dst->pages[i] = d;
 
 		reloc_offset += PAGE_SIZE;
 	}
-	dst->page_count = page_count;
+	dst->page_count = count;
 	dst->gtt_offset = src->gtt_offset;
 
 	return dst;
 
 unwind:
-	while (page--)
-		kfree(dst->pages[page]);
+	while (i--)
+		kfree(dst->pages[i]);
 	kfree(dst);
 	return NULL;
 }
@@ -949,7 +996,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 {
 	err->size = obj->base.size;
 	err->name = obj->base.name;
-	err->seqno = obj->last_rendering_seqno;
+	err->rseqno = obj->last_read_seqno;
+	err->wseqno = obj->last_write_seqno;
 	err->gtt_offset = obj->gtt_offset;
 	err->read_domains = obj->base.read_domains;
 	err->write_domain = obj->base.write_domain;
@@ -1039,12 +1087,12 @@ i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
 	if (!ring->get_seqno)
 		return NULL;
 
-	seqno = ring->get_seqno(ring);
+	seqno = ring->get_seqno(ring, false);
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
 		if (obj->ring != ring)
 			continue;
 
-		if (i915_seqno_passed(seqno, obj->last_rendering_seqno))
+		if (i915_seqno_passed(seqno, obj->last_read_seqno))
 			continue;
 
 		if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
@@ -1080,10 +1128,8 @@ static void i915_record_ring_state(struct drm_device *dev,
 		error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
 		error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
 		error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base));
-		if (ring->id == RCS) {
-			error->instdone1 = I915_READ(INSTDONE1);
+		if (ring->id == RCS)
 			error->bbaddr = I915_READ64(BB_ADDR);
-		}
 	} else {
 		error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
 		error->ipeir[ring->id] = I915_READ(IPEIR);
@@ -1093,7 +1139,7 @@ static void i915_record_ring_state(struct drm_device *dev,
 
 	error->waiting[ring->id] = waitqueue_active(&ring->irq_queue);
 	error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
-	error->seqno[ring->id] = ring->get_seqno(ring);
+	error->seqno[ring->id] = ring->get_seqno(ring, false);
 	error->acthd[ring->id] = intel_ring_get_active_head(ring);
 	error->head[ring->id] = I915_READ_HEAD(ring);
 	error->tail[ring->id] = I915_READ_TAIL(ring);
@@ -1199,6 +1245,11 @@ static void i915_capture_error_state(struct drm_device *dev)
 		error->done_reg = I915_READ(DONE_REG);
 	}
 
+	if (INTEL_INFO(dev)->gen == 7)
+		error->err_int = I915_READ(GEN7_ERR_INT);
+
+	i915_get_extra_instdone(dev, error->extra_instdone);
+
 	i915_gem_record_fences(dev, error);
 	i915_gem_record_rings(dev, error);
 
@@ -1210,7 +1261,7 @@ static void i915_capture_error_state(struct drm_device *dev)
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
 		i++;
 	error->active_bo_count = i;
-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
+	list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
 		if (obj->pin_count)
 			i++;
 	error->pinned_bo_count = i - error->active_bo_count;
@@ -1235,7 +1286,7 @@ static void i915_capture_error_state(struct drm_device *dev)
 		error->pinned_bo_count =
 			capture_pinned_bo(error->pinned_bo,
 					  error->pinned_bo_count,
-					  &dev_priv->mm.gtt_list);
+					  &dev_priv->mm.bound_list);
 
 	do_gettimeofday(&error->time);
 
@@ -1274,24 +1325,26 @@ void i915_destroy_error_state(struct drm_device *dev)
 static void i915_report_and_clear_eir(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t instdone[I915_NUM_INSTDONE_REG];
 	u32 eir = I915_READ(EIR);
-	int pipe;
+	int pipe, i;
 
 	if (!eir)
 		return;
 
 	pr_err("render error detected, EIR: 0x%08x\n", eir);
 
+	i915_get_extra_instdone(dev, instdone);
+
 	if (IS_G4X(dev)) {
 		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
 			u32 ipeir = I915_READ(IPEIR_I965);
 
 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-			pr_err("  INSTDONE: 0x%08x\n",
-			       I915_READ(INSTDONE_I965));
+			for (i = 0; i < ARRAY_SIZE(instdone); i++)
+				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
 			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-			pr_err("  INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1));
 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
 			I915_WRITE(IPEIR_I965, ipeir);
 			POSTING_READ(IPEIR_I965);
@@ -1325,12 +1378,13 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 	if (eir & I915_ERROR_INSTRUCTION) {
 		pr_err("instruction error\n");
 		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
+		for (i = 0; i < ARRAY_SIZE(instdone); i++)
+			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
 		if (INTEL_INFO(dev)->gen < 4) {
 			u32 ipeir = I915_READ(IPEIR);
 
 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
-			pr_err("  INSTDONE: 0x%08x\n", I915_READ(INSTDONE));
 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
 			I915_WRITE(IPEIR, ipeir);
 			POSTING_READ(IPEIR);
@@ -1339,10 +1393,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 
 			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
 			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
-			pr_err("  INSTDONE: 0x%08x\n",
-			       I915_READ(INSTDONE_I965));
 			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
-			pr_err("  INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1));
 			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
 			I915_WRITE(IPEIR_I965, ipeir);
 			POSTING_READ(IPEIR_I965);
@@ -1590,7 +1641,8 @@ ring_last_seqno(struct intel_ring_buffer *ring)
 static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
 {
 	if (list_empty(&ring->request_list) ||
-	    i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
+	    i915_seqno_passed(ring->get_seqno(ring, false),
+			      ring_last_seqno(ring))) {
 		/* Issue a wake-up to catch stuck h/w. */
 		if (waitqueue_active(&ring->irq_queue)) {
 			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
@@ -1656,7 +1708,7 @@ void i915_hangcheck_elapsed(unsigned long data)
 {
 	struct drm_device *dev = (struct drm_device *)data;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	uint32_t acthd[I915_NUM_RINGS], instdone, instdone1;
+	uint32_t acthd[I915_NUM_RINGS], instdone[I915_NUM_INSTDONE_REG];
 	struct intel_ring_buffer *ring;
 	bool err = false, idle;
 	int i;
@@ -1684,25 +1736,16 @@ void i915_hangcheck_elapsed(unsigned long data)
 		return;
 	}
 
-	if (INTEL_INFO(dev)->gen < 4) {
-		instdone = I915_READ(INSTDONE);
-		instdone1 = 0;
-	} else {
-		instdone = I915_READ(INSTDONE_I965);
-		instdone1 = I915_READ(INSTDONE1);
-	}
-
+	i915_get_extra_instdone(dev, instdone);
 	if (memcmp(dev_priv->last_acthd, acthd, sizeof(acthd)) == 0 &&
-	    dev_priv->last_instdone == instdone &&
-	    dev_priv->last_instdone1 == instdone1) {
+	    memcmp(dev_priv->prev_instdone, instdone, sizeof(instdone)) == 0) {
 		if (i915_hangcheck_hung(dev))
 			return;
 	} else {
 		dev_priv->hangcheck_count = 0;
 
 		memcpy(dev_priv->last_acthd, acthd, sizeof(acthd));
-		dev_priv->last_instdone = instdone;
-		dev_priv->last_instdone1 = instdone1;
+		memcpy(dev_priv->prev_instdone, instdone, sizeof(instdone));
 	}
 
 repeat:
@@ -2647,7 +2690,7 @@ void intel_irq_init(struct drm_device *dev)
 
 	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
 	INIT_WORK(&dev_priv->error_work, i915_error_work_func);
-	INIT_WORK(&dev_priv->rps_work, gen6_pm_rps_work);
+	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
 	INIT_WORK(&dev_priv->parity_error_work, ivybridge_parity_work);
 
 	dev->driver->get_vblank_counter = i915_get_vblank_counter;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 28725ce5b82..7637824c6a7 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -450,6 +450,7 @@
 #define RING_ACTHD(base)	((base)+0x74)
 #define RING_NOPID(base)	((base)+0x94)
 #define RING_IMR(base)		((base)+0xa8)
+#define RING_TIMESTAMP(base)	((base)+0x358)
 #define   TAIL_ADDR		0x001FFFF8
 #define   HEAD_WRAP_COUNT	0xFFE00000
 #define   HEAD_WRAP_ONE		0x00200000
@@ -478,6 +479,11 @@
 #define IPEIR_I965	0x02064
 #define IPEHR_I965	0x02068
 #define INSTDONE_I965	0x0206c
+#define GEN7_INSTDONE_1		0x0206c
+#define GEN7_SC_INSTDONE	0x07100
+#define GEN7_SAMPLER_INSTDONE	0x0e160
+#define GEN7_ROW_INSTDONE	0x0e164
+#define I915_NUM_INSTDONE_REG	4
 #define RING_IPEIR(base)	((base)+0x64)
 #define RING_IPEHR(base)	((base)+0x68)
 #define RING_INSTDONE(base)	((base)+0x6c)
@@ -500,6 +506,8 @@
 #define DMA_FADD_I8XX	0x020d0
 
 #define ERROR_GEN6	0x040a0
+#define GEN7_ERR_INT	0x44040
+#define   ERR_INT_MMIO_UNCLAIMED (1<<13)
 
 /* GM45+ chicken bits -- debug workaround bits that may be required
  * for various sorts of correct behavior.  The top 16 bits of each are
@@ -529,6 +537,8 @@
 #define   GFX_PSMI_GRANULARITY		(1<<10)
 #define   GFX_PPGTT_ENABLE		(1<<9)
 
+#define VLV_DISPLAY_BASE 0x180000
+
 #define SCPD0		0x0209c /* 915+ only */
 #define IER		0x020a0
 #define IIR		0x020a4
@@ -1496,6 +1506,14 @@
 					 GEN7_CXT_EXTENDED_SIZE(ctx_reg) + \
 					 GEN7_CXT_GT1_SIZE(ctx_reg) + \
 					 GEN7_CXT_VFSTATE_SIZE(ctx_reg))
+#define HSW_CXT_POWER_SIZE(ctx_reg)	((ctx_reg >> 26) & 0x3f)
+#define HSW_CXT_RING_SIZE(ctx_reg)	((ctx_reg >> 23) & 0x7)
+#define HSW_CXT_RENDER_SIZE(ctx_reg)	((ctx_reg >> 15) & 0xff)
+#define HSW_CXT_TOTAL_SIZE(ctx_reg)	(HSW_CXT_POWER_SIZE(ctx_reg) + \
+					 HSW_CXT_RING_SIZE(ctx_reg) + \
+					 HSW_CXT_RENDER_SIZE(ctx_reg) + \
+					 GEN7_CXT_VFSTATE_SIZE(ctx_reg))
+
 
 /*
  * Overlay regs
@@ -1549,12 +1567,35 @@
 
 /* VGA port control */
 #define ADPA			0x61100
+#define PCH_ADPA                0xe1100
+#define VLV_ADPA		(VLV_DISPLAY_BASE + ADPA)
+
 #define   ADPA_DAC_ENABLE	(1<<31)
 #define   ADPA_DAC_DISABLE	0
 #define   ADPA_PIPE_SELECT_MASK	(1<<30)
 #define   ADPA_PIPE_A_SELECT	0
 #define   ADPA_PIPE_B_SELECT	(1<<30)
 #define   ADPA_PIPE_SELECT(pipe) ((pipe) << 30)
+/* CPT uses bits 29:30 for pch transcoder select */
+#define   ADPA_CRT_HOTPLUG_MASK  0x03ff0000 /* bit 25-16 */
+#define   ADPA_CRT_HOTPLUG_MONITOR_NONE  (0<<24)
+#define   ADPA_CRT_HOTPLUG_MONITOR_MASK  (3<<24)
+#define   ADPA_CRT_HOTPLUG_MONITOR_COLOR (3<<24)
+#define   ADPA_CRT_HOTPLUG_MONITOR_MONO  (2<<24)
+#define   ADPA_CRT_HOTPLUG_ENABLE        (1<<23)
+#define   ADPA_CRT_HOTPLUG_PERIOD_64     (0<<22)
+#define   ADPA_CRT_HOTPLUG_PERIOD_128    (1<<22)
+#define   ADPA_CRT_HOTPLUG_WARMUP_5MS    (0<<21)
+#define   ADPA_CRT_HOTPLUG_WARMUP_10MS   (1<<21)
+#define   ADPA_CRT_HOTPLUG_SAMPLE_2S     (0<<20)
+#define   ADPA_CRT_HOTPLUG_SAMPLE_4S     (1<<20)
+#define   ADPA_CRT_HOTPLUG_VOLTAGE_40    (0<<18)
+#define   ADPA_CRT_HOTPLUG_VOLTAGE_50    (1<<18)
+#define   ADPA_CRT_HOTPLUG_VOLTAGE_60    (2<<18)
+#define   ADPA_CRT_HOTPLUG_VOLTAGE_70    (3<<18)
+#define   ADPA_CRT_HOTPLUG_VOLREF_325MV  (0<<17)
+#define   ADPA_CRT_HOTPLUG_VOLREF_475MV  (1<<17)
+#define   ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
 #define   ADPA_USE_VGA_HVPOLARITY (1<<15)
 #define   ADPA_SETS_HVPOLARITY	0
 #define   ADPA_VSYNC_CNTL_DISABLE (1<<11)
@@ -1753,6 +1794,10 @@
 
 /* Video Data Island Packet control */
 #define VIDEO_DIP_DATA		0x61178
+/* Read the description of VIDEO_DIP_DATA (before Haswel) or VIDEO_DIP_ECC
+ * (Haswell and newer) to see which VIDEO_DIP_DATA byte corresponds to each byte
+ * of the infoframe structure specified by CEA-861. */
+#define   VIDEO_DIP_DATA_SIZE	32
 #define VIDEO_DIP_CTL		0x61170
 /* Pre HSW: */
 #define   VIDEO_DIP_ENABLE		(1 << 31)
@@ -3889,31 +3934,6 @@
 #define FDI_PLL_CTL_1           0xfe000
 #define FDI_PLL_CTL_2           0xfe004
 
-/* CRT */
-#define PCH_ADPA                0xe1100
-#define  ADPA_TRANS_SELECT_MASK (1<<30)
-#define  ADPA_TRANS_A_SELECT    0
-#define  ADPA_TRANS_B_SELECT    (1<<30)
-#define  ADPA_CRT_HOTPLUG_MASK  0x03ff0000 /* bit 25-16 */
-#define  ADPA_CRT_HOTPLUG_MONITOR_NONE  (0<<24)
-#define  ADPA_CRT_HOTPLUG_MONITOR_MASK  (3<<24)
-#define  ADPA_CRT_HOTPLUG_MONITOR_COLOR (3<<24)
-#define  ADPA_CRT_HOTPLUG_MONITOR_MONO  (2<<24)
-#define  ADPA_CRT_HOTPLUG_ENABLE        (1<<23)
-#define  ADPA_CRT_HOTPLUG_PERIOD_64     (0<<22)
-#define  ADPA_CRT_HOTPLUG_PERIOD_128    (1<<22)
-#define  ADPA_CRT_HOTPLUG_WARMUP_5MS    (0<<21)
-#define  ADPA_CRT_HOTPLUG_WARMUP_10MS   (1<<21)
-#define  ADPA_CRT_HOTPLUG_SAMPLE_2S     (0<<20)
-#define  ADPA_CRT_HOTPLUG_SAMPLE_4S     (1<<20)
-#define  ADPA_CRT_HOTPLUG_VOLTAGE_40    (0<<18)
-#define  ADPA_CRT_HOTPLUG_VOLTAGE_50    (1<<18)
-#define  ADPA_CRT_HOTPLUG_VOLTAGE_60    (2<<18)
-#define  ADPA_CRT_HOTPLUG_VOLTAGE_70    (3<<18)
-#define  ADPA_CRT_HOTPLUG_VOLREF_325MV  (0<<17)
-#define  ADPA_CRT_HOTPLUG_VOLREF_475MV  (1<<17)
-#define  ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
-
 /* or SDVOB */
 #define HDMIB   0xe1140
 #define  PORT_ENABLE    (1 << 31)
@@ -4021,6 +4041,8 @@
 #define  PORT_TRANS_C_SEL_CPT	(2<<29)
 #define  PORT_TRANS_SEL_MASK	(3<<29)
 #define  PORT_TRANS_SEL_CPT(pipe)	((pipe) << 29)
+#define  PORT_TO_PIPE(val)	(((val) & (1<<30)) >> 30)
+#define  PORT_TO_PIPE_CPT(val)	(((val) & PORT_TRANS_SEL_MASK) >> 29)
 
 #define TRANS_DP_CTL_A		0xe0300
 #define TRANS_DP_CTL_B		0xe1300
@@ -4239,7 +4261,15 @@
 #define G4X_HDMIW_HDMIEDID		0x6210C
 
 #define IBX_HDMIW_HDMIEDID_A		0xE2050
+#define IBX_HDMIW_HDMIEDID_B		0xE2150
+#define IBX_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
+					IBX_HDMIW_HDMIEDID_A, \
+					IBX_HDMIW_HDMIEDID_B)
 #define IBX_AUD_CNTL_ST_A		0xE20B4
+#define IBX_AUD_CNTL_ST_B		0xE21B4
+#define IBX_AUD_CNTL_ST(pipe) _PIPE(pipe, \
+					IBX_AUD_CNTL_ST_A, \
+					IBX_AUD_CNTL_ST_B)
 #define IBX_ELD_BUFFER_SIZE		(0x1f << 10)
 #define IBX_ELD_ADDRESS			(0x1f << 5)
 #define IBX_ELD_ACK			(1 << 4)
@@ -4248,7 +4278,15 @@
 #define IBX_CP_READYB			(1 << 1)
 
 #define CPT_HDMIW_HDMIEDID_A		0xE5050
+#define CPT_HDMIW_HDMIEDID_B		0xE5150
+#define CPT_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
+					CPT_HDMIW_HDMIEDID_A, \
+					CPT_HDMIW_HDMIEDID_B)
 #define CPT_AUD_CNTL_ST_A		0xE50B4
+#define CPT_AUD_CNTL_ST_B		0xE51B4
+#define CPT_AUD_CNTL_ST(pipe) _PIPE(pipe, \
+					CPT_AUD_CNTL_ST_A, \
+					CPT_AUD_CNTL_ST_B)
 #define CPT_AUD_CNTRL_ST2		0xE50C0
 
 /* These are the 4 32-bit write offset registers for each stream
@@ -4258,7 +4296,15 @@
 #define GEN7_SO_WRITE_OFFSET(n)		(0x5280 + (n) * 4)
 
 #define IBX_AUD_CONFIG_A			0xe2000
+#define IBX_AUD_CONFIG_B			0xe2100
+#define IBX_AUD_CFG(pipe) _PIPE(pipe, \
+					IBX_AUD_CONFIG_A, \
+					IBX_AUD_CONFIG_B)
 #define CPT_AUD_CONFIG_A			0xe5000
+#define CPT_AUD_CONFIG_B			0xe5100
+#define CPT_AUD_CFG(pipe) _PIPE(pipe, \
+					CPT_AUD_CONFIG_A, \
+					CPT_AUD_CONFIG_B)
 #define   AUD_CONFIG_N_VALUE_INDEX		(1 << 29)
 #define   AUD_CONFIG_N_PROG_ENABLE		(1 << 28)
 #define   AUD_CONFIG_UPPER_N_SHIFT		20
@@ -4269,195 +4315,233 @@
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI		(0xf << 16)
 #define   AUD_CONFIG_DISABLE_NCTS		(1 << 3)
 
+/* HSW Audio */
+#define   HSW_AUD_CONFIG_A		0x65000 /* Audio Configuration Transcoder A */
+#define   HSW_AUD_CONFIG_B		0x65100 /* Audio Configuration Transcoder B */
+#define   HSW_AUD_CFG(pipe) _PIPE(pipe, \
+					HSW_AUD_CONFIG_A, \
+					HSW_AUD_CONFIG_B)
+
+#define   HSW_AUD_MISC_CTRL_A		0x65010 /* Audio Misc Control Convert 1 */
+#define   HSW_AUD_MISC_CTRL_B		0x65110 /* Audio Misc Control Convert 2 */
+#define   HSW_AUD_MISC_CTRL(pipe) _PIPE(pipe, \
+					HSW_AUD_MISC_CTRL_A, \
+					HSW_AUD_MISC_CTRL_B)
+
+#define   HSW_AUD_DIP_ELD_CTRL_ST_A	0x650b4 /* Audio DIP and ELD Control State Transcoder A */
+#define   HSW_AUD_DIP_ELD_CTRL_ST_B	0x651b4 /* Audio DIP and ELD Control State Transcoder B */
+#define   HSW_AUD_DIP_ELD_CTRL(pipe) _PIPE(pipe, \
+					HSW_AUD_DIP_ELD_CTRL_ST_A, \
+					HSW_AUD_DIP_ELD_CTRL_ST_B)
+
+/* Audio Digital Converter */
+#define   HSW_AUD_DIG_CNVT_1		0x65080 /* Audio Converter 1 */
+#define   HSW_AUD_DIG_CNVT_2		0x65180 /* Audio Converter 1 */
+#define   AUD_DIG_CNVT(pipe) _PIPE(pipe, \
+					HSW_AUD_DIG_CNVT_1, \
+					HSW_AUD_DIG_CNVT_2)
+#define   DIP_PORT_SEL_MASK		0x3
+
+#define   HSW_AUD_EDID_DATA_A		0x65050
+#define   HSW_AUD_EDID_DATA_B		0x65150
+#define   HSW_AUD_EDID_DATA(pipe) _PIPE(pipe, \
+					HSW_AUD_EDID_DATA_A, \
+					HSW_AUD_EDID_DATA_B)
+
+#define   HSW_AUD_PIPE_CONV_CFG		0x6507c /* Audio pipe and converter configs */
+#define   HSW_AUD_PIN_ELD_CP_VLD	0x650c0 /* Audio ELD and CP Ready Status */
+#define   AUDIO_INACTIVE_C		(1<<11)
+#define   AUDIO_INACTIVE_B		(1<<7)
+#define   AUDIO_INACTIVE_A		(1<<3)
+#define   AUDIO_OUTPUT_ENABLE_A		(1<<2)
+#define   AUDIO_OUTPUT_ENABLE_B		(1<<6)
+#define   AUDIO_OUTPUT_ENABLE_C		(1<<10)
+#define   AUDIO_ELD_VALID_A		(1<<0)
+#define   AUDIO_ELD_VALID_B		(1<<4)
+#define   AUDIO_ELD_VALID_C		(1<<8)
+#define   AUDIO_CP_READY_A		(1<<1)
+#define   AUDIO_CP_READY_B		(1<<5)
+#define   AUDIO_CP_READY_C		(1<<9)
+
 /* HSW Power Wells */
-#define HSW_PWR_WELL_CTL1		0x45400		/* BIOS */
-#define HSW_PWR_WELL_CTL2		0x45404		/* Driver */
-#define HSW_PWR_WELL_CTL3		0x45408		/* KVMR */
-#define HSW_PWR_WELL_CTL4		0x4540C		/* Debug */
-#define   HSW_PWR_WELL_ENABLE				(1<<31)
-#define   HSW_PWR_WELL_STATE				(1<<30)
-#define HSW_PWR_WELL_CTL5		0x45410
+#define HSW_PWR_WELL_CTL1			0x45400 /* BIOS */
+#define HSW_PWR_WELL_CTL2			0x45404 /* Driver */
+#define HSW_PWR_WELL_CTL3			0x45408 /* KVMR */
+#define HSW_PWR_WELL_CTL4			0x4540C /* Debug */
+#define   HSW_PWR_WELL_ENABLE			(1<<31)
+#define   HSW_PWR_WELL_STATE			(1<<30)
+#define HSW_PWR_WELL_CTL5			0x45410
 #define   HSW_PWR_WELL_ENABLE_SINGLE_STEP	(1<<31)
 #define   HSW_PWR_WELL_PWR_GATE_OVERRIDE	(1<<20)
-#define   HSW_PWR_WELL_FORCE_ON				(1<<19)
-#define HSW_PWR_WELL_CTL6		0x45414
+#define   HSW_PWR_WELL_FORCE_ON			(1<<19)
+#define HSW_PWR_WELL_CTL6			0x45414
 
 /* Per-pipe DDI Function Control */
-#define PIPE_DDI_FUNC_CTL_A			0x60400
-#define PIPE_DDI_FUNC_CTL_B			0x61400
-#define PIPE_DDI_FUNC_CTL_C			0x62400
+#define PIPE_DDI_FUNC_CTL_A		0x60400
+#define PIPE_DDI_FUNC_CTL_B		0x61400
+#define PIPE_DDI_FUNC_CTL_C		0x62400
 #define PIPE_DDI_FUNC_CTL_EDP		0x6F400
-#define DDI_FUNC_CTL(pipe) _PIPE(pipe, \
-					PIPE_DDI_FUNC_CTL_A, \
-					PIPE_DDI_FUNC_CTL_B)
+#define DDI_FUNC_CTL(pipe) _PIPE(pipe, PIPE_DDI_FUNC_CTL_A, \
+				       PIPE_DDI_FUNC_CTL_B)
 #define  PIPE_DDI_FUNC_ENABLE		(1<<31)
 /* Those bits are ignored by pipe EDP since it can only connect to DDI A */
-#define  PIPE_DDI_PORT_MASK			(7<<28)
-#define  PIPE_DDI_SELECT_PORT(x)		((x)<<28)
-#define  PIPE_DDI_MODE_SELECT_HDMI		(0<<24)
-#define  PIPE_DDI_MODE_SELECT_DVI		(1<<24)
+#define  PIPE_DDI_PORT_MASK		(7<<28)
+#define  PIPE_DDI_SELECT_PORT(x)	((x)<<28)
+#define  PIPE_DDI_MODE_SELECT_MASK	(7<<24)
+#define  PIPE_DDI_MODE_SELECT_HDMI	(0<<24)
+#define  PIPE_DDI_MODE_SELECT_DVI	(1<<24)
 #define  PIPE_DDI_MODE_SELECT_DP_SST	(2<<24)
 #define  PIPE_DDI_MODE_SELECT_DP_MST	(3<<24)
-#define  PIPE_DDI_MODE_SELECT_FDI		(4<<24)
-#define  PIPE_DDI_BPC_8					(0<<20)
-#define  PIPE_DDI_BPC_10				(1<<20)
-#define  PIPE_DDI_BPC_6					(2<<20)
-#define  PIPE_DDI_BPC_12				(3<<20)
-#define  PIPE_DDI_BFI_ENABLE			(1<<4)
-#define  PIPE_DDI_PORT_WIDTH_X1			(0<<1)
-#define  PIPE_DDI_PORT_WIDTH_X2			(1<<1)
-#define  PIPE_DDI_PORT_WIDTH_X4			(3<<1)
+#define  PIPE_DDI_MODE_SELECT_FDI	(4<<24)
+#define  PIPE_DDI_BPC_MASK		(7<<20)
+#define  PIPE_DDI_BPC_8			(0<<20)
+#define  PIPE_DDI_BPC_10		(1<<20)
+#define  PIPE_DDI_BPC_6			(2<<20)
+#define  PIPE_DDI_BPC_12		(3<<20)
+#define  PIPE_DDI_PVSYNC		(1<<17)
+#define  PIPE_DDI_PHSYNC		(1<<16)
+#define  PIPE_DDI_BFI_ENABLE		(1<<4)
+#define  PIPE_DDI_PORT_WIDTH_X1		(0<<1)
+#define  PIPE_DDI_PORT_WIDTH_X2		(1<<1)
+#define  PIPE_DDI_PORT_WIDTH_X4		(3<<1)
 
 /* DisplayPort Transport Control */
 #define DP_TP_CTL_A			0x64040
 #define DP_TP_CTL_B			0x64140
-#define DP_TP_CTL(port) _PORT(port, \
-					DP_TP_CTL_A, \
-					DP_TP_CTL_B)
-#define  DP_TP_CTL_ENABLE		(1<<31)
-#define  DP_TP_CTL_MODE_SST	(0<<27)
-#define  DP_TP_CTL_MODE_MST	(1<<27)
+#define DP_TP_CTL(port) _PORT(port, DP_TP_CTL_A, DP_TP_CTL_B)
+#define  DP_TP_CTL_ENABLE			(1<<31)
+#define  DP_TP_CTL_MODE_SST			(0<<27)
+#define  DP_TP_CTL_MODE_MST			(1<<27)
 #define  DP_TP_CTL_ENHANCED_FRAME_ENABLE	(1<<18)
-#define  DP_TP_CTL_FDI_AUTOTRAIN	(1<<15)
+#define  DP_TP_CTL_FDI_AUTOTRAIN		(1<<15)
 #define  DP_TP_CTL_LINK_TRAIN_MASK		(7<<8)
 #define  DP_TP_CTL_LINK_TRAIN_PAT1		(0<<8)
 #define  DP_TP_CTL_LINK_TRAIN_PAT2		(1<<8)
-#define  DP_TP_CTL_LINK_TRAIN_NORMAL	(3<<8)
+#define  DP_TP_CTL_LINK_TRAIN_NORMAL		(3<<8)
 
 /* DisplayPort Transport Status */
 #define DP_TP_STATUS_A			0x64044
 #define DP_TP_STATUS_B			0x64144
-#define DP_TP_STATUS(port) _PORT(port, \
-					DP_TP_STATUS_A, \
-					DP_TP_STATUS_B)
+#define DP_TP_STATUS(port) _PORT(port, DP_TP_STATUS_A, DP_TP_STATUS_B)
 #define  DP_TP_STATUS_AUTOTRAIN_DONE	(1<<12)
 
 /* DDI Buffer Control */
 #define DDI_BUF_CTL_A				0x64000
 #define DDI_BUF_CTL_B				0x64100
-#define DDI_BUF_CTL(port) _PORT(port, \
-					DDI_BUF_CTL_A, \
-					DDI_BUF_CTL_B)
-#define  DDI_BUF_CTL_ENABLE				(1<<31)
+#define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B)
+#define  DDI_BUF_CTL_ENABLE			(1<<31)
 #define  DDI_BUF_EMP_400MV_0DB_HSW		(0<<24)   /* Sel0 */
-#define  DDI_BUF_EMP_400MV_3_5DB_HSW	(1<<24)   /* Sel1 */
+#define  DDI_BUF_EMP_400MV_3_5DB_HSW		(1<<24)   /* Sel1 */
 #define  DDI_BUF_EMP_400MV_6DB_HSW		(2<<24)   /* Sel2 */
-#define  DDI_BUF_EMP_400MV_9_5DB_HSW	(3<<24)   /* Sel3 */
+#define  DDI_BUF_EMP_400MV_9_5DB_HSW		(3<<24)   /* Sel3 */
 #define  DDI_BUF_EMP_600MV_0DB_HSW		(4<<24)   /* Sel4 */
-#define  DDI_BUF_EMP_600MV_3_5DB_HSW	(5<<24)   /* Sel5 */
+#define  DDI_BUF_EMP_600MV_3_5DB_HSW		(5<<24)   /* Sel5 */
 #define  DDI_BUF_EMP_600MV_6DB_HSW		(6<<24)   /* Sel6 */
 #define  DDI_BUF_EMP_800MV_0DB_HSW		(7<<24)   /* Sel7 */
-#define  DDI_BUF_EMP_800MV_3_5DB_HSW	(8<<24)   /* Sel8 */
-#define  DDI_BUF_EMP_MASK				(0xf<<24)
-#define  DDI_BUF_IS_IDLE				(1<<7)
-#define  DDI_PORT_WIDTH_X1				(0<<1)
-#define  DDI_PORT_WIDTH_X2				(1<<1)
-#define  DDI_PORT_WIDTH_X4				(3<<1)
+#define  DDI_BUF_EMP_800MV_3_5DB_HSW		(8<<24)   /* Sel8 */
+#define  DDI_BUF_EMP_MASK			(0xf<<24)
+#define  DDI_BUF_IS_IDLE			(1<<7)
+#define  DDI_PORT_WIDTH_X1			(0<<1)
+#define  DDI_PORT_WIDTH_X2			(1<<1)
+#define  DDI_PORT_WIDTH_X4			(3<<1)
 #define  DDI_INIT_DISPLAY_DETECTED		(1<<0)
 
 /* DDI Buffer Translations */
 #define DDI_BUF_TRANS_A				0x64E00
 #define DDI_BUF_TRANS_B				0x64E60
-#define DDI_BUF_TRANS(port) _PORT(port, \
-					DDI_BUF_TRANS_A, \
-					DDI_BUF_TRANS_B)
+#define DDI_BUF_TRANS(port) _PORT(port, DDI_BUF_TRANS_A, DDI_BUF_TRANS_B)
 
 /* Sideband Interface (SBI) is programmed indirectly, via
  * SBI_ADDR, which contains the register offset; and SBI_DATA,
  * which contains the payload */
-#define SBI_ADDR				0xC6000
-#define SBI_DATA				0xC6004
+#define SBI_ADDR			0xC6000
+#define SBI_DATA			0xC6004
 #define SBI_CTL_STAT			0xC6008
 #define  SBI_CTL_OP_CRRD		(0x6<<8)
 #define  SBI_CTL_OP_CRWR		(0x7<<8)
 #define  SBI_RESPONSE_FAIL		(0x1<<1)
-#define  SBI_RESPONSE_SUCCESS	(0x0<<1)
-#define  SBI_BUSY				(0x1<<0)
-#define  SBI_READY				(0x0<<0)
+#define  SBI_RESPONSE_SUCCESS		(0x0<<1)
+#define  SBI_BUSY			(0x1<<0)
+#define  SBI_READY			(0x0<<0)
 
 /* SBI offsets */
-#define  SBI_SSCDIVINTPHASE6		0x0600
+#define  SBI_SSCDIVINTPHASE6			0x0600
 #define   SBI_SSCDIVINTPHASE_DIVSEL_MASK	((0x7f)<<1)
 #define   SBI_SSCDIVINTPHASE_DIVSEL(x)		((x)<<1)
 #define   SBI_SSCDIVINTPHASE_INCVAL_MASK	((0x7f)<<8)
 #define   SBI_SSCDIVINTPHASE_INCVAL(x)		((x)<<8)
-#define   SBI_SSCDIVINTPHASE_DIR(x)			((x)<<15)
+#define   SBI_SSCDIVINTPHASE_DIR(x)		((x)<<15)
 #define   SBI_SSCDIVINTPHASE_PROPAGATE		(1<<0)
-#define  SBI_SSCCTL					0x020c
+#define  SBI_SSCCTL				0x020c
 #define  SBI_SSCCTL6				0x060C
-#define   SBI_SSCCTL_DISABLE		(1<<0)
+#define   SBI_SSCCTL_DISABLE			(1<<0)
 #define  SBI_SSCAUXDIV6				0x0610
 #define   SBI_SSCAUXDIV_FINALDIV2SEL(x)		((x)<<4)
-#define  SBI_DBUFF0					0x2a00
+#define  SBI_DBUFF0				0x2a00
 
 /* LPT PIXCLK_GATE */
-#define PIXCLK_GATE				0xC6020
-#define  PIXCLK_GATE_UNGATE		1<<0
-#define  PIXCLK_GATE_GATE		0<<0
+#define PIXCLK_GATE			0xC6020
+#define  PIXCLK_GATE_UNGATE		(1<<0)
+#define  PIXCLK_GATE_GATE		(0<<0)
 
 /* SPLL */
-#define SPLL_CTL				0x46020
+#define SPLL_CTL			0x46020
 #define  SPLL_PLL_ENABLE		(1<<31)
 #define  SPLL_PLL_SCC			(1<<28)
 #define  SPLL_PLL_NON_SCC		(2<<28)
-#define  SPLL_PLL_FREQ_810MHz	(0<<26)
-#define  SPLL_PLL_FREQ_1350MHz	(1<<26)
+#define  SPLL_PLL_FREQ_810MHz		(0<<26)
+#define  SPLL_PLL_FREQ_1350MHz		(1<<26)
 
 /* WRPLL */
-#define WRPLL_CTL1				0x46040
-#define WRPLL_CTL2				0x46060
-#define  WRPLL_PLL_ENABLE				(1<<31)
-#define  WRPLL_PLL_SELECT_SSC			(0x01<<28)
-#define  WRPLL_PLL_SELECT_NON_SCC		(0x02<<28)
+#define WRPLL_CTL1			0x46040
+#define WRPLL_CTL2			0x46060
+#define  WRPLL_PLL_ENABLE		(1<<31)
+#define  WRPLL_PLL_SELECT_SSC		(0x01<<28)
+#define  WRPLL_PLL_SELECT_NON_SCC	(0x02<<28)
 #define  WRPLL_PLL_SELECT_LCPLL_2700	(0x03<<28)
 /* WRPLL divider programming */
-#define  WRPLL_DIVIDER_REFERENCE(x)		((x)<<0)
-#define  WRPLL_DIVIDER_POST(x)			((x)<<8)
-#define  WRPLL_DIVIDER_FEEDBACK(x)		((x)<<16)
+#define  WRPLL_DIVIDER_REFERENCE(x)	((x)<<0)
+#define  WRPLL_DIVIDER_POST(x)		((x)<<8)
+#define  WRPLL_DIVIDER_FEEDBACK(x)	((x)<<16)
 
 /* Port clock selection */
 #define PORT_CLK_SEL_A			0x46100
 #define PORT_CLK_SEL_B			0x46104
-#define PORT_CLK_SEL(port) _PORT(port, \
-					PORT_CLK_SEL_A, \
-					PORT_CLK_SEL_B)
+#define PORT_CLK_SEL(port) _PORT(port, PORT_CLK_SEL_A, PORT_CLK_SEL_B)
 #define  PORT_CLK_SEL_LCPLL_2700	(0<<29)
 #define  PORT_CLK_SEL_LCPLL_1350	(1<<29)
 #define  PORT_CLK_SEL_LCPLL_810		(2<<29)
-#define  PORT_CLK_SEL_SPLL			(3<<29)
+#define  PORT_CLK_SEL_SPLL		(3<<29)
 #define  PORT_CLK_SEL_WRPLL1		(4<<29)
 #define  PORT_CLK_SEL_WRPLL2		(5<<29)
 
 /* Pipe clock selection */
 #define PIPE_CLK_SEL_A			0x46140
 #define PIPE_CLK_SEL_B			0x46144
-#define PIPE_CLK_SEL(pipe) _PIPE(pipe, \
-					PIPE_CLK_SEL_A, \
-					PIPE_CLK_SEL_B)
+#define PIPE_CLK_SEL(pipe) _PIPE(pipe, PIPE_CLK_SEL_A, PIPE_CLK_SEL_B)
 /* For each pipe, we need to select the corresponding port clock */
-#define  PIPE_CLK_SEL_DISABLED	(0x0<<29)
-#define  PIPE_CLK_SEL_PORT(x)	((x+1)<<29)
+#define  PIPE_CLK_SEL_DISABLED		(0x0<<29)
+#define  PIPE_CLK_SEL_PORT(x)		((x+1)<<29)
 
 /* LCPLL Control */
-#define LCPLL_CTL				0x130040
+#define LCPLL_CTL			0x130040
 #define  LCPLL_PLL_DISABLE		(1<<31)
 #define  LCPLL_PLL_LOCK			(1<<30)
-#define  LCPLL_CD_CLOCK_DISABLE	(1<<25)
+#define  LCPLL_CD_CLOCK_DISABLE		(1<<25)
 #define  LCPLL_CD2X_CLOCK_DISABLE	(1<<23)
 
 /* Pipe WM_LINETIME - watermark line time */
 #define PIPE_WM_LINETIME_A		0x45270
 #define PIPE_WM_LINETIME_B		0x45274
-#define PIPE_WM_LINETIME(pipe) _PIPE(pipe, \
-					PIPE_WM_LINETIME_A, \
-					PIPE_WM_LINETIME_B)
-#define   PIPE_WM_LINETIME_MASK		(0x1ff)
-#define   PIPE_WM_LINETIME_TIME(x)			((x))
+#define PIPE_WM_LINETIME(pipe) _PIPE(pipe, PIPE_WM_LINETIME_A, \
+					   PIPE_WM_LINETIME_B)
+#define   PIPE_WM_LINETIME_MASK			(0x1ff)
+#define   PIPE_WM_LINETIME_TIME(x)		((x))
 #define   PIPE_WM_LINETIME_IPS_LINETIME_MASK	(0x1ff<<16)
-#define   PIPE_WM_LINETIME_IPS_LINETIME(x)		((x)<<16)
+#define   PIPE_WM_LINETIME_IPS_LINETIME(x)	((x)<<16)
 
 /* SFUSE_STRAP */
-#define SFUSE_STRAP				0xc2014
+#define SFUSE_STRAP			0xc2014
 #define  SFUSE_STRAP_DDIB_DETECTED	(1<<2)
 #define  SFUSE_STRAP_DDIC_DETECTED	(1<<1)
 #define  SFUSE_STRAP_DDID_DETECTED	(1<<0)
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 4776ccf1b3c..5854bddb1e9 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -24,9 +24,8 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "intel_drv.h"
 #include "i915_reg.h"
 
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 7631807a278..903eebd2117 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -46,32 +46,32 @@ static u32 calc_residency(struct drm_device *dev, const u32 reg)
 }
 
 static ssize_t
-show_rc6_mask(struct device *dev, struct device_attribute *attr, char *buf)
+show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+	struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
 	return snprintf(buf, PAGE_SIZE, "%x", intel_enable_rc6(dminor->dev));
 }
 
 static ssize_t
-show_rc6_ms(struct device *dev, struct device_attribute *attr, char *buf)
+show_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+	struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
 	u32 rc6_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6);
 	return snprintf(buf, PAGE_SIZE, "%u", rc6_residency);
 }
 
 static ssize_t
-show_rc6p_ms(struct device *dev, struct device_attribute *attr, char *buf)
+show_rc6p_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+	struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
 	u32 rc6p_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6p);
 	return snprintf(buf, PAGE_SIZE, "%u", rc6p_residency);
 }
 
 static ssize_t
-show_rc6pp_ms(struct device *dev, struct device_attribute *attr, char *buf)
+show_rc6pp_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+	struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
 	u32 rc6pp_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6pp);
 	return snprintf(buf, PAGE_SIZE, "%u", rc6pp_residency);
 }
@@ -93,6 +93,7 @@ static struct attribute_group rc6_attr_group = {
 	.name = power_group_name,
 	.attrs =  rc6_attrs
 };
+#endif
 
 static int l3_access_valid(struct drm_device *dev, loff_t offset)
 {
@@ -202,37 +203,214 @@ static struct bin_attribute dpf_attrs = {
 	.mmap = NULL
 };
 
+static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	ret = dev_priv->rps.cur_delay * GT_FREQUENCY_MULTIPLIER;
+	mutex_unlock(&dev->struct_mutex);
+
+	return snprintf(buf, PAGE_SIZE, "%d", ret);
+}
+
+static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	ret = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
+	mutex_unlock(&dev->struct_mutex);
+
+	return snprintf(buf, PAGE_SIZE, "%d", ret);
+}
+
+static ssize_t gt_max_freq_mhz_store(struct device *kdev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 val, rp_state_cap, hw_max, hw_min;
+	ssize_t ret;
+
+	ret = kstrtou32(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	val /= GT_FREQUENCY_MULTIPLIER;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+	hw_max = (rp_state_cap & 0xff);
+	hw_min = ((rp_state_cap & 0xff0000) >> 16);
+
+	if (val < hw_min || val > hw_max || val < dev_priv->rps.min_delay) {
+		mutex_unlock(&dev->struct_mutex);
+		return -EINVAL;
+	}
+
+	if (dev_priv->rps.cur_delay > val)
+		gen6_set_rps(dev_priv->dev, val);
+
+	dev_priv->rps.max_delay = val;
+
+	mutex_unlock(&dev->struct_mutex);
+
+	return count;
+}
+
+static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int ret;
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	ret = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
+	mutex_unlock(&dev->struct_mutex);
+
+	return snprintf(buf, PAGE_SIZE, "%d", ret);
+}
+
+static ssize_t gt_min_freq_mhz_store(struct device *kdev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 val, rp_state_cap, hw_max, hw_min;
+	ssize_t ret;
+
+	ret = kstrtou32(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	val /= GT_FREQUENCY_MULTIPLIER;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+	hw_max = (rp_state_cap & 0xff);
+	hw_min = ((rp_state_cap & 0xff0000) >> 16);
+
+	if (val < hw_min || val > hw_max || val > dev_priv->rps.max_delay) {
+		mutex_unlock(&dev->struct_mutex);
+		return -EINVAL;
+	}
+
+	if (dev_priv->rps.cur_delay < val)
+		gen6_set_rps(dev_priv->dev, val);
+
+	dev_priv->rps.min_delay = val;
+
+	mutex_unlock(&dev->struct_mutex);
+
+	return count;
+
+}
+
+static DEVICE_ATTR(gt_cur_freq_mhz, S_IRUGO, gt_cur_freq_mhz_show, NULL);
+static DEVICE_ATTR(gt_max_freq_mhz, S_IRUGO | S_IWUSR, gt_max_freq_mhz_show, gt_max_freq_mhz_store);
+static DEVICE_ATTR(gt_min_freq_mhz, S_IRUGO | S_IWUSR, gt_min_freq_mhz_show, gt_min_freq_mhz_store);
+
+
+static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf);
+static DEVICE_ATTR(gt_RP0_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
+static DEVICE_ATTR(gt_RP1_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
+static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
+
+/* For now we have a static number of RP states */
+static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+	struct drm_device *dev = minor->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 val, rp_state_cap;
+	ssize_t ret;
+
+	ret = mutex_lock_interruptible(&dev->struct_mutex);
+	if (ret)
+		return ret;
+	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+	mutex_unlock(&dev->struct_mutex);
+
+	if (attr == &dev_attr_gt_RP0_freq_mhz) {
+		val = ((rp_state_cap & 0x0000ff) >> 0) * GT_FREQUENCY_MULTIPLIER;
+	} else if (attr == &dev_attr_gt_RP1_freq_mhz) {
+		val = ((rp_state_cap & 0x00ff00) >> 8) * GT_FREQUENCY_MULTIPLIER;
+	} else if (attr == &dev_attr_gt_RPn_freq_mhz) {
+		val = ((rp_state_cap & 0xff0000) >> 16) * GT_FREQUENCY_MULTIPLIER;
+	} else {
+		BUG();
+	}
+	return snprintf(buf, PAGE_SIZE, "%d", val);
+}
+
+static const struct attribute *gen6_attrs[] = {
+	&dev_attr_gt_cur_freq_mhz.attr,
+	&dev_attr_gt_max_freq_mhz.attr,
+	&dev_attr_gt_min_freq_mhz.attr,
+	&dev_attr_gt_RP0_freq_mhz.attr,
+	&dev_attr_gt_RP1_freq_mhz.attr,
+	&dev_attr_gt_RPn_freq_mhz.attr,
+	NULL,
+};
+
 void i915_setup_sysfs(struct drm_device *dev)
 {
 	int ret;
 
+#ifdef CONFIG_PM
 	if (INTEL_INFO(dev)->gen >= 6) {
 		ret = sysfs_merge_group(&dev->primary->kdev.kobj,
 					&rc6_attr_group);
 		if (ret)
 			DRM_ERROR("RC6 residency sysfs setup failed\n");
 	}
-
-	if (IS_IVYBRIDGE(dev)) {
+#endif
+	if (HAS_L3_GPU_CACHE(dev)) {
 		ret = device_create_bin_file(&dev->primary->kdev, &dpf_attrs);
 		if (ret)
 			DRM_ERROR("l3 parity sysfs setup failed\n");
 	}
+
+	if (INTEL_INFO(dev)->gen >= 6) {
+		ret = sysfs_create_files(&dev->primary->kdev.kobj, gen6_attrs);
+		if (ret)
+			DRM_ERROR("gen6 sysfs setup failed\n");
+	}
 }
 
 void i915_teardown_sysfs(struct drm_device *dev)
 {
+	sysfs_remove_files(&dev->primary->kdev.kobj, gen6_attrs);
 	device_remove_bin_file(&dev->primary->kdev,  &dpf_attrs);
+#ifdef CONFIG_PM
 	sysfs_unmerge_group(&dev->primary->kdev.kobj, &rc6_attr_group);
+#endif
 }
-#else
-void i915_setup_sysfs(struct drm_device *dev)
-{
-	return;
-}
-
-void i915_teardown_sysfs(struct drm_device *dev)
-{
-	return;
-}
-#endif /* CONFIG_PM */
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index fe90b3a84a6..8134421b89a 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -214,22 +214,18 @@ TRACE_EVENT(i915_gem_evict,
 );
 
 TRACE_EVENT(i915_gem_evict_everything,
-	    TP_PROTO(struct drm_device *dev, bool purgeable),
-	    TP_ARGS(dev, purgeable),
+	    TP_PROTO(struct drm_device *dev),
+	    TP_ARGS(dev),
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
-			     __field(bool, purgeable)
 			    ),
 
 	    TP_fast_assign(
 			   __entry->dev = dev->primary->index;
-			   __entry->purgeable = purgeable;
 			  ),
 
-	    TP_printk("dev=%d%s",
-		      __entry->dev,
-		      __entry->purgeable ? ", purgeable only" : "")
+	    TP_printk("dev=%d", __entry->dev)
 );
 
 TRACE_EVENT(i915_gem_ring_dispatch,
@@ -434,6 +430,21 @@ TRACE_EVENT(i915_reg_rw,
 		(u32)(__entry->val >> 32))
 );
 
+TRACE_EVENT(intel_gpu_freq_change,
+	    TP_PROTO(u32 freq),
+	    TP_ARGS(freq),
+
+	    TP_STRUCT__entry(
+			     __field(u32, freq)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->freq = freq;
+			   ),
+
+	    TP_printk("new_freq=%u", __entry->freq)
+);
+
 #endif /* _I915_TRACE_H_ */
 
 /* This part must be outside protection */
diff --git a/drivers/gpu/drm/i915/intel_acpi.c b/drivers/gpu/drm/i915/intel_acpi.c
index f413899475e..bcbbaea2a78 100644
--- a/drivers/gpu/drm/i915/intel_acpi.c
+++ b/drivers/gpu/drm/i915/intel_acpi.c
@@ -8,7 +8,7 @@
 #include <linux/vga_switcheroo.h>
 #include <acpi/acpi_drivers.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "i915_drv.h"
 
 #define INTEL_DSM_REVISION_ID 1 /* For Calpella anyway... */
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 8c6074154bf..0ed6baff4b0 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -26,9 +26,8 @@
  */
 #include <linux/dmi.h>
 #include <drm/drm_dp_helper.h>
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "intel_bios.h"
 
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index 31c2107e782..36e57f93437 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -28,7 +28,7 @@
 #ifndef _I830_BIOS_H_
 #define _I830_BIOS_H_
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 struct vbt_header {
 	u8 signature[20];		/**< Always starts with 'VBT$' */
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 23bdc8cd145..893f30164b7 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -27,13 +27,12 @@
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 /* Here's the desired hotplug mode */
@@ -47,6 +46,7 @@
 struct intel_crt {
 	struct intel_encoder base;
 	bool force_hotplug_required;
+	u32 adpa_reg;
 };
 
 static struct intel_crt *intel_attached_crt(struct drm_connector *connector)
@@ -55,42 +55,68 @@ static struct intel_crt *intel_attached_crt(struct drm_connector *connector)
 			    struct intel_crt, base);
 }
 
-static void pch_crt_dpms(struct drm_encoder *encoder, int mode)
+static struct intel_crt *intel_encoder_to_crt(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->dev;
+	return container_of(encoder, struct intel_crt, base);
+}
+
+static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
+				   enum pipe *pipe)
+{
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crt *crt = intel_encoder_to_crt(encoder);
+	u32 tmp;
+
+	tmp = I915_READ(crt->adpa_reg);
+
+	if (!(tmp & ADPA_DAC_ENABLE))
+		return false;
+
+	if (HAS_PCH_CPT(dev))
+		*pipe = PORT_TO_PIPE_CPT(tmp);
+	else
+		*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
+static void intel_disable_crt(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+	struct intel_crt *crt = intel_encoder_to_crt(encoder);
 	u32 temp;
 
-	temp = I915_READ(PCH_ADPA);
+	temp = I915_READ(crt->adpa_reg);
+	temp &= ~(ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE);
 	temp &= ~ADPA_DAC_ENABLE;
+	I915_WRITE(crt->adpa_reg, temp);
+}
 
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		temp |= ADPA_DAC_ENABLE;
-		break;
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-	case DRM_MODE_DPMS_OFF:
-		/* Just leave port enable cleared */
-		break;
-	}
+static void intel_enable_crt(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+	struct intel_crt *crt = intel_encoder_to_crt(encoder);
+	u32 temp;
 
-	I915_WRITE(PCH_ADPA, temp);
+	temp = I915_READ(crt->adpa_reg);
+	temp |= ADPA_DAC_ENABLE;
+	I915_WRITE(crt->adpa_reg, temp);
 }
 
-static void gmch_crt_dpms(struct drm_encoder *encoder, int mode)
+/* Note: The caller is required to filter out dpms modes not supported by the
+ * platform. */
+static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crt *crt = intel_encoder_to_crt(encoder);
 	u32 temp;
 
-	temp = I915_READ(ADPA);
+	temp = I915_READ(crt->adpa_reg);
 	temp &= ~(ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE);
 	temp &= ~ADPA_DAC_ENABLE;
 
-	if (IS_VALLEYVIEW(dev) && mode != DRM_MODE_DPMS_ON)
-		mode = DRM_MODE_DPMS_OFF;
-
 	switch (mode) {
 	case DRM_MODE_DPMS_ON:
 		temp |= ADPA_DAC_ENABLE;
@@ -106,7 +132,51 @@ static void gmch_crt_dpms(struct drm_encoder *encoder, int mode)
 		break;
 	}
 
-	I915_WRITE(ADPA, temp);
+	I915_WRITE(crt->adpa_reg, temp);
+}
+
+static void intel_crt_dpms(struct drm_connector *connector, int mode)
+{
+	struct drm_device *dev = connector->dev;
+	struct intel_encoder *encoder = intel_attached_encoder(connector);
+	struct drm_crtc *crtc;
+	int old_dpms;
+
+	/* PCH platforms and VLV only support on/off. */
+	if (INTEL_INFO(dev)->gen < 5 && mode != DRM_MODE_DPMS_ON)
+		mode = DRM_MODE_DPMS_OFF;
+
+	if (mode == connector->dpms)
+		return;
+
+	old_dpms = connector->dpms;
+	connector->dpms = mode;
+
+	/* Only need to change hw state when actually enabled */
+	crtc = encoder->base.crtc;
+	if (!crtc) {
+		encoder->connectors_active = false;
+		return;
+	}
+
+	/* We need the pipe to run for anything but OFF. */
+	if (mode == DRM_MODE_DPMS_OFF)
+		encoder->connectors_active = false;
+	else
+		encoder->connectors_active = true;
+
+	if (mode < old_dpms) {
+		/* From off to on, enable the pipe first. */
+		intel_crtc_update_dpms(crtc);
+
+		intel_crt_set_dpms(encoder, mode);
+	} else {
+		intel_crt_set_dpms(encoder, mode);
+
+		intel_crtc_update_dpms(crtc);
+	}
+
+	intel_modeset_check_state(connector->dev);
 }
 
 static int intel_crt_mode_valid(struct drm_connector *connector,
@@ -145,19 +215,15 @@ static void intel_crt_mode_set(struct drm_encoder *encoder,
 
 	struct drm_device *dev = encoder->dev;
 	struct drm_crtc *crtc = encoder->crtc;
+	struct intel_crt *crt =
+		intel_encoder_to_crt(to_intel_encoder(encoder));
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int dpll_md_reg;
 	u32 adpa, dpll_md;
-	u32 adpa_reg;
 
 	dpll_md_reg = DPLL_MD(intel_crtc->pipe);
 
-	if (HAS_PCH_SPLIT(dev))
-		adpa_reg = PCH_ADPA;
-	else
-		adpa_reg = ADPA;
-
 	/*
 	 * Disable separate mode multiplier used when cloning SDVO to CRT
 	 * XXX this needs to be adjusted when we really are cloning
@@ -185,7 +251,7 @@ static void intel_crt_mode_set(struct drm_encoder *encoder,
 	if (!HAS_PCH_SPLIT(dev))
 		I915_WRITE(BCLRPAT(intel_crtc->pipe), 0);
 
-	I915_WRITE(adpa_reg, adpa);
+	I915_WRITE(crt->adpa_reg, adpa);
 }
 
 static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
@@ -545,14 +611,12 @@ intel_crt_detect(struct drm_connector *connector, bool force)
 		return connector->status;
 
 	/* for pre-945g platforms use load detect */
-	if (intel_get_load_detect_pipe(&crt->base, connector, NULL,
-				       &tmp)) {
+	if (intel_get_load_detect_pipe(connector, NULL, &tmp)) {
 		if (intel_crt_detect_ddc(connector))
 			status = connector_status_connected;
 		else
 			status = intel_crt_load_detect(crt);
-		intel_release_load_detect_pipe(&crt->base, connector,
-					       &tmp);
+		intel_release_load_detect_pipe(connector, &tmp);
 	} else
 		status = connector_status_unknown;
 
@@ -603,25 +667,15 @@ static void intel_crt_reset(struct drm_connector *connector)
  * Routines for controlling stuff on the analog port
  */
 
-static const struct drm_encoder_helper_funcs pch_encoder_funcs = {
+static const struct drm_encoder_helper_funcs crt_encoder_funcs = {
 	.mode_fixup = intel_crt_mode_fixup,
-	.prepare = intel_encoder_prepare,
-	.commit = intel_encoder_commit,
 	.mode_set = intel_crt_mode_set,
-	.dpms = pch_crt_dpms,
-};
-
-static const struct drm_encoder_helper_funcs gmch_encoder_funcs = {
-	.mode_fixup = intel_crt_mode_fixup,
-	.prepare = intel_encoder_prepare,
-	.commit = intel_encoder_commit,
-	.mode_set = intel_crt_mode_set,
-	.dpms = gmch_crt_dpms,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
 	.reset = intel_crt_reset,
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_crt_dpms,
 	.detect = intel_crt_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.destroy = intel_crt_destroy,
@@ -662,7 +716,6 @@ void intel_crt_init(struct drm_device *dev)
 	struct intel_crt *crt;
 	struct intel_connector *intel_connector;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	const struct drm_encoder_helper_funcs *encoder_helper_funcs;
 
 	/* Skip machines without VGA that falsely report hotplug events */
 	if (dmi_check_system(intel_no_crt))
@@ -688,13 +741,11 @@ void intel_crt_init(struct drm_device *dev)
 	intel_connector_attach_encoder(intel_connector, &crt->base);
 
 	crt->base.type = INTEL_OUTPUT_ANALOG;
-	crt->base.clone_mask = (1 << INTEL_SDVO_NON_TV_CLONE_BIT |
-				1 << INTEL_ANALOG_CLONE_BIT |
-				1 << INTEL_SDVO_LVDS_CLONE_BIT);
+	crt->base.cloneable = true;
 	if (IS_HASWELL(dev))
 		crt->base.crtc_mask = (1 << 0);
 	else
-		crt->base.crtc_mask = (1 << 0) | (1 << 1);
+		crt->base.crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 
 	if (IS_GEN2(dev))
 		connector->interlace_allowed = 0;
@@ -703,11 +754,18 @@ void intel_crt_init(struct drm_device *dev)
 	connector->doublescan_allowed = 0;
 
 	if (HAS_PCH_SPLIT(dev))
-		encoder_helper_funcs = &pch_encoder_funcs;
+		crt->adpa_reg = PCH_ADPA;
+	else if (IS_VALLEYVIEW(dev))
+		crt->adpa_reg = VLV_ADPA;
 	else
-		encoder_helper_funcs = &gmch_encoder_funcs;
+		crt->adpa_reg = ADPA;
+
+	crt->base.disable = intel_disable_crt;
+	crt->base.enable = intel_enable_crt;
+	crt->base.get_hw_state = intel_crt_get_hw_state;
+	intel_connector->get_hw_state = intel_connector_get_hw_state;
 
-	drm_encoder_helper_add(&crt->base.base, encoder_helper_funcs);
+	drm_encoder_helper_add(&crt->base.base, &crt_encoder_funcs);
 	drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
 	drm_sysfs_connector_add(connector);
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 933c7485917..bfe375466a0 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -250,7 +250,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 	case PORT_B:
 	case PORT_C:
 	case PORT_D:
-		intel_hdmi_init(dev, DDI_BUF_CTL(port));
+		intel_hdmi_init(dev, DDI_BUF_CTL(port), port);
 		break;
 	default:
 		DRM_DEBUG_DRIVER("No handlers defined for port %d, skipping DDI initialization\n",
@@ -267,7 +267,8 @@ struct wrpll_tmds_clock {
 	u16 r2;		/* Reference divider */
 };
 
-/* Table of matching values for WRPLL clocks programming for each frequency */
+/* Table of matching values for WRPLL clocks programming for each frequency.
+ * The code assumes this table is sorted. */
 static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
 	{19750,	38,	25,	18},
 	{20000,	48,	32,	18},
@@ -277,7 +278,6 @@ static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
 	{23000,	36,	23,	15},
 	{23500,	40,	40,	23},
 	{23750,	26,	16,	14},
-	{23750,	26,	16,	14},
 	{24000,	36,	24,	15},
 	{25000,	36,	25,	15},
 	{25175,	26,	40,	33},
@@ -437,7 +437,6 @@ static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
 	{108000,	8,	24,	15},
 	{108108,	8,	173,	108},
 	{109000,	6,	23,	19},
-	{109000,	6,	23,	19},
 	{110000,	6,	22,	18},
 	{110013,	6,	22,	18},
 	{110250,	8,	49,	30},
@@ -614,7 +613,6 @@ static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
 	{218250,	4,	42,	26},
 	{218750,	4,	34,	21},
 	{219000,	4,	47,	29},
-	{219000,	4,	47,	29},
 	{220000,	4,	44,	27},
 	{220640,	4,	49,	30},
 	{220750,	4,	36,	22},
@@ -658,7 +656,7 @@ void intel_ddi_mode_set(struct drm_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	int port = intel_hdmi->ddi_port;
 	int pipe = intel_crtc->pipe;
-	int p, n2, r2, valid=0;
+	int p, n2, r2;
 	u32 temp, i;
 
 	/* On Haswell, we need to enable the clocks and prepare DDI function to
@@ -666,26 +664,23 @@ void intel_ddi_mode_set(struct drm_encoder *encoder,
 	 */
 	DRM_DEBUG_KMS("Preparing HDMI DDI mode for Haswell on port %c, pipe %c\n", port_name(port), pipe_name(pipe));
 
-	for (i=0; i < ARRAY_SIZE(wrpll_tmds_clock_table); i++) {
-		if (crtc->mode.clock == wrpll_tmds_clock_table[i].clock) {
-			p = wrpll_tmds_clock_table[i].p;
-			n2 = wrpll_tmds_clock_table[i].n2;
-			r2 = wrpll_tmds_clock_table[i].r2;
+	for (i = 0; i < ARRAY_SIZE(wrpll_tmds_clock_table); i++)
+		if (crtc->mode.clock <= wrpll_tmds_clock_table[i].clock)
+			break;
 
-			DRM_DEBUG_KMS("WR PLL clock: found settings for %dKHz refresh rate: p=%d, n2=%d, r2=%d\n",
-					crtc->mode.clock,
-					p, n2, r2);
+	if (i == ARRAY_SIZE(wrpll_tmds_clock_table))
+		i--;
 
-			valid = 1;
-			break;
-		}
-	}
+	p = wrpll_tmds_clock_table[i].p;
+	n2 = wrpll_tmds_clock_table[i].n2;
+	r2 = wrpll_tmds_clock_table[i].r2;
 
-	if (!valid) {
-		DRM_ERROR("Unable to find WR PLL clock settings for %dKHz refresh rate\n",
-				crtc->mode.clock);
-		return;
-	}
+	if (wrpll_tmds_clock_table[i].clock != crtc->mode.clock)
+		DRM_INFO("WR PLL: using settings for %dKHz on %dKHz mode\n",
+			 wrpll_tmds_clock_table[i].clock, crtc->mode.clock);
+
+	DRM_DEBUG_KMS("WR PLL: %dKHz refresh rate with p=%d, n2=%d r2=%d\n",
+		      crtc->mode.clock, p, n2, r2);
 
 	/* Enable LCPLL if disabled */
 	temp = I915_READ(LCPLL_CTL);
@@ -718,46 +713,107 @@ void intel_ddi_mode_set(struct drm_encoder *encoder,
 		/* Proper support for digital audio needs a new logic and a new set
 		 * of registers, so we leave it for future patch bombing.
 		 */
-		DRM_DEBUG_DRIVER("HDMI audio on pipe %c not yet supported on DDI\n",
+		DRM_DEBUG_DRIVER("HDMI audio on pipe %c on DDI\n",
 				 pipe_name(intel_crtc->pipe));
+
+		/* write eld */
+		DRM_DEBUG_DRIVER("HDMI audio: write eld information\n");
+		intel_write_eld(encoder, adjusted_mode);
 	}
 
 	/* Enable PIPE_DDI_FUNC_CTL for the pipe to work in HDMI mode */
-	temp = I915_READ(DDI_FUNC_CTL(pipe));
-	temp &= ~PIPE_DDI_PORT_MASK;
-	temp &= ~PIPE_DDI_BPC_12;
-	temp |= PIPE_DDI_SELECT_PORT(port) |
-			PIPE_DDI_MODE_SELECT_HDMI |
-			((intel_crtc->bpp > 24) ?
-				PIPE_DDI_BPC_12 :
-				PIPE_DDI_BPC_8) |
-			PIPE_DDI_FUNC_ENABLE;
+	temp = PIPE_DDI_FUNC_ENABLE | PIPE_DDI_SELECT_PORT(port);
+
+	switch (intel_crtc->bpp) {
+	case 18:
+		temp |= PIPE_DDI_BPC_6;
+		break;
+	case 24:
+		temp |= PIPE_DDI_BPC_8;
+		break;
+	case 30:
+		temp |= PIPE_DDI_BPC_10;
+		break;
+	case 36:
+		temp |= PIPE_DDI_BPC_12;
+		break;
+	default:
+		WARN(1, "%d bpp unsupported by pipe DDI function\n",
+		     intel_crtc->bpp);
+	}
+
+	if (intel_hdmi->has_hdmi_sink)
+		temp |= PIPE_DDI_MODE_SELECT_HDMI;
+	else
+		temp |= PIPE_DDI_MODE_SELECT_DVI;
+
+	if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
+		temp |= PIPE_DDI_PVSYNC;
+	if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
+		temp |= PIPE_DDI_PHSYNC;
 
 	I915_WRITE(DDI_FUNC_CTL(pipe), temp);
 
 	intel_hdmi->set_infoframes(encoder, adjusted_mode);
 }
 
-void intel_ddi_dpms(struct drm_encoder *encoder, int mode)
+bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
+			    enum pipe *pipe)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+	u32 tmp;
+	int i;
+
+	tmp = I915_READ(DDI_BUF_CTL(intel_hdmi->ddi_port));
+
+	if (!(tmp & DDI_BUF_CTL_ENABLE))
+		return false;
+
+	for_each_pipe(i) {
+		tmp = I915_READ(DDI_FUNC_CTL(i));
+
+		if ((tmp & PIPE_DDI_PORT_MASK)
+		    == PIPE_DDI_SELECT_PORT(intel_hdmi->ddi_port)) {
+			*pipe = i;
+			return true;
+		}
+	}
+
+	DRM_DEBUG_KMS("No pipe for ddi port %i found\n", intel_hdmi->ddi_port);
+
+	return true;
+}
+
+void intel_enable_ddi(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	int port = intel_hdmi->ddi_port;
 	u32 temp;
 
 	temp = I915_READ(DDI_BUF_CTL(port));
-
-	if (mode != DRM_MODE_DPMS_ON) {
-		temp &= ~DDI_BUF_CTL_ENABLE;
-	} else {
-		temp |= DDI_BUF_CTL_ENABLE;
-	}
+	temp |= DDI_BUF_CTL_ENABLE;
 
 	/* Enable DDI_BUF_CTL. In HDMI/DVI mode, the port width,
 	 * and swing/emphasis values are ignored so nothing special needs
 	 * to be done besides enabling the port.
 	 */
-	I915_WRITE(DDI_BUF_CTL(port),
-			temp);
+	I915_WRITE(DDI_BUF_CTL(port), temp);
+}
+
+void intel_disable_ddi(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+	int port = intel_hdmi->ddi_port;
+	u32 temp;
+
+	temp = I915_READ(DDI_BUF_CTL(port));
+	temp &= ~DDI_BUF_CTL_ENABLE;
+
+	I915_WRITE(DDI_BUF_CTL(port), temp);
 }
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index c040aee1341..e3c02655d36 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -32,13 +32,13 @@
 #include <linux/slab.h>
 #include <linux/vgaarb.h>
 #include <drm/drm_edid.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
-#include "drm_dp_helper.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_crtc_helper.h>
 #include <linux/dma_remapping.h>
 
 #define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
@@ -1006,7 +1006,7 @@ void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 		/* Wait for the Pipe State to go off */
 		if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
 			     100))
-			DRM_DEBUG_KMS("pipe_off wait timed out\n");
+			WARN(1, "pipe_off wait timed out\n");
 	} else {
 		u32 last_line, line_mask;
 		int reg = PIPEDSL(pipe);
@@ -1024,7 +1024,7 @@ void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 		} while (((I915_READ(reg) & line_mask) != last_line) &&
 			 time_after(timeout, jiffies));
 		if (time_after(jiffies, timeout))
-			DRM_DEBUG_KMS("pipe_off wait timed out\n");
+			WARN(1, "pipe_off wait timed out\n");
 	}
 }
 
@@ -1431,6 +1431,8 @@ static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
  * protect mechanism may be enabled.
  *
  * Note!  This is for pre-ILK only.
+ *
+ * Unfortunately needed by dvo_ns2501 since the dvo depends on it running.
  */
 static void intel_enable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
@@ -1860,59 +1862,6 @@ static void intel_disable_plane(struct drm_i915_private *dev_priv,
 	intel_wait_for_vblank(dev_priv->dev, pipe);
 }
 
-static void disable_pch_dp(struct drm_i915_private *dev_priv,
-			   enum pipe pipe, int reg, u32 port_sel)
-{
-	u32 val = I915_READ(reg);
-	if (dp_pipe_enabled(dev_priv, pipe, port_sel, val)) {
-		DRM_DEBUG_KMS("Disabling pch dp %x on pipe %d\n", reg, pipe);
-		I915_WRITE(reg, val & ~DP_PORT_EN);
-	}
-}
-
-static void disable_pch_hdmi(struct drm_i915_private *dev_priv,
-			     enum pipe pipe, int reg)
-{
-	u32 val = I915_READ(reg);
-	if (hdmi_pipe_enabled(dev_priv, pipe, val)) {
-		DRM_DEBUG_KMS("Disabling pch HDMI %x on pipe %d\n",
-			      reg, pipe);
-		I915_WRITE(reg, val & ~PORT_ENABLE);
-	}
-}
-
-/* Disable any ports connected to this transcoder */
-static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
-				    enum pipe pipe)
-{
-	u32 reg, val;
-
-	val = I915_READ(PCH_PP_CONTROL);
-	I915_WRITE(PCH_PP_CONTROL, val | PANEL_UNLOCK_REGS);
-
-	disable_pch_dp(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
-	disable_pch_dp(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
-	disable_pch_dp(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
-
-	reg = PCH_ADPA;
-	val = I915_READ(reg);
-	if (adpa_pipe_enabled(dev_priv, pipe, val))
-		I915_WRITE(reg, val & ~ADPA_DAC_ENABLE);
-
-	reg = PCH_LVDS;
-	val = I915_READ(reg);
-	if (lvds_pipe_enabled(dev_priv, pipe, val)) {
-		DRM_DEBUG_KMS("disable lvds on pipe %d val 0x%08x\n", pipe, val);
-		I915_WRITE(reg, val & ~LVDS_PORT_EN);
-		POSTING_READ(reg);
-		udelay(100);
-	}
-
-	disable_pch_hdmi(dev_priv, pipe, HDMIB);
-	disable_pch_hdmi(dev_priv, pipe, HDMIC);
-	disable_pch_hdmi(dev_priv, pipe, HDMID);
-}
-
 int
 intel_pin_and_fence_fb_obj(struct drm_device *dev,
 			   struct drm_i915_gem_object *obj,
@@ -2201,16 +2150,17 @@ intel_finish_fb(struct drm_framebuffer *old_fb)
 
 static int
 intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
-		    struct drm_framebuffer *old_fb)
+		    struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct drm_framebuffer *old_fb;
 	int ret;
 
 	/* no fb bound */
-	if (!crtc->fb) {
+	if (!fb) {
 		DRM_ERROR("No FB bound\n");
 		return 0;
 	}
@@ -2224,7 +2174,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 
 	mutex_lock(&dev->struct_mutex);
 	ret = intel_pin_and_fence_fb_obj(dev,
-					 to_intel_framebuffer(crtc->fb)->obj,
+					 to_intel_framebuffer(fb)->obj,
 					 NULL);
 	if (ret != 0) {
 		mutex_unlock(&dev->struct_mutex);
@@ -2232,17 +2182,22 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 		return ret;
 	}
 
-	if (old_fb)
-		intel_finish_fb(old_fb);
+	if (crtc->fb)
+		intel_finish_fb(crtc->fb);
 
-	ret = dev_priv->display.update_plane(crtc, crtc->fb, x, y);
+	ret = dev_priv->display.update_plane(crtc, fb, x, y);
 	if (ret) {
-		intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
+		intel_unpin_fb_obj(to_intel_framebuffer(fb)->obj);
 		mutex_unlock(&dev->struct_mutex);
 		DRM_ERROR("failed to update base address\n");
 		return ret;
 	}
 
+	old_fb = crtc->fb;
+	crtc->fb = fb;
+	crtc->x = x;
+	crtc->y = y;
+
 	if (old_fb) {
 		intel_wait_for_vblank(dev, intel_crtc->pipe);
 		intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
@@ -2709,11 +2664,10 @@ static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
 	DRM_DEBUG_KMS("FDI train done.\n");
 }
 
-static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
+static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
 {
-	struct drm_device *dev = crtc->dev;
+	struct drm_device *dev = intel_crtc->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
 	u32 reg, temp;
 
@@ -2754,6 +2708,35 @@ static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
 	}
 }
 
+static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
+{
+	struct drm_device *dev = intel_crtc->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int pipe = intel_crtc->pipe;
+	u32 reg, temp;
+
+	/* Switch from PCDclk to Rawclk */
+	reg = FDI_RX_CTL(pipe);
+	temp = I915_READ(reg);
+	I915_WRITE(reg, temp & ~FDI_PCDCLK);
+
+	/* Disable CPU FDI TX PLL */
+	reg = FDI_TX_CTL(pipe);
+	temp = I915_READ(reg);
+	I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
+
+	POSTING_READ(reg);
+	udelay(100);
+
+	reg = FDI_RX_CTL(pipe);
+	temp = I915_READ(reg);
+	I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
+
+	/* Wait for the clocks to turn off. */
+	POSTING_READ(reg);
+	udelay(100);
+}
+
 static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2838,13 +2821,13 @@ static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	struct intel_encoder *encoder;
+	struct intel_encoder *intel_encoder;
 
 	/*
 	 * If there's a non-PCH eDP on this crtc, it must be DP_A, and that
 	 * must be driven by its own crtc; no sharing is possible.
 	 */
-	for_each_encoder_on_crtc(dev, crtc, encoder) {
+	for_each_encoder_on_crtc(dev, crtc, intel_encoder) {
 
 		/* On Haswell, LPT PCH handles the VGA connection via FDI, and Haswell
 		 * CPU handles all others */
@@ -2852,19 +2835,19 @@ static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
 			/* It is still unclear how this will work on PPT, so throw up a warning */
 			WARN_ON(!HAS_PCH_LPT(dev));
 
-			if (encoder->type == DRM_MODE_ENCODER_DAC) {
+			if (intel_encoder->type == INTEL_OUTPUT_ANALOG) {
 				DRM_DEBUG_KMS("Haswell detected DAC encoder, assuming is PCH\n");
 				return true;
 			} else {
 				DRM_DEBUG_KMS("Haswell detected encoder %d, assuming is CPU\n",
-						encoder->type);
+					      intel_encoder->type);
 				return false;
 			}
 		}
 
-		switch (encoder->type) {
+		switch (intel_encoder->type) {
 		case INTEL_OUTPUT_EDP:
-			if (!intel_encoder_is_pch_edp(&encoder->base))
+			if (!intel_encoder_is_pch_edp(&intel_encoder->base))
 				return false;
 			continue;
 		}
@@ -3181,11 +3164,14 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 	int plane = intel_crtc->plane;
 	u32 temp;
 	bool is_pch_port;
 
+	WARN_ON(!crtc->enabled);
+
 	if (intel_crtc->active)
 		return;
 
@@ -3200,10 +3186,16 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 
 	is_pch_port = intel_crtc_driving_pch(crtc);
 
-	if (is_pch_port)
-		ironlake_fdi_pll_enable(crtc);
-	else
-		ironlake_fdi_disable(crtc);
+	if (is_pch_port) {
+		ironlake_fdi_pll_enable(intel_crtc);
+	} else {
+		assert_fdi_tx_disabled(dev_priv, pipe);
+		assert_fdi_rx_disabled(dev_priv, pipe);
+	}
+
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		if (encoder->pre_enable)
+			encoder->pre_enable(encoder);
 
 	/* Enable panel fitting for LVDS */
 	if (dev_priv->pch_pf_size &&
@@ -3234,6 +3226,12 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 	mutex_unlock(&dev->struct_mutex);
 
 	intel_crtc_update_cursor(crtc, true);
+
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		encoder->enable(encoder);
+
+	if (HAS_PCH_CPT(dev))
+		intel_cpt_verify_modeset(dev, intel_crtc->pipe);
 }
 
 static void ironlake_crtc_disable(struct drm_crtc *crtc)
@@ -3241,13 +3239,18 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 	int plane = intel_crtc->plane;
 	u32 reg, temp;
 
+
 	if (!intel_crtc->active)
 		return;
 
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		encoder->disable(encoder);
+
 	intel_crtc_wait_for_pending_flips(crtc);
 	drm_vblank_off(dev, pipe);
 	intel_crtc_update_cursor(crtc, false);
@@ -3263,14 +3266,11 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	I915_WRITE(PF_CTL(pipe), 0);
 	I915_WRITE(PF_WIN_SZ(pipe), 0);
 
-	ironlake_fdi_disable(crtc);
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		if (encoder->post_disable)
+			encoder->post_disable(encoder);
 
-	/* This is a horrible layering violation; we should be doing this in
-	 * the connector/encoder ->prepare instead, but we don't always have
-	 * enough information there about the config to know whether it will
-	 * actually be necessary or just cause undesired flicker.
-	 */
-	intel_disable_pch_ports(dev_priv, pipe);
+	ironlake_fdi_disable(crtc);
 
 	intel_disable_transcoder(dev_priv, pipe);
 
@@ -3304,26 +3304,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	/* disable PCH DPLL */
 	intel_disable_pch_pll(intel_crtc);
 
-	/* Switch from PCDclk to Rawclk */
-	reg = FDI_RX_CTL(pipe);
-	temp = I915_READ(reg);
-	I915_WRITE(reg, temp & ~FDI_PCDCLK);
-
-	/* Disable CPU FDI TX PLL */
-	reg = FDI_TX_CTL(pipe);
-	temp = I915_READ(reg);
-	I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
-
-	POSTING_READ(reg);
-	udelay(100);
-
-	reg = FDI_RX_CTL(pipe);
-	temp = I915_READ(reg);
-	I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
-
-	/* Wait for the clocks to turn off. */
-	POSTING_READ(reg);
-	udelay(100);
+	ironlake_fdi_pll_disable(intel_crtc);
 
 	intel_crtc->active = false;
 	intel_update_watermarks(dev);
@@ -3333,30 +3314,6 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	mutex_unlock(&dev->struct_mutex);
 }
 
-static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
-{
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
-	int plane = intel_crtc->plane;
-
-	/* XXX: When our outputs are all unaware of DPMS modes other than off
-	 * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
-	 */
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-		DRM_DEBUG_KMS("crtc %d/%d dpms on\n", pipe, plane);
-		ironlake_crtc_enable(crtc);
-		break;
-
-	case DRM_MODE_DPMS_OFF:
-		DRM_DEBUG_KMS("crtc %d/%d dpms off\n", pipe, plane);
-		ironlake_crtc_disable(crtc);
-		break;
-	}
-}
-
 static void ironlake_crtc_off(struct drm_crtc *crtc)
 {
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -3386,9 +3343,12 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 	int plane = intel_crtc->plane;
 
+	WARN_ON(!crtc->enabled);
+
 	if (intel_crtc->active)
 		return;
 
@@ -3405,6 +3365,9 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 	/* Give the overlay scaler a chance to enable if it's on this pipe */
 	intel_crtc_dpms_overlay(intel_crtc, true);
 	intel_crtc_update_cursor(crtc, true);
+
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		encoder->enable(encoder);
 }
 
 static void i9xx_crtc_disable(struct drm_crtc *crtc)
@@ -3412,12 +3375,17 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_encoder *encoder;
 	int pipe = intel_crtc->pipe;
 	int plane = intel_crtc->plane;
 
+
 	if (!intel_crtc->active)
 		return;
 
+	for_each_encoder_on_crtc(dev, crtc, encoder)
+		encoder->disable(encoder);
+
 	/* Give the overlay scaler a chance to disable if it's on this pipe */
 	intel_crtc_wait_for_pending_flips(crtc);
 	drm_vblank_off(dev, pipe);
@@ -3436,45 +3404,17 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 	intel_update_watermarks(dev);
 }
 
-static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
-{
-	/* XXX: When our outputs are all unaware of DPMS modes other than off
-	 * and on, we should map those modes to DRM_MODE_DPMS_OFF in the CRTC.
-	 */
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-		i9xx_crtc_enable(crtc);
-		break;
-	case DRM_MODE_DPMS_OFF:
-		i9xx_crtc_disable(crtc);
-		break;
-	}
-}
-
 static void i9xx_crtc_off(struct drm_crtc *crtc)
 {
 }
 
-/**
- * Sets the power management mode of the pipe and plane.
- */
-static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
+static void intel_crtc_update_sarea(struct drm_crtc *crtc,
+				    bool enabled)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_master_private *master_priv;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	bool enabled;
-
-	if (intel_crtc->dpms_mode == mode)
-		return;
-
-	intel_crtc->dpms_mode = mode;
-
-	dev_priv->display.dpms(crtc, mode);
 
 	if (!dev->primary->master)
 		return;
@@ -3483,8 +3423,6 @@ static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 	if (!master_priv->sarea_priv)
 		return;
 
-	enabled = crtc->enabled && mode != DRM_MODE_DPMS_OFF;
-
 	switch (pipe) {
 	case 0:
 		master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0;
@@ -3500,13 +3438,42 @@ static void intel_crtc_dpms(struct drm_crtc *crtc, int mode)
 	}
 }
 
+/**
+ * Sets the power management mode of the pipe and plane.
+ */
+void intel_crtc_update_dpms(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_encoder *intel_encoder;
+	bool enable = false;
+
+	for_each_encoder_on_crtc(dev, crtc, intel_encoder)
+		enable |= intel_encoder->connectors_active;
+
+	if (enable)
+		dev_priv->display.crtc_enable(crtc);
+	else
+		dev_priv->display.crtc_disable(crtc);
+
+	intel_crtc_update_sarea(crtc, enable);
+}
+
+static void intel_crtc_noop(struct drm_crtc *crtc)
+{
+}
+
 static void intel_crtc_disable(struct drm_crtc *crtc)
 {
-	struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
 	struct drm_device *dev = crtc->dev;
+	struct drm_connector *connector;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
+	/* crtc should still be enabled when we disable it. */
+	WARN_ON(!crtc->enabled);
+
+	dev_priv->display.crtc_disable(crtc);
+	intel_crtc_update_sarea(crtc, false);
 	dev_priv->display.off(crtc);
 
 	assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
@@ -3516,63 +3483,128 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
 		mutex_lock(&dev->struct_mutex);
 		intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
 		mutex_unlock(&dev->struct_mutex);
+		crtc->fb = NULL;
+	}
+
+	/* Update computed state. */
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		if (!connector->encoder || !connector->encoder->crtc)
+			continue;
+
+		if (connector->encoder->crtc != crtc)
+			continue;
+
+		connector->dpms = DRM_MODE_DPMS_OFF;
+		to_intel_encoder(connector->encoder)->connectors_active = false;
 	}
 }
 
-/* Prepare for a mode set.
- *
- * Note we could be a lot smarter here.  We need to figure out which outputs
- * will be enabled, which disabled (in short, how the config will changes)
- * and perform the minimum necessary steps to accomplish that, e.g. updating
- * watermarks, FBC configuration, making sure PLLs are programmed correctly,
- * panel fitting is in the proper state, etc.
- */
-static void i9xx_crtc_prepare(struct drm_crtc *crtc)
+void intel_modeset_disable(struct drm_device *dev)
 {
-	i9xx_crtc_disable(crtc);
+	struct drm_crtc *crtc;
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc->enabled)
+			intel_crtc_disable(crtc);
+	}
 }
 
-static void i9xx_crtc_commit(struct drm_crtc *crtc)
+void intel_encoder_noop(struct drm_encoder *encoder)
 {
-	i9xx_crtc_enable(crtc);
 }
 
-static void ironlake_crtc_prepare(struct drm_crtc *crtc)
+void intel_encoder_destroy(struct drm_encoder *encoder)
 {
-	ironlake_crtc_disable(crtc);
+	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
+
+	drm_encoder_cleanup(encoder);
+	kfree(intel_encoder);
 }
 
-static void ironlake_crtc_commit(struct drm_crtc *crtc)
+/* Simple dpms helper for encodres with just one connector, no cloning and only
+ * one kind of off state. It clamps all !ON modes to fully OFF and changes the
+ * state of the entire output pipe. */
+void intel_encoder_dpms(struct intel_encoder *encoder, int mode)
 {
-	ironlake_crtc_enable(crtc);
+	if (mode == DRM_MODE_DPMS_ON) {
+		encoder->connectors_active = true;
+
+		intel_crtc_update_dpms(encoder->base.crtc);
+	} else {
+		encoder->connectors_active = false;
+
+		intel_crtc_update_dpms(encoder->base.crtc);
+	}
 }
 
-void intel_encoder_prepare(struct drm_encoder *encoder)
+/* Cross check the actual hw state with our own modeset state tracking (and it's
+ * internal consistency). */
+static void intel_connector_check_state(struct intel_connector *connector)
 {
-	struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
-	/* lvds has its own version of prepare see intel_lvds_prepare */
-	encoder_funcs->dpms(encoder, DRM_MODE_DPMS_OFF);
+	if (connector->get_hw_state(connector)) {
+		struct intel_encoder *encoder = connector->encoder;
+		struct drm_crtc *crtc;
+		bool encoder_enabled;
+		enum pipe pipe;
+
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
+			      connector->base.base.id,
+			      drm_get_connector_name(&connector->base));
+
+		WARN(connector->base.dpms == DRM_MODE_DPMS_OFF,
+		     "wrong connector dpms state\n");
+		WARN(connector->base.encoder != &encoder->base,
+		     "active connector not linked to encoder\n");
+		WARN(!encoder->connectors_active,
+		     "encoder->connectors_active not set\n");
+
+		encoder_enabled = encoder->get_hw_state(encoder, &pipe);
+		WARN(!encoder_enabled, "encoder not enabled\n");
+		if (WARN_ON(!encoder->base.crtc))
+			return;
+
+		crtc = encoder->base.crtc;
+
+		WARN(!crtc->enabled, "crtc not enabled\n");
+		WARN(!to_intel_crtc(crtc)->active, "crtc not active\n");
+		WARN(pipe != to_intel_crtc(crtc)->pipe,
+		     "encoder active on the wrong pipe\n");
+	}
 }
 
-void intel_encoder_commit(struct drm_encoder *encoder)
+/* Even simpler default implementation, if there's really no special case to
+ * consider. */
+void intel_connector_dpms(struct drm_connector *connector, int mode)
 {
-	struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
-	struct drm_device *dev = encoder->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct intel_encoder *encoder = intel_attached_encoder(connector);
 
-	/* lvds has its own version of commit see intel_lvds_commit */
-	encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
+	/* All the simple cases only support two dpms states. */
+	if (mode != DRM_MODE_DPMS_ON)
+		mode = DRM_MODE_DPMS_OFF;
 
-	if (HAS_PCH_CPT(dev))
-		intel_cpt_verify_modeset(dev, intel_crtc->pipe);
+	if (mode == connector->dpms)
+		return;
+
+	connector->dpms = mode;
+
+	/* Only need to change hw state when actually enabled */
+	if (encoder->base.crtc)
+		intel_encoder_dpms(encoder, mode);
+	else
+		WARN_ON(encoder->connectors_active != false);
+
+	intel_modeset_check_state(connector->dev);
 }
 
-void intel_encoder_destroy(struct drm_encoder *encoder)
+/* Simple connector->get_hw_state implementation for encoders that support only
+ * one connector and no cloning and hence the encoder state determines the state
+ * of the connector. */
+bool intel_connector_get_hw_state(struct intel_connector *connector)
 {
-	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
+	enum pipe pipe = 0;
+	struct intel_encoder *encoder = connector->encoder;
 
-	drm_encoder_cleanup(encoder);
-	kfree(intel_encoder);
+	return encoder->get_hw_state(encoder, &pipe);
 }
 
 static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
@@ -3593,6 +3625,13 @@ static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
 	if (!(adjusted_mode->private_flags & INTEL_MODE_CRTC_TIMINGS_SET))
 		drm_mode_set_crtcinfo(adjusted_mode, 0);
 
+	/* WaPruneModeWithIncorrectHsyncOffset: Cantiga+ cannot handle modes
+	 * with a hsync front porch of 0.
+	 */
+	if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
+		adjusted_mode->hsync_start == adjusted_mode->hdisplay)
+		return false;
+
 	return true;
 }
 
@@ -3728,6 +3767,7 @@ static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
  * true if they don't match).
  */
 static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
+					 struct drm_framebuffer *fb,
 					 unsigned int *pipe_bpp,
 					 struct drm_display_mode *mode)
 {
@@ -3797,7 +3837,7 @@ static bool intel_choose_pipe_bpp_dither(struct drm_crtc *crtc,
 	 * also stays within the max display bpc discovered above.
 	 */
 
-	switch (crtc->fb->depth) {
+	switch (fb->depth) {
 	case 8:
 		bpc = 8; /* since we go through a colormap */
 		break;
@@ -4216,7 +4256,7 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 			      struct drm_display_mode *mode,
 			      struct drm_display_mode *adjusted_mode,
 			      int x, int y,
-			      struct drm_framebuffer *old_fb)
+			      struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -4406,7 +4446,7 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 	I915_WRITE(DSPCNTR(plane), dspcntr);
 	POSTING_READ(DSPCNTR(plane));
 
-	ret = intel_pipe_set_base(crtc, x, y, old_fb);
+	ret = intel_pipe_set_base(crtc, x, y, fb);
 
 	intel_update_watermarks(dev);
 
@@ -4560,24 +4600,130 @@ static int ironlake_get_refclk(struct drm_crtc *crtc)
 	return 120000;
 }
 
+static void ironlake_set_pipeconf(struct drm_crtc *crtc,
+				  struct drm_display_mode *adjusted_mode,
+				  bool dither)
+{
+	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+	uint32_t val;
+
+	val = I915_READ(PIPECONF(pipe));
+
+	val &= ~PIPE_BPC_MASK;
+	switch (intel_crtc->bpp) {
+	case 18:
+		val |= PIPE_6BPC;
+		break;
+	case 24:
+		val |= PIPE_8BPC;
+		break;
+	case 30:
+		val |= PIPE_10BPC;
+		break;
+	case 36:
+		val |= PIPE_12BPC;
+		break;
+	default:
+		val |= PIPE_8BPC;
+		break;
+	}
+
+	val &= ~(PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_MASK);
+	if (dither)
+		val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
+
+	val &= ~PIPECONF_INTERLACE_MASK;
+	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+		val |= PIPECONF_INTERLACED_ILK;
+	else
+		val |= PIPECONF_PROGRESSIVE;
+
+	I915_WRITE(PIPECONF(pipe), val);
+	POSTING_READ(PIPECONF(pipe));
+}
+
+static bool ironlake_compute_clocks(struct drm_crtc *crtc,
+				    struct drm_display_mode *adjusted_mode,
+				    intel_clock_t *clock,
+				    bool *has_reduced_clock,
+				    intel_clock_t *reduced_clock)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_encoder *intel_encoder;
+	int refclk;
+	const intel_limit_t *limit;
+	bool ret, is_sdvo = false, is_tv = false, is_lvds = false;
+
+	for_each_encoder_on_crtc(dev, crtc, intel_encoder) {
+		switch (intel_encoder->type) {
+		case INTEL_OUTPUT_LVDS:
+			is_lvds = true;
+			break;
+		case INTEL_OUTPUT_SDVO:
+		case INTEL_OUTPUT_HDMI:
+			is_sdvo = true;
+			if (intel_encoder->needs_tv_clock)
+				is_tv = true;
+			break;
+		case INTEL_OUTPUT_TVOUT:
+			is_tv = true;
+			break;
+		}
+	}
+
+	refclk = ironlake_get_refclk(crtc);
+
+	/*
+	 * Returns a set of divisors for the desired target clock with the given
+	 * refclk, or FALSE.  The returned values represent the clock equation:
+	 * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
+	 */
+	limit = intel_limit(crtc, refclk);
+	ret = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
+			      clock);
+	if (!ret)
+		return false;
+
+	if (is_lvds && dev_priv->lvds_downclock_avail) {
+		/*
+		 * Ensure we match the reduced clock's P to the target clock.
+		 * If the clocks don't match, we can't switch the display clock
+		 * by using the FP0/FP1. In such case we will disable the LVDS
+		 * downclock feature.
+		*/
+		*has_reduced_clock = limit->find_pll(limit, crtc,
+						     dev_priv->lvds_downclock,
+						     refclk,
+						     clock,
+						     reduced_clock);
+	}
+
+	if (is_sdvo && is_tv)
+		i9xx_adjust_sdvo_tv_clock(adjusted_mode, clock);
+
+	return true;
+}
+
 static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 				  struct drm_display_mode *mode,
 				  struct drm_display_mode *adjusted_mode,
 				  int x, int y,
-				  struct drm_framebuffer *old_fb)
+				  struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
 	int plane = intel_crtc->plane;
-	int refclk, num_connectors = 0;
+	int num_connectors = 0;
 	intel_clock_t clock, reduced_clock;
-	u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
+	u32 dpll, fp = 0, fp2 = 0;
 	bool ok, has_reduced_clock = false, is_sdvo = false;
 	bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
 	struct intel_encoder *encoder, *edp_encoder = NULL;
-	const intel_limit_t *limit;
 	int ret;
 	struct fdi_m_n m_n = {0};
 	u32 temp;
@@ -4619,16 +4765,8 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		num_connectors++;
 	}
 
-	refclk = ironlake_get_refclk(crtc);
-
-	/*
-	 * Returns a set of divisors for the desired target clock with the given
-	 * refclk, or FALSE.  The returned values represent the clock equation:
-	 * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
-	 */
-	limit = intel_limit(crtc, refclk);
-	ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, NULL,
-			     &clock);
+	ok = ironlake_compute_clocks(crtc, adjusted_mode, &clock,
+				     &has_reduced_clock, &reduced_clock);
 	if (!ok) {
 		DRM_ERROR("Couldn't find PLL settings for mode!\n");
 		return -EINVAL;
@@ -4637,24 +4775,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	/* Ensure that the cursor is valid for the new mode before changing... */
 	intel_crtc_update_cursor(crtc, true);
 
-	if (is_lvds && dev_priv->lvds_downclock_avail) {
-		/*
-		 * Ensure we match the reduced clock's P to the target clock.
-		 * If the clocks don't match, we can't switch the display clock
-		 * by using the FP0/FP1. In such case we will disable the LVDS
-		 * downclock feature.
-		*/
-		has_reduced_clock = limit->find_pll(limit, crtc,
-						    dev_priv->lvds_downclock,
-						    refclk,
-						    &clock,
-						    &reduced_clock);
-	}
-
-	if (is_sdvo && is_tv)
-		i9xx_adjust_sdvo_tv_clock(adjusted_mode, &clock);
-
-
 	/* FDI link */
 	pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
 	lane = 0;
@@ -4682,32 +4802,17 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		target_clock = adjusted_mode->clock;
 
 	/* determine panel color depth */
-	temp = I915_READ(PIPECONF(pipe));
-	temp &= ~PIPE_BPC_MASK;
-	dither = intel_choose_pipe_bpp_dither(crtc, &pipe_bpp, mode);
-	switch (pipe_bpp) {
-	case 18:
-		temp |= PIPE_6BPC;
-		break;
-	case 24:
-		temp |= PIPE_8BPC;
-		break;
-	case 30:
-		temp |= PIPE_10BPC;
-		break;
-	case 36:
-		temp |= PIPE_12BPC;
-		break;
-	default:
+	dither = intel_choose_pipe_bpp_dither(crtc, fb, &pipe_bpp, mode);
+	if (is_lvds && dev_priv->lvds_dither)
+		dither = true;
+
+	if (pipe_bpp != 18 && pipe_bpp != 24 && pipe_bpp != 30 &&
+	    pipe_bpp != 36) {
 		WARN(1, "intel_choose_pipe_bpp returned invalid value %d\n",
-			pipe_bpp);
-		temp |= PIPE_8BPC;
+		     pipe_bpp);
 		pipe_bpp = 24;
-		break;
 	}
-
 	intel_crtc->bpp = pipe_bpp;
-	I915_WRITE(PIPECONF(pipe), temp);
 
 	if (!lane) {
 		/*
@@ -4791,12 +4896,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	else
 		dpll |= PLL_REF_INPUT_DREFCLK;
 
-	/* setup pipeconf */
-	pipeconf = I915_READ(PIPECONF(pipe));
-
-	/* Set up the display plane register */
-	dspcntr = DISPPLANE_GAMMA_ENABLE;
-
 	DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
 	drm_mode_debug_printmodeline(mode);
 
@@ -4856,12 +4955,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		I915_WRITE(PCH_LVDS, temp);
 	}
 
-	pipeconf &= ~PIPECONF_DITHER_EN;
-	pipeconf &= ~PIPECONF_DITHER_TYPE_MASK;
-	if ((is_lvds && dev_priv->lvds_dither) || dither) {
-		pipeconf |= PIPECONF_DITHER_EN;
-		pipeconf |= PIPECONF_DITHER_TYPE_SP;
-	}
 	if (is_dp && !is_cpu_edp) {
 		intel_dp_set_m_n(crtc, mode, adjusted_mode);
 	} else {
@@ -4897,9 +4990,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		}
 	}
 
-	pipeconf &= ~PIPECONF_INTERLACE_MASK;
 	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
-		pipeconf |= PIPECONF_INTERLACED_ILK;
 		/* the chip adds 2 halflines automatically */
 		adjusted_mode->crtc_vtotal -= 1;
 		adjusted_mode->crtc_vblank_end -= 1;
@@ -4907,7 +4998,6 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 			   adjusted_mode->crtc_hsync_start
 			   - adjusted_mode->crtc_htotal/2);
 	} else {
-		pipeconf |= PIPECONF_PROGRESSIVE;
 		I915_WRITE(VSYNCSHIFT(pipe), 0);
 	}
 
@@ -4945,15 +5035,15 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	if (is_cpu_edp)
 		ironlake_set_pll_edp(crtc, adjusted_mode->clock);
 
-	I915_WRITE(PIPECONF(pipe), pipeconf);
-	POSTING_READ(PIPECONF(pipe));
+	ironlake_set_pipeconf(crtc, adjusted_mode, dither);
 
 	intel_wait_for_vblank(dev, pipe);
 
-	I915_WRITE(DSPCNTR(plane), dspcntr);
+	/* Set up the display plane register */
+	I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE);
 	POSTING_READ(DSPCNTR(plane));
 
-	ret = intel_pipe_set_base(crtc, x, y, old_fb);
+	ret = intel_pipe_set_base(crtc, x, y, fb);
 
 	intel_update_watermarks(dev);
 
@@ -4966,7 +5056,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 			       struct drm_display_mode *mode,
 			       struct drm_display_mode *adjusted_mode,
 			       int x, int y,
-			       struct drm_framebuffer *old_fb)
+			       struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -4977,14 +5067,9 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc,
 	drm_vblank_pre_modeset(dev, pipe);
 
 	ret = dev_priv->display.crtc_mode_set(crtc, mode, adjusted_mode,
-					      x, y, old_fb);
+					      x, y, fb);
 	drm_vblank_post_modeset(dev, pipe);
 
-	if (ret)
-		intel_crtc->dpms_mode = DRM_MODE_DPMS_OFF;
-	else
-		intel_crtc->dpms_mode = DRM_MODE_DPMS_ON;
-
 	return ret;
 }
 
@@ -5057,6 +5142,91 @@ static void g4x_write_eld(struct drm_connector *connector,
 	I915_WRITE(G4X_AUD_CNTL_ST, i);
 }
 
+static void haswell_write_eld(struct drm_connector *connector,
+				     struct drm_crtc *crtc)
+{
+	struct drm_i915_private *dev_priv = connector->dev->dev_private;
+	uint8_t *eld = connector->eld;
+	struct drm_device *dev = crtc->dev;
+	uint32_t eldv;
+	uint32_t i;
+	int len;
+	int pipe = to_intel_crtc(crtc)->pipe;
+	int tmp;
+
+	int hdmiw_hdmiedid = HSW_AUD_EDID_DATA(pipe);
+	int aud_cntl_st = HSW_AUD_DIP_ELD_CTRL(pipe);
+	int aud_config = HSW_AUD_CFG(pipe);
+	int aud_cntrl_st2 = HSW_AUD_PIN_ELD_CP_VLD;
+
+
+	DRM_DEBUG_DRIVER("HDMI: Haswell Audio initialize....\n");
+
+	/* Audio output enable */
+	DRM_DEBUG_DRIVER("HDMI audio: enable codec\n");
+	tmp = I915_READ(aud_cntrl_st2);
+	tmp |= (AUDIO_OUTPUT_ENABLE_A << (pipe * 4));
+	I915_WRITE(aud_cntrl_st2, tmp);
+
+	/* Wait for 1 vertical blank */
+	intel_wait_for_vblank(dev, pipe);
+
+	/* Set ELD valid state */
+	tmp = I915_READ(aud_cntrl_st2);
+	DRM_DEBUG_DRIVER("HDMI audio: pin eld vld status=0x%8x\n", tmp);
+	tmp |= (AUDIO_ELD_VALID_A << (pipe * 4));
+	I915_WRITE(aud_cntrl_st2, tmp);
+	tmp = I915_READ(aud_cntrl_st2);
+	DRM_DEBUG_DRIVER("HDMI audio: eld vld status=0x%8x\n", tmp);
+
+	/* Enable HDMI mode */
+	tmp = I915_READ(aud_config);
+	DRM_DEBUG_DRIVER("HDMI audio: audio conf: 0x%8x\n", tmp);
+	/* clear N_programing_enable and N_value_index */
+	tmp &= ~(AUD_CONFIG_N_VALUE_INDEX | AUD_CONFIG_N_PROG_ENABLE);
+	I915_WRITE(aud_config, tmp);
+
+	DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(pipe));
+
+	eldv = AUDIO_ELD_VALID_A << (pipe * 4);
+
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT)) {
+		DRM_DEBUG_DRIVER("ELD: DisplayPort detected\n");
+		eld[5] |= (1 << 2);	/* Conn_Type, 0x1 = DisplayPort */
+		I915_WRITE(aud_config, AUD_CONFIG_N_VALUE_INDEX); /* 0x1 = DP */
+	} else
+		I915_WRITE(aud_config, 0);
+
+	if (intel_eld_uptodate(connector,
+			       aud_cntrl_st2, eldv,
+			       aud_cntl_st, IBX_ELD_ADDRESS,
+			       hdmiw_hdmiedid))
+		return;
+
+	i = I915_READ(aud_cntrl_st2);
+	i &= ~eldv;
+	I915_WRITE(aud_cntrl_st2, i);
+
+	if (!eld[0])
+		return;
+
+	i = I915_READ(aud_cntl_st);
+	i &= ~IBX_ELD_ADDRESS;
+	I915_WRITE(aud_cntl_st, i);
+	i = (i >> 29) & DIP_PORT_SEL_MASK;		/* DIP_Port_Select, 0x1 = PortB */
+	DRM_DEBUG_DRIVER("port num:%d\n", i);
+
+	len = min_t(uint8_t, eld[2], 21);	/* 84 bytes of hw ELD buffer */
+	DRM_DEBUG_DRIVER("ELD size %d\n", len);
+	for (i = 0; i < len; i++)
+		I915_WRITE(hdmiw_hdmiedid, *((uint32_t *)eld + i));
+
+	i = I915_READ(aud_cntrl_st2);
+	i |= eldv;
+	I915_WRITE(aud_cntrl_st2, i);
+
+}
+
 static void ironlake_write_eld(struct drm_connector *connector,
 				     struct drm_crtc *crtc)
 {
@@ -5069,28 +5239,24 @@ static void ironlake_write_eld(struct drm_connector *connector,
 	int aud_config;
 	int aud_cntl_st;
 	int aud_cntrl_st2;
+	int pipe = to_intel_crtc(crtc)->pipe;
 
 	if (HAS_PCH_IBX(connector->dev)) {
-		hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID_A;
-		aud_config = IBX_AUD_CONFIG_A;
-		aud_cntl_st = IBX_AUD_CNTL_ST_A;
+		hdmiw_hdmiedid = IBX_HDMIW_HDMIEDID(pipe);
+		aud_config = IBX_AUD_CFG(pipe);
+		aud_cntl_st = IBX_AUD_CNTL_ST(pipe);
 		aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
 	} else {
-		hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID_A;
-		aud_config = CPT_AUD_CONFIG_A;
-		aud_cntl_st = CPT_AUD_CNTL_ST_A;
+		hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID(pipe);
+		aud_config = CPT_AUD_CFG(pipe);
+		aud_cntl_st = CPT_AUD_CNTL_ST(pipe);
 		aud_cntrl_st2 = CPT_AUD_CNTRL_ST2;
 	}
 
-	i = to_intel_crtc(crtc)->pipe;
-	hdmiw_hdmiedid += i * 0x100;
-	aud_cntl_st += i * 0x100;
-	aud_config += i * 0x100;
-
-	DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(i));
+	DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(pipe));
 
 	i = I915_READ(aud_cntl_st);
-	i = (i >> 29) & 0x3;		/* DIP_Port_Select, 0x1 = PortB */
+	i = (i >> 29) & DIP_PORT_SEL_MASK;		/* DIP_Port_Select, 0x1 = PortB */
 	if (!i) {
 		DRM_DEBUG_DRIVER("Audio directed to unknown port\n");
 		/* operate blindly on all ports */
@@ -5337,8 +5503,6 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 	uint32_t addr;
 	int ret;
 
-	DRM_DEBUG_KMS("\n");
-
 	/* if we want to turn off the cursor ignore width and height */
 	if (!handle) {
 		DRM_DEBUG_KMS("cursor off\n");
@@ -5584,17 +5748,18 @@ mode_fits_in_fbdev(struct drm_device *dev,
 	return fb;
 }
 
-bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
-				struct drm_connector *connector,
+bool intel_get_load_detect_pipe(struct drm_connector *connector,
 				struct drm_display_mode *mode,
 				struct intel_load_detect_pipe *old)
 {
 	struct intel_crtc *intel_crtc;
+	struct intel_encoder *intel_encoder =
+		intel_attached_encoder(connector);
 	struct drm_crtc *possible_crtc;
 	struct drm_encoder *encoder = &intel_encoder->base;
 	struct drm_crtc *crtc = NULL;
 	struct drm_device *dev = encoder->dev;
-	struct drm_framebuffer *old_fb;
+	struct drm_framebuffer *fb;
 	int i = -1;
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
@@ -5615,21 +5780,12 @@ bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
 	if (encoder->crtc) {
 		crtc = encoder->crtc;
 
-		intel_crtc = to_intel_crtc(crtc);
-		old->dpms_mode = intel_crtc->dpms_mode;
+		old->dpms_mode = connector->dpms;
 		old->load_detect_temp = false;
 
 		/* Make sure the crtc and connector are running */
-		if (intel_crtc->dpms_mode != DRM_MODE_DPMS_ON) {
-			struct drm_encoder_helper_funcs *encoder_funcs;
-			struct drm_crtc_helper_funcs *crtc_funcs;
-
-			crtc_funcs = crtc->helper_private;
-			crtc_funcs->dpms(crtc, DRM_MODE_DPMS_ON);
-
-			encoder_funcs = encoder->helper_private;
-			encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
-		}
+		if (connector->dpms != DRM_MODE_DPMS_ON)
+			connector->funcs->dpms(connector, DRM_MODE_DPMS_ON);
 
 		return true;
 	}
@@ -5653,19 +5809,17 @@ bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
 		return false;
 	}
 
-	encoder->crtc = crtc;
-	connector->encoder = encoder;
+	intel_encoder->new_crtc = to_intel_crtc(crtc);
+	to_intel_connector(connector)->new_encoder = intel_encoder;
 
 	intel_crtc = to_intel_crtc(crtc);
-	old->dpms_mode = intel_crtc->dpms_mode;
+	old->dpms_mode = connector->dpms;
 	old->load_detect_temp = true;
 	old->release_fb = NULL;
 
 	if (!mode)
 		mode = &load_detect_mode;
 
-	old_fb = crtc->fb;
-
 	/* We need a framebuffer large enough to accommodate all accesses
 	 * that the plane may generate whilst we perform load detection.
 	 * We can not rely on the fbcon either being present (we get called
@@ -5673,50 +5827,52 @@ bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
 	 * not even exist) or that it is large enough to satisfy the
 	 * requested mode.
 	 */
-	crtc->fb = mode_fits_in_fbdev(dev, mode);
-	if (crtc->fb == NULL) {
+	fb = mode_fits_in_fbdev(dev, mode);
+	if (fb == NULL) {
 		DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
-		crtc->fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
-		old->release_fb = crtc->fb;
+		fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
+		old->release_fb = fb;
 	} else
 		DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
-	if (IS_ERR(crtc->fb)) {
+	if (IS_ERR(fb)) {
 		DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
-		crtc->fb = old_fb;
-		return false;
+		goto fail;
 	}
 
-	if (!drm_crtc_helper_set_mode(crtc, mode, 0, 0, old_fb)) {
+	if (!intel_set_mode(crtc, mode, 0, 0, fb)) {
 		DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
 		if (old->release_fb)
 			old->release_fb->funcs->destroy(old->release_fb);
-		crtc->fb = old_fb;
-		return false;
+		goto fail;
 	}
 
 	/* let the connector get through one full cycle before testing */
 	intel_wait_for_vblank(dev, intel_crtc->pipe);
 
 	return true;
+fail:
+	connector->encoder = NULL;
+	encoder->crtc = NULL;
+	return false;
 }
 
-void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
-				    struct drm_connector *connector,
+void intel_release_load_detect_pipe(struct drm_connector *connector,
 				    struct intel_load_detect_pipe *old)
 {
+	struct intel_encoder *intel_encoder =
+		intel_attached_encoder(connector);
 	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_device *dev = encoder->dev;
-	struct drm_crtc *crtc = encoder->crtc;
-	struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
-	struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
 
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
 		      connector->base.id, drm_get_connector_name(connector),
 		      encoder->base.id, drm_get_encoder_name(encoder));
 
 	if (old->load_detect_temp) {
-		connector->encoder = NULL;
-		drm_helper_disable_unused_functions(dev);
+		struct drm_crtc *crtc = encoder->crtc;
+
+		to_intel_connector(connector)->new_encoder = NULL;
+		intel_encoder->new_crtc = NULL;
+		intel_set_mode(crtc, NULL, 0, 0, NULL);
 
 		if (old->release_fb)
 			old->release_fb->funcs->destroy(old->release_fb);
@@ -5725,10 +5881,8 @@ void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
 	}
 
 	/* Switch crtc and encoder back off if necessary */
-	if (old->dpms_mode != DRM_MODE_DPMS_ON) {
-		encoder_funcs->dpms(encoder, old->dpms_mode);
-		crtc_funcs->dpms(crtc, old->dpms_mode);
-	}
+	if (old->dpms_mode != DRM_MODE_DPMS_ON)
+		connector->funcs->dpms(connector, old->dpms_mode);
 }
 
 /* Returns the clock of the currently programmed mode of the given pipe. */
@@ -5850,46 +6004,6 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
 	return mode;
 }
 
-#define GPU_IDLE_TIMEOUT 500 /* ms */
-
-/* When this timer fires, we've been idle for awhile */
-static void intel_gpu_idle_timer(unsigned long arg)
-{
-	struct drm_device *dev = (struct drm_device *)arg;
-	drm_i915_private_t *dev_priv = dev->dev_private;
-
-	if (!list_empty(&dev_priv->mm.active_list)) {
-		/* Still processing requests, so just re-arm the timer. */
-		mod_timer(&dev_priv->idle_timer, jiffies +
-			  msecs_to_jiffies(GPU_IDLE_TIMEOUT));
-		return;
-	}
-
-	dev_priv->busy = false;
-	queue_work(dev_priv->wq, &dev_priv->idle_work);
-}
-
-#define CRTC_IDLE_TIMEOUT 1000 /* ms */
-
-static void intel_crtc_idle_timer(unsigned long arg)
-{
-	struct intel_crtc *intel_crtc = (struct intel_crtc *)arg;
-	struct drm_crtc *crtc = &intel_crtc->base;
-	drm_i915_private_t *dev_priv = crtc->dev->dev_private;
-	struct intel_framebuffer *intel_fb;
-
-	intel_fb = to_intel_framebuffer(crtc->fb);
-	if (intel_fb && intel_fb->obj->active) {
-		/* The framebuffer is still being accessed by the GPU. */
-		mod_timer(&intel_crtc->idle_timer, jiffies +
-			  msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
-		return;
-	}
-
-	intel_crtc->busy = false;
-	queue_work(dev_priv->wq, &dev_priv->idle_work);
-}
-
 static void intel_increase_pllclock(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
@@ -5919,10 +6033,6 @@ static void intel_increase_pllclock(struct drm_crtc *crtc)
 		if (dpll & DISPLAY_RATE_SELECT_FPA1)
 			DRM_DEBUG_DRIVER("failed to upclock LVDS!\n");
 	}
-
-	/* Schedule downclock */
-	mod_timer(&intel_crtc->idle_timer, jiffies +
-		  msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
 }
 
 static void intel_decrease_pllclock(struct drm_crtc *crtc)
@@ -5961,89 +6071,46 @@ static void intel_decrease_pllclock(struct drm_crtc *crtc)
 
 }
 
-/**
- * intel_idle_update - adjust clocks for idleness
- * @work: work struct
- *
- * Either the GPU or display (or both) went idle.  Check the busy status
- * here and adjust the CRTC and GPU clocks as necessary.
- */
-static void intel_idle_update(struct work_struct *work)
+void intel_mark_busy(struct drm_device *dev)
 {
-	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-						    idle_work);
-	struct drm_device *dev = dev_priv->dev;
+	i915_update_gfx_val(dev->dev_private);
+}
+
+void intel_mark_idle(struct drm_device *dev)
+{
+}
+
+void intel_mark_fb_busy(struct drm_i915_gem_object *obj)
+{
+	struct drm_device *dev = obj->base.dev;
 	struct drm_crtc *crtc;
-	struct intel_crtc *intel_crtc;
 
 	if (!i915_powersave)
 		return;
 
-	mutex_lock(&dev->struct_mutex);
-
-	i915_update_gfx_val(dev_priv);
-
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		/* Skip inactive CRTCs */
 		if (!crtc->fb)
 			continue;
 
-		intel_crtc = to_intel_crtc(crtc);
-		if (!intel_crtc->busy)
-			intel_decrease_pllclock(crtc);
+		if (to_intel_framebuffer(crtc->fb)->obj == obj)
+			intel_increase_pllclock(crtc);
 	}
-
-
-	mutex_unlock(&dev->struct_mutex);
 }
 
-/**
- * intel_mark_busy - mark the GPU and possibly the display busy
- * @dev: drm device
- * @obj: object we're operating on
- *
- * Callers can use this function to indicate that the GPU is busy processing
- * commands.  If @obj matches one of the CRTC objects (i.e. it's a scanout
- * buffer), we'll also mark the display as busy, so we know to increase its
- * clock frequency.
- */
-void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
+void intel_mark_fb_idle(struct drm_i915_gem_object *obj)
 {
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc = NULL;
-	struct intel_framebuffer *intel_fb;
-	struct intel_crtc *intel_crtc;
-
-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
-		return;
-
-	if (!dev_priv->busy) {
-		intel_sanitize_pm(dev);
-		dev_priv->busy = true;
-	} else
-		mod_timer(&dev_priv->idle_timer, jiffies +
-			  msecs_to_jiffies(GPU_IDLE_TIMEOUT));
+	struct drm_device *dev = obj->base.dev;
+	struct drm_crtc *crtc;
 
-	if (obj == NULL)
+	if (!i915_powersave)
 		return;
 
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		if (!crtc->fb)
 			continue;
 
-		intel_crtc = to_intel_crtc(crtc);
-		intel_fb = to_intel_framebuffer(crtc->fb);
-		if (intel_fb->obj == obj) {
-			if (!intel_crtc->busy) {
-				/* Non-busy -> busy, upclock */
-				intel_increase_pllclock(crtc);
-				intel_crtc->busy = true;
-			} else {
-				/* Busy -> busy, put off timer */
-				mod_timer(&intel_crtc->idle_timer, jiffies +
-					  msecs_to_jiffies(CRTC_IDLE_TIMEOUT));
-			}
-		}
+		if (to_intel_framebuffer(crtc->fb)->obj == obj)
+			intel_decrease_pllclock(crtc);
 	}
 }
 
@@ -6394,7 +6461,7 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 	default:
 		WARN_ONCE(1, "unknown plane in flip command\n");
 		ret = -ENODEV;
-		goto err;
+		goto err_unpin;
 	}
 
 	ret = intel_ring_begin(ring, 4);
@@ -6502,7 +6569,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 		goto cleanup_pending;
 
 	intel_disable_fbc(dev);
-	intel_mark_busy(dev, obj);
+	intel_mark_fb_busy(obj);
 	mutex_unlock(&dev->struct_mutex);
 
 	trace_i915_flip_request(intel_crtc->plane, obj);
@@ -6527,81 +6594,807 @@ free_work:
 	return ret;
 }
 
-static void intel_sanitize_modesetting(struct drm_device *dev,
-				       int pipe, int plane)
+static struct drm_crtc_helper_funcs intel_helper_funcs = {
+	.mode_set_base_atomic = intel_pipe_set_base_atomic,
+	.load_lut = intel_crtc_load_lut,
+	.disable = intel_crtc_noop,
+};
+
+bool intel_encoder_check_is_cloned(struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 reg, val;
-	int i;
+	struct intel_encoder *other_encoder;
+	struct drm_crtc *crtc = &encoder->new_crtc->base;
 
-	/* Clear any frame start delays used for debugging left by the BIOS */
-	for_each_pipe(i) {
-		reg = PIPECONF(i);
-		I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
+	if (WARN_ON(!crtc))
+		return false;
+
+	list_for_each_entry(other_encoder,
+			    &crtc->dev->mode_config.encoder_list,
+			    base.head) {
+
+		if (&other_encoder->new_crtc->base != crtc ||
+		    encoder == other_encoder)
+			continue;
+		else
+			return true;
 	}
 
-	if (HAS_PCH_SPLIT(dev))
-		return;
+	return false;
+}
 
-	/* Who knows what state these registers were left in by the BIOS or
-	 * grub?
-	 *
-	 * If we leave the registers in a conflicting state (e.g. with the
-	 * display plane reading from the other pipe than the one we intend
-	 * to use) then when we attempt to teardown the active mode, we will
-	 * not disable the pipes and planes in the correct order -- leaving
-	 * a plane reading from a disabled pipe and possibly leading to
-	 * undefined behaviour.
+static bool intel_encoder_crtc_ok(struct drm_encoder *encoder,
+				  struct drm_crtc *crtc)
+{
+	struct drm_device *dev;
+	struct drm_crtc *tmp;
+	int crtc_mask = 1;
+
+	WARN(!crtc, "checking null crtc?\n");
+
+	dev = crtc->dev;
+
+	list_for_each_entry(tmp, &dev->mode_config.crtc_list, head) {
+		if (tmp == crtc)
+			break;
+		crtc_mask <<= 1;
+	}
+
+	if (encoder->possible_crtcs & crtc_mask)
+		return true;
+	return false;
+}
+
+/**
+ * intel_modeset_update_staged_output_state
+ *
+ * Updates the staged output configuration state, e.g. after we've read out the
+ * current hw state.
+ */
+static void intel_modeset_update_staged_output_state(struct drm_device *dev)
+{
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		connector->new_encoder =
+			to_intel_encoder(connector->base.encoder);
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		encoder->new_crtc =
+			to_intel_crtc(encoder->base.crtc);
+	}
+}
+
+/**
+ * intel_modeset_commit_output_state
+ *
+ * This function copies the stage display pipe configuration to the real one.
+ */
+static void intel_modeset_commit_output_state(struct drm_device *dev)
+{
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		connector->base.encoder = &connector->new_encoder->base;
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		encoder->base.crtc = &encoder->new_crtc->base;
+	}
+}
+
+static struct drm_display_mode *
+intel_modeset_adjusted_mode(struct drm_crtc *crtc,
+			    struct drm_display_mode *mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_display_mode *adjusted_mode;
+	struct drm_encoder_helper_funcs *encoder_funcs;
+	struct intel_encoder *encoder;
+
+	adjusted_mode = drm_mode_duplicate(dev, mode);
+	if (!adjusted_mode)
+		return ERR_PTR(-ENOMEM);
+
+	/* Pass our mode to the connectors and the CRTC to give them a chance to
+	 * adjust it according to limitations or connector properties, and also
+	 * a chance to reject the mode entirely.
 	 */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
 
-	reg = DSPCNTR(plane);
-	val = I915_READ(reg);
+		if (&encoder->new_crtc->base != crtc)
+			continue;
+		encoder_funcs = encoder->base.helper_private;
+		if (!(encoder_funcs->mode_fixup(&encoder->base, mode,
+						adjusted_mode))) {
+			DRM_DEBUG_KMS("Encoder fixup failed\n");
+			goto fail;
+		}
+	}
 
-	if ((val & DISPLAY_PLANE_ENABLE) == 0)
-		return;
-	if (!!(val & DISPPLANE_SEL_PIPE_MASK) == pipe)
-		return;
+	if (!(intel_crtc_mode_fixup(crtc, mode, adjusted_mode))) {
+		DRM_DEBUG_KMS("CRTC fixup failed\n");
+		goto fail;
+	}
+	DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
 
-	/* This display plane is active and attached to the other CPU pipe. */
-	pipe = !pipe;
+	return adjusted_mode;
+fail:
+	drm_mode_destroy(dev, adjusted_mode);
+	return ERR_PTR(-EINVAL);
+}
 
-	/* Disable the plane and wait for it to stop reading from the pipe. */
-	intel_disable_plane(dev_priv, plane, pipe);
-	intel_disable_pipe(dev_priv, pipe);
+/* Computes which crtcs are affected and sets the relevant bits in the mask. For
+ * simplicity we use the crtc's pipe number (because it's easier to obtain). */
+static void
+intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes,
+			     unsigned *prepare_pipes, unsigned *disable_pipes)
+{
+	struct intel_crtc *intel_crtc;
+	struct drm_device *dev = crtc->dev;
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+	struct drm_crtc *tmp_crtc;
+
+	*disable_pipes = *modeset_pipes = *prepare_pipes = 0;
+
+	/* Check which crtcs have changed outputs connected to them, these need
+	 * to be part of the prepare_pipes mask. We don't (yet) support global
+	 * modeset across multiple crtcs, so modeset_pipes will only have one
+	 * bit set at most. */
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		if (connector->base.encoder == &connector->new_encoder->base)
+			continue;
+
+		if (connector->base.encoder) {
+			tmp_crtc = connector->base.encoder->crtc;
+
+			*prepare_pipes |= 1 << to_intel_crtc(tmp_crtc)->pipe;
+		}
+
+		if (connector->new_encoder)
+			*prepare_pipes |=
+				1 << connector->new_encoder->new_crtc->pipe;
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		if (encoder->base.crtc == &encoder->new_crtc->base)
+			continue;
+
+		if (encoder->base.crtc) {
+			tmp_crtc = encoder->base.crtc;
+
+			*prepare_pipes |= 1 << to_intel_crtc(tmp_crtc)->pipe;
+		}
+
+		if (encoder->new_crtc)
+			*prepare_pipes |= 1 << encoder->new_crtc->pipe;
+	}
+
+	/* Check for any pipes that will be fully disabled ... */
+	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
+			    base.head) {
+		bool used = false;
+
+		/* Don't try to disable disabled crtcs. */
+		if (!intel_crtc->base.enabled)
+			continue;
+
+		list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+				    base.head) {
+			if (encoder->new_crtc == intel_crtc)
+				used = true;
+		}
+
+		if (!used)
+			*disable_pipes |= 1 << intel_crtc->pipe;
+	}
+
+
+	/* set_mode is also used to update properties on life display pipes. */
+	intel_crtc = to_intel_crtc(crtc);
+	if (crtc->enabled)
+		*prepare_pipes |= 1 << intel_crtc->pipe;
+
+	/* We only support modeset on one single crtc, hence we need to do that
+	 * only for the passed in crtc iff we change anything else than just
+	 * disable crtcs.
+	 *
+	 * This is actually not true, to be fully compatible with the old crtc
+	 * helper we automatically disable _any_ output (i.e. doesn't need to be
+	 * connected to the crtc we're modesetting on) if it's disconnected.
+	 * Which is a rather nutty api (since changed the output configuration
+	 * without userspace's explicit request can lead to confusion), but
+	 * alas. Hence we currently need to modeset on all pipes we prepare. */
+	if (*prepare_pipes)
+		*modeset_pipes = *prepare_pipes;
+
+	/* ... and mask these out. */
+	*modeset_pipes &= ~(*disable_pipes);
+	*prepare_pipes &= ~(*disable_pipes);
 }
 
-static void intel_crtc_reset(struct drm_crtc *crtc)
+static bool intel_crtc_in_use(struct drm_crtc *crtc)
 {
+	struct drm_encoder *encoder;
 	struct drm_device *dev = crtc->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	/* Reset flags back to the 'unknown' status so that they
-	 * will be correctly set on the initial modeset.
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
+		if (encoder->crtc == crtc)
+			return true;
+
+	return false;
+}
+
+static void
+intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes)
+{
+	struct intel_encoder *intel_encoder;
+	struct intel_crtc *intel_crtc;
+	struct drm_connector *connector;
+
+	list_for_each_entry(intel_encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		if (!intel_encoder->base.crtc)
+			continue;
+
+		intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
+
+		if (prepare_pipes & (1 << intel_crtc->pipe))
+			intel_encoder->connectors_active = false;
+	}
+
+	intel_modeset_commit_output_state(dev);
+
+	/* Update computed state. */
+	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
+			    base.head) {
+		intel_crtc->base.enabled = intel_crtc_in_use(&intel_crtc->base);
+	}
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		if (!connector->encoder || !connector->encoder->crtc)
+			continue;
+
+		intel_crtc = to_intel_crtc(connector->encoder->crtc);
+
+		if (prepare_pipes & (1 << intel_crtc->pipe)) {
+			struct drm_property *dpms_property =
+				dev->mode_config.dpms_property;
+
+			connector->dpms = DRM_MODE_DPMS_ON;
+			drm_connector_property_set_value(connector,
+							 dpms_property,
+							 DRM_MODE_DPMS_ON);
+
+			intel_encoder = to_intel_encoder(connector->encoder);
+			intel_encoder->connectors_active = true;
+		}
+	}
+
+}
+
+#define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
+	list_for_each_entry((intel_crtc), \
+			    &(dev)->mode_config.crtc_list, \
+			    base.head) \
+		if (mask & (1 <<(intel_crtc)->pipe)) \
+
+void
+intel_modeset_check_state(struct drm_device *dev)
+{
+	struct intel_crtc *crtc;
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		/* This also checks the encoder/connector hw state with the
+		 * ->get_hw_state callbacks. */
+		intel_connector_check_state(connector);
+
+		WARN(&connector->new_encoder->base != connector->base.encoder,
+		     "connector's staged encoder doesn't match current encoder\n");
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		bool enabled = false;
+		bool active = false;
+		enum pipe pipe, tracked_pipe;
+
+		DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
+			      encoder->base.base.id,
+			      drm_get_encoder_name(&encoder->base));
+
+		WARN(&encoder->new_crtc->base != encoder->base.crtc,
+		     "encoder's stage crtc doesn't match current crtc\n");
+		WARN(encoder->connectors_active && !encoder->base.crtc,
+		     "encoder's active_connectors set, but no crtc\n");
+
+		list_for_each_entry(connector, &dev->mode_config.connector_list,
+				    base.head) {
+			if (connector->base.encoder != &encoder->base)
+				continue;
+			enabled = true;
+			if (connector->base.dpms != DRM_MODE_DPMS_OFF)
+				active = true;
+		}
+		WARN(!!encoder->base.crtc != enabled,
+		     "encoder's enabled state mismatch "
+		     "(expected %i, found %i)\n",
+		     !!encoder->base.crtc, enabled);
+		WARN(active && !encoder->base.crtc,
+		     "active encoder with no crtc\n");
+
+		WARN(encoder->connectors_active != active,
+		     "encoder's computed active state doesn't match tracked active state "
+		     "(expected %i, found %i)\n", active, encoder->connectors_active);
+
+		active = encoder->get_hw_state(encoder, &pipe);
+		WARN(active != encoder->connectors_active,
+		     "encoder's hw state doesn't match sw tracking "
+		     "(expected %i, found %i)\n",
+		     encoder->connectors_active, active);
+
+		if (!encoder->base.crtc)
+			continue;
+
+		tracked_pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+		WARN(active && pipe != tracked_pipe,
+		     "active encoder's pipe doesn't match"
+		     "(expected %i, found %i)\n",
+		     tracked_pipe, pipe);
+
+	}
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+			    base.head) {
+		bool enabled = false;
+		bool active = false;
+
+		DRM_DEBUG_KMS("[CRTC:%d]\n",
+			      crtc->base.base.id);
+
+		WARN(crtc->active && !crtc->base.enabled,
+		     "active crtc, but not enabled in sw tracking\n");
+
+		list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+				    base.head) {
+			if (encoder->base.crtc != &crtc->base)
+				continue;
+			enabled = true;
+			if (encoder->connectors_active)
+				active = true;
+		}
+		WARN(active != crtc->active,
+		     "crtc's computed active state doesn't match tracked active state "
+		     "(expected %i, found %i)\n", active, crtc->active);
+		WARN(enabled != crtc->base.enabled,
+		     "crtc's computed enabled state doesn't match tracked enabled state "
+		     "(expected %i, found %i)\n", enabled, crtc->base.enabled);
+
+		assert_pipe(dev->dev_private, crtc->pipe, crtc->active);
+	}
+}
+
+bool intel_set_mode(struct drm_crtc *crtc,
+		    struct drm_display_mode *mode,
+		    int x, int y, struct drm_framebuffer *fb)
+{
+	struct drm_device *dev = crtc->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_display_mode *adjusted_mode, saved_mode, saved_hwmode;
+	struct drm_encoder_helper_funcs *encoder_funcs;
+	struct drm_encoder *encoder;
+	struct intel_crtc *intel_crtc;
+	unsigned disable_pipes, prepare_pipes, modeset_pipes;
+	bool ret = true;
+
+	intel_modeset_affected_pipes(crtc, &modeset_pipes,
+				     &prepare_pipes, &disable_pipes);
+
+	DRM_DEBUG_KMS("set mode pipe masks: modeset: %x, prepare: %x, disable: %x\n",
+		      modeset_pipes, prepare_pipes, disable_pipes);
+
+	for_each_intel_crtc_masked(dev, disable_pipes, intel_crtc)
+		intel_crtc_disable(&intel_crtc->base);
+
+	saved_hwmode = crtc->hwmode;
+	saved_mode = crtc->mode;
+
+	/* Hack: Because we don't (yet) support global modeset on multiple
+	 * crtcs, we don't keep track of the new mode for more than one crtc.
+	 * Hence simply check whether any bit is set in modeset_pipes in all the
+	 * pieces of code that are not yet converted to deal with mutliple crtcs
+	 * changing their mode at the same time. */
+	adjusted_mode = NULL;
+	if (modeset_pipes) {
+		adjusted_mode = intel_modeset_adjusted_mode(crtc, mode);
+		if (IS_ERR(adjusted_mode)) {
+			return false;
+		}
+	}
+
+	for_each_intel_crtc_masked(dev, prepare_pipes, intel_crtc) {
+		if (intel_crtc->base.enabled)
+			dev_priv->display.crtc_disable(&intel_crtc->base);
+	}
+
+	/* crtc->mode is already used by the ->mode_set callbacks, hence we need
+	 * to set it here already despite that we pass it down the callchain.
 	 */
-	intel_crtc->dpms_mode = -1;
+	if (modeset_pipes)
+		crtc->mode = *mode;
 
-	/* We need to fix up any BIOS configuration that conflicts with
-	 * our expectations.
+	/* Only after disabling all output pipelines that will be changed can we
+	 * update the the output configuration. */
+	intel_modeset_update_state(dev, prepare_pipes);
+
+	/* Set up the DPLL and any encoders state that needs to adjust or depend
+	 * on the DPLL.
 	 */
-	intel_sanitize_modesetting(dev, intel_crtc->pipe, intel_crtc->plane);
+	for_each_intel_crtc_masked(dev, modeset_pipes, intel_crtc) {
+		ret = !intel_crtc_mode_set(&intel_crtc->base,
+					   mode, adjusted_mode,
+					   x, y, fb);
+		if (!ret)
+		    goto done;
+
+		list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+
+			if (encoder->crtc != &intel_crtc->base)
+				continue;
+
+			DRM_DEBUG_KMS("[ENCODER:%d:%s] set [MODE:%d:%s]\n",
+				encoder->base.id, drm_get_encoder_name(encoder),
+				mode->base.id, mode->name);
+			encoder_funcs = encoder->helper_private;
+			encoder_funcs->mode_set(encoder, mode, adjusted_mode);
+		}
+	}
+
+	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
+	for_each_intel_crtc_masked(dev, prepare_pipes, intel_crtc)
+		dev_priv->display.crtc_enable(&intel_crtc->base);
+
+	if (modeset_pipes) {
+		/* Store real post-adjustment hardware mode. */
+		crtc->hwmode = *adjusted_mode;
+
+		/* Calculate and store various constants which
+		 * are later needed by vblank and swap-completion
+		 * timestamping. They are derived from true hwmode.
+		 */
+		drm_calc_timestamping_constants(crtc);
+	}
+
+	/* FIXME: add subpixel order */
+done:
+	drm_mode_destroy(dev, adjusted_mode);
+	if (!ret && crtc->enabled) {
+		crtc->hwmode = saved_hwmode;
+		crtc->mode = saved_mode;
+	} else {
+		intel_modeset_check_state(dev);
+	}
+
+	return ret;
 }
 
-static struct drm_crtc_helper_funcs intel_helper_funcs = {
-	.dpms = intel_crtc_dpms,
-	.mode_fixup = intel_crtc_mode_fixup,
-	.mode_set = intel_crtc_mode_set,
-	.mode_set_base = intel_pipe_set_base,
-	.mode_set_base_atomic = intel_pipe_set_base_atomic,
-	.load_lut = intel_crtc_load_lut,
-	.disable = intel_crtc_disable,
-};
+#undef for_each_intel_crtc_masked
+
+static void intel_set_config_free(struct intel_set_config *config)
+{
+	if (!config)
+		return;
+
+	kfree(config->save_connector_encoders);
+	kfree(config->save_encoder_crtcs);
+	kfree(config);
+}
+
+static int intel_set_config_save_state(struct drm_device *dev,
+				       struct intel_set_config *config)
+{
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
+	int count;
+
+	config->save_encoder_crtcs =
+		kcalloc(dev->mode_config.num_encoder,
+			sizeof(struct drm_crtc *), GFP_KERNEL);
+	if (!config->save_encoder_crtcs)
+		return -ENOMEM;
+
+	config->save_connector_encoders =
+		kcalloc(dev->mode_config.num_connector,
+			sizeof(struct drm_encoder *), GFP_KERNEL);
+	if (!config->save_connector_encoders)
+		return -ENOMEM;
+
+	/* Copy data. Note that driver private data is not affected.
+	 * Should anything bad happen only the expected state is
+	 * restored, not the drivers personal bookkeeping.
+	 */
+	count = 0;
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		config->save_encoder_crtcs[count++] = encoder->crtc;
+	}
+
+	count = 0;
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		config->save_connector_encoders[count++] = connector->encoder;
+	}
+
+	return 0;
+}
+
+static void intel_set_config_restore_state(struct drm_device *dev,
+					   struct intel_set_config *config)
+{
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+	int count;
+
+	count = 0;
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+		encoder->new_crtc =
+			to_intel_crtc(config->save_encoder_crtcs[count++]);
+	}
+
+	count = 0;
+	list_for_each_entry(connector, &dev->mode_config.connector_list, base.head) {
+		connector->new_encoder =
+			to_intel_encoder(config->save_connector_encoders[count++]);
+	}
+}
+
+static void
+intel_set_config_compute_mode_changes(struct drm_mode_set *set,
+				      struct intel_set_config *config)
+{
+
+	/* We should be able to check here if the fb has the same properties
+	 * and then just flip_or_move it */
+	if (set->crtc->fb != set->fb) {
+		/* If we have no fb then treat it as a full mode set */
+		if (set->crtc->fb == NULL) {
+			DRM_DEBUG_KMS("crtc has no fb, full mode set\n");
+			config->mode_changed = true;
+		} else if (set->fb == NULL) {
+			config->mode_changed = true;
+		} else if (set->fb->depth != set->crtc->fb->depth) {
+			config->mode_changed = true;
+		} else if (set->fb->bits_per_pixel !=
+			   set->crtc->fb->bits_per_pixel) {
+			config->mode_changed = true;
+		} else
+			config->fb_changed = true;
+	}
+
+	if (set->fb && (set->x != set->crtc->x || set->y != set->crtc->y))
+		config->fb_changed = true;
+
+	if (set->mode && !drm_mode_equal(set->mode, &set->crtc->mode)) {
+		DRM_DEBUG_KMS("modes are different, full mode set\n");
+		drm_mode_debug_printmodeline(&set->crtc->mode);
+		drm_mode_debug_printmodeline(set->mode);
+		config->mode_changed = true;
+	}
+}
+
+static int
+intel_modeset_stage_output_state(struct drm_device *dev,
+				 struct drm_mode_set *set,
+				 struct intel_set_config *config)
+{
+	struct drm_crtc *new_crtc;
+	struct intel_connector *connector;
+	struct intel_encoder *encoder;
+	int count, ro;
+
+	/* The upper layers ensure that we either disabl a crtc or have a list
+	 * of connectors. For paranoia, double-check this. */
+	WARN_ON(!set->fb && (set->num_connectors != 0));
+	WARN_ON(set->fb && (set->num_connectors == 0));
+
+	count = 0;
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		/* Otherwise traverse passed in connector list and get encoders
+		 * for them. */
+		for (ro = 0; ro < set->num_connectors; ro++) {
+			if (set->connectors[ro] == &connector->base) {
+				connector->new_encoder = connector->encoder;
+				break;
+			}
+		}
+
+		/* If we disable the crtc, disable all its connectors. Also, if
+		 * the connector is on the changing crtc but not on the new
+		 * connector list, disable it. */
+		if ((!set->fb || ro == set->num_connectors) &&
+		    connector->base.encoder &&
+		    connector->base.encoder->crtc == set->crtc) {
+			connector->new_encoder = NULL;
+
+			DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n",
+				connector->base.base.id,
+				drm_get_connector_name(&connector->base));
+		}
+
+
+		if (&connector->new_encoder->base != connector->base.encoder) {
+			DRM_DEBUG_KMS("encoder changed, full mode switch\n");
+			config->mode_changed = true;
+		}
+
+		/* Disable all disconnected encoders. */
+		if (connector->base.status == connector_status_disconnected)
+			connector->new_encoder = NULL;
+	}
+	/* connector->new_encoder is now updated for all connectors. */
+
+	/* Update crtc of enabled connectors. */
+	count = 0;
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		if (!connector->new_encoder)
+			continue;
+
+		new_crtc = connector->new_encoder->base.crtc;
+
+		for (ro = 0; ro < set->num_connectors; ro++) {
+			if (set->connectors[ro] == &connector->base)
+				new_crtc = set->crtc;
+		}
+
+		/* Make sure the new CRTC will work with the encoder */
+		if (!intel_encoder_crtc_ok(&connector->new_encoder->base,
+					   new_crtc)) {
+			return -EINVAL;
+		}
+		connector->encoder->new_crtc = to_intel_crtc(new_crtc);
+
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d]\n",
+			connector->base.base.id,
+			drm_get_connector_name(&connector->base),
+			new_crtc->base.id);
+	}
+
+	/* Check for any encoders that needs to be disabled. */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		list_for_each_entry(connector,
+				    &dev->mode_config.connector_list,
+				    base.head) {
+			if (connector->new_encoder == encoder) {
+				WARN_ON(!connector->new_encoder->new_crtc);
+
+				goto next_encoder;
+			}
+		}
+		encoder->new_crtc = NULL;
+next_encoder:
+		/* Only now check for crtc changes so we don't miss encoders
+		 * that will be disabled. */
+		if (&encoder->new_crtc->base != encoder->base.crtc) {
+			DRM_DEBUG_KMS("crtc changed, full mode switch\n");
+			config->mode_changed = true;
+		}
+	}
+	/* Now we've also updated encoder->new_crtc for all encoders. */
+
+	return 0;
+}
+
+static int intel_crtc_set_config(struct drm_mode_set *set)
+{
+	struct drm_device *dev;
+	struct drm_mode_set save_set;
+	struct intel_set_config *config;
+	int ret;
+
+	BUG_ON(!set);
+	BUG_ON(!set->crtc);
+	BUG_ON(!set->crtc->helper_private);
+
+	if (!set->mode)
+		set->fb = NULL;
+
+	/* The fb helper likes to play gross jokes with ->mode_set_config.
+	 * Unfortunately the crtc helper doesn't do much at all for this case,
+	 * so we have to cope with this madness until the fb helper is fixed up. */
+	if (set->fb && set->num_connectors == 0)
+		return 0;
+
+	if (set->fb) {
+		DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n",
+				set->crtc->base.id, set->fb->base.id,
+				(int)set->num_connectors, set->x, set->y);
+	} else {
+		DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id);
+	}
+
+	dev = set->crtc->dev;
+
+	ret = -ENOMEM;
+	config = kzalloc(sizeof(*config), GFP_KERNEL);
+	if (!config)
+		goto out_config;
+
+	ret = intel_set_config_save_state(dev, config);
+	if (ret)
+		goto out_config;
+
+	save_set.crtc = set->crtc;
+	save_set.mode = &set->crtc->mode;
+	save_set.x = set->crtc->x;
+	save_set.y = set->crtc->y;
+	save_set.fb = set->crtc->fb;
+
+	/* Compute whether we need a full modeset, only an fb base update or no
+	 * change at all. In the future we might also check whether only the
+	 * mode changed, e.g. for LVDS where we only change the panel fitter in
+	 * such cases. */
+	intel_set_config_compute_mode_changes(set, config);
+
+	ret = intel_modeset_stage_output_state(dev, set, config);
+	if (ret)
+		goto fail;
+
+	if (config->mode_changed) {
+		if (set->mode) {
+			DRM_DEBUG_KMS("attempting to set mode from"
+					" userspace\n");
+			drm_mode_debug_printmodeline(set->mode);
+		}
+
+		if (!intel_set_mode(set->crtc, set->mode,
+				    set->x, set->y, set->fb)) {
+			DRM_ERROR("failed to set mode on [CRTC:%d]\n",
+				  set->crtc->base.id);
+			ret = -EINVAL;
+			goto fail;
+		}
+	} else if (config->fb_changed) {
+		ret = intel_pipe_set_base(set->crtc,
+					  set->x, set->y, set->fb);
+	}
+
+	intel_set_config_free(config);
+
+	return 0;
+
+fail:
+	intel_set_config_restore_state(dev, config);
+
+	/* Try to restore the config */
+	if (config->mode_changed &&
+	    !intel_set_mode(save_set.crtc, save_set.mode,
+			    save_set.x, save_set.y, save_set.fb))
+		DRM_ERROR("failed to restore config after modeset failure\n");
+
+out_config:
+	intel_set_config_free(config);
+	return ret;
+}
 
 static const struct drm_crtc_funcs intel_crtc_funcs = {
-	.reset = intel_crtc_reset,
 	.cursor_set = intel_crtc_cursor_set,
 	.cursor_move = intel_crtc_cursor_move,
 	.gamma_set = intel_crtc_gamma_set,
-	.set_config = drm_crtc_helper_set_config,
+	.set_config = intel_crtc_set_config,
 	.destroy = intel_crtc_destroy,
 	.page_flip = intel_crtc_page_flip,
 };
@@ -6655,24 +7448,9 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 	dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
 	dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
 
-	intel_crtc_reset(&intel_crtc->base);
-	intel_crtc->active = true; /* force the pipe off on setup_init_config */
 	intel_crtc->bpp = 24; /* default for pre-Ironlake */
 
-	if (HAS_PCH_SPLIT(dev)) {
-		intel_helper_funcs.prepare = ironlake_crtc_prepare;
-		intel_helper_funcs.commit = ironlake_crtc_commit;
-	} else {
-		intel_helper_funcs.prepare = i9xx_crtc_prepare;
-		intel_helper_funcs.commit = i9xx_crtc_commit;
-	}
-
 	drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
-
-	intel_crtc->busy = false;
-
-	setup_timer(&intel_crtc->idle_timer, intel_crtc_idle_timer,
-		    (unsigned long)intel_crtc);
 }
 
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
@@ -6699,15 +7477,23 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
 	return 0;
 }
 
-static int intel_encoder_clones(struct drm_device *dev, int type_mask)
+static int intel_encoder_clones(struct intel_encoder *encoder)
 {
-	struct intel_encoder *encoder;
+	struct drm_device *dev = encoder->base.dev;
+	struct intel_encoder *source_encoder;
 	int index_mask = 0;
 	int entry = 0;
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
-		if (type_mask & encoder->clone_mask)
+	list_for_each_entry(source_encoder,
+			    &dev->mode_config.encoder_list, base.head) {
+
+		if (encoder == source_encoder)
 			index_mask |= (1 << entry);
+
+		/* Intel hw has only one MUX where enocoders could be cloned. */
+		if (encoder->cloneable && source_encoder->cloneable)
+			index_mask |= (1 << entry);
+
 		entry++;
 	}
 
@@ -6748,10 +7534,10 @@ static void intel_setup_outputs(struct drm_device *dev)
 		dpd_is_edp = intel_dpd_is_edp(dev);
 
 		if (has_edp_a(dev))
-			intel_dp_init(dev, DP_A);
+			intel_dp_init(dev, DP_A, PORT_A);
 
 		if (dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
-			intel_dp_init(dev, PCH_DP_D);
+			intel_dp_init(dev, PCH_DP_D, PORT_D);
 	}
 
 	intel_crt_init(dev);
@@ -6782,22 +7568,22 @@ static void intel_setup_outputs(struct drm_device *dev)
 			/* PCH SDVOB multiplex with HDMIB */
 			found = intel_sdvo_init(dev, PCH_SDVOB, true);
 			if (!found)
-				intel_hdmi_init(dev, HDMIB);
+				intel_hdmi_init(dev, HDMIB, PORT_B);
 			if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
-				intel_dp_init(dev, PCH_DP_B);
+				intel_dp_init(dev, PCH_DP_B, PORT_B);
 		}
 
 		if (I915_READ(HDMIC) & PORT_DETECTED)
-			intel_hdmi_init(dev, HDMIC);
+			intel_hdmi_init(dev, HDMIC, PORT_C);
 
 		if (!dpd_is_edp && I915_READ(HDMID) & PORT_DETECTED)
-			intel_hdmi_init(dev, HDMID);
+			intel_hdmi_init(dev, HDMID, PORT_D);
 
 		if (I915_READ(PCH_DP_C) & DP_DETECTED)
-			intel_dp_init(dev, PCH_DP_C);
+			intel_dp_init(dev, PCH_DP_C, PORT_C);
 
 		if (!dpd_is_edp && (I915_READ(PCH_DP_D) & DP_DETECTED))
-			intel_dp_init(dev, PCH_DP_D);
+			intel_dp_init(dev, PCH_DP_D, PORT_D);
 	} else if (IS_VALLEYVIEW(dev)) {
 		int found;
 
@@ -6805,17 +7591,17 @@ static void intel_setup_outputs(struct drm_device *dev)
 			/* SDVOB multiplex with HDMIB */
 			found = intel_sdvo_init(dev, SDVOB, true);
 			if (!found)
-				intel_hdmi_init(dev, SDVOB);
+				intel_hdmi_init(dev, SDVOB, PORT_B);
 			if (!found && (I915_READ(DP_B) & DP_DETECTED))
-				intel_dp_init(dev, DP_B);
+				intel_dp_init(dev, DP_B, PORT_B);
 		}
 
 		if (I915_READ(SDVOC) & PORT_DETECTED)
-			intel_hdmi_init(dev, SDVOC);
+			intel_hdmi_init(dev, SDVOC, PORT_C);
 
 		/* Shares lanes with HDMI on SDVOC */
 		if (I915_READ(DP_C) & DP_DETECTED)
-			intel_dp_init(dev, DP_C);
+			intel_dp_init(dev, DP_C, PORT_C);
 	} else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) {
 		bool found = false;
 
@@ -6824,12 +7610,12 @@ static void intel_setup_outputs(struct drm_device *dev)
 			found = intel_sdvo_init(dev, SDVOB, true);
 			if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
-				intel_hdmi_init(dev, SDVOB);
+				intel_hdmi_init(dev, SDVOB, PORT_B);
 			}
 
 			if (!found && SUPPORTS_INTEGRATED_DP(dev)) {
 				DRM_DEBUG_KMS("probing DP_B\n");
-				intel_dp_init(dev, DP_B);
+				intel_dp_init(dev, DP_B, PORT_B);
 			}
 		}
 
@@ -6844,18 +7630,18 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 			if (SUPPORTS_INTEGRATED_HDMI(dev)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
-				intel_hdmi_init(dev, SDVOC);
+				intel_hdmi_init(dev, SDVOC, PORT_C);
 			}
 			if (SUPPORTS_INTEGRATED_DP(dev)) {
 				DRM_DEBUG_KMS("probing DP_C\n");
-				intel_dp_init(dev, DP_C);
+				intel_dp_init(dev, DP_C, PORT_C);
 			}
 		}
 
 		if (SUPPORTS_INTEGRATED_DP(dev) &&
 		    (I915_READ(DP_D) & DP_DETECTED)) {
 			DRM_DEBUG_KMS("probing DP_D\n");
-			intel_dp_init(dev, DP_D);
+			intel_dp_init(dev, DP_D, PORT_D);
 		}
 	} else if (IS_GEN2(dev))
 		intel_dvo_init(dev);
@@ -6866,12 +7652,9 @@ static void intel_setup_outputs(struct drm_device *dev)
 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
 		encoder->base.possible_crtcs = encoder->crtc_mask;
 		encoder->base.possible_clones =
-			intel_encoder_clones(dev, encoder->clone_mask);
+			intel_encoder_clones(encoder);
 	}
 
-	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(dev);
-
 	if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
 		ironlake_init_pch_refclk(dev);
 }
@@ -6973,13 +7756,15 @@ static void intel_init_display(struct drm_device *dev)
 
 	/* We always want a DPMS function */
 	if (HAS_PCH_SPLIT(dev)) {
-		dev_priv->display.dpms = ironlake_crtc_dpms;
 		dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
+		dev_priv->display.crtc_enable = ironlake_crtc_enable;
+		dev_priv->display.crtc_disable = ironlake_crtc_disable;
 		dev_priv->display.off = ironlake_crtc_off;
 		dev_priv->display.update_plane = ironlake_update_plane;
 	} else {
-		dev_priv->display.dpms = i9xx_crtc_dpms;
 		dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+		dev_priv->display.crtc_enable = i9xx_crtc_enable;
+		dev_priv->display.crtc_disable = i9xx_crtc_disable;
 		dev_priv->display.off = i9xx_crtc_off;
 		dev_priv->display.update_plane = i9xx_update_plane;
 	}
@@ -7023,7 +7808,7 @@ static void intel_init_display(struct drm_device *dev)
 			dev_priv->display.write_eld = ironlake_write_eld;
 		} else if (IS_HASWELL(dev)) {
 			dev_priv->display.fdi_link_train = hsw_fdi_link_train;
-			dev_priv->display.write_eld = ironlake_write_eld;
+			dev_priv->display.write_eld = haswell_write_eld;
 		} else
 			dev_priv->display.update_wm = NULL;
 	} else if (IS_G4X(dev)) {
@@ -7101,21 +7886,16 @@ static struct intel_quirk intel_quirks[] = {
 	/* HP Mini needs pipe A force quirk (LP: #322104) */
 	{ 0x27ae, 0x103c, 0x361a, quirk_pipea_force },
 
-	/* Thinkpad R31 needs pipe A force quirk */
-	{ 0x3577, 0x1014, 0x0505, quirk_pipea_force },
 	/* Toshiba Protege R-205, S-209 needs pipe A force quirk */
 	{ 0x2592, 0x1179, 0x0001, quirk_pipea_force },
 
-	/* ThinkPad X30 needs pipe A force quirk (LP: #304614) */
-	{ 0x3577,  0x1014, 0x0513, quirk_pipea_force },
-	/* ThinkPad X40 needs pipe A force quirk */
-
 	/* ThinkPad T60 needs pipe A force quirk (bug #16494) */
 	{ 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
 
 	/* 855 & before need to leave pipe A & dpll A up */
 	{ 0x3582, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
 	{ 0x2562, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
+	{ 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
 
 	/* Lenovo U160 cannot use SSC on LVDS */
 	{ 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
@@ -7231,10 +8011,251 @@ void intel_modeset_init(struct drm_device *dev)
 	/* Just disable it once at startup */
 	i915_disable_vga(dev);
 	intel_setup_outputs(dev);
+}
 
-	INIT_WORK(&dev_priv->idle_work, intel_idle_update);
-	setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer,
-		    (unsigned long)dev);
+static void
+intel_connector_break_all_links(struct intel_connector *connector)
+{
+	connector->base.dpms = DRM_MODE_DPMS_OFF;
+	connector->base.encoder = NULL;
+	connector->encoder->connectors_active = false;
+	connector->encoder->base.crtc = NULL;
+}
+
+static void intel_enable_pipe_a(struct drm_device *dev)
+{
+	struct intel_connector *connector;
+	struct drm_connector *crt = NULL;
+	struct intel_load_detect_pipe load_detect_temp;
+
+	/* We can't just switch on the pipe A, we need to set things up with a
+	 * proper mode and output configuration. As a gross hack, enable pipe A
+	 * by enabling the load detect pipe once. */
+	list_for_each_entry(connector,
+			    &dev->mode_config.connector_list,
+			    base.head) {
+		if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
+			crt = &connector->base;
+			break;
+		}
+	}
+
+	if (!crt)
+		return;
+
+	if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp))
+		intel_release_load_detect_pipe(crt, &load_detect_temp);
+
+
+}
+
+static void intel_sanitize_crtc(struct intel_crtc *crtc)
+{
+	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 reg, val;
+
+	/* Clear any frame start delays used for debugging left by the BIOS */
+	reg = PIPECONF(crtc->pipe);
+	I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
+
+	/* We need to sanitize the plane -> pipe mapping first because this will
+	 * disable the crtc (and hence change the state) if it is wrong. */
+	if (!HAS_PCH_SPLIT(dev)) {
+		struct intel_connector *connector;
+		bool plane;
+
+		reg = DSPCNTR(crtc->plane);
+		val = I915_READ(reg);
+
+		if ((val & DISPLAY_PLANE_ENABLE) == 0 &&
+		    (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
+			goto ok;
+
+		DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n",
+			      crtc->base.base.id);
+
+		/* Pipe has the wrong plane attached and the plane is active.
+		 * Temporarily change the plane mapping and disable everything
+		 * ...  */
+		plane = crtc->plane;
+		crtc->plane = !plane;
+		dev_priv->display.crtc_disable(&crtc->base);
+		crtc->plane = plane;
+
+		/* ... and break all links. */
+		list_for_each_entry(connector, &dev->mode_config.connector_list,
+				    base.head) {
+			if (connector->encoder->base.crtc != &crtc->base)
+				continue;
+
+			intel_connector_break_all_links(connector);
+		}
+
+		WARN_ON(crtc->active);
+		crtc->base.enabled = false;
+	}
+ok:
+
+	if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
+	    crtc->pipe == PIPE_A && !crtc->active) {
+		/* BIOS forgot to enable pipe A, this mostly happens after
+		 * resume. Force-enable the pipe to fix this, the update_dpms
+		 * call below we restore the pipe to the right state, but leave
+		 * the required bits on. */
+		intel_enable_pipe_a(dev);
+	}
+
+	/* Adjust the state of the output pipe according to whether we
+	 * have active connectors/encoders. */
+	intel_crtc_update_dpms(&crtc->base);
+
+	if (crtc->active != crtc->base.enabled) {
+		struct intel_encoder *encoder;
+
+		/* This can happen either due to bugs in the get_hw_state
+		 * functions or because the pipe is force-enabled due to the
+		 * pipe A quirk. */
+		DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was %s, now %s\n",
+			      crtc->base.base.id,
+			      crtc->base.enabled ? "enabled" : "disabled",
+			      crtc->active ? "enabled" : "disabled");
+
+		crtc->base.enabled = crtc->active;
+
+		/* Because we only establish the connector -> encoder ->
+		 * crtc links if something is active, this means the
+		 * crtc is now deactivated. Break the links. connector
+		 * -> encoder links are only establish when things are
+		 *  actually up, hence no need to break them. */
+		WARN_ON(crtc->active);
+
+		for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
+			WARN_ON(encoder->connectors_active);
+			encoder->base.crtc = NULL;
+		}
+	}
+}
+
+static void intel_sanitize_encoder(struct intel_encoder *encoder)
+{
+	struct intel_connector *connector;
+	struct drm_device *dev = encoder->base.dev;
+
+	/* We need to check both for a crtc link (meaning that the
+	 * encoder is active and trying to read from a pipe) and the
+	 * pipe itself being active. */
+	bool has_active_crtc = encoder->base.crtc &&
+		to_intel_crtc(encoder->base.crtc)->active;
+
+	if (encoder->connectors_active && !has_active_crtc) {
+		DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
+			      encoder->base.base.id,
+			      drm_get_encoder_name(&encoder->base));
+
+		/* Connector is active, but has no active pipe. This is
+		 * fallout from our resume register restoring. Disable
+		 * the encoder manually again. */
+		if (encoder->base.crtc) {
+			DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
+				      encoder->base.base.id,
+				      drm_get_encoder_name(&encoder->base));
+			encoder->disable(encoder);
+		}
+
+		/* Inconsistent output/port/pipe state happens presumably due to
+		 * a bug in one of the get_hw_state functions. Or someplace else
+		 * in our code, like the register restore mess on resume. Clamp
+		 * things to off as a safer default. */
+		list_for_each_entry(connector,
+				    &dev->mode_config.connector_list,
+				    base.head) {
+			if (connector->encoder != encoder)
+				continue;
+
+			intel_connector_break_all_links(connector);
+		}
+	}
+	/* Enabled encoders without active connectors will be fixed in
+	 * the crtc fixup. */
+}
+
+/* Scan out the current hw modeset state, sanitizes it and maps it into the drm
+ * and i915 state tracking structures. */
+void intel_modeset_setup_hw_state(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	enum pipe pipe;
+	u32 tmp;
+	struct intel_crtc *crtc;
+	struct intel_encoder *encoder;
+	struct intel_connector *connector;
+
+	for_each_pipe(pipe) {
+		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+
+		tmp = I915_READ(PIPECONF(pipe));
+		if (tmp & PIPECONF_ENABLE)
+			crtc->active = true;
+		else
+			crtc->active = false;
+
+		crtc->base.enabled = crtc->active;
+
+		DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
+			      crtc->base.base.id,
+			      crtc->active ? "enabled" : "disabled");
+	}
+
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		pipe = 0;
+
+		if (encoder->get_hw_state(encoder, &pipe)) {
+			encoder->base.crtc =
+				dev_priv->pipe_to_crtc_mapping[pipe];
+		} else {
+			encoder->base.crtc = NULL;
+		}
+
+		encoder->connectors_active = false;
+		DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe=%i\n",
+			      encoder->base.base.id,
+			      drm_get_encoder_name(&encoder->base),
+			      encoder->base.crtc ? "enabled" : "disabled",
+			      pipe);
+	}
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list,
+			    base.head) {
+		if (connector->get_hw_state(connector)) {
+			connector->base.dpms = DRM_MODE_DPMS_ON;
+			connector->encoder->connectors_active = true;
+			connector->base.encoder = &connector->encoder->base;
+		} else {
+			connector->base.dpms = DRM_MODE_DPMS_OFF;
+			connector->base.encoder = NULL;
+		}
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
+			      connector->base.base.id,
+			      drm_get_connector_name(&connector->base),
+			      connector->base.encoder ? "enabled" : "disabled");
+	}
+
+	/* HW state is read out, now we need to sanitize this mess. */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+			    base.head) {
+		intel_sanitize_encoder(encoder);
+	}
+
+	for_each_pipe(pipe) {
+		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
+		intel_sanitize_crtc(crtc);
+	}
+
+	intel_modeset_update_staged_output_state(dev);
+
+	intel_modeset_check_state(dev);
 }
 
 void intel_modeset_gem_init(struct drm_device *dev)
@@ -7242,6 +8263,8 @@ void intel_modeset_gem_init(struct drm_device *dev)
 	intel_modeset_init_hw(dev);
 
 	intel_setup_overlay(dev);
+
+	intel_modeset_setup_hw_state(dev);
 }
 
 void intel_modeset_cleanup(struct drm_device *dev)
@@ -7280,19 +8303,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
 	 * enqueue unpin/hotplug work. */
 	drm_irq_uninstall(dev);
 	cancel_work_sync(&dev_priv->hotplug_work);
-	cancel_work_sync(&dev_priv->rps_work);
+	cancel_work_sync(&dev_priv->rps.work);
 
 	/* flush any delayed tasks or pending work */
 	flush_scheduled_work();
 
-	/* Shut off idle work before the crtcs get freed. */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		intel_crtc = to_intel_crtc(crtc);
-		del_timer_sync(&intel_crtc->idle_timer);
-	}
-	del_timer_sync(&dev_priv->idle_timer);
-	cancel_work_sync(&dev_priv->idle_work);
-
 	drm_mode_config_cleanup(dev);
 }
 
@@ -7338,7 +8353,7 @@ struct intel_display_error_state {
 		u32 position;
 		u32 base;
 		u32 size;
-	} cursor[2];
+	} cursor[I915_MAX_PIPES];
 
 	struct intel_pipe_error_state {
 		u32 conf;
@@ -7350,7 +8365,7 @@ struct intel_display_error_state {
 		u32 vtotal;
 		u32 vblank;
 		u32 vsync;
-	} pipe[2];
+	} pipe[I915_MAX_PIPES];
 
 	struct intel_plane_error_state {
 		u32 control;
@@ -7360,7 +8375,7 @@ struct intel_display_error_state {
 		u32 addr;
 		u32 surface;
 		u32 tile_offset;
-	} plane[2];
+	} plane[I915_MAX_PIPES];
 };
 
 struct intel_display_error_state *
@@ -7374,7 +8389,7 @@ intel_display_capture_error_state(struct drm_device *dev)
 	if (error == NULL)
 		return NULL;
 
-	for (i = 0; i < 2; i++) {
+	for_each_pipe(i) {
 		error->cursor[i].control = I915_READ(CURCNTR(i));
 		error->cursor[i].position = I915_READ(CURPOS(i));
 		error->cursor[i].base = I915_READ(CURBASE(i));
@@ -7407,9 +8422,11 @@ intel_display_print_error_state(struct seq_file *m,
 				struct drm_device *dev,
 				struct intel_display_error_state *error)
 {
+	drm_i915_private_t *dev_priv = dev->dev_private;
 	int i;
 
-	for (i = 0; i < 2; i++) {
+	seq_printf(m, "Num Pipes: %d\n", dev_priv->num_pipe);
+	for_each_pipe(i) {
 		seq_printf(m, "Pipe [%d]:\n", i);
 		seq_printf(m, "  CONF: %08x\n", error->pipe[i].conf);
 		seq_printf(m, "  SRC: %08x\n", error->pipe[i].source);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index ace757af913..6c8746c030c 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -28,50 +28,17 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
-#include "drm_dp_helper.h"
 
-#define DP_RECEIVER_CAP_SIZE	0xf
 #define DP_LINK_STATUS_SIZE	6
 #define DP_LINK_CHECK_TIMEOUT	(10 * 1000)
 
-#define DP_LINK_CONFIGURATION_SIZE	9
-
-struct intel_dp {
-	struct intel_encoder base;
-	uint32_t output_reg;
-	uint32_t DP;
-	uint8_t  link_configuration[DP_LINK_CONFIGURATION_SIZE];
-	bool has_audio;
-	enum hdmi_force_audio force_audio;
-	uint32_t color_range;
-	int dpms_mode;
-	uint8_t link_bw;
-	uint8_t lane_count;
-	uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
-	struct i2c_adapter adapter;
-	struct i2c_algo_dp_aux_data algo;
-	bool is_pch_edp;
-	uint8_t	train_set[4];
-	int panel_power_up_delay;
-	int panel_power_down_delay;
-	int panel_power_cycle_delay;
-	int backlight_on_delay;
-	int backlight_off_delay;
-	struct drm_display_mode *panel_fixed_mode;  /* for eDP */
-	struct delayed_work panel_vdd_work;
-	bool want_panel_vdd;
-	struct edid *edid; /* cached EDID for eDP */
-	int edid_mode_count;
-};
-
 /**
  * is_edp - is the given port attached to an eDP panel (either CPU or PCH)
  * @intel_dp: DP struct
@@ -840,9 +807,6 @@ intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
 	}
 }
 
-static void ironlake_edp_pll_on(struct drm_encoder *encoder);
-static void ironlake_edp_pll_off(struct drm_encoder *encoder);
-
 static void
 intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		  struct drm_display_mode *adjusted_mode)
@@ -853,14 +817,6 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 	struct drm_crtc *crtc = intel_dp->base.base.crtc;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-	/* Turn on the eDP PLL if needed */
-	if (is_edp(intel_dp)) {
-		if (!is_pch_edp(intel_dp))
-			ironlake_edp_pll_on(encoder);
-		else
-			ironlake_edp_pll_off(encoder);
-	}
-
 	/*
 	 * There are four kinds of DP registers:
 	 *
@@ -882,10 +838,8 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 	 * supposed to be read-only.
 	 */
 	intel_dp->DP = I915_READ(intel_dp->output_reg) & DP_DETECTED;
-	intel_dp->DP |=  DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
 
 	/* Handle DP bits in common between all three register formats */
-
 	intel_dp->DP |= DP_VOLTAGE_0_4 | DP_PRE_EMPHASIS_0;
 
 	switch (intel_dp->lane_count) {
@@ -932,7 +886,6 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		intel_dp->DP |= intel_crtc->pipe << 29;
 
 		/* don't miss out required setting for eDP */
-		intel_dp->DP |= DP_PLL_ENABLE;
 		if (adjusted_mode->clock < 200000)
 			intel_dp->DP |= DP_PLL_FREQ_160MHZ;
 		else
@@ -954,7 +907,6 @@ intel_dp_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
 		if (is_cpu_edp(intel_dp)) {
 			/* don't miss out required setting for eDP */
-			intel_dp->DP |= DP_PLL_ENABLE;
 			if (adjusted_mode->clock < 200000)
 				intel_dp->DP |= DP_PLL_FREQ_160MHZ;
 			else
@@ -1225,27 +1177,49 @@ static void ironlake_edp_backlight_off(struct intel_dp *intel_dp)
 	msleep(intel_dp->backlight_off_delay);
 }
 
-static void ironlake_edp_pll_on(struct drm_encoder *encoder)
+static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = intel_dp->base.base.dev;
+	struct drm_crtc *crtc = intel_dp->base.base.crtc;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 dpa_ctl;
 
+	assert_pipe_disabled(dev_priv,
+			     to_intel_crtc(crtc)->pipe);
+
 	DRM_DEBUG_KMS("\n");
 	dpa_ctl = I915_READ(DP_A);
-	dpa_ctl |= DP_PLL_ENABLE;
-	I915_WRITE(DP_A, dpa_ctl);
+	WARN(dpa_ctl & DP_PLL_ENABLE, "dp pll on, should be off\n");
+	WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n");
+
+	/* We don't adjust intel_dp->DP while tearing down the link, to
+	 * facilitate link retraining (e.g. after hotplug). Hence clear all
+	 * enable bits here to ensure that we don't enable too much. */
+	intel_dp->DP &= ~(DP_PORT_EN | DP_AUDIO_OUTPUT_ENABLE);
+	intel_dp->DP |= DP_PLL_ENABLE;
+	I915_WRITE(DP_A, intel_dp->DP);
 	POSTING_READ(DP_A);
 	udelay(200);
 }
 
-static void ironlake_edp_pll_off(struct drm_encoder *encoder)
+static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = intel_dp->base.base.dev;
+	struct drm_crtc *crtc = intel_dp->base.base.crtc;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 dpa_ctl;
 
+	assert_pipe_disabled(dev_priv,
+			     to_intel_crtc(crtc)->pipe);
+
 	dpa_ctl = I915_READ(DP_A);
+	WARN((dpa_ctl & DP_PLL_ENABLE) == 0,
+	     "dp pll off, should be on\n");
+	WARN(dpa_ctl & DP_PORT_EN, "dp port still on, should be off\n");
+
+	/* We can't rely on the value tracked for the DP register in
+	 * intel_dp->DP because link_down must not change that (otherwise link
+	 * re-training will fail. */
 	dpa_ctl &= ~DP_PLL_ENABLE;
 	I915_WRITE(DP_A, dpa_ctl);
 	POSTING_READ(DP_A);
@@ -1282,10 +1256,57 @@ static void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 	}
 }
 
-static void intel_dp_prepare(struct drm_encoder *encoder)
+static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
+				  enum pipe *pipe)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 tmp = I915_READ(intel_dp->output_reg);
 
+	if (!(tmp & DP_PORT_EN))
+		return false;
+
+	if (is_cpu_edp(intel_dp) && IS_GEN7(dev)) {
+		*pipe = PORT_TO_PIPE_CPT(tmp);
+	} else if (!HAS_PCH_CPT(dev) || is_cpu_edp(intel_dp)) {
+		*pipe = PORT_TO_PIPE(tmp);
+	} else {
+		u32 trans_sel;
+		u32 trans_dp;
+		int i;
+
+		switch (intel_dp->output_reg) {
+		case PCH_DP_B:
+			trans_sel = TRANS_DP_PORT_SEL_B;
+			break;
+		case PCH_DP_C:
+			trans_sel = TRANS_DP_PORT_SEL_C;
+			break;
+		case PCH_DP_D:
+			trans_sel = TRANS_DP_PORT_SEL_D;
+			break;
+		default:
+			return true;
+		}
+
+		for_each_pipe(i) {
+			trans_dp = I915_READ(TRANS_DP_CTL(i));
+			if ((trans_dp & TRANS_DP_PORT_SEL_MASK) == trans_sel) {
+				*pipe = i;
+				return true;
+			}
+		}
+	}
+
+	DRM_DEBUG_KMS("No pipe for dp port 0x%x found\n", intel_dp->output_reg);
+
+	return true;
+}
+
+static void intel_disable_dp(struct intel_encoder *encoder)
+{
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 	/* Make sure the panel is off before trying to change the mode. But also
 	 * ensure that we have vdd while we switch off the panel. */
@@ -1293,14 +1314,31 @@ static void intel_dp_prepare(struct drm_encoder *encoder)
 	ironlake_edp_backlight_off(intel_dp);
 	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
 	ironlake_edp_panel_off(intel_dp);
-	intel_dp_link_down(intel_dp);
+
+	/* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */
+	if (!is_cpu_edp(intel_dp))
+		intel_dp_link_down(intel_dp);
 }
 
-static void intel_dp_commit(struct drm_encoder *encoder)
+static void intel_post_disable_dp(struct intel_encoder *encoder)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	struct drm_device *dev = encoder->dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_dp->base.base.crtc);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+
+	if (is_cpu_edp(intel_dp)) {
+		intel_dp_link_down(intel_dp);
+		ironlake_edp_pll_off(intel_dp);
+	}
+}
+
+static void intel_enable_dp(struct intel_encoder *encoder)
+{
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dp_reg = I915_READ(intel_dp->output_reg);
+
+	if (WARN_ON(dp_reg & DP_PORT_EN))
+		return;
 
 	ironlake_edp_panel_vdd_on(intel_dp);
 	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
@@ -1309,47 +1347,14 @@ static void intel_dp_commit(struct drm_encoder *encoder)
 	ironlake_edp_panel_vdd_off(intel_dp, true);
 	intel_dp_complete_link_train(intel_dp);
 	ironlake_edp_backlight_on(intel_dp);
-
-	intel_dp->dpms_mode = DRM_MODE_DPMS_ON;
-
-	if (HAS_PCH_CPT(dev))
-		intel_cpt_verify_modeset(dev, intel_crtc->pipe);
 }
 
-static void
-intel_dp_dpms(struct drm_encoder *encoder, int mode)
+static void intel_pre_enable_dp(struct intel_encoder *encoder)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	struct drm_device *dev = encoder->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dp_reg = I915_READ(intel_dp->output_reg);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-	if (mode != DRM_MODE_DPMS_ON) {
-		/* Switching the panel off requires vdd. */
-		ironlake_edp_panel_vdd_on(intel_dp);
-		ironlake_edp_backlight_off(intel_dp);
-		intel_dp_sink_dpms(intel_dp, mode);
-		ironlake_edp_panel_off(intel_dp);
-		intel_dp_link_down(intel_dp);
-
-		if (is_cpu_edp(intel_dp))
-			ironlake_edp_pll_off(encoder);
-	} else {
-		if (is_cpu_edp(intel_dp))
-			ironlake_edp_pll_on(encoder);
-
-		ironlake_edp_panel_vdd_on(intel_dp);
-		intel_dp_sink_dpms(intel_dp, mode);
-		if (!(dp_reg & DP_PORT_EN)) {
-			intel_dp_start_link_train(intel_dp);
-			ironlake_edp_panel_on(intel_dp);
-			ironlake_edp_panel_vdd_off(intel_dp, true);
-			intel_dp_complete_link_train(intel_dp);
-		} else
-			ironlake_edp_panel_vdd_off(intel_dp, false);
-		ironlake_edp_backlight_on(intel_dp);
-	}
-	intel_dp->dpms_mode = mode;
+	if (is_cpu_edp(intel_dp))
+		ironlake_edp_pll_on(intel_dp);
 }
 
 /*
@@ -1668,6 +1673,45 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
 
+	if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) {
+		dp_reg_value &= ~DP_LINK_TRAIN_MASK_CPT;
+
+		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
+		case DP_TRAINING_PATTERN_DISABLE:
+			dp_reg_value |= DP_LINK_TRAIN_OFF_CPT;
+			break;
+		case DP_TRAINING_PATTERN_1:
+			dp_reg_value |= DP_LINK_TRAIN_PAT_1_CPT;
+			break;
+		case DP_TRAINING_PATTERN_2:
+			dp_reg_value |= DP_LINK_TRAIN_PAT_2_CPT;
+			break;
+		case DP_TRAINING_PATTERN_3:
+			DRM_ERROR("DP training pattern 3 not supported\n");
+			dp_reg_value |= DP_LINK_TRAIN_PAT_2_CPT;
+			break;
+		}
+
+	} else {
+		dp_reg_value &= ~DP_LINK_TRAIN_MASK;
+
+		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
+		case DP_TRAINING_PATTERN_DISABLE:
+			dp_reg_value |= DP_LINK_TRAIN_OFF;
+			break;
+		case DP_TRAINING_PATTERN_1:
+			dp_reg_value |= DP_LINK_TRAIN_PAT_1;
+			break;
+		case DP_TRAINING_PATTERN_2:
+			dp_reg_value |= DP_LINK_TRAIN_PAT_2;
+			break;
+		case DP_TRAINING_PATTERN_3:
+			DRM_ERROR("DP training pattern 3 not supported\n");
+			dp_reg_value |= DP_LINK_TRAIN_PAT_2;
+			break;
+		}
+	}
+
 	I915_WRITE(intel_dp->output_reg, dp_reg_value);
 	POSTING_READ(intel_dp->output_reg);
 
@@ -1675,12 +1719,15 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
 				    DP_TRAINING_PATTERN_SET,
 				    dp_train_pat);
 
-	ret = intel_dp_aux_native_write(intel_dp,
-					DP_TRAINING_LANE0_SET,
-					intel_dp->train_set,
-					intel_dp->lane_count);
-	if (ret != intel_dp->lane_count)
-		return false;
+	if ((dp_train_pat & DP_TRAINING_PATTERN_MASK) !=
+	    DP_TRAINING_PATTERN_DISABLE) {
+		ret = intel_dp_aux_native_write(intel_dp,
+						DP_TRAINING_LANE0_SET,
+						intel_dp->train_set,
+						intel_dp->lane_count);
+		if (ret != intel_dp->lane_count)
+			return false;
+	}
 
 	return true;
 }
@@ -1690,26 +1737,12 @@ static void
 intel_dp_start_link_train(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp->base.base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_dp->base.base.crtc);
 	int i;
 	uint8_t voltage;
 	bool clock_recovery = false;
 	int voltage_tries, loop_tries;
-	u32 reg;
 	uint32_t DP = intel_dp->DP;
 
-	/*
-	 * On CPT we have to enable the port in training pattern 1, which
-	 * will happen below in intel_dp_set_link_train.  Otherwise, enable
-	 * the port and wait for it to become active.
-	 */
-	if (!HAS_PCH_CPT(dev)) {
-		I915_WRITE(intel_dp->output_reg, intel_dp->DP);
-		POSTING_READ(intel_dp->output_reg);
-		intel_wait_for_vblank(dev, intel_crtc->pipe);
-	}
-
 	/* Write the link configuration data */
 	intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET,
 				  intel_dp->link_configuration,
@@ -1717,10 +1750,6 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
 
 	DP |= DP_PORT_EN;
 
-	if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp)))
-		DP &= ~DP_LINK_TRAIN_MASK_CPT;
-	else
-		DP &= ~DP_LINK_TRAIN_MASK;
 	memset(intel_dp->train_set, 0, 4);
 	voltage = 0xff;
 	voltage_tries = 0;
@@ -1744,12 +1773,7 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
 			DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
 		}
 
-		if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp)))
-			reg = DP | DP_LINK_TRAIN_PAT_1_CPT;
-		else
-			reg = DP | DP_LINK_TRAIN_PAT_1;
-
-		if (!intel_dp_set_link_train(intel_dp, reg,
+		if (!intel_dp_set_link_train(intel_dp, DP,
 					     DP_TRAINING_PATTERN_1 |
 					     DP_LINK_SCRAMBLING_DISABLE))
 			break;
@@ -1804,10 +1828,8 @@ static void
 intel_dp_complete_link_train(struct intel_dp *intel_dp)
 {
 	struct drm_device *dev = intel_dp->base.base.dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	bool channel_eq = false;
 	int tries, cr_tries;
-	u32 reg;
 	uint32_t DP = intel_dp->DP;
 
 	/* channel equalization */
@@ -1836,13 +1858,8 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
 			DP = (DP & ~(DP_VOLTAGE_MASK|DP_PRE_EMPHASIS_MASK)) | signal_levels;
 		}
 
-		if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp)))
-			reg = DP | DP_LINK_TRAIN_PAT_2_CPT;
-		else
-			reg = DP | DP_LINK_TRAIN_PAT_2;
-
 		/* channel eq pattern */
-		if (!intel_dp_set_link_train(intel_dp, reg,
+		if (!intel_dp_set_link_train(intel_dp, DP,
 					     DP_TRAINING_PATTERN_2 |
 					     DP_LINK_SCRAMBLING_DISABLE))
 			break;
@@ -1877,15 +1894,7 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
 		++tries;
 	}
 
-	if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp)))
-		reg = DP | DP_LINK_TRAIN_OFF_CPT;
-	else
-		reg = DP | DP_LINK_TRAIN_OFF;
-
-	I915_WRITE(intel_dp->output_reg, reg);
-	POSTING_READ(intel_dp->output_reg);
-	intel_dp_aux_native_write_1(intel_dp,
-				    DP_TRAINING_PATTERN_SET, DP_TRAINING_PATTERN_DISABLE);
+	intel_dp_set_link_train(intel_dp, DP, DP_TRAINING_PATTERN_DISABLE);
 }
 
 static void
@@ -1895,18 +1904,11 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	uint32_t DP = intel_dp->DP;
 
-	if ((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0)
+	if (WARN_ON((I915_READ(intel_dp->output_reg) & DP_PORT_EN) == 0))
 		return;
 
 	DRM_DEBUG_KMS("\n");
 
-	if (is_edp(intel_dp)) {
-		DP &= ~DP_PLL_ENABLE;
-		I915_WRITE(intel_dp->output_reg, DP);
-		POSTING_READ(intel_dp->output_reg);
-		udelay(100);
-	}
-
 	if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp))) {
 		DP &= ~DP_LINK_TRAIN_MASK_CPT;
 		I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT);
@@ -1918,13 +1920,6 @@ intel_dp_link_down(struct intel_dp *intel_dp)
 
 	msleep(17);
 
-	if (is_edp(intel_dp)) {
-		if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || !is_cpu_edp(intel_dp)))
-			DP |= DP_LINK_TRAIN_OFF_CPT;
-		else
-			DP |= DP_LINK_TRAIN_OFF;
-	}
-
 	if (HAS_PCH_IBX(dev) &&
 	    I915_READ(intel_dp->output_reg) & DP_PIPEB_SELECT) {
 		struct drm_crtc *crtc = intel_dp->base.base.crtc;
@@ -2033,10 +2028,10 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 	u8 sink_irq_vector;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 
-	if (intel_dp->dpms_mode != DRM_MODE_DPMS_ON)
+	if (!intel_dp->base.connectors_active)
 		return;
 
-	if (!intel_dp->base.base.crtc)
+	if (WARN_ON(!intel_dp->base.base.crtc))
 		return;
 
 	/* Try to read receiver status if the link appears to be up */
@@ -2160,7 +2155,6 @@ intel_dp_get_edid_modes(struct drm_connector *connector, struct i2c_adapter *ada
 		ret = drm_add_edid_modes(connector, intel_dp->edid);
 		drm_edid_to_eld(connector,
 				intel_dp->edid);
-		connector->display_info.raw_edid = NULL;
 		return intel_dp->edid_mode_count;
 	}
 
@@ -2206,7 +2200,6 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 		edid = intel_dp_get_edid(connector, &intel_dp->adapter);
 		if (edid) {
 			intel_dp->has_audio = drm_detect_monitor_audio(edid);
-			connector->display_info.raw_edid = NULL;
 			kfree(edid);
 		}
 	}
@@ -2271,8 +2264,6 @@ intel_dp_detect_audio(struct drm_connector *connector)
 	edid = intel_dp_get_edid(connector, &intel_dp->adapter);
 	if (edid) {
 		has_audio = drm_detect_monitor_audio(edid);
-
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 
@@ -2326,9 +2317,8 @@ intel_dp_set_property(struct drm_connector *connector,
 done:
 	if (intel_dp->base.base.crtc) {
 		struct drm_crtc *crtc = intel_dp->base.base.crtc;
-		drm_crtc_helper_set_mode(crtc, &crtc->mode,
-					 crtc->x, crtc->y,
-					 crtc->fb);
+		intel_set_mode(crtc, &crtc->mode,
+			       crtc->x, crtc->y, crtc->fb);
 	}
 
 	return 0;
@@ -2362,15 +2352,13 @@ static void intel_dp_encoder_destroy(struct drm_encoder *encoder)
 }
 
 static const struct drm_encoder_helper_funcs intel_dp_helper_funcs = {
-	.dpms = intel_dp_dpms,
 	.mode_fixup = intel_dp_mode_fixup,
-	.prepare = intel_dp_prepare,
 	.mode_set = intel_dp_mode_set,
-	.commit = intel_dp_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_dp_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_connector_dpms,
 	.detect = intel_dp_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = intel_dp_set_property,
@@ -2441,7 +2429,7 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect
 }
 
 void
-intel_dp_init(struct drm_device *dev, int output_reg)
+intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_connector *connector;
@@ -2456,7 +2444,9 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 		return;
 
 	intel_dp->output_reg = output_reg;
-	intel_dp->dpms_mode = -1;
+	intel_dp->port = port;
+	/* Preserve the current hw state. */
+	intel_dp->DP = I915_READ(intel_dp->output_reg);
 
 	intel_connector = kzalloc(sizeof(struct intel_connector), GFP_KERNEL);
 	if (!intel_connector) {
@@ -2483,18 +2473,10 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 
 	connector->polled = DRM_CONNECTOR_POLL_HPD;
 
-	if (output_reg == DP_B || output_reg == PCH_DP_B)
-		intel_encoder->clone_mask = (1 << INTEL_DP_B_CLONE_BIT);
-	else if (output_reg == DP_C || output_reg == PCH_DP_C)
-		intel_encoder->clone_mask = (1 << INTEL_DP_C_CLONE_BIT);
-	else if (output_reg == DP_D || output_reg == PCH_DP_D)
-		intel_encoder->clone_mask = (1 << INTEL_DP_D_CLONE_BIT);
+	intel_encoder->cloneable = false;
 
-	if (is_edp(intel_dp)) {
-		intel_encoder->clone_mask = (1 << INTEL_EDP_CLONE_BIT);
-		INIT_DELAYED_WORK(&intel_dp->panel_vdd_work,
-				  ironlake_panel_vdd_work);
-	}
+	INIT_DELAYED_WORK(&intel_dp->panel_vdd_work,
+			  ironlake_panel_vdd_work);
 
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 
@@ -2508,29 +2490,33 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
 	drm_sysfs_connector_add(connector);
 
+	intel_encoder->enable = intel_enable_dp;
+	intel_encoder->pre_enable = intel_pre_enable_dp;
+	intel_encoder->disable = intel_disable_dp;
+	intel_encoder->post_disable = intel_post_disable_dp;
+	intel_encoder->get_hw_state = intel_dp_get_hw_state;
+	intel_connector->get_hw_state = intel_connector_get_hw_state;
+
 	/* Set up the DDC bus. */
-	switch (output_reg) {
-		case DP_A:
-			name = "DPDDC-A";
-			break;
-		case DP_B:
-		case PCH_DP_B:
-			dev_priv->hotplug_supported_mask |=
-				DPB_HOTPLUG_INT_STATUS;
-			name = "DPDDC-B";
-			break;
-		case DP_C:
-		case PCH_DP_C:
-			dev_priv->hotplug_supported_mask |=
-				DPC_HOTPLUG_INT_STATUS;
-			name = "DPDDC-C";
-			break;
-		case DP_D:
-		case PCH_DP_D:
-			dev_priv->hotplug_supported_mask |=
-				DPD_HOTPLUG_INT_STATUS;
-			name = "DPDDC-D";
-			break;
+	switch (port) {
+	case PORT_A:
+		name = "DPDDC-A";
+		break;
+	case PORT_B:
+		dev_priv->hotplug_supported_mask |= DPB_HOTPLUG_INT_STATUS;
+		name = "DPDDC-B";
+		break;
+	case PORT_C:
+		dev_priv->hotplug_supported_mask |= DPC_HOTPLUG_INT_STATUS;
+		name = "DPDDC-C";
+		break;
+	case PORT_D:
+		dev_priv->hotplug_supported_mask |= DPD_HOTPLUG_INT_STATUS;
+		name = "DPDDC-D";
+		break;
+	default:
+		WARN(1, "Invalid port %c\n", port_name(port));
+		break;
 	}
 
 	/* Cache some DPCD data in the eDP case */
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index cd54cf88a28..05cc7c372fc 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -26,11 +26,12 @@
 #define __INTEL_DRV_H__
 
 #include <linux/i2c.h>
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_dp_helper.h>
 
 #define _wait_for(COND, MS, W) ({ \
 	unsigned long timeout__ = jiffies + msecs_to_jiffies(MS);	\
@@ -40,7 +41,11 @@
 			ret__ = -ETIMEDOUT;				\
 			break;						\
 		}							\
-		if (W && drm_can_sleep()) msleep(W);	\
+		if (W && drm_can_sleep())  {				\
+			msleep(W);					\
+		} else {						\
+			cpu_relax();					\
+		}							\
 	}								\
 	ret__;								\
 })
@@ -90,25 +95,6 @@
 #define INTEL_OUTPUT_DISPLAYPORT 7
 #define INTEL_OUTPUT_EDP 8
 
-/* Intel Pipe Clone Bit */
-#define INTEL_HDMIB_CLONE_BIT 1
-#define INTEL_HDMIC_CLONE_BIT 2
-#define INTEL_HDMID_CLONE_BIT 3
-#define INTEL_HDMIE_CLONE_BIT 4
-#define INTEL_HDMIF_CLONE_BIT 5
-#define INTEL_SDVO_NON_TV_CLONE_BIT 6
-#define INTEL_SDVO_TV_CLONE_BIT 7
-#define INTEL_SDVO_LVDS_CLONE_BIT 8
-#define INTEL_ANALOG_CLONE_BIT 9
-#define INTEL_TV_CLONE_BIT 10
-#define INTEL_DP_B_CLONE_BIT 11
-#define INTEL_DP_C_CLONE_BIT 12
-#define INTEL_DP_D_CLONE_BIT 13
-#define INTEL_LVDS_CLONE_BIT 14
-#define INTEL_DVO_TMDS_CLONE_BIT 15
-#define INTEL_DVO_LVDS_CLONE_BIT 16
-#define INTEL_EDP_CLONE_BIT 17
-
 #define INTEL_DVO_CHIP_NONE 0
 #define INTEL_DVO_CHIP_LVDS 1
 #define INTEL_DVO_CHIP_TMDS 2
@@ -151,16 +137,48 @@ struct intel_fbdev {
 
 struct intel_encoder {
 	struct drm_encoder base;
+	/*
+	 * The new crtc this encoder will be driven from. Only differs from
+	 * base->crtc while a modeset is in progress.
+	 */
+	struct intel_crtc *new_crtc;
+
 	int type;
 	bool needs_tv_clock;
+	/*
+	 * Intel hw has only one MUX where encoders could be clone, hence a
+	 * simple flag is enough to compute the possible_clones mask.
+	 */
+	bool cloneable;
+	bool connectors_active;
 	void (*hot_plug)(struct intel_encoder *);
+	void (*pre_enable)(struct intel_encoder *);
+	void (*enable)(struct intel_encoder *);
+	void (*disable)(struct intel_encoder *);
+	void (*post_disable)(struct intel_encoder *);
+	/* Read out the current hw state of this connector, returning true if
+	 * the encoder is active. If the encoder is enabled it also set the pipe
+	 * it is connected to in the pipe parameter. */
+	bool (*get_hw_state)(struct intel_encoder *, enum pipe *pipe);
 	int crtc_mask;
-	int clone_mask;
 };
 
 struct intel_connector {
 	struct drm_connector base;
+	/*
+	 * The fixed encoder this connector is connected to.
+	 */
 	struct intel_encoder *encoder;
+
+	/*
+	 * The new encoder this connector will be driven. Only differs from
+	 * encoder while a modeset is in progress.
+	 */
+	struct intel_encoder *new_encoder;
+
+	/* Reads out the current hw, returning true if the connector is enabled
+	 * and active (i.e. dpms ON state). */
+	bool (*get_hw_state)(struct intel_connector *);
 };
 
 struct intel_crtc {
@@ -168,11 +186,13 @@ struct intel_crtc {
 	enum pipe pipe;
 	enum plane plane;
 	u8 lut_r[256], lut_g[256], lut_b[256];
-	int dpms_mode;
-	bool active; /* is the crtc on? independent of the dpms mode */
+	/*
+	 * Whether the crtc and the connected output pipeline is active. Implies
+	 * that crtc->enabled is set, i.e. the current mode configuration has
+	 * some outputs connected to this crtc.
+	 */
+	bool active;
 	bool primary_disabled; /* is the crtc obscured by a plane? */
-	bool busy; /* is scanout buffer being updated frequently? */
-	struct timer_list idle_timer;
 	bool lowfreq_avail;
 	struct intel_overlay *overlay;
 	struct intel_unpin_work *unpin_work;
@@ -311,6 +331,37 @@ struct intel_hdmi {
 			       struct drm_display_mode *adjusted_mode);
 };
 
+#define DP_RECEIVER_CAP_SIZE		0xf
+#define DP_LINK_CONFIGURATION_SIZE	9
+
+struct intel_dp {
+	struct intel_encoder base;
+	uint32_t output_reg;
+	uint32_t DP;
+	uint8_t  link_configuration[DP_LINK_CONFIGURATION_SIZE];
+	bool has_audio;
+	enum hdmi_force_audio force_audio;
+	enum port port;
+	uint32_t color_range;
+	uint8_t link_bw;
+	uint8_t lane_count;
+	uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
+	struct i2c_adapter adapter;
+	struct i2c_algo_dp_aux_data algo;
+	bool is_pch_edp;
+	uint8_t train_set[4];
+	int panel_power_up_delay;
+	int panel_power_down_delay;
+	int panel_power_cycle_delay;
+	int backlight_on_delay;
+	int backlight_off_delay;
+	struct drm_display_mode *panel_fixed_mode;  /* for eDP */
+	struct delayed_work panel_vdd_work;
+	bool want_panel_vdd;
+	struct edid *edid; /* cached EDID for eDP */
+	int edid_mode_count;
+};
+
 static inline struct drm_crtc *
 intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
 {
@@ -350,17 +401,21 @@ extern void intel_attach_force_audio_property(struct drm_connector *connector);
 extern void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
 
 extern void intel_crt_init(struct drm_device *dev);
-extern void intel_hdmi_init(struct drm_device *dev, int sdvox_reg);
+extern void intel_hdmi_init(struct drm_device *dev,
+			    int sdvox_reg, enum port port);
 extern struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
 extern void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
 extern bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg,
 			    bool is_sdvob);
 extern void intel_dvo_init(struct drm_device *dev);
 extern void intel_tv_init(struct drm_device *dev);
-extern void intel_mark_busy(struct drm_device *dev,
-			    struct drm_i915_gem_object *obj);
+extern void intel_mark_busy(struct drm_device *dev);
+extern void intel_mark_idle(struct drm_device *dev);
+extern void intel_mark_fb_busy(struct drm_i915_gem_object *obj);
+extern void intel_mark_fb_idle(struct drm_i915_gem_object *obj);
 extern bool intel_lvds_init(struct drm_device *dev);
-extern void intel_dp_init(struct drm_device *dev, int dp_reg);
+extern void intel_dp_init(struct drm_device *dev, int output_reg,
+			  enum port port);
 void
 intel_dp_set_m_n(struct drm_crtc *crtc, struct drm_display_mode *mode,
 		 struct drm_display_mode *adjusted_mode);
@@ -373,8 +428,6 @@ extern int intel_plane_init(struct drm_device *dev, enum pipe pipe);
 extern void intel_flush_display_plane(struct drm_i915_private *dev_priv,
 				      enum plane plane);
 
-void intel_sanitize_pm(struct drm_device *dev);
-
 /* intel_panel.c */
 extern void intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
 				   struct drm_display_mode *adjusted_mode);
@@ -391,10 +444,27 @@ extern void intel_panel_disable_backlight(struct drm_device *dev);
 extern void intel_panel_destroy_backlight(struct drm_device *dev);
 extern enum drm_connector_status intel_panel_detect(struct drm_device *dev);
 
+struct intel_set_config {
+	struct drm_encoder **save_connector_encoders;
+	struct drm_crtc **save_encoder_crtcs;
+
+	bool fb_changed;
+	bool mode_changed;
+};
+
+extern bool intel_set_mode(struct drm_crtc *crtc, struct drm_display_mode *mode,
+			   int x, int y, struct drm_framebuffer *old_fb);
+extern void intel_modeset_disable(struct drm_device *dev);
 extern void intel_crtc_load_lut(struct drm_crtc *crtc);
-extern void intel_encoder_prepare(struct drm_encoder *encoder);
-extern void intel_encoder_commit(struct drm_encoder *encoder);
+extern void intel_crtc_update_dpms(struct drm_crtc *crtc);
+extern void intel_encoder_noop(struct drm_encoder *encoder);
 extern void intel_encoder_destroy(struct drm_encoder *encoder);
+extern void intel_encoder_dpms(struct intel_encoder *encoder, int mode);
+extern bool intel_encoder_check_is_cloned(struct intel_encoder *encoder);
+extern void intel_connector_dpms(struct drm_connector *, int mode);
+extern bool intel_connector_get_hw_state(struct intel_connector *connector);
+extern void intel_modeset_check_state(struct drm_device *dev);
+
 
 static inline struct intel_encoder *intel_attached_encoder(struct drm_connector *connector)
 {
@@ -417,12 +487,10 @@ struct intel_load_detect_pipe {
 	bool load_detect_temp;
 	int dpms_mode;
 };
-extern bool intel_get_load_detect_pipe(struct intel_encoder *intel_encoder,
-				       struct drm_connector *connector,
+extern bool intel_get_load_detect_pipe(struct drm_connector *connector,
 				       struct drm_display_mode *mode,
 				       struct intel_load_detect_pipe *old);
-extern void intel_release_load_detect_pipe(struct intel_encoder *intel_encoder,
-					   struct drm_connector *connector,
+extern void intel_release_load_detect_pipe(struct drm_connector *connector,
 					   struct intel_load_detect_pipe *old);
 
 extern void intelfb_restore(void);
@@ -503,7 +571,10 @@ extern void intel_disable_gt_powersave(struct drm_device *dev);
 extern void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv);
 extern void ironlake_teardown_rc6(struct drm_device *dev);
 
-extern void intel_ddi_dpms(struct drm_encoder *encoder, int mode);
+extern void intel_enable_ddi(struct intel_encoder *encoder);
+extern void intel_disable_ddi(struct intel_encoder *encoder);
+extern bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
+				   enum pipe *pipe);
 extern void intel_ddi_mode_set(struct drm_encoder *encoder,
 				struct drm_display_mode *mode,
 				struct drm_display_mode *adjusted_mode);
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 36c542e5036..15da99533e5 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -26,17 +26,17 @@
  */
 #include <linux/i2c.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "dvo.h"
 
 #define SIL164_ADDR	0x38
 #define CH7xxx_ADDR	0x76
 #define TFP410_ADDR	0x38
+#define NS2501_ADDR     0x38
 
 static const struct intel_dvo_device intel_dvo_devices[] = {
 	{
@@ -74,7 +74,14 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
 		.slave_addr = 0x75,
 		.gpio = GMBUS_PORT_DPB,
 		.dev_ops = &ch7017_ops,
-	}
+	},
+	{
+	        .type = INTEL_DVO_CHIP_TMDS,
+		.name = "ns2501",
+		.dvo_reg = DVOC,
+		.slave_addr = NS2501_ADDR,
+		.dev_ops = &ns2501_ops,
+       }
 };
 
 struct intel_dvo {
@@ -97,22 +104,91 @@ static struct intel_dvo *intel_attached_dvo(struct drm_connector *connector)
 			    struct intel_dvo, base);
 }
 
-static void intel_dvo_dpms(struct drm_encoder *encoder, int mode)
+static bool intel_dvo_connector_get_hw_state(struct intel_connector *connector)
 {
-	struct drm_i915_private *dev_priv = encoder->dev->dev_private;
-	struct intel_dvo *intel_dvo = enc_to_intel_dvo(encoder);
+	struct intel_dvo *intel_dvo = intel_attached_dvo(&connector->base);
+
+	return intel_dvo->dev.dev_ops->get_hw_state(&intel_dvo->dev);
+}
+
+static bool intel_dvo_get_hw_state(struct intel_encoder *encoder,
+				   enum pipe *pipe)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_dvo *intel_dvo = enc_to_intel_dvo(&encoder->base);
+	u32 tmp;
+
+	tmp = I915_READ(intel_dvo->dev.dvo_reg);
+
+	if (!(tmp & DVO_ENABLE))
+		return false;
+
+	*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
+static void intel_disable_dvo(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+	struct intel_dvo *intel_dvo = enc_to_intel_dvo(&encoder->base);
+	u32 dvo_reg = intel_dvo->dev.dvo_reg;
+	u32 temp = I915_READ(dvo_reg);
+
+	intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false);
+	I915_WRITE(dvo_reg, temp & ~DVO_ENABLE);
+	I915_READ(dvo_reg);
+}
+
+static void intel_enable_dvo(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+	struct intel_dvo *intel_dvo = enc_to_intel_dvo(&encoder->base);
 	u32 dvo_reg = intel_dvo->dev.dvo_reg;
 	u32 temp = I915_READ(dvo_reg);
 
+	I915_WRITE(dvo_reg, temp | DVO_ENABLE);
+	I915_READ(dvo_reg);
+	intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
+}
+
+static void intel_dvo_dpms(struct drm_connector *connector, int mode)
+{
+	struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
+	struct drm_crtc *crtc;
+
+	/* dvo supports only 2 dpms states. */
+	if (mode != DRM_MODE_DPMS_ON)
+		mode = DRM_MODE_DPMS_OFF;
+
+	if (mode == connector->dpms)
+		return;
+
+	connector->dpms = mode;
+
+	/* Only need to change hw state when actually enabled */
+	crtc = intel_dvo->base.base.crtc;
+	if (!crtc) {
+		intel_dvo->base.connectors_active = false;
+		return;
+	}
+
 	if (mode == DRM_MODE_DPMS_ON) {
-		I915_WRITE(dvo_reg, temp | DVO_ENABLE);
-		I915_READ(dvo_reg);
-		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, mode);
+		intel_dvo->base.connectors_active = true;
+
+		intel_crtc_update_dpms(crtc);
+
+		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
 	} else {
-		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, mode);
-		I915_WRITE(dvo_reg, temp & ~DVO_ENABLE);
-		I915_READ(dvo_reg);
+		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false);
+
+		intel_dvo->base.connectors_active = false;
+
+		intel_crtc_update_dpms(crtc);
 	}
+
+	intel_modeset_check_state(connector->dev);
 }
 
 static int intel_dvo_mode_valid(struct drm_connector *connector,
@@ -267,15 +343,13 @@ static void intel_dvo_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_encoder_helper_funcs intel_dvo_helper_funcs = {
-	.dpms = intel_dvo_dpms,
 	.mode_fixup = intel_dvo_mode_fixup,
-	.prepare = intel_encoder_prepare,
 	.mode_set = intel_dvo_mode_set,
-	.commit = intel_encoder_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_dvo_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_dvo_dpms,
 	.detect = intel_dvo_detect,
 	.destroy = intel_dvo_destroy,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -364,6 +438,11 @@ void intel_dvo_init(struct drm_device *dev)
 	drm_encoder_init(dev, &intel_encoder->base,
 			 &intel_dvo_enc_funcs, encoder_type);
 
+	intel_encoder->disable = intel_disable_dvo;
+	intel_encoder->enable = intel_enable_dvo;
+	intel_encoder->get_hw_state = intel_dvo_get_hw_state;
+	intel_connector->get_hw_state = intel_dvo_connector_get_hw_state;
+
 	/* Now, try to find a controller */
 	for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) {
 		struct drm_connector *connector = &intel_connector->base;
@@ -396,17 +475,14 @@ void intel_dvo_init(struct drm_device *dev)
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
 		switch (dvo->type) {
 		case INTEL_DVO_CHIP_TMDS:
-			intel_encoder->clone_mask =
-				(1 << INTEL_DVO_TMDS_CLONE_BIT) |
-				(1 << INTEL_ANALOG_CLONE_BIT);
+			intel_encoder->cloneable = true;
 			drm_connector_init(dev, connector,
 					   &intel_dvo_connector_funcs,
 					   DRM_MODE_CONNECTOR_DVII);
 			encoder_type = DRM_MODE_ENCODER_TMDS;
 			break;
 		case INTEL_DVO_CHIP_LVDS:
-			intel_encoder->clone_mask =
-				(1 << INTEL_DVO_LVDS_CLONE_BIT);
+			intel_encoder->cloneable = false;
 			drm_connector_init(dev, connector,
 					   &intel_dvo_connector_funcs,
 					   DRM_MODE_CONNECTOR_LVDS);
diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c
index 97f673523b9..7b30b5c2c4e 100644
--- a/drivers/gpu/drm/i915/intel_fb.c
+++ b/drivers/gpu/drm/i915/intel_fb.c
@@ -36,12 +36,11 @@
 #include <linux/init.h>
 #include <linux/vga_switcheroo.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fb_helper.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 static struct fb_ops intelfb_ops = {
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 12dc3308ab8..9ba0aaed7ee 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -29,12 +29,11 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 static void
@@ -151,6 +150,9 @@ static void g4x_write_infoframe(struct drm_encoder *encoder,
 		I915_WRITE(VIDEO_DIP_DATA, *data);
 		data++;
 	}
+	/* Write every possible data byte to force correct ECC calculation. */
+	for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
+		I915_WRITE(VIDEO_DIP_DATA, 0);
 	mmiowb();
 
 	val |= g4x_infoframe_enable(frame);
@@ -186,6 +188,9 @@ static void ibx_write_infoframe(struct drm_encoder *encoder,
 		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
 		data++;
 	}
+	/* Write every possible data byte to force correct ECC calculation. */
+	for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
+		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), 0);
 	mmiowb();
 
 	val |= g4x_infoframe_enable(frame);
@@ -224,6 +229,9 @@ static void cpt_write_infoframe(struct drm_encoder *encoder,
 		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
 		data++;
 	}
+	/* Write every possible data byte to force correct ECC calculation. */
+	for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
+		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), 0);
 	mmiowb();
 
 	val |= g4x_infoframe_enable(frame);
@@ -259,6 +267,9 @@ static void vlv_write_infoframe(struct drm_encoder *encoder,
 		I915_WRITE(VLV_TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
 		data++;
 	}
+	/* Write every possible data byte to force correct ECC calculation. */
+	for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
+		I915_WRITE(VLV_TVIDEO_DIP_DATA(intel_crtc->pipe), 0);
 	mmiowb();
 
 	val |= g4x_infoframe_enable(frame);
@@ -292,6 +303,9 @@ static void hsw_write_infoframe(struct drm_encoder *encoder,
 		I915_WRITE(data_reg + i, *data);
 		data++;
 	}
+	/* Write every possible data byte to force correct ECC calculation. */
+	for (; i < VIDEO_DIP_DATA_SIZE; i += 4)
+		I915_WRITE(data_reg + i, 0);
 	mmiowb();
 
 	val |= hsw_infoframe_enable(frame);
@@ -377,6 +391,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
 		port = VIDEO_DIP_PORT_C;
 		break;
 	default:
+		BUG();
 		return;
 	}
 
@@ -435,6 +450,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
 		port = VIDEO_DIP_PORT_D;
 		break;
 	default:
+		BUG();
 		return;
 	}
 
@@ -601,15 +617,36 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
 	intel_hdmi->set_infoframes(encoder, adjusted_mode);
 }
 
-static void intel_hdmi_dpms(struct drm_encoder *encoder, int mode)
+static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
+				    enum pipe *pipe)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+	u32 tmp;
+
+	tmp = I915_READ(intel_hdmi->sdvox_reg);
+
+	if (!(tmp & SDVO_ENABLE))
+		return false;
+
+	if (HAS_PCH_CPT(dev))
+		*pipe = PORT_TO_PIPE_CPT(tmp);
+	else
+		*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
+static void intel_enable_hdmi(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
 	u32 temp;
 	u32 enable_bits = SDVO_ENABLE;
 
-	if (intel_hdmi->has_audio || mode != DRM_MODE_DPMS_ON)
+	if (intel_hdmi->has_audio)
 		enable_bits |= SDVO_AUDIO_ENABLE;
 
 	temp = I915_READ(intel_hdmi->sdvox_reg);
@@ -617,31 +654,12 @@ static void intel_hdmi_dpms(struct drm_encoder *encoder, int mode)
 	/* HW workaround for IBX, we need to move the port to transcoder A
 	 * before disabling it. */
 	if (HAS_PCH_IBX(dev)) {
-		struct drm_crtc *crtc = encoder->crtc;
+		struct drm_crtc *crtc = encoder->base.crtc;
 		int pipe = crtc ? to_intel_crtc(crtc)->pipe : -1;
 
-		if (mode != DRM_MODE_DPMS_ON) {
-			if (temp & SDVO_PIPE_B_SELECT) {
-				temp &= ~SDVO_PIPE_B_SELECT;
-				I915_WRITE(intel_hdmi->sdvox_reg, temp);
-				POSTING_READ(intel_hdmi->sdvox_reg);
-
-				/* Again we need to write this twice. */
-				I915_WRITE(intel_hdmi->sdvox_reg, temp);
-				POSTING_READ(intel_hdmi->sdvox_reg);
-
-				/* Transcoder selection bits only update
-				 * effectively on vblank. */
-				if (crtc)
-					intel_wait_for_vblank(dev, pipe);
-				else
-					msleep(50);
-			}
-		} else {
-			/* Restore the transcoder select bit. */
-			if (pipe == PIPE_B)
-				enable_bits |= SDVO_PIPE_B_SELECT;
-		}
+		/* Restore the transcoder select bit. */
+		if (pipe == PIPE_B)
+			enable_bits |= SDVO_PIPE_B_SELECT;
 	}
 
 	/* HW workaround, need to toggle enable bit off and on for 12bpc, but
@@ -652,12 +670,64 @@ static void intel_hdmi_dpms(struct drm_encoder *encoder, int mode)
 		POSTING_READ(intel_hdmi->sdvox_reg);
 	}
 
-	if (mode != DRM_MODE_DPMS_ON) {
-		temp &= ~enable_bits;
-	} else {
-		temp |= enable_bits;
+	temp |= enable_bits;
+
+	I915_WRITE(intel_hdmi->sdvox_reg, temp);
+	POSTING_READ(intel_hdmi->sdvox_reg);
+
+	/* HW workaround, need to write this twice for issue that may result
+	 * in first write getting masked.
+	 */
+	if (HAS_PCH_SPLIT(dev)) {
+		I915_WRITE(intel_hdmi->sdvox_reg, temp);
+		POSTING_READ(intel_hdmi->sdvox_reg);
+	}
+}
+
+static void intel_disable_hdmi(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+	u32 temp;
+	u32 enable_bits = SDVO_ENABLE | SDVO_AUDIO_ENABLE;
+
+	temp = I915_READ(intel_hdmi->sdvox_reg);
+
+	/* HW workaround for IBX, we need to move the port to transcoder A
+	 * before disabling it. */
+	if (HAS_PCH_IBX(dev)) {
+		struct drm_crtc *crtc = encoder->base.crtc;
+		int pipe = crtc ? to_intel_crtc(crtc)->pipe : -1;
+
+		if (temp & SDVO_PIPE_B_SELECT) {
+			temp &= ~SDVO_PIPE_B_SELECT;
+			I915_WRITE(intel_hdmi->sdvox_reg, temp);
+			POSTING_READ(intel_hdmi->sdvox_reg);
+
+			/* Again we need to write this twice. */
+			I915_WRITE(intel_hdmi->sdvox_reg, temp);
+			POSTING_READ(intel_hdmi->sdvox_reg);
+
+			/* Transcoder selection bits only update
+			 * effectively on vblank. */
+			if (crtc)
+				intel_wait_for_vblank(dev, pipe);
+			else
+				msleep(50);
+		}
+	}
+
+	/* HW workaround, need to toggle enable bit off and on for 12bpc, but
+	 * we do this anyway which shows more stable in testing.
+	 */
+	if (HAS_PCH_SPLIT(dev)) {
+		I915_WRITE(intel_hdmi->sdvox_reg, temp & ~SDVO_ENABLE);
+		POSTING_READ(intel_hdmi->sdvox_reg);
 	}
 
+	temp &= ~enable_bits;
+
 	I915_WRITE(intel_hdmi->sdvox_reg, temp);
 	POSTING_READ(intel_hdmi->sdvox_reg);
 
@@ -737,7 +807,6 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 						drm_detect_hdmi_monitor(edid);
 			intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
 		}
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 
@@ -778,8 +847,6 @@ intel_hdmi_detect_audio(struct drm_connector *connector)
 	if (edid) {
 		if (edid->input & DRM_EDID_INPUT_DIGITAL)
 			has_audio = drm_detect_monitor_audio(edid);
-
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 
@@ -833,9 +900,8 @@ intel_hdmi_set_property(struct drm_connector *connector,
 done:
 	if (intel_hdmi->base.base.crtc) {
 		struct drm_crtc *crtc = intel_hdmi->base.base.crtc;
-		drm_crtc_helper_set_mode(crtc, &crtc->mode,
-					 crtc->x, crtc->y,
-					 crtc->fb);
+		intel_set_mode(crtc, &crtc->mode,
+			       crtc->x, crtc->y, crtc->fb);
 	}
 
 	return 0;
@@ -849,23 +915,19 @@ static void intel_hdmi_destroy(struct drm_connector *connector)
 }
 
 static const struct drm_encoder_helper_funcs intel_hdmi_helper_funcs_hsw = {
-	.dpms = intel_ddi_dpms,
 	.mode_fixup = intel_hdmi_mode_fixup,
-	.prepare = intel_encoder_prepare,
 	.mode_set = intel_ddi_mode_set,
-	.commit = intel_encoder_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_encoder_helper_funcs intel_hdmi_helper_funcs = {
-	.dpms = intel_hdmi_dpms,
 	.mode_fixup = intel_hdmi_mode_fixup,
-	.prepare = intel_encoder_prepare,
 	.mode_set = intel_hdmi_mode_set,
-	.commit = intel_encoder_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_connector_dpms,
 	.detect = intel_hdmi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = intel_hdmi_set_property,
@@ -889,7 +951,7 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *c
 	intel_attach_broadcast_rgb_property(connector);
 }
 
-void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
+void intel_hdmi_init(struct drm_device *dev, int sdvox_reg, enum port port)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_connector *connector;
@@ -923,48 +985,25 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
 	connector->doublescan_allowed = 0;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 
-	/* Set up the DDC bus. */
-	if (sdvox_reg == SDVOB) {
-		intel_encoder->clone_mask = (1 << INTEL_HDMIB_CLONE_BIT);
-		intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
-		dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == SDVOC) {
-		intel_encoder->clone_mask = (1 << INTEL_HDMIC_CLONE_BIT);
-		intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
-		dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == HDMIB) {
-		intel_encoder->clone_mask = (1 << INTEL_HDMID_CLONE_BIT);
-		intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
-		dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == HDMIC) {
-		intel_encoder->clone_mask = (1 << INTEL_HDMIE_CLONE_BIT);
-		intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
-		dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == HDMID) {
-		intel_encoder->clone_mask = (1 << INTEL_HDMIF_CLONE_BIT);
-		intel_hdmi->ddc_bus = GMBUS_PORT_DPD;
-		dev_priv->hotplug_supported_mask |= HDMID_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == DDI_BUF_CTL(PORT_B)) {
-		DRM_DEBUG_DRIVER("LPT: detected output on DDI B\n");
-		intel_encoder->clone_mask = (1 << INTEL_HDMIB_CLONE_BIT);
+	intel_encoder->cloneable = false;
+
+	intel_hdmi->ddi_port = port;
+	switch (port) {
+	case PORT_B:
 		intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
-		intel_hdmi->ddi_port = PORT_B;
 		dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == DDI_BUF_CTL(PORT_C)) {
-		DRM_DEBUG_DRIVER("LPT: detected output on DDI C\n");
-		intel_encoder->clone_mask = (1 << INTEL_HDMIC_CLONE_BIT);
+		break;
+	case PORT_C:
 		intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
-		intel_hdmi->ddi_port = PORT_C;
 		dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
-	} else if (sdvox_reg == DDI_BUF_CTL(PORT_D)) {
-		DRM_DEBUG_DRIVER("LPT: detected output on DDI D\n");
-		intel_encoder->clone_mask = (1 << INTEL_HDMID_CLONE_BIT);
+		break;
+	case PORT_D:
 		intel_hdmi->ddc_bus = GMBUS_PORT_DPD;
-		intel_hdmi->ddi_port = PORT_D;
 		dev_priv->hotplug_supported_mask |= HDMID_HOTPLUG_INT_STATUS;
-	} else {
-		/* If we got an unknown sdvox_reg, things are pretty much broken
-		 * in a way that we should let the kernel know about it */
+		break;
+	case PORT_A:
+		/* Internal port only for eDP. */
+	default:
 		BUG();
 	}
 
@@ -987,10 +1026,21 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
 		intel_hdmi->set_infoframes = cpt_set_infoframes;
 	}
 
-	if (IS_HASWELL(dev))
-		drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs_hsw);
-	else
-		drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs);
+	if (IS_HASWELL(dev)) {
+		intel_encoder->enable = intel_enable_ddi;
+		intel_encoder->disable = intel_disable_ddi;
+		intel_encoder->get_hw_state = intel_ddi_get_hw_state;
+		drm_encoder_helper_add(&intel_encoder->base,
+				       &intel_hdmi_helper_funcs_hsw);
+	} else {
+		intel_encoder->enable = intel_enable_hdmi;
+		intel_encoder->disable = intel_disable_hdmi;
+		intel_encoder->get_hw_state = intel_hdmi_get_hw_state;
+		drm_encoder_helper_add(&intel_encoder->base,
+				       &intel_hdmi_helper_funcs);
+	}
+	intel_connector->get_hw_state = intel_connector_get_hw_state;
+
 
 	intel_hdmi_add_properties(intel_hdmi, connector);
 
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index b9755f6378d..c2c6dbc0971 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -29,10 +29,9 @@
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 struct gmbus_port {
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index e9a6f6aaed8..e3166df55da 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -31,12 +31,11 @@
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include <linux/acpi.h>
 
@@ -65,13 +64,40 @@ static struct intel_lvds *intel_attached_lvds(struct drm_connector *connector)
 			    struct intel_lvds, base);
 }
 
+static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
+				    enum pipe *pipe)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 lvds_reg, tmp;
+
+	if (HAS_PCH_SPLIT(dev)) {
+		lvds_reg = PCH_LVDS;
+	} else {
+		lvds_reg = LVDS;
+	}
+
+	tmp = I915_READ(lvds_reg);
+
+	if (!(tmp & LVDS_PORT_EN))
+		return false;
+
+	if (HAS_PCH_CPT(dev))
+		*pipe = PORT_TO_PIPE_CPT(tmp);
+	else
+		*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
 /**
  * Sets the power state for the panel.
  */
-static void intel_lvds_enable(struct intel_lvds *intel_lvds)
+static void intel_enable_lvds(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = intel_lvds->base.base.dev;
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_lvds->base.base.crtc);
+	struct drm_device *dev = encoder->base.dev;
+	struct intel_lvds *intel_lvds = to_intel_lvds(&encoder->base);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 ctl_reg, lvds_reg, stat_reg;
 
@@ -111,9 +137,10 @@ static void intel_lvds_enable(struct intel_lvds *intel_lvds)
 	intel_panel_enable_backlight(dev, intel_crtc->pipe);
 }
 
-static void intel_lvds_disable(struct intel_lvds *intel_lvds)
+static void intel_disable_lvds(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = intel_lvds->base.base.dev;
+	struct drm_device *dev = encoder->base.dev;
+	struct intel_lvds *intel_lvds = to_intel_lvds(&encoder->base);
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 ctl_reg, lvds_reg, stat_reg;
 
@@ -142,18 +169,6 @@ static void intel_lvds_disable(struct intel_lvds *intel_lvds)
 	POSTING_READ(lvds_reg);
 }
 
-static void intel_lvds_dpms(struct drm_encoder *encoder, int mode)
-{
-	struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
-
-	if (mode == DRM_MODE_DPMS_ON)
-		intel_lvds_enable(intel_lvds);
-	else
-		intel_lvds_disable(intel_lvds);
-
-	/* XXX: We never power down the LVDS pairs. */
-}
-
 static int intel_lvds_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
@@ -234,9 +249,8 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
-	struct intel_encoder *tmp_encoder;
+	struct intel_crtc *intel_crtc = intel_lvds->base.new_crtc;
 	u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0;
 	int pipe;
 
@@ -246,14 +260,8 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
 		return false;
 	}
 
-	/* Should never happen!! */
-	for_each_encoder_on_crtc(dev, encoder->crtc, tmp_encoder) {
-		if (&tmp_encoder->base != encoder) {
-			DRM_ERROR("Can't enable LVDS and another "
-			       "encoder on the same pipe\n");
-			return false;
-		}
-	}
+	if (intel_encoder_check_is_cloned(&intel_lvds->base))
+		return false;
 
 	/*
 	 * We have timings from the BIOS for the panel, put them in
@@ -405,23 +413,6 @@ out:
 	return true;
 }
 
-static void intel_lvds_prepare(struct drm_encoder *encoder)
-{
-	struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
-
-	intel_lvds_disable(intel_lvds);
-}
-
-static void intel_lvds_commit(struct drm_encoder *encoder)
-{
-	struct intel_lvds *intel_lvds = to_intel_lvds(encoder);
-
-	/* Always do a full power on as we do not know what state
-	 * we were left in.
-	 */
-	intel_lvds_enable(intel_lvds);
-}
-
 static void intel_lvds_mode_set(struct drm_encoder *encoder,
 				struct drm_display_mode *mode,
 				struct drm_display_mode *adjusted_mode)
@@ -535,7 +526,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
 	dev_priv->modeset_on_lid = 0;
 
 	mutex_lock(&dev->mode_config.mutex);
-	drm_helper_resume_force_mode(dev);
+	intel_modeset_check_state(dev);
 	mutex_unlock(&dev->mode_config.mutex);
 
 	return NOTIFY_OK;
@@ -587,8 +578,8 @@ static int intel_lvds_set_property(struct drm_connector *connector,
 			 * If the CRTC is enabled, the display will be changed
 			 * according to the new panel fitting mode.
 			 */
-			drm_crtc_helper_set_mode(crtc, &crtc->mode,
-				crtc->x, crtc->y, crtc->fb);
+			intel_set_mode(crtc, &crtc->mode,
+				       crtc->x, crtc->y, crtc->fb);
 		}
 	}
 
@@ -596,11 +587,9 @@ static int intel_lvds_set_property(struct drm_connector *connector,
 }
 
 static const struct drm_encoder_helper_funcs intel_lvds_helper_funcs = {
-	.dpms = intel_lvds_dpms,
 	.mode_fixup = intel_lvds_mode_fixup,
-	.prepare = intel_lvds_prepare,
 	.mode_set = intel_lvds_mode_set,
-	.commit = intel_lvds_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = {
@@ -610,7 +599,7 @@ static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs
 };
 
 static const struct drm_connector_funcs intel_lvds_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_connector_dpms,
 	.detect = intel_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = intel_lvds_set_property,
@@ -972,10 +961,15 @@ bool intel_lvds_init(struct drm_device *dev)
 	drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs,
 			 DRM_MODE_ENCODER_LVDS);
 
+	intel_encoder->enable = intel_enable_lvds;
+	intel_encoder->disable = intel_disable_lvds;
+	intel_encoder->get_hw_state = intel_lvds_get_hw_state;
+	intel_connector->get_hw_state = intel_connector_get_hw_state;
+
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
 	intel_encoder->type = INTEL_OUTPUT_LVDS;
 
-	intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT);
+	intel_encoder->cloneable = false;
 	if (HAS_PCH_SPLIT(dev))
 		intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
 	else if (IS_GEN4(dev))
diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c
index 29b72593fbb..cabd84bf66e 100644
--- a/drivers/gpu/drm/i915/intel_modes.c
+++ b/drivers/gpu/drm/i915/intel_modes.c
@@ -27,8 +27,8 @@
 #include <linux/i2c.h>
 #include <linux/fb.h>
 #include <drm/drm_edid.h>
-#include "drmP.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
 #include "i915_drv.h"
 
@@ -45,7 +45,6 @@ int intel_connector_update_modes(struct drm_connector *connector,
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
 	drm_edid_to_eld(connector, edid);
-	connector->display_info.raw_edid = NULL;
 	kfree(edid);
 
 	return ret;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 18bd0af855d..5530413213d 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -31,8 +31,8 @@
 #include <linux/acpi_io.h>
 #include <acpi/video.h>
 
-#include "drmP.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "intel_drv.h"
 
@@ -427,6 +427,25 @@ blind_set:
 	goto end;
 }
 
+static void intel_setup_cadls(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_opregion *opregion = &dev_priv->opregion;
+	int i = 0;
+	u32 disp_id;
+
+	/* Initialize the CADL field by duplicating the DIDL values.
+	 * Technically, this is not always correct as display outputs may exist,
+	 * but not active. This initialization is necessary for some Clevo
+	 * laptops that check this field before processing the brightness and
+	 * display switching hotkeys. Just like DIDL, CADL is NULL-terminated if
+	 * there are less than eight devices. */
+	do {
+		disp_id = ioread32(&opregion->acpi->didl[i]);
+		iowrite32(disp_id, &opregion->acpi->cadl[i]);
+	} while (++i < 8 && disp_id != 0);
+}
+
 void intel_opregion_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -436,8 +455,10 @@ void intel_opregion_init(struct drm_device *dev)
 		return;
 
 	if (opregion->acpi) {
-		if (drm_core_check_feature(dev, DRIVER_MODESET))
+		if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 			intel_didl_outputs(dev);
+			intel_setup_cadls(dev);
+		}
 
 		/* Notify BIOS we are ready to handle ACPI video ext notifs.
 		 * Right now, all the events are handled by the ACPI video module.
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 830d0dd610e..ebff850a9ab 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -25,9 +25,8 @@
  *
  * Derived from Xorg ddx, xf86-video-intel, src/i830_video.c
  */
-#include "drmP.h"
-#include "drm.h"
-#include "i915_drm.h"
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_drv.h"
@@ -235,54 +234,6 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 	return 0;
 }
 
-/* Workaround for i830 bug where pipe a must be enable to change control regs */
-static int
-i830_activate_pipe_a(struct drm_device *dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct intel_crtc *crtc;
-	struct drm_crtc_helper_funcs *crtc_funcs;
-	struct drm_display_mode vesa_640x480 = {
-		DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
-			 752, 800, 0, 480, 489, 492, 525, 0,
-			 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
-	}, *mode;
-
-	crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[0]);
-	if (crtc->dpms_mode == DRM_MODE_DPMS_ON)
-		return 0;
-
-	/* most i8xx have pipe a forced on, so don't trust dpms mode */
-	if (I915_READ(_PIPEACONF) & PIPECONF_ENABLE)
-		return 0;
-
-	crtc_funcs = crtc->base.helper_private;
-	if (crtc_funcs->dpms == NULL)
-		return 0;
-
-	DRM_DEBUG_DRIVER("Enabling pipe A in order to enable overlay\n");
-
-	mode = drm_mode_duplicate(dev, &vesa_640x480);
-
-	if (!drm_crtc_helper_set_mode(&crtc->base, mode,
-				       crtc->base.x, crtc->base.y,
-				       crtc->base.fb))
-		return 0;
-
-	crtc_funcs->dpms(&crtc->base, DRM_MODE_DPMS_ON);
-	return 1;
-}
-
-static void
-i830_deactivate_pipe_a(struct drm_device *dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[0];
-	struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
-
-	crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
-}
-
 /* overlay needs to be disable in OCMD reg */
 static int intel_overlay_on(struct intel_overlay *overlay)
 {
@@ -290,17 +241,12 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
 	struct drm_i915_gem_request *request;
-	int pipe_a_quirk = 0;
 	int ret;
 
 	BUG_ON(overlay->active);
 	overlay->active = 1;
 
-	if (IS_I830(dev)) {
-		pipe_a_quirk = i830_activate_pipe_a(dev);
-		if (pipe_a_quirk < 0)
-			return pipe_a_quirk;
-	}
+	WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
 
 	request = kzalloc(sizeof(*request), GFP_KERNEL);
 	if (request == NULL) {
@@ -322,9 +268,6 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 
 	ret = intel_overlay_do_wait_request(overlay, request, NULL);
 out:
-	if (pipe_a_quirk)
-		i830_deactivate_pipe_a(dev);
-
 	return ret;
 }
 
@@ -1439,7 +1382,7 @@ void intel_setup_overlay(struct drm_device *dev)
 		}
 		overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
 	} else {
-		ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true);
+		ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true, false);
 		if (ret) {
 			DRM_ERROR("failed to pin overlay register bo\n");
 			goto out_free_bo;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index ba8a27b1757..d69f8f49beb 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -31,6 +31,8 @@
 #include "../../../platform/x86/intel_ips.h"
 #include <linux/module.h>
 
+#define FORCEWAKE_ACK_TIMEOUT_MS 2
+
 /* FBC, or Frame Buffer Compression, is a technique employed to compress the
  * framebuffer contents in-memory, aiming at reducing the required bandwidth
  * during in-memory transfers and, therefore, reduce the power packet.
@@ -593,7 +595,7 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 		break;
 	}
 
-	dev_priv->r_t = dev_priv->mem_freq;
+	dev_priv->ips.r_t = dev_priv->mem_freq;
 
 	switch (csipll & 0x3ff) {
 	case 0x00c:
@@ -625,11 +627,11 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 	}
 
 	if (dev_priv->fsb_freq == 3200) {
-		dev_priv->c_m = 0;
+		dev_priv->ips.c_m = 0;
 	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
-		dev_priv->c_m = 1;
+		dev_priv->ips.c_m = 1;
 	} else {
-		dev_priv->c_m = 2;
+		dev_priv->ips.c_m = 2;
 	}
 }
 
@@ -2138,7 +2140,7 @@ intel_alloc_context_page(struct drm_device *dev)
 		return NULL;
 	}
 
-	ret = i915_gem_object_pin(ctx, 4096, true);
+	ret = i915_gem_object_pin(ctx, 4096, true, false);
 	if (ret) {
 		DRM_ERROR("failed to pin power context: %d\n", ret);
 		goto err_unref;
@@ -2160,11 +2162,22 @@ err_unref:
 	return NULL;
 }
 
+/**
+ * Lock protecting IPS related data structures
+ */
+DEFINE_SPINLOCK(mchdev_lock);
+
+/* Global for IPS driver to get at the current i915 device. Protected by
+ * mchdev_lock. */
+static struct drm_i915_private *i915_mch_dev;
+
 bool ironlake_set_drps(struct drm_device *dev, u8 val)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u16 rgvswctl;
 
+	assert_spin_locked(&mchdev_lock);
+
 	rgvswctl = I915_READ16(MEMSWCTL);
 	if (rgvswctl & MEMCTL_CMD_STS) {
 		DRM_DEBUG("gpu busy, RCS change rejected\n");
@@ -2188,6 +2201,8 @@ static void ironlake_enable_drps(struct drm_device *dev)
 	u32 rgvmodectl = I915_READ(MEMMODECTL);
 	u8 fmax, fmin, fstart, vstart;
 
+	spin_lock_irq(&mchdev_lock);
+
 	/* Enable temp reporting */
 	I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
 	I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
@@ -2211,12 +2226,12 @@ static void ironlake_enable_drps(struct drm_device *dev)
 	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
 		PXVFREQ_PX_SHIFT;
 
-	dev_priv->fmax = fmax; /* IPS callback will increase this */
-	dev_priv->fstart = fstart;
+	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
+	dev_priv->ips.fstart = fstart;
 
-	dev_priv->max_delay = fstart;
-	dev_priv->min_delay = fmin;
-	dev_priv->cur_delay = fstart;
+	dev_priv->ips.max_delay = fstart;
+	dev_priv->ips.min_delay = fmin;
+	dev_priv->ips.cur_delay = fstart;
 
 	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
 			 fmax, fmin, fstart);
@@ -2233,23 +2248,29 @@ static void ironlake_enable_drps(struct drm_device *dev)
 	rgvmodectl |= MEMMODE_SWMODE_EN;
 	I915_WRITE(MEMMODECTL, rgvmodectl);
 
-	if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
+	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
 		DRM_ERROR("stuck trying to change perf mode\n");
-	msleep(1);
+	mdelay(1);
 
 	ironlake_set_drps(dev, fstart);
 
-	dev_priv->last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
+	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
 		I915_READ(0x112e0);
-	dev_priv->last_time1 = jiffies_to_msecs(jiffies);
-	dev_priv->last_count2 = I915_READ(0x112f4);
-	getrawmonotonic(&dev_priv->last_time2);
+	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
+	dev_priv->ips.last_count2 = I915_READ(0x112f4);
+	getrawmonotonic(&dev_priv->ips.last_time2);
+
+	spin_unlock_irq(&mchdev_lock);
 }
 
 static void ironlake_disable_drps(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u16 rgvswctl = I915_READ16(MEMSWCTL);
+	u16 rgvswctl;
+
+	spin_lock_irq(&mchdev_lock);
+
+	rgvswctl = I915_READ16(MEMSWCTL);
 
 	/* Ack interrupts, disable EFC interrupt */
 	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
@@ -2259,31 +2280,54 @@ static void ironlake_disable_drps(struct drm_device *dev)
 	I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
 
 	/* Go back to the starting frequency */
-	ironlake_set_drps(dev, dev_priv->fstart);
-	msleep(1);
+	ironlake_set_drps(dev, dev_priv->ips.fstart);
+	mdelay(1);
 	rgvswctl |= MEMCTL_CMD_STS;
 	I915_WRITE(MEMSWCTL, rgvswctl);
-	msleep(1);
+	mdelay(1);
 
+	spin_unlock_irq(&mchdev_lock);
 }
 
-void gen6_set_rps(struct drm_device *dev, u8 val)
+/* There's a funny hw issue where the hw returns all 0 when reading from
+ * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
+ * ourselves, instead of doing a rmw cycle (which might result in us clearing
+ * all limits and the gpu stuck at whatever frequency it is at atm).
+ */
+static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 *val)
 {
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 limits;
 
 	limits = 0;
-	if (val >= dev_priv->max_delay)
-		val = dev_priv->max_delay;
-	else
-		limits |= dev_priv->max_delay << 24;
 
-	if (val <= dev_priv->min_delay)
-		val = dev_priv->min_delay;
-	else
-		limits |= dev_priv->min_delay << 16;
+	if (*val >= dev_priv->rps.max_delay)
+		*val = dev_priv->rps.max_delay;
+	limits |= dev_priv->rps.max_delay << 24;
+
+	/* Only set the down limit when we've reached the lowest level to avoid
+	 * getting more interrupts, otherwise leave this clear. This prevents a
+	 * race in the hw when coming out of rc6: There's a tiny window where
+	 * the hw runs at the minimal clock before selecting the desired
+	 * frequency, if the down threshold expires in that window we will not
+	 * receive a down interrupt. */
+	if (*val <= dev_priv->rps.min_delay) {
+		*val = dev_priv->rps.min_delay;
+		limits |= dev_priv->rps.min_delay << 16;
+	}
 
-	if (val == dev_priv->cur_delay)
+	return limits;
+}
+
+void gen6_set_rps(struct drm_device *dev, u8 val)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 limits = gen6_rps_limits(dev_priv, &val);
+
+	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+	WARN_ON(val > dev_priv->rps.max_delay);
+	WARN_ON(val < dev_priv->rps.min_delay);
+
+	if (val == dev_priv->rps.cur_delay)
 		return;
 
 	I915_WRITE(GEN6_RPNSWREQ,
@@ -2296,7 +2340,11 @@ void gen6_set_rps(struct drm_device *dev, u8 val)
 	 */
 	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
 
-	dev_priv->cur_delay = val;
+	POSTING_READ(GEN6_RPNSWREQ);
+
+	dev_priv->rps.cur_delay = val;
+
+	trace_intel_gpu_freq_change(val * 50);
 }
 
 static void gen6_disable_rps(struct drm_device *dev)
@@ -2312,40 +2360,40 @@ static void gen6_disable_rps(struct drm_device *dev)
 	 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
 	 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
 
-	spin_lock_irq(&dev_priv->rps_lock);
-	dev_priv->pm_iir = 0;
-	spin_unlock_irq(&dev_priv->rps_lock);
+	spin_lock_irq(&dev_priv->rps.lock);
+	dev_priv->rps.pm_iir = 0;
+	spin_unlock_irq(&dev_priv->rps.lock);
 
 	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
 }
 
 int intel_enable_rc6(const struct drm_device *dev)
 {
-	/*
-	 * Respect the kernel parameter if it is set
-	 */
+	/* Respect the kernel parameter if it is set */
 	if (i915_enable_rc6 >= 0)
 		return i915_enable_rc6;
 
-	/*
-	 * Disable RC6 on Ironlake
-	 */
-	if (INTEL_INFO(dev)->gen == 5)
-		return 0;
+	if (INTEL_INFO(dev)->gen == 5) {
+#ifdef CONFIG_INTEL_IOMMU
+		/* Disable rc6 on ilk if VT-d is on. */
+		if (intel_iommu_gfx_mapped)
+			return false;
+#endif
+		DRM_DEBUG_DRIVER("Ironlake: only RC6 available\n");
+		return INTEL_RC6_ENABLE;
+	}
 
-	/* On Haswell, only RC6 is available. So let's enable it by default to
-	 * provide better testing and coverage since the beginning.
-	 */
-	if (IS_HASWELL(dev))
+	if (IS_HASWELL(dev)) {
+		DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
 		return INTEL_RC6_ENABLE;
+	}
 
-	/*
-	 * Disable rc6 on Sandybridge
-	 */
+	/* snb/ivb have more than one rc6 state. */
 	if (INTEL_INFO(dev)->gen == 6) {
 		DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
 		return INTEL_RC6_ENABLE;
 	}
+
 	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
 	return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
 }
@@ -2383,9 +2431,9 @@ static void gen6_enable_rps(struct drm_device *dev)
 	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
 
 	/* In units of 100MHz */
-	dev_priv->max_delay = rp_state_cap & 0xff;
-	dev_priv->min_delay = (rp_state_cap & 0xff0000) >> 16;
-	dev_priv->cur_delay = 0;
+	dev_priv->rps.max_delay = rp_state_cap & 0xff;
+	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
+	dev_priv->rps.cur_delay = 0;
 
 	/* disable the counters and set deterministic thresholds */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -2438,8 +2486,8 @@ static void gen6_enable_rps(struct drm_device *dev)
 
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
 	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
-		   dev_priv->max_delay << 24 |
-		   dev_priv->min_delay << 16);
+		   dev_priv->rps.max_delay << 24 |
+		   dev_priv->rps.min_delay << 16);
 
 	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
 	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
@@ -2477,7 +2525,7 @@ static void gen6_enable_rps(struct drm_device *dev)
 		     500))
 		DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
 	if (pcu_mbox & (1<<31)) { /* OC supported */
-		dev_priv->max_delay = pcu_mbox & 0xff;
+		dev_priv->rps.max_delay = pcu_mbox & 0xff;
 		DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
 	}
 
@@ -2485,10 +2533,10 @@ static void gen6_enable_rps(struct drm_device *dev)
 
 	/* requires MSI enabled */
 	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
-	spin_lock_irq(&dev_priv->rps_lock);
-	WARN_ON(dev_priv->pm_iir != 0);
+	spin_lock_irq(&dev_priv->rps.lock);
+	WARN_ON(dev_priv->rps.pm_iir != 0);
 	I915_WRITE(GEN6_PMIMR, 0);
-	spin_unlock_irq(&dev_priv->rps_lock);
+	spin_unlock_irq(&dev_priv->rps.lock);
 	/* enable all PM interrupts */
 	I915_WRITE(GEN6_PMINTRMSK, 0);
 
@@ -2520,9 +2568,9 @@ static void gen6_update_ring_freq(struct drm_device *dev)
 	 * to use for memory access.  We do this by specifying the IA frequency
 	 * the PCU should use as a reference to determine the ring frequency.
 	 */
-	for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
+	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
 	     gpu_freq--) {
-		int diff = dev_priv->max_delay - gpu_freq;
+		int diff = dev_priv->rps.max_delay - gpu_freq;
 
 		/*
 		 * For GPU frequencies less than 750MHz, just use the lowest
@@ -2686,14 +2734,16 @@ static const struct cparams {
 	{ 0, 800, 231, 23784 },
 };
 
-unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
+static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
 {
 	u64 total_count, diff, ret;
 	u32 count1, count2, count3, m = 0, c = 0;
 	unsigned long now = jiffies_to_msecs(jiffies), diff1;
 	int i;
 
-	diff1 = now - dev_priv->last_time1;
+	assert_spin_locked(&mchdev_lock);
+
+	diff1 = now - dev_priv->ips.last_time1;
 
 	/* Prevent division-by-zero if we are asking too fast.
 	 * Also, we don't get interesting results if we are polling
@@ -2701,7 +2751,7 @@ unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 	 * in such cases.
 	 */
 	if (diff1 <= 10)
-		return dev_priv->chipset_power;
+		return dev_priv->ips.chipset_power;
 
 	count1 = I915_READ(DMIEC);
 	count2 = I915_READ(DDREC);
@@ -2710,16 +2760,16 @@ unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 	total_count = count1 + count2 + count3;
 
 	/* FIXME: handle per-counter overflow */
-	if (total_count < dev_priv->last_count1) {
-		diff = ~0UL - dev_priv->last_count1;
+	if (total_count < dev_priv->ips.last_count1) {
+		diff = ~0UL - dev_priv->ips.last_count1;
 		diff += total_count;
 	} else {
-		diff = total_count - dev_priv->last_count1;
+		diff = total_count - dev_priv->ips.last_count1;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
-		if (cparams[i].i == dev_priv->c_m &&
-		    cparams[i].t == dev_priv->r_t) {
+		if (cparams[i].i == dev_priv->ips.c_m &&
+		    cparams[i].t == dev_priv->ips.r_t) {
 			m = cparams[i].m;
 			c = cparams[i].c;
 			break;
@@ -2730,14 +2780,30 @@ unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 	ret = ((m * diff) + c);
 	ret = div_u64(ret, 10);
 
-	dev_priv->last_count1 = total_count;
-	dev_priv->last_time1 = now;
+	dev_priv->ips.last_count1 = total_count;
+	dev_priv->ips.last_time1 = now;
 
-	dev_priv->chipset_power = ret;
+	dev_priv->ips.chipset_power = ret;
 
 	return ret;
 }
 
+unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
+{
+	unsigned long val;
+
+	if (dev_priv->info->gen != 5)
+		return 0;
+
+	spin_lock_irq(&mchdev_lock);
+
+	val = __i915_chipset_val(dev_priv);
+
+	spin_unlock_irq(&mchdev_lock);
+
+	return val;
+}
+
 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
 {
 	unsigned long m, x, b;
@@ -2894,18 +2960,17 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 		return v_table[pxvid].vd;
 }
 
-void i915_update_gfx_val(struct drm_i915_private *dev_priv)
+static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
 {
 	struct timespec now, diff1;
 	u64 diff;
 	unsigned long diffms;
 	u32 count;
 
-	if (dev_priv->info->gen != 5)
-		return;
+	assert_spin_locked(&mchdev_lock);
 
 	getrawmonotonic(&now);
-	diff1 = timespec_sub(now, dev_priv->last_time2);
+	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
 
 	/* Don't divide by 0 */
 	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
@@ -2914,28 +2979,42 @@ void i915_update_gfx_val(struct drm_i915_private *dev_priv)
 
 	count = I915_READ(GFXEC);
 
-	if (count < dev_priv->last_count2) {
-		diff = ~0UL - dev_priv->last_count2;
+	if (count < dev_priv->ips.last_count2) {
+		diff = ~0UL - dev_priv->ips.last_count2;
 		diff += count;
 	} else {
-		diff = count - dev_priv->last_count2;
+		diff = count - dev_priv->ips.last_count2;
 	}
 
-	dev_priv->last_count2 = count;
-	dev_priv->last_time2 = now;
+	dev_priv->ips.last_count2 = count;
+	dev_priv->ips.last_time2 = now;
 
 	/* More magic constants... */
 	diff = diff * 1181;
 	diff = div_u64(diff, diffms * 10);
-	dev_priv->gfx_power = diff;
+	dev_priv->ips.gfx_power = diff;
 }
 
-unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
+void i915_update_gfx_val(struct drm_i915_private *dev_priv)
+{
+	if (dev_priv->info->gen != 5)
+		return;
+
+	spin_lock_irq(&mchdev_lock);
+
+	__i915_update_gfx_val(dev_priv);
+
+	spin_unlock_irq(&mchdev_lock);
+}
+
+static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
 {
 	unsigned long t, corr, state1, corr2, state2;
 	u32 pxvid, ext_v;
 
-	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->cur_delay * 4));
+	assert_spin_locked(&mchdev_lock);
+
+	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
 	pxvid = (pxvid >> 24) & 0x7f;
 	ext_v = pvid_to_extvid(dev_priv, pxvid);
 
@@ -2955,27 +3034,31 @@ unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
 
 	corr = corr * ((150142 * state1) / 10000 - 78642);
 	corr /= 100000;
-	corr2 = (corr * dev_priv->corr);
+	corr2 = (corr * dev_priv->ips.corr);
 
 	state2 = (corr2 * state1) / 10000;
 	state2 /= 100; /* convert to mW */
 
-	i915_update_gfx_val(dev_priv);
+	__i915_update_gfx_val(dev_priv);
 
-	return dev_priv->gfx_power + state2;
+	return dev_priv->ips.gfx_power + state2;
 }
 
-/* Global for IPS driver to get at the current i915 device */
-static struct drm_i915_private *i915_mch_dev;
-/*
- * Lock protecting IPS related data structures
- *   - i915_mch_dev
- *   - dev_priv->max_delay
- *   - dev_priv->min_delay
- *   - dev_priv->fmax
- *   - dev_priv->gpu_busy
- */
-static DEFINE_SPINLOCK(mchdev_lock);
+unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
+{
+	unsigned long val;
+
+	if (dev_priv->info->gen != 5)
+		return 0;
+
+	spin_lock_irq(&mchdev_lock);
+
+	val = __i915_gfx_val(dev_priv);
+
+	spin_unlock_irq(&mchdev_lock);
+
+	return val;
+}
 
 /**
  * i915_read_mch_val - return value for IPS use
@@ -2988,18 +3071,18 @@ unsigned long i915_read_mch_val(void)
 	struct drm_i915_private *dev_priv;
 	unsigned long chipset_val, graphics_val, ret = 0;
 
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	if (!i915_mch_dev)
 		goto out_unlock;
 	dev_priv = i915_mch_dev;
 
-	chipset_val = i915_chipset_val(dev_priv);
-	graphics_val = i915_gfx_val(dev_priv);
+	chipset_val = __i915_chipset_val(dev_priv);
+	graphics_val = __i915_gfx_val(dev_priv);
 
 	ret = chipset_val + graphics_val;
 
 out_unlock:
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
 }
@@ -3015,18 +3098,18 @@ bool i915_gpu_raise(void)
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 
-	if (dev_priv->max_delay > dev_priv->fmax)
-		dev_priv->max_delay--;
+	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
+		dev_priv->ips.max_delay--;
 
 out_unlock:
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
 }
@@ -3043,18 +3126,18 @@ bool i915_gpu_lower(void)
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 
-	if (dev_priv->max_delay < dev_priv->min_delay)
-		dev_priv->max_delay++;
+	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
+		dev_priv->ips.max_delay++;
 
 out_unlock:
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
 }
@@ -3068,17 +3151,20 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower);
 bool i915_gpu_busy(void)
 {
 	struct drm_i915_private *dev_priv;
+	struct intel_ring_buffer *ring;
 	bool ret = false;
+	int i;
 
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	if (!i915_mch_dev)
 		goto out_unlock;
 	dev_priv = i915_mch_dev;
 
-	ret = dev_priv->busy;
+	for_each_ring(ring, dev_priv, i)
+		ret |= !list_empty(&ring->request_list);
 
 out_unlock:
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
 }
@@ -3095,20 +3181,20 @@ bool i915_gpu_turbo_disable(void)
 	struct drm_i915_private *dev_priv;
 	bool ret = true;
 
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	if (!i915_mch_dev) {
 		ret = false;
 		goto out_unlock;
 	}
 	dev_priv = i915_mch_dev;
 
-	dev_priv->max_delay = dev_priv->fstart;
+	dev_priv->ips.max_delay = dev_priv->ips.fstart;
 
-	if (!ironlake_set_drps(dev_priv->dev, dev_priv->fstart))
+	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
 		ret = false;
 
 out_unlock:
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	return ret;
 }
@@ -3136,19 +3222,20 @@ ips_ping_for_i915_load(void)
 
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
 {
-	spin_lock(&mchdev_lock);
+	/* We only register the i915 ips part with intel-ips once everything is
+	 * set up, to avoid intel-ips sneaking in and reading bogus values. */
+	spin_lock_irq(&mchdev_lock);
 	i915_mch_dev = dev_priv;
-	dev_priv->mchdev_lock = &mchdev_lock;
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 
 	ips_ping_for_i915_load();
 }
 
 void intel_gpu_ips_teardown(void)
 {
-	spin_lock(&mchdev_lock);
+	spin_lock_irq(&mchdev_lock);
 	i915_mch_dev = NULL;
-	spin_unlock(&mchdev_lock);
+	spin_unlock_irq(&mchdev_lock);
 }
 static void intel_init_emon(struct drm_device *dev)
 {
@@ -3218,7 +3305,7 @@ static void intel_init_emon(struct drm_device *dev)
 
 	lcfuse = I915_READ(LCFUSE02);
 
-	dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
+	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
 void intel_disable_gt_powersave(struct drm_device *dev)
@@ -3731,42 +3818,6 @@ void intel_init_clock_gating(struct drm_device *dev)
 		dev_priv->display.init_pch_clock_gating(dev);
 }
 
-static void gen6_sanitize_pm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 limits, delay, old;
-
-	gen6_gt_force_wake_get(dev_priv);
-
-	old = limits = I915_READ(GEN6_RP_INTERRUPT_LIMITS);
-	/* Make sure we continue to get interrupts
-	 * until we hit the minimum or maximum frequencies.
-	 */
-	limits &= ~(0x3f << 16 | 0x3f << 24);
-	delay = dev_priv->cur_delay;
-	if (delay < dev_priv->max_delay)
-		limits |= (dev_priv->max_delay & 0x3f) << 24;
-	if (delay > dev_priv->min_delay)
-		limits |= (dev_priv->min_delay & 0x3f) << 16;
-
-	if (old != limits) {
-		/* Note that the known failure case is to read back 0. */
-		DRM_DEBUG_DRIVER("Power management discrepancy: GEN6_RP_INTERRUPT_LIMITS "
-				 "expected %08x, was %08x\n", limits, old);
-		I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
-	}
-
-	gen6_gt_force_wake_put(dev_priv);
-}
-
-void intel_sanitize_pm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->display.sanitize_pm)
-		dev_priv->display.sanitize_pm(dev);
-}
-
 /* Starting with Haswell, we have different power wells for
  * different parts of the GPU. This attempts to enable them all.
  */
@@ -3852,7 +3903,6 @@ void intel_init_pm(struct drm_device *dev)
 				dev_priv->display.update_wm = NULL;
 			}
 			dev_priv->display.init_clock_gating = gen6_init_clock_gating;
-			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
 		} else if (IS_IVYBRIDGE(dev)) {
 			/* FIXME: detect B0+ stepping and use auto training */
 			if (SNB_READ_WM0_LATENCY()) {
@@ -3864,7 +3914,6 @@ void intel_init_pm(struct drm_device *dev)
 				dev_priv->display.update_wm = NULL;
 			}
 			dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
-			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
 		} else if (IS_HASWELL(dev)) {
 			if (SNB_READ_WM0_LATENCY()) {
 				dev_priv->display.update_wm = sandybridge_update_wm;
@@ -3876,7 +3925,6 @@ void intel_init_pm(struct drm_device *dev)
 				dev_priv->display.update_wm = NULL;
 			}
 			dev_priv->display.init_clock_gating = haswell_init_clock_gating;
-			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
 		} else
 			dev_priv->display.update_wm = NULL;
 	} else if (IS_VALLEYVIEW(dev)) {
@@ -3955,14 +4003,16 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 	else
 		forcewake_ack = FORCEWAKE_ACK;
 
-	if (wait_for_atomic_us((I915_READ_NOTRACE(forcewake_ack) & 1) == 0, 500))
-		DRM_ERROR("Force wake wait timed out\n");
+	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
 	I915_WRITE_NOTRACE(FORCEWAKE, 1);
-	POSTING_READ(FORCEWAKE);
+	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
-	if (wait_for_atomic_us((I915_READ_NOTRACE(forcewake_ack) & 1), 500))
-		DRM_ERROR("Force wake wait timed out\n");
+	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
 	__gen6_gt_wait_for_thread_c0(dev_priv);
 }
@@ -3976,14 +4026,16 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 	else
 		forcewake_ack = FORCEWAKE_MT_ACK;
 
-	if (wait_for_atomic_us((I915_READ_NOTRACE(forcewake_ack) & 1) == 0, 500))
-		DRM_ERROR("Force wake wait timed out\n");
+	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
 	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(1));
-	POSTING_READ(FORCEWAKE_MT);
+	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
 
-	if (wait_for_atomic_us((I915_READ_NOTRACE(forcewake_ack) & 1), 500))
-		DRM_ERROR("Force wake wait timed out\n");
+	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
 	__gen6_gt_wait_for_thread_c0(dev_priv);
 }
@@ -4016,14 +4068,14 @@ void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE_NOTRACE(FORCEWAKE, 0);
-	POSTING_READ(FORCEWAKE);
+	/* gen6_gt_check_fifodbg doubles as the POSTING_READ */
 	gen6_gt_check_fifodbg(dev_priv);
 }
 
 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(1));
-	POSTING_READ(FORCEWAKE_MT);
+	/* gen6_gt_check_fifodbg doubles as the POSTING_READ */
 	gen6_gt_check_fifodbg(dev_priv);
 }
 
@@ -4062,24 +4114,24 @@ int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 
 static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
 {
-	/* Already awake? */
-	if ((I915_READ(0x130094) & 0xa1) == 0xa1)
-		return;
+	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
 
-	I915_WRITE_NOTRACE(FORCEWAKE_VLV, 0xffffffff);
-	POSTING_READ(FORCEWAKE_VLV);
+	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(1));
 
-	if (wait_for_atomic_us((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1), 500))
-		DRM_ERROR("Force wake wait timed out\n");
+	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
+			    FORCEWAKE_ACK_TIMEOUT_MS))
+		DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
 	__gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
 static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
 {
-	I915_WRITE_NOTRACE(FORCEWAKE_VLV, 0xffff0000);
-	/* FIXME: confirm VLV behavior with Punit folks */
-	POSTING_READ(FORCEWAKE_VLV);
+	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(1));
+	/* The below doubles as a POSTING_READ */
+	gen6_gt_check_fifodbg(dev_priv);
 }
 
 void intel_gt_init(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index e2a73b38abe..ecbc5c5dbbb 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -27,10 +27,9 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "i915_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_trace.h"
 #include "intel_drv.h"
 
@@ -262,6 +261,83 @@ gen6_render_ring_flush(struct intel_ring_buffer *ring,
 	return 0;
 }
 
+static int
+gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
+{
+	int ret;
+
+	ret = intel_ring_begin(ring, 4);
+	if (ret)
+		return ret;
+
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
+	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
+			      PIPE_CONTROL_STALL_AT_SCOREBOARD);
+	intel_ring_emit(ring, 0);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
+
+	return 0;
+}
+
+static int
+gen7_render_ring_flush(struct intel_ring_buffer *ring,
+		       u32 invalidate_domains, u32 flush_domains)
+{
+	u32 flags = 0;
+	struct pipe_control *pc = ring->private;
+	u32 scratch_addr = pc->gtt_offset + 128;
+	int ret;
+
+	/*
+	 * Ensure that any following seqno writes only happen when the render
+	 * cache is indeed flushed.
+	 *
+	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
+	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
+	 * don't try to be clever and just set it unconditionally.
+	 */
+	flags |= PIPE_CONTROL_CS_STALL;
+
+	/* Just flush everything.  Experiments have shown that reducing the
+	 * number of bits based on the write domains has little performance
+	 * impact.
+	 */
+	if (flush_domains) {
+		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
+		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
+	}
+	if (invalidate_domains) {
+		flags |= PIPE_CONTROL_TLB_INVALIDATE;
+		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
+		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
+		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
+		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
+		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
+		/*
+		 * TLB invalidate requires a post-sync write.
+		 */
+		flags |= PIPE_CONTROL_QW_WRITE;
+
+		/* Workaround: we must issue a pipe_control with CS-stall bit
+		 * set before a pipe_control command that has the state cache
+		 * invalidate bit set. */
+		gen7_render_ring_cs_stall_wa(ring);
+	}
+
+	ret = intel_ring_begin(ring, 4);
+	if (ret)
+		return ret;
+
+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
+	intel_ring_emit(ring, flags);
+	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
+
+	return 0;
+}
+
 static void ring_write_tail(struct intel_ring_buffer *ring,
 			    u32 value)
 {
@@ -382,12 +458,12 @@ init_pipe_control(struct intel_ring_buffer *ring)
 
 	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
-	ret = i915_gem_object_pin(obj, 4096, true);
+	ret = i915_gem_object_pin(obj, 4096, true, false);
 	if (ret)
 		goto err_unref;
 
 	pc->gtt_offset = obj->gtt_offset;
-	pc->cpu_page =  kmap(obj->pages[0]);
+	pc->cpu_page =  kmap(sg_page(obj->pages->sgl));
 	if (pc->cpu_page == NULL)
 		goto err_unpin;
 
@@ -414,7 +490,8 @@ cleanup_pipe_control(struct intel_ring_buffer *ring)
 		return;
 
 	obj = pc->obj;
-	kunmap(obj->pages[0]);
+
+	kunmap(sg_page(obj->pages->sgl));
 	i915_gem_object_unpin(obj);
 	drm_gem_object_unreference(&obj->base);
 
@@ -462,7 +539,7 @@ static int init_render_ring(struct intel_ring_buffer *ring)
 	if (INTEL_INFO(dev)->gen >= 6)
 		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
-	if (IS_IVYBRIDGE(dev))
+	if (HAS_L3_GPU_CACHE(dev))
 		I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 
 	return ret;
@@ -628,26 +705,24 @@ pc_render_add_request(struct intel_ring_buffer *ring,
 }
 
 static u32
-gen6_ring_get_seqno(struct intel_ring_buffer *ring)
+gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
 {
-	struct drm_device *dev = ring->dev;
-
 	/* Workaround to force correct ordering between irq and seqno writes on
 	 * ivb (and maybe also on snb) by reading from a CS register (like
 	 * ACTHD) before reading the status page. */
-	if (IS_GEN6(dev) || IS_GEN7(dev))
+	if (!lazy_coherency)
 		intel_ring_get_active_head(ring);
 	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 }
 
 static u32
-ring_get_seqno(struct intel_ring_buffer *ring)
+ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
 {
 	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 }
 
 static u32
-pc_render_get_seqno(struct intel_ring_buffer *ring)
+pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
 {
 	struct pipe_control *pc = ring->private;
 	return pc->cpu_page[0];
@@ -852,7 +927,7 @@ gen6_ring_get_irq(struct intel_ring_buffer *ring)
 
 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
 	if (ring->irq_refcount++ == 0) {
-		if (IS_IVYBRIDGE(dev) && ring->id == RCS)
+		if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
 			I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
 						GEN6_RENDER_L3_PARITY_ERROR));
 		else
@@ -875,7 +950,7 @@ gen6_ring_put_irq(struct intel_ring_buffer *ring)
 
 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
 	if (--ring->irq_refcount == 0) {
-		if (IS_IVYBRIDGE(dev) && ring->id == RCS)
+		if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
 			I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 		else
 			I915_WRITE_IMR(ring, ~0);
@@ -951,7 +1026,7 @@ static void cleanup_status_page(struct intel_ring_buffer *ring)
 	if (obj == NULL)
 		return;
 
-	kunmap(obj->pages[0]);
+	kunmap(sg_page(obj->pages->sgl));
 	i915_gem_object_unpin(obj);
 	drm_gem_object_unreference(&obj->base);
 	ring->status_page.obj = NULL;
@@ -972,13 +1047,13 @@ static int init_status_page(struct intel_ring_buffer *ring)
 
 	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 
-	ret = i915_gem_object_pin(obj, 4096, true);
+	ret = i915_gem_object_pin(obj, 4096, true, false);
 	if (ret != 0) {
 		goto err_unref;
 	}
 
 	ring->status_page.gfx_addr = obj->gtt_offset;
-	ring->status_page.page_addr = kmap(obj->pages[0]);
+	ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
 	if (ring->status_page.page_addr == NULL) {
 		ret = -ENOMEM;
 		goto err_unpin;
@@ -1010,7 +1085,6 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 	ring->dev = dev;
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
-	INIT_LIST_HEAD(&ring->gpu_write_list);
 	ring->size = 32 * PAGE_SIZE;
 
 	init_waitqueue_head(&ring->irq_queue);
@@ -1030,7 +1104,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 
 	ring->obj = obj;
 
-	ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
+	ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
 	if (ret)
 		goto err_unref;
 
@@ -1379,7 +1453,9 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
 
 	if (INTEL_INFO(dev)->gen >= 6) {
 		ring->add_request = gen6_add_request;
-		ring->flush = gen6_render_ring_flush;
+		ring->flush = gen7_render_ring_flush;
+		if (INTEL_INFO(dev)->gen == 6)
+			ring->flush = gen6_render_ring_flush;
 		ring->irq_get = gen6_ring_get_irq;
 		ring->irq_put = gen6_ring_put_irq;
 		ring->irq_enable_mask = GT_USER_INTERRUPT;
@@ -1481,7 +1557,6 @@ int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
 	ring->dev = dev;
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
-	INIT_LIST_HEAD(&ring->gpu_write_list);
 
 	ring->size = size;
 	ring->effective_size = ring->size;
@@ -1574,3 +1649,41 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
 
 	return intel_init_ring_buffer(dev, ring);
 }
+
+int
+intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
+{
+	int ret;
+
+	if (!ring->gpu_caches_dirty)
+		return 0;
+
+	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
+	if (ret)
+		return ret;
+
+	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
+
+	ring->gpu_caches_dirty = false;
+	return 0;
+}
+
+int
+intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
+{
+	uint32_t flush_domains;
+	int ret;
+
+	flush_domains = 0;
+	if (ring->gpu_caches_dirty)
+		flush_domains = I915_GEM_GPU_DOMAINS;
+
+	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
+	if (ret)
+		return ret;
+
+	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
+
+	ring->gpu_caches_dirty = false;
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 1d3c81fdad9..2ea7a311a1f 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -72,7 +72,14 @@ struct  intel_ring_buffer {
 				  u32	flush_domains);
 	int		(*add_request)(struct intel_ring_buffer *ring,
 				       u32 *seqno);
-	u32		(*get_seqno)(struct intel_ring_buffer *ring);
+	/* Some chipsets are not quite as coherent as advertised and need
+	 * an expensive kick to force a true read of the up-to-date seqno.
+	 * However, the up-to-date seqno is not always required and the last
+	 * seen value is good enough. Note that the seqno will always be
+	 * monotonic, even if not coherent.
+	 */
+	u32		(*get_seqno)(struct intel_ring_buffer *ring,
+				     bool lazy_coherency);
 	int		(*dispatch_execbuffer)(struct intel_ring_buffer *ring,
 					       u32 offset, u32 length);
 	void		(*cleanup)(struct intel_ring_buffer *ring);
@@ -101,15 +108,6 @@ struct  intel_ring_buffer {
 	struct list_head request_list;
 
 	/**
-	 * List of objects currently pending a GPU write flush.
-	 *
-	 * All elements on this list will belong to either the
-	 * active_list or flushing_list, last_rendering_seqno can
-	 * be used to differentiate between the two elements.
-	 */
-	struct list_head gpu_write_list;
-
-	/**
 	 * Do we have some not yet emitted requests outstanding?
 	 */
 	u32 outstanding_lazy_request;
@@ -204,6 +202,8 @@ static inline void intel_ring_emit(struct intel_ring_buffer *ring,
 void intel_ring_advance(struct intel_ring_buffer *ring);
 
 u32 intel_ring_get_seqno(struct intel_ring_buffer *ring);
+int intel_ring_flush_all_caches(struct intel_ring_buffer *ring);
+int intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring);
 
 int intel_init_render_ring_buffer(struct drm_device *dev);
 int intel_init_bsd_ring_buffer(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 123afd35761..0007a4d9bf6 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -29,12 +29,11 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/export.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "intel_sdvo_regs.h"
 
@@ -97,7 +96,7 @@ struct intel_sdvo {
 	/*
 	 * Hotplug activation bits for this device
 	 */
-	uint8_t hotplug_active[2];
+	uint16_t hotplug_active;
 
 	/**
 	 * This is used to select the color range of RBG outputs in HDMI mode.
@@ -628,6 +627,14 @@ static bool intel_sdvo_set_active_outputs(struct intel_sdvo *intel_sdvo,
 				    &outputs, sizeof(outputs));
 }
 
+static bool intel_sdvo_get_active_outputs(struct intel_sdvo *intel_sdvo,
+					  u16 *outputs)
+{
+	return intel_sdvo_get_value(intel_sdvo,
+				    SDVO_CMD_GET_ACTIVE_OUTPUTS,
+				    outputs, sizeof(*outputs));
+}
+
 static bool intel_sdvo_set_encoder_power_state(struct intel_sdvo *intel_sdvo,
 					       int mode)
 {
@@ -1142,51 +1149,132 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder,
 	intel_sdvo_write_sdvox(intel_sdvo, sdvox);
 }
 
-static void intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
+static bool intel_sdvo_connector_get_hw_state(struct intel_connector *connector)
 {
-	struct drm_device *dev = encoder->dev;
+	struct intel_sdvo_connector *intel_sdvo_connector =
+		to_intel_sdvo_connector(&connector->base);
+	struct intel_sdvo *intel_sdvo = intel_attached_sdvo(&connector->base);
+	u16 active_outputs;
+
+	intel_sdvo_get_active_outputs(intel_sdvo, &active_outputs);
+
+	if (active_outputs & intel_sdvo_connector->output_flag)
+		return true;
+	else
+		return false;
+}
+
+static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
+				    enum pipe *pipe)
+{
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
+	u32 tmp;
+
+	tmp = I915_READ(intel_sdvo->sdvo_reg);
+
+	if (!(tmp & SDVO_ENABLE))
+		return false;
+
+	if (HAS_PCH_CPT(dev))
+		*pipe = PORT_TO_PIPE_CPT(tmp);
+	else
+		*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
+static void intel_disable_sdvo(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+	struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
+	u32 temp;
+
+	intel_sdvo_set_active_outputs(intel_sdvo, 0);
+	if (0)
+		intel_sdvo_set_encoder_power_state(intel_sdvo,
+						   DRM_MODE_DPMS_OFF);
+
+	temp = I915_READ(intel_sdvo->sdvo_reg);
+	if ((temp & SDVO_ENABLE) != 0) {
+		intel_sdvo_write_sdvox(intel_sdvo, temp & ~SDVO_ENABLE);
+	}
+}
+
+static void intel_enable_sdvo(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	u32 temp;
+	bool input1, input2;
+	int i;
+	u8 status;
+
+	temp = I915_READ(intel_sdvo->sdvo_reg);
+	if ((temp & SDVO_ENABLE) == 0)
+		intel_sdvo_write_sdvox(intel_sdvo, temp | SDVO_ENABLE);
+	for (i = 0; i < 2; i++)
+		intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+	status = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2);
+	/* Warn if the device reported failure to sync.
+	 * A lot of SDVO devices fail to notify of sync, but it's
+	 * a given it the status is a success, we succeeded.
+	 */
+	if (status == SDVO_CMD_STATUS_SUCCESS && !input1) {
+		DRM_DEBUG_KMS("First %s output reported failure to "
+				"sync\n", SDVO_NAME(intel_sdvo));
+	}
+
+	if (0)
+		intel_sdvo_set_encoder_power_state(intel_sdvo,
+						   DRM_MODE_DPMS_ON);
+	intel_sdvo_set_active_outputs(intel_sdvo, intel_sdvo->attached_output);
+}
+
+static void intel_sdvo_dpms(struct drm_connector *connector, int mode)
+{
+	struct drm_crtc *crtc;
+	struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
+
+	/* dvo supports only 2 dpms states. */
+	if (mode != DRM_MODE_DPMS_ON)
+		mode = DRM_MODE_DPMS_OFF;
+
+	if (mode == connector->dpms)
+		return;
+
+	connector->dpms = mode;
+
+	/* Only need to change hw state when actually enabled */
+	crtc = intel_sdvo->base.base.crtc;
+	if (!crtc) {
+		intel_sdvo->base.connectors_active = false;
+		return;
+	}
 
 	if (mode != DRM_MODE_DPMS_ON) {
 		intel_sdvo_set_active_outputs(intel_sdvo, 0);
 		if (0)
 			intel_sdvo_set_encoder_power_state(intel_sdvo, mode);
 
-		if (mode == DRM_MODE_DPMS_OFF) {
-			temp = I915_READ(intel_sdvo->sdvo_reg);
-			if ((temp & SDVO_ENABLE) != 0) {
-				intel_sdvo_write_sdvox(intel_sdvo, temp & ~SDVO_ENABLE);
-			}
-		}
+		intel_sdvo->base.connectors_active = false;
+
+		intel_crtc_update_dpms(crtc);
 	} else {
-		bool input1, input2;
-		int i;
-		u8 status;
-
-		temp = I915_READ(intel_sdvo->sdvo_reg);
-		if ((temp & SDVO_ENABLE) == 0)
-			intel_sdvo_write_sdvox(intel_sdvo, temp | SDVO_ENABLE);
-		for (i = 0; i < 2; i++)
-			intel_wait_for_vblank(dev, intel_crtc->pipe);
-
-		status = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2);
-		/* Warn if the device reported failure to sync.
-		 * A lot of SDVO devices fail to notify of sync, but it's
-		 * a given it the status is a success, we succeeded.
-		 */
-		if (status == SDVO_CMD_STATUS_SUCCESS && !input1) {
-			DRM_DEBUG_KMS("First %s output reported failure to "
-					"sync\n", SDVO_NAME(intel_sdvo));
-		}
+		intel_sdvo->base.connectors_active = true;
+
+		intel_crtc_update_dpms(crtc);
 
 		if (0)
 			intel_sdvo_set_encoder_power_state(intel_sdvo, mode);
 		intel_sdvo_set_active_outputs(intel_sdvo, intel_sdvo->attached_output);
 	}
-	return;
+
+	intel_modeset_check_state(connector->dev);
 }
 
 static int intel_sdvo_mode_valid(struct drm_connector *connector,
@@ -1251,25 +1339,29 @@ static bool intel_sdvo_get_capabilities(struct intel_sdvo *intel_sdvo, struct in
 	return true;
 }
 
-static int intel_sdvo_supports_hotplug(struct intel_sdvo *intel_sdvo)
+static uint16_t intel_sdvo_get_hotplug_support(struct intel_sdvo *intel_sdvo)
 {
 	struct drm_device *dev = intel_sdvo->base.base.dev;
-	u8 response[2];
+	uint16_t hotplug;
 
 	/* HW Erratum: SDVO Hotplug is broken on all i945G chips, there's noise
 	 * on the line. */
 	if (IS_I945G(dev) || IS_I945GM(dev))
-		return false;
+		return 0;
 
-	return intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT,
-				    &response, 2) && response[0];
+	if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HOT_PLUG_SUPPORT,
+					&hotplug, sizeof(hotplug)))
+		return 0;
+
+	return hotplug;
 }
 
 static void intel_sdvo_enable_hotplug(struct intel_encoder *encoder)
 {
 	struct intel_sdvo *intel_sdvo = to_intel_sdvo(&encoder->base);
 
-	intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG, &intel_sdvo->hotplug_active, 2);
+	intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_ACTIVE_HOT_PLUG,
+			&intel_sdvo->hotplug_active, 2);
 }
 
 static bool
@@ -1345,7 +1437,6 @@ intel_sdvo_tmds_sink_detect(struct drm_connector *connector)
 			}
 		} else
 			status = connector_status_disconnected;
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 
@@ -1419,7 +1510,6 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
 			else
 				ret = connector_status_disconnected;
 
-			connector->display_info.raw_edid = NULL;
 			kfree(edid);
 		} else
 			ret = connector_status_connected;
@@ -1465,7 +1555,6 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector)
 			drm_add_edid_modes(connector, edid);
 		}
 
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 }
@@ -1837,8 +1926,8 @@ set_value:
 done:
 	if (intel_sdvo->base.base.crtc) {
 		struct drm_crtc *crtc = intel_sdvo->base.base.crtc;
-		drm_crtc_helper_set_mode(crtc, &crtc->mode, crtc->x,
-					 crtc->y, crtc->fb);
+		intel_set_mode(crtc, &crtc->mode,
+			       crtc->x, crtc->y, crtc->fb);
 	}
 
 	return 0;
@@ -1846,15 +1935,13 @@ done:
 }
 
 static const struct drm_encoder_helper_funcs intel_sdvo_helper_funcs = {
-	.dpms = intel_sdvo_dpms,
 	.mode_fixup = intel_sdvo_mode_fixup,
-	.prepare = intel_encoder_prepare,
 	.mode_set = intel_sdvo_mode_set,
-	.commit = intel_encoder_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_sdvo_dpms,
 	.detect = intel_sdvo_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
 	.set_property = intel_sdvo_set_property,
@@ -2026,6 +2113,7 @@ intel_sdvo_connector_init(struct intel_sdvo_connector *connector,
 	connector->base.base.interlace_allowed = 1;
 	connector->base.base.doublescan_allowed = 0;
 	connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB;
+	connector->base.get_hw_state = intel_sdvo_connector_get_hw_state;
 
 	intel_connector_attach_encoder(&connector->base, &encoder->base);
 	drm_sysfs_connector_add(&connector->base.base);
@@ -2064,17 +2152,18 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 
 	intel_connector = &intel_sdvo_connector->base;
 	connector = &intel_connector->base;
-	if (intel_sdvo_supports_hotplug(intel_sdvo) & (1 << device)) {
+	if (intel_sdvo_get_hotplug_support(intel_sdvo) &
+		intel_sdvo_connector->output_flag) {
 		connector->polled = DRM_CONNECTOR_POLL_HPD;
-		intel_sdvo->hotplug_active[0] |= 1 << device;
+		intel_sdvo->hotplug_active |= intel_sdvo_connector->output_flag;
 		/* Some SDVO devices have one-shot hotplug interrupts.
 		 * Ensure that they get re-enabled when an interrupt happens.
 		 */
 		intel_encoder->hot_plug = intel_sdvo_enable_hotplug;
 		intel_sdvo_enable_hotplug(intel_encoder);
-	}
-	else
+	} else {
 		connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
+	}
 	encoder->encoder_type = DRM_MODE_ENCODER_TMDS;
 	connector->connector_type = DRM_MODE_CONNECTOR_DVID;
 
@@ -2082,8 +2171,7 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
 		connector->connector_type = DRM_MODE_CONNECTOR_HDMIA;
 		intel_sdvo->is_hdmi = true;
 	}
-	intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
-				       (1 << INTEL_ANALOG_CLONE_BIT));
+	intel_sdvo->base.cloneable = true;
 
 	intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
 	if (intel_sdvo->is_hdmi)
@@ -2114,7 +2202,7 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type)
 
 	intel_sdvo->is_tv = true;
 	intel_sdvo->base.needs_tv_clock = true;
-	intel_sdvo->base.clone_mask = 1 << INTEL_SDVO_TV_CLONE_BIT;
+	intel_sdvo->base.cloneable = false;
 
 	intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
 
@@ -2157,8 +2245,7 @@ intel_sdvo_analog_init(struct intel_sdvo *intel_sdvo, int device)
 		intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1;
 	}
 
-	intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) |
-				       (1 << INTEL_ANALOG_CLONE_BIT));
+	intel_sdvo->base.cloneable = true;
 
 	intel_sdvo_connector_init(intel_sdvo_connector,
 				  intel_sdvo);
@@ -2190,8 +2277,10 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device)
 		intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1;
 	}
 
-	intel_sdvo->base.clone_mask = ((1 << INTEL_ANALOG_CLONE_BIT) |
-				       (1 << INTEL_SDVO_LVDS_CLONE_BIT));
+	/* SDVO LVDS is cloneable because the SDVO encoder does the upscaling,
+	 * as opposed to native LVDS, where we upscale with the panel-fitter
+	 * (and hence only the native LVDS resolution could be cloned). */
+	intel_sdvo->base.cloneable = true;
 
 	intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
 	if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
@@ -2576,6 +2665,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
 
 	drm_encoder_helper_add(&intel_encoder->base, &intel_sdvo_helper_funcs);
 
+	intel_encoder->disable = intel_disable_sdvo;
+	intel_encoder->enable = intel_enable_sdvo;
+	intel_encoder->get_hw_state = intel_sdvo_get_hw_state;
+
 	/* In default case sdvo lvds is false */
 	if (!intel_sdvo_get_capabilities(intel_sdvo, &intel_sdvo->caps))
 		goto err;
@@ -2590,7 +2683,7 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
 	/* Only enable the hotplug irq if we need it, to work around noisy
 	 * hotplug lines.
 	 */
-	if (intel_sdvo->hotplug_active[0])
+	if (intel_sdvo->hotplug_active)
 		dev_priv->hotplug_supported_mask |= hotplug_mask;
 
 	intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg);
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 7644f31a377..82f5e5c7009 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -29,11 +29,11 @@
  * registers; newer ones are much simpler and we can use the new DRM plane
  * support.
  */
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_fourcc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 static void
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index befce6c4970..62bb048c135 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -30,12 +30,11 @@
  * Integrated TV-out support for the 915GM and 945GM.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
 #include "intel_drv.h"
-#include "i915_drm.h"
+#include <drm/i915_drm.h>
 #include "i915_drv.h"
 
 enum tv_margin {
@@ -836,22 +835,37 @@ static struct intel_tv *intel_attached_tv(struct drm_connector *connector)
 			    base);
 }
 
+static bool
+intel_tv_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 tmp = I915_READ(TV_CTL);
+
+	if (!(tmp & TV_ENC_ENABLE))
+		return false;
+
+	*pipe = PORT_TO_PIPE(tmp);
+
+	return true;
+}
+
 static void
-intel_tv_dpms(struct drm_encoder *encoder, int mode)
+intel_enable_tv(struct intel_encoder *encoder)
 {
-	struct drm_device *dev = encoder->dev;
+	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE);
-		break;
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-	case DRM_MODE_DPMS_OFF:
-		I915_WRITE(TV_CTL, I915_READ(TV_CTL) & ~TV_ENC_ENABLE);
-		break;
-	}
+	I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE);
+}
+
+static void
+intel_disable_tv(struct intel_encoder *encoder)
+{
+	struct drm_device *dev = encoder->base.dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(TV_CTL, I915_READ(TV_CTL) & ~TV_ENC_ENABLE);
 }
 
 static const struct tv_mode *
@@ -895,17 +909,14 @@ intel_tv_mode_fixup(struct drm_encoder *encoder,
 		    const struct drm_display_mode *mode,
 		    struct drm_display_mode *adjusted_mode)
 {
-	struct drm_device *dev = encoder->dev;
 	struct intel_tv *intel_tv = enc_to_intel_tv(encoder);
 	const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
-	struct intel_encoder *other_encoder;
 
 	if (!tv_mode)
 		return false;
 
-	for_each_encoder_on_crtc(dev, encoder->crtc, other_encoder)
-		if (&other_encoder->base != encoder)
-			return false;
+	if (intel_encoder_check_is_cloned(&intel_tv->base))
+		return false;
 
 	adjusted_mode->clock = tv_mode->clock;
 	return true;
@@ -1303,12 +1314,9 @@ intel_tv_detect(struct drm_connector *connector, bool force)
 	if (force) {
 		struct intel_load_detect_pipe tmp;
 
-		if (intel_get_load_detect_pipe(&intel_tv->base, connector,
-					       &mode, &tmp)) {
+		if (intel_get_load_detect_pipe(connector, &mode, &tmp)) {
 			type = intel_tv_detect_type(intel_tv, connector);
-			intel_release_load_detect_pipe(&intel_tv->base,
-						       connector,
-						       &tmp);
+			intel_release_load_detect_pipe(connector, &tmp);
 		} else
 			return connector_status_unknown;
 	} else
@@ -1474,22 +1482,20 @@ intel_tv_set_property(struct drm_connector *connector, struct drm_property *prop
 	}
 
 	if (changed && crtc)
-		drm_crtc_helper_set_mode(crtc, &crtc->mode, crtc->x,
-				crtc->y, crtc->fb);
+		intel_set_mode(crtc, &crtc->mode,
+			       crtc->x, crtc->y, crtc->fb);
 out:
 	return ret;
 }
 
 static const struct drm_encoder_helper_funcs intel_tv_helper_funcs = {
-	.dpms = intel_tv_dpms,
 	.mode_fixup = intel_tv_mode_fixup,
-	.prepare = intel_encoder_prepare,
 	.mode_set = intel_tv_mode_set,
-	.commit = intel_encoder_commit,
+	.disable = intel_encoder_noop,
 };
 
 static const struct drm_connector_funcs intel_tv_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
+	.dpms = intel_connector_dpms,
 	.detect = intel_tv_detect,
 	.destroy = intel_tv_destroy,
 	.set_property = intel_tv_set_property,
@@ -1619,10 +1625,15 @@ intel_tv_init(struct drm_device *dev)
 	drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs,
 			 DRM_MODE_ENCODER_TVDAC);
 
+	intel_encoder->enable = intel_enable_tv;
+	intel_encoder->disable = intel_disable_tv;
+	intel_encoder->get_hw_state = intel_tv_get_hw_state;
+	intel_connector->get_hw_state = intel_connector_get_hw_state;
+
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
 	intel_encoder->type = INTEL_OUTPUT_TVOUT;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
-	intel_encoder->clone_mask = (1 << INTEL_TV_CLONE_BIT);
+	intel_encoder->cloneable = false;
 	intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1));
 	intel_encoder->base.possible_clones = (1 << INTEL_OUTPUT_TVOUT);
 	intel_tv->type = DRM_MODE_CONNECTOR_Unknown;
diff --git a/drivers/gpu/drm/mga/mga_dma.c b/drivers/gpu/drm/mga/mga_dma.c
index 507aa3df016..cc3166dd445 100644
--- a/drivers/gpu/drm/mga/mga_dma.c
+++ b/drivers/gpu/drm/mga/mga_dma.c
@@ -35,10 +35,8 @@
  * \author Gareth Hughes <gareth@valinux.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 #include "mga_drv.h"
 
 #define MGA_DEFAULT_USEC_TIMEOUT	10000
diff --git a/drivers/gpu/drm/mga/mga_drv.c b/drivers/gpu/drm/mga/mga_drv.c
index b1bb46de3f5..17d0a637e4f 100644
--- a/drivers/gpu/drm/mga/mga_drv.c
+++ b/drivers/gpu/drm/mga/mga_drv.c
@@ -31,12 +31,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 #include "mga_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static int mga_driver_device_is_agp(struct drm_device *dev);
 
diff --git a/drivers/gpu/drm/mga/mga_ioc32.c b/drivers/gpu/drm/mga/mga_ioc32.c
index c1f877b7bac..709e90db8c4 100644
--- a/drivers/gpu/drm/mga/mga_ioc32.c
+++ b/drivers/gpu/drm/mga/mga_ioc32.c
@@ -32,9 +32,8 @@
  */
 #include <linux/compat.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 
 typedef struct drm32_mga_init {
 	int func;
diff --git a/drivers/gpu/drm/mga/mga_irq.c b/drivers/gpu/drm/mga/mga_irq.c
index 2581202297e..598c281def0 100644
--- a/drivers/gpu/drm/mga/mga_irq.c
+++ b/drivers/gpu/drm/mga/mga_irq.c
@@ -31,9 +31,8 @@
  *    Eric Anholt <anholt@FreeBSD.org>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 #include "mga_drv.h"
 
 u32 mga_get_vblank_counter(struct drm_device *dev, int crtc)
diff --git a/drivers/gpu/drm/mga/mga_state.c b/drivers/gpu/drm/mga/mga_state.c
index 9ce2827f8c0..9c145143ad0 100644
--- a/drivers/gpu/drm/mga/mga_state.c
+++ b/drivers/gpu/drm/mga/mga_state.c
@@ -32,9 +32,8 @@
  *    Gareth Hughes <gareth@valinux.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 #include "mga_drv.h"
 
 /* ================================================================
diff --git a/drivers/gpu/drm/mga/mga_warp.c b/drivers/gpu/drm/mga/mga_warp.c
index 722a91b69b0..0b76352260a 100644
--- a/drivers/gpu/drm/mga/mga_warp.c
+++ b/drivers/gpu/drm/mga/mga_warp.c
@@ -32,9 +32,8 @@
 #include <linux/platform_device.h>
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "mga_drm.h"
+#include <drm/drmP.h>
+#include <drm/mga_drm.h>
 #include "mga_drv.h"
 
 #define FIRMWARE_G200 "matrox/g200_warp.fw"
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index e5f145d2cb3..1e910117b0a 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -10,12 +10,11 @@
  */
 #include <linux/module.h>
 #include <linux/console.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include "mgag200_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 /*
  * This is the generic driver code. This binds the driver to the drm core,
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 6f13b356323..5ea5033eae0 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -15,12 +15,12 @@
 
 #include <video/vga.h>
 
-#include "drm/drm_fb_helper.h"
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
+#include <drm/drm_fb_helper.h>
+#include <drm/ttm/ttm_bo_api.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_memory.h>
+#include <drm/ttm/ttm_module.h>
 
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
@@ -195,7 +195,6 @@ struct mga_device {
 		struct drm_global_reference mem_global_ref;
 		struct ttm_bo_global_ref bo_global_ref;
 		struct ttm_bo_device bdev;
-		atomic_t validate_sequence;
 	} ttm;
 
 	u32 reg_1e24; /* SE model number */
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index 880d3369760..2f486481d79 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -11,9 +11,8 @@
  *          Dave Airlie
  */
 #include <linux/module.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_fb_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_fb_helper.h>
 
 #include <linux/fb.h>
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_i2c.c b/drivers/gpu/drm/mgag200/mgag200_i2c.c
index dd3568a1b6b..5a88ec51b51 100644
--- a/drivers/gpu/drm/mgag200/mgag200_i2c.c
+++ b/drivers/gpu/drm/mgag200/mgag200_i2c.c
@@ -28,8 +28,7 @@
 #include <linux/export.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include "mgag200_drv.h"
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index 636a81cd2f3..d6a1aae3370 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -10,9 +10,8 @@
  *          Matt Turner
  *          Dave Airlie
  */
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 #include "mgag200_drv.h"
 
 static void mga_user_framebuffer_destroy(struct drm_framebuffer *fb)
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index b69642d5d85..d3d99a28dde 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -13,9 +13,8 @@
 
 #include <linux/delay.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "mgag200_drv.h"
 
@@ -1399,7 +1398,6 @@ static int mga_vga_get_modes(struct drm_connector *connector)
 	if (edid) {
 		drm_mode_connector_update_edid_property(connector, edid);
 		ret = drm_add_edid_modes(connector, edid);
-		connector->display_info.raw_edid = NULL;
 		kfree(edid);
 	}
 	return ret;
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index b223dcb7a71..1504699666c 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -25,7 +25,7 @@
 /*
  * Authors: Dave Airlie <airlied@redhat.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "mgag200_drv.h"
 #include <ttm/ttm_page_alloc.h>
 
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 97a81260485..8a55beeb8bd 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -17,6 +17,34 @@ config DRM_NOUVEAU
 	help
 	  Choose this option for open-source nVidia support.
 
+config NOUVEAU_DEBUG
+	int "Maximum debug level"
+	depends on DRM_NOUVEAU
+	range 0 7
+	default 5
+	help
+	  Selects the maximum debug level to compile support for.
+
+	  0 - fatal
+	  1 - error
+	  2 - warning
+	  3 - info
+	  4 - debug
+	  5 - trace (recommended)
+	  6 - paranoia
+	  7 - spam
+
+	  The paranoia and spam levels will add a lot of extra checks which
+	  may potentially slow down driver operation.
+
+config NOUVEAU_DEBUG_DEFAULT
+	int "Default debug level"
+	depends on DRM_NOUVEAU
+	range 0 7
+	default 3
+	help
+	  Selects the default debug level
+
 config DRM_NOUVEAU_BACKLIGHT
 	bool "Support for backlight control"
 	depends on DRM_NOUVEAU
@@ -25,14 +53,6 @@ config DRM_NOUVEAU_BACKLIGHT
 	  Say Y here if you want to control the backlight of your display
 	  (e.g. a laptop panel).
 
-config DRM_NOUVEAU_DEBUG
-	bool "Build in Nouveau's debugfs support"
-	depends on DRM_NOUVEAU && DEBUG_FS
-	default y
-	help
-	  Say Y here if you want Nouveau to output debugging information
-	  via debugfs.
-
 menu "I2C encoder or helper chips"
      depends on DRM && DRM_KMS_HELPER && I2C
 
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index 1cece6a78f3..a990df4d6c0 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -3,49 +3,190 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 ccflags-y := -Iinclude/drm
-nouveau-y := nouveau_drv.o nouveau_state.o nouveau_channel.o nouveau_mem.o \
-             nouveau_gpuobj.o nouveau_irq.o nouveau_notifier.o \
-             nouveau_sgdma.o nouveau_dma.o nouveau_util.o \
-             nouveau_bo.o nouveau_fence.o nouveau_gem.o nouveau_ttm.o \
-             nouveau_hw.o nouveau_calc.o nouveau_bios.o nouveau_i2c.o \
-             nouveau_display.o nouveau_connector.o nouveau_fbcon.o \
-             nouveau_hdmi.o nouveau_dp.o nouveau_ramht.o \
-	     nouveau_pm.o nouveau_volt.o nouveau_perf.o nouveau_temp.o \
-	     nouveau_mm.o nouveau_vm.o nouveau_mxm.o nouveau_gpio.o \
-	     nouveau_abi16.o \
-             nv04_timer.o \
-             nv04_mc.o nv40_mc.o nv50_mc.o \
-             nv04_fb.o nv10_fb.o nv20_fb.o nv30_fb.o nv40_fb.o \
-             nv50_fb.o nvc0_fb.o \
-             nv04_fifo.o nv10_fifo.o nv17_fifo.o nv40_fifo.o nv50_fifo.o \
-             nv84_fifo.o nvc0_fifo.o nve0_fifo.o \
-             nv04_fence.o nv10_fence.o nv84_fence.o nvc0_fence.o \
-             nv04_software.o nv50_software.o nvc0_software.o \
-             nv04_graph.o nv10_graph.o nv20_graph.o \
-             nv40_graph.o nv50_graph.o nvc0_graph.o nve0_graph.o \
-             nv40_grctx.o nv50_grctx.o nvc0_grctx.o nve0_grctx.o \
-             nv84_crypt.o nv98_crypt.o \
-             nva3_copy.o nvc0_copy.o \
-             nv31_mpeg.o nv50_mpeg.o \
-             nv84_bsp.o \
-             nv84_vp.o \
-             nv98_ppp.o \
-             nv04_instmem.o nv50_instmem.o nvc0_instmem.o \
-             nv04_dac.o nv04_dfp.o nv04_tv.o nv17_tv.o nv17_tv_modes.o \
-             nv04_crtc.o nv04_display.o nv04_cursor.o \
-             nv50_evo.o nv50_crtc.o nv50_dac.o nv50_sor.o \
-             nv50_cursor.o nv50_display.o \
-             nvd0_display.o \
-             nv04_fbcon.o nv50_fbcon.o nvc0_fbcon.o \
-             nv10_gpio.o nv50_gpio.o \
-	     nv50_calc.o \
-	     nv04_pm.o nv40_pm.o nv50_pm.o nva3_pm.o nvc0_pm.o \
-	     nv50_vram.o nvc0_vram.o \
-	     nv50_vm.o nvc0_vm.o nouveau_prime.o
-
-nouveau-$(CONFIG_DRM_NOUVEAU_DEBUG) += nouveau_debugfs.o
+ccflags-y += -I$(src)/core/include
+ccflags-y += -I$(src)/core
+ccflags-y += -I$(src)
+
+nouveau-y := core/core/client.o
+nouveau-y += core/core/engctx.o
+nouveau-y += core/core/engine.o
+nouveau-y += core/core/enum.o
+nouveau-y += core/core/gpuobj.o
+nouveau-y += core/core/handle.o
+nouveau-y += core/core/mm.o
+nouveau-y += core/core/namedb.o
+nouveau-y += core/core/object.o
+nouveau-y += core/core/option.o
+nouveau-y += core/core/parent.o
+nouveau-y += core/core/printk.o
+nouveau-y += core/core/ramht.o
+nouveau-y += core/core/subdev.o
+
+nouveau-y += core/subdev/bar/base.o
+nouveau-y += core/subdev/bar/nv50.o
+nouveau-y += core/subdev/bar/nvc0.o
+nouveau-y += core/subdev/bios/base.o
+nouveau-y += core/subdev/bios/bit.o
+nouveau-y += core/subdev/bios/conn.o
+nouveau-y += core/subdev/bios/dcb.o
+nouveau-y += core/subdev/bios/dp.o
+nouveau-y += core/subdev/bios/extdev.o
+nouveau-y += core/subdev/bios/gpio.o
+nouveau-y += core/subdev/bios/i2c.o
+nouveau-y += core/subdev/bios/init.o
+nouveau-y += core/subdev/bios/mxm.o
+nouveau-y += core/subdev/bios/perf.o
+nouveau-y += core/subdev/bios/pll.o
+nouveau-y += core/subdev/bios/therm.o
+nouveau-y += core/subdev/clock/nv04.o
+nouveau-y += core/subdev/clock/nv40.o
+nouveau-y += core/subdev/clock/nv50.o
+nouveau-y += core/subdev/clock/nva3.o
+nouveau-y += core/subdev/clock/nvc0.o
+nouveau-y += core/subdev/clock/pllnv04.o
+nouveau-y += core/subdev/clock/pllnva3.o
+nouveau-y += core/subdev/device/base.o
+nouveau-y += core/subdev/device/nv04.o
+nouveau-y += core/subdev/device/nv10.o
+nouveau-y += core/subdev/device/nv20.o
+nouveau-y += core/subdev/device/nv30.o
+nouveau-y += core/subdev/device/nv40.o
+nouveau-y += core/subdev/device/nv50.o
+nouveau-y += core/subdev/device/nvc0.o
+nouveau-y += core/subdev/device/nve0.o
+nouveau-y += core/subdev/devinit/base.o
+nouveau-y += core/subdev/devinit/nv04.o
+nouveau-y += core/subdev/devinit/nv05.o
+nouveau-y += core/subdev/devinit/nv10.o
+nouveau-y += core/subdev/devinit/nv1a.o
+nouveau-y += core/subdev/devinit/nv20.o
+nouveau-y += core/subdev/devinit/nv50.o
+nouveau-y += core/subdev/fb/base.o
+nouveau-y += core/subdev/fb/nv04.o
+nouveau-y += core/subdev/fb/nv10.o
+nouveau-y += core/subdev/fb/nv20.o
+nouveau-y += core/subdev/fb/nv30.o
+nouveau-y += core/subdev/fb/nv40.o
+nouveau-y += core/subdev/fb/nv50.o
+nouveau-y += core/subdev/fb/nvc0.o
+nouveau-y += core/subdev/gpio/base.o
+nouveau-y += core/subdev/gpio/nv10.o
+nouveau-y += core/subdev/gpio/nv50.o
+nouveau-y += core/subdev/gpio/nvd0.o
+nouveau-y += core/subdev/i2c/base.o
+nouveau-y += core/subdev/i2c/aux.o
+nouveau-y += core/subdev/i2c/bit.o
+nouveau-y += core/subdev/ibus/nvc0.o
+nouveau-y += core/subdev/ibus/nve0.o
+nouveau-y += core/subdev/instmem/base.o
+nouveau-y += core/subdev/instmem/nv04.o
+nouveau-y += core/subdev/instmem/nv40.o
+nouveau-y += core/subdev/instmem/nv50.o
+nouveau-y += core/subdev/ltcg/nvc0.o
+nouveau-y += core/subdev/mc/base.o
+nouveau-y += core/subdev/mc/nv04.o
+nouveau-y += core/subdev/mc/nv44.o
+nouveau-y += core/subdev/mc/nv50.o
+nouveau-y += core/subdev/mc/nv98.o
+nouveau-y += core/subdev/mc/nvc0.o
+nouveau-y += core/subdev/mxm/base.o
+nouveau-y += core/subdev/mxm/mxms.o
+nouveau-y += core/subdev/mxm/nv50.o
+nouveau-y += core/subdev/therm/base.o
+nouveau-y += core/subdev/therm/fan.o
+nouveau-y += core/subdev/therm/ic.o
+nouveau-y += core/subdev/therm/nv40.o
+nouveau-y += core/subdev/therm/nv50.o
+nouveau-y += core/subdev/therm/temp.o
+nouveau-y += core/subdev/timer/base.o
+nouveau-y += core/subdev/timer/nv04.o
+nouveau-y += core/subdev/vm/base.o
+nouveau-y += core/subdev/vm/nv04.o
+nouveau-y += core/subdev/vm/nv41.o
+nouveau-y += core/subdev/vm/nv44.o
+nouveau-y += core/subdev/vm/nv50.o
+nouveau-y += core/subdev/vm/nvc0.o
+
+nouveau-y += core/engine/dmaobj/base.o
+nouveau-y += core/engine/dmaobj/nv04.o
+nouveau-y += core/engine/dmaobj/nv50.o
+nouveau-y += core/engine/dmaobj/nvc0.o
+nouveau-y += core/engine/bsp/nv84.o
+nouveau-y += core/engine/copy/nva3.o
+nouveau-y += core/engine/copy/nvc0.o
+nouveau-y += core/engine/copy/nve0.o
+nouveau-y += core/engine/crypt/nv84.o
+nouveau-y += core/engine/crypt/nv98.o
+nouveau-y += core/engine/disp/nv04.o
+nouveau-y += core/engine/disp/nv50.o
+nouveau-y += core/engine/disp/nvd0.o
+nouveau-y += core/engine/disp/vga.o
+nouveau-y += core/engine/fifo/base.o
+nouveau-y += core/engine/fifo/nv04.o
+nouveau-y += core/engine/fifo/nv10.o
+nouveau-y += core/engine/fifo/nv17.o
+nouveau-y += core/engine/fifo/nv40.o
+nouveau-y += core/engine/fifo/nv50.o
+nouveau-y += core/engine/fifo/nv84.o
+nouveau-y += core/engine/fifo/nvc0.o
+nouveau-y += core/engine/fifo/nve0.o
+nouveau-y += core/engine/graph/ctxnv40.o
+nouveau-y += core/engine/graph/ctxnv50.o
+nouveau-y += core/engine/graph/ctxnvc0.o
+nouveau-y += core/engine/graph/ctxnve0.o
+nouveau-y += core/engine/graph/nv04.o
+nouveau-y += core/engine/graph/nv10.o
+nouveau-y += core/engine/graph/nv20.o
+nouveau-y += core/engine/graph/nv25.o
+nouveau-y += core/engine/graph/nv2a.o
+nouveau-y += core/engine/graph/nv30.o
+nouveau-y += core/engine/graph/nv34.o
+nouveau-y += core/engine/graph/nv35.o
+nouveau-y += core/engine/graph/nv40.o
+nouveau-y += core/engine/graph/nv50.o
+nouveau-y += core/engine/graph/nvc0.o
+nouveau-y += core/engine/graph/nve0.o
+nouveau-y += core/engine/mpeg/nv31.o
+nouveau-y += core/engine/mpeg/nv40.o
+nouveau-y += core/engine/mpeg/nv50.o
+nouveau-y += core/engine/mpeg/nv84.o
+nouveau-y += core/engine/ppp/nv98.o
+nouveau-y += core/engine/software/nv04.o
+nouveau-y += core/engine/software/nv10.o
+nouveau-y += core/engine/software/nv50.o
+nouveau-y += core/engine/software/nvc0.o
+nouveau-y += core/engine/vp/nv84.o
+
+# drm/core
+nouveau-y += nouveau_drm.o nouveau_chan.o nouveau_dma.o nouveau_fence.o
+nouveau-y += nouveau_irq.o nouveau_vga.o nouveau_agp.o
+nouveau-y += nouveau_ttm.o nouveau_sgdma.o nouveau_bo.o nouveau_gem.o
+nouveau-y += nouveau_prime.o nouveau_abi16.o
+nouveau-y += nv04_fence.o nv10_fence.o nv50_fence.o nv84_fence.o nvc0_fence.o
+
+# drm/kms
+nouveau-y += nouveau_bios.o nouveau_fbcon.o nouveau_display.o
+nouveau-y += nouveau_connector.o nouveau_hdmi.o nouveau_dp.o
+nouveau-y += nv04_fbcon.o nv50_fbcon.o nvc0_fbcon.o
+
+# drm/kms/nv04:nv50
+nouveau-y += nouveau_hw.o nouveau_calc.o
+nouveau-y += nv04_dac.o nv04_dfp.o nv04_tv.o nv17_tv.o nv17_tv_modes.o
+nouveau-y += nv04_crtc.o nv04_display.o nv04_cursor.o
+
+# drm/kms/nv50-
+nouveau-y += nv50_display.o nvd0_display.o
+nouveau-y += nv50_crtc.o nv50_dac.o nv50_sor.o nv50_cursor.o
+nouveau-y += nv50_evo.o
+
+# drm/pm
+nouveau-y += nouveau_pm.o nouveau_volt.o nouveau_perf.o
+nouveau-y += nv04_pm.o nv40_pm.o nv50_pm.o nva3_pm.o nvc0_pm.o
+nouveau-y += nouveau_mem.o
+
+# other random bits
 nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
-nouveau-$(CONFIG_DRM_NOUVEAU_BACKLIGHT) += nouveau_backlight.o
 nouveau-$(CONFIG_ACPI) += nouveau_acpi.o
+nouveau-$(CONFIG_DRM_NOUVEAU_BACKLIGHT) += nouveau_backlight.o
 
 obj-$(CONFIG_DRM_NOUVEAU)+= nouveau.o
diff --git a/drivers/gpu/drm/nouveau/core/core/client.c b/drivers/gpu/drm/nouveau/core/core/client.c
new file mode 100644
index 00000000000..c617f048007
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/client.c
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/client.h>
+#include <core/handle.h>
+#include <core/option.h>
+
+#include <subdev/device.h>
+
+static void
+nouveau_client_dtor(struct nouveau_object *object)
+{
+	struct nouveau_client *client = (void *)object;
+	nouveau_object_ref(NULL, &client->device);
+	nouveau_handle_destroy(client->root);
+	nouveau_namedb_destroy(&client->base);
+}
+
+static struct nouveau_oclass
+nouveau_client_oclass = {
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.dtor = nouveau_client_dtor,
+	},
+};
+
+int
+nouveau_client_create_(const char *name, u64 devname, const char *cfg,
+		       const char *dbg, int length, void **pobject)
+{
+	struct nouveau_object *device;
+	struct nouveau_client *client;
+	int ret;
+
+	device = (void *)nouveau_device_find(devname);
+	if (!device)
+		return -ENODEV;
+
+	ret = nouveau_namedb_create_(NULL, NULL, &nouveau_client_oclass,
+				     NV_CLIENT_CLASS, nouveau_device_sclass,
+				     0, length, pobject);
+	client = *pobject;
+	if (ret)
+		return ret;
+
+	ret = nouveau_handle_create(nv_object(client), ~0, ~0,
+				    nv_object(client), &client->root);
+	if (ret) {
+		nouveau_namedb_destroy(&client->base);
+		return ret;
+	}
+
+	/* prevent init/fini being called, os in in charge of this */
+	atomic_set(&nv_object(client)->usecount, 2);
+
+	nouveau_object_ref(device, &client->device);
+	snprintf(client->name, sizeof(client->name), "%s", name);
+	client->debug = nouveau_dbgopt(dbg, "CLIENT");
+	return 0;
+}
+
+int
+nouveau_client_init(struct nouveau_client *client)
+{
+	int ret;
+	nv_debug(client, "init running\n");
+	ret = nouveau_handle_init(client->root);
+	nv_debug(client, "init completed with %d\n", ret);
+	return ret;
+}
+
+int
+nouveau_client_fini(struct nouveau_client *client, bool suspend)
+{
+	const char *name[2] = { "fini", "suspend" };
+	int ret;
+
+	nv_debug(client, "%s running\n", name[suspend]);
+	ret = nouveau_handle_fini(client->root, suspend);
+	nv_debug(client, "%s completed with %d\n", name[suspend], ret);
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/engctx.c b/drivers/gpu/drm/nouveau/core/core/engctx.c
new file mode 100644
index 00000000000..e41b10d5eb5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/engctx.c
@@ -0,0 +1,236 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/namedb.h>
+#include <core/handle.h>
+#include <core/client.h>
+#include <core/engctx.h>
+
+#include <subdev/vm.h>
+
+static inline int
+nouveau_engctx_exists(struct nouveau_object *parent,
+		      struct nouveau_engine *engine, void **pobject)
+{
+	struct nouveau_engctx *engctx;
+	struct nouveau_object *parctx;
+
+	list_for_each_entry(engctx, &engine->contexts, head) {
+		parctx = nv_pclass(nv_object(engctx), NV_PARENT_CLASS);
+		if (parctx == parent) {
+			atomic_inc(&nv_object(engctx)->refcount);
+			*pobject = engctx;
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+int
+nouveau_engctx_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engobj,
+		       struct nouveau_oclass *oclass,
+		       struct nouveau_object *pargpu,
+		       u32 size, u32 align, u32 flags,
+		       int length, void **pobject)
+{
+	struct nouveau_client *client = nouveau_client(parent);
+	struct nouveau_engine *engine = nv_engine(engobj);
+	struct nouveau_object *engctx;
+	unsigned long save;
+	int ret;
+
+	/* check if this engine already has a context for the parent object,
+	 * and reference it instead of creating a new one
+	 */
+	spin_lock_irqsave(&engine->lock, save);
+	ret = nouveau_engctx_exists(parent, engine, pobject);
+	spin_unlock_irqrestore(&engine->lock, save);
+	if (ret)
+		return ret;
+
+	/* create the new context, supports creating both raw objects and
+	 * objects backed by instance memory
+	 */
+	if (size) {
+		ret = nouveau_gpuobj_create_(parent, engobj, oclass,
+					     NV_ENGCTX_CLASS,
+					     pargpu, size, align, flags,
+					     length, pobject);
+	} else {
+		ret = nouveau_object_create_(parent, engobj, oclass,
+					     NV_ENGCTX_CLASS, length, pobject);
+	}
+
+	engctx = *pobject;
+	if (ret)
+		return ret;
+
+	/* must take the lock again and re-check a context doesn't already
+	 * exist (in case of a race) - the lock had to be dropped before as
+	 * it's not possible to allocate the object with it held.
+	 */
+	spin_lock_irqsave(&engine->lock, save);
+	ret = nouveau_engctx_exists(parent, engine, pobject);
+	if (ret) {
+		spin_unlock_irqrestore(&engine->lock, save);
+		nouveau_object_ref(NULL, &engctx);
+		return ret;
+	}
+
+	if (client->vm)
+		atomic_inc(&client->vm->engref[nv_engidx(engobj)]);
+	list_add(&nv_engctx(engctx)->head, &engine->contexts);
+	nv_engctx(engctx)->addr = ~0ULL;
+	spin_unlock_irqrestore(&engine->lock, save);
+	return 0;
+}
+
+void
+nouveau_engctx_destroy(struct nouveau_engctx *engctx)
+{
+	struct nouveau_object *engobj = nv_object(engctx)->engine;
+	struct nouveau_engine *engine = nv_engine(engobj);
+	struct nouveau_client *client = nouveau_client(engctx);
+	unsigned long save;
+
+	nouveau_gpuobj_unmap(&engctx->vma);
+	spin_lock_irqsave(&engine->lock, save);
+	list_del(&engctx->head);
+	spin_unlock_irqrestore(&engine->lock, save);
+
+	if (client->vm)
+		atomic_dec(&client->vm->engref[nv_engidx(engobj)]);
+
+	if (engctx->base.size)
+		nouveau_gpuobj_destroy(&engctx->base);
+	else
+		nouveau_object_destroy(&engctx->base.base);
+}
+
+int
+nouveau_engctx_init(struct nouveau_engctx *engctx)
+{
+	struct nouveau_object *object = nv_object(engctx);
+	struct nouveau_subdev *subdev = nv_subdev(object->engine);
+	struct nouveau_object *parent;
+	struct nouveau_subdev *pardev;
+	int ret;
+
+	ret = nouveau_gpuobj_init(&engctx->base);
+	if (ret)
+		return ret;
+
+	parent = nv_pclass(object->parent, NV_PARENT_CLASS);
+	pardev = nv_subdev(parent->engine);
+	if (nv_parent(parent)->context_attach) {
+		mutex_lock(&pardev->mutex);
+		ret = nv_parent(parent)->context_attach(parent, object);
+		mutex_unlock(&pardev->mutex);
+	}
+
+	if (ret) {
+		nv_error(parent, "failed to attach %s context, %d\n",
+			 subdev->name, ret);
+		return ret;
+	}
+
+	nv_debug(parent, "attached %s context\n", subdev->name);
+	return 0;
+}
+
+int
+nouveau_engctx_fini(struct nouveau_engctx *engctx, bool suspend)
+{
+	struct nouveau_object *object = nv_object(engctx);
+	struct nouveau_subdev *subdev = nv_subdev(object->engine);
+	struct nouveau_object *parent;
+	struct nouveau_subdev *pardev;
+	int ret = 0;
+
+	parent = nv_pclass(object->parent, NV_PARENT_CLASS);
+	pardev = nv_subdev(parent->engine);
+	if (nv_parent(parent)->context_detach) {
+		mutex_lock(&pardev->mutex);
+		ret = nv_parent(parent)->context_detach(parent, suspend, object);
+		mutex_unlock(&pardev->mutex);
+	}
+
+	if (ret) {
+		nv_error(parent, "failed to detach %s context, %d\n",
+			 subdev->name, ret);
+		return ret;
+	}
+
+	nv_debug(parent, "detached %s context\n", subdev->name);
+	return nouveau_gpuobj_fini(&engctx->base, suspend);
+}
+
+void
+_nouveau_engctx_dtor(struct nouveau_object *object)
+{
+	nouveau_engctx_destroy(nv_engctx(object));
+}
+
+int
+_nouveau_engctx_init(struct nouveau_object *object)
+{
+	return nouveau_engctx_init(nv_engctx(object));
+}
+
+
+int
+_nouveau_engctx_fini(struct nouveau_object *object, bool suspend)
+{
+	return nouveau_engctx_fini(nv_engctx(object), suspend);
+}
+
+struct nouveau_object *
+nouveau_engctx_get(struct nouveau_engine *engine, u64 addr)
+{
+	struct nouveau_engctx *engctx;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->lock, flags);
+	list_for_each_entry(engctx, &engine->contexts, head) {
+		if (engctx->addr == addr) {
+			engctx->save = flags;
+			return nv_object(engctx);
+		}
+	}
+	spin_unlock_irqrestore(&engine->lock, flags);
+	return NULL;
+}
+
+void
+nouveau_engctx_put(struct nouveau_object *object)
+{
+	if (object) {
+		struct nouveau_engine *engine = nv_engine(object->engine);
+		struct nouveau_engctx *engctx = nv_engctx(object);
+		spin_unlock_irqrestore(&engine->lock, engctx->save);
+	}
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/engine.c b/drivers/gpu/drm/nouveau/core/core/engine.c
new file mode 100644
index 00000000000..09b3bd502fd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/engine.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/device.h>
+#include <core/engine.h>
+#include <core/option.h>
+
+int
+nouveau_engine_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engobj,
+		       struct nouveau_oclass *oclass, bool enable,
+		       const char *iname, const char *fname,
+		       int length, void **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_engine *engine;
+	int ret;
+
+	ret = nouveau_subdev_create_(parent, engobj, oclass, NV_ENGINE_CLASS,
+				     iname, fname, length, pobject);
+	engine = *pobject;
+	if (ret)
+		return ret;
+
+	if (!nouveau_boolopt(device->cfgopt, iname, enable)) {
+		if (!enable)
+			nv_warn(engine, "disabled, %s=1 to enable\n", iname);
+		return -ENODEV;
+	}
+
+	INIT_LIST_HEAD(&engine->contexts);
+	spin_lock_init(&engine->lock);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.c b/drivers/gpu/drm/nouveau/core/core/enum.c
index e51b51503ba..7cc7133d82d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_util.c
+++ b/drivers/gpu/drm/nouveau/core/core/enum.c
@@ -25,27 +25,8 @@
  *
  */
 
-#include <linux/ratelimit.h>
-
-#include "nouveau_util.h"
-
-static DEFINE_RATELIMIT_STATE(nouveau_ratelimit_state, 3 * HZ, 20);
-
-void
-nouveau_bitfield_print(const struct nouveau_bitfield *bf, u32 value)
-{
-	while (bf->name) {
-		if (value & bf->mask) {
-			printk(" %s", bf->name);
-			value &= ~bf->mask;
-		}
-
-		bf++;
-	}
-
-	if (value)
-		printk(" (unknown bits 0x%08x)", value);
-}
+#include <core/os.h>
+#include <core/enum.h>
 
 const struct nouveau_enum *
 nouveau_enum_find(const struct nouveau_enum *en, u32 value)
@@ -63,16 +44,24 @@ void
 nouveau_enum_print(const struct nouveau_enum *en, u32 value)
 {
 	en = nouveau_enum_find(en, value);
-	if (en) {
+	if (en)
 		printk("%s", en->name);
-		return;
-	}
-
-	printk("(unknown enum 0x%08x)", value);
+	else
+		printk("(unknown enum 0x%08x)", value);
 }
 
-int
-nouveau_ratelimit(void)
+void
+nouveau_bitfield_print(const struct nouveau_bitfield *bf, u32 value)
 {
-	return __ratelimit(&nouveau_ratelimit_state);
+	while (bf->name) {
+		if (value & bf->mask) {
+			printk(" %s", bf->name);
+			value &= ~bf->mask;
+		}
+
+		bf++;
+	}
+
+	if (value)
+		printk(" (unknown bits 0x%08x)", value);
 }
diff --git a/drivers/gpu/drm/nouveau/core/core/gpuobj.c b/drivers/gpu/drm/nouveau/core/core/gpuobj.c
new file mode 100644
index 00000000000..1f34549aff1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/gpuobj.c
@@ -0,0 +1,318 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/gpuobj.h>
+
+#include <subdev/instmem.h>
+#include <subdev/bar.h>
+#include <subdev/vm.h>
+
+void
+nouveau_gpuobj_destroy(struct nouveau_gpuobj *gpuobj)
+{
+	int i;
+
+	if (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE) {
+		for (i = 0; i < gpuobj->size; i += 4)
+			nv_wo32(gpuobj, i, 0x00000000);
+	}
+
+	if (gpuobj->heap.block_size)
+		nouveau_mm_fini(&gpuobj->heap);
+
+	nouveau_object_destroy(&gpuobj->base);
+}
+
+int
+nouveau_gpuobj_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, u32 pclass,
+		       struct nouveau_object *pargpu,
+		       u32 size, u32 align, u32 flags,
+		       int length, void **pobject)
+{
+	struct nouveau_instmem *imem = nouveau_instmem(parent);
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nouveau_gpuobj *gpuobj;
+	struct nouveau_mm *heap = NULL;
+	int ret, i;
+	u64 addr;
+
+	*pobject = NULL;
+
+	if (pargpu) {
+		while ((pargpu = nv_pclass(pargpu, NV_GPUOBJ_CLASS))) {
+			if (nv_gpuobj(pargpu)->heap.block_size)
+				break;
+			pargpu = pargpu->parent;
+		}
+
+		if (unlikely(pargpu == NULL)) {
+			nv_error(parent, "no gpuobj heap\n");
+			return -EINVAL;
+		}
+
+		addr =  nv_gpuobj(pargpu)->addr;
+		heap = &nv_gpuobj(pargpu)->heap;
+		atomic_inc(&parent->refcount);
+	} else {
+		ret = imem->alloc(imem, parent, size, align, &parent);
+		pargpu = parent;
+		if (ret)
+			return ret;
+
+		addr = nv_memobj(pargpu)->addr;
+		size = nv_memobj(pargpu)->size;
+
+		if (bar && bar->alloc) {
+			struct nouveau_instobj *iobj = (void *)parent;
+			struct nouveau_mem **mem = (void *)(iobj + 1);
+			struct nouveau_mem *node = *mem;
+			if (!bar->alloc(bar, parent, node, &pargpu)) {
+				nouveau_object_ref(NULL, &parent);
+				parent = pargpu;
+			}
+		}
+	}
+
+	ret = nouveau_object_create_(parent, engine, oclass, pclass |
+				     NV_GPUOBJ_CLASS, length, pobject);
+	nouveau_object_ref(NULL, &parent);
+	gpuobj = *pobject;
+	if (ret)
+		return ret;
+
+	gpuobj->parent = pargpu;
+	gpuobj->flags = flags;
+	gpuobj->addr = addr;
+	gpuobj->size = size;
+
+	if (heap) {
+		ret = nouveau_mm_head(heap, 1, size, size,
+				      max(align, (u32)1), &gpuobj->node);
+		if (ret)
+			return ret;
+
+		gpuobj->addr += gpuobj->node->offset;
+	}
+
+	if (gpuobj->flags & NVOBJ_FLAG_HEAP) {
+		ret = nouveau_mm_init(&gpuobj->heap, 0, gpuobj->size, 1);
+		if (ret)
+			return ret;
+	}
+
+	if (flags & NVOBJ_FLAG_ZERO_ALLOC) {
+		for (i = 0; i < gpuobj->size; i += 4)
+			nv_wo32(gpuobj, i, 0x00000000);
+	}
+
+	return ret;
+}
+
+struct nouveau_gpuobj_class {
+	struct nouveau_object *pargpu;
+	u64 size;
+	u32 align;
+	u32 flags;
+};
+
+static int
+_nouveau_gpuobj_ctor(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, void *data, u32 size,
+		     struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj_class *args = data;
+	struct nouveau_gpuobj *object;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, args->pargpu,
+				    args->size, args->align, args->flags,
+				    &object);
+	*pobject = nv_object(object);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void
+_nouveau_gpuobj_dtor(struct nouveau_object *object)
+{
+	nouveau_gpuobj_destroy(nv_gpuobj(object));
+}
+
+int
+_nouveau_gpuobj_init(struct nouveau_object *object)
+{
+	return nouveau_gpuobj_init(nv_gpuobj(object));
+}
+
+int
+_nouveau_gpuobj_fini(struct nouveau_object *object, bool suspend)
+{
+	return nouveau_gpuobj_fini(nv_gpuobj(object), suspend);
+}
+
+u32
+_nouveau_gpuobj_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_gpuobj *gpuobj = nv_gpuobj(object);
+	struct nouveau_ofuncs *pfuncs = nv_ofuncs(gpuobj->parent);
+	if (gpuobj->node)
+		addr += gpuobj->node->offset;
+	return pfuncs->rd32(gpuobj->parent, addr);
+}
+
+void
+_nouveau_gpuobj_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nouveau_gpuobj *gpuobj = nv_gpuobj(object);
+	struct nouveau_ofuncs *pfuncs = nv_ofuncs(gpuobj->parent);
+	if (gpuobj->node)
+		addr += gpuobj->node->offset;
+	pfuncs->wr32(gpuobj->parent, addr, data);
+}
+
+static struct nouveau_oclass
+_nouveau_gpuobj_oclass = {
+	.handle = 0x00000000,
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = _nouveau_gpuobj_ctor,
+		.dtor = _nouveau_gpuobj_dtor,
+		.init = _nouveau_gpuobj_init,
+		.fini = _nouveau_gpuobj_fini,
+		.rd32 = _nouveau_gpuobj_rd32,
+		.wr32 = _nouveau_gpuobj_wr32,
+	},
+};
+
+int
+nouveau_gpuobj_new(struct nouveau_object *parent, struct nouveau_object *pargpu,
+		   u32 size, u32 align, u32 flags,
+		   struct nouveau_gpuobj **pgpuobj)
+{
+	struct nouveau_object *engine = parent;
+	struct nouveau_gpuobj_class args = {
+		.pargpu = pargpu,
+		.size = size,
+		.align = align,
+		.flags = flags,
+	};
+
+	if (!nv_iclass(engine, NV_SUBDEV_CLASS))
+		engine = engine->engine;
+	BUG_ON(engine == NULL);
+
+	return nouveau_object_ctor(parent, engine, &_nouveau_gpuobj_oclass,
+				   &args, sizeof(args),
+				   (struct nouveau_object **)pgpuobj);
+}
+
+int
+nouveau_gpuobj_map(struct nouveau_gpuobj *gpuobj, u32 access,
+		   struct nouveau_vma *vma)
+{
+	struct nouveau_bar *bar = nouveau_bar(gpuobj);
+	int ret = -EINVAL;
+
+	if (bar && bar->umap) {
+		struct nouveau_instobj *iobj = (void *)
+			nv_pclass(nv_object(gpuobj), NV_MEMOBJ_CLASS);
+		struct nouveau_mem **mem = (void *)(iobj + 1);
+		ret = bar->umap(bar, *mem, access, vma);
+	}
+
+	return ret;
+}
+
+int
+nouveau_gpuobj_map_vm(struct nouveau_gpuobj *gpuobj, struct nouveau_vm *vm,
+		      u32 access, struct nouveau_vma *vma)
+{
+	struct nouveau_instobj *iobj = (void *)
+		nv_pclass(nv_object(gpuobj), NV_MEMOBJ_CLASS);
+	struct nouveau_mem **mem = (void *)(iobj + 1);
+	int ret;
+
+	ret = nouveau_vm_get(vm, gpuobj->size, 12, access, vma);
+	if (ret)
+		return ret;
+
+	nouveau_vm_map(vma, *mem);
+	return 0;
+}
+
+void
+nouveau_gpuobj_unmap(struct nouveau_vma *vma)
+{
+	if (vma->node) {
+		nouveau_vm_unmap(vma);
+		nouveau_vm_put(vma);
+	}
+}
+
+/* the below is basically only here to support sharing the paged dma object
+ * for PCI(E)GART on <=nv4x chipsets, and should *not* be expected to work
+ * anywhere else.
+ */
+
+static void
+nouveau_gpudup_dtor(struct nouveau_object *object)
+{
+	struct nouveau_gpuobj *gpuobj = (void *)object;
+	nouveau_object_ref(NULL, &gpuobj->parent);
+	nouveau_object_destroy(&gpuobj->base);
+}
+
+static struct nouveau_oclass
+nouveau_gpudup_oclass = {
+	.handle = NV_GPUOBJ_CLASS,
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.dtor = nouveau_gpudup_dtor,
+		.init = nouveau_object_init,
+		.fini = nouveau_object_fini,
+	},
+};
+
+int
+nouveau_gpuobj_dup(struct nouveau_object *parent, struct nouveau_gpuobj *base,
+		   struct nouveau_gpuobj **pgpuobj)
+{
+	struct nouveau_gpuobj *gpuobj;
+	int ret;
+
+	ret = nouveau_object_create(parent, parent->engine,
+				   &nouveau_gpudup_oclass, 0, &gpuobj);
+	*pgpuobj = gpuobj;
+	if (ret)
+		return ret;
+
+	nouveau_object_ref(nv_object(base), &gpuobj->parent);
+	gpuobj->addr = base->addr;
+	gpuobj->size = base->size;
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/handle.c b/drivers/gpu/drm/nouveau/core/core/handle.c
new file mode 100644
index 00000000000..b8d2cbf8a7a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/handle.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/handle.h>
+#include <core/client.h>
+
+#define hprintk(h,l,f,a...) do {                                               \
+	struct nouveau_client *c = nouveau_client((h)->object);                \
+	struct nouveau_handle *p = (h)->parent; u32 n = p ? p->name : ~0;      \
+	nv_printk((c), l, "0x%08x:0x%08x "f, n, (h)->name, ##a);               \
+} while(0)
+
+int
+nouveau_handle_init(struct nouveau_handle *handle)
+{
+	struct nouveau_handle *item;
+	int ret;
+
+	hprintk(handle, TRACE, "init running\n");
+	ret = nouveau_object_inc(handle->object);
+	if (ret)
+		return ret;
+
+	hprintk(handle, TRACE, "init children\n");
+	list_for_each_entry(item, &handle->tree, head) {
+		ret = nouveau_handle_init(item);
+		if (ret)
+			goto fail;
+	}
+
+	hprintk(handle, TRACE, "init completed\n");
+	return 0;
+fail:
+	hprintk(handle, ERROR, "init failed with %d\n", ret);
+	list_for_each_entry_continue_reverse(item, &handle->tree, head) {
+		nouveau_handle_fini(item, false);
+	}
+
+	nouveau_object_dec(handle->object, false);
+	return ret;
+}
+
+int
+nouveau_handle_fini(struct nouveau_handle *handle, bool suspend)
+{
+	static char *name[2] = { "fini", "suspend" };
+	struct nouveau_handle *item;
+	int ret;
+
+	hprintk(handle, TRACE, "%s children\n", name[suspend]);
+	list_for_each_entry(item, &handle->tree, head) {
+		ret = nouveau_handle_fini(item, suspend);
+		if (ret && suspend)
+			goto fail;
+	}
+
+	hprintk(handle, TRACE, "%s running\n", name[suspend]);
+	if (handle->object) {
+		ret = nouveau_object_dec(handle->object, suspend);
+		if (ret && suspend)
+			goto fail;
+	}
+
+	hprintk(handle, TRACE, "%s completed\n", name[suspend]);
+	return 0;
+fail:
+	hprintk(handle, ERROR, "%s failed with %d\n", name[suspend], ret);
+	list_for_each_entry_continue_reverse(item, &handle->tree, head) {
+		int rret = nouveau_handle_init(item);
+		if (rret)
+			hprintk(handle, FATAL, "failed to restart, %d\n", rret);
+	}
+
+	return ret;
+}
+
+int
+nouveau_handle_create(struct nouveau_object *parent, u32 _parent, u32 _handle,
+		      struct nouveau_object *object,
+		      struct nouveau_handle **phandle)
+{
+	struct nouveau_object *namedb;
+	struct nouveau_handle *handle;
+	int ret;
+
+	namedb = parent;
+	while (!nv_iclass(namedb, NV_NAMEDB_CLASS))
+		namedb = namedb->parent;
+
+	handle = *phandle = kzalloc(sizeof(*handle), GFP_KERNEL);
+	if (!handle)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&handle->head);
+	INIT_LIST_HEAD(&handle->tree);
+	handle->name = _handle;
+	handle->priv = ~0;
+
+	ret = nouveau_namedb_insert(nv_namedb(namedb), _handle, object, handle);
+	if (ret) {
+		kfree(handle);
+		return ret;
+	}
+
+	if (nv_parent(parent)->object_attach) {
+		ret = nv_parent(parent)->object_attach(parent, object, _handle);
+		if (ret < 0) {
+			nouveau_handle_destroy(handle);
+			return ret;
+		}
+
+		handle->priv = ret;
+	}
+
+	if (object != namedb) {
+		while (!nv_iclass(namedb, NV_CLIENT_CLASS))
+			namedb = namedb->parent;
+
+		handle->parent = nouveau_namedb_get(nv_namedb(namedb), _parent);
+		if (handle->parent) {
+			list_add(&handle->head, &handle->parent->tree);
+			nouveau_namedb_put(handle->parent);
+		}
+	}
+
+	hprintk(handle, TRACE, "created\n");
+	return 0;
+}
+
+void
+nouveau_handle_destroy(struct nouveau_handle *handle)
+{
+	struct nouveau_handle *item, *temp;
+
+	hprintk(handle, TRACE, "destroy running\n");
+	list_for_each_entry_safe(item, temp, &handle->tree, head) {
+		nouveau_handle_destroy(item);
+	}
+	list_del(&handle->head);
+
+	if (handle->priv != ~0) {
+		struct nouveau_object *parent = handle->parent->object;
+		nv_parent(parent)->object_detach(parent, handle->priv);
+	}
+
+	hprintk(handle, TRACE, "destroy completed\n");
+	nouveau_namedb_remove(handle);
+	kfree(handle);
+}
+
+struct nouveau_object *
+nouveau_handle_ref(struct nouveau_object *parent, u32 name)
+{
+	struct nouveau_object *object = NULL;
+	struct nouveau_handle *handle;
+
+	while (!nv_iclass(parent, NV_NAMEDB_CLASS))
+		parent = parent->parent;
+
+	handle = nouveau_namedb_get(nv_namedb(parent), name);
+	if (handle) {
+		nouveau_object_ref(handle->object, &object);
+		nouveau_namedb_put(handle);
+	}
+
+	return object;
+}
+
+struct nouveau_handle *
+nouveau_handle_get_class(struct nouveau_object *engctx, u16 oclass)
+{
+	struct nouveau_namedb *namedb;
+	if (engctx && (namedb = (void *)nv_pclass(engctx, NV_NAMEDB_CLASS)))
+		return nouveau_namedb_get_class(namedb, oclass);
+	return NULL;
+}
+
+struct nouveau_handle *
+nouveau_handle_get_vinst(struct nouveau_object *engctx, u64 vinst)
+{
+	struct nouveau_namedb *namedb;
+	if (engctx && (namedb = (void *)nv_pclass(engctx, NV_NAMEDB_CLASS)))
+		return nouveau_namedb_get_vinst(namedb, vinst);
+	return NULL;
+}
+
+struct nouveau_handle *
+nouveau_handle_get_cinst(struct nouveau_object *engctx, u32 cinst)
+{
+	struct nouveau_namedb *namedb;
+	if (engctx && (namedb = (void *)nv_pclass(engctx, NV_NAMEDB_CLASS)))
+		return nouveau_namedb_get_cinst(namedb, cinst);
+	return NULL;
+}
+
+void
+nouveau_handle_put(struct nouveau_handle *handle)
+{
+	if (handle)
+		nouveau_namedb_put(handle);
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.c b/drivers/gpu/drm/nouveau/core/core/mm.c
index b29ffb3d140..bfddf87926d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mm.c
+++ b/drivers/gpu/drm/nouveau/core/core/mm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2010 Red Hat Inc.
+ * Copyright 2012 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -22,20 +22,52 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
+#include "core/os.h"
+#include "core/mm.h"
 
-static inline void
-region_put(struct nouveau_mm *mm, struct nouveau_mm_node *a)
+#define node(root, dir) ((root)->nl_entry.dir == &mm->nodes) ? NULL : \
+	list_entry((root)->nl_entry.dir, struct nouveau_mm_node, nl_entry)
+
+void
+nouveau_mm_free(struct nouveau_mm *mm, struct nouveau_mm_node **pthis)
 {
-	list_del(&a->nl_entry);
-	list_del(&a->fl_entry);
-	kfree(a);
+	struct nouveau_mm_node *this = *pthis;
+
+	if (this) {
+		struct nouveau_mm_node *prev = node(this, prev);
+		struct nouveau_mm_node *next = node(this, next);
+
+		if (prev && prev->type == 0) {
+			prev->length += this->length;
+			list_del(&this->nl_entry);
+			kfree(this); this = prev;
+		}
+
+		if (next && next->type == 0) {
+			next->offset  = this->offset;
+			next->length += this->length;
+			if (this->type == 0)
+				list_del(&this->fl_entry);
+			list_del(&this->nl_entry);
+			kfree(this); this = NULL;
+		}
+
+		if (this && this->type != 0) {
+			list_for_each_entry(prev, &mm->free, fl_entry) {
+				if (this->offset < prev->offset)
+					break;
+			}
+
+			list_add_tail(&this->fl_entry, &prev->fl_entry);
+			this->type = 0;
+		}
+	}
+
+	*pthis = NULL;
 }
 
 static struct nouveau_mm_node *
-region_split(struct nouveau_mm *mm, struct nouveau_mm_node *a, u32 size)
+region_head(struct nouveau_mm *mm, struct nouveau_mm_node *a, u32 size)
 {
 	struct nouveau_mm_node *b;
 
@@ -57,38 +89,12 @@ region_split(struct nouveau_mm *mm, struct nouveau_mm_node *a, u32 size)
 	return b;
 }
 
-#define node(root, dir) ((root)->nl_entry.dir == &mm->nodes) ? NULL : \
-	list_entry((root)->nl_entry.dir, struct nouveau_mm_node, nl_entry)
-
-void
-nouveau_mm_put(struct nouveau_mm *mm, struct nouveau_mm_node *this)
-{
-	struct nouveau_mm_node *prev = node(this, prev);
-	struct nouveau_mm_node *next = node(this, next);
-
-	list_add(&this->fl_entry, &mm->free);
-	this->type = 0;
-
-	if (prev && prev->type == 0) {
-		prev->length += this->length;
-		region_put(mm, this);
-		this = prev;
-	}
-
-	if (next && next->type == 0) {
-		next->offset  = this->offset;
-		next->length += this->length;
-		region_put(mm, this);
-	}
-}
-
 int
-nouveau_mm_get(struct nouveau_mm *mm, int type, u32 size, u32 size_nc,
-	       u32 align, struct nouveau_mm_node **pnode)
+nouveau_mm_head(struct nouveau_mm *mm, u8 type, u32 size_max, u32 size_min,
+		u32 align, struct nouveau_mm_node **pnode)
 {
 	struct nouveau_mm_node *prev, *this, *next;
-	u32 min = size_nc ? size_nc : size;
-	u32 align_mask = align - 1;
+	u32 mask = align - 1;
 	u32 splitoff;
 	u32 s, e;
 
@@ -104,16 +110,86 @@ nouveau_mm_get(struct nouveau_mm *mm, int type, u32 size, u32 size_nc,
 		if (next && next->type != type)
 			e = rounddown(e, mm->block_size);
 
-		s  = (s + align_mask) & ~align_mask;
-		e &= ~align_mask;
-		if (s > e || e - s < min)
+		s  = (s + mask) & ~mask;
+		e &= ~mask;
+		if (s > e || e - s < size_min)
 			continue;
 
 		splitoff = s - this->offset;
-		if (splitoff && !region_split(mm, this, splitoff))
+		if (splitoff && !region_head(mm, this, splitoff))
+			return -ENOMEM;
+
+		this = region_head(mm, this, min(size_max, e - s));
+		if (!this)
+			return -ENOMEM;
+
+		this->type = type;
+		list_del(&this->fl_entry);
+		*pnode = this;
+		return 0;
+	}
+
+	return -ENOSPC;
+}
+
+static struct nouveau_mm_node *
+region_tail(struct nouveau_mm *mm, struct nouveau_mm_node *a, u32 size)
+{
+	struct nouveau_mm_node *b;
+
+	if (a->length == size)
+		return a;
+
+	b = kmalloc(sizeof(*b), GFP_KERNEL);
+	if (unlikely(b == NULL))
+		return NULL;
+
+	a->length -= size;
+	b->offset  = a->offset + a->length;
+	b->length  = size;
+	b->type    = a->type;
+
+	list_add(&b->nl_entry, &a->nl_entry);
+	if (b->type == 0)
+		list_add(&b->fl_entry, &a->fl_entry);
+	return b;
+}
+
+int
+nouveau_mm_tail(struct nouveau_mm *mm, u8 type, u32 size_max, u32 size_min,
+		u32 align, struct nouveau_mm_node **pnode)
+{
+	struct nouveau_mm_node *prev, *this, *next;
+	u32 mask = align - 1;
+
+	list_for_each_entry_reverse(this, &mm->free, fl_entry) {
+		u32 e = this->offset + this->length;
+		u32 s = this->offset;
+		u32 c = 0, a;
+
+		prev = node(this, prev);
+		if (prev && prev->type != type)
+			s = roundup(s, mm->block_size);
+
+		next = node(this, next);
+		if (next && next->type != type) {
+			e = rounddown(e, mm->block_size);
+			c = next->offset - e;
+		}
+
+		s = (s + mask) & ~mask;
+		a = e - s;
+		if (s > e || a < size_min)
+			continue;
+
+		a  = min(a, size_max);
+		s  = (e - a) & ~mask;
+		c += (e - s) - a;
+
+		if (c && !region_tail(mm, this, c))
 			return -ENOMEM;
 
-		this = region_split(mm, this, min(size, e - s));
+		this = region_tail(mm, this, a);
 		if (!this)
 			return -ENOMEM;
 
@@ -148,6 +224,7 @@ nouveau_mm_init(struct nouveau_mm *mm, u32 offset, u32 length, u32 block)
 	list_add_tail(&node->nl_entry, &mm->nodes);
 	list_add_tail(&node->fl_entry, &mm->free);
 	mm->heap_nodes++;
+	mm->heap_size += length;
 	return 0;
 }
 
@@ -159,15 +236,8 @@ nouveau_mm_fini(struct nouveau_mm *mm)
 	int nodes = 0;
 
 	list_for_each_entry(node, &mm->nodes, nl_entry) {
-		if (nodes++ == mm->heap_nodes) {
-			printk(KERN_ERR "nouveau_mm in use at destroy time!\n");
-			list_for_each_entry(node, &mm->nodes, nl_entry) {
-				printk(KERN_ERR "0x%02x: 0x%08x 0x%08x\n",
-				       node->type, node->offset, node->length);
-			}
-			WARN_ON(1);
+		if (nodes++ == mm->heap_nodes)
 			return -EBUSY;
-		}
 	}
 
 	kfree(heap);
diff --git a/drivers/gpu/drm/nouveau/core/core/namedb.c b/drivers/gpu/drm/nouveau/core/core/namedb.c
new file mode 100644
index 00000000000..1ce95a8709d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/namedb.c
@@ -0,0 +1,203 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/namedb.h>
+#include <core/handle.h>
+#include <core/gpuobj.h>
+
+static struct nouveau_handle *
+nouveau_namedb_lookup(struct nouveau_namedb *namedb, u32 name)
+{
+	struct nouveau_handle *handle;
+
+	list_for_each_entry(handle, &namedb->list, node) {
+		if (handle->name == name)
+			return handle;
+	}
+
+	return NULL;
+}
+
+static struct nouveau_handle *
+nouveau_namedb_lookup_class(struct nouveau_namedb *namedb, u16 oclass)
+{
+	struct nouveau_handle *handle;
+
+	list_for_each_entry(handle, &namedb->list, node) {
+		if (nv_mclass(handle->object) == oclass)
+			return handle;
+	}
+
+	return NULL;
+}
+
+static struct nouveau_handle *
+nouveau_namedb_lookup_vinst(struct nouveau_namedb *namedb, u64 vinst)
+{
+	struct nouveau_handle *handle;
+
+	list_for_each_entry(handle, &namedb->list, node) {
+		if (nv_iclass(handle->object, NV_GPUOBJ_CLASS)) {
+			if (nv_gpuobj(handle->object)->addr == vinst)
+				return handle;
+		}
+	}
+
+	return NULL;
+}
+
+static struct nouveau_handle *
+nouveau_namedb_lookup_cinst(struct nouveau_namedb *namedb, u32 cinst)
+{
+	struct nouveau_handle *handle;
+
+	list_for_each_entry(handle, &namedb->list, node) {
+		if (nv_iclass(handle->object, NV_GPUOBJ_CLASS)) {
+			if (nv_gpuobj(handle->object)->node &&
+			    nv_gpuobj(handle->object)->node->offset == cinst)
+				return handle;
+		}
+	}
+
+	return NULL;
+}
+
+int
+nouveau_namedb_insert(struct nouveau_namedb *namedb, u32 name,
+		      struct nouveau_object *object,
+		      struct nouveau_handle *handle)
+{
+	int ret = -EEXIST;
+	write_lock_irq(&namedb->lock);
+	if (!nouveau_namedb_lookup(namedb, name)) {
+		nouveau_object_ref(object, &handle->object);
+		handle->namedb = namedb;
+		list_add(&handle->node, &namedb->list);
+		ret = 0;
+	}
+	write_unlock_irq(&namedb->lock);
+	return ret;
+}
+
+void
+nouveau_namedb_remove(struct nouveau_handle *handle)
+{
+	struct nouveau_namedb *namedb = handle->namedb;
+	struct nouveau_object *object = handle->object;
+	write_lock_irq(&namedb->lock);
+	list_del(&handle->node);
+	write_unlock_irq(&namedb->lock);
+	nouveau_object_ref(NULL, &object);
+}
+
+struct nouveau_handle *
+nouveau_namedb_get(struct nouveau_namedb *namedb, u32 name)
+{
+	struct nouveau_handle *handle;
+	read_lock(&namedb->lock);
+	handle = nouveau_namedb_lookup(namedb, name);
+	if (handle == NULL)
+		read_unlock(&namedb->lock);
+	return handle;
+}
+
+struct nouveau_handle *
+nouveau_namedb_get_class(struct nouveau_namedb *namedb, u16 oclass)
+{
+	struct nouveau_handle *handle;
+	read_lock(&namedb->lock);
+	handle = nouveau_namedb_lookup_class(namedb, oclass);
+	if (handle == NULL)
+		read_unlock(&namedb->lock);
+	return handle;
+}
+
+struct nouveau_handle *
+nouveau_namedb_get_vinst(struct nouveau_namedb *namedb, u64 vinst)
+{
+	struct nouveau_handle *handle;
+	read_lock(&namedb->lock);
+	handle = nouveau_namedb_lookup_vinst(namedb, vinst);
+	if (handle == NULL)
+		read_unlock(&namedb->lock);
+	return handle;
+}
+
+struct nouveau_handle *
+nouveau_namedb_get_cinst(struct nouveau_namedb *namedb, u32 cinst)
+{
+	struct nouveau_handle *handle;
+	read_lock(&namedb->lock);
+	handle = nouveau_namedb_lookup_cinst(namedb, cinst);
+	if (handle == NULL)
+		read_unlock(&namedb->lock);
+	return handle;
+}
+
+void
+nouveau_namedb_put(struct nouveau_handle *handle)
+{
+	if (handle)
+		read_unlock(&handle->namedb->lock);
+}
+
+int
+nouveau_namedb_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, u32 pclass,
+		       struct nouveau_oclass *sclass, u32 engcls,
+		       int length, void **pobject)
+{
+	struct nouveau_namedb *namedb;
+	int ret;
+
+	ret = nouveau_parent_create_(parent, engine, oclass, pclass |
+				     NV_NAMEDB_CLASS, sclass, engcls,
+				     length, pobject);
+	namedb = *pobject;
+	if (ret)
+		return ret;
+
+	rwlock_init(&namedb->lock);
+	INIT_LIST_HEAD(&namedb->list);
+	return 0;
+}
+
+int
+_nouveau_namedb_ctor(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, void *data, u32 size,
+		     struct nouveau_object **pobject)
+{
+	struct nouveau_namedb *object;
+	int ret;
+
+	ret = nouveau_namedb_create(parent, engine, oclass, 0, NULL, 0, &object);
+	*pobject = nv_object(object);
+	if (ret)
+		return ret;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/object.c b/drivers/gpu/drm/nouveau/core/core/object.c
new file mode 100644
index 00000000000..0daab62ea14
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/object.c
@@ -0,0 +1,468 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/parent.h>
+#include <core/namedb.h>
+#include <core/handle.h>
+#include <core/engine.h>
+
+#ifdef NOUVEAU_OBJECT_MAGIC
+static struct list_head _objlist = LIST_HEAD_INIT(_objlist);
+static DEFINE_SPINLOCK(_objlist_lock);
+#endif
+
+int
+nouveau_object_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, u32 pclass,
+		       int size, void **pobject)
+{
+	struct nouveau_object *object;
+
+	object = *pobject = kzalloc(size, GFP_KERNEL);
+	if (!object)
+		return -ENOMEM;
+
+	nouveau_object_ref(parent, &object->parent);
+	nouveau_object_ref(engine, &object->engine);
+	object->oclass = oclass;
+	object->oclass->handle |= pclass;
+	atomic_set(&object->refcount, 1);
+	atomic_set(&object->usecount, 0);
+
+#ifdef NOUVEAU_OBJECT_MAGIC
+	object->_magic = NOUVEAU_OBJECT_MAGIC;
+	spin_lock(&_objlist_lock);
+	list_add(&object->list, &_objlist);
+	spin_unlock(&_objlist_lock);
+#endif
+	return 0;
+}
+
+static int
+_nouveau_object_ctor(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, void *data, u32 size,
+		     struct nouveau_object **pobject)
+{
+	struct nouveau_object *object;
+	int ret;
+
+	ret = nouveau_object_create(parent, engine, oclass, 0, &object);
+	*pobject = nv_object(object);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void
+nouveau_object_destroy(struct nouveau_object *object)
+{
+#ifdef NOUVEAU_OBJECT_MAGIC
+	spin_lock(&_objlist_lock);
+	list_del(&object->list);
+	spin_unlock(&_objlist_lock);
+#endif
+	nouveau_object_ref(NULL, &object->engine);
+	nouveau_object_ref(NULL, &object->parent);
+	kfree(object);
+}
+
+static void
+_nouveau_object_dtor(struct nouveau_object *object)
+{
+	nouveau_object_destroy(object);
+}
+
+int
+nouveau_object_init(struct nouveau_object *object)
+{
+	return 0;
+}
+
+static int
+_nouveau_object_init(struct nouveau_object *object)
+{
+	return nouveau_object_init(object);
+}
+
+int
+nouveau_object_fini(struct nouveau_object *object, bool suspend)
+{
+	return 0;
+}
+
+static int
+_nouveau_object_fini(struct nouveau_object *object, bool suspend)
+{
+	return nouveau_object_fini(object, suspend);
+}
+
+struct nouveau_ofuncs
+nouveau_object_ofuncs = {
+	.ctor = _nouveau_object_ctor,
+	.dtor = _nouveau_object_dtor,
+	.init = _nouveau_object_init,
+	.fini = _nouveau_object_fini,
+};
+
+int
+nouveau_object_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nouveau_ofuncs *ofuncs = oclass->ofuncs;
+	int ret;
+
+	*pobject = NULL;
+
+	ret = ofuncs->ctor(parent, engine, oclass, data, size, pobject);
+	if (ret < 0) {
+		if (ret != -ENODEV) {
+			nv_error(parent, "failed to create 0x%08x, %d\n",
+				 oclass->handle, ret);
+		}
+
+		if (*pobject) {
+			ofuncs->dtor(*pobject);
+			*pobject = NULL;
+		}
+
+		return ret;
+	}
+
+	nv_debug(*pobject, "created\n");
+	return 0;
+}
+
+static void
+nouveau_object_dtor(struct nouveau_object *object)
+{
+	nv_debug(object, "destroying\n");
+	nv_ofuncs(object)->dtor(object);
+}
+
+void
+nouveau_object_ref(struct nouveau_object *obj, struct nouveau_object **ref)
+{
+	if (obj) {
+		atomic_inc(&obj->refcount);
+		nv_trace(obj, "inc() == %d\n", atomic_read(&obj->refcount));
+	}
+
+	if (*ref) {
+		int dead = atomic_dec_and_test(&(*ref)->refcount);
+		nv_trace(*ref, "dec() == %d\n", atomic_read(&(*ref)->refcount));
+		if (dead)
+			nouveau_object_dtor(*ref);
+	}
+
+	*ref = obj;
+}
+
+int
+nouveau_object_new(struct nouveau_object *client, u32 _parent, u32 _handle,
+		   u16 _oclass, void *data, u32 size,
+		   struct nouveau_object **pobject)
+{
+	struct nouveau_object *parent = NULL;
+	struct nouveau_object *engctx = NULL;
+	struct nouveau_object *object = NULL;
+	struct nouveau_object *engine;
+	struct nouveau_oclass *oclass;
+	struct nouveau_handle *handle;
+	int ret;
+
+	/* lookup parent object and ensure it *is* a parent */
+	parent = nouveau_handle_ref(client, _parent);
+	if (!parent) {
+		nv_error(client, "parent 0x%08x not found\n", _parent);
+		return -ENOENT;
+	}
+
+	if (!nv_iclass(parent, NV_PARENT_CLASS)) {
+		nv_error(parent, "cannot have children\n");
+		ret = -EINVAL;
+		goto fail_class;
+	}
+
+	/* check that parent supports the requested subclass */
+	ret = nouveau_parent_sclass(parent, _oclass, &engine, &oclass);
+	if (ret) {
+		nv_debug(parent, "illegal class 0x%04x\n", _oclass);
+		goto fail_class;
+	}
+
+	/* make sure engine init has been completed *before* any objects
+	 * it controls are created - the constructors may depend on
+	 * state calculated at init (ie. default context construction)
+	 */
+	if (engine) {
+		ret = nouveau_object_inc(engine);
+		if (ret)
+			goto fail_class;
+	}
+
+	/* if engine requires it, create a context object to insert
+	 * between the parent and its children (eg. PGRAPH context)
+	 */
+	if (engine && nv_engine(engine)->cclass) {
+		ret = nouveau_object_ctor(parent, engine,
+					  nv_engine(engine)->cclass,
+					  data, size, &engctx);
+		if (ret)
+			goto fail_engctx;
+	} else {
+		nouveau_object_ref(parent, &engctx);
+	}
+
+	/* finally, create new object and bind it to its handle */
+	ret = nouveau_object_ctor(engctx, engine, oclass, data, size, &object);
+	*pobject = object;
+	if (ret)
+		goto fail_ctor;
+
+	ret = nouveau_object_inc(object);
+	if (ret)
+		goto fail_init;
+
+	ret = nouveau_handle_create(parent, _parent, _handle, object, &handle);
+	if (ret)
+		goto fail_handle;
+
+	ret = nouveau_handle_init(handle);
+	if (ret)
+		nouveau_handle_destroy(handle);
+
+fail_handle:
+	nouveau_object_dec(object, false);
+fail_init:
+	nouveau_object_ref(NULL, &object);
+fail_ctor:
+	nouveau_object_ref(NULL, &engctx);
+fail_engctx:
+	if (engine)
+		nouveau_object_dec(engine, false);
+fail_class:
+	nouveau_object_ref(NULL, &parent);
+	return ret;
+}
+
+int
+nouveau_object_del(struct nouveau_object *client, u32 _parent, u32 _handle)
+{
+	struct nouveau_object *parent = NULL;
+	struct nouveau_object *namedb = NULL;
+	struct nouveau_handle *handle = NULL;
+	int ret = -EINVAL;
+
+	parent = nouveau_handle_ref(client, _parent);
+	if (!parent)
+		return -ENOENT;
+
+	namedb = nv_pclass(parent, NV_NAMEDB_CLASS);
+	if (namedb) {
+		handle = nouveau_namedb_get(nv_namedb(namedb), _handle);
+		if (handle) {
+			nouveau_namedb_put(handle);
+			nouveau_handle_fini(handle, false);
+			nouveau_handle_destroy(handle);
+		}
+	}
+
+	nouveau_object_ref(NULL, &parent);
+	return ret;
+}
+
+int
+nouveau_object_inc(struct nouveau_object *object)
+{
+	int ref = atomic_add_return(1, &object->usecount);
+	int ret;
+
+	nv_trace(object, "use(+1) == %d\n", atomic_read(&object->usecount));
+	if (ref != 1)
+		return 0;
+
+	nv_trace(object, "initialising...\n");
+	if (object->parent) {
+		ret = nouveau_object_inc(object->parent);
+		if (ret) {
+			nv_error(object, "parent failed, %d\n", ret);
+			goto fail_parent;
+		}
+	}
+
+	if (object->engine) {
+		mutex_lock(&nv_subdev(object->engine)->mutex);
+		ret = nouveau_object_inc(object->engine);
+		mutex_unlock(&nv_subdev(object->engine)->mutex);
+		if (ret) {
+			nv_error(object, "engine failed, %d\n", ret);
+			goto fail_engine;
+		}
+	}
+
+	ret = nv_ofuncs(object)->init(object);
+	if (ret) {
+		nv_error(object, "init failed, %d\n", ret);
+		goto fail_self;
+	}
+
+	nv_debug(object, "initialised\n");
+	return 0;
+
+fail_self:
+	if (object->engine) {
+		mutex_lock(&nv_subdev(object->engine)->mutex);
+		nouveau_object_dec(object->engine, false);
+		mutex_unlock(&nv_subdev(object->engine)->mutex);
+	}
+fail_engine:
+	if (object->parent)
+		 nouveau_object_dec(object->parent, false);
+fail_parent:
+	atomic_dec(&object->usecount);
+	return ret;
+}
+
+static int
+nouveau_object_decf(struct nouveau_object *object)
+{
+	int ret;
+
+	nv_trace(object, "stopping...\n");
+
+	ret = nv_ofuncs(object)->fini(object, false);
+	if (ret)
+		nv_warn(object, "failed fini, %d\n", ret);
+
+	if (object->engine) {
+		mutex_lock(&nv_subdev(object->engine)->mutex);
+		nouveau_object_dec(object->engine, false);
+		mutex_unlock(&nv_subdev(object->engine)->mutex);
+	}
+
+	if (object->parent)
+		nouveau_object_dec(object->parent, false);
+
+	nv_debug(object, "stopped\n");
+	return 0;
+}
+
+static int
+nouveau_object_decs(struct nouveau_object *object)
+{
+	int ret, rret;
+
+	nv_trace(object, "suspending...\n");
+
+	ret = nv_ofuncs(object)->fini(object, true);
+	if (ret) {
+		nv_error(object, "failed suspend, %d\n", ret);
+		return ret;
+	}
+
+	if (object->engine) {
+		mutex_lock(&nv_subdev(object->engine)->mutex);
+		ret = nouveau_object_dec(object->engine, true);
+		mutex_unlock(&nv_subdev(object->engine)->mutex);
+		if (ret) {
+			nv_warn(object, "engine failed suspend, %d\n", ret);
+			goto fail_engine;
+		}
+	}
+
+	if (object->parent) {
+		ret = nouveau_object_dec(object->parent, true);
+		if (ret) {
+			nv_warn(object, "parent failed suspend, %d\n", ret);
+			goto fail_parent;
+		}
+	}
+
+	nv_debug(object, "suspended\n");
+	return 0;
+
+fail_parent:
+	if (object->engine) {
+		mutex_lock(&nv_subdev(object->engine)->mutex);
+		rret = nouveau_object_inc(object->engine);
+		mutex_unlock(&nv_subdev(object->engine)->mutex);
+		if (rret)
+			nv_fatal(object, "engine failed to reinit, %d\n", rret);
+	}
+
+fail_engine:
+	rret = nv_ofuncs(object)->init(object);
+	if (rret)
+		nv_fatal(object, "failed to reinit, %d\n", rret);
+
+	return ret;
+}
+
+int
+nouveau_object_dec(struct nouveau_object *object, bool suspend)
+{
+	int ref = atomic_add_return(-1, &object->usecount);
+	int ret;
+
+	nv_trace(object, "use(-1) == %d\n", atomic_read(&object->usecount));
+
+	if (ref == 0) {
+		if (suspend)
+			ret = nouveau_object_decs(object);
+		else
+			ret = nouveau_object_decf(object);
+
+		if (ret) {
+			atomic_inc(&object->usecount);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+void
+nouveau_object_debug(void)
+{
+#ifdef NOUVEAU_OBJECT_MAGIC
+	struct nouveau_object *object;
+	if (!list_empty(&_objlist)) {
+		nv_fatal(NULL, "*******************************************\n");
+		nv_fatal(NULL, "* AIIIII! object(s) still exist!!!\n");
+		nv_fatal(NULL, "*******************************************\n");
+		list_for_each_entry(object, &_objlist, list) {
+			nv_fatal(object, "%p/%p/%d/%d\n",
+				 object->parent, object->engine,
+				 atomic_read(&object->refcount),
+				 atomic_read(&object->usecount));
+		}
+	}
+#endif
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/option.c b/drivers/gpu/drm/nouveau/core/core/option.c
new file mode 100644
index 00000000000..62a432ea39e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/option.c
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/option.h>
+#include <core/debug.h>
+
+/* compares unterminated string 'str' with zero-terminated string 'cmp' */
+static inline int
+strncasecmpz(const char *str, const char *cmp, size_t len)
+{
+	if (strlen(cmp) != len)
+		return len;
+	return strncasecmp(str, cmp, len);
+}
+
+const char *
+nouveau_stropt(const char *optstr, const char *opt, int *arglen)
+{
+	while (optstr && *optstr != '\0') {
+		int len = strcspn(optstr, ",=");
+		switch (optstr[len]) {
+		case '=':
+			if (!strncasecmpz(optstr, opt, len)) {
+				optstr += len + 1;
+				*arglen = strcspn(optstr, ",=");
+				return *arglen ? optstr : NULL;
+			}
+			optstr++;
+			break;
+		case ',':
+			optstr++;
+			break;
+		default:
+			break;
+		}
+		optstr += len;
+	}
+
+	return NULL;
+}
+
+bool
+nouveau_boolopt(const char *optstr, const char *opt, bool value)
+{
+	int arglen;
+
+	optstr = nouveau_stropt(optstr, opt, &arglen);
+	if (optstr) {
+		if (!strncasecmpz(optstr, "0", arglen) ||
+		    !strncasecmpz(optstr, "no", arglen) ||
+		    !strncasecmpz(optstr, "off", arglen) ||
+		    !strncasecmpz(optstr, "false", arglen))
+			value = false;
+		else
+		if (!strncasecmpz(optstr, "1", arglen) ||
+		    !strncasecmpz(optstr, "yes", arglen) ||
+		    !strncasecmpz(optstr, "on", arglen) ||
+		    !strncasecmpz(optstr, "true", arglen))
+			value = true;
+	}
+
+	return value;
+}
+
+int
+nouveau_dbgopt(const char *optstr, const char *sub)
+{
+	int mode = 1, level = CONFIG_NOUVEAU_DEBUG_DEFAULT;
+
+	while (optstr) {
+		int len = strcspn(optstr, ",=");
+		switch (optstr[len]) {
+		case '=':
+			if (strncasecmpz(optstr, sub, len))
+				mode = 0;
+			optstr++;
+			break;
+		default:
+			if (mode) {
+				if (!strncasecmpz(optstr, "fatal", len))
+					level = NV_DBG_FATAL;
+				else if (!strncasecmpz(optstr, "error", len))
+					level = NV_DBG_ERROR;
+				else if (!strncasecmpz(optstr, "warn", len))
+					level = NV_DBG_WARN;
+				else if (!strncasecmpz(optstr, "info", len))
+					level = NV_DBG_INFO;
+				else if (!strncasecmpz(optstr, "debug", len))
+					level = NV_DBG_DEBUG;
+				else if (!strncasecmpz(optstr, "trace", len))
+					level = NV_DBG_TRACE;
+				else if (!strncasecmpz(optstr, "paranoia", len))
+					level = NV_DBG_PARANOIA;
+				else if (!strncasecmpz(optstr, "spam", len))
+					level = NV_DBG_SPAM;
+			}
+
+			if (optstr[len] != '\0') {
+				optstr++;
+				mode = 1;
+				break;
+			}
+
+			return level;
+		}
+		optstr += len;
+	}
+
+	return level;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/parent.c b/drivers/gpu/drm/nouveau/core/core/parent.c
new file mode 100644
index 00000000000..a1ea034611d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/parent.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/parent.h>
+
+int
+nouveau_parent_sclass(struct nouveau_object *parent, u16 handle,
+		      struct nouveau_object **pengine,
+		      struct nouveau_oclass **poclass)
+{
+	struct nouveau_sclass *sclass;
+	struct nouveau_engine *engine;
+	struct nouveau_oclass *oclass;
+	u64 mask;
+
+	sclass = nv_parent(parent)->sclass;
+	while (sclass) {
+		if ((sclass->oclass->handle & 0xffff) == handle) {
+			*pengine = parent->engine;
+			*poclass = sclass->oclass;
+			return 0;
+		}
+
+		sclass = sclass->sclass;
+	}
+
+	mask = nv_parent(parent)->engine;
+	while (mask) {
+		int i = ffsll(mask) - 1;
+
+		if ((engine = nouveau_engine(parent, i))) {
+			oclass = engine->sclass;
+			while (oclass->ofuncs) {
+				if ((oclass->handle & 0xffff) == handle) {
+					*pengine = nv_object(engine);
+					*poclass = oclass;
+					return 0;
+				}
+				oclass++;
+			}
+		}
+
+		mask &= ~(1ULL << i);
+	}
+
+	return -EINVAL;
+}
+
+int
+nouveau_parent_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, u32 pclass,
+		       struct nouveau_oclass *sclass, u64 engcls,
+		       int size, void **pobject)
+{
+	struct nouveau_parent *object;
+	struct nouveau_sclass *nclass;
+	int ret;
+
+	ret = nouveau_object_create_(parent, engine, oclass, pclass |
+				     NV_PARENT_CLASS, size, pobject);
+	object = *pobject;
+	if (ret)
+		return ret;
+
+	while (sclass && sclass->ofuncs) {
+		nclass = kzalloc(sizeof(*nclass), GFP_KERNEL);
+		if (!nclass)
+			return -ENOMEM;
+
+		nclass->sclass = object->sclass;
+		object->sclass = nclass;
+		nclass->engine = engine ? nv_engine(engine) : NULL;
+		nclass->oclass = sclass;
+		sclass++;
+	}
+
+	object->engine = engcls;
+	return 0;
+}
+
+int
+_nouveau_parent_ctor(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, void *data, u32 size,
+		     struct nouveau_object **pobject)
+{
+	struct nouveau_parent *object;
+	int ret;
+
+	ret = nouveau_parent_create(parent, engine, oclass, 0, NULL, 0, &object);
+	*pobject = nv_object(object);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void
+nouveau_parent_destroy(struct nouveau_parent *parent)
+{
+	struct nouveau_sclass *sclass;
+
+	while ((sclass = parent->sclass)) {
+		parent->sclass = sclass->sclass;
+		kfree(sclass);
+	}
+
+	nouveau_object_destroy(&parent->base);
+}
+
+
+void
+_nouveau_parent_dtor(struct nouveau_object *object)
+{
+	nouveau_parent_destroy(nv_parent(object));
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/printk.c b/drivers/gpu/drm/nouveau/core/core/printk.c
new file mode 100644
index 00000000000..6161eaf5447
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/printk.c
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/client.h>
+#include <core/subdev.h>
+#include <core/printk.h>
+
+void
+nv_printk_(struct nouveau_object *object, const char *pfx, int level,
+	   const char *fmt, ...)
+{
+	static const char name[] = { '!', 'E', 'W', ' ', 'D', 'T', 'P', 'S' };
+	char mfmt[256];
+	va_list args;
+
+	if (object && !nv_iclass(object, NV_CLIENT_CLASS)) {
+		struct nouveau_object *device = object;
+		struct nouveau_object *subdev = object;
+		char obuf[64], *ofmt = "";
+
+		if (object->engine) {
+			snprintf(obuf, sizeof(obuf), "[0x%08x][%p]",
+				 nv_hclass(object), object);
+			ofmt = obuf;
+			subdev = object->engine;
+			device = object->engine;
+		}
+
+		if (subdev->parent)
+			device = subdev->parent;
+
+		if (level > nv_subdev(subdev)->debug)
+			return;
+
+		snprintf(mfmt, sizeof(mfmt), "%snouveau %c[%8s][%s]%s %s", pfx,
+			 name[level], nv_subdev(subdev)->name,
+			 nv_device(device)->name, ofmt, fmt);
+	} else
+	if (object && nv_iclass(object, NV_CLIENT_CLASS)) {
+		if (level > nv_client(object)->debug)
+			return;
+
+		snprintf(mfmt, sizeof(mfmt), "%snouveau %c[%8s] %s", pfx,
+			 name[level], nv_client(object)->name, fmt);
+	} else {
+		snprintf(mfmt, sizeof(mfmt), "%snouveau: %s", pfx, fmt);
+	}
+
+	va_start(args, fmt);
+	vprintk(mfmt, args);
+	va_end(args);
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/ramht.c b/drivers/gpu/drm/nouveau/core/core/ramht.c
new file mode 100644
index 00000000000..86a64045dd6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/ramht.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <core/object.h>
+#include <core/ramht.h>
+#include <core/math.h>
+
+#include <subdev/bar.h>
+
+static u32
+nouveau_ramht_hash(struct nouveau_ramht *ramht, int chid, u32 handle)
+{
+	u32 hash = 0;
+
+	while (handle) {
+		hash ^= (handle & ((1 << ramht->bits) - 1));
+		handle >>= ramht->bits;
+	}
+
+	hash ^= chid << (ramht->bits - 4);
+	hash  = hash << 3;
+	return hash;
+}
+
+int
+nouveau_ramht_insert(struct nouveau_ramht *ramht, int chid,
+		     u32 handle, u32 context)
+{
+	struct nouveau_bar *bar = nouveau_bar(ramht);
+	u32 co, ho;
+
+	co = ho = nouveau_ramht_hash(ramht, chid, handle);
+	do {
+		if (!nv_ro32(ramht, co + 4)) {
+			nv_wo32(ramht, co + 0, handle);
+			nv_wo32(ramht, co + 4, context);
+			if (bar)
+				bar->flush(bar);
+			return co;
+		}
+
+		co += 8;
+		if (co >= nv_gpuobj(ramht)->size)
+			co = 0;
+	} while (co != ho);
+
+	return -ENOMEM;
+}
+
+void
+nouveau_ramht_remove(struct nouveau_ramht *ramht, int cookie)
+{
+	struct nouveau_bar *bar = nouveau_bar(ramht);
+	nv_wo32(ramht, cookie + 0, 0x00000000);
+	nv_wo32(ramht, cookie + 4, 0x00000000);
+	if (bar)
+		bar->flush(bar);
+}
+
+static struct nouveau_oclass
+nouveau_ramht_oclass = {
+	.handle = 0x0000abcd,
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = NULL,
+		.dtor = _nouveau_gpuobj_dtor,
+		.init = _nouveau_gpuobj_init,
+		.fini = _nouveau_gpuobj_fini,
+		.rd32 = _nouveau_gpuobj_rd32,
+		.wr32 = _nouveau_gpuobj_wr32,
+	},
+};
+
+int
+nouveau_ramht_new(struct nouveau_object *parent, struct nouveau_object *pargpu,
+		  u32 size, u32 align, struct nouveau_ramht **pramht)
+{
+	struct nouveau_ramht *ramht;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, parent->engine ?
+				    parent->engine : parent, /* <nv50 ramht */
+				    &nouveau_ramht_oclass, 0, pargpu, size,
+				    align, NVOBJ_FLAG_ZERO_ALLOC, &ramht);
+	*pramht = ramht;
+	if (ret)
+		return ret;
+
+	ramht->bits = log2i(nv_gpuobj(ramht)->size >> 3);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/core/subdev.c b/drivers/gpu/drm/nouveau/core/core/subdev.c
new file mode 100644
index 00000000000..f74c30aa33a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/core/subdev.c
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/subdev.h>
+#include <core/device.h>
+#include <core/option.h>
+
+void
+nouveau_subdev_reset(struct nouveau_object *subdev)
+{
+	nv_trace(subdev, "resetting...\n");
+	nv_ofuncs(subdev)->fini(subdev, false);
+	nv_debug(subdev, "reset\n");
+}
+
+int
+nouveau_subdev_init(struct nouveau_subdev *subdev)
+{
+	int ret = nouveau_object_init(&subdev->base);
+	if (ret)
+		return ret;
+
+	nouveau_subdev_reset(&subdev->base);
+	return 0;
+}
+
+int
+_nouveau_subdev_init(struct nouveau_object *object)
+{
+	return nouveau_subdev_init(nv_subdev(object));
+}
+
+int
+nouveau_subdev_fini(struct nouveau_subdev *subdev, bool suspend)
+{
+	if (subdev->unit) {
+		nv_mask(subdev, 0x000200, subdev->unit, 0x00000000);
+		nv_mask(subdev, 0x000200, subdev->unit, subdev->unit);
+	}
+
+	return nouveau_object_fini(&subdev->base, suspend);
+}
+
+int
+_nouveau_subdev_fini(struct nouveau_object *object, bool suspend)
+{
+	return nouveau_subdev_fini(nv_subdev(object), suspend);
+}
+
+void
+nouveau_subdev_destroy(struct nouveau_subdev *subdev)
+{
+	int subidx = nv_hclass(subdev) & 0xff;
+	nv_device(subdev)->subdev[subidx] = NULL;
+	nouveau_object_destroy(&subdev->base);
+}
+
+void
+_nouveau_subdev_dtor(struct nouveau_object *object)
+{
+	nouveau_subdev_destroy(nv_subdev(object));
+}
+
+int
+nouveau_subdev_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, u32 pclass,
+		       const char *subname, const char *sysname,
+		       int size, void **pobject)
+{
+	struct nouveau_subdev *subdev;
+	int ret;
+
+	ret = nouveau_object_create_(parent, engine, oclass, pclass |
+				     NV_SUBDEV_CLASS, size, pobject);
+	subdev = *pobject;
+	if (ret)
+		return ret;
+
+	mutex_init(&subdev->mutex);
+	subdev->name = subname;
+
+	if (parent) {
+		struct nouveau_device *device = nv_device(parent);
+		int subidx = nv_hclass(subdev) & 0xff;
+
+		subdev->debug = nouveau_dbgopt(device->dbgopt, subname);
+		subdev->mmio  = nv_subdev(device)->mmio;
+		device->subdev[subidx] = *pobject;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c
new file mode 100644
index 00000000000..66f7dfd907e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/bsp.h>
+
+struct nv84_bsp_priv {
+	struct nouveau_bsp base;
+};
+
+struct nv84_bsp_chan {
+	struct nouveau_bsp_chan base;
+};
+
+/*******************************************************************************
+ * BSP object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv84_bsp_sclass[] = {
+	{},
+};
+
+/*******************************************************************************
+ * BSP context
+ ******************************************************************************/
+
+static int
+nv84_bsp_context_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nv84_bsp_chan *priv;
+	int ret;
+
+	ret = nouveau_bsp_context_create(parent, engine, oclass, NULL,
+					 0, 0, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nv84_bsp_context_dtor(struct nouveau_object *object)
+{
+	struct nv84_bsp_chan *priv = (void *)object;
+	nouveau_bsp_context_destroy(&priv->base);
+}
+
+static int
+nv84_bsp_context_init(struct nouveau_object *object)
+{
+	struct nv84_bsp_chan *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_bsp_context_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv84_bsp_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv84_bsp_chan *priv = (void *)object;
+	return nouveau_bsp_context_fini(&priv->base, suspend);
+}
+
+static struct nouveau_oclass
+nv84_bsp_cclass = {
+	.handle = NV_ENGCTX(BSP, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_bsp_context_ctor,
+		.dtor = nv84_bsp_context_dtor,
+		.init = nv84_bsp_context_init,
+		.fini = nv84_bsp_context_fini,
+		.rd32 = _nouveau_bsp_context_rd32,
+		.wr32 = _nouveau_bsp_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * BSP engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv84_bsp_intr(struct nouveau_subdev *subdev)
+{
+}
+
+static int
+nv84_bsp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv84_bsp_priv *priv;
+	int ret;
+
+	ret = nouveau_bsp_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x04008000;
+	nv_subdev(priv)->intr = nv84_bsp_intr;
+	nv_engine(priv)->cclass = &nv84_bsp_cclass;
+	nv_engine(priv)->sclass = nv84_bsp_sclass;
+	return 0;
+}
+
+static void
+nv84_bsp_dtor(struct nouveau_object *object)
+{
+	struct nv84_bsp_priv *priv = (void *)object;
+	nouveau_bsp_destroy(&priv->base);
+}
+
+static int
+nv84_bsp_init(struct nouveau_object *object)
+{
+	struct nv84_bsp_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_bsp_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv84_bsp_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv84_bsp_priv *priv = (void *)object;
+	return nouveau_bsp_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv84_bsp_oclass = {
+	.handle = NV_ENGINE(BSP, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_bsp_ctor,
+		.dtor = nv84_bsp_dtor,
+		.init = nv84_bsp_init,
+		.fini = nv84_bsp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.fuc b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nva3.fuc
index 219850d5328..219850d5328 100644
--- a/drivers/gpu/drm/nouveau/nva3_copy.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nva3.fuc
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.fuc.h b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nva3.fuc.h
index 37d6de3c9d6..c92520f3ed4 100644
--- a/drivers/gpu/drm/nouveau/nva3_copy.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nva3.fuc.h
@@ -1,4 +1,4 @@
-u32 nva3_pcopy_data[] = {
+static u32 nva3_pcopy_data[] = {
 /* 0x0000: ctx_object */
 	0x00000000,
 /* 0x0004: ctx_dma */
@@ -183,7 +183,7 @@ u32 nva3_pcopy_data[] = {
 	0x00000800,
 };
 
-u32 nva3_pcopy_code[] = {
+static u32 nva3_pcopy_code[] = {
 /* 0x0000: main */
 	0x04fe04bd,
 	0x3517f000,
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.fuc.h b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nvc0.fuc.h
index cd879f31bb3..0d98c6c0958 100644
--- a/drivers/gpu/drm/nouveau/nvc0_copy.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/fuc/nvc0.fuc.h
@@ -1,4 +1,4 @@
-u32 nvc0_pcopy_data[] = {
+static u32 nvc0_pcopy_data[] = {
 /* 0x0000: ctx_object */
 	0x00000000,
 /* 0x0004: ctx_query_address_high */
@@ -171,7 +171,7 @@ u32 nvc0_pcopy_data[] = {
 	0x00000800,
 };
 
-u32 nvc0_pcopy_code[] = {
+static u32 nvc0_pcopy_code[] = {
 /* 0x0000: main */
 	0x04fe04bd,
 	0x3517f000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
new file mode 100644
index 00000000000..4df6da0af74
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/enum.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+
+#include <engine/fifo.h>
+#include <engine/copy.h>
+
+#include "fuc/nva3.fuc.h"
+
+struct nva3_copy_priv {
+	struct nouveau_copy base;
+};
+
+struct nva3_copy_chan {
+	struct nouveau_copy_chan base;
+};
+
+/*******************************************************************************
+ * Copy object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nva3_copy_sclass[] = {
+	{ 0x85b5, &nouveau_object_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PCOPY context
+ ******************************************************************************/
+
+static int
+nva3_copy_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nva3_copy_chan *priv;
+	int ret;
+
+	ret = nouveau_copy_context_create(parent, engine, oclass, NULL, 256, 0,
+					  NVOBJ_FLAG_ZERO_ALLOC, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nva3_copy_cclass = {
+	.handle = NV_ENGCTX(COPY0, 0xa3),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nva3_copy_context_ctor,
+		.dtor = _nouveau_copy_context_dtor,
+		.init = _nouveau_copy_context_init,
+		.fini = _nouveau_copy_context_fini,
+		.rd32 = _nouveau_copy_context_rd32,
+		.wr32 = _nouveau_copy_context_wr32,
+
+	},
+};
+
+/*******************************************************************************
+ * PCOPY engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_enum nva3_copy_isr_error_name[] = {
+	{ 0x0001, "ILLEGAL_MTHD" },
+	{ 0x0002, "INVALID_ENUM" },
+	{ 0x0003, "INVALID_BITFIELD" },
+	{}
+};
+
+static void
+nva3_copy_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nva3_copy_priv *priv = (void *)subdev;
+	u32 dispatch = nv_rd32(priv, 0x10401c);
+	u32 stat = nv_rd32(priv, 0x104008) & dispatch & ~(dispatch >> 16);
+	u64 inst = nv_rd32(priv, 0x104050) & 0x3fffffff;
+	u32 ssta = nv_rd32(priv, 0x104040) & 0x0000ffff;
+	u32 addr = nv_rd32(priv, 0x104040) >> 16;
+	u32 mthd = (addr & 0x07ff) << 2;
+	u32 subc = (addr & 0x3800) >> 11;
+	u32 data = nv_rd32(priv, 0x104044);
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000040) {
+		nv_error(priv, "DISPATCH_ERROR [");
+		nouveau_enum_print(nva3_copy_isr_error_name, ssta);
+		printk("] ch %d [0x%010llx] subc %d mthd 0x%04x data 0x%08x\n",
+		       chid, inst << 12, subc, mthd, data);
+		nv_wr32(priv, 0x104004, 0x00000040);
+		stat &= ~0x00000040;
+	}
+
+	if (stat) {
+		nv_error(priv, "unhandled intr 0x%08x\n", stat);
+		nv_wr32(priv, 0x104004, stat);
+	}
+
+	nv50_fb_trap(nouveau_fb(priv), 1);
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nva3_copy_tlb_flush(struct nouveau_engine *engine)
+{
+	nv50_vm_flush_engine(&engine->base, 0x0d);
+	return 0;
+}
+
+static int
+nva3_copy_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	bool enable = (nv_device(parent)->chipset != 0xaf);
+	struct nva3_copy_priv *priv;
+	int ret;
+
+	ret = nouveau_copy_create(parent, engine, oclass, enable, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00802000;
+	nv_subdev(priv)->intr = nva3_copy_intr;
+	nv_engine(priv)->cclass = &nva3_copy_cclass;
+	nv_engine(priv)->sclass = nva3_copy_sclass;
+	nv_engine(priv)->tlb_flush = nva3_copy_tlb_flush;
+	return 0;
+}
+
+static int
+nva3_copy_init(struct nouveau_object *object)
+{
+	struct nva3_copy_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_copy_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* disable all interrupts */
+	nv_wr32(priv, 0x104014, 0xffffffff);
+
+	/* upload ucode */
+	nv_wr32(priv, 0x1041c0, 0x01000000);
+	for (i = 0; i < sizeof(nva3_pcopy_data) / 4; i++)
+		nv_wr32(priv, 0x1041c4, nva3_pcopy_data[i]);
+
+	nv_wr32(priv, 0x104180, 0x01000000);
+	for (i = 0; i < sizeof(nva3_pcopy_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x104188, i >> 6);
+		nv_wr32(priv, 0x104184, nva3_pcopy_code[i]);
+	}
+
+	/* start it running */
+	nv_wr32(priv, 0x10410c, 0x00000000);
+	nv_wr32(priv, 0x104104, 0x00000000); /* ENTRY */
+	nv_wr32(priv, 0x104100, 0x00000002); /* TRIGGER */
+	return 0;
+}
+
+static int
+nva3_copy_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nva3_copy_priv *priv = (void *)object;
+
+	nv_mask(priv, 0x104048, 0x00000003, 0x00000000);
+	nv_wr32(priv, 0x104014, 0xffffffff);
+
+	return nouveau_copy_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nva3_copy_oclass = {
+	.handle = NV_ENGINE(COPY0, 0xa3),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nva3_copy_ctor,
+		.dtor = _nouveau_copy_dtor,
+		.init = nva3_copy_init,
+		.fini = nva3_copy_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
new file mode 100644
index 00000000000..06d4a879105
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
@@ -0,0 +1,265 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/enum.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/fifo.h>
+#include <engine/copy.h>
+
+#include "fuc/nvc0.fuc.h"
+
+struct nvc0_copy_priv {
+	struct nouveau_copy base;
+};
+
+struct nvc0_copy_chan {
+	struct nouveau_copy_chan base;
+};
+
+/*******************************************************************************
+ * Copy object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nvc0_copy0_sclass[] = {
+	{ 0x90b5, &nouveau_object_ofuncs },
+	{},
+};
+
+static struct nouveau_oclass
+nvc0_copy1_sclass[] = {
+	{ 0x90b8, &nouveau_object_ofuncs },
+	{},
+};
+
+/*******************************************************************************
+ * PCOPY context
+ ******************************************************************************/
+
+static int
+nvc0_copy_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nvc0_copy_chan *priv;
+	int ret;
+
+	ret = nouveau_copy_context_create(parent, engine, oclass, NULL, 256,
+					  256, NVOBJ_FLAG_ZERO_ALLOC, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nvc0_copy_context_ofuncs = {
+	.ctor = nvc0_copy_context_ctor,
+	.dtor = _nouveau_copy_context_dtor,
+	.init = _nouveau_copy_context_init,
+	.fini = _nouveau_copy_context_fini,
+	.rd32 = _nouveau_copy_context_rd32,
+	.wr32 = _nouveau_copy_context_wr32,
+};
+
+static struct nouveau_oclass
+nvc0_copy0_cclass = {
+	.handle = NV_ENGCTX(COPY0, 0xc0),
+	.ofuncs = &nvc0_copy_context_ofuncs,
+};
+
+static struct nouveau_oclass
+nvc0_copy1_cclass = {
+	.handle = NV_ENGCTX(COPY1, 0xc0),
+	.ofuncs = &nvc0_copy_context_ofuncs,
+};
+
+/*******************************************************************************
+ * PCOPY engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_enum nvc0_copy_isr_error_name[] = {
+	{ 0x0001, "ILLEGAL_MTHD" },
+	{ 0x0002, "INVALID_ENUM" },
+	{ 0x0003, "INVALID_BITFIELD" },
+	{}
+};
+
+static void
+nvc0_copy_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	int idx = nv_engidx(nv_object(subdev)) - NVDEV_ENGINE_COPY0;
+	struct nvc0_copy_priv *priv = (void *)subdev;
+	u32 disp = nv_rd32(priv, 0x10401c + (idx * 0x1000));
+	u32 intr = nv_rd32(priv, 0x104008 + (idx * 0x1000));
+	u32 stat = intr & disp & ~(disp >> 16);
+	u64 inst = nv_rd32(priv, 0x104050 + (idx * 0x1000)) & 0x0fffffff;
+	u32 ssta = nv_rd32(priv, 0x104040 + (idx * 0x1000)) & 0x0000ffff;
+	u32 addr = nv_rd32(priv, 0x104040 + (idx * 0x1000)) >> 16;
+	u32 mthd = (addr & 0x07ff) << 2;
+	u32 subc = (addr & 0x3800) >> 11;
+	u32 data = nv_rd32(priv, 0x104044 + (idx * 0x1000));
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000040) {
+		nv_error(priv, "DISPATCH_ERROR [");
+		nouveau_enum_print(nvc0_copy_isr_error_name, ssta);
+		printk("] ch %d [0x%010llx] subc %d mthd 0x%04x data 0x%08x\n",
+		       chid, (u64)inst << 12, subc, mthd, data);
+		nv_wr32(priv, 0x104004 + (idx * 0x1000), 0x00000040);
+		stat &= ~0x00000040;
+	}
+
+	if (stat) {
+		nv_error(priv, "unhandled intr 0x%08x\n", stat);
+		nv_wr32(priv, 0x104004 + (idx * 0x1000), stat);
+	}
+
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nvc0_copy0_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nvc0_copy_priv *priv;
+	int ret;
+
+	if (nv_rd32(parent, 0x022500) & 0x00000100)
+		return -ENODEV;
+
+	ret = nouveau_copy_create(parent, engine, oclass, true, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000040;
+	nv_subdev(priv)->intr = nvc0_copy_intr;
+	nv_engine(priv)->cclass = &nvc0_copy0_cclass;
+	nv_engine(priv)->sclass = nvc0_copy0_sclass;
+	return 0;
+}
+
+static int
+nvc0_copy1_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nvc0_copy_priv *priv;
+	int ret;
+
+	if (nv_rd32(parent, 0x022500) & 0x00000200)
+		return -ENODEV;
+
+	ret = nouveau_copy_create(parent, engine, oclass, true, 1, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000080;
+	nv_subdev(priv)->intr = nvc0_copy_intr;
+	nv_engine(priv)->cclass = &nvc0_copy1_cclass;
+	nv_engine(priv)->sclass = nvc0_copy1_sclass;
+	return 0;
+}
+
+static int
+nvc0_copy_init(struct nouveau_object *object)
+{
+	int idx = nv_engidx(object) - NVDEV_ENGINE_COPY0;
+	struct nvc0_copy_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_copy_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* disable all interrupts */
+	nv_wr32(priv, 0x104014 + (idx * 0x1000), 0xffffffff);
+
+	/* upload ucode */
+	nv_wr32(priv, 0x1041c0 + (idx * 0x1000), 0x01000000);
+	for (i = 0; i < sizeof(nvc0_pcopy_data) / 4; i++)
+		nv_wr32(priv, 0x1041c4 + (idx * 0x1000), nvc0_pcopy_data[i]);
+
+	nv_wr32(priv, 0x104180 + (idx * 0x1000), 0x01000000);
+	for (i = 0; i < sizeof(nvc0_pcopy_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x104188 + (idx * 0x1000), i >> 6);
+		nv_wr32(priv, 0x104184 + (idx * 0x1000), nvc0_pcopy_code[i]);
+	}
+
+	/* start it running */
+	nv_wr32(priv, 0x104084 + (idx * 0x1000), idx);
+	nv_wr32(priv, 0x10410c + (idx * 0x1000), 0x00000000);
+	nv_wr32(priv, 0x104104 + (idx * 0x1000), 0x00000000); /* ENTRY */
+	nv_wr32(priv, 0x104100 + (idx * 0x1000), 0x00000002); /* TRIGGER */
+	return 0;
+}
+
+static int
+nvc0_copy_fini(struct nouveau_object *object, bool suspend)
+{
+	int idx = nv_engidx(object) - NVDEV_ENGINE_COPY0;
+	struct nvc0_copy_priv *priv = (void *)object;
+
+	nv_mask(priv, 0x104048 + (idx * 0x1000), 0x00000003, 0x00000000);
+	nv_wr32(priv, 0x104014 + (idx * 0x1000), 0xffffffff);
+
+	return nouveau_copy_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nvc0_copy0_oclass = {
+	.handle = NV_ENGINE(COPY0, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_copy0_ctor,
+		.dtor = _nouveau_copy_dtor,
+		.init = nvc0_copy_init,
+		.fini = nvc0_copy_fini,
+	},
+};
+
+struct nouveau_oclass
+nvc0_copy1_oclass = {
+	.handle = NV_ENGINE(COPY1, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_copy1_ctor,
+		.dtor = _nouveau_copy_dtor,
+		.init = nvc0_copy_init,
+		.fini = nvc0_copy_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c b/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c
new file mode 100644
index 00000000000..2017c1579ac
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nve0.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/enum.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/copy.h>
+
+struct nve0_copy_priv {
+	struct nouveau_copy base;
+};
+
+struct nve0_copy_chan {
+	struct nouveau_copy_chan base;
+};
+
+/*******************************************************************************
+ * Copy object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nve0_copy_sclass[] = {
+	{ 0xa0b5, &nouveau_object_ofuncs },
+	{},
+};
+
+/*******************************************************************************
+ * PCOPY context
+ ******************************************************************************/
+
+static int
+nve0_copy_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nve0_copy_chan *priv;
+	int ret;
+
+	ret = nouveau_copy_context_create(parent, engine, oclass, NULL, 256,
+					  256, NVOBJ_FLAG_ZERO_ALLOC, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nve0_copy_context_ofuncs = {
+	.ctor = nve0_copy_context_ctor,
+	.dtor = _nouveau_copy_context_dtor,
+	.init = _nouveau_copy_context_init,
+	.fini = _nouveau_copy_context_fini,
+	.rd32 = _nouveau_copy_context_rd32,
+	.wr32 = _nouveau_copy_context_wr32,
+};
+
+static struct nouveau_oclass
+nve0_copy_cclass = {
+	.handle = NV_ENGCTX(COPY0, 0xc0),
+	.ofuncs = &nve0_copy_context_ofuncs,
+};
+
+/*******************************************************************************
+ * PCOPY engine/subdev functions
+ ******************************************************************************/
+
+static int
+nve0_copy0_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nve0_copy_priv *priv;
+	int ret;
+
+	if (nv_rd32(parent, 0x022500) & 0x00000100)
+		return -ENODEV;
+
+	ret = nouveau_copy_create(parent, engine, oclass, true, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000040;
+	nv_engine(priv)->cclass = &nve0_copy_cclass;
+	nv_engine(priv)->sclass = nve0_copy_sclass;
+	return 0;
+}
+
+static int
+nve0_copy1_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nve0_copy_priv *priv;
+	int ret;
+
+	if (nv_rd32(parent, 0x022500) & 0x00000200)
+		return -ENODEV;
+
+	ret = nouveau_copy_create(parent, engine, oclass, true, 1, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000080;
+	nv_engine(priv)->cclass = &nve0_copy_cclass;
+	nv_engine(priv)->sclass = nve0_copy_sclass;
+	return 0;
+}
+
+struct nouveau_oclass
+nve0_copy0_oclass = {
+	.handle = NV_ENGINE(COPY0, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_copy0_ctor,
+		.dtor = _nouveau_copy_dtor,
+		.init = _nouveau_copy_init,
+		.fini = _nouveau_copy_fini,
+	},
+};
+
+struct nouveau_oclass
+nve0_copy1_oclass = {
+	.handle = NV_ENGINE(COPY1, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_copy1_ctor,
+		.dtor = _nouveau_copy_dtor,
+		.init = _nouveau_copy_init,
+		.fini = _nouveau_copy_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nv98_crypt.fuc b/drivers/gpu/drm/nouveau/core/engine/crypt/fuc/nv98.fuc
index 7393813044d..629da02dc35 100644
--- a/drivers/gpu/drm/nouveau/nv98_crypt.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/fuc/nv98.fuc
@@ -238,7 +238,7 @@ ih:
 			cmpu b32 $r4 0x60+#dma_count
 			bra nc #illegal_mthd
 			shl b32 $r5 $r4 2
-			add b32 $r5 (#ctx_dma - 0x60 * 4) & 0xffff
+			add b32 $r5 ((#ctx_dma - 0x60 * 4) & 0xffff)
 			bset $r3 0x1e
 			st b32 D[$r5] $r3
 			add b32 $r4 0x180 - 0x60
diff --git a/drivers/gpu/drm/nouveau/nv98_crypt.fuc.h b/drivers/gpu/drm/nouveau/core/engine/crypt/fuc/nv98.fuc.h
index 38676c74e6e..09962e4210e 100644
--- a/drivers/gpu/drm/nouveau/nv98_crypt.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/fuc/nv98.fuc.h
@@ -1,4 +1,4 @@
-uint32_t nv98_pcrypt_data[] = {
+static uint32_t nv98_pcrypt_data[] = {
 /* 0x0000: ctx_dma */
 /* 0x0000: ctx_dma_query */
 	0x00000000,
@@ -150,7 +150,7 @@ uint32_t nv98_pcrypt_data[] = {
 	0x00000000,
 };
 
-uint32_t nv98_pcrypt_code[] = {
+static uint32_t nv98_pcrypt_code[] = {
 	0x17f004bd,
 	0x0010fe35,
 	0xf10004fe,
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
new file mode 100644
index 00000000000..1d85e5b66ca
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/enum.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/gpuobj.h>
+
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+#include <engine/crypt.h>
+
+struct nv84_crypt_priv {
+	struct nouveau_crypt base;
+};
+
+struct nv84_crypt_chan {
+	struct nouveau_crypt_chan base;
+};
+
+/*******************************************************************************
+ * Crypt object classes
+ ******************************************************************************/
+
+static int
+nv84_crypt_object_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    16, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+	nv_wo32(obj, 0x0c, 0x00000000);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv84_crypt_ofuncs = {
+	.ctor = nv84_crypt_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_oclass
+nv84_crypt_sclass[] = {
+	{ 0x74c1, &nv84_crypt_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PCRYPT context
+ ******************************************************************************/
+
+static int
+nv84_crypt_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv84_crypt_chan *priv;
+	int ret;
+
+	ret = nouveau_crypt_context_create(parent, engine, oclass, NULL, 256,
+					   0, NVOBJ_FLAG_ZERO_ALLOC, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv84_crypt_cclass = {
+	.handle = NV_ENGCTX(CRYPT, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_crypt_context_ctor,
+		.dtor = _nouveau_crypt_context_dtor,
+		.init = _nouveau_crypt_context_init,
+		.fini = _nouveau_crypt_context_fini,
+		.rd32 = _nouveau_crypt_context_rd32,
+		.wr32 = _nouveau_crypt_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PCRYPT engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_bitfield nv84_crypt_intr_mask[] = {
+	{ 0x00000001, "INVALID_STATE" },
+	{ 0x00000002, "ILLEGAL_MTHD" },
+	{ 0x00000004, "ILLEGAL_CLASS" },
+	{ 0x00000080, "QUERY" },
+	{ 0x00000100, "FAULT" },
+	{}
+};
+
+static void
+nv84_crypt_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nv84_crypt_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, 0x102130);
+	u32 mthd = nv_rd32(priv, 0x102190);
+	u32 data = nv_rd32(priv, 0x102194);
+	u32 inst = nv_rd32(priv, 0x102188) & 0x7fffffff;
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat) {
+		nv_error(priv, "");
+		nouveau_bitfield_print(nv84_crypt_intr_mask, stat);
+		printk(" ch %d [0x%010llx] mthd 0x%04x data 0x%08x\n",
+		       chid, (u64)inst << 12, mthd, data);
+	}
+
+	nv_wr32(priv, 0x102130, stat);
+	nv_wr32(priv, 0x10200c, 0x10);
+
+	nv50_fb_trap(nouveau_fb(priv), 1);
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv84_crypt_tlb_flush(struct nouveau_engine *engine)
+{
+	nv50_vm_flush_engine(&engine->base, 0x0a);
+	return 0;
+}
+
+static int
+nv84_crypt_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv84_crypt_priv *priv;
+	int ret;
+
+	ret = nouveau_crypt_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00004000;
+	nv_subdev(priv)->intr = nv84_crypt_intr;
+	nv_engine(priv)->cclass = &nv84_crypt_cclass;
+	nv_engine(priv)->sclass = nv84_crypt_sclass;
+	nv_engine(priv)->tlb_flush = nv84_crypt_tlb_flush;
+	return 0;
+}
+
+static int
+nv84_crypt_init(struct nouveau_object *object)
+{
+	struct nv84_crypt_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_crypt_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x102130, 0xffffffff);
+	nv_wr32(priv, 0x102140, 0xffffffbf);
+	nv_wr32(priv, 0x10200c, 0x00000010);
+	return 0;
+}
+
+struct nouveau_oclass
+nv84_crypt_oclass = {
+	.handle = NV_ENGINE(CRYPT, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_crypt_ctor,
+		.dtor = _nouveau_crypt_dtor,
+		.init = nv84_crypt_init,
+		.fini = _nouveau_crypt_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
new file mode 100644
index 00000000000..9e3876c89b9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/enum.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+#include <engine/crypt.h>
+
+#include "fuc/nv98.fuc.h"
+
+struct nv98_crypt_priv {
+	struct nouveau_crypt base;
+};
+
+struct nv98_crypt_chan {
+	struct nouveau_crypt_chan base;
+};
+
+/*******************************************************************************
+ * Crypt object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv98_crypt_sclass[] = {
+	{ 0x88b4, &nouveau_object_ofuncs },
+	{},
+};
+
+/*******************************************************************************
+ * PCRYPT context
+ ******************************************************************************/
+
+static int
+nv98_crypt_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv98_crypt_chan *priv;
+	int ret;
+
+	ret = nouveau_crypt_context_create(parent, engine, oclass, NULL, 256,
+					   256, NVOBJ_FLAG_ZERO_ALLOC, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv98_crypt_cclass = {
+	.handle = NV_ENGCTX(CRYPT, 0x98),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv98_crypt_context_ctor,
+		.dtor = _nouveau_crypt_context_dtor,
+		.init = _nouveau_crypt_context_init,
+		.fini = _nouveau_crypt_context_fini,
+		.rd32 = _nouveau_crypt_context_rd32,
+		.wr32 = _nouveau_crypt_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PCRYPT engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_enum nv98_crypt_isr_error_name[] = {
+	{ 0x0000, "ILLEGAL_MTHD" },
+	{ 0x0001, "INVALID_BITFIELD" },
+	{ 0x0002, "INVALID_ENUM" },
+	{ 0x0003, "QUERY" },
+	{}
+};
+
+static void
+nv98_crypt_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nv98_crypt_priv *priv = (void *)subdev;
+	u32 disp = nv_rd32(priv, 0x08701c);
+	u32 stat = nv_rd32(priv, 0x087008) & disp & ~(disp >> 16);
+	u32 inst = nv_rd32(priv, 0x087050) & 0x3fffffff;
+	u32 ssta = nv_rd32(priv, 0x087040) & 0x0000ffff;
+	u32 addr = nv_rd32(priv, 0x087040) >> 16;
+	u32 mthd = (addr & 0x07ff) << 2;
+	u32 subc = (addr & 0x3800) >> 11;
+	u32 data = nv_rd32(priv, 0x087044);
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000040) {
+		nv_error(priv, "DISPATCH_ERROR [");
+		nouveau_enum_print(nv98_crypt_isr_error_name, ssta);
+		printk("] ch %d [0x%010llx] subc %d mthd 0x%04x data 0x%08x\n",
+		       chid, (u64)inst << 12, subc, mthd, data);
+		nv_wr32(priv, 0x087004, 0x00000040);
+		stat &= ~0x00000040;
+	}
+
+	if (stat) {
+		nv_error(priv, "unhandled intr 0x%08x\n", stat);
+		nv_wr32(priv, 0x087004, stat);
+	}
+
+	nv50_fb_trap(nouveau_fb(priv), 1);
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv98_crypt_tlb_flush(struct nouveau_engine *engine)
+{
+	nv50_vm_flush_engine(&engine->base, 0x0a);
+	return 0;
+}
+
+static int
+nv98_crypt_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv98_crypt_priv *priv;
+	int ret;
+
+	ret = nouveau_crypt_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00004000;
+	nv_subdev(priv)->intr = nv98_crypt_intr;
+	nv_engine(priv)->cclass = &nv98_crypt_cclass;
+	nv_engine(priv)->sclass = nv98_crypt_sclass;
+	nv_engine(priv)->tlb_flush = nv98_crypt_tlb_flush;
+	return 0;
+}
+
+static int
+nv98_crypt_init(struct nouveau_object *object)
+{
+	struct nv98_crypt_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_crypt_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* wait for exit interrupt to signal */
+	nv_wait(priv, 0x087008, 0x00000010, 0x00000010);
+	nv_wr32(priv, 0x087004, 0x00000010);
+
+	/* upload microcode code and data segments */
+	nv_wr32(priv, 0x087ff8, 0x00100000);
+	for (i = 0; i < ARRAY_SIZE(nv98_pcrypt_code); i++)
+		nv_wr32(priv, 0x087ff4, nv98_pcrypt_code[i]);
+
+	nv_wr32(priv, 0x087ff8, 0x00000000);
+	for (i = 0; i < ARRAY_SIZE(nv98_pcrypt_data); i++)
+		nv_wr32(priv, 0x087ff4, nv98_pcrypt_data[i]);
+
+	/* start it running */
+	nv_wr32(priv, 0x08710c, 0x00000000);
+	nv_wr32(priv, 0x087104, 0x00000000); /* ENTRY */
+	nv_wr32(priv, 0x087100, 0x00000002); /* TRIGGER */
+	return 0;
+}
+
+struct nouveau_oclass
+nv98_crypt_oclass = {
+	.handle = NV_ENGINE(CRYPT, 0x98),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv98_crypt_ctor,
+		.dtor = _nouveau_crypt_dtor,
+		.init = nv98_crypt_init,
+		.fini = _nouveau_crypt_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
new file mode 100644
index 00000000000..1c919f2af89
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <engine/disp.h>
+
+struct nv04_disp_priv {
+	struct nouveau_disp base;
+};
+
+static struct nouveau_oclass
+nv04_disp_sclass[] = {
+	{},
+};
+
+static void
+nv04_disp_intr_vblank(struct nv04_disp_priv *priv, int crtc)
+{
+	struct nouveau_disp *disp = &priv->base;
+	if (disp->vblank.notify)
+		disp->vblank.notify(disp->vblank.data, crtc);
+}
+
+static void
+nv04_disp_intr(struct nouveau_subdev *subdev)
+{
+	struct nv04_disp_priv *priv = (void *)subdev;
+	u32 crtc0 = nv_rd32(priv, 0x600100);
+	u32 crtc1 = nv_rd32(priv, 0x602100);
+
+	if (crtc0 & 0x00000001) {
+		nv04_disp_intr_vblank(priv, 0);
+		nv_wr32(priv, 0x600100, 0x00000001);
+	}
+
+	if (crtc1 & 0x00000001) {
+		nv04_disp_intr_vblank(priv, 1);
+		nv_wr32(priv, 0x602100, 0x00000001);
+	}
+}
+
+static int
+nv04_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv04_disp_priv *priv;
+	int ret;
+
+	ret = nouveau_disp_create(parent, engine, oclass, "DISPLAY",
+				  "display", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->sclass = nv04_disp_sclass;
+	nv_subdev(priv)->intr = nv04_disp_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_disp_oclass = {
+	.handle = NV_ENGINE(DISP, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_disp_ctor,
+		.dtor = _nouveau_disp_dtor,
+		.init = _nouveau_disp_init,
+		.fini = _nouveau_disp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
new file mode 100644
index 00000000000..16a9afb1060
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+struct nv50_disp_priv {
+	struct nouveau_disp base;
+};
+
+static struct nouveau_oclass
+nv50_disp_sclass[] = {
+	{},
+};
+
+static void
+nv50_disp_intr_vblank(struct nv50_disp_priv *priv, int crtc)
+{
+	struct nouveau_disp *disp = &priv->base;
+	struct nouveau_software_chan *chan, *temp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&disp->vblank.lock, flags);
+	list_for_each_entry_safe(chan, temp, &disp->vblank.list, vblank.head) {
+		if (chan->vblank.crtc != crtc)
+			continue;
+
+		nv_wr32(priv, 0x001704, chan->vblank.channel);
+		nv_wr32(priv, 0x001710, 0x80000000 | chan->vblank.ctxdma);
+
+		if (nv_device(priv)->chipset == 0x50) {
+			nv_wr32(priv, 0x001570, chan->vblank.offset);
+			nv_wr32(priv, 0x001574, chan->vblank.value);
+		} else {
+			if (nv_device(priv)->chipset >= 0xc0) {
+				nv_wr32(priv, 0x06000c,
+					upper_32_bits(chan->vblank.offset));
+			}
+			nv_wr32(priv, 0x060010, chan->vblank.offset);
+			nv_wr32(priv, 0x060014, chan->vblank.value);
+		}
+
+		list_del(&chan->vblank.head);
+		if (disp->vblank.put)
+			disp->vblank.put(disp->vblank.data, crtc);
+	}
+	spin_unlock_irqrestore(&disp->vblank.lock, flags);
+
+	if (disp->vblank.notify)
+		disp->vblank.notify(disp->vblank.data, crtc);
+}
+
+static void
+nv50_disp_intr(struct nouveau_subdev *subdev)
+{
+	struct nv50_disp_priv *priv = (void *)subdev;
+	u32 stat1 = nv_rd32(priv, 0x610024);
+
+	if (stat1 & 0x00000004) {
+		nv50_disp_intr_vblank(priv, 0);
+		nv_wr32(priv, 0x610024, 0x00000004);
+		stat1 &= ~0x00000004;
+	}
+
+	if (stat1 & 0x00000008) {
+		nv50_disp_intr_vblank(priv, 1);
+		nv_wr32(priv, 0x610024, 0x00000008);
+		stat1 &= ~0x00000008;
+	}
+
+}
+
+static int
+nv50_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv50_disp_priv *priv;
+	int ret;
+
+	ret = nouveau_disp_create(parent, engine, oclass, "PDISP",
+				  "display", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->sclass = nv50_disp_sclass;
+	nv_subdev(priv)->intr = nv50_disp_intr;
+
+	INIT_LIST_HEAD(&priv->base.vblank.list);
+	spin_lock_init(&priv->base.vblank.lock);
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_disp_oclass = {
+	.handle = NV_ENGINE(DISP, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_disp_ctor,
+		.dtor = _nouveau_disp_dtor,
+		.init = _nouveau_disp_init,
+		.fini = _nouveau_disp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
new file mode 100644
index 00000000000..d93efbcf75b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bar.h>
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+struct nvd0_disp_priv {
+	struct nouveau_disp base;
+};
+
+static struct nouveau_oclass
+nvd0_disp_sclass[] = {
+	{},
+};
+
+static void
+nvd0_disp_intr_vblank(struct nvd0_disp_priv *priv, int crtc)
+{
+	struct nouveau_bar *bar = nouveau_bar(priv);
+	struct nouveau_disp *disp = &priv->base;
+	struct nouveau_software_chan *chan, *temp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&disp->vblank.lock, flags);
+	list_for_each_entry_safe(chan, temp, &disp->vblank.list, vblank.head) {
+		if (chan->vblank.crtc != crtc)
+			continue;
+
+		nv_wr32(priv, 0x001718, 0x80000000 | chan->vblank.channel);
+		bar->flush(bar);
+		nv_wr32(priv, 0x06000c, upper_32_bits(chan->vblank.offset));
+		nv_wr32(priv, 0x060010, lower_32_bits(chan->vblank.offset));
+		nv_wr32(priv, 0x060014, chan->vblank.value);
+
+		list_del(&chan->vblank.head);
+		if (disp->vblank.put)
+			disp->vblank.put(disp->vblank.data, crtc);
+	}
+	spin_unlock_irqrestore(&disp->vblank.lock, flags);
+
+	if (disp->vblank.notify)
+		disp->vblank.notify(disp->vblank.data, crtc);
+}
+
+static void
+nvd0_disp_intr(struct nouveau_subdev *subdev)
+{
+	struct nvd0_disp_priv *priv = (void *)subdev;
+	u32 intr = nv_rd32(priv, 0x610088);
+	int i;
+
+	for (i = 0; i < 4; i++) {
+		u32 mask = 0x01000000 << i;
+		if (mask & intr) {
+			u32 stat = nv_rd32(priv, 0x6100bc + (i * 0x800));
+			if (stat & 0x00000001)
+				nvd0_disp_intr_vblank(priv, i);
+			nv_mask(priv, 0x6100bc + (i * 0x800), 0, 0);
+			nv_rd32(priv, 0x6100c0 + (i * 0x800));
+		}
+	}
+}
+
+static int
+nvd0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nvd0_disp_priv *priv;
+	int ret;
+
+	ret = nouveau_disp_create(parent, engine, oclass, "PDISP",
+				  "display", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->sclass = nvd0_disp_sclass;
+	nv_subdev(priv)->intr = nvd0_disp_intr;
+
+	INIT_LIST_HEAD(&priv->base.vblank.list);
+	spin_lock_init(&priv->base.vblank.lock);
+	return 0;
+}
+
+struct nouveau_oclass
+nvd0_disp_oclass = {
+	.handle = NV_ENGINE(DISP, 0xd0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvd0_disp_ctor,
+		.dtor = _nouveau_disp_dtor,
+		.init = _nouveau_disp_init,
+		.fini = _nouveau_disp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/vga.c b/drivers/gpu/drm/nouveau/core/engine/disp/vga.c
new file mode 100644
index 00000000000..5a1c6847459
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/vga.c
@@ -0,0 +1,215 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/subdev.h>
+#include <core/device.h>
+#include <subdev/vga.h>
+
+u8
+nv_rdport(void *obj, int head, u16 port)
+{
+	struct nouveau_device *device = nv_device(obj);
+
+	if (device->card_type >= NV_50)
+		return nv_rd08(obj, 0x601000 + port);
+
+	if (port == 0x03c0 || port == 0x03c1 ||	/* AR */
+	    port == 0x03c2 || port == 0x03da ||	/* INP0 */
+	    port == 0x03d4 || port == 0x03d5)	/* CR */
+		return nv_rd08(obj, 0x601000 + (head * 0x2000) + port);
+
+	if (port == 0x03c2 || port == 0x03cc ||	/* MISC */
+	    port == 0x03c4 || port == 0x03c5 ||	/* SR */
+	    port == 0x03ce || port == 0x03cf) {	/* GR */
+		if (device->card_type < NV_40)
+			head = 0; /* CR44 selects head */
+		return nv_rd08(obj, 0x0c0000 + (head * 0x2000) + port);
+	}
+
+	nv_error(obj, "unknown vga port 0x%04x\n", port);
+	return 0x00;
+}
+
+void
+nv_wrport(void *obj, int head, u16 port, u8 data)
+{
+	struct nouveau_device *device = nv_device(obj);
+
+	if (device->card_type >= NV_50)
+		nv_wr08(obj, 0x601000 + port, data);
+	else
+	if (port == 0x03c0 || port == 0x03c1 ||	/* AR */
+	    port == 0x03c2 || port == 0x03da ||	/* INP0 */
+	    port == 0x03d4 || port == 0x03d5)	/* CR */
+		nv_wr08(obj, 0x601000 + (head * 0x2000) + port, data);
+	else
+	if (port == 0x03c2 || port == 0x03cc ||	/* MISC */
+	    port == 0x03c4 || port == 0x03c5 ||	/* SR */
+	    port == 0x03ce || port == 0x03cf) {	/* GR */
+		if (device->card_type < NV_40)
+			head = 0; /* CR44 selects head */
+		nv_wr08(obj, 0x0c0000 + (head * 0x2000) + port, data);
+	} else
+		nv_error(obj, "unknown vga port 0x%04x\n", port);
+}
+
+u8
+nv_rdvgas(void *obj, int head, u8 index)
+{
+	nv_wrport(obj, head, 0x03c4, index);
+	return nv_rdport(obj, head, 0x03c5);
+}
+
+void
+nv_wrvgas(void *obj, int head, u8 index, u8 value)
+{
+	nv_wrport(obj, head, 0x03c4, index);
+	nv_wrport(obj, head, 0x03c5, value);
+}
+
+u8
+nv_rdvgag(void *obj, int head, u8 index)
+{
+	nv_wrport(obj, head, 0x03ce, index);
+	return nv_rdport(obj, head, 0x03cf);
+}
+
+void
+nv_wrvgag(void *obj, int head, u8 index, u8 value)
+{
+	nv_wrport(obj, head, 0x03ce, index);
+	nv_wrport(obj, head, 0x03cf, value);
+}
+
+u8
+nv_rdvgac(void *obj, int head, u8 index)
+{
+	nv_wrport(obj, head, 0x03d4, index);
+	return nv_rdport(obj, head, 0x03d5);
+}
+
+void
+nv_wrvgac(void *obj, int head, u8 index, u8 value)
+{
+	nv_wrport(obj, head, 0x03d4, index);
+	nv_wrport(obj, head, 0x03d5, value);
+}
+
+u8
+nv_rdvgai(void *obj, int head, u16 port, u8 index)
+{
+	if (port == 0x03c4) return nv_rdvgas(obj, head, index);
+	if (port == 0x03ce) return nv_rdvgag(obj, head, index);
+	if (port == 0x03d4) return nv_rdvgac(obj, head, index);
+	nv_error(obj, "unknown indexed vga port 0x%04x\n", port);
+	return 0x00;
+}
+
+void
+nv_wrvgai(void *obj, int head, u16 port, u8 index, u8 value)
+{
+	if      (port == 0x03c4) nv_wrvgas(obj, head, index, value);
+	else if (port == 0x03ce) nv_wrvgag(obj, head, index, value);
+	else if (port == 0x03d4) nv_wrvgac(obj, head, index, value);
+	else nv_error(obj, "unknown indexed vga port 0x%04x\n", port);
+}
+
+bool
+nv_lockvgac(void *obj, bool lock)
+{
+	bool locked = !nv_rdvgac(obj, 0, 0x1f);
+	u8 data = lock ? 0x99 : 0x57;
+	nv_wrvgac(obj, 0, 0x1f, data);
+	if (nv_device(obj)->chipset == 0x11) {
+		if (!(nv_rd32(obj, 0x001084) & 0x10000000))
+			nv_wrvgac(obj, 1, 0x1f, data);
+	}
+	return locked;
+}
+
+/* CR44 takes values 0 (head A), 3 (head B) and 4 (heads tied)
+ * it affects only the 8 bit vga io regs, which we access using mmio at
+ * 0xc{0,2}3c*, 0x60{1,3}3*, and 0x68{1,3}3d*
+ * in general, the set value of cr44 does not matter: reg access works as
+ * expected and values can be set for the appropriate head by using a 0x2000
+ * offset as required
+ * however:
+ * a) pre nv40, the head B range of PRMVIO regs at 0xc23c* was not exposed and
+ *    cr44 must be set to 0 or 3 for accessing values on the correct head
+ *    through the common 0xc03c* addresses
+ * b) in tied mode (4) head B is programmed to the values set on head A, and
+ *    access using the head B addresses can have strange results, ergo we leave
+ *    tied mode in init once we know to what cr44 should be restored on exit
+ *
+ * the owner parameter is slightly abused:
+ * 0 and 1 are treated as head values and so the set value is (owner * 3)
+ * other values are treated as literal values to set
+ */
+u8
+nv_rdvgaowner(void *obj)
+{
+	if (nv_device(obj)->card_type < NV_50) {
+		if (nv_device(obj)->chipset == 0x11) {
+			u32 tied = nv_rd32(obj, 0x001084) & 0x10000000;
+			if (tied == 0) {
+				u8 slA = nv_rdvgac(obj, 0, 0x28) & 0x80;
+				u8 tvA = nv_rdvgac(obj, 0, 0x33) & 0x01;
+				u8 slB = nv_rdvgac(obj, 1, 0x28) & 0x80;
+				u8 tvB = nv_rdvgac(obj, 1, 0x33) & 0x01;
+				if (slA && !tvA) return 0x00;
+				if (slB && !tvB) return 0x03;
+				if (slA) return 0x00;
+				if (slB) return 0x03;
+				return 0x00;
+			}
+			return 0x04;
+		}
+
+		return nv_rdvgac(obj, 0, 0x44);
+	}
+
+	nv_error(obj, "rdvgaowner after nv4x\n");
+	return 0x00;
+}
+
+void
+nv_wrvgaowner(void *obj, u8 select)
+{
+	if (nv_device(obj)->card_type < NV_50) {
+		u8 owner = (select == 1) ? 3 : select;
+		if (nv_device(obj)->chipset == 0x11) {
+			/* workaround hw lockup bug */
+			nv_rdvgac(obj, 0, 0x1f);
+			nv_rdvgac(obj, 1, 0x1f);
+		}
+
+		nv_wrvgac(obj, 0, 0x44, owner);
+
+		if (nv_device(obj)->chipset == 0x11) {
+			nv_wrvgac(obj, 0, 0x2e, owner);
+			nv_wrvgac(obj, 0, 0x2e, owner);
+		}
+	} else
+		nv_error(obj, "wrvgaowner after nv4x\n");
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c
new file mode 100644
index 00000000000..e1f013d3976
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/base.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/class.h>
+
+#include <subdev/fb.h>
+#include <engine/dmaobj.h>
+
+int
+nouveau_dmaobj_create_(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass,
+		       void *data, u32 size, int len, void **pobject)
+{
+	struct nv_dma_class *args = data;
+	struct nouveau_dmaobj *object;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_object_create_(parent, engine, oclass, 0, len, pobject);
+	object = *pobject;
+	if (ret)
+		return ret;
+
+	switch (args->flags & NV_DMA_TARGET_MASK) {
+	case NV_DMA_TARGET_VM:
+		object->target = NV_MEM_TARGET_VM;
+		break;
+	case NV_DMA_TARGET_VRAM:
+		object->target = NV_MEM_TARGET_VRAM;
+		break;
+	case NV_DMA_TARGET_PCI:
+		object->target = NV_MEM_TARGET_PCI;
+		break;
+	case NV_DMA_TARGET_PCI_US:
+	case NV_DMA_TARGET_AGP:
+		object->target = NV_MEM_TARGET_PCI_NOSNOOP;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (args->flags & NV_DMA_ACCESS_MASK) {
+	case NV_DMA_ACCESS_VM:
+		object->access = NV_MEM_ACCESS_VM;
+		break;
+	case NV_DMA_ACCESS_RD:
+		object->access = NV_MEM_ACCESS_RO;
+		break;
+	case NV_DMA_ACCESS_WR:
+		object->access = NV_MEM_ACCESS_WO;
+		break;
+	case NV_DMA_ACCESS_RDWR:
+		object->access = NV_MEM_ACCESS_RW;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	object->start = args->start;
+	object->limit = args->limit;
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c
new file mode 100644
index 00000000000..9f4cc2f3199
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv04.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+#include <core/class.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm/nv04.h>
+
+#include <engine/dmaobj.h>
+
+struct nv04_dmaeng_priv {
+	struct nouveau_dmaeng base;
+};
+
+struct nv04_dmaobj_priv {
+	struct nouveau_dmaobj base;
+};
+
+static int
+nv04_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+		 struct nouveau_object *parent,
+		 struct nouveau_dmaobj *dmaobj,
+		 struct nouveau_gpuobj **pgpuobj)
+{
+	struct nv04_vmmgr_priv *vmm = nv04_vmmgr(dmaeng);
+	struct nouveau_gpuobj *gpuobj;
+	u32 flags0 = nv_mclass(dmaobj);
+	u32 flags2 = 0x00000000;
+	u64 offset = dmaobj->start & 0xfffff000;
+	u64 adjust = dmaobj->start & 0x00000fff;
+	u32 length = dmaobj->limit - dmaobj->start;
+	int ret;
+
+	if (dmaobj->target == NV_MEM_TARGET_VM) {
+		if (nv_object(vmm)->oclass == &nv04_vmmgr_oclass) {
+			struct nouveau_gpuobj *pgt = vmm->vm->pgt[0].obj[0];
+			if (!dmaobj->start)
+				return nouveau_gpuobj_dup(parent, pgt, pgpuobj);
+			offset  = nv_ro32(pgt, 8 + (offset >> 10));
+			offset &= 0xfffff000;
+		}
+
+		dmaobj->target = NV_MEM_TARGET_PCI;
+		dmaobj->access = NV_MEM_ACCESS_RW;
+	}
+
+	switch (dmaobj->target) {
+	case NV_MEM_TARGET_VRAM:
+		flags0 |= 0x00003000;
+		break;
+	case NV_MEM_TARGET_PCI:
+		flags0 |= 0x00023000;
+		break;
+	case NV_MEM_TARGET_PCI_NOSNOOP:
+		flags0 |= 0x00033000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dmaobj->access) {
+	case NV_MEM_ACCESS_RO:
+		flags0 |= 0x00004000;
+		break;
+	case NV_MEM_ACCESS_WO:
+		flags0 |= 0x00008000;
+	case NV_MEM_ACCESS_RW:
+		flags2 |= 0x00000002;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = nouveau_gpuobj_new(parent, parent, 16, 16, 0, &gpuobj);
+	*pgpuobj = gpuobj;
+	if (ret == 0) {
+		nv_wo32(*pgpuobj, 0x00, flags0 | (adjust << 20));
+		nv_wo32(*pgpuobj, 0x04, length);
+		nv_wo32(*pgpuobj, 0x08, flags2 | offset);
+		nv_wo32(*pgpuobj, 0x0c, flags2 | offset);
+	}
+
+	return ret;
+}
+
+static int
+nv04_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nouveau_dmaeng *dmaeng = (void *)engine;
+	struct nv04_dmaobj_priv *dmaobj;
+	struct nouveau_gpuobj *gpuobj;
+	int ret;
+
+	ret = nouveau_dmaobj_create(parent, engine, oclass,
+				    data, size, &dmaobj);
+	*pobject = nv_object(dmaobj);
+	if (ret)
+		return ret;
+
+	switch (nv_mclass(parent)) {
+	case NV_DEVICE_CLASS:
+		break;
+	case NV03_CHANNEL_DMA_CLASS:
+	case NV10_CHANNEL_DMA_CLASS:
+	case NV17_CHANNEL_DMA_CLASS:
+	case NV40_CHANNEL_DMA_CLASS:
+		ret = dmaeng->bind(dmaeng, *pobject, &dmaobj->base, &gpuobj);
+		nouveau_object_ref(NULL, pobject);
+		*pobject = nv_object(gpuobj);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static struct nouveau_ofuncs
+nv04_dmaobj_ofuncs = {
+	.ctor = nv04_dmaobj_ctor,
+	.dtor = _nouveau_dmaobj_dtor,
+	.init = _nouveau_dmaobj_init,
+	.fini = _nouveau_dmaobj_fini,
+};
+
+static struct nouveau_oclass
+nv04_dmaobj_sclass[] = {
+	{ 0x0002, &nv04_dmaobj_ofuncs },
+	{ 0x0003, &nv04_dmaobj_ofuncs },
+	{ 0x003d, &nv04_dmaobj_ofuncs },
+	{}
+};
+
+static int
+nv04_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nv04_dmaeng_priv *priv;
+	int ret;
+
+	ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.base.sclass = nv04_dmaobj_sclass;
+	priv->base.bind = nv04_dmaobj_bind;
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_dmaeng_oclass = {
+	.handle = NV_ENGINE(DMAOBJ, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_dmaeng_ctor,
+		.dtor = _nouveau_dmaeng_dtor,
+		.init = _nouveau_dmaeng_init,
+		.fini = _nouveau_dmaeng_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c
new file mode 100644
index 00000000000..045d2565e28
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nv50.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+#include <core/class.h>
+
+#include <subdev/fb.h>
+#include <engine/dmaobj.h>
+
+struct nv50_dmaeng_priv {
+	struct nouveau_dmaeng base;
+};
+
+struct nv50_dmaobj_priv {
+	struct nouveau_dmaobj base;
+};
+
+static int
+nv50_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
+		 struct nouveau_object *parent,
+		 struct nouveau_dmaobj *dmaobj,
+		 struct nouveau_gpuobj **pgpuobj)
+{
+	u32 flags = nv_mclass(dmaobj);
+	int ret;
+
+	switch (dmaobj->target) {
+	case NV_MEM_TARGET_VM:
+		flags |= 0x00000000;
+		flags |= 0x60000000; /* COMPRESSION_USEVM */
+		flags |= 0x1fc00000; /* STORAGE_TYPE_USEVM */
+		break;
+	case NV_MEM_TARGET_VRAM:
+		flags |= 0x00010000;
+		flags |= 0x00100000; /* ACCESSUS_USER_SYSTEM */
+		break;
+	case NV_MEM_TARGET_PCI:
+		flags |= 0x00020000;
+		flags |= 0x00100000; /* ACCESSUS_USER_SYSTEM */
+		break;
+	case NV_MEM_TARGET_PCI_NOSNOOP:
+		flags |= 0x00030000;
+		flags |= 0x00100000; /* ACCESSUS_USER_SYSTEM */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (dmaobj->access) {
+	case NV_MEM_ACCESS_VM:
+		break;
+	case NV_MEM_ACCESS_RO:
+		flags |= 0x00040000;
+		break;
+	case NV_MEM_ACCESS_WO:
+	case NV_MEM_ACCESS_RW:
+		flags |= 0x00080000;
+		break;
+	}
+
+	ret = nouveau_gpuobj_new(parent, parent, 24, 32, 0, pgpuobj);
+	if (ret == 0) {
+		nv_wo32(*pgpuobj, 0x00, flags);
+		nv_wo32(*pgpuobj, 0x04, lower_32_bits(dmaobj->limit));
+		nv_wo32(*pgpuobj, 0x08, lower_32_bits(dmaobj->start));
+		nv_wo32(*pgpuobj, 0x0c, upper_32_bits(dmaobj->limit) << 24 |
+					upper_32_bits(dmaobj->start));
+		nv_wo32(*pgpuobj, 0x10, 0x00000000);
+		nv_wo32(*pgpuobj, 0x14, 0x00000000);
+	}
+
+	return ret;
+}
+
+static int
+nv50_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nouveau_dmaeng *dmaeng = (void *)engine;
+	struct nv50_dmaobj_priv *dmaobj;
+	struct nouveau_gpuobj *gpuobj;
+	int ret;
+
+	ret = nouveau_dmaobj_create(parent, engine, oclass,
+				    data, size, &dmaobj);
+	*pobject = nv_object(dmaobj);
+	if (ret)
+		return ret;
+
+	switch (nv_mclass(parent)) {
+	case NV_DEVICE_CLASS:
+		break;
+	case NV50_CHANNEL_DMA_CLASS:
+	case NV84_CHANNEL_DMA_CLASS:
+	case NV50_CHANNEL_IND_CLASS:
+	case NV84_CHANNEL_IND_CLASS:
+		ret = dmaeng->bind(dmaeng, *pobject, &dmaobj->base, &gpuobj);
+		nouveau_object_ref(NULL, pobject);
+		*pobject = nv_object(gpuobj);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static struct nouveau_ofuncs
+nv50_dmaobj_ofuncs = {
+	.ctor = nv50_dmaobj_ctor,
+	.dtor = _nouveau_dmaobj_dtor,
+	.init = _nouveau_dmaobj_init,
+	.fini = _nouveau_dmaobj_fini,
+};
+
+static struct nouveau_oclass
+nv50_dmaobj_sclass[] = {
+	{ 0x0002, &nv50_dmaobj_ofuncs },
+	{ 0x0003, &nv50_dmaobj_ofuncs },
+	{ 0x003d, &nv50_dmaobj_ofuncs },
+	{}
+};
+
+static int
+nv50_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nv50_dmaeng_priv *priv;
+	int ret;
+
+	ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.base.sclass = nv50_dmaobj_sclass;
+	priv->base.bind = nv50_dmaobj_bind;
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_dmaeng_oclass = {
+	.handle = NV_ENGINE(DMAOBJ, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_dmaeng_ctor,
+		.dtor = _nouveau_dmaeng_dtor,
+		.init = _nouveau_dmaeng_init,
+		.fini = _nouveau_dmaeng_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c
new file mode 100644
index 00000000000..5baa0869553
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvc0.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+
+#include <subdev/fb.h>
+#include <engine/dmaobj.h>
+
+struct nvc0_dmaeng_priv {
+	struct nouveau_dmaeng base;
+};
+
+struct nvc0_dmaobj_priv {
+	struct nouveau_dmaobj base;
+};
+
+static int
+nvc0_dmaobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nvc0_dmaobj_priv *dmaobj;
+	int ret;
+
+	ret = nouveau_dmaobj_create(parent, engine, oclass, data, size, &dmaobj);
+	*pobject = nv_object(dmaobj);
+	if (ret)
+		return ret;
+
+	if (dmaobj->base.target != NV_MEM_TARGET_VM || dmaobj->base.start)
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nvc0_dmaobj_ofuncs = {
+	.ctor = nvc0_dmaobj_ctor,
+	.dtor = _nouveau_dmaobj_dtor,
+	.init = _nouveau_dmaobj_init,
+	.fini = _nouveau_dmaobj_fini,
+};
+
+static struct nouveau_oclass
+nvc0_dmaobj_sclass[] = {
+	{ 0x0002, &nvc0_dmaobj_ofuncs },
+	{ 0x0003, &nvc0_dmaobj_ofuncs },
+	{ 0x003d, &nvc0_dmaobj_ofuncs },
+	{}
+};
+
+static int
+nvc0_dmaeng_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nvc0_dmaeng_priv *priv;
+	int ret;
+
+	ret = nouveau_dmaeng_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.base.sclass = nvc0_dmaobj_sclass;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_dmaeng_oclass = {
+	.handle = NV_ENGINE(DMAOBJ, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_dmaeng_ctor,
+		.dtor = _nouveau_dmaeng_dtor,
+		.init = _nouveau_dmaeng_init,
+		.fini = _nouveau_dmaeng_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/base.c b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
new file mode 100644
index 00000000000..bbb43c67c2a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/handle.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+
+int
+nouveau_fifo_channel_create_(struct nouveau_object *parent,
+			     struct nouveau_object *engine,
+			     struct nouveau_oclass *oclass,
+			     int bar, u32 addr, u32 size, u32 pushbuf,
+			     u32 engmask, int len, void **ptr)
+{
+	struct nouveau_device *device = nv_device(engine);
+	struct nouveau_fifo *priv = (void *)engine;
+	struct nouveau_fifo_chan *chan;
+	struct nouveau_dmaeng *dmaeng;
+	unsigned long flags;
+	int ret;
+
+	/* create base object class */
+	ret = nouveau_namedb_create_(parent, engine, oclass, 0, NULL,
+				     engmask, len, ptr);
+	chan = *ptr;
+	if (ret)
+		return ret;
+
+	/* validate dma object representing push buffer */
+	chan->pushdma = (void *)nouveau_handle_ref(parent, pushbuf);
+	if (!chan->pushdma)
+		return -ENOENT;
+
+	dmaeng = (void *)chan->pushdma->base.engine;
+	switch (chan->pushdma->base.oclass->handle) {
+	case 0x0002:
+	case 0x003d:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (dmaeng->bind) {
+		ret = dmaeng->bind(dmaeng, parent, chan->pushdma, &chan->pushgpu);
+		if (ret)
+			return ret;
+	}
+
+	/* find a free fifo channel */
+	spin_lock_irqsave(&priv->lock, flags);
+	for (chan->chid = priv->min; chan->chid < priv->max; chan->chid++) {
+		if (!priv->channel[chan->chid]) {
+			priv->channel[chan->chid] = nv_object(chan);
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	if (chan->chid == priv->max) {
+		nv_error(priv, "no free channels\n");
+		return -ENOSPC;
+	}
+
+	/* map fifo control registers */
+	chan->user = ioremap(pci_resource_start(device->pdev, bar) + addr +
+			     (chan->chid * size), size);
+	if (!chan->user)
+		return -EFAULT;
+
+	chan->size = size;
+	return 0;
+}
+
+void
+nouveau_fifo_channel_destroy(struct nouveau_fifo_chan *chan)
+{
+	struct nouveau_fifo *priv = (void *)nv_object(chan)->engine;
+	unsigned long flags;
+
+	iounmap(chan->user);
+
+	spin_lock_irqsave(&priv->lock, flags);
+	priv->channel[chan->chid] = NULL;
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	nouveau_gpuobj_ref(NULL, &chan->pushgpu);
+	nouveau_object_ref(NULL, (struct nouveau_object **)&chan->pushdma);
+	nouveau_namedb_destroy(&chan->base);
+}
+
+void
+_nouveau_fifo_channel_dtor(struct nouveau_object *object)
+{
+	struct nouveau_fifo_chan *chan = (void *)object;
+	nouveau_fifo_channel_destroy(chan);
+}
+
+u32
+_nouveau_fifo_channel_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_fifo_chan *chan = (void *)object;
+	return ioread32_native(chan->user + addr);
+}
+
+void
+_nouveau_fifo_channel_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nouveau_fifo_chan *chan = (void *)object;
+	iowrite32_native(data, chan->user + addr);
+}
+
+static int
+nouveau_fifo_chid(struct nouveau_fifo *priv, struct nouveau_object *object)
+{
+	int engidx = nv_hclass(priv) & 0xff;
+
+	while (object && object->parent) {
+		if ( nv_iclass(object->parent, NV_ENGCTX_CLASS) &&
+		    (nv_hclass(object->parent) & 0xff) == engidx)
+			return nouveau_fifo_chan(object)->chid;
+		object = object->parent;
+	}
+
+	return -1;
+}
+
+void
+nouveau_fifo_destroy(struct nouveau_fifo *priv)
+{
+	kfree(priv->channel);
+	nouveau_engine_destroy(&priv->base);
+}
+
+int
+nouveau_fifo_create_(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass,
+		     int min, int max, int length, void **pobject)
+{
+	struct nouveau_fifo *priv;
+	int ret;
+
+	ret = nouveau_engine_create_(parent, engine, oclass, true, "PFIFO",
+				     "fifo", length, pobject);
+	priv = *pobject;
+	if (ret)
+		return ret;
+
+	priv->min = min;
+	priv->max = max;
+	priv->channel = kzalloc(sizeof(*priv->channel) * (max + 1), GFP_KERNEL);
+	if (!priv->channel)
+		return -ENOMEM;
+
+	priv->chid = nouveau_fifo_chid;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
new file mode 100644
index 00000000000..ea76e3e8c9c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
@@ -0,0 +1,630 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/namedb.h>
+#include <core/handle.h>
+#include <core/ramht.h>
+
+#include <subdev/instmem.h>
+#include <subdev/instmem/nv04.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+
+#include "nv04.h"
+
+static struct ramfc_desc
+nv04_ramfc[] = {
+	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
+	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
+	{ 16,  0, 0x08,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
+	{ 16, 16, 0x08,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
+	{ 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_STATE },
+	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
+	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_ENGINE },
+	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_PULL1 },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+int
+nv04_fifo_object_attach(struct nouveau_object *parent,
+			struct nouveau_object *object, u32 handle)
+{
+	struct nv04_fifo_priv *priv = (void *)parent->engine;
+	struct nv04_fifo_chan *chan = (void *)parent;
+	u32 context, chid = chan->base.chid;
+	int ret;
+
+	if (nv_iclass(object, NV_GPUOBJ_CLASS))
+		context = nv_gpuobj(object)->addr >> 4;
+	else
+		context = 0x00000004; /* just non-zero */
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_DMAOBJ:
+	case NVDEV_ENGINE_SW:
+		context |= 0x00000000;
+		break;
+	case NVDEV_ENGINE_GR:
+		context |= 0x00010000;
+		break;
+	case NVDEV_ENGINE_MPEG:
+		context |= 0x00020000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	context |= 0x80000000; /* valid */
+	context |= chid << 24;
+
+	mutex_lock(&nv_subdev(priv)->mutex);
+	ret = nouveau_ramht_insert(priv->ramht, chid, handle, context);
+	mutex_unlock(&nv_subdev(priv)->mutex);
+	return ret;
+}
+
+void
+nv04_fifo_object_detach(struct nouveau_object *parent, int cookie)
+{
+	struct nv04_fifo_priv *priv = (void *)parent->engine;
+	mutex_lock(&nv_subdev(priv)->mutex);
+	nouveau_ramht_remove(priv->ramht, cookie);
+	mutex_unlock(&nv_subdev(priv)->mutex);
+}
+
+int
+nv04_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *object)
+{
+	nv_engctx(object)->addr = nouveau_fifo_chan(parent)->chid;
+	return 0;
+}
+
+static int
+nv04_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nv04_fifo_priv *priv = (void *)engine;
+	struct nv04_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
+					  0x10000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->object_attach = nv04_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv04_fifo_object_detach;
+	nv_parent(chan)->context_attach = nv04_fifo_context_attach;
+	chan->ramfc = chan->base.chid * 32;
+
+	nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x10,
+			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
+			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
+#ifdef __BIG_ENDIAN
+			     NV_PFIFO_CACHE1_BIG_ENDIAN |
+#endif
+			     NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
+	return 0;
+}
+
+void
+nv04_fifo_chan_dtor(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object->engine;
+	struct nv04_fifo_chan *chan = (void *)object;
+	struct ramfc_desc *c = priv->ramfc_desc;
+
+	do {
+		nv_wo32(priv->ramfc, chan->ramfc + c->ctxp, 0x00000000);
+	} while ((++c)->bits);
+
+	nouveau_fifo_channel_destroy(&chan->base);
+}
+
+int
+nv04_fifo_chan_init(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object->engine;
+	struct nv04_fifo_chan *chan = (void *)object;
+	u32 mask = 1 << chan->base.chid;
+	unsigned long flags;
+	int ret;
+
+	ret = nouveau_fifo_channel_init(&chan->base);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	nv_mask(priv, NV04_PFIFO_MODE, mask, mask);
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return 0;
+}
+
+int
+nv04_fifo_chan_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv04_fifo_priv *priv = (void *)object->engine;
+	struct nv04_fifo_chan *chan = (void *)object;
+	struct nouveau_gpuobj *fctx = priv->ramfc;
+	struct ramfc_desc *c;
+	unsigned long flags;
+	u32 data = chan->ramfc;
+	u32 chid;
+
+	/* prevent fifo context switches */
+	spin_lock_irqsave(&priv->base.lock, flags);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 0);
+
+	/* if this channel is active, replace it with a null context */
+	chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
+	if (chid == chan->base.chid) {
+		nv_mask(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
+		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 0);
+		nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
+
+		c = priv->ramfc_desc;
+		do {
+			u32 rm = ((1ULL << c->bits) - 1) << c->regs;
+			u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
+			u32 rv = (nv_rd32(priv, c->regp) &  rm) >> c->regs;
+			u32 cv = (nv_ro32(fctx, c->ctxp + data) & ~cm);
+			nv_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
+		} while ((++c)->bits);
+
+		c = priv->ramfc_desc;
+		do {
+			nv_wr32(priv, c->regp, 0x00000000);
+		} while ((++c)->bits);
+
+		nv_wr32(priv, NV03_PFIFO_CACHE1_GET, 0);
+		nv_wr32(priv, NV03_PFIFO_CACHE1_PUT, 0);
+		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
+		nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
+		nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+	}
+
+	/* restore normal operation, after disabling dma mode */
+	nv_mask(priv, NV04_PFIFO_MODE, 1 << chan->base.chid, 0);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+
+	return nouveau_fifo_channel_fini(&chan->base, suspend);
+}
+
+static struct nouveau_ofuncs
+nv04_fifo_ofuncs = {
+	.ctor = nv04_fifo_chan_ctor,
+	.dtor = nv04_fifo_chan_dtor,
+	.init = nv04_fifo_chan_init,
+	.fini = nv04_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv04_fifo_sclass[] = {
+	{ NV03_CHANNEL_DMA_CLASS, &nv04_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+int
+nv04_fifo_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nv04_fifo_base *base;
+	int ret;
+
+	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
+				          0x1000, NVOBJ_FLAG_HEAP, &base);
+	*pobject = nv_object(base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv04_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fifo_context_ctor,
+		.dtor = _nouveau_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+void
+nv04_fifo_pause(struct nouveau_fifo *pfifo, unsigned long *pflags)
+__acquires(priv->base.lock)
+{
+	struct nv04_fifo_priv *priv = (void *)pfifo;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	*pflags = flags;
+
+	nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000000);
+	nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000);
+
+	/* in some cases the puller may be left in an inconsistent state
+	 * if you try to stop it while it's busy translating handles.
+	 * sometimes you get a CACHE_ERROR, sometimes it just fails
+	 * silently; sending incorrect instance offsets to PGRAPH after
+	 * it's started up again.
+	 *
+	 * to avoid this, we invalidate the most recently calculated
+	 * instance.
+	 */
+	if (!nv_wait(priv, NV04_PFIFO_CACHE1_PULL0,
+			   NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0x00000000))
+		nv_warn(priv, "timeout idling puller\n");
+
+	if (nv_rd32(priv, NV04_PFIFO_CACHE1_PULL0) &
+			  NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
+		nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);
+
+	nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0x00000000);
+}
+
+void
+nv04_fifo_start(struct nouveau_fifo *pfifo, unsigned long *pflags)
+__releases(priv->base.lock)
+{
+	struct nv04_fifo_priv *priv = (void *)pfifo;
+	unsigned long flags = *pflags;
+
+	nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000001);
+
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+}
+
+static const char *
+nv_dma_state_err(u32 state)
+{
+	static const char * const desc[] = {
+		"NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
+		"INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
+	};
+	return desc[(state >> 29) & 0x7];
+}
+
+static bool
+nv04_fifo_swmthd(struct nv04_fifo_priv *priv, u32 chid, u32 addr, u32 data)
+{
+	struct nv04_fifo_chan *chan = NULL;
+	struct nouveau_handle *bind;
+	const int subc = (addr >> 13) & 0x7;
+	const int mthd = addr & 0x1ffc;
+	bool handled = false;
+	unsigned long flags;
+	u32 engine;
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	if (likely(chid >= priv->base.min && chid <= priv->base.max))
+		chan = (void *)priv->base.channel[chid];
+	if (unlikely(!chan))
+		goto out;
+
+	switch (mthd) {
+	case 0x0000:
+		bind = nouveau_namedb_get(nv_namedb(chan), data);
+		if (unlikely(!bind))
+			break;
+
+		if (nv_engidx(bind->object->engine) == NVDEV_ENGINE_SW) {
+			engine = 0x0000000f << (subc * 4);
+			chan->subc[subc] = data;
+			handled = true;
+
+			nv_mask(priv, NV04_PFIFO_CACHE1_ENGINE, engine, 0);
+		}
+
+		nouveau_namedb_put(bind);
+		break;
+	default:
+		engine = nv_rd32(priv, NV04_PFIFO_CACHE1_ENGINE);
+		if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
+			break;
+
+		bind = nouveau_namedb_get(nv_namedb(chan), chan->subc[subc]);
+		if (likely(bind)) {
+			if (!nv_call(bind->object, mthd, data))
+				handled = true;
+			nouveau_namedb_put(bind);
+		}
+		break;
+	}
+
+out:
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return handled;
+}
+
+void
+nv04_fifo_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_device *device = nv_device(subdev);
+	struct nv04_fifo_priv *priv = (void *)subdev;
+	uint32_t status, reassign;
+	int cnt = 0;
+
+	reassign = nv_rd32(priv, NV03_PFIFO_CACHES) & 1;
+	while ((status = nv_rd32(priv, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
+		uint32_t chid, get;
+
+		nv_wr32(priv, NV03_PFIFO_CACHES, 0);
+
+		chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
+		get  = nv_rd32(priv, NV03_PFIFO_CACHE1_GET);
+
+		if (status & NV_PFIFO_INTR_CACHE_ERROR) {
+			uint32_t mthd, data;
+			int ptr;
+
+			/* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
+			 * wrapping on my G80 chips, but CACHE1 isn't big
+			 * enough for this much data.. Tests show that it
+			 * wraps around to the start at GET=0x800.. No clue
+			 * as to why..
+			 */
+			ptr = (get & 0x7ff) >> 2;
+
+			if (device->card_type < NV_40) {
+				mthd = nv_rd32(priv,
+					NV04_PFIFO_CACHE1_METHOD(ptr));
+				data = nv_rd32(priv,
+					NV04_PFIFO_CACHE1_DATA(ptr));
+			} else {
+				mthd = nv_rd32(priv,
+					NV40_PFIFO_CACHE1_METHOD(ptr));
+				data = nv_rd32(priv,
+					NV40_PFIFO_CACHE1_DATA(ptr));
+			}
+
+			if (!nv04_fifo_swmthd(priv, chid, mthd, data)) {
+				nv_info(priv, "CACHE_ERROR - Ch %d/%d "
+					      "Mthd 0x%04x Data 0x%08x\n",
+					chid, (mthd >> 13) & 7, mthd & 0x1ffc,
+					data);
+			}
+
+			nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
+			nv_wr32(priv, NV03_PFIFO_INTR_0,
+						NV_PFIFO_INTR_CACHE_ERROR);
+
+			nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
+				nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) & ~1);
+			nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
+			nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
+				nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) | 1);
+			nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0);
+
+			nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH,
+				nv_rd32(priv, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
+			nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+
+			status &= ~NV_PFIFO_INTR_CACHE_ERROR;
+		}
+
+		if (status & NV_PFIFO_INTR_DMA_PUSHER) {
+			u32 dma_get = nv_rd32(priv, 0x003244);
+			u32 dma_put = nv_rd32(priv, 0x003240);
+			u32 push = nv_rd32(priv, 0x003220);
+			u32 state = nv_rd32(priv, 0x003228);
+
+			if (device->card_type == NV_50) {
+				u32 ho_get = nv_rd32(priv, 0x003328);
+				u32 ho_put = nv_rd32(priv, 0x003320);
+				u32 ib_get = nv_rd32(priv, 0x003334);
+				u32 ib_put = nv_rd32(priv, 0x003330);
+
+				nv_info(priv, "DMA_PUSHER - Ch %d Get 0x%02x%08x "
+				     "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
+				     "State 0x%08x (err: %s) Push 0x%08x\n",
+					chid, ho_get, dma_get, ho_put,
+					dma_put, ib_get, ib_put, state,
+					nv_dma_state_err(state),
+					push);
+
+				/* METHOD_COUNT, in DMA_STATE on earlier chipsets */
+				nv_wr32(priv, 0x003364, 0x00000000);
+				if (dma_get != dma_put || ho_get != ho_put) {
+					nv_wr32(priv, 0x003244, dma_put);
+					nv_wr32(priv, 0x003328, ho_put);
+				} else
+				if (ib_get != ib_put) {
+					nv_wr32(priv, 0x003334, ib_put);
+				}
+			} else {
+				nv_info(priv, "DMA_PUSHER - Ch %d Get 0x%08x "
+					     "Put 0x%08x State 0x%08x (err: %s) Push 0x%08x\n",
+					chid, dma_get, dma_put, state,
+					nv_dma_state_err(state), push);
+
+				if (dma_get != dma_put)
+					nv_wr32(priv, 0x003244, dma_put);
+			}
+
+			nv_wr32(priv, 0x003228, 0x00000000);
+			nv_wr32(priv, 0x003220, 0x00000001);
+			nv_wr32(priv, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
+			status &= ~NV_PFIFO_INTR_DMA_PUSHER;
+		}
+
+		if (status & NV_PFIFO_INTR_SEMAPHORE) {
+			uint32_t sem;
+
+			status &= ~NV_PFIFO_INTR_SEMAPHORE;
+			nv_wr32(priv, NV03_PFIFO_INTR_0,
+				NV_PFIFO_INTR_SEMAPHORE);
+
+			sem = nv_rd32(priv, NV10_PFIFO_CACHE1_SEMAPHORE);
+			nv_wr32(priv, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
+
+			nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
+			nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+		}
+
+		if (device->card_type == NV_50) {
+			if (status & 0x00000010) {
+				nv50_fb_trap(nouveau_fb(priv), 1);
+				status &= ~0x00000010;
+				nv_wr32(priv, 0x002100, 0x00000010);
+			}
+		}
+
+		if (status) {
+			nv_info(priv, "unknown intr 0x%08x, ch %d\n",
+				status, chid);
+			nv_wr32(priv, NV03_PFIFO_INTR_0, status);
+			status = 0;
+		}
+
+		nv_wr32(priv, NV03_PFIFO_CACHES, reassign);
+	}
+
+	if (status) {
+		nv_info(priv, "still angry after %d spins, halt\n", cnt);
+		nv_wr32(priv, 0x002140, 0);
+		nv_wr32(priv, 0x000140, 0);
+	}
+
+	nv_wr32(priv, 0x000100, 0x00000100);
+}
+
+static int
+nv04_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *imem = nv04_instmem(parent);
+	struct nv04_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 15, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nouveau_ramht_ref(imem->ramht, &priv->ramht);
+	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
+	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv04_fifo_cclass;
+	nv_engine(priv)->sclass = nv04_fifo_sclass;
+	priv->base.pause = nv04_fifo_pause;
+	priv->base.start = nv04_fifo_start;
+	priv->ramfc_desc = nv04_ramfc;
+	return 0;
+}
+
+void
+nv04_fifo_dtor(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object;
+	nouveau_gpuobj_ref(NULL, &priv->ramfc);
+	nouveau_gpuobj_ref(NULL, &priv->ramro);
+	nouveau_ramht_ref(NULL, &priv->ramht);
+	nouveau_fifo_destroy(&priv->base);
+}
+
+int
+nv04_fifo_init(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff);
+	nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
+
+	nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
+				       ((priv->ramht->bits - 9) << 16) |
+				        (priv->ramht->base.addr >> 8));
+	nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
+	nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8);
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
+
+	nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff);
+	nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff);
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
+	nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fifo_ctor,
+		.dtor = nv04_fifo_dtor,
+		.init = nv04_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.h b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.h
new file mode 100644
index 00000000000..496a4b4fdfa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.h
@@ -0,0 +1,178 @@
+#ifndef __NV04_FIFO_H__
+#define __NV04_FIFO_H__
+
+#include <engine/fifo.h>
+
+#define NV04_PFIFO_DELAY_0                                 0x00002040
+#define NV04_PFIFO_DMA_TIMESLICE                           0x00002044
+#define NV04_PFIFO_NEXT_CHANNEL                            0x00002050
+#define NV03_PFIFO_INTR_0                                  0x00002100
+#define NV03_PFIFO_INTR_EN_0                               0x00002140
+#    define NV_PFIFO_INTR_CACHE_ERROR                          (1<<0)
+#    define NV_PFIFO_INTR_RUNOUT                               (1<<4)
+#    define NV_PFIFO_INTR_RUNOUT_OVERFLOW                      (1<<8)
+#    define NV_PFIFO_INTR_DMA_PUSHER                          (1<<12)
+#    define NV_PFIFO_INTR_DMA_PT                              (1<<16)
+#    define NV_PFIFO_INTR_SEMAPHORE                           (1<<20)
+#    define NV_PFIFO_INTR_ACQUIRE_TIMEOUT                     (1<<24)
+#define NV03_PFIFO_RAMHT                                   0x00002210
+#define NV03_PFIFO_RAMFC                                   0x00002214
+#define NV03_PFIFO_RAMRO                                   0x00002218
+#define NV40_PFIFO_RAMFC                                   0x00002220
+#define NV03_PFIFO_CACHES                                  0x00002500
+#define NV04_PFIFO_MODE                                    0x00002504
+#define NV04_PFIFO_DMA                                     0x00002508
+#define NV04_PFIFO_SIZE                                    0x0000250c
+#define NV50_PFIFO_CTX_TABLE(c)                        (0x2600+(c)*4)
+#define NV50_PFIFO_CTX_TABLE__SIZE                                128
+#define NV50_PFIFO_CTX_TABLE_CHANNEL_ENABLED                  (1<<31)
+#define NV50_PFIFO_CTX_TABLE_UNK30_BAD                        (1<<30)
+#define NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G80             0x0FFFFFFF
+#define NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G84             0x00FFFFFF
+#define NV03_PFIFO_CACHE0_PUSH0                            0x00003000
+#define NV03_PFIFO_CACHE0_PULL0                            0x00003040
+#define NV04_PFIFO_CACHE0_PULL0                            0x00003050
+#define NV04_PFIFO_CACHE0_PULL1                            0x00003054
+#define NV03_PFIFO_CACHE1_PUSH0                            0x00003200
+#define NV03_PFIFO_CACHE1_PUSH1                            0x00003204
+#define NV03_PFIFO_CACHE1_PUSH1_DMA                            (1<<8)
+#define NV40_PFIFO_CACHE1_PUSH1_DMA                           (1<<16)
+#define NV03_PFIFO_CACHE1_PUSH1_CHID_MASK                  0x0000000f
+#define NV10_PFIFO_CACHE1_PUSH1_CHID_MASK                  0x0000001f
+#define NV50_PFIFO_CACHE1_PUSH1_CHID_MASK                  0x0000007f
+#define NV03_PFIFO_CACHE1_PUT                              0x00003210
+#define NV04_PFIFO_CACHE1_DMA_PUSH                         0x00003220
+#define NV04_PFIFO_CACHE1_DMA_FETCH                        0x00003224
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_8_BYTES         0x00000000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_16_BYTES        0x00000008
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_24_BYTES        0x00000010
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_32_BYTES        0x00000018
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_40_BYTES        0x00000020
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_48_BYTES        0x00000028
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_56_BYTES        0x00000030
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_64_BYTES        0x00000038
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_72_BYTES        0x00000040
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_80_BYTES        0x00000048
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_88_BYTES        0x00000050
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_96_BYTES        0x00000058
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_104_BYTES       0x00000060
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_112_BYTES       0x00000068
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_120_BYTES       0x00000070
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES       0x00000078
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_136_BYTES       0x00000080
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_144_BYTES       0x00000088
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_152_BYTES       0x00000090
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_160_BYTES       0x00000098
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_168_BYTES       0x000000A0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_176_BYTES       0x000000A8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_184_BYTES       0x000000B0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_192_BYTES       0x000000B8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_200_BYTES       0x000000C0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_208_BYTES       0x000000C8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_216_BYTES       0x000000D0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_224_BYTES       0x000000D8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_232_BYTES       0x000000E0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_240_BYTES       0x000000E8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_248_BYTES       0x000000F0
+#    define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_256_BYTES       0x000000F8
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE                 0x0000E000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_32_BYTES        0x00000000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_64_BYTES        0x00002000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_96_BYTES        0x00004000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES       0x00006000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_160_BYTES       0x00008000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_192_BYTES       0x0000A000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_224_BYTES       0x0000C000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_256_BYTES       0x0000E000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS             0x001F0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_0           0x00000000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_1           0x00010000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_2           0x00020000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_3           0x00030000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_4           0x00040000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_5           0x00050000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_6           0x00060000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_7           0x00070000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8           0x00080000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_9           0x00090000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_10          0x000A0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_11          0x000B0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_12          0x000C0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_13          0x000D0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_14          0x000E0000
+#    define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_15          0x000F0000
+#    define NV_PFIFO_CACHE1_ENDIAN                         0x80000000
+#    define NV_PFIFO_CACHE1_LITTLE_ENDIAN                  0x7FFFFFFF
+#    define NV_PFIFO_CACHE1_BIG_ENDIAN                     0x80000000
+#define NV04_PFIFO_CACHE1_DMA_STATE                        0x00003228
+#define NV04_PFIFO_CACHE1_DMA_INSTANCE                     0x0000322c
+#define NV04_PFIFO_CACHE1_DMA_CTL                          0x00003230
+#define NV04_PFIFO_CACHE1_DMA_PUT                          0x00003240
+#define NV04_PFIFO_CACHE1_DMA_GET                          0x00003244
+#define NV10_PFIFO_CACHE1_REF_CNT                          0x00003248
+#define NV10_PFIFO_CACHE1_DMA_SUBROUTINE                   0x0000324C
+#define NV03_PFIFO_CACHE1_PULL0                            0x00003240
+#define NV04_PFIFO_CACHE1_PULL0                            0x00003250
+#    define NV04_PFIFO_CACHE1_PULL0_HASH_FAILED            0x00000010
+#    define NV04_PFIFO_CACHE1_PULL0_HASH_BUSY              0x00001000
+#define NV03_PFIFO_CACHE1_PULL1                            0x00003250
+#define NV04_PFIFO_CACHE1_PULL1                            0x00003254
+#define NV04_PFIFO_CACHE1_HASH                             0x00003258
+#define NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT                  0x00003260
+#define NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP                0x00003264
+#define NV10_PFIFO_CACHE1_ACQUIRE_VALUE                    0x00003268
+#define NV10_PFIFO_CACHE1_SEMAPHORE                        0x0000326C
+#define NV03_PFIFO_CACHE1_GET                              0x00003270
+#define NV04_PFIFO_CACHE1_ENGINE                           0x00003280
+#define NV04_PFIFO_CACHE1_DMA_DCOUNT                       0x000032A0
+#define NV40_PFIFO_GRCTX_INSTANCE                          0x000032E0
+#define NV40_PFIFO_UNK32E4                                 0x000032E4
+#define NV04_PFIFO_CACHE1_METHOD(i)                (0x00003800+(i*8))
+#define NV04_PFIFO_CACHE1_DATA(i)                  (0x00003804+(i*8))
+#define NV40_PFIFO_CACHE1_METHOD(i)                (0x00090000+(i*8))
+#define NV40_PFIFO_CACHE1_DATA(i)                  (0x00090004+(i*8))
+
+struct ramfc_desc {
+	unsigned bits:6;
+	unsigned ctxs:5;
+	unsigned ctxp:8;
+	unsigned regs:5;
+	unsigned regp;
+};
+
+struct nv04_fifo_priv {
+	struct nouveau_fifo base;
+	struct ramfc_desc *ramfc_desc;
+	struct nouveau_ramht  *ramht;
+	struct nouveau_gpuobj *ramro;
+	struct nouveau_gpuobj *ramfc;
+};
+
+struct nv04_fifo_base {
+	struct nouveau_fifo_base base;
+};
+
+struct nv04_fifo_chan {
+	struct nouveau_fifo_chan base;
+	u32 subc[8];
+	u32 ramfc;
+};
+
+int  nv04_fifo_object_attach(struct nouveau_object *,
+			     struct nouveau_object *, u32);
+void nv04_fifo_object_detach(struct nouveau_object *, int);
+
+void nv04_fifo_chan_dtor(struct nouveau_object *);
+int  nv04_fifo_chan_init(struct nouveau_object *);
+int  nv04_fifo_chan_fini(struct nouveau_object *, bool suspend);
+
+int  nv04_fifo_context_ctor(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, void *, u32,
+			    struct nouveau_object **);
+
+void nv04_fifo_dtor(struct nouveau_object *);
+int  nv04_fifo_init(struct nouveau_object *);
+void nv04_fifo_pause(struct nouveau_fifo *, unsigned long *);
+void nv04_fifo_start(struct nouveau_fifo *, unsigned long *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
new file mode 100644
index 00000000000..4ba75422b89
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/ramht.h>
+
+#include <subdev/instmem.h>
+#include <subdev/instmem/nv04.h>
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+
+#include "nv04.h"
+
+static struct ramfc_desc
+nv10_ramfc[] = {
+	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
+	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
+	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
+	{ 16,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
+	{ 16, 16, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
+	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_STATE },
+	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
+	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_ENGINE },
+	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_PULL1 },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static int
+nv10_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nv04_fifo_priv *priv = (void *)engine;
+	struct nv04_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
+					  0x10000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->object_attach = nv04_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv04_fifo_object_detach;
+	nv_parent(chan)->context_attach = nv04_fifo_context_attach;
+	chan->ramfc = chan->base.chid * 32;
+
+	nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x14,
+			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
+			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
+#ifdef __BIG_ENDIAN
+			     NV_PFIFO_CACHE1_BIG_ENDIAN |
+#endif
+			     NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv10_fifo_ofuncs = {
+	.ctor = nv10_fifo_chan_ctor,
+	.dtor = nv04_fifo_chan_dtor,
+	.init = nv04_fifo_chan_init,
+	.fini = nv04_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv10_fifo_sclass[] = {
+	{ NV10_CHANNEL_DMA_CLASS, &nv10_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv10_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fifo_context_ctor,
+		.dtor = _nouveau_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static int
+nv10_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *imem = nv04_instmem(parent);
+	struct nv04_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nouveau_ramht_ref(imem->ramht, &priv->ramht);
+	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
+	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv10_fifo_cclass;
+	nv_engine(priv)->sclass = nv10_fifo_sclass;
+	priv->base.pause = nv04_fifo_pause;
+	priv->base.start = nv04_fifo_start;
+	priv->ramfc_desc = nv10_ramfc;
+	return 0;
+}
+
+struct nouveau_oclass
+nv10_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_fifo_ctor,
+		.dtor = nv04_fifo_dtor,
+		.init = nv04_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
new file mode 100644
index 00000000000..b96e6b0ae2b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/ramht.h>
+
+#include <subdev/instmem.h>
+#include <subdev/instmem/nv04.h>
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+
+#include "nv04.h"
+
+static struct ramfc_desc
+nv17_ramfc[] = {
+	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
+	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
+	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
+	{ 16,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
+	{ 16, 16, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
+	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_STATE },
+	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
+	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_ENGINE },
+	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_PULL1 },
+	{ 32,  0, 0x20,  0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE },
+	{ 32,  0, 0x24,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP },
+	{ 32,  0, 0x28,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT },
+	{ 32,  0, 0x2c,  0, NV10_PFIFO_CACHE1_SEMAPHORE },
+	{ 32,  0, 0x30,  0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static int
+nv17_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nv04_fifo_priv *priv = (void *)engine;
+	struct nv04_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
+					  0x10000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG), /* NV31- */
+					  &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->object_attach = nv04_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv04_fifo_object_detach;
+	nv_parent(chan)->context_attach = nv04_fifo_context_attach;
+	chan->ramfc = chan->base.chid * 64;
+
+	nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x14,
+			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
+			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
+#ifdef __BIG_ENDIAN
+			     NV_PFIFO_CACHE1_BIG_ENDIAN |
+#endif
+			     NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv17_fifo_ofuncs = {
+	.ctor = nv17_fifo_chan_ctor,
+	.dtor = nv04_fifo_chan_dtor,
+	.init = nv04_fifo_chan_init,
+	.fini = nv04_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv17_fifo_sclass[] = {
+	{ NV17_CHANNEL_DMA_CLASS, &nv17_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv17_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x17),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fifo_context_ctor,
+		.dtor = _nouveau_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static int
+nv17_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *imem = nv04_instmem(parent);
+	struct nv04_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nouveau_ramht_ref(imem->ramht, &priv->ramht);
+	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
+	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv17_fifo_cclass;
+	nv_engine(priv)->sclass = nv17_fifo_sclass;
+	priv->base.pause = nv04_fifo_pause;
+	priv->base.start = nv04_fifo_start;
+	priv->ramfc_desc = nv17_ramfc;
+	return 0;
+}
+
+static int
+nv17_fifo_init(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff);
+	nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
+
+	nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
+				       ((priv->ramht->bits - 9) << 16) |
+				        (priv->ramht->base.addr >> 8));
+	nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
+	nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8 | 0x00010000);
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
+
+	nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff);
+	nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff);
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
+	nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
+	return 0;
+}
+
+struct nouveau_oclass
+nv17_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x17),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv17_fifo_ctor,
+		.dtor = nv04_fifo_dtor,
+		.init = nv17_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
new file mode 100644
index 00000000000..559c3b4e1b8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
@@ -0,0 +1,349 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/ramht.h>
+
+#include <subdev/instmem.h>
+#include <subdev/instmem/nv04.h>
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+
+#include "nv04.h"
+
+static struct ramfc_desc
+nv40_ramfc[] = {
+	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
+	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
+	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
+	{ 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
+	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
+	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_STATE },
+	{ 28,  0, 0x18,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
+	{  2, 28, 0x18, 28, 0x002058 },
+	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_ENGINE },
+	{ 32,  0, 0x20,  0, NV04_PFIFO_CACHE1_PULL1 },
+	{ 32,  0, 0x24,  0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE },
+	{ 32,  0, 0x28,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP },
+	{ 32,  0, 0x2c,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT },
+	{ 32,  0, 0x30,  0, NV10_PFIFO_CACHE1_SEMAPHORE },
+	{ 32,  0, 0x34,  0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE },
+	{ 32,  0, 0x38,  0, NV40_PFIFO_GRCTX_INSTANCE },
+	{ 17,  0, 0x3c,  0, NV04_PFIFO_DMA_TIMESLICE },
+	{ 32,  0, 0x40,  0, 0x0032e4 },
+	{ 32,  0, 0x44,  0, 0x0032e8 },
+	{ 32,  0, 0x4c,  0, 0x002088 },
+	{ 32,  0, 0x50,  0, 0x003300 },
+	{ 32,  0, 0x54,  0, 0x00330c },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static int
+nv40_fifo_object_attach(struct nouveau_object *parent,
+			struct nouveau_object *object, u32 handle)
+{
+	struct nv04_fifo_priv *priv = (void *)parent->engine;
+	struct nv04_fifo_chan *chan = (void *)parent;
+	u32 context, chid = chan->base.chid;
+	int ret;
+
+	if (nv_iclass(object, NV_GPUOBJ_CLASS))
+		context = nv_gpuobj(object)->addr >> 4;
+	else
+		context = 0x00000004; /* just non-zero */
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_DMAOBJ:
+	case NVDEV_ENGINE_SW:
+		context |= 0x00000000;
+		break;
+	case NVDEV_ENGINE_GR:
+		context |= 0x00100000;
+		break;
+	case NVDEV_ENGINE_MPEG:
+		context |= 0x00200000;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	context |= chid << 23;
+
+	mutex_lock(&nv_subdev(priv)->mutex);
+	ret = nouveau_ramht_insert(priv->ramht, chid, handle, context);
+	mutex_unlock(&nv_subdev(priv)->mutex);
+	return ret;
+}
+
+static int
+nv40_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *engctx)
+{
+	struct nv04_fifo_priv *priv = (void *)parent->engine;
+	struct nv04_fifo_chan *chan = (void *)parent;
+	unsigned long flags;
+	u32 reg, ctx;
+
+	switch (nv_engidx(engctx->engine)) {
+	case NVDEV_ENGINE_SW:
+		return 0;
+	case NVDEV_ENGINE_GR:
+		reg = 0x32e0;
+		ctx = 0x38;
+		break;
+	case NVDEV_ENGINE_MPEG:
+		reg = 0x330c;
+		ctx = 0x54;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	nv_engctx(engctx)->addr = nv_gpuobj(engctx)->addr >> 4;
+	nv_mask(priv, 0x002500, 0x00000001, 0x00000000);
+
+	if ((nv_rd32(priv, 0x003204) & priv->base.max) == chan->base.chid)
+		nv_wr32(priv, reg, nv_engctx(engctx)->addr);
+	nv_wo32(priv->ramfc, chan->ramfc + ctx, nv_engctx(engctx)->addr);
+
+	nv_mask(priv, 0x002500, 0x00000001, 0x00000001);
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return 0;
+}
+
+static int
+nv40_fifo_context_detach(struct nouveau_object *parent, bool suspend,
+			 struct nouveau_object *engctx)
+{
+	struct nv04_fifo_priv *priv = (void *)parent->engine;
+	struct nv04_fifo_chan *chan = (void *)parent;
+	unsigned long flags;
+	u32 reg, ctx;
+
+	switch (nv_engidx(engctx->engine)) {
+	case NVDEV_ENGINE_SW:
+		return 0;
+	case NVDEV_ENGINE_GR:
+		reg = 0x32e0;
+		ctx = 0x38;
+		break;
+	case NVDEV_ENGINE_MPEG:
+		reg = 0x330c;
+		ctx = 0x54;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	nv_mask(priv, 0x002500, 0x00000001, 0x00000000);
+
+	if ((nv_rd32(priv, 0x003204) & priv->base.max) == chan->base.chid)
+		nv_wr32(priv, reg, 0x00000000);
+	nv_wo32(priv->ramfc, chan->ramfc + ctx, 0x00000000);
+
+	nv_mask(priv, 0x002500, 0x00000001, 0x00000001);
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return 0;
+}
+
+static int
+nv40_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nv04_fifo_priv *priv = (void *)engine;
+	struct nv04_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
+					  0x1000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nv40_fifo_context_attach;
+	nv_parent(chan)->context_detach = nv40_fifo_context_detach;
+	nv_parent(chan)->object_attach = nv40_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv04_fifo_object_detach;
+	chan->ramfc = chan->base.chid * 128;
+
+	nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->offset);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x18, 0x30000000 |
+			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
+			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
+#ifdef __BIG_ENDIAN
+			     NV_PFIFO_CACHE1_BIG_ENDIAN |
+#endif
+			     NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
+	nv_wo32(priv->ramfc, chan->ramfc + 0x3c, 0x0001ffff);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv40_fifo_ofuncs = {
+	.ctor = nv40_fifo_chan_ctor,
+	.dtor = nv04_fifo_chan_dtor,
+	.init = nv04_fifo_chan_init,
+	.fini = nv04_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv40_fifo_sclass[] = {
+	{ NV40_CHANNEL_DMA_CLASS, &nv40_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv40_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fifo_context_ctor,
+		.dtor = _nouveau_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static int
+nv40_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *imem = nv04_instmem(parent);
+	struct nv04_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nouveau_ramht_ref(imem->ramht, &priv->ramht);
+	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
+	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv40_fifo_cclass;
+	nv_engine(priv)->sclass = nv40_fifo_sclass;
+	priv->base.pause = nv04_fifo_pause;
+	priv->base.start = nv04_fifo_start;
+	priv->ramfc_desc = nv40_ramfc;
+	return 0;
+}
+
+static int
+nv40_fifo_init(struct nouveau_object *object)
+{
+	struct nv04_fifo_priv *priv = (void *)object;
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	int ret;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x002040, 0x000000ff);
+	nv_wr32(priv, 0x002044, 0x2101ffff);
+	nv_wr32(priv, 0x002058, 0x00000001);
+
+	nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
+				       ((priv->ramht->bits - 9) << 16) |
+				        (priv->ramht->base.addr >> 8));
+	nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
+
+	switch (nv_device(priv)->chipset) {
+	case 0x47:
+	case 0x49:
+	case 0x4b:
+		nv_wr32(priv, 0x002230, 0x00000001);
+	case 0x40:
+	case 0x41:
+	case 0x42:
+	case 0x43:
+	case 0x45:
+	case 0x48:
+		nv_wr32(priv, 0x002220, 0x00030002);
+		break;
+	default:
+		nv_wr32(priv, 0x002230, 0x00000000);
+		nv_wr32(priv, 0x002220, ((pfb->ram.size - 512 * 1024 +
+					 priv->ramfc->addr) >> 16) |
+					0x00030000);
+		break;
+	}
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
+
+	nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff);
+	nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff);
+
+	nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
+	nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
+	nv_wr32(priv, NV03_PFIFO_CACHES, 1);
+	return 0;
+}
+
+struct nouveau_oclass
+nv40_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_fifo_ctor,
+		.dtor = nv04_fifo_dtor,
+		.init = nv40_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
new file mode 100644
index 00000000000..536e7634a00
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
@@ -0,0 +1,502 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/client.h>
+#include <core/engctx.h>
+#include <core/ramht.h>
+#include <core/class.h>
+#include <core/math.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+
+#include "nv50.h"
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+void
+nv50_fifo_playlist_update(struct nv50_fifo_priv *priv)
+{
+	struct nouveau_bar *bar = nouveau_bar(priv);
+	struct nouveau_gpuobj *cur;
+	int i, p;
+
+	cur = priv->playlist[priv->cur_playlist];
+	priv->cur_playlist = !priv->cur_playlist;
+
+	for (i = priv->base.min, p = 0; i < priv->base.max; i++) {
+		if (nv_rd32(priv, 0x002600 + (i * 4)) & 0x80000000)
+			nv_wo32(cur, p++ * 4, i);
+	}
+
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x0032f4, cur->addr >> 12);
+	nv_wr32(priv, 0x0032ec, p);
+	nv_wr32(priv, 0x002500, 0x00000101);
+}
+
+static int
+nv50_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent->parent;
+	struct nouveau_gpuobj *ectx = (void *)object;
+	u64 limit = ectx->addr + ectx->size - 1;
+	u64 start = ectx->addr;
+	u32 addr;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : addr = 0x0000; break;
+	case NVDEV_ENGINE_MPEG : addr = 0x0060; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
+	nv_wo32(base->eng, addr + 0x00, 0x00190000);
+	nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit));
+	nv_wo32(base->eng, addr + 0x08, lower_32_bits(start));
+	nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 |
+					upper_32_bits(start));
+	nv_wo32(base->eng, addr + 0x10, 0x00000000);
+	nv_wo32(base->eng, addr + 0x14, 0x00000000);
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nv50_fifo_context_detach(struct nouveau_object *parent, bool suspend,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_priv *priv = (void *)parent->engine;
+	struct nv50_fifo_base *base = (void *)parent->parent;
+	struct nv50_fifo_chan *chan = (void *)parent;
+	u32 addr, me;
+	int ret = 0;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : addr = 0x0000; break;
+	case NVDEV_ENGINE_MPEG : addr = 0x0060; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_wo32(base->eng, addr + 0x00, 0x00000000);
+	nv_wo32(base->eng, addr + 0x04, 0x00000000);
+	nv_wo32(base->eng, addr + 0x08, 0x00000000);
+	nv_wo32(base->eng, addr + 0x0c, 0x00000000);
+	nv_wo32(base->eng, addr + 0x10, 0x00000000);
+	nv_wo32(base->eng, addr + 0x14, 0x00000000);
+	bar->flush(bar);
+
+	/* HW bug workaround:
+	 *
+	 * PFIFO will hang forever if the connected engines don't report
+	 * that they've processed the context switch request.
+	 *
+	 * In order for the kickoff to work, we need to ensure all the
+	 * connected engines are in a state where they can answer.
+	 *
+	 * Newer chipsets don't seem to suffer from this issue, and well,
+	 * there's also a "ignore these engines" bitmask reg we can use
+	 * if we hit the issue there..
+	 */
+	me = nv_mask(priv, 0x00b860, 0x00000001, 0x00000001);
+
+	/* do the kickoff... */
+	nv_wr32(priv, 0x0032fc, nv_gpuobj(base)->addr >> 12);
+	if (!nv_wait_ne(priv, 0x0032fc, 0xffffffff, 0xffffffff)) {
+		nv_error(priv, "channel %d unload timeout\n", chan->base.chid);
+		if (suspend)
+			ret = -EBUSY;
+	}
+
+	nv_wr32(priv, 0x00b860, me);
+	return ret;
+}
+
+static int
+nv50_fifo_object_attach(struct nouveau_object *parent,
+			struct nouveau_object *object, u32 handle)
+{
+	struct nv50_fifo_chan *chan = (void *)parent;
+	u32 context;
+
+	if (nv_iclass(object, NV_GPUOBJ_CLASS))
+		context = nv_gpuobj(object)->node->offset >> 4;
+	else
+		context = 0x00000004; /* just non-zero */
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_DMAOBJ:
+	case NVDEV_ENGINE_SW    : context |= 0x00000000; break;
+	case NVDEV_ENGINE_GR    : context |= 0x00100000; break;
+	case NVDEV_ENGINE_MPEG  : context |= 0x00200000; break;
+	default:
+		return -EINVAL;
+	}
+
+	return nouveau_ramht_insert(chan->ramht, 0, handle, context);
+}
+
+void
+nv50_fifo_object_detach(struct nouveau_object *parent, int cookie)
+{
+	struct nv50_fifo_chan *chan = (void *)parent;
+	nouveau_ramht_remove(chan->ramht, cookie);
+}
+
+static int
+nv50_fifo_chan_ctor_dma(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent;
+	struct nv50_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
+					  0x2000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nv50_fifo_context_attach;
+	nv_parent(chan)->context_detach = nv50_fifo_context_detach;
+	nv_parent(chan)->object_attach = nv50_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv50_fifo_object_detach;
+
+	ret = nouveau_ramht_new(parent, parent, 0x8000, 16, &chan->ramht);
+	if (ret)
+		return ret;
+
+	nv_wo32(base->ramfc, 0x08, lower_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x10, lower_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x14, upper_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x3c, 0x003f6078);
+	nv_wo32(base->ramfc, 0x44, 0x01003fff);
+	nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
+	nv_wo32(base->ramfc, 0x4c, 0xffffffff);
+	nv_wo32(base->ramfc, 0x60, 0x7fffffff);
+	nv_wo32(base->ramfc, 0x78, 0x00000000);
+	nv_wo32(base->ramfc, 0x7c, 0x30000001);
+	nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
+				   (4 << 24) /* SEARCH_FULL */ |
+				   (chan->ramht->base.node->offset >> 4));
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nv50_fifo_chan_ctor_ind(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv50_channel_ind_class *args = data;
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent;
+	struct nv50_fifo_chan *chan;
+	u64 ioffset, ilength;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
+					  0x2000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nv50_fifo_context_attach;
+	nv_parent(chan)->context_detach = nv50_fifo_context_detach;
+	nv_parent(chan)->object_attach = nv50_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv50_fifo_object_detach;
+
+	ret = nouveau_ramht_new(parent, parent, 0x8000, 16, &chan->ramht);
+	if (ret)
+		return ret;
+
+	ioffset = args->ioffset;
+	ilength = log2i(args->ilength / 8);
+
+	nv_wo32(base->ramfc, 0x3c, 0x403f6078);
+	nv_wo32(base->ramfc, 0x44, 0x01003fff);
+	nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
+	nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset));
+	nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16));
+	nv_wo32(base->ramfc, 0x60, 0x7fffffff);
+	nv_wo32(base->ramfc, 0x78, 0x00000000);
+	nv_wo32(base->ramfc, 0x7c, 0x30000001);
+	nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
+				   (4 << 24) /* SEARCH_FULL */ |
+				   (chan->ramht->base.node->offset >> 4));
+	bar->flush(bar);
+	return 0;
+}
+
+void
+nv50_fifo_chan_dtor(struct nouveau_object *object)
+{
+	struct nv50_fifo_chan *chan = (void *)object;
+	nouveau_ramht_ref(NULL, &chan->ramht);
+	nouveau_fifo_channel_destroy(&chan->base);
+}
+
+static int
+nv50_fifo_chan_init(struct nouveau_object *object)
+{
+	struct nv50_fifo_priv *priv = (void *)object->engine;
+	struct nv50_fifo_base *base = (void *)object->parent;
+	struct nv50_fifo_chan *chan = (void *)object;
+	struct nouveau_gpuobj *ramfc = base->ramfc;
+	u32 chid = chan->base.chid;
+	int ret;
+
+	ret = nouveau_fifo_channel_init(&chan->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 12);
+	nv50_fifo_playlist_update(priv);
+	return 0;
+}
+
+int
+nv50_fifo_chan_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv50_fifo_priv *priv = (void *)object->engine;
+	struct nv50_fifo_chan *chan = (void *)object;
+	u32 chid = chan->base.chid;
+
+	/* remove channel from playlist, fifo will unload context */
+	nv_mask(priv, 0x002600 + (chid * 4), 0x80000000, 0x00000000);
+	nv50_fifo_playlist_update(priv);
+	nv_wr32(priv, 0x002600 + (chid * 4), 0x00000000);
+
+	return nouveau_fifo_channel_fini(&chan->base, suspend);
+}
+
+static struct nouveau_ofuncs
+nv50_fifo_ofuncs_dma = {
+	.ctor = nv50_fifo_chan_ctor_dma,
+	.dtor = nv50_fifo_chan_dtor,
+	.init = nv50_fifo_chan_init,
+	.fini = nv50_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_ofuncs
+nv50_fifo_ofuncs_ind = {
+	.ctor = nv50_fifo_chan_ctor_ind,
+	.dtor = nv50_fifo_chan_dtor,
+	.init = nv50_fifo_chan_init,
+	.fini = nv50_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv50_fifo_sclass[] = {
+	{ NV50_CHANNEL_DMA_CLASS, &nv50_fifo_ofuncs_dma },
+	{ NV50_CHANNEL_IND_CLASS, &nv50_fifo_ofuncs_ind },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+static int
+nv50_fifo_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nv50_fifo_base *base;
+	int ret;
+
+	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x10000,
+				          0x1000, NVOBJ_FLAG_HEAP, &base);
+	*pobject = nv_object(base);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x0200, 0x1000,
+				 NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x1200, 0,
+				 NVOBJ_FLAG_ZERO_ALLOC, &base->eng);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x4000, 0, 0,
+				&base->pgd);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+void
+nv50_fifo_context_dtor(struct nouveau_object *object)
+{
+	struct nv50_fifo_base *base = (void *)object;
+	nouveau_vm_ref(NULL, &base->vm, base->pgd);
+	nouveau_gpuobj_ref(NULL, &base->pgd);
+	nouveau_gpuobj_ref(NULL, &base->eng);
+	nouveau_gpuobj_ref(NULL, &base->ramfc);
+	nouveau_gpuobj_ref(NULL, &base->cache);
+	nouveau_fifo_context_destroy(&base->base);
+}
+
+static struct nouveau_oclass
+nv50_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_fifo_context_ctor,
+		.dtor = nv50_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static int
+nv50_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv50_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 1, 127, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 128 * 4, 0x1000, 0,
+				&priv->playlist[0]);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 128 * 4, 0x1000, 0,
+				&priv->playlist[1]);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv50_fifo_cclass;
+	nv_engine(priv)->sclass = nv50_fifo_sclass;
+	return 0;
+}
+
+void
+nv50_fifo_dtor(struct nouveau_object *object)
+{
+	struct nv50_fifo_priv *priv = (void *)object;
+
+	nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
+	nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
+
+	nouveau_fifo_destroy(&priv->base);
+}
+
+int
+nv50_fifo_init(struct nouveau_object *object)
+{
+	struct nv50_fifo_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000000);
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000100);
+	nv_wr32(priv, 0x00250c, 0x6f3cfc34);
+	nv_wr32(priv, 0x002044, 0x01003fff);
+
+	nv_wr32(priv, 0x002100, 0xffffffff);
+	nv_wr32(priv, 0x002140, 0xffffffff);
+
+	for (i = 0; i < 128; i++)
+		nv_wr32(priv, 0x002600 + (i * 4), 0x00000000);
+	nv50_fifo_playlist_update(priv);
+
+	nv_wr32(priv, 0x003200, 0x00000001);
+	nv_wr32(priv, 0x003250, 0x00000001);
+	nv_wr32(priv, 0x002500, 0x00000001);
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_fifo_ctor,
+		.dtor = nv50_fifo_dtor,
+		.init = nv50_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.h
new file mode 100644
index 00000000000..3a9ceb315c2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.h
@@ -0,0 +1,36 @@
+#ifndef __NV50_FIFO_H__
+#define __NV50_FIFO_H__
+
+struct nv50_fifo_priv {
+	struct nouveau_fifo base;
+	struct nouveau_gpuobj *playlist[2];
+	int cur_playlist;
+};
+
+struct nv50_fifo_base {
+	struct nouveau_fifo_base base;
+	struct nouveau_gpuobj *ramfc;
+	struct nouveau_gpuobj *cache;
+	struct nouveau_gpuobj *eng;
+	struct nouveau_gpuobj *pgd;
+	struct nouveau_vm *vm;
+};
+
+struct nv50_fifo_chan {
+	struct nouveau_fifo_chan base;
+	u32 subc[8];
+	struct nouveau_ramht *ramht;
+};
+
+void nv50_fifo_playlist_update(struct nv50_fifo_priv *);
+
+void nv50_fifo_object_detach(struct nouveau_object *, int);
+void nv50_fifo_chan_dtor(struct nouveau_object *);
+int  nv50_fifo_chan_fini(struct nouveau_object *, bool);
+
+void nv50_fifo_context_dtor(struct nouveau_object *);
+
+void nv50_fifo_dtor(struct nouveau_object *);
+int  nv50_fifo_init(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
new file mode 100644
index 00000000000..b4fd26d8f16
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
@@ -0,0 +1,420 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/client.h>
+#include <core/engctx.h>
+#include <core/ramht.h>
+#include <core/class.h>
+#include <core/math.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+
+#include "nv50.h"
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static int
+nv84_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent->parent;
+	struct nouveau_gpuobj *ectx = (void *)object;
+	u64 limit = ectx->addr + ectx->size - 1;
+	u64 start = ectx->addr;
+	u32 addr;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : addr = 0x0020; break;
+	case NVDEV_ENGINE_MPEG : addr = 0x0060; break;
+	case NVDEV_ENGINE_CRYPT: addr = 0x00a0; break;
+	case NVDEV_ENGINE_COPY0: addr = 0x00c0; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
+	nv_wo32(base->eng, addr + 0x00, 0x00190000);
+	nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit));
+	nv_wo32(base->eng, addr + 0x08, lower_32_bits(start));
+	nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 |
+					upper_32_bits(start));
+	nv_wo32(base->eng, addr + 0x10, 0x00000000);
+	nv_wo32(base->eng, addr + 0x14, 0x00000000);
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nv84_fifo_context_detach(struct nouveau_object *parent, bool suspend,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_priv *priv = (void *)parent->engine;
+	struct nv50_fifo_base *base = (void *)parent->parent;
+	struct nv50_fifo_chan *chan = (void *)parent;
+	u32 addr, save, engn;
+	bool done;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : engn = 0; addr = 0x0020; break;
+	case NVDEV_ENGINE_MPEG : engn = 1; addr = 0x0060; break;
+	case NVDEV_ENGINE_CRYPT: engn = 4; addr = 0x00a0; break;
+	case NVDEV_ENGINE_COPY0: engn = 2; addr = 0x00c0; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_wo32(base->eng, addr + 0x00, 0x00000000);
+	nv_wo32(base->eng, addr + 0x04, 0x00000000);
+	nv_wo32(base->eng, addr + 0x08, 0x00000000);
+	nv_wo32(base->eng, addr + 0x0c, 0x00000000);
+	nv_wo32(base->eng, addr + 0x10, 0x00000000);
+	nv_wo32(base->eng, addr + 0x14, 0x00000000);
+	bar->flush(bar);
+
+	save = nv_mask(priv, 0x002520, 0x0000003f, 1 << engn);
+	nv_wr32(priv, 0x0032fc, nv_gpuobj(base)->addr >> 12);
+	done = nv_wait_ne(priv, 0x0032fc, 0xffffffff, 0xffffffff);
+	nv_wr32(priv, 0x002520, save);
+	if (!done) {
+		nv_error(priv, "channel %d unload timeout\n", chan->base.chid);
+		if (suspend)
+			return -EBUSY;
+	}
+	return 0;
+}
+
+static int
+nv84_fifo_object_attach(struct nouveau_object *parent,
+			struct nouveau_object *object, u32 handle)
+{
+	struct nv50_fifo_chan *chan = (void *)parent;
+	u32 context;
+
+	if (nv_iclass(object, NV_GPUOBJ_CLASS))
+		context = nv_gpuobj(object)->node->offset >> 4;
+	else
+		context = 0x00000004; /* just non-zero */
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_DMAOBJ:
+	case NVDEV_ENGINE_SW    : context |= 0x00000000; break;
+	case NVDEV_ENGINE_GR    : context |= 0x00100000; break;
+	case NVDEV_ENGINE_MPEG  :
+	case NVDEV_ENGINE_PPP   : context |= 0x00200000; break;
+	case NVDEV_ENGINE_ME    :
+	case NVDEV_ENGINE_COPY0 : context |= 0x00300000; break;
+	case NVDEV_ENGINE_VP    : context |= 0x00400000; break;
+	case NVDEV_ENGINE_CRYPT :
+	case NVDEV_ENGINE_UNK1C1: context |= 0x00500000; break;
+	case NVDEV_ENGINE_BSP   : context |= 0x00600000; break;
+	default:
+		return -EINVAL;
+	}
+
+	return nouveau_ramht_insert(chan->ramht, 0, handle, context);
+}
+
+static int
+nv84_fifo_chan_ctor_dma(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent;
+	struct nv50_fifo_chan *chan;
+	struct nv03_channel_dma_class *args = data;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
+					  0x2000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG) |
+					  (1 << NVDEV_ENGINE_ME) |
+					  (1 << NVDEV_ENGINE_VP) |
+					  (1 << NVDEV_ENGINE_CRYPT) |
+					  (1 << NVDEV_ENGINE_BSP) |
+					  (1 << NVDEV_ENGINE_PPP) |
+					  (1 << NVDEV_ENGINE_COPY0) |
+					  (1 << NVDEV_ENGINE_UNK1C1), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	ret = nouveau_ramht_new(parent, parent, 0x8000, 16, &chan->ramht);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nv84_fifo_context_attach;
+	nv_parent(chan)->context_detach = nv84_fifo_context_detach;
+	nv_parent(chan)->object_attach = nv84_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv50_fifo_object_detach;
+
+	nv_wo32(base->ramfc, 0x08, lower_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x10, lower_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x14, upper_32_bits(args->offset));
+	nv_wo32(base->ramfc, 0x3c, 0x003f6078);
+	nv_wo32(base->ramfc, 0x44, 0x01003fff);
+	nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
+	nv_wo32(base->ramfc, 0x4c, 0xffffffff);
+	nv_wo32(base->ramfc, 0x60, 0x7fffffff);
+	nv_wo32(base->ramfc, 0x78, 0x00000000);
+	nv_wo32(base->ramfc, 0x7c, 0x30000001);
+	nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
+				   (4 << 24) /* SEARCH_FULL */ |
+				   (chan->ramht->base.node->offset >> 4));
+	nv_wo32(base->ramfc, 0x88, base->cache->addr >> 10);
+	nv_wo32(base->ramfc, 0x98, nv_gpuobj(base)->addr >> 12);
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nv84_fifo_chan_ctor_ind(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_fifo_base *base = (void *)parent;
+	struct nv50_fifo_chan *chan;
+	struct nv50_channel_ind_class *args = data;
+	u64 ioffset, ilength;
+	int ret;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000,
+					  0x2000, args->pushbuf,
+					  (1 << NVDEV_ENGINE_DMAOBJ) |
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_MPEG) |
+					  (1 << NVDEV_ENGINE_ME) |
+					  (1 << NVDEV_ENGINE_VP) |
+					  (1 << NVDEV_ENGINE_CRYPT) |
+					  (1 << NVDEV_ENGINE_BSP) |
+					  (1 << NVDEV_ENGINE_PPP) |
+					  (1 << NVDEV_ENGINE_COPY0) |
+					  (1 << NVDEV_ENGINE_UNK1C1), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	ret = nouveau_ramht_new(parent, parent, 0x8000, 16, &chan->ramht);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nv84_fifo_context_attach;
+	nv_parent(chan)->context_detach = nv84_fifo_context_detach;
+	nv_parent(chan)->object_attach = nv84_fifo_object_attach;
+	nv_parent(chan)->object_detach = nv50_fifo_object_detach;
+
+	ioffset = args->ioffset;
+	ilength = log2i(args->ilength / 8);
+
+	nv_wo32(base->ramfc, 0x3c, 0x403f6078);
+	nv_wo32(base->ramfc, 0x44, 0x01003fff);
+	nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4);
+	nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset));
+	nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16));
+	nv_wo32(base->ramfc, 0x60, 0x7fffffff);
+	nv_wo32(base->ramfc, 0x78, 0x00000000);
+	nv_wo32(base->ramfc, 0x7c, 0x30000001);
+	nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
+				   (4 << 24) /* SEARCH_FULL */ |
+				   (chan->ramht->base.node->offset >> 4));
+	nv_wo32(base->ramfc, 0x88, base->cache->addr >> 10);
+	nv_wo32(base->ramfc, 0x98, nv_gpuobj(base)->addr >> 12);
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nv84_fifo_chan_init(struct nouveau_object *object)
+{
+	struct nv50_fifo_priv *priv = (void *)object->engine;
+	struct nv50_fifo_base *base = (void *)object->parent;
+	struct nv50_fifo_chan *chan = (void *)object;
+	struct nouveau_gpuobj *ramfc = base->ramfc;
+	u32 chid = chan->base.chid;
+	int ret;
+
+	ret = nouveau_fifo_channel_init(&chan->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 8);
+	nv50_fifo_playlist_update(priv);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv84_fifo_ofuncs_dma = {
+	.ctor = nv84_fifo_chan_ctor_dma,
+	.dtor = nv50_fifo_chan_dtor,
+	.init = nv84_fifo_chan_init,
+	.fini = nv50_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_ofuncs
+nv84_fifo_ofuncs_ind = {
+	.ctor = nv84_fifo_chan_ctor_ind,
+	.dtor = nv50_fifo_chan_dtor,
+	.init = nv84_fifo_chan_init,
+	.fini = nv50_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nv84_fifo_sclass[] = {
+	{ NV84_CHANNEL_DMA_CLASS, &nv84_fifo_ofuncs_dma },
+	{ NV84_CHANNEL_IND_CLASS, &nv84_fifo_ofuncs_ind },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - basically just the instmem reserved for the channel
+ ******************************************************************************/
+
+static int
+nv84_fifo_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nv50_fifo_base *base;
+	int ret;
+
+	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x10000,
+				          0x1000, NVOBJ_FLAG_HEAP, &base);
+	*pobject = nv_object(base);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x0200, 0,
+				 NVOBJ_FLAG_ZERO_ALLOC, &base->eng);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x4000, 0,
+				 0, &base->pgd);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x1000, 0x400,
+				 NVOBJ_FLAG_ZERO_ALLOC, &base->cache);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, nv_object(base), 0x0100, 0x100,
+				 NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv84_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_fifo_context_ctor,
+		.dtor = nv50_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static int
+nv84_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv50_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 1, 127, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 128 * 4, 0x1000, 0,
+				&priv->playlist[0]);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 128 * 4, 0x1000, 0,
+				&priv->playlist[1]);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nv04_fifo_intr;
+	nv_engine(priv)->cclass = &nv84_fifo_cclass;
+	nv_engine(priv)->sclass = nv84_fifo_sclass;
+	return 0;
+}
+
+struct nouveau_oclass
+nv84_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_fifo_ctor,
+		.dtor = nv50_fifo_dtor,
+		.init = nv50_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
new file mode 100644
index 00000000000..6f21be60055
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
@@ -0,0 +1,647 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/client.h>
+#include <core/handle.h>
+#include <core/namedb.h>
+#include <core/gpuobj.h>
+#include <core/engctx.h>
+#include <core/class.h>
+#include <core/math.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+
+struct nvc0_fifo_priv {
+	struct nouveau_fifo base;
+	struct nouveau_gpuobj *playlist[2];
+	int cur_playlist;
+	struct {
+		struct nouveau_gpuobj *mem;
+		struct nouveau_vma bar;
+	} user;
+	int spoon_nr;
+};
+
+struct nvc0_fifo_base {
+	struct nouveau_fifo_base base;
+	struct nouveau_gpuobj *pgd;
+	struct nouveau_vm *vm;
+};
+
+struct nvc0_fifo_chan {
+	struct nouveau_fifo_chan base;
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static void
+nvc0_fifo_playlist_update(struct nvc0_fifo_priv *priv)
+{
+	struct nouveau_bar *bar = nouveau_bar(priv);
+	struct nouveau_gpuobj *cur;
+	int i, p;
+
+	cur = priv->playlist[priv->cur_playlist];
+	priv->cur_playlist = !priv->cur_playlist;
+
+	for (i = 0, p = 0; i < 128; i++) {
+		if (!(nv_rd32(priv, 0x003004 + (i * 8)) & 1))
+			continue;
+		nv_wo32(cur, p + 0, i);
+		nv_wo32(cur, p + 4, 0x00000004);
+		p += 8;
+	}
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x002270, cur->addr >> 12);
+	nv_wr32(priv, 0x002274, 0x01f00000 | (p >> 3));
+	if (!nv_wait(priv, 0x00227c, 0x00100000, 0x00000000))
+		nv_error(priv, "playlist update failed\n");
+}
+
+static int
+nvc0_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nvc0_fifo_base *base = (void *)parent->parent;
+	struct nouveau_engctx *ectx = (void *)object;
+	u32 addr;
+	int ret;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : addr = 0x0210; break;
+	case NVDEV_ENGINE_COPY0: addr = 0x0230; break;
+	case NVDEV_ENGINE_COPY1: addr = 0x0240; break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!ectx->vma.node) {
+		ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm,
+					    NV_MEM_ACCESS_RW, &ectx->vma);
+		if (ret)
+			return ret;
+
+		nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
+	}
+
+	nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4);
+	nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset));
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nvc0_fifo_context_detach(struct nouveau_object *parent, bool suspend,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nvc0_fifo_priv *priv = (void *)parent->engine;
+	struct nvc0_fifo_base *base = (void *)parent->parent;
+	struct nvc0_fifo_chan *chan = (void *)parent;
+	u32 addr;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   : addr = 0x0210; break;
+	case NVDEV_ENGINE_COPY0: addr = 0x0230; break;
+	case NVDEV_ENGINE_COPY1: addr = 0x0240; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_wo32(base, addr + 0x00, 0x00000000);
+	nv_wo32(base, addr + 0x04, 0x00000000);
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x002634, chan->base.chid);
+	if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) {
+		nv_error(priv, "channel %d kick timeout\n", chan->base.chid);
+		if (suspend)
+			return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int
+nvc0_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nvc0_fifo_priv *priv = (void *)engine;
+	struct nvc0_fifo_base *base = (void *)parent;
+	struct nvc0_fifo_chan *chan;
+	struct nv50_channel_ind_class *args = data;
+	u64 usermem, ioffset, ilength;
+	int ret, i;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 1,
+					  priv->user.bar.offset, 0x1000,
+					  args->pushbuf,
+					  (1 << NVDEV_ENGINE_SW) |
+					  (1 << NVDEV_ENGINE_GR) |
+					  (1 << NVDEV_ENGINE_COPY0) |
+					  (1 << NVDEV_ENGINE_COPY1), &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nvc0_fifo_context_attach;
+	nv_parent(chan)->context_detach = nvc0_fifo_context_detach;
+
+	usermem = chan->base.chid * 0x1000;
+	ioffset = args->ioffset;
+	ilength = log2i(args->ilength / 8);
+
+	for (i = 0; i < 0x1000; i += 4)
+		nv_wo32(priv->user.mem, usermem + i, 0x00000000);
+
+	nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem));
+	nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem));
+	nv_wo32(base, 0x10, 0x0000face);
+	nv_wo32(base, 0x30, 0xfffff902);
+	nv_wo32(base, 0x48, lower_32_bits(ioffset));
+	nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16));
+	nv_wo32(base, 0x54, 0x00000002);
+	nv_wo32(base, 0x84, 0x20400000);
+	nv_wo32(base, 0x94, 0x30000001);
+	nv_wo32(base, 0x9c, 0x00000100);
+	nv_wo32(base, 0xa4, 0x1f1f1f1f);
+	nv_wo32(base, 0xa8, 0x1f1f1f1f);
+	nv_wo32(base, 0xac, 0x0000001f);
+	nv_wo32(base, 0xb8, 0xf8000000);
+	nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */
+	nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nvc0_fifo_chan_init(struct nouveau_object *object)
+{
+	struct nouveau_gpuobj *base = nv_gpuobj(object->parent);
+	struct nvc0_fifo_priv *priv = (void *)object->engine;
+	struct nvc0_fifo_chan *chan = (void *)object;
+	u32 chid = chan->base.chid;
+	int ret;
+
+	ret = nouveau_fifo_channel_init(&chan->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x003000 + (chid * 8), 0xc0000000 | base->addr >> 12);
+	nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001);
+	nvc0_fifo_playlist_update(priv);
+	return 0;
+}
+
+static int
+nvc0_fifo_chan_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nvc0_fifo_priv *priv = (void *)object->engine;
+	struct nvc0_fifo_chan *chan = (void *)object;
+	u32 chid = chan->base.chid;
+
+	nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000);
+	nvc0_fifo_playlist_update(priv);
+	nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000);
+
+	return nouveau_fifo_channel_fini(&chan->base, suspend);
+}
+
+static struct nouveau_ofuncs
+nvc0_fifo_ofuncs = {
+	.ctor = nvc0_fifo_chan_ctor,
+	.dtor = _nouveau_fifo_channel_dtor,
+	.init = nvc0_fifo_chan_init,
+	.fini = nvc0_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nvc0_fifo_sclass[] = {
+	{ NVC0_CHANNEL_IND_CLASS, &nvc0_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - instmem heap and vm setup
+ ******************************************************************************/
+
+static int
+nvc0_fifo_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nvc0_fifo_base *base;
+	int ret;
+
+	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
+				          0x1000, NVOBJ_FLAG_ZERO_ALLOC |
+					  NVOBJ_FLAG_HEAP, &base);
+	*pobject = nv_object(base);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x10000, 0x1000, 0, &base->pgd);
+	if (ret)
+		return ret;
+
+	nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr));
+	nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr));
+	nv_wo32(base, 0x0208, 0xffffffff);
+	nv_wo32(base, 0x020c, 0x000000ff);
+
+	ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nvc0_fifo_context_dtor(struct nouveau_object *object)
+{
+	struct nvc0_fifo_base *base = (void *)object;
+	nouveau_vm_ref(NULL, &base->vm, base->pgd);
+	nouveau_gpuobj_ref(NULL, &base->pgd);
+	nouveau_fifo_context_destroy(&base->base);
+}
+
+static struct nouveau_oclass
+nvc0_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_fifo_context_ctor,
+		.dtor = nvc0_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static const struct nouveau_enum nvc0_fifo_fault_unit[] = {
+	{ 0x00, "PGRAPH" },
+	{ 0x03, "PEEPHOLE" },
+	{ 0x04, "BAR1" },
+	{ 0x05, "BAR3" },
+	{ 0x07, "PFIFO" },
+	{ 0x10, "PBSP" },
+	{ 0x11, "PPPP" },
+	{ 0x13, "PCOUNTER" },
+	{ 0x14, "PVP" },
+	{ 0x15, "PCOPY0" },
+	{ 0x16, "PCOPY1" },
+	{ 0x17, "PDAEMON" },
+	{}
+};
+
+static const struct nouveau_enum nvc0_fifo_fault_reason[] = {
+	{ 0x00, "PT_NOT_PRESENT" },
+	{ 0x01, "PT_TOO_SHORT" },
+	{ 0x02, "PAGE_NOT_PRESENT" },
+	{ 0x03, "VM_LIMIT_EXCEEDED" },
+	{ 0x04, "NO_CHANNEL" },
+	{ 0x05, "PAGE_SYSTEM_ONLY" },
+	{ 0x06, "PAGE_READ_ONLY" },
+	{ 0x0a, "COMPRESSED_SYSRAM" },
+	{ 0x0c, "INVALID_STORAGE_TYPE" },
+	{}
+};
+
+static const struct nouveau_enum nvc0_fifo_fault_hubclient[] = {
+	{ 0x01, "PCOPY0" },
+	{ 0x02, "PCOPY1" },
+	{ 0x04, "DISPATCH" },
+	{ 0x05, "CTXCTL" },
+	{ 0x06, "PFIFO" },
+	{ 0x07, "BAR_READ" },
+	{ 0x08, "BAR_WRITE" },
+	{ 0x0b, "PVP" },
+	{ 0x0c, "PPPP" },
+	{ 0x0d, "PBSP" },
+	{ 0x11, "PCOUNTER" },
+	{ 0x12, "PDAEMON" },
+	{ 0x14, "CCACHE" },
+	{ 0x15, "CCACHE_POST" },
+	{}
+};
+
+static const struct nouveau_enum nvc0_fifo_fault_gpcclient[] = {
+	{ 0x01, "TEX" },
+	{ 0x0c, "ESETUP" },
+	{ 0x0e, "CTXCTL" },
+	{ 0x0f, "PROP" },
+	{}
+};
+
+static const struct nouveau_bitfield nvc0_fifo_subfifo_intr[] = {
+/*	{ 0x00008000, "" }	seen with null ib push */
+	{ 0x00200000, "ILLEGAL_MTHD" },
+	{ 0x00800000, "EMPTY_SUBC" },
+	{}
+};
+
+static void
+nvc0_fifo_isr_vm_fault(struct nvc0_fifo_priv *priv, int unit)
+{
+	u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10));
+	u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10));
+	u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10));
+	u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10));
+	u32 client = (stat & 0x00001f00) >> 8;
+
+	switch (unit) {
+	case 3: /* PEEPHOLE */
+		nv_mask(priv, 0x001718, 0x00000000, 0x00000000);
+		break;
+	case 4: /* BAR1 */
+		nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
+		break;
+	case 5: /* BAR3 */
+		nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
+		break;
+	default:
+		break;
+	}
+
+	nv_error(priv, "%s fault at 0x%010llx [", (stat & 0x00000080) ?
+		 "write" : "read", (u64)vahi << 32 | valo);
+	nouveau_enum_print(nvc0_fifo_fault_reason, stat & 0x0000000f);
+	printk("] from ");
+	nouveau_enum_print(nvc0_fifo_fault_unit, unit);
+	if (stat & 0x00000040) {
+		printk("/");
+		nouveau_enum_print(nvc0_fifo_fault_hubclient, client);
+	} else {
+		printk("/GPC%d/", (stat & 0x1f000000) >> 24);
+		nouveau_enum_print(nvc0_fifo_fault_gpcclient, client);
+	}
+	printk(" on channel 0x%010llx\n", (u64)inst << 12);
+}
+
+static int
+nvc0_fifo_swmthd(struct nvc0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
+{
+	struct nvc0_fifo_chan *chan = NULL;
+	struct nouveau_handle *bind;
+	unsigned long flags;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	if (likely(chid >= priv->base.min && chid <= priv->base.max))
+		chan = (void *)priv->base.channel[chid];
+	if (unlikely(!chan))
+		goto out;
+
+	bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
+	if (likely(bind)) {
+		if (!mthd || !nv_call(bind->object, mthd, data))
+			ret = 0;
+		nouveau_namedb_put(bind);
+	}
+
+out:
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return ret;
+}
+
+static void
+nvc0_fifo_isr_subfifo_intr(struct nvc0_fifo_priv *priv, int unit)
+{
+	u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
+	u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
+	u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));
+	u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0x7f;
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00003ffc);
+	u32 show = stat;
+
+	if (stat & 0x00200000) {
+		if (mthd == 0x0054) {
+			if (!nvc0_fifo_swmthd(priv, chid, 0x0500, 0x00000000))
+				show &= ~0x00200000;
+		}
+	}
+
+	if (stat & 0x00800000) {
+		if (!nvc0_fifo_swmthd(priv, chid, mthd, data))
+			show &= ~0x00800000;
+	}
+
+	if (show) {
+		nv_error(priv, "SUBFIFO%d:", unit);
+		nouveau_bitfield_print(nvc0_fifo_subfifo_intr, show);
+		printk("\n");
+		nv_error(priv, "SUBFIFO%d: ch %d subc %d mthd 0x%04x "
+			       "data 0x%08x\n",
+			 unit, chid, subc, mthd, data);
+	}
+
+	nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
+	nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);
+}
+
+static void
+nvc0_fifo_intr(struct nouveau_subdev *subdev)
+{
+	struct nvc0_fifo_priv *priv = (void *)subdev;
+	u32 mask = nv_rd32(priv, 0x002140);
+	u32 stat = nv_rd32(priv, 0x002100) & mask;
+
+	if (stat & 0x00000100) {
+		nv_info(priv, "unknown status 0x00000100\n");
+		nv_wr32(priv, 0x002100, 0x00000100);
+		stat &= ~0x00000100;
+	}
+
+	if (stat & 0x10000000) {
+		u32 units = nv_rd32(priv, 0x00259c);
+		u32 u = units;
+
+		while (u) {
+			int i = ffs(u) - 1;
+			nvc0_fifo_isr_vm_fault(priv, i);
+			u &= ~(1 << i);
+		}
+
+		nv_wr32(priv, 0x00259c, units);
+		stat &= ~0x10000000;
+	}
+
+	if (stat & 0x20000000) {
+		u32 units = nv_rd32(priv, 0x0025a0);
+		u32 u = units;
+
+		while (u) {
+			int i = ffs(u) - 1;
+			nvc0_fifo_isr_subfifo_intr(priv, i);
+			u &= ~(1 << i);
+		}
+
+		nv_wr32(priv, 0x0025a0, units);
+		stat &= ~0x20000000;
+	}
+
+	if (stat & 0x40000000) {
+		nv_warn(priv, "unknown status 0x40000000\n");
+		nv_mask(priv, 0x002a00, 0x00000000, 0x00000000);
+		stat &= ~0x40000000;
+	}
+
+	if (stat) {
+		nv_fatal(priv, "unhandled status 0x%08x\n", stat);
+		nv_wr32(priv, 0x002100, stat);
+		nv_wr32(priv, 0x002140, 0);
+	}
+}
+
+static int
+nvc0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nvc0_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 127, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0x1000, 0,
+				&priv->playlist[0]);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0x1000, 0,
+				&priv->playlist[1]);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 128 * 0x1000, 0x1000, 0,
+				&priv->user.mem);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW,
+				&priv->user.bar);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nvc0_fifo_intr;
+	nv_engine(priv)->cclass = &nvc0_fifo_cclass;
+	nv_engine(priv)->sclass = nvc0_fifo_sclass;
+	return 0;
+}
+
+static void
+nvc0_fifo_dtor(struct nouveau_object *object)
+{
+	struct nvc0_fifo_priv *priv = (void *)object;
+
+	nouveau_gpuobj_unmap(&priv->user.bar);
+	nouveau_gpuobj_ref(NULL, &priv->user.mem);
+	nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
+	nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
+
+	nouveau_fifo_destroy(&priv->base);
+}
+
+static int
+nvc0_fifo_init(struct nouveau_object *object)
+{
+	struct nvc0_fifo_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x000204, 0xffffffff);
+	nv_wr32(priv, 0x002204, 0xffffffff);
+
+	priv->spoon_nr = hweight32(nv_rd32(priv, 0x002204));
+	nv_debug(priv, "%d subfifo(s)\n", priv->spoon_nr);
+
+	/* assign engines to subfifos */
+	if (priv->spoon_nr >= 3) {
+		nv_wr32(priv, 0x002208, ~(1 << 0)); /* PGRAPH */
+		nv_wr32(priv, 0x00220c, ~(1 << 1)); /* PVP */
+		nv_wr32(priv, 0x002210, ~(1 << 1)); /* PPP */
+		nv_wr32(priv, 0x002214, ~(1 << 1)); /* PBSP */
+		nv_wr32(priv, 0x002218, ~(1 << 2)); /* PCE0 */
+		nv_wr32(priv, 0x00221c, ~(1 << 1)); /* PCE1 */
+	}
+
+	/* PSUBFIFO[n] */
+	for (i = 0; i < priv->spoon_nr; i++) {
+		nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
+		nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
+		nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
+	}
+
+	nv_mask(priv, 0x002200, 0x00000001, 0x00000001);
+	nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
+
+	nv_wr32(priv, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
+	nv_wr32(priv, 0x002100, 0xffffffff);
+	nv_wr32(priv, 0x002140, 0xbfffffff);
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_fifo_ctor,
+		.dtor = nvc0_fifo_dtor,
+		.init = nvc0_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
new file mode 100644
index 00000000000..36e81b6fafb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -0,0 +1,628 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/client.h>
+#include <core/handle.h>
+#include <core/namedb.h>
+#include <core/gpuobj.h>
+#include <core/engctx.h>
+#include <core/class.h>
+#include <core/math.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+
+#define _(a,b) { (a), ((1 << (a)) | (b)) }
+static const struct {
+	int subdev;
+	u32 mask;
+} fifo_engine[] = {
+	_(NVDEV_ENGINE_GR      , (1 << NVDEV_ENGINE_SW)),
+	_(NVDEV_ENGINE_VP      , 0),
+	_(NVDEV_ENGINE_PPP     , 0),
+	_(NVDEV_ENGINE_BSP     , 0),
+	_(NVDEV_ENGINE_COPY0   , 0),
+	_(NVDEV_ENGINE_COPY1   , 0),
+	_(NVDEV_ENGINE_VENC    , 0),
+};
+#undef _
+#define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine)
+
+struct nve0_fifo_engn {
+	struct nouveau_gpuobj *playlist[2];
+	int cur_playlist;
+};
+
+struct nve0_fifo_priv {
+	struct nouveau_fifo base;
+	struct nve0_fifo_engn engine[FIFO_ENGINE_NR];
+	struct {
+		struct nouveau_gpuobj *mem;
+		struct nouveau_vma bar;
+	} user;
+	int spoon_nr;
+};
+
+struct nve0_fifo_base {
+	struct nouveau_fifo_base base;
+	struct nouveau_gpuobj *pgd;
+	struct nouveau_vm *vm;
+};
+
+struct nve0_fifo_chan {
+	struct nouveau_fifo_chan base;
+	u32 engine;
+};
+
+/*******************************************************************************
+ * FIFO channel objects
+ ******************************************************************************/
+
+static void
+nve0_fifo_playlist_update(struct nve0_fifo_priv *priv, u32 engine)
+{
+	struct nouveau_bar *bar = nouveau_bar(priv);
+	struct nve0_fifo_engn *engn = &priv->engine[engine];
+	struct nouveau_gpuobj *cur;
+	u32 match = (engine << 16) | 0x00000001;
+	int i, p;
+
+	cur = engn->playlist[engn->cur_playlist];
+	if (unlikely(cur == NULL)) {
+		int ret = nouveau_gpuobj_new(nv_object(priv)->parent, NULL,
+					     0x8000, 0x1000, 0, &cur);
+		if (ret) {
+			nv_error(priv, "playlist alloc failed\n");
+			return;
+		}
+
+		engn->playlist[engn->cur_playlist] = cur;
+	}
+
+	engn->cur_playlist = !engn->cur_playlist;
+
+	for (i = 0, p = 0; i < priv->base.max; i++) {
+		u32 ctrl = nv_rd32(priv, 0x800004 + (i * 8)) & 0x001f0001;
+		if (ctrl != match)
+			continue;
+		nv_wo32(cur, p + 0, i);
+		nv_wo32(cur, p + 4, 0x00000000);
+		p += 8;
+	}
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x002270, cur->addr >> 12);
+	nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3));
+	if (!nv_wait(priv, 0x002284 + (engine * 4), 0x00100000, 0x00000000))
+		nv_error(priv, "playlist %d update timeout\n", engine);
+}
+
+static int
+nve0_fifo_context_attach(struct nouveau_object *parent,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nve0_fifo_base *base = (void *)parent->parent;
+	struct nouveau_engctx *ectx = (void *)object;
+	u32 addr;
+	int ret;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   :
+	case NVDEV_ENGINE_COPY0:
+	case NVDEV_ENGINE_COPY1: addr = 0x0210; break;
+	default:
+		return -EINVAL;
+	}
+
+	if (!ectx->vma.node) {
+		ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm,
+					    NV_MEM_ACCESS_RW, &ectx->vma);
+		if (ret)
+			return ret;
+
+		nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
+	}
+
+	nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4);
+	nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset));
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend,
+			 struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nve0_fifo_priv *priv = (void *)parent->engine;
+	struct nve0_fifo_base *base = (void *)parent->parent;
+	struct nve0_fifo_chan *chan = (void *)parent;
+	u32 addr;
+
+	switch (nv_engidx(object->engine)) {
+	case NVDEV_ENGINE_SW   : return 0;
+	case NVDEV_ENGINE_GR   :
+	case NVDEV_ENGINE_COPY0:
+	case NVDEV_ENGINE_COPY1: addr = 0x0210; break;
+	default:
+		return -EINVAL;
+	}
+
+	nv_wo32(base, addr + 0x00, 0x00000000);
+	nv_wo32(base, addr + 0x04, 0x00000000);
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x002634, chan->base.chid);
+	if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) {
+		nv_error(priv, "channel %d kick timeout\n", chan->base.chid);
+		if (suspend)
+			return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int
+nve0_fifo_chan_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nve0_fifo_priv *priv = (void *)engine;
+	struct nve0_fifo_base *base = (void *)parent;
+	struct nve0_fifo_chan *chan;
+	struct nve0_channel_ind_class *args = data;
+	u64 usermem, ioffset, ilength;
+	int ret, i;
+
+	if (size < sizeof(*args))
+		return -EINVAL;
+
+	for (i = 0; i < FIFO_ENGINE_NR; i++) {
+		if (args->engine & (1 << i)) {
+			if (nouveau_engine(parent, fifo_engine[i].subdev)) {
+				args->engine = (1 << i);
+				break;
+			}
+		}
+	}
+
+	if (i == FIFO_ENGINE_NR)
+		return -ENODEV;
+
+	ret = nouveau_fifo_channel_create(parent, engine, oclass, 1,
+					  priv->user.bar.offset, 0x200,
+					  args->pushbuf,
+					  fifo_engine[i].mask, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_parent(chan)->context_attach = nve0_fifo_context_attach;
+	nv_parent(chan)->context_detach = nve0_fifo_context_detach;
+	chan->engine = i;
+
+	usermem = chan->base.chid * 0x200;
+	ioffset = args->ioffset;
+	ilength = log2i(args->ilength / 8);
+
+	for (i = 0; i < 0x200; i += 4)
+		nv_wo32(priv->user.mem, usermem + i, 0x00000000);
+
+	nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem));
+	nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem));
+	nv_wo32(base, 0x10, 0x0000face);
+	nv_wo32(base, 0x30, 0xfffff902);
+	nv_wo32(base, 0x48, lower_32_bits(ioffset));
+	nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16));
+	nv_wo32(base, 0x84, 0x20400000);
+	nv_wo32(base, 0x94, 0x30000001);
+	nv_wo32(base, 0x9c, 0x00000100);
+	nv_wo32(base, 0xac, 0x0000001f);
+	nv_wo32(base, 0xe8, chan->base.chid);
+	nv_wo32(base, 0xb8, 0xf8000000);
+	nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */
+	nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */
+	bar->flush(bar);
+	return 0;
+}
+
+static int
+nve0_fifo_chan_init(struct nouveau_object *object)
+{
+	struct nouveau_gpuobj *base = nv_gpuobj(object->parent);
+	struct nve0_fifo_priv *priv = (void *)object->engine;
+	struct nve0_fifo_chan *chan = (void *)object;
+	u32 chid = chan->base.chid;
+	int ret;
+
+	ret = nouveau_fifo_channel_init(&chan->base);
+	if (ret)
+		return ret;
+
+	nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16);
+	nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12);
+	nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
+	nve0_fifo_playlist_update(priv, chan->engine);
+	nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
+	return 0;
+}
+
+static int
+nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nve0_fifo_priv *priv = (void *)object->engine;
+	struct nve0_fifo_chan *chan = (void *)object;
+	u32 chid = chan->base.chid;
+
+	nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800);
+	nve0_fifo_playlist_update(priv, chan->engine);
+	nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000);
+
+	return nouveau_fifo_channel_fini(&chan->base, suspend);
+}
+
+static struct nouveau_ofuncs
+nve0_fifo_ofuncs = {
+	.ctor = nve0_fifo_chan_ctor,
+	.dtor = _nouveau_fifo_channel_dtor,
+	.init = nve0_fifo_chan_init,
+	.fini = nve0_fifo_chan_fini,
+	.rd32 = _nouveau_fifo_channel_rd32,
+	.wr32 = _nouveau_fifo_channel_wr32,
+};
+
+static struct nouveau_oclass
+nve0_fifo_sclass[] = {
+	{ NVE0_CHANNEL_IND_CLASS, &nve0_fifo_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * FIFO context - instmem heap and vm setup
+ ******************************************************************************/
+
+static int
+nve0_fifo_context_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nve0_fifo_base *base;
+	int ret;
+
+	ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
+				          0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base);
+	*pobject = nv_object(base);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x10000, 0x1000, 0, &base->pgd);
+	if (ret)
+		return ret;
+
+	nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr));
+	nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr));
+	nv_wo32(base, 0x0208, 0xffffffff);
+	nv_wo32(base, 0x020c, 0x000000ff);
+
+	ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nve0_fifo_context_dtor(struct nouveau_object *object)
+{
+	struct nve0_fifo_base *base = (void *)object;
+	nouveau_vm_ref(NULL, &base->vm, base->pgd);
+	nouveau_gpuobj_ref(NULL, &base->pgd);
+	nouveau_fifo_context_destroy(&base->base);
+}
+
+static struct nouveau_oclass
+nve0_fifo_cclass = {
+	.handle = NV_ENGCTX(FIFO, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_fifo_context_ctor,
+		.dtor = nve0_fifo_context_dtor,
+		.init = _nouveau_fifo_context_init,
+		.fini = _nouveau_fifo_context_fini,
+		.rd32 = _nouveau_fifo_context_rd32,
+		.wr32 = _nouveau_fifo_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PFIFO engine
+ ******************************************************************************/
+
+static const struct nouveau_enum nve0_fifo_fault_unit[] = {
+	{}
+};
+
+static const struct nouveau_enum nve0_fifo_fault_reason[] = {
+	{ 0x00, "PT_NOT_PRESENT" },
+	{ 0x01, "PT_TOO_SHORT" },
+	{ 0x02, "PAGE_NOT_PRESENT" },
+	{ 0x03, "VM_LIMIT_EXCEEDED" },
+	{ 0x04, "NO_CHANNEL" },
+	{ 0x05, "PAGE_SYSTEM_ONLY" },
+	{ 0x06, "PAGE_READ_ONLY" },
+	{ 0x0a, "COMPRESSED_SYSRAM" },
+	{ 0x0c, "INVALID_STORAGE_TYPE" },
+	{}
+};
+
+static const struct nouveau_enum nve0_fifo_fault_hubclient[] = {
+	{}
+};
+
+static const struct nouveau_enum nve0_fifo_fault_gpcclient[] = {
+	{}
+};
+
+static const struct nouveau_bitfield nve0_fifo_subfifo_intr[] = {
+	{ 0x00200000, "ILLEGAL_MTHD" },
+	{ 0x00800000, "EMPTY_SUBC" },
+	{}
+};
+
+static void
+nve0_fifo_isr_vm_fault(struct nve0_fifo_priv *priv, int unit)
+{
+	u32 inst = nv_rd32(priv, 0x2800 + (unit * 0x10));
+	u32 valo = nv_rd32(priv, 0x2804 + (unit * 0x10));
+	u32 vahi = nv_rd32(priv, 0x2808 + (unit * 0x10));
+	u32 stat = nv_rd32(priv, 0x280c + (unit * 0x10));
+	u32 client = (stat & 0x00001f00) >> 8;
+
+	nv_error(priv, "PFIFO: %s fault at 0x%010llx [", (stat & 0x00000080) ?
+		       "write" : "read", (u64)vahi << 32 | valo);
+	nouveau_enum_print(nve0_fifo_fault_reason, stat & 0x0000000f);
+	printk("] from ");
+	nouveau_enum_print(nve0_fifo_fault_unit, unit);
+	if (stat & 0x00000040) {
+		printk("/");
+		nouveau_enum_print(nve0_fifo_fault_hubclient, client);
+	} else {
+		printk("/GPC%d/", (stat & 0x1f000000) >> 24);
+		nouveau_enum_print(nve0_fifo_fault_gpcclient, client);
+	}
+	printk(" on channel 0x%010llx\n", (u64)inst << 12);
+}
+
+static int
+nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
+{
+	struct nve0_fifo_chan *chan = NULL;
+	struct nouveau_handle *bind;
+	unsigned long flags;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&priv->base.lock, flags);
+	if (likely(chid >= priv->base.min && chid <= priv->base.max))
+		chan = (void *)priv->base.channel[chid];
+	if (unlikely(!chan))
+		goto out;
+
+	bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
+	if (likely(bind)) {
+		if (!mthd || !nv_call(bind->object, mthd, data))
+			ret = 0;
+		nouveau_namedb_put(bind);
+	}
+
+out:
+	spin_unlock_irqrestore(&priv->base.lock, flags);
+	return ret;
+}
+
+static void
+nve0_fifo_isr_subfifo_intr(struct nve0_fifo_priv *priv, int unit)
+{
+	u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
+	u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
+	u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000));
+	u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff;
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00003ffc);
+	u32 show = stat;
+
+	if (stat & 0x00200000) {
+		if (mthd == 0x0054) {
+			if (!nve0_fifo_swmthd(priv, chid, 0x0500, 0x00000000))
+				show &= ~0x00200000;
+		}
+	}
+
+	if (stat & 0x00800000) {
+		if (!nve0_fifo_swmthd(priv, chid, mthd, data))
+			show &= ~0x00800000;
+	}
+
+	if (show) {
+		nv_error(priv, "SUBFIFO%d:", unit);
+		nouveau_bitfield_print(nve0_fifo_subfifo_intr, show);
+		printk("\n");
+		nv_error(priv, "SUBFIFO%d: ch %d subc %d mthd 0x%04x "
+			       "data 0x%08x\n",
+			 unit, chid, subc, mthd, data);
+	}
+
+	nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008);
+	nv_wr32(priv, 0x040108 + (unit * 0x2000), stat);
+}
+
+static void
+nve0_fifo_intr(struct nouveau_subdev *subdev)
+{
+	struct nve0_fifo_priv *priv = (void *)subdev;
+	u32 mask = nv_rd32(priv, 0x002140);
+	u32 stat = nv_rd32(priv, 0x002100) & mask;
+
+	if (stat & 0x00000100) {
+		nv_warn(priv, "unknown status 0x00000100\n");
+		nv_wr32(priv, 0x002100, 0x00000100);
+		stat &= ~0x00000100;
+	}
+
+	if (stat & 0x10000000) {
+		u32 units = nv_rd32(priv, 0x00259c);
+		u32 u = units;
+
+		while (u) {
+			int i = ffs(u) - 1;
+			nve0_fifo_isr_vm_fault(priv, i);
+			u &= ~(1 << i);
+		}
+
+		nv_wr32(priv, 0x00259c, units);
+		stat &= ~0x10000000;
+	}
+
+	if (stat & 0x20000000) {
+		u32 units = nv_rd32(priv, 0x0025a0);
+		u32 u = units;
+
+		while (u) {
+			int i = ffs(u) - 1;
+			nve0_fifo_isr_subfifo_intr(priv, i);
+			u &= ~(1 << i);
+		}
+
+		nv_wr32(priv, 0x0025a0, units);
+		stat &= ~0x20000000;
+	}
+
+	if (stat & 0x40000000) {
+		nv_warn(priv, "unknown status 0x40000000\n");
+		nv_mask(priv, 0x002a00, 0x00000000, 0x00000000);
+		stat &= ~0x40000000;
+	}
+
+	if (stat) {
+		nv_fatal(priv, "unhandled status 0x%08x\n", stat);
+		nv_wr32(priv, 0x002100, stat);
+		nv_wr32(priv, 0x002140, 0);
+	}
+}
+
+static int
+nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nve0_fifo_priv *priv;
+	int ret;
+
+	ret = nouveau_fifo_create(parent, engine, oclass, 0, 4095, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 4096 * 0x200, 0x1000,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW,
+				&priv->user.bar);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000100;
+	nv_subdev(priv)->intr = nve0_fifo_intr;
+	nv_engine(priv)->cclass = &nve0_fifo_cclass;
+	nv_engine(priv)->sclass = nve0_fifo_sclass;
+	return 0;
+}
+
+static void
+nve0_fifo_dtor(struct nouveau_object *object)
+{
+	struct nve0_fifo_priv *priv = (void *)object;
+	int i;
+
+	nouveau_gpuobj_unmap(&priv->user.bar);
+	nouveau_gpuobj_ref(NULL, &priv->user.mem);
+
+	for (i = 0; i < ARRAY_SIZE(priv->engine); i++) {
+		nouveau_gpuobj_ref(NULL, &priv->engine[i].playlist[1]);
+		nouveau_gpuobj_ref(NULL, &priv->engine[i].playlist[0]);
+	}
+
+	nouveau_fifo_destroy(&priv->base);
+}
+
+static int
+nve0_fifo_init(struct nouveau_object *object)
+{
+	struct nve0_fifo_priv *priv = (void *)object;
+	int ret, i;
+
+	ret = nouveau_fifo_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* enable all available PSUBFIFOs */
+	nv_wr32(priv, 0x000204, 0xffffffff);
+	priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204));
+	nv_debug(priv, "%d subfifo(s)\n", priv->spoon_nr);
+
+	/* PSUBFIFO[n] */
+	for (i = 0; i < priv->spoon_nr; i++) {
+		nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
+		nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
+		nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */
+	}
+
+	nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
+
+	nv_wr32(priv, 0x002a00, 0xffffffff);
+	nv_wr32(priv, 0x002100, 0xffffffff);
+	nv_wr32(priv, 0x002140, 0xbfffffff);
+	return 0;
+}
+
+struct nouveau_oclass
+nve0_fifo_oclass = {
+	.handle = NV_ENGINE(FIFO, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_fifo_ctor,
+		.dtor = nve0_fifo_dtor,
+		.init = nve0_fifo_init,
+		.fini = _nouveau_fifo_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_grctx.h b/drivers/gpu/drm/nouveau/core/engine/graph/ctx.h
index b0795ececbd..e1947013d3b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_grctx.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctx.h
@@ -2,7 +2,7 @@
 #define __NOUVEAU_GRCTX_H__
 
 struct nouveau_grctx {
-	struct drm_device *dev;
+	struct nouveau_device *device;
 
 	enum {
 		NOUVEAU_GRCTX_PROG,
@@ -10,18 +10,18 @@ struct nouveau_grctx {
 	} mode;
 	void *data;
 
-	uint32_t ctxprog_max;
-	uint32_t ctxprog_len;
-	uint32_t ctxprog_reg;
-	int      ctxprog_label[32];
-	uint32_t ctxvals_pos;
-	uint32_t ctxvals_base;
+	u32 ctxprog_max;
+	u32 ctxprog_len;
+	u32 ctxprog_reg;
+	int ctxprog_label[32];
+	u32 ctxvals_pos;
+	u32 ctxvals_base;
 };
 
 static inline void
-cp_out(struct nouveau_grctx *ctx, uint32_t inst)
+cp_out(struct nouveau_grctx *ctx, u32 inst)
 {
-	uint32_t *ctxprog = ctx->data;
+	u32 *ctxprog = ctx->data;
 
 	if (ctx->mode != NOUVEAU_GRCTX_PROG)
 		return;
@@ -31,13 +31,13 @@ cp_out(struct nouveau_grctx *ctx, uint32_t inst)
 }
 
 static inline void
-cp_lsr(struct nouveau_grctx *ctx, uint32_t val)
+cp_lsr(struct nouveau_grctx *ctx, u32 val)
 {
 	cp_out(ctx, CP_LOAD_SR | val);
 }
 
 static inline void
-cp_ctx(struct nouveau_grctx *ctx, uint32_t reg, uint32_t length)
+cp_ctx(struct nouveau_grctx *ctx, u32 reg, u32 length)
 {
 	ctx->ctxprog_reg = (reg - 0x00400000) >> 2;
 
@@ -55,7 +55,7 @@ cp_ctx(struct nouveau_grctx *ctx, uint32_t reg, uint32_t length)
 static inline void
 cp_name(struct nouveau_grctx *ctx, int name)
 {
-	uint32_t *ctxprog = ctx->data;
+	u32 *ctxprog = ctx->data;
 	int i;
 
 	if (ctx->mode != NOUVEAU_GRCTX_PROG)
@@ -115,7 +115,7 @@ cp_pos(struct nouveau_grctx *ctx, int offset)
 }
 
 static inline void
-gr_def(struct nouveau_grctx *ctx, uint32_t reg, uint32_t val)
+gr_def(struct nouveau_grctx *ctx, u32 reg, u32 val)
 {
 	if (ctx->mode != NOUVEAU_GRCTX_VALS)
 		return;
diff --git a/drivers/gpu/drm/nouveau/nv40_grctx.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv40.c
index be0a74750fb..e45035efb8c 100644
--- a/drivers/gpu/drm/nouveau/nv40_grctx.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv40.c
@@ -22,6 +22,8 @@
  * Authors: Ben Skeggs
  */
 
+#include <core/gpuobj.h>
+
 /* NVIDIA context programs handle a number of other conditions which are
  * not implemented in our versions.  It's not clear why NVIDIA context
  * programs have this code, nor whether it's strictly necessary for
@@ -109,20 +111,18 @@
 #define CP_LOAD_MAGIC_NV44TCL    0x00800029 /* per-vs state (0x4497) */
 #define CP_LOAD_MAGIC_NV40TCL    0x00800041 /* per-vs state (0x4097) */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_grctx.h"
+#include "nv40.h"
+#include "ctx.h"
 
 /* TODO:
  *  - get vs count from 0x1540
  */
 
 static int
-nv40_graph_vs_count(struct drm_device *dev)
+nv40_graph_vs_count(struct nouveau_device *device)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x47:
 	case 0x49:
 	case 0x4b:
@@ -160,7 +160,7 @@ enum cp_label {
 static void
 nv40_graph_construct_general(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 
 	cp_ctx(ctx, 0x4000a4, 1);
@@ -187,7 +187,7 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
 	cp_ctx(ctx, 0x400724, 1);
 	gr_def(ctx, 0x400724, 0x02008821);
 	cp_ctx(ctx, 0x400770, 3);
-	if (dev_priv->chipset == 0x40) {
+	if (device->chipset == 0x40) {
 		cp_ctx(ctx, 0x400814, 4);
 		cp_ctx(ctx, 0x400828, 5);
 		cp_ctx(ctx, 0x400840, 5);
@@ -208,7 +208,7 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
 		gr_def(ctx, 0x4009dc, 0x80000000);
 	} else {
 		cp_ctx(ctx, 0x400840, 20);
-		if (nv44_graph_class(ctx->dev)) {
+		if (nv44_graph_class(ctx->device)) {
 			for (i = 0; i < 8; i++)
 				gr_def(ctx, 0x400860 + (i * 4), 0x00000001);
 		}
@@ -217,21 +217,21 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
 		gr_def(ctx, 0x400888, 0x00000040);
 		cp_ctx(ctx, 0x400894, 11);
 		gr_def(ctx, 0x400894, 0x00000040);
-		if (!nv44_graph_class(ctx->dev)) {
+		if (!nv44_graph_class(ctx->device)) {
 			for (i = 0; i < 8; i++)
 				gr_def(ctx, 0x4008a0 + (i * 4), 0x80000000);
 		}
 		cp_ctx(ctx, 0x4008e0, 2);
 		cp_ctx(ctx, 0x4008f8, 2);
-		if (dev_priv->chipset == 0x4c ||
-		    (dev_priv->chipset & 0xf0) == 0x60)
+		if (device->chipset == 0x4c ||
+		    (device->chipset & 0xf0) == 0x60)
 			cp_ctx(ctx, 0x4009f8, 1);
 	}
 	cp_ctx(ctx, 0x400a00, 73);
 	gr_def(ctx, 0x400b0c, 0x0b0b0b0c);
 	cp_ctx(ctx, 0x401000, 4);
 	cp_ctx(ctx, 0x405004, 1);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x47:
 	case 0x49:
 	case 0x4b:
@@ -240,7 +240,7 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
 		break;
 	default:
 		cp_ctx(ctx, 0x403440, 1);
-		switch (dev_priv->chipset) {
+		switch (device->chipset) {
 		case 0x40:
 			gr_def(ctx, 0x403440, 0x00000010);
 			break;
@@ -266,19 +266,19 @@ nv40_graph_construct_general(struct nouveau_grctx *ctx)
 static void
 nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 
-	if (dev_priv->chipset == 0x40) {
+	if (device->chipset == 0x40) {
 		cp_ctx(ctx, 0x401880, 51);
 		gr_def(ctx, 0x401940, 0x00000100);
 	} else
-	if (dev_priv->chipset == 0x46 || dev_priv->chipset == 0x47 ||
-	    dev_priv->chipset == 0x49 || dev_priv->chipset == 0x4b) {
+	if (device->chipset == 0x46 || device->chipset == 0x47 ||
+	    device->chipset == 0x49 || device->chipset == 0x4b) {
 		cp_ctx(ctx, 0x401880, 32);
 		for (i = 0; i < 16; i++)
 			gr_def(ctx, 0x401880 + (i * 4), 0x00000111);
-		if (dev_priv->chipset == 0x46)
+		if (device->chipset == 0x46)
 			cp_ctx(ctx, 0x401900, 16);
 		cp_ctx(ctx, 0x401940, 3);
 	}
@@ -289,7 +289,7 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 	gr_def(ctx, 0x401978, 0xffff0000);
 	gr_def(ctx, 0x40197c, 0x00000001);
 	gr_def(ctx, 0x401990, 0x46400000);
-	if (dev_priv->chipset == 0x40) {
+	if (device->chipset == 0x40) {
 		cp_ctx(ctx, 0x4019a0, 2);
 		cp_ctx(ctx, 0x4019ac, 5);
 	} else {
@@ -297,7 +297,7 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 		cp_ctx(ctx, 0x4019b4, 3);
 	}
 	gr_def(ctx, 0x4019bc, 0xffff0000);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x46:
 	case 0x47:
 	case 0x49:
@@ -316,7 +316,7 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 	for (i = 0; i < 16; i++)
 		gr_def(ctx, 0x401a44 + (i * 4), 0x07ff0000);
 	gr_def(ctx, 0x401a8c, 0x4b7fffff);
-	if (dev_priv->chipset == 0x40) {
+	if (device->chipset == 0x40) {
 		cp_ctx(ctx, 0x401ab8, 3);
 	} else {
 		cp_ctx(ctx, 0x401ab8, 1);
@@ -327,10 +327,10 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 	gr_def(ctx, 0x401ad4, 0x70605040);
 	gr_def(ctx, 0x401ad8, 0xb8a89888);
 	gr_def(ctx, 0x401adc, 0xf8e8d8c8);
-	cp_ctx(ctx, 0x401b10, dev_priv->chipset == 0x40 ? 2 : 1);
+	cp_ctx(ctx, 0x401b10, device->chipset == 0x40 ? 2 : 1);
 	gr_def(ctx, 0x401b10, 0x40100000);
-	cp_ctx(ctx, 0x401b18, dev_priv->chipset == 0x40 ? 6 : 5);
-	gr_def(ctx, 0x401b28, dev_priv->chipset == 0x40 ?
+	cp_ctx(ctx, 0x401b18, device->chipset == 0x40 ? 6 : 5);
+	gr_def(ctx, 0x401b28, device->chipset == 0x40 ?
 			      0x00000004 : 0x00000000);
 	cp_ctx(ctx, 0x401b30, 25);
 	gr_def(ctx, 0x401b34, 0x0000ffff);
@@ -341,8 +341,8 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 	gr_def(ctx, 0x401b84, 0xffffffff);
 	gr_def(ctx, 0x401b88, 0x00ff7000);
 	gr_def(ctx, 0x401b8c, 0x0000ffff);
-	if (dev_priv->chipset != 0x44 && dev_priv->chipset != 0x4a &&
-	    dev_priv->chipset != 0x4e)
+	if (device->chipset != 0x44 && device->chipset != 0x4a &&
+	    device->chipset != 0x4e)
 		cp_ctx(ctx, 0x401b94, 1);
 	cp_ctx(ctx, 0x401b98, 8);
 	gr_def(ctx, 0x401b9c, 0x00ff0000);
@@ -371,12 +371,12 @@ nv40_graph_construct_state3d(struct nouveau_grctx *ctx)
 static void
 nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 
 	cp_ctx(ctx, 0x402000, 1);
-	cp_ctx(ctx, 0x402404, dev_priv->chipset == 0x40 ? 1 : 2);
-	switch (dev_priv->chipset) {
+	cp_ctx(ctx, 0x402404, device->chipset == 0x40 ? 1 : 2);
+	switch (device->chipset) {
 	case 0x40:
 		gr_def(ctx, 0x402404, 0x00000001);
 		break;
@@ -393,9 +393,9 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 	default:
 		gr_def(ctx, 0x402404, 0x00000021);
 	}
-	if (dev_priv->chipset != 0x40)
+	if (device->chipset != 0x40)
 		gr_def(ctx, 0x402408, 0x030c30c3);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x44:
 	case 0x46:
 	case 0x4a:
@@ -408,10 +408,10 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 	default:
 		break;
 	}
-	cp_ctx(ctx, 0x402480, dev_priv->chipset == 0x40 ? 8 : 9);
+	cp_ctx(ctx, 0x402480, device->chipset == 0x40 ? 8 : 9);
 	gr_def(ctx, 0x402488, 0x3e020200);
 	gr_def(ctx, 0x40248c, 0x00ffffff);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x40:
 		gr_def(ctx, 0x402490, 0x60103f00);
 		break;
@@ -428,16 +428,16 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 		gr_def(ctx, 0x402490, 0x0c103f00);
 		break;
 	}
-	gr_def(ctx, 0x40249c, dev_priv->chipset <= 0x43 ?
+	gr_def(ctx, 0x40249c, device->chipset <= 0x43 ?
 			      0x00020000 : 0x00040000);
 	cp_ctx(ctx, 0x402500, 31);
 	gr_def(ctx, 0x402530, 0x00008100);
-	if (dev_priv->chipset == 0x40)
+	if (device->chipset == 0x40)
 		cp_ctx(ctx, 0x40257c, 6);
 	cp_ctx(ctx, 0x402594, 16);
 	cp_ctx(ctx, 0x402800, 17);
 	gr_def(ctx, 0x402800, 0x00000001);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x47:
 	case 0x49:
 	case 0x4b:
@@ -445,7 +445,7 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 		gr_def(ctx, 0x402864, 0x00001001);
 		cp_ctx(ctx, 0x402870, 3);
 		gr_def(ctx, 0x402878, 0x00000003);
-		if (dev_priv->chipset != 0x47) { /* belong at end!! */
+		if (device->chipset != 0x47) { /* belong at end!! */
 			cp_ctx(ctx, 0x402900, 1);
 			cp_ctx(ctx, 0x402940, 1);
 			cp_ctx(ctx, 0x402980, 1);
@@ -470,9 +470,9 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 	}
 
 	cp_ctx(ctx, 0x402c00, 4);
-	gr_def(ctx, 0x402c00, dev_priv->chipset == 0x40 ?
+	gr_def(ctx, 0x402c00, device->chipset == 0x40 ?
 			      0x80800001 : 0x00888001);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x47:
 	case 0x49:
 	case 0x4b:
@@ -485,30 +485,30 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 		break;
 	default:
 		cp_ctx(ctx, 0x402c10, 4);
-		if (dev_priv->chipset == 0x40)
+		if (device->chipset == 0x40)
 			cp_ctx(ctx, 0x402c20, 36);
 		else
-		if (dev_priv->chipset <= 0x42)
+		if (device->chipset <= 0x42)
 			cp_ctx(ctx, 0x402c20, 24);
 		else
-		if (dev_priv->chipset <= 0x4a)
+		if (device->chipset <= 0x4a)
 			cp_ctx(ctx, 0x402c20, 16);
 		else
 			cp_ctx(ctx, 0x402c20, 8);
-		cp_ctx(ctx, 0x402cb0, dev_priv->chipset == 0x40 ? 12 : 13);
+		cp_ctx(ctx, 0x402cb0, device->chipset == 0x40 ? 12 : 13);
 		gr_def(ctx, 0x402cd4, 0x00000005);
-		if (dev_priv->chipset != 0x40)
+		if (device->chipset != 0x40)
 			gr_def(ctx, 0x402ce0, 0x0000ffff);
 		break;
 	}
 
-	cp_ctx(ctx, 0x403400, dev_priv->chipset == 0x40 ? 4 : 3);
-	cp_ctx(ctx, 0x403410, dev_priv->chipset == 0x40 ? 4 : 3);
-	cp_ctx(ctx, 0x403420, nv40_graph_vs_count(ctx->dev));
-	for (i = 0; i < nv40_graph_vs_count(ctx->dev); i++)
+	cp_ctx(ctx, 0x403400, device->chipset == 0x40 ? 4 : 3);
+	cp_ctx(ctx, 0x403410, device->chipset == 0x40 ? 4 : 3);
+	cp_ctx(ctx, 0x403420, nv40_graph_vs_count(ctx->device));
+	for (i = 0; i < nv40_graph_vs_count(ctx->device); i++)
 		gr_def(ctx, 0x403420 + (i * 4), 0x00005555);
 
-	if (dev_priv->chipset != 0x40) {
+	if (device->chipset != 0x40) {
 		cp_ctx(ctx, 0x403600, 1);
 		gr_def(ctx, 0x403600, 0x00000001);
 	}
@@ -516,7 +516,7 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 
 	cp_ctx(ctx, 0x403c18, 1);
 	gr_def(ctx, 0x403c18, 0x00000001);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x46:
 	case 0x47:
 	case 0x49:
@@ -527,7 +527,7 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 		gr_def(ctx, 0x405c24, 0x000e3000);
 		break;
 	}
-	if (dev_priv->chipset != 0x4e)
+	if (device->chipset != 0x4e)
 		cp_ctx(ctx, 0x405800, 11);
 	cp_ctx(ctx, 0x407000, 1);
 }
@@ -535,7 +535,7 @@ nv40_graph_construct_state3d_2(struct nouveau_grctx *ctx)
 static void
 nv40_graph_construct_state3d_3(struct nouveau_grctx *ctx)
 {
-	int len = nv44_graph_class(ctx->dev) ? 0x0084 : 0x0684;
+	int len = nv44_graph_class(ctx->device) ? 0x0084 : 0x0684;
 
 	cp_out (ctx, 0x300000);
 	cp_lsr (ctx, len - 4);
@@ -550,32 +550,31 @@ nv40_graph_construct_state3d_3(struct nouveau_grctx *ctx)
 static void
 nv40_graph_construct_shader(struct nouveau_grctx *ctx)
 {
-	struct drm_device *dev = ctx->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	struct nouveau_gpuobj *obj = ctx->data;
 	int vs, vs_nr, vs_len, vs_nr_b0, vs_nr_b1, b0_offset, b1_offset;
 	int offset, i;
 
-	vs_nr    = nv40_graph_vs_count(ctx->dev);
+	vs_nr    = nv40_graph_vs_count(ctx->device);
 	vs_nr_b0 = 363;
-	vs_nr_b1 = dev_priv->chipset == 0x40 ? 128 : 64;
-	if (dev_priv->chipset == 0x40) {
+	vs_nr_b1 = device->chipset == 0x40 ? 128 : 64;
+	if (device->chipset == 0x40) {
 		b0_offset = 0x2200/4; /* 33a0 */
 		b1_offset = 0x55a0/4; /* 1500 */
 		vs_len = 0x6aa0/4;
 	} else
-	if (dev_priv->chipset == 0x41 || dev_priv->chipset == 0x42) {
+	if (device->chipset == 0x41 || device->chipset == 0x42) {
 		b0_offset = 0x2200/4; /* 2200 */
 		b1_offset = 0x4400/4; /* 0b00 */
 		vs_len = 0x4f00/4;
 	} else {
 		b0_offset = 0x1d40/4; /* 2200 */
 		b1_offset = 0x3f40/4; /* 0b00 : 0a40 */
-		vs_len = nv44_graph_class(dev) ? 0x4980/4 : 0x4a40/4;
+		vs_len = nv44_graph_class(device) ? 0x4980/4 : 0x4a40/4;
 	}
 
 	cp_lsr(ctx, vs_len * vs_nr + 0x300/4);
-	cp_out(ctx, nv44_graph_class(dev) ? 0x800029 : 0x800041);
+	cp_out(ctx, nv44_graph_class(device) ? 0x800029 : 0x800041);
 
 	offset = ctx->ctxvals_pos;
 	ctx->ctxvals_pos += (0x0300/4 + (vs_nr * vs_len));
@@ -661,21 +660,21 @@ nv40_grctx_generate(struct nouveau_grctx *ctx)
 }
 
 void
-nv40_grctx_fill(struct drm_device *dev, struct nouveau_gpuobj *mem)
+nv40_grctx_fill(struct nouveau_device *device, struct nouveau_gpuobj *mem)
 {
 	nv40_grctx_generate(&(struct nouveau_grctx) {
-			     .dev = dev,
+			     .device = device,
 			     .mode = NOUVEAU_GRCTX_VALS,
 			     .data = mem,
 			   });
 }
 
 void
-nv40_grctx_init(struct drm_device *dev, u32 *size)
+nv40_grctx_init(struct nouveau_device *device, u32 *size)
 {
 	u32 ctxprog[256], i;
 	struct nouveau_grctx ctx = {
-		.dev = dev,
+		.device = device,
 		.mode = NOUVEAU_GRCTX_PROG,
 		.data = ctxprog,
 		.ctxprog_max = ARRAY_SIZE(ctxprog)
@@ -683,8 +682,8 @@ nv40_grctx_init(struct drm_device *dev, u32 *size)
 
 	nv40_grctx_generate(&ctx);
 
-	nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_INDEX, 0);
+	nv_wr32(device, 0x400324, 0);
 	for (i = 0; i < ctx.ctxprog_len; i++)
-		nv_wr32(dev, NV40_PGRAPH_CTXCTL_UCODE_DATA, ctxprog[i]);
+		nv_wr32(device, 0x400328, ctxprog[i]);
 	*size = ctx.ctxvals_pos * 4;
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_grctx.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv50.c
index 881e22b249f..552fdbd45eb 100644
--- a/drivers/gpu/drm/nouveau/nv50_grctx.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv50.c
@@ -20,6 +20,8 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <core/gpuobj.h>
+
 #define CP_FLAG_CLEAR                 0
 #define CP_FLAG_SET                   1
 #define CP_FLAG_SWAP_DIRECTION        ((0 * 32) + 0)
@@ -105,9 +107,8 @@
 #define CP_SEEK_1      0x00c000ff
 #define CP_SEEK_2      0x00c800ff
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_grctx.h"
+#include "nv50.h"
+#include "ctx.h"
 
 #define IS_NVA3F(x) (((x) > 0xa0 && (x) < 0xaa) || (x) == 0xaf)
 #define IS_NVAAF(x) ((x) >= 0xaa && (x) <= 0xac)
@@ -175,32 +176,6 @@ static void nv50_graph_construct_xfer2(struct nouveau_grctx *ctx);
 static int
 nv50_grctx_generate(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-
-	switch (dev_priv->chipset) {
-	case 0x50:
-	case 0x84:
-	case 0x86:
-	case 0x92:
-	case 0x94:
-	case 0x96:
-	case 0x98:
-	case 0xa0:
-	case 0xa3:
-	case 0xa5:
-	case 0xa8:
-	case 0xaa:
-	case 0xac:
-	case 0xaf:
-		break;
-	default:
-		NV_ERROR(ctx->dev, "I don't know how to make a ctxprog for "
-				   "your NV%x card.\n", dev_priv->chipset);
-		NV_ERROR(ctx->dev, "Disabling acceleration. Please contact "
-				   "the devs.\n");
-		return -ENOSYS;
-	}
-
 	cp_set (ctx, STATE, RUNNING);
 	cp_set (ctx, XFER_SWITCH, ENABLE);
 	/* decide whether we're loading/unloading the context */
@@ -278,30 +253,36 @@ nv50_grctx_generate(struct nouveau_grctx *ctx)
 }
 
 void
-nv50_grctx_fill(struct drm_device *dev, struct nouveau_gpuobj *mem)
+nv50_grctx_fill(struct nouveau_device *device, struct nouveau_gpuobj *mem)
 {
 	nv50_grctx_generate(&(struct nouveau_grctx) {
-			     .dev = dev,
+			     .device = device,
 			     .mode = NOUVEAU_GRCTX_VALS,
 			     .data = mem,
 			   });
 }
 
 int
-nv50_grctx_init(struct drm_device *dev, u32 *data, u32 max, u32 *len, u32 *cnt)
+nv50_grctx_init(struct nouveau_device *device, u32 *size)
 {
+	u32 *ctxprog = kmalloc(512 * 4, GFP_KERNEL), i;
 	struct nouveau_grctx ctx = {
-		.dev = dev,
+		.device = device,
 		.mode = NOUVEAU_GRCTX_PROG,
-		.data = data,
-		.ctxprog_max = max
+		.data = ctxprog,
+		.ctxprog_max = 512,
 	};
-	int ret;
 
-	ret = nv50_grctx_generate(&ctx);
-	*cnt = ctx.ctxvals_pos * 4;
-	*len = ctx.ctxprog_len;
-	return ret;
+	if (!ctxprog)
+		return -ENOMEM;
+	nv50_grctx_generate(&ctx);
+
+	nv_wr32(device, 0x400324, 0);
+	for (i = 0; i < ctx.ctxprog_len; i++)
+		nv_wr32(device, 0x400328, ctxprog[i]);
+	*size = ctx.ctxvals_pos * 4;
+	kfree(ctxprog);
+	return 0;
 }
 
 /*
@@ -315,36 +296,36 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx);
 static void
 nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i, j;
 	int offset, base;
-	uint32_t units = nv_rd32 (ctx->dev, 0x1540);
+	u32 units = nv_rd32 (ctx->device, 0x1540);
 
 	/* 0800: DISPATCH */
 	cp_ctx(ctx, 0x400808, 7);
 	gr_def(ctx, 0x400814, 0x00000030);
 	cp_ctx(ctx, 0x400834, 0x32);
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		gr_def(ctx, 0x400834, 0xff400040);
 		gr_def(ctx, 0x400838, 0xfff00080);
 		gr_def(ctx, 0x40083c, 0xfff70090);
 		gr_def(ctx, 0x400840, 0xffe806a8);
 	}
 	gr_def(ctx, 0x400844, 0x00000002);
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		gr_def(ctx, 0x400894, 0x00001000);
 	gr_def(ctx, 0x4008e8, 0x00000003);
 	gr_def(ctx, 0x4008ec, 0x00001000);
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		cp_ctx(ctx, 0x400908, 0xb);
-	else if (dev_priv->chipset < 0xa0)
+	else if (device->chipset < 0xa0)
 		cp_ctx(ctx, 0x400908, 0xc);
 	else
 		cp_ctx(ctx, 0x400908, 0xe);
 
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		cp_ctx(ctx, 0x400b00, 0x1);
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		cp_ctx(ctx, 0x400b10, 0x1);
 		gr_def(ctx, 0x400b10, 0x0001629d);
 		cp_ctx(ctx, 0x400b20, 0x1);
@@ -358,10 +339,10 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	gr_def(ctx, 0x400c08, 0x0000fe0c);
 
 	/* 1000 */
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		cp_ctx(ctx, 0x401008, 0x4);
 		gr_def(ctx, 0x401014, 0x00001000);
-	} else if (!IS_NVA3F(dev_priv->chipset)) {
+	} else if (!IS_NVA3F(device->chipset)) {
 		cp_ctx(ctx, 0x401008, 0x5);
 		gr_def(ctx, 0x401018, 0x00001000);
 	} else {
@@ -372,7 +353,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	/* 1400 */
 	cp_ctx(ctx, 0x401400, 0x8);
 	cp_ctx(ctx, 0x401424, 0x3);
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		gr_def(ctx, 0x40142c, 0x0001fd87);
 	else
 		gr_def(ctx, 0x40142c, 0x00000187);
@@ -382,10 +363,10 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	/* 1800: STREAMOUT */
 	cp_ctx(ctx, 0x401814, 0x1);
 	gr_def(ctx, 0x401814, 0x000000ff);
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		cp_ctx(ctx, 0x40181c, 0xe);
 		gr_def(ctx, 0x401850, 0x00000004);
-	} else if (dev_priv->chipset < 0xa0) {
+	} else if (device->chipset < 0xa0) {
 		cp_ctx(ctx, 0x40181c, 0xf);
 		gr_def(ctx, 0x401854, 0x00000004);
 	} else {
@@ -395,7 +376,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
 	/* 1C00 */
 	cp_ctx(ctx, 0x401c00, 0x1);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x50:
 		gr_def(ctx, 0x401c00, 0x0001005f);
 		break;
@@ -424,7 +405,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
 	/* 2400 */
 	cp_ctx(ctx, 0x402400, 0x1);
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		cp_ctx(ctx, 0x402408, 0x1);
 	else
 		cp_ctx(ctx, 0x402408, 0x2);
@@ -432,21 +413,21 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 
 	/* 2800: CSCHED */
 	cp_ctx(ctx, 0x402800, 0x1);
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		gr_def(ctx, 0x402800, 0x00000006);
 
 	/* 2C00: ZCULL */
 	cp_ctx(ctx, 0x402c08, 0x6);
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		gr_def(ctx, 0x402c14, 0x01000000);
 	gr_def(ctx, 0x402c18, 0x000000ff);
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		cp_ctx(ctx, 0x402ca0, 0x1);
 	else
 		cp_ctx(ctx, 0x402ca0, 0x2);
-	if (dev_priv->chipset < 0xa0)
+	if (device->chipset < 0xa0)
 		gr_def(ctx, 0x402ca0, 0x00000400);
-	else if (!IS_NVA3F(dev_priv->chipset))
+	else if (!IS_NVA3F(device->chipset))
 		gr_def(ctx, 0x402ca0, 0x00000800);
 	else
 		gr_def(ctx, 0x402ca0, 0x00000400);
@@ -457,14 +438,14 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	gr_def(ctx, 0x403004, 0x00000001);
 
 	/* 3400 */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		cp_ctx(ctx, 0x403404, 0x1);
 		gr_def(ctx, 0x403404, 0x00000001);
 	}
 
 	/* 5000: CCACHE */
 	cp_ctx(ctx, 0x405000, 0x1);
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x50:
 		gr_def(ctx, 0x405000, 0x00300080);
 		break;
@@ -493,22 +474,22 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	cp_ctx(ctx, 0x40502c, 0x1);
 
 	/* 6000? */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		cp_ctx(ctx, 0x4063e0, 0x1);
 
 	/* 6800: M2MF */
-	if (dev_priv->chipset < 0x90) {
+	if (device->chipset < 0x90) {
 		cp_ctx(ctx, 0x406814, 0x2b);
 		gr_def(ctx, 0x406818, 0x00000f80);
 		gr_def(ctx, 0x406860, 0x007f0080);
 		gr_def(ctx, 0x40689c, 0x007f0080);
 	} else {
 		cp_ctx(ctx, 0x406814, 0x4);
-		if (dev_priv->chipset == 0x98)
+		if (device->chipset == 0x98)
 			gr_def(ctx, 0x406818, 0x00000f80);
 		else
 			gr_def(ctx, 0x406818, 0x00001f80);
-		if (IS_NVA3F(dev_priv->chipset))
+		if (IS_NVA3F(device->chipset))
 			gr_def(ctx, 0x40681c, 0x00000030);
 		cp_ctx(ctx, 0x406830, 0x3);
 	}
@@ -517,43 +498,43 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	for (i = 0; i < 8; i++) {
 		if (units & (1<<(i+16))) {
 			cp_ctx(ctx, 0x407000 + (i<<8), 3);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, 0x407000 + (i<<8), 0x1b74f820);
-			else if (dev_priv->chipset != 0xa5)
+			else if (device->chipset != 0xa5)
 				gr_def(ctx, 0x407000 + (i<<8), 0x3b74f821);
 			else
 				gr_def(ctx, 0x407000 + (i<<8), 0x7b74f821);
 			gr_def(ctx, 0x407004 + (i<<8), 0x89058001);
 
-			if (dev_priv->chipset == 0x50) {
+			if (device->chipset == 0x50) {
 				cp_ctx(ctx, 0x407010 + (i<<8), 1);
-			} else if (dev_priv->chipset < 0xa0) {
+			} else if (device->chipset < 0xa0) {
 				cp_ctx(ctx, 0x407010 + (i<<8), 2);
 				gr_def(ctx, 0x407010 + (i<<8), 0x00001000);
 				gr_def(ctx, 0x407014 + (i<<8), 0x0000001f);
 			} else {
 				cp_ctx(ctx, 0x407010 + (i<<8), 3);
 				gr_def(ctx, 0x407010 + (i<<8), 0x00001000);
-				if (dev_priv->chipset != 0xa5)
+				if (device->chipset != 0xa5)
 					gr_def(ctx, 0x407014 + (i<<8), 0x000000ff);
 				else
 					gr_def(ctx, 0x407014 + (i<<8), 0x000001ff);
 			}
 
 			cp_ctx(ctx, 0x407080 + (i<<8), 4);
-			if (dev_priv->chipset != 0xa5)
+			if (device->chipset != 0xa5)
 				gr_def(ctx, 0x407080 + (i<<8), 0x027c10fa);
 			else
 				gr_def(ctx, 0x407080 + (i<<8), 0x827c10fa);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, 0x407084 + (i<<8), 0x000000c0);
 			else
 				gr_def(ctx, 0x407084 + (i<<8), 0x400000c0);
 			gr_def(ctx, 0x407088 + (i<<8), 0xb7892080);
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				cp_ctx(ctx, 0x407094 + (i<<8), 1);
-			else if (!IS_NVA3F(dev_priv->chipset))
+			else if (!IS_NVA3F(device->chipset))
 				cp_ctx(ctx, 0x407094 + (i<<8), 3);
 			else {
 				cp_ctx(ctx, 0x407094 + (i<<8), 4);
@@ -563,30 +544,30 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	}
 
 	cp_ctx(ctx, 0x407c00, 0x3);
-	if (dev_priv->chipset < 0x90)
+	if (device->chipset < 0x90)
 		gr_def(ctx, 0x407c00, 0x00010040);
-	else if (dev_priv->chipset < 0xa0)
+	else if (device->chipset < 0xa0)
 		gr_def(ctx, 0x407c00, 0x00390040);
 	else
 		gr_def(ctx, 0x407c00, 0x003d0040);
 	gr_def(ctx, 0x407c08, 0x00000022);
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		cp_ctx(ctx, 0x407c10, 0x3);
 		cp_ctx(ctx, 0x407c20, 0x1);
 		cp_ctx(ctx, 0x407c2c, 0x1);
 	}
 
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		cp_ctx(ctx, 0x407d00, 0x9);
 	} else {
 		cp_ctx(ctx, 0x407d00, 0x15);
 	}
-	if (dev_priv->chipset == 0x98)
+	if (device->chipset == 0x98)
 		gr_def(ctx, 0x407d08, 0x00380040);
 	else {
-		if (dev_priv->chipset < 0x90)
+		if (device->chipset < 0x90)
 			gr_def(ctx, 0x407d08, 0x00010040);
-		else if (dev_priv->chipset < 0xa0)
+		else if (device->chipset < 0xa0)
 			gr_def(ctx, 0x407d08, 0x00390040);
 		else
 			gr_def(ctx, 0x407d08, 0x003d0040);
@@ -596,11 +577,11 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 	/* 8000+: per-TP state */
 	for (i = 0; i < 10; i++) {
 		if (units & (1<<i)) {
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				base = 0x408000 + (i<<12);
 			else
 				base = 0x408000 + (i<<11);
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				offset = base + 0xc00;
 			else
 				offset = base + 0x80;
@@ -609,9 +590,9 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 			cp_ctx(ctx, offset + 0x08, 1);
 
 			/* per-MP state */
-			for (j = 0; j < (dev_priv->chipset < 0xa0 ? 2 : 4); j++) {
+			for (j = 0; j < (device->chipset < 0xa0 ? 2 : 4); j++) {
 				if (!(units & (1 << (j+24)))) continue;
-				if (dev_priv->chipset < 0xa0)
+				if (device->chipset < 0xa0)
 					offset = base + 0x200 + (j<<7);
 				else
 					offset = base + 0x100 + (j<<7);
@@ -620,7 +601,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 				gr_def(ctx, offset + 0x04, 0x00160000);
 				gr_def(ctx, offset + 0x08, 0x01800000);
 				gr_def(ctx, offset + 0x18, 0x0003ffff);
-				switch (dev_priv->chipset) {
+				switch (device->chipset) {
 				case 0x50:
 					gr_def(ctx, offset + 0x1c, 0x00080000);
 					break;
@@ -651,53 +632,53 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 					break;
 				}
 				gr_def(ctx, offset + 0x40, 0x00010401);
-				if (dev_priv->chipset == 0x50)
+				if (device->chipset == 0x50)
 					gr_def(ctx, offset + 0x48, 0x00000040);
 				else
 					gr_def(ctx, offset + 0x48, 0x00000078);
 				gr_def(ctx, offset + 0x50, 0x000000bf);
 				gr_def(ctx, offset + 0x58, 0x00001210);
-				if (dev_priv->chipset == 0x50)
+				if (device->chipset == 0x50)
 					gr_def(ctx, offset + 0x5c, 0x00000080);
 				else
 					gr_def(ctx, offset + 0x5c, 0x08000080);
-				if (dev_priv->chipset >= 0xa0)
+				if (device->chipset >= 0xa0)
 					gr_def(ctx, offset + 0x68, 0x0000003e);
 			}
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				cp_ctx(ctx, base + 0x300, 0x4);
 			else
 				cp_ctx(ctx, base + 0x300, 0x5);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, base + 0x304, 0x00007070);
-			else if (dev_priv->chipset < 0xa0)
+			else if (device->chipset < 0xa0)
 				gr_def(ctx, base + 0x304, 0x00027070);
-			else if (!IS_NVA3F(dev_priv->chipset))
+			else if (!IS_NVA3F(device->chipset))
 				gr_def(ctx, base + 0x304, 0x01127070);
 			else
 				gr_def(ctx, base + 0x304, 0x05127070);
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				cp_ctx(ctx, base + 0x318, 1);
 			else
 				cp_ctx(ctx, base + 0x320, 1);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, base + 0x318, 0x0003ffff);
-			else if (dev_priv->chipset < 0xa0)
+			else if (device->chipset < 0xa0)
 				gr_def(ctx, base + 0x318, 0x03ffffff);
 			else
 				gr_def(ctx, base + 0x320, 0x07ffffff);
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				cp_ctx(ctx, base + 0x324, 5);
 			else
 				cp_ctx(ctx, base + 0x328, 4);
 
-			if (dev_priv->chipset < 0xa0) {
+			if (device->chipset < 0xa0) {
 				cp_ctx(ctx, base + 0x340, 9);
 				offset = base + 0x340;
-			} else if (!IS_NVA3F(dev_priv->chipset)) {
+			} else if (!IS_NVA3F(device->chipset)) {
 				cp_ctx(ctx, base + 0x33c, 0xb);
 				offset = base + 0x344;
 			} else {
@@ -706,12 +687,12 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 			}
 			gr_def(ctx, offset + 0x0, 0x00120407);
 			gr_def(ctx, offset + 0x4, 0x05091507);
-			if (dev_priv->chipset == 0x84)
+			if (device->chipset == 0x84)
 				gr_def(ctx, offset + 0x8, 0x05100202);
 			else
 				gr_def(ctx, offset + 0x8, 0x05010202);
 			gr_def(ctx, offset + 0xc, 0x00030201);
-			if (dev_priv->chipset == 0xa3)
+			if (device->chipset == 0xa3)
 				cp_ctx(ctx, base + 0x36c, 1);
 
 			cp_ctx(ctx, base + 0x400, 2);
@@ -720,7 +701,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 			gr_def(ctx, base + 0x40c, 0x0d0c0b0a);
 			gr_def(ctx, base + 0x410, 0x00141210);
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				offset = base + 0x800;
 			else
 				offset = base + 0x500;
@@ -728,55 +709,55 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 			gr_def(ctx, offset + 0x0, 0x000001f0);
 			gr_def(ctx, offset + 0x4, 0x00000001);
 			gr_def(ctx, offset + 0x8, 0x00000003);
-			if (dev_priv->chipset == 0x50 || IS_NVAAF(dev_priv->chipset))
+			if (device->chipset == 0x50 || IS_NVAAF(device->chipset))
 				gr_def(ctx, offset + 0xc, 0x00008000);
 			gr_def(ctx, offset + 0x14, 0x00039e00);
 			cp_ctx(ctx, offset + 0x1c, 2);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, offset + 0x1c, 0x00000040);
 			else
 				gr_def(ctx, offset + 0x1c, 0x00000100);
 			gr_def(ctx, offset + 0x20, 0x00003800);
 
-			if (dev_priv->chipset >= 0xa0) {
+			if (device->chipset >= 0xa0) {
 				cp_ctx(ctx, base + 0x54c, 2);
-				if (!IS_NVA3F(dev_priv->chipset))
+				if (!IS_NVA3F(device->chipset))
 					gr_def(ctx, base + 0x54c, 0x003fe006);
 				else
 					gr_def(ctx, base + 0x54c, 0x003fe007);
 				gr_def(ctx, base + 0x550, 0x003fe000);
 			}
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				offset = base + 0xa00;
 			else
 				offset = base + 0x680;
 			cp_ctx(ctx, offset, 1);
 			gr_def(ctx, offset, 0x00404040);
 
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				offset = base + 0xe00;
 			else
 				offset = base + 0x700;
 			cp_ctx(ctx, offset, 2);
-			if (dev_priv->chipset < 0xa0)
+			if (device->chipset < 0xa0)
 				gr_def(ctx, offset, 0x0077f005);
-			else if (dev_priv->chipset == 0xa5)
+			else if (device->chipset == 0xa5)
 				gr_def(ctx, offset, 0x6cf7f007);
-			else if (dev_priv->chipset == 0xa8)
+			else if (device->chipset == 0xa8)
 				gr_def(ctx, offset, 0x6cfff007);
-			else if (dev_priv->chipset == 0xac)
+			else if (device->chipset == 0xac)
 				gr_def(ctx, offset, 0x0cfff007);
 			else
 				gr_def(ctx, offset, 0x0cf7f007);
-			if (dev_priv->chipset == 0x50)
+			if (device->chipset == 0x50)
 				gr_def(ctx, offset + 0x4, 0x00007fff);
-			else if (dev_priv->chipset < 0xa0)
+			else if (device->chipset < 0xa0)
 				gr_def(ctx, offset + 0x4, 0x003f7fff);
 			else
 				gr_def(ctx, offset + 0x4, 0x02bf7fff);
 			cp_ctx(ctx, offset + 0x2c, 1);
-			if (dev_priv->chipset == 0x50) {
+			if (device->chipset == 0x50) {
 				cp_ctx(ctx, offset + 0x50, 9);
 				gr_def(ctx, offset + 0x54, 0x000003ff);
 				gr_def(ctx, offset + 0x58, 0x00000003);
@@ -785,7 +766,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 				gr_def(ctx, offset + 0x64, 0x0000001f);
 				gr_def(ctx, offset + 0x68, 0x0000000f);
 				gr_def(ctx, offset + 0x6c, 0x0000000f);
-			} else if (dev_priv->chipset < 0xa0) {
+			} else if (device->chipset < 0xa0) {
 				cp_ctx(ctx, offset + 0x50, 1);
 				cp_ctx(ctx, offset + 0x70, 1);
 			} else {
@@ -797,7 +778,7 @@ nv50_graph_construct_mmio(struct nouveau_grctx *ctx)
 }
 
 static void
-dd_emit(struct nouveau_grctx *ctx, int num, uint32_t val) {
+dd_emit(struct nouveau_grctx *ctx, int num, u32 val) {
 	int i;
 	if (val && ctx->mode == NOUVEAU_GRCTX_VALS)
 		for (i = 0; i < num; i++)
@@ -808,7 +789,7 @@ dd_emit(struct nouveau_grctx *ctx, int num, uint32_t val) {
 static void
 nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int base, num;
 	base = ctx->ctxvals_pos;
 
@@ -822,7 +803,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 1);	/* 00000001 SRC_LINEAR #1 */
 	dd_emit(ctx, 1, 0);	/* 000000ff SRC_ADDRESS_HIGH */
 	dd_emit(ctx, 1, 0);	/* 00000001 SRC_SRGB */
-	if (dev_priv->chipset >= 0x94)
+	if (device->chipset >= 0x94)
 		dd_emit(ctx, 1, 0);	/* 00000003 eng2d UNK0258 */
 	dd_emit(ctx, 1, 1);	/* 00000fff SRC_DEPTH */
 	dd_emit(ctx, 1, 0x100);	/* 0000ffff SRC_HEIGHT */
@@ -851,7 +832,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 1);		/* 0000007f BLOCKDIM_Z */
 	dd_emit(ctx, 1, 4);		/* 000000ff CP_REG_ALLOC_TEMP */
 	dd_emit(ctx, 1, 1);		/* 00000001 BLOCKDIM_DIRTY */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		dd_emit(ctx, 1, 0);	/* 00000003 UNK03E8 */
 	dd_emit(ctx, 1, 1);		/* 0000007f BLOCK_ALLOC_HALFWARPS */
 	dd_emit(ctx, 1, 1);		/* 00000007 LOCAL_WARPS_NO_CLAMP */
@@ -863,7 +844,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 1);		/* 000007ff BLOCK_ALLOC_THREADS */
 
 	/* compat 2d state */
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		dd_emit(ctx, 4, 0);		/* 0000ffff clip X, Y, W, H */
 
 		dd_emit(ctx, 1, 1);		/* ffffffff chroma COLOR_FORMAT */
@@ -923,7 +904,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 0x100);		/* ffffffff m2mf TILING_PITCH_IN */
 
 	/* more compat 2d state */
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		dd_emit(ctx, 1, 1);		/* ffffffff line COLOR_FORMAT */
 		dd_emit(ctx, 1, 0);		/* ffffffff line OPERATION */
 
@@ -957,18 +938,18 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 0);		/* 000000ff UNK12B0_2 */
 	dd_emit(ctx, 1, 0);		/* 0000000f FP_TEXTURES_LOG2 */
 	dd_emit(ctx, 1, 0);		/* 0000000f FP_SAMPLERS_LOG2 */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		dd_emit(ctx, 1, 0);	/* ffffffff */
 		dd_emit(ctx, 1, 0);	/* 0000007f MULTISAMPLE_SAMPLES_LOG2 */
 	} else {
 		dd_emit(ctx, 1, 0);	/* 0000000f MULTISAMPLE_SAMPLES_LOG2 */
 	}
 	dd_emit(ctx, 1, 0xc);		/* 000000ff SEMANTIC_COLOR.BFC0_ID */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		dd_emit(ctx, 1, 0);	/* 00000001 SEMANTIC_COLOR.CLMP_EN */
 	dd_emit(ctx, 1, 8);		/* 000000ff SEMANTIC_COLOR.COLR_NR */
 	dd_emit(ctx, 1, 0x14);		/* 000000ff SEMANTIC_COLOR.FFC0_ID */
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		dd_emit(ctx, 1, 0);	/* 000000ff SEMANTIC_LAYER */
 		dd_emit(ctx, 1, 0);	/* 00000001 */
 	} else {
@@ -994,7 +975,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 8, 0);		/* ffffffff RT_ADDRESS_LOW */
 	dd_emit(ctx, 1, 0xcf);		/* 000000ff RT_FORMAT */
 	dd_emit(ctx, 7, 0);		/* 000000ff RT_FORMAT */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		dd_emit(ctx, 3, 0);	/* 1, 1, 1 */
 	else
 		dd_emit(ctx, 2, 0);	/* 1, 1 */
@@ -1002,15 +983,15 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 0x80);		/* 0000ffff GP_VERTEX_OUTPUT_COUNT*/
 	dd_emit(ctx, 1, 4);		/* 000000ff GP_REG_ALLOC_RESULT */
 	dd_emit(ctx, 1, 4);		/* 000000ff GP_RESULT_MAP_SIZE */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		dd_emit(ctx, 1, 3);	/* 00000003 */
 		dd_emit(ctx, 1, 0);	/* 00000001 UNK1418. Alone. */
 	}
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		dd_emit(ctx, 1, 3);	/* 00000003 UNK15AC */
 	dd_emit(ctx, 1, 1);		/* ffffffff RASTERIZE_ENABLE */
 	dd_emit(ctx, 1, 0);		/* 00000001 FP_CONTROL.EXPORTS_Z */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		dd_emit(ctx, 1, 0);	/* 00000001 FP_CONTROL.MULTIPLE_RESULTS */
 	dd_emit(ctx, 1, 0x12);		/* 000000ff FP_INTERPOLANT_CTRL.COUNT */
 	dd_emit(ctx, 1, 0x10);		/* 000000ff FP_INTERPOLANT_CTRL.COUNT_NONFLAT */
@@ -1022,16 +1003,16 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 4);		/* 000000ff FP_RESULT_COUNT */
 	dd_emit(ctx, 1, 2);		/* ffffffff REG_MODE */
 	dd_emit(ctx, 1, 4);		/* 000000ff FP_REG_ALLOC_TEMP */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		dd_emit(ctx, 1, 0);	/* ffffffff */
 	dd_emit(ctx, 1, 0);		/* 00000001 GP_BUILTIN_RESULT_EN.LAYER_IDX */
 	dd_emit(ctx, 1, 0);		/* ffffffff STRMOUT_ENABLE */
 	dd_emit(ctx, 1, 0x3fffff);	/* 003fffff TIC_LIMIT */
 	dd_emit(ctx, 1, 0x1fff);	/* 000fffff TSC_LIMIT */
 	dd_emit(ctx, 1, 0);		/* 00000001 VERTEX_TWO_SIDE_ENABLE*/
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		dd_emit(ctx, 8, 0);	/* 00000001 */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		dd_emit(ctx, 1, 1);	/* 00000007 VTX_ATTR_DEFINE.COMP */
 		dd_emit(ctx, 1, 1);	/* 00000007 VTX_ATTR_DEFINE.SIZE */
 		dd_emit(ctx, 1, 2);	/* 00000007 VTX_ATTR_DEFINE.TYPE */
@@ -1042,20 +1023,20 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
 	dd_emit(ctx, 1, 0);		/* 0000000f VP_TEXTURES_LOG2 */
 	dd_emit(ctx, 1, 0);		/* 0000000f VP_SAMPLERS_LOG2 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		dd_emit(ctx, 1, 0);	/* 00000001 */
 	dd_emit(ctx, 1, 2);		/* 00000003 POLYGON_MODE_BACK */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		dd_emit(ctx, 1, 0);	/* 00000003 VTX_ATTR_DEFINE.SIZE - 1 */
 	dd_emit(ctx, 1, 0);		/* 0000ffff CB_ADDR_INDEX */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		dd_emit(ctx, 1, 0);	/* 00000003 */
 	dd_emit(ctx, 1, 0);		/* 00000001 CULL_FACE_ENABLE */
 	dd_emit(ctx, 1, 1);		/* 00000003 CULL_FACE */
 	dd_emit(ctx, 1, 0);		/* 00000001 FRONT_FACE */
 	dd_emit(ctx, 1, 2);		/* 00000003 POLYGON_MODE_FRONT */
 	dd_emit(ctx, 1, 0x1000);	/* 00007fff UNK141C */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		dd_emit(ctx, 1, 0xe00);		/* 7fff */
 		dd_emit(ctx, 1, 0x1000);	/* 7fff */
 		dd_emit(ctx, 1, 0x1e00);	/* 7fff */
@@ -1070,10 +1051,10 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 	dd_emit(ctx, 1, 0);		/* 00000001 VTX_ATTR_MASK_UNK0 nonempty */
 	dd_emit(ctx, 1, 0);		/* 00000001 VTX_ATTR_MASK_UNK1 nonempty */
 	dd_emit(ctx, 1, 0x200);		/* 0003ffff GP_VERTEX_OUTPUT_COUNT*GP_REG_ALLOC_RESULT */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		dd_emit(ctx, 1, 0x200);
 	dd_emit(ctx, 1, 0);		/* 00000001 */
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		dd_emit(ctx, 1, 1);	/* 00000001 */
 		dd_emit(ctx, 1, 0x70);	/* 000000ff */
 		dd_emit(ctx, 1, 0x80);	/* 000000ff */
@@ -1120,7 +1101,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
 
 	num = ctx->ctxvals_pos - base;
 	ctx->ctxvals_pos = base;
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		cp_ctx(ctx, 0x404800, num);
 	else
 		cp_ctx(ctx, 0x405400, num);
@@ -1169,7 +1150,7 @@ nv50_graph_construct_mmio_ddata(struct nouveau_grctx *ctx)
  */
 
 static void
-xf_emit(struct nouveau_grctx *ctx, int num, uint32_t val) {
+xf_emit(struct nouveau_grctx *ctx, int num, u32 val) {
 	int i;
 	if (val && ctx->mode == NOUVEAU_GRCTX_VALS)
 		for (i = 0; i < num; i++)
@@ -1201,16 +1182,16 @@ static void nv50_graph_construct_xfer_tp(struct nouveau_grctx *ctx);
 static void
 nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 	int offset;
 	int size = 0;
-	uint32_t units = nv_rd32 (ctx->dev, 0x1540);
+	u32 units = nv_rd32 (ctx->device, 0x1540);
 
 	offset = (ctx->ctxvals_pos+0x3f)&~0x3f;
 	ctx->ctxvals_base = offset;
 
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		/* Strand 0 */
 		ctx->ctxvals_pos = offset;
 		nv50_graph_construct_gene_dispatch(ctx);
@@ -1280,7 +1261,7 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
 
 		/* Strand 2 */
 		ctx->ctxvals_pos = offset + 2;
-		if (dev_priv->chipset == 0xa0)
+		if (device->chipset == 0xa0)
 			nv50_graph_construct_gene_unk14xx(ctx);
 		nv50_graph_construct_gene_unk24xx(ctx);
 		if ((ctx->ctxvals_pos-offset)/8 > size)
@@ -1327,7 +1308,7 @@ nv50_graph_construct_xfer1(struct nouveau_grctx *ctx)
 
 		/* Strand 7 */
 		ctx->ctxvals_pos = offset + 7;
-		if (dev_priv->chipset == 0xa0) {
+		if (device->chipset == 0xa0) {
 			if (units & (1 << 4))
 				nv50_graph_construct_xfer_tp(ctx);
 			if (units & (1 << 5))
@@ -1365,24 +1346,24 @@ static void
 nv50_graph_construct_gene_dispatch(struct nouveau_grctx *ctx)
 {
 	/* start of strand 0 */
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* SEEK */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 5, 0);
-	else if (!IS_NVA3F(dev_priv->chipset))
+	else if (!IS_NVA3F(device->chipset))
 		xf_emit(ctx, 6, 0);
 	else
 		xf_emit(ctx, 4, 0);
 	/* SEEK */
 	/* the PGRAPH's internal FIFO */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 8*3, 0);
 	else
 		xf_emit(ctx, 0x100*3, 0);
 	/* and another bonus slot?!? */
 	xf_emit(ctx, 3, 0);
 	/* and YET ANOTHER bonus slot? */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 3, 0);
 	/* SEEK */
 	/* CTX_SWITCH: caches of gr objects bound to subchannels. 8 values, last used index */
@@ -1394,7 +1375,7 @@ nv50_graph_construct_gene_dispatch(struct nouveau_grctx *ctx)
 	/* SEEK */
 	xf_emit(ctx, 9, 0);
 	/* SEEK */
-	if (dev_priv->chipset < 0x90)
+	if (device->chipset < 0x90)
 		xf_emit(ctx, 4, 0);
 	/* SEEK */
 	xf_emit(ctx, 2, 0);
@@ -1407,9 +1388,9 @@ nv50_graph_construct_gene_dispatch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 6*2, 0);
 	xf_emit(ctx, 2, 0);
 	/* SEEK */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 0x1c, 0);
-	else if (dev_priv->chipset < 0xa0)
+	else if (device->chipset < 0xa0)
 		xf_emit(ctx, 0x1e, 0);
 	else
 		xf_emit(ctx, 0x22, 0);
@@ -1421,9 +1402,9 @@ static void
 nv50_graph_construct_gene_m2mf(struct nouveau_grctx *ctx)
 {
 	/* Strand 0, right after dispatch */
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int smallm2mf = 0;
-	if (dev_priv->chipset < 0x92 || dev_priv->chipset == 0x98)
+	if (device->chipset < 0x92 || device->chipset == 0x98)
 		smallm2mf = 1;
 	/* SEEK */
 	xf_emit (ctx, 1, 0);		/* DMA_NOTIFY instance >> 4 */
@@ -1472,10 +1453,10 @@ nv50_graph_construct_gene_m2mf(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_ccache(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 2, 0);		/* RO */
 	xf_emit(ctx, 0x800, 0);		/* ffffffff */
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x50:
 	case 0x92:
 	case 0xa0:
@@ -1540,7 +1521,7 @@ nv50_graph_construct_gene_ccache(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_unk10xx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 	/* end of area 2 on pre-NVA0, area 1 on NVAx */
 	xf_emit(ctx, 1, 4);		/* 000000ff GP_RESULT_MAP_SIZE */
@@ -1550,14 +1531,14 @@ nv50_graph_construct_gene_unk10xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 4);		/* 000000ff GP_REG_ALLOC_RESULT */
 	xf_emit(ctx, 1, 0x80c14);	/* 01ffffff SEMANTIC_COLOR */
 	xf_emit(ctx, 1, 0);		/* 00000001 VERTEX_TWO_SIDE_ENABLE */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0x3ff);
 	else
 		xf_emit(ctx, 1, 0x7ff);	/* 000007ff */
 	xf_emit(ctx, 1, 0);		/* 111/113 */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
 	for (i = 0; i < 8; i++) {
-		switch (dev_priv->chipset) {
+		switch (device->chipset) {
 		case 0x50:
 		case 0x86:
 		case 0x98:
@@ -1600,7 +1581,7 @@ nv50_graph_construct_gene_unk10xx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_unk34xx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* end of area 2 on pre-NVA0, area 1 on NVAx */
 	xf_emit(ctx, 1, 0);		/* 00000001 VIEWPORT_CLIP_RECTS_EN */
 	xf_emit(ctx, 1, 0);		/* 00000003 VIEWPORT_CLIP_MODE */
@@ -1614,9 +1595,9 @@ nv50_graph_construct_gene_unk34xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000003 WINDOW_ORIGIN */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
 	xf_emit(ctx, 1, 0x1fe21);	/* 0001ffff tesla UNK0FAC */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 0x0fac6881);
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 1);
 		xf_emit(ctx, 3, 0);
 	}
@@ -1625,9 +1606,9 @@ nv50_graph_construct_gene_unk34xx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* middle of area 2 on pre-NVA0, beginning of area 2 on NVA0, area 7 on >NVA0 */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		xf_emit(ctx, 5, 0);		/* ffffffff */
 		xf_emit(ctx, 1, 0x80c14);	/* 01ffffff SEMANTIC_COLOR */
 		xf_emit(ctx, 1, 0);		/* 00000001 */
@@ -1643,14 +1624,14 @@ nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);			/* 00000001 GP_ENABLE */
 	xf_emit(ctx, 1, 0x10);			/* 7f/ff VIEW_VOLUME_CLIP_CTRL */
 	xf_emit(ctx, 1, 0);			/* 000000ff VP_CLIP_DISTANCE_ENABLE */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0);		/* 3ff */
 	xf_emit(ctx, 1, 0);			/* 000000ff tesla UNK1940 */
 	xf_emit(ctx, 1, 0);			/* 00000001 tesla UNK0D7C */
 	xf_emit(ctx, 1, 0x804);			/* 00000fff SEMANTIC_CLIP */
 	xf_emit(ctx, 1, 1);			/* 00000001 VIEWPORT_TRANSFORM_EN */
 	xf_emit(ctx, 1, 0x1a);			/* 0000001f POLYGON_MODE */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0x7f);		/* 000000ff tesla UNK0FFC */
 	xf_emit(ctx, 1, 0);			/* ffffffff tesla UNK1A30 */
 	xf_emit(ctx, 1, 1);			/* 00000001 SHADE_MODEL */
@@ -1669,7 +1650,7 @@ nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 4, 0);			/* ffffffff NOPERSPECTIVE_BITMAP */
 	xf_emit(ctx, 1, 0);			/* 00000001 tesla UNK1900 */
 	xf_emit(ctx, 1, 0);			/* 0000000f */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0x3ff);		/* 000003ff tesla UNK0D68 */
 	else
 		xf_emit(ctx, 1, 0x7ff);		/* 000007ff tesla UNK0D68 */
@@ -1704,11 +1685,11 @@ nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);			/* 00000001 LINE_STIPPLE_ENABLE */
 	xf_emit(ctx, 1, 0);			/* 00000001 LINE_SMOOTH_ENABLE */
 	xf_emit(ctx, 1, 0);			/* 00000007 MULTISAMPLE_SAMPLES_LOG2 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 0);		/* 00000001 */
 	xf_emit(ctx, 1, 0x1a);			/* 0000001f POLYGON_MODE */
 	xf_emit(ctx, 1, 0x10);			/* 000000ff VIEW_VOLUME_CLIP_CTRL */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		xf_emit(ctx, 1, 0);		/* ffffffff */
 		xf_emit(ctx, 1, 0);		/* 00000001 */
 		xf_emit(ctx, 1, 0);		/* 000003ff */
@@ -1736,7 +1717,7 @@ nv50_graph_construct_gene_unk14xx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_zcull(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* end of strand 0 on pre-NVA0, beginning of strand 6 on NVAx */
 	/* SEEK */
 	xf_emit(ctx, 1, 0x3f);		/* 0000003f UNK1590 */
@@ -1774,7 +1755,7 @@ nv50_graph_construct_gene_zcull(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 STENCIL_BACK_ENABLE */
 	xf_emit(ctx, 1, 0);		/* ffffffff CLEAR_DEPTH */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0);	/* 00000003 tesla UNK1108 */
 	xf_emit(ctx, 1, 0);		/* 00000001 SAMPLECNT_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 0000000f ZETA_FORMAT */
@@ -1789,7 +1770,7 @@ nv50_graph_construct_gene_zcull(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 VIEWPORT_CLIP_RECTS_EN */
 	xf_emit(ctx, 1, 3);		/* 00000003 FP_CTRL_UNK196C */
 	xf_emit(ctx, 1, 0);		/* 00000003 tesla UNK1968 */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0);	/* 0fffffff tesla UNK1104 */
 	xf_emit(ctx, 1, 0);		/* 00000001 tesla UNK151C */
 }
@@ -1817,7 +1798,7 @@ nv50_graph_construct_gene_clipid(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
 	/* middle of strand 0 on pre-NVA0 [after m2mf], end of strand 2 on NVAx */
 	/* SEEK */
@@ -1829,7 +1810,7 @@ nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 4);		/* 0000007f VP_RESULT_MAP_SIZE */
 	xf_emit(ctx, 1, 4);		/* 000000ff GP_RESULT_MAP_SIZE */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 4, 0);	/* RO */
 		xf_emit(ctx, 0xe10, 0); /* 190 * 9: 8*ffffffff, 7ff */
 		xf_emit(ctx, 1, 0);	/* 1ff */
@@ -1860,7 +1841,7 @@ nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 4);		/* 000000ff GP_RESULT_MAP_SIZE */
 	xf_emit(ctx, 1, 4);		/* 0000007f VP_RESULT_MAP_SIZE */
 	xf_emit(ctx, 1, 0x8100c12);	/* 1fffffff FP_INTERPOLANT_CTRL */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 3);	/* 00000003 tesla UNK1100 */
 	/* SEEK */
 	xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
@@ -1869,7 +1850,7 @@ nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0x80c14);	/* 01ffffff SEMANTIC_COLOR */
 	xf_emit(ctx, 1, 1);		/* 00000001 */
 	/* SEEK */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 2, 4);	/* 000000ff */
 	xf_emit(ctx, 1, 0x80c14);	/* 01ffffff SEMANTIC_COLOR */
 	xf_emit(ctx, 1, 0);		/* 00000001 VERTEX_TWO_SIDE_ENABLE */
@@ -1893,20 +1874,20 @@ nv50_graph_construct_gene_unk24xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 0x10, 0);		/* 00ffffff POINT_COORD_REPLACE_MAP */
 	xf_emit(ctx, 1, 0);		/* 00000003 WINDOW_ORIGIN */
 	xf_emit(ctx, 1, 0x8100c12);	/* 1fffffff FP_INTERPOLANT_CTRL */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0);	/* 000003ff */
 }
 
 static void
 nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int acnt = 0x10, rep, i;
 	/* beginning of strand 1 on pre-NVA0, strand 3 on NVAx */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		acnt = 0x20;
 	/* SEEK */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 1, 0);	/* ffffffff tesla UNK13A4 */
 		xf_emit(ctx, 1, 1);	/* 00000fff tesla UNK1318 */
 	}
@@ -1923,9 +1904,9 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 0000ffff turing USER_PARAM_COUNT */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0xb, 0);	/* RO */
-	else if (dev_priv->chipset >= 0xa0)
+	else if (device->chipset >= 0xa0)
 		xf_emit(ctx, 0x9, 0);	/* RO */
 	else
 		xf_emit(ctx, 0x8, 0);	/* RO */
@@ -1944,11 +1925,11 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 4);		/* 000001ff UNK1A28 */
 	xf_emit(ctx, 1, 8);		/* 000001ff UNK0DF0 */
 	xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0x3ff);	/* 3ff tesla UNK0D68 */
 	else
 		xf_emit(ctx, 1, 0x7ff);	/* 7ff tesla UNK0D68 */
-	if (dev_priv->chipset == 0xa8)
+	if (device->chipset == 0xa8)
 		xf_emit(ctx, 1, 0x1e00);	/* 7fff */
 	/* SEEK */
 	xf_emit(ctx, 0xc, 0);		/* RO or close */
@@ -1956,13 +1937,13 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0xf);		/* ffffffff VP_ATTR_EN */
 	xf_emit(ctx, (acnt/8)-1, 0);	/* ffffffff VP_ATTR_EN */
 	xf_emit(ctx, 1, 0);		/* 0000000f VP_GP_BUILTIN_ATTR_EN */
-	if (dev_priv->chipset > 0x50 && dev_priv->chipset < 0xa0)
+	if (device->chipset > 0x50 && device->chipset < 0xa0)
 		xf_emit(ctx, 2, 0);	/* ffffffff */
 	else
 		xf_emit(ctx, 1, 0);	/* ffffffff */
 	xf_emit(ctx, 1, 0);		/* 00000003 tesla UNK0FD8 */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 0x10, 0);	/* 0? */
 		xf_emit(ctx, 2, 0);	/* weird... */
 		xf_emit(ctx, 2, 0);	/* RO */
@@ -1975,7 +1956,7 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* ffffffff VB_ELEMENT_BASE */
 	xf_emit(ctx, 1, 0);		/* ffffffff UNK1438 */
 	xf_emit(ctx, acnt, 0);		/* 1 tesla UNK1000 */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 0);	/* ffffffff tesla UNK1118? */
 	/* SEEK */
 	xf_emit(ctx, acnt, 0);		/* ffffffff VERTEX_ARRAY_UNK90C */
@@ -2013,23 +1994,23 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, acnt, 0);		/* 000000ff VERTEX_LIMIT_HIGH */
 	xf_emit(ctx, 3, 0);		/* f/1f */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, acnt, 0);		/* f */
 		xf_emit(ctx, 3, 0);		/* f/1f */
 	}
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 2, 0);	/* RO */
 	else
 		xf_emit(ctx, 5, 0);	/* RO */
 	/* SEEK */
 	xf_emit(ctx, 1, 0);		/* ffff DMA_VTXBUF */
 	/* SEEK */
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		xf_emit(ctx, 0x41, 0);	/* RO */
 		/* SEEK */
 		xf_emit(ctx, 0x11, 0);	/* RO */
-	} else if (!IS_NVA3F(dev_priv->chipset))
+	} else if (!IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0x50, 0);	/* RO */
 	else
 		xf_emit(ctx, 0x58, 0);	/* RO */
@@ -2041,7 +2022,7 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, acnt*4, 0);	/* ffffffff VTX_ATTR */
 	xf_emit(ctx, 4, 0);		/* f/1f, 0, 0, 0 */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0x1d, 0);	/* RO */
 	else
 		xf_emit(ctx, 0x16, 0);	/* RO */
@@ -2049,21 +2030,21 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0xf);		/* ffffffff VP_ATTR_EN */
 	xf_emit(ctx, (acnt/8)-1, 0);	/* ffffffff VP_ATTR_EN */
 	/* SEEK */
-	if (dev_priv->chipset < 0xa0)
+	if (device->chipset < 0xa0)
 		xf_emit(ctx, 8, 0);	/* RO */
-	else if (IS_NVA3F(dev_priv->chipset))
+	else if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0xc, 0);	/* RO */
 	else
 		xf_emit(ctx, 7, 0);	/* RO */
 	/* SEEK */
 	xf_emit(ctx, 0xa, 0);		/* RO */
-	if (dev_priv->chipset == 0xa0)
+	if (device->chipset == 0xa0)
 		rep = 0xc;
 	else
 		rep = 4;
 	for (i = 0; i < rep; i++) {
 		/* SEEK */
-		if (IS_NVA3F(dev_priv->chipset))
+		if (IS_NVA3F(device->chipset))
 			xf_emit(ctx, 0x20, 0);	/* ffffffff */
 		xf_emit(ctx, 0x200, 0);	/* ffffffff */
 		xf_emit(ctx, 4, 0);	/* 7f/ff, 0, 0, 0 */
@@ -2077,7 +2058,7 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 0000000f VP_GP_BUILTIN_ATTR_EN */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
 	/* SEEK */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 7, 0);	/* weird... */
 	else
 		xf_emit(ctx, 5, 0);	/* weird... */
@@ -2086,13 +2067,13 @@ nv50_graph_construct_gene_vfetch(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_eng2d(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* middle of strand 1 on pre-NVA0 [after vfetch], middle of strand 6 on NVAx */
 	/* SEEK */
 	xf_emit(ctx, 2, 0);		/* 0001ffff CLIP_X, CLIP_Y */
 	xf_emit(ctx, 2, 0);		/* 0000ffff CLIP_W, CLIP_H */
 	xf_emit(ctx, 1, 0);		/* 00000001 CLIP_ENABLE */
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		/* this is useless on everything but the original NV50,
 		 * guess they forgot to nuke it. Or just didn't bother. */
 		xf_emit(ctx, 2, 0);	/* 0000ffff IFC_CLIP_X, Y */
@@ -2148,7 +2129,7 @@ nv50_graph_construct_gene_eng2d(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_csched(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* middle of strand 1 on pre-NVA0 [after eng2d], middle of strand 0 on NVAx */
 	/* SEEK */
 	xf_emit(ctx, 2, 0);		/* 00007fff WINDOW_OFFSET_XY... what is it doing here??? */
@@ -2173,7 +2154,7 @@ nv50_graph_construct_gene_csched(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 2);		/* 00000003 REG_MODE */
 	/* SEEK */
 	xf_emit(ctx, 0x40, 0);		/* ffffffff USER_PARAM */
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x50:
 	case 0x92:
 		xf_emit(ctx, 8, 0);	/* 7, 0, 0, 0, ... */
@@ -2247,7 +2228,7 @@ nv50_graph_construct_gene_csched(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 2, 0);		/* 00007fff WINDOW_OFFSET_XY */
 	xf_emit(ctx, 1, 0x3f800000);	/* ffffffff LINE_WIDTH */
 	xf_emit(ctx, 1, 0);		/* 00000001 LINE_SMOOTH_ENABLE */
@@ -2277,9 +2258,9 @@ nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 4);		/* 00000007 FP_CONTROL */
 	xf_emit(ctx, 1, 0);		/* 00000001 ALPHA_TEST_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 00000007 ALPHA_TEST_FUNC */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 3);	/* 00000003 UNK16B4 */
-	else if (dev_priv->chipset >= 0xa0)
+	else if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 1);	/* 00000001 UNK16B4 */
 	xf_emit(ctx, 1, 0);		/* 00000003 MULTISAMPLE_CTRL */
 	xf_emit(ctx, 1, 0);		/* 00000003 tesla UNK0F90 */
@@ -2293,11 +2274,11 @@ nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* ffffffff POINT_SIZE */
 	xf_emit(ctx, 1, 0);		/* 00000001 */
 	xf_emit(ctx, 1, 0);		/* 00000007 tesla UNK0FB4 */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		xf_emit(ctx, 1, 0);	/* 3ff */
 		xf_emit(ctx, 1, 1);	/* 00000001 tesla UNK1110 */
 	}
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 0);	/* 00000003 tesla UNK1928 */
 	xf_emit(ctx, 0x10, 0);		/* ffffffff DEPTH_RANGE_NEAR */
 	xf_emit(ctx, 0x10, 0x3f800000);	/* ffffffff DEPTH_RANGE_FAR */
@@ -2316,11 +2297,11 @@ nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 00000001 VERTEX_TWO_SIDE_ENABLE */
 	xf_emit(ctx, 4, 0xffff);	/* 0000ffff MSAA_MASK */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 3);	/* 00000003 tesla UNK1100 */
-	if (dev_priv->chipset < 0xa0)
+	if (device->chipset < 0xa0)
 		xf_emit(ctx, 0x1c, 0);	/* RO */
-	else if (IS_NVA3F(dev_priv->chipset))
+	else if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0x9, 0);
 	xf_emit(ctx, 1, 0);		/* 00000001 UNK1534 */
 	xf_emit(ctx, 1, 0);		/* 00000001 LINE_SMOOTH_ENABLE */
@@ -2328,13 +2309,13 @@ nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0x00ffff00);	/* 00ffffff LINE_STIPPLE_PATTERN */
 	xf_emit(ctx, 1, 0x1a);		/* 0000001f POLYGON_MODE */
 	xf_emit(ctx, 1, 0);		/* 00000003 WINDOW_ORIGIN */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		xf_emit(ctx, 1, 3);	/* 00000003 tesla UNK1100 */
 		xf_emit(ctx, 1, 0);	/* 3ff */
 	}
 	/* XXX: the following block could belong either to unk1cxx, or
 	 * to STRMOUT. Rather hard to tell. */
-	if (dev_priv->chipset < 0xa0)
+	if (device->chipset < 0xa0)
 		xf_emit(ctx, 0x25, 0);
 	else
 		xf_emit(ctx, 0x3b, 0);
@@ -2343,18 +2324,18 @@ nv50_graph_construct_gene_unk1cxx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_strmout(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 1, 0x102);		/* 0000ffff STRMOUT_BUFFER_CTRL */
 	xf_emit(ctx, 1, 0);		/* ffffffff STRMOUT_PRIMITIVE_COUNT */
 	xf_emit(ctx, 4, 4);		/* 000000ff STRMOUT_NUM_ATTRIBS */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 4, 0);	/* ffffffff UNK1A8C */
 		xf_emit(ctx, 4, 0);	/* ffffffff UNK1780 */
 	}
 	xf_emit(ctx, 1, 4);		/* 000000ff GP_RESULT_MAP_SIZE */
 	xf_emit(ctx, 1, 4);		/* 0000007f VP_RESULT_MAP_SIZE */
 	xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0x3ff);	/* 000003ff tesla UNK0D68 */
 	else
 		xf_emit(ctx, 1, 0x7ff);	/* 000007ff tesla UNK0D68 */
@@ -2365,7 +2346,7 @@ nv50_graph_construct_gene_strmout(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 4, 0);		/* 000000ff STRMOUT_ADDRESS_HIGH */
 	xf_emit(ctx, 4, 0);		/* ffffffff STRMOUT_ADDRESS_LOW */
 	xf_emit(ctx, 4, 4);		/* 000000ff STRMOUT_NUM_ATTRIBS */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 4, 0);	/* ffffffff UNK1A8C */
 		xf_emit(ctx, 4, 0);	/* ffffffff UNK1780 */
 	}
@@ -2385,12 +2366,12 @@ nv50_graph_construct_gene_strmout(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_ropm1(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 1, 0x4e3bfdf);	/* ffffffff UNK0D64 */
 	xf_emit(ctx, 1, 0x4e3bfdf);	/* ffffffff UNK0DF4 */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
 	xf_emit(ctx, 1, 0);		/* 000003ff */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 0x11);	/* 000000ff tesla UNK1968 */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
 }
@@ -2398,7 +2379,7 @@ nv50_graph_construct_gene_ropm1(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_ropm2(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	/* SEEK */
 	xf_emit(ctx, 1, 0);		/* 0000ffff DMA_QUERY */
 	xf_emit(ctx, 1, 0x0fac6881);	/* 0fffffff RT_CONTROL */
@@ -2416,7 +2397,7 @@ nv50_graph_construct_gene_ropm2(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 eng2d UNK260 */
 	xf_emit(ctx, 1, 0);		/* ff/3ff */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 0x11);	/* 000000ff tesla UNK1968 */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
 }
@@ -2424,11 +2405,11 @@ nv50_graph_construct_gene_ropm2(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int magic2;
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		magic2 = 0x00003e60;
-	} else if (!IS_NVA3F(dev_priv->chipset)) {
+	} else if (!IS_NVA3F(device->chipset)) {
 		magic2 = 0x001ffe67;
 	} else {
 		magic2 = 0x00087e67;
@@ -2446,14 +2427,14 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000007 DEPTH_TEST_FUNC */
 	xf_emit(ctx, 1, 0);		/* 00000001 DEPTH_TEST_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 00000001 DEPTH_WRITE_ENABLE */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 1, 0);		/* 00000007 STENCIL_FRONT_FUNC_FUNC */
 	xf_emit(ctx, 1, 0);		/* 000000ff STENCIL_FRONT_FUNC_MASK */
 	xf_emit(ctx, 1, 0);		/* 000000ff STENCIL_FRONT_MASK */
 	xf_emit(ctx, 3, 0);		/* 00000007 STENCIL_FRONT_OP_FAIL, ZFAIL, ZPASS */
 	xf_emit(ctx, 1, 0);		/* 00000001 STENCIL_FRONT_ENABLE */
-	if (dev_priv->chipset >= 0xa0 && !IS_NVAAF(dev_priv->chipset))
+	if (device->chipset >= 0xa0 && !IS_NVAAF(device->chipset))
 		xf_emit(ctx, 1, 0x15);	/* 000000ff */
 	xf_emit(ctx, 1, 0);		/* 00000001 STENCIL_BACK_ENABLE */
 	xf_emit(ctx, 1, 1);		/* 00000001 tesla UNK15B4 */
@@ -2462,14 +2443,14 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 0000000f ZETA_FORMAT */
 	xf_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
-	if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x92 || dev_priv->chipset == 0x98 || dev_priv->chipset >= 0xa0) {
+	if (device->chipset == 0x86 || device->chipset == 0x92 || device->chipset == 0x98 || device->chipset >= 0xa0) {
 		xf_emit(ctx, 3, 0);	/* ff, ffffffff, ffffffff */
 		xf_emit(ctx, 1, 4);	/* 7 */
 		xf_emit(ctx, 1, 0x400);	/* fffffff */
 		xf_emit(ctx, 1, 0x300);	/* ffff */
 		xf_emit(ctx, 1, 0x1001);	/* 1fff */
-		if (dev_priv->chipset != 0xa0) {
-			if (IS_NVA3F(dev_priv->chipset))
+		if (device->chipset != 0xa0) {
+			if (IS_NVA3F(device->chipset))
 				xf_emit(ctx, 1, 0);	/* 0000000f UNK15C8 */
 			else
 				xf_emit(ctx, 1, 0x15);	/* ff */
@@ -2547,7 +2528,7 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
 	xf_emit(ctx, 1, 0);		/* ffffffff CLEAR_DEPTH */
 	xf_emit(ctx, 1, 1);		/* 00000001 tesla UNK19CC */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 2, 0);
 		xf_emit(ctx, 1, 0x1001);
 		xf_emit(ctx, 0xb, 0);
@@ -2564,7 +2545,7 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 7, 0);		/* 0000000f COLOR_MASK */
 	xf_emit(ctx, 1, 0x11);		/* 3f/7f */
 	xf_emit(ctx, 1, 0);		/* 00000001 LOGIC_OP_ENABLE */
-	if (dev_priv->chipset != 0x50) {
+	if (device->chipset != 0x50) {
 		xf_emit(ctx, 1, 0);	/* 0000000f LOGIC_OP */
 		xf_emit(ctx, 1, 0);	/* 000000ff */
 	}
@@ -2581,7 +2562,7 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, magic2);	/* 001fffff tesla UNK0F78 */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
 	xf_emit(ctx, 1, 0x0fac6881);	/* 0fffffff RT_CONTROL */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 0);	/* 00000001 tesla UNK12E4 */
 		xf_emit(ctx, 8, 1);	/* 00000007 IBLEND_EQUATION_RGB */
 		xf_emit(ctx, 8, 1);	/* 00000007 IBLEND_EQUATION_ALPHA */
@@ -2600,7 +2581,7 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 		xf_emit(ctx, 1, 0);	/* 00000001 */
 		xf_emit(ctx, 1, 0);	/* 000003ff */
-	} else if (dev_priv->chipset >= 0xa0) {
+	} else if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 2, 0);	/* 00000001 */
 		xf_emit(ctx, 1, 0);	/* 00000007 */
 		xf_emit(ctx, 1, 0);	/* 00000003 */
@@ -2614,7 +2595,7 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 4, 0);		/* ffffffff CLEAR_COLOR */
 	xf_emit(ctx, 4, 0);		/* ffffffff BLEND_COLOR A R G B */
 	xf_emit(ctx, 1, 0);		/* 00000fff eng2d UNK2B0 */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 2, 0);	/* 00000001 */
 	xf_emit(ctx, 1, 0);		/* 000003ff */
 	xf_emit(ctx, 8, 0);		/* 00000001 BLEND_ENABLE */
@@ -2628,9 +2609,9 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 UNK19C0 */
 	xf_emit(ctx, 1, 0);		/* 00000001 LOGIC_OP_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 0000000f LOGIC_OP */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 0);	/* 00000001 UNK12E4? NVA3+ only? */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 8, 1);	/* 00000001 IBLEND_UNK00 */
 		xf_emit(ctx, 8, 1);	/* 00000007 IBLEND_EQUATION_RGB */
 		xf_emit(ctx, 8, 2);	/* 0000001f IBLEND_FUNC_SRC_RGB */
@@ -2659,9 +2640,9 @@ nv50_graph_construct_gene_ropc(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_unk84xx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int magic3;
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 	case 0x50:
 		magic3 = 0x1000;
 		break;
@@ -2681,16 +2662,16 @@ nv50_graph_construct_xfer_unk84xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
 	xf_emit(ctx, 1, 0);		/* 111/113[NVA0+] */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0x1f, 0);	/* ffffffff */
-	else if (dev_priv->chipset >= 0xa0)
+	else if (device->chipset >= 0xa0)
 		xf_emit(ctx, 0x0f, 0);	/* ffffffff */
 	else
 		xf_emit(ctx, 0x10, 0);	/* fffffff VP_RESULT_MAP_1 up */
 	xf_emit(ctx, 2, 0);		/* f/1f[NVA3], fffffff/ffffffff[NVA0+] */
 	xf_emit(ctx, 1, 4);		/* 7f/ff VP_REG_ALLOC_RESULT */
 	xf_emit(ctx, 1, 4);		/* 7f/ff VP_RESULT_MAP_SIZE */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 0x03020100);	/* ffffffff */
 	else
 		xf_emit(ctx, 1, 0x00608080);	/* fffffff VP_RESULT_MAP_0 */
@@ -2733,11 +2714,11 @@ nv50_graph_construct_xfer_unk84xx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
 	xf_emit(ctx, 1, 0);		/* 111/113 */
-	if (dev_priv->chipset == 0x94 || dev_priv->chipset == 0x96)
+	if (device->chipset == 0x94 || device->chipset == 0x96)
 		xf_emit(ctx, 0x1020, 0);	/* 4 x (0x400 x 0xffffffff, ff, 0, 0, 0, 4 x ffffffff) */
-	else if (dev_priv->chipset < 0xa0)
+	else if (device->chipset < 0xa0)
 		xf_emit(ctx, 0xa20, 0);	/* 4 x (0x280 x 0xffffffff, ff, 0, 0, 0, 4 x ffffffff) */
-	else if (!IS_NVA3F(dev_priv->chipset))
+	else if (!IS_NVA3F(device->chipset))
 		xf_emit(ctx, 0x210, 0);	/* ffffffff */
 	else
 		xf_emit(ctx, 0x410, 0);	/* ffffffff */
@@ -2751,12 +2732,12 @@ nv50_graph_construct_xfer_unk84xx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int magic1, magic2;
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		magic1 = 0x3ff;
 		magic2 = 0x00003e60;
-	} else if (!IS_NVA3F(dev_priv->chipset)) {
+	} else if (!IS_NVA3F(device->chipset)) {
 		magic1 = 0x7ff;
 		magic2 = 0x001ffe67;
 	} else {
@@ -2766,7 +2747,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000007 ALPHA_TEST_FUNC */
 	xf_emit(ctx, 1, 0);		/* ffffffff ALPHA_TEST_REF */
 	xf_emit(ctx, 1, 0);		/* 00000001 ALPHA_TEST_ENABLE */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000000f UNK16A0 */
 	xf_emit(ctx, 1, 0);		/* 7/f MULTISAMPLE_SAMPLES_LOG2 */
 	xf_emit(ctx, 1, 0);		/* 00000001 tesla UNK1534 */
@@ -2800,11 +2781,11 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);		/* 00000001 SIFC_BITMAP_WRITE_BIT0_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 00000007 ALPHA_TEST_FUNC */
 	xf_emit(ctx, 1, 0);		/* 00000001 ALPHA_TEST_ENABLE */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 3);	/* 00000003 tesla UNK16B4 */
 		xf_emit(ctx, 1, 0);	/* 00000003 */
 		xf_emit(ctx, 1, 0);	/* 00000003 tesla UNK1298 */
-	} else if (dev_priv->chipset >= 0xa0) {
+	} else if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 1, 1);	/* 00000001 tesla UNK16B4 */
 		xf_emit(ctx, 1, 0);	/* 00000003 */
 	} else {
@@ -2818,7 +2799,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);		/* 0000001f BLEND_FUNC_DST_RGB */
 	xf_emit(ctx, 1, 1);		/* 00000007 BLEND_EQUATION_RGB */
 	xf_emit(ctx, 1, 2);		/* 0000001f BLEND_FUNC_SRC_RGB */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 0);	/* 00000001 UNK12E4 */
 		xf_emit(ctx, 8, 1);	/* 00000007 IBLEND_EQUATION_RGB */
 		xf_emit(ctx, 8, 1);	/* 00000007 IBLEND_EQUATION_ALPHA */
@@ -2846,7 +2827,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0xcf);		/* 000000ff SIFC_FORMAT */
 	xf_emit(ctx, 1, 0xcf);		/* 000000ff DRAW_COLOR_FORMAT */
 	xf_emit(ctx, 1, 0xcf);		/* 000000ff SRC_FORMAT */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
 	xf_emit(ctx, 1, 0);		/* 7/f[NVA3] MULTISAMPLE_SAMPLES_LOG2 */
@@ -2870,9 +2851,9 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 DEPTH_TEST_ENABLE */
 	xf_emit(ctx, 1, 0x11);		/* 3f/7f DST_FORMAT */
 	xf_emit(ctx, 1, 1);		/* 00000001 DST_LINEAR */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0);	/* ff */
 	else
 		xf_emit(ctx, 3, 0);	/* 1, 7, 3ff */
@@ -2907,7 +2888,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 8, 0);		/* 0000ffff DMA_COLOR */
 	xf_emit(ctx, 1, 0);		/* 0000ffff DMA_GLOBAL */
@@ -2945,7 +2926,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 0001ffff GP_BUILTIN_RESULT_EN */
 	xf_emit(ctx, 1, 0);		/* 00000003 UNK0F90 */
 	xf_emit(ctx, 1, 0);		/* 00000007 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 1, magic2);	/* 001fffff tesla UNK0F78 */
 	xf_emit(ctx, 1, 0);		/* 7/f MULTISAMPLE_SAMPLES_LOG2 */
@@ -2974,7 +2955,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0x1001);	/* 00001fff ZETA_ARRAY_MODE */
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
 	xf_emit(ctx, 1, 0);		/* 7/f MULTISAMPLE_SAMPLES_LOG2 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 0);	/* 00000001 */
 	xf_emit(ctx, 1, 0);		/* ffff0ff3 */
 	xf_emit(ctx, 1, 0x11);		/* 3f/7f RT_FORMAT */
@@ -2988,14 +2969,14 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 FRAMEBUFFER_SRGB */
 	xf_emit(ctx, 1, 0);		/* 7 */
 	xf_emit(ctx, 1, 0);		/* 00000001 LOGIC_OP_ENABLE */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 0);	/* 00000001 UNK1140 */
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	}
 	xf_emit(ctx, 1, 0);		/* 7/f MULTISAMPLE_SAMPLES_LOG2 */
 	xf_emit(ctx, 1, 0);		/* 00000001 UNK1534 */
 	xf_emit(ctx, 1, 0);		/* ffff0ff3 */
-	if (dev_priv->chipset >= 0xa0)
+	if (device->chipset >= 0xa0)
 		xf_emit(ctx, 1, 0x0fac6881);	/* fffffff */
 	xf_emit(ctx, 1, magic2);	/* 001fffff tesla UNK0F78 */
 	xf_emit(ctx, 1, 0);		/* 00000001 DEPTH_BOUNDS_EN */
@@ -3012,12 +2993,12 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 SAMPLECNT_ENABLE */
 	xf_emit(ctx, 1, 0);		/* 0000000f ZETA_FORMAT */
 	xf_emit(ctx, 1, 1);		/* 00000001 ZETA_ENABLE */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 		xf_emit(ctx, 1, 0);	/* 0000000f tesla UNK15C8 */
 	}
 	xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A3C */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 3, 0);		/* 7/f, 1, ffff0ff3 */
 		xf_emit(ctx, 1, 0xfac6881);	/* fffffff */
 		xf_emit(ctx, 4, 0);		/* 1, 1, 1, 3ff */
@@ -3027,7 +3008,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 		xf_emit(ctx, 2, 0);		/* 7, f */
 		xf_emit(ctx, 1, 1);		/* 1 */
 		xf_emit(ctx, 1, 0);		/* 7/f */
-		if (IS_NVA3F(dev_priv->chipset))
+		if (IS_NVA3F(device->chipset))
 			xf_emit(ctx, 0x9, 0);	/* 1 */
 		else
 			xf_emit(ctx, 0x8, 0);	/* 1 */
@@ -3041,7 +3022,7 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 		xf_emit(ctx, 1, 0x11);		/* 7f */
 		xf_emit(ctx, 1, 1);		/* 1 */
 		xf_emit(ctx, 5, 0);		/* 1, 7, 3ff, 3, 7 */
-		if (IS_NVA3F(dev_priv->chipset)) {
+		if (IS_NVA3F(device->chipset)) {
 			xf_emit(ctx, 1, 0);	/* 00000001 UNK1140 */
 			xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 		}
@@ -3051,15 +3032,15 @@ nv50_graph_construct_xfer_tprop(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_tex(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 2, 0);		/* 1 LINKED_TSC. yes, 2. */
-	if (dev_priv->chipset != 0x50)
+	if (device->chipset != 0x50)
 		xf_emit(ctx, 1, 0);	/* 3 */
 	xf_emit(ctx, 1, 1);		/* 1ffff BLIT_DU_DX_INT */
 	xf_emit(ctx, 1, 0);		/* fffff BLIT_DU_DX_FRACT */
 	xf_emit(ctx, 1, 1);		/* 1ffff BLIT_DV_DY_INT */
 	xf_emit(ctx, 1, 0);		/* fffff BLIT_DV_DY_FRACT */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 1, 0);	/* 3 BLIT_CONTROL */
 	else
 		xf_emit(ctx, 2, 0);	/* 3ff, 1 */
@@ -3071,13 +3052,13 @@ nv50_graph_construct_xfer_tex(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0x10100);	/* ffffffff SRC_TIC_5 */
 	xf_emit(ctx, 1, 0x02800000);	/* ffffffff SRC_TIC_6 */
 	xf_emit(ctx, 1, 0);		/* ffffffff SRC_TIC_7 */
-	if (dev_priv->chipset == 0x50) {
+	if (device->chipset == 0x50) {
 		xf_emit(ctx, 1, 0);	/* 00000001 turing UNK358 */
 		xf_emit(ctx, 1, 0);	/* ffffffff tesla UNK1A34? */
 		xf_emit(ctx, 1, 0);	/* 00000003 turing UNK37C tesla UNK1690 */
 		xf_emit(ctx, 1, 0);	/* 00000003 BLIT_CONTROL */
 		xf_emit(ctx, 1, 0);	/* 00000001 turing UNK32C tesla UNK0F94 */
-	} else if (!IS_NVAAF(dev_priv->chipset)) {
+	} else if (!IS_NVAAF(device->chipset)) {
 		xf_emit(ctx, 1, 0);	/* ffffffff tesla UNK1A34? */
 		xf_emit(ctx, 1, 0);	/* 00000003 */
 		xf_emit(ctx, 1, 0);	/* 000003ff */
@@ -3097,7 +3078,7 @@ nv50_graph_construct_xfer_tex(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_unk8cxx(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	xf_emit(ctx, 1, 0);		/* 00000001 UNK1534 */
 	xf_emit(ctx, 1, 0);		/* 7/f MULTISAMPLE_SAMPLES_LOG2 */
 	xf_emit(ctx, 2, 0);		/* 7, ffff0ff3 */
@@ -3109,7 +3090,7 @@ nv50_graph_construct_xfer_unk8cxx(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);		/* 00000001 LINE_STIPPLE_ENABLE */
 	xf_emit(ctx, 1, 0x00ffff00);	/* 00ffffff LINE_STIPPLE_PATTERN */
 	xf_emit(ctx, 1, 1);		/* 00000001 tesla UNK0F98 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);	/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 1, 0);		/* 00000003 tesla UNK1668 */
 	xf_emit(ctx, 1, 0);		/* 00000001 LINE_STIPPLE_ENABLE */
@@ -3136,8 +3117,8 @@ nv50_graph_construct_xfer_unk8cxx(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_tp(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
-	if (dev_priv->chipset < 0xa0) {
+	struct nouveau_device *device = ctx->device;
+	if (device->chipset < 0xa0) {
 		nv50_graph_construct_xfer_unk84xx(ctx);
 		nv50_graph_construct_xfer_tprop(ctx);
 		nv50_graph_construct_xfer_tex(ctx);
@@ -3153,9 +3134,9 @@ nv50_graph_construct_xfer_tp(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i, mpcnt = 2;
-	switch (dev_priv->chipset) {
+	switch (device->chipset) {
 		case 0x98:
 		case 0xaa:
 			mpcnt = 1;
@@ -3182,34 +3163,34 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 		xf_emit(ctx, 1, 0x80);		/* ffffffff tesla UNK1404 */
 		xf_emit(ctx, 1, 0x80007004);	/* ffffffff tesla UNK12B0 */
 		xf_emit(ctx, 1, 0x04000400);	/* ffffffff */
-		if (dev_priv->chipset >= 0xa0)
+		if (device->chipset >= 0xa0)
 			xf_emit(ctx, 1, 0xc0);	/* 00007fff tesla UNK152C */
 		xf_emit(ctx, 1, 0x1000);	/* 0000ffff tesla UNK0D60 */
 		xf_emit(ctx, 1, 0);		/* ff/3ff */
 		xf_emit(ctx, 1, 0);		/* ffffffff tesla UNK1A30 */
-		if (dev_priv->chipset == 0x86 || dev_priv->chipset == 0x98 || dev_priv->chipset == 0xa8 || IS_NVAAF(dev_priv->chipset)) {
+		if (device->chipset == 0x86 || device->chipset == 0x98 || device->chipset == 0xa8 || IS_NVAAF(device->chipset)) {
 			xf_emit(ctx, 1, 0xe00);		/* 7fff */
 			xf_emit(ctx, 1, 0x1e00);	/* 7fff */
 		}
 		xf_emit(ctx, 1, 1);		/* 000000ff VP_REG_ALLOC_TEMP */
 		xf_emit(ctx, 1, 0);		/* 00000001 LINKED_TSC */
 		xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
-		if (dev_priv->chipset == 0x50)
+		if (device->chipset == 0x50)
 			xf_emit(ctx, 2, 0x1000);	/* 7fff tesla UNK141C */
 		xf_emit(ctx, 1, 1);		/* 000000ff GP_REG_ALLOC_TEMP */
 		xf_emit(ctx, 1, 0);		/* 00000001 GP_ENABLE */
 		xf_emit(ctx, 1, 4);		/* 000000ff FP_REG_ALLOC_TEMP */
 		xf_emit(ctx, 1, 2);		/* 00000003 REG_MODE */
-		if (IS_NVAAF(dev_priv->chipset))
+		if (IS_NVAAF(device->chipset))
 			xf_emit(ctx, 0xb, 0);	/* RO */
-		else if (dev_priv->chipset >= 0xa0)
+		else if (device->chipset >= 0xa0)
 			xf_emit(ctx, 0xc, 0);	/* RO */
 		else
 			xf_emit(ctx, 0xa, 0);	/* RO */
 	}
 	xf_emit(ctx, 1, 0x08100c12);		/* 1fffffff FP_INTERPOLANT_CTRL */
 	xf_emit(ctx, 1, 0);			/* ff/3ff */
-	if (dev_priv->chipset >= 0xa0) {
+	if (device->chipset >= 0xa0) {
 		xf_emit(ctx, 1, 0x1fe21);	/* 0003ffff tesla UNK0FAC */
 	}
 	xf_emit(ctx, 3, 0);			/* 7fff, 0, 0 */
@@ -3223,7 +3204,7 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);			/* ffffffff SHARED_SIZE */
 	xf_emit(ctx, 1, 0x1fe21);		/* 1ffff/3ffff[NVA0+] tesla UNk0FAC */
 	xf_emit(ctx, 1, 0);			/* ffffffff tesla UNK1A34 */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 1);		/* 0000001f tesla UNK169C */
 	xf_emit(ctx, 1, 0);			/* ff/3ff */
 	xf_emit(ctx, 1, 0);			/* 1 LINKED_TSC */
@@ -3238,7 +3219,7 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);			/* 00000007 */
 	xf_emit(ctx, 1, 0xfac6881);		/* 0fffffff RT_CONTROL */
 	xf_emit(ctx, 1, 0);			/* 00000003 MULTISAMPLE_CTRL */
-	if (IS_NVA3F(dev_priv->chipset))
+	if (IS_NVA3F(device->chipset))
 		xf_emit(ctx, 1, 3);		/* 00000003 tesla UNK16B4 */
 	xf_emit(ctx, 1, 0);			/* 00000001 ALPHA_TEST_ENABLE */
 	xf_emit(ctx, 1, 0);			/* 00000007 ALPHA_TEST_FUNC */
@@ -3253,7 +3234,7 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 1);			/* 0000001f BLEND_FUNC_DST_ALPHA */
 	xf_emit(ctx, 1, 1);			/* 00000007 BLEND_EQUATION_ALPHA */
 	xf_emit(ctx, 1, 1);			/* 00000001 UNK133C */
-	if (IS_NVA3F(dev_priv->chipset)) {
+	if (IS_NVA3F(device->chipset)) {
 		xf_emit(ctx, 1, 0);		/* 00000001 UNK12E4 */
 		xf_emit(ctx, 8, 2);		/* 0000001f IBLEND_FUNC_SRC_RGB */
 		xf_emit(ctx, 8, 1);		/* 0000001f IBLEND_FUNC_DST_RGB */
@@ -3268,11 +3249,11 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 	xf_emit(ctx, 1, 0);			/* 00000003 tesla UNK0F90 */
 	xf_emit(ctx, 1, 4);			/* 000000ff FP_RESULT_COUNT */
 	/* XXX: demagic this part some day */
-	if (dev_priv->chipset == 0x50)
+	if (device->chipset == 0x50)
 		xf_emit(ctx, 0x3a0, 0);
-	else if (dev_priv->chipset < 0x94)
+	else if (device->chipset < 0x94)
 		xf_emit(ctx, 0x3a2, 0);
-	else if (dev_priv->chipset == 0x98 || dev_priv->chipset == 0xaa)
+	else if (device->chipset == 0x98 || device->chipset == 0xaa)
 		xf_emit(ctx, 0x39f, 0);
 	else
 		xf_emit(ctx, 0x3a3, 0);
@@ -3285,15 +3266,15 @@ nv50_graph_construct_xfer_mpc(struct nouveau_grctx *ctx)
 static void
 nv50_graph_construct_xfer2(struct nouveau_grctx *ctx)
 {
-	struct drm_nouveau_private *dev_priv = ctx->dev->dev_private;
+	struct nouveau_device *device = ctx->device;
 	int i;
-	uint32_t offset;
-	uint32_t units = nv_rd32 (ctx->dev, 0x1540);
+	u32 offset;
+	u32 units = nv_rd32 (ctx->device, 0x1540);
 	int size = 0;
 
 	offset = (ctx->ctxvals_pos+0x3f)&~0x3f;
 
-	if (dev_priv->chipset < 0xa0) {
+	if (device->chipset < 0xa0) {
 		for (i = 0; i < 8; i++) {
 			ctx->ctxvals_pos = offset + i;
 			/* that little bugger belongs to csched. No idea
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
new file mode 100644
index 00000000000..0b7951a8594
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
@@ -0,0 +1,3039 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+void
+nv_icmd(struct nvc0_graph_priv *priv, u32 icmd, u32 data)
+{
+	nv_wr32(priv, 0x400204, data);
+	nv_wr32(priv, 0x400200, icmd);
+	while (nv_rd32(priv, 0x400700) & 2) {}
+}
+
+int
+nvc0_grctx_init(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+{
+	struct nouveau_bar *bar = nouveau_bar(priv);
+	struct nouveau_object *parent = nv_object(priv);
+	struct nouveau_gpuobj *chan;
+	u32 size = (0x80000 + priv->size + 4095) & ~4095;
+	int ret, i;
+
+	/* allocate memory to for a "channel", which we'll use to generate
+	 * the default context values
+	 */
+	ret = nouveau_gpuobj_new(parent, NULL, size, 0x1000,
+				 NVOBJ_FLAG_ZERO_ALLOC, &info->chan);
+	chan = info->chan;
+	if (ret) {
+		nv_error(priv, "failed to allocate channel memory, %d\n", ret);
+		return ret;
+	}
+
+	/* PGD pointer */
+	nv_wo32(chan, 0x0200, lower_32_bits(chan->addr + 0x1000));
+	nv_wo32(chan, 0x0204, upper_32_bits(chan->addr + 0x1000));
+	nv_wo32(chan, 0x0208, 0xffffffff);
+	nv_wo32(chan, 0x020c, 0x000000ff);
+
+	/* PGT[0] pointer */
+	nv_wo32(chan, 0x1000, 0x00000000);
+	nv_wo32(chan, 0x1004, 0x00000001 | (chan->addr + 0x2000) >> 8);
+
+	/* identity-map the whole "channel" into its own vm */
+	for (i = 0; i < size / 4096; i++) {
+		u64 addr = ((chan->addr + (i * 4096)) >> 8) | 1;
+		nv_wo32(chan, 0x2000 + (i * 8), lower_32_bits(addr));
+		nv_wo32(chan, 0x2004 + (i * 8), upper_32_bits(addr));
+	}
+
+	/* context pointer (virt) */
+	nv_wo32(chan, 0x0210, 0x00080004);
+	nv_wo32(chan, 0x0214, 0x00000000);
+
+	bar->flush(bar);
+
+	nv_wr32(priv, 0x100cb8, (chan->addr + 0x1000) >> 8);
+	nv_wr32(priv, 0x100cbc, 0x80000001);
+	nv_wait(priv, 0x100c80, 0x00008000, 0x00008000);
+
+	/* setup default state for mmio list construction */
+	info->data = priv->mmio_data;
+	info->mmio = priv->mmio_list;
+	info->addr = 0x2000 + (i * 8);
+	info->priv = priv;
+	info->buffer_nr = 0;
+
+	if (priv->firmware) {
+		nv_wr32(priv, 0x409840, 0x00000030);
+		nv_wr32(priv, 0x409500, 0x80000000 | chan->addr >> 12);
+		nv_wr32(priv, 0x409504, 0x00000003);
+		if (!nv_wait(priv, 0x409800, 0x00000010, 0x00000010))
+			nv_error(priv, "load_ctx timeout\n");
+
+		nv_wo32(chan, 0x8001c, 1);
+		nv_wo32(chan, 0x80020, 0);
+		nv_wo32(chan, 0x80028, 0);
+		nv_wo32(chan, 0x8002c, 0);
+		bar->flush(bar);
+		return 0;
+	}
+
+	/* HUB_FUC(SET_CHAN) */
+	nv_wr32(priv, 0x409840, 0x80000000);
+	nv_wr32(priv, 0x409500, 0x80000000 | chan->addr >> 12);
+	nv_wr32(priv, 0x409504, 0x00000001);
+	if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
+		nv_error(priv, "HUB_SET_CHAN timeout\n");
+		nvc0_graph_ctxctl_debug(priv);
+		nouveau_gpuobj_ref(NULL, &info->chan);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+void
+nvc0_grctx_data(struct nvc0_grctx *info, u32 size, u32 align, u32 access)
+{
+	info->buffer[info->buffer_nr]  = info->addr;
+	info->buffer[info->buffer_nr] +=  (align - 1);
+	info->buffer[info->buffer_nr] &= ~(align - 1);
+	info->addr = info->buffer[info->buffer_nr++] + size;
+
+	info->data->size = size;
+	info->data->align = align;
+	info->data->access = access;
+	info->data++;
+}
+
+void
+nvc0_grctx_mmio(struct nvc0_grctx *info, u32 addr, u32 data, u32 shift, u32 buf)
+{
+	struct nvc0_graph_priv *priv = info->priv;
+
+	info->mmio->addr = addr;
+	info->mmio->data = data;
+	info->mmio->shift = shift;
+	info->mmio->buffer = buf;
+	info->mmio++;
+
+	if (shift)
+		data |= info->buffer[buf] >> shift;
+	nv_wr32(priv, addr, data);
+}
+
+int
+nvc0_grctx_fini(struct nvc0_grctx *info)
+{
+	struct nvc0_graph_priv *priv = info->priv;
+	int i;
+
+	/* trigger a context unload by unsetting the "next channel valid" bit
+	 * and faking a context switch interrupt
+	 */
+	nv_mask(priv, 0x409b04, 0x80000000, 0x00000000);
+	nv_wr32(priv, 0x409000, 0x00000100);
+	if (!nv_wait(priv, 0x409b00, 0x80000000, 0x00000000)) {
+		nv_error(priv, "grctx template channel unload timeout\n");
+		return -EBUSY;
+	}
+
+	priv->data = kmalloc(priv->size, GFP_KERNEL);
+	if (priv->data) {
+		for (i = 0; i < priv->size; i += 4)
+			priv->data[i / 4] = nv_ro32(info->chan, 0x80000 + i);
+	}
+
+	nouveau_gpuobj_ref(NULL, &info->chan);
+	return priv->data ? 0 : -ENOMEM;
+}
+
+static void
+nvc0_grctx_generate_9097(struct nvc0_graph_priv *priv)
+{
+	u32 fermi = nvc0_graph_class(priv);
+	u32 mthd;
+
+	nv_mthd(priv, 0x9097, 0x0800, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0840, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0880, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0900, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0940, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0980, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0804, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0844, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0884, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0904, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0944, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0984, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0808, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x0848, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x0888, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x08c8, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x0908, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x0948, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x0988, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x09c8, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x080c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x084c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x088c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x08cc, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x090c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x094c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x098c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x09cc, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x0810, 0x000000cf);
+	nv_mthd(priv, 0x9097, 0x0850, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0890, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0910, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0950, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0990, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0814, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0854, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0894, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x08d4, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0914, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0954, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0994, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x09d4, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0818, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0858, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0898, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x08d8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0918, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0958, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0998, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x09d8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x081c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x085c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x089c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x091c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x095c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x099c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0820, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0860, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x08e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0920, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0960, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x09e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2700, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2720, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2740, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2760, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2780, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2704, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2724, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2744, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2764, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2784, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2708, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2728, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2748, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2768, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2788, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x270c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x272c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x274c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x276c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x278c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x27ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2710, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x2730, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x2750, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x2770, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x2790, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x27b0, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x27d0, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x27f0, 0x00014000);
+	nv_mthd(priv, 0x9097, 0x2714, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x2734, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x2754, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x2774, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x2794, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x27b4, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x27d4, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x27f4, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x1c00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ca0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ce0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cf0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ca4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cb4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cd4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ce4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cf4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c18, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c38, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c78, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c98, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ca8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cb8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cd8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ce8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cf8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c1c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c2c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c3c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c5c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c6c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c7c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1c9c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cbc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ccc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cdc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1cfc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1da0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1db0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1de0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1df0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1da4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1db4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dd4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1de4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1df4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d18, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d38, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d78, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d98, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1da8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1db8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dd8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1de8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1df8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d1c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d2c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d3c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d5c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d6c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d7c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1d9c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dbc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dcc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ddc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1dfc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f18, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f38, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f78, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f1c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f2c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f3c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f5c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f6c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f7c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f98, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fa0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fa8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fb8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fd8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fe0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fe8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ff0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ff8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1f9c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fa4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fb4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fbc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fcc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fd4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fdc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fe4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1fec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ff4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1ffc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2200, 0x00000022);
+	nv_mthd(priv, 0x9097, 0x2210, 0x00000022);
+	nv_mthd(priv, 0x9097, 0x2220, 0x00000022);
+	nv_mthd(priv, 0x9097, 0x2230, 0x00000022);
+	nv_mthd(priv, 0x9097, 0x2240, 0x00000022);
+	nv_mthd(priv, 0x9097, 0x2000, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2040, 0x00000011);
+	nv_mthd(priv, 0x9097, 0x2080, 0x00000020);
+	nv_mthd(priv, 0x9097, 0x20c0, 0x00000030);
+	nv_mthd(priv, 0x9097, 0x2100, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x2140, 0x00000051);
+	nv_mthd(priv, 0x9097, 0x200c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x204c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x208c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x20cc, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x210c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x214c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x2010, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2050, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2090, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x20d0, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x2110, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x2150, 0x00000004);
+	nv_mthd(priv, 0x9097, 0x0380, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0384, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0388, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x038c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x03ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0700, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0710, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0720, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0730, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0704, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0714, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0724, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0734, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0708, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0718, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0728, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0738, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2800, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2804, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2808, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x280c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2810, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2814, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2818, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x281c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2820, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2824, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2828, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x282c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2830, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2834, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2838, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x283c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2840, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2844, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2848, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x284c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2850, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2854, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2858, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x285c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2860, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2864, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2868, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x286c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2870, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2874, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2878, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x287c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2880, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2884, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2888, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x288c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2890, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2894, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2898, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x289c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28b0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28b4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28b8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28d4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28d8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28f0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28f4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28f8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x28fc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2900, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2904, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2908, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x290c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2910, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2914, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2918, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x291c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2920, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2924, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2928, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x292c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2930, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2934, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2938, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x293c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2940, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2944, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2948, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x294c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2950, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2954, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2958, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x295c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2960, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2964, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2968, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x296c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2970, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2974, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2978, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x297c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2980, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2984, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2988, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x298c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2990, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2994, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2998, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x299c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29b0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29b4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29b8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29d4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29d8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29f0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29f4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29f8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x29fc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0aa0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ac0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ae0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ba0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0be0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0aa4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ac4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ae4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ba4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0be4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0aa8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ac8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ae8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ba8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0be8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a2c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a6c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0aac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0acc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0aec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b2c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b6c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bcc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ab0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ad0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0af0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bf0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0a94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ab4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ad4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0af4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0b94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bb4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bd4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0bf4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ca0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ce0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cf0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c24, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c34, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c64, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c94, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ca4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cb4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cd4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ce4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cf4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c18, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c28, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c38, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c68, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c78, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c98, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ca8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cb8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cd8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ce8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0cf8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0c0c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c1c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c2c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c3c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c4c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c5c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c6c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c7c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c8c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0c9c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0cac, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0cbc, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0ccc, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0cdc, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0cec, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0cfc, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0d00, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d08, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d10, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d18, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d20, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d28, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d30, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d38, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d04, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d0c, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d14, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d1c, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d24, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d2c, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d34, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d3c, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e00, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e20, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e30, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e60, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e70, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ea0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0eb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ec0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ed0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ee0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ef0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0e04, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e14, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e24, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e34, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e44, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e54, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e64, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e74, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e84, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e94, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ea4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0eb4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ec4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ed4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ee4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ef4, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e08, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e18, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e28, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e38, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e48, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e58, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e68, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e78, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e88, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0e98, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ea8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0eb8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ec8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ed8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ee8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0ef8, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d40, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d48, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d50, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d44, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d4c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d5c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1e00, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e20, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e40, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e60, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e80, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ea0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ec0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ee0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e04, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e24, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e44, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e64, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e84, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ea4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ec4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ee4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e08, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e28, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e48, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e68, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e88, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1ea8, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1ec8, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1ee8, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e0c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e2c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e4c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e6c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e8c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1eac, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ecc, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1eec, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e10, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e30, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e50, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e70, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e90, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1eb0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ed0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ef0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e14, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e34, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e54, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e74, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e94, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1eb4, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1ed4, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1ef4, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1e18, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e38, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e58, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e78, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1e98, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1eb8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ed8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1ef8, 0x00000001);
+	if (fermi == 0x9097) {
+		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+			nv_mthd(priv, 0x9097, mthd, 0x00000000);
+	}
+	nv_mthd(priv, 0x9097, 0x030c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1944, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1514, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d68, 0x0000ffff);
+	nv_mthd(priv, 0x9097, 0x121c, 0x0fac6881);
+	nv_mthd(priv, 0x9097, 0x0fac, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1538, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0fe0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fe4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fe8, 0x00000014);
+	nv_mthd(priv, 0x9097, 0x0fec, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x0ff0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x179c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1228, 0x00000400);
+	nv_mthd(priv, 0x9097, 0x122c, 0x00000300);
+	nv_mthd(priv, 0x9097, 0x1230, 0x00010001);
+	nv_mthd(priv, 0x9097, 0x07f8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15b4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x15cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1534, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fb0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x153c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x16b4, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x0fbc, 0x0000ffff);
+	nv_mthd(priv, 0x9097, 0x0fc0, 0x0000ffff);
+	nv_mthd(priv, 0x9097, 0x0fc4, 0x0000ffff);
+	nv_mthd(priv, 0x9097, 0x0fc8, 0x0000ffff);
+	nv_mthd(priv, 0x9097, 0x0df8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0dfc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1948, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1970, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x161c, 0x000009f0);
+	nv_mthd(priv, 0x9097, 0x0dcc, 0x00000010);
+	nv_mthd(priv, 0x9097, 0x163c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1160, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1164, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1168, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x116c, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1170, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1174, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1178, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x117c, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1180, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1184, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1188, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x118c, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1190, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1194, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1198, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x119c, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11a0, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11a4, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11a8, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11ac, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11b0, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11b4, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11b8, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11bc, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11c0, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11c4, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11c8, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11cc, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11d0, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11d4, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11d8, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x11dc, 0x25e00040);
+	nv_mthd(priv, 0x9097, 0x1880, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1884, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1888, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x188c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1890, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1894, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1898, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x189c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18b0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18b4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18b8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18d0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18d4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18d8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18e0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18f0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18f4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18f8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x18fc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x17c8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x17cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x17d0, 0x000000ff);
+	nv_mthd(priv, 0x9097, 0x17d4, 0xffffffff);
+	nv_mthd(priv, 0x9097, 0x17d8, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x17dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15f4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15f8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1434, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1438, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d74, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0dec, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x13a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1318, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1644, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0748, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0de8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1648, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12a4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1120, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1124, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1128, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x112c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1118, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x164c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1658, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1910, 0x00000290);
+	nv_mthd(priv, 0x9097, 0x1518, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x165c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1520, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1604, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1570, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x13b0, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x13b4, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x020c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1670, 0x30201000);
+	nv_mthd(priv, 0x9097, 0x1674, 0x70605040);
+	nv_mthd(priv, 0x9097, 0x1678, 0xb8a89888);
+	nv_mthd(priv, 0x9097, 0x167c, 0xf8e8d8c8);
+	nv_mthd(priv, 0x9097, 0x166c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1680, 0x00ffff00);
+	nv_mthd(priv, 0x9097, 0x12d0, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x12d4, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1684, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1688, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0dac, 0x00001b02);
+	nv_mthd(priv, 0x9097, 0x0db0, 0x00001b02);
+	nv_mthd(priv, 0x9097, 0x0db4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x168c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x15bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x156c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x187c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1110, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0dc0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0dc4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0dc8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1234, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1690, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12ac, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x02c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0790, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0794, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0798, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x079c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x07a0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x077c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1000, 0x00000010);
+	nv_mthd(priv, 0x9097, 0x10fc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1290, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0218, 0x00000010);
+	nv_mthd(priv, 0x9097, 0x12d8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12dc, 0x00000010);
+	nv_mthd(priv, 0x9097, 0x0d94, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x155c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1560, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1564, 0x00001fff);
+	nv_mthd(priv, 0x9097, 0x1574, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1578, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x157c, 0x003fffff);
+	nv_mthd(priv, 0x9097, 0x1354, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1664, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1610, 0x00000012);
+	nv_mthd(priv, 0x9097, 0x1608, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x160c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x162c, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x0210, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0320, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0324, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0328, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x032c, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0330, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0334, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0338, 0x3f800000);
+	nv_mthd(priv, 0x9097, 0x0750, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0760, 0x39291909);
+	nv_mthd(priv, 0x9097, 0x0764, 0x79695949);
+	nv_mthd(priv, 0x9097, 0x0768, 0xb9a99989);
+	nv_mthd(priv, 0x9097, 0x076c, 0xf9e9d9c9);
+	nv_mthd(priv, 0x9097, 0x0770, 0x30201000);
+	nv_mthd(priv, 0x9097, 0x0774, 0x70605040);
+	nv_mthd(priv, 0x9097, 0x0778, 0x00009080);
+	nv_mthd(priv, 0x9097, 0x0780, 0x39291909);
+	nv_mthd(priv, 0x9097, 0x0784, 0x79695949);
+	nv_mthd(priv, 0x9097, 0x0788, 0xb9a99989);
+	nv_mthd(priv, 0x9097, 0x078c, 0xf9e9d9c9);
+	nv_mthd(priv, 0x9097, 0x07d0, 0x30201000);
+	nv_mthd(priv, 0x9097, 0x07d4, 0x70605040);
+	nv_mthd(priv, 0x9097, 0x07d8, 0x00009080);
+	nv_mthd(priv, 0x9097, 0x037c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0740, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0744, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x2600, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1918, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x191c, 0x00000900);
+	nv_mthd(priv, 0x9097, 0x1920, 0x00000405);
+	nv_mthd(priv, 0x9097, 0x1308, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1924, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x13ac, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x192c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x193c, 0x00002c1c);
+	nv_mthd(priv, 0x9097, 0x0d7c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x02c0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1510, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1940, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ff4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0ff8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x194c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1950, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1968, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1590, 0x0000003f);
+	nv_mthd(priv, 0x9097, 0x07e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x07ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x07f0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x07f4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x196c, 0x00000011);
+	nv_mthd(priv, 0x9097, 0x197c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fcc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fd0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x02d8, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x1980, 0x00000080);
+	nv_mthd(priv, 0x9097, 0x1504, 0x00000080);
+	nv_mthd(priv, 0x9097, 0x1984, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0300, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x13a8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12ec, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1310, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1314, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1380, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1384, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1388, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x138c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1390, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1394, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x139c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1398, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1594, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1598, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x159c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x15a0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x15a4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x0f54, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f58, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f5c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x19bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f9c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0fa0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12cc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x12e8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x130c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1360, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1364, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1368, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x136c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1370, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1374, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1378, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x137c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x133c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1340, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1344, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1348, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x134c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1350, 0x00000002);
+	nv_mthd(priv, 0x9097, 0x1358, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x12e4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x131c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1320, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1324, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1328, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x19c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1140, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x19c4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x19c8, 0x00001500);
+	nv_mthd(priv, 0x9097, 0x135c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x19e0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19e4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19e8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19ec, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19f0, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19f4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19f8, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19fc, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x19cc, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x15b8, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a00, 0x00001111);
+	nv_mthd(priv, 0x9097, 0x1a04, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a08, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a0c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a10, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a14, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a18, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1a1c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d6c, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x0d70, 0xffff0000);
+	nv_mthd(priv, 0x9097, 0x10f8, 0x00001010);
+	nv_mthd(priv, 0x9097, 0x0d80, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d84, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d88, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d8c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0d90, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0da0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1508, 0x80000000);
+	nv_mthd(priv, 0x9097, 0x150c, 0x40000000);
+	nv_mthd(priv, 0x9097, 0x1668, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0318, 0x00000008);
+	nv_mthd(priv, 0x9097, 0x031c, 0x00000008);
+	nv_mthd(priv, 0x9097, 0x0d9c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x07dc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x074c, 0x00000055);
+	nv_mthd(priv, 0x9097, 0x1420, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x17bc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x17c0, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x17c4, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1008, 0x00000008);
+	nv_mthd(priv, 0x9097, 0x100c, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x1010, 0x0000012c);
+	nv_mthd(priv, 0x9097, 0x0d60, 0x00000040);
+	nv_mthd(priv, 0x9097, 0x075c, 0x00000003);
+	nv_mthd(priv, 0x9097, 0x1018, 0x00000020);
+	nv_mthd(priv, 0x9097, 0x101c, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1020, 0x00000020);
+	nv_mthd(priv, 0x9097, 0x1024, 0x00000001);
+	nv_mthd(priv, 0x9097, 0x1444, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x1448, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x144c, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0360, 0x20164010);
+	nv_mthd(priv, 0x9097, 0x0364, 0x00000020);
+	nv_mthd(priv, 0x9097, 0x0368, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0de4, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0204, 0x00000006);
+	nv_mthd(priv, 0x9097, 0x0208, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x02cc, 0x003fffff);
+	nv_mthd(priv, 0x9097, 0x02d0, 0x00000c48);
+	nv_mthd(priv, 0x9097, 0x1220, 0x00000005);
+	nv_mthd(priv, 0x9097, 0x0fdc, 0x00000000);
+	nv_mthd(priv, 0x9097, 0x0f98, 0x00300008);
+	nv_mthd(priv, 0x9097, 0x1284, 0x04000080);
+	nv_mthd(priv, 0x9097, 0x1450, 0x00300008);
+	nv_mthd(priv, 0x9097, 0x1454, 0x04000080);
+	nv_mthd(priv, 0x9097, 0x0214, 0x00000000);
+	/* in trace, right after 0x90c0, not here */
+	nv_mthd(priv, 0x9097, 0x3410, 0x80002006);
+}
+
+static void
+nvc0_grctx_generate_9197(struct nvc0_graph_priv *priv)
+{
+	u32 fermi = nvc0_graph_class(priv);
+	u32 mthd;
+
+	if (fermi == 0x9197) {
+		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+			nv_mthd(priv, 0x9197, mthd, 0x00000000);
+	}
+	nv_mthd(priv, 0x9197, 0x02e4, 0x0000b001);
+}
+
+static void
+nvc0_grctx_generate_9297(struct nvc0_graph_priv *priv)
+{
+	u32 fermi = nvc0_graph_class(priv);
+	u32 mthd;
+
+	if (fermi == 0x9297) {
+		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
+			nv_mthd(priv, 0x9297, mthd, 0x00000000);
+	}
+	nv_mthd(priv, 0x9297, 0x036c, 0x00000000);
+	nv_mthd(priv, 0x9297, 0x0370, 0x00000000);
+	nv_mthd(priv, 0x9297, 0x07a4, 0x00000000);
+	nv_mthd(priv, 0x9297, 0x07a8, 0x00000000);
+	nv_mthd(priv, 0x9297, 0x0374, 0x00000000);
+	nv_mthd(priv, 0x9297, 0x0378, 0x00000020);
+}
+
+static void
+nvc0_grctx_generate_902d(struct nvc0_graph_priv *priv)
+{
+	nv_mthd(priv, 0x902d, 0x0200, 0x000000cf);
+	nv_mthd(priv, 0x902d, 0x0204, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0208, 0x00000020);
+	nv_mthd(priv, 0x902d, 0x020c, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0210, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0214, 0x00000080);
+	nv_mthd(priv, 0x902d, 0x0218, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x021c, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x0220, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0224, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0230, 0x000000cf);
+	nv_mthd(priv, 0x902d, 0x0234, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0238, 0x00000020);
+	nv_mthd(priv, 0x902d, 0x023c, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0244, 0x00000080);
+	nv_mthd(priv, 0x902d, 0x0248, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x024c, 0x00000100);
+}
+
+static void
+nvc0_grctx_generate_9039(struct nvc0_graph_priv *priv)
+{
+	nv_mthd(priv, 0x9039, 0x030c, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x0310, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x0314, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x0320, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x0238, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x023c, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x0318, 0x00000000);
+	nv_mthd(priv, 0x9039, 0x031c, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_90c0(struct nvc0_graph_priv *priv)
+{
+	int i;
+
+	for (i = 0; nv_device(priv)->chipset == 0xd9 && i < 4; i++) {
+		nv_mthd(priv, 0x90c0, 0x2700 + (i * 0x40), 0x00000000);
+		nv_mthd(priv, 0x90c0, 0x2720 + (i * 0x40), 0x00000000);
+		nv_mthd(priv, 0x90c0, 0x2704 + (i * 0x40), 0x00000000);
+		nv_mthd(priv, 0x90c0, 0x2724 + (i * 0x40), 0x00000000);
+		nv_mthd(priv, 0x90c0, 0x2708 + (i * 0x40), 0x00000000);
+		nv_mthd(priv, 0x90c0, 0x2728 + (i * 0x40), 0x00000000);
+	}
+	nv_mthd(priv, 0x90c0, 0x270c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x272c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x274c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x276c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x278c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x27ac, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x27cc, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x27ec, 0x00000000);
+	for (i = 0; nv_device(priv)->chipset == 0xd9 && i < 4; i++) {
+		nv_mthd(priv, 0x90c0, 0x2710 + (i * 0x40), 0x00014000);
+		nv_mthd(priv, 0x90c0, 0x2730 + (i * 0x40), 0x00014000);
+		nv_mthd(priv, 0x90c0, 0x2714 + (i * 0x40), 0x00000040);
+		nv_mthd(priv, 0x90c0, 0x2734 + (i * 0x40), 0x00000040);
+	}
+	nv_mthd(priv, 0x90c0, 0x030c, 0x00000001);
+	nv_mthd(priv, 0x90c0, 0x1944, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0758, 0x00000100);
+	nv_mthd(priv, 0x90c0, 0x02c4, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0790, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0794, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0798, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x079c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x07a0, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x077c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0204, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0208, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x020c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0214, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x024c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x0d94, 0x00000001);
+	nv_mthd(priv, 0x90c0, 0x1608, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x160c, 0x00000000);
+	nv_mthd(priv, 0x90c0, 0x1664, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_dispatch(struct nvc0_graph_priv *priv)
+{
+	int i;
+
+	nv_wr32(priv, 0x404004, 0x00000000);
+	nv_wr32(priv, 0x404008, 0x00000000);
+	nv_wr32(priv, 0x40400c, 0x00000000);
+	nv_wr32(priv, 0x404010, 0x00000000);
+	nv_wr32(priv, 0x404014, 0x00000000);
+	nv_wr32(priv, 0x404018, 0x00000000);
+	nv_wr32(priv, 0x40401c, 0x00000000);
+	nv_wr32(priv, 0x404020, 0x00000000);
+	nv_wr32(priv, 0x404024, 0x00000000);
+	nv_wr32(priv, 0x404028, 0x00000000);
+	nv_wr32(priv, 0x40402c, 0x00000000);
+	nv_wr32(priv, 0x404044, 0x00000000);
+	nv_wr32(priv, 0x404094, 0x00000000);
+	nv_wr32(priv, 0x404098, 0x00000000);
+	nv_wr32(priv, 0x40409c, 0x00000000);
+	nv_wr32(priv, 0x4040a0, 0x00000000);
+	nv_wr32(priv, 0x4040a4, 0x00000000);
+	nv_wr32(priv, 0x4040a8, 0x00000000);
+	nv_wr32(priv, 0x4040ac, 0x00000000);
+	nv_wr32(priv, 0x4040b0, 0x00000000);
+	nv_wr32(priv, 0x4040b4, 0x00000000);
+	nv_wr32(priv, 0x4040b8, 0x00000000);
+	nv_wr32(priv, 0x4040bc, 0x00000000);
+	nv_wr32(priv, 0x4040c0, 0x00000000);
+	nv_wr32(priv, 0x4040c4, 0x00000000);
+	nv_wr32(priv, 0x4040c8, 0xf0000087);
+	nv_wr32(priv, 0x4040d4, 0x00000000);
+	nv_wr32(priv, 0x4040d8, 0x00000000);
+	nv_wr32(priv, 0x4040dc, 0x00000000);
+	nv_wr32(priv, 0x4040e0, 0x00000000);
+	nv_wr32(priv, 0x4040e4, 0x00000000);
+	nv_wr32(priv, 0x4040e8, 0x00001000);
+	nv_wr32(priv, 0x4040f8, 0x00000000);
+	nv_wr32(priv, 0x404130, 0x00000000);
+	nv_wr32(priv, 0x404134, 0x00000000);
+	nv_wr32(priv, 0x404138, 0x20000040);
+	nv_wr32(priv, 0x404150, 0x0000002e);
+	nv_wr32(priv, 0x404154, 0x00000400);
+	nv_wr32(priv, 0x404158, 0x00000200);
+	nv_wr32(priv, 0x404164, 0x00000055);
+	nv_wr32(priv, 0x404168, 0x00000000);
+	nv_wr32(priv, 0x404174, 0x00000000);
+	nv_wr32(priv, 0x404178, 0x00000000);
+	nv_wr32(priv, 0x40417c, 0x00000000);
+	for (i = 0; i < 8; i++)
+		nv_wr32(priv, 0x404200 + (i * 4), 0x00000000); /* subc */
+}
+
+static void
+nvc0_grctx_generate_macro(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404404, 0x00000000);
+	nv_wr32(priv, 0x404408, 0x00000000);
+	nv_wr32(priv, 0x40440c, 0x00000000);
+	nv_wr32(priv, 0x404410, 0x00000000);
+	nv_wr32(priv, 0x404414, 0x00000000);
+	nv_wr32(priv, 0x404418, 0x00000000);
+	nv_wr32(priv, 0x40441c, 0x00000000);
+	nv_wr32(priv, 0x404420, 0x00000000);
+	nv_wr32(priv, 0x404424, 0x00000000);
+	nv_wr32(priv, 0x404428, 0x00000000);
+	nv_wr32(priv, 0x40442c, 0x00000000);
+	nv_wr32(priv, 0x404430, 0x00000000);
+	nv_wr32(priv, 0x404434, 0x00000000);
+	nv_wr32(priv, 0x404438, 0x00000000);
+	nv_wr32(priv, 0x404460, 0x00000000);
+	nv_wr32(priv, 0x404464, 0x00000000);
+	nv_wr32(priv, 0x404468, 0x00ffffff);
+	nv_wr32(priv, 0x40446c, 0x00000000);
+	nv_wr32(priv, 0x404480, 0x00000001);
+	nv_wr32(priv, 0x404498, 0x00000001);
+}
+
+static void
+nvc0_grctx_generate_m2mf(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404604, 0x00000015);
+	nv_wr32(priv, 0x404608, 0x00000000);
+	nv_wr32(priv, 0x40460c, 0x00002e00);
+	nv_wr32(priv, 0x404610, 0x00000100);
+	nv_wr32(priv, 0x404618, 0x00000000);
+	nv_wr32(priv, 0x40461c, 0x00000000);
+	nv_wr32(priv, 0x404620, 0x00000000);
+	nv_wr32(priv, 0x404624, 0x00000000);
+	nv_wr32(priv, 0x404628, 0x00000000);
+	nv_wr32(priv, 0x40462c, 0x00000000);
+	nv_wr32(priv, 0x404630, 0x00000000);
+	nv_wr32(priv, 0x404634, 0x00000000);
+	nv_wr32(priv, 0x404638, 0x00000004);
+	nv_wr32(priv, 0x40463c, 0x00000000);
+	nv_wr32(priv, 0x404640, 0x00000000);
+	nv_wr32(priv, 0x404644, 0x00000000);
+	nv_wr32(priv, 0x404648, 0x00000000);
+	nv_wr32(priv, 0x40464c, 0x00000000);
+	nv_wr32(priv, 0x404650, 0x00000000);
+	nv_wr32(priv, 0x404654, 0x00000000);
+	nv_wr32(priv, 0x404658, 0x00000000);
+	nv_wr32(priv, 0x40465c, 0x007f0100);
+	nv_wr32(priv, 0x404660, 0x00000000);
+	nv_wr32(priv, 0x404664, 0x00000000);
+	nv_wr32(priv, 0x404668, 0x00000000);
+	nv_wr32(priv, 0x40466c, 0x00000000);
+	nv_wr32(priv, 0x404670, 0x00000000);
+	nv_wr32(priv, 0x404674, 0x00000000);
+	nv_wr32(priv, 0x404678, 0x00000000);
+	nv_wr32(priv, 0x40467c, 0x00000002);
+	nv_wr32(priv, 0x404680, 0x00000000);
+	nv_wr32(priv, 0x404684, 0x00000000);
+	nv_wr32(priv, 0x404688, 0x00000000);
+	nv_wr32(priv, 0x40468c, 0x00000000);
+	nv_wr32(priv, 0x404690, 0x00000000);
+	nv_wr32(priv, 0x404694, 0x00000000);
+	nv_wr32(priv, 0x404698, 0x00000000);
+	nv_wr32(priv, 0x40469c, 0x00000000);
+	nv_wr32(priv, 0x4046a0, 0x007f0080);
+	nv_wr32(priv, 0x4046a4, 0x00000000);
+	nv_wr32(priv, 0x4046a8, 0x00000000);
+	nv_wr32(priv, 0x4046ac, 0x00000000);
+	nv_wr32(priv, 0x4046b0, 0x00000000);
+	nv_wr32(priv, 0x4046b4, 0x00000000);
+	nv_wr32(priv, 0x4046b8, 0x00000000);
+	nv_wr32(priv, 0x4046bc, 0x00000000);
+	nv_wr32(priv, 0x4046c0, 0x00000000);
+	nv_wr32(priv, 0x4046c4, 0x00000000);
+	nv_wr32(priv, 0x4046c8, 0x00000000);
+	nv_wr32(priv, 0x4046cc, 0x00000000);
+	nv_wr32(priv, 0x4046d0, 0x00000000);
+	nv_wr32(priv, 0x4046d4, 0x00000000);
+	nv_wr32(priv, 0x4046d8, 0x00000000);
+	nv_wr32(priv, 0x4046dc, 0x00000000);
+	nv_wr32(priv, 0x4046e0, 0x00000000);
+	nv_wr32(priv, 0x4046e4, 0x00000000);
+	nv_wr32(priv, 0x4046e8, 0x00000000);
+	nv_wr32(priv, 0x4046f0, 0x00000000);
+	nv_wr32(priv, 0x4046f4, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_unk47xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404700, 0x00000000);
+	nv_wr32(priv, 0x404704, 0x00000000);
+	nv_wr32(priv, 0x404708, 0x00000000);
+	nv_wr32(priv, 0x40470c, 0x00000000);
+	nv_wr32(priv, 0x404710, 0x00000000);
+	nv_wr32(priv, 0x404714, 0x00000000);
+	nv_wr32(priv, 0x404718, 0x00000000);
+	nv_wr32(priv, 0x40471c, 0x00000000);
+	nv_wr32(priv, 0x404720, 0x00000000);
+	nv_wr32(priv, 0x404724, 0x00000000);
+	nv_wr32(priv, 0x404728, 0x00000000);
+	nv_wr32(priv, 0x40472c, 0x00000000);
+	nv_wr32(priv, 0x404730, 0x00000000);
+	nv_wr32(priv, 0x404734, 0x00000100);
+	nv_wr32(priv, 0x404738, 0x00000000);
+	nv_wr32(priv, 0x40473c, 0x00000000);
+	nv_wr32(priv, 0x404740, 0x00000000);
+	nv_wr32(priv, 0x404744, 0x00000000);
+	nv_wr32(priv, 0x404748, 0x00000000);
+	nv_wr32(priv, 0x40474c, 0x00000000);
+	nv_wr32(priv, 0x404750, 0x00000000);
+	nv_wr32(priv, 0x404754, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_shaders(struct nvc0_graph_priv *priv)
+{
+
+	if (nv_device(priv)->chipset == 0xd9) {
+		nv_wr32(priv, 0x405800, 0x0f8000bf);
+		nv_wr32(priv, 0x405830, 0x02180218);
+		nv_wr32(priv, 0x405834, 0x08000000);
+	} else
+	if (nv_device(priv)->chipset == 0xc1) {
+		nv_wr32(priv, 0x405800, 0x0f8000bf);
+		nv_wr32(priv, 0x405830, 0x02180218);
+		nv_wr32(priv, 0x405834, 0x00000000);
+	} else {
+		nv_wr32(priv, 0x405800, 0x078000bf);
+		nv_wr32(priv, 0x405830, 0x02180000);
+		nv_wr32(priv, 0x405834, 0x00000000);
+	}
+	nv_wr32(priv, 0x405838, 0x00000000);
+	nv_wr32(priv, 0x405854, 0x00000000);
+	nv_wr32(priv, 0x405870, 0x00000001);
+	nv_wr32(priv, 0x405874, 0x00000001);
+	nv_wr32(priv, 0x405878, 0x00000001);
+	nv_wr32(priv, 0x40587c, 0x00000001);
+	nv_wr32(priv, 0x405a00, 0x00000000);
+	nv_wr32(priv, 0x405a04, 0x00000000);
+	nv_wr32(priv, 0x405a18, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_unk60xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x406020, 0x000103c1);
+	nv_wr32(priv, 0x406028, 0x00000001);
+	nv_wr32(priv, 0x40602c, 0x00000001);
+	nv_wr32(priv, 0x406030, 0x00000001);
+	nv_wr32(priv, 0x406034, 0x00000001);
+}
+
+static void
+nvc0_grctx_generate_unk64xx(struct nvc0_graph_priv *priv)
+{
+
+	nv_wr32(priv, 0x4064a8, 0x00000000);
+	nv_wr32(priv, 0x4064ac, 0x00003fff);
+	nv_wr32(priv, 0x4064b4, 0x00000000);
+	nv_wr32(priv, 0x4064b8, 0x00000000);
+	if (nv_device(priv)->chipset == 0xd9)
+		nv_wr32(priv, 0x4064bc, 0x00000000);
+	if (nv_device(priv)->chipset == 0xc1 ||
+	    nv_device(priv)->chipset == 0xd9) {
+		nv_wr32(priv, 0x4064c0, 0x80140078);
+		nv_wr32(priv, 0x4064c4, 0x0086ffff);
+	}
+}
+
+static void
+nvc0_grctx_generate_tpbus(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x407804, 0x00000023);
+	nv_wr32(priv, 0x40780c, 0x0a418820);
+	nv_wr32(priv, 0x407810, 0x062080e6);
+	nv_wr32(priv, 0x407814, 0x020398a4);
+	nv_wr32(priv, 0x407818, 0x0e629062);
+	nv_wr32(priv, 0x40781c, 0x0a418820);
+	nv_wr32(priv, 0x407820, 0x000000e6);
+	nv_wr32(priv, 0x4078bc, 0x00000103);
+}
+
+static void
+nvc0_grctx_generate_ccache(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x408000, 0x00000000);
+	nv_wr32(priv, 0x408004, 0x00000000);
+	nv_wr32(priv, 0x408008, 0x00000018);
+	nv_wr32(priv, 0x40800c, 0x00000000);
+	nv_wr32(priv, 0x408010, 0x00000000);
+	nv_wr32(priv, 0x408014, 0x00000069);
+	nv_wr32(priv, 0x408018, 0xe100e100);
+	nv_wr32(priv, 0x408064, 0x00000000);
+}
+
+static void
+nvc0_grctx_generate_rop(struct nvc0_graph_priv *priv)
+{
+	int chipset = nv_device(priv)->chipset;
+
+	/* ROPC_BROADCAST */
+	nv_wr32(priv, 0x408800, 0x02802a3c);
+	nv_wr32(priv, 0x408804, 0x00000040);
+	if (chipset == 0xd9) {
+		nv_wr32(priv, 0x408808, 0x1043e005);
+		nv_wr32(priv, 0x408900, 0x3080b801);
+		nv_wr32(priv, 0x408904, 0x1043e005);
+		nv_wr32(priv, 0x408908, 0x00c8102f);
+	} else
+	if (chipset == 0xc1) {
+		nv_wr32(priv, 0x408808, 0x1003e005);
+		nv_wr32(priv, 0x408900, 0x3080b801);
+		nv_wr32(priv, 0x408904, 0x62000001);
+		nv_wr32(priv, 0x408908, 0x00c80929);
+	} else {
+		nv_wr32(priv, 0x408808, 0x0003e00d);
+		nv_wr32(priv, 0x408900, 0x3080b801);
+		nv_wr32(priv, 0x408904, 0x02000001);
+		nv_wr32(priv, 0x408908, 0x00c80929);
+	}
+	nv_wr32(priv, 0x40890c, 0x00000000);
+	nv_wr32(priv, 0x408980, 0x0000011d);
+}
+
+static void
+nvc0_grctx_generate_gpc(struct nvc0_graph_priv *priv)
+{
+	int chipset = nv_device(priv)->chipset;
+	int i;
+
+	/* GPC_BROADCAST */
+	nv_wr32(priv, 0x418380, 0x00000016);
+	nv_wr32(priv, 0x418400, 0x38004e00);
+	nv_wr32(priv, 0x418404, 0x71e0ffff);
+	nv_wr32(priv, 0x418408, 0x00000000);
+	nv_wr32(priv, 0x41840c, 0x00001008);
+	nv_wr32(priv, 0x418410, 0x0fff0fff);
+	nv_wr32(priv, 0x418414, chipset != 0xd9 ? 0x00200fff : 0x02200fff);
+	nv_wr32(priv, 0x418450, 0x00000000);
+	nv_wr32(priv, 0x418454, 0x00000000);
+	nv_wr32(priv, 0x418458, 0x00000000);
+	nv_wr32(priv, 0x41845c, 0x00000000);
+	nv_wr32(priv, 0x418460, 0x00000000);
+	nv_wr32(priv, 0x418464, 0x00000000);
+	nv_wr32(priv, 0x418468, 0x00000001);
+	nv_wr32(priv, 0x41846c, 0x00000000);
+	nv_wr32(priv, 0x418470, 0x00000000);
+	nv_wr32(priv, 0x418600, 0x0000001f);
+	nv_wr32(priv, 0x418684, 0x0000000f);
+	nv_wr32(priv, 0x418700, 0x00000002);
+	nv_wr32(priv, 0x418704, 0x00000080);
+	nv_wr32(priv, 0x418708, 0x00000000);
+	nv_wr32(priv, 0x41870c, chipset != 0xd9 ? 0x07c80000 : 0x00000000);
+	nv_wr32(priv, 0x418710, 0x00000000);
+	nv_wr32(priv, 0x418800, chipset != 0xd9 ? 0x0006860a : 0x7006860a);
+	nv_wr32(priv, 0x418808, 0x00000000);
+	nv_wr32(priv, 0x41880c, 0x00000000);
+	nv_wr32(priv, 0x418810, 0x00000000);
+	nv_wr32(priv, 0x418828, 0x00008442);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x418830, 0x10000001);
+	else
+		nv_wr32(priv, 0x418830, 0x00000001);
+	nv_wr32(priv, 0x4188d8, 0x00000008);
+	nv_wr32(priv, 0x4188e0, 0x01000000);
+	nv_wr32(priv, 0x4188e8, 0x00000000);
+	nv_wr32(priv, 0x4188ec, 0x00000000);
+	nv_wr32(priv, 0x4188f0, 0x00000000);
+	nv_wr32(priv, 0x4188f4, 0x00000000);
+	nv_wr32(priv, 0x4188f8, 0x00000000);
+	if (chipset == 0xd9)
+		nv_wr32(priv, 0x4188fc, 0x20100008);
+	else if (chipset == 0xc1)
+		nv_wr32(priv, 0x4188fc, 0x00100018);
+	else
+		nv_wr32(priv, 0x4188fc, 0x00100000);
+	nv_wr32(priv, 0x41891c, 0x00ff00ff);
+	nv_wr32(priv, 0x418924, 0x00000000);
+	nv_wr32(priv, 0x418928, 0x00ffff00);
+	nv_wr32(priv, 0x41892c, 0x0000ff00);
+	for (i = 0; i < 8; i++) {
+		nv_wr32(priv, 0x418a00 + (i * 0x20), 0x00000000);
+		nv_wr32(priv, 0x418a04 + (i * 0x20), 0x00000000);
+		nv_wr32(priv, 0x418a08 + (i * 0x20), 0x00000000);
+		nv_wr32(priv, 0x418a0c + (i * 0x20), 0x00010000);
+		nv_wr32(priv, 0x418a10 + (i * 0x20), 0x00000000);
+		nv_wr32(priv, 0x418a14 + (i * 0x20), 0x00000000);
+		nv_wr32(priv, 0x418a18 + (i * 0x20), 0x00000000);
+	}
+	nv_wr32(priv, 0x418b00, chipset != 0xd9 ? 0x00000000 : 0x00000006);
+	nv_wr32(priv, 0x418b08, 0x0a418820);
+	nv_wr32(priv, 0x418b0c, 0x062080e6);
+	nv_wr32(priv, 0x418b10, 0x020398a4);
+	nv_wr32(priv, 0x418b14, 0x0e629062);
+	nv_wr32(priv, 0x418b18, 0x0a418820);
+	nv_wr32(priv, 0x418b1c, 0x000000e6);
+	nv_wr32(priv, 0x418bb8, 0x00000103);
+	nv_wr32(priv, 0x418c08, 0x00000001);
+	nv_wr32(priv, 0x418c10, 0x00000000);
+	nv_wr32(priv, 0x418c14, 0x00000000);
+	nv_wr32(priv, 0x418c18, 0x00000000);
+	nv_wr32(priv, 0x418c1c, 0x00000000);
+	nv_wr32(priv, 0x418c20, 0x00000000);
+	nv_wr32(priv, 0x418c24, 0x00000000);
+	nv_wr32(priv, 0x418c28, 0x00000000);
+	nv_wr32(priv, 0x418c2c, 0x00000000);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x418c6c, 0x00000001);
+	nv_wr32(priv, 0x418c80, 0x20200004);
+	nv_wr32(priv, 0x418c8c, 0x00000001);
+	nv_wr32(priv, 0x419000, 0x00000780);
+	nv_wr32(priv, 0x419004, 0x00000000);
+	nv_wr32(priv, 0x419008, 0x00000000);
+	nv_wr32(priv, 0x419014, 0x00000004);
+}
+
+static void
+nvc0_grctx_generate_tp(struct nvc0_graph_priv *priv)
+{
+	int chipset = nv_device(priv)->chipset;
+
+	/* GPC_BROADCAST.TP_BROADCAST */
+	nv_wr32(priv, 0x419818, 0x00000000);
+	nv_wr32(priv, 0x41983c, 0x00038bc7);
+	nv_wr32(priv, 0x419848, 0x00000000);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x419864, 0x00000129);
+	else
+		nv_wr32(priv, 0x419864, 0x0000012a);
+	nv_wr32(priv, 0x419888, 0x00000000);
+	nv_wr32(priv, 0x419a00, 0x000001f0);
+	nv_wr32(priv, 0x419a04, 0x00000001);
+	nv_wr32(priv, 0x419a08, 0x00000023);
+	nv_wr32(priv, 0x419a0c, 0x00020000);
+	nv_wr32(priv, 0x419a10, 0x00000000);
+	nv_wr32(priv, 0x419a14, 0x00000200);
+	nv_wr32(priv, 0x419a1c, 0x00000000);
+	nv_wr32(priv, 0x419a20, 0x00000800);
+	if (chipset == 0xd9)
+		nv_wr32(priv, 0x00419ac4, 0x0017f440);
+	else if (chipset != 0xc0 && chipset != 0xc8)
+		nv_wr32(priv, 0x00419ac4, 0x0007f440);
+	nv_wr32(priv, 0x419b00, 0x0a418820);
+	nv_wr32(priv, 0x419b04, 0x062080e6);
+	nv_wr32(priv, 0x419b08, 0x020398a4);
+	nv_wr32(priv, 0x419b0c, 0x0e629062);
+	nv_wr32(priv, 0x419b10, 0x0a418820);
+	nv_wr32(priv, 0x419b14, 0x000000e6);
+	nv_wr32(priv, 0x419bd0, 0x00900103);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x419be0, 0x00400001);
+	else
+		nv_wr32(priv, 0x419be0, 0x00000001);
+	nv_wr32(priv, 0x419be4, 0x00000000);
+	nv_wr32(priv, 0x419c00, chipset != 0xd9 ? 0x00000002 : 0x0000000a);
+	nv_wr32(priv, 0x419c04, 0x00000006);
+	nv_wr32(priv, 0x419c08, 0x00000002);
+	nv_wr32(priv, 0x419c20, 0x00000000);
+	if (nv_device(priv)->chipset == 0xd9) {
+		nv_wr32(priv, 0x419c24, 0x00084210);
+		nv_wr32(priv, 0x419c28, 0x3cf3cf3c);
+		nv_wr32(priv, 0x419cb0, 0x00020048);
+	} else
+	if (chipset == 0xce || chipset == 0xcf) {
+		nv_wr32(priv, 0x419cb0, 0x00020048);
+	} else {
+		nv_wr32(priv, 0x419cb0, 0x00060048);
+	}
+	nv_wr32(priv, 0x419ce8, 0x00000000);
+	nv_wr32(priv, 0x419cf4, 0x00000183);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x419d20, 0x12180000);
+	else
+		nv_wr32(priv, 0x419d20, 0x02180000);
+	nv_wr32(priv, 0x419d24, 0x00001fff);
+	if (chipset == 0xc1 || chipset == 0xd9)
+		nv_wr32(priv, 0x419d44, 0x02180218);
+	nv_wr32(priv, 0x419e04, 0x00000000);
+	nv_wr32(priv, 0x419e08, 0x00000000);
+	nv_wr32(priv, 0x419e0c, 0x00000000);
+	nv_wr32(priv, 0x419e10, 0x00000002);
+	nv_wr32(priv, 0x419e44, 0x001beff2);
+	nv_wr32(priv, 0x419e48, 0x00000000);
+	nv_wr32(priv, 0x419e4c, 0x0000000f);
+	nv_wr32(priv, 0x419e50, 0x00000000);
+	nv_wr32(priv, 0x419e54, 0x00000000);
+	nv_wr32(priv, 0x419e58, 0x00000000);
+	nv_wr32(priv, 0x419e5c, 0x00000000);
+	nv_wr32(priv, 0x419e60, 0x00000000);
+	nv_wr32(priv, 0x419e64, 0x00000000);
+	nv_wr32(priv, 0x419e68, 0x00000000);
+	nv_wr32(priv, 0x419e6c, 0x00000000);
+	nv_wr32(priv, 0x419e70, 0x00000000);
+	nv_wr32(priv, 0x419e74, 0x00000000);
+	nv_wr32(priv, 0x419e78, 0x00000000);
+	nv_wr32(priv, 0x419e7c, 0x00000000);
+	nv_wr32(priv, 0x419e80, 0x00000000);
+	nv_wr32(priv, 0x419e84, 0x00000000);
+	nv_wr32(priv, 0x419e88, 0x00000000);
+	nv_wr32(priv, 0x419e8c, 0x00000000);
+	nv_wr32(priv, 0x419e90, 0x00000000);
+	nv_wr32(priv, 0x419e98, 0x00000000);
+	if (chipset != 0xc0 && chipset != 0xc8)
+		nv_wr32(priv, 0x419ee0, 0x00011110);
+	nv_wr32(priv, 0x419f50, 0x00000000);
+	nv_wr32(priv, 0x419f54, 0x00000000);
+	if (chipset != 0xc0 && chipset != 0xc8)
+		nv_wr32(priv, 0x419f58, 0x00000000);
+}
+
+int
+nvc0_grctx_generate(struct nvc0_graph_priv *priv)
+{
+	struct nvc0_grctx info;
+	int ret, i, gpc, tpc, id;
+	u32 fermi = nvc0_graph_class(priv);
+	u32 r000260, tmp;
+
+	ret = nvc0_grctx_init(priv, &info);
+	if (ret)
+		return ret;
+
+	r000260 = nv_rd32(priv, 0x000260);
+	nv_wr32(priv, 0x000260, r000260 & ~1);
+	nv_wr32(priv, 0x400208, 0x00000000);
+
+	nvc0_grctx_generate_dispatch(priv);
+	nvc0_grctx_generate_macro(priv);
+	nvc0_grctx_generate_m2mf(priv);
+	nvc0_grctx_generate_unk47xx(priv);
+	nvc0_grctx_generate_shaders(priv);
+	nvc0_grctx_generate_unk60xx(priv);
+	nvc0_grctx_generate_unk64xx(priv);
+	nvc0_grctx_generate_tpbus(priv);
+	nvc0_grctx_generate_ccache(priv);
+	nvc0_grctx_generate_rop(priv);
+	nvc0_grctx_generate_gpc(priv);
+	nvc0_grctx_generate_tp(priv);
+
+	nv_wr32(priv, 0x404154, 0x00000000);
+
+	/* generate per-context mmio list data */
+	mmio_data(0x002000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+	mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+	mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
+	mmio_list(0x408004, 0x00000000,  8, 0);
+	mmio_list(0x408008, 0x80000018,  0, 0);
+	mmio_list(0x40800c, 0x00000000,  8, 1);
+	mmio_list(0x408010, 0x80000000,  0, 0);
+	mmio_list(0x418810, 0x80000000, 12, 2);
+	mmio_list(0x419848, 0x10000000, 12, 2);
+	mmio_list(0x419004, 0x00000000,  8, 1);
+	mmio_list(0x419008, 0x00000000,  0, 0);
+	mmio_list(0x418808, 0x00000000,  8, 0);
+	mmio_list(0x41880c, 0x80000018,  0, 0);
+	if (nv_device(priv)->chipset != 0xc1) {
+		tmp = 0x02180000;
+		mmio_list(0x405830, tmp, 0, 0);
+		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+			for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+				u32 reg = TPC_UNIT(gpc, tpc, 0x0520);
+				mmio_list(reg, tmp, 0, 0);
+				tmp += 0x0324;
+			}
+		}
+	} else {
+		tmp = 0x02180000;
+		mmio_list(0x405830, 0x00000218 | tmp, 0, 0);
+		mmio_list(0x4064c4, 0x0086ffff, 0, 0);
+		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+			for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+				u32 reg = TPC_UNIT(gpc, tpc, 0x0520);
+				mmio_list(reg, 0x10000000 | tmp, 0, 0);
+				tmp += 0x0324;
+			}
+			for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+				u32 reg = TPC_UNIT(gpc, tpc, 0x0544);
+				mmio_list(reg, tmp, 0, 0);
+				tmp += 0x0324;
+			}
+		}
+	}
+
+	for (tpc = 0, id = 0; tpc < 4; tpc++) {
+		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+			if (tpc < priv->tpc_nr[gpc]) {
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x698), id);
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x4e8), id);
+				nv_wr32(priv, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x088), id);
+				id++;
+			}
+
+			nv_wr32(priv, GPC_UNIT(gpc, 0x0c08), priv->tpc_nr[gpc]);
+			nv_wr32(priv, GPC_UNIT(gpc, 0x0c8c), priv->tpc_nr[gpc]);
+		}
+	}
+
+	tmp = 0;
+	for (i = 0; i < priv->gpc_nr; i++)
+		tmp |= priv->tpc_nr[i] << (i * 4);
+	nv_wr32(priv, 0x406028, tmp);
+	nv_wr32(priv, 0x405870, tmp);
+
+	nv_wr32(priv, 0x40602c, 0x00000000);
+	nv_wr32(priv, 0x405874, 0x00000000);
+	nv_wr32(priv, 0x406030, 0x00000000);
+	nv_wr32(priv, 0x405878, 0x00000000);
+	nv_wr32(priv, 0x406034, 0x00000000);
+	nv_wr32(priv, 0x40587c, 0x00000000);
+
+	if (1) {
+		u8 tpcnr[GPC_MAX], data[TPC_MAX];
+
+		memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+		memset(data, 0x1f, sizeof(data));
+
+		gpc = -1;
+		for (tpc = 0; tpc < priv->tpc_total; tpc++) {
+			do {
+				gpc = (gpc + 1) % priv->gpc_nr;
+			} while (!tpcnr[gpc]);
+			tpcnr[gpc]--;
+			data[tpc] = gpc;
+		}
+
+		for (i = 0; i < 4; i++)
+			nv_wr32(priv, 0x4060a8 + (i * 4), ((u32 *)data)[i]);
+	}
+
+	if (1) {
+		u32 data[6] = {}, data2[2] = {};
+		u8 tpcnr[GPC_MAX];
+		u8 shift, ntpcv;
+
+		/* calculate first set of magics */
+		memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+
+		gpc = -1;
+		for (tpc = 0; tpc < priv->tpc_total; tpc++) {
+			do {
+				gpc = (gpc + 1) % priv->gpc_nr;
+			} while (!tpcnr[gpc]);
+			tpcnr[gpc]--;
+
+			data[tpc / 6] |= gpc << ((tpc % 6) * 5);
+		}
+
+		for (; tpc < 32; tpc++)
+			data[tpc / 6] |= 7 << ((tpc % 6) * 5);
+
+		/* and the second... */
+		shift = 0;
+		ntpcv = priv->tpc_total;
+		while (!(ntpcv & (1 << 4))) {
+			ntpcv <<= 1;
+			shift++;
+		}
+
+		data2[0]  = (ntpcv << 16);
+		data2[0] |= (shift << 21);
+		data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
+		for (i = 1; i < 7; i++)
+			data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
+
+		/* GPC_BROADCAST */
+		nv_wr32(priv, 0x418bb8, (priv->tpc_total << 8) |
+					priv->magic_not_rop_nr);
+		for (i = 0; i < 6; i++)
+			nv_wr32(priv, 0x418b08 + (i * 4), data[i]);
+
+		/* GPC_BROADCAST.TP_BROADCAST */
+		nv_wr32(priv, 0x419bd0, (priv->tpc_total << 8) |
+				       priv->magic_not_rop_nr |
+				       data2[0]);
+		nv_wr32(priv, 0x419be4, data2[1]);
+		for (i = 0; i < 6; i++)
+			nv_wr32(priv, 0x419b00 + (i * 4), data[i]);
+
+		/* UNK78xx */
+		nv_wr32(priv, 0x4078bc, (priv->tpc_total << 8) |
+					priv->magic_not_rop_nr);
+		for (i = 0; i < 6; i++)
+			nv_wr32(priv, 0x40780c + (i * 4), data[i]);
+	}
+
+	if (1) {
+		u32 tpc_mask = 0, tpc_set = 0;
+		u8  tpcnr[GPC_MAX], a, b;
+
+		memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+		for (gpc = 0; gpc < priv->gpc_nr; gpc++)
+			tpc_mask |= ((1 << priv->tpc_nr[gpc]) - 1) << (gpc * 8);
+
+		for (i = 0, gpc = -1, b = -1; i < 32; i++) {
+			a = (i * (priv->tpc_total - 1)) / 32;
+			if (a != b) {
+				b = a;
+				do {
+					gpc = (gpc + 1) % priv->gpc_nr;
+				} while (!tpcnr[gpc]);
+				tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
+
+				tpc_set |= 1 << ((gpc * 8) + tpc);
+			}
+
+			nv_wr32(priv, 0x406800 + (i * 0x20), tpc_set);
+			nv_wr32(priv, 0x406c00 + (i * 0x20), tpc_set ^ tpc_mask);
+		}
+	}
+
+	nv_wr32(priv, 0x400208, 0x80000000);
+
+	nv_icmd(priv, 0x00001000, 0x00000004);
+	nv_icmd(priv, 0x000000a9, 0x0000ffff);
+	nv_icmd(priv, 0x00000038, 0x0fac6881);
+	nv_icmd(priv, 0x0000003d, 0x00000001);
+	nv_icmd(priv, 0x000000e8, 0x00000400);
+	nv_icmd(priv, 0x000000e9, 0x00000400);
+	nv_icmd(priv, 0x000000ea, 0x00000400);
+	nv_icmd(priv, 0x000000eb, 0x00000400);
+	nv_icmd(priv, 0x000000ec, 0x00000400);
+	nv_icmd(priv, 0x000000ed, 0x00000400);
+	nv_icmd(priv, 0x000000ee, 0x00000400);
+	nv_icmd(priv, 0x000000ef, 0x00000400);
+	nv_icmd(priv, 0x00000078, 0x00000300);
+	nv_icmd(priv, 0x00000079, 0x00000300);
+	nv_icmd(priv, 0x0000007a, 0x00000300);
+	nv_icmd(priv, 0x0000007b, 0x00000300);
+	nv_icmd(priv, 0x0000007c, 0x00000300);
+	nv_icmd(priv, 0x0000007d, 0x00000300);
+	nv_icmd(priv, 0x0000007e, 0x00000300);
+	nv_icmd(priv, 0x0000007f, 0x00000300);
+	nv_icmd(priv, 0x00000050, 0x00000011);
+	nv_icmd(priv, 0x00000058, 0x00000008);
+	nv_icmd(priv, 0x00000059, 0x00000008);
+	nv_icmd(priv, 0x0000005a, 0x00000008);
+	nv_icmd(priv, 0x0000005b, 0x00000008);
+	nv_icmd(priv, 0x0000005c, 0x00000008);
+	nv_icmd(priv, 0x0000005d, 0x00000008);
+	nv_icmd(priv, 0x0000005e, 0x00000008);
+	nv_icmd(priv, 0x0000005f, 0x00000008);
+	nv_icmd(priv, 0x00000208, 0x00000001);
+	nv_icmd(priv, 0x00000209, 0x00000001);
+	nv_icmd(priv, 0x0000020a, 0x00000001);
+	nv_icmd(priv, 0x0000020b, 0x00000001);
+	nv_icmd(priv, 0x0000020c, 0x00000001);
+	nv_icmd(priv, 0x0000020d, 0x00000001);
+	nv_icmd(priv, 0x0000020e, 0x00000001);
+	nv_icmd(priv, 0x0000020f, 0x00000001);
+	nv_icmd(priv, 0x00000081, 0x00000001);
+	nv_icmd(priv, 0x00000085, 0x00000004);
+	nv_icmd(priv, 0x00000088, 0x00000400);
+	nv_icmd(priv, 0x00000090, 0x00000300);
+	nv_icmd(priv, 0x00000098, 0x00001001);
+	nv_icmd(priv, 0x000000e3, 0x00000001);
+	nv_icmd(priv, 0x000000da, 0x00000001);
+	nv_icmd(priv, 0x000000f8, 0x00000003);
+	nv_icmd(priv, 0x000000fa, 0x00000001);
+	nv_icmd(priv, 0x0000009f, 0x0000ffff);
+	nv_icmd(priv, 0x000000a0, 0x0000ffff);
+	nv_icmd(priv, 0x000000a1, 0x0000ffff);
+	nv_icmd(priv, 0x000000a2, 0x0000ffff);
+	nv_icmd(priv, 0x000000b1, 0x00000001);
+	nv_icmd(priv, 0x000000b2, 0x00000000);
+	nv_icmd(priv, 0x000000b3, 0x00000000);
+	nv_icmd(priv, 0x000000b4, 0x00000000);
+	nv_icmd(priv, 0x000000b5, 0x00000000);
+	nv_icmd(priv, 0x000000b6, 0x00000000);
+	nv_icmd(priv, 0x000000b7, 0x00000000);
+	nv_icmd(priv, 0x000000b8, 0x00000000);
+	nv_icmd(priv, 0x000000b9, 0x00000000);
+	nv_icmd(priv, 0x000000ba, 0x00000000);
+	nv_icmd(priv, 0x000000bb, 0x00000000);
+	nv_icmd(priv, 0x000000bc, 0x00000000);
+	nv_icmd(priv, 0x000000bd, 0x00000000);
+	nv_icmd(priv, 0x000000be, 0x00000000);
+	nv_icmd(priv, 0x000000bf, 0x00000000);
+	nv_icmd(priv, 0x000000c0, 0x00000000);
+	nv_icmd(priv, 0x000000c1, 0x00000000);
+	nv_icmd(priv, 0x000000c2, 0x00000000);
+	nv_icmd(priv, 0x000000c3, 0x00000000);
+	nv_icmd(priv, 0x000000c4, 0x00000000);
+	nv_icmd(priv, 0x000000c5, 0x00000000);
+	nv_icmd(priv, 0x000000c6, 0x00000000);
+	nv_icmd(priv, 0x000000c7, 0x00000000);
+	nv_icmd(priv, 0x000000c8, 0x00000000);
+	nv_icmd(priv, 0x000000c9, 0x00000000);
+	nv_icmd(priv, 0x000000ca, 0x00000000);
+	nv_icmd(priv, 0x000000cb, 0x00000000);
+	nv_icmd(priv, 0x000000cc, 0x00000000);
+	nv_icmd(priv, 0x000000cd, 0x00000000);
+	nv_icmd(priv, 0x000000ce, 0x00000000);
+	nv_icmd(priv, 0x000000cf, 0x00000000);
+	nv_icmd(priv, 0x000000d0, 0x00000000);
+	nv_icmd(priv, 0x000000d1, 0x00000000);
+	nv_icmd(priv, 0x000000d2, 0x00000000);
+	nv_icmd(priv, 0x000000d3, 0x00000000);
+	nv_icmd(priv, 0x000000d4, 0x00000000);
+	nv_icmd(priv, 0x000000d5, 0x00000000);
+	nv_icmd(priv, 0x000000d6, 0x00000000);
+	nv_icmd(priv, 0x000000d7, 0x00000000);
+	nv_icmd(priv, 0x000000d8, 0x00000000);
+	nv_icmd(priv, 0x000000d9, 0x00000000);
+	nv_icmd(priv, 0x00000210, 0x00000040);
+	nv_icmd(priv, 0x00000211, 0x00000040);
+	nv_icmd(priv, 0x00000212, 0x00000040);
+	nv_icmd(priv, 0x00000213, 0x00000040);
+	nv_icmd(priv, 0x00000214, 0x00000040);
+	nv_icmd(priv, 0x00000215, 0x00000040);
+	nv_icmd(priv, 0x00000216, 0x00000040);
+	nv_icmd(priv, 0x00000217, 0x00000040);
+	if (nv_device(priv)->chipset == 0xd9) {
+		for (i = 0x0400; i <= 0x0417; i++)
+			nv_icmd(priv, i, 0x00000040);
+	}
+	nv_icmd(priv, 0x00000218, 0x0000c080);
+	nv_icmd(priv, 0x00000219, 0x0000c080);
+	nv_icmd(priv, 0x0000021a, 0x0000c080);
+	nv_icmd(priv, 0x0000021b, 0x0000c080);
+	nv_icmd(priv, 0x0000021c, 0x0000c080);
+	nv_icmd(priv, 0x0000021d, 0x0000c080);
+	nv_icmd(priv, 0x0000021e, 0x0000c080);
+	nv_icmd(priv, 0x0000021f, 0x0000c080);
+	if (nv_device(priv)->chipset == 0xd9) {
+		for (i = 0x0440; i <= 0x0457; i++)
+			nv_icmd(priv, i, 0x0000c080);
+	}
+	nv_icmd(priv, 0x000000ad, 0x0000013e);
+	nv_icmd(priv, 0x000000e1, 0x00000010);
+	nv_icmd(priv, 0x00000290, 0x00000000);
+	nv_icmd(priv, 0x00000291, 0x00000000);
+	nv_icmd(priv, 0x00000292, 0x00000000);
+	nv_icmd(priv, 0x00000293, 0x00000000);
+	nv_icmd(priv, 0x00000294, 0x00000000);
+	nv_icmd(priv, 0x00000295, 0x00000000);
+	nv_icmd(priv, 0x00000296, 0x00000000);
+	nv_icmd(priv, 0x00000297, 0x00000000);
+	nv_icmd(priv, 0x00000298, 0x00000000);
+	nv_icmd(priv, 0x00000299, 0x00000000);
+	nv_icmd(priv, 0x0000029a, 0x00000000);
+	nv_icmd(priv, 0x0000029b, 0x00000000);
+	nv_icmd(priv, 0x0000029c, 0x00000000);
+	nv_icmd(priv, 0x0000029d, 0x00000000);
+	nv_icmd(priv, 0x0000029e, 0x00000000);
+	nv_icmd(priv, 0x0000029f, 0x00000000);
+	nv_icmd(priv, 0x000003b0, 0x00000000);
+	nv_icmd(priv, 0x000003b1, 0x00000000);
+	nv_icmd(priv, 0x000003b2, 0x00000000);
+	nv_icmd(priv, 0x000003b3, 0x00000000);
+	nv_icmd(priv, 0x000003b4, 0x00000000);
+	nv_icmd(priv, 0x000003b5, 0x00000000);
+	nv_icmd(priv, 0x000003b6, 0x00000000);
+	nv_icmd(priv, 0x000003b7, 0x00000000);
+	nv_icmd(priv, 0x000003b8, 0x00000000);
+	nv_icmd(priv, 0x000003b9, 0x00000000);
+	nv_icmd(priv, 0x000003ba, 0x00000000);
+	nv_icmd(priv, 0x000003bb, 0x00000000);
+	nv_icmd(priv, 0x000003bc, 0x00000000);
+	nv_icmd(priv, 0x000003bd, 0x00000000);
+	nv_icmd(priv, 0x000003be, 0x00000000);
+	nv_icmd(priv, 0x000003bf, 0x00000000);
+	nv_icmd(priv, 0x000002a0, 0x00000000);
+	nv_icmd(priv, 0x000002a1, 0x00000000);
+	nv_icmd(priv, 0x000002a2, 0x00000000);
+	nv_icmd(priv, 0x000002a3, 0x00000000);
+	nv_icmd(priv, 0x000002a4, 0x00000000);
+	nv_icmd(priv, 0x000002a5, 0x00000000);
+	nv_icmd(priv, 0x000002a6, 0x00000000);
+	nv_icmd(priv, 0x000002a7, 0x00000000);
+	nv_icmd(priv, 0x000002a8, 0x00000000);
+	nv_icmd(priv, 0x000002a9, 0x00000000);
+	nv_icmd(priv, 0x000002aa, 0x00000000);
+	nv_icmd(priv, 0x000002ab, 0x00000000);
+	nv_icmd(priv, 0x000002ac, 0x00000000);
+	nv_icmd(priv, 0x000002ad, 0x00000000);
+	nv_icmd(priv, 0x000002ae, 0x00000000);
+	nv_icmd(priv, 0x000002af, 0x00000000);
+	nv_icmd(priv, 0x00000420, 0x00000000);
+	nv_icmd(priv, 0x00000421, 0x00000000);
+	nv_icmd(priv, 0x00000422, 0x00000000);
+	nv_icmd(priv, 0x00000423, 0x00000000);
+	nv_icmd(priv, 0x00000424, 0x00000000);
+	nv_icmd(priv, 0x00000425, 0x00000000);
+	nv_icmd(priv, 0x00000426, 0x00000000);
+	nv_icmd(priv, 0x00000427, 0x00000000);
+	nv_icmd(priv, 0x00000428, 0x00000000);
+	nv_icmd(priv, 0x00000429, 0x00000000);
+	nv_icmd(priv, 0x0000042a, 0x00000000);
+	nv_icmd(priv, 0x0000042b, 0x00000000);
+	nv_icmd(priv, 0x0000042c, 0x00000000);
+	nv_icmd(priv, 0x0000042d, 0x00000000);
+	nv_icmd(priv, 0x0000042e, 0x00000000);
+	nv_icmd(priv, 0x0000042f, 0x00000000);
+	nv_icmd(priv, 0x000002b0, 0x00000000);
+	nv_icmd(priv, 0x000002b1, 0x00000000);
+	nv_icmd(priv, 0x000002b2, 0x00000000);
+	nv_icmd(priv, 0x000002b3, 0x00000000);
+	nv_icmd(priv, 0x000002b4, 0x00000000);
+	nv_icmd(priv, 0x000002b5, 0x00000000);
+	nv_icmd(priv, 0x000002b6, 0x00000000);
+	nv_icmd(priv, 0x000002b7, 0x00000000);
+	nv_icmd(priv, 0x000002b8, 0x00000000);
+	nv_icmd(priv, 0x000002b9, 0x00000000);
+	nv_icmd(priv, 0x000002ba, 0x00000000);
+	nv_icmd(priv, 0x000002bb, 0x00000000);
+	nv_icmd(priv, 0x000002bc, 0x00000000);
+	nv_icmd(priv, 0x000002bd, 0x00000000);
+	nv_icmd(priv, 0x000002be, 0x00000000);
+	nv_icmd(priv, 0x000002bf, 0x00000000);
+	nv_icmd(priv, 0x00000430, 0x00000000);
+	nv_icmd(priv, 0x00000431, 0x00000000);
+	nv_icmd(priv, 0x00000432, 0x00000000);
+	nv_icmd(priv, 0x00000433, 0x00000000);
+	nv_icmd(priv, 0x00000434, 0x00000000);
+	nv_icmd(priv, 0x00000435, 0x00000000);
+	nv_icmd(priv, 0x00000436, 0x00000000);
+	nv_icmd(priv, 0x00000437, 0x00000000);
+	nv_icmd(priv, 0x00000438, 0x00000000);
+	nv_icmd(priv, 0x00000439, 0x00000000);
+	nv_icmd(priv, 0x0000043a, 0x00000000);
+	nv_icmd(priv, 0x0000043b, 0x00000000);
+	nv_icmd(priv, 0x0000043c, 0x00000000);
+	nv_icmd(priv, 0x0000043d, 0x00000000);
+	nv_icmd(priv, 0x0000043e, 0x00000000);
+	nv_icmd(priv, 0x0000043f, 0x00000000);
+	nv_icmd(priv, 0x000002c0, 0x00000000);
+	nv_icmd(priv, 0x000002c1, 0x00000000);
+	nv_icmd(priv, 0x000002c2, 0x00000000);
+	nv_icmd(priv, 0x000002c3, 0x00000000);
+	nv_icmd(priv, 0x000002c4, 0x00000000);
+	nv_icmd(priv, 0x000002c5, 0x00000000);
+	nv_icmd(priv, 0x000002c6, 0x00000000);
+	nv_icmd(priv, 0x000002c7, 0x00000000);
+	nv_icmd(priv, 0x000002c8, 0x00000000);
+	nv_icmd(priv, 0x000002c9, 0x00000000);
+	nv_icmd(priv, 0x000002ca, 0x00000000);
+	nv_icmd(priv, 0x000002cb, 0x00000000);
+	nv_icmd(priv, 0x000002cc, 0x00000000);
+	nv_icmd(priv, 0x000002cd, 0x00000000);
+	nv_icmd(priv, 0x000002ce, 0x00000000);
+	nv_icmd(priv, 0x000002cf, 0x00000000);
+	nv_icmd(priv, 0x000004d0, 0x00000000);
+	nv_icmd(priv, 0x000004d1, 0x00000000);
+	nv_icmd(priv, 0x000004d2, 0x00000000);
+	nv_icmd(priv, 0x000004d3, 0x00000000);
+	nv_icmd(priv, 0x000004d4, 0x00000000);
+	nv_icmd(priv, 0x000004d5, 0x00000000);
+	nv_icmd(priv, 0x000004d6, 0x00000000);
+	nv_icmd(priv, 0x000004d7, 0x00000000);
+	nv_icmd(priv, 0x000004d8, 0x00000000);
+	nv_icmd(priv, 0x000004d9, 0x00000000);
+	nv_icmd(priv, 0x000004da, 0x00000000);
+	nv_icmd(priv, 0x000004db, 0x00000000);
+	nv_icmd(priv, 0x000004dc, 0x00000000);
+	nv_icmd(priv, 0x000004dd, 0x00000000);
+	nv_icmd(priv, 0x000004de, 0x00000000);
+	nv_icmd(priv, 0x000004df, 0x00000000);
+	nv_icmd(priv, 0x00000720, 0x00000000);
+	nv_icmd(priv, 0x00000721, 0x00000000);
+	nv_icmd(priv, 0x00000722, 0x00000000);
+	nv_icmd(priv, 0x00000723, 0x00000000);
+	nv_icmd(priv, 0x00000724, 0x00000000);
+	nv_icmd(priv, 0x00000725, 0x00000000);
+	nv_icmd(priv, 0x00000726, 0x00000000);
+	nv_icmd(priv, 0x00000727, 0x00000000);
+	nv_icmd(priv, 0x00000728, 0x00000000);
+	nv_icmd(priv, 0x00000729, 0x00000000);
+	nv_icmd(priv, 0x0000072a, 0x00000000);
+	nv_icmd(priv, 0x0000072b, 0x00000000);
+	nv_icmd(priv, 0x0000072c, 0x00000000);
+	nv_icmd(priv, 0x0000072d, 0x00000000);
+	nv_icmd(priv, 0x0000072e, 0x00000000);
+	nv_icmd(priv, 0x0000072f, 0x00000000);
+	nv_icmd(priv, 0x000008c0, 0x00000000);
+	nv_icmd(priv, 0x000008c1, 0x00000000);
+	nv_icmd(priv, 0x000008c2, 0x00000000);
+	nv_icmd(priv, 0x000008c3, 0x00000000);
+	nv_icmd(priv, 0x000008c4, 0x00000000);
+	nv_icmd(priv, 0x000008c5, 0x00000000);
+	nv_icmd(priv, 0x000008c6, 0x00000000);
+	nv_icmd(priv, 0x000008c7, 0x00000000);
+	nv_icmd(priv, 0x000008c8, 0x00000000);
+	nv_icmd(priv, 0x000008c9, 0x00000000);
+	nv_icmd(priv, 0x000008ca, 0x00000000);
+	nv_icmd(priv, 0x000008cb, 0x00000000);
+	nv_icmd(priv, 0x000008cc, 0x00000000);
+	nv_icmd(priv, 0x000008cd, 0x00000000);
+	nv_icmd(priv, 0x000008ce, 0x00000000);
+	nv_icmd(priv, 0x000008cf, 0x00000000);
+	nv_icmd(priv, 0x00000890, 0x00000000);
+	nv_icmd(priv, 0x00000891, 0x00000000);
+	nv_icmd(priv, 0x00000892, 0x00000000);
+	nv_icmd(priv, 0x00000893, 0x00000000);
+	nv_icmd(priv, 0x00000894, 0x00000000);
+	nv_icmd(priv, 0x00000895, 0x00000000);
+	nv_icmd(priv, 0x00000896, 0x00000000);
+	nv_icmd(priv, 0x00000897, 0x00000000);
+	nv_icmd(priv, 0x00000898, 0x00000000);
+	nv_icmd(priv, 0x00000899, 0x00000000);
+	nv_icmd(priv, 0x0000089a, 0x00000000);
+	nv_icmd(priv, 0x0000089b, 0x00000000);
+	nv_icmd(priv, 0x0000089c, 0x00000000);
+	nv_icmd(priv, 0x0000089d, 0x00000000);
+	nv_icmd(priv, 0x0000089e, 0x00000000);
+	nv_icmd(priv, 0x0000089f, 0x00000000);
+	nv_icmd(priv, 0x000008e0, 0x00000000);
+	nv_icmd(priv, 0x000008e1, 0x00000000);
+	nv_icmd(priv, 0x000008e2, 0x00000000);
+	nv_icmd(priv, 0x000008e3, 0x00000000);
+	nv_icmd(priv, 0x000008e4, 0x00000000);
+	nv_icmd(priv, 0x000008e5, 0x00000000);
+	nv_icmd(priv, 0x000008e6, 0x00000000);
+	nv_icmd(priv, 0x000008e7, 0x00000000);
+	nv_icmd(priv, 0x000008e8, 0x00000000);
+	nv_icmd(priv, 0x000008e9, 0x00000000);
+	nv_icmd(priv, 0x000008ea, 0x00000000);
+	nv_icmd(priv, 0x000008eb, 0x00000000);
+	nv_icmd(priv, 0x000008ec, 0x00000000);
+	nv_icmd(priv, 0x000008ed, 0x00000000);
+	nv_icmd(priv, 0x000008ee, 0x00000000);
+	nv_icmd(priv, 0x000008ef, 0x00000000);
+	nv_icmd(priv, 0x000008a0, 0x00000000);
+	nv_icmd(priv, 0x000008a1, 0x00000000);
+	nv_icmd(priv, 0x000008a2, 0x00000000);
+	nv_icmd(priv, 0x000008a3, 0x00000000);
+	nv_icmd(priv, 0x000008a4, 0x00000000);
+	nv_icmd(priv, 0x000008a5, 0x00000000);
+	nv_icmd(priv, 0x000008a6, 0x00000000);
+	nv_icmd(priv, 0x000008a7, 0x00000000);
+	nv_icmd(priv, 0x000008a8, 0x00000000);
+	nv_icmd(priv, 0x000008a9, 0x00000000);
+	nv_icmd(priv, 0x000008aa, 0x00000000);
+	nv_icmd(priv, 0x000008ab, 0x00000000);
+	nv_icmd(priv, 0x000008ac, 0x00000000);
+	nv_icmd(priv, 0x000008ad, 0x00000000);
+	nv_icmd(priv, 0x000008ae, 0x00000000);
+	nv_icmd(priv, 0x000008af, 0x00000000);
+	nv_icmd(priv, 0x000008f0, 0x00000000);
+	nv_icmd(priv, 0x000008f1, 0x00000000);
+	nv_icmd(priv, 0x000008f2, 0x00000000);
+	nv_icmd(priv, 0x000008f3, 0x00000000);
+	nv_icmd(priv, 0x000008f4, 0x00000000);
+	nv_icmd(priv, 0x000008f5, 0x00000000);
+	nv_icmd(priv, 0x000008f6, 0x00000000);
+	nv_icmd(priv, 0x000008f7, 0x00000000);
+	nv_icmd(priv, 0x000008f8, 0x00000000);
+	nv_icmd(priv, 0x000008f9, 0x00000000);
+	nv_icmd(priv, 0x000008fa, 0x00000000);
+	nv_icmd(priv, 0x000008fb, 0x00000000);
+	nv_icmd(priv, 0x000008fc, 0x00000000);
+	nv_icmd(priv, 0x000008fd, 0x00000000);
+	nv_icmd(priv, 0x000008fe, 0x00000000);
+	nv_icmd(priv, 0x000008ff, 0x00000000);
+	nv_icmd(priv, 0x0000094c, 0x000000ff);
+	nv_icmd(priv, 0x0000094d, 0xffffffff);
+	nv_icmd(priv, 0x0000094e, 0x00000002);
+	nv_icmd(priv, 0x000002ec, 0x00000001);
+	nv_icmd(priv, 0x00000303, 0x00000001);
+	nv_icmd(priv, 0x000002e6, 0x00000001);
+	nv_icmd(priv, 0x00000466, 0x00000052);
+	nv_icmd(priv, 0x00000301, 0x3f800000);
+	nv_icmd(priv, 0x00000304, 0x30201000);
+	nv_icmd(priv, 0x00000305, 0x70605040);
+	nv_icmd(priv, 0x00000306, 0xb8a89888);
+	nv_icmd(priv, 0x00000307, 0xf8e8d8c8);
+	nv_icmd(priv, 0x0000030a, 0x00ffff00);
+	nv_icmd(priv, 0x0000030b, 0x0000001a);
+	nv_icmd(priv, 0x0000030c, 0x00000001);
+	nv_icmd(priv, 0x00000318, 0x00000001);
+	nv_icmd(priv, 0x00000340, 0x00000000);
+	nv_icmd(priv, 0x00000375, 0x00000001);
+	nv_icmd(priv, 0x00000351, 0x00000100);
+	nv_icmd(priv, 0x0000037d, 0x00000006);
+	nv_icmd(priv, 0x000003a0, 0x00000002);
+	nv_icmd(priv, 0x000003aa, 0x00000001);
+	nv_icmd(priv, 0x000003a9, 0x00000001);
+	nv_icmd(priv, 0x00000380, 0x00000001);
+	nv_icmd(priv, 0x00000360, 0x00000040);
+	nv_icmd(priv, 0x00000366, 0x00000000);
+	nv_icmd(priv, 0x00000367, 0x00000000);
+	nv_icmd(priv, 0x00000368, 0x00001fff);
+	nv_icmd(priv, 0x00000370, 0x00000000);
+	nv_icmd(priv, 0x00000371, 0x00000000);
+	nv_icmd(priv, 0x00000372, 0x003fffff);
+	nv_icmd(priv, 0x0000037a, 0x00000012);
+	nv_icmd(priv, 0x000005e0, 0x00000022);
+	nv_icmd(priv, 0x000005e1, 0x00000022);
+	nv_icmd(priv, 0x000005e2, 0x00000022);
+	nv_icmd(priv, 0x000005e3, 0x00000022);
+	nv_icmd(priv, 0x000005e4, 0x00000022);
+	nv_icmd(priv, 0x00000619, 0x00000003);
+	nv_icmd(priv, 0x00000811, 0x00000003);
+	nv_icmd(priv, 0x00000812, 0x00000004);
+	nv_icmd(priv, 0x00000813, 0x00000006);
+	nv_icmd(priv, 0x00000814, 0x00000008);
+	nv_icmd(priv, 0x00000815, 0x0000000b);
+	nv_icmd(priv, 0x00000800, 0x00000001);
+	nv_icmd(priv, 0x00000801, 0x00000001);
+	nv_icmd(priv, 0x00000802, 0x00000001);
+	nv_icmd(priv, 0x00000803, 0x00000001);
+	nv_icmd(priv, 0x00000804, 0x00000001);
+	nv_icmd(priv, 0x00000805, 0x00000001);
+	nv_icmd(priv, 0x00000632, 0x00000001);
+	nv_icmd(priv, 0x00000633, 0x00000002);
+	nv_icmd(priv, 0x00000634, 0x00000003);
+	nv_icmd(priv, 0x00000635, 0x00000004);
+	nv_icmd(priv, 0x00000654, 0x3f800000);
+	nv_icmd(priv, 0x00000657, 0x3f800000);
+	nv_icmd(priv, 0x00000655, 0x3f800000);
+	nv_icmd(priv, 0x00000656, 0x3f800000);
+	nv_icmd(priv, 0x000006cd, 0x3f800000);
+	nv_icmd(priv, 0x000007f5, 0x3f800000);
+	nv_icmd(priv, 0x000007dc, 0x39291909);
+	nv_icmd(priv, 0x000007dd, 0x79695949);
+	nv_icmd(priv, 0x000007de, 0xb9a99989);
+	nv_icmd(priv, 0x000007df, 0xf9e9d9c9);
+	nv_icmd(priv, 0x000007e8, 0x00003210);
+	nv_icmd(priv, 0x000007e9, 0x00007654);
+	nv_icmd(priv, 0x000007ea, 0x00000098);
+	nv_icmd(priv, 0x000007ec, 0x39291909);
+	nv_icmd(priv, 0x000007ed, 0x79695949);
+	nv_icmd(priv, 0x000007ee, 0xb9a99989);
+	nv_icmd(priv, 0x000007ef, 0xf9e9d9c9);
+	nv_icmd(priv, 0x000007f0, 0x00003210);
+	nv_icmd(priv, 0x000007f1, 0x00007654);
+	nv_icmd(priv, 0x000007f2, 0x00000098);
+	nv_icmd(priv, 0x000005a5, 0x00000001);
+	nv_icmd(priv, 0x00000980, 0x00000000);
+	nv_icmd(priv, 0x00000981, 0x00000000);
+	nv_icmd(priv, 0x00000982, 0x00000000);
+	nv_icmd(priv, 0x00000983, 0x00000000);
+	nv_icmd(priv, 0x00000984, 0x00000000);
+	nv_icmd(priv, 0x00000985, 0x00000000);
+	nv_icmd(priv, 0x00000986, 0x00000000);
+	nv_icmd(priv, 0x00000987, 0x00000000);
+	nv_icmd(priv, 0x00000988, 0x00000000);
+	nv_icmd(priv, 0x00000989, 0x00000000);
+	nv_icmd(priv, 0x0000098a, 0x00000000);
+	nv_icmd(priv, 0x0000098b, 0x00000000);
+	nv_icmd(priv, 0x0000098c, 0x00000000);
+	nv_icmd(priv, 0x0000098d, 0x00000000);
+	nv_icmd(priv, 0x0000098e, 0x00000000);
+	nv_icmd(priv, 0x0000098f, 0x00000000);
+	nv_icmd(priv, 0x00000990, 0x00000000);
+	nv_icmd(priv, 0x00000991, 0x00000000);
+	nv_icmd(priv, 0x00000992, 0x00000000);
+	nv_icmd(priv, 0x00000993, 0x00000000);
+	nv_icmd(priv, 0x00000994, 0x00000000);
+	nv_icmd(priv, 0x00000995, 0x00000000);
+	nv_icmd(priv, 0x00000996, 0x00000000);
+	nv_icmd(priv, 0x00000997, 0x00000000);
+	nv_icmd(priv, 0x00000998, 0x00000000);
+	nv_icmd(priv, 0x00000999, 0x00000000);
+	nv_icmd(priv, 0x0000099a, 0x00000000);
+	nv_icmd(priv, 0x0000099b, 0x00000000);
+	nv_icmd(priv, 0x0000099c, 0x00000000);
+	nv_icmd(priv, 0x0000099d, 0x00000000);
+	nv_icmd(priv, 0x0000099e, 0x00000000);
+	nv_icmd(priv, 0x0000099f, 0x00000000);
+	nv_icmd(priv, 0x000009a0, 0x00000000);
+	nv_icmd(priv, 0x000009a1, 0x00000000);
+	nv_icmd(priv, 0x000009a2, 0x00000000);
+	nv_icmd(priv, 0x000009a3, 0x00000000);
+	nv_icmd(priv, 0x000009a4, 0x00000000);
+	nv_icmd(priv, 0x000009a5, 0x00000000);
+	nv_icmd(priv, 0x000009a6, 0x00000000);
+	nv_icmd(priv, 0x000009a7, 0x00000000);
+	nv_icmd(priv, 0x000009a8, 0x00000000);
+	nv_icmd(priv, 0x000009a9, 0x00000000);
+	nv_icmd(priv, 0x000009aa, 0x00000000);
+	nv_icmd(priv, 0x000009ab, 0x00000000);
+	nv_icmd(priv, 0x000009ac, 0x00000000);
+	nv_icmd(priv, 0x000009ad, 0x00000000);
+	nv_icmd(priv, 0x000009ae, 0x00000000);
+	nv_icmd(priv, 0x000009af, 0x00000000);
+	nv_icmd(priv, 0x000009b0, 0x00000000);
+	nv_icmd(priv, 0x000009b1, 0x00000000);
+	nv_icmd(priv, 0x000009b2, 0x00000000);
+	nv_icmd(priv, 0x000009b3, 0x00000000);
+	nv_icmd(priv, 0x000009b4, 0x00000000);
+	nv_icmd(priv, 0x000009b5, 0x00000000);
+	nv_icmd(priv, 0x000009b6, 0x00000000);
+	nv_icmd(priv, 0x000009b7, 0x00000000);
+	nv_icmd(priv, 0x000009b8, 0x00000000);
+	nv_icmd(priv, 0x000009b9, 0x00000000);
+	nv_icmd(priv, 0x000009ba, 0x00000000);
+	nv_icmd(priv, 0x000009bb, 0x00000000);
+	nv_icmd(priv, 0x000009bc, 0x00000000);
+	nv_icmd(priv, 0x000009bd, 0x00000000);
+	nv_icmd(priv, 0x000009be, 0x00000000);
+	nv_icmd(priv, 0x000009bf, 0x00000000);
+	nv_icmd(priv, 0x000009c0, 0x00000000);
+	nv_icmd(priv, 0x000009c1, 0x00000000);
+	nv_icmd(priv, 0x000009c2, 0x00000000);
+	nv_icmd(priv, 0x000009c3, 0x00000000);
+	nv_icmd(priv, 0x000009c4, 0x00000000);
+	nv_icmd(priv, 0x000009c5, 0x00000000);
+	nv_icmd(priv, 0x000009c6, 0x00000000);
+	nv_icmd(priv, 0x000009c7, 0x00000000);
+	nv_icmd(priv, 0x000009c8, 0x00000000);
+	nv_icmd(priv, 0x000009c9, 0x00000000);
+	nv_icmd(priv, 0x000009ca, 0x00000000);
+	nv_icmd(priv, 0x000009cb, 0x00000000);
+	nv_icmd(priv, 0x000009cc, 0x00000000);
+	nv_icmd(priv, 0x000009cd, 0x00000000);
+	nv_icmd(priv, 0x000009ce, 0x00000000);
+	nv_icmd(priv, 0x000009cf, 0x00000000);
+	nv_icmd(priv, 0x000009d0, 0x00000000);
+	nv_icmd(priv, 0x000009d1, 0x00000000);
+	nv_icmd(priv, 0x000009d2, 0x00000000);
+	nv_icmd(priv, 0x000009d3, 0x00000000);
+	nv_icmd(priv, 0x000009d4, 0x00000000);
+	nv_icmd(priv, 0x000009d5, 0x00000000);
+	nv_icmd(priv, 0x000009d6, 0x00000000);
+	nv_icmd(priv, 0x000009d7, 0x00000000);
+	nv_icmd(priv, 0x000009d8, 0x00000000);
+	nv_icmd(priv, 0x000009d9, 0x00000000);
+	nv_icmd(priv, 0x000009da, 0x00000000);
+	nv_icmd(priv, 0x000009db, 0x00000000);
+	nv_icmd(priv, 0x000009dc, 0x00000000);
+	nv_icmd(priv, 0x000009dd, 0x00000000);
+	nv_icmd(priv, 0x000009de, 0x00000000);
+	nv_icmd(priv, 0x000009df, 0x00000000);
+	nv_icmd(priv, 0x000009e0, 0x00000000);
+	nv_icmd(priv, 0x000009e1, 0x00000000);
+	nv_icmd(priv, 0x000009e2, 0x00000000);
+	nv_icmd(priv, 0x000009e3, 0x00000000);
+	nv_icmd(priv, 0x000009e4, 0x00000000);
+	nv_icmd(priv, 0x000009e5, 0x00000000);
+	nv_icmd(priv, 0x000009e6, 0x00000000);
+	nv_icmd(priv, 0x000009e7, 0x00000000);
+	nv_icmd(priv, 0x000009e8, 0x00000000);
+	nv_icmd(priv, 0x000009e9, 0x00000000);
+	nv_icmd(priv, 0x000009ea, 0x00000000);
+	nv_icmd(priv, 0x000009eb, 0x00000000);
+	nv_icmd(priv, 0x000009ec, 0x00000000);
+	nv_icmd(priv, 0x000009ed, 0x00000000);
+	nv_icmd(priv, 0x000009ee, 0x00000000);
+	nv_icmd(priv, 0x000009ef, 0x00000000);
+	nv_icmd(priv, 0x000009f0, 0x00000000);
+	nv_icmd(priv, 0x000009f1, 0x00000000);
+	nv_icmd(priv, 0x000009f2, 0x00000000);
+	nv_icmd(priv, 0x000009f3, 0x00000000);
+	nv_icmd(priv, 0x000009f4, 0x00000000);
+	nv_icmd(priv, 0x000009f5, 0x00000000);
+	nv_icmd(priv, 0x000009f6, 0x00000000);
+	nv_icmd(priv, 0x000009f7, 0x00000000);
+	nv_icmd(priv, 0x000009f8, 0x00000000);
+	nv_icmd(priv, 0x000009f9, 0x00000000);
+	nv_icmd(priv, 0x000009fa, 0x00000000);
+	nv_icmd(priv, 0x000009fb, 0x00000000);
+	nv_icmd(priv, 0x000009fc, 0x00000000);
+	nv_icmd(priv, 0x000009fd, 0x00000000);
+	nv_icmd(priv, 0x000009fe, 0x00000000);
+	nv_icmd(priv, 0x000009ff, 0x00000000);
+	nv_icmd(priv, 0x00000468, 0x00000004);
+	nv_icmd(priv, 0x0000046c, 0x00000001);
+	nv_icmd(priv, 0x00000470, 0x00000000);
+	nv_icmd(priv, 0x00000471, 0x00000000);
+	nv_icmd(priv, 0x00000472, 0x00000000);
+	nv_icmd(priv, 0x00000473, 0x00000000);
+	nv_icmd(priv, 0x00000474, 0x00000000);
+	nv_icmd(priv, 0x00000475, 0x00000000);
+	nv_icmd(priv, 0x00000476, 0x00000000);
+	nv_icmd(priv, 0x00000477, 0x00000000);
+	nv_icmd(priv, 0x00000478, 0x00000000);
+	nv_icmd(priv, 0x00000479, 0x00000000);
+	nv_icmd(priv, 0x0000047a, 0x00000000);
+	nv_icmd(priv, 0x0000047b, 0x00000000);
+	nv_icmd(priv, 0x0000047c, 0x00000000);
+	nv_icmd(priv, 0x0000047d, 0x00000000);
+	nv_icmd(priv, 0x0000047e, 0x00000000);
+	nv_icmd(priv, 0x0000047f, 0x00000000);
+	nv_icmd(priv, 0x00000480, 0x00000000);
+	nv_icmd(priv, 0x00000481, 0x00000000);
+	nv_icmd(priv, 0x00000482, 0x00000000);
+	nv_icmd(priv, 0x00000483, 0x00000000);
+	nv_icmd(priv, 0x00000484, 0x00000000);
+	nv_icmd(priv, 0x00000485, 0x00000000);
+	nv_icmd(priv, 0x00000486, 0x00000000);
+	nv_icmd(priv, 0x00000487, 0x00000000);
+	nv_icmd(priv, 0x00000488, 0x00000000);
+	nv_icmd(priv, 0x00000489, 0x00000000);
+	nv_icmd(priv, 0x0000048a, 0x00000000);
+	nv_icmd(priv, 0x0000048b, 0x00000000);
+	nv_icmd(priv, 0x0000048c, 0x00000000);
+	nv_icmd(priv, 0x0000048d, 0x00000000);
+	nv_icmd(priv, 0x0000048e, 0x00000000);
+	nv_icmd(priv, 0x0000048f, 0x00000000);
+	nv_icmd(priv, 0x00000490, 0x00000000);
+	nv_icmd(priv, 0x00000491, 0x00000000);
+	nv_icmd(priv, 0x00000492, 0x00000000);
+	nv_icmd(priv, 0x00000493, 0x00000000);
+	nv_icmd(priv, 0x00000494, 0x00000000);
+	nv_icmd(priv, 0x00000495, 0x00000000);
+	nv_icmd(priv, 0x00000496, 0x00000000);
+	nv_icmd(priv, 0x00000497, 0x00000000);
+	nv_icmd(priv, 0x00000498, 0x00000000);
+	nv_icmd(priv, 0x00000499, 0x00000000);
+	nv_icmd(priv, 0x0000049a, 0x00000000);
+	nv_icmd(priv, 0x0000049b, 0x00000000);
+	nv_icmd(priv, 0x0000049c, 0x00000000);
+	nv_icmd(priv, 0x0000049d, 0x00000000);
+	nv_icmd(priv, 0x0000049e, 0x00000000);
+	nv_icmd(priv, 0x0000049f, 0x00000000);
+	nv_icmd(priv, 0x000004a0, 0x00000000);
+	nv_icmd(priv, 0x000004a1, 0x00000000);
+	nv_icmd(priv, 0x000004a2, 0x00000000);
+	nv_icmd(priv, 0x000004a3, 0x00000000);
+	nv_icmd(priv, 0x000004a4, 0x00000000);
+	nv_icmd(priv, 0x000004a5, 0x00000000);
+	nv_icmd(priv, 0x000004a6, 0x00000000);
+	nv_icmd(priv, 0x000004a7, 0x00000000);
+	nv_icmd(priv, 0x000004a8, 0x00000000);
+	nv_icmd(priv, 0x000004a9, 0x00000000);
+	nv_icmd(priv, 0x000004aa, 0x00000000);
+	nv_icmd(priv, 0x000004ab, 0x00000000);
+	nv_icmd(priv, 0x000004ac, 0x00000000);
+	nv_icmd(priv, 0x000004ad, 0x00000000);
+	nv_icmd(priv, 0x000004ae, 0x00000000);
+	nv_icmd(priv, 0x000004af, 0x00000000);
+	nv_icmd(priv, 0x000004b0, 0x00000000);
+	nv_icmd(priv, 0x000004b1, 0x00000000);
+	nv_icmd(priv, 0x000004b2, 0x00000000);
+	nv_icmd(priv, 0x000004b3, 0x00000000);
+	nv_icmd(priv, 0x000004b4, 0x00000000);
+	nv_icmd(priv, 0x000004b5, 0x00000000);
+	nv_icmd(priv, 0x000004b6, 0x00000000);
+	nv_icmd(priv, 0x000004b7, 0x00000000);
+	nv_icmd(priv, 0x000004b8, 0x00000000);
+	nv_icmd(priv, 0x000004b9, 0x00000000);
+	nv_icmd(priv, 0x000004ba, 0x00000000);
+	nv_icmd(priv, 0x000004bb, 0x00000000);
+	nv_icmd(priv, 0x000004bc, 0x00000000);
+	nv_icmd(priv, 0x000004bd, 0x00000000);
+	nv_icmd(priv, 0x000004be, 0x00000000);
+	nv_icmd(priv, 0x000004bf, 0x00000000);
+	nv_icmd(priv, 0x000004c0, 0x00000000);
+	nv_icmd(priv, 0x000004c1, 0x00000000);
+	nv_icmd(priv, 0x000004c2, 0x00000000);
+	nv_icmd(priv, 0x000004c3, 0x00000000);
+	nv_icmd(priv, 0x000004c4, 0x00000000);
+	nv_icmd(priv, 0x000004c5, 0x00000000);
+	nv_icmd(priv, 0x000004c6, 0x00000000);
+	nv_icmd(priv, 0x000004c7, 0x00000000);
+	nv_icmd(priv, 0x000004c8, 0x00000000);
+	nv_icmd(priv, 0x000004c9, 0x00000000);
+	nv_icmd(priv, 0x000004ca, 0x00000000);
+	nv_icmd(priv, 0x000004cb, 0x00000000);
+	nv_icmd(priv, 0x000004cc, 0x00000000);
+	nv_icmd(priv, 0x000004cd, 0x00000000);
+	nv_icmd(priv, 0x000004ce, 0x00000000);
+	nv_icmd(priv, 0x000004cf, 0x00000000);
+	nv_icmd(priv, 0x00000510, 0x3f800000);
+	nv_icmd(priv, 0x00000511, 0x3f800000);
+	nv_icmd(priv, 0x00000512, 0x3f800000);
+	nv_icmd(priv, 0x00000513, 0x3f800000);
+	nv_icmd(priv, 0x00000514, 0x3f800000);
+	nv_icmd(priv, 0x00000515, 0x3f800000);
+	nv_icmd(priv, 0x00000516, 0x3f800000);
+	nv_icmd(priv, 0x00000517, 0x3f800000);
+	nv_icmd(priv, 0x00000518, 0x3f800000);
+	nv_icmd(priv, 0x00000519, 0x3f800000);
+	nv_icmd(priv, 0x0000051a, 0x3f800000);
+	nv_icmd(priv, 0x0000051b, 0x3f800000);
+	nv_icmd(priv, 0x0000051c, 0x3f800000);
+	nv_icmd(priv, 0x0000051d, 0x3f800000);
+	nv_icmd(priv, 0x0000051e, 0x3f800000);
+	nv_icmd(priv, 0x0000051f, 0x3f800000);
+	nv_icmd(priv, 0x00000520, 0x000002b6);
+	nv_icmd(priv, 0x00000529, 0x00000001);
+	nv_icmd(priv, 0x00000530, 0xffff0000);
+	nv_icmd(priv, 0x00000531, 0xffff0000);
+	nv_icmd(priv, 0x00000532, 0xffff0000);
+	nv_icmd(priv, 0x00000533, 0xffff0000);
+	nv_icmd(priv, 0x00000534, 0xffff0000);
+	nv_icmd(priv, 0x00000535, 0xffff0000);
+	nv_icmd(priv, 0x00000536, 0xffff0000);
+	nv_icmd(priv, 0x00000537, 0xffff0000);
+	nv_icmd(priv, 0x00000538, 0xffff0000);
+	nv_icmd(priv, 0x00000539, 0xffff0000);
+	nv_icmd(priv, 0x0000053a, 0xffff0000);
+	nv_icmd(priv, 0x0000053b, 0xffff0000);
+	nv_icmd(priv, 0x0000053c, 0xffff0000);
+	nv_icmd(priv, 0x0000053d, 0xffff0000);
+	nv_icmd(priv, 0x0000053e, 0xffff0000);
+	nv_icmd(priv, 0x0000053f, 0xffff0000);
+	nv_icmd(priv, 0x00000585, 0x0000003f);
+	nv_icmd(priv, 0x00000576, 0x00000003);
+	if (nv_device(priv)->chipset == 0xc1 ||
+	    nv_device(priv)->chipset == 0xd9)
+		nv_icmd(priv, 0x0000057b, 0x00000059);
+	nv_icmd(priv, 0x00000586, 0x00000040);
+	nv_icmd(priv, 0x00000582, 0x00000080);
+	nv_icmd(priv, 0x00000583, 0x00000080);
+	nv_icmd(priv, 0x000005c2, 0x00000001);
+	nv_icmd(priv, 0x00000638, 0x00000001);
+	nv_icmd(priv, 0x00000639, 0x00000001);
+	nv_icmd(priv, 0x0000063a, 0x00000002);
+	nv_icmd(priv, 0x0000063b, 0x00000001);
+	nv_icmd(priv, 0x0000063c, 0x00000001);
+	nv_icmd(priv, 0x0000063d, 0x00000002);
+	nv_icmd(priv, 0x0000063e, 0x00000001);
+	nv_icmd(priv, 0x000008b8, 0x00000001);
+	nv_icmd(priv, 0x000008b9, 0x00000001);
+	nv_icmd(priv, 0x000008ba, 0x00000001);
+	nv_icmd(priv, 0x000008bb, 0x00000001);
+	nv_icmd(priv, 0x000008bc, 0x00000001);
+	nv_icmd(priv, 0x000008bd, 0x00000001);
+	nv_icmd(priv, 0x000008be, 0x00000001);
+	nv_icmd(priv, 0x000008bf, 0x00000001);
+	nv_icmd(priv, 0x00000900, 0x00000001);
+	nv_icmd(priv, 0x00000901, 0x00000001);
+	nv_icmd(priv, 0x00000902, 0x00000001);
+	nv_icmd(priv, 0x00000903, 0x00000001);
+	nv_icmd(priv, 0x00000904, 0x00000001);
+	nv_icmd(priv, 0x00000905, 0x00000001);
+	nv_icmd(priv, 0x00000906, 0x00000001);
+	nv_icmd(priv, 0x00000907, 0x00000001);
+	nv_icmd(priv, 0x00000908, 0x00000002);
+	nv_icmd(priv, 0x00000909, 0x00000002);
+	nv_icmd(priv, 0x0000090a, 0x00000002);
+	nv_icmd(priv, 0x0000090b, 0x00000002);
+	nv_icmd(priv, 0x0000090c, 0x00000002);
+	nv_icmd(priv, 0x0000090d, 0x00000002);
+	nv_icmd(priv, 0x0000090e, 0x00000002);
+	nv_icmd(priv, 0x0000090f, 0x00000002);
+	nv_icmd(priv, 0x00000910, 0x00000001);
+	nv_icmd(priv, 0x00000911, 0x00000001);
+	nv_icmd(priv, 0x00000912, 0x00000001);
+	nv_icmd(priv, 0x00000913, 0x00000001);
+	nv_icmd(priv, 0x00000914, 0x00000001);
+	nv_icmd(priv, 0x00000915, 0x00000001);
+	nv_icmd(priv, 0x00000916, 0x00000001);
+	nv_icmd(priv, 0x00000917, 0x00000001);
+	nv_icmd(priv, 0x00000918, 0x00000001);
+	nv_icmd(priv, 0x00000919, 0x00000001);
+	nv_icmd(priv, 0x0000091a, 0x00000001);
+	nv_icmd(priv, 0x0000091b, 0x00000001);
+	nv_icmd(priv, 0x0000091c, 0x00000001);
+	nv_icmd(priv, 0x0000091d, 0x00000001);
+	nv_icmd(priv, 0x0000091e, 0x00000001);
+	nv_icmd(priv, 0x0000091f, 0x00000001);
+	nv_icmd(priv, 0x00000920, 0x00000002);
+	nv_icmd(priv, 0x00000921, 0x00000002);
+	nv_icmd(priv, 0x00000922, 0x00000002);
+	nv_icmd(priv, 0x00000923, 0x00000002);
+	nv_icmd(priv, 0x00000924, 0x00000002);
+	nv_icmd(priv, 0x00000925, 0x00000002);
+	nv_icmd(priv, 0x00000926, 0x00000002);
+	nv_icmd(priv, 0x00000927, 0x00000002);
+	nv_icmd(priv, 0x00000928, 0x00000001);
+	nv_icmd(priv, 0x00000929, 0x00000001);
+	nv_icmd(priv, 0x0000092a, 0x00000001);
+	nv_icmd(priv, 0x0000092b, 0x00000001);
+	nv_icmd(priv, 0x0000092c, 0x00000001);
+	nv_icmd(priv, 0x0000092d, 0x00000001);
+	nv_icmd(priv, 0x0000092e, 0x00000001);
+	nv_icmd(priv, 0x0000092f, 0x00000001);
+	nv_icmd(priv, 0x00000648, 0x00000001);
+	nv_icmd(priv, 0x00000649, 0x00000001);
+	nv_icmd(priv, 0x0000064a, 0x00000001);
+	nv_icmd(priv, 0x0000064b, 0x00000001);
+	nv_icmd(priv, 0x0000064c, 0x00000001);
+	nv_icmd(priv, 0x0000064d, 0x00000001);
+	nv_icmd(priv, 0x0000064e, 0x00000001);
+	nv_icmd(priv, 0x0000064f, 0x00000001);
+	nv_icmd(priv, 0x00000650, 0x00000001);
+	nv_icmd(priv, 0x00000658, 0x0000000f);
+	nv_icmd(priv, 0x000007ff, 0x0000000a);
+	nv_icmd(priv, 0x0000066a, 0x40000000);
+	nv_icmd(priv, 0x0000066b, 0x10000000);
+	nv_icmd(priv, 0x0000066c, 0xffff0000);
+	nv_icmd(priv, 0x0000066d, 0xffff0000);
+	nv_icmd(priv, 0x000007af, 0x00000008);
+	nv_icmd(priv, 0x000007b0, 0x00000008);
+	nv_icmd(priv, 0x000007f6, 0x00000001);
+	nv_icmd(priv, 0x000006b2, 0x00000055);
+	nv_icmd(priv, 0x000007ad, 0x00000003);
+	nv_icmd(priv, 0x00000937, 0x00000001);
+	nv_icmd(priv, 0x00000971, 0x00000008);
+	nv_icmd(priv, 0x00000972, 0x00000040);
+	nv_icmd(priv, 0x00000973, 0x0000012c);
+	nv_icmd(priv, 0x0000097c, 0x00000040);
+	nv_icmd(priv, 0x00000979, 0x00000003);
+	nv_icmd(priv, 0x00000975, 0x00000020);
+	nv_icmd(priv, 0x00000976, 0x00000001);
+	nv_icmd(priv, 0x00000977, 0x00000020);
+	nv_icmd(priv, 0x00000978, 0x00000001);
+	nv_icmd(priv, 0x00000957, 0x00000003);
+	nv_icmd(priv, 0x0000095e, 0x20164010);
+	nv_icmd(priv, 0x0000095f, 0x00000020);
+	if (nv_device(priv)->chipset == 0xd9)
+		nv_icmd(priv, 0x0000097d, 0x00000020);
+	nv_icmd(priv, 0x00000683, 0x00000006);
+	nv_icmd(priv, 0x00000685, 0x003fffff);
+	nv_icmd(priv, 0x00000687, 0x00000c48);
+	nv_icmd(priv, 0x000006a0, 0x00000005);
+	nv_icmd(priv, 0x00000840, 0x00300008);
+	nv_icmd(priv, 0x00000841, 0x04000080);
+	nv_icmd(priv, 0x00000842, 0x00300008);
+	nv_icmd(priv, 0x00000843, 0x04000080);
+	nv_icmd(priv, 0x00000818, 0x00000000);
+	nv_icmd(priv, 0x00000819, 0x00000000);
+	nv_icmd(priv, 0x0000081a, 0x00000000);
+	nv_icmd(priv, 0x0000081b, 0x00000000);
+	nv_icmd(priv, 0x0000081c, 0x00000000);
+	nv_icmd(priv, 0x0000081d, 0x00000000);
+	nv_icmd(priv, 0x0000081e, 0x00000000);
+	nv_icmd(priv, 0x0000081f, 0x00000000);
+	nv_icmd(priv, 0x00000848, 0x00000000);
+	nv_icmd(priv, 0x00000849, 0x00000000);
+	nv_icmd(priv, 0x0000084a, 0x00000000);
+	nv_icmd(priv, 0x0000084b, 0x00000000);
+	nv_icmd(priv, 0x0000084c, 0x00000000);
+	nv_icmd(priv, 0x0000084d, 0x00000000);
+	nv_icmd(priv, 0x0000084e, 0x00000000);
+	nv_icmd(priv, 0x0000084f, 0x00000000);
+	nv_icmd(priv, 0x00000850, 0x00000000);
+	nv_icmd(priv, 0x00000851, 0x00000000);
+	nv_icmd(priv, 0x00000852, 0x00000000);
+	nv_icmd(priv, 0x00000853, 0x00000000);
+	nv_icmd(priv, 0x00000854, 0x00000000);
+	nv_icmd(priv, 0x00000855, 0x00000000);
+	nv_icmd(priv, 0x00000856, 0x00000000);
+	nv_icmd(priv, 0x00000857, 0x00000000);
+	nv_icmd(priv, 0x00000738, 0x00000000);
+	nv_icmd(priv, 0x000006aa, 0x00000001);
+	nv_icmd(priv, 0x000006ab, 0x00000002);
+	nv_icmd(priv, 0x000006ac, 0x00000080);
+	nv_icmd(priv, 0x000006ad, 0x00000100);
+	nv_icmd(priv, 0x000006ae, 0x00000100);
+	nv_icmd(priv, 0x000006b1, 0x00000011);
+	nv_icmd(priv, 0x000006bb, 0x000000cf);
+	nv_icmd(priv, 0x000006ce, 0x2a712488);
+	nv_icmd(priv, 0x00000739, 0x4085c000);
+	nv_icmd(priv, 0x0000073a, 0x00000080);
+	nv_icmd(priv, 0x00000786, 0x80000100);
+	nv_icmd(priv, 0x0000073c, 0x00010100);
+	nv_icmd(priv, 0x0000073d, 0x02800000);
+	nv_icmd(priv, 0x00000787, 0x000000cf);
+	nv_icmd(priv, 0x0000078c, 0x00000008);
+	nv_icmd(priv, 0x00000792, 0x00000001);
+	nv_icmd(priv, 0x00000794, 0x00000001);
+	nv_icmd(priv, 0x00000795, 0x00000001);
+	nv_icmd(priv, 0x00000796, 0x00000001);
+	nv_icmd(priv, 0x00000797, 0x000000cf);
+	nv_icmd(priv, 0x00000836, 0x00000001);
+	nv_icmd(priv, 0x0000079a, 0x00000002);
+	nv_icmd(priv, 0x00000833, 0x04444480);
+	nv_icmd(priv, 0x000007a1, 0x00000001);
+	nv_icmd(priv, 0x000007a3, 0x00000001);
+	nv_icmd(priv, 0x000007a4, 0x00000001);
+	nv_icmd(priv, 0x000007a5, 0x00000001);
+	nv_icmd(priv, 0x00000831, 0x00000004);
+	nv_icmd(priv, 0x0000080c, 0x00000002);
+	nv_icmd(priv, 0x0000080d, 0x00000100);
+	nv_icmd(priv, 0x0000080e, 0x00000100);
+	nv_icmd(priv, 0x0000080f, 0x00000001);
+	nv_icmd(priv, 0x00000823, 0x00000002);
+	nv_icmd(priv, 0x00000824, 0x00000100);
+	nv_icmd(priv, 0x00000825, 0x00000100);
+	nv_icmd(priv, 0x00000826, 0x00000001);
+	nv_icmd(priv, 0x0000095d, 0x00000001);
+	nv_icmd(priv, 0x0000082b, 0x00000004);
+	nv_icmd(priv, 0x00000942, 0x00010001);
+	nv_icmd(priv, 0x00000943, 0x00000001);
+	nv_icmd(priv, 0x00000944, 0x00000022);
+	nv_icmd(priv, 0x000007c5, 0x00010001);
+	nv_icmd(priv, 0x00000834, 0x00000001);
+	nv_icmd(priv, 0x000007c7, 0x00000001);
+	nv_icmd(priv, 0x0000c1b0, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b1, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b2, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b3, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b4, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b5, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b6, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b7, 0x0000000f);
+	nv_icmd(priv, 0x0000c1b8, 0x0fac6881);
+	nv_icmd(priv, 0x0000c1b9, 0x00fac688);
+	nv_icmd(priv, 0x0001e100, 0x00000001);
+	nv_icmd(priv, 0x00001000, 0x00000002);
+	nv_icmd(priv, 0x000006aa, 0x00000001);
+	nv_icmd(priv, 0x000006ad, 0x00000100);
+	nv_icmd(priv, 0x000006ae, 0x00000100);
+	nv_icmd(priv, 0x000006b1, 0x00000011);
+	nv_icmd(priv, 0x0000078c, 0x00000008);
+	nv_icmd(priv, 0x00000792, 0x00000001);
+	nv_icmd(priv, 0x00000794, 0x00000001);
+	nv_icmd(priv, 0x00000795, 0x00000001);
+	nv_icmd(priv, 0x00000796, 0x00000001);
+	nv_icmd(priv, 0x00000797, 0x000000cf);
+	nv_icmd(priv, 0x0000079a, 0x00000002);
+	nv_icmd(priv, 0x00000833, 0x04444480);
+	nv_icmd(priv, 0x000007a1, 0x00000001);
+	nv_icmd(priv, 0x000007a3, 0x00000001);
+	nv_icmd(priv, 0x000007a4, 0x00000001);
+	nv_icmd(priv, 0x000007a5, 0x00000001);
+	nv_icmd(priv, 0x00000831, 0x00000004);
+	nv_icmd(priv, 0x0001e100, 0x00000001);
+	nv_icmd(priv, 0x00001000, 0x00000014);
+	nv_icmd(priv, 0x00000351, 0x00000100);
+	nv_icmd(priv, 0x00000957, 0x00000003);
+	nv_icmd(priv, 0x0000095d, 0x00000001);
+	nv_icmd(priv, 0x0000082b, 0x00000004);
+	nv_icmd(priv, 0x00000942, 0x00010001);
+	nv_icmd(priv, 0x00000943, 0x00000001);
+	nv_icmd(priv, 0x000007c5, 0x00010001);
+	nv_icmd(priv, 0x00000834, 0x00000001);
+	nv_icmd(priv, 0x000007c7, 0x00000001);
+	nv_icmd(priv, 0x0001e100, 0x00000001);
+	nv_icmd(priv, 0x00001000, 0x00000001);
+	nv_icmd(priv, 0x0000080c, 0x00000002);
+	nv_icmd(priv, 0x0000080d, 0x00000100);
+	nv_icmd(priv, 0x0000080e, 0x00000100);
+	nv_icmd(priv, 0x0000080f, 0x00000001);
+	nv_icmd(priv, 0x00000823, 0x00000002);
+	nv_icmd(priv, 0x00000824, 0x00000100);
+	nv_icmd(priv, 0x00000825, 0x00000100);
+	nv_icmd(priv, 0x00000826, 0x00000001);
+	nv_icmd(priv, 0x0001e100, 0x00000001);
+	nv_wr32(priv, 0x400208, 0x00000000);
+	nv_wr32(priv, 0x404154, 0x00000400);
+
+	nvc0_grctx_generate_9097(priv);
+	if (fermi >= 0x9197)
+		nvc0_grctx_generate_9197(priv);
+	if (fermi >= 0x9297)
+		nvc0_grctx_generate_9297(priv);
+	nvc0_grctx_generate_902d(priv);
+	nvc0_grctx_generate_9039(priv);
+	nvc0_grctx_generate_90c0(priv);
+
+	nv_wr32(priv, 0x000260, r000260);
+
+	return nvc0_grctx_fini(&info);
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve0.c
new file mode 100644
index 00000000000..6d8c63931ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve0.c
@@ -0,0 +1,2788 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+static void
+nve0_grctx_generate_icmd(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x400208, 0x80000000);
+	nv_icmd(priv, 0x001000, 0x00000004);
+	nv_icmd(priv, 0x000039, 0x00000000);
+	nv_icmd(priv, 0x00003a, 0x00000000);
+	nv_icmd(priv, 0x00003b, 0x00000000);
+	nv_icmd(priv, 0x0000a9, 0x0000ffff);
+	nv_icmd(priv, 0x000038, 0x0fac6881);
+	nv_icmd(priv, 0x00003d, 0x00000001);
+	nv_icmd(priv, 0x0000e8, 0x00000400);
+	nv_icmd(priv, 0x0000e9, 0x00000400);
+	nv_icmd(priv, 0x0000ea, 0x00000400);
+	nv_icmd(priv, 0x0000eb, 0x00000400);
+	nv_icmd(priv, 0x0000ec, 0x00000400);
+	nv_icmd(priv, 0x0000ed, 0x00000400);
+	nv_icmd(priv, 0x0000ee, 0x00000400);
+	nv_icmd(priv, 0x0000ef, 0x00000400);
+	nv_icmd(priv, 0x000078, 0x00000300);
+	nv_icmd(priv, 0x000079, 0x00000300);
+	nv_icmd(priv, 0x00007a, 0x00000300);
+	nv_icmd(priv, 0x00007b, 0x00000300);
+	nv_icmd(priv, 0x00007c, 0x00000300);
+	nv_icmd(priv, 0x00007d, 0x00000300);
+	nv_icmd(priv, 0x00007e, 0x00000300);
+	nv_icmd(priv, 0x00007f, 0x00000300);
+	nv_icmd(priv, 0x000050, 0x00000011);
+	nv_icmd(priv, 0x000058, 0x00000008);
+	nv_icmd(priv, 0x000059, 0x00000008);
+	nv_icmd(priv, 0x00005a, 0x00000008);
+	nv_icmd(priv, 0x00005b, 0x00000008);
+	nv_icmd(priv, 0x00005c, 0x00000008);
+	nv_icmd(priv, 0x00005d, 0x00000008);
+	nv_icmd(priv, 0x00005e, 0x00000008);
+	nv_icmd(priv, 0x00005f, 0x00000008);
+	nv_icmd(priv, 0x000208, 0x00000001);
+	nv_icmd(priv, 0x000209, 0x00000001);
+	nv_icmd(priv, 0x00020a, 0x00000001);
+	nv_icmd(priv, 0x00020b, 0x00000001);
+	nv_icmd(priv, 0x00020c, 0x00000001);
+	nv_icmd(priv, 0x00020d, 0x00000001);
+	nv_icmd(priv, 0x00020e, 0x00000001);
+	nv_icmd(priv, 0x00020f, 0x00000001);
+	nv_icmd(priv, 0x000081, 0x00000001);
+	nv_icmd(priv, 0x000085, 0x00000004);
+	nv_icmd(priv, 0x000088, 0x00000400);
+	nv_icmd(priv, 0x000090, 0x00000300);
+	nv_icmd(priv, 0x000098, 0x00001001);
+	nv_icmd(priv, 0x0000e3, 0x00000001);
+	nv_icmd(priv, 0x0000da, 0x00000001);
+	nv_icmd(priv, 0x0000f8, 0x00000003);
+	nv_icmd(priv, 0x0000fa, 0x00000001);
+	nv_icmd(priv, 0x00009f, 0x0000ffff);
+	nv_icmd(priv, 0x0000a0, 0x0000ffff);
+	nv_icmd(priv, 0x0000a1, 0x0000ffff);
+	nv_icmd(priv, 0x0000a2, 0x0000ffff);
+	nv_icmd(priv, 0x0000b1, 0x00000001);
+	nv_icmd(priv, 0x0000ad, 0x0000013e);
+	nv_icmd(priv, 0x0000e1, 0x00000010);
+	nv_icmd(priv, 0x000290, 0x00000000);
+	nv_icmd(priv, 0x000291, 0x00000000);
+	nv_icmd(priv, 0x000292, 0x00000000);
+	nv_icmd(priv, 0x000293, 0x00000000);
+	nv_icmd(priv, 0x000294, 0x00000000);
+	nv_icmd(priv, 0x000295, 0x00000000);
+	nv_icmd(priv, 0x000296, 0x00000000);
+	nv_icmd(priv, 0x000297, 0x00000000);
+	nv_icmd(priv, 0x000298, 0x00000000);
+	nv_icmd(priv, 0x000299, 0x00000000);
+	nv_icmd(priv, 0x00029a, 0x00000000);
+	nv_icmd(priv, 0x00029b, 0x00000000);
+	nv_icmd(priv, 0x00029c, 0x00000000);
+	nv_icmd(priv, 0x00029d, 0x00000000);
+	nv_icmd(priv, 0x00029e, 0x00000000);
+	nv_icmd(priv, 0x00029f, 0x00000000);
+	nv_icmd(priv, 0x0003b0, 0x00000000);
+	nv_icmd(priv, 0x0003b1, 0x00000000);
+	nv_icmd(priv, 0x0003b2, 0x00000000);
+	nv_icmd(priv, 0x0003b3, 0x00000000);
+	nv_icmd(priv, 0x0003b4, 0x00000000);
+	nv_icmd(priv, 0x0003b5, 0x00000000);
+	nv_icmd(priv, 0x0003b6, 0x00000000);
+	nv_icmd(priv, 0x0003b7, 0x00000000);
+	nv_icmd(priv, 0x0003b8, 0x00000000);
+	nv_icmd(priv, 0x0003b9, 0x00000000);
+	nv_icmd(priv, 0x0003ba, 0x00000000);
+	nv_icmd(priv, 0x0003bb, 0x00000000);
+	nv_icmd(priv, 0x0003bc, 0x00000000);
+	nv_icmd(priv, 0x0003bd, 0x00000000);
+	nv_icmd(priv, 0x0003be, 0x00000000);
+	nv_icmd(priv, 0x0003bf, 0x00000000);
+	nv_icmd(priv, 0x0002a0, 0x00000000);
+	nv_icmd(priv, 0x0002a1, 0x00000000);
+	nv_icmd(priv, 0x0002a2, 0x00000000);
+	nv_icmd(priv, 0x0002a3, 0x00000000);
+	nv_icmd(priv, 0x0002a4, 0x00000000);
+	nv_icmd(priv, 0x0002a5, 0x00000000);
+	nv_icmd(priv, 0x0002a6, 0x00000000);
+	nv_icmd(priv, 0x0002a7, 0x00000000);
+	nv_icmd(priv, 0x0002a8, 0x00000000);
+	nv_icmd(priv, 0x0002a9, 0x00000000);
+	nv_icmd(priv, 0x0002aa, 0x00000000);
+	nv_icmd(priv, 0x0002ab, 0x00000000);
+	nv_icmd(priv, 0x0002ac, 0x00000000);
+	nv_icmd(priv, 0x0002ad, 0x00000000);
+	nv_icmd(priv, 0x0002ae, 0x00000000);
+	nv_icmd(priv, 0x0002af, 0x00000000);
+	nv_icmd(priv, 0x000420, 0x00000000);
+	nv_icmd(priv, 0x000421, 0x00000000);
+	nv_icmd(priv, 0x000422, 0x00000000);
+	nv_icmd(priv, 0x000423, 0x00000000);
+	nv_icmd(priv, 0x000424, 0x00000000);
+	nv_icmd(priv, 0x000425, 0x00000000);
+	nv_icmd(priv, 0x000426, 0x00000000);
+	nv_icmd(priv, 0x000427, 0x00000000);
+	nv_icmd(priv, 0x000428, 0x00000000);
+	nv_icmd(priv, 0x000429, 0x00000000);
+	nv_icmd(priv, 0x00042a, 0x00000000);
+	nv_icmd(priv, 0x00042b, 0x00000000);
+	nv_icmd(priv, 0x00042c, 0x00000000);
+	nv_icmd(priv, 0x00042d, 0x00000000);
+	nv_icmd(priv, 0x00042e, 0x00000000);
+	nv_icmd(priv, 0x00042f, 0x00000000);
+	nv_icmd(priv, 0x0002b0, 0x00000000);
+	nv_icmd(priv, 0x0002b1, 0x00000000);
+	nv_icmd(priv, 0x0002b2, 0x00000000);
+	nv_icmd(priv, 0x0002b3, 0x00000000);
+	nv_icmd(priv, 0x0002b4, 0x00000000);
+	nv_icmd(priv, 0x0002b5, 0x00000000);
+	nv_icmd(priv, 0x0002b6, 0x00000000);
+	nv_icmd(priv, 0x0002b7, 0x00000000);
+	nv_icmd(priv, 0x0002b8, 0x00000000);
+	nv_icmd(priv, 0x0002b9, 0x00000000);
+	nv_icmd(priv, 0x0002ba, 0x00000000);
+	nv_icmd(priv, 0x0002bb, 0x00000000);
+	nv_icmd(priv, 0x0002bc, 0x00000000);
+	nv_icmd(priv, 0x0002bd, 0x00000000);
+	nv_icmd(priv, 0x0002be, 0x00000000);
+	nv_icmd(priv, 0x0002bf, 0x00000000);
+	nv_icmd(priv, 0x000430, 0x00000000);
+	nv_icmd(priv, 0x000431, 0x00000000);
+	nv_icmd(priv, 0x000432, 0x00000000);
+	nv_icmd(priv, 0x000433, 0x00000000);
+	nv_icmd(priv, 0x000434, 0x00000000);
+	nv_icmd(priv, 0x000435, 0x00000000);
+	nv_icmd(priv, 0x000436, 0x00000000);
+	nv_icmd(priv, 0x000437, 0x00000000);
+	nv_icmd(priv, 0x000438, 0x00000000);
+	nv_icmd(priv, 0x000439, 0x00000000);
+	nv_icmd(priv, 0x00043a, 0x00000000);
+	nv_icmd(priv, 0x00043b, 0x00000000);
+	nv_icmd(priv, 0x00043c, 0x00000000);
+	nv_icmd(priv, 0x00043d, 0x00000000);
+	nv_icmd(priv, 0x00043e, 0x00000000);
+	nv_icmd(priv, 0x00043f, 0x00000000);
+	nv_icmd(priv, 0x0002c0, 0x00000000);
+	nv_icmd(priv, 0x0002c1, 0x00000000);
+	nv_icmd(priv, 0x0002c2, 0x00000000);
+	nv_icmd(priv, 0x0002c3, 0x00000000);
+	nv_icmd(priv, 0x0002c4, 0x00000000);
+	nv_icmd(priv, 0x0002c5, 0x00000000);
+	nv_icmd(priv, 0x0002c6, 0x00000000);
+	nv_icmd(priv, 0x0002c7, 0x00000000);
+	nv_icmd(priv, 0x0002c8, 0x00000000);
+	nv_icmd(priv, 0x0002c9, 0x00000000);
+	nv_icmd(priv, 0x0002ca, 0x00000000);
+	nv_icmd(priv, 0x0002cb, 0x00000000);
+	nv_icmd(priv, 0x0002cc, 0x00000000);
+	nv_icmd(priv, 0x0002cd, 0x00000000);
+	nv_icmd(priv, 0x0002ce, 0x00000000);
+	nv_icmd(priv, 0x0002cf, 0x00000000);
+	nv_icmd(priv, 0x0004d0, 0x00000000);
+	nv_icmd(priv, 0x0004d1, 0x00000000);
+	nv_icmd(priv, 0x0004d2, 0x00000000);
+	nv_icmd(priv, 0x0004d3, 0x00000000);
+	nv_icmd(priv, 0x0004d4, 0x00000000);
+	nv_icmd(priv, 0x0004d5, 0x00000000);
+	nv_icmd(priv, 0x0004d6, 0x00000000);
+	nv_icmd(priv, 0x0004d7, 0x00000000);
+	nv_icmd(priv, 0x0004d8, 0x00000000);
+	nv_icmd(priv, 0x0004d9, 0x00000000);
+	nv_icmd(priv, 0x0004da, 0x00000000);
+	nv_icmd(priv, 0x0004db, 0x00000000);
+	nv_icmd(priv, 0x0004dc, 0x00000000);
+	nv_icmd(priv, 0x0004dd, 0x00000000);
+	nv_icmd(priv, 0x0004de, 0x00000000);
+	nv_icmd(priv, 0x0004df, 0x00000000);
+	nv_icmd(priv, 0x000720, 0x00000000);
+	nv_icmd(priv, 0x000721, 0x00000000);
+	nv_icmd(priv, 0x000722, 0x00000000);
+	nv_icmd(priv, 0x000723, 0x00000000);
+	nv_icmd(priv, 0x000724, 0x00000000);
+	nv_icmd(priv, 0x000725, 0x00000000);
+	nv_icmd(priv, 0x000726, 0x00000000);
+	nv_icmd(priv, 0x000727, 0x00000000);
+	nv_icmd(priv, 0x000728, 0x00000000);
+	nv_icmd(priv, 0x000729, 0x00000000);
+	nv_icmd(priv, 0x00072a, 0x00000000);
+	nv_icmd(priv, 0x00072b, 0x00000000);
+	nv_icmd(priv, 0x00072c, 0x00000000);
+	nv_icmd(priv, 0x00072d, 0x00000000);
+	nv_icmd(priv, 0x00072e, 0x00000000);
+	nv_icmd(priv, 0x00072f, 0x00000000);
+	nv_icmd(priv, 0x0008c0, 0x00000000);
+	nv_icmd(priv, 0x0008c1, 0x00000000);
+	nv_icmd(priv, 0x0008c2, 0x00000000);
+	nv_icmd(priv, 0x0008c3, 0x00000000);
+	nv_icmd(priv, 0x0008c4, 0x00000000);
+	nv_icmd(priv, 0x0008c5, 0x00000000);
+	nv_icmd(priv, 0x0008c6, 0x00000000);
+	nv_icmd(priv, 0x0008c7, 0x00000000);
+	nv_icmd(priv, 0x0008c8, 0x00000000);
+	nv_icmd(priv, 0x0008c9, 0x00000000);
+	nv_icmd(priv, 0x0008ca, 0x00000000);
+	nv_icmd(priv, 0x0008cb, 0x00000000);
+	nv_icmd(priv, 0x0008cc, 0x00000000);
+	nv_icmd(priv, 0x0008cd, 0x00000000);
+	nv_icmd(priv, 0x0008ce, 0x00000000);
+	nv_icmd(priv, 0x0008cf, 0x00000000);
+	nv_icmd(priv, 0x000890, 0x00000000);
+	nv_icmd(priv, 0x000891, 0x00000000);
+	nv_icmd(priv, 0x000892, 0x00000000);
+	nv_icmd(priv, 0x000893, 0x00000000);
+	nv_icmd(priv, 0x000894, 0x00000000);
+	nv_icmd(priv, 0x000895, 0x00000000);
+	nv_icmd(priv, 0x000896, 0x00000000);
+	nv_icmd(priv, 0x000897, 0x00000000);
+	nv_icmd(priv, 0x000898, 0x00000000);
+	nv_icmd(priv, 0x000899, 0x00000000);
+	nv_icmd(priv, 0x00089a, 0x00000000);
+	nv_icmd(priv, 0x00089b, 0x00000000);
+	nv_icmd(priv, 0x00089c, 0x00000000);
+	nv_icmd(priv, 0x00089d, 0x00000000);
+	nv_icmd(priv, 0x00089e, 0x00000000);
+	nv_icmd(priv, 0x00089f, 0x00000000);
+	nv_icmd(priv, 0x0008e0, 0x00000000);
+	nv_icmd(priv, 0x0008e1, 0x00000000);
+	nv_icmd(priv, 0x0008e2, 0x00000000);
+	nv_icmd(priv, 0x0008e3, 0x00000000);
+	nv_icmd(priv, 0x0008e4, 0x00000000);
+	nv_icmd(priv, 0x0008e5, 0x00000000);
+	nv_icmd(priv, 0x0008e6, 0x00000000);
+	nv_icmd(priv, 0x0008e7, 0x00000000);
+	nv_icmd(priv, 0x0008e8, 0x00000000);
+	nv_icmd(priv, 0x0008e9, 0x00000000);
+	nv_icmd(priv, 0x0008ea, 0x00000000);
+	nv_icmd(priv, 0x0008eb, 0x00000000);
+	nv_icmd(priv, 0x0008ec, 0x00000000);
+	nv_icmd(priv, 0x0008ed, 0x00000000);
+	nv_icmd(priv, 0x0008ee, 0x00000000);
+	nv_icmd(priv, 0x0008ef, 0x00000000);
+	nv_icmd(priv, 0x0008a0, 0x00000000);
+	nv_icmd(priv, 0x0008a1, 0x00000000);
+	nv_icmd(priv, 0x0008a2, 0x00000000);
+	nv_icmd(priv, 0x0008a3, 0x00000000);
+	nv_icmd(priv, 0x0008a4, 0x00000000);
+	nv_icmd(priv, 0x0008a5, 0x00000000);
+	nv_icmd(priv, 0x0008a6, 0x00000000);
+	nv_icmd(priv, 0x0008a7, 0x00000000);
+	nv_icmd(priv, 0x0008a8, 0x00000000);
+	nv_icmd(priv, 0x0008a9, 0x00000000);
+	nv_icmd(priv, 0x0008aa, 0x00000000);
+	nv_icmd(priv, 0x0008ab, 0x00000000);
+	nv_icmd(priv, 0x0008ac, 0x00000000);
+	nv_icmd(priv, 0x0008ad, 0x00000000);
+	nv_icmd(priv, 0x0008ae, 0x00000000);
+	nv_icmd(priv, 0x0008af, 0x00000000);
+	nv_icmd(priv, 0x0008f0, 0x00000000);
+	nv_icmd(priv, 0x0008f1, 0x00000000);
+	nv_icmd(priv, 0x0008f2, 0x00000000);
+	nv_icmd(priv, 0x0008f3, 0x00000000);
+	nv_icmd(priv, 0x0008f4, 0x00000000);
+	nv_icmd(priv, 0x0008f5, 0x00000000);
+	nv_icmd(priv, 0x0008f6, 0x00000000);
+	nv_icmd(priv, 0x0008f7, 0x00000000);
+	nv_icmd(priv, 0x0008f8, 0x00000000);
+	nv_icmd(priv, 0x0008f9, 0x00000000);
+	nv_icmd(priv, 0x0008fa, 0x00000000);
+	nv_icmd(priv, 0x0008fb, 0x00000000);
+	nv_icmd(priv, 0x0008fc, 0x00000000);
+	nv_icmd(priv, 0x0008fd, 0x00000000);
+	nv_icmd(priv, 0x0008fe, 0x00000000);
+	nv_icmd(priv, 0x0008ff, 0x00000000);
+	nv_icmd(priv, 0x00094c, 0x000000ff);
+	nv_icmd(priv, 0x00094d, 0xffffffff);
+	nv_icmd(priv, 0x00094e, 0x00000002);
+	nv_icmd(priv, 0x0002ec, 0x00000001);
+	nv_icmd(priv, 0x000303, 0x00000001);
+	nv_icmd(priv, 0x0002e6, 0x00000001);
+	nv_icmd(priv, 0x000466, 0x00000052);
+	nv_icmd(priv, 0x000301, 0x3f800000);
+	nv_icmd(priv, 0x000304, 0x30201000);
+	nv_icmd(priv, 0x000305, 0x70605040);
+	nv_icmd(priv, 0x000306, 0xb8a89888);
+	nv_icmd(priv, 0x000307, 0xf8e8d8c8);
+	nv_icmd(priv, 0x00030a, 0x00ffff00);
+	nv_icmd(priv, 0x00030b, 0x0000001a);
+	nv_icmd(priv, 0x00030c, 0x00000001);
+	nv_icmd(priv, 0x000318, 0x00000001);
+	nv_icmd(priv, 0x000340, 0x00000000);
+	nv_icmd(priv, 0x000375, 0x00000001);
+	nv_icmd(priv, 0x00037d, 0x00000006);
+	nv_icmd(priv, 0x0003a0, 0x00000002);
+	nv_icmd(priv, 0x0003aa, 0x00000001);
+	nv_icmd(priv, 0x0003a9, 0x00000001);
+	nv_icmd(priv, 0x000380, 0x00000001);
+	nv_icmd(priv, 0x000383, 0x00000011);
+	nv_icmd(priv, 0x000360, 0x00000040);
+	nv_icmd(priv, 0x000366, 0x00000000);
+	nv_icmd(priv, 0x000367, 0x00000000);
+	nv_icmd(priv, 0x000368, 0x00000fff);
+	nv_icmd(priv, 0x000370, 0x00000000);
+	nv_icmd(priv, 0x000371, 0x00000000);
+	nv_icmd(priv, 0x000372, 0x000fffff);
+	nv_icmd(priv, 0x00037a, 0x00000012);
+	nv_icmd(priv, 0x000619, 0x00000003);
+	nv_icmd(priv, 0x000811, 0x00000003);
+	nv_icmd(priv, 0x000812, 0x00000004);
+	nv_icmd(priv, 0x000813, 0x00000006);
+	nv_icmd(priv, 0x000814, 0x00000008);
+	nv_icmd(priv, 0x000815, 0x0000000b);
+	nv_icmd(priv, 0x000800, 0x00000001);
+	nv_icmd(priv, 0x000801, 0x00000001);
+	nv_icmd(priv, 0x000802, 0x00000001);
+	nv_icmd(priv, 0x000803, 0x00000001);
+	nv_icmd(priv, 0x000804, 0x00000001);
+	nv_icmd(priv, 0x000805, 0x00000001);
+	nv_icmd(priv, 0x000632, 0x00000001);
+	nv_icmd(priv, 0x000633, 0x00000002);
+	nv_icmd(priv, 0x000634, 0x00000003);
+	nv_icmd(priv, 0x000635, 0x00000004);
+	nv_icmd(priv, 0x000654, 0x3f800000);
+	nv_icmd(priv, 0x000657, 0x3f800000);
+	nv_icmd(priv, 0x000655, 0x3f800000);
+	nv_icmd(priv, 0x000656, 0x3f800000);
+	nv_icmd(priv, 0x0006cd, 0x3f800000);
+	nv_icmd(priv, 0x0007f5, 0x3f800000);
+	nv_icmd(priv, 0x0007dc, 0x39291909);
+	nv_icmd(priv, 0x0007dd, 0x79695949);
+	nv_icmd(priv, 0x0007de, 0xb9a99989);
+	nv_icmd(priv, 0x0007df, 0xf9e9d9c9);
+	nv_icmd(priv, 0x0007e8, 0x00003210);
+	nv_icmd(priv, 0x0007e9, 0x00007654);
+	nv_icmd(priv, 0x0007ea, 0x00000098);
+	nv_icmd(priv, 0x0007ec, 0x39291909);
+	nv_icmd(priv, 0x0007ed, 0x79695949);
+	nv_icmd(priv, 0x0007ee, 0xb9a99989);
+	nv_icmd(priv, 0x0007ef, 0xf9e9d9c9);
+	nv_icmd(priv, 0x0007f0, 0x00003210);
+	nv_icmd(priv, 0x0007f1, 0x00007654);
+	nv_icmd(priv, 0x0007f2, 0x00000098);
+	nv_icmd(priv, 0x0005a5, 0x00000001);
+	nv_icmd(priv, 0x000980, 0x00000000);
+	nv_icmd(priv, 0x000981, 0x00000000);
+	nv_icmd(priv, 0x000982, 0x00000000);
+	nv_icmd(priv, 0x000983, 0x00000000);
+	nv_icmd(priv, 0x000984, 0x00000000);
+	nv_icmd(priv, 0x000985, 0x00000000);
+	nv_icmd(priv, 0x000986, 0x00000000);
+	nv_icmd(priv, 0x000987, 0x00000000);
+	nv_icmd(priv, 0x000988, 0x00000000);
+	nv_icmd(priv, 0x000989, 0x00000000);
+	nv_icmd(priv, 0x00098a, 0x00000000);
+	nv_icmd(priv, 0x00098b, 0x00000000);
+	nv_icmd(priv, 0x00098c, 0x00000000);
+	nv_icmd(priv, 0x00098d, 0x00000000);
+	nv_icmd(priv, 0x00098e, 0x00000000);
+	nv_icmd(priv, 0x00098f, 0x00000000);
+	nv_icmd(priv, 0x000990, 0x00000000);
+	nv_icmd(priv, 0x000991, 0x00000000);
+	nv_icmd(priv, 0x000992, 0x00000000);
+	nv_icmd(priv, 0x000993, 0x00000000);
+	nv_icmd(priv, 0x000994, 0x00000000);
+	nv_icmd(priv, 0x000995, 0x00000000);
+	nv_icmd(priv, 0x000996, 0x00000000);
+	nv_icmd(priv, 0x000997, 0x00000000);
+	nv_icmd(priv, 0x000998, 0x00000000);
+	nv_icmd(priv, 0x000999, 0x00000000);
+	nv_icmd(priv, 0x00099a, 0x00000000);
+	nv_icmd(priv, 0x00099b, 0x00000000);
+	nv_icmd(priv, 0x00099c, 0x00000000);
+	nv_icmd(priv, 0x00099d, 0x00000000);
+	nv_icmd(priv, 0x00099e, 0x00000000);
+	nv_icmd(priv, 0x00099f, 0x00000000);
+	nv_icmd(priv, 0x0009a0, 0x00000000);
+	nv_icmd(priv, 0x0009a1, 0x00000000);
+	nv_icmd(priv, 0x0009a2, 0x00000000);
+	nv_icmd(priv, 0x0009a3, 0x00000000);
+	nv_icmd(priv, 0x0009a4, 0x00000000);
+	nv_icmd(priv, 0x0009a5, 0x00000000);
+	nv_icmd(priv, 0x0009a6, 0x00000000);
+	nv_icmd(priv, 0x0009a7, 0x00000000);
+	nv_icmd(priv, 0x0009a8, 0x00000000);
+	nv_icmd(priv, 0x0009a9, 0x00000000);
+	nv_icmd(priv, 0x0009aa, 0x00000000);
+	nv_icmd(priv, 0x0009ab, 0x00000000);
+	nv_icmd(priv, 0x0009ac, 0x00000000);
+	nv_icmd(priv, 0x0009ad, 0x00000000);
+	nv_icmd(priv, 0x0009ae, 0x00000000);
+	nv_icmd(priv, 0x0009af, 0x00000000);
+	nv_icmd(priv, 0x0009b0, 0x00000000);
+	nv_icmd(priv, 0x0009b1, 0x00000000);
+	nv_icmd(priv, 0x0009b2, 0x00000000);
+	nv_icmd(priv, 0x0009b3, 0x00000000);
+	nv_icmd(priv, 0x0009b4, 0x00000000);
+	nv_icmd(priv, 0x0009b5, 0x00000000);
+	nv_icmd(priv, 0x0009b6, 0x00000000);
+	nv_icmd(priv, 0x0009b7, 0x00000000);
+	nv_icmd(priv, 0x0009b8, 0x00000000);
+	nv_icmd(priv, 0x0009b9, 0x00000000);
+	nv_icmd(priv, 0x0009ba, 0x00000000);
+	nv_icmd(priv, 0x0009bb, 0x00000000);
+	nv_icmd(priv, 0x0009bc, 0x00000000);
+	nv_icmd(priv, 0x0009bd, 0x00000000);
+	nv_icmd(priv, 0x0009be, 0x00000000);
+	nv_icmd(priv, 0x0009bf, 0x00000000);
+	nv_icmd(priv, 0x0009c0, 0x00000000);
+	nv_icmd(priv, 0x0009c1, 0x00000000);
+	nv_icmd(priv, 0x0009c2, 0x00000000);
+	nv_icmd(priv, 0x0009c3, 0x00000000);
+	nv_icmd(priv, 0x0009c4, 0x00000000);
+	nv_icmd(priv, 0x0009c5, 0x00000000);
+	nv_icmd(priv, 0x0009c6, 0x00000000);
+	nv_icmd(priv, 0x0009c7, 0x00000000);
+	nv_icmd(priv, 0x0009c8, 0x00000000);
+	nv_icmd(priv, 0x0009c9, 0x00000000);
+	nv_icmd(priv, 0x0009ca, 0x00000000);
+	nv_icmd(priv, 0x0009cb, 0x00000000);
+	nv_icmd(priv, 0x0009cc, 0x00000000);
+	nv_icmd(priv, 0x0009cd, 0x00000000);
+	nv_icmd(priv, 0x0009ce, 0x00000000);
+	nv_icmd(priv, 0x0009cf, 0x00000000);
+	nv_icmd(priv, 0x0009d0, 0x00000000);
+	nv_icmd(priv, 0x0009d1, 0x00000000);
+	nv_icmd(priv, 0x0009d2, 0x00000000);
+	nv_icmd(priv, 0x0009d3, 0x00000000);
+	nv_icmd(priv, 0x0009d4, 0x00000000);
+	nv_icmd(priv, 0x0009d5, 0x00000000);
+	nv_icmd(priv, 0x0009d6, 0x00000000);
+	nv_icmd(priv, 0x0009d7, 0x00000000);
+	nv_icmd(priv, 0x0009d8, 0x00000000);
+	nv_icmd(priv, 0x0009d9, 0x00000000);
+	nv_icmd(priv, 0x0009da, 0x00000000);
+	nv_icmd(priv, 0x0009db, 0x00000000);
+	nv_icmd(priv, 0x0009dc, 0x00000000);
+	nv_icmd(priv, 0x0009dd, 0x00000000);
+	nv_icmd(priv, 0x0009de, 0x00000000);
+	nv_icmd(priv, 0x0009df, 0x00000000);
+	nv_icmd(priv, 0x0009e0, 0x00000000);
+	nv_icmd(priv, 0x0009e1, 0x00000000);
+	nv_icmd(priv, 0x0009e2, 0x00000000);
+	nv_icmd(priv, 0x0009e3, 0x00000000);
+	nv_icmd(priv, 0x0009e4, 0x00000000);
+	nv_icmd(priv, 0x0009e5, 0x00000000);
+	nv_icmd(priv, 0x0009e6, 0x00000000);
+	nv_icmd(priv, 0x0009e7, 0x00000000);
+	nv_icmd(priv, 0x0009e8, 0x00000000);
+	nv_icmd(priv, 0x0009e9, 0x00000000);
+	nv_icmd(priv, 0x0009ea, 0x00000000);
+	nv_icmd(priv, 0x0009eb, 0x00000000);
+	nv_icmd(priv, 0x0009ec, 0x00000000);
+	nv_icmd(priv, 0x0009ed, 0x00000000);
+	nv_icmd(priv, 0x0009ee, 0x00000000);
+	nv_icmd(priv, 0x0009ef, 0x00000000);
+	nv_icmd(priv, 0x0009f0, 0x00000000);
+	nv_icmd(priv, 0x0009f1, 0x00000000);
+	nv_icmd(priv, 0x0009f2, 0x00000000);
+	nv_icmd(priv, 0x0009f3, 0x00000000);
+	nv_icmd(priv, 0x0009f4, 0x00000000);
+	nv_icmd(priv, 0x0009f5, 0x00000000);
+	nv_icmd(priv, 0x0009f6, 0x00000000);
+	nv_icmd(priv, 0x0009f7, 0x00000000);
+	nv_icmd(priv, 0x0009f8, 0x00000000);
+	nv_icmd(priv, 0x0009f9, 0x00000000);
+	nv_icmd(priv, 0x0009fa, 0x00000000);
+	nv_icmd(priv, 0x0009fb, 0x00000000);
+	nv_icmd(priv, 0x0009fc, 0x00000000);
+	nv_icmd(priv, 0x0009fd, 0x00000000);
+	nv_icmd(priv, 0x0009fe, 0x00000000);
+	nv_icmd(priv, 0x0009ff, 0x00000000);
+	nv_icmd(priv, 0x000468, 0x00000004);
+	nv_icmd(priv, 0x00046c, 0x00000001);
+	nv_icmd(priv, 0x000470, 0x00000000);
+	nv_icmd(priv, 0x000471, 0x00000000);
+	nv_icmd(priv, 0x000472, 0x00000000);
+	nv_icmd(priv, 0x000473, 0x00000000);
+	nv_icmd(priv, 0x000474, 0x00000000);
+	nv_icmd(priv, 0x000475, 0x00000000);
+	nv_icmd(priv, 0x000476, 0x00000000);
+	nv_icmd(priv, 0x000477, 0x00000000);
+	nv_icmd(priv, 0x000478, 0x00000000);
+	nv_icmd(priv, 0x000479, 0x00000000);
+	nv_icmd(priv, 0x00047a, 0x00000000);
+	nv_icmd(priv, 0x00047b, 0x00000000);
+	nv_icmd(priv, 0x00047c, 0x00000000);
+	nv_icmd(priv, 0x00047d, 0x00000000);
+	nv_icmd(priv, 0x00047e, 0x00000000);
+	nv_icmd(priv, 0x00047f, 0x00000000);
+	nv_icmd(priv, 0x000480, 0x00000000);
+	nv_icmd(priv, 0x000481, 0x00000000);
+	nv_icmd(priv, 0x000482, 0x00000000);
+	nv_icmd(priv, 0x000483, 0x00000000);
+	nv_icmd(priv, 0x000484, 0x00000000);
+	nv_icmd(priv, 0x000485, 0x00000000);
+	nv_icmd(priv, 0x000486, 0x00000000);
+	nv_icmd(priv, 0x000487, 0x00000000);
+	nv_icmd(priv, 0x000488, 0x00000000);
+	nv_icmd(priv, 0x000489, 0x00000000);
+	nv_icmd(priv, 0x00048a, 0x00000000);
+	nv_icmd(priv, 0x00048b, 0x00000000);
+	nv_icmd(priv, 0x00048c, 0x00000000);
+	nv_icmd(priv, 0x00048d, 0x00000000);
+	nv_icmd(priv, 0x00048e, 0x00000000);
+	nv_icmd(priv, 0x00048f, 0x00000000);
+	nv_icmd(priv, 0x000490, 0x00000000);
+	nv_icmd(priv, 0x000491, 0x00000000);
+	nv_icmd(priv, 0x000492, 0x00000000);
+	nv_icmd(priv, 0x000493, 0x00000000);
+	nv_icmd(priv, 0x000494, 0x00000000);
+	nv_icmd(priv, 0x000495, 0x00000000);
+	nv_icmd(priv, 0x000496, 0x00000000);
+	nv_icmd(priv, 0x000497, 0x00000000);
+	nv_icmd(priv, 0x000498, 0x00000000);
+	nv_icmd(priv, 0x000499, 0x00000000);
+	nv_icmd(priv, 0x00049a, 0x00000000);
+	nv_icmd(priv, 0x00049b, 0x00000000);
+	nv_icmd(priv, 0x00049c, 0x00000000);
+	nv_icmd(priv, 0x00049d, 0x00000000);
+	nv_icmd(priv, 0x00049e, 0x00000000);
+	nv_icmd(priv, 0x00049f, 0x00000000);
+	nv_icmd(priv, 0x0004a0, 0x00000000);
+	nv_icmd(priv, 0x0004a1, 0x00000000);
+	nv_icmd(priv, 0x0004a2, 0x00000000);
+	nv_icmd(priv, 0x0004a3, 0x00000000);
+	nv_icmd(priv, 0x0004a4, 0x00000000);
+	nv_icmd(priv, 0x0004a5, 0x00000000);
+	nv_icmd(priv, 0x0004a6, 0x00000000);
+	nv_icmd(priv, 0x0004a7, 0x00000000);
+	nv_icmd(priv, 0x0004a8, 0x00000000);
+	nv_icmd(priv, 0x0004a9, 0x00000000);
+	nv_icmd(priv, 0x0004aa, 0x00000000);
+	nv_icmd(priv, 0x0004ab, 0x00000000);
+	nv_icmd(priv, 0x0004ac, 0x00000000);
+	nv_icmd(priv, 0x0004ad, 0x00000000);
+	nv_icmd(priv, 0x0004ae, 0x00000000);
+	nv_icmd(priv, 0x0004af, 0x00000000);
+	nv_icmd(priv, 0x0004b0, 0x00000000);
+	nv_icmd(priv, 0x0004b1, 0x00000000);
+	nv_icmd(priv, 0x0004b2, 0x00000000);
+	nv_icmd(priv, 0x0004b3, 0x00000000);
+	nv_icmd(priv, 0x0004b4, 0x00000000);
+	nv_icmd(priv, 0x0004b5, 0x00000000);
+	nv_icmd(priv, 0x0004b6, 0x00000000);
+	nv_icmd(priv, 0x0004b7, 0x00000000);
+	nv_icmd(priv, 0x0004b8, 0x00000000);
+	nv_icmd(priv, 0x0004b9, 0x00000000);
+	nv_icmd(priv, 0x0004ba, 0x00000000);
+	nv_icmd(priv, 0x0004bb, 0x00000000);
+	nv_icmd(priv, 0x0004bc, 0x00000000);
+	nv_icmd(priv, 0x0004bd, 0x00000000);
+	nv_icmd(priv, 0x0004be, 0x00000000);
+	nv_icmd(priv, 0x0004bf, 0x00000000);
+	nv_icmd(priv, 0x0004c0, 0x00000000);
+	nv_icmd(priv, 0x0004c1, 0x00000000);
+	nv_icmd(priv, 0x0004c2, 0x00000000);
+	nv_icmd(priv, 0x0004c3, 0x00000000);
+	nv_icmd(priv, 0x0004c4, 0x00000000);
+	nv_icmd(priv, 0x0004c5, 0x00000000);
+	nv_icmd(priv, 0x0004c6, 0x00000000);
+	nv_icmd(priv, 0x0004c7, 0x00000000);
+	nv_icmd(priv, 0x0004c8, 0x00000000);
+	nv_icmd(priv, 0x0004c9, 0x00000000);
+	nv_icmd(priv, 0x0004ca, 0x00000000);
+	nv_icmd(priv, 0x0004cb, 0x00000000);
+	nv_icmd(priv, 0x0004cc, 0x00000000);
+	nv_icmd(priv, 0x0004cd, 0x00000000);
+	nv_icmd(priv, 0x0004ce, 0x00000000);
+	nv_icmd(priv, 0x0004cf, 0x00000000);
+	nv_icmd(priv, 0x000510, 0x3f800000);
+	nv_icmd(priv, 0x000511, 0x3f800000);
+	nv_icmd(priv, 0x000512, 0x3f800000);
+	nv_icmd(priv, 0x000513, 0x3f800000);
+	nv_icmd(priv, 0x000514, 0x3f800000);
+	nv_icmd(priv, 0x000515, 0x3f800000);
+	nv_icmd(priv, 0x000516, 0x3f800000);
+	nv_icmd(priv, 0x000517, 0x3f800000);
+	nv_icmd(priv, 0x000518, 0x3f800000);
+	nv_icmd(priv, 0x000519, 0x3f800000);
+	nv_icmd(priv, 0x00051a, 0x3f800000);
+	nv_icmd(priv, 0x00051b, 0x3f800000);
+	nv_icmd(priv, 0x00051c, 0x3f800000);
+	nv_icmd(priv, 0x00051d, 0x3f800000);
+	nv_icmd(priv, 0x00051e, 0x3f800000);
+	nv_icmd(priv, 0x00051f, 0x3f800000);
+	nv_icmd(priv, 0x000520, 0x000002b6);
+	nv_icmd(priv, 0x000529, 0x00000001);
+	nv_icmd(priv, 0x000530, 0xffff0000);
+	nv_icmd(priv, 0x000531, 0xffff0000);
+	nv_icmd(priv, 0x000532, 0xffff0000);
+	nv_icmd(priv, 0x000533, 0xffff0000);
+	nv_icmd(priv, 0x000534, 0xffff0000);
+	nv_icmd(priv, 0x000535, 0xffff0000);
+	nv_icmd(priv, 0x000536, 0xffff0000);
+	nv_icmd(priv, 0x000537, 0xffff0000);
+	nv_icmd(priv, 0x000538, 0xffff0000);
+	nv_icmd(priv, 0x000539, 0xffff0000);
+	nv_icmd(priv, 0x00053a, 0xffff0000);
+	nv_icmd(priv, 0x00053b, 0xffff0000);
+	nv_icmd(priv, 0x00053c, 0xffff0000);
+	nv_icmd(priv, 0x00053d, 0xffff0000);
+	nv_icmd(priv, 0x00053e, 0xffff0000);
+	nv_icmd(priv, 0x00053f, 0xffff0000);
+	nv_icmd(priv, 0x000585, 0x0000003f);
+	nv_icmd(priv, 0x000576, 0x00000003);
+	nv_icmd(priv, 0x00057b, 0x00000059);
+	nv_icmd(priv, 0x000586, 0x00000040);
+	nv_icmd(priv, 0x000582, 0x00000080);
+	nv_icmd(priv, 0x000583, 0x00000080);
+	nv_icmd(priv, 0x0005c2, 0x00000001);
+	nv_icmd(priv, 0x000638, 0x00000001);
+	nv_icmd(priv, 0x000639, 0x00000001);
+	nv_icmd(priv, 0x00063a, 0x00000002);
+	nv_icmd(priv, 0x00063b, 0x00000001);
+	nv_icmd(priv, 0x00063c, 0x00000001);
+	nv_icmd(priv, 0x00063d, 0x00000002);
+	nv_icmd(priv, 0x00063e, 0x00000001);
+	nv_icmd(priv, 0x0008b8, 0x00000001);
+	nv_icmd(priv, 0x0008b9, 0x00000001);
+	nv_icmd(priv, 0x0008ba, 0x00000001);
+	nv_icmd(priv, 0x0008bb, 0x00000001);
+	nv_icmd(priv, 0x0008bc, 0x00000001);
+	nv_icmd(priv, 0x0008bd, 0x00000001);
+	nv_icmd(priv, 0x0008be, 0x00000001);
+	nv_icmd(priv, 0x0008bf, 0x00000001);
+	nv_icmd(priv, 0x000900, 0x00000001);
+	nv_icmd(priv, 0x000901, 0x00000001);
+	nv_icmd(priv, 0x000902, 0x00000001);
+	nv_icmd(priv, 0x000903, 0x00000001);
+	nv_icmd(priv, 0x000904, 0x00000001);
+	nv_icmd(priv, 0x000905, 0x00000001);
+	nv_icmd(priv, 0x000906, 0x00000001);
+	nv_icmd(priv, 0x000907, 0x00000001);
+	nv_icmd(priv, 0x000908, 0x00000002);
+	nv_icmd(priv, 0x000909, 0x00000002);
+	nv_icmd(priv, 0x00090a, 0x00000002);
+	nv_icmd(priv, 0x00090b, 0x00000002);
+	nv_icmd(priv, 0x00090c, 0x00000002);
+	nv_icmd(priv, 0x00090d, 0x00000002);
+	nv_icmd(priv, 0x00090e, 0x00000002);
+	nv_icmd(priv, 0x00090f, 0x00000002);
+	nv_icmd(priv, 0x000910, 0x00000001);
+	nv_icmd(priv, 0x000911, 0x00000001);
+	nv_icmd(priv, 0x000912, 0x00000001);
+	nv_icmd(priv, 0x000913, 0x00000001);
+	nv_icmd(priv, 0x000914, 0x00000001);
+	nv_icmd(priv, 0x000915, 0x00000001);
+	nv_icmd(priv, 0x000916, 0x00000001);
+	nv_icmd(priv, 0x000917, 0x00000001);
+	nv_icmd(priv, 0x000918, 0x00000001);
+	nv_icmd(priv, 0x000919, 0x00000001);
+	nv_icmd(priv, 0x00091a, 0x00000001);
+	nv_icmd(priv, 0x00091b, 0x00000001);
+	nv_icmd(priv, 0x00091c, 0x00000001);
+	nv_icmd(priv, 0x00091d, 0x00000001);
+	nv_icmd(priv, 0x00091e, 0x00000001);
+	nv_icmd(priv, 0x00091f, 0x00000001);
+	nv_icmd(priv, 0x000920, 0x00000002);
+	nv_icmd(priv, 0x000921, 0x00000002);
+	nv_icmd(priv, 0x000922, 0x00000002);
+	nv_icmd(priv, 0x000923, 0x00000002);
+	nv_icmd(priv, 0x000924, 0x00000002);
+	nv_icmd(priv, 0x000925, 0x00000002);
+	nv_icmd(priv, 0x000926, 0x00000002);
+	nv_icmd(priv, 0x000927, 0x00000002);
+	nv_icmd(priv, 0x000928, 0x00000001);
+	nv_icmd(priv, 0x000929, 0x00000001);
+	nv_icmd(priv, 0x00092a, 0x00000001);
+	nv_icmd(priv, 0x00092b, 0x00000001);
+	nv_icmd(priv, 0x00092c, 0x00000001);
+	nv_icmd(priv, 0x00092d, 0x00000001);
+	nv_icmd(priv, 0x00092e, 0x00000001);
+	nv_icmd(priv, 0x00092f, 0x00000001);
+	nv_icmd(priv, 0x000648, 0x00000001);
+	nv_icmd(priv, 0x000649, 0x00000001);
+	nv_icmd(priv, 0x00064a, 0x00000001);
+	nv_icmd(priv, 0x00064b, 0x00000001);
+	nv_icmd(priv, 0x00064c, 0x00000001);
+	nv_icmd(priv, 0x00064d, 0x00000001);
+	nv_icmd(priv, 0x00064e, 0x00000001);
+	nv_icmd(priv, 0x00064f, 0x00000001);
+	nv_icmd(priv, 0x000650, 0x00000001);
+	nv_icmd(priv, 0x000658, 0x0000000f);
+	nv_icmd(priv, 0x0007ff, 0x0000000a);
+	nv_icmd(priv, 0x00066a, 0x40000000);
+	nv_icmd(priv, 0x00066b, 0x10000000);
+	nv_icmd(priv, 0x00066c, 0xffff0000);
+	nv_icmd(priv, 0x00066d, 0xffff0000);
+	nv_icmd(priv, 0x0007af, 0x00000008);
+	nv_icmd(priv, 0x0007b0, 0x00000008);
+	nv_icmd(priv, 0x0007f6, 0x00000001);
+	nv_icmd(priv, 0x0006b2, 0x00000055);
+	nv_icmd(priv, 0x0007ad, 0x00000003);
+	nv_icmd(priv, 0x000937, 0x00000001);
+	nv_icmd(priv, 0x000971, 0x00000008);
+	nv_icmd(priv, 0x000972, 0x00000040);
+	nv_icmd(priv, 0x000973, 0x0000012c);
+	nv_icmd(priv, 0x00097c, 0x00000040);
+	nv_icmd(priv, 0x000979, 0x00000003);
+	nv_icmd(priv, 0x000975, 0x00000020);
+	nv_icmd(priv, 0x000976, 0x00000001);
+	nv_icmd(priv, 0x000977, 0x00000020);
+	nv_icmd(priv, 0x000978, 0x00000001);
+	nv_icmd(priv, 0x000957, 0x00000003);
+	nv_icmd(priv, 0x00095e, 0x20164010);
+	nv_icmd(priv, 0x00095f, 0x00000020);
+	nv_icmd(priv, 0x00097d, 0x00000020);
+	nv_icmd(priv, 0x000683, 0x00000006);
+	nv_icmd(priv, 0x000685, 0x003fffff);
+	nv_icmd(priv, 0x000687, 0x003fffff);
+	nv_icmd(priv, 0x0006a0, 0x00000005);
+	nv_icmd(priv, 0x000840, 0x00400008);
+	nv_icmd(priv, 0x000841, 0x08000080);
+	nv_icmd(priv, 0x000842, 0x00400008);
+	nv_icmd(priv, 0x000843, 0x08000080);
+	nv_icmd(priv, 0x000818, 0x00000000);
+	nv_icmd(priv, 0x000819, 0x00000000);
+	nv_icmd(priv, 0x00081a, 0x00000000);
+	nv_icmd(priv, 0x00081b, 0x00000000);
+	nv_icmd(priv, 0x00081c, 0x00000000);
+	nv_icmd(priv, 0x00081d, 0x00000000);
+	nv_icmd(priv, 0x00081e, 0x00000000);
+	nv_icmd(priv, 0x00081f, 0x00000000);
+	nv_icmd(priv, 0x000848, 0x00000000);
+	nv_icmd(priv, 0x000849, 0x00000000);
+	nv_icmd(priv, 0x00084a, 0x00000000);
+	nv_icmd(priv, 0x00084b, 0x00000000);
+	nv_icmd(priv, 0x00084c, 0x00000000);
+	nv_icmd(priv, 0x00084d, 0x00000000);
+	nv_icmd(priv, 0x00084e, 0x00000000);
+	nv_icmd(priv, 0x00084f, 0x00000000);
+	nv_icmd(priv, 0x000850, 0x00000000);
+	nv_icmd(priv, 0x000851, 0x00000000);
+	nv_icmd(priv, 0x000852, 0x00000000);
+	nv_icmd(priv, 0x000853, 0x00000000);
+	nv_icmd(priv, 0x000854, 0x00000000);
+	nv_icmd(priv, 0x000855, 0x00000000);
+	nv_icmd(priv, 0x000856, 0x00000000);
+	nv_icmd(priv, 0x000857, 0x00000000);
+	nv_icmd(priv, 0x000738, 0x00000000);
+	nv_icmd(priv, 0x0006aa, 0x00000001);
+	nv_icmd(priv, 0x0006ab, 0x00000002);
+	nv_icmd(priv, 0x0006ac, 0x00000080);
+	nv_icmd(priv, 0x0006ad, 0x00000100);
+	nv_icmd(priv, 0x0006ae, 0x00000100);
+	nv_icmd(priv, 0x0006b1, 0x00000011);
+	nv_icmd(priv, 0x0006bb, 0x000000cf);
+	nv_icmd(priv, 0x0006ce, 0x2a712488);
+	nv_icmd(priv, 0x000739, 0x4085c000);
+	nv_icmd(priv, 0x00073a, 0x00000080);
+	nv_icmd(priv, 0x000786, 0x80000100);
+	nv_icmd(priv, 0x00073c, 0x00010100);
+	nv_icmd(priv, 0x00073d, 0x02800000);
+	nv_icmd(priv, 0x000787, 0x000000cf);
+	nv_icmd(priv, 0x00078c, 0x00000008);
+	nv_icmd(priv, 0x000792, 0x00000001);
+	nv_icmd(priv, 0x000794, 0x00000001);
+	nv_icmd(priv, 0x000795, 0x00000001);
+	nv_icmd(priv, 0x000796, 0x00000001);
+	nv_icmd(priv, 0x000797, 0x000000cf);
+	nv_icmd(priv, 0x000836, 0x00000001);
+	nv_icmd(priv, 0x00079a, 0x00000002);
+	nv_icmd(priv, 0x000833, 0x04444480);
+	nv_icmd(priv, 0x0007a1, 0x00000001);
+	nv_icmd(priv, 0x0007a3, 0x00000001);
+	nv_icmd(priv, 0x0007a4, 0x00000001);
+	nv_icmd(priv, 0x0007a5, 0x00000001);
+	nv_icmd(priv, 0x000831, 0x00000004);
+	nv_icmd(priv, 0x000b07, 0x00000002);
+	nv_icmd(priv, 0x000b08, 0x00000100);
+	nv_icmd(priv, 0x000b09, 0x00000100);
+	nv_icmd(priv, 0x000b0a, 0x00000001);
+	nv_icmd(priv, 0x000a04, 0x000000ff);
+	nv_icmd(priv, 0x000a0b, 0x00000040);
+	nv_icmd(priv, 0x00097f, 0x00000100);
+	nv_icmd(priv, 0x000a02, 0x00000001);
+	nv_icmd(priv, 0x000809, 0x00000007);
+	nv_icmd(priv, 0x00c221, 0x00000040);
+	nv_icmd(priv, 0x00c1b0, 0x0000000f);
+	nv_icmd(priv, 0x00c1b1, 0x0000000f);
+	nv_icmd(priv, 0x00c1b2, 0x0000000f);
+	nv_icmd(priv, 0x00c1b3, 0x0000000f);
+	nv_icmd(priv, 0x00c1b4, 0x0000000f);
+	nv_icmd(priv, 0x00c1b5, 0x0000000f);
+	nv_icmd(priv, 0x00c1b6, 0x0000000f);
+	nv_icmd(priv, 0x00c1b7, 0x0000000f);
+	nv_icmd(priv, 0x00c1b8, 0x0fac6881);
+	nv_icmd(priv, 0x00c1b9, 0x00fac688);
+	nv_icmd(priv, 0x00c401, 0x00000001);
+	nv_icmd(priv, 0x00c402, 0x00010001);
+	nv_icmd(priv, 0x00c403, 0x00000001);
+	nv_icmd(priv, 0x00c404, 0x00000001);
+	nv_icmd(priv, 0x00c40e, 0x00000020);
+	nv_icmd(priv, 0x00c500, 0x00000003);
+	nv_icmd(priv, 0x01e100, 0x00000001);
+	nv_icmd(priv, 0x001000, 0x00000002);
+	nv_icmd(priv, 0x0006aa, 0x00000001);
+	nv_icmd(priv, 0x0006ad, 0x00000100);
+	nv_icmd(priv, 0x0006ae, 0x00000100);
+	nv_icmd(priv, 0x0006b1, 0x00000011);
+	nv_icmd(priv, 0x00078c, 0x00000008);
+	nv_icmd(priv, 0x000792, 0x00000001);
+	nv_icmd(priv, 0x000794, 0x00000001);
+	nv_icmd(priv, 0x000795, 0x00000001);
+	nv_icmd(priv, 0x000796, 0x00000001);
+	nv_icmd(priv, 0x000797, 0x000000cf);
+	nv_icmd(priv, 0x00079a, 0x00000002);
+	nv_icmd(priv, 0x000833, 0x04444480);
+	nv_icmd(priv, 0x0007a1, 0x00000001);
+	nv_icmd(priv, 0x0007a3, 0x00000001);
+	nv_icmd(priv, 0x0007a4, 0x00000001);
+	nv_icmd(priv, 0x0007a5, 0x00000001);
+	nv_icmd(priv, 0x000831, 0x00000004);
+	nv_icmd(priv, 0x01e100, 0x00000001);
+	nv_icmd(priv, 0x001000, 0x00000008);
+	nv_icmd(priv, 0x000039, 0x00000000);
+	nv_icmd(priv, 0x00003a, 0x00000000);
+	nv_icmd(priv, 0x00003b, 0x00000000);
+	nv_icmd(priv, 0x000380, 0x00000001);
+	nv_icmd(priv, 0x000366, 0x00000000);
+	nv_icmd(priv, 0x000367, 0x00000000);
+	nv_icmd(priv, 0x000368, 0x00000fff);
+	nv_icmd(priv, 0x000370, 0x00000000);
+	nv_icmd(priv, 0x000371, 0x00000000);
+	nv_icmd(priv, 0x000372, 0x000fffff);
+	nv_icmd(priv, 0x000813, 0x00000006);
+	nv_icmd(priv, 0x000814, 0x00000008);
+	nv_icmd(priv, 0x000957, 0x00000003);
+	nv_icmd(priv, 0x000818, 0x00000000);
+	nv_icmd(priv, 0x000819, 0x00000000);
+	nv_icmd(priv, 0x00081a, 0x00000000);
+	nv_icmd(priv, 0x00081b, 0x00000000);
+	nv_icmd(priv, 0x00081c, 0x00000000);
+	nv_icmd(priv, 0x00081d, 0x00000000);
+	nv_icmd(priv, 0x00081e, 0x00000000);
+	nv_icmd(priv, 0x00081f, 0x00000000);
+	nv_icmd(priv, 0x000848, 0x00000000);
+	nv_icmd(priv, 0x000849, 0x00000000);
+	nv_icmd(priv, 0x00084a, 0x00000000);
+	nv_icmd(priv, 0x00084b, 0x00000000);
+	nv_icmd(priv, 0x00084c, 0x00000000);
+	nv_icmd(priv, 0x00084d, 0x00000000);
+	nv_icmd(priv, 0x00084e, 0x00000000);
+	nv_icmd(priv, 0x00084f, 0x00000000);
+	nv_icmd(priv, 0x000850, 0x00000000);
+	nv_icmd(priv, 0x000851, 0x00000000);
+	nv_icmd(priv, 0x000852, 0x00000000);
+	nv_icmd(priv, 0x000853, 0x00000000);
+	nv_icmd(priv, 0x000854, 0x00000000);
+	nv_icmd(priv, 0x000855, 0x00000000);
+	nv_icmd(priv, 0x000856, 0x00000000);
+	nv_icmd(priv, 0x000857, 0x00000000);
+	nv_icmd(priv, 0x000738, 0x00000000);
+	nv_icmd(priv, 0x000b07, 0x00000002);
+	nv_icmd(priv, 0x000b08, 0x00000100);
+	nv_icmd(priv, 0x000b09, 0x00000100);
+	nv_icmd(priv, 0x000b0a, 0x00000001);
+	nv_icmd(priv, 0x000a04, 0x000000ff);
+	nv_icmd(priv, 0x00097f, 0x00000100);
+	nv_icmd(priv, 0x000a02, 0x00000001);
+	nv_icmd(priv, 0x000809, 0x00000007);
+	nv_icmd(priv, 0x00c221, 0x00000040);
+	nv_icmd(priv, 0x00c401, 0x00000001);
+	nv_icmd(priv, 0x00c402, 0x00010001);
+	nv_icmd(priv, 0x00c403, 0x00000001);
+	nv_icmd(priv, 0x00c404, 0x00000001);
+	nv_icmd(priv, 0x00c40e, 0x00000020);
+	nv_icmd(priv, 0x00c500, 0x00000003);
+	nv_icmd(priv, 0x01e100, 0x00000001);
+	nv_icmd(priv, 0x001000, 0x00000001);
+	nv_icmd(priv, 0x000b07, 0x00000002);
+	nv_icmd(priv, 0x000b08, 0x00000100);
+	nv_icmd(priv, 0x000b09, 0x00000100);
+	nv_icmd(priv, 0x000b0a, 0x00000001);
+	nv_icmd(priv, 0x01e100, 0x00000001);
+	nv_wr32(priv, 0x400208, 0x00000000);
+}
+
+static void
+nve0_grctx_generate_a097(struct nvc0_graph_priv *priv)
+{
+	nv_mthd(priv, 0xa097, 0x0800, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0840, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0880, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0900, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0940, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0980, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0804, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0844, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0884, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0904, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0944, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0984, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0808, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x0848, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x0888, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x08c8, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x0908, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x0948, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x0988, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x09c8, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x080c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x084c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x088c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x08cc, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x090c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x094c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x098c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x09cc, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x0810, 0x000000cf);
+	nv_mthd(priv, 0xa097, 0x0850, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0890, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0910, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0950, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0990, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0814, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0854, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0894, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x08d4, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0914, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0954, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0994, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x09d4, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0818, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0858, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0898, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x08d8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0918, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0958, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0998, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x09d8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x081c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x085c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x089c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x091c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x095c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x099c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0820, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0860, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x08e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0920, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0960, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x09e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ca0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ce0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cf0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ca4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cb4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cd4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ce4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cf4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c18, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c38, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c78, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c98, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ca8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cb8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cd8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ce8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cf8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c1c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c2c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c3c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c5c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c6c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c7c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1c9c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cbc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ccc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cdc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1cfc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1da0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1db0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1de0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1df0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1da4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1db4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dd4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1de4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1df4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d18, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d38, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d78, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d98, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1da8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1db8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dd8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1de8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1df8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d1c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d2c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d3c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d5c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d6c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d7c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1d9c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dbc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dcc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ddc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1dfc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f18, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f38, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f78, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f1c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f2c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f3c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f5c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f6c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f7c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f98, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fa0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fa8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fb8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fd8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fe0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fe8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ff0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ff8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1f9c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fa4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fb4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fbc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fcc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fd4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fdc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fe4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1fec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ff4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1ffc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2000, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2040, 0x00000011);
+	nv_mthd(priv, 0xa097, 0x2080, 0x00000020);
+	nv_mthd(priv, 0xa097, 0x20c0, 0x00000030);
+	nv_mthd(priv, 0xa097, 0x2100, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x2140, 0x00000051);
+	nv_mthd(priv, 0xa097, 0x200c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x204c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x208c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x20cc, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x210c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x214c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x2010, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2050, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2090, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x20d0, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x2110, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x2150, 0x00000004);
+	nv_mthd(priv, 0xa097, 0x0380, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0384, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0388, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x038c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x03ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0700, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0710, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0720, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0730, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0704, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0714, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0724, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0734, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0708, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0718, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0728, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0738, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2800, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2804, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2808, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x280c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2810, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2814, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2818, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x281c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2820, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2824, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2828, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x282c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2830, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2834, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2838, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x283c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2840, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2844, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2848, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x284c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2850, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2854, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2858, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x285c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2860, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2864, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2868, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x286c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2870, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2874, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2878, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x287c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2880, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2884, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2888, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x288c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2890, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2894, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2898, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x289c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28b0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28b4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28d4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x28fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2900, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2904, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2908, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x290c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2910, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2914, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2918, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x291c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2920, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2924, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2928, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x292c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2930, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2934, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2938, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x293c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2940, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2944, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2948, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x294c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2950, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2954, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2958, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x295c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2960, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2964, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2968, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x296c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2970, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2974, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2978, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x297c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2980, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2984, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2988, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x298c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2990, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2994, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2998, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x299c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29b0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29b4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29d4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x29fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0aa0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ac0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ae0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ba0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0be0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0aa4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ac4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ae4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ba4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0be4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0aa8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ac8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ae8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ba8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0be8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a2c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a6c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0aac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0acc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0aec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b2c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b6c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bcc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ab0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ad0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0af0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bf0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0a94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ab4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ad4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0af4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0b94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bb4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bd4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0bf4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ca0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ce0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cf0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c24, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c34, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c64, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c94, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ca4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cb4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cd4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ce4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cf4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c18, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c28, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c38, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c68, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c78, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c98, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ca8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cb8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cd8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ce8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0cf8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0c0c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c1c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c2c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c3c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c4c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c5c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c6c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c7c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c8c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0c9c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0cac, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0cbc, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0ccc, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0cdc, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0cec, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0cfc, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0d00, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d08, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d10, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d18, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d20, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d28, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d30, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d38, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d04, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d0c, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d14, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d1c, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d24, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d2c, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d34, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d3c, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e00, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e20, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e30, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e60, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e70, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ea0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0eb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ec0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ed0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ee0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ef0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0e04, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e14, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e24, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e34, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e44, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e54, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e64, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e74, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e84, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e94, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ea4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0eb4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ec4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ed4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ee4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ef4, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e08, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e18, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e28, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e38, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e48, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e58, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e68, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e78, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e88, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0e98, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ea8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0eb8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ec8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ed8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ee8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0ef8, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d40, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d48, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d50, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d44, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d4c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d5c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1e00, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e20, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e40, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e60, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e80, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ea0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ec0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ee0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e04, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e24, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e44, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e64, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e84, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ea4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ec4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ee4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e08, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e28, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e48, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e68, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e88, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1ea8, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1ec8, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1ee8, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e0c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e2c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e4c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e6c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e8c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1eac, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ecc, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1eec, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e10, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e30, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e50, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e70, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e90, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1eb0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ed0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ef0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e14, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e34, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e54, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e74, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e94, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1eb4, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1ed4, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1ef4, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1e18, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e38, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e58, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e78, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1e98, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1eb8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ed8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1ef8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x3400, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3404, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3408, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x340c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3410, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3414, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3418, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x341c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3420, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3424, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3428, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x342c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3430, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3434, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3438, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x343c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3440, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3444, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3448, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x344c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3450, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3454, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3458, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x345c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3460, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3464, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3468, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x346c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3470, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3474, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3478, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x347c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3480, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3484, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3488, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x348c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3490, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3494, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3498, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x349c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34b0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34b4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34d4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x34fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3500, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3504, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3508, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x350c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3510, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3514, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3518, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x351c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3520, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3524, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3528, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x352c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3530, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3534, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3538, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x353c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3540, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3544, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3548, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x354c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3550, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3554, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3558, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x355c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3560, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3564, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3568, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x356c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3570, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3574, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3578, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x357c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3580, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3584, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3588, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x358c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3590, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3594, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x3598, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x359c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35b0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35b4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35d4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x35fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x030c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1944, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1514, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d68, 0x0000ffff);
+	nv_mthd(priv, 0xa097, 0x121c, 0x0fac6881);
+	nv_mthd(priv, 0xa097, 0x0fac, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1538, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0fe0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fe4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fe8, 0x00000014);
+	nv_mthd(priv, 0xa097, 0x0fec, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x0ff0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x179c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1228, 0x00000400);
+	nv_mthd(priv, 0xa097, 0x122c, 0x00000300);
+	nv_mthd(priv, 0xa097, 0x1230, 0x00010001);
+	nv_mthd(priv, 0xa097, 0x07f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15b4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x15cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1534, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fb0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x153c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x16b4, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x0fbc, 0x0000ffff);
+	nv_mthd(priv, 0xa097, 0x0fc0, 0x0000ffff);
+	nv_mthd(priv, 0xa097, 0x0fc4, 0x0000ffff);
+	nv_mthd(priv, 0xa097, 0x0fc8, 0x0000ffff);
+	nv_mthd(priv, 0xa097, 0x0df8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0dfc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1948, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1970, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x161c, 0x000009f0);
+	nv_mthd(priv, 0xa097, 0x0dcc, 0x00000010);
+	nv_mthd(priv, 0xa097, 0x163c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1160, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1164, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1168, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x116c, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1170, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1174, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1178, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x117c, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1180, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1184, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1188, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x118c, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1190, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1194, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1198, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x119c, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11a0, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11a4, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11a8, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11ac, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11b0, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11b4, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11b8, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11bc, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11c0, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11c4, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11c8, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11cc, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11d0, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11d4, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11d8, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x11dc, 0x25e00040);
+	nv_mthd(priv, 0xa097, 0x1880, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1884, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1888, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x188c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1890, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1894, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1898, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x189c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18b0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18b4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18d0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18d4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18e0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x18fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x17c8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x17cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x17d0, 0x000000ff);
+	nv_mthd(priv, 0xa097, 0x17d4, 0xffffffff);
+	nv_mthd(priv, 0xa097, 0x17d8, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x17dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15f8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1434, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1438, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d74, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0dec, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x13a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1318, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1644, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0748, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0de8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1648, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1120, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1124, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1128, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x112c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1118, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x164c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1658, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1910, 0x00000290);
+	nv_mthd(priv, 0xa097, 0x1518, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x165c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1520, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1604, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1570, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x13b0, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x13b4, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x020c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1670, 0x30201000);
+	nv_mthd(priv, 0xa097, 0x1674, 0x70605040);
+	nv_mthd(priv, 0xa097, 0x1678, 0xb8a89888);
+	nv_mthd(priv, 0xa097, 0x167c, 0xf8e8d8c8);
+	nv_mthd(priv, 0xa097, 0x166c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1680, 0x00ffff00);
+	nv_mthd(priv, 0xa097, 0x12d0, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x12d4, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1684, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1688, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0dac, 0x00001b02);
+	nv_mthd(priv, 0xa097, 0x0db0, 0x00001b02);
+	nv_mthd(priv, 0xa097, 0x0db4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x168c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x15bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x156c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x187c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1110, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0dc0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0dc4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0dc8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1234, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1690, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12ac, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0790, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0794, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0798, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x079c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07a0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x077c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1000, 0x00000010);
+	nv_mthd(priv, 0xa097, 0x10fc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1290, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0218, 0x00000010);
+	nv_mthd(priv, 0xa097, 0x12d8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12dc, 0x00000010);
+	nv_mthd(priv, 0xa097, 0x0d94, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x155c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1560, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1564, 0x00000fff);
+	nv_mthd(priv, 0xa097, 0x1574, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1578, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x157c, 0x000fffff);
+	nv_mthd(priv, 0xa097, 0x1354, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1610, 0x00000012);
+	nv_mthd(priv, 0xa097, 0x1608, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x160c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x260c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x162c, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x0210, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0320, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0324, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0328, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x032c, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0330, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0334, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0338, 0x3f800000);
+	nv_mthd(priv, 0xa097, 0x0750, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0760, 0x39291909);
+	nv_mthd(priv, 0xa097, 0x0764, 0x79695949);
+	nv_mthd(priv, 0xa097, 0x0768, 0xb9a99989);
+	nv_mthd(priv, 0xa097, 0x076c, 0xf9e9d9c9);
+	nv_mthd(priv, 0xa097, 0x0770, 0x30201000);
+	nv_mthd(priv, 0xa097, 0x0774, 0x70605040);
+	nv_mthd(priv, 0xa097, 0x0778, 0x00009080);
+	nv_mthd(priv, 0xa097, 0x0780, 0x39291909);
+	nv_mthd(priv, 0xa097, 0x0784, 0x79695949);
+	nv_mthd(priv, 0xa097, 0x0788, 0xb9a99989);
+	nv_mthd(priv, 0xa097, 0x078c, 0xf9e9d9c9);
+	nv_mthd(priv, 0xa097, 0x07d0, 0x30201000);
+	nv_mthd(priv, 0xa097, 0x07d4, 0x70605040);
+	nv_mthd(priv, 0xa097, 0x07d8, 0x00009080);
+	nv_mthd(priv, 0xa097, 0x037c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0740, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0744, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x2600, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1918, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x191c, 0x00000900);
+	nv_mthd(priv, 0xa097, 0x1920, 0x00000405);
+	nv_mthd(priv, 0xa097, 0x1308, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1924, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x13ac, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x192c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x193c, 0x00002c1c);
+	nv_mthd(priv, 0xa097, 0x0d7c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x02c0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1510, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1940, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ff4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0ff8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x194c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1950, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1968, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1590, 0x0000003f);
+	nv_mthd(priv, 0xa097, 0x07e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07f0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07f4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x196c, 0x00000011);
+	nv_mthd(priv, 0xa097, 0x02e4, 0x0000b001);
+	nv_mthd(priv, 0xa097, 0x036c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0370, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x197c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fcc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fd0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x02d8, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x1980, 0x00000080);
+	nv_mthd(priv, 0xa097, 0x1504, 0x00000080);
+	nv_mthd(priv, 0xa097, 0x1984, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0300, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x13a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12ec, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1310, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1314, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1380, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1384, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1388, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x138c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1390, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1394, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x139c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1398, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1594, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1598, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x159c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x15a0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x15a4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0f54, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f58, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f5c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x19bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f9c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0fa0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12cc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x12e8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x130c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1360, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1364, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1368, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x136c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1370, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1374, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1378, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x137c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x133c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1340, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1344, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1348, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x134c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1350, 0x00000002);
+	nv_mthd(priv, 0xa097, 0x1358, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x12e4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x131c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1320, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1324, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1328, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x19c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1140, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x19c4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x19c8, 0x00001500);
+	nv_mthd(priv, 0xa097, 0x135c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x19e0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19e4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19e8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19ec, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19f0, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19f4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19f8, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19fc, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x19cc, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x15b8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a00, 0x00001111);
+	nv_mthd(priv, 0xa097, 0x1a04, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a08, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a0c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a10, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a14, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a18, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1a1c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d6c, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x0d70, 0xffff0000);
+	nv_mthd(priv, 0xa097, 0x10f8, 0x00001010);
+	nv_mthd(priv, 0xa097, 0x0d80, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d84, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d88, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d8c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0d90, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0da0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07a4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x07a8, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1508, 0x80000000);
+	nv_mthd(priv, 0xa097, 0x150c, 0x40000000);
+	nv_mthd(priv, 0xa097, 0x1668, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0318, 0x00000008);
+	nv_mthd(priv, 0xa097, 0x031c, 0x00000008);
+	nv_mthd(priv, 0xa097, 0x0d9c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x0374, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0378, 0x00000020);
+	nv_mthd(priv, 0xa097, 0x07dc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x074c, 0x00000055);
+	nv_mthd(priv, 0xa097, 0x1420, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x17bc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x17c0, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x17c4, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1008, 0x00000008);
+	nv_mthd(priv, 0xa097, 0x100c, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x1010, 0x0000012c);
+	nv_mthd(priv, 0xa097, 0x0d60, 0x00000040);
+	nv_mthd(priv, 0xa097, 0x075c, 0x00000003);
+	nv_mthd(priv, 0xa097, 0x1018, 0x00000020);
+	nv_mthd(priv, 0xa097, 0x101c, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1020, 0x00000020);
+	nv_mthd(priv, 0xa097, 0x1024, 0x00000001);
+	nv_mthd(priv, 0xa097, 0x1444, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x1448, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x144c, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0360, 0x20164010);
+	nv_mthd(priv, 0xa097, 0x0364, 0x00000020);
+	nv_mthd(priv, 0xa097, 0x0368, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0de4, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0204, 0x00000006);
+	nv_mthd(priv, 0xa097, 0x0208, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x02cc, 0x003fffff);
+	nv_mthd(priv, 0xa097, 0x02d0, 0x003fffff);
+	nv_mthd(priv, 0xa097, 0x1220, 0x00000005);
+	nv_mthd(priv, 0xa097, 0x0fdc, 0x00000000);
+	nv_mthd(priv, 0xa097, 0x0f98, 0x00400008);
+	nv_mthd(priv, 0xa097, 0x1284, 0x08000080);
+	nv_mthd(priv, 0xa097, 0x1450, 0x00400008);
+	nv_mthd(priv, 0xa097, 0x1454, 0x08000080);
+	nv_mthd(priv, 0xa097, 0x0214, 0x00000000);
+}
+
+static void
+nve0_grctx_generate_902d(struct nvc0_graph_priv *priv)
+{
+	nv_mthd(priv, 0x902d, 0x0200, 0x000000cf);
+	nv_mthd(priv, 0x902d, 0x0204, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0208, 0x00000020);
+	nv_mthd(priv, 0x902d, 0x020c, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0210, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0214, 0x00000080);
+	nv_mthd(priv, 0x902d, 0x0218, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x021c, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x0220, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0224, 0x00000000);
+	nv_mthd(priv, 0x902d, 0x0230, 0x000000cf);
+	nv_mthd(priv, 0x902d, 0x0234, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0238, 0x00000020);
+	nv_mthd(priv, 0x902d, 0x023c, 0x00000001);
+	nv_mthd(priv, 0x902d, 0x0244, 0x00000080);
+	nv_mthd(priv, 0x902d, 0x0248, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x024c, 0x00000100);
+	nv_mthd(priv, 0x902d, 0x3410, 0x00000000);
+}
+
+static void
+nve0_graph_generate_unk40xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404010, 0x0);
+	nv_wr32(priv, 0x404014, 0x0);
+	nv_wr32(priv, 0x404018, 0x0);
+	nv_wr32(priv, 0x40401c, 0x0);
+	nv_wr32(priv, 0x404020, 0x0);
+	nv_wr32(priv, 0x404024, 0xe000);
+	nv_wr32(priv, 0x404028, 0x0);
+	nv_wr32(priv, 0x4040a8, 0x0);
+	nv_wr32(priv, 0x4040ac, 0x0);
+	nv_wr32(priv, 0x4040b0, 0x0);
+	nv_wr32(priv, 0x4040b4, 0x0);
+	nv_wr32(priv, 0x4040b8, 0x0);
+	nv_wr32(priv, 0x4040bc, 0x0);
+	nv_wr32(priv, 0x4040c0, 0x0);
+	nv_wr32(priv, 0x4040c4, 0x0);
+	nv_wr32(priv, 0x4040c8, 0xf800008f);
+	nv_wr32(priv, 0x4040d0, 0x0);
+	nv_wr32(priv, 0x4040d4, 0x0);
+	nv_wr32(priv, 0x4040d8, 0x0);
+	nv_wr32(priv, 0x4040dc, 0x0);
+	nv_wr32(priv, 0x4040e0, 0x0);
+	nv_wr32(priv, 0x4040e4, 0x0);
+	nv_wr32(priv, 0x4040e8, 0x1000);
+	nv_wr32(priv, 0x4040f8, 0x0);
+	nv_wr32(priv, 0x404130, 0x0);
+	nv_wr32(priv, 0x404134, 0x0);
+	nv_wr32(priv, 0x404138, 0x20000040);
+	nv_wr32(priv, 0x404150, 0x2e);
+	nv_wr32(priv, 0x404154, 0x400);
+	nv_wr32(priv, 0x404158, 0x200);
+	nv_wr32(priv, 0x404164, 0x55);
+	nv_wr32(priv, 0x4041a0, 0x0);
+	nv_wr32(priv, 0x4041a4, 0x0);
+	nv_wr32(priv, 0x4041a8, 0x0);
+	nv_wr32(priv, 0x4041ac, 0x0);
+	nv_wr32(priv, 0x404200, 0x0);
+	nv_wr32(priv, 0x404204, 0x0);
+	nv_wr32(priv, 0x404208, 0x0);
+	nv_wr32(priv, 0x40420c, 0x0);
+}
+
+static void
+nve0_graph_generate_unk44xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404404, 0x0);
+	nv_wr32(priv, 0x404408, 0x0);
+	nv_wr32(priv, 0x40440c, 0x0);
+	nv_wr32(priv, 0x404410, 0x0);
+	nv_wr32(priv, 0x404414, 0x0);
+	nv_wr32(priv, 0x404418, 0x0);
+	nv_wr32(priv, 0x40441c, 0x0);
+	nv_wr32(priv, 0x404420, 0x0);
+	nv_wr32(priv, 0x404424, 0x0);
+	nv_wr32(priv, 0x404428, 0x0);
+	nv_wr32(priv, 0x40442c, 0x0);
+	nv_wr32(priv, 0x404430, 0x0);
+	nv_wr32(priv, 0x404434, 0x0);
+	nv_wr32(priv, 0x404438, 0x0);
+	nv_wr32(priv, 0x404460, 0x0);
+	nv_wr32(priv, 0x404464, 0x0);
+	nv_wr32(priv, 0x404468, 0xffffff);
+	nv_wr32(priv, 0x40446c, 0x0);
+	nv_wr32(priv, 0x404480, 0x1);
+	nv_wr32(priv, 0x404498, 0x1);
+}
+
+static void
+nve0_graph_generate_unk46xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404604, 0x14);
+	nv_wr32(priv, 0x404608, 0x0);
+	nv_wr32(priv, 0x40460c, 0x3fff);
+	nv_wr32(priv, 0x404610, 0x100);
+	nv_wr32(priv, 0x404618, 0x0);
+	nv_wr32(priv, 0x40461c, 0x0);
+	nv_wr32(priv, 0x404620, 0x0);
+	nv_wr32(priv, 0x404624, 0x0);
+	nv_wr32(priv, 0x40462c, 0x0);
+	nv_wr32(priv, 0x404630, 0x0);
+	nv_wr32(priv, 0x404640, 0x0);
+	nv_wr32(priv, 0x404654, 0x0);
+	nv_wr32(priv, 0x404660, 0x0);
+	nv_wr32(priv, 0x404678, 0x0);
+	nv_wr32(priv, 0x40467c, 0x2);
+	nv_wr32(priv, 0x404680, 0x0);
+	nv_wr32(priv, 0x404684, 0x0);
+	nv_wr32(priv, 0x404688, 0x0);
+	nv_wr32(priv, 0x40468c, 0x0);
+	nv_wr32(priv, 0x404690, 0x0);
+	nv_wr32(priv, 0x404694, 0x0);
+	nv_wr32(priv, 0x404698, 0x0);
+	nv_wr32(priv, 0x40469c, 0x0);
+	nv_wr32(priv, 0x4046a0, 0x7f0080);
+	nv_wr32(priv, 0x4046a4, 0x0);
+	nv_wr32(priv, 0x4046a8, 0x0);
+	nv_wr32(priv, 0x4046ac, 0x0);
+	nv_wr32(priv, 0x4046b0, 0x0);
+	nv_wr32(priv, 0x4046b4, 0x0);
+	nv_wr32(priv, 0x4046b8, 0x0);
+	nv_wr32(priv, 0x4046bc, 0x0);
+	nv_wr32(priv, 0x4046c0, 0x0);
+	nv_wr32(priv, 0x4046c8, 0x0);
+	nv_wr32(priv, 0x4046cc, 0x0);
+	nv_wr32(priv, 0x4046d0, 0x0);
+}
+
+static void
+nve0_graph_generate_unk47xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x404700, 0x0);
+	nv_wr32(priv, 0x404704, 0x0);
+	nv_wr32(priv, 0x404708, 0x0);
+	nv_wr32(priv, 0x404718, 0x0);
+	nv_wr32(priv, 0x40471c, 0x0);
+	nv_wr32(priv, 0x404720, 0x0);
+	nv_wr32(priv, 0x404724, 0x0);
+	nv_wr32(priv, 0x404728, 0x0);
+	nv_wr32(priv, 0x40472c, 0x0);
+	nv_wr32(priv, 0x404730, 0x0);
+	nv_wr32(priv, 0x404734, 0x100);
+	nv_wr32(priv, 0x404738, 0x0);
+	nv_wr32(priv, 0x40473c, 0x0);
+	nv_wr32(priv, 0x404744, 0x0);
+	nv_wr32(priv, 0x404748, 0x0);
+	nv_wr32(priv, 0x404754, 0x0);
+}
+
+static void
+nve0_graph_generate_unk58xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x405800, 0xf8000bf);
+	nv_wr32(priv, 0x405830, 0x2180648);
+	nv_wr32(priv, 0x405834, 0x8000000);
+	nv_wr32(priv, 0x405838, 0x0);
+	nv_wr32(priv, 0x405854, 0x0);
+	nv_wr32(priv, 0x405870, 0x1);
+	nv_wr32(priv, 0x405874, 0x1);
+	nv_wr32(priv, 0x405878, 0x1);
+	nv_wr32(priv, 0x40587c, 0x1);
+	nv_wr32(priv, 0x405a00, 0x0);
+	nv_wr32(priv, 0x405a04, 0x0);
+	nv_wr32(priv, 0x405a18, 0x0);
+	nv_wr32(priv, 0x405b00, 0x0);
+	nv_wr32(priv, 0x405b10, 0x1000);
+}
+
+static void
+nve0_graph_generate_unk60xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x406020, 0x4103c1);
+	nv_wr32(priv, 0x406028, 0x1);
+	nv_wr32(priv, 0x40602c, 0x1);
+	nv_wr32(priv, 0x406030, 0x1);
+	nv_wr32(priv, 0x406034, 0x1);
+}
+
+static void
+nve0_graph_generate_unk64xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x4064a8, 0x0);
+	nv_wr32(priv, 0x4064ac, 0x3fff);
+	nv_wr32(priv, 0x4064b4, 0x0);
+	nv_wr32(priv, 0x4064b8, 0x0);
+	nv_wr32(priv, 0x4064c0, 0x801a00f0);
+	nv_wr32(priv, 0x4064c4, 0x192ffff);
+	nv_wr32(priv, 0x4064c8, 0x1800600);
+	nv_wr32(priv, 0x4064cc, 0x0);
+	nv_wr32(priv, 0x4064d0, 0x0);
+	nv_wr32(priv, 0x4064d4, 0x0);
+	nv_wr32(priv, 0x4064d8, 0x0);
+	nv_wr32(priv, 0x4064dc, 0x0);
+	nv_wr32(priv, 0x4064e0, 0x0);
+	nv_wr32(priv, 0x4064e4, 0x0);
+	nv_wr32(priv, 0x4064e8, 0x0);
+	nv_wr32(priv, 0x4064ec, 0x0);
+	nv_wr32(priv, 0x4064fc, 0x22a);
+}
+
+static void
+nve0_graph_generate_unk70xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x407040, 0x0);
+}
+
+static void
+nve0_graph_generate_unk78xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x407804, 0x23);
+	nv_wr32(priv, 0x40780c, 0xa418820);
+	nv_wr32(priv, 0x407810, 0x62080e6);
+	nv_wr32(priv, 0x407814, 0x20398a4);
+	nv_wr32(priv, 0x407818, 0xe629062);
+	nv_wr32(priv, 0x40781c, 0xa418820);
+	nv_wr32(priv, 0x407820, 0xe6);
+	nv_wr32(priv, 0x4078bc, 0x103);
+}
+
+static void
+nve0_graph_generate_unk80xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x408000, 0x0);
+	nv_wr32(priv, 0x408004, 0x0);
+	nv_wr32(priv, 0x408008, 0x30);
+	nv_wr32(priv, 0x40800c, 0x0);
+	nv_wr32(priv, 0x408010, 0x0);
+	nv_wr32(priv, 0x408014, 0x69);
+	nv_wr32(priv, 0x408018, 0xe100e100);
+	nv_wr32(priv, 0x408064, 0x0);
+}
+
+static void
+nve0_graph_generate_unk88xx(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x408800, 0x2802a3c);
+	nv_wr32(priv, 0x408804, 0x40);
+	nv_wr32(priv, 0x408808, 0x1043e005);
+	nv_wr32(priv, 0x408840, 0xb);
+	nv_wr32(priv, 0x408900, 0x3080b801);
+	nv_wr32(priv, 0x408904, 0x62000001);
+	nv_wr32(priv, 0x408908, 0xc8102f);
+	nv_wr32(priv, 0x408980, 0x11d);
+}
+
+static void
+nve0_graph_generate_gpc(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x418380, 0x16);
+	nv_wr32(priv, 0x418400, 0x38004e00);
+	nv_wr32(priv, 0x418404, 0x71e0ffff);
+	nv_wr32(priv, 0x41840c, 0x1008);
+	nv_wr32(priv, 0x418410, 0xfff0fff);
+	nv_wr32(priv, 0x418414, 0x2200fff);
+	nv_wr32(priv, 0x418450, 0x0);
+	nv_wr32(priv, 0x418454, 0x0);
+	nv_wr32(priv, 0x418458, 0x0);
+	nv_wr32(priv, 0x41845c, 0x0);
+	nv_wr32(priv, 0x418460, 0x0);
+	nv_wr32(priv, 0x418464, 0x0);
+	nv_wr32(priv, 0x418468, 0x1);
+	nv_wr32(priv, 0x41846c, 0x0);
+	nv_wr32(priv, 0x418470, 0x0);
+	nv_wr32(priv, 0x418600, 0x1f);
+	nv_wr32(priv, 0x418684, 0xf);
+	nv_wr32(priv, 0x418700, 0x2);
+	nv_wr32(priv, 0x418704, 0x80);
+	nv_wr32(priv, 0x418708, 0x0);
+	nv_wr32(priv, 0x41870c, 0x0);
+	nv_wr32(priv, 0x418710, 0x0);
+	nv_wr32(priv, 0x418800, 0x7006860a);
+	nv_wr32(priv, 0x418808, 0x0);
+	nv_wr32(priv, 0x41880c, 0x0);
+	nv_wr32(priv, 0x418810, 0x0);
+	nv_wr32(priv, 0x418828, 0x44);
+	nv_wr32(priv, 0x418830, 0x10000001);
+	nv_wr32(priv, 0x4188d8, 0x8);
+	nv_wr32(priv, 0x4188e0, 0x1000000);
+	nv_wr32(priv, 0x4188e8, 0x0);
+	nv_wr32(priv, 0x4188ec, 0x0);
+	nv_wr32(priv, 0x4188f0, 0x0);
+	nv_wr32(priv, 0x4188f4, 0x0);
+	nv_wr32(priv, 0x4188f8, 0x0);
+	nv_wr32(priv, 0x4188fc, 0x20100018);
+	nv_wr32(priv, 0x41891c, 0xff00ff);
+	nv_wr32(priv, 0x418924, 0x0);
+	nv_wr32(priv, 0x418928, 0xffff00);
+	nv_wr32(priv, 0x41892c, 0xff00);
+	nv_wr32(priv, 0x418a00, 0x0);
+	nv_wr32(priv, 0x418a04, 0x0);
+	nv_wr32(priv, 0x418a08, 0x0);
+	nv_wr32(priv, 0x418a0c, 0x10000);
+	nv_wr32(priv, 0x418a10, 0x0);
+	nv_wr32(priv, 0x418a14, 0x0);
+	nv_wr32(priv, 0x418a18, 0x0);
+	nv_wr32(priv, 0x418a20, 0x0);
+	nv_wr32(priv, 0x418a24, 0x0);
+	nv_wr32(priv, 0x418a28, 0x0);
+	nv_wr32(priv, 0x418a2c, 0x10000);
+	nv_wr32(priv, 0x418a30, 0x0);
+	nv_wr32(priv, 0x418a34, 0x0);
+	nv_wr32(priv, 0x418a38, 0x0);
+	nv_wr32(priv, 0x418a40, 0x0);
+	nv_wr32(priv, 0x418a44, 0x0);
+	nv_wr32(priv, 0x418a48, 0x0);
+	nv_wr32(priv, 0x418a4c, 0x10000);
+	nv_wr32(priv, 0x418a50, 0x0);
+	nv_wr32(priv, 0x418a54, 0x0);
+	nv_wr32(priv, 0x418a58, 0x0);
+	nv_wr32(priv, 0x418a60, 0x0);
+	nv_wr32(priv, 0x418a64, 0x0);
+	nv_wr32(priv, 0x418a68, 0x0);
+	nv_wr32(priv, 0x418a6c, 0x10000);
+	nv_wr32(priv, 0x418a70, 0x0);
+	nv_wr32(priv, 0x418a74, 0x0);
+	nv_wr32(priv, 0x418a78, 0x0);
+	nv_wr32(priv, 0x418a80, 0x0);
+	nv_wr32(priv, 0x418a84, 0x0);
+	nv_wr32(priv, 0x418a88, 0x0);
+	nv_wr32(priv, 0x418a8c, 0x10000);
+	nv_wr32(priv, 0x418a90, 0x0);
+	nv_wr32(priv, 0x418a94, 0x0);
+	nv_wr32(priv, 0x418a98, 0x0);
+	nv_wr32(priv, 0x418aa0, 0x0);
+	nv_wr32(priv, 0x418aa4, 0x0);
+	nv_wr32(priv, 0x418aa8, 0x0);
+	nv_wr32(priv, 0x418aac, 0x10000);
+	nv_wr32(priv, 0x418ab0, 0x0);
+	nv_wr32(priv, 0x418ab4, 0x0);
+	nv_wr32(priv, 0x418ab8, 0x0);
+	nv_wr32(priv, 0x418ac0, 0x0);
+	nv_wr32(priv, 0x418ac4, 0x0);
+	nv_wr32(priv, 0x418ac8, 0x0);
+	nv_wr32(priv, 0x418acc, 0x10000);
+	nv_wr32(priv, 0x418ad0, 0x0);
+	nv_wr32(priv, 0x418ad4, 0x0);
+	nv_wr32(priv, 0x418ad8, 0x0);
+	nv_wr32(priv, 0x418ae0, 0x0);
+	nv_wr32(priv, 0x418ae4, 0x0);
+	nv_wr32(priv, 0x418ae8, 0x0);
+	nv_wr32(priv, 0x418aec, 0x10000);
+	nv_wr32(priv, 0x418af0, 0x0);
+	nv_wr32(priv, 0x418af4, 0x0);
+	nv_wr32(priv, 0x418af8, 0x0);
+	nv_wr32(priv, 0x418b00, 0x6);
+	nv_wr32(priv, 0x418b08, 0xa418820);
+	nv_wr32(priv, 0x418b0c, 0x62080e6);
+	nv_wr32(priv, 0x418b10, 0x20398a4);
+	nv_wr32(priv, 0x418b14, 0xe629062);
+	nv_wr32(priv, 0x418b18, 0xa418820);
+	nv_wr32(priv, 0x418b1c, 0xe6);
+	nv_wr32(priv, 0x418bb8, 0x103);
+	nv_wr32(priv, 0x418c08, 0x1);
+	nv_wr32(priv, 0x418c10, 0x0);
+	nv_wr32(priv, 0x418c14, 0x0);
+	nv_wr32(priv, 0x418c18, 0x0);
+	nv_wr32(priv, 0x418c1c, 0x0);
+	nv_wr32(priv, 0x418c20, 0x0);
+	nv_wr32(priv, 0x418c24, 0x0);
+	nv_wr32(priv, 0x418c28, 0x0);
+	nv_wr32(priv, 0x418c2c, 0x0);
+	nv_wr32(priv, 0x418c40, 0xffffffff);
+	nv_wr32(priv, 0x418c6c, 0x1);
+	nv_wr32(priv, 0x418c80, 0x20200004);
+	nv_wr32(priv, 0x418c8c, 0x1);
+	nv_wr32(priv, 0x419000, 0x780);
+	nv_wr32(priv, 0x419004, 0x0);
+	nv_wr32(priv, 0x419008, 0x0);
+	nv_wr32(priv, 0x419014, 0x4);
+}
+
+static void
+nve0_graph_generate_tpc(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x419848, 0x0);
+	nv_wr32(priv, 0x419864, 0x129);
+	nv_wr32(priv, 0x419888, 0x0);
+	nv_wr32(priv, 0x419a00, 0xf0);
+	nv_wr32(priv, 0x419a04, 0x1);
+	nv_wr32(priv, 0x419a08, 0x21);
+	nv_wr32(priv, 0x419a0c, 0x20000);
+	nv_wr32(priv, 0x419a10, 0x0);
+	nv_wr32(priv, 0x419a14, 0x200);
+	nv_wr32(priv, 0x419a1c, 0xc000);
+	nv_wr32(priv, 0x419a20, 0x800);
+	nv_wr32(priv, 0x419a30, 0x1);
+	nv_wr32(priv, 0x419ac4, 0x37f440);
+	nv_wr32(priv, 0x419c00, 0xa);
+	nv_wr32(priv, 0x419c04, 0x80000006);
+	nv_wr32(priv, 0x419c08, 0x2);
+	nv_wr32(priv, 0x419c20, 0x0);
+	nv_wr32(priv, 0x419c24, 0x84210);
+	nv_wr32(priv, 0x419c28, 0x3efbefbe);
+	nv_wr32(priv, 0x419ce8, 0x0);
+	nv_wr32(priv, 0x419cf4, 0x3203);
+	nv_wr32(priv, 0x419e04, 0x0);
+	nv_wr32(priv, 0x419e08, 0x0);
+	nv_wr32(priv, 0x419e0c, 0x0);
+	nv_wr32(priv, 0x419e10, 0x402);
+	nv_wr32(priv, 0x419e44, 0x13eff2);
+	nv_wr32(priv, 0x419e48, 0x0);
+	nv_wr32(priv, 0x419e4c, 0x7f);
+	nv_wr32(priv, 0x419e50, 0x0);
+	nv_wr32(priv, 0x419e54, 0x0);
+	nv_wr32(priv, 0x419e58, 0x0);
+	nv_wr32(priv, 0x419e5c, 0x0);
+	nv_wr32(priv, 0x419e60, 0x0);
+	nv_wr32(priv, 0x419e64, 0x0);
+	nv_wr32(priv, 0x419e68, 0x0);
+	nv_wr32(priv, 0x419e6c, 0x0);
+	nv_wr32(priv, 0x419e70, 0x0);
+	nv_wr32(priv, 0x419e74, 0x0);
+	nv_wr32(priv, 0x419e78, 0x0);
+	nv_wr32(priv, 0x419e7c, 0x0);
+	nv_wr32(priv, 0x419e80, 0x0);
+	nv_wr32(priv, 0x419e84, 0x0);
+	nv_wr32(priv, 0x419e88, 0x0);
+	nv_wr32(priv, 0x419e8c, 0x0);
+	nv_wr32(priv, 0x419e90, 0x0);
+	nv_wr32(priv, 0x419e94, 0x0);
+	nv_wr32(priv, 0x419e98, 0x0);
+	nv_wr32(priv, 0x419eac, 0x1fcf);
+	nv_wr32(priv, 0x419eb0, 0xd3f);
+	nv_wr32(priv, 0x419ec8, 0x1304f);
+	nv_wr32(priv, 0x419f30, 0x0);
+	nv_wr32(priv, 0x419f34, 0x0);
+	nv_wr32(priv, 0x419f38, 0x0);
+	nv_wr32(priv, 0x419f3c, 0x0);
+	nv_wr32(priv, 0x419f40, 0x0);
+	nv_wr32(priv, 0x419f44, 0x0);
+	nv_wr32(priv, 0x419f48, 0x0);
+	nv_wr32(priv, 0x419f4c, 0x0);
+	nv_wr32(priv, 0x419f58, 0x0);
+	nv_wr32(priv, 0x419f78, 0xb);
+}
+
+static void
+nve0_graph_generate_tpcunk(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x41be24, 0x6);
+	nv_wr32(priv, 0x41bec0, 0x12180000);
+	nv_wr32(priv, 0x41bec4, 0x37f7f);
+	nv_wr32(priv, 0x41bee4, 0x6480430);
+	nv_wr32(priv, 0x41bf00, 0xa418820);
+	nv_wr32(priv, 0x41bf04, 0x62080e6);
+	nv_wr32(priv, 0x41bf08, 0x20398a4);
+	nv_wr32(priv, 0x41bf0c, 0xe629062);
+	nv_wr32(priv, 0x41bf10, 0xa418820);
+	nv_wr32(priv, 0x41bf14, 0xe6);
+	nv_wr32(priv, 0x41bfd0, 0x900103);
+	nv_wr32(priv, 0x41bfe0, 0x400001);
+	nv_wr32(priv, 0x41bfe4, 0x0);
+}
+
+int
+nve0_grctx_generate(struct nvc0_graph_priv *priv)
+{
+	struct nvc0_grctx info;
+	int ret, i, gpc, tpc, id;
+	u32 data[6] = {}, data2[2] = {}, tmp;
+	u32 tpc_set = 0, tpc_mask = 0;
+	u32 magic[GPC_MAX][2], offset;
+	u8 tpcnr[GPC_MAX], a, b;
+	u8 shift, ntpcv;
+
+	ret = nvc0_grctx_init(priv, &info);
+	if (ret)
+		return ret;
+
+	nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+	nv_wr32(priv, 0x400204, 0x00000000);
+	nv_wr32(priv, 0x400208, 0x00000000);
+
+	nve0_graph_generate_unk40xx(priv);
+	nve0_graph_generate_unk44xx(priv);
+	nve0_graph_generate_unk46xx(priv);
+	nve0_graph_generate_unk47xx(priv);
+	nve0_graph_generate_unk58xx(priv);
+	nve0_graph_generate_unk60xx(priv);
+	nve0_graph_generate_unk64xx(priv);
+	nve0_graph_generate_unk70xx(priv);
+	nve0_graph_generate_unk78xx(priv);
+	nve0_graph_generate_unk80xx(priv);
+	nve0_graph_generate_unk88xx(priv);
+	nve0_graph_generate_gpc(priv);
+	nve0_graph_generate_tpc(priv);
+	nve0_graph_generate_tpcunk(priv);
+
+	nv_wr32(priv, 0x404154, 0x0);
+
+	mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+	mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+	mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
+	mmio_list(0x40800c, 0x00000000,  8, 1);
+	mmio_list(0x408010, 0x80000000,  0, 0);
+	mmio_list(0x419004, 0x00000000,  8, 1);
+	mmio_list(0x419008, 0x00000000,  0, 0);
+	mmio_list(0x4064cc, 0x80000000,  0, 0);
+	mmio_list(0x408004, 0x00000000,  8, 0);
+	mmio_list(0x408008, 0x80000030,  0, 0);
+	mmio_list(0x418808, 0x00000000,  8, 0);
+	mmio_list(0x41880c, 0x80000030,  0, 0);
+	mmio_list(0x4064c8, 0x01800600,  0, 0);
+	mmio_list(0x418810, 0x80000000, 12, 2);
+	mmio_list(0x419848, 0x10000000, 12, 2);
+	mmio_list(0x405830, 0x02180648,  0, 0);
+	mmio_list(0x4064c4, 0x0192ffff,  0, 0);
+	for (gpc = 0, offset = 0; gpc < priv->gpc_nr; gpc++) {
+		u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
+		u16 magic1 = 0x0648 * priv->tpc_nr[gpc];
+		magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
+		magic[gpc][1]  = 0x00000000 | (magic1 << 16);
+		offset += 0x0324 * priv->tpc_nr[gpc];
+	}
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+		mmio_list(GPC_UNIT(gpc, 0x30c0), magic[gpc][0], 0, 0);
+		mmio_list(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset, 0, 0);
+		offset += 0x07ff * priv->tpc_nr[gpc];
+	}
+	mmio_list(0x17e91c, 0x06060609, 0, 0);
+	mmio_list(0x17e920, 0x00090a05, 0, 0);
+
+	nv_wr32(priv, 0x418c6c, 0x1);
+	nv_wr32(priv, 0x41980c, 0x10);
+	nv_wr32(priv, 0x41be08, 0x4);
+	nv_wr32(priv, 0x4064c0, 0x801a00f0);
+	nv_wr32(priv, 0x405800, 0xf8000bf);
+	nv_wr32(priv, 0x419c00, 0xa);
+
+	for (tpc = 0, id = 0; tpc < 4; tpc++) {
+		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+			if (tpc < priv->tpc_nr[gpc]) {
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0698), id);
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x04e8), id);
+				nv_wr32(priv, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
+				nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0088), id++);
+			}
+
+			nv_wr32(priv, GPC_UNIT(gpc, 0x0c08), priv->tpc_nr[gpc]);
+			nv_wr32(priv, GPC_UNIT(gpc, 0x0c8c), priv->tpc_nr[gpc]);
+		}
+	}
+
+	tmp = 0;
+	for (i = 0; i < priv->gpc_nr; i++)
+		tmp |= priv->tpc_nr[i] << (i * 4);
+	nv_wr32(priv, 0x406028, tmp);
+	nv_wr32(priv, 0x405870, tmp);
+
+	nv_wr32(priv, 0x40602c, 0x0);
+	nv_wr32(priv, 0x405874, 0x0);
+	nv_wr32(priv, 0x406030, 0x0);
+	nv_wr32(priv, 0x405878, 0x0);
+	nv_wr32(priv, 0x406034, 0x0);
+	nv_wr32(priv, 0x40587c, 0x0);
+
+	/* calculate first set of magics */
+	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+
+	gpc = -1;
+	for (tpc = 0; tpc < priv->tpc_total; tpc++) {
+		do {
+			gpc = (gpc + 1) % priv->gpc_nr;
+		} while (!tpcnr[gpc]);
+		tpcnr[gpc]--;
+
+		data[tpc / 6] |= gpc << ((tpc % 6) * 5);
+	}
+
+	for (; tpc < 32; tpc++)
+		data[tpc / 6] |= 7 << ((tpc % 6) * 5);
+
+	/* and the second... */
+	shift = 0;
+	ntpcv = priv->tpc_total;
+	while (!(ntpcv & (1 << 4))) {
+		ntpcv <<= 1;
+		shift++;
+	}
+
+	data2[0]  = ntpcv << 16;
+	data2[0] |= shift << 21;
+	data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
+	data2[0] |= priv->tpc_total << 8;
+	data2[0] |= priv->magic_not_rop_nr;
+	for (i = 1; i < 7; i++)
+		data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
+
+	/* and write it all the various parts of PGRAPH */
+	nv_wr32(priv, 0x418bb8, (priv->tpc_total << 8) | priv->magic_not_rop_nr);
+	for (i = 0; i < 6; i++)
+		nv_wr32(priv, 0x418b08 + (i * 4), data[i]);
+
+	nv_wr32(priv, 0x41bfd0, data2[0]);
+	nv_wr32(priv, 0x41bfe4, data2[1]);
+	for (i = 0; i < 6; i++)
+		nv_wr32(priv, 0x41bf00 + (i * 4), data[i]);
+
+	nv_wr32(priv, 0x4078bc, (priv->tpc_total << 8) | priv->magic_not_rop_nr);
+	for (i = 0; i < 6; i++)
+		nv_wr32(priv, 0x40780c + (i * 4), data[i]);
+
+
+	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++)
+		tpc_mask |= ((1 << priv->tpc_nr[gpc]) - 1) << (gpc * 8);
+
+	for (i = 0, gpc = -1, b = -1; i < 32; i++) {
+		a = (i * (priv->tpc_total - 1)) / 32;
+		if (a != b) {
+			b = a;
+			do {
+				gpc = (gpc + 1) % priv->gpc_nr;
+			} while (!tpcnr[gpc]);
+			tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
+
+			tpc_set |= 1 << ((gpc * 8) + tpc);
+		}
+
+		nv_wr32(priv, 0x406800 + (i * 0x20), tpc_set);
+		nv_wr32(priv, 0x406c00 + (i * 0x20), tpc_set ^ tpc_mask);
+	}
+
+	for (i = 0; i < 8; i++)
+		nv_wr32(priv, 0x4064d0 + (i * 0x04), 0x00000000);
+
+	nv_wr32(priv, 0x405b00, 0x201);
+	nv_wr32(priv, 0x408850, 0x2);
+	nv_wr32(priv, 0x408958, 0x2);
+	nv_wr32(priv, 0x419f78, 0xa);
+
+	nve0_grctx_generate_icmd(priv);
+	nve0_grctx_generate_a097(priv);
+	nve0_grctx_generate_902d(priv);
+
+	nv_mask(priv, 0x000260, 0x00000001, 0x00000001);
+	nv_wr32(priv, 0x418800, 0x7026860a); //XXX
+	nv_wr32(priv, 0x41be10, 0x00bb8bc7); //XXX
+	return nvc0_grctx_fini(&info);
+}
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc
index 15272be33b6..b86cc60dcd5 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc
@@ -24,7 +24,7 @@
  */
 
 /* To build:
- *    m4 nvc0_grgpc.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grgpc.fuc.h
+ *    m4 gpcnvc0.fuc | envyas -a -w -m fuc -V fuc3 -o gpcnvc0.fuc.h
  */
 
 /* TODO
@@ -33,7 +33,7 @@
  */
 
 .section #nvc0_grgpc_data
-include(`nvc0_graph.fuc')
+include(`nvc0.fuc')
 gpc_id:			.b32 0
 gpc_mmio_list_head:	.b32 0
 gpc_mmio_list_tail:	.b32 0
@@ -209,11 +209,11 @@ nvd9_tpc_mmio_tail:
 .section #nvc0_grgpc_code
 bra #init
 define(`include_code')
-include(`nvc0_graph.fuc')
+include(`nvc0.fuc')
 
 // reports an exception to the host
 //
-// In: $r15 error code (see nvc0_graph.fuc)
+// In: $r15 error code (see nvc0.fuc)
 //
 error:
 	push $r14
diff --git a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h
index a988b8ad00a..96050ddb22c 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grgpc.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h
@@ -1,11 +1,19 @@
 uint32_t nvc0_grgpc_data[] = {
+/* 0x0000: gpc_id */
 	0x00000000,
+/* 0x0004: gpc_mmio_list_head */
 	0x00000000,
+/* 0x0008: gpc_mmio_list_tail */
 	0x00000000,
+/* 0x000c: tpc_count */
 	0x00000000,
+/* 0x0010: tpc_mask */
 	0x00000000,
+/* 0x0014: tpc_mmio_list_head */
 	0x00000000,
+/* 0x0018: tpc_mmio_list_tail */
 	0x00000000,
+/* 0x001c: cmd_queue */
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -24,6 +32,7 @@ uint32_t nvc0_grgpc_data[] = {
 	0x00000000,
 	0x00000000,
 	0x00000000,
+/* 0x0064: chipsets */
 	0x000000c0,
 	0x012800c8,
 	0x01e40194,
@@ -49,6 +58,7 @@ uint32_t nvc0_grgpc_data[] = {
 	0x0194012c,
 	0x025401f8,
 	0x00000000,
+/* 0x00c8: nvc0_gpc_mmio_head */
 	0x00000380,
 	0x14000400,
 	0x20000450,
@@ -73,7 +83,10 @@ uint32_t nvc0_grgpc_data[] = {
 	0x00000c8c,
 	0x08001000,
 	0x00001014,
+/* 0x0128: nvc0_gpc_mmio_tail */
 	0x00000c6c,
+/* 0x012c: nvc1_gpc_mmio_tail */
+/* 0x012c: nvd9_gpc_mmio_head */
 	0x00000380,
 	0x04000400,
 	0x0800040c,
@@ -100,6 +113,8 @@ uint32_t nvc0_grgpc_data[] = {
 	0x00000c8c,
 	0x08001000,
 	0x00001014,
+/* 0x0194: nvd9_gpc_mmio_tail */
+/* 0x0194: nvc0_tpc_mmio_head */
 	0x00000018,
 	0x0000003c,
 	0x00000048,
@@ -120,11 +135,16 @@ uint32_t nvc0_grgpc_data[] = {
 	0x4c000644,
 	0x00000698,
 	0x04000750,
+/* 0x01e4: nvc0_tpc_mmio_tail */
 	0x00000758,
 	0x000002c4,
 	0x000006e0,
+/* 0x01f0: nvcf_tpc_mmio_tail */
 	0x000004bc,
+/* 0x01f4: nvc3_tpc_mmio_tail */
 	0x00000544,
+/* 0x01f8: nvc1_tpc_mmio_tail */
+/* 0x01f8: nvd9_tpc_mmio_head */
 	0x00000018,
 	0x0000003c,
 	0x00000048,
@@ -152,12 +172,14 @@ uint32_t nvc0_grgpc_data[] = {
 
 uint32_t nvc0_grgpc_code[] = {
 	0x03060ef5,
+/* 0x0004: queue_put */
 	0x9800d898,
 	0x86f001d9,
 	0x0489b808,
 	0xf00c1bf4,
 	0x21f502f7,
 	0x00f802ec,
+/* 0x001c: queue_put_next */
 	0xb60798c4,
 	0x8dbb0384,
 	0x0880b600,
@@ -165,6 +187,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x90b6018f,
 	0x0f94f001,
 	0xf801d980,
+/* 0x0039: queue_get */
 	0x0131f400,
 	0x9800d898,
 	0x89b801d9,
@@ -176,37 +199,46 @@ uint32_t nvc0_grgpc_code[] = {
 	0x80b6019f,
 	0x0f84f001,
 	0xf400d880,
+/* 0x0066: queue_get_done */
 	0x00f80132,
+/* 0x0068: nv_rd32 */
 	0x0728b7f1,
 	0xb906b4b6,
 	0xc9f002ec,
 	0x00bcd01f,
+/* 0x0078: nv_rd32_wait */
 	0xc800bccf,
 	0x1bf41fcc,
 	0x06a7f0fa,
 	0x010321f5,
 	0xf840bfcf,
+/* 0x008d: nv_wr32 */
 	0x28b7f100,
 	0x06b4b607,
 	0xb980bfd0,
 	0xc9f002ec,
 	0x1ec9f01f,
+/* 0x00a3: nv_wr32_wait */
 	0xcf00bcd0,
 	0xccc800bc,
 	0xfa1bf41f,
+/* 0x00ae: watchdog_reset */
 	0x87f100f8,
 	0x84b60430,
 	0x1ff9f006,
 	0xf8008fd0,
+/* 0x00bd: watchdog_clear */
 	0x3087f100,
 	0x0684b604,
 	0xf80080d0,
+/* 0x00c9: wait_donez */
 	0x3c87f100,
 	0x0684b608,
 	0x99f094bd,
 	0x0089d000,
 	0x081887f1,
 	0xd00684b6,
+/* 0x00e2: wait_done_wait_donez */
 	0x87f1008a,
 	0x84b60400,
 	0x0088cf06,
@@ -215,6 +247,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x84b6085c,
 	0xf094bd06,
 	0x89d00099,
+/* 0x0103: wait_doneo */
 	0xf100f800,
 	0xb6083c87,
 	0x94bd0684,
@@ -222,6 +255,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x87f10089,
 	0x84b60818,
 	0x008ad006,
+/* 0x011c: wait_done_wait_doneo */
 	0x040087f1,
 	0xcf0684b6,
 	0x8aff0088,
@@ -230,6 +264,8 @@ uint32_t nvc0_grgpc_code[] = {
 	0xbd0684b6,
 	0x0099f094,
 	0xf80089d0,
+/* 0x013d: mmctx_size */
+/* 0x013f: nv_mmctx_size_loop */
 	0x9894bd00,
 	0x85b600e8,
 	0x0180b61a,
@@ -238,6 +274,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x04efb804,
 	0xb9eb1bf4,
 	0x00f8029f,
+/* 0x015c: mmctx_xfer */
 	0x083c87f1,
 	0xbd0684b6,
 	0x0199f094,
@@ -247,9 +284,11 @@ uint32_t nvc0_grgpc_code[] = {
 	0xf405bbfd,
 	0x8bd0090b,
 	0x0099f000,
+/* 0x0180: mmctx_base_disabled */
 	0xf405eefd,
 	0x8ed00c0b,
 	0xc08fd080,
+/* 0x018f: mmctx_multi_disabled */
 	0xb70199f0,
 	0xc8010080,
 	0xb4b600ab,
@@ -257,6 +296,8 @@ uint32_t nvc0_grgpc_code[] = {
 	0xb601aec8,
 	0xbefd11e4,
 	0x008bd005,
+/* 0x01a8: mmctx_exec_loop */
+/* 0x01a8: mmctx_wait_free */
 	0xf0008ecf,
 	0x0bf41fe4,
 	0x00ce98fa,
@@ -265,34 +306,42 @@ uint32_t nvc0_grgpc_code[] = {
 	0x04cdb804,
 	0xc8e81bf4,
 	0x1bf402ab,
+/* 0x01c9: mmctx_fini_wait */
 	0x008bcf18,
 	0xb01fb4f0,
 	0x1bf410b4,
 	0x02a7f0f7,
 	0xf4c921f4,
+/* 0x01de: mmctx_stop */
 	0xabc81b0e,
 	0x10b4b600,
 	0xf00cb9f0,
 	0x8bd012b9,
+/* 0x01ed: mmctx_stop_wait */
 	0x008bcf00,
 	0xf412bbc8,
+/* 0x01f6: mmctx_done */
 	0x87f1fa1b,
 	0x84b6085c,
 	0xf094bd06,
 	0x89d00199,
+/* 0x0207: strand_wait */
 	0xf900f800,
 	0x02a7f0a0,
 	0xfcc921f4,
+/* 0x0213: strand_pre */
 	0xf100f8a0,
 	0xf04afc87,
 	0x97f00283,
 	0x0089d00c,
 	0x020721f5,
+/* 0x0226: strand_post */
 	0x87f100f8,
 	0x83f04afc,
 	0x0d97f002,
 	0xf50089d0,
 	0xf8020721,
+/* 0x0239: strand_set */
 	0xfca7f100,
 	0x02a3f04f,
 	0x0500aba2,
@@ -303,6 +352,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0xf000aed0,
 	0xbcd00ac7,
 	0x0721f500,
+/* 0x0263: strand_ctx_init */
 	0xf100f802,
 	0xb6083c87,
 	0x94bd0684,
@@ -325,6 +375,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x0684b608,
 	0xb70089cf,
 	0x95220080,
+/* 0x02ba: ctx_init_strand_loop */
 	0x8ed008fe,
 	0x408ed000,
 	0xb6808acf,
@@ -338,12 +389,14 @@ uint32_t nvc0_grgpc_code[] = {
 	0x94bd0684,
 	0xd00399f0,
 	0x00f80089,
+/* 0x02ec: error */
 	0xe7f1e0f9,
 	0xe3f09814,
 	0x8d21f440,
 	0x041ce0b7,
 	0xf401f7f0,
 	0xe0fc8d21,
+/* 0x0306: init */
 	0x04bd00f8,
 	0xf10004fe,
 	0xf0120017,
@@ -366,11 +419,13 @@ uint32_t nvc0_grgpc_code[] = {
 	0x27f10002,
 	0x24b60800,
 	0x0022cf06,
+/* 0x035f: init_find_chipset */
 	0xb65817f0,
 	0x13980c10,
 	0x0432b800,
 	0xb00b0bf4,
 	0x1bf40034,
+/* 0x0373: init_context */
 	0xf100f8f1,
 	0xb6080027,
 	0x22cf0624,
@@ -407,6 +462,7 @@ uint32_t nvc0_grgpc_code[] = {
 	0x0010b740,
 	0xf024bd08,
 	0x12d01f29,
+/* 0x0401: main */
 	0x0031f400,
 	0xf00028f4,
 	0x21f41cd7,
@@ -419,9 +475,11 @@ uint32_t nvc0_grgpc_code[] = {
 	0xfe051efd,
 	0x21f50018,
 	0x0ef404c3,
+/* 0x0431: main_not_ctx_xfer */
 	0x10ef94d3,
 	0xf501f5f0,
 	0xf402ec21,
+/* 0x043e: ih */
 	0x80f9c60e,
 	0xf90188fe,
 	0xf990f980,
@@ -436,30 +494,36 @@ uint32_t nvc0_grgpc_code[] = {
 	0xb0b70421,
 	0xe7f00400,
 	0x00bed001,
+/* 0x0474: ih_no_fifo */
 	0xfc400ad0,
 	0xfce0fcf0,
 	0xfcb0fcd0,
 	0xfc90fca0,
 	0x0088fe80,
 	0x32f480fc,
+/* 0x048f: hub_barrier_done */
 	0xf001f800,
 	0x0e9801f7,
 	0x04febb00,
 	0x9418e7f1,
 	0xf440e3f0,
 	0x00f88d21,
+/* 0x04a4: ctx_redswitch */
 	0x0614e7f1,
 	0xf006e4b6,
 	0xefd020f7,
 	0x08f7f000,
+/* 0x04b4: ctx_redswitch_delay */
 	0xf401f2b6,
 	0xf7f1fd1b,
 	0xefd00a20,
+/* 0x04c3: ctx_xfer */
 	0xf100f800,
 	0xb60a0417,
 	0x1fd00614,
 	0x0711f400,
 	0x04a421f5,
+/* 0x04d4: ctx_xfer_not_load */
 	0x4afc17f1,
 	0xf00213f0,
 	0x12d00c27,
@@ -489,11 +553,13 @@ uint32_t nvc0_grgpc_code[] = {
 	0x5c21f508,
 	0x0721f501,
 	0x0601f402,
+/* 0x054b: ctx_xfer_post */
 	0xf11412f4,
 	0xf04afc17,
 	0x27f00213,
 	0x0012d00d,
 	0x020721f5,
+/* 0x055c: ctx_xfer_done */
 	0x048f21f5,
 	0x000000f8,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc
new file mode 100644
index 00000000000..7b715fda276
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc
@@ -0,0 +1,451 @@
+/* fuc microcode for nve0 PGRAPH/GPC
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/* To build:
+ *    m4 nve0_grgpc.fuc | envyas -a -w -m fuc -V nva3 -o nve0_grgpc.fuc.h
+ */
+
+/* TODO
+ * - bracket certain functions with scratch writes, useful for debugging
+ * - watchdog timer around ctx operations
+ */
+
+.section #nve0_grgpc_data
+include(`nve0.fuc')
+gpc_id:			.b32 0
+gpc_mmio_list_head:	.b32 0
+gpc_mmio_list_tail:	.b32 0
+
+tpc_count:		.b32 0
+tpc_mask:		.b32 0
+tpc_mmio_list_head:	.b32 0
+tpc_mmio_list_tail:	.b32 0
+
+cmd_queue:		queue_init
+
+// chipset descriptions
+chipsets:
+.b8  0xe4 0 0 0
+.b16 #nve4_gpc_mmio_head
+.b16 #nve4_gpc_mmio_tail
+.b16 #nve4_tpc_mmio_head
+.b16 #nve4_tpc_mmio_tail
+.b8  0xe7 0 0 0
+.b16 #nve4_gpc_mmio_head
+.b16 #nve4_gpc_mmio_tail
+.b16 #nve4_tpc_mmio_head
+.b16 #nve4_tpc_mmio_tail
+.b8  0 0 0 0
+
+// GPC mmio lists
+nve4_gpc_mmio_head:
+mmctx_data(0x000380, 1)
+mmctx_data(0x000400, 2)
+mmctx_data(0x00040c, 3)
+mmctx_data(0x000450, 9)
+mmctx_data(0x000600, 1)
+mmctx_data(0x000684, 1)
+mmctx_data(0x000700, 5)
+mmctx_data(0x000800, 1)
+mmctx_data(0x000808, 3)
+mmctx_data(0x000828, 1)
+mmctx_data(0x000830, 1)
+mmctx_data(0x0008d8, 1)
+mmctx_data(0x0008e0, 1)
+mmctx_data(0x0008e8, 6)
+mmctx_data(0x00091c, 1)
+mmctx_data(0x000924, 3)
+mmctx_data(0x000b00, 1)
+mmctx_data(0x000b08, 6)
+mmctx_data(0x000bb8, 1)
+mmctx_data(0x000c08, 1)
+mmctx_data(0x000c10, 8)
+mmctx_data(0x000c40, 1)
+mmctx_data(0x000c6c, 1)
+mmctx_data(0x000c80, 1)
+mmctx_data(0x000c8c, 1)
+mmctx_data(0x001000, 3)
+mmctx_data(0x001014, 1)
+mmctx_data(0x003024, 1)
+mmctx_data(0x0030c0, 2)
+mmctx_data(0x0030e4, 1)
+mmctx_data(0x003100, 6)
+mmctx_data(0x0031d0, 1)
+mmctx_data(0x0031e0, 2)
+nve4_gpc_mmio_tail:
+
+// TPC mmio lists
+nve4_tpc_mmio_head:
+mmctx_data(0x000048, 1)
+mmctx_data(0x000064, 1)
+mmctx_data(0x000088, 1)
+mmctx_data(0x000200, 6)
+mmctx_data(0x00021c, 2)
+mmctx_data(0x000230, 1)
+mmctx_data(0x0002c4, 1)
+mmctx_data(0x000400, 3)
+mmctx_data(0x000420, 3)
+mmctx_data(0x0004e8, 1)
+mmctx_data(0x0004f4, 1)
+mmctx_data(0x000604, 4)
+mmctx_data(0x000644, 22)
+mmctx_data(0x0006ac, 2)
+mmctx_data(0x0006c8, 1)
+mmctx_data(0x000730, 8)
+mmctx_data(0x000758, 1)
+mmctx_data(0x000778, 1)
+nve4_tpc_mmio_tail:
+
+.section #nve0_grgpc_code
+bra #init
+define(`include_code')
+include(`nve0.fuc')
+
+// reports an exception to the host
+//
+// In: $r15 error code (see nve0.fuc)
+//
+error:
+	push $r14
+	mov $r14 -0x67ec 	// 0x9814
+	sethi $r14 0x400000
+	call #nv_wr32		// HUB_CTXCTL_CC_SCRATCH[5] = error code
+	add b32 $r14 0x41c
+	mov $r15 1
+	call #nv_wr32		// HUB_CTXCTL_INTR_UP_SET
+	pop $r14
+	ret
+
+// GPC fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//   CC_SCRATCH[1]: context base
+//
+// Output:
+//   CC_SCRATCH[0]:
+//	     31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//	      31:0: GPC context size
+//
+init:
+	clear b32 $r0
+	mov $sp $r0
+
+	// enable fifo access
+	mov $r1 0x1200
+	mov $r2 2
+	iowr I[$r1 + 0x000] $r2		// FIFO_ENABLE
+
+	// setup i0 handler, and route all interrupts to it
+	mov $r1 #ih
+	mov $iv0 $r1
+	mov $r1 0x400
+	iowr I[$r1 + 0x300] $r0		// INTR_DISPATCH
+
+	// enable fifo interrupt
+	mov $r2 4
+	iowr I[$r1 + 0x000] $r2		// INTR_EN_SET
+
+	// enable interrupts
+	bset $flags ie0
+
+	// figure out which GPC we are, and how many TPCs we have
+	mov $r1 0x608
+	shl b32 $r1 6
+	iord $r2 I[$r1 + 0x000]		// UNITS
+	mov $r3 1
+	and $r2 0x1f
+	shl b32 $r3 $r2
+	sub b32 $r3 1
+	st b32 D[$r0 + #tpc_count] $r2
+	st b32 D[$r0 + #tpc_mask] $r3
+	add b32 $r1 0x400
+	iord $r2 I[$r1 + 0x000]		// MYINDEX
+	st b32 D[$r0 + #gpc_id] $r2
+
+	// find context data for this chipset
+	mov $r2 0x800
+	shl b32 $r2 6
+	iord $r2 I[$r2 + 0x000]		// CC_SCRATCH[0]
+	mov $r1 #chipsets - 12
+	init_find_chipset:
+		add b32 $r1 12
+		ld b32 $r3 D[$r1 + 0x00]
+		cmpu b32 $r3 $r2
+		bra e #init_context
+		cmpu b32 $r3 0
+		bra ne #init_find_chipset
+		// unknown chipset
+		ret
+
+	// initialise context base, and size tracking
+	init_context:
+	mov $r2 0x800
+	shl b32 $r2 6
+	iord $r2 I[$r2 + 0x100]	// CC_SCRATCH[1], initial base
+	clear b32 $r3		// track GPC context size here
+
+	// set mmctx base addresses now so we don't have to do it later,
+	// they don't currently ever change
+	mov $r4 0x700
+	shl b32 $r4 6
+	shr b32 $r5 $r2 8
+	iowr I[$r4 + 0x000] $r5		// MMCTX_SAVE_SWBASE
+	iowr I[$r4 + 0x100] $r5		// MMCTX_LOAD_SWBASE
+
+	// calculate GPC mmio context size, store the chipset-specific
+	// mmio list pointers somewhere we can get at them later without
+	// re-parsing the chipset list
+	clear b32 $r14
+	clear b32 $r15
+	ld b16 $r14 D[$r1 + 4]
+	ld b16 $r15 D[$r1 + 6]
+	st b16 D[$r0 + #gpc_mmio_list_head] $r14
+	st b16 D[$r0 + #gpc_mmio_list_tail] $r15
+	call #mmctx_size
+	add b32 $r2 $r15
+	add b32 $r3 $r15
+
+	// calculate per-TPC mmio context size, store the list pointers
+	ld b16 $r14 D[$r1 + 8]
+	ld b16 $r15 D[$r1 + 10]
+	st b16 D[$r0 + #tpc_mmio_list_head] $r14
+	st b16 D[$r0 + #tpc_mmio_list_tail] $r15
+	call #mmctx_size
+	ld b32 $r14 D[$r0 + #tpc_count]
+	mulu $r14 $r15
+	add b32 $r2 $r14
+	add b32 $r3 $r14
+
+	// round up base/size to 256 byte boundary (for strand SWBASE)
+	add b32 $r4 0x1300
+	shr b32 $r3 2
+	iowr I[$r4 + 0x000] $r3		// MMCTX_LOAD_COUNT, wtf for?!?
+	shr b32 $r2 8
+	shr b32 $r3 6
+	add b32 $r2 1
+	add b32 $r3 1
+	shl b32 $r2 8
+	shl b32 $r3 8
+
+	// calculate size of strand context data
+	mov b32 $r15 $r2
+	call #strand_ctx_init
+	add b32 $r3 $r15
+
+	// save context size, and tell HUB we're done
+	mov $r1 0x800
+	shl b32 $r1 6
+	iowr I[$r1 + 0x100] $r3		// CC_SCRATCH[1]  = context size
+	add b32 $r1 0x800
+	clear b32 $r2
+	bset $r2 31
+	iowr I[$r1 + 0x000] $r2		// CC_SCRATCH[0] |= 0x80000000
+
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+	bset $flags $p0
+	sleep $p0
+	mov $r13 #cmd_queue
+	call #queue_get
+	bra $p1 #main
+
+	// 0x0000-0x0003 are all context transfers
+	cmpu b32 $r14 0x04
+	bra nc #main_not_ctx_xfer
+		// fetch $flags and mask off $p1/$p2
+		mov $r1 $flags
+		mov $r2 0x0006
+		not b32 $r2
+		and $r1 $r2
+		// set $p1/$p2 according to transfer type
+		shl b32 $r14 1
+		or $r1 $r14
+		mov $flags $r1
+		// transfer context data
+		call #ctx_xfer
+		bra #main
+
+	main_not_ctx_xfer:
+	shl b32 $r15 $r14 16
+	or $r15 E_BAD_COMMAND
+	call #error
+	bra #main
+
+// interrupt handler
+ih:
+	push $r8
+	mov $r8 $flags
+	push $r8
+	push $r9
+	push $r10
+	push $r11
+	push $r13
+	push $r14
+	push $r15
+
+	// incoming fifo command?
+	iord $r10 I[$r0 + 0x200]	// INTR
+	and $r11 $r10 0x00000004
+	bra e #ih_no_fifo
+		// queue incoming fifo command for later processing
+		mov $r11 0x1900
+		mov $r13 #cmd_queue
+		iord $r14 I[$r11 + 0x100]	// FIFO_CMD
+		iord $r15 I[$r11 + 0x000]	// FIFO_DATA
+		call #queue_put
+		add b32 $r11 0x400
+		mov $r14 1
+		iowr I[$r11 + 0x000] $r14	// FIFO_ACK
+
+	// ack, and wake up main()
+	ih_no_fifo:
+	iowr I[$r0 + 0x100] $r10	// INTR_ACK
+
+	pop $r15
+	pop $r14
+	pop $r13
+	pop $r11
+	pop $r10
+	pop $r9
+	pop $r8
+	mov $flags $r8
+	pop $r8
+	bclr $flags $p0
+	iret
+
+// Set this GPC's bit in HUB_BAR, used to signal completion of various
+// activities to the HUB fuc
+//
+hub_barrier_done:
+	mov $r15 1
+	ld b32 $r14 D[$r0 + #gpc_id]
+	shl b32 $r15 $r14
+	mov $r14 -0x6be8 	// 0x409418 - HUB_BAR_SET
+	sethi $r14 0x400000
+	call #nv_wr32
+	ret
+
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+	mov $r14 0x614
+	shl b32 $r14 6
+	mov $r15 0x020
+	iowr I[$r14] $r15	// GPC_RED_SWITCH = POWER
+	mov $r15 8
+	ctx_redswitch_delay:
+		sub b32 $r15 1
+		bra ne #ctx_redswitch_delay
+	mov $r15 0xa20
+	iowr I[$r14] $r15	// GPC_RED_SWITCH = UNK11, ENABLE, POWER
+	ret
+
+// Transfer GPC context data between GPU and storage area
+//
+// In: $r15 context base address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//		on save it means: "a load will follow this save"
+//		on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+	// set context base address
+	mov $r1 0xa04
+	shl b32 $r1 6
+	iowr I[$r1 + 0x000] $r15// MEM_BASE
+	bra not $p1 #ctx_xfer_not_load
+		call #ctx_redswitch
+	ctx_xfer_not_load:
+
+	// strands
+	mov $r1 0x4afc
+	sethi $r1 0x20000
+	mov $r2 0xc
+	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x0c
+	call #strand_wait
+	mov $r2 0x47fc
+	sethi $r2 0x20000
+	iowr I[$r2] $r0		// STRAND_FIRST_GENE(0x3f) = 0x00
+	xbit $r2 $flags $p1
+	add b32 $r2 3
+	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+
+	// mmio context
+	xbit $r10 $flags $p1	// direction
+	or $r10 2		// first
+	mov $r11 0x0000
+	sethi $r11 0x500000
+	ld b32 $r12 D[$r0 + #gpc_id]
+	shl b32 $r12 15
+	add b32 $r11 $r12	// base = NV_PGRAPH_GPCn
+	ld b32 $r12 D[$r0 + #gpc_mmio_list_head]
+	ld b32 $r13 D[$r0 + #gpc_mmio_list_tail]
+	mov $r14 0		// not multi
+	call #mmctx_xfer
+
+	// per-TPC mmio context
+	xbit $r10 $flags $p1	// direction
+	or $r10 4		// last
+	mov $r11 0x4000
+	sethi $r11 0x500000	// base = NV_PGRAPH_GPC0_TPC0
+	ld b32 $r12 D[$r0 + #gpc_id]
+	shl b32 $r12 15
+	add b32 $r11 $r12	// base = NV_PGRAPH_GPCn_TPC0
+	ld b32 $r12 D[$r0 + #tpc_mmio_list_head]
+	ld b32 $r13 D[$r0 + #tpc_mmio_list_tail]
+	ld b32 $r15 D[$r0 + #tpc_mask]
+	mov $r14 0x800		// stride = 0x800
+	call #mmctx_xfer
+
+	// wait for strands to finish
+	call #strand_wait
+
+	// if load, or a save without a load following, do some
+	// unknown stuff that's done after finishing a block of
+	// strand commands
+	bra $p1 #ctx_xfer_post
+	bra not $p2 #ctx_xfer_done
+	ctx_xfer_post:
+		mov $r1 0x4afc
+		sethi $r1 0x20000
+		mov $r2 0xd
+		iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x0d
+		call #strand_wait
+
+	// mark completion in HUB's barrier
+	ctx_xfer_done:
+	call #hub_barrier_done
+	ret
+
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h
new file mode 100644
index 00000000000..26c2165bad0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h
@@ -0,0 +1,530 @@
+uint32_t nve0_grgpc_data[] = {
+/* 0x0000: gpc_id */
+	0x00000000,
+/* 0x0004: gpc_mmio_list_head */
+	0x00000000,
+/* 0x0008: gpc_mmio_list_tail */
+	0x00000000,
+/* 0x000c: tpc_count */
+	0x00000000,
+/* 0x0010: tpc_mask */
+	0x00000000,
+/* 0x0014: tpc_mmio_list_head */
+	0x00000000,
+/* 0x0018: tpc_mmio_list_tail */
+	0x00000000,
+/* 0x001c: cmd_queue */
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+/* 0x0064: chipsets */
+	0x000000e4,
+	0x01040080,
+	0x014c0104,
+	0x000000e7,
+	0x01040080,
+	0x014c0104,
+	0x00000000,
+/* 0x0080: nve4_gpc_mmio_head */
+	0x00000380,
+	0x04000400,
+	0x0800040c,
+	0x20000450,
+	0x00000600,
+	0x00000684,
+	0x10000700,
+	0x00000800,
+	0x08000808,
+	0x00000828,
+	0x00000830,
+	0x000008d8,
+	0x000008e0,
+	0x140008e8,
+	0x0000091c,
+	0x08000924,
+	0x00000b00,
+	0x14000b08,
+	0x00000bb8,
+	0x00000c08,
+	0x1c000c10,
+	0x00000c40,
+	0x00000c6c,
+	0x00000c80,
+	0x00000c8c,
+	0x08001000,
+	0x00001014,
+	0x00003024,
+	0x040030c0,
+	0x000030e4,
+	0x14003100,
+	0x000031d0,
+	0x040031e0,
+/* 0x0104: nve4_gpc_mmio_tail */
+/* 0x0104: nve4_tpc_mmio_head */
+	0x00000048,
+	0x00000064,
+	0x00000088,
+	0x14000200,
+	0x0400021c,
+	0x00000230,
+	0x000002c4,
+	0x08000400,
+	0x08000420,
+	0x000004e8,
+	0x000004f4,
+	0x0c000604,
+	0x54000644,
+	0x040006ac,
+	0x000006c8,
+	0x1c000730,
+	0x00000758,
+	0x00000778,
+};
+
+uint32_t nve0_grgpc_code[] = {
+	0x03060ef5,
+/* 0x0004: queue_put */
+	0x9800d898,
+	0x86f001d9,
+	0x0489b808,
+	0xf00c1bf4,
+	0x21f502f7,
+	0x00f802ec,
+/* 0x001c: queue_put_next */
+	0xb60798c4,
+	0x8dbb0384,
+	0x0880b600,
+	0x80008e80,
+	0x90b6018f,
+	0x0f94f001,
+	0xf801d980,
+/* 0x0039: queue_get */
+	0x0131f400,
+	0x9800d898,
+	0x89b801d9,
+	0x210bf404,
+	0xb60789c4,
+	0x9dbb0394,
+	0x0890b600,
+	0x98009e98,
+	0x80b6019f,
+	0x0f84f001,
+	0xf400d880,
+/* 0x0066: queue_get_done */
+	0x00f80132,
+/* 0x0068: nv_rd32 */
+	0x0728b7f1,
+	0xb906b4b6,
+	0xc9f002ec,
+	0x00bcd01f,
+/* 0x0078: nv_rd32_wait */
+	0xc800bccf,
+	0x1bf41fcc,
+	0x06a7f0fa,
+	0x010321f5,
+	0xf840bfcf,
+/* 0x008d: nv_wr32 */
+	0x28b7f100,
+	0x06b4b607,
+	0xb980bfd0,
+	0xc9f002ec,
+	0x1ec9f01f,
+/* 0x00a3: nv_wr32_wait */
+	0xcf00bcd0,
+	0xccc800bc,
+	0xfa1bf41f,
+/* 0x00ae: watchdog_reset */
+	0x87f100f8,
+	0x84b60430,
+	0x1ff9f006,
+	0xf8008fd0,
+/* 0x00bd: watchdog_clear */
+	0x3087f100,
+	0x0684b604,
+	0xf80080d0,
+/* 0x00c9: wait_donez */
+	0x3c87f100,
+	0x0684b608,
+	0x99f094bd,
+	0x0089d000,
+	0x081887f1,
+	0xd00684b6,
+/* 0x00e2: wait_done_wait_donez */
+	0x87f1008a,
+	0x84b60400,
+	0x0088cf06,
+	0xf4888aff,
+	0x87f1f31b,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00099,
+/* 0x0103: wait_doneo */
+	0xf100f800,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00099f0,
+	0x87f10089,
+	0x84b60818,
+	0x008ad006,
+/* 0x011c: wait_done_wait_doneo */
+	0x040087f1,
+	0xcf0684b6,
+	0x8aff0088,
+	0xf30bf488,
+	0x085c87f1,
+	0xbd0684b6,
+	0x0099f094,
+	0xf80089d0,
+/* 0x013d: mmctx_size */
+/* 0x013f: nv_mmctx_size_loop */
+	0x9894bd00,
+	0x85b600e8,
+	0x0180b61a,
+	0xbb0284b6,
+	0xe0b60098,
+	0x04efb804,
+	0xb9eb1bf4,
+	0x00f8029f,
+/* 0x015c: mmctx_xfer */
+	0x083c87f1,
+	0xbd0684b6,
+	0x0199f094,
+	0xf10089d0,
+	0xb6071087,
+	0x94bd0684,
+	0xf405bbfd,
+	0x8bd0090b,
+	0x0099f000,
+/* 0x0180: mmctx_base_disabled */
+	0xf405eefd,
+	0x8ed00c0b,
+	0xc08fd080,
+/* 0x018f: mmctx_multi_disabled */
+	0xb70199f0,
+	0xc8010080,
+	0xb4b600ab,
+	0x0cb9f010,
+	0xb601aec8,
+	0xbefd11e4,
+	0x008bd005,
+/* 0x01a8: mmctx_exec_loop */
+/* 0x01a8: mmctx_wait_free */
+	0xf0008ecf,
+	0x0bf41fe4,
+	0x00ce98fa,
+	0xd005e9fd,
+	0xc0b6c08e,
+	0x04cdb804,
+	0xc8e81bf4,
+	0x1bf402ab,
+/* 0x01c9: mmctx_fini_wait */
+	0x008bcf18,
+	0xb01fb4f0,
+	0x1bf410b4,
+	0x02a7f0f7,
+	0xf4c921f4,
+/* 0x01de: mmctx_stop */
+	0xabc81b0e,
+	0x10b4b600,
+	0xf00cb9f0,
+	0x8bd012b9,
+/* 0x01ed: mmctx_stop_wait */
+	0x008bcf00,
+	0xf412bbc8,
+/* 0x01f6: mmctx_done */
+	0x87f1fa1b,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00199,
+/* 0x0207: strand_wait */
+	0xf900f800,
+	0x02a7f0a0,
+	0xfcc921f4,
+/* 0x0213: strand_pre */
+	0xf100f8a0,
+	0xf04afc87,
+	0x97f00283,
+	0x0089d00c,
+	0x020721f5,
+/* 0x0226: strand_post */
+	0x87f100f8,
+	0x83f04afc,
+	0x0d97f002,
+	0xf50089d0,
+	0xf8020721,
+/* 0x0239: strand_set */
+	0xfca7f100,
+	0x02a3f04f,
+	0x0500aba2,
+	0xd00fc7f0,
+	0xc7f000ac,
+	0x00bcd00b,
+	0x020721f5,
+	0xf000aed0,
+	0xbcd00ac7,
+	0x0721f500,
+/* 0x0263: strand_ctx_init */
+	0xf100f802,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00399f0,
+	0x21f50089,
+	0xe7f00213,
+	0x3921f503,
+	0xfca7f102,
+	0x02a3f046,
+	0x0400aba0,
+	0xf040a0d0,
+	0xbcd001c7,
+	0x0721f500,
+	0x010c9202,
+	0xf000acd0,
+	0xbcd002c7,
+	0x0721f500,
+	0x2621f502,
+	0x8087f102,
+	0x0684b608,
+	0xb70089cf,
+	0x95220080,
+/* 0x02ba: ctx_init_strand_loop */
+	0x8ed008fe,
+	0x408ed000,
+	0xb6808acf,
+	0xa0b606a5,
+	0x00eabb01,
+	0xb60480b6,
+	0x1bf40192,
+	0x08e4b6e8,
+	0xf1f2efbc,
+	0xb6085c87,
+	0x94bd0684,
+	0xd00399f0,
+	0x00f80089,
+/* 0x02ec: error */
+	0xe7f1e0f9,
+	0xe3f09814,
+	0x8d21f440,
+	0x041ce0b7,
+	0xf401f7f0,
+	0xe0fc8d21,
+/* 0x0306: init */
+	0x04bd00f8,
+	0xf10004fe,
+	0xf0120017,
+	0x12d00227,
+	0x3e17f100,
+	0x0010fe04,
+	0x040017f1,
+	0xf0c010d0,
+	0x12d00427,
+	0x1031f400,
+	0x060817f1,
+	0xcf0614b6,
+	0x37f00012,
+	0x1f24f001,
+	0xb60432bb,
+	0x02800132,
+	0x04038003,
+	0x040010b7,
+	0x800012cf,
+	0x27f10002,
+	0x24b60800,
+	0x0022cf06,
+/* 0x035f: init_find_chipset */
+	0xb65817f0,
+	0x13980c10,
+	0x0432b800,
+	0xb00b0bf4,
+	0x1bf40034,
+/* 0x0373: init_context */
+	0xf100f8f1,
+	0xb6080027,
+	0x22cf0624,
+	0xf134bd40,
+	0xb6070047,
+	0x25950644,
+	0x0045d008,
+	0xbd4045d0,
+	0x58f4bde4,
+	0x1f58021e,
+	0x020e4003,
+	0xf5040f40,
+	0xbb013d21,
+	0x3fbb002f,
+	0x041e5800,
+	0x40051f58,
+	0x0f400a0e,
+	0x3d21f50c,
+	0x030e9801,
+	0xbb00effd,
+	0x3ebb002e,
+	0x0040b700,
+	0x0235b613,
+	0xb60043d0,
+	0x35b60825,
+	0x0120b606,
+	0xb60130b6,
+	0x34b60824,
+	0x022fb908,
+	0x026321f5,
+	0xf1003fbb,
+	0xb6080017,
+	0x13d00614,
+	0x0010b740,
+	0xf024bd08,
+	0x12d01f29,
+/* 0x0401: main */
+	0x0031f400,
+	0xf00028f4,
+	0x21f41cd7,
+	0xf401f439,
+	0xf404e4b0,
+	0x81fe1e18,
+	0x0627f001,
+	0x12fd20bd,
+	0x01e4b604,
+	0xfe051efd,
+	0x21f50018,
+	0x0ef404c3,
+/* 0x0431: main_not_ctx_xfer */
+	0x10ef94d3,
+	0xf501f5f0,
+	0xf402ec21,
+/* 0x043e: ih */
+	0x80f9c60e,
+	0xf90188fe,
+	0xf990f980,
+	0xf9b0f9a0,
+	0xf9e0f9d0,
+	0x800acff0,
+	0xf404abc4,
+	0xb7f11d0b,
+	0xd7f01900,
+	0x40becf1c,
+	0xf400bfcf,
+	0xb0b70421,
+	0xe7f00400,
+	0x00bed001,
+/* 0x0474: ih_no_fifo */
+	0xfc400ad0,
+	0xfce0fcf0,
+	0xfcb0fcd0,
+	0xfc90fca0,
+	0x0088fe80,
+	0x32f480fc,
+/* 0x048f: hub_barrier_done */
+	0xf001f800,
+	0x0e9801f7,
+	0x04febb00,
+	0x9418e7f1,
+	0xf440e3f0,
+	0x00f88d21,
+/* 0x04a4: ctx_redswitch */
+	0x0614e7f1,
+	0xf006e4b6,
+	0xefd020f7,
+	0x08f7f000,
+/* 0x04b4: ctx_redswitch_delay */
+	0xf401f2b6,
+	0xf7f1fd1b,
+	0xefd00a20,
+/* 0x04c3: ctx_xfer */
+	0xf100f800,
+	0xb60a0417,
+	0x1fd00614,
+	0x0711f400,
+	0x04a421f5,
+/* 0x04d4: ctx_xfer_not_load */
+	0x4afc17f1,
+	0xf00213f0,
+	0x12d00c27,
+	0x0721f500,
+	0xfc27f102,
+	0x0223f047,
+	0xf00020d0,
+	0x20b6012c,
+	0x0012d003,
+	0xf001acf0,
+	0xb7f002a5,
+	0x50b3f000,
+	0xb6000c98,
+	0xbcbb0fc4,
+	0x010c9800,
+	0xf0020d98,
+	0x21f500e7,
+	0xacf0015c,
+	0x04a5f001,
+	0x4000b7f1,
+	0x9850b3f0,
+	0xc4b6000c,
+	0x00bcbb0f,
+	0x98050c98,
+	0x0f98060d,
+	0x00e7f104,
+	0x5c21f508,
+	0x0721f501,
+	0x0601f402,
+/* 0x054b: ctx_xfer_post */
+	0xf11412f4,
+	0xf04afc17,
+	0x27f00213,
+	0x0012d00d,
+	0x020721f5,
+/* 0x055c: ctx_xfer_done */
+	0x048f21f5,
+	0x000000f8,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc
index 98acddb2c5b..acfc457654b 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc
@@ -24,11 +24,11 @@
  */
 
 /* To build:
- *    m4 nvc0_grhub.fuc | envyas -a -w -m fuc -V nva3 -o nvc0_grhub.fuc.h
+ *    m4 hubnvc0.fuc | envyas -a -w -m fuc -V fuc3 -o hubnvc0.fuc.h
  */
 
 .section #nvc0_grhub_data
-include(`nvc0_graph.fuc')
+include(`nvc0.fuc')
 gpc_count:		.b32 0
 rop_count:		.b32 0
 cmd_queue:		queue_init
@@ -161,11 +161,11 @@ xfer_data: 		.b32 0
 .section #nvc0_grhub_code
 bra #init
 define(`include_code')
-include(`nvc0_graph.fuc')
+include(`nvc0.fuc')
 
 // reports an exception to the host
 //
-// In: $r15 error code (see nvc0_graph.fuc)
+// In: $r15 error code (see nvc0.fuc)
 //
 error:
 	push $r14
diff --git a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h
index c5ed307abeb..85a8d556f48 100644
--- a/drivers/gpu/drm/nouveau/nvc0_grhub.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h
@@ -1,6 +1,9 @@
 uint32_t nvc0_grhub_data[] = {
+/* 0x0000: gpc_count */
 	0x00000000,
+/* 0x0004: rop_count */
 	0x00000000,
+/* 0x0008: cmd_queue */
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -19,9 +22,13 @@ uint32_t nvc0_grhub_data[] = {
 	0x00000000,
 	0x00000000,
 	0x00000000,
+/* 0x0050: hub_mmio_list_head */
 	0x00000000,
+/* 0x0054: hub_mmio_list_tail */
 	0x00000000,
+/* 0x0058: ctx_current */
 	0x00000000,
+/* 0x005c: chipsets */
 	0x000000c0,
 	0x013c00a0,
 	0x000000c1,
@@ -39,6 +46,7 @@ uint32_t nvc0_grhub_data[] = {
 	0x000000d9,
 	0x01dc0140,
 	0x00000000,
+/* 0x00a0: nvc0_hub_mmio_head */
 	0x0417e91c,
 	0x04400204,
 	0x28404004,
@@ -78,7 +86,10 @@ uint32_t nvc0_grhub_data[] = {
 	0x08408800,
 	0x0c408900,
 	0x00408980,
+/* 0x013c: nvc0_hub_mmio_tail */
 	0x044064c0,
+/* 0x0140: nvc1_hub_mmio_tail */
+/* 0x0140: nvd9_hub_mmio_head */
 	0x0417e91c,
 	0x04400204,
 	0x24404004,
@@ -118,6 +129,7 @@ uint32_t nvc0_grhub_data[] = {
 	0x08408800,
 	0x0c408900,
 	0x00408980,
+/* 0x01dc: nvd9_hub_mmio_tail */
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -127,7 +139,10 @@ uint32_t nvc0_grhub_data[] = {
 	0x00000000,
 	0x00000000,
 	0x00000000,
+/* 0x0200: chan_data */
+/* 0x0200: chan_mmio_count */
 	0x00000000,
+/* 0x0204: chan_mmio_address */
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -191,17 +206,20 @@ uint32_t nvc0_grhub_data[] = {
 	0x00000000,
 	0x00000000,
 	0x00000000,
+/* 0x0300: xfer_data */
 	0x00000000,
 };
 
 uint32_t nvc0_grhub_code[] = {
 	0x03090ef5,
+/* 0x0004: queue_put */
 	0x9800d898,
 	0x86f001d9,
 	0x0489b808,
 	0xf00c1bf4,
 	0x21f502f7,
 	0x00f802ec,
+/* 0x001c: queue_put_next */
 	0xb60798c4,
 	0x8dbb0384,
 	0x0880b600,
@@ -209,6 +227,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x90b6018f,
 	0x0f94f001,
 	0xf801d980,
+/* 0x0039: queue_get */
 	0x0131f400,
 	0x9800d898,
 	0x89b801d9,
@@ -220,37 +239,46 @@ uint32_t nvc0_grhub_code[] = {
 	0x80b6019f,
 	0x0f84f001,
 	0xf400d880,
+/* 0x0066: queue_get_done */
 	0x00f80132,
+/* 0x0068: nv_rd32 */
 	0x0728b7f1,
 	0xb906b4b6,
 	0xc9f002ec,
 	0x00bcd01f,
+/* 0x0078: nv_rd32_wait */
 	0xc800bccf,
 	0x1bf41fcc,
 	0x06a7f0fa,
 	0x010321f5,
 	0xf840bfcf,
+/* 0x008d: nv_wr32 */
 	0x28b7f100,
 	0x06b4b607,
 	0xb980bfd0,
 	0xc9f002ec,
 	0x1ec9f01f,
+/* 0x00a3: nv_wr32_wait */
 	0xcf00bcd0,
 	0xccc800bc,
 	0xfa1bf41f,
+/* 0x00ae: watchdog_reset */
 	0x87f100f8,
 	0x84b60430,
 	0x1ff9f006,
 	0xf8008fd0,
+/* 0x00bd: watchdog_clear */
 	0x3087f100,
 	0x0684b604,
 	0xf80080d0,
+/* 0x00c9: wait_donez */
 	0x3c87f100,
 	0x0684b608,
 	0x99f094bd,
 	0x0089d000,
 	0x081887f1,
 	0xd00684b6,
+/* 0x00e2: wait_done_wait_donez */
 	0x87f1008a,
 	0x84b60400,
 	0x0088cf06,
@@ -259,6 +287,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x84b6085c,
 	0xf094bd06,
 	0x89d00099,
+/* 0x0103: wait_doneo */
 	0xf100f800,
 	0xb6083c87,
 	0x94bd0684,
@@ -266,6 +295,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x87f10089,
 	0x84b60818,
 	0x008ad006,
+/* 0x011c: wait_done_wait_doneo */
 	0x040087f1,
 	0xcf0684b6,
 	0x8aff0088,
@@ -274,6 +304,8 @@ uint32_t nvc0_grhub_code[] = {
 	0xbd0684b6,
 	0x0099f094,
 	0xf80089d0,
+/* 0x013d: mmctx_size */
+/* 0x013f: nv_mmctx_size_loop */
 	0x9894bd00,
 	0x85b600e8,
 	0x0180b61a,
@@ -282,6 +314,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x04efb804,
 	0xb9eb1bf4,
 	0x00f8029f,
+/* 0x015c: mmctx_xfer */
 	0x083c87f1,
 	0xbd0684b6,
 	0x0199f094,
@@ -291,9 +324,11 @@ uint32_t nvc0_grhub_code[] = {
 	0xf405bbfd,
 	0x8bd0090b,
 	0x0099f000,
+/* 0x0180: mmctx_base_disabled */
 	0xf405eefd,
 	0x8ed00c0b,
 	0xc08fd080,
+/* 0x018f: mmctx_multi_disabled */
 	0xb70199f0,
 	0xc8010080,
 	0xb4b600ab,
@@ -301,6 +336,8 @@ uint32_t nvc0_grhub_code[] = {
 	0xb601aec8,
 	0xbefd11e4,
 	0x008bd005,
+/* 0x01a8: mmctx_exec_loop */
+/* 0x01a8: mmctx_wait_free */
 	0xf0008ecf,
 	0x0bf41fe4,
 	0x00ce98fa,
@@ -309,34 +346,42 @@ uint32_t nvc0_grhub_code[] = {
 	0x04cdb804,
 	0xc8e81bf4,
 	0x1bf402ab,
+/* 0x01c9: mmctx_fini_wait */
 	0x008bcf18,
 	0xb01fb4f0,
 	0x1bf410b4,
 	0x02a7f0f7,
 	0xf4c921f4,
+/* 0x01de: mmctx_stop */
 	0xabc81b0e,
 	0x10b4b600,
 	0xf00cb9f0,
 	0x8bd012b9,
+/* 0x01ed: mmctx_stop_wait */
 	0x008bcf00,
 	0xf412bbc8,
+/* 0x01f6: mmctx_done */
 	0x87f1fa1b,
 	0x84b6085c,
 	0xf094bd06,
 	0x89d00199,
+/* 0x0207: strand_wait */
 	0xf900f800,
 	0x02a7f0a0,
 	0xfcc921f4,
+/* 0x0213: strand_pre */
 	0xf100f8a0,
 	0xf04afc87,
 	0x97f00283,
 	0x0089d00c,
 	0x020721f5,
+/* 0x0226: strand_post */
 	0x87f100f8,
 	0x83f04afc,
 	0x0d97f002,
 	0xf50089d0,
 	0xf8020721,
+/* 0x0239: strand_set */
 	0xfca7f100,
 	0x02a3f04f,
 	0x0500aba2,
@@ -347,6 +392,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xf000aed0,
 	0xbcd00ac7,
 	0x0721f500,
+/* 0x0263: strand_ctx_init */
 	0xf100f802,
 	0xb6083c87,
 	0x94bd0684,
@@ -369,6 +415,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x0684b608,
 	0xb70089cf,
 	0x95220080,
+/* 0x02ba: ctx_init_strand_loop */
 	0x8ed008fe,
 	0x408ed000,
 	0xb6808acf,
@@ -382,6 +429,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x94bd0684,
 	0xd00399f0,
 	0x00f80089,
+/* 0x02ec: error */
 	0xe7f1e0f9,
 	0xe4b60814,
 	0x00efd006,
@@ -389,6 +437,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xf006e4b6,
 	0xefd001f7,
 	0xf8e0fc00,
+/* 0x0309: init */
 	0xfe04bd00,
 	0x07fe0004,
 	0x0017f100,
@@ -429,11 +478,13 @@ uint32_t nvc0_grhub_code[] = {
 	0x080027f1,
 	0xcf0624b6,
 	0xf7f00022,
+/* 0x03a9: init_find_chipset */
 	0x08f0b654,
 	0xb800f398,
 	0x0bf40432,
 	0x0034b00b,
 	0xf8f11bf4,
+/* 0x03bd: init_context */
 	0x0017f100,
 	0x02fe5801,
 	0xf003ff58,
@@ -454,6 +505,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x001fbb02,
 	0xf1000398,
 	0xf0200047,
+/* 0x040e: init_gpc */
 	0x4ea05043,
 	0x1fb90804,
 	0x8d21f402,
@@ -467,6 +519,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xf7f00100,
 	0x8d21f402,
 	0x08004ea0,
+/* 0x0440: init_gpc_wait */
 	0xc86821f4,
 	0x0bf41fff,
 	0x044ea0fa,
@@ -479,6 +532,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xb74021d0,
 	0xbd080020,
 	0x1f19f014,
+/* 0x0473: main */
 	0xf40021d0,
 	0x28f40031,
 	0x08d7f000,
@@ -517,6 +571,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x94bd0684,
 	0xd00699f0,
 	0x0ef40089,
+/* 0x0509: chsw_prev_no_next */
 	0xb920f931,
 	0x32f40212,
 	0x0232f401,
@@ -524,10 +579,12 @@ uint32_t nvc0_grhub_code[] = {
 	0x17f120fc,
 	0x14b60b00,
 	0x0012d006,
+/* 0x0527: chsw_no_prev */
 	0xc8130ef4,
 	0x0bf41f23,
 	0x0131f40d,
 	0xf50232f4,
+/* 0x0537: chsw_done */
 	0xf1082921,
 	0xb60b0c17,
 	0x27f00614,
@@ -536,10 +593,12 @@ uint32_t nvc0_grhub_code[] = {
 	0xbd0684b6,
 	0x0499f094,
 	0xf50089d0,
+/* 0x0557: main_not_ctx_switch */
 	0xb0ff200e,
 	0x1bf401e4,
 	0x02f2b90d,
 	0x07b521f5,
+/* 0x0567: main_not_ctx_chan */
 	0xb0420ef4,
 	0x1bf402e4,
 	0x3c87f12e,
@@ -553,14 +612,17 @@ uint32_t nvc0_grhub_code[] = {
 	0xf094bd06,
 	0x89d00799,
 	0x110ef400,
+/* 0x0598: main_not_ctx_save */
 	0xf010ef94,
 	0x21f501f5,
 	0x0ef502ec,
+/* 0x05a6: main_done */
 	0x17f1fed1,
 	0x14b60820,
 	0xf024bd06,
 	0x12d01f29,
 	0xbe0ef500,
+/* 0x05b9: ih */
 	0xfe80f9fe,
 	0x80f90188,
 	0xa0f990f9,
@@ -574,16 +636,19 @@ uint32_t nvc0_grhub_code[] = {
 	0x21f400bf,
 	0x00b0b704,
 	0x01e7f004,
+/* 0x05ef: ih_no_fifo */
 	0xe400bed0,
 	0xf40100ab,
 	0xd7f00d0b,
 	0x01e7f108,
 	0x0421f440,
+/* 0x0600: ih_no_ctxsw */
 	0x0104b7f1,
 	0xabffb0bd,
 	0x0d0bf4b4,
 	0x0c1ca7f1,
 	0xd006a4b6,
+/* 0x0616: ih_no_other */
 	0x0ad000ab,
 	0xfcf0fc40,
 	0xfcd0fce0,
@@ -591,32 +656,40 @@ uint32_t nvc0_grhub_code[] = {
 	0xfe80fc90,
 	0x80fc0088,
 	0xf80032f4,
+/* 0x0631: ctx_4160s */
 	0x60e7f101,
 	0x40e3f041,
 	0xf401f7f0,
+/* 0x063e: ctx_4160s_wait */
 	0x21f48d21,
 	0x04ffc868,
 	0xf8fa0bf4,
+/* 0x0649: ctx_4160c */
 	0x60e7f100,
 	0x40e3f041,
 	0x21f4f4bd,
+/* 0x0657: ctx_4170s */
 	0xf100f88d,
 	0xf04170e7,
 	0xf5f040e3,
 	0x8d21f410,
+/* 0x0666: ctx_4170w */
 	0xe7f100f8,
 	0xe3f04170,
 	0x6821f440,
 	0xf410f4f0,
 	0x00f8f31b,
+/* 0x0678: ctx_redswitch */
 	0x0614e7f1,
 	0xf106e4b6,
 	0xd00270f7,
 	0xf7f000ef,
+/* 0x0689: ctx_redswitch_delay */
 	0x01f2b608,
 	0xf1fd1bf4,
 	0xd00770f7,
 	0x00f800ef,
+/* 0x0698: ctx_86c */
 	0x086ce7f1,
 	0xd006e4b6,
 	0xe7f100ef,
@@ -625,6 +698,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xa86ce7f1,
 	0xf441e3f0,
 	0x00f88d21,
+/* 0x06b8: ctx_load */
 	0x083c87f1,
 	0xbd0684b6,
 	0x0599f094,
@@ -639,6 +713,7 @@ uint32_t nvc0_grhub_code[] = {
 	0x0614b60a,
 	0xd00747f0,
 	0x14d00012,
+/* 0x06f1: ctx_chan_wait_0 */
 	0x4014cf40,
 	0xf41f44f0,
 	0x32d0fa1b,
@@ -688,6 +763,7 @@ uint32_t nvc0_grhub_code[] = {
 	0xbd0684b6,
 	0x0599f094,
 	0xf80089d0,
+/* 0x07b5: ctx_chan */
 	0x3121f500,
 	0xb821f506,
 	0x0ca7f006,
@@ -695,39 +771,48 @@ uint32_t nvc0_grhub_code[] = {
 	0xb60a1017,
 	0x27f00614,
 	0x0012d005,
+/* 0x07d0: ctx_chan_wait */
 	0xfd0012cf,
 	0x1bf40522,
 	0x4921f5fa,
+/* 0x07df: ctx_mmio_exec */
 	0x9800f806,
 	0x27f18103,
 	0x24b60a04,
 	0x0023d006,
+/* 0x07ee: ctx_mmio_loop */
 	0x34c434bd,
 	0x0f1bf4ff,
 	0x030057f1,
 	0xfa0653f0,
 	0x03f80535,
+/* 0x0800: ctx_mmio_pull */
 	0x98c04e98,
 	0x21f4c14f,
 	0x0830b68d,
 	0xf40112b6,
+/* 0x0812: ctx_mmio_done */
 	0x0398df1b,
 	0x0023d016,
 	0xf1800080,
 	0xf0020017,
 	0x01fa0613,
 	0xf803f806,
+/* 0x0829: ctx_xfer */
 	0x0611f400,
+/* 0x082f: ctx_xfer_pre */
 	0xf01102f4,
 	0x21f510f7,
 	0x21f50698,
 	0x11f40631,
+/* 0x083d: ctx_xfer_pre_load */
 	0x02f7f01c,
 	0x065721f5,
 	0x066621f5,
 	0x067821f5,
 	0x21f5f4bd,
 	0x21f50657,
+/* 0x0856: ctx_xfer_exec */
 	0x019806b8,
 	0x1427f116,
 	0x0624b604,
@@ -762,9 +847,11 @@ uint32_t nvc0_grhub_code[] = {
 	0x0a1017f1,
 	0xf00614b6,
 	0x12d00527,
+/* 0x08dd: ctx_xfer_post_save_wait */
 	0x0012cf00,
 	0xf40522fd,
 	0x02f4fa1b,
+/* 0x08e9: ctx_xfer_post */
 	0x02f7f032,
 	0x065721f5,
 	0x21f5f4bd,
@@ -776,7 +863,9 @@ uint32_t nvc0_grhub_code[] = {
 	0x11fd8001,
 	0x070bf405,
 	0x07df21f5,
+/* 0x0914: ctx_xfer_no_post_mmio */
 	0x064921f5,
+/* 0x0918: ctx_xfer_done */
 	0x000000f8,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc
new file mode 100644
index 00000000000..138eeaa2866
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc
@@ -0,0 +1,780 @@
+/* fuc microcode for nve0 PGRAPH/HUB
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/* To build:
+ *    m4 nve0_grhub.fuc | envyas -a -w -m fuc -V nva3 -o nve0_grhub.fuc.h
+ */
+
+.section #nve0_grhub_data
+include(`nve0.fuc')
+gpc_count:		.b32 0
+rop_count:		.b32 0
+cmd_queue:		queue_init
+hub_mmio_list_head:	.b32 0
+hub_mmio_list_tail:	.b32 0
+
+ctx_current:		.b32 0
+
+chipsets:
+.b8  0xe4 0 0 0
+.b16 #nve4_hub_mmio_head
+.b16 #nve4_hub_mmio_tail
+.b8  0xe7 0 0 0
+.b16 #nve4_hub_mmio_head
+.b16 #nve4_hub_mmio_tail
+.b8  0 0 0 0
+
+nve4_hub_mmio_head:
+mmctx_data(0x17e91c, 2)
+mmctx_data(0x400204, 2)
+mmctx_data(0x404010, 7)
+mmctx_data(0x4040a8, 9)
+mmctx_data(0x4040d0, 7)
+mmctx_data(0x4040f8, 1)
+mmctx_data(0x404130, 3)
+mmctx_data(0x404150, 3)
+mmctx_data(0x404164, 1)
+mmctx_data(0x4041a0, 4)
+mmctx_data(0x404200, 4)
+mmctx_data(0x404404, 14)
+mmctx_data(0x404460, 4)
+mmctx_data(0x404480, 1)
+mmctx_data(0x404498, 1)
+mmctx_data(0x404604, 4)
+mmctx_data(0x404618, 4)
+mmctx_data(0x40462c, 2)
+mmctx_data(0x404640, 1)
+mmctx_data(0x404654, 1)
+mmctx_data(0x404660, 1)
+mmctx_data(0x404678, 19)
+mmctx_data(0x4046c8, 3)
+mmctx_data(0x404700, 3)
+mmctx_data(0x404718, 10)
+mmctx_data(0x404744, 2)
+mmctx_data(0x404754, 1)
+mmctx_data(0x405800, 1)
+mmctx_data(0x405830, 3)
+mmctx_data(0x405854, 1)
+mmctx_data(0x405870, 4)
+mmctx_data(0x405a00, 2)
+mmctx_data(0x405a18, 1)
+mmctx_data(0x405b00, 1)
+mmctx_data(0x405b10, 1)
+mmctx_data(0x406020, 1)
+mmctx_data(0x406028, 4)
+mmctx_data(0x4064a8, 2)
+mmctx_data(0x4064b4, 2)
+mmctx_data(0x4064c0, 12)
+mmctx_data(0x4064fc, 1)
+mmctx_data(0x407040, 1)
+mmctx_data(0x407804, 1)
+mmctx_data(0x40780c, 6)
+mmctx_data(0x4078bc, 1)
+mmctx_data(0x408000, 7)
+mmctx_data(0x408064, 1)
+mmctx_data(0x408800, 3)
+mmctx_data(0x408840, 1)
+mmctx_data(0x408900, 3)
+mmctx_data(0x408980, 1)
+nve4_hub_mmio_tail:
+
+.align 256
+chan_data:
+chan_mmio_count:	.b32 0
+chan_mmio_address:	.b32 0
+
+.align 256
+xfer_data: 		.b32 0
+
+.section #nve0_grhub_code
+bra #init
+define(`include_code')
+include(`nve0.fuc')
+
+// reports an exception to the host
+//
+// In: $r15 error code (see nve0.fuc)
+//
+error:
+	push $r14
+	mov $r14 0x814
+	shl b32 $r14 6
+	iowr I[$r14 + 0x000] $r15	// CC_SCRATCH[5] = error code
+	mov $r14 0xc1c
+	shl b32 $r14 6
+	mov $r15 1
+	iowr I[$r14 + 0x000] $r15	// INTR_UP_SET
+	pop $r14
+	ret
+
+// HUB fuc initialisation, executed by triggering ucode start, will
+// fall through to main loop after completion.
+//
+// Input:
+//   CC_SCRATCH[0]: chipset (PMC_BOOT_0 read returns 0x0bad0bad... sigh)
+//
+// Output:
+//   CC_SCRATCH[0]:
+//	     31:31: set to signal completion
+//   CC_SCRATCH[1]:
+//	      31:0: total PGRAPH context size
+//
+init:
+	clear b32 $r0
+	mov $sp $r0
+	mov $xdbase $r0
+
+	// enable fifo access
+	mov $r1 0x1200
+	mov $r2 2
+	iowr I[$r1 + 0x000] $r2	// FIFO_ENABLE
+
+	// setup i0 handler, and route all interrupts to it
+	mov $r1 #ih
+	mov $iv0 $r1
+	mov $r1 0x400
+	iowr I[$r1 + 0x300] $r0	// INTR_DISPATCH
+
+	// route HUB_CHANNEL_SWITCH to fuc interrupt 8
+	mov $r3 0x404
+	shl b32 $r3 6
+	mov $r2 0x2003		// { HUB_CHANNEL_SWITCH, ZERO } -> intr 8
+	iowr I[$r3 + 0x000] $r2
+
+	// not sure what these are, route them because NVIDIA does, and
+	// the IRQ handler will signal the host if we ever get one.. we
+	// may find out if/why we need to handle these if so..
+	//
+	mov $r2 0x2004
+	iowr I[$r3 + 0x004] $r2 // { 0x04, ZERO } -> intr 9
+	mov $r2 0x200b
+	iowr I[$r3 + 0x008] $r2 // { 0x0b, ZERO } -> intr 10
+	mov $r2 0x200c
+	iowr I[$r3 + 0x01c] $r2 // { 0x0c, ZERO } -> intr 15
+
+	// enable all INTR_UP interrupts
+	mov $r2 0xc24
+	shl b32 $r2 6
+	not b32 $r3 $r0
+	iowr I[$r2] $r3
+
+	// enable fifo, ctxsw, 9, 10, 15 interrupts
+	mov $r2 -0x78fc		// 0x8704
+	sethi $r2 0
+	iowr I[$r1 + 0x000] $r2	// INTR_EN_SET
+
+	// fifo level triggered, rest edge
+	sub b32 $r1 0x100
+	mov $r2 4
+	iowr I[$r1] $r2
+
+	// enable interrupts
+	bset $flags ie0
+
+	// fetch enabled GPC/ROP counts
+	mov $r14 -0x69fc	// 0x409604
+	sethi $r14 0x400000
+	call #nv_rd32
+	extr $r1 $r15 16:20
+	st b32 D[$r0 + #rop_count] $r1
+	and $r15 0x1f
+	st b32 D[$r0 + #gpc_count] $r15
+
+	// set BAR_REQMASK to GPC mask
+	mov $r1 1
+	shl b32 $r1 $r15
+	sub b32 $r1 1
+	mov $r2 0x40c
+	shl b32 $r2 6
+	iowr I[$r2 + 0x000] $r1
+	iowr I[$r2 + 0x100] $r1
+
+	// find context data for this chipset
+	mov $r2 0x800
+	shl b32 $r2 6
+	iord $r2 I[$r2 + 0x000]		// CC_SCRATCH[0]
+	mov $r15 #chipsets - 8
+	init_find_chipset:
+		add b32 $r15 8
+		ld b32 $r3 D[$r15 + 0x00]
+		cmpu b32 $r3 $r2
+		bra e #init_context
+		cmpu b32 $r3 0
+		bra ne #init_find_chipset
+		// unknown chipset
+		ret
+
+	// context size calculation, reserve first 256 bytes for use by fuc
+	init_context:
+	mov $r1 256
+
+	// calculate size of mmio context data
+	ld b16 $r14 D[$r15 + 4]
+	ld b16 $r15 D[$r15 + 6]
+	sethi $r14 0
+	st b32 D[$r0 + #hub_mmio_list_head] $r14
+	st b32 D[$r0 + #hub_mmio_list_tail] $r15
+	call #mmctx_size
+
+	// set mmctx base addresses now so we don't have to do it later,
+	// they don't (currently) ever change
+	mov $r3 0x700
+	shl b32 $r3 6
+	shr b32 $r4 $r1 8
+	iowr I[$r3 + 0x000] $r4		// MMCTX_SAVE_SWBASE
+	iowr I[$r3 + 0x100] $r4		// MMCTX_LOAD_SWBASE
+	add b32 $r3 0x1300
+	add b32 $r1 $r15
+	shr b32 $r15 2
+	iowr I[$r3 + 0x000] $r15	// MMCTX_LOAD_COUNT, wtf for?!?
+
+	// strands, base offset needs to be aligned to 256 bytes
+	shr b32 $r1 8
+	add b32 $r1 1
+	shl b32 $r1 8
+	mov b32 $r15 $r1
+	call #strand_ctx_init
+	add b32 $r1 $r15
+
+	// initialise each GPC in sequence by passing in the offset of its
+	// context data in GPCn_CC_SCRATCH[1], and starting its FUC (which
+	// has previously been uploaded by the host) running.
+	//
+	// the GPC fuc init sequence will set GPCn_CC_SCRATCH[0] bit 31
+	// when it has completed, and return the size of its context data
+	// in GPCn_CC_SCRATCH[1]
+	//
+	ld b32 $r3 D[$r0 + #gpc_count]
+	mov $r4 0x2000
+	sethi $r4 0x500000
+	init_gpc:
+		// setup, and start GPC ucode running
+		add b32 $r14 $r4 0x804
+		mov b32 $r15 $r1
+		call #nv_wr32			// CC_SCRATCH[1] = ctx offset
+		add b32 $r14 $r4 0x800
+		mov b32 $r15 $r2
+		call #nv_wr32			// CC_SCRATCH[0] = chipset
+		add b32 $r14 $r4 0x10c
+		clear b32 $r15
+		call #nv_wr32
+		add b32 $r14 $r4 0x104
+		call #nv_wr32			// ENTRY
+		add b32 $r14 $r4 0x100
+		mov $r15 2			// CTRL_START_TRIGGER
+		call #nv_wr32			// CTRL
+
+		// wait for it to complete, and adjust context size
+		add b32 $r14 $r4 0x800
+		init_gpc_wait:
+			call #nv_rd32
+			xbit $r15 $r15 31
+			bra e #init_gpc_wait
+		add b32 $r14 $r4 0x804
+		call #nv_rd32
+		add b32 $r1 $r15
+
+		// next!
+		add b32 $r4 0x8000
+		sub b32 $r3 1
+		bra ne #init_gpc
+
+	// save context size, and tell host we're ready
+	mov $r2 0x800
+	shl b32 $r2 6
+	iowr I[$r2 + 0x100] $r1		// CC_SCRATCH[1]  = context size
+	add b32 $r2 0x800
+	clear b32 $r1
+	bset $r1 31
+	iowr I[$r2 + 0x000] $r1		// CC_SCRATCH[0] |= 0x80000000
+
+// Main program loop, very simple, sleeps until woken up by the interrupt
+// handler, pulls a command from the queue and executes its handler
+//
+main:
+	// sleep until we have something to do
+	bset $flags $p0
+	sleep $p0
+	mov $r13 #cmd_queue
+	call #queue_get
+	bra $p1 #main
+
+	// context switch, requested by GPU?
+	cmpu b32 $r14 0x4001
+	bra ne #main_not_ctx_switch
+		trace_set(T_AUTO)
+		mov $r1 0xb00
+		shl b32 $r1 6
+		iord $r2 I[$r1 + 0x100]		// CHAN_NEXT
+		iord $r1 I[$r1 + 0x000]		// CHAN_CUR
+
+		xbit $r3 $r1 31
+		bra e #chsw_no_prev
+			xbit $r3 $r2 31
+			bra e #chsw_prev_no_next
+				push $r2
+				mov b32 $r2 $r1
+				trace_set(T_SAVE)
+				bclr $flags $p1
+				bset $flags $p2
+				call #ctx_xfer
+				trace_clr(T_SAVE);
+				pop $r2
+				trace_set(T_LOAD);
+				bset $flags $p1
+				call #ctx_xfer
+				trace_clr(T_LOAD);
+				bra #chsw_done
+			chsw_prev_no_next:
+				push $r2
+				mov b32 $r2 $r1
+				bclr $flags $p1
+				bclr $flags $p2
+				call #ctx_xfer
+				pop $r2
+				mov $r1 0xb00
+				shl b32 $r1 6
+				iowr I[$r1] $r2
+				bra #chsw_done
+		chsw_no_prev:
+			xbit $r3 $r2 31
+			bra e #chsw_done
+				bset $flags $p1
+				bclr $flags $p2
+				call #ctx_xfer
+
+		// ack the context switch request
+		chsw_done:
+		mov $r1 0xb0c
+		shl b32 $r1 6
+		mov $r2 1
+		iowr I[$r1 + 0x000] $r2		// 0x409b0c
+		trace_clr(T_AUTO)
+		bra #main
+
+	// request to set current channel? (*not* a context switch)
+	main_not_ctx_switch:
+	cmpu b32 $r14 0x0001
+	bra ne #main_not_ctx_chan
+		mov b32 $r2 $r15
+		call #ctx_chan
+		bra #main_done
+
+	// request to store current channel context?
+	main_not_ctx_chan:
+	cmpu b32 $r14 0x0002
+	bra ne #main_not_ctx_save
+		trace_set(T_SAVE)
+		bclr $flags $p1
+		bclr $flags $p2
+		call #ctx_xfer
+		trace_clr(T_SAVE)
+		bra #main_done
+
+	main_not_ctx_save:
+		shl b32 $r15 $r14 16
+		or $r15 E_BAD_COMMAND
+		call #error
+		bra #main
+
+	main_done:
+	mov $r1 0x820
+	shl b32 $r1 6
+	clear b32 $r2
+	bset $r2 31
+	iowr I[$r1 + 0x000] $r2		// CC_SCRATCH[0] |= 0x80000000
+	bra #main
+
+// interrupt handler
+ih:
+	push $r8
+	mov $r8 $flags
+	push $r8
+	push $r9
+	push $r10
+	push $r11
+	push $r13
+	push $r14
+	push $r15
+
+	// incoming fifo command?
+	iord $r10 I[$r0 + 0x200]	// INTR
+	and $r11 $r10 0x00000004
+	bra e #ih_no_fifo
+		// queue incoming fifo command for later processing
+		mov $r11 0x1900
+		mov $r13 #cmd_queue
+		iord $r14 I[$r11 + 0x100]	// FIFO_CMD
+		iord $r15 I[$r11 + 0x000]	// FIFO_DATA
+		call #queue_put
+		add b32 $r11 0x400
+		mov $r14 1
+		iowr I[$r11 + 0x000] $r14	// FIFO_ACK
+
+	// context switch request?
+	ih_no_fifo:
+	and $r11 $r10 0x00000100
+	bra e #ih_no_ctxsw
+		// enqueue a context switch for later processing
+		mov $r13 #cmd_queue
+		mov $r14 0x4001
+		call #queue_put
+
+	// anything we didn't handle, bring it to the host's attention
+	ih_no_ctxsw:
+	mov $r11 0x104
+	not b32 $r11
+	and $r11 $r10 $r11
+	bra e #ih_no_other
+		mov $r10 0xc1c
+		shl b32 $r10 6
+		iowr I[$r10] $r11	// INTR_UP_SET
+
+	// ack, and wake up main()
+	ih_no_other:
+	iowr I[$r0 + 0x100] $r10	// INTR_ACK
+
+	pop $r15
+	pop $r14
+	pop $r13
+	pop $r11
+	pop $r10
+	pop $r9
+	pop $r8
+	mov $flags $r8
+	pop $r8
+	bclr $flags $p0
+	iret
+
+// Again, not real sure
+//
+// In: $r15 value to set 0x404170 to
+//
+ctx_4170s:
+	mov $r14 0x4170
+	sethi $r14 0x400000
+	or $r15 0x10
+	call #nv_wr32
+	ret
+
+// Waits for a ctx_4170s() call to complete
+//
+ctx_4170w:
+	mov $r14 0x4170
+	sethi $r14 0x400000
+	call #nv_rd32
+	and $r15 0x10
+	bra ne #ctx_4170w
+	ret
+
+// Disables various things, waits a bit, and re-enables them..
+//
+// Not sure how exactly this helps, perhaps "ENABLE" is not such a
+// good description for the bits we turn off?  Anyways, without this,
+// funny things happen.
+//
+ctx_redswitch:
+	mov $r14 0x614
+	shl b32 $r14 6
+	mov $r15 0x270
+	iowr I[$r14] $r15	// HUB_RED_SWITCH = ENABLE_GPC, POWER_ALL
+	mov $r15 8
+	ctx_redswitch_delay:
+		sub b32 $r15 1
+		bra ne #ctx_redswitch_delay
+	mov $r15 0x770
+	iowr I[$r14] $r15	// HUB_RED_SWITCH = ENABLE_ALL, POWER_ALL
+	ret
+
+// Not a clue what this is for, except that unless the value is 0x10, the
+// strand context is saved (and presumably restored) incorrectly..
+//
+// In: $r15 value to set to (0x00/0x10 are used)
+//
+ctx_86c:
+	mov $r14 0x86c
+	shl b32 $r14 6
+	iowr I[$r14] $r15	// HUB(0x86c) = val
+	mov $r14 -0x75ec
+	sethi $r14 0x400000
+	call #nv_wr32		// ROP(0xa14) = val
+	mov $r14 -0x5794
+	sethi $r14 0x410000
+	call #nv_wr32		// GPC(0x86c) = val
+	ret
+
+// ctx_load - load's a channel's ctxctl data, and selects its vm
+//
+// In: $r2 channel address
+//
+ctx_load:
+	trace_set(T_CHAN)
+
+	// switch to channel, somewhat magic in parts..
+	mov $r10 12		// DONE_UNK12
+	call #wait_donez
+	mov $r1 0xa24
+	shl b32 $r1 6
+	iowr I[$r1 + 0x000] $r0	// 0x409a24
+	mov $r3 0xb00
+	shl b32 $r3 6
+	iowr I[$r3 + 0x100] $r2	// CHAN_NEXT
+	mov $r1 0xa0c
+	shl b32 $r1 6
+	mov $r4 7
+	iowr I[$r1 + 0x000] $r2 // MEM_CHAN
+	iowr I[$r1 + 0x100] $r4	// MEM_CMD
+	ctx_chan_wait_0:
+		iord $r4 I[$r1 + 0x100]
+		and $r4 0x1f
+		bra ne #ctx_chan_wait_0
+	iowr I[$r3 + 0x000] $r2	// CHAN_CUR
+
+	// load channel header, fetch PGRAPH context pointer
+	mov $xtargets $r0
+	bclr $r2 31
+	shl b32 $r2 4
+	add b32 $r2 2
+
+	trace_set(T_LCHAN)
+	mov $r1 0xa04
+	shl b32 $r1 6
+	iowr I[$r1 + 0x000] $r2		// MEM_BASE
+	mov $r1 0xa20
+	shl b32 $r1 6
+	mov $r2 0x0002
+	sethi $r2 0x80000000
+	iowr I[$r1 + 0x000] $r2		// MEM_TARGET = vram
+	mov $r1 0x10			// chan + 0x0210
+	mov $r2 #xfer_data
+	sethi $r2 0x00020000		// 16 bytes
+	xdld $r1 $r2
+	xdwait
+	trace_clr(T_LCHAN)
+
+	// update current context
+	ld b32 $r1 D[$r0 + #xfer_data + 4]
+	shl b32 $r1 24
+	ld b32 $r2 D[$r0 + #xfer_data + 0]
+	shr b32 $r2 8
+	or $r1 $r2
+	st b32 D[$r0 + #ctx_current] $r1
+
+	// set transfer base to start of context, and fetch context header
+	trace_set(T_LCTXH)
+	mov $r2 0xa04
+	shl b32 $r2 6
+	iowr I[$r2 + 0x000] $r1		// MEM_BASE
+	mov $r2 1
+	mov $r1 0xa20
+	shl b32 $r1 6
+	iowr I[$r1 + 0x000] $r2		// MEM_TARGET = vm
+	mov $r1 #chan_data
+	sethi $r1 0x00060000		// 256 bytes
+	xdld $r0 $r1
+	xdwait
+	trace_clr(T_LCTXH)
+
+	trace_clr(T_CHAN)
+	ret
+
+// ctx_chan - handler for HUB_SET_CHAN command, will set a channel as
+//            the active channel for ctxctl, but not actually transfer
+//            any context data.  intended for use only during initial
+//            context construction.
+//
+// In: $r2 channel address
+//
+ctx_chan:
+	call #ctx_load
+	mov $r10 12			// DONE_UNK12
+	call #wait_donez
+	mov $r1 0xa10
+	shl b32 $r1 6
+	mov $r2 5
+	iowr I[$r1 + 0x000] $r2		// MEM_CMD = 5 (???)
+	ctx_chan_wait:
+		iord $r2 I[$r1 + 0x000]
+		or $r2 $r2
+		bra ne #ctx_chan_wait
+	ret
+
+// Execute per-context state overrides list
+//
+// Only executed on the first load of a channel.  Might want to look into
+// removing this and having the host directly modify the channel's context
+// to change this state...  The nouveau DRM already builds this list as
+// it's definitely needed for NVIDIA's, so we may as well use it for now
+//
+// Input: $r1 mmio list length
+//
+ctx_mmio_exec:
+	// set transfer base to be the mmio list
+	ld b32 $r3 D[$r0 + #chan_mmio_address]
+	mov $r2 0xa04
+	shl b32 $r2 6
+	iowr I[$r2 + 0x000] $r3		// MEM_BASE
+
+	clear b32 $r3
+	ctx_mmio_loop:
+		// fetch next 256 bytes of mmio list if necessary
+		and $r4 $r3 0xff
+		bra ne #ctx_mmio_pull
+			mov $r5 #xfer_data
+			sethi $r5 0x00060000	// 256 bytes
+			xdld $r3 $r5
+			xdwait
+
+		// execute a single list entry
+		ctx_mmio_pull:
+		ld b32 $r14 D[$r4 + #xfer_data + 0x00]
+		ld b32 $r15 D[$r4 + #xfer_data + 0x04]
+		call #nv_wr32
+
+		// next!
+		add b32 $r3 8
+		sub b32 $r1 1
+		bra ne #ctx_mmio_loop
+
+	// set transfer base back to the current context
+	ctx_mmio_done:
+	ld b32 $r3 D[$r0 + #ctx_current]
+	iowr I[$r2 + 0x000] $r3		// MEM_BASE
+
+	// disable the mmio list now, we don't need/want to execute it again
+	st b32 D[$r0 + #chan_mmio_count] $r0
+	mov $r1 #chan_data
+	sethi $r1 0x00060000		// 256 bytes
+	xdst $r0 $r1
+	xdwait
+	ret
+
+// Transfer HUB context data between GPU and storage area
+//
+// In: $r2 channel address
+//     $p1 clear on save, set on load
+//     $p2 set if opposite direction done/will be done, so:
+//		on save it means: "a load will follow this save"
+//		on load it means: "a save preceeded this load"
+//
+ctx_xfer:
+	bra not $p1 #ctx_xfer_pre
+	bra $p2 #ctx_xfer_pre_load
+	ctx_xfer_pre:
+		mov $r15 0x10
+		call #ctx_86c
+		bra not $p1 #ctx_xfer_exec
+
+	ctx_xfer_pre_load:
+		mov $r15 2
+		call #ctx_4170s
+		call #ctx_4170w
+		call #ctx_redswitch
+		clear b32 $r15
+		call #ctx_4170s
+		call #ctx_load
+
+	// fetch context pointer, and initiate xfer on all GPCs
+	ctx_xfer_exec:
+	ld b32 $r1 D[$r0 + #ctx_current]
+	mov $r2 0x414
+	shl b32 $r2 6
+	iowr I[$r2 + 0x000] $r0	// BAR_STATUS = reset
+	mov $r14 -0x5b00
+	sethi $r14 0x410000
+	mov b32 $r15 $r1
+	call #nv_wr32		// GPC_BCAST_WRCMD_DATA = ctx pointer
+	add b32 $r14 4
+	xbit $r15 $flags $p1
+	xbit $r2 $flags $p2
+	shl b32 $r2 1
+	or $r15 $r2
+	call #nv_wr32		// GPC_BCAST_WRCMD_CMD = GPC_XFER(type)
+
+	// strands
+	mov $r1 0x4afc
+	sethi $r1 0x20000
+	mov $r2 0xc
+	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x0c
+	call #strand_wait
+	mov $r2 0x47fc
+	sethi $r2 0x20000
+	iowr I[$r2] $r0		// STRAND_FIRST_GENE(0x3f) = 0x00
+	xbit $r2 $flags $p1
+	add b32 $r2 3
+	iowr I[$r1] $r2		// STRAND_CMD(0x3f) = 0x03/0x04 (SAVE/LOAD)
+
+	// mmio context
+	xbit $r10 $flags $p1	// direction
+	or $r10 6		// first, last
+	mov $r11 0		// base = 0
+	ld b32 $r12 D[$r0 + #hub_mmio_list_head]
+	ld b32 $r13 D[$r0 + #hub_mmio_list_tail]
+	mov $r14 0		// not multi
+	call #mmctx_xfer
+
+	// wait for GPCs to all complete
+	mov $r10 8		// DONE_BAR
+	call #wait_doneo
+
+	// wait for strand xfer to complete
+	call #strand_wait
+
+	// post-op
+	bra $p1 #ctx_xfer_post
+		mov $r10 12		// DONE_UNK12
+		call #wait_donez
+		mov $r1 0xa10
+		shl b32 $r1 6
+		mov $r2 5
+		iowr I[$r1] $r2		// MEM_CMD
+		ctx_xfer_post_save_wait:
+			iord $r2 I[$r1]
+			or $r2 $r2
+			bra ne #ctx_xfer_post_save_wait
+
+	bra $p2 #ctx_xfer_done
+	ctx_xfer_post:
+		mov $r15 2
+		call #ctx_4170s
+		clear b32 $r15
+		call #ctx_86c
+		call #strand_post
+		call #ctx_4170w
+		clear b32 $r15
+		call #ctx_4170s
+
+		bra not $p1 #ctx_xfer_no_post_mmio
+		ld b32 $r1 D[$r0 + #chan_mmio_count]
+		or $r1 $r1
+		bra e #ctx_xfer_no_post_mmio
+			call #ctx_mmio_exec
+
+		ctx_xfer_no_post_mmio:
+
+	ctx_xfer_done:
+	ret
+
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h
new file mode 100644
index 00000000000..decf0c60ca3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h
@@ -0,0 +1,857 @@
+uint32_t nve0_grhub_data[] = {
+/* 0x0000: gpc_count */
+	0x00000000,
+/* 0x0004: rop_count */
+	0x00000000,
+/* 0x0008: cmd_queue */
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+/* 0x0050: hub_mmio_list_head */
+	0x00000000,
+/* 0x0054: hub_mmio_list_tail */
+	0x00000000,
+/* 0x0058: ctx_current */
+	0x00000000,
+/* 0x005c: chipsets */
+	0x000000e4,
+	0x013c0070,
+	0x000000e7,
+	0x013c0070,
+	0x00000000,
+/* 0x0070: nve4_hub_mmio_head */
+	0x0417e91c,
+	0x04400204,
+	0x18404010,
+	0x204040a8,
+	0x184040d0,
+	0x004040f8,
+	0x08404130,
+	0x08404150,
+	0x00404164,
+	0x0c4041a0,
+	0x0c404200,
+	0x34404404,
+	0x0c404460,
+	0x00404480,
+	0x00404498,
+	0x0c404604,
+	0x0c404618,
+	0x0440462c,
+	0x00404640,
+	0x00404654,
+	0x00404660,
+	0x48404678,
+	0x084046c8,
+	0x08404700,
+	0x24404718,
+	0x04404744,
+	0x00404754,
+	0x00405800,
+	0x08405830,
+	0x00405854,
+	0x0c405870,
+	0x04405a00,
+	0x00405a18,
+	0x00405b00,
+	0x00405b10,
+	0x00406020,
+	0x0c406028,
+	0x044064a8,
+	0x044064b4,
+	0x2c4064c0,
+	0x004064fc,
+	0x00407040,
+	0x00407804,
+	0x1440780c,
+	0x004078bc,
+	0x18408000,
+	0x00408064,
+	0x08408800,
+	0x00408840,
+	0x08408900,
+	0x00408980,
+/* 0x013c: nve4_hub_mmio_tail */
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+/* 0x0200: chan_data */
+/* 0x0200: chan_mmio_count */
+	0x00000000,
+/* 0x0204: chan_mmio_address */
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+/* 0x0300: xfer_data */
+	0x00000000,
+};
+
+uint32_t nve0_grhub_code[] = {
+	0x03090ef5,
+/* 0x0004: queue_put */
+	0x9800d898,
+	0x86f001d9,
+	0x0489b808,
+	0xf00c1bf4,
+	0x21f502f7,
+	0x00f802ec,
+/* 0x001c: queue_put_next */
+	0xb60798c4,
+	0x8dbb0384,
+	0x0880b600,
+	0x80008e80,
+	0x90b6018f,
+	0x0f94f001,
+	0xf801d980,
+/* 0x0039: queue_get */
+	0x0131f400,
+	0x9800d898,
+	0x89b801d9,
+	0x210bf404,
+	0xb60789c4,
+	0x9dbb0394,
+	0x0890b600,
+	0x98009e98,
+	0x80b6019f,
+	0x0f84f001,
+	0xf400d880,
+/* 0x0066: queue_get_done */
+	0x00f80132,
+/* 0x0068: nv_rd32 */
+	0x0728b7f1,
+	0xb906b4b6,
+	0xc9f002ec,
+	0x00bcd01f,
+/* 0x0078: nv_rd32_wait */
+	0xc800bccf,
+	0x1bf41fcc,
+	0x06a7f0fa,
+	0x010321f5,
+	0xf840bfcf,
+/* 0x008d: nv_wr32 */
+	0x28b7f100,
+	0x06b4b607,
+	0xb980bfd0,
+	0xc9f002ec,
+	0x1ec9f01f,
+/* 0x00a3: nv_wr32_wait */
+	0xcf00bcd0,
+	0xccc800bc,
+	0xfa1bf41f,
+/* 0x00ae: watchdog_reset */
+	0x87f100f8,
+	0x84b60430,
+	0x1ff9f006,
+	0xf8008fd0,
+/* 0x00bd: watchdog_clear */
+	0x3087f100,
+	0x0684b604,
+	0xf80080d0,
+/* 0x00c9: wait_donez */
+	0x3c87f100,
+	0x0684b608,
+	0x99f094bd,
+	0x0089d000,
+	0x081887f1,
+	0xd00684b6,
+/* 0x00e2: wait_done_wait_donez */
+	0x87f1008a,
+	0x84b60400,
+	0x0088cf06,
+	0xf4888aff,
+	0x87f1f31b,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00099,
+/* 0x0103: wait_doneo */
+	0xf100f800,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00099f0,
+	0x87f10089,
+	0x84b60818,
+	0x008ad006,
+/* 0x011c: wait_done_wait_doneo */
+	0x040087f1,
+	0xcf0684b6,
+	0x8aff0088,
+	0xf30bf488,
+	0x085c87f1,
+	0xbd0684b6,
+	0x0099f094,
+	0xf80089d0,
+/* 0x013d: mmctx_size */
+/* 0x013f: nv_mmctx_size_loop */
+	0x9894bd00,
+	0x85b600e8,
+	0x0180b61a,
+	0xbb0284b6,
+	0xe0b60098,
+	0x04efb804,
+	0xb9eb1bf4,
+	0x00f8029f,
+/* 0x015c: mmctx_xfer */
+	0x083c87f1,
+	0xbd0684b6,
+	0x0199f094,
+	0xf10089d0,
+	0xb6071087,
+	0x94bd0684,
+	0xf405bbfd,
+	0x8bd0090b,
+	0x0099f000,
+/* 0x0180: mmctx_base_disabled */
+	0xf405eefd,
+	0x8ed00c0b,
+	0xc08fd080,
+/* 0x018f: mmctx_multi_disabled */
+	0xb70199f0,
+	0xc8010080,
+	0xb4b600ab,
+	0x0cb9f010,
+	0xb601aec8,
+	0xbefd11e4,
+	0x008bd005,
+/* 0x01a8: mmctx_exec_loop */
+/* 0x01a8: mmctx_wait_free */
+	0xf0008ecf,
+	0x0bf41fe4,
+	0x00ce98fa,
+	0xd005e9fd,
+	0xc0b6c08e,
+	0x04cdb804,
+	0xc8e81bf4,
+	0x1bf402ab,
+/* 0x01c9: mmctx_fini_wait */
+	0x008bcf18,
+	0xb01fb4f0,
+	0x1bf410b4,
+	0x02a7f0f7,
+	0xf4c921f4,
+/* 0x01de: mmctx_stop */
+	0xabc81b0e,
+	0x10b4b600,
+	0xf00cb9f0,
+	0x8bd012b9,
+/* 0x01ed: mmctx_stop_wait */
+	0x008bcf00,
+	0xf412bbc8,
+/* 0x01f6: mmctx_done */
+	0x87f1fa1b,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00199,
+/* 0x0207: strand_wait */
+	0xf900f800,
+	0x02a7f0a0,
+	0xfcc921f4,
+/* 0x0213: strand_pre */
+	0xf100f8a0,
+	0xf04afc87,
+	0x97f00283,
+	0x0089d00c,
+	0x020721f5,
+/* 0x0226: strand_post */
+	0x87f100f8,
+	0x83f04afc,
+	0x0d97f002,
+	0xf50089d0,
+	0xf8020721,
+/* 0x0239: strand_set */
+	0xfca7f100,
+	0x02a3f04f,
+	0x0500aba2,
+	0xd00fc7f0,
+	0xc7f000ac,
+	0x00bcd00b,
+	0x020721f5,
+	0xf000aed0,
+	0xbcd00ac7,
+	0x0721f500,
+/* 0x0263: strand_ctx_init */
+	0xf100f802,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00399f0,
+	0x21f50089,
+	0xe7f00213,
+	0x3921f503,
+	0xfca7f102,
+	0x02a3f046,
+	0x0400aba0,
+	0xf040a0d0,
+	0xbcd001c7,
+	0x0721f500,
+	0x010c9202,
+	0xf000acd0,
+	0xbcd002c7,
+	0x0721f500,
+	0x2621f502,
+	0x8087f102,
+	0x0684b608,
+	0xb70089cf,
+	0x95220080,
+/* 0x02ba: ctx_init_strand_loop */
+	0x8ed008fe,
+	0x408ed000,
+	0xb6808acf,
+	0xa0b606a5,
+	0x00eabb01,
+	0xb60480b6,
+	0x1bf40192,
+	0x08e4b6e8,
+	0xf1f2efbc,
+	0xb6085c87,
+	0x94bd0684,
+	0xd00399f0,
+	0x00f80089,
+/* 0x02ec: error */
+	0xe7f1e0f9,
+	0xe4b60814,
+	0x00efd006,
+	0x0c1ce7f1,
+	0xf006e4b6,
+	0xefd001f7,
+	0xf8e0fc00,
+/* 0x0309: init */
+	0xfe04bd00,
+	0x07fe0004,
+	0x0017f100,
+	0x0227f012,
+	0xf10012d0,
+	0xfe05b917,
+	0x17f10010,
+	0x10d00400,
+	0x0437f1c0,
+	0x0634b604,
+	0x200327f1,
+	0xf10032d0,
+	0xd0200427,
+	0x27f10132,
+	0x32d0200b,
+	0x0c27f102,
+	0x0732d020,
+	0x0c2427f1,
+	0xb90624b6,
+	0x23d00003,
+	0x0427f100,
+	0x0023f087,
+	0xb70012d0,
+	0xf0010012,
+	0x12d00427,
+	0x1031f400,
+	0x9604e7f1,
+	0xf440e3f0,
+	0xf1c76821,
+	0x01018090,
+	0x801ff4f0,
+	0x17f0000f,
+	0x041fbb01,
+	0xf10112b6,
+	0xb6040c27,
+	0x21d00624,
+	0x4021d000,
+	0x080027f1,
+	0xcf0624b6,
+	0xf7f00022,
+/* 0x03a9: init_find_chipset */
+	0x08f0b654,
+	0xb800f398,
+	0x0bf40432,
+	0x0034b00b,
+	0xf8f11bf4,
+/* 0x03bd: init_context */
+	0x0017f100,
+	0x02fe5801,
+	0xf003ff58,
+	0x0e8000e3,
+	0x150f8014,
+	0x013d21f5,
+	0x070037f1,
+	0x950634b6,
+	0x34d00814,
+	0x4034d000,
+	0x130030b7,
+	0xb6001fbb,
+	0x3fd002f5,
+	0x0815b600,
+	0xb60110b6,
+	0x1fb90814,
+	0x6321f502,
+	0x001fbb02,
+	0xf1000398,
+	0xf0200047,
+/* 0x040e: init_gpc */
+	0x4ea05043,
+	0x1fb90804,
+	0x8d21f402,
+	0x08004ea0,
+	0xf4022fb9,
+	0x4ea08d21,
+	0xf4bd010c,
+	0xa08d21f4,
+	0xf401044e,
+	0x4ea08d21,
+	0xf7f00100,
+	0x8d21f402,
+	0x08004ea0,
+/* 0x0440: init_gpc_wait */
+	0xc86821f4,
+	0x0bf41fff,
+	0x044ea0fa,
+	0x6821f408,
+	0xb7001fbb,
+	0xb6800040,
+	0x1bf40132,
+	0x0027f1b4,
+	0x0624b608,
+	0xb74021d0,
+	0xbd080020,
+	0x1f19f014,
+/* 0x0473: main */
+	0xf40021d0,
+	0x28f40031,
+	0x08d7f000,
+	0xf43921f4,
+	0xe4b1f401,
+	0x1bf54001,
+	0x87f100d1,
+	0x84b6083c,
+	0xf094bd06,
+	0x89d00499,
+	0x0017f100,
+	0x0614b60b,
+	0xcf4012cf,
+	0x13c80011,
+	0x7e0bf41f,
+	0xf41f23c8,
+	0x20f95a0b,
+	0xf10212b9,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00799f0,
+	0x32f40089,
+	0x0231f401,
+	0x07fb21f5,
+	0x085c87f1,
+	0xbd0684b6,
+	0x0799f094,
+	0xfc0089d0,
+	0x3c87f120,
+	0x0684b608,
+	0x99f094bd,
+	0x0089d006,
+	0xf50131f4,
+	0xf107fb21,
+	0xb6085c87,
+	0x94bd0684,
+	0xd00699f0,
+	0x0ef40089,
+/* 0x0509: chsw_prev_no_next */
+	0xb920f931,
+	0x32f40212,
+	0x0232f401,
+	0x07fb21f5,
+	0x17f120fc,
+	0x14b60b00,
+	0x0012d006,
+/* 0x0527: chsw_no_prev */
+	0xc8130ef4,
+	0x0bf41f23,
+	0x0131f40d,
+	0xf50232f4,
+/* 0x0537: chsw_done */
+	0xf107fb21,
+	0xb60b0c17,
+	0x27f00614,
+	0x0012d001,
+	0x085c87f1,
+	0xbd0684b6,
+	0x0499f094,
+	0xf50089d0,
+/* 0x0557: main_not_ctx_switch */
+	0xb0ff200e,
+	0x1bf401e4,
+	0x02f2b90d,
+	0x078f21f5,
+/* 0x0567: main_not_ctx_chan */
+	0xb0420ef4,
+	0x1bf402e4,
+	0x3c87f12e,
+	0x0684b608,
+	0x99f094bd,
+	0x0089d007,
+	0xf40132f4,
+	0x21f50232,
+	0x87f107fb,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00799,
+	0x110ef400,
+/* 0x0598: main_not_ctx_save */
+	0xf010ef94,
+	0x21f501f5,
+	0x0ef502ec,
+/* 0x05a6: main_done */
+	0x17f1fed1,
+	0x14b60820,
+	0xf024bd06,
+	0x12d01f29,
+	0xbe0ef500,
+/* 0x05b9: ih */
+	0xfe80f9fe,
+	0x80f90188,
+	0xa0f990f9,
+	0xd0f9b0f9,
+	0xf0f9e0f9,
+	0xc4800acf,
+	0x0bf404ab,
+	0x00b7f11d,
+	0x08d7f019,
+	0xcf40becf,
+	0x21f400bf,
+	0x00b0b704,
+	0x01e7f004,
+/* 0x05ef: ih_no_fifo */
+	0xe400bed0,
+	0xf40100ab,
+	0xd7f00d0b,
+	0x01e7f108,
+	0x0421f440,
+/* 0x0600: ih_no_ctxsw */
+	0x0104b7f1,
+	0xabffb0bd,
+	0x0d0bf4b4,
+	0x0c1ca7f1,
+	0xd006a4b6,
+/* 0x0616: ih_no_other */
+	0x0ad000ab,
+	0xfcf0fc40,
+	0xfcd0fce0,
+	0xfca0fcb0,
+	0xfe80fc90,
+	0x80fc0088,
+	0xf80032f4,
+/* 0x0631: ctx_4170s */
+	0x70e7f101,
+	0x40e3f041,
+	0xf410f5f0,
+	0x00f88d21,
+/* 0x0640: ctx_4170w */
+	0x4170e7f1,
+	0xf440e3f0,
+	0xf4f06821,
+	0xf31bf410,
+/* 0x0652: ctx_redswitch */
+	0xe7f100f8,
+	0xe4b60614,
+	0x70f7f106,
+	0x00efd002,
+/* 0x0663: ctx_redswitch_delay */
+	0xb608f7f0,
+	0x1bf401f2,
+	0x70f7f1fd,
+	0x00efd007,
+/* 0x0672: ctx_86c */
+	0xe7f100f8,
+	0xe4b6086c,
+	0x00efd006,
+	0x8a14e7f1,
+	0xf440e3f0,
+	0xe7f18d21,
+	0xe3f0a86c,
+	0x8d21f441,
+/* 0x0692: ctx_load */
+	0x87f100f8,
+	0x84b6083c,
+	0xf094bd06,
+	0x89d00599,
+	0x0ca7f000,
+	0xf1c921f4,
+	0xb60a2417,
+	0x10d00614,
+	0x0037f100,
+	0x0634b60b,
+	0xf14032d0,
+	0xb60a0c17,
+	0x47f00614,
+	0x0012d007,
+/* 0x06cb: ctx_chan_wait_0 */
+	0xcf4014d0,
+	0x44f04014,
+	0xfa1bf41f,
+	0xfe0032d0,
+	0x2af0000b,
+	0x0424b61f,
+	0xf10220b6,
+	0xb6083c87,
+	0x94bd0684,
+	0xd00899f0,
+	0x17f10089,
+	0x14b60a04,
+	0x0012d006,
+	0x0a2017f1,
+	0xf00614b6,
+	0x23f10227,
+	0x12d08000,
+	0x1017f000,
+	0x030027f1,
+	0xfa0223f0,
+	0x03f80512,
+	0x085c87f1,
+	0xbd0684b6,
+	0x0899f094,
+	0x980089d0,
+	0x14b6c101,
+	0xc0029818,
+	0xfd0825b6,
+	0x01800512,
+	0x3c87f116,
+	0x0684b608,
+	0x99f094bd,
+	0x0089d009,
+	0x0a0427f1,
+	0xd00624b6,
+	0x27f00021,
+	0x2017f101,
+	0x0614b60a,
+	0xf10012d0,
+	0xf0020017,
+	0x01fa0613,
+	0xf103f805,
+	0xb6085c87,
+	0x94bd0684,
+	0xd00999f0,
+	0x87f10089,
+	0x84b6085c,
+	0xf094bd06,
+	0x89d00599,
+/* 0x078f: ctx_chan */
+	0xf500f800,
+	0xf0069221,
+	0x21f40ca7,
+	0x1017f1c9,
+	0x0614b60a,
+	0xd00527f0,
+/* 0x07a6: ctx_chan_wait */
+	0x12cf0012,
+	0x0522fd00,
+	0xf8fa1bf4,
+/* 0x07b1: ctx_mmio_exec */
+	0x81039800,
+	0x0a0427f1,
+	0xd00624b6,
+	0x34bd0023,
+/* 0x07c0: ctx_mmio_loop */
+	0xf4ff34c4,
+	0x57f10f1b,
+	0x53f00300,
+	0x0535fa06,
+/* 0x07d2: ctx_mmio_pull */
+	0x4e9803f8,
+	0xc14f98c0,
+	0xb68d21f4,
+	0x12b60830,
+	0xdf1bf401,
+/* 0x07e4: ctx_mmio_done */
+	0xd0160398,
+	0x00800023,
+	0x0017f180,
+	0x0613f002,
+	0xf80601fa,
+/* 0x07fb: ctx_xfer */
+	0xf400f803,
+	0x02f40611,
+/* 0x0801: ctx_xfer_pre */
+	0x10f7f00d,
+	0x067221f5,
+/* 0x080b: ctx_xfer_pre_load */
+	0xf01c11f4,
+	0x21f502f7,
+	0x21f50631,
+	0x21f50640,
+	0xf4bd0652,
+	0x063121f5,
+	0x069221f5,
+/* 0x0824: ctx_xfer_exec */
+	0xf1160198,
+	0xb6041427,
+	0x20d00624,
+	0x00e7f100,
+	0x41e3f0a5,
+	0xf4021fb9,
+	0xe0b68d21,
+	0x01fcf004,
+	0xb6022cf0,
+	0xf2fd0124,
+	0x8d21f405,
+	0x4afc17f1,
+	0xf00213f0,
+	0x12d00c27,
+	0x0721f500,
+	0xfc27f102,
+	0x0223f047,
+	0xf00020d0,
+	0x20b6012c,
+	0x0012d003,
+	0xf001acf0,
+	0xb7f006a5,
+	0x140c9800,
+	0xf0150d98,
+	0x21f500e7,
+	0xa7f0015c,
+	0x0321f508,
+	0x0721f501,
+	0x2201f402,
+	0xf40ca7f0,
+	0x17f1c921,
+	0x14b60a10,
+	0x0527f006,
+/* 0x08ab: ctx_xfer_post_save_wait */
+	0xcf0012d0,
+	0x22fd0012,
+	0xfa1bf405,
+/* 0x08b7: ctx_xfer_post */
+	0xf02e02f4,
+	0x21f502f7,
+	0xf4bd0631,
+	0x067221f5,
+	0x022621f5,
+	0x064021f5,
+	0x21f5f4bd,
+	0x11f40631,
+	0x80019810,
+	0xf40511fd,
+	0x21f5070b,
+/* 0x08e2: ctx_xfer_no_post_mmio */
+/* 0x08e2: ctx_xfer_done */
+	0x00f807b1,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+	0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/nvc0.fuc
index e6b228844a3..e6b228844a3 100644
--- a/drivers/gpu/drm/nouveau/nvc0_graph.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/nvc0.fuc
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/nve0.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/nve0.fuc
new file mode 100644
index 00000000000..f16a5d53319
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/nve0.fuc
@@ -0,0 +1,400 @@
+/* fuc microcode util functions for nve0 PGRAPH
+ *
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+define(`mmctx_data', `.b32 eval((($2 - 1) << 26) | $1)')
+define(`queue_init', `.skip eval((2 * 4) + ((8 * 4) * 2))')
+
+ifdef(`include_code', `
+// Error codes
+define(`E_BAD_COMMAND', 0x01)
+define(`E_CMD_OVERFLOW', 0x02)
+
+// Util macros to help with debugging ucode hangs etc
+define(`T_WAIT', 0)
+define(`T_MMCTX', 1)
+define(`T_STRWAIT', 2)
+define(`T_STRINIT', 3)
+define(`T_AUTO', 4)
+define(`T_CHAN', 5)
+define(`T_LOAD', 6)
+define(`T_SAVE', 7)
+define(`T_LCHAN', 8)
+define(`T_LCTXH', 9)
+
+define(`trace_set', `
+	mov $r8 0x83c
+	shl b32 $r8 6
+	clear b32 $r9
+	bset $r9 $1
+	iowr I[$r8 + 0x000] $r9		// CC_SCRATCH[7]
+')
+
+define(`trace_clr', `
+	mov $r8 0x85c
+	shl b32 $r8 6
+	clear b32 $r9
+	bset $r9 $1
+	iowr I[$r8 + 0x000] $r9		// CC_SCRATCH[7]
+')
+
+// queue_put - add request to queue
+//
+// In : $r13 queue pointer
+//	$r14 command
+//	$r15 data
+//
+queue_put:
+	// make sure we have space..
+	ld b32 $r8 D[$r13 + 0x0]	// GET
+	ld b32 $r9 D[$r13 + 0x4]	// PUT
+	xor $r8 8
+	cmpu b32 $r8 $r9
+	bra ne #queue_put_next
+		mov $r15 E_CMD_OVERFLOW
+		call #error
+		ret
+
+	// store cmd/data on queue
+	queue_put_next:
+	and $r8 $r9 7
+	shl b32 $r8 3
+	add b32 $r8 $r13
+	add b32 $r8 8
+	st b32 D[$r8 + 0x0] $r14
+	st b32 D[$r8 + 0x4] $r15
+
+	// update PUT
+	add b32 $r9 1
+	and $r9 0xf
+	st b32 D[$r13 + 0x4] $r9
+	ret
+
+// queue_get - fetch request from queue
+//
+// In : $r13 queue pointer
+//
+// Out:	$p1  clear on success (data available)
+//	$r14 command
+// 	$r15 data
+//
+queue_get:
+	bset $flags $p1
+	ld b32 $r8 D[$r13 + 0x0]	// GET
+	ld b32 $r9 D[$r13 + 0x4]	// PUT
+	cmpu b32 $r8 $r9
+	bra e #queue_get_done
+		// fetch first cmd/data pair
+		and $r9 $r8 7
+		shl b32 $r9 3
+		add b32 $r9 $r13
+		add b32 $r9 8
+		ld b32 $r14 D[$r9 + 0x0]
+		ld b32 $r15 D[$r9 + 0x4]
+
+		// update GET
+		add b32 $r8 1
+		and $r8 0xf
+		st b32 D[$r13 + 0x0] $r8
+		bclr $flags $p1
+queue_get_done:
+	ret
+
+// nv_rd32 - read 32-bit value from nv register
+//
+// In : $r14 register
+// Out: $r15 value
+//
+nv_rd32:
+	mov $r11 0x728
+	shl b32 $r11 6
+	mov b32 $r12 $r14
+	bset $r12 31			// MMIO_CTRL_PENDING
+	iowr I[$r11 + 0x000] $r12	// MMIO_CTRL
+	nv_rd32_wait:
+		iord $r12 I[$r11 + 0x000]
+		xbit $r12 $r12 31
+		bra ne #nv_rd32_wait
+	mov $r10 6			// DONE_MMIO_RD
+	call #wait_doneo
+	iord $r15 I[$r11 + 0x100]	// MMIO_RDVAL
+	ret
+
+// nv_wr32 - write 32-bit value to nv register
+//
+// In : $r14 register
+//      $r15 value
+//
+nv_wr32:
+	mov $r11 0x728
+	shl b32 $r11 6
+	iowr I[$r11 + 0x200] $r15	// MMIO_WRVAL
+	mov b32 $r12 $r14
+	bset $r12 31			// MMIO_CTRL_PENDING
+	bset $r12 30			// MMIO_CTRL_WRITE
+	iowr I[$r11 + 0x000] $r12	// MMIO_CTRL
+	nv_wr32_wait:
+		iord $r12 I[$r11 + 0x000]
+		xbit $r12 $r12 31
+		bra ne #nv_wr32_wait
+	ret
+
+// (re)set watchdog timer
+//
+// In : $r15 timeout
+//
+watchdog_reset:
+	mov $r8 0x430
+	shl b32 $r8 6
+	bset $r15 31
+	iowr I[$r8 + 0x000] $r15
+	ret
+
+// clear watchdog timer
+watchdog_clear:
+	mov $r8 0x430
+	shl b32 $r8 6
+	iowr I[$r8 + 0x000] $r0
+	ret
+
+// wait_done{z,o} - wait on FUC_DONE bit to become clear/set
+//
+// In : $r10 bit to wait on
+//
+define(`wait_done', `
+$1:
+	trace_set(T_WAIT);
+	mov $r8 0x818
+	shl b32 $r8 6
+	iowr I[$r8 + 0x000] $r10	// CC_SCRATCH[6] = wait bit
+	wait_done_$1:
+		mov $r8 0x400
+		shl b32 $r8 6
+		iord $r8 I[$r8 + 0x000]	// DONE
+		xbit $r8 $r8 $r10
+		bra $2 #wait_done_$1
+	trace_clr(T_WAIT)
+	ret
+')
+wait_done(wait_donez, ne)
+wait_done(wait_doneo, e)
+
+// mmctx_size - determine size of a mmio list transfer
+//
+// In : $r14 mmio list head
+//      $r15 mmio list tail
+// Out: $r15 transfer size (in bytes)
+//
+mmctx_size:
+	clear b32 $r9
+	nv_mmctx_size_loop:
+		ld b32 $r8 D[$r14]
+		shr b32 $r8 26
+		add b32 $r8 1
+		shl b32 $r8 2
+		add b32 $r9 $r8
+		add b32 $r14 4
+		cmpu b32 $r14 $r15
+		bra ne #nv_mmctx_size_loop
+	mov b32 $r15 $r9
+	ret
+
+// mmctx_xfer - execute a list of mmio transfers
+//
+// In : $r10 flags
+//		bit 0: direction (0 = save, 1 = load)
+//		bit 1: set if first transfer
+//		bit 2: set if last transfer
+//	$r11 base
+//	$r12 mmio list head
+//	$r13 mmio list tail
+//	$r14 multi_stride
+//	$r15 multi_mask
+//
+mmctx_xfer:
+	trace_set(T_MMCTX)
+	mov $r8 0x710
+	shl b32 $r8 6
+	clear b32 $r9
+	or $r11 $r11
+	bra e #mmctx_base_disabled
+		iowr I[$r8 + 0x000] $r11	// MMCTX_BASE
+		bset $r9 0			// BASE_EN
+	mmctx_base_disabled:
+	or $r14 $r14
+	bra e #mmctx_multi_disabled
+		iowr I[$r8 + 0x200] $r14 	// MMCTX_MULTI_STRIDE
+		iowr I[$r8 + 0x300] $r15 	// MMCTX_MULTI_MASK
+		bset $r9 1			// MULTI_EN
+	mmctx_multi_disabled:
+	add b32 $r8 0x100
+
+	xbit $r11 $r10 0
+	shl b32 $r11 16			// DIR
+	bset $r11 12			// QLIMIT = 0x10
+	xbit $r14 $r10 1
+	shl b32 $r14 17
+	or $r11 $r14			// START_TRIGGER
+	iowr I[$r8 + 0x000] $r11	// MMCTX_CTRL
+
+	// loop over the mmio list, and send requests to the hw
+	mmctx_exec_loop:
+		// wait for space in mmctx queue
+		mmctx_wait_free:
+			iord $r14 I[$r8 + 0x000] // MMCTX_CTRL
+			and $r14 0x1f
+			bra e #mmctx_wait_free
+
+		// queue up an entry
+		ld b32 $r14 D[$r12]
+		or $r14 $r9
+		iowr I[$r8 + 0x300] $r14
+		add b32 $r12 4
+		cmpu b32 $r12 $r13
+		bra ne #mmctx_exec_loop
+
+	xbit $r11 $r10 2
+	bra ne #mmctx_stop
+		// wait for queue to empty
+		mmctx_fini_wait:
+			iord $r11 I[$r8 + 0x000]	// MMCTX_CTRL
+			and $r11 0x1f
+			cmpu b32 $r11 0x10
+			bra ne #mmctx_fini_wait
+		mov $r10 2				// DONE_MMCTX
+		call #wait_donez
+		bra #mmctx_done
+	mmctx_stop:
+		xbit $r11 $r10 0
+		shl b32 $r11 16			// DIR
+		bset $r11 12			// QLIMIT = 0x10
+		bset $r11 18			// STOP_TRIGGER
+		iowr I[$r8 + 0x000] $r11	// MMCTX_CTRL
+		mmctx_stop_wait:
+			// wait for STOP_TRIGGER to clear
+			iord $r11 I[$r8 + 0x000] // MMCTX_CTRL
+			xbit $r11 $r11 18
+			bra ne #mmctx_stop_wait
+	mmctx_done:
+	trace_clr(T_MMCTX)
+	ret
+
+// Wait for DONE_STRAND
+//
+strand_wait:
+	push $r10
+	mov $r10 2
+	call #wait_donez
+	pop $r10
+	ret
+
+// unknown - call before issuing strand commands
+//
+strand_pre:
+	mov $r8 0x4afc
+	sethi $r8 0x20000
+	mov $r9 0xc
+	iowr I[$r8] $r9
+	call #strand_wait
+	ret
+
+// unknown - call after issuing strand commands
+//
+strand_post:
+	mov $r8 0x4afc
+	sethi $r8 0x20000
+	mov $r9 0xd
+	iowr I[$r8] $r9
+	call #strand_wait
+	ret
+
+// Selects strand set?!
+//
+// In: $r14 id
+//
+strand_set:
+	mov $r10 0x4ffc
+	sethi $r10 0x20000
+	sub b32 $r11 $r10 0x500
+	mov $r12 0xf
+	iowr I[$r10 + 0x000] $r12		// 0x93c = 0xf
+	mov $r12 0xb
+	iowr I[$r11 + 0x000] $r12		// 0x928 = 0xb
+	call #strand_wait
+	iowr I[$r10 + 0x000] $r14		// 0x93c = <id>
+	mov $r12 0xa
+	iowr I[$r11 + 0x000] $r12		// 0x928 = 0xa
+	call #strand_wait
+	ret
+
+// Initialise strand context data
+//
+// In : $r15 context base
+// Out: $r15 context size (in bytes)
+//
+// Strandset(?) 3 hardcoded currently
+//
+strand_ctx_init:
+	trace_set(T_STRINIT)
+	call #strand_pre
+	mov $r14 3
+	call #strand_set
+	mov $r10 0x46fc
+	sethi $r10 0x20000
+	add b32 $r11 $r10 0x400
+	iowr I[$r10 + 0x100] $r0	// STRAND_FIRST_GENE = 0
+	mov $r12 1
+	iowr I[$r11 + 0x000] $r12	// STRAND_CMD = LATCH_FIRST_GENE
+	call #strand_wait
+	sub b32 $r12 $r0 1
+	iowr I[$r10 + 0x000] $r12	// STRAND_GENE_CNT = 0xffffffff
+	mov $r12 2
+	iowr I[$r11 + 0x000] $r12	// STRAND_CMD = LATCH_GENE_CNT
+	call #strand_wait
+	call #strand_post
+
+	// read the size of each strand, poke the context offset of
+	// each into STRAND_{SAVE,LOAD}_SWBASE now, no need to worry
+	// about it later then.
+	mov $r8 0x880
+	shl b32 $r8 6
+	iord $r9 I[$r8 + 0x000]		// STRANDS
+	add b32 $r8 0x2200
+	shr b32 $r14 $r15 8
+	ctx_init_strand_loop:
+		iowr I[$r8 + 0x000] $r14	// STRAND_SAVE_SWBASE
+		iowr I[$r8 + 0x100] $r14	// STRAND_LOAD_SWBASE
+		iord $r10 I[$r8 + 0x200]	// STRAND_SIZE
+		shr b32 $r10 6
+		add b32 $r10 1
+		add b32 $r14 $r10
+		add b32 $r8 4
+		sub b32 $r9 1
+		bra ne #ctx_init_strand_loop
+
+	shl b32 $r14 8
+	sub b32 $r15 $r14 $r15
+	trace_clr(T_STRINIT)
+	ret
+')
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c
new file mode 100644
index 00000000000..61852824845
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv04.c
@@ -0,0 +1,1387 @@
+/*
+ * Copyright 2007 Stephane Marchesin
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/handle.h>
+#include <core/namedb.h>
+
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
+#include <engine/graph.h>
+
+#include "regs.h"
+
+static u32
+nv04_graph_ctx_regs[] = {
+	0x0040053c,
+	0x00400544,
+	0x00400540,
+	0x00400548,
+	NV04_PGRAPH_CTX_SWITCH1,
+	NV04_PGRAPH_CTX_SWITCH2,
+	NV04_PGRAPH_CTX_SWITCH3,
+	NV04_PGRAPH_CTX_SWITCH4,
+	NV04_PGRAPH_CTX_CACHE1,
+	NV04_PGRAPH_CTX_CACHE2,
+	NV04_PGRAPH_CTX_CACHE3,
+	NV04_PGRAPH_CTX_CACHE4,
+	0x00400184,
+	0x004001a4,
+	0x004001c4,
+	0x004001e4,
+	0x00400188,
+	0x004001a8,
+	0x004001c8,
+	0x004001e8,
+	0x0040018c,
+	0x004001ac,
+	0x004001cc,
+	0x004001ec,
+	0x00400190,
+	0x004001b0,
+	0x004001d0,
+	0x004001f0,
+	0x00400194,
+	0x004001b4,
+	0x004001d4,
+	0x004001f4,
+	0x00400198,
+	0x004001b8,
+	0x004001d8,
+	0x004001f8,
+	0x0040019c,
+	0x004001bc,
+	0x004001dc,
+	0x004001fc,
+	0x00400174,
+	NV04_PGRAPH_DMA_START_0,
+	NV04_PGRAPH_DMA_START_1,
+	NV04_PGRAPH_DMA_LENGTH,
+	NV04_PGRAPH_DMA_MISC,
+	NV04_PGRAPH_DMA_PITCH,
+	NV04_PGRAPH_BOFFSET0,
+	NV04_PGRAPH_BBASE0,
+	NV04_PGRAPH_BLIMIT0,
+	NV04_PGRAPH_BOFFSET1,
+	NV04_PGRAPH_BBASE1,
+	NV04_PGRAPH_BLIMIT1,
+	NV04_PGRAPH_BOFFSET2,
+	NV04_PGRAPH_BBASE2,
+	NV04_PGRAPH_BLIMIT2,
+	NV04_PGRAPH_BOFFSET3,
+	NV04_PGRAPH_BBASE3,
+	NV04_PGRAPH_BLIMIT3,
+	NV04_PGRAPH_BOFFSET4,
+	NV04_PGRAPH_BBASE4,
+	NV04_PGRAPH_BLIMIT4,
+	NV04_PGRAPH_BOFFSET5,
+	NV04_PGRAPH_BBASE5,
+	NV04_PGRAPH_BLIMIT5,
+	NV04_PGRAPH_BPITCH0,
+	NV04_PGRAPH_BPITCH1,
+	NV04_PGRAPH_BPITCH2,
+	NV04_PGRAPH_BPITCH3,
+	NV04_PGRAPH_BPITCH4,
+	NV04_PGRAPH_SURFACE,
+	NV04_PGRAPH_STATE,
+	NV04_PGRAPH_BSWIZZLE2,
+	NV04_PGRAPH_BSWIZZLE5,
+	NV04_PGRAPH_BPIXEL,
+	NV04_PGRAPH_NOTIFY,
+	NV04_PGRAPH_PATT_COLOR0,
+	NV04_PGRAPH_PATT_COLOR1,
+	NV04_PGRAPH_PATT_COLORRAM+0x00,
+	NV04_PGRAPH_PATT_COLORRAM+0x04,
+	NV04_PGRAPH_PATT_COLORRAM+0x08,
+	NV04_PGRAPH_PATT_COLORRAM+0x0c,
+	NV04_PGRAPH_PATT_COLORRAM+0x10,
+	NV04_PGRAPH_PATT_COLORRAM+0x14,
+	NV04_PGRAPH_PATT_COLORRAM+0x18,
+	NV04_PGRAPH_PATT_COLORRAM+0x1c,
+	NV04_PGRAPH_PATT_COLORRAM+0x20,
+	NV04_PGRAPH_PATT_COLORRAM+0x24,
+	NV04_PGRAPH_PATT_COLORRAM+0x28,
+	NV04_PGRAPH_PATT_COLORRAM+0x2c,
+	NV04_PGRAPH_PATT_COLORRAM+0x30,
+	NV04_PGRAPH_PATT_COLORRAM+0x34,
+	NV04_PGRAPH_PATT_COLORRAM+0x38,
+	NV04_PGRAPH_PATT_COLORRAM+0x3c,
+	NV04_PGRAPH_PATT_COLORRAM+0x40,
+	NV04_PGRAPH_PATT_COLORRAM+0x44,
+	NV04_PGRAPH_PATT_COLORRAM+0x48,
+	NV04_PGRAPH_PATT_COLORRAM+0x4c,
+	NV04_PGRAPH_PATT_COLORRAM+0x50,
+	NV04_PGRAPH_PATT_COLORRAM+0x54,
+	NV04_PGRAPH_PATT_COLORRAM+0x58,
+	NV04_PGRAPH_PATT_COLORRAM+0x5c,
+	NV04_PGRAPH_PATT_COLORRAM+0x60,
+	NV04_PGRAPH_PATT_COLORRAM+0x64,
+	NV04_PGRAPH_PATT_COLORRAM+0x68,
+	NV04_PGRAPH_PATT_COLORRAM+0x6c,
+	NV04_PGRAPH_PATT_COLORRAM+0x70,
+	NV04_PGRAPH_PATT_COLORRAM+0x74,
+	NV04_PGRAPH_PATT_COLORRAM+0x78,
+	NV04_PGRAPH_PATT_COLORRAM+0x7c,
+	NV04_PGRAPH_PATT_COLORRAM+0x80,
+	NV04_PGRAPH_PATT_COLORRAM+0x84,
+	NV04_PGRAPH_PATT_COLORRAM+0x88,
+	NV04_PGRAPH_PATT_COLORRAM+0x8c,
+	NV04_PGRAPH_PATT_COLORRAM+0x90,
+	NV04_PGRAPH_PATT_COLORRAM+0x94,
+	NV04_PGRAPH_PATT_COLORRAM+0x98,
+	NV04_PGRAPH_PATT_COLORRAM+0x9c,
+	NV04_PGRAPH_PATT_COLORRAM+0xa0,
+	NV04_PGRAPH_PATT_COLORRAM+0xa4,
+	NV04_PGRAPH_PATT_COLORRAM+0xa8,
+	NV04_PGRAPH_PATT_COLORRAM+0xac,
+	NV04_PGRAPH_PATT_COLORRAM+0xb0,
+	NV04_PGRAPH_PATT_COLORRAM+0xb4,
+	NV04_PGRAPH_PATT_COLORRAM+0xb8,
+	NV04_PGRAPH_PATT_COLORRAM+0xbc,
+	NV04_PGRAPH_PATT_COLORRAM+0xc0,
+	NV04_PGRAPH_PATT_COLORRAM+0xc4,
+	NV04_PGRAPH_PATT_COLORRAM+0xc8,
+	NV04_PGRAPH_PATT_COLORRAM+0xcc,
+	NV04_PGRAPH_PATT_COLORRAM+0xd0,
+	NV04_PGRAPH_PATT_COLORRAM+0xd4,
+	NV04_PGRAPH_PATT_COLORRAM+0xd8,
+	NV04_PGRAPH_PATT_COLORRAM+0xdc,
+	NV04_PGRAPH_PATT_COLORRAM+0xe0,
+	NV04_PGRAPH_PATT_COLORRAM+0xe4,
+	NV04_PGRAPH_PATT_COLORRAM+0xe8,
+	NV04_PGRAPH_PATT_COLORRAM+0xec,
+	NV04_PGRAPH_PATT_COLORRAM+0xf0,
+	NV04_PGRAPH_PATT_COLORRAM+0xf4,
+	NV04_PGRAPH_PATT_COLORRAM+0xf8,
+	NV04_PGRAPH_PATT_COLORRAM+0xfc,
+	NV04_PGRAPH_PATTERN,
+	0x0040080c,
+	NV04_PGRAPH_PATTERN_SHAPE,
+	0x00400600,
+	NV04_PGRAPH_ROP3,
+	NV04_PGRAPH_CHROMA,
+	NV04_PGRAPH_BETA_AND,
+	NV04_PGRAPH_BETA_PREMULT,
+	NV04_PGRAPH_CONTROL0,
+	NV04_PGRAPH_CONTROL1,
+	NV04_PGRAPH_CONTROL2,
+	NV04_PGRAPH_BLEND,
+	NV04_PGRAPH_STORED_FMT,
+	NV04_PGRAPH_SOURCE_COLOR,
+	0x00400560,
+	0x00400568,
+	0x00400564,
+	0x0040056c,
+	0x00400400,
+	0x00400480,
+	0x00400404,
+	0x00400484,
+	0x00400408,
+	0x00400488,
+	0x0040040c,
+	0x0040048c,
+	0x00400410,
+	0x00400490,
+	0x00400414,
+	0x00400494,
+	0x00400418,
+	0x00400498,
+	0x0040041c,
+	0x0040049c,
+	0x00400420,
+	0x004004a0,
+	0x00400424,
+	0x004004a4,
+	0x00400428,
+	0x004004a8,
+	0x0040042c,
+	0x004004ac,
+	0x00400430,
+	0x004004b0,
+	0x00400434,
+	0x004004b4,
+	0x00400438,
+	0x004004b8,
+	0x0040043c,
+	0x004004bc,
+	0x00400440,
+	0x004004c0,
+	0x00400444,
+	0x004004c4,
+	0x00400448,
+	0x004004c8,
+	0x0040044c,
+	0x004004cc,
+	0x00400450,
+	0x004004d0,
+	0x00400454,
+	0x004004d4,
+	0x00400458,
+	0x004004d8,
+	0x0040045c,
+	0x004004dc,
+	0x00400460,
+	0x004004e0,
+	0x00400464,
+	0x004004e4,
+	0x00400468,
+	0x004004e8,
+	0x0040046c,
+	0x004004ec,
+	0x00400470,
+	0x004004f0,
+	0x00400474,
+	0x004004f4,
+	0x00400478,
+	0x004004f8,
+	0x0040047c,
+	0x004004fc,
+	0x00400534,
+	0x00400538,
+	0x00400514,
+	0x00400518,
+	0x0040051c,
+	0x00400520,
+	0x00400524,
+	0x00400528,
+	0x0040052c,
+	0x00400530,
+	0x00400d00,
+	0x00400d40,
+	0x00400d80,
+	0x00400d04,
+	0x00400d44,
+	0x00400d84,
+	0x00400d08,
+	0x00400d48,
+	0x00400d88,
+	0x00400d0c,
+	0x00400d4c,
+	0x00400d8c,
+	0x00400d10,
+	0x00400d50,
+	0x00400d90,
+	0x00400d14,
+	0x00400d54,
+	0x00400d94,
+	0x00400d18,
+	0x00400d58,
+	0x00400d98,
+	0x00400d1c,
+	0x00400d5c,
+	0x00400d9c,
+	0x00400d20,
+	0x00400d60,
+	0x00400da0,
+	0x00400d24,
+	0x00400d64,
+	0x00400da4,
+	0x00400d28,
+	0x00400d68,
+	0x00400da8,
+	0x00400d2c,
+	0x00400d6c,
+	0x00400dac,
+	0x00400d30,
+	0x00400d70,
+	0x00400db0,
+	0x00400d34,
+	0x00400d74,
+	0x00400db4,
+	0x00400d38,
+	0x00400d78,
+	0x00400db8,
+	0x00400d3c,
+	0x00400d7c,
+	0x00400dbc,
+	0x00400590,
+	0x00400594,
+	0x00400598,
+	0x0040059c,
+	0x004005a8,
+	0x004005ac,
+	0x004005b0,
+	0x004005b4,
+	0x004005c0,
+	0x004005c4,
+	0x004005c8,
+	0x004005cc,
+	0x004005d0,
+	0x004005d4,
+	0x004005d8,
+	0x004005dc,
+	0x004005e0,
+	NV04_PGRAPH_PASSTHRU_0,
+	NV04_PGRAPH_PASSTHRU_1,
+	NV04_PGRAPH_PASSTHRU_2,
+	NV04_PGRAPH_DVD_COLORFMT,
+	NV04_PGRAPH_SCALED_FORMAT,
+	NV04_PGRAPH_MISC24_0,
+	NV04_PGRAPH_MISC24_1,
+	NV04_PGRAPH_MISC24_2,
+	0x00400500,
+	0x00400504,
+	NV04_PGRAPH_VALID1,
+	NV04_PGRAPH_VALID2,
+	NV04_PGRAPH_DEBUG_3
+};
+
+struct nv04_graph_priv {
+	struct nouveau_graph base;
+	struct nv04_graph_chan *chan[16];
+	spinlock_t lock;
+};
+
+struct nv04_graph_chan {
+	struct nouveau_object base;
+	int chid;
+	u32 nv04[ARRAY_SIZE(nv04_graph_ctx_regs)];
+};
+
+
+static inline struct nv04_graph_priv *
+nv04_graph_priv(struct nv04_graph_chan *chan)
+{
+	return (void *)nv_object(chan)->engine;
+}
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+/*
+ * Software methods, why they are needed, and how they all work:
+ *
+ * NV04 and NV05 keep most of the state in PGRAPH context itself, but some
+ * 2d engine settings are kept inside the grobjs themselves. The grobjs are
+ * 3 words long on both. grobj format on NV04 is:
+ *
+ * word 0:
+ *  - bits 0-7: class
+ *  - bit 12: color key active
+ *  - bit 13: clip rect active
+ *  - bit 14: if set, destination surface is swizzled and taken from buffer 5
+ *            [set by NV04_SWIZZLED_SURFACE], otherwise it's linear and taken
+ *            from buffer 0 [set by NV04_CONTEXT_SURFACES_2D or
+ *            NV03_CONTEXT_SURFACE_DST].
+ *  - bits 15-17: 2d operation [aka patch config]
+ *  - bit 24: patch valid [enables rendering using this object]
+ *  - bit 25: surf3d valid [for tex_tri and multitex_tri only]
+ * word 1:
+ *  - bits 0-1: mono format
+ *  - bits 8-13: color format
+ *  - bits 16-31: DMA_NOTIFY instance
+ * word 2:
+ *  - bits 0-15: DMA_A instance
+ *  - bits 16-31: DMA_B instance
+ *
+ * On NV05 it's:
+ *
+ * word 0:
+ *  - bits 0-7: class
+ *  - bit 12: color key active
+ *  - bit 13: clip rect active
+ *  - bit 14: if set, destination surface is swizzled and taken from buffer 5
+ *            [set by NV04_SWIZZLED_SURFACE], otherwise it's linear and taken
+ *            from buffer 0 [set by NV04_CONTEXT_SURFACES_2D or
+ *            NV03_CONTEXT_SURFACE_DST].
+ *  - bits 15-17: 2d operation [aka patch config]
+ *  - bits 20-22: dither mode
+ *  - bit 24: patch valid [enables rendering using this object]
+ *  - bit 25: surface_dst/surface_color/surf2d/surf3d valid
+ *  - bit 26: surface_src/surface_zeta valid
+ *  - bit 27: pattern valid
+ *  - bit 28: rop valid
+ *  - bit 29: beta1 valid
+ *  - bit 30: beta4 valid
+ * word 1:
+ *  - bits 0-1: mono format
+ *  - bits 8-13: color format
+ *  - bits 16-31: DMA_NOTIFY instance
+ * word 2:
+ *  - bits 0-15: DMA_A instance
+ *  - bits 16-31: DMA_B instance
+ *
+ * NV05 will set/unset the relevant valid bits when you poke the relevant
+ * object-binding methods with object of the proper type, or with the NULL
+ * type. It'll only allow rendering using the grobj if all needed objects
+ * are bound. The needed set of objects depends on selected operation: for
+ * example rop object is needed by ROP_AND, but not by SRCCOPY_AND.
+ *
+ * NV04 doesn't have these methods implemented at all, and doesn't have the
+ * relevant bits in grobj. Instead, it'll allow rendering whenever bit 24
+ * is set. So we have to emulate them in software, internally keeping the
+ * same bits as NV05 does. Since grobjs are aligned to 16 bytes on nv04,
+ * but the last word isn't actually used for anything, we abuse it for this
+ * purpose.
+ *
+ * Actually, NV05 can optionally check bit 24 too, but we disable this since
+ * there's no use for it.
+ *
+ * For unknown reasons, NV04 implements surf3d binding in hardware as an
+ * exception. Also for unknown reasons, NV04 doesn't implement the clipping
+ * methods on the surf3d object, so we have to emulate them too.
+ */
+
+static void
+nv04_graph_set_ctx1(struct nouveau_object *object, u32 mask, u32 value)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	int subc = (nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR) >> 13) & 0x7;
+	u32 tmp;
+
+	tmp  = nv_ro32(object, 0x00);
+	tmp &= ~mask;
+	tmp |= value;
+	nv_wo32(object, 0x00, tmp);
+
+	nv_wr32(priv, NV04_PGRAPH_CTX_SWITCH1, tmp);
+	nv_wr32(priv, NV04_PGRAPH_CTX_CACHE1 + (subc<<2), tmp);
+}
+
+static void
+nv04_graph_set_ctx_val(struct nouveau_object *object, u32 mask, u32 value)
+{
+	int class, op, valid = 1;
+	u32 tmp, ctx1;
+
+	ctx1 = nv_ro32(object, 0x00);
+	class = ctx1 & 0xff;
+	op = (ctx1 >> 15) & 7;
+
+	tmp = nv_ro32(object, 0x0c);
+	tmp &= ~mask;
+	tmp |= value;
+	nv_wo32(object, 0x0c, tmp);
+
+	/* check for valid surf2d/surf_dst/surf_color */
+	if (!(tmp & 0x02000000))
+		valid = 0;
+	/* check for valid surf_src/surf_zeta */
+	if ((class == 0x1f || class == 0x48) && !(tmp & 0x04000000))
+		valid = 0;
+
+	switch (op) {
+	/* SRCCOPY_AND, SRCCOPY: no extra objects required */
+	case 0:
+	case 3:
+		break;
+	/* ROP_AND: requires pattern and rop */
+	case 1:
+		if (!(tmp & 0x18000000))
+			valid = 0;
+		break;
+	/* BLEND_AND: requires beta1 */
+	case 2:
+		if (!(tmp & 0x20000000))
+			valid = 0;
+		break;
+	/* SRCCOPY_PREMULT, BLEND_PREMULT: beta4 required */
+	case 4:
+	case 5:
+		if (!(tmp & 0x40000000))
+			valid = 0;
+		break;
+	}
+
+	nv04_graph_set_ctx1(object, 0x01000000, valid << 24);
+}
+
+static int
+nv04_graph_mthd_set_operation(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	u32 class = nv_ro32(object, 0) & 0xff;
+	u32 data = *(u32 *)args;
+	if (data > 5)
+		return 1;
+	/* Old versions of the objects only accept first three operations. */
+	if (data > 2 && class < 0x40)
+		return 1;
+	nv04_graph_set_ctx1(object, 0x00038000, data << 15);
+	/* changing operation changes set of objects needed for validation */
+	nv04_graph_set_ctx_val(object, 0, 0);
+	return 0;
+}
+
+static int
+nv04_graph_mthd_surf3d_clip_h(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	u32 data = *(u32 *)args;
+	u32 min = data & 0xffff, max;
+	u32 w = data >> 16;
+	if (min & 0x8000)
+		/* too large */
+		return 1;
+	if (w & 0x8000)
+		/* yes, it accepts negative for some reason. */
+		w |= 0xffff0000;
+	max = min + w;
+	max &= 0x3ffff;
+	nv_wr32(priv, 0x40053c, min);
+	nv_wr32(priv, 0x400544, max);
+	return 0;
+}
+
+static int
+nv04_graph_mthd_surf3d_clip_v(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	u32 data = *(u32 *)args;
+	u32 min = data & 0xffff, max;
+	u32 w = data >> 16;
+	if (min & 0x8000)
+		/* too large */
+		return 1;
+	if (w & 0x8000)
+		/* yes, it accepts negative for some reason. */
+		w |= 0xffff0000;
+	max = min + w;
+	max &= 0x3ffff;
+	nv_wr32(priv, 0x400540, min);
+	nv_wr32(priv, 0x400548, max);
+	return 0;
+}
+
+static u16
+nv04_graph_mthd_bind_class(struct nouveau_object *object, u32 *args, u32 size)
+{
+	struct nouveau_instmem *imem = nouveau_instmem(object);
+	u32 inst = *(u32 *)args << 4;
+	return nv_ro32(imem, inst);
+}
+
+static int
+nv04_graph_mthd_bind_surf2d(struct nouveau_object *object, u32 mthd,
+			    void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx1(object, 0x00004000, 0);
+		nv04_graph_set_ctx_val(object, 0x02000000, 0);
+		return 0;
+	case 0x42:
+		nv04_graph_set_ctx1(object, 0x00004000, 0);
+		nv04_graph_set_ctx_val(object, 0x02000000, 0x02000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_object *object, u32 mthd,
+				    void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx1(object, 0x00004000, 0);
+		nv04_graph_set_ctx_val(object, 0x02000000, 0);
+		return 0;
+	case 0x42:
+		nv04_graph_set_ctx1(object, 0x00004000, 0);
+		nv04_graph_set_ctx_val(object, 0x02000000, 0x02000000);
+		return 0;
+	case 0x52:
+		nv04_graph_set_ctx1(object, 0x00004000, 0x00004000);
+		nv04_graph_set_ctx_val(object, 0x02000000, 0x02000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv01_graph_mthd_bind_patt(struct nouveau_object *object, u32 mthd,
+			  void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x08000000, 0);
+		return 0;
+	case 0x18:
+		nv04_graph_set_ctx_val(object, 0x08000000, 0x08000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_patt(struct nouveau_object *object, u32 mthd,
+			  void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x08000000, 0);
+		return 0;
+	case 0x44:
+		nv04_graph_set_ctx_val(object, 0x08000000, 0x08000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_rop(struct nouveau_object *object, u32 mthd,
+			 void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x10000000, 0);
+		return 0;
+	case 0x43:
+		nv04_graph_set_ctx_val(object, 0x10000000, 0x10000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_beta1(struct nouveau_object *object, u32 mthd,
+			   void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x20000000, 0);
+		return 0;
+	case 0x12:
+		nv04_graph_set_ctx_val(object, 0x20000000, 0x20000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_beta4(struct nouveau_object *object, u32 mthd,
+			   void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x40000000, 0);
+		return 0;
+	case 0x72:
+		nv04_graph_set_ctx_val(object, 0x40000000, 0x40000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_surf_dst(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x02000000, 0);
+		return 0;
+	case 0x58:
+		nv04_graph_set_ctx_val(object, 0x02000000, 0x02000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_surf_src(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x04000000, 0);
+		return 0;
+	case 0x59:
+		nv04_graph_set_ctx_val(object, 0x04000000, 0x04000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_surf_color(struct nouveau_object *object, u32 mthd,
+				void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x02000000, 0);
+		return 0;
+	case 0x5a:
+		nv04_graph_set_ctx_val(object, 0x02000000, 0x02000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv04_graph_mthd_bind_surf_zeta(struct nouveau_object *object, u32 mthd,
+			       void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx_val(object, 0x04000000, 0);
+		return 0;
+	case 0x5b:
+		nv04_graph_set_ctx_val(object, 0x04000000, 0x04000000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv01_graph_mthd_bind_clip(struct nouveau_object *object, u32 mthd,
+			  void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx1(object, 0x2000, 0);
+		return 0;
+	case 0x19:
+		nv04_graph_set_ctx1(object, 0x2000, 0x2000);
+		return 0;
+	}
+	return 1;
+}
+
+static int
+nv01_graph_mthd_bind_chroma(struct nouveau_object *object, u32 mthd,
+			    void *args, u32 size)
+{
+	switch (nv04_graph_mthd_bind_class(object, args, size)) {
+	case 0x30:
+		nv04_graph_set_ctx1(object, 0x1000, 0);
+		return 0;
+	/* Yes, for some reason even the old versions of objects
+	 * accept 0x57 and not 0x17. Consistency be damned.
+	 */
+	case 0x57:
+		nv04_graph_set_ctx1(object, 0x1000, 0x1000);
+		return 0;
+	}
+	return 1;
+}
+
+static struct nouveau_omthds
+nv03_graph_gdi_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_patt },
+	{ 0x0188, nv04_graph_mthd_bind_rop },
+	{ 0x018c, nv04_graph_mthd_bind_beta1 },
+	{ 0x0190, nv04_graph_mthd_bind_surf_dst },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_gdi_omthds[] = {
+	{ 0x0188, nv04_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_beta4 },
+	{ 0x0198, nv04_graph_mthd_bind_surf2d },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv01_graph_blit_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv01_graph_mthd_bind_clip },
+	{ 0x018c, nv01_graph_mthd_bind_patt },
+	{ 0x0190, nv04_graph_mthd_bind_rop },
+	{ 0x0194, nv04_graph_mthd_bind_beta1 },
+	{ 0x0198, nv04_graph_mthd_bind_surf_dst },
+	{ 0x019c, nv04_graph_mthd_bind_surf_src },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_blit_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv01_graph_mthd_bind_clip },
+	{ 0x018c, nv04_graph_mthd_bind_patt },
+	{ 0x0190, nv04_graph_mthd_bind_rop },
+	{ 0x0194, nv04_graph_mthd_bind_beta1 },
+	{ 0x0198, nv04_graph_mthd_bind_beta4 },
+	{ 0x019c, nv04_graph_mthd_bind_surf2d },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_iifc_omthds[] = {
+	{ 0x0188, nv01_graph_mthd_bind_chroma },
+	{ 0x018c, nv01_graph_mthd_bind_clip },
+	{ 0x0190, nv04_graph_mthd_bind_patt },
+	{ 0x0194, nv04_graph_mthd_bind_rop },
+	{ 0x0198, nv04_graph_mthd_bind_beta1 },
+	{ 0x019c, nv04_graph_mthd_bind_beta4 },
+	{ 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf },
+	{ 0x03e4, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv01_graph_ifc_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv01_graph_mthd_bind_clip },
+	{ 0x018c, nv01_graph_mthd_bind_patt },
+	{ 0x0190, nv04_graph_mthd_bind_rop },
+	{ 0x0194, nv04_graph_mthd_bind_beta1 },
+	{ 0x0198, nv04_graph_mthd_bind_surf_dst },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_ifc_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv01_graph_mthd_bind_clip },
+	{ 0x018c, nv04_graph_mthd_bind_patt },
+	{ 0x0190, nv04_graph_mthd_bind_rop },
+	{ 0x0194, nv04_graph_mthd_bind_beta1 },
+	{ 0x0198, nv04_graph_mthd_bind_beta4 },
+	{ 0x019c, nv04_graph_mthd_bind_surf2d },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv03_graph_sifc_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv01_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_surf_dst },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_sifc_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_chroma },
+	{ 0x0188, nv04_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_beta4 },
+	{ 0x0198, nv04_graph_mthd_bind_surf2d },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv03_graph_sifm_omthds[] = {
+	{ 0x0188, nv01_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_surf_dst },
+	{ 0x0304, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_sifm_omthds[] = {
+	{ 0x0188, nv04_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_beta4 },
+	{ 0x0198, nv04_graph_mthd_bind_surf2d },
+	{ 0x0304, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_surf3d_omthds[] = {
+	{ 0x02f8, nv04_graph_mthd_surf3d_clip_h },
+	{ 0x02fc, nv04_graph_mthd_surf3d_clip_v },
+	{}
+};
+
+static struct nouveau_omthds
+nv03_graph_ttri_omthds[] = {
+	{ 0x0188, nv01_graph_mthd_bind_clip },
+	{ 0x018c, nv04_graph_mthd_bind_surf_color },
+	{ 0x0190, nv04_graph_mthd_bind_surf_zeta },
+	{}
+};
+
+static struct nouveau_omthds
+nv01_graph_prim_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_clip },
+	{ 0x0188, nv01_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_surf_dst },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static struct nouveau_omthds
+nv04_graph_prim_omthds[] = {
+	{ 0x0184, nv01_graph_mthd_bind_clip },
+	{ 0x0188, nv04_graph_mthd_bind_patt },
+	{ 0x018c, nv04_graph_mthd_bind_rop },
+	{ 0x0190, nv04_graph_mthd_bind_beta1 },
+	{ 0x0194, nv04_graph_mthd_bind_beta4 },
+	{ 0x0198, nv04_graph_mthd_bind_surf2d },
+	{ 0x02fc, nv04_graph_mthd_set_operation },
+	{}
+};
+
+static int
+nv04_graph_object_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    16, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+#ifdef __BIG_ENDIAN
+	nv_mo32(obj, 0x00, 0x00080000, 0x00080000);
+#endif
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+	nv_wo32(obj, 0x0c, 0x00000000);
+	return 0;
+}
+
+struct nouveau_ofuncs
+nv04_graph_ofuncs = {
+	.ctor = nv04_graph_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_oclass
+nv04_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs }, /* beta1 */
+	{ 0x0017, &nv04_graph_ofuncs }, /* chroma */
+	{ 0x0018, &nv04_graph_ofuncs }, /* pattern (nv01) */
+	{ 0x0019, &nv04_graph_ofuncs }, /* clip */
+	{ 0x001c, &nv04_graph_ofuncs, nv01_graph_prim_omthds }, /* line */
+	{ 0x001d, &nv04_graph_ofuncs, nv01_graph_prim_omthds }, /* tri */
+	{ 0x001e, &nv04_graph_ofuncs, nv01_graph_prim_omthds }, /* rect */
+	{ 0x001f, &nv04_graph_ofuncs, nv01_graph_blit_omthds },
+	{ 0x0021, &nv04_graph_ofuncs, nv01_graph_ifc_omthds },
+	{ 0x0030, &nv04_graph_ofuncs }, /* null */
+	{ 0x0036, &nv04_graph_ofuncs, nv03_graph_sifc_omthds },
+	{ 0x0037, &nv04_graph_ofuncs, nv03_graph_sifm_omthds },
+	{ 0x0038, &nv04_graph_ofuncs }, /* dvd subpicture */
+	{ 0x0039, &nv04_graph_ofuncs }, /* m2mf */
+	{ 0x0042, &nv04_graph_ofuncs }, /* surf2d */
+	{ 0x0043, &nv04_graph_ofuncs }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs }, /* pattern */
+	{ 0x0048, &nv04_graph_ofuncs, nv03_graph_ttri_omthds },
+	{ 0x004a, &nv04_graph_ofuncs, nv04_graph_gdi_omthds },
+	{ 0x004b, &nv04_graph_ofuncs, nv03_graph_gdi_omthds },
+	{ 0x0052, &nv04_graph_ofuncs }, /* swzsurf */
+	{ 0x0053, &nv04_graph_ofuncs, nv04_graph_surf3d_omthds },
+	{ 0x0054, &nv04_graph_ofuncs }, /* ttri */
+	{ 0x0055, &nv04_graph_ofuncs }, /* mtri */
+	{ 0x0057, &nv04_graph_ofuncs }, /* chroma */
+	{ 0x0058, &nv04_graph_ofuncs }, /* surf_dst */
+	{ 0x0059, &nv04_graph_ofuncs }, /* surf_src */
+	{ 0x005a, &nv04_graph_ofuncs }, /* surf_color */
+	{ 0x005b, &nv04_graph_ofuncs }, /* surf_zeta */
+	{ 0x005c, &nv04_graph_ofuncs, nv04_graph_prim_omthds }, /* line */
+	{ 0x005d, &nv04_graph_ofuncs, nv04_graph_prim_omthds }, /* tri */
+	{ 0x005e, &nv04_graph_ofuncs, nv04_graph_prim_omthds }, /* rect */
+	{ 0x005f, &nv04_graph_ofuncs, nv04_graph_blit_omthds },
+	{ 0x0060, &nv04_graph_ofuncs, nv04_graph_iifc_omthds },
+	{ 0x0061, &nv04_graph_ofuncs, nv04_graph_ifc_omthds },
+	{ 0x0064, &nv04_graph_ofuncs }, /* iifc (nv05) */
+	{ 0x0065, &nv04_graph_ofuncs }, /* ifc (nv05) */
+	{ 0x0066, &nv04_graph_ofuncs }, /* sifc (nv05) */
+	{ 0x0072, &nv04_graph_ofuncs }, /* beta4 */
+	{ 0x0076, &nv04_graph_ofuncs, nv04_graph_sifc_omthds },
+	{ 0x0077, &nv04_graph_ofuncs, nv04_graph_sifm_omthds },
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static struct nv04_graph_chan *
+nv04_graph_channel(struct nv04_graph_priv *priv)
+{
+	struct nv04_graph_chan *chan = NULL;
+	if (nv_rd32(priv, NV04_PGRAPH_CTX_CONTROL) & 0x00010000) {
+		int chid = nv_rd32(priv, NV04_PGRAPH_CTX_USER) >> 24;
+		if (chid < ARRAY_SIZE(priv->chan))
+			chan = priv->chan[chid];
+	}
+	return chan;
+}
+
+static int
+nv04_graph_load_context(struct nv04_graph_chan *chan, int chid)
+{
+	struct nv04_graph_priv *priv = nv04_graph_priv(chan);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++)
+		nv_wr32(priv, nv04_graph_ctx_regs[i], chan->nv04[i]);
+
+	nv_wr32(priv, NV04_PGRAPH_CTX_CONTROL, 0x10010100);
+	nv_mask(priv, NV04_PGRAPH_CTX_USER, 0xff000000, chid << 24);
+	nv_mask(priv, NV04_PGRAPH_FFINTFC_ST2, 0xfff00000, 0x00000000);
+	return 0;
+}
+
+static int
+nv04_graph_unload_context(struct nv04_graph_chan *chan)
+{
+	struct nv04_graph_priv *priv = nv04_graph_priv(chan);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++)
+		chan->nv04[i] = nv_rd32(priv, nv04_graph_ctx_regs[i]);
+
+	nv_wr32(priv, NV04_PGRAPH_CTX_CONTROL, 0x10000000);
+	nv_mask(priv, NV04_PGRAPH_CTX_USER, 0xff000000, 0x0f000000);
+	return 0;
+}
+
+static void
+nv04_graph_context_switch(struct nv04_graph_priv *priv)
+{
+	struct nv04_graph_chan *prev = NULL;
+	struct nv04_graph_chan *next = NULL;
+	unsigned long flags;
+	int chid;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	nv04_graph_idle(priv);
+
+	/* If previous context is valid, we need to save it */
+	prev = nv04_graph_channel(priv);
+	if (prev)
+		nv04_graph_unload_context(prev);
+
+	/* load context for next channel */
+	chid = (nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR) >> 24) & 0x0f;
+	next = priv->chan[chid];
+	if (next)
+		nv04_graph_load_context(next, chid);
+
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static u32 *ctx_reg(struct nv04_graph_chan *chan, u32 reg)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++) {
+		if (nv04_graph_ctx_regs[i] == reg)
+			return &chan->nv04[i];
+	}
+
+	return NULL;
+}
+
+static int
+nv04_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_fifo_chan *fifo = (void *)parent;
+	struct nv04_graph_priv *priv = (void *)engine;
+	struct nv04_graph_chan *chan;
+	unsigned long flags;
+	int ret;
+
+	ret = nouveau_object_create(parent, engine, oclass, 0, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	if (priv->chan[fifo->chid]) {
+		*pobject = nv_object(priv->chan[fifo->chid]);
+		atomic_inc(&(*pobject)->refcount);
+		spin_unlock_irqrestore(&priv->lock, flags);
+		nouveau_object_destroy(&chan->base);
+		return 1;
+	}
+
+	*ctx_reg(chan, NV04_PGRAPH_DEBUG_3) = 0xfad4ff31;
+
+	priv->chan[fifo->chid] = chan;
+	chan->chid = fifo->chid;
+	spin_unlock_irqrestore(&priv->lock, flags);
+	return 0;
+}
+
+static void
+nv04_graph_context_dtor(struct nouveau_object *object)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	struct nv04_graph_chan *chan = (void *)object;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	priv->chan[chan->chid] = NULL;
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	nouveau_object_destroy(&chan->base);
+}
+
+static int
+nv04_graph_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	struct nv04_graph_chan *chan = (void *)object;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+	if (nv04_graph_channel(priv) == chan)
+		nv04_graph_unload_context(chan);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	return nouveau_object_fini(&chan->base, suspend);
+}
+
+static struct nouveau_oclass
+nv04_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_graph_context_ctor,
+		.dtor = nv04_graph_context_dtor,
+		.init = nouveau_object_init,
+		.fini = nv04_graph_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+bool
+nv04_graph_idle(void *obj)
+{
+	struct nouveau_graph *graph = nouveau_graph(obj);
+	u32 mask = 0xffffffff;
+
+	if (nv_device(obj)->card_type == NV_40)
+		mask &= ~NV40_PGRAPH_STATUS_SYNC_STALL;
+
+	if (!nv_wait(graph, NV04_PGRAPH_STATUS, mask, 0)) {
+		nv_error(graph, "idle timed out with status 0x%08x\n",
+			 nv_rd32(graph, NV04_PGRAPH_STATUS));
+		return false;
+	}
+
+	return true;
+}
+
+static const struct nouveau_bitfield
+nv04_graph_intr_name[] = {
+	{ NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
+	{}
+};
+
+static const struct nouveau_bitfield
+nv04_graph_nstatus[] = {
+	{ NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
+	{ NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
+	{ NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
+	{ NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
+	{}
+};
+
+const struct nouveau_bitfield
+nv04_graph_nsource[] = {
+	{ NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
+	{ NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
+	{ NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
+	{ NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
+	{ NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
+	{ NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
+	{ NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
+	{ NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
+	{ NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
+	{ NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
+	{ NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
+	{ NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
+	{ NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
+	{ NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
+	{ NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
+	{ NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
+	{ NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
+	{ NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
+	{ NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
+	{}
+};
+
+static void
+nv04_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nv04_graph_priv *priv = (void *)subdev;
+	struct nv04_graph_chan *chan = NULL;
+	struct nouveau_namedb *namedb = NULL;
+	struct nouveau_handle *handle = NULL;
+	u32 stat = nv_rd32(priv, NV03_PGRAPH_INTR);
+	u32 nsource = nv_rd32(priv, NV03_PGRAPH_NSOURCE);
+	u32 nstatus = nv_rd32(priv, NV03_PGRAPH_NSTATUS);
+	u32 addr = nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR);
+	u32 chid = (addr & 0x0f000000) >> 24;
+	u32 subc = (addr & 0x0000e000) >> 13;
+	u32 mthd = (addr & 0x00001ffc);
+	u32 data = nv_rd32(priv, NV04_PGRAPH_TRAPPED_DATA);
+	u32 class = nv_rd32(priv, 0x400180 + subc * 4) & 0xff;
+	u32 inst = (nv_rd32(priv, 0x40016c) & 0xffff) << 4;
+	u32 show = stat;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	chan = priv->chan[chid];
+	if (chan)
+		namedb = (void *)nv_pclass(nv_object(chan), NV_NAMEDB_CLASS);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	if (stat & NV_PGRAPH_INTR_NOTIFY) {
+		if (chan && (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD)) {
+			handle = nouveau_namedb_get_vinst(namedb, inst);
+			if (handle && !nv_call(handle->object, mthd, data))
+				show &= ~NV_PGRAPH_INTR_NOTIFY;
+		}
+	}
+
+	if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
+		nv_wr32(priv, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
+		stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+		show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+		nv04_graph_context_switch(priv);
+	}
+
+	nv_wr32(priv, NV03_PGRAPH_INTR, stat);
+	nv_wr32(priv, NV04_PGRAPH_FIFO, 0x00000001);
+
+	if (show) {
+		nv_error(priv, "");
+		nouveau_bitfield_print(nv04_graph_intr_name, show);
+		printk(" nsource:");
+		nouveau_bitfield_print(nv04_graph_nsource, nsource);
+		printk(" nstatus:");
+		nouveau_bitfield_print(nv04_graph_nstatus, nstatus);
+		printk("\n");
+		nv_error(priv, "ch %d/%d class 0x%04x "
+			       "mthd 0x%04x data 0x%08x\n",
+			 chid, subc, class, mthd, data);
+	}
+
+	nouveau_namedb_put(handle);
+}
+
+static int
+nv04_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv04_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv04_graph_intr;
+	nv_engine(priv)->cclass = &nv04_graph_cclass;
+	nv_engine(priv)->sclass = nv04_graph_sclass;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static int
+nv04_graph_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object);
+	struct nv04_graph_priv *priv = (void *)engine;
+	int ret;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* Enable PGRAPH interrupts */
+	nv_wr32(priv, NV03_PGRAPH_INTR, 0xFFFFFFFF);
+	nv_wr32(priv, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV04_PGRAPH_VALID1, 0);
+	nv_wr32(priv, NV04_PGRAPH_VALID2, 0);
+	/*nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x000001FF);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x001FFFFF);*/
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x1231c000);
+	/*1231C000 blob, 001 haiku*/
+	/*V_WRITE(NV04_PGRAPH_DEBUG_1, 0xf2d91100);*/
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_1, 0x72111100);
+	/*0x72111100 blob , 01 haiku*/
+	/*nv_wr32(priv, NV04_PGRAPH_DEBUG_2, 0x11d5f870);*/
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_2, 0x11d5f071);
+	/*haiku same*/
+
+	/*nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0xfad4ff31);*/
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0xf0d4ff31);
+	/*haiku and blob 10d4*/
+
+	nv_wr32(priv, NV04_PGRAPH_STATE        , 0xFFFFFFFF);
+	nv_wr32(priv, NV04_PGRAPH_CTX_CONTROL  , 0x10000100);
+	nv_mask(priv, NV04_PGRAPH_CTX_USER, 0xff000000, 0x0f000000);
+
+	/* These don't belong here, they're part of a per-channel context */
+	nv_wr32(priv, NV04_PGRAPH_PATTERN_SHAPE, 0x00000000);
+	nv_wr32(priv, NV04_PGRAPH_BETA_AND     , 0xFFFFFFFF);
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_graph_ctor,
+		.dtor = _nouveau_graph_dtor,
+		.init = nv04_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
new file mode 100644
index 00000000000..92521c89e77
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
@@ -0,0 +1,1314 @@
+/*
+ * Copyright 2007 Matthieu CASTET <castet.matthieu@free.fr>
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/handle.h>
+
+#include <subdev/fb.h>
+
+#include <engine/fifo.h>
+#include <engine/graph.h>
+
+#include "regs.h"
+
+struct pipe_state {
+	u32 pipe_0x0000[0x040/4];
+	u32 pipe_0x0040[0x010/4];
+	u32 pipe_0x0200[0x0c0/4];
+	u32 pipe_0x4400[0x080/4];
+	u32 pipe_0x6400[0x3b0/4];
+	u32 pipe_0x6800[0x2f0/4];
+	u32 pipe_0x6c00[0x030/4];
+	u32 pipe_0x7000[0x130/4];
+	u32 pipe_0x7400[0x0c0/4];
+	u32 pipe_0x7800[0x0c0/4];
+};
+
+static int nv10_graph_ctx_regs[] = {
+	NV10_PGRAPH_CTX_SWITCH(0),
+	NV10_PGRAPH_CTX_SWITCH(1),
+	NV10_PGRAPH_CTX_SWITCH(2),
+	NV10_PGRAPH_CTX_SWITCH(3),
+	NV10_PGRAPH_CTX_SWITCH(4),
+	NV10_PGRAPH_CTX_CACHE(0, 0),
+	NV10_PGRAPH_CTX_CACHE(0, 1),
+	NV10_PGRAPH_CTX_CACHE(0, 2),
+	NV10_PGRAPH_CTX_CACHE(0, 3),
+	NV10_PGRAPH_CTX_CACHE(0, 4),
+	NV10_PGRAPH_CTX_CACHE(1, 0),
+	NV10_PGRAPH_CTX_CACHE(1, 1),
+	NV10_PGRAPH_CTX_CACHE(1, 2),
+	NV10_PGRAPH_CTX_CACHE(1, 3),
+	NV10_PGRAPH_CTX_CACHE(1, 4),
+	NV10_PGRAPH_CTX_CACHE(2, 0),
+	NV10_PGRAPH_CTX_CACHE(2, 1),
+	NV10_PGRAPH_CTX_CACHE(2, 2),
+	NV10_PGRAPH_CTX_CACHE(2, 3),
+	NV10_PGRAPH_CTX_CACHE(2, 4),
+	NV10_PGRAPH_CTX_CACHE(3, 0),
+	NV10_PGRAPH_CTX_CACHE(3, 1),
+	NV10_PGRAPH_CTX_CACHE(3, 2),
+	NV10_PGRAPH_CTX_CACHE(3, 3),
+	NV10_PGRAPH_CTX_CACHE(3, 4),
+	NV10_PGRAPH_CTX_CACHE(4, 0),
+	NV10_PGRAPH_CTX_CACHE(4, 1),
+	NV10_PGRAPH_CTX_CACHE(4, 2),
+	NV10_PGRAPH_CTX_CACHE(4, 3),
+	NV10_PGRAPH_CTX_CACHE(4, 4),
+	NV10_PGRAPH_CTX_CACHE(5, 0),
+	NV10_PGRAPH_CTX_CACHE(5, 1),
+	NV10_PGRAPH_CTX_CACHE(5, 2),
+	NV10_PGRAPH_CTX_CACHE(5, 3),
+	NV10_PGRAPH_CTX_CACHE(5, 4),
+	NV10_PGRAPH_CTX_CACHE(6, 0),
+	NV10_PGRAPH_CTX_CACHE(6, 1),
+	NV10_PGRAPH_CTX_CACHE(6, 2),
+	NV10_PGRAPH_CTX_CACHE(6, 3),
+	NV10_PGRAPH_CTX_CACHE(6, 4),
+	NV10_PGRAPH_CTX_CACHE(7, 0),
+	NV10_PGRAPH_CTX_CACHE(7, 1),
+	NV10_PGRAPH_CTX_CACHE(7, 2),
+	NV10_PGRAPH_CTX_CACHE(7, 3),
+	NV10_PGRAPH_CTX_CACHE(7, 4),
+	NV10_PGRAPH_CTX_USER,
+	NV04_PGRAPH_DMA_START_0,
+	NV04_PGRAPH_DMA_START_1,
+	NV04_PGRAPH_DMA_LENGTH,
+	NV04_PGRAPH_DMA_MISC,
+	NV10_PGRAPH_DMA_PITCH,
+	NV04_PGRAPH_BOFFSET0,
+	NV04_PGRAPH_BBASE0,
+	NV04_PGRAPH_BLIMIT0,
+	NV04_PGRAPH_BOFFSET1,
+	NV04_PGRAPH_BBASE1,
+	NV04_PGRAPH_BLIMIT1,
+	NV04_PGRAPH_BOFFSET2,
+	NV04_PGRAPH_BBASE2,
+	NV04_PGRAPH_BLIMIT2,
+	NV04_PGRAPH_BOFFSET3,
+	NV04_PGRAPH_BBASE3,
+	NV04_PGRAPH_BLIMIT3,
+	NV04_PGRAPH_BOFFSET4,
+	NV04_PGRAPH_BBASE4,
+	NV04_PGRAPH_BLIMIT4,
+	NV04_PGRAPH_BOFFSET5,
+	NV04_PGRAPH_BBASE5,
+	NV04_PGRAPH_BLIMIT5,
+	NV04_PGRAPH_BPITCH0,
+	NV04_PGRAPH_BPITCH1,
+	NV04_PGRAPH_BPITCH2,
+	NV04_PGRAPH_BPITCH3,
+	NV04_PGRAPH_BPITCH4,
+	NV10_PGRAPH_SURFACE,
+	NV10_PGRAPH_STATE,
+	NV04_PGRAPH_BSWIZZLE2,
+	NV04_PGRAPH_BSWIZZLE5,
+	NV04_PGRAPH_BPIXEL,
+	NV10_PGRAPH_NOTIFY,
+	NV04_PGRAPH_PATT_COLOR0,
+	NV04_PGRAPH_PATT_COLOR1,
+	NV04_PGRAPH_PATT_COLORRAM, /* 64 values from 0x400900 to 0x4009fc */
+	0x00400904,
+	0x00400908,
+	0x0040090c,
+	0x00400910,
+	0x00400914,
+	0x00400918,
+	0x0040091c,
+	0x00400920,
+	0x00400924,
+	0x00400928,
+	0x0040092c,
+	0x00400930,
+	0x00400934,
+	0x00400938,
+	0x0040093c,
+	0x00400940,
+	0x00400944,
+	0x00400948,
+	0x0040094c,
+	0x00400950,
+	0x00400954,
+	0x00400958,
+	0x0040095c,
+	0x00400960,
+	0x00400964,
+	0x00400968,
+	0x0040096c,
+	0x00400970,
+	0x00400974,
+	0x00400978,
+	0x0040097c,
+	0x00400980,
+	0x00400984,
+	0x00400988,
+	0x0040098c,
+	0x00400990,
+	0x00400994,
+	0x00400998,
+	0x0040099c,
+	0x004009a0,
+	0x004009a4,
+	0x004009a8,
+	0x004009ac,
+	0x004009b0,
+	0x004009b4,
+	0x004009b8,
+	0x004009bc,
+	0x004009c0,
+	0x004009c4,
+	0x004009c8,
+	0x004009cc,
+	0x004009d0,
+	0x004009d4,
+	0x004009d8,
+	0x004009dc,
+	0x004009e0,
+	0x004009e4,
+	0x004009e8,
+	0x004009ec,
+	0x004009f0,
+	0x004009f4,
+	0x004009f8,
+	0x004009fc,
+	NV04_PGRAPH_PATTERN,	/* 2 values from 0x400808 to 0x40080c */
+	0x0040080c,
+	NV04_PGRAPH_PATTERN_SHAPE,
+	NV03_PGRAPH_MONO_COLOR0,
+	NV04_PGRAPH_ROP3,
+	NV04_PGRAPH_CHROMA,
+	NV04_PGRAPH_BETA_AND,
+	NV04_PGRAPH_BETA_PREMULT,
+	0x00400e70,
+	0x00400e74,
+	0x00400e78,
+	0x00400e7c,
+	0x00400e80,
+	0x00400e84,
+	0x00400e88,
+	0x00400e8c,
+	0x00400ea0,
+	0x00400ea4,
+	0x00400ea8,
+	0x00400e90,
+	0x00400e94,
+	0x00400e98,
+	0x00400e9c,
+	NV10_PGRAPH_WINDOWCLIP_HORIZONTAL, /* 8 values from 0x400f00-0x400f1c */
+	NV10_PGRAPH_WINDOWCLIP_VERTICAL,   /* 8 values from 0x400f20-0x400f3c */
+	0x00400f04,
+	0x00400f24,
+	0x00400f08,
+	0x00400f28,
+	0x00400f0c,
+	0x00400f2c,
+	0x00400f10,
+	0x00400f30,
+	0x00400f14,
+	0x00400f34,
+	0x00400f18,
+	0x00400f38,
+	0x00400f1c,
+	0x00400f3c,
+	NV10_PGRAPH_XFMODE0,
+	NV10_PGRAPH_XFMODE1,
+	NV10_PGRAPH_GLOBALSTATE0,
+	NV10_PGRAPH_GLOBALSTATE1,
+	NV04_PGRAPH_STORED_FMT,
+	NV04_PGRAPH_SOURCE_COLOR,
+	NV03_PGRAPH_ABS_X_RAM,	/* 32 values from 0x400400 to 0x40047c */
+	NV03_PGRAPH_ABS_Y_RAM,	/* 32 values from 0x400480 to 0x4004fc */
+	0x00400404,
+	0x00400484,
+	0x00400408,
+	0x00400488,
+	0x0040040c,
+	0x0040048c,
+	0x00400410,
+	0x00400490,
+	0x00400414,
+	0x00400494,
+	0x00400418,
+	0x00400498,
+	0x0040041c,
+	0x0040049c,
+	0x00400420,
+	0x004004a0,
+	0x00400424,
+	0x004004a4,
+	0x00400428,
+	0x004004a8,
+	0x0040042c,
+	0x004004ac,
+	0x00400430,
+	0x004004b0,
+	0x00400434,
+	0x004004b4,
+	0x00400438,
+	0x004004b8,
+	0x0040043c,
+	0x004004bc,
+	0x00400440,
+	0x004004c0,
+	0x00400444,
+	0x004004c4,
+	0x00400448,
+	0x004004c8,
+	0x0040044c,
+	0x004004cc,
+	0x00400450,
+	0x004004d0,
+	0x00400454,
+	0x004004d4,
+	0x00400458,
+	0x004004d8,
+	0x0040045c,
+	0x004004dc,
+	0x00400460,
+	0x004004e0,
+	0x00400464,
+	0x004004e4,
+	0x00400468,
+	0x004004e8,
+	0x0040046c,
+	0x004004ec,
+	0x00400470,
+	0x004004f0,
+	0x00400474,
+	0x004004f4,
+	0x00400478,
+	0x004004f8,
+	0x0040047c,
+	0x004004fc,
+	NV03_PGRAPH_ABS_UCLIP_XMIN,
+	NV03_PGRAPH_ABS_UCLIP_XMAX,
+	NV03_PGRAPH_ABS_UCLIP_YMIN,
+	NV03_PGRAPH_ABS_UCLIP_YMAX,
+	0x00400550,
+	0x00400558,
+	0x00400554,
+	0x0040055c,
+	NV03_PGRAPH_ABS_UCLIPA_XMIN,
+	NV03_PGRAPH_ABS_UCLIPA_XMAX,
+	NV03_PGRAPH_ABS_UCLIPA_YMIN,
+	NV03_PGRAPH_ABS_UCLIPA_YMAX,
+	NV03_PGRAPH_ABS_ICLIP_XMAX,
+	NV03_PGRAPH_ABS_ICLIP_YMAX,
+	NV03_PGRAPH_XY_LOGIC_MISC0,
+	NV03_PGRAPH_XY_LOGIC_MISC1,
+	NV03_PGRAPH_XY_LOGIC_MISC2,
+	NV03_PGRAPH_XY_LOGIC_MISC3,
+	NV03_PGRAPH_CLIPX_0,
+	NV03_PGRAPH_CLIPX_1,
+	NV03_PGRAPH_CLIPY_0,
+	NV03_PGRAPH_CLIPY_1,
+	NV10_PGRAPH_COMBINER0_IN_ALPHA,
+	NV10_PGRAPH_COMBINER1_IN_ALPHA,
+	NV10_PGRAPH_COMBINER0_IN_RGB,
+	NV10_PGRAPH_COMBINER1_IN_RGB,
+	NV10_PGRAPH_COMBINER_COLOR0,
+	NV10_PGRAPH_COMBINER_COLOR1,
+	NV10_PGRAPH_COMBINER0_OUT_ALPHA,
+	NV10_PGRAPH_COMBINER1_OUT_ALPHA,
+	NV10_PGRAPH_COMBINER0_OUT_RGB,
+	NV10_PGRAPH_COMBINER1_OUT_RGB,
+	NV10_PGRAPH_COMBINER_FINAL0,
+	NV10_PGRAPH_COMBINER_FINAL1,
+	0x00400e00,
+	0x00400e04,
+	0x00400e08,
+	0x00400e0c,
+	0x00400e10,
+	0x00400e14,
+	0x00400e18,
+	0x00400e1c,
+	0x00400e20,
+	0x00400e24,
+	0x00400e28,
+	0x00400e2c,
+	0x00400e30,
+	0x00400e34,
+	0x00400e38,
+	0x00400e3c,
+	NV04_PGRAPH_PASSTHRU_0,
+	NV04_PGRAPH_PASSTHRU_1,
+	NV04_PGRAPH_PASSTHRU_2,
+	NV10_PGRAPH_DIMX_TEXTURE,
+	NV10_PGRAPH_WDIMX_TEXTURE,
+	NV10_PGRAPH_DVD_COLORFMT,
+	NV10_PGRAPH_SCALED_FORMAT,
+	NV04_PGRAPH_MISC24_0,
+	NV04_PGRAPH_MISC24_1,
+	NV04_PGRAPH_MISC24_2,
+	NV03_PGRAPH_X_MISC,
+	NV03_PGRAPH_Y_MISC,
+	NV04_PGRAPH_VALID1,
+	NV04_PGRAPH_VALID2,
+};
+
+static int nv17_graph_ctx_regs[] = {
+	NV10_PGRAPH_DEBUG_4,
+	0x004006b0,
+	0x00400eac,
+	0x00400eb0,
+	0x00400eb4,
+	0x00400eb8,
+	0x00400ebc,
+	0x00400ec0,
+	0x00400ec4,
+	0x00400ec8,
+	0x00400ecc,
+	0x00400ed0,
+	0x00400ed4,
+	0x00400ed8,
+	0x00400edc,
+	0x00400ee0,
+	0x00400a00,
+	0x00400a04,
+};
+
+struct nv10_graph_priv {
+	struct nouveau_graph base;
+	struct nv10_graph_chan *chan[32];
+	spinlock_t lock;
+};
+
+struct nv10_graph_chan {
+	struct nouveau_object base;
+	int chid;
+	int nv10[ARRAY_SIZE(nv10_graph_ctx_regs)];
+	int nv17[ARRAY_SIZE(nv17_graph_ctx_regs)];
+	struct pipe_state pipe_state;
+	u32 lma_window[4];
+};
+
+
+static inline struct nv10_graph_priv *
+nv10_graph_priv(struct nv10_graph_chan *chan)
+{
+	return (void *)nv_object(chan)->engine;
+}
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+#define PIPE_SAVE(priv, state, addr)					\
+	do {								\
+		int __i;						\
+		nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
+		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
+			state[__i] = nv_rd32(priv, NV10_PGRAPH_PIPE_DATA); \
+	} while (0)
+
+#define PIPE_RESTORE(priv, state, addr)					\
+	do {								\
+		int __i;						\
+		nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
+		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
+			nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, state[__i]); \
+	} while (0)
+
+static struct nouveau_oclass
+nv10_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs }, /* pattern */
+	{ 0x004a, &nv04_graph_ofuncs }, /* gdi */
+	{ 0x0052, &nv04_graph_ofuncs }, /* swzsurf */
+	{ 0x005f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0062, &nv04_graph_ofuncs }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0093, &nv04_graph_ofuncs }, /* surf3d */
+	{ 0x0094, &nv04_graph_ofuncs }, /* ttri */
+	{ 0x0095, &nv04_graph_ofuncs }, /* mtri */
+	{ 0x0056, &nv04_graph_ofuncs }, /* celcius */
+	{},
+};
+
+static struct nouveau_oclass
+nv15_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs }, /* pattern */
+	{ 0x004a, &nv04_graph_ofuncs }, /* gdi */
+	{ 0x0052, &nv04_graph_ofuncs }, /* swzsurf */
+	{ 0x005f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0062, &nv04_graph_ofuncs }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0093, &nv04_graph_ofuncs }, /* surf3d */
+	{ 0x0094, &nv04_graph_ofuncs }, /* ttri */
+	{ 0x0095, &nv04_graph_ofuncs }, /* mtri */
+	{ 0x0096, &nv04_graph_ofuncs }, /* celcius */
+	{},
+};
+
+static int
+nv17_graph_mthd_lma_window(struct nouveau_object *object, u32 mthd,
+			   void *args, u32 size)
+{
+	struct nv10_graph_chan *chan = (void *)object->parent;
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	struct pipe_state *pipe = &chan->pipe_state;
+	u32 pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3];
+	u32 xfmode0, xfmode1;
+	u32 data = *(u32 *)args;
+	int i;
+
+	chan->lma_window[(mthd - 0x1638) / 4] = data;
+
+	if (mthd != 0x1644)
+		return 0;
+
+	nv04_graph_idle(priv);
+
+	PIPE_SAVE(priv, pipe_0x0040, 0x0040);
+	PIPE_SAVE(priv, pipe->pipe_0x0200, 0x0200);
+
+	PIPE_RESTORE(priv, chan->lma_window, 0x6790);
+
+	nv04_graph_idle(priv);
+
+	xfmode0 = nv_rd32(priv, NV10_PGRAPH_XFMODE0);
+	xfmode1 = nv_rd32(priv, NV10_PGRAPH_XFMODE1);
+
+	PIPE_SAVE(priv, pipe->pipe_0x4400, 0x4400);
+	PIPE_SAVE(priv, pipe_0x64c0, 0x64c0);
+	PIPE_SAVE(priv, pipe_0x6ab0, 0x6ab0);
+	PIPE_SAVE(priv, pipe_0x6a80, 0x6a80);
+
+	nv04_graph_idle(priv);
+
+	nv_wr32(priv, NV10_PGRAPH_XFMODE0, 0x10000000);
+	nv_wr32(priv, NV10_PGRAPH_XFMODE1, 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
+	for (i = 0; i < 3; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
+	for (i = 0; i < 3; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
+	nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000008);
+
+	PIPE_RESTORE(priv, pipe->pipe_0x0200, 0x0200);
+
+	nv04_graph_idle(priv);
+
+	PIPE_RESTORE(priv, pipe_0x0040, 0x0040);
+
+	nv_wr32(priv, NV10_PGRAPH_XFMODE0, xfmode0);
+	nv_wr32(priv, NV10_PGRAPH_XFMODE1, xfmode1);
+
+	PIPE_RESTORE(priv, pipe_0x64c0, 0x64c0);
+	PIPE_RESTORE(priv, pipe_0x6ab0, 0x6ab0);
+	PIPE_RESTORE(priv, pipe_0x6a80, 0x6a80);
+	PIPE_RESTORE(priv, pipe->pipe_0x4400, 0x4400);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x000000c0);
+	nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000000);
+
+	nv04_graph_idle(priv);
+
+	return 0;
+}
+
+static int
+nv17_graph_mthd_lma_enable(struct nouveau_object *object, u32 mthd,
+			   void *args, u32 size)
+{
+	struct nv10_graph_chan *chan = (void *)object->parent;
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+
+	nv04_graph_idle(priv);
+
+	nv_mask(priv, NV10_PGRAPH_DEBUG_4, 0x00000100, 0x00000100);
+	nv_mask(priv, 0x4006b0, 0x08000000, 0x08000000);
+	return 0;
+}
+
+static struct nouveau_omthds
+nv17_celcius_omthds[] = {
+	{ 0x1638, nv17_graph_mthd_lma_window },
+	{ 0x163c, nv17_graph_mthd_lma_window },
+	{ 0x1640, nv17_graph_mthd_lma_window },
+	{ 0x1644, nv17_graph_mthd_lma_window },
+	{ 0x1658, nv17_graph_mthd_lma_enable },
+	{}
+};
+
+static struct nouveau_oclass
+nv17_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs }, /* pattern */
+	{ 0x004a, &nv04_graph_ofuncs }, /* gdi */
+	{ 0x0052, &nv04_graph_ofuncs }, /* swzsurf */
+	{ 0x005f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0062, &nv04_graph_ofuncs }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs }, /* blit */
+	{ 0x0093, &nv04_graph_ofuncs }, /* surf3d */
+	{ 0x0094, &nv04_graph_ofuncs }, /* ttri */
+	{ 0x0095, &nv04_graph_ofuncs }, /* mtri */
+	{ 0x0099, &nv04_graph_ofuncs, nv17_celcius_omthds },
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static struct nv10_graph_chan *
+nv10_graph_channel(struct nv10_graph_priv *priv)
+{
+	struct nv10_graph_chan *chan = NULL;
+	if (nv_rd32(priv, 0x400144) & 0x00010000) {
+		int chid = nv_rd32(priv, 0x400148) >> 24;
+		if (chid < ARRAY_SIZE(priv->chan))
+			chan = priv->chan[chid];
+	}
+	return chan;
+}
+
+static void
+nv10_graph_save_pipe(struct nv10_graph_chan *chan)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	struct pipe_state *pipe = &chan->pipe_state;
+
+	PIPE_SAVE(priv, pipe->pipe_0x4400, 0x4400);
+	PIPE_SAVE(priv, pipe->pipe_0x0200, 0x0200);
+	PIPE_SAVE(priv, pipe->pipe_0x6400, 0x6400);
+	PIPE_SAVE(priv, pipe->pipe_0x6800, 0x6800);
+	PIPE_SAVE(priv, pipe->pipe_0x6c00, 0x6c00);
+	PIPE_SAVE(priv, pipe->pipe_0x7000, 0x7000);
+	PIPE_SAVE(priv, pipe->pipe_0x7400, 0x7400);
+	PIPE_SAVE(priv, pipe->pipe_0x7800, 0x7800);
+	PIPE_SAVE(priv, pipe->pipe_0x0040, 0x0040);
+	PIPE_SAVE(priv, pipe->pipe_0x0000, 0x0000);
+}
+
+static void
+nv10_graph_load_pipe(struct nv10_graph_chan *chan)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	struct pipe_state *pipe = &chan->pipe_state;
+	u32 xfmode0, xfmode1;
+	int i;
+
+	nv04_graph_idle(priv);
+	/* XXX check haiku comments */
+	xfmode0 = nv_rd32(priv, NV10_PGRAPH_XFMODE0);
+	xfmode1 = nv_rd32(priv, NV10_PGRAPH_XFMODE1);
+	nv_wr32(priv, NV10_PGRAPH_XFMODE0, 0x10000000);
+	nv_wr32(priv, NV10_PGRAPH_XFMODE1, 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
+	for (i = 0; i < 3; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
+	for (i = 0; i < 3; i++)
+		nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000000);
+
+	nv_wr32(priv, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
+	nv_wr32(priv, NV10_PGRAPH_PIPE_DATA, 0x00000008);
+
+
+	PIPE_RESTORE(priv, pipe->pipe_0x0200, 0x0200);
+	nv04_graph_idle(priv);
+
+	/* restore XFMODE */
+	nv_wr32(priv, NV10_PGRAPH_XFMODE0, xfmode0);
+	nv_wr32(priv, NV10_PGRAPH_XFMODE1, xfmode1);
+	PIPE_RESTORE(priv, pipe->pipe_0x6400, 0x6400);
+	PIPE_RESTORE(priv, pipe->pipe_0x6800, 0x6800);
+	PIPE_RESTORE(priv, pipe->pipe_0x6c00, 0x6c00);
+	PIPE_RESTORE(priv, pipe->pipe_0x7000, 0x7000);
+	PIPE_RESTORE(priv, pipe->pipe_0x7400, 0x7400);
+	PIPE_RESTORE(priv, pipe->pipe_0x7800, 0x7800);
+	PIPE_RESTORE(priv, pipe->pipe_0x4400, 0x4400);
+	PIPE_RESTORE(priv, pipe->pipe_0x0000, 0x0000);
+	PIPE_RESTORE(priv, pipe->pipe_0x0040, 0x0040);
+	nv04_graph_idle(priv);
+}
+
+static void
+nv10_graph_create_pipe(struct nv10_graph_chan *chan)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	struct pipe_state *pipe_state = &chan->pipe_state;
+	u32 *pipe_state_addr;
+	int i;
+#define PIPE_INIT(addr) \
+	do { \
+		pipe_state_addr = pipe_state->pipe_##addr; \
+	} while (0)
+#define PIPE_INIT_END(addr) \
+	do { \
+		u32 *__end_addr = pipe_state->pipe_##addr + \
+				ARRAY_SIZE(pipe_state->pipe_##addr); \
+		if (pipe_state_addr != __end_addr) \
+			nv_error(priv, "incomplete pipe init for 0x%x :  %p/%p\n", \
+				addr, pipe_state_addr, __end_addr); \
+	} while (0)
+#define NV_WRITE_PIPE_INIT(value) *(pipe_state_addr++) = value
+
+	PIPE_INIT(0x0200);
+	for (i = 0; i < 48; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x0200);
+
+	PIPE_INIT(0x6400);
+	for (i = 0; i < 211; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x40000000);
+	NV_WRITE_PIPE_INIT(0x40000000);
+	NV_WRITE_PIPE_INIT(0x40000000);
+	NV_WRITE_PIPE_INIT(0x40000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f000000);
+	NV_WRITE_PIPE_INIT(0x3f000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	PIPE_INIT_END(0x6400);
+
+	PIPE_INIT(0x6800);
+	for (i = 0; i < 162; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x3f800000);
+	for (i = 0; i < 25; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x6800);
+
+	PIPE_INIT(0x6c00);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0xbf800000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x6c00);
+
+	PIPE_INIT(0x7000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x00000000);
+	NV_WRITE_PIPE_INIT(0x7149f2ca);
+	for (i = 0; i < 35; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x7000);
+
+	PIPE_INIT(0x7400);
+	for (i = 0; i < 48; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x7400);
+
+	PIPE_INIT(0x7800);
+	for (i = 0; i < 48; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x7800);
+
+	PIPE_INIT(0x4400);
+	for (i = 0; i < 32; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x4400);
+
+	PIPE_INIT(0x0000);
+	for (i = 0; i < 16; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x0000);
+
+	PIPE_INIT(0x0040);
+	for (i = 0; i < 4; i++)
+		NV_WRITE_PIPE_INIT(0x00000000);
+	PIPE_INIT_END(0x0040);
+
+#undef PIPE_INIT
+#undef PIPE_INIT_END
+#undef NV_WRITE_PIPE_INIT
+}
+
+static int
+nv10_graph_ctx_regs_find_offset(struct nv10_graph_priv *priv, int reg)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++) {
+		if (nv10_graph_ctx_regs[i] == reg)
+			return i;
+	}
+	nv_error(priv, "unknow offset nv10_ctx_regs %d\n", reg);
+	return -1;
+}
+
+static int
+nv17_graph_ctx_regs_find_offset(struct nv10_graph_priv *priv, int reg)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++) {
+		if (nv17_graph_ctx_regs[i] == reg)
+			return i;
+	}
+	nv_error(priv, "unknow offset nv17_ctx_regs %d\n", reg);
+	return -1;
+}
+
+static void
+nv10_graph_load_dma_vtxbuf(struct nv10_graph_chan *chan, int chid, u32 inst)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	u32 st2, st2_dl, st2_dh, fifo_ptr, fifo[0x60/4];
+	u32 ctx_user, ctx_switch[5];
+	int i, subchan = -1;
+
+	/* NV10TCL_DMA_VTXBUF (method 0x18c) modifies hidden state
+	 * that cannot be restored via MMIO. Do it through the FIFO
+	 * instead.
+	 */
+
+	/* Look for a celsius object */
+	for (i = 0; i < 8; i++) {
+		int class = nv_rd32(priv, NV10_PGRAPH_CTX_CACHE(i, 0)) & 0xfff;
+
+		if (class == 0x56 || class == 0x96 || class == 0x99) {
+			subchan = i;
+			break;
+		}
+	}
+
+	if (subchan < 0 || !inst)
+		return;
+
+	/* Save the current ctx object */
+	ctx_user = nv_rd32(priv, NV10_PGRAPH_CTX_USER);
+	for (i = 0; i < 5; i++)
+		ctx_switch[i] = nv_rd32(priv, NV10_PGRAPH_CTX_SWITCH(i));
+
+	/* Save the FIFO state */
+	st2 = nv_rd32(priv, NV10_PGRAPH_FFINTFC_ST2);
+	st2_dl = nv_rd32(priv, NV10_PGRAPH_FFINTFC_ST2_DL);
+	st2_dh = nv_rd32(priv, NV10_PGRAPH_FFINTFC_ST2_DH);
+	fifo_ptr = nv_rd32(priv, NV10_PGRAPH_FFINTFC_FIFO_PTR);
+
+	for (i = 0; i < ARRAY_SIZE(fifo); i++)
+		fifo[i] = nv_rd32(priv, 0x4007a0 + 4 * i);
+
+	/* Switch to the celsius subchannel */
+	for (i = 0; i < 5; i++)
+		nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(i),
+			nv_rd32(priv, NV10_PGRAPH_CTX_CACHE(subchan, i)));
+	nv_mask(priv, NV10_PGRAPH_CTX_USER, 0xe000, subchan << 13);
+
+	/* Inject NV10TCL_DMA_VTXBUF */
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_FIFO_PTR, 0);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2,
+		0x2c000000 | chid << 20 | subchan << 16 | 0x18c);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2_DL, inst);
+	nv_mask(priv, NV10_PGRAPH_CTX_CONTROL, 0, 0x10000);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+
+	/* Restore the FIFO state */
+	for (i = 0; i < ARRAY_SIZE(fifo); i++)
+		nv_wr32(priv, 0x4007a0 + 4 * i, fifo[i]);
+
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_FIFO_PTR, fifo_ptr);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2, st2);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2_DL, st2_dl);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2_DH, st2_dh);
+
+	/* Restore the current ctx object */
+	for (i = 0; i < 5; i++)
+		nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(i), ctx_switch[i]);
+	nv_wr32(priv, NV10_PGRAPH_CTX_USER, ctx_user);
+}
+
+static int
+nv10_graph_load_context(struct nv10_graph_chan *chan, int chid)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	u32 inst;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
+		nv_wr32(priv, nv10_graph_ctx_regs[i], chan->nv10[i]);
+
+	if (nv_device(priv)->chipset >= 0x17) {
+		for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
+			nv_wr32(priv, nv17_graph_ctx_regs[i], chan->nv17[i]);
+	}
+
+	nv10_graph_load_pipe(chan);
+
+	inst = nv_rd32(priv, NV10_PGRAPH_GLOBALSTATE1) & 0xffff;
+	nv10_graph_load_dma_vtxbuf(chan, chid, inst);
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
+	nv_mask(priv, NV10_PGRAPH_CTX_USER, 0xff000000, chid << 24);
+	nv_mask(priv, NV10_PGRAPH_FFINTFC_ST2, 0x30000000, 0x00000000);
+	return 0;
+}
+
+static int
+nv10_graph_unload_context(struct nv10_graph_chan *chan)
+{
+	struct nv10_graph_priv *priv = nv10_graph_priv(chan);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
+		chan->nv10[i] = nv_rd32(priv, nv10_graph_ctx_regs[i]);
+
+	if (nv_device(priv)->chipset >= 0x17) {
+		for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
+			chan->nv17[i] = nv_rd32(priv, nv17_graph_ctx_regs[i]);
+	}
+
+	nv10_graph_save_pipe(chan);
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
+	nv_mask(priv, NV10_PGRAPH_CTX_USER, 0xff000000, 0x1f000000);
+	return 0;
+}
+
+static void
+nv10_graph_context_switch(struct nv10_graph_priv *priv)
+{
+	struct nv10_graph_chan *prev = NULL;
+	struct nv10_graph_chan *next = NULL;
+	unsigned long flags;
+	int chid;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	nv04_graph_idle(priv);
+
+	/* If previous context is valid, we need to save it */
+	prev = nv10_graph_channel(priv);
+	if (prev)
+		nv10_graph_unload_context(prev);
+
+	/* load context for next channel */
+	chid = (nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
+	next = priv->chan[chid];
+	if (next)
+		nv10_graph_load_context(next, chid);
+
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+#define NV_WRITE_CTX(reg, val) do { \
+	int offset = nv10_graph_ctx_regs_find_offset(priv, reg); \
+	if (offset > 0) \
+		chan->nv10[offset] = val; \
+	} while (0)
+
+#define NV17_WRITE_CTX(reg, val) do { \
+	int offset = nv17_graph_ctx_regs_find_offset(priv, reg); \
+	if (offset > 0) \
+		chan->nv17[offset] = val; \
+	} while (0)
+
+static int
+nv10_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_fifo_chan *fifo = (void *)parent;
+	struct nv10_graph_priv *priv = (void *)engine;
+	struct nv10_graph_chan *chan;
+	unsigned long flags;
+	int ret;
+
+	ret = nouveau_object_create(parent, engine, oclass, 0, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	if (priv->chan[fifo->chid]) {
+		*pobject = nv_object(priv->chan[fifo->chid]);
+		atomic_inc(&(*pobject)->refcount);
+		spin_unlock_irqrestore(&priv->lock, flags);
+		nouveau_object_destroy(&chan->base);
+		return 1;
+	}
+
+	NV_WRITE_CTX(0x00400e88, 0x08000000);
+	NV_WRITE_CTX(0x00400e9c, 0x4b7fffff);
+	NV_WRITE_CTX(NV03_PGRAPH_XY_LOGIC_MISC0, 0x0001ffff);
+	NV_WRITE_CTX(0x00400e10, 0x00001000);
+	NV_WRITE_CTX(0x00400e14, 0x00001000);
+	NV_WRITE_CTX(0x00400e30, 0x00080008);
+	NV_WRITE_CTX(0x00400e34, 0x00080008);
+	if (nv_device(priv)->chipset >= 0x17) {
+		/* is it really needed ??? */
+		NV17_WRITE_CTX(NV10_PGRAPH_DEBUG_4,
+					nv_rd32(priv, NV10_PGRAPH_DEBUG_4));
+		NV17_WRITE_CTX(0x004006b0, nv_rd32(priv, 0x004006b0));
+		NV17_WRITE_CTX(0x00400eac, 0x0fff0000);
+		NV17_WRITE_CTX(0x00400eb0, 0x0fff0000);
+		NV17_WRITE_CTX(0x00400ec0, 0x00000080);
+		NV17_WRITE_CTX(0x00400ed0, 0x00000080);
+	}
+	NV_WRITE_CTX(NV10_PGRAPH_CTX_USER, chan->chid << 24);
+
+	nv10_graph_create_pipe(chan);
+
+	priv->chan[fifo->chid] = chan;
+	chan->chid = fifo->chid;
+	spin_unlock_irqrestore(&priv->lock, flags);
+	return 0;
+}
+
+static void
+nv10_graph_context_dtor(struct nouveau_object *object)
+{
+	struct nv10_graph_priv *priv = (void *)object->engine;
+	struct nv10_graph_chan *chan = (void *)object;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	priv->chan[chan->chid] = NULL;
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	nouveau_object_destroy(&chan->base);
+}
+
+static int
+nv10_graph_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv10_graph_priv *priv = (void *)object->engine;
+	struct nv10_graph_chan *chan = (void *)object;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
+	if (nv10_graph_channel(priv) == chan)
+		nv10_graph_unload_context(chan);
+	nv_mask(priv, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	return nouveau_object_fini(&chan->base, suspend);
+}
+
+static struct nouveau_oclass
+nv10_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_graph_context_ctor,
+		.dtor = nv10_graph_context_dtor,
+		.init = nouveau_object_init,
+		.fini = nv10_graph_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv10_graph_tile_prog(struct nouveau_engine *engine, int i)
+{
+	struct nouveau_fb_tile *tile = &nouveau_fb(engine)->tile.region[i];
+	struct nouveau_fifo *pfifo = nouveau_fifo(engine);
+	struct nv10_graph_priv *priv = (void *)engine;
+	unsigned long flags;
+
+	pfifo->pause(pfifo, &flags);
+	nv04_graph_idle(priv);
+
+	nv_wr32(priv, NV10_PGRAPH_TLIMIT(i), tile->limit);
+	nv_wr32(priv, NV10_PGRAPH_TSIZE(i), tile->pitch);
+	nv_wr32(priv, NV10_PGRAPH_TILE(i), tile->addr);
+
+	pfifo->start(pfifo, &flags);
+}
+
+const struct nouveau_bitfield nv10_graph_intr_name[] = {
+	{ NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
+	{ NV_PGRAPH_INTR_ERROR,  "ERROR"  },
+	{}
+};
+
+const struct nouveau_bitfield nv10_graph_nstatus[] = {
+	{ NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
+	{ NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
+	{ NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
+	{ NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
+	{}
+};
+
+static void
+nv10_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nv10_graph_priv *priv = (void *)subdev;
+	struct nv10_graph_chan *chan = NULL;
+	struct nouveau_namedb *namedb = NULL;
+	struct nouveau_handle *handle = NULL;
+	u32 stat = nv_rd32(priv, NV03_PGRAPH_INTR);
+	u32 nsource = nv_rd32(priv, NV03_PGRAPH_NSOURCE);
+	u32 nstatus = nv_rd32(priv, NV03_PGRAPH_NSTATUS);
+	u32 addr = nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR);
+	u32 chid = (addr & 0x01f00000) >> 20;
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00001ffc);
+	u32 data = nv_rd32(priv, NV04_PGRAPH_TRAPPED_DATA);
+	u32 class = nv_rd32(priv, 0x400160 + subc * 4) & 0xfff;
+	u32 show = stat;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	chan = priv->chan[chid];
+	if (chan)
+		namedb = (void *)nv_pclass(nv_object(chan), NV_NAMEDB_CLASS);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	if (stat & NV_PGRAPH_INTR_ERROR) {
+		if (chan && (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD)) {
+			handle = nouveau_namedb_get_class(namedb, class);
+			if (handle && !nv_call(handle->object, mthd, data))
+				show &= ~NV_PGRAPH_INTR_ERROR;
+		}
+	}
+
+	if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
+		nv_wr32(priv, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
+		stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+		show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
+		nv10_graph_context_switch(priv);
+	}
+
+	nv_wr32(priv, NV03_PGRAPH_INTR, stat);
+	nv_wr32(priv, NV04_PGRAPH_FIFO, 0x00000001);
+
+	if (show) {
+		nv_error(priv, "");
+		nouveau_bitfield_print(nv10_graph_intr_name, show);
+		printk(" nsource:");
+		nouveau_bitfield_print(nv04_graph_nsource, nsource);
+		printk(" nstatus:");
+		nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+		printk("\n");
+		nv_error(priv, "ch %d/%d class 0x%04x "
+			       "mthd 0x%04x data 0x%08x\n",
+			 chid, subc, class, mthd, data);
+	}
+
+	nouveau_namedb_put(handle);
+}
+
+static int
+nv10_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv10_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv10_graph_intr;
+	nv_engine(priv)->cclass = &nv10_graph_cclass;
+
+	if (nv_device(priv)->chipset <= 0x10)
+		nv_engine(priv)->sclass = nv10_graph_sclass;
+	else
+	if (nv_device(priv)->chipset <  0x17 ||
+	    nv_device(priv)->chipset == 0x1a)
+		nv_engine(priv)->sclass = nv15_graph_sclass;
+	else
+		nv_engine(priv)->sclass = nv17_graph_sclass;
+
+	nv_engine(priv)->tile_prog = nv10_graph_tile_prog;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static void
+nv10_graph_dtor(struct nouveau_object *object)
+{
+	struct nv10_graph_priv *priv = (void *)object;
+	nouveau_graph_destroy(&priv->base);
+}
+
+static int
+nv10_graph_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object);
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	struct nv10_graph_priv *priv = (void *)engine;
+	int ret, i;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
+	nv_wr32(priv, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x00000000);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_1, 0x00118700);
+	/* nv_wr32(priv, NV04_PGRAPH_DEBUG_2, 0x24E00810); */ /* 0x25f92ad9 */
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_2, 0x25f92ad9);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0x55DE0830 | (1 << 29) | (1 << 31));
+
+	if (nv_device(priv)->chipset >= 0x17) {
+		nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x1f000000);
+		nv_wr32(priv, 0x400a10, 0x03ff3fb6);
+		nv_wr32(priv, 0x400838, 0x002f8684);
+		nv_wr32(priv, 0x40083c, 0x00115f3f);
+		nv_wr32(priv, 0x4006b0, 0x40000020);
+	} else {
+		nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x00000000);
+	}
+
+	/* Turn all the tiling regions off. */
+	for (i = 0; i < pfb->tile.regions; i++)
+		engine->tile_prog(engine, i);
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(2), 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(3), 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_CTX_SWITCH(4), 0x00000000);
+	nv_wr32(priv, NV10_PGRAPH_STATE, 0xFFFFFFFF);
+
+	nv_mask(priv, NV10_PGRAPH_CTX_USER, 0xff000000, 0x1f000000);
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
+	nv_wr32(priv, NV10_PGRAPH_FFINTFC_ST2, 0x08000000);
+	return 0;
+}
+
+static int
+nv10_graph_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv10_graph_priv *priv = (void *)object;
+	return nouveau_graph_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv10_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_graph_ctor,
+		.dtor = nv10_graph_dtor,
+		.init = nv10_graph_init,
+		.fini = nv10_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
new file mode 100644
index 00000000000..8f3f619c4a7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
@@ -0,0 +1,381 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/handle.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/graph.h>
+#include <engine/fifo.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv20_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv04_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv04_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs, NULL }, /* ifc */
+	{ 0x0096, &nv04_graph_ofuncs, NULL }, /* celcius */
+	{ 0x0097, &nv04_graph_ofuncs, NULL }, /* kelvin */
+	{ 0x009e, &nv04_graph_ofuncs, NULL }, /* swzsurf */
+	{ 0x009f, &nv04_graph_ofuncs, NULL }, /* imageblit */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv20_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
+					   0x37f0, 16, NVOBJ_FLAG_ZERO_ALLOC,
+					   &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0000, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x033c, 0xffff0000);
+	nv_wo32(chan, 0x03a0, 0x0fff0000);
+	nv_wo32(chan, 0x03a4, 0x0fff0000);
+	nv_wo32(chan, 0x047c, 0x00000101);
+	nv_wo32(chan, 0x0490, 0x00000111);
+	nv_wo32(chan, 0x04a8, 0x44400000);
+	for (i = 0x04d4; i <= 0x04e0; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x04f4; i <= 0x0500; i += 4)
+		nv_wo32(chan, i, 0x00080000);
+	for (i = 0x050c; i <= 0x0518; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x051c; i <= 0x0528; i += 4)
+		nv_wo32(chan, i, 0x000105b8);
+	for (i = 0x052c; i <= 0x0538; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	for (i = 0x055c; i <= 0x0598; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x05a4, 0x4b7fffff);
+	nv_wo32(chan, 0x05fc, 0x00000001);
+	nv_wo32(chan, 0x0604, 0x00004000);
+	nv_wo32(chan, 0x0610, 0x00000001);
+	nv_wo32(chan, 0x0618, 0x00040000);
+	nv_wo32(chan, 0x061c, 0x00010000);
+	for (i = 0x1c1c; i <= 0x248c; i += 16) {
+		nv_wo32(chan, (i + 0), 0x10700ff9);
+		nv_wo32(chan, (i + 4), 0x0436086c);
+		nv_wo32(chan, (i + 8), 0x000c001b);
+	}
+	nv_wo32(chan, 0x281c, 0x3f800000);
+	nv_wo32(chan, 0x2830, 0x3f800000);
+	nv_wo32(chan, 0x285c, 0x40000000);
+	nv_wo32(chan, 0x2860, 0x3f800000);
+	nv_wo32(chan, 0x2864, 0x3f000000);
+	nv_wo32(chan, 0x286c, 0x40000000);
+	nv_wo32(chan, 0x2870, 0x3f800000);
+	nv_wo32(chan, 0x2878, 0xbf800000);
+	nv_wo32(chan, 0x2880, 0xbf800000);
+	nv_wo32(chan, 0x34a4, 0x000fe000);
+	nv_wo32(chan, 0x3530, 0x000003f8);
+	nv_wo32(chan, 0x3540, 0x002fe000);
+	for (i = 0x355c; i <= 0x3578; i += 4)
+		nv_wo32(chan, i, 0x001c527c);
+	return 0;
+}
+
+int
+nv20_graph_context_init(struct nouveau_object *object)
+{
+	struct nv20_graph_priv *priv = (void *)object->engine;
+	struct nv20_graph_chan *chan = (void *)object;
+	int ret;
+
+	ret = nouveau_graph_context_init(&chan->base);
+	if (ret)
+		return ret;
+
+	nv_wo32(priv->ctxtab, chan->chid * 4, nv_gpuobj(chan)->addr >> 4);
+	return 0;
+}
+
+int
+nv20_graph_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv20_graph_priv *priv = (void *)object->engine;
+	struct nv20_graph_chan *chan = (void *)object;
+	int chid = -1;
+
+	nv_mask(priv, 0x400720, 0x00000001, 0x00000000);
+	if (nv_rd32(priv, 0x400144) & 0x00010000)
+		chid = (nv_rd32(priv, 0x400148) & 0x1f000000) >> 24;
+	if (chan->chid == chid) {
+		nv_wr32(priv, 0x400784, nv_gpuobj(chan)->addr >> 4);
+		nv_wr32(priv, 0x400788, 0x00000002);
+		nv_wait(priv, 0x400700, 0xffffffff, 0x00000000);
+		nv_wr32(priv, 0x400144, 0x10000000);
+		nv_mask(priv, 0x400148, 0xff000000, 0x1f000000);
+	}
+	nv_mask(priv, 0x400720, 0x00000001, 0x00000001);
+
+	nv_wo32(priv->ctxtab, chan->chid * 4, 0x00000000);
+	return nouveau_graph_context_fini(&chan->base, suspend);
+}
+
+static struct nouveau_oclass
+nv20_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x20),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv20_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+void
+nv20_graph_tile_prog(struct nouveau_engine *engine, int i)
+{
+	struct nouveau_fb_tile *tile = &nouveau_fb(engine)->tile.region[i];
+	struct nouveau_fifo *pfifo = nouveau_fifo(engine);
+	struct nv20_graph_priv *priv = (void *)engine;
+	unsigned long flags;
+
+	pfifo->pause(pfifo, &flags);
+	nv04_graph_idle(priv);
+
+	nv_wr32(priv, NV20_PGRAPH_TLIMIT(i), tile->limit);
+	nv_wr32(priv, NV20_PGRAPH_TSIZE(i), tile->pitch);
+	nv_wr32(priv, NV20_PGRAPH_TILE(i), tile->addr);
+
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0030 + 4 * i);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA, tile->limit);
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0050 + 4 * i);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA, tile->pitch);
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0010 + 4 * i);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA, tile->addr);
+
+	if (nv_device(engine)->card_type == NV_20) {
+		nv_wr32(priv, NV20_PGRAPH_ZCOMP(i), tile->zcomp);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00ea0090 + 4 * i);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA, tile->zcomp);
+	}
+
+	pfifo->start(pfifo, &flags);
+}
+
+void
+nv20_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle;
+	struct nv20_graph_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, NV03_PGRAPH_INTR);
+	u32 nsource = nv_rd32(priv, NV03_PGRAPH_NSOURCE);
+	u32 nstatus = nv_rd32(priv, NV03_PGRAPH_NSTATUS);
+	u32 addr = nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR);
+	u32 chid = (addr & 0x01f00000) >> 20;
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00001ffc);
+	u32 data = nv_rd32(priv, NV04_PGRAPH_TRAPPED_DATA);
+	u32 class = nv_rd32(priv, 0x400160 + subc * 4) & 0xfff;
+	u32 show = stat;
+
+	engctx = nouveau_engctx_get(engine, chid);
+	if (stat & NV_PGRAPH_INTR_ERROR) {
+		if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+			handle = nouveau_handle_get_class(engctx, class);
+			if (handle && !nv_call(handle->object, mthd, data))
+				show &= ~NV_PGRAPH_INTR_ERROR;
+			nouveau_handle_put(handle);
+		}
+	}
+
+	nv_wr32(priv, NV03_PGRAPH_INTR, stat);
+	nv_wr32(priv, NV04_PGRAPH_FIFO, 0x00000001);
+
+	if (show) {
+		nv_info(priv, "");
+		nouveau_bitfield_print(nv10_graph_intr_name, show);
+		printk(" nsource:");
+		nouveau_bitfield_print(nv04_graph_nsource, nsource);
+		printk(" nstatus:");
+		nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+		printk("\n");
+		nv_info(priv, "ch %d/%d class 0x%04x mthd 0x%04x data 0x%08x\n",
+			chid, subc, class, mthd, data);
+	}
+
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv20_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv20_graph_cclass;
+	nv_engine(priv)->sclass = nv20_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+void
+nv20_graph_dtor(struct nouveau_object *object)
+{
+	struct nv20_graph_priv *priv = (void *)object;
+	nouveau_gpuobj_ref(NULL, &priv->ctxtab);
+	nouveau_graph_destroy(&priv->base);
+}
+
+int
+nv20_graph_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object);
+	struct nv20_graph_priv *priv = (void *)engine;
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	u32 tmp, vramsz;
+	int ret, i;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, NV20_PGRAPH_CHANNEL_CTX_TABLE, priv->ctxtab->addr >> 4);
+
+	if (nv_device(priv)->chipset == 0x20) {
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x003d0000);
+		for (i = 0; i < 15; i++)
+			nv_wr32(priv, NV10_PGRAPH_RDI_DATA, 0x00000000);
+		nv_wait(priv, 0x400700, 0xffffffff, 0x00000000);
+	} else {
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x02c80000);
+		for (i = 0; i < 32; i++)
+			nv_wr32(priv, NV10_PGRAPH_RDI_DATA, 0x00000000);
+		nv_wait(priv, 0x400700, 0xffffffff, 0x00000000);
+	}
+
+	nv_wr32(priv, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
+	nv_wr32(priv, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x00000000);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_1, 0x00118700);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0xF3CE0475); /* 0x4 = auto ctx switch */
+	nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x00000000);
+	nv_wr32(priv, 0x40009C           , 0x00000040);
+
+	if (nv_device(priv)->chipset >= 0x25) {
+		nv_wr32(priv, 0x400890, 0x00a8cfff);
+		nv_wr32(priv, 0x400610, 0x304B1FB6);
+		nv_wr32(priv, 0x400B80, 0x1cbd3883);
+		nv_wr32(priv, 0x400B84, 0x44000000);
+		nv_wr32(priv, 0x400098, 0x40000080);
+		nv_wr32(priv, 0x400B88, 0x000000ff);
+
+	} else {
+		nv_wr32(priv, 0x400880, 0x0008c7df);
+		nv_wr32(priv, 0x400094, 0x00000005);
+		nv_wr32(priv, 0x400B80, 0x45eae20e);
+		nv_wr32(priv, 0x400B84, 0x24000000);
+		nv_wr32(priv, 0x400098, 0x00000040);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00E00038);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00000030);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00E10038);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00000030);
+	}
+
+	/* Turn all the tiling regions off. */
+	for (i = 0; i < pfb->tile.regions; i++)
+		engine->tile_prog(engine, i);
+
+	nv_wr32(priv, 0x4009a0, nv_rd32(priv, 0x100324));
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA000C);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA, nv_rd32(priv, 0x100324));
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
+	nv_wr32(priv, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
+
+	tmp = nv_rd32(priv, NV10_PGRAPH_SURFACE) & 0x0007ff00;
+	nv_wr32(priv, NV10_PGRAPH_SURFACE, tmp);
+	tmp = nv_rd32(priv, NV10_PGRAPH_SURFACE) | 0x00020100;
+	nv_wr32(priv, NV10_PGRAPH_SURFACE, tmp);
+
+	/* begin RAM config */
+	vramsz = pci_resource_len(nv_device(priv)->pdev, 0) - 1;
+	nv_wr32(priv, 0x4009A4, nv_rd32(priv, 0x100200));
+	nv_wr32(priv, 0x4009A8, nv_rd32(priv, 0x100204));
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0000);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA , nv_rd32(priv, 0x100200));
+	nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0004);
+	nv_wr32(priv, NV10_PGRAPH_RDI_DATA , nv_rd32(priv, 0x100204));
+	nv_wr32(priv, 0x400820, 0);
+	nv_wr32(priv, 0x400824, 0);
+	nv_wr32(priv, 0x400864, vramsz - 1);
+	nv_wr32(priv, 0x400868, vramsz - 1);
+
+	/* interesting.. the below overwrites some of the tile setup above.. */
+	nv_wr32(priv, 0x400B20, 0x00000000);
+	nv_wr32(priv, 0x400B04, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV03_PGRAPH_ABS_UCLIP_XMIN, 0);
+	nv_wr32(priv, NV03_PGRAPH_ABS_UCLIP_YMIN, 0);
+	nv_wr32(priv, NV03_PGRAPH_ABS_UCLIP_XMAX, 0x7fff);
+	nv_wr32(priv, NV03_PGRAPH_ABS_UCLIP_YMAX, 0x7fff);
+	return 0;
+}
+
+struct nouveau_oclass
+nv20_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x20),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv20_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv20_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.h b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.h
new file mode 100644
index 00000000000..2bea7313e03
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.h
@@ -0,0 +1,31 @@
+#ifndef __NV20_GRAPH_H__
+#define __NV20_GRAPH_H__
+
+#include <core/enum.h>
+
+#include <engine/graph.h>
+#include <engine/fifo.h>
+
+struct nv20_graph_priv {
+	struct nouveau_graph base;
+	struct nouveau_gpuobj *ctxtab;
+};
+
+struct nv20_graph_chan {
+	struct nouveau_graph_chan base;
+	int chid;
+};
+
+extern struct nouveau_oclass nv25_graph_sclass[];
+int  nv20_graph_context_init(struct nouveau_object *);
+int  nv20_graph_context_fini(struct nouveau_object *, bool);
+
+void nv20_graph_tile_prog(struct nouveau_engine *, int);
+void nv20_graph_intr(struct nouveau_subdev *);
+
+void nv20_graph_dtor(struct nouveau_object *);
+int  nv20_graph_init(struct nouveau_object *);
+
+int  nv30_graph_init(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c
new file mode 100644
index 00000000000..b2b650dd8b2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv25.c
@@ -0,0 +1,167 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/graph.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+struct nouveau_oclass
+nv25_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv04_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv04_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs, NULL }, /* ifc */
+	{ 0x0096, &nv04_graph_ofuncs, NULL }, /* celcius */
+	{ 0x009e, &nv04_graph_ofuncs, NULL }, /* swzsurf */
+	{ 0x009f, &nv04_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x0597, &nv04_graph_ofuncs, NULL }, /* kelvin */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv25_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL, 0x3724,
+					   16, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0028, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x035c, 0xffff0000);
+	nv_wo32(chan, 0x03c0, 0x0fff0000);
+	nv_wo32(chan, 0x03c4, 0x0fff0000);
+	nv_wo32(chan, 0x049c, 0x00000101);
+	nv_wo32(chan, 0x04b0, 0x00000111);
+	nv_wo32(chan, 0x04c8, 0x00000080);
+	nv_wo32(chan, 0x04cc, 0xffff0000);
+	nv_wo32(chan, 0x04d0, 0x00000001);
+	nv_wo32(chan, 0x04e4, 0x44400000);
+	nv_wo32(chan, 0x04fc, 0x4b800000);
+	for (i = 0x0510; i <= 0x051c; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x0530; i <= 0x053c; i += 4)
+		nv_wo32(chan, i, 0x00080000);
+	for (i = 0x0548; i <= 0x0554; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x0558; i <= 0x0564; i += 4)
+		nv_wo32(chan, i, 0x000105b8);
+	for (i = 0x0568; i <= 0x0574; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	for (i = 0x0598; i <= 0x05d4; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x05e0, 0x4b7fffff);
+	nv_wo32(chan, 0x0620, 0x00000080);
+	nv_wo32(chan, 0x0624, 0x30201000);
+	nv_wo32(chan, 0x0628, 0x70605040);
+	nv_wo32(chan, 0x062c, 0xb0a09080);
+	nv_wo32(chan, 0x0630, 0xf0e0d0c0);
+	nv_wo32(chan, 0x0664, 0x00000001);
+	nv_wo32(chan, 0x066c, 0x00004000);
+	nv_wo32(chan, 0x0678, 0x00000001);
+	nv_wo32(chan, 0x0680, 0x00040000);
+	nv_wo32(chan, 0x0684, 0x00010000);
+	for (i = 0x1b04; i <= 0x2374; i += 16) {
+		nv_wo32(chan, (i + 0), 0x10700ff9);
+		nv_wo32(chan, (i + 4), 0x0436086c);
+		nv_wo32(chan, (i + 8), 0x000c001b);
+	}
+	nv_wo32(chan, 0x2704, 0x3f800000);
+	nv_wo32(chan, 0x2718, 0x3f800000);
+	nv_wo32(chan, 0x2744, 0x40000000);
+	nv_wo32(chan, 0x2748, 0x3f800000);
+	nv_wo32(chan, 0x274c, 0x3f000000);
+	nv_wo32(chan, 0x2754, 0x40000000);
+	nv_wo32(chan, 0x2758, 0x3f800000);
+	nv_wo32(chan, 0x2760, 0xbf800000);
+	nv_wo32(chan, 0x2768, 0xbf800000);
+	nv_wo32(chan, 0x308c, 0x000fe000);
+	nv_wo32(chan, 0x3108, 0x000003f8);
+	nv_wo32(chan, 0x3468, 0x002fe000);
+	for (i = 0x3484; i <= 0x34a0; i += 4)
+		nv_wo32(chan, i, 0x001c527c);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv25_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x25),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv25_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv25_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv25_graph_cclass;
+	nv_engine(priv)->sclass = nv25_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv25_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x25),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv25_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv20_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c
new file mode 100644
index 00000000000..700462fa0ae
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv2a.c
@@ -0,0 +1,134 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/graph.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv2a_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL, 0x36b0,
+					   16, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0000, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x033c, 0xffff0000);
+	nv_wo32(chan, 0x03a0, 0x0fff0000);
+	nv_wo32(chan, 0x03a4, 0x0fff0000);
+	nv_wo32(chan, 0x047c, 0x00000101);
+	nv_wo32(chan, 0x0490, 0x00000111);
+	nv_wo32(chan, 0x04a8, 0x44400000);
+	for (i = 0x04d4; i <= 0x04e0; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x04f4; i <= 0x0500; i += 4)
+		nv_wo32(chan, i, 0x00080000);
+	for (i = 0x050c; i <= 0x0518; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x051c; i <= 0x0528; i += 4)
+		nv_wo32(chan, i, 0x000105b8);
+	for (i = 0x052c; i <= 0x0538; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	for (i = 0x055c; i <= 0x0598; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x05a4, 0x4b7fffff);
+	nv_wo32(chan, 0x05fc, 0x00000001);
+	nv_wo32(chan, 0x0604, 0x00004000);
+	nv_wo32(chan, 0x0610, 0x00000001);
+	nv_wo32(chan, 0x0618, 0x00040000);
+	nv_wo32(chan, 0x061c, 0x00010000);
+	for (i = 0x1a9c; i <= 0x22fc; i += 16) { /*XXX: check!! */
+		nv_wo32(chan, (i + 0), 0x10700ff9);
+		nv_wo32(chan, (i + 4), 0x0436086c);
+		nv_wo32(chan, (i + 8), 0x000c001b);
+	}
+	nv_wo32(chan, 0x269c, 0x3f800000);
+	nv_wo32(chan, 0x26b0, 0x3f800000);
+	nv_wo32(chan, 0x26dc, 0x40000000);
+	nv_wo32(chan, 0x26e0, 0x3f800000);
+	nv_wo32(chan, 0x26e4, 0x3f000000);
+	nv_wo32(chan, 0x26ec, 0x40000000);
+	nv_wo32(chan, 0x26f0, 0x3f800000);
+	nv_wo32(chan, 0x26f8, 0xbf800000);
+	nv_wo32(chan, 0x2700, 0xbf800000);
+	nv_wo32(chan, 0x3024, 0x000fe000);
+	nv_wo32(chan, 0x30a0, 0x000003f8);
+	nv_wo32(chan, 0x33fc, 0x002fe000);
+	for (i = 0x341c; i <= 0x3438; i += 4)
+		nv_wo32(chan, i, 0x001c527c);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv2a_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x2a),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv2a_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv2a_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv2a_graph_cclass;
+	nv_engine(priv)->sclass = nv25_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv2a_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x2a),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv2a_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv20_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c
new file mode 100644
index 00000000000..cedadaa92d3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv30.c
@@ -0,0 +1,238 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/graph.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv30_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv04_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv04_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs, NULL }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x0362, &nv04_graph_ofuncs, NULL }, /* surf2d (nv30) */
+	{ 0x0389, &nv04_graph_ofuncs, NULL }, /* sifm (nv30) */
+	{ 0x038a, &nv04_graph_ofuncs, NULL }, /* ifc (nv30) */
+	{ 0x039e, &nv04_graph_ofuncs, NULL }, /* swzsurf (nv30) */
+	{ 0x0397, &nv04_graph_ofuncs, NULL }, /* rankine */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv30_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL, 0x5f48,
+					   16, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0028, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x0410, 0x00000101);
+	nv_wo32(chan, 0x0424, 0x00000111);
+	nv_wo32(chan, 0x0428, 0x00000060);
+	nv_wo32(chan, 0x0444, 0x00000080);
+	nv_wo32(chan, 0x0448, 0xffff0000);
+	nv_wo32(chan, 0x044c, 0x00000001);
+	nv_wo32(chan, 0x0460, 0x44400000);
+	nv_wo32(chan, 0x048c, 0xffff0000);
+	for (i = 0x04e0; i < 0x04e8; i += 4)
+		nv_wo32(chan, i, 0x0fff0000);
+	nv_wo32(chan, 0x04ec, 0x00011100);
+	for (i = 0x0508; i < 0x0548; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x0550, 0x4b7fffff);
+	nv_wo32(chan, 0x058c, 0x00000080);
+	nv_wo32(chan, 0x0590, 0x30201000);
+	nv_wo32(chan, 0x0594, 0x70605040);
+	nv_wo32(chan, 0x0598, 0xb8a89888);
+	nv_wo32(chan, 0x059c, 0xf8e8d8c8);
+	nv_wo32(chan, 0x05b0, 0xb0000000);
+	for (i = 0x0600; i < 0x0640; i += 4)
+		nv_wo32(chan, i, 0x00010588);
+	for (i = 0x0640; i < 0x0680; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x06c0; i < 0x0700; i += 4)
+		nv_wo32(chan, i, 0x0008aae4);
+	for (i = 0x0700; i < 0x0740; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x0740; i < 0x0780; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	nv_wo32(chan, 0x085c, 0x00040000);
+	nv_wo32(chan, 0x0860, 0x00010000);
+	for (i = 0x0864; i < 0x0874; i += 4)
+		nv_wo32(chan, i, 0x00040004);
+	for (i = 0x1f18; i <= 0x3088 ; i += 16) {
+		nv_wo32(chan, i + 0, 0x10700ff9);
+		nv_wo32(chan, i + 1, 0x0436086c);
+		nv_wo32(chan, i + 2, 0x000c001b);
+	}
+	for (i = 0x30b8; i < 0x30c8; i += 4)
+		nv_wo32(chan, i, 0x0000ffff);
+	nv_wo32(chan, 0x344c, 0x3f800000);
+	nv_wo32(chan, 0x3808, 0x3f800000);
+	nv_wo32(chan, 0x381c, 0x3f800000);
+	nv_wo32(chan, 0x3848, 0x40000000);
+	nv_wo32(chan, 0x384c, 0x3f800000);
+	nv_wo32(chan, 0x3850, 0x3f000000);
+	nv_wo32(chan, 0x3858, 0x40000000);
+	nv_wo32(chan, 0x385c, 0x3f800000);
+	nv_wo32(chan, 0x3864, 0xbf800000);
+	nv_wo32(chan, 0x386c, 0xbf800000);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv30_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x30),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv30_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv30_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv30_graph_cclass;
+	nv_engine(priv)->sclass = nv30_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+int
+nv30_graph_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object);
+	struct nv20_graph_priv *priv = (void *)engine;
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	int ret, i;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, NV20_PGRAPH_CHANNEL_CTX_TABLE, priv->ctxtab->addr >> 4);
+
+	nv_wr32(priv, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
+	nv_wr32(priv, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x00000000);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_1, 0x401287c0);
+	nv_wr32(priv, 0x400890, 0x01b463ff);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0xf2de0475);
+	nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x00008000);
+	nv_wr32(priv, NV04_PGRAPH_LIMIT_VIOL_PIX, 0xf04bdff6);
+	nv_wr32(priv, 0x400B80, 0x1003d888);
+	nv_wr32(priv, 0x400B84, 0x0c000000);
+	nv_wr32(priv, 0x400098, 0x00000000);
+	nv_wr32(priv, 0x40009C, 0x0005ad00);
+	nv_wr32(priv, 0x400B88, 0x62ff00ff); /* suspiciously like PGRAPH_DEBUG_2 */
+	nv_wr32(priv, 0x4000a0, 0x00000000);
+	nv_wr32(priv, 0x4000a4, 0x00000008);
+	nv_wr32(priv, 0x4008a8, 0xb784a400);
+	nv_wr32(priv, 0x400ba0, 0x002f8685);
+	nv_wr32(priv, 0x400ba4, 0x00231f3f);
+	nv_wr32(priv, 0x4008a4, 0x40000020);
+
+	if (nv_device(priv)->chipset == 0x34) {
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0004);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00200201);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0008);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00000008);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0000);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00000032);
+		nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00E00004);
+		nv_wr32(priv, NV10_PGRAPH_RDI_DATA , 0x00000002);
+	}
+
+	nv_wr32(priv, 0x4000c0, 0x00000016);
+
+	/* Turn all the tiling regions off. */
+	for (i = 0; i < pfb->tile.regions; i++)
+		engine->tile_prog(engine, i);
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
+	nv_wr32(priv, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
+	nv_wr32(priv, 0x0040075c             , 0x00000001);
+
+	/* begin RAM config */
+	/* vramsz = pci_resource_len(priv->dev->pdev, 0) - 1; */
+	nv_wr32(priv, 0x4009A4, nv_rd32(priv, 0x100200));
+	nv_wr32(priv, 0x4009A8, nv_rd32(priv, 0x100204));
+	if (nv_device(priv)->chipset != 0x34) {
+		nv_wr32(priv, 0x400750, 0x00EA0000);
+		nv_wr32(priv, 0x400754, nv_rd32(priv, 0x100200));
+		nv_wr32(priv, 0x400750, 0x00EA0004);
+		nv_wr32(priv, 0x400754, nv_rd32(priv, 0x100204));
+	}
+	return 0;
+}
+
+struct nouveau_oclass
+nv30_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x30),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv30_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv30_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c
new file mode 100644
index 00000000000..273f6320027
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv34.c
@@ -0,0 +1,168 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include <engine/graph.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv34_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv04_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv04_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs, NULL }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x0362, &nv04_graph_ofuncs, NULL }, /* surf2d (nv30) */
+	{ 0x0389, &nv04_graph_ofuncs, NULL }, /* sifm (nv30) */
+	{ 0x038a, &nv04_graph_ofuncs, NULL }, /* ifc (nv30) */
+	{ 0x039e, &nv04_graph_ofuncs, NULL }, /* swzsurf (nv30) */
+	{ 0x0697, &nv04_graph_ofuncs, NULL }, /* rankine */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv34_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL, 0x46dc,
+					   16, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0028, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x040c, 0x01000101);
+	nv_wo32(chan, 0x0420, 0x00000111);
+	nv_wo32(chan, 0x0424, 0x00000060);
+	nv_wo32(chan, 0x0440, 0x00000080);
+	nv_wo32(chan, 0x0444, 0xffff0000);
+	nv_wo32(chan, 0x0448, 0x00000001);
+	nv_wo32(chan, 0x045c, 0x44400000);
+	nv_wo32(chan, 0x0480, 0xffff0000);
+	for (i = 0x04d4; i < 0x04dc; i += 4)
+		nv_wo32(chan, i, 0x0fff0000);
+	nv_wo32(chan, 0x04e0, 0x00011100);
+	for (i = 0x04fc; i < 0x053c; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x0544, 0x4b7fffff);
+	nv_wo32(chan, 0x057c, 0x00000080);
+	nv_wo32(chan, 0x0580, 0x30201000);
+	nv_wo32(chan, 0x0584, 0x70605040);
+	nv_wo32(chan, 0x0588, 0xb8a89888);
+	nv_wo32(chan, 0x058c, 0xf8e8d8c8);
+	nv_wo32(chan, 0x05a0, 0xb0000000);
+	for (i = 0x05f0; i < 0x0630; i += 4)
+		nv_wo32(chan, i, 0x00010588);
+	for (i = 0x0630; i < 0x0670; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x06b0; i < 0x06f0; i += 4)
+		nv_wo32(chan, i, 0x0008aae4);
+	for (i = 0x06f0; i < 0x0730; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x0730; i < 0x0770; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	nv_wo32(chan, 0x0850, 0x00040000);
+	nv_wo32(chan, 0x0854, 0x00010000);
+	for (i = 0x0858; i < 0x0868; i += 4)
+		nv_wo32(chan, i, 0x00040004);
+	for (i = 0x15ac; i <= 0x271c ; i += 16) {
+		nv_wo32(chan, i + 0, 0x10700ff9);
+		nv_wo32(chan, i + 1, 0x0436086c);
+		nv_wo32(chan, i + 2, 0x000c001b);
+	}
+	for (i = 0x274c; i < 0x275c; i += 4)
+		nv_wo32(chan, i, 0x0000ffff);
+	nv_wo32(chan, 0x2ae0, 0x3f800000);
+	nv_wo32(chan, 0x2e9c, 0x3f800000);
+	nv_wo32(chan, 0x2eb0, 0x3f800000);
+	nv_wo32(chan, 0x2edc, 0x40000000);
+	nv_wo32(chan, 0x2ee0, 0x3f800000);
+	nv_wo32(chan, 0x2ee4, 0x3f000000);
+	nv_wo32(chan, 0x2eec, 0x40000000);
+	nv_wo32(chan, 0x2ef0, 0x3f800000);
+	nv_wo32(chan, 0x2ef8, 0xbf800000);
+	nv_wo32(chan, 0x2f00, 0xbf800000);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv34_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x34),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv34_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv34_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv34_graph_cclass;
+	nv_engine(priv)->sclass = nv34_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv34_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x34),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv34_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv30_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c
new file mode 100644
index 00000000000..f40ee2116ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv35.c
@@ -0,0 +1,166 @@
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+#include "nv20.h"
+#include "regs.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv35_graph_sclass[] = {
+	{ 0x0012, &nv04_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv04_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv04_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv04_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv04_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv04_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv04_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv04_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv04_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv04_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv04_graph_ofuncs, NULL }, /* ifc */
+	{ 0x009f, &nv04_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x0362, &nv04_graph_ofuncs, NULL }, /* surf2d (nv30) */
+	{ 0x0389, &nv04_graph_ofuncs, NULL }, /* sifm (nv30) */
+	{ 0x038a, &nv04_graph_ofuncs, NULL }, /* ifc (nv30) */
+	{ 0x039e, &nv04_graph_ofuncs, NULL }, /* swzsurf (nv30) */
+	{ 0x0497, &nv04_graph_ofuncs, NULL }, /* rankine */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv35_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv20_graph_chan *chan;
+	int ret, i;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL, 0x577c,
+					   16, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->chid = nouveau_fifo_chan(parent)->chid;
+
+	nv_wo32(chan, 0x0028, 0x00000001 | (chan->chid << 24));
+	nv_wo32(chan, 0x040c, 0x00000101);
+	nv_wo32(chan, 0x0420, 0x00000111);
+	nv_wo32(chan, 0x0424, 0x00000060);
+	nv_wo32(chan, 0x0440, 0x00000080);
+	nv_wo32(chan, 0x0444, 0xffff0000);
+	nv_wo32(chan, 0x0448, 0x00000001);
+	nv_wo32(chan, 0x045c, 0x44400000);
+	nv_wo32(chan, 0x0488, 0xffff0000);
+	for (i = 0x04dc; i < 0x04e4; i += 4)
+		nv_wo32(chan, i, 0x0fff0000);
+	nv_wo32(chan, 0x04e8, 0x00011100);
+	for (i = 0x0504; i < 0x0544; i += 4)
+		nv_wo32(chan, i, 0x07ff0000);
+	nv_wo32(chan, 0x054c, 0x4b7fffff);
+	nv_wo32(chan, 0x0588, 0x00000080);
+	nv_wo32(chan, 0x058c, 0x30201000);
+	nv_wo32(chan, 0x0590, 0x70605040);
+	nv_wo32(chan, 0x0594, 0xb8a89888);
+	nv_wo32(chan, 0x0598, 0xf8e8d8c8);
+	nv_wo32(chan, 0x05ac, 0xb0000000);
+	for (i = 0x0604; i < 0x0644; i += 4)
+		nv_wo32(chan, i, 0x00010588);
+	for (i = 0x0644; i < 0x0684; i += 4)
+		nv_wo32(chan, i, 0x00030303);
+	for (i = 0x06c4; i < 0x0704; i += 4)
+		nv_wo32(chan, i, 0x0008aae4);
+	for (i = 0x0704; i < 0x0744; i += 4)
+		nv_wo32(chan, i, 0x01012000);
+	for (i = 0x0744; i < 0x0784; i += 4)
+		nv_wo32(chan, i, 0x00080008);
+	nv_wo32(chan, 0x0860, 0x00040000);
+	nv_wo32(chan, 0x0864, 0x00010000);
+	for (i = 0x0868; i < 0x0878; i += 4)
+		nv_wo32(chan, i, 0x00040004);
+	for (i = 0x1f1c; i <= 0x308c ; i += 16) {
+		nv_wo32(chan, i + 0, 0x10700ff9);
+		nv_wo32(chan, i + 4, 0x0436086c);
+		nv_wo32(chan, i + 8, 0x000c001b);
+	}
+	for (i = 0x30bc; i < 0x30cc; i += 4)
+		nv_wo32(chan, i, 0x0000ffff);
+	nv_wo32(chan, 0x3450, 0x3f800000);
+	nv_wo32(chan, 0x380c, 0x3f800000);
+	nv_wo32(chan, 0x3820, 0x3f800000);
+	nv_wo32(chan, 0x384c, 0x40000000);
+	nv_wo32(chan, 0x3850, 0x3f800000);
+	nv_wo32(chan, 0x3854, 0x3f000000);
+	nv_wo32(chan, 0x385c, 0x40000000);
+	nv_wo32(chan, 0x3860, 0x3f800000);
+	nv_wo32(chan, 0x3868, 0xbf800000);
+	nv_wo32(chan, 0x3870, 0xbf800000);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv35_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x35),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv35_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = nv20_graph_context_init,
+		.fini = nv20_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv35_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv20_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 32 * 4, 16,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ctxtab);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv20_graph_intr;
+	nv_engine(priv)->cclass = &nv35_graph_cclass;
+	nv_engine(priv)->sclass = nv35_graph_sclass;
+	nv_engine(priv)->tile_prog = nv20_graph_tile_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv35_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x35),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv35_graph_ctor,
+		.dtor = nv20_graph_dtor,
+		.init = nv30_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
new file mode 100644
index 00000000000..8d0021049ec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
@@ -0,0 +1,495 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/handle.h>
+#include <core/engctx.h>
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+
+#include <engine/graph.h>
+#include <engine/fifo.h>
+
+#include "nv40.h"
+#include "regs.h"
+
+struct nv40_graph_priv {
+	struct nouveau_graph base;
+	u32 size;
+};
+
+struct nv40_graph_chan {
+	struct nouveau_graph_chan base;
+};
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static int
+nv40_graph_object_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    20, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+#ifdef __BIG_ENDIAN
+	nv_mo32(obj, 0x08, 0x01000000, 0x01000000);
+#endif
+	nv_wo32(obj, 0x0c, 0x00000000);
+	nv_wo32(obj, 0x10, 0x00000000);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv40_graph_ofuncs = {
+	.ctor = nv40_graph_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_oclass
+nv40_graph_sclass[] = {
+	{ 0x0012, &nv40_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv40_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv40_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv40_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv40_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv40_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv40_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv40_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv40_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv40_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv40_graph_ofuncs, NULL }, /* ifc */
+	{ 0x009f, &nv40_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x3062, &nv40_graph_ofuncs, NULL }, /* surf2d (nv40) */
+	{ 0x3089, &nv40_graph_ofuncs, NULL }, /* sifm (nv40) */
+	{ 0x309e, &nv40_graph_ofuncs, NULL }, /* swzsurf (nv40) */
+	{ 0x4097, &nv40_graph_ofuncs, NULL }, /* curie */
+	{},
+};
+
+static struct nouveau_oclass
+nv44_graph_sclass[] = {
+	{ 0x0012, &nv40_graph_ofuncs, NULL }, /* beta1 */
+	{ 0x0019, &nv40_graph_ofuncs, NULL }, /* clip */
+	{ 0x0030, &nv40_graph_ofuncs, NULL }, /* null */
+	{ 0x0039, &nv40_graph_ofuncs, NULL }, /* m2mf */
+	{ 0x0043, &nv40_graph_ofuncs, NULL }, /* rop */
+	{ 0x0044, &nv40_graph_ofuncs, NULL }, /* patt */
+	{ 0x004a, &nv40_graph_ofuncs, NULL }, /* gdi */
+	{ 0x0062, &nv40_graph_ofuncs, NULL }, /* surf2d */
+	{ 0x0072, &nv40_graph_ofuncs, NULL }, /* beta4 */
+	{ 0x0089, &nv40_graph_ofuncs, NULL }, /* sifm */
+	{ 0x008a, &nv40_graph_ofuncs, NULL }, /* ifc */
+	{ 0x009f, &nv40_graph_ofuncs, NULL }, /* imageblit */
+	{ 0x3062, &nv40_graph_ofuncs, NULL }, /* surf2d (nv40) */
+	{ 0x3089, &nv40_graph_ofuncs, NULL }, /* sifm (nv40) */
+	{ 0x309e, &nv40_graph_ofuncs, NULL }, /* swzsurf (nv40) */
+	{ 0x4497, &nv40_graph_ofuncs, NULL }, /* curie */
+	{},
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv40_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv40_graph_priv *priv = (void *)engine;
+	struct nv40_graph_chan *chan;
+	int ret;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
+					   priv->size, 16,
+					   NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv40_grctx_fill(nv_device(priv), nv_gpuobj(chan));
+	nv_wo32(chan, 0x00000, nv_gpuobj(chan)->addr >> 4);
+	return 0;
+}
+
+static int
+nv40_graph_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv04_graph_priv *priv = (void *)object->engine;
+	struct nv04_graph_chan *chan = (void *)object;
+	u32 inst = 0x01000000 | nv_gpuobj(chan)->addr >> 4;
+	int ret = 0;
+
+	nv_mask(priv, 0x400720, 0x00000001, 0x00000000);
+
+	if (nv_rd32(priv, 0x40032c) == inst) {
+		if (suspend) {
+			nv_wr32(priv, 0x400720, 0x00000000);
+			nv_wr32(priv, 0x400784, inst);
+			nv_mask(priv, 0x400310, 0x00000020, 0x00000020);
+			nv_mask(priv, 0x400304, 0x00000001, 0x00000001);
+			if (!nv_wait(priv, 0x400300, 0x00000001, 0x00000000)) {
+				u32 insn = nv_rd32(priv, 0x400308);
+				nv_warn(priv, "ctxprog timeout 0x%08x\n", insn);
+				ret = -EBUSY;
+			}
+		}
+
+		nv_mask(priv, 0x40032c, 0x01000000, 0x00000000);
+	}
+
+	if (nv_rd32(priv, 0x400330) == inst)
+		nv_mask(priv, 0x400330, 0x01000000, 0x00000000);
+
+	nv_mask(priv, 0x400720, 0x00000001, 0x00000001);
+	return ret;
+}
+
+static struct nouveau_oclass
+nv40_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = _nouveau_graph_context_init,
+		.fini = nv40_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv40_graph_tile_prog(struct nouveau_engine *engine, int i)
+{
+	struct nouveau_fb_tile *tile = &nouveau_fb(engine)->tile.region[i];
+	struct nouveau_fifo *pfifo = nouveau_fifo(engine);
+	struct nv40_graph_priv *priv = (void *)engine;
+	unsigned long flags;
+
+	pfifo->pause(pfifo, &flags);
+	nv04_graph_idle(priv);
+
+	switch (nv_device(priv)->chipset) {
+	case 0x40:
+	case 0x41: /* guess */
+	case 0x42:
+	case 0x43:
+	case 0x45: /* guess */
+	case 0x4e:
+		nv_wr32(priv, NV20_PGRAPH_TSIZE(i), tile->pitch);
+		nv_wr32(priv, NV20_PGRAPH_TLIMIT(i), tile->limit);
+		nv_wr32(priv, NV20_PGRAPH_TILE(i), tile->addr);
+		nv_wr32(priv, NV40_PGRAPH_TSIZE1(i), tile->pitch);
+		nv_wr32(priv, NV40_PGRAPH_TLIMIT1(i), tile->limit);
+		nv_wr32(priv, NV40_PGRAPH_TILE1(i), tile->addr);
+		break;
+	case 0x44:
+	case 0x4a:
+		nv_wr32(priv, NV20_PGRAPH_TSIZE(i), tile->pitch);
+		nv_wr32(priv, NV20_PGRAPH_TLIMIT(i), tile->limit);
+		nv_wr32(priv, NV20_PGRAPH_TILE(i), tile->addr);
+		break;
+	case 0x46:
+	case 0x47:
+	case 0x49:
+	case 0x4b:
+	case 0x4c:
+	case 0x67:
+	default:
+		nv_wr32(priv, NV47_PGRAPH_TSIZE(i), tile->pitch);
+		nv_wr32(priv, NV47_PGRAPH_TLIMIT(i), tile->limit);
+		nv_wr32(priv, NV47_PGRAPH_TILE(i), tile->addr);
+		nv_wr32(priv, NV40_PGRAPH_TSIZE1(i), tile->pitch);
+		nv_wr32(priv, NV40_PGRAPH_TLIMIT1(i), tile->limit);
+		nv_wr32(priv, NV40_PGRAPH_TILE1(i), tile->addr);
+		break;
+	}
+
+	pfifo->start(pfifo, &flags);
+}
+
+static void
+nv40_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle = NULL;
+	struct nv40_graph_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, NV03_PGRAPH_INTR);
+	u32 nsource = nv_rd32(priv, NV03_PGRAPH_NSOURCE);
+	u32 nstatus = nv_rd32(priv, NV03_PGRAPH_NSTATUS);
+	u32 inst = nv_rd32(priv, 0x40032c) & 0x000fffff;
+	u32 addr = nv_rd32(priv, NV04_PGRAPH_TRAPPED_ADDR);
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00001ffc);
+	u32 data = nv_rd32(priv, NV04_PGRAPH_TRAPPED_DATA);
+	u32 class = nv_rd32(priv, 0x400160 + subc * 4) & 0xffff;
+	u32 show = stat;
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & NV_PGRAPH_INTR_ERROR) {
+		if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
+			handle = nouveau_handle_get_class(engctx, class);
+			if (handle && !nv_call(handle->object, mthd, data))
+				show &= ~NV_PGRAPH_INTR_ERROR;
+			nouveau_handle_put(handle);
+		}
+
+		if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
+			nv_mask(priv, 0x402000, 0, 0);
+		}
+	}
+
+	nv_wr32(priv, NV03_PGRAPH_INTR, stat);
+	nv_wr32(priv, NV04_PGRAPH_FIFO, 0x00000001);
+
+	if (show) {
+		nv_info(priv, "");
+		nouveau_bitfield_print(nv10_graph_intr_name, show);
+		printk(" nsource:");
+		nouveau_bitfield_print(nv04_graph_nsource, nsource);
+		printk(" nstatus:");
+		nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
+		printk("\n");
+		nv_error(priv, "ch %d [0x%08x] subc %d class 0x%04x "
+			       "mthd 0x%04x data 0x%08x\n",
+			 chid, inst << 4, subc, class, mthd, data);
+	}
+
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv40_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv40_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00001000;
+	nv_subdev(priv)->intr = nv40_graph_intr;
+	nv_engine(priv)->cclass = &nv40_graph_cclass;
+	if (nv44_graph_class(priv))
+		nv_engine(priv)->sclass = nv44_graph_sclass;
+	else
+		nv_engine(priv)->sclass = nv40_graph_sclass;
+	nv_engine(priv)->tile_prog = nv40_graph_tile_prog;
+	return 0;
+}
+
+static int
+nv40_graph_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object);
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	struct nv40_graph_priv *priv = (void *)engine;
+	int ret, i, j;
+	u32 vramsz;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* generate and upload context program */
+	nv40_grctx_init(nv_device(priv), &priv->size);
+
+	/* No context present currently */
+	nv_wr32(priv, NV40_PGRAPH_CTXCTL_CUR, 0x00000000);
+
+	nv_wr32(priv, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
+	nv_wr32(priv, NV40_PGRAPH_INTR_EN, 0xFFFFFFFF);
+
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_0, 0x00000000);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_1, 0x401287c0);
+	nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0xe0de8055);
+	nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x00008000);
+	nv_wr32(priv, NV04_PGRAPH_LIMIT_VIOL_PIX, 0x00be3c5f);
+
+	nv_wr32(priv, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
+	nv_wr32(priv, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
+
+	j = nv_rd32(priv, 0x1540) & 0xff;
+	if (j) {
+		for (i = 0; !(j & 1); j >>= 1, i++)
+			;
+		nv_wr32(priv, 0x405000, i);
+	}
+
+	if (nv_device(priv)->chipset == 0x40) {
+		nv_wr32(priv, 0x4009b0, 0x83280fff);
+		nv_wr32(priv, 0x4009b4, 0x000000a0);
+	} else {
+		nv_wr32(priv, 0x400820, 0x83280eff);
+		nv_wr32(priv, 0x400824, 0x000000a0);
+	}
+
+	switch (nv_device(priv)->chipset) {
+	case 0x40:
+	case 0x45:
+		nv_wr32(priv, 0x4009b8, 0x0078e366);
+		nv_wr32(priv, 0x4009bc, 0x0000014c);
+		break;
+	case 0x41:
+	case 0x42: /* pciid also 0x00Cx */
+	/* case 0x0120: XXX (pciid) */
+		nv_wr32(priv, 0x400828, 0x007596ff);
+		nv_wr32(priv, 0x40082c, 0x00000108);
+		break;
+	case 0x43:
+		nv_wr32(priv, 0x400828, 0x0072cb77);
+		nv_wr32(priv, 0x40082c, 0x00000108);
+		break;
+	case 0x44:
+	case 0x46: /* G72 */
+	case 0x4a:
+	case 0x4c: /* G7x-based C51 */
+	case 0x4e:
+		nv_wr32(priv, 0x400860, 0);
+		nv_wr32(priv, 0x400864, 0);
+		break;
+	case 0x47: /* G70 */
+	case 0x49: /* G71 */
+	case 0x4b: /* G73 */
+		nv_wr32(priv, 0x400828, 0x07830610);
+		nv_wr32(priv, 0x40082c, 0x0000016A);
+		break;
+	default:
+		break;
+	}
+
+	nv_wr32(priv, 0x400b38, 0x2ffff800);
+	nv_wr32(priv, 0x400b3c, 0x00006000);
+
+	/* Tiling related stuff. */
+	switch (nv_device(priv)->chipset) {
+	case 0x44:
+	case 0x4a:
+		nv_wr32(priv, 0x400bc4, 0x1003d888);
+		nv_wr32(priv, 0x400bbc, 0xb7a7b500);
+		break;
+	case 0x46:
+		nv_wr32(priv, 0x400bc4, 0x0000e024);
+		nv_wr32(priv, 0x400bbc, 0xb7a7b520);
+		break;
+	case 0x4c:
+	case 0x4e:
+	case 0x67:
+		nv_wr32(priv, 0x400bc4, 0x1003d888);
+		nv_wr32(priv, 0x400bbc, 0xb7a7b540);
+		break;
+	default:
+		break;
+	}
+
+	/* Turn all the tiling regions off. */
+	for (i = 0; i < pfb->tile.regions; i++)
+		engine->tile_prog(engine, i);
+
+	/* begin RAM config */
+	vramsz = pci_resource_len(nv_device(priv)->pdev, 0) - 1;
+	switch (nv_device(priv)->chipset) {
+	case 0x40:
+		nv_wr32(priv, 0x4009A4, nv_rd32(priv, 0x100200));
+		nv_wr32(priv, 0x4009A8, nv_rd32(priv, 0x100204));
+		nv_wr32(priv, 0x4069A4, nv_rd32(priv, 0x100200));
+		nv_wr32(priv, 0x4069A8, nv_rd32(priv, 0x100204));
+		nv_wr32(priv, 0x400820, 0);
+		nv_wr32(priv, 0x400824, 0);
+		nv_wr32(priv, 0x400864, vramsz);
+		nv_wr32(priv, 0x400868, vramsz);
+		break;
+	default:
+		switch (nv_device(priv)->chipset) {
+		case 0x41:
+		case 0x42:
+		case 0x43:
+		case 0x45:
+		case 0x4e:
+		case 0x44:
+		case 0x4a:
+			nv_wr32(priv, 0x4009F0, nv_rd32(priv, 0x100200));
+			nv_wr32(priv, 0x4009F4, nv_rd32(priv, 0x100204));
+			break;
+		default:
+			nv_wr32(priv, 0x400DF0, nv_rd32(priv, 0x100200));
+			nv_wr32(priv, 0x400DF4, nv_rd32(priv, 0x100204));
+			break;
+		}
+		nv_wr32(priv, 0x4069F0, nv_rd32(priv, 0x100200));
+		nv_wr32(priv, 0x4069F4, nv_rd32(priv, 0x100204));
+		nv_wr32(priv, 0x400840, 0);
+		nv_wr32(priv, 0x400844, 0);
+		nv_wr32(priv, 0x4008A0, vramsz);
+		nv_wr32(priv, 0x4008A4, vramsz);
+		break;
+	}
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv40_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_graph_ctor,
+		.dtor = _nouveau_graph_dtor,
+		.init = nv40_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.h b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.h
new file mode 100644
index 00000000000..d2ac975afc2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.h
@@ -0,0 +1,21 @@
+#ifndef __NV40_GRAPH_H__
+#define __NV40_GRAPH_H__
+
+/* returns 1 if device is one of the nv4x using the 0x4497 object class,
+ * helpful to determine a number of other hardware features
+ */
+static inline int
+nv44_graph_class(void *priv)
+{
+	struct nouveau_device *device = nv_device(priv);
+
+	if ((device->chipset & 0xf0) == 0x60)
+		return 1;
+
+	return !(0x0baf & (1 << (device->chipset & 0x0f)));
+}
+
+void nv40_grctx_init(struct nouveau_device *, u32 *size);
+void nv40_grctx_fill(struct nouveau_device *, struct nouveau_gpuobj *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
new file mode 100644
index 00000000000..ab3b9dcaf47
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
@@ -0,0 +1,888 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/handle.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
+#include <engine/graph.h>
+
+#include "nv50.h"
+
+struct nv50_graph_priv {
+	struct nouveau_graph base;
+	spinlock_t lock;
+	u32 size;
+};
+
+struct nv50_graph_chan {
+	struct nouveau_graph_chan base;
+};
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static int
+nv50_graph_object_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    16, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+	nv_wo32(obj, 0x0c, 0x00000000);
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv50_graph_ofuncs = {
+	.ctor = nv50_graph_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_oclass
+nv50_graph_sclass[] = {
+	{ 0x0030, &nv50_graph_ofuncs },
+	{ 0x502d, &nv50_graph_ofuncs },
+	{ 0x5039, &nv50_graph_ofuncs },
+	{ 0x5097, &nv50_graph_ofuncs },
+	{ 0x50c0, &nv50_graph_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nv84_graph_sclass[] = {
+	{ 0x0030, &nv50_graph_ofuncs },
+	{ 0x502d, &nv50_graph_ofuncs },
+	{ 0x5039, &nv50_graph_ofuncs },
+	{ 0x50c0, &nv50_graph_ofuncs },
+	{ 0x8297, &nv50_graph_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nva0_graph_sclass[] = {
+	{ 0x0030, &nv50_graph_ofuncs },
+	{ 0x502d, &nv50_graph_ofuncs },
+	{ 0x5039, &nv50_graph_ofuncs },
+	{ 0x50c0, &nv50_graph_ofuncs },
+	{ 0x8397, &nv50_graph_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nva3_graph_sclass[] = {
+	{ 0x0030, &nv50_graph_ofuncs },
+	{ 0x502d, &nv50_graph_ofuncs },
+	{ 0x5039, &nv50_graph_ofuncs },
+	{ 0x50c0, &nv50_graph_ofuncs },
+	{ 0x8597, &nv50_graph_ofuncs },
+	{ 0x85c0, &nv50_graph_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nvaf_graph_sclass[] = {
+	{ 0x0030, &nv50_graph_ofuncs },
+	{ 0x502d, &nv50_graph_ofuncs },
+	{ 0x5039, &nv50_graph_ofuncs },
+	{ 0x50c0, &nv50_graph_ofuncs },
+	{ 0x85c0, &nv50_graph_ofuncs },
+	{ 0x8697, &nv50_graph_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static int
+nv50_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *data, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nv50_graph_priv *priv = (void *)engine;
+	struct nv50_graph_chan *chan;
+	int ret;
+
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
+					   priv->size, 0,
+					   NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv50_grctx_fill(nv_device(priv), nv_gpuobj(chan));
+	return 0;
+}
+
+static struct nouveau_oclass
+nv50_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_graph_context_ctor,
+		.dtor = _nouveau_graph_context_dtor,
+		.init = _nouveau_graph_context_init,
+		.fini = _nouveau_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv50_graph_tlb_flush(struct nouveau_engine *engine)
+{
+	nv50_vm_flush_engine(&engine->base, 0x00);
+	return 0;
+}
+
+static int
+nv84_graph_tlb_flush(struct nouveau_engine *engine)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(engine);
+	struct nv50_graph_priv *priv = (void *)engine;
+	bool idle, timeout = false;
+	unsigned long flags;
+	u64 start;
+	u32 tmp;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_mask(priv, 0x400500, 0x00000001, 0x00000000);
+
+	start = ptimer->read(ptimer);
+	do {
+		idle = true;
+
+		for (tmp = nv_rd32(priv, 0x400380); tmp && idle; tmp >>= 3) {
+			if ((tmp & 7) == 1)
+				idle = false;
+		}
+
+		for (tmp = nv_rd32(priv, 0x400384); tmp && idle; tmp >>= 3) {
+			if ((tmp & 7) == 1)
+				idle = false;
+		}
+
+		for (tmp = nv_rd32(priv, 0x400388); tmp && idle; tmp >>= 3) {
+			if ((tmp & 7) == 1)
+				idle = false;
+		}
+	} while (!idle &&
+		 !(timeout = ptimer->read(ptimer) - start > 2000000000));
+
+	if (timeout) {
+		nv_error(priv, "PGRAPH TLB flush idle timeout fail: "
+			      "0x%08x 0x%08x 0x%08x 0x%08x\n",
+			 nv_rd32(priv, 0x400700), nv_rd32(priv, 0x400380),
+			 nv_rd32(priv, 0x400384), nv_rd32(priv, 0x400388));
+	}
+
+	nv50_vm_flush_engine(&engine->base, 0x00);
+
+	nv_mask(priv, 0x400500, 0x00000001, 0x00000001);
+	spin_unlock_irqrestore(&priv->lock, flags);
+	return timeout ? -EBUSY : 0;
+}
+
+static const struct nouveau_enum nv50_mp_exec_error_names[] = {
+	{ 3, "STACK_UNDERFLOW", NULL },
+	{ 4, "QUADON_ACTIVE", NULL },
+	{ 8, "TIMEOUT", NULL },
+	{ 0x10, "INVALID_OPCODE", NULL },
+	{ 0x40, "BREAKPOINT", NULL },
+	{}
+};
+
+static const struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
+	{ 0x00000001, "NOTIFY" },
+	{ 0x00000002, "IN" },
+	{ 0x00000004, "OUT" },
+	{}
+};
+
+static const struct nouveau_bitfield nv50_graph_trap_vfetch[] = {
+	{ 0x00000001, "FAULT" },
+	{}
+};
+
+static const struct nouveau_bitfield nv50_graph_trap_strmout[] = {
+	{ 0x00000001, "FAULT" },
+	{}
+};
+
+static const struct nouveau_bitfield nv50_graph_trap_ccache[] = {
+	{ 0x00000001, "FAULT" },
+	{}
+};
+
+/* There must be a *lot* of these. Will take some time to gather them up. */
+const struct nouveau_enum nv50_data_error_names[] = {
+	{ 0x00000003, "INVALID_OPERATION", NULL },
+	{ 0x00000004, "INVALID_VALUE", NULL },
+	{ 0x00000005, "INVALID_ENUM", NULL },
+	{ 0x00000008, "INVALID_OBJECT", NULL },
+	{ 0x00000009, "READ_ONLY_OBJECT", NULL },
+	{ 0x0000000a, "SUPERVISOR_OBJECT", NULL },
+	{ 0x0000000b, "INVALID_ADDRESS_ALIGNMENT", NULL },
+	{ 0x0000000c, "INVALID_BITFIELD", NULL },
+	{ 0x0000000d, "BEGIN_END_ACTIVE", NULL },
+	{ 0x0000000e, "SEMANTIC_COLOR_BACK_OVER_LIMIT", NULL },
+	{ 0x0000000f, "VIEWPORT_ID_NEEDS_GP", NULL },
+	{ 0x00000010, "RT_DOUBLE_BIND", NULL },
+	{ 0x00000011, "RT_TYPES_MISMATCH", NULL },
+	{ 0x00000012, "RT_LINEAR_WITH_ZETA", NULL },
+	{ 0x00000015, "FP_TOO_FEW_REGS", NULL },
+	{ 0x00000016, "ZETA_FORMAT_CSAA_MISMATCH", NULL },
+	{ 0x00000017, "RT_LINEAR_WITH_MSAA", NULL },
+	{ 0x00000018, "FP_INTERPOLANT_START_OVER_LIMIT", NULL },
+	{ 0x00000019, "SEMANTIC_LAYER_OVER_LIMIT", NULL },
+	{ 0x0000001a, "RT_INVALID_ALIGNMENT", NULL },
+	{ 0x0000001b, "SAMPLER_OVER_LIMIT", NULL },
+	{ 0x0000001c, "TEXTURE_OVER_LIMIT", NULL },
+	{ 0x0000001e, "GP_TOO_MANY_OUTPUTS", NULL },
+	{ 0x0000001f, "RT_BPP128_WITH_MS8", NULL },
+	{ 0x00000021, "Z_OUT_OF_BOUNDS", NULL },
+	{ 0x00000023, "XY_OUT_OF_BOUNDS", NULL },
+	{ 0x00000024, "VP_ZERO_INPUTS", NULL },
+	{ 0x00000027, "CP_MORE_PARAMS_THAN_SHARED", NULL },
+	{ 0x00000028, "CP_NO_REG_SPACE_STRIPED", NULL },
+	{ 0x00000029, "CP_NO_REG_SPACE_PACKED", NULL },
+	{ 0x0000002a, "CP_NOT_ENOUGH_WARPS", NULL },
+	{ 0x0000002b, "CP_BLOCK_SIZE_MISMATCH", NULL },
+	{ 0x0000002c, "CP_NOT_ENOUGH_LOCAL_WARPS", NULL },
+	{ 0x0000002d, "CP_NOT_ENOUGH_STACK_WARPS", NULL },
+	{ 0x0000002e, "CP_NO_BLOCKDIM_LATCH", NULL },
+	{ 0x00000031, "ENG2D_FORMAT_MISMATCH", NULL },
+	{ 0x0000003f, "PRIMITIVE_ID_NEEDS_GP", NULL },
+	{ 0x00000044, "SEMANTIC_VIEWPORT_OVER_LIMIT", NULL },
+	{ 0x00000045, "SEMANTIC_COLOR_FRONT_OVER_LIMIT", NULL },
+	{ 0x00000046, "LAYER_ID_NEEDS_GP", NULL },
+	{ 0x00000047, "SEMANTIC_CLIP_OVER_LIMIT", NULL },
+	{ 0x00000048, "SEMANTIC_PTSZ_OVER_LIMIT", NULL },
+	{}
+};
+
+static const struct nouveau_bitfield nv50_graph_intr_name[] = {
+	{ 0x00000001, "NOTIFY" },
+	{ 0x00000002, "COMPUTE_QUERY" },
+	{ 0x00000010, "ILLEGAL_MTHD" },
+	{ 0x00000020, "ILLEGAL_CLASS" },
+	{ 0x00000040, "DOUBLE_NOTIFY" },
+	{ 0x00001000, "CONTEXT_SWITCH" },
+	{ 0x00010000, "BUFFER_NOTIFY" },
+	{ 0x00100000, "DATA_ERROR" },
+	{ 0x00200000, "TRAP" },
+	{ 0x01000000, "SINGLE_STEP" },
+	{}
+};
+
+static void
+nv50_priv_mp_trap(struct nv50_graph_priv *priv, int tpid, int display)
+{
+	u32 units = nv_rd32(priv, 0x1540);
+	u32 addr, mp10, status, pc, oplow, ophigh;
+	int i;
+	int mps = 0;
+	for (i = 0; i < 4; i++) {
+		if (!(units & 1 << (i+24)))
+			continue;
+		if (nv_device(priv)->chipset < 0xa0)
+			addr = 0x408200 + (tpid << 12) + (i << 7);
+		else
+			addr = 0x408100 + (tpid << 11) + (i << 7);
+		mp10 = nv_rd32(priv, addr + 0x10);
+		status = nv_rd32(priv, addr + 0x14);
+		if (!status)
+			continue;
+		if (display) {
+			nv_rd32(priv, addr + 0x20);
+			pc = nv_rd32(priv, addr + 0x24);
+			oplow = nv_rd32(priv, addr + 0x70);
+			ophigh = nv_rd32(priv, addr + 0x74);
+			nv_error(priv, "TRAP_MP_EXEC - "
+					"TP %d MP %d: ", tpid, i);
+			nouveau_enum_print(nv50_mp_exec_error_names, status);
+			printk(" at %06x warp %d, opcode %08x %08x\n",
+					pc&0xffffff, pc >> 24,
+					oplow, ophigh);
+		}
+		nv_wr32(priv, addr + 0x10, mp10);
+		nv_wr32(priv, addr + 0x14, 0);
+		mps++;
+	}
+	if (!mps && display)
+		nv_error(priv, "TRAP_MP_EXEC - TP %d: "
+				"No MPs claiming errors?\n", tpid);
+}
+
+static void
+nv50_priv_tp_trap(struct nv50_graph_priv *priv, int type, u32 ustatus_old,
+		u32 ustatus_new, int display, const char *name)
+{
+	int tps = 0;
+	u32 units = nv_rd32(priv, 0x1540);
+	int i, r;
+	u32 ustatus_addr, ustatus;
+	for (i = 0; i < 16; i++) {
+		if (!(units & (1 << i)))
+			continue;
+		if (nv_device(priv)->chipset < 0xa0)
+			ustatus_addr = ustatus_old + (i << 12);
+		else
+			ustatus_addr = ustatus_new + (i << 11);
+		ustatus = nv_rd32(priv, ustatus_addr) & 0x7fffffff;
+		if (!ustatus)
+			continue;
+		tps++;
+		switch (type) {
+		case 6: /* texture error... unknown for now */
+			if (display) {
+				nv_error(priv, "magic set %d:\n", i);
+				for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
+					nv_error(priv, "\t0x%08x: 0x%08x\n", r,
+						nv_rd32(priv, r));
+			}
+			break;
+		case 7: /* MP error */
+			if (ustatus & 0x04030000) {
+				nv50_priv_mp_trap(priv, i, display);
+				ustatus &= ~0x04030000;
+			}
+			break;
+		case 8: /* TPDMA error */
+			{
+			u32 e0c = nv_rd32(priv, ustatus_addr + 4);
+			u32 e10 = nv_rd32(priv, ustatus_addr + 8);
+			u32 e14 = nv_rd32(priv, ustatus_addr + 0xc);
+			u32 e18 = nv_rd32(priv, ustatus_addr + 0x10);
+			u32 e1c = nv_rd32(priv, ustatus_addr + 0x14);
+			u32 e20 = nv_rd32(priv, ustatus_addr + 0x18);
+			u32 e24 = nv_rd32(priv, ustatus_addr + 0x1c);
+			/* 2d engine destination */
+			if (ustatus & 0x00000010) {
+				if (display) {
+					nv_error(priv, "TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
+							i, e14, e10);
+					nv_error(priv, "TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+							i, e0c, e18, e1c, e20, e24);
+				}
+				ustatus &= ~0x00000010;
+			}
+			/* Render target */
+			if (ustatus & 0x00000040) {
+				if (display) {
+					nv_error(priv, "TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
+							i, e14, e10);
+					nv_error(priv, "TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+							i, e0c, e18, e1c, e20, e24);
+				}
+				ustatus &= ~0x00000040;
+			}
+			/* CUDA memory: l[], g[] or stack. */
+			if (ustatus & 0x00000080) {
+				if (display) {
+					if (e18 & 0x80000000) {
+						/* g[] read fault? */
+						nv_error(priv, "TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
+								i, e14, e10 | ((e18 >> 24) & 0x1f));
+						e18 &= ~0x1f000000;
+					} else if (e18 & 0xc) {
+						/* g[] write fault? */
+						nv_error(priv, "TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
+								i, e14, e10 | ((e18 >> 7) & 0x1f));
+						e18 &= ~0x00000f80;
+					} else {
+						nv_error(priv, "TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
+								i, e14, e10);
+					}
+					nv_error(priv, "TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
+							i, e0c, e18, e1c, e20, e24);
+				}
+				ustatus &= ~0x00000080;
+			}
+			}
+			break;
+		}
+		if (ustatus) {
+			if (display)
+				nv_info(priv, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
+		}
+		nv_wr32(priv, ustatus_addr, 0xc0000000);
+	}
+
+	if (!tps && display)
+		nv_info(priv, "%s - No TPs claiming errors?\n", name);
+}
+
+static int
+nv50_graph_trap_handler(struct nv50_graph_priv *priv, u32 display,
+			int chid, u64 inst)
+{
+	u32 status = nv_rd32(priv, 0x400108);
+	u32 ustatus;
+
+	if (!status && display) {
+		nv_error(priv, "TRAP: no units reporting traps?\n");
+		return 1;
+	}
+
+	/* DISPATCH: Relays commands to other units and handles NOTIFY,
+	 * COND, QUERY. If you get a trap from it, the command is still stuck
+	 * in DISPATCH and you need to do something about it. */
+	if (status & 0x001) {
+		ustatus = nv_rd32(priv, 0x400804) & 0x7fffffff;
+		if (!ustatus && display) {
+			nv_error(priv, "TRAP_DISPATCH - no ustatus?\n");
+		}
+
+		nv_wr32(priv, 0x400500, 0x00000000);
+
+		/* Known to be triggered by screwed up NOTIFY and COND... */
+		if (ustatus & 0x00000001) {
+			u32 addr = nv_rd32(priv, 0x400808);
+			u32 subc = (addr & 0x00070000) >> 16;
+			u32 mthd = (addr & 0x00001ffc);
+			u32 datal = nv_rd32(priv, 0x40080c);
+			u32 datah = nv_rd32(priv, 0x400810);
+			u32 class = nv_rd32(priv, 0x400814);
+			u32 r848 = nv_rd32(priv, 0x400848);
+
+			nv_error(priv, "TRAP DISPATCH_FAULT\n");
+			if (display && (addr & 0x80000000)) {
+				nv_error(priv, "ch %d [0x%010llx] "
+					     "subc %d class 0x%04x mthd 0x%04x "
+					     "data 0x%08x%08x "
+					     "400808 0x%08x 400848 0x%08x\n",
+					chid, inst, subc, class, mthd, datah,
+					datal, addr, r848);
+			} else
+			if (display) {
+				nv_error(priv, "no stuck command?\n");
+			}
+
+			nv_wr32(priv, 0x400808, 0);
+			nv_wr32(priv, 0x4008e8, nv_rd32(priv, 0x4008e8) & 3);
+			nv_wr32(priv, 0x400848, 0);
+			ustatus &= ~0x00000001;
+		}
+
+		if (ustatus & 0x00000002) {
+			u32 addr = nv_rd32(priv, 0x40084c);
+			u32 subc = (addr & 0x00070000) >> 16;
+			u32 mthd = (addr & 0x00001ffc);
+			u32 data = nv_rd32(priv, 0x40085c);
+			u32 class = nv_rd32(priv, 0x400814);
+
+			nv_error(priv, "TRAP DISPATCH_QUERY\n");
+			if (display && (addr & 0x80000000)) {
+				nv_error(priv, "ch %d [0x%010llx] "
+					     "subc %d class 0x%04x mthd 0x%04x "
+					     "data 0x%08x 40084c 0x%08x\n",
+					chid, inst, subc, class, mthd,
+					data, addr);
+			} else
+			if (display) {
+				nv_error(priv, "no stuck command?\n");
+			}
+
+			nv_wr32(priv, 0x40084c, 0);
+			ustatus &= ~0x00000002;
+		}
+
+		if (ustatus && display) {
+			nv_error(priv, "TRAP_DISPATCH (unknown "
+				      "0x%08x)\n", ustatus);
+		}
+
+		nv_wr32(priv, 0x400804, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x001);
+		status &= ~0x001;
+		if (!status)
+			return 0;
+	}
+
+	/* M2MF: Memory to memory copy engine. */
+	if (status & 0x002) {
+		u32 ustatus = nv_rd32(priv, 0x406800) & 0x7fffffff;
+		if (display) {
+			nv_error(priv, "TRAP_M2MF");
+			nouveau_bitfield_print(nv50_graph_trap_m2mf, ustatus);
+			printk("\n");
+			nv_error(priv, "TRAP_M2MF %08x %08x %08x %08x\n",
+				nv_rd32(priv, 0x406804), nv_rd32(priv, 0x406808),
+				nv_rd32(priv, 0x40680c), nv_rd32(priv, 0x406810));
+
+		}
+
+		/* No sane way found yet -- just reset the bugger. */
+		nv_wr32(priv, 0x400040, 2);
+		nv_wr32(priv, 0x400040, 0);
+		nv_wr32(priv, 0x406800, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x002);
+		status &= ~0x002;
+	}
+
+	/* VFETCH: Fetches data from vertex buffers. */
+	if (status & 0x004) {
+		u32 ustatus = nv_rd32(priv, 0x400c04) & 0x7fffffff;
+		if (display) {
+			nv_error(priv, "TRAP_VFETCH");
+			nouveau_bitfield_print(nv50_graph_trap_vfetch, ustatus);
+			printk("\n");
+			nv_error(priv, "TRAP_VFETCH %08x %08x %08x %08x\n",
+				nv_rd32(priv, 0x400c00), nv_rd32(priv, 0x400c08),
+				nv_rd32(priv, 0x400c0c), nv_rd32(priv, 0x400c10));
+		}
+
+		nv_wr32(priv, 0x400c04, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x004);
+		status &= ~0x004;
+	}
+
+	/* STRMOUT: DirectX streamout / OpenGL transform feedback. */
+	if (status & 0x008) {
+		ustatus = nv_rd32(priv, 0x401800) & 0x7fffffff;
+		if (display) {
+			nv_error(priv, "TRAP_STRMOUT");
+			nouveau_bitfield_print(nv50_graph_trap_strmout, ustatus);
+			printk("\n");
+			nv_error(priv, "TRAP_STRMOUT %08x %08x %08x %08x\n",
+				nv_rd32(priv, 0x401804), nv_rd32(priv, 0x401808),
+				nv_rd32(priv, 0x40180c), nv_rd32(priv, 0x401810));
+
+		}
+
+		/* No sane way found yet -- just reset the bugger. */
+		nv_wr32(priv, 0x400040, 0x80);
+		nv_wr32(priv, 0x400040, 0);
+		nv_wr32(priv, 0x401800, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x008);
+		status &= ~0x008;
+	}
+
+	/* CCACHE: Handles code and c[] caches and fills them. */
+	if (status & 0x010) {
+		ustatus = nv_rd32(priv, 0x405018) & 0x7fffffff;
+		if (display) {
+			nv_error(priv, "TRAP_CCACHE");
+			nouveau_bitfield_print(nv50_graph_trap_ccache, ustatus);
+			printk("\n");
+			nv_error(priv, "TRAP_CCACHE %08x %08x %08x %08x"
+				     " %08x %08x %08x\n",
+				nv_rd32(priv, 0x405000), nv_rd32(priv, 0x405004),
+				nv_rd32(priv, 0x405008), nv_rd32(priv, 0x40500c),
+				nv_rd32(priv, 0x405010), nv_rd32(priv, 0x405014),
+				nv_rd32(priv, 0x40501c));
+
+		}
+
+		nv_wr32(priv, 0x405018, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x010);
+		status &= ~0x010;
+	}
+
+	/* Unknown, not seen yet... 0x402000 is the only trap status reg
+	 * remaining, so try to handle it anyway. Perhaps related to that
+	 * unknown DMA slot on tesla? */
+	if (status & 0x20) {
+		ustatus = nv_rd32(priv, 0x402000) & 0x7fffffff;
+		if (display)
+			nv_error(priv, "TRAP_UNKC04 0x%08x\n", ustatus);
+		nv_wr32(priv, 0x402000, 0xc0000000);
+		/* no status modifiction on purpose */
+	}
+
+	/* TEXTURE: CUDA texturing units */
+	if (status & 0x040) {
+		nv50_priv_tp_trap(priv, 6, 0x408900, 0x408600, display,
+				    "TRAP_TEXTURE");
+		nv_wr32(priv, 0x400108, 0x040);
+		status &= ~0x040;
+	}
+
+	/* MP: CUDA execution engines. */
+	if (status & 0x080) {
+		nv50_priv_tp_trap(priv, 7, 0x408314, 0x40831c, display,
+				    "TRAP_MP");
+		nv_wr32(priv, 0x400108, 0x080);
+		status &= ~0x080;
+	}
+
+	/* TPDMA:  Handles TP-initiated uncached memory accesses:
+	 * l[], g[], stack, 2d surfaces, render targets. */
+	if (status & 0x100) {
+		nv50_priv_tp_trap(priv, 8, 0x408e08, 0x408708, display,
+				    "TRAP_TPDMA");
+		nv_wr32(priv, 0x400108, 0x100);
+		status &= ~0x100;
+	}
+
+	if (status) {
+		if (display)
+			nv_error(priv, "TRAP: unknown 0x%08x\n", status);
+		nv_wr32(priv, 0x400108, status);
+	}
+
+	return 1;
+}
+
+static void
+nv50_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle = NULL;
+	struct nv50_graph_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, 0x400100);
+	u32 inst = nv_rd32(priv, 0x40032c) & 0x0fffffff;
+	u32 addr = nv_rd32(priv, 0x400704);
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 mthd = (addr & 0x00001ffc);
+	u32 data = nv_rd32(priv, 0x400708);
+	u32 class = nv_rd32(priv, 0x400814);
+	u32 show = stat;
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000010) {
+		handle = nouveau_handle_get_class(engctx, class);
+		if (handle && !nv_call(handle->object, mthd, data))
+			show &= ~0x00000010;
+		nouveau_handle_put(handle);
+	}
+
+	if (show & 0x00100000) {
+		u32 ecode = nv_rd32(priv, 0x400110);
+		nv_error(priv, "DATA_ERROR ");
+		nouveau_enum_print(nv50_data_error_names, ecode);
+		printk("\n");
+	}
+
+	if (stat & 0x00200000) {
+		if (!nv50_graph_trap_handler(priv, show, chid, (u64)inst << 12))
+			show &= ~0x00200000;
+	}
+
+	nv_wr32(priv, 0x400100, stat);
+	nv_wr32(priv, 0x400500, 0x00010001);
+
+	if (show) {
+		nv_info(priv, "");
+		nouveau_bitfield_print(nv50_graph_intr_name, show);
+		printk("\n");
+		nv_error(priv, "ch %d [0x%010llx] subc %d class 0x%04x "
+			       "mthd 0x%04x data 0x%08x\n",
+			 chid, (u64)inst << 12, subc, class, mthd, data);
+		nv50_fb_trap(nouveau_fb(priv), 1);
+	}
+
+	if (nv_rd32(priv, 0x400824) & (1 << 31))
+		nv_wr32(priv, 0x400824, nv_rd32(priv, 0x400824) & ~(1 << 31));
+
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv50_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv50_graph_priv *priv;
+	int ret;
+
+	ret = nouveau_graph_create(parent, engine, oclass, true, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00201000;
+	nv_subdev(priv)->intr = nv50_graph_intr;
+	nv_engine(priv)->cclass = &nv50_graph_cclass;
+
+	switch (nv_device(priv)->chipset) {
+	case 0x50:
+		nv_engine(priv)->sclass = nv50_graph_sclass;
+		break;
+	case 0x84:
+	case 0x86:
+	case 0x92:
+	case 0x94:
+	case 0x96:
+	case 0x98:
+		nv_engine(priv)->sclass = nv84_graph_sclass;
+		break;
+	case 0xa0:
+	case 0xaa:
+	case 0xac:
+		nv_engine(priv)->sclass = nva0_graph_sclass;
+		break;
+	case 0xa3:
+	case 0xa5:
+	case 0xa8:
+		nv_engine(priv)->sclass = nva3_graph_sclass;
+		break;
+	case 0xaf:
+		nv_engine(priv)->sclass = nvaf_graph_sclass;
+		break;
+
+	};
+
+	if (nv_device(priv)->chipset == 0x50 ||
+	    nv_device(priv)->chipset == 0xac)
+		nv_engine(priv)->tlb_flush = nv50_graph_tlb_flush;
+	else
+		nv_engine(priv)->tlb_flush = nv84_graph_tlb_flush;
+
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static int
+nv50_graph_init(struct nouveau_object *object)
+{
+	struct nv50_graph_priv *priv = (void *)object;
+	int ret, units, i;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* NV_PGRAPH_DEBUG_3_HW_CTX_SWITCH_ENABLED */
+	nv_wr32(priv, 0x40008c, 0x00000004);
+
+	/* reset/enable traps and interrupts */
+	nv_wr32(priv, 0x400804, 0xc0000000);
+	nv_wr32(priv, 0x406800, 0xc0000000);
+	nv_wr32(priv, 0x400c04, 0xc0000000);
+	nv_wr32(priv, 0x401800, 0xc0000000);
+	nv_wr32(priv, 0x405018, 0xc0000000);
+	nv_wr32(priv, 0x402000, 0xc0000000);
+
+	units = nv_rd32(priv, 0x001540);
+	for (i = 0; i < 16; i++) {
+		if (!(units & (1 << i)))
+			continue;
+
+		if (nv_device(priv)->chipset < 0xa0) {
+			nv_wr32(priv, 0x408900 + (i << 12), 0xc0000000);
+			nv_wr32(priv, 0x408e08 + (i << 12), 0xc0000000);
+			nv_wr32(priv, 0x408314 + (i << 12), 0xc0000000);
+		} else {
+			nv_wr32(priv, 0x408600 + (i << 11), 0xc0000000);
+			nv_wr32(priv, 0x408708 + (i << 11), 0xc0000000);
+			nv_wr32(priv, 0x40831c + (i << 11), 0xc0000000);
+		}
+	}
+
+	nv_wr32(priv, 0x400108, 0xffffffff);
+	nv_wr32(priv, 0x400138, 0xffffffff);
+	nv_wr32(priv, 0x400100, 0xffffffff);
+	nv_wr32(priv, 0x40013c, 0xffffffff);
+	nv_wr32(priv, 0x400500, 0x00010001);
+
+	/* upload context program, initialise ctxctl defaults */
+	ret = nv50_grctx_init(nv_device(priv), &priv->size);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x400824, 0x00000000);
+	nv_wr32(priv, 0x400828, 0x00000000);
+	nv_wr32(priv, 0x40082c, 0x00000000);
+	nv_wr32(priv, 0x400830, 0x00000000);
+	nv_wr32(priv, 0x400724, 0x00000000);
+	nv_wr32(priv, 0x40032c, 0x00000000);
+	nv_wr32(priv, 0x400320, 4);	/* CTXCTL_CMD = NEWCTXDMA */
+
+	/* some unknown zcull magic */
+	switch (nv_device(priv)->chipset & 0xf0) {
+	case 0x50:
+	case 0x80:
+	case 0x90:
+		nv_wr32(priv, 0x402ca8, 0x00000800);
+		break;
+	case 0xa0:
+	default:
+		nv_wr32(priv, 0x402cc0, 0x00000000);
+		if (nv_device(priv)->chipset == 0xa0 ||
+		    nv_device(priv)->chipset == 0xaa ||
+		    nv_device(priv)->chipset == 0xac) {
+			nv_wr32(priv, 0x402ca8, 0x00000802);
+		} else {
+			nv_wr32(priv, 0x402cc0, 0x00000000);
+			nv_wr32(priv, 0x402ca8, 0x00000002);
+		}
+
+		break;
+	}
+
+	/* zero out zcull regions */
+	for (i = 0; i < 8; i++) {
+		nv_wr32(priv, 0x402c20 + (i * 8), 0x00000000);
+		nv_wr32(priv, 0x402c24 + (i * 8), 0x00000000);
+		nv_wr32(priv, 0x402c28 + (i * 8), 0x00000000);
+		nv_wr32(priv, 0x402c2c + (i * 8), 0x00000000);
+	}
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_graph_ctor,
+		.dtor = _nouveau_graph_dtor,
+		.init = nv50_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.h b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.h
new file mode 100644
index 00000000000..0505fb419bd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.h
@@ -0,0 +1,7 @@
+#ifndef __NV50_GRAPH_H__
+#define __NV50_GRAPH_H__
+
+int  nv50_grctx_init(struct nouveau_device *, u32 *size);
+void nv50_grctx_fill(struct nouveau_device *, struct nouveau_gpuobj *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
new file mode 100644
index 00000000000..c62f2d0f5f0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -0,0 +1,955 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+#include "fuc/hubnvc0.fuc.h"
+#include "fuc/gpcnvc0.fuc.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nvc0_graph_sclass[] = {
+	{ 0x902d, &nouveau_object_ofuncs },
+	{ 0x9039, &nouveau_object_ofuncs },
+	{ 0x9097, &nouveau_object_ofuncs },
+	{ 0x90c0, &nouveau_object_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nvc1_graph_sclass[] = {
+	{ 0x902d, &nouveau_object_ofuncs },
+	{ 0x9039, &nouveau_object_ofuncs },
+	{ 0x9097, &nouveau_object_ofuncs },
+	{ 0x90c0, &nouveau_object_ofuncs },
+	{ 0x9197, &nouveau_object_ofuncs },
+	{}
+};
+
+static struct nouveau_oclass
+nvc8_graph_sclass[] = {
+	{ 0x902d, &nouveau_object_ofuncs },
+	{ 0x9039, &nouveau_object_ofuncs },
+	{ 0x9097, &nouveau_object_ofuncs },
+	{ 0x90c0, &nouveau_object_ofuncs },
+	{ 0x9197, &nouveau_object_ofuncs },
+	{ 0x9297, &nouveau_object_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+int
+nvc0_graph_context_ctor(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass, void *args, u32 size,
+			struct nouveau_object **pobject)
+{
+	struct nouveau_vm *vm = nouveau_client(parent)->vm;
+	struct nvc0_graph_priv *priv = (void *)engine;
+	struct nvc0_graph_data *data = priv->mmio_data;
+	struct nvc0_graph_mmio *mmio = priv->mmio_list;
+	struct nvc0_graph_chan *chan;
+	int ret, i;
+
+	/* allocate memory for context, and fill with default values */
+	ret = nouveau_graph_context_create(parent, engine, oclass, NULL,
+					   priv->size, 0x100,
+					   NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	/* allocate memory for a "mmio list" buffer that's used by the HUB
+	 * fuc to modify some per-context register settings on first load
+	 * of the context.
+	 */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0x100, 0, &chan->mmio);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_map_vm(nv_gpuobj(chan->mmio), vm,
+				    NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS,
+				    &chan->mmio_vma);
+	if (ret)
+		return ret;
+
+	/* allocate buffers referenced by mmio list */
+	for (i = 0; data->size && i < ARRAY_SIZE(priv->mmio_data); i++) {
+		ret = nouveau_gpuobj_new(parent, NULL, data->size, data->align,
+					 0, &chan->data[i].mem);
+		if (ret)
+			return ret;
+
+		ret = nouveau_gpuobj_map_vm(chan->data[i].mem, vm, data->access,
+					   &chan->data[i].vma);
+		if (ret)
+			return ret;
+
+		data++;
+	}
+
+	/* finally, fill in the mmio list and point the context at it */
+	for (i = 0; mmio->addr && i < ARRAY_SIZE(priv->mmio_list); i++) {
+		u32 addr = mmio->addr;
+		u32 data = mmio->data;
+
+		if (mmio->shift) {
+			u64 info = chan->data[mmio->buffer].vma.offset;
+			data |= info >> mmio->shift;
+		}
+
+		nv_wo32(chan->mmio, chan->mmio_nr++ * 4, addr);
+		nv_wo32(chan->mmio, chan->mmio_nr++ * 4, data);
+		mmio++;
+	}
+
+	for (i = 0; i < priv->size; i += 4)
+		nv_wo32(chan, i, priv->data[i / 4]);
+
+	if (!priv->firmware) {
+		nv_wo32(chan, 0x00, chan->mmio_nr / 2);
+		nv_wo32(chan, 0x04, chan->mmio_vma.offset >> 8);
+	} else {
+		nv_wo32(chan, 0xf4, 0);
+		nv_wo32(chan, 0xf8, 0);
+		nv_wo32(chan, 0x10, chan->mmio_nr / 2);
+		nv_wo32(chan, 0x14, lower_32_bits(chan->mmio_vma.offset));
+		nv_wo32(chan, 0x18, upper_32_bits(chan->mmio_vma.offset));
+		nv_wo32(chan, 0x1c, 1);
+		nv_wo32(chan, 0x20, 0);
+		nv_wo32(chan, 0x28, 0);
+		nv_wo32(chan, 0x2c, 0);
+	}
+
+	return 0;
+}
+
+void
+nvc0_graph_context_dtor(struct nouveau_object *object)
+{
+	struct nvc0_graph_chan *chan = (void *)object;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(chan->data); i++) {
+		nouveau_gpuobj_unmap(&chan->data[i].vma);
+		nouveau_gpuobj_ref(NULL, &chan->data[i].mem);
+	}
+
+	nouveau_gpuobj_unmap(&chan->mmio_vma);
+	nouveau_gpuobj_ref(NULL, &chan->mmio);
+
+	nouveau_graph_context_destroy(&chan->base);
+}
+
+static struct nouveau_oclass
+nvc0_graph_cclass = {
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_graph_context_ctor,
+		.dtor = nvc0_graph_context_dtor,
+		.init = _nouveau_graph_context_init,
+		.fini = _nouveau_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+nvc0_graph_ctxctl_debug_unit(struct nvc0_graph_priv *priv, u32 base)
+{
+	nv_error(priv, "%06x - done 0x%08x\n", base,
+		 nv_rd32(priv, base + 0x400));
+	nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+		 nv_rd32(priv, base + 0x800), nv_rd32(priv, base + 0x804),
+		 nv_rd32(priv, base + 0x808), nv_rd32(priv, base + 0x80c));
+	nv_error(priv, "%06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
+		 nv_rd32(priv, base + 0x810), nv_rd32(priv, base + 0x814),
+		 nv_rd32(priv, base + 0x818), nv_rd32(priv, base + 0x81c));
+}
+
+void
+nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *priv)
+{
+	u32 gpcnr = nv_rd32(priv, 0x409604) & 0xffff;
+	u32 gpc;
+
+	nvc0_graph_ctxctl_debug_unit(priv, 0x409000);
+	for (gpc = 0; gpc < gpcnr; gpc++)
+		nvc0_graph_ctxctl_debug_unit(priv, 0x502000 + (gpc * 0x8000));
+}
+
+static void
+nvc0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
+{
+	u32 ustat = nv_rd32(priv, 0x409c18);
+
+	if (ustat & 0x00000001)
+		nv_error(priv, "CTXCTRL ucode error\n");
+	if (ustat & 0x00080000)
+		nv_error(priv, "CTXCTRL watchdog timeout\n");
+	if (ustat & ~0x00080001)
+		nv_error(priv, "CTXCTRL 0x%08x\n", ustat);
+
+	nvc0_graph_ctxctl_debug(priv);
+	nv_wr32(priv, 0x409c20, ustat);
+}
+
+static void
+nvc0_graph_trap_tpc(struct nvc0_graph_priv *priv, int gpc, int tpc)
+{
+	u32 stat = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0508));
+
+	if (stat & 0x00000001) {
+		u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0224));
+		nv_error(priv, "GPC%d/TPC%d/TEX: 0x%08x\n", gpc, tpc, trap);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000001);
+		stat &= ~0x00000001;
+	}
+
+	if (stat & 0x00000002) {
+		u32 trap0 = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0644));
+		u32 trap1 = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x064c));
+		nv_error(priv, "GPC%d/TPC%d/MP: 0x%08x 0x%08x\n",
+			       gpc, tpc, trap0, trap1);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0644), 0x001ffffe);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x064c), 0x0000000f);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000002);
+		stat &= ~0x00000002;
+	}
+
+	if (stat & 0x00000004) {
+		u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x0084));
+		nv_error(priv, "GPC%d/TPC%d/POLY: 0x%08x\n", gpc, tpc, trap);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000004);
+		stat &= ~0x00000004;
+	}
+
+	if (stat & 0x00000008) {
+		u32 trap = nv_rd32(priv, TPC_UNIT(gpc, tpc, 0x048c));
+		nv_error(priv, "GPC%d/TPC%d/L1C: 0x%08x\n", gpc, tpc, trap);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), 0x00000008);
+		stat &= ~0x00000008;
+	}
+
+	if (stat) {
+		nv_error(priv, "GPC%d/TPC%d/0x%08x: unknown\n", gpc, tpc, stat);
+		nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x0508), stat);
+	}
+}
+
+static void
+nvc0_graph_trap_gpc(struct nvc0_graph_priv *priv, int gpc)
+{
+	u32 stat = nv_rd32(priv, GPC_UNIT(gpc, 0x2c90));
+	int tpc;
+
+	if (stat & 0x00000001) {
+		u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0420));
+		nv_error(priv, "GPC%d/PROP: 0x%08x\n", gpc, trap);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000001);
+		stat &= ~0x00000001;
+	}
+
+	if (stat & 0x00000002) {
+		u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0900));
+		nv_error(priv, "GPC%d/ZCULL: 0x%08x\n", gpc, trap);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000002);
+		stat &= ~0x00000002;
+	}
+
+	if (stat & 0x00000004) {
+		u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x1028));
+		nv_error(priv, "GPC%d/CCACHE: 0x%08x\n", gpc, trap);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000004);
+		stat &= ~0x00000004;
+	}
+
+	if (stat & 0x00000008) {
+		u32 trap = nv_rd32(priv, GPC_UNIT(gpc, 0x0824));
+		nv_error(priv, "GPC%d/ESETUP: 0x%08x\n", gpc, trap);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0x00000008);
+		stat &= ~0x00000009;
+	}
+
+	for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+		u32 mask = 0x00010000 << tpc;
+		if (stat & mask) {
+			nvc0_graph_trap_tpc(priv, gpc, tpc);
+			nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), mask);
+			stat &= ~mask;
+		}
+	}
+
+	if (stat) {
+		nv_error(priv, "GPC%d/0x%08x: unknown\n", gpc, stat);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), stat);
+	}
+}
+
+static void
+nvc0_graph_trap_intr(struct nvc0_graph_priv *priv)
+{
+	u32 trap = nv_rd32(priv, 0x400108);
+	int rop, gpc;
+
+	if (trap & 0x00000001) {
+		u32 stat = nv_rd32(priv, 0x404000);
+		nv_error(priv, "DISPATCH 0x%08x\n", stat);
+		nv_wr32(priv, 0x404000, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000001);
+		trap &= ~0x00000001;
+	}
+
+	if (trap & 0x00000002) {
+		u32 stat = nv_rd32(priv, 0x404600);
+		nv_error(priv, "M2MF 0x%08x\n", stat);
+		nv_wr32(priv, 0x404600, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000002);
+		trap &= ~0x00000002;
+	}
+
+	if (trap & 0x00000008) {
+		u32 stat = nv_rd32(priv, 0x408030);
+		nv_error(priv, "CCACHE 0x%08x\n", stat);
+		nv_wr32(priv, 0x408030, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000008);
+		trap &= ~0x00000008;
+	}
+
+	if (trap & 0x00000010) {
+		u32 stat = nv_rd32(priv, 0x405840);
+		nv_error(priv, "SHADER 0x%08x\n", stat);
+		nv_wr32(priv, 0x405840, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000010);
+		trap &= ~0x00000010;
+	}
+
+	if (trap & 0x00000040) {
+		u32 stat = nv_rd32(priv, 0x40601c);
+		nv_error(priv, "UNK6 0x%08x\n", stat);
+		nv_wr32(priv, 0x40601c, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000040);
+		trap &= ~0x00000040;
+	}
+
+	if (trap & 0x00000080) {
+		u32 stat = nv_rd32(priv, 0x404490);
+		nv_error(priv, "MACRO 0x%08x\n", stat);
+		nv_wr32(priv, 0x404490, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000080);
+		trap &= ~0x00000080;
+	}
+
+	if (trap & 0x01000000) {
+		u32 stat = nv_rd32(priv, 0x400118);
+		for (gpc = 0; stat && gpc < priv->gpc_nr; gpc++) {
+			u32 mask = 0x00000001 << gpc;
+			if (stat & mask) {
+				nvc0_graph_trap_gpc(priv, gpc);
+				nv_wr32(priv, 0x400118, mask);
+				stat &= ~mask;
+			}
+		}
+		nv_wr32(priv, 0x400108, 0x01000000);
+		trap &= ~0x01000000;
+	}
+
+	if (trap & 0x02000000) {
+		for (rop = 0; rop < priv->rop_nr; rop++) {
+			u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
+			u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
+			nv_error(priv, "ROP%d 0x%08x 0x%08x\n",
+				 rop, statz, statc);
+			nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
+			nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
+		}
+		nv_wr32(priv, 0x400108, 0x02000000);
+		trap &= ~0x02000000;
+	}
+
+	if (trap) {
+		nv_error(priv, "TRAP UNHANDLED 0x%08x\n", trap);
+		nv_wr32(priv, 0x400108, trap);
+	}
+}
+
+static void
+nvc0_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle;
+	struct nvc0_graph_priv *priv = (void *)subdev;
+	u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
+	u32 stat = nv_rd32(priv, 0x400100);
+	u32 addr = nv_rd32(priv, 0x400704);
+	u32 mthd = (addr & 0x00003ffc);
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 data = nv_rd32(priv, 0x400708);
+	u32 code = nv_rd32(priv, 0x400110);
+	u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000010) {
+		handle = nouveau_handle_get_class(engctx, class);
+		if (!handle || nv_call(handle->object, mthd, data)) {
+			nv_error(priv, "ILLEGAL_MTHD ch %d [0x%010llx] "
+				     "subc %d class 0x%04x mthd 0x%04x "
+				     "data 0x%08x\n",
+				 chid, inst << 12, subc, class, mthd, data);
+		}
+		nouveau_handle_put(handle);
+		nv_wr32(priv, 0x400100, 0x00000010);
+		stat &= ~0x00000010;
+	}
+
+	if (stat & 0x00000020) {
+		nv_error(priv, "ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
+			     "class 0x%04x mthd 0x%04x data 0x%08x\n",
+			chid, inst << 12, subc, class, mthd, data);
+		nv_wr32(priv, 0x400100, 0x00000020);
+		stat &= ~0x00000020;
+	}
+
+	if (stat & 0x00100000) {
+		nv_error(priv, "DATA_ERROR [");
+		nouveau_enum_print(nv50_data_error_names, code);
+		printk("] ch %d [0x%010llx] subc %d class 0x%04x "
+		       "mthd 0x%04x data 0x%08x\n",
+		       chid, inst << 12, subc, class, mthd, data);
+		nv_wr32(priv, 0x400100, 0x00100000);
+		stat &= ~0x00100000;
+	}
+
+	if (stat & 0x00200000) {
+		nv_error(priv, "TRAP ch %d [0x%010llx]\n", chid, inst << 12);
+		nvc0_graph_trap_intr(priv);
+		nv_wr32(priv, 0x400100, 0x00200000);
+		stat &= ~0x00200000;
+	}
+
+	if (stat & 0x00080000) {
+		nvc0_graph_ctxctl_isr(priv);
+		nv_wr32(priv, 0x400100, 0x00080000);
+		stat &= ~0x00080000;
+	}
+
+	if (stat) {
+		nv_error(priv, "unknown stat 0x%08x\n", stat);
+		nv_wr32(priv, 0x400100, stat);
+	}
+
+	nv_wr32(priv, 0x400500, 0x00010001);
+	nouveau_engctx_put(engctx);
+}
+
+int
+nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
+		   struct nvc0_graph_fuc *fuc)
+{
+	struct nouveau_device *device = nv_device(priv);
+	const struct firmware *fw;
+	char f[32];
+	int ret;
+
+	snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
+	ret = request_firmware(&fw, f, &device->pdev->dev);
+	if (ret) {
+		snprintf(f, sizeof(f), "nouveau/%s", fwname);
+		ret = request_firmware(&fw, f, &device->pdev->dev);
+		if (ret) {
+			nv_error(priv, "failed to load %s\n", fwname);
+			return ret;
+		}
+	}
+
+	fuc->size = fw->size;
+	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
+	release_firmware(fw);
+	return (fuc->data != NULL) ? 0 : -ENOMEM;
+}
+
+static int
+nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nvc0_graph_priv *priv;
+	bool enable = true;
+	int ret, i;
+
+	switch (device->chipset) {
+	case 0xd9: /* known broken without binary driver firmware */
+		enable = false;
+		break;
+	default:
+		break;
+	}
+
+	ret = nouveau_graph_create(parent, engine, oclass, enable, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x18001000;
+	nv_subdev(priv)->intr = nvc0_graph_intr;
+	nv_engine(priv)->cclass = &nvc0_graph_cclass;
+
+	if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
+		nv_info(priv, "using external firmware\n");
+		if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
+		    nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
+		    nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
+		    nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
+			return -EINVAL;
+		priv->firmware = true;
+	}
+
+	switch (nvc0_graph_class(priv)) {
+	case 0x9097:
+		nv_engine(priv)->sclass = nvc0_graph_sclass;
+		break;
+	case 0x9197:
+		nv_engine(priv)->sclass = nvc1_graph_sclass;
+		break;
+	case 0x9297:
+		nv_engine(priv)->sclass = nvc8_graph_sclass;
+		break;
+	}
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b4);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b8);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < 0x1000; i += 4) {
+		nv_wo32(priv->unk4188b4, i, 0x00000010);
+		nv_wo32(priv->unk4188b8, i, 0x00000010);
+	}
+
+	priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
+	priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
+	for (i = 0; i < priv->gpc_nr; i++) {
+		priv->tpc_nr[i]  = nv_rd32(priv, GPC_UNIT(i, 0x2608));
+		priv->tpc_total += priv->tpc_nr[i];
+	}
+
+	/*XXX: these need figuring out... though it might not even matter */
+	switch (nv_device(priv)->chipset) {
+	case 0xc0:
+		if (priv->tpc_total == 11) { /* 465, 3/4/4/0, 4 */
+			priv->magic_not_rop_nr = 0x07;
+		} else
+		if (priv->tpc_total == 14) { /* 470, 3/3/4/4, 5 */
+			priv->magic_not_rop_nr = 0x05;
+		} else
+		if (priv->tpc_total == 15) { /* 480, 3/4/4/4, 6 */
+			priv->magic_not_rop_nr = 0x06;
+		}
+		break;
+	case 0xc3: /* 450, 4/0/0/0, 2 */
+		priv->magic_not_rop_nr = 0x03;
+		break;
+	case 0xc4: /* 460, 3/4/0/0, 4 */
+		priv->magic_not_rop_nr = 0x01;
+		break;
+	case 0xc1: /* 2/0/0/0, 1 */
+		priv->magic_not_rop_nr = 0x01;
+		break;
+	case 0xc8: /* 4/4/3/4, 5 */
+		priv->magic_not_rop_nr = 0x06;
+		break;
+	case 0xce: /* 4/4/0/0, 4 */
+		priv->magic_not_rop_nr = 0x03;
+		break;
+	case 0xcf: /* 4/0/0/0, 3 */
+		priv->magic_not_rop_nr = 0x03;
+		break;
+	case 0xd9: /* 1/0/0/0, 1 */
+		priv->magic_not_rop_nr = 0x01;
+		break;
+	}
+
+	return 0;
+}
+
+static void
+nvc0_graph_dtor_fw(struct nvc0_graph_fuc *fuc)
+{
+	if (fuc->data) {
+		kfree(fuc->data);
+		fuc->data = NULL;
+	}
+}
+
+void
+nvc0_graph_dtor(struct nouveau_object *object)
+{
+	struct nvc0_graph_priv *priv = (void *)object;
+
+	if (priv->data)
+		kfree(priv->data);
+
+	nvc0_graph_dtor_fw(&priv->fuc409c);
+	nvc0_graph_dtor_fw(&priv->fuc409d);
+	nvc0_graph_dtor_fw(&priv->fuc41ac);
+	nvc0_graph_dtor_fw(&priv->fuc41ad);
+
+	nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
+	nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
+
+	nouveau_graph_destroy(&priv->base);
+}
+
+static void
+nvc0_graph_init_obj418880(struct nvc0_graph_priv *priv)
+{
+	int i;
+
+	nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
+	nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
+	nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
+	nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
+}
+
+static void
+nvc0_graph_init_regs(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x400080, 0x003083c2);
+	nv_wr32(priv, 0x400088, 0x00006fe7);
+	nv_wr32(priv, 0x40008c, 0x00000000);
+	nv_wr32(priv, 0x400090, 0x00000030);
+	nv_wr32(priv, 0x40013c, 0x013901f7);
+	nv_wr32(priv, 0x400140, 0x00000100);
+	nv_wr32(priv, 0x400144, 0x00000000);
+	nv_wr32(priv, 0x400148, 0x00000110);
+	nv_wr32(priv, 0x400138, 0x00000000);
+	nv_wr32(priv, 0x400130, 0x00000000);
+	nv_wr32(priv, 0x400134, 0x00000000);
+	nv_wr32(priv, 0x400124, 0x00000002);
+}
+
+static void
+nvc0_graph_init_gpc_0(struct nvc0_graph_priv *priv)
+{
+	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
+	u32 data[TPC_MAX / 8];
+	u8  tpcnr[GPC_MAX];
+	int i, gpc, tpc;
+
+	nv_wr32(priv, TPC_UNIT(0, 0, 0x5c), 1); /* affects TFB offset queries */
+
+	/*
+	 *      TP      ROP UNKVAL(magic_not_rop_nr)
+	 * 450: 4/0/0/0 2        3
+	 * 460: 3/4/0/0 4        1
+	 * 465: 3/4/4/0 4        7
+	 * 470: 3/3/4/4 5        5
+	 * 480: 3/4/4/4 6        6
+	 */
+
+	memset(data, 0x00, sizeof(data));
+	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+	for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
+		do {
+			gpc = (gpc + 1) % priv->gpc_nr;
+		} while (!tpcnr[gpc]);
+		tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
+
+		data[i / 8] |= tpc << ((i % 8) * 4);
+	}
+
+	nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
+	nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
+	nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
+	nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
+
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
+						  priv->tpc_nr[gpc]);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
+	}
+
+	nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
+	nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
+}
+
+static void
+nvc0_graph_init_units(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x409c24, 0x000f0000);
+	nv_wr32(priv, 0x404000, 0xc0000000); /* DISPATCH */
+	nv_wr32(priv, 0x404600, 0xc0000000); /* M2MF */
+	nv_wr32(priv, 0x408030, 0xc0000000);
+	nv_wr32(priv, 0x40601c, 0xc0000000);
+	nv_wr32(priv, 0x404490, 0xc0000000); /* MACRO */
+	nv_wr32(priv, 0x406018, 0xc0000000);
+	nv_wr32(priv, 0x405840, 0xc0000000);
+	nv_wr32(priv, 0x405844, 0x00ffffff);
+	nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
+	nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
+}
+
+static void
+nvc0_graph_init_gpc_1(struct nvc0_graph_priv *priv)
+{
+	int gpc, tpc;
+
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+		for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
+		}
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
+	}
+}
+
+static void
+nvc0_graph_init_rop(struct nvc0_graph_priv *priv)
+{
+	int rop;
+
+	for (rop = 0; rop < priv->rop_nr; rop++) {
+		nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
+		nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
+		nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
+		nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
+	}
+}
+
+void
+nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
+		   struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
+{
+	int i;
+
+	nv_wr32(priv, fuc_base + 0x01c0, 0x01000000);
+	for (i = 0; i < data->size / 4; i++)
+		nv_wr32(priv, fuc_base + 0x01c4, data->data[i]);
+
+	nv_wr32(priv, fuc_base + 0x0180, 0x01000000);
+	for (i = 0; i < code->size / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, fuc_base + 0x0188, i >> 6);
+		nv_wr32(priv, fuc_base + 0x0184, code->data[i]);
+	}
+}
+
+static int
+nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
+{
+	u32 r000260;
+	int i;
+
+	if (priv->firmware) {
+		/* load fuc microcode */
+		r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+		nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c,
+						   &priv->fuc409d);
+		nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac,
+						   &priv->fuc41ad);
+		nv_wr32(priv, 0x000260, r000260);
+
+		/* start both of them running */
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x41a10c, 0x00000000);
+		nv_wr32(priv, 0x40910c, 0x00000000);
+		nv_wr32(priv, 0x41a100, 0x00000002);
+		nv_wr32(priv, 0x409100, 0x00000002);
+		if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
+			nv_info(priv, "0x409800 wait failed\n");
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x7fffffff);
+		nv_wr32(priv, 0x409504, 0x00000021);
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000010);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x10 timeout\n");
+			return -EBUSY;
+		}
+		priv->size = nv_rd32(priv, 0x409800);
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000016);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x16 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000025);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x25 timeout\n");
+			return -EBUSY;
+		}
+
+		if (priv->data == NULL) {
+			int ret = nvc0_grctx_generate(priv);
+			if (ret) {
+				nv_error(priv, "failed to construct context\n");
+				return ret;
+			}
+		}
+
+		return 0;
+	}
+
+	/* load HUB microcode */
+	r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+	nv_wr32(priv, 0x4091c0, 0x01000000);
+	for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
+		nv_wr32(priv, 0x4091c4, nvc0_grhub_data[i]);
+
+	nv_wr32(priv, 0x409180, 0x01000000);
+	for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x409188, i >> 6);
+		nv_wr32(priv, 0x409184, nvc0_grhub_code[i]);
+	}
+
+	/* load GPC microcode */
+	nv_wr32(priv, 0x41a1c0, 0x01000000);
+	for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
+		nv_wr32(priv, 0x41a1c4, nvc0_grgpc_data[i]);
+
+	nv_wr32(priv, 0x41a180, 0x01000000);
+	for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x41a188, i >> 6);
+		nv_wr32(priv, 0x41a184, nvc0_grgpc_code[i]);
+	}
+	nv_wr32(priv, 0x000260, r000260);
+
+	/* start HUB ucode running, it'll init the GPCs */
+	nv_wr32(priv, 0x409800, nv_device(priv)->chipset);
+	nv_wr32(priv, 0x40910c, 0x00000000);
+	nv_wr32(priv, 0x409100, 0x00000002);
+	if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
+		nv_error(priv, "HUB_INIT timed out\n");
+		nvc0_graph_ctxctl_debug(priv);
+		return -EBUSY;
+	}
+
+	priv->size = nv_rd32(priv, 0x409804);
+	if (priv->data == NULL) {
+		int ret = nvc0_grctx_generate(priv);
+		if (ret) {
+			nv_error(priv, "failed to construct context\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+nvc0_graph_init(struct nouveau_object *object)
+{
+	struct nvc0_graph_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nvc0_graph_init_obj418880(priv);
+	nvc0_graph_init_regs(priv);
+	/*nvc0_graph_init_unitplemented_magics(priv);*/
+	nvc0_graph_init_gpc_0(priv);
+	/*nvc0_graph_init_unitplemented_c242(priv);*/
+
+	nv_wr32(priv, 0x400500, 0x00010001);
+	nv_wr32(priv, 0x400100, 0xffffffff);
+	nv_wr32(priv, 0x40013c, 0xffffffff);
+
+	nvc0_graph_init_units(priv);
+	nvc0_graph_init_gpc_1(priv);
+	nvc0_graph_init_rop(priv);
+
+	nv_wr32(priv, 0x400108, 0xffffffff);
+	nv_wr32(priv, 0x400138, 0xffffffff);
+	nv_wr32(priv, 0x400118, 0xffffffff);
+	nv_wr32(priv, 0x400130, 0xffffffff);
+	nv_wr32(priv, 0x40011c, 0xffffffff);
+	nv_wr32(priv, 0x400134, 0xffffffff);
+	nv_wr32(priv, 0x400054, 0x34ce3464);
+
+	ret = nvc0_graph_init_ctxctl(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_graph_ctor,
+		.dtor = nvc0_graph_dtor,
+		.init = nvc0_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
new file mode 100644
index 00000000000..18d2210e12e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NVC0_GRAPH_H__
+#define __NVC0_GRAPH_H__
+
+#include <core/client.h>
+#include <core/handle.h>
+#include <core/gpuobj.h>
+#include <core/option.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
+#include <engine/graph.h>
+
+#define GPC_MAX 4
+#define TPC_MAX 32
+
+#define ROP_BCAST(r)      (0x408800 + (r))
+#define ROP_UNIT(u, r)    (0x410000 + (u) * 0x400 + (r))
+#define GPC_BCAST(r)      (0x418000 + (r))
+#define GPC_UNIT(t, r)    (0x500000 + (t) * 0x8000 + (r))
+#define TPC_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
+
+struct nvc0_graph_data {
+	u32 size;
+	u32 align;
+	u32 access;
+};
+
+struct nvc0_graph_mmio {
+	u32 addr;
+	u32 data;
+	u32 shift;
+	u32 buffer;
+};
+
+struct nvc0_graph_fuc {
+	u32 *data;
+	u32  size;
+};
+
+struct nvc0_graph_priv {
+	struct nouveau_graph base;
+
+	struct nvc0_graph_fuc fuc409c;
+	struct nvc0_graph_fuc fuc409d;
+	struct nvc0_graph_fuc fuc41ac;
+	struct nvc0_graph_fuc fuc41ad;
+	bool firmware;
+
+	u8 rop_nr;
+	u8 gpc_nr;
+	u8 tpc_nr[GPC_MAX];
+	u8 tpc_total;
+
+	struct nouveau_gpuobj *unk4188b4;
+	struct nouveau_gpuobj *unk4188b8;
+
+	struct nvc0_graph_data mmio_data[4];
+	struct nvc0_graph_mmio mmio_list[4096/8];
+	u32  size;
+	u32 *data;
+
+	u8 magic_not_rop_nr;
+};
+
+struct nvc0_graph_chan {
+	struct nouveau_graph_chan base;
+
+	struct nouveau_gpuobj *mmio;
+	struct nouveau_vma mmio_vma;
+	int mmio_nr;
+	struct {
+		struct nouveau_gpuobj *mem;
+		struct nouveau_vma vma;
+	} data[4];
+};
+
+static inline u32
+nvc0_graph_class(void *obj)
+{
+	struct nouveau_device *device = nv_device(obj);
+
+	switch (device->chipset) {
+	case 0xc0:
+	case 0xc3:
+	case 0xc4:
+	case 0xce: /* guess, mmio trace shows only 0x9097 state */
+	case 0xcf: /* guess, mmio trace shows only 0x9097 state */
+		return 0x9097;
+	case 0xc1:
+		return 0x9197;
+	case 0xc8:
+	case 0xd9:
+		return 0x9297;
+	case 0xe4:
+	case 0xe7:
+		return 0xa097;
+	default:
+		return 0;
+	}
+}
+
+void nv_icmd(struct nvc0_graph_priv *priv, u32 icmd, u32 data);
+
+static inline void
+nv_mthd(struct nvc0_graph_priv *priv, u32 class, u32 mthd, u32 data)
+{
+	nv_wr32(priv, 0x40448c, data);
+	nv_wr32(priv, 0x404488, 0x80000000 | (mthd << 14) | class);
+}
+
+struct nvc0_grctx {
+	struct nvc0_graph_priv *priv;
+	struct nvc0_graph_data *data;
+	struct nvc0_graph_mmio *mmio;
+	struct nouveau_gpuobj *chan;
+	int buffer_nr;
+	u64 buffer[4];
+	u64 addr;
+};
+
+int  nvc0_grctx_generate(struct nvc0_graph_priv *);
+int  nvc0_grctx_init(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc0_grctx_data(struct nvc0_grctx *, u32, u32, u32);
+void nvc0_grctx_mmio(struct nvc0_grctx *, u32, u32, u32, u32);
+int  nvc0_grctx_fini(struct nvc0_grctx *);
+
+int  nve0_grctx_generate(struct nvc0_graph_priv *);
+
+#define mmio_data(s,a,p) nvc0_grctx_data(&info, (s), (a), (p))
+#define mmio_list(r,d,s,b) nvc0_grctx_mmio(&info, (r), (d), (s), (b))
+
+void nvc0_graph_ctxctl_debug(struct nvc0_graph_priv *);
+int  nvc0_graph_ctor_fw(struct nvc0_graph_priv *, const char *,
+			struct nvc0_graph_fuc *);
+void nvc0_graph_dtor(struct nouveau_object *);
+void nvc0_graph_init_fw(struct nvc0_graph_priv *, u32 base,
+			struct nvc0_graph_fuc *, struct nvc0_graph_fuc *);
+int  nvc0_graph_context_ctor(struct nouveau_object *, struct nouveau_object *,
+			     struct nouveau_oclass *, void *, u32,
+			     struct nouveau_object **);
+void nvc0_graph_context_dtor(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nve0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nve0.c
new file mode 100644
index 00000000000..539d4c72f19
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nve0.c
@@ -0,0 +1,576 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+#include "fuc/hubnve0.fuc.h"
+#include "fuc/gpcnve0.fuc.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nve0_graph_sclass[] = {
+	{ 0x902d, &nouveau_object_ofuncs },
+	{ 0xa040, &nouveau_object_ofuncs },
+	{ 0xa097, &nouveau_object_ofuncs },
+	{ 0xa0c0, &nouveau_object_ofuncs },
+	{ 0xa0b5, &nouveau_object_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PGRAPH context
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nve0_graph_cclass = {
+	.handle = NV_ENGCTX(GR, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_graph_context_ctor,
+		.dtor = nvc0_graph_context_dtor,
+		.init = _nouveau_graph_context_init,
+		.fini = _nouveau_graph_context_fini,
+		.rd32 = _nouveau_graph_context_rd32,
+		.wr32 = _nouveau_graph_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+nve0_graph_ctxctl_isr(struct nvc0_graph_priv *priv)
+{
+	u32 ustat = nv_rd32(priv, 0x409c18);
+
+	if (ustat & 0x00000001)
+		nv_error(priv, "CTXCTRL ucode error\n");
+	if (ustat & 0x00080000)
+		nv_error(priv, "CTXCTRL watchdog timeout\n");
+	if (ustat & ~0x00080001)
+		nv_error(priv, "CTXCTRL 0x%08x\n", ustat);
+
+	nvc0_graph_ctxctl_debug(priv);
+	nv_wr32(priv, 0x409c20, ustat);
+}
+
+static void
+nve0_graph_trap_isr(struct nvc0_graph_priv *priv, int chid, u64 inst)
+{
+	u32 trap = nv_rd32(priv, 0x400108);
+	int rop;
+
+	if (trap & 0x00000001) {
+		u32 stat = nv_rd32(priv, 0x404000);
+		nv_error(priv, "DISPATCH ch %d [0x%010llx] 0x%08x\n",
+			 chid, inst, stat);
+		nv_wr32(priv, 0x404000, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000001);
+		trap &= ~0x00000001;
+	}
+
+	if (trap & 0x00000010) {
+		u32 stat = nv_rd32(priv, 0x405840);
+		nv_error(priv, "SHADER ch %d [0x%010llx] 0x%08x\n",
+			 chid, inst, stat);
+		nv_wr32(priv, 0x405840, 0xc0000000);
+		nv_wr32(priv, 0x400108, 0x00000010);
+		trap &= ~0x00000010;
+	}
+
+	if (trap & 0x02000000) {
+		for (rop = 0; rop < priv->rop_nr; rop++) {
+			u32 statz = nv_rd32(priv, ROP_UNIT(rop, 0x070));
+			u32 statc = nv_rd32(priv, ROP_UNIT(rop, 0x144));
+			nv_error(priv, "ROP%d ch %d [0x%010llx] 0x%08x 0x%08x\n",
+				 rop, chid, inst, statz, statc);
+			nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
+			nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
+		}
+		nv_wr32(priv, 0x400108, 0x02000000);
+		trap &= ~0x02000000;
+	}
+
+	if (trap) {
+		nv_error(priv, "TRAP ch %d [0x%010llx] 0x%08x\n",
+			 chid, inst, trap);
+		nv_wr32(priv, 0x400108, trap);
+	}
+}
+
+static void
+nve0_graph_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle;
+	struct nvc0_graph_priv *priv = (void *)subdev;
+	u64 inst = nv_rd32(priv, 0x409b00) & 0x0fffffff;
+	u32 stat = nv_rd32(priv, 0x400100);
+	u32 addr = nv_rd32(priv, 0x400704);
+	u32 mthd = (addr & 0x00003ffc);
+	u32 subc = (addr & 0x00070000) >> 16;
+	u32 data = nv_rd32(priv, 0x400708);
+	u32 code = nv_rd32(priv, 0x400110);
+	u32 class = nv_rd32(priv, 0x404200 + (subc * 4));
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x00000010) {
+		handle = nouveau_handle_get_class(engctx, class);
+		if (!handle || nv_call(handle->object, mthd, data)) {
+			nv_error(priv, "ILLEGAL_MTHD ch %d [0x%010llx] "
+				     "subc %d class 0x%04x mthd 0x%04x "
+				     "data 0x%08x\n",
+				 chid, inst, subc, class, mthd, data);
+		}
+		nouveau_handle_put(handle);
+		nv_wr32(priv, 0x400100, 0x00000010);
+		stat &= ~0x00000010;
+	}
+
+	if (stat & 0x00000020) {
+		nv_error(priv, "ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
+			     "class 0x%04x mthd 0x%04x data 0x%08x\n",
+			 chid, inst, subc, class, mthd, data);
+		nv_wr32(priv, 0x400100, 0x00000020);
+		stat &= ~0x00000020;
+	}
+
+	if (stat & 0x00100000) {
+		nv_error(priv, "DATA_ERROR [");
+		nouveau_enum_print(nv50_data_error_names, code);
+		printk("] ch %d [0x%010llx] subc %d class 0x%04x "
+		       "mthd 0x%04x data 0x%08x\n",
+		       chid, inst, subc, class, mthd, data);
+		nv_wr32(priv, 0x400100, 0x00100000);
+		stat &= ~0x00100000;
+	}
+
+	if (stat & 0x00200000) {
+		nve0_graph_trap_isr(priv, chid, inst);
+		nv_wr32(priv, 0x400100, 0x00200000);
+		stat &= ~0x00200000;
+	}
+
+	if (stat & 0x00080000) {
+		nve0_graph_ctxctl_isr(priv);
+		nv_wr32(priv, 0x400100, 0x00080000);
+		stat &= ~0x00080000;
+	}
+
+	if (stat) {
+		nv_error(priv, "unknown stat 0x%08x\n", stat);
+		nv_wr32(priv, 0x400100, stat);
+	}
+
+	nv_wr32(priv, 0x400500, 0x00010001);
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nve0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nvc0_graph_priv *priv;
+	int ret, i;
+
+	ret = nouveau_graph_create(parent, engine, oclass, false, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x18001000;
+	nv_subdev(priv)->intr = nve0_graph_intr;
+	nv_engine(priv)->cclass = &nve0_graph_cclass;
+	nv_engine(priv)->sclass = nve0_graph_sclass;
+
+	if (nouveau_boolopt(device->cfgopt, "NvGrUseFW", false)) {
+		nv_info(priv, "using external firmware\n");
+		if (nvc0_graph_ctor_fw(priv, "fuc409c", &priv->fuc409c) ||
+		    nvc0_graph_ctor_fw(priv, "fuc409d", &priv->fuc409d) ||
+		    nvc0_graph_ctor_fw(priv, "fuc41ac", &priv->fuc41ac) ||
+		    nvc0_graph_ctor_fw(priv, "fuc41ad", &priv->fuc41ad))
+			return -EINVAL;
+		priv->firmware = true;
+	}
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b4);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 256, 0, &priv->unk4188b8);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < 0x1000; i += 4) {
+		nv_wo32(priv->unk4188b4, i, 0x00000010);
+		nv_wo32(priv->unk4188b8, i, 0x00000010);
+	}
+
+	priv->gpc_nr =  nv_rd32(priv, 0x409604) & 0x0000001f;
+	priv->rop_nr = (nv_rd32(priv, 0x409604) & 0x001f0000) >> 16;
+	for (i = 0; i < priv->gpc_nr; i++) {
+		priv->tpc_nr[i] = nv_rd32(priv, GPC_UNIT(i, 0x2608));
+		priv->tpc_total += priv->tpc_nr[i];
+	}
+
+	switch (nv_device(priv)->chipset) {
+	case 0xe4:
+		if (priv->tpc_total == 8)
+			priv->magic_not_rop_nr = 3;
+		else
+		if (priv->tpc_total == 7)
+			priv->magic_not_rop_nr = 1;
+		break;
+	case 0xe7:
+		priv->magic_not_rop_nr = 1;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static void
+nve0_graph_init_obj418880(struct nvc0_graph_priv *priv)
+{
+	int i;
+
+	nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
+	nv_wr32(priv, GPC_BCAST(0x08a4), 0x00000000);
+	for (i = 0; i < 4; i++)
+		nv_wr32(priv, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
+	nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
+	nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
+}
+
+static void
+nve0_graph_init_regs(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x400080, 0x003083c2);
+	nv_wr32(priv, 0x400088, 0x0001ffe7);
+	nv_wr32(priv, 0x40008c, 0x00000000);
+	nv_wr32(priv, 0x400090, 0x00000030);
+	nv_wr32(priv, 0x40013c, 0x003901f7);
+	nv_wr32(priv, 0x400140, 0x00000100);
+	nv_wr32(priv, 0x400144, 0x00000000);
+	nv_wr32(priv, 0x400148, 0x00000110);
+	nv_wr32(priv, 0x400138, 0x00000000);
+	nv_wr32(priv, 0x400130, 0x00000000);
+	nv_wr32(priv, 0x400134, 0x00000000);
+	nv_wr32(priv, 0x400124, 0x00000002);
+}
+
+static void
+nve0_graph_init_units(struct nvc0_graph_priv *priv)
+{
+	nv_wr32(priv, 0x409ffc, 0x00000000);
+	nv_wr32(priv, 0x409c14, 0x00003e3e);
+	nv_wr32(priv, 0x409c24, 0x000f0000);
+
+	nv_wr32(priv, 0x404000, 0xc0000000);
+	nv_wr32(priv, 0x404600, 0xc0000000);
+	nv_wr32(priv, 0x408030, 0xc0000000);
+	nv_wr32(priv, 0x404490, 0xc0000000);
+	nv_wr32(priv, 0x406018, 0xc0000000);
+	nv_wr32(priv, 0x407020, 0xc0000000);
+	nv_wr32(priv, 0x405840, 0xc0000000);
+	nv_wr32(priv, 0x405844, 0x00ffffff);
+
+	nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
+	nv_mask(priv, 0x419eb4, 0x00001000, 0x00001000);
+
+}
+
+static void
+nve0_graph_init_gpc_0(struct nvc0_graph_priv *priv)
+{
+	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
+	u32 data[TPC_MAX / 8];
+	u8  tpcnr[GPC_MAX];
+	int i, gpc, tpc;
+
+	nv_wr32(priv, GPC_UNIT(0, 0x3018), 0x00000001);
+
+	memset(data, 0x00, sizeof(data));
+	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+	for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
+		do {
+			gpc = (gpc + 1) % priv->gpc_nr;
+		} while (!tpcnr[gpc]);
+		tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
+
+		data[i / 8] |= tpc << ((i % 8) * 4);
+	}
+
+	nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
+	nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
+	nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
+	nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
+
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
+						  priv->tpc_nr[gpc]);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
+	}
+
+	nv_wr32(priv, GPC_BCAST(0x1bd4), magicgpc918);
+	nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
+}
+
+static void
+nve0_graph_init_gpc_1(struct nvc0_graph_priv *priv)
+{
+	int gpc, tpc;
+
+	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+		nv_wr32(priv, GPC_UNIT(gpc, 0x3038), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+		for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
+			nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
+		}
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
+		nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
+	}
+}
+
+static void
+nve0_graph_init_rop(struct nvc0_graph_priv *priv)
+{
+	int rop;
+
+	for (rop = 0; rop < priv->rop_nr; rop++) {
+		nv_wr32(priv, ROP_UNIT(rop, 0x144), 0xc0000000);
+		nv_wr32(priv, ROP_UNIT(rop, 0x070), 0xc0000000);
+		nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
+		nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
+	}
+}
+
+static int
+nve0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
+{
+	u32 r000260;
+	int i;
+
+	if (priv->firmware) {
+		/* load fuc microcode */
+		r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+		nvc0_graph_init_fw(priv, 0x409000, &priv->fuc409c, &priv->fuc409d);
+		nvc0_graph_init_fw(priv, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
+		nv_wr32(priv, 0x000260, r000260);
+
+		/* start both of them running */
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x41a10c, 0x00000000);
+		nv_wr32(priv, 0x40910c, 0x00000000);
+		nv_wr32(priv, 0x41a100, 0x00000002);
+		nv_wr32(priv, 0x409100, 0x00000002);
+		if (!nv_wait(priv, 0x409800, 0x00000001, 0x00000001))
+			nv_error(priv, "0x409800 wait failed\n");
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x7fffffff);
+		nv_wr32(priv, 0x409504, 0x00000021);
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000010);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x10 timeout\n");
+			return -EBUSY;
+		}
+		priv->size = nv_rd32(priv, 0x409800);
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000016);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x16 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409840, 0xffffffff);
+		nv_wr32(priv, 0x409500, 0x00000000);
+		nv_wr32(priv, 0x409504, 0x00000025);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x25 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409800, 0x00000000);
+		nv_wr32(priv, 0x409500, 0x00000001);
+		nv_wr32(priv, 0x409504, 0x00000030);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x30 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409810, 0xb00095c8);
+		nv_wr32(priv, 0x409800, 0x00000000);
+		nv_wr32(priv, 0x409500, 0x00000001);
+		nv_wr32(priv, 0x409504, 0x00000031);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x31 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409810, 0x00080420);
+		nv_wr32(priv, 0x409800, 0x00000000);
+		nv_wr32(priv, 0x409500, 0x00000001);
+		nv_wr32(priv, 0x409504, 0x00000032);
+		if (!nv_wait_ne(priv, 0x409800, 0xffffffff, 0x00000000)) {
+			nv_error(priv, "fuc09 req 0x32 timeout\n");
+			return -EBUSY;
+		}
+
+		nv_wr32(priv, 0x409614, 0x00000070);
+		nv_wr32(priv, 0x409614, 0x00000770);
+		nv_wr32(priv, 0x40802c, 0x00000001);
+
+		if (priv->data == NULL) {
+			int ret = nve0_grctx_generate(priv);
+			if (ret) {
+				nv_error(priv, "failed to construct context\n");
+				return ret;
+			}
+		}
+
+		return 0;
+	}
+
+	/* load HUB microcode */
+	r000260 = nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
+	nv_wr32(priv, 0x4091c0, 0x01000000);
+	for (i = 0; i < sizeof(nve0_grhub_data) / 4; i++)
+		nv_wr32(priv, 0x4091c4, nve0_grhub_data[i]);
+
+	nv_wr32(priv, 0x409180, 0x01000000);
+	for (i = 0; i < sizeof(nve0_grhub_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x409188, i >> 6);
+		nv_wr32(priv, 0x409184, nve0_grhub_code[i]);
+	}
+
+	/* load GPC microcode */
+	nv_wr32(priv, 0x41a1c0, 0x01000000);
+	for (i = 0; i < sizeof(nve0_grgpc_data) / 4; i++)
+		nv_wr32(priv, 0x41a1c4, nve0_grgpc_data[i]);
+
+	nv_wr32(priv, 0x41a180, 0x01000000);
+	for (i = 0; i < sizeof(nve0_grgpc_code) / 4; i++) {
+		if ((i & 0x3f) == 0)
+			nv_wr32(priv, 0x41a188, i >> 6);
+		nv_wr32(priv, 0x41a184, nve0_grgpc_code[i]);
+	}
+	nv_wr32(priv, 0x000260, r000260);
+
+	/* start HUB ucode running, it'll init the GPCs */
+	nv_wr32(priv, 0x409800, nv_device(priv)->chipset);
+	nv_wr32(priv, 0x40910c, 0x00000000);
+	nv_wr32(priv, 0x409100, 0x00000002);
+	if (!nv_wait(priv, 0x409800, 0x80000000, 0x80000000)) {
+		nv_error(priv, "HUB_INIT timed out\n");
+		nvc0_graph_ctxctl_debug(priv);
+		return -EBUSY;
+	}
+
+	priv->size = nv_rd32(priv, 0x409804);
+	if (priv->data == NULL) {
+		int ret = nve0_grctx_generate(priv);
+		if (ret) {
+			nv_error(priv, "failed to construct context\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+nve0_graph_init(struct nouveau_object *object)
+{
+	struct nvc0_graph_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_graph_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nve0_graph_init_obj418880(priv);
+	nve0_graph_init_regs(priv);
+	nve0_graph_init_gpc_0(priv);
+
+	nv_wr32(priv, 0x400500, 0x00010001);
+	nv_wr32(priv, 0x400100, 0xffffffff);
+	nv_wr32(priv, 0x40013c, 0xffffffff);
+
+	nve0_graph_init_units(priv);
+	nve0_graph_init_gpc_1(priv);
+	nve0_graph_init_rop(priv);
+
+	nv_wr32(priv, 0x400108, 0xffffffff);
+	nv_wr32(priv, 0x400138, 0xffffffff);
+	nv_wr32(priv, 0x400118, 0xffffffff);
+	nv_wr32(priv, 0x400130, 0xffffffff);
+	nv_wr32(priv, 0x40011c, 0xffffffff);
+	nv_wr32(priv, 0x400134, 0xffffffff);
+	nv_wr32(priv, 0x400054, 0x34ce3464);
+
+	ret = nve0_graph_init_ctxctl(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+struct nouveau_oclass
+nve0_graph_oclass = {
+	.handle = NV_ENGINE(GR, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_graph_ctor,
+		.dtor = nvc0_graph_dtor,
+		.init = nve0_graph_init,
+		.fini = _nouveau_graph_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/regs.h b/drivers/gpu/drm/nouveau/core/engine/graph/regs.h
new file mode 100644
index 00000000000..9c715a25cec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/regs.h
@@ -0,0 +1,269 @@
+#ifndef __NOUVEAU_GRAPH_REGS_H__
+#define __NOUVEAU_GRAPH_REGS_H__
+
+#define NV04_PGRAPH_DEBUG_0                                0x00400080
+#define NV04_PGRAPH_DEBUG_1                                0x00400084
+#define NV04_PGRAPH_DEBUG_2                                0x00400088
+#define NV04_PGRAPH_DEBUG_3                                0x0040008c
+#define NV10_PGRAPH_DEBUG_4                                0x00400090
+#define NV03_PGRAPH_INTR                                   0x00400100
+#define NV03_PGRAPH_NSTATUS                                0x00400104
+#    define NV04_PGRAPH_NSTATUS_STATE_IN_USE                  (1<<11)
+#    define NV04_PGRAPH_NSTATUS_INVALID_STATE                 (1<<12)
+#    define NV04_PGRAPH_NSTATUS_BAD_ARGUMENT                  (1<<13)
+#    define NV04_PGRAPH_NSTATUS_PROTECTION_FAULT              (1<<14)
+#    define NV10_PGRAPH_NSTATUS_STATE_IN_USE                  (1<<23)
+#    define NV10_PGRAPH_NSTATUS_INVALID_STATE                 (1<<24)
+#    define NV10_PGRAPH_NSTATUS_BAD_ARGUMENT                  (1<<25)
+#    define NV10_PGRAPH_NSTATUS_PROTECTION_FAULT              (1<<26)
+#define NV03_PGRAPH_NSOURCE                                0x00400108
+#    define NV03_PGRAPH_NSOURCE_NOTIFICATION                   (1<<0)
+#    define NV03_PGRAPH_NSOURCE_DATA_ERROR                     (1<<1)
+#    define NV03_PGRAPH_NSOURCE_PROTECTION_ERROR               (1<<2)
+#    define NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION                (1<<3)
+#    define NV03_PGRAPH_NSOURCE_LIMIT_COLOR                    (1<<4)
+#    define NV03_PGRAPH_NSOURCE_LIMIT_ZETA                     (1<<5)
+#    define NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD                   (1<<6)
+#    define NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION               (1<<7)
+#    define NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION               (1<<8)
+#    define NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION               (1<<9)
+#    define NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION               (1<<10)
+#    define NV03_PGRAPH_NSOURCE_STATE_INVALID                 (1<<11)
+#    define NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY                 (1<<12)
+#    define NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE                 (1<<13)
+#    define NV03_PGRAPH_NSOURCE_METHOD_CNT                    (1<<14)
+#    define NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION              (1<<15)
+#    define NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION            (1<<16)
+#    define NV03_PGRAPH_NSOURCE_DMA_WIDTH_A                   (1<<17)
+#    define NV03_PGRAPH_NSOURCE_DMA_WIDTH_B                   (1<<18)
+#define NV03_PGRAPH_INTR_EN                                0x00400140
+#define NV40_PGRAPH_INTR_EN                                0x0040013C
+#    define NV_PGRAPH_INTR_NOTIFY                              (1<<0)
+#    define NV_PGRAPH_INTR_MISSING_HW                          (1<<4)
+#    define NV_PGRAPH_INTR_CONTEXT_SWITCH                     (1<<12)
+#    define NV_PGRAPH_INTR_BUFFER_NOTIFY                      (1<<16)
+#    define NV_PGRAPH_INTR_ERROR                              (1<<20)
+#define NV10_PGRAPH_CTX_CONTROL                            0x00400144
+#define NV10_PGRAPH_CTX_USER                               0x00400148
+#define NV10_PGRAPH_CTX_SWITCH(i)                         (0x0040014C + 0x4*(i))
+#define NV04_PGRAPH_CTX_SWITCH1                            0x00400160
+#define NV10_PGRAPH_CTX_CACHE(i, j)                       (0x00400160	\
+							   + 0x4*(i) + 0x20*(j))
+#define NV04_PGRAPH_CTX_SWITCH2                            0x00400164
+#define NV04_PGRAPH_CTX_SWITCH3                            0x00400168
+#define NV04_PGRAPH_CTX_SWITCH4                            0x0040016C
+#define NV04_PGRAPH_CTX_CONTROL                            0x00400170
+#define NV04_PGRAPH_CTX_USER                               0x00400174
+#define NV04_PGRAPH_CTX_CACHE1                             0x00400180
+#define NV03_PGRAPH_CTX_CONTROL                            0x00400190
+#define NV03_PGRAPH_CTX_USER                               0x00400194
+#define NV04_PGRAPH_CTX_CACHE2                             0x004001A0
+#define NV04_PGRAPH_CTX_CACHE3                             0x004001C0
+#define NV04_PGRAPH_CTX_CACHE4                             0x004001E0
+#define NV40_PGRAPH_CTXCTL_0304                            0x00400304
+#define NV40_PGRAPH_CTXCTL_0304_XFER_CTX                   0x00000001
+#define NV40_PGRAPH_CTXCTL_UCODE_STAT                      0x00400308
+#define NV40_PGRAPH_CTXCTL_UCODE_STAT_IP_MASK              0xff000000
+#define NV40_PGRAPH_CTXCTL_UCODE_STAT_IP_SHIFT                     24
+#define NV40_PGRAPH_CTXCTL_UCODE_STAT_OP_MASK              0x00ffffff
+#define NV40_PGRAPH_CTXCTL_0310                            0x00400310
+#define NV40_PGRAPH_CTXCTL_0310_XFER_SAVE                  0x00000020
+#define NV40_PGRAPH_CTXCTL_0310_XFER_LOAD                  0x00000040
+#define NV40_PGRAPH_CTXCTL_030C                            0x0040030c
+#define NV40_PGRAPH_CTXCTL_UCODE_INDEX                     0x00400324
+#define NV40_PGRAPH_CTXCTL_UCODE_DATA                      0x00400328
+#define NV40_PGRAPH_CTXCTL_CUR                             0x0040032c
+#define NV40_PGRAPH_CTXCTL_CUR_LOADED                      0x01000000
+#define NV40_PGRAPH_CTXCTL_CUR_INSTANCE                    0x000FFFFF
+#define NV40_PGRAPH_CTXCTL_NEXT                            0x00400330
+#define NV40_PGRAPH_CTXCTL_NEXT_INSTANCE                   0x000fffff
+#define NV50_PGRAPH_CTXCTL_CUR                             0x0040032c
+#define NV50_PGRAPH_CTXCTL_CUR_LOADED                      0x80000000
+#define NV50_PGRAPH_CTXCTL_CUR_INSTANCE                    0x00ffffff
+#define NV50_PGRAPH_CTXCTL_NEXT                            0x00400330
+#define NV50_PGRAPH_CTXCTL_NEXT_INSTANCE                   0x00ffffff
+#define NV03_PGRAPH_ABS_X_RAM                              0x00400400
+#define NV03_PGRAPH_ABS_Y_RAM                              0x00400480
+#define NV03_PGRAPH_X_MISC                                 0x00400500
+#define NV03_PGRAPH_Y_MISC                                 0x00400504
+#define NV04_PGRAPH_VALID1                                 0x00400508
+#define NV04_PGRAPH_SOURCE_COLOR                           0x0040050C
+#define NV04_PGRAPH_MISC24_0                               0x00400510
+#define NV03_PGRAPH_XY_LOGIC_MISC0                         0x00400514
+#define NV03_PGRAPH_XY_LOGIC_MISC1                         0x00400518
+#define NV03_PGRAPH_XY_LOGIC_MISC2                         0x0040051C
+#define NV03_PGRAPH_XY_LOGIC_MISC3                         0x00400520
+#define NV03_PGRAPH_CLIPX_0                                0x00400524
+#define NV03_PGRAPH_CLIPX_1                                0x00400528
+#define NV03_PGRAPH_CLIPY_0                                0x0040052C
+#define NV03_PGRAPH_CLIPY_1                                0x00400530
+#define NV03_PGRAPH_ABS_ICLIP_XMAX                         0x00400534
+#define NV03_PGRAPH_ABS_ICLIP_YMAX                         0x00400538
+#define NV03_PGRAPH_ABS_UCLIP_XMIN                         0x0040053C
+#define NV03_PGRAPH_ABS_UCLIP_YMIN                         0x00400540
+#define NV03_PGRAPH_ABS_UCLIP_XMAX                         0x00400544
+#define NV03_PGRAPH_ABS_UCLIP_YMAX                         0x00400548
+#define NV03_PGRAPH_ABS_UCLIPA_XMIN                        0x00400560
+#define NV03_PGRAPH_ABS_UCLIPA_YMIN                        0x00400564
+#define NV03_PGRAPH_ABS_UCLIPA_XMAX                        0x00400568
+#define NV03_PGRAPH_ABS_UCLIPA_YMAX                        0x0040056C
+#define NV04_PGRAPH_MISC24_1                               0x00400570
+#define NV04_PGRAPH_MISC24_2                               0x00400574
+#define NV04_PGRAPH_VALID2                                 0x00400578
+#define NV04_PGRAPH_PASSTHRU_0                             0x0040057C
+#define NV04_PGRAPH_PASSTHRU_1                             0x00400580
+#define NV04_PGRAPH_PASSTHRU_2                             0x00400584
+#define NV10_PGRAPH_DIMX_TEXTURE                           0x00400588
+#define NV10_PGRAPH_WDIMX_TEXTURE                          0x0040058C
+#define NV04_PGRAPH_COMBINE_0_ALPHA                        0x00400590
+#define NV04_PGRAPH_COMBINE_0_COLOR                        0x00400594
+#define NV04_PGRAPH_COMBINE_1_ALPHA                        0x00400598
+#define NV04_PGRAPH_COMBINE_1_COLOR                        0x0040059C
+#define NV04_PGRAPH_FORMAT_0                               0x004005A8
+#define NV04_PGRAPH_FORMAT_1                               0x004005AC
+#define NV04_PGRAPH_FILTER_0                               0x004005B0
+#define NV04_PGRAPH_FILTER_1                               0x004005B4
+#define NV03_PGRAPH_MONO_COLOR0                            0x00400600
+#define NV04_PGRAPH_ROP3                                   0x00400604
+#define NV04_PGRAPH_BETA_AND                               0x00400608
+#define NV04_PGRAPH_BETA_PREMULT                           0x0040060C
+#define NV04_PGRAPH_LIMIT_VIOL_PIX                         0x00400610
+#define NV04_PGRAPH_FORMATS                                0x00400618
+#define NV10_PGRAPH_DEBUG_2                                0x00400620
+#define NV04_PGRAPH_BOFFSET0                               0x00400640
+#define NV04_PGRAPH_BOFFSET1                               0x00400644
+#define NV04_PGRAPH_BOFFSET2                               0x00400648
+#define NV04_PGRAPH_BOFFSET3                               0x0040064C
+#define NV04_PGRAPH_BOFFSET4                               0x00400650
+#define NV04_PGRAPH_BOFFSET5                               0x00400654
+#define NV04_PGRAPH_BBASE0                                 0x00400658
+#define NV04_PGRAPH_BBASE1                                 0x0040065C
+#define NV04_PGRAPH_BBASE2                                 0x00400660
+#define NV04_PGRAPH_BBASE3                                 0x00400664
+#define NV04_PGRAPH_BBASE4                                 0x00400668
+#define NV04_PGRAPH_BBASE5                                 0x0040066C
+#define NV04_PGRAPH_BPITCH0                                0x00400670
+#define NV04_PGRAPH_BPITCH1                                0x00400674
+#define NV04_PGRAPH_BPITCH2                                0x00400678
+#define NV04_PGRAPH_BPITCH3                                0x0040067C
+#define NV04_PGRAPH_BPITCH4                                0x00400680
+#define NV04_PGRAPH_BLIMIT0                                0x00400684
+#define NV04_PGRAPH_BLIMIT1                                0x00400688
+#define NV04_PGRAPH_BLIMIT2                                0x0040068C
+#define NV04_PGRAPH_BLIMIT3                                0x00400690
+#define NV04_PGRAPH_BLIMIT4                                0x00400694
+#define NV04_PGRAPH_BLIMIT5                                0x00400698
+#define NV04_PGRAPH_BSWIZZLE2                              0x0040069C
+#define NV04_PGRAPH_BSWIZZLE5                              0x004006A0
+#define NV03_PGRAPH_STATUS                                 0x004006B0
+#define NV04_PGRAPH_STATUS                                 0x00400700
+#    define NV40_PGRAPH_STATUS_SYNC_STALL                  0x00004000
+#define NV04_PGRAPH_TRAPPED_ADDR                           0x00400704
+#define NV04_PGRAPH_TRAPPED_DATA                           0x00400708
+#define NV04_PGRAPH_SURFACE                                0x0040070C
+#define NV10_PGRAPH_TRAPPED_DATA_HIGH                      0x0040070C
+#define NV04_PGRAPH_STATE                                  0x00400710
+#define NV10_PGRAPH_SURFACE                                0x00400710
+#define NV04_PGRAPH_NOTIFY                                 0x00400714
+#define NV10_PGRAPH_STATE                                  0x00400714
+#define NV10_PGRAPH_NOTIFY                                 0x00400718
+
+#define NV04_PGRAPH_FIFO                                   0x00400720
+
+#define NV04_PGRAPH_BPIXEL                                 0x00400724
+#define NV10_PGRAPH_RDI_INDEX                              0x00400750
+#define NV04_PGRAPH_FFINTFC_ST2                            0x00400754
+#define NV10_PGRAPH_RDI_DATA                               0x00400754
+#define NV04_PGRAPH_DMA_PITCH                              0x00400760
+#define NV10_PGRAPH_FFINTFC_FIFO_PTR                       0x00400760
+#define NV04_PGRAPH_DVD_COLORFMT                           0x00400764
+#define NV10_PGRAPH_FFINTFC_ST2                            0x00400764
+#define NV04_PGRAPH_SCALED_FORMAT                          0x00400768
+#define NV10_PGRAPH_FFINTFC_ST2_DL                         0x00400768
+#define NV10_PGRAPH_FFINTFC_ST2_DH                         0x0040076c
+#define NV10_PGRAPH_DMA_PITCH                              0x00400770
+#define NV10_PGRAPH_DVD_COLORFMT                           0x00400774
+#define NV10_PGRAPH_SCALED_FORMAT                          0x00400778
+#define NV20_PGRAPH_CHANNEL_CTX_TABLE                      0x00400780
+#define NV20_PGRAPH_CHANNEL_CTX_POINTER                    0x00400784
+#define NV20_PGRAPH_CHANNEL_CTX_XFER                       0x00400788
+#define NV20_PGRAPH_CHANNEL_CTX_XFER_LOAD                  0x00000001
+#define NV20_PGRAPH_CHANNEL_CTX_XFER_SAVE                  0x00000002
+#define NV04_PGRAPH_PATT_COLOR0                            0x00400800
+#define NV04_PGRAPH_PATT_COLOR1                            0x00400804
+#define NV04_PGRAPH_PATTERN                                0x00400808
+#define NV04_PGRAPH_PATTERN_SHAPE                          0x00400810
+#define NV04_PGRAPH_CHROMA                                 0x00400814
+#define NV04_PGRAPH_CONTROL0                               0x00400818
+#define NV04_PGRAPH_CONTROL1                               0x0040081C
+#define NV04_PGRAPH_CONTROL2                               0x00400820
+#define NV04_PGRAPH_BLEND                                  0x00400824
+#define NV04_PGRAPH_STORED_FMT                             0x00400830
+#define NV04_PGRAPH_PATT_COLORRAM                          0x00400900
+#define NV20_PGRAPH_TILE(i)                                (0x00400900 + (i*16))
+#define NV20_PGRAPH_TLIMIT(i)                              (0x00400904 + (i*16))
+#define NV20_PGRAPH_TSIZE(i)                               (0x00400908 + (i*16))
+#define NV20_PGRAPH_TSTATUS(i)                             (0x0040090C + (i*16))
+#define NV20_PGRAPH_ZCOMP(i)                               (0x00400980 + 4*(i))
+#define NV10_PGRAPH_TILE(i)                                (0x00400B00 + (i*16))
+#define NV10_PGRAPH_TLIMIT(i)                              (0x00400B04 + (i*16))
+#define NV10_PGRAPH_TSIZE(i)                               (0x00400B08 + (i*16))
+#define NV10_PGRAPH_TSTATUS(i)                             (0x00400B0C + (i*16))
+#define NV04_PGRAPH_U_RAM                                  0x00400D00
+#define NV47_PGRAPH_TILE(i)                                (0x00400D00 + (i*16))
+#define NV47_PGRAPH_TLIMIT(i)                              (0x00400D04 + (i*16))
+#define NV47_PGRAPH_TSIZE(i)                               (0x00400D08 + (i*16))
+#define NV47_PGRAPH_TSTATUS(i)                             (0x00400D0C + (i*16))
+#define NV04_PGRAPH_V_RAM                                  0x00400D40
+#define NV04_PGRAPH_W_RAM                                  0x00400D80
+#define NV10_PGRAPH_COMBINER0_IN_ALPHA                     0x00400E40
+#define NV10_PGRAPH_COMBINER1_IN_ALPHA                     0x00400E44
+#define NV10_PGRAPH_COMBINER0_IN_RGB                       0x00400E48
+#define NV10_PGRAPH_COMBINER1_IN_RGB                       0x00400E4C
+#define NV10_PGRAPH_COMBINER_COLOR0                        0x00400E50
+#define NV10_PGRAPH_COMBINER_COLOR1                        0x00400E54
+#define NV10_PGRAPH_COMBINER0_OUT_ALPHA                    0x00400E58
+#define NV10_PGRAPH_COMBINER1_OUT_ALPHA                    0x00400E5C
+#define NV10_PGRAPH_COMBINER0_OUT_RGB                      0x00400E60
+#define NV10_PGRAPH_COMBINER1_OUT_RGB                      0x00400E64
+#define NV10_PGRAPH_COMBINER_FINAL0                        0x00400E68
+#define NV10_PGRAPH_COMBINER_FINAL1                        0x00400E6C
+#define NV10_PGRAPH_WINDOWCLIP_HORIZONTAL                  0x00400F00
+#define NV10_PGRAPH_WINDOWCLIP_VERTICAL                    0x00400F20
+#define NV10_PGRAPH_XFMODE0                                0x00400F40
+#define NV10_PGRAPH_XFMODE1                                0x00400F44
+#define NV10_PGRAPH_GLOBALSTATE0                           0x00400F48
+#define NV10_PGRAPH_GLOBALSTATE1                           0x00400F4C
+#define NV10_PGRAPH_PIPE_ADDRESS                           0x00400F50
+#define NV10_PGRAPH_PIPE_DATA                              0x00400F54
+#define NV04_PGRAPH_DMA_START_0                            0x00401000
+#define NV04_PGRAPH_DMA_START_1                            0x00401004
+#define NV04_PGRAPH_DMA_LENGTH                             0x00401008
+#define NV04_PGRAPH_DMA_MISC                               0x0040100C
+#define NV04_PGRAPH_DMA_DATA_0                             0x00401020
+#define NV04_PGRAPH_DMA_DATA_1                             0x00401024
+#define NV04_PGRAPH_DMA_RM                                 0x00401030
+#define NV04_PGRAPH_DMA_A_XLATE_INST                       0x00401040
+#define NV04_PGRAPH_DMA_A_CONTROL                          0x00401044
+#define NV04_PGRAPH_DMA_A_LIMIT                            0x00401048
+#define NV04_PGRAPH_DMA_A_TLB_PTE                          0x0040104C
+#define NV04_PGRAPH_DMA_A_TLB_TAG                          0x00401050
+#define NV04_PGRAPH_DMA_A_ADJ_OFFSET                       0x00401054
+#define NV04_PGRAPH_DMA_A_OFFSET                           0x00401058
+#define NV04_PGRAPH_DMA_A_SIZE                             0x0040105C
+#define NV04_PGRAPH_DMA_A_Y_SIZE                           0x00401060
+#define NV04_PGRAPH_DMA_B_XLATE_INST                       0x00401080
+#define NV04_PGRAPH_DMA_B_CONTROL                          0x00401084
+#define NV04_PGRAPH_DMA_B_LIMIT                            0x00401088
+#define NV04_PGRAPH_DMA_B_TLB_PTE                          0x0040108C
+#define NV04_PGRAPH_DMA_B_TLB_TAG                          0x00401090
+#define NV04_PGRAPH_DMA_B_ADJ_OFFSET                       0x00401094
+#define NV04_PGRAPH_DMA_B_OFFSET                           0x00401098
+#define NV04_PGRAPH_DMA_B_SIZE                             0x0040109C
+#define NV04_PGRAPH_DMA_B_Y_SIZE                           0x004010A0
+#define NV40_PGRAPH_TILE1(i)                               (0x00406900 + (i*16))
+#define NV40_PGRAPH_TLIMIT1(i)                             (0x00406904 + (i*16))
+#define NV40_PGRAPH_TSIZE1(i)                              (0x00406908 + (i*16))
+#define NV40_PGRAPH_TSTATUS1(i)                            (0x0040690C + (i*16))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
new file mode 100644
index 00000000000..1f394a2629e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
@@ -0,0 +1,308 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/handle.h>
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+#include <subdev/instmem.h>
+
+#include <engine/fifo.h>
+#include <engine/mpeg.h>
+#include <engine/graph/nv40.h>
+
+struct nv31_mpeg_priv {
+	struct nouveau_mpeg base;
+	atomic_t refcount;
+};
+
+struct nv31_mpeg_chan {
+	struct nouveau_object base;
+};
+
+/*******************************************************************************
+ * MPEG object classes
+ ******************************************************************************/
+
+static int
+nv31_mpeg_object_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    20, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+	nv_wo32(obj, 0x0c, 0x00000000);
+	return 0;
+}
+
+static int
+nv31_mpeg_mthd_dma(struct nouveau_object *object, u32 mthd, void *arg, u32 len)
+{
+	struct nouveau_instmem *imem = nouveau_instmem(object);
+	struct nv31_mpeg_priv *priv = (void *)object->engine;
+	u32 inst = *(u32 *)arg << 4;
+	u32 dma0 = nv_ro32(imem, inst + 0);
+	u32 dma1 = nv_ro32(imem, inst + 4);
+	u32 dma2 = nv_ro32(imem, inst + 8);
+	u32 base = (dma2 & 0xfffff000) | (dma0 >> 20);
+	u32 size = dma1 + 1;
+
+	/* only allow linear DMA objects */
+	if (!(dma0 & 0x00002000))
+		return -EINVAL;
+
+	if (mthd == 0x0190) {
+		/* DMA_CMD */
+		nv_mask(priv, 0x00b300, 0x00030000, (dma0 & 0x00030000));
+		nv_wr32(priv, 0x00b334, base);
+		nv_wr32(priv, 0x00b324, size);
+	} else
+	if (mthd == 0x01a0) {
+		/* DMA_DATA */
+		nv_mask(priv, 0x00b300, 0x000c0000, (dma0 & 0x00030000) << 2);
+		nv_wr32(priv, 0x00b360, base);
+		nv_wr32(priv, 0x00b364, size);
+	} else {
+		/* DMA_IMAGE, VRAM only */
+		if (dma0 & 0x000c0000)
+			return -EINVAL;
+
+		nv_wr32(priv, 0x00b370, base);
+		nv_wr32(priv, 0x00b374, size);
+	}
+
+	return 0;
+}
+
+static struct nouveau_ofuncs
+nv31_mpeg_ofuncs = {
+	.ctor = nv31_mpeg_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_omthds
+nv31_mpeg_omthds[] = {
+	{ 0x0190, nv31_mpeg_mthd_dma },
+	{ 0x01a0, nv31_mpeg_mthd_dma },
+	{ 0x01b0, nv31_mpeg_mthd_dma },
+	{}
+};
+
+struct nouveau_oclass
+nv31_mpeg_sclass[] = {
+	{ 0x3174, &nv31_mpeg_ofuncs, nv31_mpeg_omthds },
+	{}
+};
+
+/*******************************************************************************
+ * PMPEG context
+ ******************************************************************************/
+
+static int
+nv31_mpeg_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nv31_mpeg_priv *priv = (void *)engine;
+	struct nv31_mpeg_chan *chan;
+	int ret;
+
+	if (!atomic_add_unless(&priv->refcount, 1, 1))
+		return -EBUSY;
+
+	ret = nouveau_object_create(parent, engine, oclass, 0, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nv31_mpeg_context_dtor(struct nouveau_object *object)
+{
+	struct nv31_mpeg_priv *priv = (void *)object->engine;
+	struct nv31_mpeg_chan *chan = (void *)object;
+	atomic_dec(&priv->refcount);
+	nouveau_object_destroy(&chan->base);
+}
+
+static struct nouveau_oclass
+nv31_mpeg_cclass = {
+	.handle = NV_ENGCTX(MPEG, 0x31),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv31_mpeg_context_ctor,
+		.dtor = nv31_mpeg_context_dtor,
+		.init = nouveau_object_init,
+		.fini = nouveau_object_fini,
+	},
+};
+
+/*******************************************************************************
+ * PMPEG engine/subdev functions
+ ******************************************************************************/
+
+void
+nv31_mpeg_tile_prog(struct nouveau_engine *engine, int i)
+{
+	struct nouveau_fb_tile *tile = &nouveau_fb(engine)->tile.region[i];
+	struct nv31_mpeg_priv *priv = (void *)engine;
+
+	nv_wr32(priv, 0x00b008 + (i * 0x10), tile->pitch);
+	nv_wr32(priv, 0x00b004 + (i * 0x10), tile->limit);
+	nv_wr32(priv, 0x00b000 + (i * 0x10), tile->addr);
+}
+
+void
+nv31_mpeg_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+	struct nouveau_engine *engine = nv_engine(subdev);
+	struct nouveau_object *engctx;
+	struct nouveau_handle *handle;
+	struct nv31_mpeg_priv *priv = (void *)subdev;
+	u32 inst = nv_rd32(priv, 0x00b318) & 0x000fffff;
+	u32 stat = nv_rd32(priv, 0x00b100);
+	u32 type = nv_rd32(priv, 0x00b230);
+	u32 mthd = nv_rd32(priv, 0x00b234);
+	u32 data = nv_rd32(priv, 0x00b238);
+	u32 show = stat;
+	int chid;
+
+	engctx = nouveau_engctx_get(engine, inst);
+	chid   = pfifo->chid(pfifo, engctx);
+
+	if (stat & 0x01000000) {
+		/* happens on initial binding of the object */
+		if (type == 0x00000020 && mthd == 0x0000) {
+			nv_mask(priv, 0x00b308, 0x00000000, 0x00000000);
+			show &= ~0x01000000;
+		}
+
+		if (type == 0x00000010) {
+			handle = nouveau_handle_get_class(engctx, 0x3174);
+			if (handle && !nv_call(handle->object, mthd, data))
+				show &= ~0x01000000;
+			nouveau_handle_put(handle);
+		}
+	}
+
+	nv_wr32(priv, 0x00b100, stat);
+	nv_wr32(priv, 0x00b230, 0x00000001);
+
+	if (show) {
+		nv_error(priv, "ch %d [0x%08x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+			 chid, inst << 4, stat, type, mthd, data);
+	}
+
+	nouveau_engctx_put(engctx);
+}
+
+static int
+nv31_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv31_mpeg_priv *priv;
+	int ret;
+
+	ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000002;
+	nv_subdev(priv)->intr = nv31_mpeg_intr;
+	nv_engine(priv)->cclass = &nv31_mpeg_cclass;
+	nv_engine(priv)->sclass = nv31_mpeg_sclass;
+	nv_engine(priv)->tile_prog = nv31_mpeg_tile_prog;
+	return 0;
+}
+
+int
+nv31_mpeg_init(struct nouveau_object *object)
+{
+	struct nouveau_engine *engine = nv_engine(object->engine);
+	struct nv31_mpeg_priv *priv = (void *)engine;
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	int ret, i;
+
+	ret = nouveau_mpeg_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* VPE init */
+	nv_wr32(priv, 0x00b0e0, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
+	nv_wr32(priv, 0x00b0e8, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
+
+	for (i = 0; i < pfb->tile.regions; i++)
+		engine->tile_prog(engine, i);
+
+	/* PMPEG init */
+	nv_wr32(priv, 0x00b32c, 0x00000000);
+	nv_wr32(priv, 0x00b314, 0x00000100);
+	nv_wr32(priv, 0x00b220, nv44_graph_class(priv) ? 0x00000044 : 0x00000031);
+	nv_wr32(priv, 0x00b300, 0x02001ec1);
+	nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
+
+	nv_wr32(priv, 0x00b100, 0xffffffff);
+	nv_wr32(priv, 0x00b140, 0xffffffff);
+
+	if (!nv_wait(priv, 0x00b200, 0x00000001, 0x00000000)) {
+		nv_error(priv, "timeout 0x%08x\n", nv_rd32(priv, 0x00b200));
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv31_mpeg_oclass = {
+	.handle = NV_ENGINE(MPEG, 0x31),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv31_mpeg_ctor,
+		.dtor = _nouveau_mpeg_dtor,
+		.init = nv31_mpeg_init,
+		.fini = _nouveau_mpeg_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
new file mode 100644
index 00000000000..12418574efe
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+#include <subdev/instmem.h>
+
+#include <engine/mpeg.h>
+#include <engine/graph/nv40.h>
+
+struct nv40_mpeg_priv {
+	struct nouveau_mpeg base;
+};
+
+struct nv40_mpeg_chan {
+	struct nouveau_mpeg base;
+};
+
+/*******************************************************************************
+ * PMPEG context
+ ******************************************************************************/
+
+static int
+nv40_mpeg_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nv40_mpeg_chan *chan;
+	int ret;
+
+	ret = nouveau_mpeg_context_create(parent, engine, oclass, NULL,
+					  264 * 4, 16,
+					  NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv40_mpeg_context_fini(struct nouveau_object *object, bool suspend)
+{
+
+	struct nv40_mpeg_priv *priv = (void *)object->engine;
+	struct nv40_mpeg_chan *chan = (void *)object;
+	u32 inst = 0x80000000 | nv_gpuobj(chan)->addr >> 4;
+
+	nv_mask(priv, 0x00b32c, 0x00000001, 0x00000000);
+	if (nv_rd32(priv, 0x00b318) == inst)
+		nv_mask(priv, 0x00b318, 0x80000000, 0x00000000);
+	nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv40_mpeg_cclass = {
+	.handle = NV_ENGCTX(MPEG, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_mpeg_context_ctor,
+		.dtor = _nouveau_mpeg_context_dtor,
+		.init = _nouveau_mpeg_context_init,
+		.fini = nv40_mpeg_context_fini,
+		.rd32 = _nouveau_mpeg_context_rd32,
+		.wr32 = _nouveau_mpeg_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PMPEG engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv40_mpeg_intr(struct nouveau_subdev *subdev)
+{
+	struct nv40_mpeg_priv *priv = (void *)subdev;
+	u32 stat;
+
+	if ((stat = nv_rd32(priv, 0x00b100)))
+		nv31_mpeg_intr(subdev);
+
+	if ((stat = nv_rd32(priv, 0x00b800))) {
+		nv_error(priv, "PMSRCH 0x%08x\n", stat);
+		nv_wr32(priv, 0x00b800, stat);
+	}
+}
+
+static int
+nv40_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv40_mpeg_priv *priv;
+	int ret;
+
+	ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000002;
+	nv_subdev(priv)->intr = nv40_mpeg_intr;
+	nv_engine(priv)->cclass = &nv40_mpeg_cclass;
+	nv_engine(priv)->sclass = nv31_mpeg_sclass;
+	nv_engine(priv)->tile_prog = nv31_mpeg_tile_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv40_mpeg_oclass = {
+	.handle = NV_ENGINE(MPEG, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_mpeg_ctor,
+		.dtor = _nouveau_mpeg_dtor,
+		.init = nv31_mpeg_init,
+		.fini = _nouveau_mpeg_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c
new file mode 100644
index 00000000000..8678a9996d5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv50.c
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+#include <subdev/timer.h>
+
+#include <engine/mpeg.h>
+
+struct nv50_mpeg_priv {
+	struct nouveau_mpeg base;
+};
+
+struct nv50_mpeg_chan {
+	struct nouveau_mpeg_chan base;
+};
+
+/*******************************************************************************
+ * MPEG object classes
+ ******************************************************************************/
+
+static int
+nv50_mpeg_object_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nouveau_gpuobj *obj;
+	int ret;
+
+	ret = nouveau_gpuobj_create(parent, engine, oclass, 0, parent,
+				    16, 16, 0, &obj);
+	*pobject = nv_object(obj);
+	if (ret)
+		return ret;
+
+	nv_wo32(obj, 0x00, nv_mclass(obj));
+	nv_wo32(obj, 0x04, 0x00000000);
+	nv_wo32(obj, 0x08, 0x00000000);
+	nv_wo32(obj, 0x0c, 0x00000000);
+	return 0;
+}
+
+struct nouveau_ofuncs
+nv50_mpeg_ofuncs = {
+	.ctor = nv50_mpeg_object_ctor,
+	.dtor = _nouveau_gpuobj_dtor,
+	.init = _nouveau_gpuobj_init,
+	.fini = _nouveau_gpuobj_fini,
+	.rd32 = _nouveau_gpuobj_rd32,
+	.wr32 = _nouveau_gpuobj_wr32,
+};
+
+static struct nouveau_oclass
+nv50_mpeg_sclass[] = {
+	{ 0x3174, &nv50_mpeg_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PMPEG context
+ ******************************************************************************/
+
+int
+nv50_mpeg_context_ctor(struct nouveau_object *parent,
+		       struct nouveau_object *engine,
+		       struct nouveau_oclass *oclass, void *data, u32 size,
+		       struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = nouveau_bar(parent);
+	struct nv50_mpeg_chan *chan;
+	int ret;
+
+	ret = nouveau_mpeg_context_create(parent, engine, oclass, NULL, 128 * 4,
+					  0, NVOBJ_FLAG_ZERO_ALLOC, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	nv_wo32(chan, 0x0070, 0x00801ec1);
+	nv_wo32(chan, 0x007c, 0x0000037c);
+	bar->flush(bar);
+	return 0;
+}
+
+static struct nouveau_oclass
+nv50_mpeg_cclass = {
+	.handle = NV_ENGCTX(MPEG, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_mpeg_context_ctor,
+		.dtor = _nouveau_mpeg_context_dtor,
+		.init = _nouveau_mpeg_context_init,
+		.fini = _nouveau_mpeg_context_fini,
+		.rd32 = _nouveau_mpeg_context_rd32,
+		.wr32 = _nouveau_mpeg_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PMPEG engine/subdev functions
+ ******************************************************************************/
+
+int
+nv50_mpeg_tlb_flush(struct nouveau_engine *engine)
+{
+	nv50_vm_flush_engine(&engine->base, 0x08);
+	return 0;
+}
+
+void
+nv50_mpeg_intr(struct nouveau_subdev *subdev)
+{
+	struct nv50_mpeg_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, 0x00b100);
+	u32 type = nv_rd32(priv, 0x00b230);
+	u32 mthd = nv_rd32(priv, 0x00b234);
+	u32 data = nv_rd32(priv, 0x00b238);
+	u32 show = stat;
+
+	if (stat & 0x01000000) {
+		/* happens on initial binding of the object */
+		if (type == 0x00000020 && mthd == 0x0000) {
+			nv_wr32(priv, 0x00b308, 0x00000100);
+			show &= ~0x01000000;
+		}
+	}
+
+	if (show) {
+		nv_info(priv, "0x%08x 0x%08x 0x%08x 0x%08x\n",
+			stat, type, mthd, data);
+	}
+
+	nv_wr32(priv, 0x00b100, stat);
+	nv_wr32(priv, 0x00b230, 0x00000001);
+	nv50_fb_trap(nouveau_fb(priv), 1);
+}
+
+static void
+nv50_vpe_intr(struct nouveau_subdev *subdev)
+{
+	struct nv50_mpeg_priv *priv = (void *)subdev;
+
+	if (nv_rd32(priv, 0x00b100))
+		nv50_mpeg_intr(subdev);
+
+	if (nv_rd32(priv, 0x00b800)) {
+		u32 stat = nv_rd32(priv, 0x00b800);
+		nv_info(priv, "PMSRCH: 0x%08x\n", stat);
+		nv_wr32(priv, 0xb800, stat);
+	}
+}
+
+static int
+nv50_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv50_mpeg_priv *priv;
+	int ret;
+
+	ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00400002;
+	nv_subdev(priv)->intr = nv50_vpe_intr;
+	nv_engine(priv)->cclass = &nv50_mpeg_cclass;
+	nv_engine(priv)->sclass = nv50_mpeg_sclass;
+	nv_engine(priv)->tlb_flush = nv50_mpeg_tlb_flush;
+	return 0;
+}
+
+int
+nv50_mpeg_init(struct nouveau_object *object)
+{
+	struct nv50_mpeg_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_mpeg_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x00b32c, 0x00000000);
+	nv_wr32(priv, 0x00b314, 0x00000100);
+	nv_wr32(priv, 0x00b0e0, 0x0000001a);
+
+	nv_wr32(priv, 0x00b220, 0x00000044);
+	nv_wr32(priv, 0x00b300, 0x00801ec1);
+	nv_wr32(priv, 0x00b390, 0x00000000);
+	nv_wr32(priv, 0x00b394, 0x00000000);
+	nv_wr32(priv, 0x00b398, 0x00000000);
+	nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
+
+	nv_wr32(priv, 0x00b100, 0xffffffff);
+	nv_wr32(priv, 0x00b140, 0xffffffff);
+
+	if (!nv_wait(priv, 0x00b200, 0x00000001, 0x00000000)) {
+		nv_error(priv, "timeout 0x%08x\n", nv_rd32(priv, 0x00b200));
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_mpeg_oclass = {
+	.handle = NV_ENGINE(MPEG, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_mpeg_ctor,
+		.dtor = _nouveau_mpeg_dtor,
+		.init = nv50_mpeg_init,
+		.fini = _nouveau_mpeg_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c
new file mode 100644
index 00000000000..8f805b44d59
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv84.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+#include <subdev/timer.h>
+
+#include <engine/mpeg.h>
+
+struct nv84_mpeg_priv {
+	struct nouveau_mpeg base;
+};
+
+struct nv84_mpeg_chan {
+	struct nouveau_mpeg_chan base;
+};
+
+/*******************************************************************************
+ * MPEG object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv84_mpeg_sclass[] = {
+	{ 0x8274, &nv50_mpeg_ofuncs },
+	{}
+};
+
+/*******************************************************************************
+ * PMPEG context
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv84_mpeg_cclass = {
+	.handle = NV_ENGCTX(MPEG, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_mpeg_context_ctor,
+		.dtor = _nouveau_mpeg_context_dtor,
+		.init = _nouveau_mpeg_context_init,
+		.fini = _nouveau_mpeg_context_fini,
+		.rd32 = _nouveau_mpeg_context_rd32,
+		.wr32 = _nouveau_mpeg_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PMPEG engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv84_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv84_mpeg_priv *priv;
+	int ret;
+
+	ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00000002;
+	nv_subdev(priv)->intr = nv50_mpeg_intr;
+	nv_engine(priv)->cclass = &nv84_mpeg_cclass;
+	nv_engine(priv)->sclass = nv84_mpeg_sclass;
+	nv_engine(priv)->tlb_flush = nv50_mpeg_tlb_flush;
+	return 0;
+}
+
+struct nouveau_oclass
+nv84_mpeg_oclass = {
+	.handle = NV_ENGINE(MPEG, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_mpeg_ctor,
+		.dtor = _nouveau_mpeg_dtor,
+		.init = nv50_mpeg_init,
+		.fini = _nouveau_mpeg_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/ppp/nv98.c b/drivers/gpu/drm/nouveau/core/engine/ppp/nv98.c
new file mode 100644
index 00000000000..50e7e0da198
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/ppp/nv98.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/ppp.h>
+
+struct nv98_ppp_priv {
+	struct nouveau_ppp base;
+};
+
+struct nv98_ppp_chan {
+	struct nouveau_ppp_chan base;
+};
+
+/*******************************************************************************
+ * PPP object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv98_ppp_sclass[] = {
+	{},
+};
+
+/*******************************************************************************
+ * PPPP context
+ ******************************************************************************/
+
+static int
+nv98_ppp_context_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nv98_ppp_chan *priv;
+	int ret;
+
+	ret = nouveau_ppp_context_create(parent, engine, oclass, NULL,
+					 0, 0, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nv98_ppp_context_dtor(struct nouveau_object *object)
+{
+	struct nv98_ppp_chan *priv = (void *)object;
+	nouveau_ppp_context_destroy(&priv->base);
+}
+
+static int
+nv98_ppp_context_init(struct nouveau_object *object)
+{
+	struct nv98_ppp_chan *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_ppp_context_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv98_ppp_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv98_ppp_chan *priv = (void *)object;
+	return nouveau_ppp_context_fini(&priv->base, suspend);
+}
+
+static struct nouveau_oclass
+nv98_ppp_cclass = {
+	.handle = NV_ENGCTX(PPP, 0x98),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv98_ppp_context_ctor,
+		.dtor = nv98_ppp_context_dtor,
+		.init = nv98_ppp_context_init,
+		.fini = nv98_ppp_context_fini,
+		.rd32 = _nouveau_ppp_context_rd32,
+		.wr32 = _nouveau_ppp_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PPPP engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv98_ppp_intr(struct nouveau_subdev *subdev)
+{
+}
+
+static int
+nv98_ppp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv98_ppp_priv *priv;
+	int ret;
+
+	ret = nouveau_ppp_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x00400002;
+	nv_subdev(priv)->intr = nv98_ppp_intr;
+	nv_engine(priv)->cclass = &nv98_ppp_cclass;
+	nv_engine(priv)->sclass = nv98_ppp_sclass;
+	return 0;
+}
+
+static void
+nv98_ppp_dtor(struct nouveau_object *object)
+{
+	struct nv98_ppp_priv *priv = (void *)object;
+	nouveau_ppp_destroy(&priv->base);
+}
+
+static int
+nv98_ppp_init(struct nouveau_object *object)
+{
+	struct nv98_ppp_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_ppp_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv98_ppp_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv98_ppp_priv *priv = (void *)object;
+	return nouveau_ppp_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv98_ppp_oclass = {
+	.handle = NV_ENGINE(PPP, 0x98),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv98_ppp_ctor,
+		.dtor = nv98_ppp_dtor,
+		.init = nv98_ppp_init,
+		.fini = nv98_ppp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv04.c b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
new file mode 100644
index 00000000000..3ca4c3aa90b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/software.h>
+#include <engine/fifo.h>
+
+struct nv04_software_priv {
+	struct nouveau_software base;
+};
+
+struct nv04_software_chan {
+	struct nouveau_software_chan base;
+};
+
+/*******************************************************************************
+ * software object classes
+ ******************************************************************************/
+
+static int
+nv04_software_set_ref(struct nouveau_object *object, u32 mthd,
+		      void *data, u32 size)
+{
+	struct nouveau_object *channel = (void *)nv_engctx(object->parent);
+	struct nouveau_fifo_chan *fifo = (void *)channel->parent;
+	atomic_set(&fifo->refcnt, *(u32*)data);
+	return 0;
+}
+
+static int
+nv04_software_flip(struct nouveau_object *object, u32 mthd,
+		   void *args, u32 size)
+{
+	struct nv04_software_chan *chan = (void *)nv_engctx(object->parent);
+	if (chan->base.flip)
+		return chan->base.flip(chan->base.flip_data);
+	return -EINVAL;
+}
+
+static struct nouveau_omthds
+nv04_software_omthds[] = {
+	{ 0x0150, nv04_software_set_ref },
+	{ 0x0500, nv04_software_flip },
+	{}
+};
+
+static struct nouveau_oclass
+nv04_software_sclass[] = {
+	{ 0x006e, &nouveau_object_ofuncs, nv04_software_omthds },
+	{}
+};
+
+/*******************************************************************************
+ * software context
+ ******************************************************************************/
+
+static int
+nv04_software_context_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nv04_software_chan *chan;
+	int ret;
+
+	ret = nouveau_software_context_create(parent, engine, oclass, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv04_software_cclass = {
+	.handle = NV_ENGCTX(SW, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_software_context_ctor,
+		.dtor = _nouveau_software_context_dtor,
+		.init = _nouveau_software_context_init,
+		.fini = _nouveau_software_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * software engine/subdev functions
+ ******************************************************************************/
+
+void
+nv04_software_intr(struct nouveau_subdev *subdev)
+{
+	nv_mask(subdev, 0x000100, 0x80000000, 0x00000000);
+}
+
+static int
+nv04_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv04_software_priv *priv;
+	int ret;
+
+	ret = nouveau_software_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->cclass = &nv04_software_cclass;
+	nv_engine(priv)->sclass = nv04_software_sclass;
+	nv_subdev(priv)->intr = nv04_software_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_software_oclass = {
+	.handle = NV_ENGINE(SW, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_software_ctor,
+		.dtor = _nouveau_software_dtor,
+		.init = _nouveau_software_init,
+		.fini = _nouveau_software_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv10.c b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
new file mode 100644
index 00000000000..6e699afbfdb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/software.h>
+
+struct nv10_software_priv {
+	struct nouveau_software base;
+};
+
+struct nv10_software_chan {
+	struct nouveau_software_chan base;
+};
+
+/*******************************************************************************
+ * software object classes
+ ******************************************************************************/
+
+static int
+nv10_software_flip(struct nouveau_object *object, u32 mthd,
+		   void *args, u32 size)
+{
+	struct nv10_software_chan *chan = (void *)nv_engctx(object->parent);
+	if (chan->base.flip)
+		return chan->base.flip(chan->base.flip_data);
+	return -EINVAL;
+}
+
+static struct nouveau_omthds
+nv10_software_omthds[] = {
+	{ 0x0500, nv10_software_flip },
+	{}
+};
+
+static struct nouveau_oclass
+nv10_software_sclass[] = {
+	{ 0x016e, &nouveau_object_ofuncs, nv10_software_omthds },
+	{}
+};
+
+/*******************************************************************************
+ * software context
+ ******************************************************************************/
+
+static int
+nv10_software_context_ctor(struct nouveau_object *parent,
+		      struct nouveau_object *engine,
+		      struct nouveau_oclass *oclass, void *data, u32 size,
+		      struct nouveau_object **pobject)
+{
+	struct nv10_software_chan *chan;
+	int ret;
+
+	ret = nouveau_software_context_create(parent, engine, oclass, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static struct nouveau_oclass
+nv10_software_cclass = {
+	.handle = NV_ENGCTX(SW, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_software_context_ctor,
+		.dtor = _nouveau_software_context_dtor,
+		.init = _nouveau_software_context_init,
+		.fini = _nouveau_software_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * software engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv10_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv10_software_priv *priv;
+	int ret;
+
+	ret = nouveau_software_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->cclass = &nv10_software_cclass;
+	nv_engine(priv)->sclass = nv10_software_sclass;
+	nv_subdev(priv)->intr = nv04_software_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nv10_software_oclass = {
+	.handle = NV_ENGINE(SW, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_software_ctor,
+		.dtor = _nouveau_software_dtor,
+		.init = _nouveau_software_init,
+		.fini = _nouveau_software_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv50.c b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
new file mode 100644
index 00000000000..a2edcd38544
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+#include <core/namedb.h>
+#include <core/handle.h>
+#include <core/gpuobj.h>
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+struct nv50_software_priv {
+	struct nouveau_software base;
+};
+
+struct nv50_software_chan {
+	struct nouveau_software_chan base;
+};
+
+/*******************************************************************************
+ * software object classes
+ ******************************************************************************/
+
+static int
+nv50_software_mthd_dma_vblsem(struct nouveau_object *object, u32 mthd,
+			      void *args, u32 size)
+{
+	struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+	struct nouveau_fifo_chan *fifo = (void *)nv_object(chan)->parent;
+	struct nouveau_handle *handle;
+	int ret = -EINVAL;
+
+	handle = nouveau_namedb_get(nv_namedb(fifo), *(u32 *)args);
+	if (!handle)
+		return -ENOENT;
+
+	if (nv_iclass(handle->object, NV_GPUOBJ_CLASS)) {
+		struct nouveau_gpuobj *gpuobj = nv_gpuobj(handle->object);
+		chan->base.vblank.ctxdma = gpuobj->node->offset >> 4;
+		ret = 0;
+	}
+	nouveau_namedb_put(handle);
+	return ret;
+}
+
+static int
+nv50_software_mthd_vblsem_offset(struct nouveau_object *object, u32 mthd,
+				 void *args, u32 size)
+{
+	struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+	chan->base.vblank.offset = *(u32 *)args;
+	return 0;
+}
+
+static int
+nv50_software_mthd_vblsem_value(struct nouveau_object *object, u32 mthd,
+				void *args, u32 size)
+{
+	struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+	chan->base.vblank.value = *(u32 *)args;
+	return 0;
+}
+
+static int
+nv50_software_mthd_vblsem_release(struct nouveau_object *object, u32 mthd,
+				  void *args, u32 size)
+{
+	struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+	struct nouveau_disp *disp = nouveau_disp(object);
+	unsigned long flags;
+	u32 crtc = *(u32 *)args;
+
+	if (crtc > 1)
+		return -EINVAL;
+
+	disp->vblank.get(disp->vblank.data, crtc);
+
+	spin_lock_irqsave(&disp->vblank.lock, flags);
+	list_add(&chan->base.vblank.head, &disp->vblank.list);
+	chan->base.vblank.crtc = crtc;
+	spin_unlock_irqrestore(&disp->vblank.lock, flags);
+	return 0;
+}
+
+static int
+nv50_software_mthd_flip(struct nouveau_object *object, u32 mthd,
+			void *args, u32 size)
+{
+	struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+	if (chan->base.flip)
+		return chan->base.flip(chan->base.flip_data);
+	return -EINVAL;
+}
+
+static struct nouveau_omthds
+nv50_software_omthds[] = {
+	{ 0x018c, nv50_software_mthd_dma_vblsem },
+	{ 0x0400, nv50_software_mthd_vblsem_offset },
+	{ 0x0404, nv50_software_mthd_vblsem_value },
+	{ 0x0408, nv50_software_mthd_vblsem_release },
+	{ 0x0500, nv50_software_mthd_flip },
+	{}
+};
+
+static struct nouveau_oclass
+nv50_software_sclass[] = {
+	{ 0x506e, &nouveau_object_ofuncs, nv50_software_omthds },
+	{}
+};
+
+/*******************************************************************************
+ * software context
+ ******************************************************************************/
+
+static int
+nv50_software_context_ctor(struct nouveau_object *parent,
+			   struct nouveau_object *engine,
+			   struct nouveau_oclass *oclass, void *data, u32 size,
+			   struct nouveau_object **pobject)
+{
+	struct nv50_software_chan *chan;
+	int ret;
+
+	ret = nouveau_software_context_create(parent, engine, oclass, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->base.vblank.channel = nv_gpuobj(parent->parent)->addr >> 12;
+	return 0;
+}
+
+static struct nouveau_oclass
+nv50_software_cclass = {
+	.handle = NV_ENGCTX(SW, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_software_context_ctor,
+		.dtor = _nouveau_software_context_dtor,
+		.init = _nouveau_software_context_init,
+		.fini = _nouveau_software_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * software engine/subdev functions
+ ******************************************************************************/
+
+static int
+nv50_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv50_software_priv *priv;
+	int ret;
+
+	ret = nouveau_software_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->cclass = &nv50_software_cclass;
+	nv_engine(priv)->sclass = nv50_software_sclass;
+	nv_subdev(priv)->intr = nv04_software_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_software_oclass = {
+	.handle = NV_ENGINE(SW, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_software_ctor,
+		.dtor = _nouveau_software_dtor,
+		.init = _nouveau_software_init,
+		.fini = _nouveau_software_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
new file mode 100644
index 00000000000..b7b0d7e330d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+struct nvc0_software_priv {
+	struct nouveau_software base;
+};
+
+struct nvc0_software_chan {
+	struct nouveau_software_chan base;
+};
+
+/*******************************************************************************
+ * software object classes
+ ******************************************************************************/
+
+static int
+nvc0_software_mthd_vblsem_offset(struct nouveau_object *object, u32 mthd,
+				 void *args, u32 size)
+{
+	struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
+	u64 data = *(u32 *)args;
+	if (mthd == 0x0400) {
+		chan->base.vblank.offset &= 0x00ffffffffULL;
+		chan->base.vblank.offset |= data << 32;
+	} else {
+		chan->base.vblank.offset &= 0xff00000000ULL;
+		chan->base.vblank.offset |= data;
+	}
+	return 0;
+}
+
+static int
+nvc0_software_mthd_vblsem_value(struct nouveau_object *object, u32 mthd,
+				void *args, u32 size)
+{
+	struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
+	chan->base.vblank.value = *(u32 *)args;
+	return 0;
+}
+
+static int
+nvc0_software_mthd_vblsem_release(struct nouveau_object *object, u32 mthd,
+				  void *args, u32 size)
+{
+	struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
+	struct nouveau_disp *disp = nouveau_disp(object);
+	unsigned long flags;
+	u32 crtc = *(u32 *)args;
+
+	if ((nv_device(object)->card_type < NV_E0 && crtc > 1) || crtc > 3)
+		return -EINVAL;
+
+	disp->vblank.get(disp->vblank.data, crtc);
+
+	spin_lock_irqsave(&disp->vblank.lock, flags);
+	list_add(&chan->base.vblank.head, &disp->vblank.list);
+	chan->base.vblank.crtc = crtc;
+	spin_unlock_irqrestore(&disp->vblank.lock, flags);
+	return 0;
+}
+
+static int
+nvc0_software_mthd_flip(struct nouveau_object *object, u32 mthd,
+			void *args, u32 size)
+{
+	struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
+	if (chan->base.flip)
+		return chan->base.flip(chan->base.flip_data);
+	return -EINVAL;
+}
+
+static struct nouveau_omthds
+nvc0_software_omthds[] = {
+	{ 0x0400, nvc0_software_mthd_vblsem_offset },
+	{ 0x0404, nvc0_software_mthd_vblsem_offset },
+	{ 0x0408, nvc0_software_mthd_vblsem_value },
+	{ 0x040c, nvc0_software_mthd_vblsem_release },
+	{ 0x0500, nvc0_software_mthd_flip },
+	{}
+};
+
+static struct nouveau_oclass
+nvc0_software_sclass[] = {
+	{ 0x906e, &nouveau_object_ofuncs, nvc0_software_omthds },
+	{}
+};
+
+/*******************************************************************************
+ * software context
+ ******************************************************************************/
+
+static int
+nvc0_software_context_ctor(struct nouveau_object *parent,
+			   struct nouveau_object *engine,
+			   struct nouveau_oclass *oclass, void *data, u32 size,
+			   struct nouveau_object **pobject)
+{
+	struct nvc0_software_chan *chan;
+	int ret;
+
+	ret = nouveau_software_context_create(parent, engine, oclass, &chan);
+	*pobject = nv_object(chan);
+	if (ret)
+		return ret;
+
+	chan->base.vblank.channel = nv_gpuobj(parent->parent)->addr >> 12;
+	return 0;
+}
+
+static struct nouveau_oclass
+nvc0_software_cclass = {
+	.handle = NV_ENGCTX(SW, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_software_context_ctor,
+		.dtor = _nouveau_software_context_dtor,
+		.init = _nouveau_software_context_init,
+		.fini = _nouveau_software_context_fini,
+	},
+};
+
+/*******************************************************************************
+ * software engine/subdev functions
+ ******************************************************************************/
+
+static int
+nvc0_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		   struct nouveau_oclass *oclass, void *data, u32 size,
+		   struct nouveau_object **pobject)
+{
+	struct nvc0_software_priv *priv;
+	int ret;
+
+	ret = nouveau_software_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_engine(priv)->cclass = &nvc0_software_cclass;
+	nv_engine(priv)->sclass = nvc0_software_sclass;
+	nv_subdev(priv)->intr = nv04_software_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_software_oclass = {
+	.handle = NV_ENGINE(SW, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_software_ctor,
+		.dtor = _nouveau_software_dtor,
+		.init = _nouveau_software_init,
+		.fini = _nouveau_software_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/vp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/vp/nv84.c
new file mode 100644
index 00000000000..dd23c80e540
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/vp/nv84.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/engctx.h>
+
+#include <engine/vp.h>
+
+struct nv84_vp_priv {
+	struct nouveau_vp base;
+};
+
+struct nv84_vp_chan {
+	struct nouveau_vp_chan base;
+};
+
+/*******************************************************************************
+ * VP object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+nv84_vp_sclass[] = {
+	{},
+};
+
+/*******************************************************************************
+ * PVP context
+ ******************************************************************************/
+
+static int
+nv84_vp_context_ctor(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, void *data, u32 size,
+		     struct nouveau_object **pobject)
+{
+	struct nv84_vp_chan *priv;
+	int ret;
+
+	ret = nouveau_vp_context_create(parent, engine, oclass, NULL,
+					0, 0, 0, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nv84_vp_context_dtor(struct nouveau_object *object)
+{
+	struct nv84_vp_chan *priv = (void *)object;
+	nouveau_vp_context_destroy(&priv->base);
+}
+
+static int
+nv84_vp_context_init(struct nouveau_object *object)
+{
+	struct nv84_vp_chan *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_vp_context_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv84_vp_context_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv84_vp_chan *priv = (void *)object;
+	return nouveau_vp_context_fini(&priv->base, suspend);
+}
+
+static struct nouveau_oclass
+nv84_vp_cclass = {
+	.handle = NV_ENGCTX(VP, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_vp_context_ctor,
+		.dtor = nv84_vp_context_dtor,
+		.init = nv84_vp_context_init,
+		.fini = nv84_vp_context_fini,
+		.rd32 = _nouveau_vp_context_rd32,
+		.wr32 = _nouveau_vp_context_wr32,
+	},
+};
+
+/*******************************************************************************
+ * PVP engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv84_vp_intr(struct nouveau_subdev *subdev)
+{
+}
+
+static int
+nv84_vp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv84_vp_priv *priv;
+	int ret;
+
+	ret = nouveau_vp_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->unit = 0x01020000;
+	nv_subdev(priv)->intr = nv84_vp_intr;
+	nv_engine(priv)->cclass = &nv84_vp_cclass;
+	nv_engine(priv)->sclass = nv84_vp_sclass;
+	return 0;
+}
+
+static void
+nv84_vp_dtor(struct nouveau_object *object)
+{
+	struct nv84_vp_priv *priv = (void *)object;
+	nouveau_vp_destroy(&priv->base);
+}
+
+static int
+nv84_vp_init(struct nouveau_object *object)
+{
+	struct nv84_vp_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_vp_init(&priv->base);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv84_vp_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv84_vp_priv *priv = (void *)object;
+	return nouveau_vp_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv84_vp_oclass = {
+	.handle = NV_ENGINE(VP, 0x84),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv84_vp_ctor,
+		.dtor = nv84_vp_dtor,
+		.init = nv84_vp_init,
+		.fini = nv84_vp_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/include/core/class.h b/drivers/gpu/drm/nouveau/core/include/core/class.h
new file mode 100644
index 00000000000..6180ae9800f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/class.h
@@ -0,0 +1,118 @@
+#ifndef __NOUVEAU_CLASS_H__
+#define __NOUVEAU_CLASS_H__
+
+/* Device class
+ *
+ * 0080: NV_DEVICE
+ */
+#define NV_DEVICE_CLASS                                              0x00000080
+
+#define NV_DEVICE_DISABLE_IDENTIFY                        0x0000000000000001ULL
+#define NV_DEVICE_DISABLE_MMIO                            0x0000000000000002ULL
+#define NV_DEVICE_DISABLE_VBIOS                           0x0000000000000004ULL
+#define NV_DEVICE_DISABLE_CORE                            0x0000000000000008ULL
+#define NV_DEVICE_DISABLE_DISP                            0x0000000000010000ULL
+#define NV_DEVICE_DISABLE_FIFO                            0x0000000000020000ULL
+#define NV_DEVICE_DISABLE_GRAPH                           0x0000000100000000ULL
+#define NV_DEVICE_DISABLE_MPEG                            0x0000000200000000ULL
+#define NV_DEVICE_DISABLE_ME                              0x0000000400000000ULL
+#define NV_DEVICE_DISABLE_VP                              0x0000000800000000ULL
+#define NV_DEVICE_DISABLE_CRYPT                           0x0000001000000000ULL
+#define NV_DEVICE_DISABLE_BSP                             0x0000002000000000ULL
+#define NV_DEVICE_DISABLE_PPP                             0x0000004000000000ULL
+#define NV_DEVICE_DISABLE_COPY0                           0x0000008000000000ULL
+#define NV_DEVICE_DISABLE_COPY1                           0x0000010000000000ULL
+#define NV_DEVICE_DISABLE_UNK1C1                          0x0000020000000000ULL
+
+struct nv_device_class {
+	u64 device;	/* device identifier, ~0 for client default */
+	u64 disable;	/* disable particular subsystems */
+	u64 debug0;	/* as above, but *internal* ids, and *NOT* ABI */
+};
+
+/* DMA object classes
+ *
+ * 0002: NV_DMA_FROM_MEMORY
+ * 0003: NV_DMA_TO_MEMORY
+ * 003d: NV_DMA_IN_MEMORY
+ */
+#define NV_DMA_FROM_MEMORY_CLASS                                     0x00000002
+#define NV_DMA_TO_MEMORY_CLASS                                       0x00000003
+#define NV_DMA_IN_MEMORY_CLASS                                       0x0000003d
+
+#define NV_DMA_TARGET_MASK                                           0x000000ff
+#define NV_DMA_TARGET_VM                                             0x00000000
+#define NV_DMA_TARGET_VRAM                                           0x00000001
+#define NV_DMA_TARGET_PCI                                            0x00000002
+#define NV_DMA_TARGET_PCI_US                                         0x00000003
+#define NV_DMA_TARGET_AGP                                            0x00000004
+#define NV_DMA_ACCESS_MASK                                           0x00000f00
+#define NV_DMA_ACCESS_VM                                             0x00000000
+#define NV_DMA_ACCESS_RD                                             0x00000100
+#define NV_DMA_ACCESS_WR                                             0x00000200
+#define NV_DMA_ACCESS_RDWR                                           0x00000300
+
+struct nv_dma_class {
+	u32 flags;
+	u32 pad0;
+	u64 start;
+	u64 limit;
+};
+
+/* DMA FIFO channel classes
+ *
+ * 006b: NV03_CHANNEL_DMA
+ * 006e: NV10_CHANNEL_DMA
+ * 176e: NV17_CHANNEL_DMA
+ * 406e: NV40_CHANNEL_DMA
+ * 506e: NV50_CHANNEL_DMA
+ * 826e: NV84_CHANNEL_DMA
+ */
+#define NV03_CHANNEL_DMA_CLASS                                       0x0000006b
+#define NV10_CHANNEL_DMA_CLASS                                       0x0000006e
+#define NV17_CHANNEL_DMA_CLASS                                       0x0000176e
+#define NV40_CHANNEL_DMA_CLASS                                       0x0000406e
+#define NV50_CHANNEL_DMA_CLASS                                       0x0000506e
+#define NV84_CHANNEL_DMA_CLASS                                       0x0000826e
+
+struct nv03_channel_dma_class {
+	u32 pushbuf;
+	u32 pad0;
+	u64 offset;
+};
+
+/* Indirect FIFO channel classes
+ *
+ * 506f: NV50_CHANNEL_IND
+ * 826f: NV84_CHANNEL_IND
+ * 906f: NVC0_CHANNEL_IND
+ * a06f: NVE0_CHANNEL_IND
+ */
+
+#define NV50_CHANNEL_IND_CLASS                                       0x0000506f
+#define NV84_CHANNEL_IND_CLASS                                       0x0000826f
+#define NVC0_CHANNEL_IND_CLASS                                       0x0000906f
+#define NVE0_CHANNEL_IND_CLASS                                       0x0000a06f
+
+struct nv50_channel_ind_class {
+	u32 pushbuf;
+	u32 ilength;
+	u64 ioffset;
+};
+
+#define NVE0_CHANNEL_IND_ENGINE_GR                                   0x00000001
+#define NVE0_CHANNEL_IND_ENGINE_VP                                   0x00000002
+#define NVE0_CHANNEL_IND_ENGINE_PPP                                  0x00000004
+#define NVE0_CHANNEL_IND_ENGINE_BSP                                  0x00000008
+#define NVE0_CHANNEL_IND_ENGINE_CE0                                  0x00000010
+#define NVE0_CHANNEL_IND_ENGINE_CE1                                  0x00000020
+#define NVE0_CHANNEL_IND_ENGINE_ENC                                  0x00000040
+
+struct nve0_channel_ind_class {
+	u32 pushbuf;
+	u32 ilength;
+	u64 ioffset;
+	u32 engine;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/client.h b/drivers/gpu/drm/nouveau/core/include/core/client.h
new file mode 100644
index 00000000000..0193532ceac
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/client.h
@@ -0,0 +1,42 @@
+#ifndef __NOUVEAU_CLIENT_H__
+#define __NOUVEAU_CLIENT_H__
+
+#include <core/namedb.h>
+
+struct nouveau_client {
+	struct nouveau_namedb base;
+	struct nouveau_handle *root;
+	struct nouveau_object *device;
+	char name[16];
+	u32 debug;
+	struct nouveau_vm *vm;
+};
+
+static inline struct nouveau_client *
+nv_client(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_CLIENT_CLASS)))
+		nv_assert("BAD CAST -> NvClient, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+static inline struct nouveau_client *
+nouveau_client(void *obj)
+{
+	struct nouveau_object *client = nv_object(obj);
+	while (client && !(nv_iclass(client, NV_CLIENT_CLASS)))
+		client = client->parent;
+	return (void *)client;
+}
+
+#define nouveau_client_create(n,c,oc,od,d)                                     \
+	nouveau_client_create_((n), (c), (oc), (od), sizeof(**d), (void **)d)
+
+int  nouveau_client_create_(const char *name, u64 device, const char *cfg,
+			    const char *dbg, int, void **);
+int  nouveau_client_init(struct nouveau_client *);
+int  nouveau_client_fini(struct nouveau_client *, bool suspend);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/debug.h b/drivers/gpu/drm/nouveau/core/include/core/debug.h
new file mode 100644
index 00000000000..9ea18dfcb4d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/debug.h
@@ -0,0 +1,13 @@
+#ifndef __NOUVEAU_DEBUG_H__
+#define __NOUVEAU_DEBUG_H__
+
+#define NV_DBG_FATAL    0
+#define NV_DBG_ERROR    1
+#define NV_DBG_WARN     2
+#define NV_DBG_INFO     3
+#define NV_DBG_DEBUG    4
+#define NV_DBG_TRACE    5
+#define NV_DBG_PARANOIA 6
+#define NV_DBG_SPAM     7
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
new file mode 100644
index 00000000000..e58b6f0984c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -0,0 +1,136 @@
+#ifndef __NOUVEAU_DEVICE_H__
+#define __NOUVEAU_DEVICE_H__
+
+#include <core/object.h>
+#include <core/subdev.h>
+#include <core/engine.h>
+
+enum nv_subdev_type {
+	NVDEV_SUBDEV_DEVICE,
+	NVDEV_SUBDEV_VBIOS,
+
+	/* All subdevs from DEVINIT to DEVINIT_LAST will be created before
+	 * *any* of them are initialised.  This subdev category is used
+	 * for any subdevs that the VBIOS init table parsing may call out
+	 * to during POST.
+	 */
+	NVDEV_SUBDEV_DEVINIT,
+	NVDEV_SUBDEV_GPIO,
+	NVDEV_SUBDEV_I2C,
+	NVDEV_SUBDEV_CLOCK,
+	NVDEV_SUBDEV_DEVINIT_LAST = NVDEV_SUBDEV_CLOCK,
+
+	/* This grouping of subdevs are initialised right after they've
+	 * been created, and are allowed to assume any subdevs in the
+	 * list above them exist and have been initialised.
+	 */
+	NVDEV_SUBDEV_MXM,
+	NVDEV_SUBDEV_MC,
+	NVDEV_SUBDEV_TIMER,
+	NVDEV_SUBDEV_FB,
+	NVDEV_SUBDEV_LTCG,
+	NVDEV_SUBDEV_IBUS,
+	NVDEV_SUBDEV_INSTMEM,
+	NVDEV_SUBDEV_VM,
+	NVDEV_SUBDEV_BAR,
+	NVDEV_SUBDEV_VOLT,
+	NVDEV_SUBDEV_THERM,
+
+	NVDEV_ENGINE_DMAOBJ,
+	NVDEV_ENGINE_FIFO,
+	NVDEV_ENGINE_SW,
+	NVDEV_ENGINE_GR,
+	NVDEV_ENGINE_MPEG,
+	NVDEV_ENGINE_ME,
+	NVDEV_ENGINE_VP,
+	NVDEV_ENGINE_CRYPT,
+	NVDEV_ENGINE_BSP,
+	NVDEV_ENGINE_PPP,
+	NVDEV_ENGINE_COPY0,
+	NVDEV_ENGINE_COPY1,
+	NVDEV_ENGINE_UNK1C1,
+	NVDEV_ENGINE_VENC,
+	NVDEV_ENGINE_DISP,
+
+	NVDEV_SUBDEV_NR,
+};
+
+struct nouveau_device {
+	struct nouveau_subdev base;
+	struct list_head head;
+
+	struct pci_dev *pdev;
+	u64 handle;
+
+	const char *cfgopt;
+	const char *dbgopt;
+	const char *name;
+	const char *cname;
+
+	enum {
+		NV_04    = 0x04,
+		NV_10    = 0x10,
+		NV_20    = 0x20,
+		NV_30    = 0x30,
+		NV_40    = 0x40,
+		NV_50    = 0x50,
+		NV_C0    = 0xc0,
+		NV_D0    = 0xd0,
+		NV_E0    = 0xe0,
+	} card_type;
+	u32 chipset;
+	u32 crystal;
+
+	struct nouveau_oclass *oclass[NVDEV_SUBDEV_NR];
+	struct nouveau_object *subdev[NVDEV_SUBDEV_NR];
+};
+
+static inline struct nouveau_device *
+nv_device(void *obj)
+{
+	struct nouveau_object *object = nv_object(obj);
+	struct nouveau_object *device = object;
+
+	if (device->engine)
+		device = device->engine;
+	if (device->parent)
+		device = device->parent;
+
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(device, NV_SUBDEV_CLASS) ||
+		     (nv_hclass(device) & 0xff) != NVDEV_SUBDEV_DEVICE)) {
+		nv_assert("BAD CAST -> NvDevice, 0x%08x 0x%08x",
+			  nv_hclass(object), nv_hclass(device));
+	}
+#endif
+
+	return (void *)device;
+}
+
+static inline struct nouveau_subdev *
+nouveau_subdev(void *obj, int sub)
+{
+	if (nv_device(obj)->subdev[sub])
+		return nv_subdev(nv_device(obj)->subdev[sub]);
+	return NULL;
+}
+
+static inline struct nouveau_engine *
+nouveau_engine(void *obj, int sub)
+{
+	struct nouveau_subdev *subdev = nouveau_subdev(obj, sub);
+	if (subdev && nv_iclass(subdev, NV_ENGINE_CLASS))
+		return nv_engine(subdev);
+	return NULL;
+}
+
+static inline bool
+nv_device_match(struct nouveau_object *object, u16 dev, u16 ven, u16 sub)
+{
+	struct nouveau_device *device = nv_device(object);
+	return device->pdev->device == dev &&
+	       device->pdev->subsystem_vendor == ven &&
+	       device->pdev->subsystem_device == sub;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/engctx.h b/drivers/gpu/drm/nouveau/core/include/core/engctx.h
new file mode 100644
index 00000000000..8a947b6872e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/engctx.h
@@ -0,0 +1,51 @@
+#ifndef __NOUVEAU_ENGCTX_H__
+#define __NOUVEAU_ENGCTX_H__
+
+#include <core/object.h>
+#include <core/gpuobj.h>
+
+#include <subdev/vm.h>
+
+#define NV_ENGCTX_(eng,var) (NV_ENGCTX_CLASS | ((var) << 8) | (eng))
+#define NV_ENGCTX(name,var)  NV_ENGCTX_(NVDEV_ENGINE_##name, (var))
+
+struct nouveau_engctx {
+	struct nouveau_gpuobj base;
+	struct nouveau_vma vma;
+	struct list_head head;
+	unsigned long save;
+	u64 addr;
+};
+
+static inline struct nouveau_engctx *
+nv_engctx(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_ENGCTX_CLASS)))
+		nv_assert("BAD CAST -> NvEngCtx, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+#define nouveau_engctx_create(p,e,c,g,s,a,f,d)                                 \
+	nouveau_engctx_create_((p), (e), (c), (g), (s), (a), (f),              \
+			       sizeof(**d), (void **)d)
+
+int  nouveau_engctx_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, struct nouveau_object *,
+			    u32 size, u32 align, u32 flags,
+			    int length, void **data);
+void nouveau_engctx_destroy(struct nouveau_engctx *);
+int  nouveau_engctx_init(struct nouveau_engctx *);
+int  nouveau_engctx_fini(struct nouveau_engctx *, bool suspend);
+
+void _nouveau_engctx_dtor(struct nouveau_object *);
+int  _nouveau_engctx_init(struct nouveau_object *);
+int  _nouveau_engctx_fini(struct nouveau_object *, bool suspend);
+#define _nouveau_engctx_rd32 _nouveau_gpuobj_rd32
+#define _nouveau_engctx_wr32 _nouveau_gpuobj_wr32
+
+struct nouveau_object *nouveau_engctx_get(struct nouveau_engine *, u64 addr);
+void nouveau_engctx_put(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/engine.h b/drivers/gpu/drm/nouveau/core/include/core/engine.h
new file mode 100644
index 00000000000..666d06de77e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/engine.h
@@ -0,0 +1,57 @@
+#ifndef __NOUVEAU_ENGINE_H__
+#define __NOUVEAU_ENGINE_H__
+
+#include <core/object.h>
+#include <core/subdev.h>
+
+#define NV_ENGINE_(eng,var) (NV_ENGINE_CLASS | ((var) << 8) | (eng))
+#define NV_ENGINE(name,var)  NV_ENGINE_(NVDEV_ENGINE_##name, (var))
+
+struct nouveau_engine {
+	struct nouveau_subdev base;
+	struct nouveau_oclass *cclass;
+	struct nouveau_oclass *sclass;
+
+	struct list_head contexts;
+	spinlock_t lock;
+
+	void (*tile_prog)(struct nouveau_engine *, int region);
+	int  (*tlb_flush)(struct nouveau_engine *);
+};
+
+static inline struct nouveau_engine *
+nv_engine(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_ENGINE_CLASS)))
+		nv_assert("BAD CAST -> NvEngine, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+static inline int
+nv_engidx(struct nouveau_object *object)
+{
+	return nv_subidx(object);
+}
+
+#define nouveau_engine_create(p,e,c,d,i,f,r)                                   \
+	nouveau_engine_create_((p), (e), (c), (d), (i), (f),                   \
+			       sizeof(**r),(void **)r)
+
+#define nouveau_engine_destroy(p)                                              \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_engine_init(p)                                                 \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_engine_fini(p,s)                                               \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+int nouveau_engine_create_(struct nouveau_object *, struct nouveau_object *,
+			   struct nouveau_oclass *, bool, const char *,
+			   const char *, int, void **);
+
+#define _nouveau_engine_dtor _nouveau_subdev_dtor
+#define _nouveau_engine_init _nouveau_subdev_init
+#define _nouveau_engine_fini _nouveau_subdev_fini
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/enum.h b/drivers/gpu/drm/nouveau/core/include/core/enum.h
new file mode 100644
index 00000000000..e7b1e181943
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/enum.h
@@ -0,0 +1,23 @@
+#ifndef __NOUVEAU_ENUM_H__
+#define __NOUVEAU_ENUM_H__
+
+struct nouveau_enum {
+	u32 value;
+	const char *name;
+	const void *data;
+};
+
+const struct nouveau_enum *
+nouveau_enum_find(const struct nouveau_enum *, u32 value);
+
+void
+nouveau_enum_print(const struct nouveau_enum *en, u32 value);
+
+struct nouveau_bitfield {
+	u32 mask;
+	const char *name;
+};
+
+void nouveau_bitfield_print(const struct nouveau_bitfield *, u32 value);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/gpuobj.h b/drivers/gpu/drm/nouveau/core/include/core/gpuobj.h
new file mode 100644
index 00000000000..6eaff79377a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/gpuobj.h
@@ -0,0 +1,71 @@
+#ifndef __NOUVEAU_GPUOBJ_H__
+#define __NOUVEAU_GPUOBJ_H__
+
+#include <core/object.h>
+#include <core/device.h>
+#include <core/parent.h>
+#include <core/mm.h>
+
+struct nouveau_vma;
+struct nouveau_vm;
+
+#define NVOBJ_FLAG_ZERO_ALLOC 0x00000001
+#define NVOBJ_FLAG_ZERO_FREE  0x00000002
+#define NVOBJ_FLAG_HEAP       0x00000004
+
+struct nouveau_gpuobj {
+	struct nouveau_object base;
+	struct nouveau_object *parent;
+	struct nouveau_mm_node *node;
+	struct nouveau_mm heap;
+
+	u32 flags;
+	u64 addr;
+	u32 size;
+};
+
+static inline struct nouveau_gpuobj *
+nv_gpuobj(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_GPUOBJ_CLASS)))
+		nv_assert("BAD CAST -> NvGpuObj, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+#define nouveau_gpuobj_create(p,e,c,v,g,s,a,f,d)                               \
+	nouveau_gpuobj_create_((p), (e), (c), (v), (g), (s), (a), (f),         \
+			       sizeof(**d), (void **)d)
+#define nouveau_gpuobj_init(p) nouveau_object_init(&(p)->base)
+#define nouveau_gpuobj_fini(p,s) nouveau_object_fini(&(p)->base, (s))
+int  nouveau_gpuobj_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, u32 pclass,
+			    struct nouveau_object *, u32 size, u32 align,
+			    u32 flags, int length, void **);
+void nouveau_gpuobj_destroy(struct nouveau_gpuobj *);
+
+int nouveau_gpuobj_new(struct nouveau_object *, struct nouveau_object *,
+		       u32 size, u32 align, u32 flags,
+		       struct nouveau_gpuobj **);
+int nouveau_gpuobj_dup(struct nouveau_object *, struct nouveau_gpuobj *,
+		       struct nouveau_gpuobj **);
+
+int nouveau_gpuobj_map(struct nouveau_gpuobj *, u32 acc, struct nouveau_vma *);
+int nouveau_gpuobj_map_vm(struct nouveau_gpuobj *, struct nouveau_vm *,
+			  u32 access, struct nouveau_vma *);
+void nouveau_gpuobj_unmap(struct nouveau_vma *);
+
+static inline void
+nouveau_gpuobj_ref(struct nouveau_gpuobj *obj, struct nouveau_gpuobj **ref)
+{
+	nouveau_object_ref(&obj->base, (struct nouveau_object **)ref);
+}
+
+void _nouveau_gpuobj_dtor(struct nouveau_object *);
+int  _nouveau_gpuobj_init(struct nouveau_object *);
+int  _nouveau_gpuobj_fini(struct nouveau_object *, bool);
+u32  _nouveau_gpuobj_rd32(struct nouveau_object *, u32);
+void _nouveau_gpuobj_wr32(struct nouveau_object *, u32, u32);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/handle.h b/drivers/gpu/drm/nouveau/core/include/core/handle.h
new file mode 100644
index 00000000000..363674cdf8a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/handle.h
@@ -0,0 +1,31 @@
+#ifndef __NOUVEAU_HANDLE_H__
+#define __NOUVEAU_HANDLE_H__
+
+struct nouveau_handle {
+	struct nouveau_namedb *namedb;
+	struct list_head node;
+
+	struct list_head head;
+	struct list_head tree;
+	u32 name;
+	u32 priv;
+
+	struct nouveau_handle *parent;
+	struct nouveau_object *object;
+};
+
+int  nouveau_handle_create(struct nouveau_object *, u32 parent, u32 handle,
+			   struct nouveau_object *, struct nouveau_handle **);
+void nouveau_handle_destroy(struct nouveau_handle *);
+int  nouveau_handle_init(struct nouveau_handle *);
+int  nouveau_handle_fini(struct nouveau_handle *, bool suspend);
+
+struct nouveau_object *
+nouveau_handle_ref(struct nouveau_object *, u32 name);
+
+struct nouveau_handle *nouveau_handle_get_class(struct nouveau_object *, u16);
+struct nouveau_handle *nouveau_handle_get_vinst(struct nouveau_object *, u64);
+struct nouveau_handle *nouveau_handle_get_cinst(struct nouveau_object *, u32);
+void nouveau_handle_put(struct nouveau_handle *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/math.h b/drivers/gpu/drm/nouveau/core/include/core/math.h
new file mode 100644
index 00000000000..f808131c5cd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/math.h
@@ -0,0 +1,16 @@
+#ifndef __NOUVEAU_MATH_H__
+#define __NOUVEAU_MATH_H__
+
+static inline int
+log2i(u64 base)
+{
+	u64 temp = base >> 1;
+	int log2;
+
+	for (log2 = 0; temp; log2++, temp >>= 1) {
+	}
+
+	return (base & (base - 1)) ? log2 + 1: log2;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/mm.h b/drivers/gpu/drm/nouveau/core/include/core/mm.h
new file mode 100644
index 00000000000..9ee9bf4028c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/mm.h
@@ -0,0 +1,33 @@
+#ifndef __NOUVEAU_MM_H__
+#define __NOUVEAU_MM_H__
+
+struct nouveau_mm_node {
+	struct list_head nl_entry;
+	struct list_head fl_entry;
+	struct list_head rl_entry;
+
+	u8  type;
+	u32 offset;
+	u32 length;
+};
+
+struct nouveau_mm {
+	struct list_head nodes;
+	struct list_head free;
+
+	struct mutex mutex;
+
+	u32 block_size;
+	int heap_nodes;
+	u32 heap_size;
+};
+
+int  nouveau_mm_init(struct nouveau_mm *, u32 offset, u32 length, u32 block);
+int  nouveau_mm_fini(struct nouveau_mm *);
+int  nouveau_mm_head(struct nouveau_mm *, u8 type, u32 size_max, u32 size_min,
+		     u32 align, struct nouveau_mm_node **);
+int  nouveau_mm_tail(struct nouveau_mm *, u8 type, u32 size_max, u32 size_min,
+		     u32 align, struct nouveau_mm_node **);
+void nouveau_mm_free(struct nouveau_mm *, struct nouveau_mm_node **);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/namedb.h b/drivers/gpu/drm/nouveau/core/include/core/namedb.h
new file mode 100644
index 00000000000..8897e088608
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/namedb.h
@@ -0,0 +1,56 @@
+#ifndef __NOUVEAU_NAMEDB_H__
+#define __NOUVEAU_NAMEDB_H__
+
+#include <core/parent.h>
+
+struct nouveau_handle;
+
+struct nouveau_namedb {
+	struct nouveau_parent base;
+	rwlock_t lock;
+	struct list_head list;
+};
+
+static inline struct nouveau_namedb *
+nv_namedb(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_NAMEDB_CLASS)))
+		nv_assert("BAD CAST -> NvNameDB, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+#define nouveau_namedb_create(p,e,c,v,s,m,d)                                   \
+	nouveau_namedb_create_((p), (e), (c), (v), (s), (m),                   \
+			       sizeof(**d), (void **)d)
+#define nouveau_namedb_init(p)                                                 \
+	nouveau_parent_init(&(p)->base)
+#define nouveau_namedb_fini(p,s)                                               \
+	nouveau_parent_fini(&(p)->base, (s))
+#define nouveau_namedb_destroy(p)                                              \
+	nouveau_parent_destroy(&(p)->base)
+
+int  nouveau_namedb_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, u32 pclass,
+			    struct nouveau_oclass *, u32 engcls,
+			    int size, void **);
+
+int  _nouveau_namedb_ctor(struct nouveau_object *, struct nouveau_object *,
+			  struct nouveau_oclass *, void *, u32,
+			  struct nouveau_object **);
+#define _nouveau_namedb_dtor _nouveau_parent_dtor
+#define _nouveau_namedb_init _nouveau_parent_init
+#define _nouveau_namedb_fini _nouveau_parent_fini
+
+int  nouveau_namedb_insert(struct nouveau_namedb *, u32 name,
+			   struct nouveau_object *, struct nouveau_handle *);
+void nouveau_namedb_remove(struct nouveau_handle *);
+
+struct nouveau_handle *nouveau_namedb_get(struct nouveau_namedb *, u32);
+struct nouveau_handle *nouveau_namedb_get_class(struct nouveau_namedb *, u16);
+struct nouveau_handle *nouveau_namedb_get_vinst(struct nouveau_namedb *, u64);
+struct nouveau_handle *nouveau_namedb_get_cinst(struct nouveau_namedb *, u32);
+void nouveau_namedb_put(struct nouveau_handle *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/object.h b/drivers/gpu/drm/nouveau/core/include/core/object.h
new file mode 100644
index 00000000000..818feabbf4a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/object.h
@@ -0,0 +1,188 @@
+#ifndef __NOUVEAU_OBJECT_H__
+#define __NOUVEAU_OBJECT_H__
+
+#include <core/os.h>
+#include <core/printk.h>
+
+#define NV_PARENT_CLASS 0x80000000
+#define NV_NAMEDB_CLASS 0x40000000
+#define NV_CLIENT_CLASS 0x20000000
+#define NV_SUBDEV_CLASS 0x10000000
+#define NV_ENGINE_CLASS 0x08000000
+#define NV_MEMOBJ_CLASS 0x04000000
+#define NV_GPUOBJ_CLASS 0x02000000
+#define NV_ENGCTX_CLASS 0x01000000
+#define NV_OBJECT_CLASS 0x0000ffff
+
+struct nouveau_object {
+	struct nouveau_oclass *oclass;
+	struct nouveau_object *parent;
+	struct nouveau_object *engine;
+	atomic_t refcount;
+	atomic_t usecount;
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+#define NOUVEAU_OBJECT_MAGIC 0x75ef0bad
+	struct list_head list;
+	u32 _magic;
+#endif
+};
+
+static inline struct nouveau_object *
+nv_object(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (likely(obj)) {
+		struct nouveau_object *object = obj;
+		if (unlikely(object->_magic != NOUVEAU_OBJECT_MAGIC))
+			nv_assert("BAD CAST -> NvObject, invalid magic");
+	}
+#endif
+	return obj;
+}
+
+#define nouveau_object_create(p,e,c,s,d)                                       \
+	nouveau_object_create_((p), (e), (c), (s), sizeof(**d), (void **)d)
+int  nouveau_object_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, u32, int size, void **);
+void nouveau_object_destroy(struct nouveau_object *);
+int  nouveau_object_init(struct nouveau_object *);
+int  nouveau_object_fini(struct nouveau_object *, bool suspend);
+
+extern struct nouveau_ofuncs nouveau_object_ofuncs;
+
+struct nouveau_oclass {
+	u32 handle;
+	struct nouveau_ofuncs *ofuncs;
+	struct nouveau_omthds *omthds;
+};
+
+#define nv_oclass(o)    nv_object(o)->oclass
+#define nv_hclass(o)    nv_oclass(o)->handle
+#define nv_iclass(o,i) (nv_hclass(o) & (i))
+#define nv_mclass(o)    nv_iclass(o, NV_OBJECT_CLASS)
+
+static inline struct nouveau_object *
+nv_pclass(struct nouveau_object *parent, u32 oclass)
+{
+	while (parent && !nv_iclass(parent, oclass))
+		parent = parent->parent;
+	return parent;
+}
+
+struct nouveau_omthds {
+	u32 method;
+	int (*call)(struct nouveau_object *, u32, void *, u32);
+};
+
+struct nouveau_ofuncs {
+	int  (*ctor)(struct nouveau_object *, struct nouveau_object *,
+		     struct nouveau_oclass *, void *data, u32 size,
+		     struct nouveau_object **);
+	void (*dtor)(struct nouveau_object *);
+	int  (*init)(struct nouveau_object *);
+	int  (*fini)(struct nouveau_object *, bool suspend);
+	u8   (*rd08)(struct nouveau_object *, u32 offset);
+	u16  (*rd16)(struct nouveau_object *, u32 offset);
+	u32  (*rd32)(struct nouveau_object *, u32 offset);
+	void (*wr08)(struct nouveau_object *, u32 offset, u8 data);
+	void (*wr16)(struct nouveau_object *, u32 offset, u16 data);
+	void (*wr32)(struct nouveau_object *, u32 offset, u32 data);
+};
+
+static inline struct nouveau_ofuncs *
+nv_ofuncs(void *obj)
+{
+	return nv_oclass(obj)->ofuncs;
+}
+
+int  nouveau_object_ctor(struct nouveau_object *, struct nouveau_object *,
+			 struct nouveau_oclass *, void *, u32,
+			 struct nouveau_object **);
+void nouveau_object_ref(struct nouveau_object *, struct nouveau_object **);
+int nouveau_object_inc(struct nouveau_object *);
+int nouveau_object_dec(struct nouveau_object *, bool suspend);
+
+int nouveau_object_new(struct nouveau_object *, u32 parent, u32 handle,
+		       u16 oclass, void *data, u32 size,
+		       struct nouveau_object **);
+int nouveau_object_del(struct nouveau_object *, u32 parent, u32 handle);
+void nouveau_object_debug(void);
+
+static inline int
+nv_call(void *obj, u32 mthd, u32 data)
+{
+	struct nouveau_omthds *method = nv_oclass(obj)->omthds;
+
+	while (method && method->call) {
+		if (method->method == mthd)
+			return method->call(obj, mthd, &data, sizeof(data));
+		method++;
+	}
+
+	return -EINVAL;
+}
+
+static inline u8
+nv_ro08(void *obj, u32 addr)
+{
+	u8 data = nv_ofuncs(obj)->rd08(obj, addr);
+	nv_spam(obj, "nv_ro08 0x%08x 0x%02x\n", addr, data);
+	return data;
+}
+
+static inline u16
+nv_ro16(void *obj, u32 addr)
+{
+	u16 data = nv_ofuncs(obj)->rd16(obj, addr);
+	nv_spam(obj, "nv_ro16 0x%08x 0x%04x\n", addr, data);
+	return data;
+}
+
+static inline u32
+nv_ro32(void *obj, u32 addr)
+{
+	u32 data = nv_ofuncs(obj)->rd32(obj, addr);
+	nv_spam(obj, "nv_ro32 0x%08x 0x%08x\n", addr, data);
+	return data;
+}
+
+static inline void
+nv_wo08(void *obj, u32 addr, u8 data)
+{
+	nv_spam(obj, "nv_wo08 0x%08x 0x%02x\n", addr, data);
+	nv_ofuncs(obj)->wr08(obj, addr, data);
+}
+
+static inline void
+nv_wo16(void *obj, u32 addr, u16 data)
+{
+	nv_spam(obj, "nv_wo16 0x%08x 0x%04x\n", addr, data);
+	nv_ofuncs(obj)->wr16(obj, addr, data);
+}
+
+static inline void
+nv_wo32(void *obj, u32 addr, u32 data)
+{
+	nv_spam(obj, "nv_wo32 0x%08x 0x%08x\n", addr, data);
+	nv_ofuncs(obj)->wr32(obj, addr, data);
+}
+
+static inline u32
+nv_mo32(void *obj, u32 addr, u32 mask, u32 data)
+{
+	u32 temp = nv_ro32(obj, addr);
+	nv_wo32(obj, addr, (temp & ~mask) | data);
+	return temp;
+}
+
+static inline bool
+nv_strncmp(void *obj, u32 addr, u32 len, const char *str)
+{
+	while (len--) {
+		if (nv_ro08(obj, addr++) != *(str++))
+			return false;
+	}
+	return true;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/option.h b/drivers/gpu/drm/nouveau/core/include/core/option.h
new file mode 100644
index 00000000000..27074957fd2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/option.h
@@ -0,0 +1,11 @@
+#ifndef __NOUVEAU_OPTION_H__
+#define __NOUVEAU_OPTION_H__
+
+#include <core/os.h>
+
+const char *nouveau_stropt(const char *optstr, const char *opt, int *len);
+bool nouveau_boolopt(const char *optstr, const char *opt, bool value);
+
+int nouveau_dbgopt(const char *optstr, const char *sub);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/parent.h b/drivers/gpu/drm/nouveau/core/include/core/parent.h
new file mode 100644
index 00000000000..d3aa251a5eb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/parent.h
@@ -0,0 +1,64 @@
+#ifndef __NOUVEAU_PARENT_H__
+#define __NOUVEAU_PARENT_H__
+
+#include <core/device.h>
+#include <core/object.h>
+
+struct nouveau_sclass {
+	struct nouveau_sclass *sclass;
+	struct nouveau_engine *engine;
+	struct nouveau_oclass *oclass;
+};
+
+struct nouveau_parent {
+	struct nouveau_object base;
+
+	struct nouveau_sclass *sclass;
+	u32 engine;
+
+	int  (*context_attach)(struct nouveau_object *,
+			       struct nouveau_object *);
+	int  (*context_detach)(struct nouveau_object *, bool suspend,
+			       struct nouveau_object *);
+
+	int  (*object_attach)(struct nouveau_object *parent,
+			      struct nouveau_object *object, u32 name);
+	void (*object_detach)(struct nouveau_object *parent, int cookie);
+};
+
+static inline struct nouveau_parent *
+nv_parent(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!(nv_iclass(obj, NV_PARENT_CLASS))))
+		nv_assert("BAD CAST -> NvParent, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+#define nouveau_parent_create(p,e,c,v,s,m,d)                                   \
+	nouveau_parent_create_((p), (e), (c), (v), (s), (m),                   \
+			       sizeof(**d), (void **)d)
+#define nouveau_parent_init(p)                                                 \
+	nouveau_object_init(&(p)->base)
+#define nouveau_parent_fini(p,s)                                               \
+	nouveau_object_fini(&(p)->base, (s))
+
+int  nouveau_parent_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, u32 pclass,
+			    struct nouveau_oclass *, u64 engcls,
+			    int size, void **);
+void nouveau_parent_destroy(struct nouveau_parent *);
+
+int  _nouveau_parent_ctor(struct nouveau_object *, struct nouveau_object *,
+			  struct nouveau_oclass *, void *, u32,
+			  struct nouveau_object **);
+void _nouveau_parent_dtor(struct nouveau_object *);
+#define _nouveau_parent_init _nouveau_object_init
+#define _nouveau_parent_fini _nouveau_object_fini
+
+int nouveau_parent_sclass(struct nouveau_object *, u16 handle,
+			  struct nouveau_object **pengine,
+			  struct nouveau_oclass **poclass);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/printk.h b/drivers/gpu/drm/nouveau/core/include/core/printk.h
new file mode 100644
index 00000000000..1d629664f32
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/printk.h
@@ -0,0 +1,39 @@
+#ifndef __NOUVEAU_PRINTK_H__
+#define __NOUVEAU_PRINTK_H__
+
+#include <core/os.h>
+#include <core/debug.h>
+
+struct nouveau_object;
+
+#define NV_PRINTK_FATAL    KERN_CRIT
+#define NV_PRINTK_ERROR    KERN_ERR
+#define NV_PRINTK_WARN     KERN_WARNING
+#define NV_PRINTK_INFO     KERN_INFO
+#define NV_PRINTK_DEBUG    KERN_DEBUG
+#define NV_PRINTK_PARANOIA KERN_DEBUG
+#define NV_PRINTK_TRACE    KERN_DEBUG
+#define NV_PRINTK_SPAM     KERN_DEBUG
+
+void nv_printk_(struct nouveau_object *, const char *, int, const char *, ...);
+
+#define nv_printk(o,l,f,a...) do {                                             \
+	if (NV_DBG_##l <= CONFIG_NOUVEAU_DEBUG)                                \
+		nv_printk_(nv_object(o), NV_PRINTK_##l, NV_DBG_##l, f, ##a);   \
+} while(0)
+
+#define nv_fatal(o,f,a...) nv_printk((o), FATAL, f, ##a)
+#define nv_error(o,f,a...) nv_printk((o), ERROR, f, ##a)
+#define nv_warn(o,f,a...) nv_printk((o), WARN, f, ##a)
+#define nv_info(o,f,a...) nv_printk((o), INFO, f, ##a)
+#define nv_debug(o,f,a...) nv_printk((o), DEBUG, f, ##a)
+#define nv_trace(o,f,a...) nv_printk((o), TRACE, f, ##a)
+#define nv_spam(o,f,a...) nv_printk((o), SPAM, f, ##a)
+
+#define nv_assert(f,a...) do {                                                 \
+	if (NV_DBG_FATAL <= CONFIG_NOUVEAU_DEBUG)                              \
+		nv_printk_(NULL, NV_PRINTK_FATAL, NV_DBG_FATAL, f "\n", ##a);  \
+	BUG_ON(1);                                                             \
+} while(0)
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/ramht.h b/drivers/gpu/drm/nouveau/core/include/core/ramht.h
new file mode 100644
index 00000000000..47e4cacbca3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/ramht.h
@@ -0,0 +1,23 @@
+#ifndef __NOUVEAU_RAMHT_H__
+#define __NOUVEAU_RAMHT_H__
+
+#include <core/gpuobj.h>
+
+struct nouveau_ramht {
+	struct nouveau_gpuobj base;
+	int bits;
+};
+
+int  nouveau_ramht_insert(struct nouveau_ramht *, int chid,
+			  u32 handle, u32 context);
+void nouveau_ramht_remove(struct nouveau_ramht *, int cookie);
+int  nouveau_ramht_new(struct nouveau_object *, struct nouveau_object *,
+		       u32 size, u32 align, struct nouveau_ramht **);
+
+static inline void
+nouveau_ramht_ref(struct nouveau_ramht *obj, struct nouveau_ramht **ref)
+{
+	nouveau_gpuobj_ref(&obj->base, (struct nouveau_gpuobj **)ref);
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/subdev.h b/drivers/gpu/drm/nouveau/core/include/core/subdev.h
new file mode 100644
index 00000000000..e9632e93161
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/core/subdev.h
@@ -0,0 +1,118 @@
+#ifndef __NOUVEAU_SUBDEV_H__
+#define __NOUVEAU_SUBDEV_H__
+
+#include <core/object.h>
+
+#define NV_SUBDEV_(sub,var) (NV_SUBDEV_CLASS | ((var) << 8) | (sub))
+#define NV_SUBDEV(name,var)  NV_SUBDEV_(NVDEV_SUBDEV_##name, (var))
+
+struct nouveau_subdev {
+	struct nouveau_object base;
+	struct mutex mutex;
+	const char *name;
+	void __iomem *mmio;
+	u32 debug;
+	u32 unit;
+
+	void (*intr)(struct nouveau_subdev *);
+};
+
+static inline struct nouveau_subdev *
+nv_subdev(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_SUBDEV_CLASS)))
+		nv_assert("BAD CAST -> NvSubDev, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+static inline int
+nv_subidx(struct nouveau_object *object)
+{
+	return nv_hclass(nv_subdev(object)) & 0xff;
+}
+
+#define nouveau_subdev_create(p,e,o,v,s,f,d)                                   \
+	nouveau_subdev_create_((p), (e), (o), (v), (s), (f),                   \
+			       sizeof(**d),(void **)d)
+
+int  nouveau_subdev_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, u32 pclass,
+			    const char *sname, const char *fname,
+			    int size, void **);
+void nouveau_subdev_destroy(struct nouveau_subdev *);
+int  nouveau_subdev_init(struct nouveau_subdev *);
+int  nouveau_subdev_fini(struct nouveau_subdev *, bool suspend);
+void nouveau_subdev_reset(struct nouveau_object *);
+
+void _nouveau_subdev_dtor(struct nouveau_object *);
+int  _nouveau_subdev_init(struct nouveau_object *);
+int  _nouveau_subdev_fini(struct nouveau_object *, bool suspend);
+
+#define s_printk(s,l,f,a...) do {                                              \
+	if ((s)->debug >= OS_DBG_##l) {                                        \
+		nv_printk((s)->base.parent, (s)->name, l, f, ##a);             \
+	}                                                                      \
+} while(0)
+
+static inline u8
+nv_rd08(void *obj, u32 addr)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	u8 data = ioread8(subdev->mmio + addr);
+	nv_spam(subdev, "nv_rd08 0x%06x 0x%02x\n", addr, data);
+	return data;
+}
+
+static inline u16
+nv_rd16(void *obj, u32 addr)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	u16 data = ioread16_native(subdev->mmio + addr);
+	nv_spam(subdev, "nv_rd16 0x%06x 0x%04x\n", addr, data);
+	return data;
+}
+
+static inline u32
+nv_rd32(void *obj, u32 addr)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	u32 data = ioread32_native(subdev->mmio + addr);
+	nv_spam(subdev, "nv_rd32 0x%06x 0x%08x\n", addr, data);
+	return data;
+}
+
+static inline void
+nv_wr08(void *obj, u32 addr, u8 data)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	nv_spam(subdev, "nv_wr08 0x%06x 0x%02x\n", addr, data);
+	iowrite8(data, subdev->mmio + addr);
+}
+
+static inline void
+nv_wr16(void *obj, u32 addr, u16 data)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	nv_spam(subdev, "nv_wr16 0x%06x 0x%04x\n", addr, data);
+	iowrite16_native(data, subdev->mmio + addr);
+}
+
+static inline void
+nv_wr32(void *obj, u32 addr, u32 data)
+{
+	struct nouveau_subdev *subdev = nv_subdev(obj);
+	nv_spam(subdev, "nv_wr32 0x%06x 0x%08x\n", addr, data);
+	iowrite32_native(data, subdev->mmio + addr);
+}
+
+static inline u32
+nv_mask(void *obj, u32 addr, u32 mask, u32 data)
+{
+	u32 temp = nv_rd32(obj, addr);
+	nv_wr32(obj, addr, (temp & ~mask) | data);
+	return temp;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/bsp.h b/drivers/gpu/drm/nouveau/core/include/engine/bsp.h
new file mode 100644
index 00000000000..75d1ed5f85f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/bsp.h
@@ -0,0 +1,45 @@
+#ifndef __NOUVEAU_BSP_H__
+#define __NOUVEAU_BSP_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_bsp_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_bsp_context_create(p,e,c,g,s,a,f,d)                            \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_bsp_context_destroy(d)                                         \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_bsp_context_init(d)                                            \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_bsp_context_fini(d,s)                                          \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_bsp_context_dtor _nouveau_engctx_dtor
+#define _nouveau_bsp_context_init _nouveau_engctx_init
+#define _nouveau_bsp_context_fini _nouveau_engctx_fini
+#define _nouveau_bsp_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_bsp_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_bsp {
+	struct nouveau_engine base;
+};
+
+#define nouveau_bsp_create(p,e,c,d)                                            \
+	nouveau_engine_create((p), (e), (c), true, "PBSP", "bsp", (d))
+#define nouveau_bsp_destroy(d)                                                 \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_bsp_init(d)                                                    \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_bsp_fini(d,s)                                                  \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_bsp_dtor _nouveau_engine_dtor
+#define _nouveau_bsp_init _nouveau_engine_init
+#define _nouveau_bsp_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv84_bsp_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/copy.h b/drivers/gpu/drm/nouveau/core/include/engine/copy.h
new file mode 100644
index 00000000000..70b9d8c5fcf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/copy.h
@@ -0,0 +1,49 @@
+#ifndef __NOUVEAU_COPY_H__
+#define __NOUVEAU_COPY_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_copy_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_copy_context_create(p,e,c,g,s,a,f,d)                           \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_copy_context_destroy(d)                                        \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_copy_context_init(d)                                           \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_copy_context_fini(d,s)                                         \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_copy_context_dtor _nouveau_engctx_dtor
+#define _nouveau_copy_context_init _nouveau_engctx_init
+#define _nouveau_copy_context_fini _nouveau_engctx_fini
+#define _nouveau_copy_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_copy_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_copy {
+	struct nouveau_engine base;
+};
+
+#define nouveau_copy_create(p,e,c,y,i,d)                                       \
+	nouveau_engine_create((p), (e), (c), (y), "PCE"#i, "copy"#i, (d))
+#define nouveau_copy_destroy(d)                                                \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_copy_init(d)                                                   \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_copy_fini(d,s)                                                 \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_copy_dtor _nouveau_engine_dtor
+#define _nouveau_copy_init _nouveau_engine_init
+#define _nouveau_copy_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nva3_copy_oclass;
+extern struct nouveau_oclass nvc0_copy0_oclass;
+extern struct nouveau_oclass nvc0_copy1_oclass;
+extern struct nouveau_oclass nve0_copy0_oclass;
+extern struct nouveau_oclass nve0_copy1_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/crypt.h b/drivers/gpu/drm/nouveau/core/include/engine/crypt.h
new file mode 100644
index 00000000000..e3674743baa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/crypt.h
@@ -0,0 +1,46 @@
+#ifndef __NOUVEAU_CRYPT_H__
+#define __NOUVEAU_CRYPT_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_crypt_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_crypt_context_create(p,e,c,g,s,a,f,d)                          \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_crypt_context_destroy(d)                                       \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_crypt_context_init(d)                                          \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_crypt_context_fini(d,s)                                        \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_crypt_context_dtor _nouveau_engctx_dtor
+#define _nouveau_crypt_context_init _nouveau_engctx_init
+#define _nouveau_crypt_context_fini _nouveau_engctx_fini
+#define _nouveau_crypt_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_crypt_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_crypt {
+	struct nouveau_engine base;
+};
+
+#define nouveau_crypt_create(p,e,c,d)                                          \
+	nouveau_engine_create((p), (e), (c), true, "PCRYPT", "crypt", (d))
+#define nouveau_crypt_destroy(d)                                               \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_crypt_init(d)                                                  \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_crypt_fini(d,s)                                                \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_crypt_dtor _nouveau_engine_dtor
+#define _nouveau_crypt_init _nouveau_engine_init
+#define _nouveau_crypt_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv84_crypt_oclass;
+extern struct nouveau_oclass nv98_crypt_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/disp.h b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
new file mode 100644
index 00000000000..38ec1252cba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
@@ -0,0 +1,44 @@
+#ifndef __NOUVEAU_DISP_H__
+#define __NOUVEAU_DISP_H__
+
+#include <core/object.h>
+#include <core/engine.h>
+#include <core/device.h>
+
+struct nouveau_disp {
+	struct nouveau_engine base;
+
+	struct {
+		struct list_head list;
+		spinlock_t lock;
+		void (*notify)(void *, int);
+		void (*get)(void *, int);
+		void (*put)(void *, int);
+		void *data;
+	} vblank;
+};
+
+static inline struct nouveau_disp *
+nouveau_disp(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_ENGINE_DISP];
+}
+
+#define nouveau_disp_create(p,e,c,i,x,d)                                       \
+	nouveau_engine_create((p), (e), (c), true, (i), (x), (d))
+#define nouveau_disp_destroy(d)                                                \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_disp_init(d)                                                   \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_disp_fini(d,s)                                                 \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_disp_dtor _nouveau_engine_dtor
+#define _nouveau_disp_init _nouveau_engine_init
+#define _nouveau_disp_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv04_disp_oclass;
+extern struct nouveau_oclass nv50_disp_oclass;
+extern struct nouveau_oclass nvd0_disp_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h b/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h
new file mode 100644
index 00000000000..700ccbb1941
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/dmaobj.h
@@ -0,0 +1,57 @@
+#ifndef __NOUVEAU_DMAOBJ_H__
+#define __NOUVEAU_DMAOBJ_H__
+
+#include <core/object.h>
+#include <core/engine.h>
+
+struct nouveau_gpuobj;
+
+struct nouveau_dmaobj {
+	struct nouveau_object base;
+	u32 target;
+	u32 access;
+	u64 start;
+	u64 limit;
+};
+
+#define nouveau_dmaobj_create(p,e,c,a,s,d)                                     \
+	nouveau_dmaobj_create_((p), (e), (c), (a), (s), sizeof(**d), (void **)d)
+#define nouveau_dmaobj_destroy(p)                                              \
+	nouveau_object_destroy(&(p)->base)
+#define nouveau_dmaobj_init(p)                                                 \
+	nouveau_object_init(&(p)->base)
+#define nouveau_dmaobj_fini(p,s)                                               \
+	nouveau_object_fini(&(p)->base, (s))
+
+int nouveau_dmaobj_create_(struct nouveau_object *, struct nouveau_object *,
+			   struct nouveau_oclass *, void *data, u32 size,
+			   int length, void **);
+
+#define _nouveau_dmaobj_dtor nouveau_object_destroy
+#define _nouveau_dmaobj_init nouveau_object_init
+#define _nouveau_dmaobj_fini nouveau_object_fini
+
+struct nouveau_dmaeng {
+	struct nouveau_engine base;
+	int (*bind)(struct nouveau_dmaeng *, struct nouveau_object *parent,
+		    struct nouveau_dmaobj *, struct nouveau_gpuobj **);
+};
+
+#define nouveau_dmaeng_create(p,e,c,d)                                         \
+	nouveau_engine_create((p), (e), (c), true, "DMAOBJ", "dmaobj", (d))
+#define nouveau_dmaeng_destroy(p)                                              \
+	nouveau_engine_destroy(&(p)->base)
+#define nouveau_dmaeng_init(p)                                                 \
+	nouveau_engine_init(&(p)->base)
+#define nouveau_dmaeng_fini(p,s)                                               \
+	nouveau_engine_fini(&(p)->base, (s))
+
+#define _nouveau_dmaeng_dtor _nouveau_engine_dtor
+#define _nouveau_dmaeng_init _nouveau_engine_init
+#define _nouveau_dmaeng_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv04_dmaeng_oclass;
+extern struct nouveau_oclass nv50_dmaeng_oclass;
+extern struct nouveau_oclass nvc0_dmaeng_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/fifo.h b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
new file mode 100644
index 00000000000..d67fed1e397
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
@@ -0,0 +1,111 @@
+#ifndef __NOUVEAU_FIFO_H__
+#define __NOUVEAU_FIFO_H__
+
+#include <core/namedb.h>
+#include <core/gpuobj.h>
+#include <core/engine.h>
+
+struct nouveau_fifo_chan {
+	struct nouveau_namedb base;
+	struct nouveau_dmaobj *pushdma;
+	struct nouveau_gpuobj *pushgpu;
+	void __iomem *user;
+	u32 size;
+	u16 chid;
+	atomic_t refcnt; /* NV04_NVSW_SET_REF */
+};
+
+static inline struct nouveau_fifo_chan *
+nouveau_fifo_chan(void *obj)
+{
+	return (void *)nv_namedb(obj);
+}
+
+#define nouveau_fifo_channel_create(p,e,c,b,a,s,n,m,d)                         \
+	nouveau_fifo_channel_create_((p), (e), (c), (b), (a), (s), (n),        \
+				     (m), sizeof(**d), (void **)d)
+#define nouveau_fifo_channel_init(p)                                           \
+	nouveau_namedb_init(&(p)->base)
+#define nouveau_fifo_channel_fini(p,s)                                         \
+	nouveau_namedb_fini(&(p)->base, (s))
+
+int  nouveau_fifo_channel_create_(struct nouveau_object *,
+				  struct nouveau_object *,
+				  struct nouveau_oclass *,
+				  int bar, u32 addr, u32 size, u32 push,
+				  u32 engmask, int len, void **);
+void nouveau_fifo_channel_destroy(struct nouveau_fifo_chan *);
+
+#define _nouveau_fifo_channel_init _nouveau_namedb_init
+#define _nouveau_fifo_channel_fini _nouveau_namedb_fini
+
+void _nouveau_fifo_channel_dtor(struct nouveau_object *);
+u32  _nouveau_fifo_channel_rd32(struct nouveau_object *, u32);
+void _nouveau_fifo_channel_wr32(struct nouveau_object *, u32, u32);
+
+struct nouveau_fifo_base {
+	struct nouveau_gpuobj base;
+};
+
+#define nouveau_fifo_context_create(p,e,c,g,s,a,f,d)                           \
+	nouveau_gpuobj_create((p), (e), (c), 0, (g), (s), (a), (f), (d))
+#define nouveau_fifo_context_destroy(p)                                        \
+	nouveau_gpuobj_destroy(&(p)->base)
+#define nouveau_fifo_context_init(p)                                           \
+	nouveau_gpuobj_init(&(p)->base)
+#define nouveau_fifo_context_fini(p,s)                                         \
+	nouveau_gpuobj_fini(&(p)->base, (s))
+
+#define _nouveau_fifo_context_dtor _nouveau_gpuobj_dtor
+#define _nouveau_fifo_context_init _nouveau_gpuobj_init
+#define _nouveau_fifo_context_fini _nouveau_gpuobj_fini
+#define _nouveau_fifo_context_rd32 _nouveau_gpuobj_rd32
+#define _nouveau_fifo_context_wr32 _nouveau_gpuobj_wr32
+
+struct nouveau_fifo {
+	struct nouveau_engine base;
+
+	struct nouveau_object **channel;
+	spinlock_t lock;
+	u16 min;
+	u16 max;
+
+	int  (*chid)(struct nouveau_fifo *, struct nouveau_object *);
+	void (*pause)(struct nouveau_fifo *, unsigned long *);
+	void (*start)(struct nouveau_fifo *, unsigned long *);
+};
+
+static inline struct nouveau_fifo *
+nouveau_fifo(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_ENGINE_FIFO];
+}
+
+#define nouveau_fifo_create(o,e,c,fc,lc,d)                                     \
+	nouveau_fifo_create_((o), (e), (c), (fc), (lc), sizeof(**d), (void **)d)
+#define nouveau_fifo_init(p)                                                   \
+	nouveau_engine_init(&(p)->base)
+#define nouveau_fifo_fini(p,s)                                                 \
+	nouveau_engine_fini(&(p)->base, (s))
+
+int nouveau_fifo_create_(struct nouveau_object *, struct nouveau_object *,
+			 struct nouveau_oclass *, int min, int max,
+			 int size, void **);
+void nouveau_fifo_destroy(struct nouveau_fifo *);
+
+#define _nouveau_fifo_init _nouveau_engine_init
+#define _nouveau_fifo_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv04_fifo_oclass;
+extern struct nouveau_oclass nv10_fifo_oclass;
+extern struct nouveau_oclass nv17_fifo_oclass;
+extern struct nouveau_oclass nv40_fifo_oclass;
+extern struct nouveau_oclass nv50_fifo_oclass;
+extern struct nouveau_oclass nv84_fifo_oclass;
+extern struct nouveau_oclass nvc0_fifo_oclass;
+extern struct nouveau_oclass nve0_fifo_oclass;
+
+void nv04_fifo_intr(struct nouveau_subdev *);
+int  nv04_fifo_context_attach(struct nouveau_object *, struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/graph.h b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
new file mode 100644
index 00000000000..6943b40d081
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
@@ -0,0 +1,72 @@
+#ifndef __NOUVEAU_GRAPH_H__
+#define __NOUVEAU_GRAPH_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+#include <core/enum.h>
+
+struct nouveau_graph_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_graph_context_create(p,e,c,g,s,a,f,d)                          \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_graph_context_destroy(d)                                       \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_graph_context_init(d)                                          \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_graph_context_fini(d,s)                                        \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_graph_context_dtor _nouveau_engctx_dtor
+#define _nouveau_graph_context_init _nouveau_engctx_init
+#define _nouveau_graph_context_fini _nouveau_engctx_fini
+#define _nouveau_graph_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_graph_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_graph {
+	struct nouveau_engine base;
+};
+
+static inline struct nouveau_graph *
+nouveau_graph(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_ENGINE_GR];
+}
+
+#define nouveau_graph_create(p,e,c,y,d)                                        \
+	nouveau_engine_create((p), (e), (c), (y), "PGRAPH", "graphics", (d))
+#define nouveau_graph_destroy(d)                                               \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_graph_init(d)                                                  \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_graph_fini(d,s)                                                \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_graph_dtor _nouveau_engine_dtor
+#define _nouveau_graph_init _nouveau_engine_init
+#define _nouveau_graph_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv04_graph_oclass;
+extern struct nouveau_oclass nv10_graph_oclass;
+extern struct nouveau_oclass nv20_graph_oclass;
+extern struct nouveau_oclass nv25_graph_oclass;
+extern struct nouveau_oclass nv2a_graph_oclass;
+extern struct nouveau_oclass nv30_graph_oclass;
+extern struct nouveau_oclass nv34_graph_oclass;
+extern struct nouveau_oclass nv35_graph_oclass;
+extern struct nouveau_oclass nv40_graph_oclass;
+extern struct nouveau_oclass nv50_graph_oclass;
+extern struct nouveau_oclass nvc0_graph_oclass;
+extern struct nouveau_oclass nve0_graph_oclass;
+
+extern const struct nouveau_bitfield nv04_graph_nsource[];
+extern struct nouveau_ofuncs nv04_graph_ofuncs;
+bool nv04_graph_idle(void *obj);
+
+extern const struct nouveau_bitfield nv10_graph_intr_name[];
+extern const struct nouveau_bitfield nv10_graph_nstatus[];
+
+extern const struct nouveau_enum nv50_data_error_names[];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h b/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h
new file mode 100644
index 00000000000..bbf0d4a5bbd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h
@@ -0,0 +1,61 @@
+#ifndef __NOUVEAU_MPEG_H__
+#define __NOUVEAU_MPEG_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_mpeg_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_mpeg_context_create(p,e,c,g,s,a,f,d)                           \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_mpeg_context_destroy(d)                                        \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_mpeg_context_init(d)                                           \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_mpeg_context_fini(d,s)                                         \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_mpeg_context_dtor _nouveau_engctx_dtor
+#define _nouveau_mpeg_context_init _nouveau_engctx_init
+#define _nouveau_mpeg_context_fini _nouveau_engctx_fini
+#define _nouveau_mpeg_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_mpeg_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_mpeg {
+	struct nouveau_engine base;
+};
+
+#define nouveau_mpeg_create(p,e,c,d)                                           \
+	nouveau_engine_create((p), (e), (c), true, "PMPEG", "mpeg", (d))
+#define nouveau_mpeg_destroy(d)                                                \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_mpeg_init(d)                                                   \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_mpeg_fini(d,s)                                                 \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_mpeg_dtor _nouveau_engine_dtor
+#define _nouveau_mpeg_init _nouveau_engine_init
+#define _nouveau_mpeg_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv31_mpeg_oclass;
+extern struct nouveau_oclass nv40_mpeg_oclass;
+extern struct nouveau_oclass nv50_mpeg_oclass;
+extern struct nouveau_oclass nv84_mpeg_oclass;
+
+extern struct nouveau_oclass nv31_mpeg_sclass[];
+void nv31_mpeg_intr(struct nouveau_subdev *);
+void nv31_mpeg_tile_prog(struct nouveau_engine *, int);
+int  nv31_mpeg_init(struct nouveau_object *);
+
+extern struct nouveau_ofuncs nv50_mpeg_ofuncs;
+int  nv50_mpeg_context_ctor(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, void *, u32,
+			    struct nouveau_object **);
+int  nv50_mpeg_tlb_flush(struct nouveau_engine *);
+void nv50_mpeg_intr(struct nouveau_subdev *);
+int  nv50_mpeg_init(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/ppp.h b/drivers/gpu/drm/nouveau/core/include/engine/ppp.h
new file mode 100644
index 00000000000..74d554fb328
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/ppp.h
@@ -0,0 +1,45 @@
+#ifndef __NOUVEAU_PPP_H__
+#define __NOUVEAU_PPP_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_ppp_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_ppp_context_create(p,e,c,g,s,a,f,d)                            \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_ppp_context_destroy(d)                                         \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_ppp_context_init(d)                                            \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_ppp_context_fini(d,s)                                          \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_ppp_context_dtor _nouveau_engctx_dtor
+#define _nouveau_ppp_context_init _nouveau_engctx_init
+#define _nouveau_ppp_context_fini _nouveau_engctx_fini
+#define _nouveau_ppp_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_ppp_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_ppp {
+	struct nouveau_engine base;
+};
+
+#define nouveau_ppp_create(p,e,c,d)                                            \
+	nouveau_engine_create((p), (e), (c), true, "PPPP", "ppp", (d))
+#define nouveau_ppp_destroy(d)                                                 \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_ppp_init(d)                                                    \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_ppp_fini(d,s)                                                  \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_ppp_dtor _nouveau_engine_dtor
+#define _nouveau_ppp_init _nouveau_engine_init
+#define _nouveau_ppp_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv98_ppp_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/software.h b/drivers/gpu/drm/nouveau/core/include/engine/software.h
new file mode 100644
index 00000000000..c945691c856
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/software.h
@@ -0,0 +1,60 @@
+#ifndef __NOUVEAU_SOFTWARE_H__
+#define __NOUVEAU_SOFTWARE_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_software_chan {
+	struct nouveau_engctx base;
+
+	struct {
+		struct list_head head;
+		u32 channel;
+		u32 ctxdma;
+		u64 offset;
+		u32 value;
+		u32 crtc;
+	} vblank;
+
+	int (*flip)(void *);
+	void *flip_data;
+};
+
+#define nouveau_software_context_create(p,e,c,d)                               \
+	nouveau_engctx_create((p), (e), (c), (p), 0, 0, 0, (d))
+#define nouveau_software_context_destroy(d)                                    \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_software_context_init(d)                                       \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_software_context_fini(d,s)                                     \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_software_context_dtor _nouveau_engctx_dtor
+#define _nouveau_software_context_init _nouveau_engctx_init
+#define _nouveau_software_context_fini _nouveau_engctx_fini
+
+struct nouveau_software {
+	struct nouveau_engine base;
+};
+
+#define nouveau_software_create(p,e,c,d)                                       \
+	nouveau_engine_create((p), (e), (c), true, "SW", "software", (d))
+#define nouveau_software_destroy(d)                                            \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_software_init(d)                                               \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_software_fini(d,s)                                             \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_software_dtor _nouveau_engine_dtor
+#define _nouveau_software_init _nouveau_engine_init
+#define _nouveau_software_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv04_software_oclass;
+extern struct nouveau_oclass nv10_software_oclass;
+extern struct nouveau_oclass nv50_software_oclass;
+extern struct nouveau_oclass nvc0_software_oclass;
+
+void nv04_software_intr(struct nouveau_subdev *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/vp.h b/drivers/gpu/drm/nouveau/core/include/engine/vp.h
new file mode 100644
index 00000000000..05cd08fba37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/vp.h
@@ -0,0 +1,45 @@
+#ifndef __NOUVEAU_VP_H__
+#define __NOUVEAU_VP_H__
+
+#include <core/engine.h>
+#include <core/engctx.h>
+
+struct nouveau_vp_chan {
+	struct nouveau_engctx base;
+};
+
+#define nouveau_vp_context_create(p,e,c,g,s,a,f,d)                             \
+	nouveau_engctx_create((p), (e), (c), (g), (s), (a), (f), (d))
+#define nouveau_vp_context_destroy(d)                                          \
+	nouveau_engctx_destroy(&(d)->base)
+#define nouveau_vp_context_init(d)                                             \
+	nouveau_engctx_init(&(d)->base)
+#define nouveau_vp_context_fini(d,s)                                           \
+	nouveau_engctx_fini(&(d)->base, (s))
+
+#define _nouveau_vp_context_dtor _nouveau_engctx_dtor
+#define _nouveau_vp_context_init _nouveau_engctx_init
+#define _nouveau_vp_context_fini _nouveau_engctx_fini
+#define _nouveau_vp_context_rd32 _nouveau_engctx_rd32
+#define _nouveau_vp_context_wr32 _nouveau_engctx_wr32
+
+struct nouveau_vp {
+	struct nouveau_engine base;
+};
+
+#define nouveau_vp_create(p,e,c,d)                                             \
+	nouveau_engine_create((p), (e), (c), true, "PVP", "vp", (d))
+#define nouveau_vp_destroy(d)                                                  \
+	nouveau_engine_destroy(&(d)->base)
+#define nouveau_vp_init(d)                                                     \
+	nouveau_engine_init(&(d)->base)
+#define nouveau_vp_fini(d,s)                                                   \
+	nouveau_engine_fini(&(d)->base, (s))
+
+#define _nouveau_vp_dtor _nouveau_engine_dtor
+#define _nouveau_vp_init _nouveau_engine_init
+#define _nouveau_vp_fini _nouveau_engine_fini
+
+extern struct nouveau_oclass nv84_vp_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bar.h b/drivers/gpu/drm/nouveau/core/include/subdev/bar.h
new file mode 100644
index 00000000000..4f4ff4502c3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bar.h
@@ -0,0 +1,55 @@
+#ifndef __NOUVEAU_BAR_H__
+#define __NOUVEAU_BAR_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+#include <subdev/fb.h>
+
+struct nouveau_vma;
+
+struct nouveau_bar {
+	struct nouveau_subdev base;
+
+	int (*alloc)(struct nouveau_bar *, struct nouveau_object *,
+		     struct nouveau_mem *, struct nouveau_object **);
+	void __iomem *iomem;
+
+	int (*kmap)(struct nouveau_bar *, struct nouveau_mem *,
+		    u32 flags, struct nouveau_vma *);
+	int (*umap)(struct nouveau_bar *, struct nouveau_mem *,
+		    u32 flags, struct nouveau_vma *);
+	void (*unmap)(struct nouveau_bar *, struct nouveau_vma *);
+	void (*flush)(struct nouveau_bar *);
+};
+
+static inline struct nouveau_bar *
+nouveau_bar(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_BAR];
+}
+
+#define nouveau_bar_create(p,e,o,d)                                            \
+	nouveau_bar_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_bar_init(p)                                                    \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_bar_fini(p,s)                                                  \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+int nouveau_bar_create_(struct nouveau_object *, struct nouveau_object *,
+			struct nouveau_oclass *, int, void **);
+void nouveau_bar_destroy(struct nouveau_bar *);
+
+void _nouveau_bar_dtor(struct nouveau_object *);
+#define _nouveau_bar_init _nouveau_subdev_init
+#define _nouveau_bar_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv50_bar_oclass;
+extern struct nouveau_oclass nvc0_bar_oclass;
+
+int nouveau_bar_alloc(struct nouveau_bar *, struct nouveau_object *,
+		      struct nouveau_mem *, struct nouveau_object **);
+
+void nv84_bar_flush(struct nouveau_bar *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios.h
new file mode 100644
index 00000000000..d145b25e6be
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios.h
@@ -0,0 +1,34 @@
+#ifndef __NOUVEAU_BIOS_H__
+#define __NOUVEAU_BIOS_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_bios {
+	struct nouveau_subdev base;
+	u32 size;
+	u8 *data;
+
+	u32 bmp_offset;
+	u32 bit_offset;
+
+	struct {
+		u8 major;
+		u8 chip;
+		u8 minor;
+		u8 micro;
+	} version;
+};
+
+static inline struct nouveau_bios *
+nouveau_bios(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_VBIOS];
+}
+
+u8  nvbios_checksum(const u8 *data, int size);
+u16 nvbios_findstr(const u8 *data, int size, const char *str, int len);
+
+extern struct nouveau_oclass nouveau_bios_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/bit.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/bit.h
new file mode 100644
index 00000000000..73f060b0798
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/bit.h
@@ -0,0 +1,13 @@
+#ifndef __NVBIOS_BIT_H__
+#define __NVBIOS_BIT_H__
+
+struct bit_entry {
+	u8  id;
+	u8  version;
+	u16 length;
+	u16 offset;
+};
+
+int bit_entry(struct nouveau_bios *, u8 id, struct bit_entry *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/bmp.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/bmp.h
new file mode 100644
index 00000000000..10e4dbca649
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/bmp.h
@@ -0,0 +1,39 @@
+#ifndef __NVBIOS_BMP_H__
+#define __NVBIOS_BMP_H__
+
+static inline u16
+bmp_version(struct nouveau_bios *bios)
+{
+	if (bios->bmp_offset) {
+		return nv_ro08(bios, bios->bmp_offset + 5) << 8 |
+		       nv_ro08(bios, bios->bmp_offset + 6);
+	}
+
+	return 0x0000;
+}
+
+static inline u16
+bmp_mem_init_table(struct nouveau_bios *bios)
+{
+	if (bmp_version(bios) >= 0x0300)
+		return nv_ro16(bios, bios->bmp_offset + 24);
+	return 0x0000;
+}
+
+static inline u16
+bmp_sdr_seq_table(struct nouveau_bios *bios)
+{
+	if (bmp_version(bios) >= 0x0300)
+		return nv_ro16(bios, bios->bmp_offset + 26);
+	return 0x0000;
+}
+
+static inline u16
+bmp_ddr_seq_table(struct nouveau_bios *bios)
+{
+	if (bmp_version(bios) >= 0x0300)
+		return nv_ro16(bios, bios->bmp_offset + 28);
+	return 0x0000;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h
new file mode 100644
index 00000000000..c1270548fd0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h
@@ -0,0 +1,27 @@
+#ifndef __NVBIOS_CONN_H__
+#define __NVBIOS_CONN_H__
+
+enum dcb_connector_type {
+	DCB_CONNECTOR_VGA = 0x00,
+	DCB_CONNECTOR_TV_0 = 0x10,
+	DCB_CONNECTOR_TV_1 = 0x11,
+	DCB_CONNECTOR_TV_3 = 0x13,
+	DCB_CONNECTOR_DVI_I = 0x30,
+	DCB_CONNECTOR_DVI_D = 0x31,
+	DCB_CONNECTOR_DMS59_0 = 0x38,
+	DCB_CONNECTOR_DMS59_1 = 0x39,
+	DCB_CONNECTOR_LVDS = 0x40,
+	DCB_CONNECTOR_LVDS_SPWG = 0x41,
+	DCB_CONNECTOR_DP = 0x46,
+	DCB_CONNECTOR_eDP = 0x47,
+	DCB_CONNECTOR_HDMI_0 = 0x60,
+	DCB_CONNECTOR_HDMI_1 = 0x61,
+	DCB_CONNECTOR_DMS59_DP0 = 0x64,
+	DCB_CONNECTOR_DMS59_DP1 = 0x65,
+	DCB_CONNECTOR_NONE = 0xff
+};
+
+u16 dcb_conntab(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 dcb_conn(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/dcb.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/dcb.h
new file mode 100644
index 00000000000..d682fb62583
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/dcb.h
@@ -0,0 +1,90 @@
+#ifndef __NVBIOS_DCB_H__
+#define __NVBIOS_DCB_H__
+
+struct nouveau_bios;
+
+enum dcb_output_type {
+	DCB_OUTPUT_ANALOG	= 0x0,
+	DCB_OUTPUT_TV		= 0x1,
+	DCB_OUTPUT_TMDS		= 0x2,
+	DCB_OUTPUT_LVDS		= 0x3,
+	DCB_OUTPUT_DP		= 0x6,
+	DCB_OUTPUT_EOL		= 0xe,
+	DCB_OUTPUT_UNUSED	= 0xf,
+	DCB_OUTPUT_ANY = -1,
+};
+
+struct dcb_output {
+	int index;	/* may not be raw dcb index if merging has happened */
+	enum dcb_output_type type;
+	uint8_t i2c_index;
+	uint8_t heads;
+	uint8_t connector;
+	uint8_t bus;
+	uint8_t location;
+	uint8_t or;
+	bool duallink_possible;
+	union {
+		struct sor_conf {
+			int link;
+		} sorconf;
+		struct {
+			int maxfreq;
+		} crtconf;
+		struct {
+			struct sor_conf sor;
+			bool use_straps_for_mode;
+			bool use_acpi_for_edid;
+			bool use_power_scripts;
+		} lvdsconf;
+		struct {
+			bool has_component_output;
+		} tvconf;
+		struct {
+			struct sor_conf sor;
+			int link_nr;
+			int link_bw;
+		} dpconf;
+		struct {
+			struct sor_conf sor;
+			int slave_addr;
+		} tmdsconf;
+	};
+	bool i2c_upper_default;
+};
+
+u16 dcb_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *ent, u8 *len);
+u16 dcb_outp(struct nouveau_bios *, u8 idx, u8 *ver, u8 *len);
+int dcb_outp_foreach(struct nouveau_bios *, void *data, int (*exec)
+		     (struct nouveau_bios *, void *, int index, u16 entry));
+
+
+/* BIT 'U'/'d' table encoder subtables have hashes matching them to
+ * a particular set of encoders.
+ *
+ * This function returns true if a particular DCB entry matches.
+ */
+static inline bool
+dcb_hash_match(struct dcb_output *dcb, u32 hash)
+{
+	if ((hash & 0x000000f0) != (dcb->location << 4))
+		return false;
+	if ((hash & 0x0000000f) != dcb->type)
+		return false;
+	if (!(hash & (dcb->or << 16)))
+		return false;
+
+	switch (dcb->type) {
+	case DCB_OUTPUT_TMDS:
+	case DCB_OUTPUT_LVDS:
+	case DCB_OUTPUT_DP:
+		if (hash & 0x00c00000) {
+			if (!(hash & (dcb->sorconf.link << 22)))
+				return false;
+		}
+	default:
+		return true;
+	}
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/dp.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/dp.h
new file mode 100644
index 00000000000..73b5e5d3e75
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/dp.h
@@ -0,0 +1,8 @@
+#ifndef __NVBIOS_DP_H__
+#define __NVBIOS_DP_H__
+
+u16 dp_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 dp_outp(struct nouveau_bios *, u8 idx, u8 *ver, u8 *len);
+u16 dp_outp_match(struct nouveau_bios *, struct dcb_output *, u8 *ver, u8 *len);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/extdev.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/extdev.h
new file mode 100644
index 00000000000..949fee3af8f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/extdev.h
@@ -0,0 +1,30 @@
+#ifndef __NVBIOS_EXTDEV_H__
+#define __NVBIOS_EXTDEV_H__
+
+struct nouveau_bios;
+
+enum nvbios_extdev_type {
+	NVBIOS_EXTDEV_LM89		= 0x02,
+	NVBIOS_EXTDEV_VT1103M		= 0x40,
+	NVBIOS_EXTDEV_PX3540		= 0x41,
+	NVBIOS_EXTDEV_VT1105M		= 0x42, /* or close enough... */
+	NVBIOS_EXTDEV_ADT7473		= 0x70, /* can also be a LM64 */
+	NVBIOS_EXTDEV_HDCP_EEPROM	= 0x90,
+	NVBIOS_EXTDEV_NONE		= 0xff,
+};
+
+struct nvbios_extdev_func {
+	u8 type;
+	u8 addr;
+	u8 bus;
+};
+
+int
+nvbios_extdev_parse(struct nouveau_bios *, int, struct nvbios_extdev_func *);
+
+int
+nvbios_extdev_find(struct nouveau_bios *, enum nvbios_extdev_type,
+		   struct nvbios_extdev_func *);
+
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h
new file mode 100644
index 00000000000..2bf178082a3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h
@@ -0,0 +1,33 @@
+#ifndef __NVBIOS_GPIO_H__
+#define __NVBIOS_GPIO_H__
+
+struct nouveau_bios;
+
+enum dcb_gpio_func_name {
+	DCB_GPIO_PANEL_POWER = 0x01,
+	DCB_GPIO_TVDAC0 = 0x0c,
+	DCB_GPIO_TVDAC1 = 0x2d,
+	DCB_GPIO_PWM_FAN = 0x09,
+	DCB_GPIO_FAN_SENSE = 0x3d,
+	DCB_GPIO_UNUSED = 0xff
+};
+
+struct dcb_gpio_func {
+	u8 func;
+	u8 line;
+	u8 log[2];
+
+	/* so far, "param" seems to only have an influence on PWM-related
+	 * GPIOs such as FAN_CONTROL and PANEL_BACKLIGHT_LEVEL.
+	 * if param equals 1, hardware PWM is available
+	 * if param equals 0, the host should toggle the GPIO itself
+	 */
+	u8 param;
+};
+
+u16 dcb_gpio_table(struct nouveau_bios *);
+u16 dcb_gpio_entry(struct nouveau_bios *, int idx, int ent, u8 *ver);
+int dcb_gpio_parse(struct nouveau_bios *, int idx, u8 func, u8 line,
+		   struct dcb_gpio_func *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
new file mode 100644
index 00000000000..5079bedfd98
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/i2c.h
@@ -0,0 +1,25 @@
+#ifndef __NVBIOS_I2C_H__
+#define __NVBIOS_I2C_H__
+
+struct nouveau_bios;
+
+enum dcb_i2c_type {
+	DCB_I2C_NV04_BIT = 0,
+	DCB_I2C_NV4E_BIT = 4,
+	DCB_I2C_NVIO_BIT = 5,
+	DCB_I2C_NVIO_AUX = 6,
+	DCB_I2C_UNUSED = 0xff
+};
+
+struct dcb_i2c_entry {
+	enum dcb_i2c_type type;
+	u8 drive;
+	u8 sense;
+	u32 data;
+};
+
+u16 dcb_i2c_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 dcb_i2c_entry(struct nouveau_bios *, u8 index, u8 *ver, u8 *len);
+int dcb_i2c_parse(struct nouveau_bios *, u8 index, struct dcb_i2c_entry *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/init.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/init.h
new file mode 100644
index 00000000000..e69a8bdc6e9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/init.h
@@ -0,0 +1,21 @@
+#ifndef __NVBIOS_INIT_H__
+#define __NVBIOS_INIT_H__
+
+struct nvbios_init {
+	struct nouveau_subdev *subdev;
+	struct nouveau_bios *bios;
+	u16 offset;
+	struct dcb_output *outp;
+	int crtc;
+
+	/* internal state used during parsing */
+	u8 execute;
+	u32 nested;
+	u16 repeat;
+	u16 repend;
+};
+
+int nvbios_exec(struct nvbios_init *);
+int nvbios_init(struct nouveau_subdev *, bool execute);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/mxm.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/mxm.h
new file mode 100644
index 00000000000..5572e60414e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/mxm.h
@@ -0,0 +1,9 @@
+#ifndef __NVBIOS_MXM_H__
+#define __NVBIOS_MXM_H__
+
+u16 mxm_table(struct nouveau_bios *, u8 *ver, u8 *hdr);
+
+u8  mxm_sor_map(struct nouveau_bios *, u8 conn);
+u8  mxm_ddc_map(struct nouveau_bios *, u8 port);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h
new file mode 100644
index 00000000000..0b285e99be5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h
@@ -0,0 +1,14 @@
+#ifndef __NVBIOS_PERF_H__
+#define __NVBIOS_PERF_H__
+
+struct nouveau_bios;
+
+struct nvbios_perf_fan {
+	u32 pwm_divisor;
+};
+
+int
+nvbios_perf_fan_parse(struct nouveau_bios *, struct nvbios_perf_fan *);
+
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/pll.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pll.h
new file mode 100644
index 00000000000..c345097592f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/pll.h
@@ -0,0 +1,77 @@
+#ifndef __NVBIOS_PLL_H__
+#define __NVBIOS_PLL_H__
+
+/*XXX: kill me */
+struct nouveau_pll_vals {
+	union {
+		struct {
+#ifdef __BIG_ENDIAN
+			uint8_t N1, M1, N2, M2;
+#else
+			uint8_t M1, N1, M2, N2;
+#endif
+		};
+		struct {
+			uint16_t NM1, NM2;
+		} __attribute__((packed));
+	};
+	int log2P;
+
+	int refclk;
+};
+
+struct nouveau_bios;
+
+/* these match types in pll limits table version 0x40,
+ * nouveau uses them on all chipsets internally where a
+ * specific pll needs to be referenced, but the exact
+ * register isn't known.
+ */
+enum nvbios_pll_type {
+	PLL_CORE   = 0x01,
+	PLL_SHADER = 0x02,
+	PLL_UNK03  = 0x03,
+	PLL_MEMORY = 0x04,
+	PLL_VDEC   = 0x05,
+	PLL_UNK40  = 0x40,
+	PLL_UNK41  = 0x41,
+	PLL_UNK42  = 0x42,
+	PLL_VPLL0  = 0x80,
+	PLL_VPLL1  = 0x81,
+	PLL_MAX    = 0xff
+};
+
+struct nvbios_pll {
+	enum nvbios_pll_type type;
+	u32 reg;
+	u32 refclk;
+
+	u8 min_p;
+	u8 max_p;
+	u8 bias_p;
+
+	/*
+	 * for most pre nv50 cards setting a log2P of 7 (the common max_log2p
+	 * value) is no different to 6 (at least for vplls) so allowing the MNP
+	 * calc to use 7 causes the generated clock to be out by a factor of 2.
+	 * however, max_log2p cannot be fixed-up during parsing as the
+	 * unmodified max_log2p value is still needed for setting mplls, hence
+	 * an additional max_usable_log2p member
+	 */
+	u8 max_p_usable;
+
+	struct {
+		u32 min_freq;
+		u32 max_freq;
+		u32 min_inputfreq;
+		u32 max_inputfreq;
+		u8  min_m;
+		u8  max_m;
+		u8  min_n;
+		u8  max_n;
+	} vco1, vco2;
+};
+
+int nvbios_pll_parse(struct nouveau_bios *, u32 type, struct nvbios_pll *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h
new file mode 100644
index 00000000000..a2c4296fc5f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h
@@ -0,0 +1,46 @@
+#ifndef __NVBIOS_THERM_H__
+#define __NVBIOS_THERM_H__
+
+struct nouveau_bios;
+
+struct nvbios_therm_threshold {
+	u8 temp;
+	u8 hysteresis;
+};
+
+struct nvbios_therm_sensor {
+	/* diode */
+	s16 slope_mult;
+	s16 slope_div;
+	s16 offset_num;
+	s16 offset_den;
+	s8 offset_constant;
+
+	/* thresholds */
+	struct nvbios_therm_threshold thrs_fan_boost;
+	struct nvbios_therm_threshold thrs_down_clock;
+	struct nvbios_therm_threshold thrs_critical;
+	struct nvbios_therm_threshold thrs_shutdown;
+};
+
+struct nvbios_therm_fan {
+	u16 pwm_freq;
+
+	u8 min_duty;
+	u8 max_duty;
+};
+
+enum nvbios_therm_domain {
+	NVBIOS_THERM_DOMAIN_CORE,
+	NVBIOS_THERM_DOMAIN_AMBIENT,
+};
+
+int
+nvbios_therm_sensor_parse(struct nouveau_bios *, enum nvbios_therm_domain,
+			  struct nvbios_therm_sensor *);
+
+int
+nvbios_therm_fan_parse(struct nouveau_bios *, struct nvbios_therm_fan *);
+
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/clock.h b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
new file mode 100644
index 00000000000..39e73b91d36
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
@@ -0,0 +1,59 @@
+#ifndef __NOUVEAU_CLOCK_H__
+#define __NOUVEAU_CLOCK_H__
+
+#include <core/device.h>
+#include <core/subdev.h>
+
+struct nouveau_pll_vals;
+struct nvbios_pll;
+
+struct nouveau_clock {
+	struct nouveau_subdev base;
+
+	int (*pll_set)(struct nouveau_clock *, u32 type, u32 freq);
+
+	/*XXX: die, these are here *only* to support the completely
+	 *     bat-shit insane what-was-nouveau_hw.c code
+	 */
+	int (*pll_calc)(struct nouveau_clock *, struct nvbios_pll *,
+			int clk, struct nouveau_pll_vals *pv);
+	int (*pll_prog)(struct nouveau_clock *, u32 reg1,
+			struct nouveau_pll_vals *pv);
+};
+
+static inline struct nouveau_clock *
+nouveau_clock(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_CLOCK];
+}
+
+#define nouveau_clock_create(p,e,o,d)                                          \
+	nouveau_subdev_create((p), (e), (o), 0, "CLOCK", "clock", d)
+#define nouveau_clock_destroy(p)                                               \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_clock_init(p)                                                  \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_clock_fini(p,s)                                                \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+int  nouveau_clock_create_(struct nouveau_object *, struct nouveau_object *,
+			   struct nouveau_oclass *, void *, u32, int, void **);
+
+#define _nouveau_clock_dtor _nouveau_subdev_dtor
+#define _nouveau_clock_init _nouveau_subdev_init
+#define _nouveau_clock_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv04_clock_oclass;
+extern struct nouveau_oclass nv40_clock_oclass;
+extern struct nouveau_oclass nv50_clock_oclass;
+extern struct nouveau_oclass nva3_clock_oclass;
+extern struct nouveau_oclass nvc0_clock_oclass;
+
+int nv04_clock_pll_set(struct nouveau_clock *, u32 type, u32 freq);
+int nv04_clock_pll_calc(struct nouveau_clock *, struct nvbios_pll *,
+			int clk, struct nouveau_pll_vals *);
+int nv04_clock_pll_prog(struct nouveau_clock *, u32 reg1,
+			struct nouveau_pll_vals *);
+
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/device.h b/drivers/gpu/drm/nouveau/core/include/subdev/device.h
new file mode 100644
index 00000000000..c9e4c4afa50
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/device.h
@@ -0,0 +1,24 @@
+#ifndef __NOUVEAU_SUBDEV_DEVICE_H__
+#define __NOUVEAU_SUBDEV_DEVICE_H__
+
+#include <core/device.h>
+
+#define nouveau_device_create(p,n,s,c,d,u)                                     \
+	nouveau_device_create_((p), (n), (s), (c), (d), sizeof(**u), (void **)u)
+
+int  nouveau_device_create_(struct pci_dev *, u64 name, const char *sname,
+			    const char *cfg, const char *dbg, int, void **);
+
+int nv04_identify(struct nouveau_device *);
+int nv10_identify(struct nouveau_device *);
+int nv20_identify(struct nouveau_device *);
+int nv30_identify(struct nouveau_device *);
+int nv40_identify(struct nouveau_device *);
+int nv50_identify(struct nouveau_device *);
+int nvc0_identify(struct nouveau_device *);
+int nve0_identify(struct nouveau_device *);
+
+extern struct nouveau_oclass nouveau_device_sclass[];
+struct nouveau_device *nouveau_device_find(u64 name);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
new file mode 100644
index 00000000000..29e4cc1f6cc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
@@ -0,0 +1,40 @@
+#ifndef __NOUVEAU_DEVINIT_H__
+#define __NOUVEAU_DEVINIT_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_devinit {
+	struct nouveau_subdev base;
+	bool post;
+	void (*meminit)(struct nouveau_devinit *);
+};
+
+static inline struct nouveau_devinit *
+nouveau_devinit(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_DEVINIT];
+}
+
+#define nouveau_devinit_create(p,e,o,d)                                        \
+	nouveau_devinit_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_devinit_destroy(p)                                             \
+	nouveau_subdev_destroy(&(p)->base)
+
+int nouveau_devinit_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, int, void **);
+int nouveau_devinit_init(struct nouveau_devinit *);
+int nouveau_devinit_fini(struct nouveau_devinit *, bool suspend);
+
+extern struct nouveau_oclass nv04_devinit_oclass;
+extern struct nouveau_oclass nv05_devinit_oclass;
+extern struct nouveau_oclass nv10_devinit_oclass;
+extern struct nouveau_oclass nv1a_devinit_oclass;
+extern struct nouveau_oclass nv20_devinit_oclass;
+extern struct nouveau_oclass nv50_devinit_oclass;
+
+void nv04_devinit_dtor(struct nouveau_object *);
+int  nv04_devinit_init(struct nouveau_object *);
+int  nv04_devinit_fini(struct nouveau_object *, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/fb.h b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
new file mode 100644
index 00000000000..5c1b5e1904f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
@@ -0,0 +1,134 @@
+#ifndef __NOUVEAU_FB_H__
+#define __NOUVEAU_FB_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+#include <core/mm.h>
+
+#include <subdev/vm.h>
+
+/* memory type/access flags, do not match hardware values */
+#define NV_MEM_ACCESS_RO  1
+#define NV_MEM_ACCESS_WO  2
+#define NV_MEM_ACCESS_RW (NV_MEM_ACCESS_RO | NV_MEM_ACCESS_WO)
+#define NV_MEM_ACCESS_SYS 4
+#define NV_MEM_ACCESS_VM  8
+#define NV_MEM_ACCESS_NOSNOOP 16
+
+#define NV_MEM_TARGET_VRAM        0
+#define NV_MEM_TARGET_PCI         1
+#define NV_MEM_TARGET_PCI_NOSNOOP 2
+#define NV_MEM_TARGET_VM          3
+#define NV_MEM_TARGET_GART        4
+
+#define NV_MEM_TYPE_VM 0x7f
+#define NV_MEM_COMP_VM 0x03
+
+struct nouveau_mem {
+	struct drm_device *dev;
+
+	struct nouveau_vma bar_vma;
+	struct nouveau_vma vma[2];
+	u8  page_shift;
+
+	struct nouveau_mm_node *tag;
+	struct list_head regions;
+	dma_addr_t *pages;
+	u32 memtype;
+	u64 offset;
+	u64 size;
+	struct sg_table *sg;
+};
+
+struct nouveau_fb_tile {
+	struct nouveau_mm_node *tag;
+	u32 addr;
+	u32 limit;
+	u32 pitch;
+	u32 zcomp;
+};
+
+struct nouveau_fb {
+	struct nouveau_subdev base;
+
+	bool (*memtype_valid)(struct nouveau_fb *, u32 memtype);
+
+	struct {
+		enum {
+			NV_MEM_TYPE_UNKNOWN = 0,
+			NV_MEM_TYPE_STOLEN,
+			NV_MEM_TYPE_SGRAM,
+			NV_MEM_TYPE_SDRAM,
+			NV_MEM_TYPE_DDR1,
+			NV_MEM_TYPE_DDR2,
+			NV_MEM_TYPE_DDR3,
+			NV_MEM_TYPE_GDDR2,
+			NV_MEM_TYPE_GDDR3,
+			NV_MEM_TYPE_GDDR4,
+			NV_MEM_TYPE_GDDR5
+		} type;
+		u64 stolen;
+		u64 size;
+		int ranks;
+
+		int  (*get)(struct nouveau_fb *, u64 size, u32 align,
+			    u32 size_nc, u32 type, struct nouveau_mem **);
+		void (*put)(struct nouveau_fb *, struct nouveau_mem **);
+	} ram;
+
+	struct nouveau_mm vram;
+	struct nouveau_mm tags;
+
+	struct {
+		struct nouveau_fb_tile region[16];
+		int regions;
+		void (*init)(struct nouveau_fb *, int i, u32 addr, u32 size,
+			     u32 pitch, u32 flags, struct nouveau_fb_tile *);
+		void (*fini)(struct nouveau_fb *, int i,
+			     struct nouveau_fb_tile *);
+		void (*prog)(struct nouveau_fb *, int i,
+			     struct nouveau_fb_tile *);
+	} tile;
+};
+
+static inline struct nouveau_fb *
+nouveau_fb(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_FB];
+}
+
+#define nouveau_fb_create(p,e,c,d)                                             \
+	nouveau_subdev_create((p), (e), (c), 0, "PFB", "fb", (d))
+int  nouveau_fb_created(struct nouveau_fb *);
+void nouveau_fb_destroy(struct nouveau_fb *);
+int  nouveau_fb_init(struct nouveau_fb *);
+#define nouveau_fb_fini(p,s)                                                   \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+void _nouveau_fb_dtor(struct nouveau_object *);
+int  _nouveau_fb_init(struct nouveau_object *);
+#define _nouveau_fb_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv04_fb_oclass;
+extern struct nouveau_oclass nv10_fb_oclass;
+extern struct nouveau_oclass nv20_fb_oclass;
+extern struct nouveau_oclass nv30_fb_oclass;
+extern struct nouveau_oclass nv40_fb_oclass;
+extern struct nouveau_oclass nv50_fb_oclass;
+extern struct nouveau_oclass nvc0_fb_oclass;
+
+struct nouveau_bios;
+int  nouveau_fb_bios_memtype(struct nouveau_bios *);
+
+bool nv04_fb_memtype_valid(struct nouveau_fb *, u32 memtype);
+
+void nv10_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
+
+void nv30_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
+		       u32 pitch, u32 flags, struct nouveau_fb_tile *);
+void nv30_fb_tile_fini(struct nouveau_fb *, int i, struct nouveau_fb_tile *);
+
+void nv50_fb_vram_del(struct nouveau_fb *, struct nouveau_mem **);
+void nv50_fb_trap(struct nouveau_fb *, int display);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h b/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h
new file mode 100644
index 00000000000..9ea2b12cc15
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/gpio.h
@@ -0,0 +1,64 @@
+#ifndef __NOUVEAU_GPIO_H__
+#define __NOUVEAU_GPIO_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/gpio.h>
+
+struct nouveau_gpio {
+	struct nouveau_subdev base;
+
+	/* hardware interfaces */
+	void (*reset)(struct nouveau_gpio *);
+	int  (*drive)(struct nouveau_gpio *, int line, int dir, int out);
+	int  (*sense)(struct nouveau_gpio *, int line);
+	void (*irq_enable)(struct nouveau_gpio *, int line, bool);
+
+	/* software interfaces */
+	int  (*find)(struct nouveau_gpio *, int idx, u8 tag, u8 line,
+		     struct dcb_gpio_func *);
+	int  (*set)(struct nouveau_gpio *, int idx, u8 tag, u8 line, int state);
+	int  (*get)(struct nouveau_gpio *, int idx, u8 tag, u8 line);
+	int  (*irq)(struct nouveau_gpio *, int idx, u8 tag, u8 line, bool on);
+
+	/* interrupt handling */
+	struct list_head isr;
+	spinlock_t lock;
+
+	void (*isr_run)(struct nouveau_gpio *, int idx, u32 mask);
+	int  (*isr_add)(struct nouveau_gpio *, int idx, u8 tag, u8 line,
+			void (*)(void *, int state), void *data);
+	void (*isr_del)(struct nouveau_gpio *, int idx, u8 tag, u8 line,
+			void (*)(void *, int state), void *data);
+};
+
+static inline struct nouveau_gpio *
+nouveau_gpio(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_GPIO];
+}
+
+#define nouveau_gpio_create(p,e,o,d)                                           \
+	nouveau_gpio_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_gpio_destroy(p)                                                \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_gpio_fini(p,s)                                                 \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+int nouveau_gpio_create_(struct nouveau_object *, struct nouveau_object *,
+			 struct nouveau_oclass *, int, void **);
+int nouveau_gpio_init(struct nouveau_gpio *);
+
+extern struct nouveau_oclass nv10_gpio_oclass;
+extern struct nouveau_oclass nv50_gpio_oclass;
+extern struct nouveau_oclass nvd0_gpio_oclass;
+
+void nv50_gpio_dtor(struct nouveau_object *);
+int  nv50_gpio_init(struct nouveau_object *);
+int  nv50_gpio_fini(struct nouveau_object *, bool);
+void nv50_gpio_intr(struct nouveau_subdev *);
+void nv50_gpio_irq_enable(struct nouveau_gpio *, int line, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
new file mode 100644
index 00000000000..b93ab01e378
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
@@ -0,0 +1,60 @@
+#ifndef __NOUVEAU_I2C_H__
+#define __NOUVEAU_I2C_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/i2c.h>
+
+#define NV_I2C_PORT(n)    (0x00 + (n))
+#define NV_I2C_DEFAULT(n) (0x80 + (n))
+
+struct nouveau_i2c_port {
+	struct i2c_adapter adapter;
+	struct nouveau_i2c *i2c;
+	struct i2c_algo_bit_data bit;
+	struct list_head head;
+	u8  index;
+	u8  type;
+	u32 dcb;
+	u32 drive;
+	u32 sense;
+	u32 state;
+};
+
+struct nouveau_i2c {
+	struct nouveau_subdev base;
+
+	struct nouveau_i2c_port *(*find)(struct nouveau_i2c *, u8 index);
+	int (*identify)(struct nouveau_i2c *, int index,
+			const char *what, struct i2c_board_info *,
+			bool (*match)(struct nouveau_i2c_port *,
+				      struct i2c_board_info *));
+	struct list_head ports;
+};
+
+static inline struct nouveau_i2c *
+nouveau_i2c(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_I2C];
+}
+
+extern struct nouveau_oclass nouveau_i2c_oclass;
+
+void nouveau_i2c_drive_scl(void *, int);
+void nouveau_i2c_drive_sda(void *, int);
+int  nouveau_i2c_sense_scl(void *);
+int  nouveau_i2c_sense_sda(void *);
+
+int  nv_rdi2cr(struct nouveau_i2c_port *, u8 addr, u8 reg);
+int  nv_wri2cr(struct nouveau_i2c_port *, u8 addr, u8 reg, u8 val);
+bool nv_probe_i2c(struct nouveau_i2c_port *, u8 addr);
+
+int nv_rdaux(struct nouveau_i2c_port *, u32 addr, u8 *data, u8 size);
+int nv_wraux(struct nouveau_i2c_port *, u32 addr, u8 *data, u8 size);
+
+extern const struct i2c_algorithm nouveau_i2c_bit_algo;
+extern const struct i2c_algorithm nouveau_i2c_aux_algo;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/ibus.h b/drivers/gpu/drm/nouveau/core/include/subdev/ibus.h
new file mode 100644
index 00000000000..88814f159d8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/ibus.h
@@ -0,0 +1,34 @@
+#ifndef __NOUVEAU_IBUS_H__
+#define __NOUVEAU_IBUS_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_ibus {
+	struct nouveau_subdev base;
+};
+
+static inline struct nouveau_ibus *
+nouveau_ibus(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_IBUS];
+}
+
+#define nouveau_ibus_create(p,e,o,d)                                           \
+	nouveau_subdev_create_((p), (e), (o), 0, "PIBUS", "ibus",              \
+			       sizeof(**d), (void **)d)
+#define nouveau_ibus_destroy(p)                                                \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_ibus_init(p)                                                   \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_ibus_fini(p,s)                                                 \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+#define _nouveau_ibus_dtor _nouveau_subdev_dtor
+#define _nouveau_ibus_init _nouveau_subdev_init
+#define _nouveau_ibus_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nvc0_ibus_oclass;
+extern struct nouveau_oclass nve0_ibus_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/instmem.h b/drivers/gpu/drm/nouveau/core/include/subdev/instmem.h
new file mode 100644
index 00000000000..ec7a54e91a0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/instmem.h
@@ -0,0 +1,73 @@
+#ifndef __NOUVEAU_INSTMEM_H__
+#define __NOUVEAU_INSTMEM_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+#include <core/mm.h>
+
+struct nouveau_instobj {
+	struct nouveau_object base;
+	struct list_head head;
+	u32 *suspend;
+	u64 addr;
+	u32 size;
+};
+
+static inline struct nouveau_instobj *
+nv_memobj(void *obj)
+{
+#if CONFIG_NOUVEAU_DEBUG >= NV_DBG_PARANOIA
+	if (unlikely(!nv_iclass(obj, NV_MEMOBJ_CLASS)))
+		nv_assert("BAD CAST -> NvMemObj, %08x", nv_hclass(obj));
+#endif
+	return obj;
+}
+
+#define nouveau_instobj_create(p,e,o,d)                                        \
+	nouveau_instobj_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_instobj_init(p)                                                \
+	nouveau_object_init(&(p)->base)
+#define nouveau_instobj_fini(p,s)                                              \
+	nouveau_object_fini(&(p)->base, (s))
+
+int  nouveau_instobj_create_(struct nouveau_object *, struct nouveau_object *,
+			     struct nouveau_oclass *, int, void **);
+void nouveau_instobj_destroy(struct nouveau_instobj *);
+
+void _nouveau_instobj_dtor(struct nouveau_object *);
+#define _nouveau_instobj_init nouveau_object_init
+#define _nouveau_instobj_fini nouveau_object_fini
+
+struct nouveau_instmem {
+	struct nouveau_subdev base;
+	struct list_head list;
+
+	u32 reserved;
+	int (*alloc)(struct nouveau_instmem *, struct nouveau_object *,
+		     u32 size, u32 align, struct nouveau_object **);
+};
+
+static inline struct nouveau_instmem *
+nouveau_instmem(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_INSTMEM];
+}
+
+#define nouveau_instmem_create(p,e,o,d)                                        \
+	nouveau_instmem_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_instmem_destroy(p)                                             \
+	nouveau_subdev_destroy(&(p)->base)
+int nouveau_instmem_create_(struct nouveau_object *, struct nouveau_object *,
+			    struct nouveau_oclass *, int, void **);
+int nouveau_instmem_init(struct nouveau_instmem *);
+int nouveau_instmem_fini(struct nouveau_instmem *, bool);
+
+#define _nouveau_instmem_dtor _nouveau_subdev_dtor
+int _nouveau_instmem_init(struct nouveau_object *);
+int _nouveau_instmem_fini(struct nouveau_object *, bool);
+
+extern struct nouveau_oclass nv04_instmem_oclass;
+extern struct nouveau_oclass nv40_instmem_oclass;
+extern struct nouveau_oclass nv50_instmem_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h b/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
new file mode 100644
index 00000000000..f351f63bc65
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
@@ -0,0 +1,33 @@
+#ifndef __NOUVEAU_LTCG_H__
+#define __NOUVEAU_LTCG_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_ltcg {
+	struct nouveau_subdev base;
+};
+
+static inline struct nouveau_ltcg *
+nouveau_ltcg(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_LTCG];
+}
+
+#define nouveau_ltcg_create(p,e,o,d)                                           \
+	nouveau_subdev_create_((p), (e), (o), 0, "PLTCG", "level2",            \
+			       sizeof(**d), (void **)d)
+#define nouveau_ltcg_destroy(p)                                                \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_ltcg_init(p)                                                   \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_ltcg_fini(p,s)                                                 \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+#define _nouveau_ltcg_dtor _nouveau_subdev_dtor
+#define _nouveau_ltcg_init _nouveau_subdev_init
+#define _nouveau_ltcg_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nvc0_ltcg_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
new file mode 100644
index 00000000000..fded97cea50
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
@@ -0,0 +1,49 @@
+#ifndef __NOUVEAU_MC_H__
+#define __NOUVEAU_MC_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_mc_intr {
+	u32 stat;
+	u32 unit;
+};
+
+struct nouveau_mc {
+	struct nouveau_subdev base;
+	const struct nouveau_mc_intr *intr_map;
+};
+
+static inline struct nouveau_mc *
+nouveau_mc(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_MC];
+}
+
+#define nouveau_mc_create(p,e,o,d)                                             \
+	nouveau_subdev_create_((p), (e), (o), 0, "PMC", "master",              \
+			       sizeof(**d), (void **)d)
+#define nouveau_mc_destroy(p)                                                  \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_mc_init(p)                                                     \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_mc_fini(p,s)                                                   \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+#define _nouveau_mc_dtor _nouveau_subdev_dtor
+#define _nouveau_mc_init _nouveau_subdev_init
+#define _nouveau_mc_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv04_mc_oclass;
+extern struct nouveau_oclass nv44_mc_oclass;
+extern struct nouveau_oclass nv50_mc_oclass;
+extern struct nouveau_oclass nv98_mc_oclass;
+extern struct nouveau_oclass nvc0_mc_oclass;
+
+void nouveau_mc_intr(struct nouveau_subdev *);
+
+extern const struct nouveau_mc_intr nv04_mc_intr[];
+int nv04_mc_init(struct nouveau_object *);
+int nv50_mc_init(struct nouveau_object *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/mxm.h b/drivers/gpu/drm/nouveau/core/include/subdev/mxm.h
new file mode 100644
index 00000000000..b93b152cb56
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/mxm.h
@@ -0,0 +1,37 @@
+#ifndef __NOUVEAU_MXM_H__
+#define __NOUVEAU_MXM_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+#define MXM_SANITISE_DCB 0x00000001
+
+struct nouveau_mxm {
+	struct nouveau_subdev base;
+	u32 action;
+	u8 *mxms;
+};
+
+static inline struct nouveau_mxm *
+nouveau_mxm(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_MXM];
+}
+
+#define nouveau_mxm_create(p,e,o,d)                                            \
+	nouveau_mxm_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_mxm_init(p)                                                    \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_mxm_fini(p,s)                                                  \
+	nouveau_subdev_fini(&(p)->base, (s))
+int  nouveau_mxm_create_(struct nouveau_object *, struct nouveau_object *,
+			 struct nouveau_oclass *, int, void **);
+void nouveau_mxm_destroy(struct nouveau_mxm *);
+
+#define _nouveau_mxm_dtor _nouveau_subdev_dtor
+#define _nouveau_mxm_init _nouveau_subdev_init
+#define _nouveau_mxm_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv50_mxm_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/therm.h b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
new file mode 100644
index 00000000000..faee569fd45
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
@@ -0,0 +1,58 @@
+#ifndef __NOUVEAU_THERM_H__
+#define __NOUVEAU_THERM_H__
+
+#include <core/device.h>
+#include <core/subdev.h>
+
+enum nouveau_therm_fan_mode {
+	FAN_CONTROL_NONE = 0,
+	FAN_CONTROL_MANUAL = 1,
+	FAN_CONTROL_NR,
+};
+
+enum nouveau_therm_attr_type {
+	NOUVEAU_THERM_ATTR_FAN_MIN_DUTY = 0,
+	NOUVEAU_THERM_ATTR_FAN_MAX_DUTY = 1,
+	NOUVEAU_THERM_ATTR_FAN_MODE = 2,
+
+	NOUVEAU_THERM_ATTR_THRS_FAN_BOOST = 10,
+	NOUVEAU_THERM_ATTR_THRS_FAN_BOOST_HYST = 11,
+	NOUVEAU_THERM_ATTR_THRS_DOWN_CLK = 12,
+	NOUVEAU_THERM_ATTR_THRS_DOWN_CLK_HYST = 13,
+	NOUVEAU_THERM_ATTR_THRS_CRITICAL = 14,
+	NOUVEAU_THERM_ATTR_THRS_CRITICAL_HYST = 15,
+	NOUVEAU_THERM_ATTR_THRS_SHUTDOWN = 16,
+	NOUVEAU_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
+};
+
+struct nouveau_therm {
+	struct nouveau_subdev base;
+
+	int (*fan_get)(struct nouveau_therm *);
+	int (*fan_set)(struct nouveau_therm *, int);
+	int (*fan_sense)(struct nouveau_therm *);
+
+	int (*temp_get)(struct nouveau_therm *);
+
+	int (*attr_get)(struct nouveau_therm *, enum nouveau_therm_attr_type);
+	int (*attr_set)(struct nouveau_therm *,
+			enum nouveau_therm_attr_type, int);
+};
+
+static inline struct nouveau_therm *
+nouveau_therm(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_THERM];
+}
+
+#define nouveau_therm_create(p,e,o,d)                                          \
+	nouveau_subdev_create((p), (e), (o), 0, "THERM", "therm", d)
+#define nouveau_therm_destroy(p)                                               \
+	nouveau_subdev_destroy(&(p)->base)
+
+#define _nouveau_therm_dtor _nouveau_subdev_dtor
+
+extern struct nouveau_oclass nv40_therm_oclass;
+extern struct nouveau_oclass nv50_therm_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/timer.h b/drivers/gpu/drm/nouveau/core/include/subdev/timer.h
new file mode 100644
index 00000000000..49bff901544
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/timer.h
@@ -0,0 +1,53 @@
+#ifndef __NOUVEAU_TIMER_H__
+#define __NOUVEAU_TIMER_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_alarm {
+	struct list_head head;
+	u64 timestamp;
+	void (*func)(struct nouveau_alarm *);
+};
+
+bool nouveau_timer_wait_eq(void *, u64 nsec, u32 addr, u32 mask, u32 data);
+bool nouveau_timer_wait_ne(void *, u64 nsec, u32 addr, u32 mask, u32 data);
+bool nouveau_timer_wait_cb(void *, u64 nsec, bool (*func)(void *), void *data);
+void nouveau_timer_alarm(void *, u32 nsec, struct nouveau_alarm *);
+
+#define NV_WAIT_DEFAULT 2000000000ULL
+#define nv_wait(o,a,m,v)                                                       \
+	nouveau_timer_wait_eq((o), NV_WAIT_DEFAULT, (a), (m), (v))
+#define nv_wait_ne(o,a,m,v)                                                    \
+	nouveau_timer_wait_ne((o), NV_WAIT_DEFAULT, (a), (m), (v))
+#define nv_wait_cb(o,c,d)                                                      \
+	nouveau_timer_wait_cb((o), NV_WAIT_DEFAULT, (c), (d))
+
+struct nouveau_timer {
+	struct nouveau_subdev base;
+	u64  (*read)(struct nouveau_timer *);
+	void (*alarm)(struct nouveau_timer *, u32 time, struct nouveau_alarm *);
+};
+
+static inline struct nouveau_timer *
+nouveau_timer(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_TIMER];
+}
+
+#define nouveau_timer_create(p,e,o,d)                                          \
+	nouveau_subdev_create_((p), (e), (o), 0, "PTIMER", "timer",            \
+			       sizeof(**d), (void **)d)
+#define nouveau_timer_destroy(p)                                               \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_timer_init(p)                                                  \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_timer_fini(p,s)                                                \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+int nouveau_timer_create_(struct nouveau_object *, struct nouveau_engine *,
+			  struct nouveau_oclass *, int size, void **);
+
+extern struct nouveau_oclass nv04_timer_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/vga.h b/drivers/gpu/drm/nouveau/core/include/subdev/vga.h
new file mode 100644
index 00000000000..fee09ad818e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/vga.h
@@ -0,0 +1,30 @@
+#ifndef __NOUVEAU_VGA_H__
+#define __NOUVEAU_VGA_H__
+
+#include <core/os.h>
+
+/* access to various legacy io ports */
+u8   nv_rdport(void *obj, int head, u16 port);
+void nv_wrport(void *obj, int head, u16 port, u8 value);
+
+/* VGA Sequencer */
+u8   nv_rdvgas(void *obj, int head, u8 index);
+void nv_wrvgas(void *obj, int head, u8 index, u8 value);
+
+/* VGA Graphics */
+u8   nv_rdvgag(void *obj, int head, u8 index);
+void nv_wrvgag(void *obj, int head, u8 index, u8 value);
+
+/* VGA CRTC */
+u8   nv_rdvgac(void *obj, int head, u8 index);
+void nv_wrvgac(void *obj, int head, u8 index, u8 value);
+
+/* VGA indexed port access dispatcher */
+u8   nv_rdvgai(void *obj, int head, u16 port, u8 index);
+void nv_wrvgai(void *obj, int head, u16 port, u8 index, u8 value);
+
+bool nv_lockvgac(void *obj, bool lock);
+u8   nv_rdvgaowner(void *obj);
+void nv_wrvgaowner(void *obj, u8);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.h b/drivers/gpu/drm/nouveau/core/include/subdev/vm.h
index a8246e7e4a8..9d595efe667 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/vm.h
@@ -25,10 +25,10 @@
 #ifndef __NOUVEAU_VM_H__
 #define __NOUVEAU_VM_H__
 
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
+#include <core/object.h>
+#include <core/subdev.h>
+#include <core/device.h>
+#include <core/mm.h>
 
 struct nouveau_vm_pgt {
 	struct nouveau_gpuobj *obj[2];
@@ -40,6 +40,9 @@ struct nouveau_vm_pgd {
 	struct nouveau_gpuobj *obj;
 };
 
+struct nouveau_gpuobj;
+struct nouveau_mem;
+
 struct nouveau_vma {
 	struct list_head head;
 	int refcount;
@@ -50,21 +53,30 @@ struct nouveau_vma {
 };
 
 struct nouveau_vm {
-	struct drm_device *dev;
+	struct nouveau_vmmgr *vmm;
 	struct nouveau_mm mm;
 	int refcount;
 
 	struct list_head pgd_list;
-	atomic_t engref[16];
+	atomic_t engref[64]; //NVDEV_SUBDEV_NR];
 
 	struct nouveau_vm_pgt *pgt;
 	u32 fpde;
 	u32 lpde;
+};
+
+struct nouveau_vmmgr {
+	struct nouveau_subdev base;
 
+	u64 limit;
+	u8  dma_bits;
 	u32 pgt_bits;
 	u8  spg_shift;
 	u8  lpg_shift;
 
+	int  (*create)(struct nouveau_vmmgr *, u64 offset, u64 length,
+		       u64 mm_offset, struct nouveau_vm **);
+
 	void (*map_pgt)(struct nouveau_gpuobj *pgd, u32 pde,
 			struct nouveau_gpuobj *pgt[2]);
 	void (*map)(struct nouveau_vma *, struct nouveau_gpuobj *,
@@ -72,16 +84,47 @@ struct nouveau_vm {
 		    u64 phys, u64 delta);
 	void (*map_sg)(struct nouveau_vma *, struct nouveau_gpuobj *,
 		       struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
-
-	void (*map_sg_table)(struct nouveau_vma *, struct nouveau_gpuobj *,
-			     struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
 	void (*unmap)(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt);
 	void (*flush)(struct nouveau_vm *);
 };
 
-/* nouveau_vm.c */
-int  nouveau_vm_new(struct drm_device *, u64 offset, u64 length, u64 mm_offset,
+static inline struct nouveau_vmmgr *
+nouveau_vmmgr(void *obj)
+{
+	return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_VM];
+}
+
+#define nouveau_vmmgr_create(p,e,o,i,f,d)                                      \
+	nouveau_subdev_create((p), (e), (o), 0, (i), (f), (d))
+#define nouveau_vmmgr_destroy(p)                                               \
+	nouveau_subdev_destroy(&(p)->base)
+#define nouveau_vmmgr_init(p)                                                  \
+	nouveau_subdev_init(&(p)->base)
+#define nouveau_vmmgr_fini(p,s)                                                \
+	nouveau_subdev_fini(&(p)->base, (s))
+
+#define _nouveau_vmmgr_dtor _nouveau_subdev_dtor
+#define _nouveau_vmmgr_init _nouveau_subdev_init
+#define _nouveau_vmmgr_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv04_vmmgr_oclass;
+extern struct nouveau_oclass nv41_vmmgr_oclass;
+extern struct nouveau_oclass nv44_vmmgr_oclass;
+extern struct nouveau_oclass nv50_vmmgr_oclass;
+extern struct nouveau_oclass nvc0_vmmgr_oclass;
+
+int  nv04_vm_create(struct nouveau_vmmgr *, u64, u64, u64,
 		    struct nouveau_vm **);
+void nv04_vmmgr_dtor(struct nouveau_object *);
+
+void nv50_vm_flush_engine(struct nouveau_subdev *, int engine);
+void nvc0_vm_flush_engine(struct nouveau_subdev *, u64 addr, int type);
+
+/* nouveau_vm.c */
+int  nouveau_vm_create(struct nouveau_vmmgr *, u64 offset, u64 length,
+		       u64 mm_offset, u32 block, struct nouveau_vm **);
+int  nouveau_vm_new(struct nouveau_device *, u64 offset, u64 length,
+		    u64 mm_offset, struct nouveau_vm **);
 int  nouveau_vm_ref(struct nouveau_vm *, struct nouveau_vm **,
 		    struct nouveau_gpuobj *pgd);
 int  nouveau_vm_get(struct nouveau_vm *, u64 size, u32 page_shift,
@@ -94,26 +137,6 @@ void nouveau_vm_unmap_at(struct nouveau_vma *, u64 offset, u64 length);
 void nouveau_vm_map_sg(struct nouveau_vma *, u64 offset, u64 length,
 		       struct nouveau_mem *);
 void nouveau_vm_map_sg_table(struct nouveau_vma *vma, u64 delta, u64 length,
-			     struct nouveau_mem *mem);
-/* nv50_vm.c */
-void nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
-		     struct nouveau_gpuobj *pgt[2]);
-void nv50_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
-		 struct nouveau_mem *, u32 pte, u32 cnt, u64 phys, u64 delta);
-void nv50_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
-		    struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
-void nv50_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
-void nv50_vm_flush(struct nouveau_vm *);
-void nv50_vm_flush_engine(struct drm_device *, int engine);
-
-/* nvc0_vm.c */
-void nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
-		     struct nouveau_gpuobj *pgt[2]);
-void nvc0_vm_map(struct nouveau_vma *, struct nouveau_gpuobj *,
-		 struct nouveau_mem *, u32 pte, u32 cnt, u64 phys, u64 delta);
-void nvc0_vm_map_sg(struct nouveau_vma *, struct nouveau_gpuobj *,
-		    struct nouveau_mem *, u32 pte, u32 cnt, dma_addr_t *);
-void nvc0_vm_unmap(struct nouveau_gpuobj *, u32 pte, u32 cnt);
-void nvc0_vm_flush(struct nouveau_vm *);
+		     struct nouveau_mem *mem);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/os.h b/drivers/gpu/drm/nouveau/core/os.h
new file mode 100644
index 00000000000..cfe3b9cad15
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/os.h
@@ -0,0 +1,47 @@
+#ifndef __NOUVEAU_OS_H__
+#define __NOUVEAU_OS_H__
+
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/printk.h>
+#include <linux/bitops.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/i2c-algo-bit.h>
+#include <linux/delay.h>
+#include <linux/io-mapping.h>
+#include <linux/vmalloc.h>
+#include <linux/acpi.h>
+#include <linux/dmi.h>
+
+#include <asm/unaligned.h>
+
+static inline int
+ffsll(u64 mask)
+{
+	int i;
+	for (i = 0; i < 64; i++) {
+		if (mask & (1ULL << i))
+			return i + 1;
+	}
+	return 0;
+}
+
+#ifndef ioread32_native
+#ifdef __BIG_ENDIAN
+#define ioread16_native ioread16be
+#define iowrite16_native iowrite16be
+#define ioread32_native  ioread32be
+#define iowrite32_native iowrite32be
+#else /* def __BIG_ENDIAN */
+#define ioread16_native ioread16
+#define iowrite16_native iowrite16
+#define ioread32_native  ioread32
+#define iowrite32_native iowrite32
+#endif /* def __BIG_ENDIAN else */
+#endif /* !ioread32_native */
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/base.c b/drivers/gpu/drm/nouveau/core/subdev/bar/base.c
new file mode 100644
index 00000000000..cd01c533007
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/base.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <subdev/bar.h>
+
+struct nouveau_barobj {
+	struct nouveau_object base;
+	struct nouveau_vma vma;
+	void __iomem *iomem;
+};
+
+static int
+nouveau_barobj_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *mem, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nouveau_bar *bar = (void *)engine;
+	struct nouveau_barobj *barobj;
+	int ret;
+
+	ret = nouveau_object_create(parent, engine, oclass, 0, &barobj);
+	*pobject = nv_object(barobj);
+	if (ret)
+		return ret;
+
+	ret = bar->kmap(bar, mem, NV_MEM_ACCESS_RW, &barobj->vma);
+	if (ret)
+		return ret;
+
+	barobj->iomem = bar->iomem + (u32)barobj->vma.offset;
+	return 0;
+}
+
+static void
+nouveau_barobj_dtor(struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = (void *)object->engine;
+	struct nouveau_barobj *barobj = (void *)object;
+	if (barobj->vma.node)
+		bar->unmap(bar, &barobj->vma);
+	nouveau_object_destroy(&barobj->base);
+}
+
+static u32
+nouveau_barobj_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_barobj *barobj = (void *)object;
+	return ioread32_native(barobj->iomem + addr);
+}
+
+static void
+nouveau_barobj_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nouveau_barobj *barobj = (void *)object;
+	iowrite32_native(data, barobj->iomem + addr);
+}
+
+static struct nouveau_oclass
+nouveau_barobj_oclass = {
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nouveau_barobj_ctor,
+		.dtor = nouveau_barobj_dtor,
+		.init = nouveau_object_init,
+		.fini = nouveau_object_fini,
+		.rd32 = nouveau_barobj_rd32,
+		.wr32 = nouveau_barobj_wr32,
+	},
+};
+
+int
+nouveau_bar_alloc(struct nouveau_bar *bar, struct nouveau_object *parent,
+		  struct nouveau_mem *mem, struct nouveau_object **pobject)
+{
+	struct nouveau_object *engine = nv_object(bar);
+	return nouveau_object_ctor(parent, engine, &nouveau_barobj_oclass,
+				   mem, 0, pobject);
+}
+
+int
+nouveau_bar_create_(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, int length, void **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_bar *bar;
+	int ret;
+
+	ret = nouveau_subdev_create_(parent, engine, oclass, 0, "BARCTL",
+				     "bar", length, pobject);
+	bar = *pobject;
+	if (ret)
+		return ret;
+
+	bar->iomem = ioremap(pci_resource_start(device->pdev, 3),
+			     pci_resource_len(device->pdev, 3));
+	return 0;
+}
+
+void
+nouveau_bar_destroy(struct nouveau_bar *bar)
+{
+	if (bar->iomem)
+		iounmap(bar->iomem);
+	nouveau_subdev_destroy(&bar->base);
+}
+
+void
+_nouveau_bar_dtor(struct nouveau_object *object)
+{
+	struct nouveau_bar *bar = (void *)object;
+	nouveau_bar_destroy(bar);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c
new file mode 100644
index 00000000000..c3acf5b70d9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+
+struct nv50_bar_priv {
+	struct nouveau_bar base;
+	spinlock_t lock;
+	struct nouveau_gpuobj *mem;
+	struct nouveau_gpuobj *pad;
+	struct nouveau_gpuobj *pgd;
+	struct nouveau_vm *bar1_vm;
+	struct nouveau_gpuobj *bar1;
+	struct nouveau_vm *bar3_vm;
+	struct nouveau_gpuobj *bar3;
+};
+
+static int
+nv50_bar_kmap(struct nouveau_bar *bar, struct nouveau_mem *mem,
+	      u32 flags, struct nouveau_vma *vma)
+{
+	struct nv50_bar_priv *priv = (void *)bar;
+	int ret;
+
+	ret = nouveau_vm_get(priv->bar3_vm, mem->size << 12, 12, flags, vma);
+	if (ret)
+		return ret;
+
+	nouveau_vm_map(vma, mem);
+	nv50_vm_flush_engine(nv_subdev(bar), 6);
+	return 0;
+}
+
+static int
+nv50_bar_umap(struct nouveau_bar *bar, struct nouveau_mem *mem,
+	      u32 flags, struct nouveau_vma *vma)
+{
+	struct nv50_bar_priv *priv = (void *)bar;
+	int ret;
+
+	ret = nouveau_vm_get(priv->bar1_vm, mem->size << 12, 12, flags, vma);
+	if (ret)
+		return ret;
+
+	nouveau_vm_map(vma, mem);
+	nv50_vm_flush_engine(nv_subdev(bar), 6);
+	return 0;
+}
+
+static void
+nv50_bar_unmap(struct nouveau_bar *bar, struct nouveau_vma *vma)
+{
+	nouveau_vm_unmap(vma);
+	nv50_vm_flush_engine(nv_subdev(bar), 6);
+	nouveau_vm_put(vma);
+}
+
+static void
+nv50_bar_flush(struct nouveau_bar *bar)
+{
+	struct nv50_bar_priv *priv = (void *)bar;
+	unsigned long flags;
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_wr32(priv, 0x00330c, 0x00000001);
+	if (!nv_wait(priv, 0x00330c, 0x00000002, 0x00000000))
+		nv_warn(priv, "flush timeout\n");
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+void
+nv84_bar_flush(struct nouveau_bar *bar)
+{
+	struct nv50_bar_priv *priv = (void *)bar;
+	unsigned long flags;
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_wr32(bar, 0x070000, 0x00000001);
+	if (!nv_wait(priv, 0x070000, 0x00000002, 0x00000000))
+		nv_warn(priv, "flush timeout\n");
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static int
+nv50_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_object *heap;
+	struct nouveau_vm *vm;
+	struct nv50_bar_priv *priv;
+	u64 start, limit;
+	int ret;
+
+	ret = nouveau_bar_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x20000, 0, NVOBJ_FLAG_HEAP,
+				&priv->mem);
+	heap = nv_object(priv->mem);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, heap, (device->chipset == 0x50) ?
+				 0x1400 : 0x0200, 0, 0, &priv->pad);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, heap, 0x4000, 0, 0, &priv->pgd);
+	if (ret)
+		return ret;
+
+	/* BAR3 */
+	start = 0x0100000000ULL;
+	limit = start + pci_resource_len(device->pdev, 3);
+
+	ret = nouveau_vm_new(device, start, limit, start, &vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, heap, ((limit-- - start) >> 12) * 8,
+				 0x1000, NVOBJ_FLAG_ZERO_ALLOC,
+				 &vm->pgt[0].obj[0]);
+	vm->pgt[0].refcount[0] = 1;
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(vm, &priv->bar3_vm, priv->pgd);
+	nouveau_vm_ref(NULL, &vm, NULL);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, heap, 24, 16, 0, &priv->bar3);
+	if (ret)
+		return ret;
+
+	nv_wo32(priv->bar3, 0x00, 0x7fc00000);
+	nv_wo32(priv->bar3, 0x04, lower_32_bits(limit));
+	nv_wo32(priv->bar3, 0x08, lower_32_bits(start));
+	nv_wo32(priv->bar3, 0x0c, upper_32_bits(limit) << 24 |
+				  upper_32_bits(start));
+	nv_wo32(priv->bar3, 0x10, 0x00000000);
+	nv_wo32(priv->bar3, 0x14, 0x00000000);
+
+	/* BAR1 */
+	start = 0x0000000000ULL;
+	limit = start + pci_resource_len(device->pdev, 1);
+
+	ret = nouveau_vm_new(device, start, limit--, start, &vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(vm, &priv->bar1_vm, priv->pgd);
+	nouveau_vm_ref(NULL, &vm, NULL);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, heap, 24, 16, 0, &priv->bar1);
+	if (ret)
+		return ret;
+
+	nv_wo32(priv->bar1, 0x00, 0x7fc00000);
+	nv_wo32(priv->bar1, 0x04, lower_32_bits(limit));
+	nv_wo32(priv->bar1, 0x08, lower_32_bits(start));
+	nv_wo32(priv->bar1, 0x0c, upper_32_bits(limit) << 24 |
+				  upper_32_bits(start));
+	nv_wo32(priv->bar1, 0x10, 0x00000000);
+	nv_wo32(priv->bar1, 0x14, 0x00000000);
+
+	priv->base.alloc = nouveau_bar_alloc;
+	priv->base.kmap = nv50_bar_kmap;
+	priv->base.umap = nv50_bar_umap;
+	priv->base.unmap = nv50_bar_unmap;
+	if (device->chipset == 0x50)
+		priv->base.flush = nv50_bar_flush;
+	else
+		priv->base.flush = nv84_bar_flush;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static void
+nv50_bar_dtor(struct nouveau_object *object)
+{
+	struct nv50_bar_priv *priv = (void *)object;
+	nouveau_gpuobj_ref(NULL, &priv->bar1);
+	nouveau_vm_ref(NULL, &priv->bar1_vm, priv->pgd);
+	nouveau_gpuobj_ref(NULL, &priv->bar3);
+	if (priv->bar3_vm) {
+		nouveau_gpuobj_ref(NULL, &priv->bar3_vm->pgt[0].obj[0]);
+		nouveau_vm_ref(NULL, &priv->bar3_vm, priv->pgd);
+	}
+	nouveau_gpuobj_ref(NULL, &priv->pgd);
+	nouveau_gpuobj_ref(NULL, &priv->pad);
+	nouveau_gpuobj_ref(NULL, &priv->mem);
+	nouveau_bar_destroy(&priv->base);
+}
+
+static int
+nv50_bar_init(struct nouveau_object *object)
+{
+	struct nv50_bar_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_bar_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000000);
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000100);
+	nv50_vm_flush_engine(nv_subdev(priv), 6);
+
+	nv_wr32(priv, 0x001704, 0x00000000 | priv->mem->addr >> 12);
+	nv_wr32(priv, 0x001704, 0x40000000 | priv->mem->addr >> 12);
+	nv_wr32(priv, 0x001708, 0x80000000 | priv->bar1->node->offset >> 4);
+	nv_wr32(priv, 0x00170c, 0x80000000 | priv->bar3->node->offset >> 4);
+	return 0;
+}
+
+static int
+nv50_bar_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv50_bar_priv *priv = (void *)object;
+	return nouveau_bar_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv50_bar_oclass = {
+	.handle = NV_SUBDEV(BAR, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_bar_ctor,
+		.dtor = nv50_bar_dtor,
+		.init = nv50_bar_init,
+		.fini = nv50_bar_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
new file mode 100644
index 00000000000..77a6fb725d3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
@@ -0,0 +1,215 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+
+struct nvc0_bar_priv {
+	struct nouveau_bar base;
+	spinlock_t lock;
+	struct {
+		struct nouveau_gpuobj *mem;
+		struct nouveau_gpuobj *pgd;
+		struct nouveau_vm *vm;
+	} bar[2];
+};
+
+static int
+nvc0_bar_kmap(struct nouveau_bar *bar, struct nouveau_mem *mem,
+	      u32 flags, struct nouveau_vma *vma)
+{
+	struct nvc0_bar_priv *priv = (void *)bar;
+	int ret;
+
+	ret = nouveau_vm_get(priv->bar[0].vm, mem->size << 12, 12, flags, vma);
+	if (ret)
+		return ret;
+
+	nouveau_vm_map(vma, mem);
+	nvc0_vm_flush_engine(nv_subdev(bar), priv->bar[0].pgd->addr, 5);
+	return 0;
+}
+
+static int
+nvc0_bar_umap(struct nouveau_bar *bar, struct nouveau_mem *mem,
+	      u32 flags, struct nouveau_vma *vma)
+{
+	struct nvc0_bar_priv *priv = (void *)bar;
+	int ret;
+
+	ret = nouveau_vm_get(priv->bar[1].vm, mem->size << 12,
+			     mem->page_shift, flags, vma);
+	if (ret)
+		return ret;
+
+	nouveau_vm_map(vma, mem);
+	nvc0_vm_flush_engine(nv_subdev(bar), priv->bar[1].pgd->addr, 5);
+	return 0;
+}
+
+static void
+nvc0_bar_unmap(struct nouveau_bar *bar, struct nouveau_vma *vma)
+{
+	struct nvc0_bar_priv *priv = (void *)bar;
+	int i = !(vma->vm == priv->bar[0].vm);
+
+	nouveau_vm_unmap(vma);
+	nvc0_vm_flush_engine(nv_subdev(bar), priv->bar[i].pgd->addr, 5);
+	nouveau_vm_put(vma);
+}
+
+static int
+nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct pci_dev *pdev = device->pdev;
+	struct nvc0_bar_priv *priv;
+	struct nouveau_gpuobj *mem;
+	struct nouveau_vm *vm;
+	int ret;
+
+	ret = nouveau_bar_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	/* BAR3 */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0, 0, &priv->bar[0].mem);
+	mem = priv->bar[0].mem;
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x8000, 0, 0, &priv->bar[0].pgd);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_new(device, 0, pci_resource_len(pdev, 3), 0, &vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL,
+				 (pci_resource_len(pdev, 3) >> 12) * 8,
+				 0x1000, NVOBJ_FLAG_ZERO_ALLOC,
+				 &vm->pgt[0].obj[0]);
+	vm->pgt[0].refcount[0] = 1;
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(vm, &priv->bar[0].vm, priv->bar[0].pgd);
+	nouveau_vm_ref(NULL, &vm, NULL);
+	if (ret)
+		return ret;
+
+	nv_wo32(mem, 0x0200, lower_32_bits(priv->bar[0].pgd->addr));
+	nv_wo32(mem, 0x0204, upper_32_bits(priv->bar[0].pgd->addr));
+	nv_wo32(mem, 0x0208, lower_32_bits(pci_resource_len(pdev, 3) - 1));
+	nv_wo32(mem, 0x020c, upper_32_bits(pci_resource_len(pdev, 3) - 1));
+
+	/* BAR1 */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x1000, 0, 0, &priv->bar[1].mem);
+	mem = priv->bar[1].mem;
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL, 0x8000, 0, 0, &priv->bar[1].pgd);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_new(device, 0, pci_resource_len(pdev, 1), 0, &vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_vm_ref(vm, &priv->bar[1].vm, priv->bar[1].pgd);
+	nouveau_vm_ref(NULL, &vm, NULL);
+	if (ret)
+		return ret;
+
+	nv_wo32(mem, 0x0200, lower_32_bits(priv->bar[1].pgd->addr));
+	nv_wo32(mem, 0x0204, upper_32_bits(priv->bar[1].pgd->addr));
+	nv_wo32(mem, 0x0208, lower_32_bits(pci_resource_len(pdev, 1) - 1));
+	nv_wo32(mem, 0x020c, upper_32_bits(pci_resource_len(pdev, 1) - 1));
+
+	priv->base.alloc = nouveau_bar_alloc;
+	priv->base.kmap = nvc0_bar_kmap;
+	priv->base.umap = nvc0_bar_umap;
+	priv->base.unmap = nvc0_bar_unmap;
+	priv->base.flush = nv84_bar_flush;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static void
+nvc0_bar_dtor(struct nouveau_object *object)
+{
+	struct nvc0_bar_priv *priv = (void *)object;
+
+	nouveau_vm_ref(NULL, &priv->bar[1].vm, priv->bar[1].pgd);
+	nouveau_gpuobj_ref(NULL, &priv->bar[1].pgd);
+	nouveau_gpuobj_ref(NULL, &priv->bar[1].mem);
+
+	if (priv->bar[0].vm) {
+		nouveau_gpuobj_ref(NULL, &priv->bar[0].vm->pgt[0].obj[0]);
+		nouveau_vm_ref(NULL, &priv->bar[0].vm, priv->bar[0].pgd);
+	}
+	nouveau_gpuobj_ref(NULL, &priv->bar[0].pgd);
+	nouveau_gpuobj_ref(NULL, &priv->bar[0].mem);
+
+	nouveau_bar_destroy(&priv->base);
+}
+
+static int
+nvc0_bar_init(struct nouveau_object *object)
+{
+	struct nvc0_bar_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_bar_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000000);
+	nv_mask(priv, 0x000200, 0x00000100, 0x00000100);
+	nv_mask(priv, 0x100c80, 0x00000001, 0x00000000);
+
+	nv_wr32(priv, 0x001704, 0x80000000 | priv->bar[1].mem->addr >> 12);
+	nv_wr32(priv, 0x001714, 0xc0000000 | priv->bar[0].mem->addr >> 12);
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_bar_oclass = {
+	.handle = NV_SUBDEV(BAR, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_bar_ctor,
+		.dtor = nvc0_bar_dtor,
+		.init = nvc0_bar_init,
+		.fini = _nouveau_bar_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
new file mode 100644
index 00000000000..2fbb6df697c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
@@ -0,0 +1,479 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/device.h>
+#include <core/subdev.h>
+#include <core/option.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/bmp.h>
+#include <subdev/bios/bit.h>
+
+u8
+nvbios_checksum(const u8 *data, int size)
+{
+	u8 sum = 0;
+	while (size--)
+		sum += *data++;
+	return sum;
+}
+
+u16
+nvbios_findstr(const u8 *data, int size, const char *str, int len)
+{
+	int i, j;
+
+	for (i = 0; i <= (size - len); i++) {
+		for (j = 0; j < len; j++)
+			if ((char)data[i + j] != str[j])
+				break;
+		if (j == len)
+			return i;
+	}
+
+	return 0;
+}
+
+#if defined(__powerpc__)
+static void
+nouveau_bios_shadow_of(struct nouveau_bios *bios)
+{
+	struct pci_dev *pdev = nv_device(bios)->pdev;
+	struct device_node *dn;
+	const u32 *data;
+	int size, i;
+
+	dn = pci_device_to_OF_node(pdev);
+	if (!dn) {
+		nv_info(bios, "Unable to get the OF node\n");
+		return;
+	}
+
+	data = of_get_property(dn, "NVDA,BMP", &size);
+	if (data) {
+		bios->size = size;
+		bios->data = kmalloc(bios->size, GFP_KERNEL);
+		if (bios->data)
+			memcpy(bios->data, data, size);
+	}
+}
+#endif
+
+static void
+nouveau_bios_shadow_pramin(struct nouveau_bios *bios)
+{
+	struct nouveau_device *device = nv_device(bios);
+	u32 bar0 = 0;
+	int i;
+
+	if (device->card_type >= NV_50) {
+		u64 addr = (u64)(nv_rd32(bios, 0x619f04) & 0xffffff00) << 8;
+		if (!addr) {
+			addr  = (u64)nv_rd32(bios, 0x001700) << 16;
+			addr += 0xf0000;
+		}
+
+		bar0 = nv_mask(bios, 0x001700, 0xffffffff, addr >> 16);
+	}
+
+	/* bail if no rom signature */
+	if (nv_rd08(bios, 0x700000) != 0x55 ||
+	    nv_rd08(bios, 0x700001) != 0xaa)
+		goto out;
+
+	bios->size = nv_rd08(bios, 0x700002) * 512;
+	bios->data = kmalloc(bios->size, GFP_KERNEL);
+	if (bios->data) {
+		for (i = 0; i < bios->size; i++)
+			nv_wo08(bios, i, nv_rd08(bios, 0x700000 + i));
+	}
+
+out:
+	if (device->card_type >= NV_50)
+		nv_wr32(bios, 0x001700, bar0);
+}
+
+static void
+nouveau_bios_shadow_prom(struct nouveau_bios *bios)
+{
+	struct nouveau_device *device = nv_device(bios);
+	u32 pcireg, access;
+	u16 pcir;
+	int i;
+
+	/* enable access to rom */
+	if (device->card_type >= NV_50)
+		pcireg = 0x088050;
+	else
+		pcireg = 0x001850;
+	access = nv_mask(bios, pcireg, 0x00000001, 0x00000000);
+
+	/* bail if no rom signature, with a workaround for a PROM reading
+	 * issue on some chipsets.  the first read after a period of
+	 * inactivity returns the wrong result, so retry the first header
+	 * byte a few times before giving up as a workaround
+	 */
+	i = 16;
+	do {
+		if (nv_rd08(bios, 0x300000) == 0x55)
+			break;
+	} while (i--);
+
+	if (!i || nv_rd08(bios, 0x300001) != 0xaa)
+		goto out;
+
+	/* additional check (see note below) - read PCI record header */
+	pcir = nv_rd08(bios, 0x300018) |
+	       nv_rd08(bios, 0x300019) << 8;
+	if (nv_rd08(bios, 0x300000 + pcir) != 'P' ||
+	    nv_rd08(bios, 0x300001 + pcir) != 'C' ||
+	    nv_rd08(bios, 0x300002 + pcir) != 'I' ||
+	    nv_rd08(bios, 0x300003 + pcir) != 'R')
+		goto out;
+
+	/* read entire bios image to system memory */
+	bios->size = nv_rd08(bios, 0x300002) * 512;
+	bios->data = kmalloc(bios->size, GFP_KERNEL);
+	if (bios->data) {
+		for (i = 0; i < bios->size; i++)
+			nv_wo08(bios, i, nv_rd08(bios, 0x300000 + i));
+	}
+
+out:
+	/* disable access to rom */
+	nv_wr32(bios, pcireg, access);
+}
+
+#if defined(CONFIG_ACPI)
+int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
+bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
+#else
+static inline bool
+nouveau_acpi_rom_supported(struct pci_dev *pdev) {
+	return false;
+}
+
+static inline int
+nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len) {
+	return -EINVAL;
+}
+#endif
+
+static void
+nouveau_bios_shadow_acpi(struct nouveau_bios *bios)
+{
+	struct pci_dev *pdev = nv_device(bios)->pdev;
+	int cnt = 65536 / 4096;
+	int ret;
+
+	if (!nouveau_acpi_rom_supported(pdev))
+		return;
+
+	bios->data = kmalloc(65536, GFP_KERNEL);
+	bios->size = 0;
+	if (!bios->data)
+		return;
+
+	while (cnt--) {
+		ret = nouveau_acpi_get_bios_chunk(bios->data, bios->size, 4096);
+		if (ret != 4096)
+			return;
+
+		bios->size += 4096;
+	}
+}
+
+static void
+nouveau_bios_shadow_pci(struct nouveau_bios *bios)
+{
+	struct pci_dev *pdev = nv_device(bios)->pdev;
+	size_t size;
+
+	if (!pci_enable_rom(pdev)) {
+		void __iomem *rom = pci_map_rom(pdev, &size);
+		if (rom && size) {
+			bios->data = kmalloc(size, GFP_KERNEL);
+			if (bios->data) {
+				memcpy_fromio(bios->data, rom, size);
+				bios->size = size;
+			}
+		}
+		if (rom)
+			pci_unmap_rom(pdev, rom);
+
+		pci_disable_rom(pdev);
+	}
+}
+
+static int
+nouveau_bios_score(struct nouveau_bios *bios, const bool writeable)
+{
+	if (!bios->data || bios->data[0] != 0x55 || bios->data[1] != 0xAA) {
+		nv_info(bios, "... signature not found\n");
+		return 0;
+	}
+
+	if (nvbios_checksum(bios->data, bios->data[2] * 512)) {
+		nv_info(bios, "... checksum invalid\n");
+		/* if a ro image is somewhat bad, it's probably all rubbish */
+		return writeable ? 2 : 1;
+	}
+
+	nv_info(bios, "... appears to be valid\n");
+	return 3;
+}
+
+struct methods {
+	const char desc[16];
+	void (*shadow)(struct nouveau_bios *);
+	const bool rw;
+	int score;
+	u32 size;
+	u8 *data;
+};
+
+static int
+nouveau_bios_shadow(struct nouveau_bios *bios)
+{
+	struct methods shadow_methods[] = {
+#if defined(__powerpc__)
+		{ "OpenFirmware", nouveau_bios_shadow_of, true, 0, 0, NULL },
+#endif
+		{ "PRAMIN", nouveau_bios_shadow_pramin, true, 0, 0, NULL },
+		{ "PROM", nouveau_bios_shadow_prom, false, 0, 0, NULL },
+		{ "ACPI", nouveau_bios_shadow_acpi, true, 0, 0, NULL },
+		{ "PCIROM", nouveau_bios_shadow_pci, true, 0, 0, NULL },
+		{}
+	};
+	struct methods *mthd, *best;
+	const struct firmware *fw;
+	const char *optarg;
+	int optlen, ret;
+	char *source;
+
+	optarg = nouveau_stropt(nv_device(bios)->cfgopt, "NvBios", &optlen);
+	source = optarg ? kstrndup(optarg, optlen, GFP_KERNEL) : NULL;
+	if (source) {
+		/* try to match one of the built-in methods */
+		mthd = shadow_methods;
+		do {
+			if (strcasecmp(source, mthd->desc))
+				continue;
+			nv_info(bios, "source: %s\n", mthd->desc);
+
+			mthd->shadow(bios);
+			mthd->score = nouveau_bios_score(bios, mthd->rw);
+			if (mthd->score) {
+				kfree(source);
+				return 0;
+			}
+		} while ((++mthd)->shadow);
+
+		/* attempt to load firmware image */
+		ret = request_firmware(&fw, source, &nv_device(bios)->pdev->dev);
+		if (ret == 0) {
+			bios->size = fw->size;
+			bios->data = kmemdup(fw->data, fw->size, GFP_KERNEL);
+			release_firmware(fw);
+
+			nv_info(bios, "image: %s\n", source);
+			if (nouveau_bios_score(bios, 1)) {
+				kfree(source);
+				return 0;
+			}
+
+			kfree(bios->data);
+			bios->data = NULL;
+		}
+
+		nv_error(bios, "source \'%s\' invalid\n", source);
+		kfree(source);
+	}
+
+	mthd = shadow_methods;
+	do {
+		nv_info(bios, "checking %s for image...\n", mthd->desc);
+		mthd->shadow(bios);
+		mthd->score = nouveau_bios_score(bios, mthd->rw);
+		mthd->size = bios->size;
+		mthd->data = bios->data;
+		bios->data = NULL;
+	} while (mthd->score != 3 && (++mthd)->shadow);
+
+	mthd = shadow_methods;
+	best = mthd;
+	do {
+		if (mthd->score > best->score) {
+			kfree(best->data);
+			best = mthd;
+		}
+	} while ((++mthd)->shadow);
+
+	if (best->score) {
+		nv_info(bios, "using image from %s\n", best->desc);
+		bios->size = best->size;
+		bios->data = best->data;
+		return 0;
+	}
+
+	nv_error(bios, "unable to locate usable image\n");
+	return -EINVAL;
+}
+
+static u8
+nouveau_bios_rd08(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_bios *bios = (void *)object;
+	return bios->data[addr];
+}
+
+static u16
+nouveau_bios_rd16(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_bios *bios = (void *)object;
+	return get_unaligned_le16(&bios->data[addr]);
+}
+
+static u32
+nouveau_bios_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nouveau_bios *bios = (void *)object;
+	return get_unaligned_le32(&bios->data[addr]);
+}
+
+static void
+nouveau_bios_wr08(struct nouveau_object *object, u32 addr, u8 data)
+{
+	struct nouveau_bios *bios = (void *)object;
+	bios->data[addr] = data;
+}
+
+static void
+nouveau_bios_wr16(struct nouveau_object *object, u32 addr, u16 data)
+{
+	struct nouveau_bios *bios = (void *)object;
+	put_unaligned_le16(data, &bios->data[addr]);
+}
+
+static void
+nouveau_bios_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nouveau_bios *bios = (void *)object;
+	put_unaligned_le32(data, &bios->data[addr]);
+}
+
+static int
+nouveau_bios_ctor(struct nouveau_object *parent,
+		  struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nouveau_bios *bios;
+	struct bit_entry bit_i;
+	int ret;
+
+	ret = nouveau_subdev_create(parent, engine, oclass, 0,
+				    "VBIOS", "bios", &bios);
+	*pobject = nv_object(bios);
+	if (ret)
+		return ret;
+
+	ret = nouveau_bios_shadow(bios);
+	if (ret)
+		return ret;
+
+	/* detect type of vbios we're dealing with */
+	bios->bmp_offset = nvbios_findstr(bios->data, bios->size,
+					  "\xff\x7f""NV\0", 5);
+	if (bios->bmp_offset) {
+		nv_info(bios, "BMP version %x.%x\n",
+			bmp_version(bios) >> 8,
+			bmp_version(bios) & 0xff);
+	}
+
+	bios->bit_offset = nvbios_findstr(bios->data, bios->size,
+					  "\xff\xb8""BIT", 5);
+	if (bios->bit_offset)
+		nv_info(bios, "BIT signature found\n");
+
+	/* determine the vbios version number */
+	if (!bit_entry(bios, 'i', &bit_i) && bit_i.length >= 4) {
+		bios->version.major = nv_ro08(bios, bit_i.offset + 3);
+		bios->version.chip  = nv_ro08(bios, bit_i.offset + 2);
+		bios->version.minor = nv_ro08(bios, bit_i.offset + 1);
+		bios->version.micro = nv_ro08(bios, bit_i.offset + 0);
+	} else
+	if (bmp_version(bios)) {
+		bios->version.major = nv_ro08(bios, bios->bmp_offset + 13);
+		bios->version.chip  = nv_ro08(bios, bios->bmp_offset + 12);
+		bios->version.minor = nv_ro08(bios, bios->bmp_offset + 11);
+		bios->version.micro = nv_ro08(bios, bios->bmp_offset + 10);
+	}
+
+	nv_info(bios, "version %02x.%02x.%02x.%02x\n",
+		bios->version.major, bios->version.chip,
+		bios->version.minor, bios->version.micro);
+
+	return 0;
+}
+
+static void
+nouveau_bios_dtor(struct nouveau_object *object)
+{
+	struct nouveau_bios *bios = (void *)object;
+	kfree(bios->data);
+	nouveau_subdev_destroy(&bios->base);
+}
+
+static int
+nouveau_bios_init(struct nouveau_object *object)
+{
+	struct nouveau_bios *bios = (void *)object;
+	return nouveau_subdev_init(&bios->base);
+}
+
+static int
+nouveau_bios_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nouveau_bios *bios = (void *)object;
+	return nouveau_subdev_fini(&bios->base, suspend);
+}
+
+struct nouveau_oclass
+nouveau_bios_oclass = {
+	.handle = NV_SUBDEV(VBIOS, 0x00),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nouveau_bios_ctor,
+		.dtor = nouveau_bios_dtor,
+		.init = nouveau_bios_init,
+		.fini = nouveau_bios_fini,
+		.rd08 = nouveau_bios_rd08,
+		.rd16 = nouveau_bios_rd16,
+		.rd32 = nouveau_bios_rd32,
+		.wr08 = nouveau_bios_wr08,
+		.wr16 = nouveau_bios_wr16,
+		.wr32 = nouveau_bios_wr32,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/bit.c b/drivers/gpu/drm/nouveau/core/subdev/bios/bit.c
new file mode 100644
index 00000000000..1d03a3f2b2d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/bit.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "core/object.h"
+
+#include "subdev/bios.h"
+#include "subdev/bios/bit.h"
+
+int
+bit_entry(struct nouveau_bios *bios, u8 id, struct bit_entry *bit)
+{
+	if (likely(bios->bit_offset)) {
+		u8  entries = nv_ro08(bios, bios->bit_offset + 10);
+		u32 entry   = bios->bit_offset + 12;
+		while (entries--) {
+			if (nv_ro08(bios, entry + 0) == id) {
+				bit->id      = nv_ro08(bios, entry + 0);
+				bit->version = nv_ro08(bios, entry + 1);
+				bit->length  = nv_ro16(bios, entry + 2);
+				bit->offset  = nv_ro16(bios, entry + 4);
+				return 0;
+			}
+
+			entry += nv_ro08(bios, bios->bit_offset + 9);
+		}
+
+		return -ENOENT;
+	}
+
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/conn.c b/drivers/gpu/drm/nouveau/core/subdev/bios/conn.c
new file mode 100644
index 00000000000..5ac010efd95
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/conn.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/device.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/conn.h>
+
+u16
+dcb_conntab(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	u16 dcb = dcb_table(bios, ver, hdr, cnt, len);
+	if (dcb && *ver >= 0x30 && *hdr >= 0x16) {
+		u16 data = nv_ro16(bios, dcb + 0x14);
+		if (data) {
+			*ver = nv_ro08(bios, data + 0);
+			*hdr = nv_ro08(bios, data + 1);
+			*cnt = nv_ro08(bios, data + 2);
+			*len = nv_ro08(bios, data + 3);
+			return data;
+		}
+	}
+	return 0x0000;
+}
+
+u16
+dcb_conn(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len)
+{
+	u8  hdr, cnt;
+	u16 data = dcb_conntab(bios, ver, &hdr, &cnt, len);
+	if (data && idx < cnt)
+		return data + hdr + (idx * *len);
+	return 0x0000;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
new file mode 100644
index 00000000000..9ed6e728a94
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "core/device.h"
+
+#include "subdev/bios.h"
+#include "subdev/bios/dcb.h"
+
+u16
+dcb_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	struct nouveau_device *device = nv_device(bios);
+	u16 dcb = 0x0000;
+
+	if (device->card_type > NV_04)
+		dcb = nv_ro16(bios, 0x36);
+	if (!dcb) {
+		nv_warn(bios, "DCB table not found\n");
+		return dcb;
+	}
+
+	*ver = nv_ro08(bios, dcb);
+
+	if (*ver >= 0x41) {
+		nv_warn(bios, "DCB *ver 0x%02x unknown\n", *ver);
+		return 0x0000;
+	} else
+	if (*ver >= 0x30) {
+		if (nv_ro32(bios, dcb + 6) == 0x4edcbdcb) {
+			*hdr = nv_ro08(bios, dcb + 1);
+			*cnt = nv_ro08(bios, dcb + 2);
+			*len = nv_ro08(bios, dcb + 3);
+			return dcb;
+		}
+	} else
+	if (*ver >= 0x20) {
+		if (nv_ro32(bios, dcb + 4) == 0x4edcbdcb) {
+			u16 i2c = nv_ro16(bios, dcb + 2);
+			*hdr = 8;
+			*cnt = (i2c - dcb) / 8;
+			*len = 8;
+			return dcb;
+		}
+	} else
+	if (*ver >= 0x15) {
+		if (!nv_strncmp(bios, dcb - 7, 7, "DEV_REC")) {
+			u16 i2c = nv_ro16(bios, dcb + 2);
+			*hdr = 4;
+			*cnt = (i2c - dcb) / 10;
+			*len = 10;
+			return dcb;
+		}
+	} else {
+		/*
+		 * v1.4 (some NV15/16, NV11+) seems the same as v1.5, but
+		 * always has the same single (crt) entry, even when tv-out
+		 * present, so the conclusion is this version cannot really
+		 * be used.
+		 *
+		 * v1.2 tables (some NV6/10, and NV15+) normally have the
+		 * same 5 entries, which are not specific to the card and so
+		 * no use.
+		 *
+		 * v1.2 does have an I2C table that read_dcb_i2c_table can
+		 * handle, but cards exist (nv11 in #14821) with a bad i2c
+		 * table pointer, so use the indices parsed in
+		 * parse_bmp_structure.
+		 *
+		 * v1.1 (NV5+, maybe some NV4) is entirely unhelpful
+		 */
+		nv_warn(bios, "DCB contains no useful data\n");
+		return 0x0000;
+	}
+
+	nv_warn(bios, "DCB header validation failed\n");
+	return 0x0000;
+}
+
+u16
+dcb_outp(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len)
+{
+	u8  hdr, cnt;
+	u16 dcb = dcb_table(bios, ver, &hdr, &cnt, len);
+	if (dcb && idx < cnt)
+		return dcb + hdr + (idx * *len);
+	return 0x0000;
+}
+
+int
+dcb_outp_foreach(struct nouveau_bios *bios, void *data,
+		 int (*exec)(struct nouveau_bios *, void *, int, u16))
+{
+	int ret, idx = -1;
+	u8  ver, len;
+	u16 outp;
+
+	while ((outp = dcb_outp(bios, ++idx, &ver, &len))) {
+		if (nv_ro32(bios, outp) == 0x00000000)
+			break; /* seen on an NV11 with DCB v1.5 */
+		if (nv_ro32(bios, outp) == 0xffffffff)
+			break; /* seen on an NV17 with DCB v2.0 */
+
+		if (nv_ro08(bios, outp) == DCB_OUTPUT_UNUSED)
+			continue;
+		if (nv_ro08(bios, outp) == DCB_OUTPUT_EOL)
+			break;
+
+		ret = exec(bios, data, idx, outp);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
new file mode 100644
index 00000000000..3cbc0f3e8d5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+
+#include "subdev/bios.h"
+#include "subdev/bios/bit.h"
+#include "subdev/bios/dcb.h"
+#include "subdev/bios/dp.h"
+
+u16
+dp_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	struct bit_entry bit_d;
+
+	if (!bit_entry(bios, 'd', &bit_d)) {
+		if (bit_d.version == 1) {
+			u16 data = nv_ro16(bios, bit_d.offset);
+			if (data) {
+				*ver = nv_ro08(bios, data + 0);
+				*hdr = nv_ro08(bios, data + 1);
+				*len = nv_ro08(bios, data + 2);
+				*cnt = nv_ro08(bios, data + 3);
+				return data;
+			}
+		}
+	}
+
+	return 0x0000;
+}
+
+u16
+dp_outp(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len)
+{
+	u8  hdr, cnt;
+	u16 table = dp_table(bios, ver, &hdr, &cnt, len);
+	if (table && idx < cnt)
+		return nv_ro16(bios, table + hdr + (idx * *len));
+	return 0xffff;
+}
+
+u16
+dp_outp_match(struct nouveau_bios *bios, struct dcb_output *outp,
+	      u8 *ver, u8 *len)
+{
+	u8  idx = 0;
+	u16 data;
+	while ((data = dp_outp(bios, idx++, ver, len)) != 0xffff) {
+		if (data) {
+			u32 hash = nv_ro32(bios, data);
+			if (dcb_hash_match(outp, hash))
+				return data;
+		}
+	}
+	return 0x0000;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
new file mode 100644
index 00000000000..5afb568b2d6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/extdev.c
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/extdev.h>
+
+static u16
+extdev_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *len, u8 *cnt)
+{
+	u8  dcb_ver, dcb_hdr, dcb_cnt, dcb_len;
+	u16 dcb, extdev = 0;
+
+	dcb = dcb_table(bios, &dcb_ver, &dcb_hdr, &dcb_cnt, &dcb_len);
+	if (!dcb || (dcb_ver != 0x30 && dcb_ver != 0x40))
+		return 0x0000;
+
+	extdev = nv_ro16(bios, dcb + 18);
+	if (!extdev)
+		return 0x0000;
+
+	*ver = nv_ro08(bios, extdev + 0);
+	*hdr = nv_ro08(bios, extdev + 1);
+	*cnt = nv_ro08(bios, extdev + 2);
+	*len = nv_ro08(bios, extdev + 3);
+
+	return extdev + *hdr;
+}
+
+u16
+nvbios_extdev_entry(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+	u8 hdr, cnt;
+	u16 extdev = extdev_table(bios, ver, &hdr, len, &cnt);
+	if (extdev && idx < cnt)
+		return extdev + idx * *len;
+	return 0x0000;
+}
+
+static void
+extdev_parse_entry(struct nouveau_bios *bios, u16 offset,
+			  struct nvbios_extdev_func *entry)
+{
+	entry->type = nv_ro08(bios, offset + 0);
+	entry->addr = nv_ro08(bios, offset + 1);
+	entry->bus = (nv_ro08(bios, offset + 2) >> 4) & 1;
+}
+
+int
+nvbios_extdev_parse(struct nouveau_bios *bios, int idx,
+		    struct nvbios_extdev_func *func)
+{
+	u8 ver, len;
+	u16 entry;
+
+	if (!(entry = nvbios_extdev_entry(bios, idx, &ver, &len)))
+		return -EINVAL;
+
+	extdev_parse_entry(bios, entry, func);
+
+	return 0;
+}
+
+int
+nvbios_extdev_find(struct nouveau_bios *bios, enum nvbios_extdev_type type,
+		   struct nvbios_extdev_func *func)
+{
+	u8 ver, len, i;
+	u16 entry;
+
+	i = 0;
+	while (!(entry = nvbios_extdev_entry(bios, i++, &ver, &len))) {
+		extdev_parse_entry(bios, entry, func);
+		if (func->type == type)
+			return 0;
+	}
+
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/gpio.c b/drivers/gpu/drm/nouveau/core/subdev/bios/gpio.c
new file mode 100644
index 00000000000..4c9f1e50816
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/gpio.c
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/gpio.h>
+
+u16
+dcb_gpio_table(struct nouveau_bios *bios)
+{
+	u8  ver, hdr, cnt, len;
+	u16 dcb = dcb_table(bios, &ver, &hdr, &cnt, &len);
+	if (dcb) {
+		if (ver >= 0x30 && hdr >= 0x0c)
+			return nv_ro16(bios, dcb + 0x0a);
+		if (ver >= 0x22 && nv_ro08(bios, dcb - 1) >= 0x13)
+			return nv_ro16(bios, dcb - 0x0f);
+	}
+	return 0x0000;
+}
+
+u16
+dcb_gpio_entry(struct nouveau_bios *bios, int idx, int ent, u8 *ver)
+{
+	u16 gpio = dcb_gpio_table(bios);
+	if (gpio) {
+		*ver = nv_ro08(bios, gpio);
+		if (*ver < 0x30 && ent < nv_ro08(bios, gpio + 2))
+			return gpio + 3 + (ent * nv_ro08(bios, gpio + 1));
+		else if (ent < nv_ro08(bios, gpio + 2))
+			return gpio + nv_ro08(bios, gpio + 1) +
+			       (ent * nv_ro08(bios, gpio + 3));
+	}
+	return 0x0000;
+}
+
+int
+dcb_gpio_parse(struct nouveau_bios *bios, int idx, u8 func, u8 line,
+	       struct dcb_gpio_func *gpio)
+{
+	u8  ver, hdr, cnt, len;
+	u16 entry;
+	int i = -1;
+
+	while ((entry = dcb_gpio_entry(bios, idx, ++i, &ver))) {
+		if (ver < 0x40) {
+			u16 data = nv_ro16(bios, entry);
+			*gpio = (struct dcb_gpio_func) {
+				.line = (data & 0x001f) >> 0,
+				.func = (data & 0x07e0) >> 5,
+				.log[0] = (data & 0x1800) >> 11,
+				.log[1] = (data & 0x6000) >> 13,
+				.param = !!(data & 0x8000),
+			};
+		} else
+		if (ver < 0x41) {
+			u32 data = nv_ro32(bios, entry);
+			*gpio = (struct dcb_gpio_func) {
+				.line = (data & 0x0000001f) >> 0,
+				.func = (data & 0x0000ff00) >> 8,
+				.log[0] = (data & 0x18000000) >> 27,
+				.log[1] = (data & 0x60000000) >> 29,
+				.param = !!(data & 0x80000000),
+			};
+		} else {
+			u32 data = nv_ro32(bios, entry + 0);
+			u8 data1 = nv_ro32(bios, entry + 4);
+			*gpio = (struct dcb_gpio_func) {
+				.line = (data & 0x0000003f) >> 0,
+				.func = (data & 0x0000ff00) >> 8,
+				.log[0] = (data1 & 0x30) >> 4,
+				.log[1] = (data1 & 0xc0) >> 6,
+				.param = !!(data & 0x80000000),
+			};
+		}
+
+		if ((line == 0xff || line == gpio->line) &&
+		    (func == 0xff || func == gpio->func))
+			return 0;
+	}
+
+	/* DCB 2.2, fixed TVDAC GPIO data */
+	if ((entry = dcb_table(bios, &ver, &hdr, &cnt, &len)) && ver >= 0x22) {
+		if (func == DCB_GPIO_TVDAC0) {
+			u8 conf = nv_ro08(bios, entry - 5);
+			u8 addr = nv_ro08(bios, entry - 4);
+			if (conf & 0x01) {
+				*gpio = (struct dcb_gpio_func) {
+					.func = DCB_GPIO_TVDAC0,
+					.line = addr >> 4,
+					.log[0] = !!(conf & 0x02),
+					.log[1] =  !(conf & 0x02),
+				};
+				return 0;
+			}
+		}
+	}
+
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
new file mode 100644
index 00000000000..ad577db8376
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/i2c.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+
+#include "subdev/bios.h"
+#include "subdev/bios/dcb.h"
+#include "subdev/bios/i2c.h"
+
+u16
+dcb_i2c_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	u16 i2c = 0x0000;
+	u16 dcb = dcb_table(bios, ver, hdr, cnt, len);
+	if (dcb) {
+		if (*ver >= 0x15)
+			i2c = nv_ro16(bios, dcb + 2);
+		if (*ver >= 0x30)
+			i2c = nv_ro16(bios, dcb + 4);
+	}
+
+	if (i2c && *ver >= 0x30) {
+		*ver = nv_ro08(bios, i2c + 0);
+		*hdr = nv_ro08(bios, i2c + 1);
+		*cnt = nv_ro08(bios, i2c + 2);
+		*len = nv_ro08(bios, i2c + 3);
+	} else {
+		*ver = *ver; /* use DCB version */
+		*hdr = 0;
+		*cnt = 16;
+		*len = 4;
+	}
+
+	return i2c;
+}
+
+u16
+dcb_i2c_entry(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len)
+{
+	u8  hdr, cnt;
+	u16 i2c = dcb_i2c_table(bios, ver, &hdr, &cnt, len);
+	if (i2c && idx < cnt)
+		return i2c + hdr + (idx * *len);
+	return 0x0000;
+}
+
+int
+dcb_i2c_parse(struct nouveau_bios *bios, u8 idx, struct dcb_i2c_entry *info)
+{
+	u8  ver, len;
+	u16 ent = dcb_i2c_entry(bios, idx, &ver, &len);
+	if (ent) {
+		info->data = nv_ro32(bios, ent + 0);
+		info->type = nv_ro08(bios, ent + 3);
+		if (ver < 0x30) {
+			info->type &= 0x07;
+			if (info->type == 0x07)
+				info->type = 0xff;
+		}
+
+		switch (info->type) {
+		case DCB_I2C_NV04_BIT:
+			info->drive = nv_ro08(bios, ent + 0);
+			info->sense = nv_ro08(bios, ent + 1);
+			return 0;
+		case DCB_I2C_NV4E_BIT:
+			info->drive = nv_ro08(bios, ent + 1);
+			return 0;
+		case DCB_I2C_NVIO_BIT:
+		case DCB_I2C_NVIO_AUX:
+			info->drive = nv_ro08(bios, ent + 0);
+			return 0;
+		case DCB_I2C_UNUSED:
+			return 0;
+		default:
+			nv_warn(bios, "unknown i2c type %d\n", info->type);
+			info->type = DCB_I2C_UNUSED;
+			return 0;
+		}
+	}
+
+	if (bios->bmp_offset && idx < 2) {
+		/* BMP (from v4.0 has i2c info in the structure, it's in a
+		 * fixed location on earlier VBIOS
+		 */
+		if (nv_ro08(bios, bios->bmp_offset + 5) < 4)
+			ent = 0x0048;
+		else
+			ent = 0x0036 + bios->bmp_offset;
+
+		if (idx == 0) {
+			info->drive = nv_ro08(bios, ent + 4);
+			if (!info->drive) info->drive = 0x3f;
+			info->sense = nv_ro08(bios, ent + 5);
+			if (!info->sense) info->sense = 0x3e;
+		} else
+		if (idx == 1) {
+			info->drive = nv_ro08(bios, ent + 6);
+			if (!info->drive) info->drive = 0x37;
+			info->sense = nv_ro08(bios, ent + 7);
+			if (!info->sense) info->sense = 0x36;
+		}
+
+		info->type = DCB_I2C_NV04_BIT;
+		return 0;
+	}
+
+	return -ENOENT;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
new file mode 100644
index 00000000000..6be8c32f6e4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
@@ -0,0 +1,2120 @@
+#include <core/engine.h>
+#include <core/device.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/conn.h>
+#include <subdev/bios/bmp.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/dp.h>
+#include <subdev/bios/init.h>
+#include <subdev/devinit.h>
+#include <subdev/clock.h>
+#include <subdev/i2c.h>
+#include <subdev/vga.h>
+#include <subdev/gpio.h>
+
+#define bioslog(lvl, fmt, args...) do {                                        \
+	nv_printk(init->bios, lvl, "0x%04x[%c]: "fmt, init->offset,            \
+		  init_exec(init) ? '0' + (init->nested - 1) : ' ', ##args);   \
+} while(0)
+#define cont(fmt, args...) do {                                                \
+	if (nv_subdev(init->bios)->debug >= NV_DBG_TRACE)                      \
+		printk(fmt, ##args);                                           \
+} while(0)
+#define trace(fmt, args...) bioslog(TRACE, fmt, ##args)
+#define warn(fmt, args...) bioslog(WARN, fmt, ##args)
+#define error(fmt, args...) bioslog(ERROR, fmt, ##args)
+
+/******************************************************************************
+ * init parser control flow helpers
+ *****************************************************************************/
+
+static inline bool
+init_exec(struct nvbios_init *init)
+{
+	return (init->execute == 1) || ((init->execute & 5) == 5);
+}
+
+static inline void
+init_exec_set(struct nvbios_init *init, bool exec)
+{
+	if (exec) init->execute &= 0xfd;
+	else      init->execute |= 0x02;
+}
+
+static inline void
+init_exec_inv(struct nvbios_init *init)
+{
+	init->execute ^= 0x02;
+}
+
+static inline void
+init_exec_force(struct nvbios_init *init, bool exec)
+{
+	if (exec) init->execute |= 0x04;
+	else      init->execute &= 0xfb;
+}
+
+/******************************************************************************
+ * init parser wrappers for normal register/i2c/whatever accessors
+ *****************************************************************************/
+
+static inline int
+init_or(struct nvbios_init *init)
+{
+	if (init->outp)
+		return ffs(init->outp->or) - 1;
+	error("script needs OR!!\n");
+	return 0;
+}
+
+static inline int
+init_link(struct nvbios_init *init)
+{
+	if (init->outp)
+		return !(init->outp->sorconf.link & 1);
+	error("script needs OR link\n");
+	return 0;
+}
+
+static inline int
+init_crtc(struct nvbios_init *init)
+{
+	if (init->crtc >= 0)
+		return init->crtc;
+	error("script needs crtc\n");
+	return 0;
+}
+
+static u8
+init_conn(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+
+	if (init->outp) {
+		u8  ver, len;
+		u16 conn = dcb_conn(bios, init->outp->connector, &ver, &len);
+		if (conn)
+			return nv_ro08(bios, conn);
+	}
+
+	error("script needs connector type\n");
+	return 0x00;
+}
+
+static inline u32
+init_nvreg(struct nvbios_init *init, u32 reg)
+{
+	/* C51 (at least) sometimes has the lower bits set which the VBIOS
+	 * interprets to mean that access needs to go through certain IO
+	 * ports instead.  The NVIDIA binary driver has been seen to access
+	 * these through the NV register address, so lets assume we can
+	 * do the same
+	 */
+	reg &= ~0x00000003;
+
+	/* GF8+ display scripts need register addresses mangled a bit to
+	 * select a specific CRTC/OR
+	 */
+	if (nv_device(init->bios)->card_type >= NV_50) {
+		if (reg & 0x80000000) {
+			reg += init_crtc(init) * 0x800;
+			reg &= ~0x80000000;
+		}
+
+		if (reg & 0x40000000) {
+			reg += init_or(init) * 0x800;
+			reg &= ~0x40000000;
+			if (reg & 0x20000000) {
+				reg += init_link(init) * 0x80;
+				reg &= ~0x20000000;
+			}
+		}
+	}
+
+	if (reg & ~0x00fffffc)
+		warn("unknown bits in register 0x%08x\n", reg);
+	return reg;
+}
+
+static u32
+init_rd32(struct nvbios_init *init, u32 reg)
+{
+	reg = init_nvreg(init, reg);
+	if (init_exec(init))
+		return nv_rd32(init->subdev, reg);
+	return 0x00000000;
+}
+
+static void
+init_wr32(struct nvbios_init *init, u32 reg, u32 val)
+{
+	reg = init_nvreg(init, reg);
+	if (init_exec(init))
+		nv_wr32(init->subdev, reg, val);
+}
+
+static u32
+init_mask(struct nvbios_init *init, u32 reg, u32 mask, u32 val)
+{
+	reg = init_nvreg(init, reg);
+	if (init_exec(init)) {
+		u32 tmp = nv_rd32(init->subdev, reg);
+		nv_wr32(init->subdev, reg, (tmp & ~mask) | val);
+		return tmp;
+	}
+	return 0x00000000;
+}
+
+static u8
+init_rdport(struct nvbios_init *init, u16 port)
+{
+	if (init_exec(init))
+		return nv_rdport(init->subdev, init->crtc, port);
+	return 0x00;
+}
+
+static void
+init_wrport(struct nvbios_init *init, u16 port, u8 value)
+{
+	if (init_exec(init))
+		nv_wrport(init->subdev, init->crtc, port, value);
+}
+
+static u8
+init_rdvgai(struct nvbios_init *init, u16 port, u8 index)
+{
+	struct nouveau_subdev *subdev = init->subdev;
+	if (init_exec(init)) {
+		int head = init->crtc < 0 ? 0 : init->crtc;
+		return nv_rdvgai(subdev, head, port, index);
+	}
+	return 0x00;
+}
+
+static void
+init_wrvgai(struct nvbios_init *init, u16 port, u8 index, u8 value)
+{
+	/* force head 0 for updates to cr44, it only exists on first head */
+	if (nv_device(init->subdev)->card_type < NV_50) {
+		if (port == 0x03d4 && index == 0x44)
+			init->crtc = 0;
+	}
+
+	if (init_exec(init)) {
+		int head = init->crtc < 0 ? 0 : init->crtc;
+		nv_wrvgai(init->subdev, head, port, index, value);
+	}
+
+	/* select head 1 if cr44 write selected it */
+	if (nv_device(init->subdev)->card_type < NV_50) {
+		if (port == 0x03d4 && index == 0x44 && value == 3)
+			init->crtc = 1;
+	}
+}
+
+static struct nouveau_i2c_port *
+init_i2c(struct nvbios_init *init, int index)
+{
+	struct nouveau_i2c *i2c = nouveau_i2c(init->bios);
+
+	if (index == 0xff) {
+		index = NV_I2C_DEFAULT(0);
+		if (init->outp && init->outp->i2c_upper_default)
+			index = NV_I2C_DEFAULT(1);
+	} else
+	if (index < 0) {
+		if (!init->outp) {
+			error("script needs output for i2c\n");
+			return NULL;
+		}
+
+		index = init->outp->i2c_index;
+	}
+
+	return i2c->find(i2c, index);
+}
+
+static int
+init_rdi2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg)
+{
+	struct nouveau_i2c_port *port = init_i2c(init, index);
+	if (port && init_exec(init))
+		return nv_rdi2cr(port, addr, reg);
+	return -ENODEV;
+}
+
+static int
+init_wri2cr(struct nvbios_init *init, u8 index, u8 addr, u8 reg, u8 val)
+{
+	struct nouveau_i2c_port *port = init_i2c(init, index);
+	if (port && init_exec(init))
+		return nv_wri2cr(port, addr, reg, val);
+	return -ENODEV;
+}
+
+static int
+init_rdauxr(struct nvbios_init *init, u32 addr)
+{
+	struct nouveau_i2c_port *port = init_i2c(init, -1);
+	u8 data;
+
+	if (port && init_exec(init)) {
+		int ret = nv_rdaux(port, addr, &data, 1);
+		if (ret)
+			return ret;
+		return data;
+	}
+
+	return -ENODEV;
+}
+
+static int
+init_wrauxr(struct nvbios_init *init, u32 addr, u8 data)
+{
+	struct nouveau_i2c_port *port = init_i2c(init, -1);
+	if (port && init_exec(init))
+		return nv_wraux(port, addr, &data, 1);
+	return -ENODEV;
+}
+
+static void
+init_prog_pll(struct nvbios_init *init, u32 id, u32 freq)
+{
+	struct nouveau_clock *clk = nouveau_clock(init->bios);
+	if (clk && clk->pll_set && init_exec(init)) {
+		int ret = clk->pll_set(clk, id, freq);
+		if (ret)
+			warn("failed to prog pll 0x%08x to %dkHz\n", id, freq);
+	}
+}
+
+/******************************************************************************
+ * parsing of bios structures that are required to execute init tables
+ *****************************************************************************/
+
+static u16
+init_table(struct nouveau_bios *bios, u16 *len)
+{
+	struct bit_entry bit_I;
+
+	if (!bit_entry(bios, 'I', &bit_I)) {
+		*len = bit_I.length;
+		return bit_I.offset;
+	}
+
+	if (bmp_version(bios) >= 0x0510) {
+		*len = 14;
+		return bios->bmp_offset + 75;
+	}
+
+	return 0x0000;
+}
+
+static u16
+init_table_(struct nvbios_init *init, u16 offset, const char *name)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 len, data = init_table(bios, &len);
+	if (data) {
+		if (len >= offset + 2) {
+			data = nv_ro16(bios, data + offset);
+			if (data)
+				return data;
+
+			warn("%s pointer invalid\n", name);
+			return 0x0000;
+		}
+
+		warn("init data too short for %s pointer", name);
+		return 0x0000;
+	}
+
+	warn("init data not found\n");
+	return 0x0000;
+}
+
+#define init_script_table(b) init_table_((b), 0x00, "script table")
+#define init_macro_index_table(b) init_table_((b), 0x02, "macro index table")
+#define init_macro_table(b) init_table_((b), 0x04, "macro table")
+#define init_condition_table(b) init_table_((b), 0x06, "condition table")
+#define init_io_condition_table(b) init_table_((b), 0x08, "io condition table")
+#define init_io_flag_condition_table(b) init_table_((b), 0x0a, "io flag conditon table")
+#define init_function_table(b) init_table_((b), 0x0c, "function table")
+#define init_xlat_table(b) init_table_((b), 0x10, "xlat table");
+
+static u16
+init_script(struct nouveau_bios *bios, int index)
+{
+	struct nvbios_init init = { .bios = bios };
+	u16 data;
+
+	if (bmp_version(bios) && bmp_version(bios) < 0x0510) {
+		if (index > 1)
+			return 0x0000;
+
+		data = bios->bmp_offset + (bios->version.major < 2 ? 14 : 18);
+		return nv_ro16(bios, data + (index * 2));
+	}
+
+	data = init_script_table(&init);
+	if (data)
+		return nv_ro16(bios, data + (index * 2));
+
+	return 0x0000;
+}
+
+static u16
+init_unknown_script(struct nouveau_bios *bios)
+{
+	u16 len, data = init_table(bios, &len);
+	if (data && len >= 16)
+		return nv_ro16(bios, data + 14);
+	return 0x0000;
+}
+
+static u16
+init_ram_restrict_table(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	struct bit_entry bit_M;
+	u16 data = 0x0000;
+
+	if (!bit_entry(bios, 'M', &bit_M)) {
+		if (bit_M.version == 1 && bit_M.length >= 5)
+			data = nv_ro16(bios, bit_M.offset + 3);
+		if (bit_M.version == 2 && bit_M.length >= 3)
+			data = nv_ro16(bios, bit_M.offset + 1);
+	}
+
+	if (data == 0x0000)
+		warn("ram restrict table not found\n");
+	return data;
+}
+
+static u8
+init_ram_restrict_group_count(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	struct bit_entry bit_M;
+
+	if (!bit_entry(bios, 'M', &bit_M)) {
+		if (bit_M.version == 1 && bit_M.length >= 5)
+			return nv_ro08(bios, bit_M.offset + 2);
+		if (bit_M.version == 2 && bit_M.length >= 3)
+			return nv_ro08(bios, bit_M.offset + 0);
+	}
+
+	return 0x00;
+}
+
+static u8
+init_ram_restrict(struct nvbios_init *init)
+{
+	u32 strap = (init_rd32(init, 0x101000) & 0x0000003c) >> 2;
+	u16 table = init_ram_restrict_table(init);
+	if (table)
+		return nv_ro08(init->bios, table + strap);
+	return 0x00;
+}
+
+static u8
+init_xlat_(struct nvbios_init *init, u8 index, u8 offset)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 table = init_xlat_table(init);
+	if (table) {
+		u16 data = nv_ro16(bios, table + (index * 2));
+		if (data)
+			return nv_ro08(bios, data + offset);
+		warn("xlat table pointer %d invalid\n", index);
+	}
+	return 0x00;
+}
+
+/******************************************************************************
+ * utility functions used by various init opcode handlers
+ *****************************************************************************/
+
+static bool
+init_condition_met(struct nvbios_init *init, u8 cond)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 table = init_condition_table(init);
+	if (table) {
+		u32 reg = nv_ro32(bios, table + (cond * 12) + 0);
+		u32 msk = nv_ro32(bios, table + (cond * 12) + 4);
+		u32 val = nv_ro32(bios, table + (cond * 12) + 8);
+		trace("\t[0x%02x] (R[0x%06x] & 0x%08x) == 0x%08x\n",
+		      cond, reg, msk, val);
+		return (init_rd32(init, reg) & msk) == val;
+	}
+	return false;
+}
+
+static bool
+init_io_condition_met(struct nvbios_init *init, u8 cond)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 table = init_io_condition_table(init);
+	if (table) {
+		u16 port = nv_ro16(bios, table + (cond * 5) + 0);
+		u8 index = nv_ro08(bios, table + (cond * 5) + 2);
+		u8  mask = nv_ro08(bios, table + (cond * 5) + 3);
+		u8 value = nv_ro08(bios, table + (cond * 5) + 4);
+		trace("\t[0x%02x] (0x%04x[0x%02x] & 0x%02x) == 0x%02x\n",
+		      cond, port, index, mask, value);
+		return (init_rdvgai(init, port, index) & mask) == value;
+	}
+	return false;
+}
+
+static bool
+init_io_flag_condition_met(struct nvbios_init *init, u8 cond)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 table = init_io_flag_condition_table(init);
+	if (table) {
+		u16 port = nv_ro16(bios, table + (cond * 9) + 0);
+		u8 index = nv_ro08(bios, table + (cond * 9) + 2);
+		u8  mask = nv_ro08(bios, table + (cond * 9) + 3);
+		u8 shift = nv_ro08(bios, table + (cond * 9) + 4);
+		u16 data = nv_ro16(bios, table + (cond * 9) + 5);
+		u8 dmask = nv_ro08(bios, table + (cond * 9) + 7);
+		u8 value = nv_ro08(bios, table + (cond * 9) + 8);
+		u8 ioval = (init_rdvgai(init, port, index) & mask) >> shift;
+		return (nv_ro08(bios, data + ioval) & dmask) == value;
+	}
+	return false;
+}
+
+static inline u32
+init_shift(u32 data, u8 shift)
+{
+	if (shift < 0x80)
+		return data >> shift;
+	return data << (0x100 - shift);
+}
+
+static u32
+init_tmds_reg(struct nvbios_init *init, u8 tmds)
+{
+	/* For mlv < 0x80, it is an index into a table of TMDS base addresses.
+	 * For mlv == 0x80 use the "or" value of the dcb_entry indexed by
+	 * CR58 for CR57 = 0 to index a table of offsets to the basic
+	 * 0x6808b0 address.
+	 * For mlv == 0x81 use the "or" value of the dcb_entry indexed by
+	 * CR58 for CR57 = 0 to index a table of offsets to the basic
+	 * 0x6808b0 address, and then flip the offset by 8.
+	 */
+
+	const int pramdac_offset[13] = {
+		0, 0, 0x8, 0, 0x2000, 0, 0, 0, 0x2008, 0, 0, 0, 0x2000 };
+	const u32 pramdac_table[4] = {
+		0x6808b0, 0x6808b8, 0x6828b0, 0x6828b8 };
+
+	if (tmds >= 0x80) {
+		if (init->outp) {
+			u32 dacoffset = pramdac_offset[init->outp->or];
+			if (tmds == 0x81)
+				dacoffset ^= 8;
+			return 0x6808b0 + dacoffset;
+		}
+
+		error("tmds opcodes need dcb\n");
+	} else {
+		if (tmds < ARRAY_SIZE(pramdac_table))
+			return pramdac_table[tmds];
+
+		error("tmds selector 0x%02x unknown\n", tmds);
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ * init opcode handlers
+ *****************************************************************************/
+
+/**
+ * init_reserved - stub for various unknown/unused single-byte opcodes
+ *
+ */
+static void
+init_reserved(struct nvbios_init *init)
+{
+	u8 opcode = nv_ro08(init->bios, init->offset);
+	trace("RESERVED\t0x%02x\n", opcode);
+	init->offset += 1;
+}
+
+/**
+ * INIT_DONE - opcode 0x71
+ *
+ */
+static void
+init_done(struct nvbios_init *init)
+{
+	trace("DONE\n");
+	init->offset = 0x0000;
+}
+
+/**
+ * INIT_IO_RESTRICT_PROG - opcode 0x32
+ *
+ */
+static void
+init_io_restrict_prog(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8 index = nv_ro08(bios, init->offset + 3);
+	u8  mask = nv_ro08(bios, init->offset + 4);
+	u8 shift = nv_ro08(bios, init->offset + 5);
+	u8 count = nv_ro08(bios, init->offset + 6);
+	u32  reg = nv_ro32(bios, init->offset + 7);
+	u8 conf, i;
+
+	trace("IO_RESTRICT_PROG\tR[0x%06x] = "
+	      "((0x%04x[0x%02x] & 0x%02x) >> %d) [{\n",
+	      reg, port, index, mask, shift);
+	init->offset += 11;
+
+	conf = (init_rdvgai(init, port, index) & mask) >> shift;
+	for (i = 0; i < count; i++) {
+		u32 data = nv_ro32(bios, init->offset);
+
+		if (i == conf) {
+			trace("\t0x%08x *\n", data);
+			init_wr32(init, reg, data);
+		} else {
+			trace("\t0x%08x\n", data);
+		}
+
+		init->offset += 4;
+	}
+	trace("}]\n");
+}
+
+/**
+ * INIT_REPEAT - opcode 0x33
+ *
+ */
+static void
+init_repeat(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 count = nv_ro08(bios, init->offset + 1);
+	u16 repeat = init->repeat;
+
+	trace("REPEAT\t0x%02x\n", count);
+	init->offset += 2;
+
+	init->repeat = init->offset;
+	init->repend = init->offset;
+	while (count--) {
+		init->offset = init->repeat;
+		nvbios_exec(init);
+		if (count)
+			trace("REPEAT\t0x%02x\n", count);
+	}
+	init->offset = init->repend;
+	init->repeat = repeat;
+}
+
+/**
+ * INIT_IO_RESTRICT_PLL - opcode 0x34
+ *
+ */
+static void
+init_io_restrict_pll(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8 index = nv_ro08(bios, init->offset + 3);
+	u8  mask = nv_ro08(bios, init->offset + 4);
+	u8 shift = nv_ro08(bios, init->offset + 5);
+	s8  iofc = nv_ro08(bios, init->offset + 6);
+	u8 count = nv_ro08(bios, init->offset + 7);
+	u32  reg = nv_ro32(bios, init->offset + 8);
+	u8 conf, i;
+
+	trace("IO_RESTRICT_PLL\tR[0x%06x] =PLL= "
+	      "((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) IOFCOND 0x%02x [{\n",
+	      reg, port, index, mask, shift, iofc);
+	init->offset += 12;
+
+	conf = (init_rdvgai(init, port, index) & mask) >> shift;
+	for (i = 0; i < count; i++) {
+		u32 freq = nv_ro16(bios, init->offset) * 10;
+
+		if (i == conf) {
+			trace("\t%dkHz *\n", freq);
+			if (iofc > 0 && init_io_flag_condition_met(init, iofc))
+				freq *= 2;
+			init_prog_pll(init, reg, freq);
+		} else {
+			trace("\t%dkHz\n", freq);
+		}
+
+		init->offset += 2;
+	}
+	trace("}]\n");
+}
+
+/**
+ * INIT_END_REPEAT - opcode 0x36
+ *
+ */
+static void
+init_end_repeat(struct nvbios_init *init)
+{
+	trace("END_REPEAT\n");
+	init->offset += 1;
+
+	if (init->repeat) {
+		init->repend = init->offset;
+		init->offset = 0;
+	}
+}
+
+/**
+ * INIT_COPY - opcode 0x37
+ *
+ */
+static void
+init_copy(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32  reg = nv_ro32(bios, init->offset + 1);
+	u8 shift = nv_ro08(bios, init->offset + 5);
+	u8 smask = nv_ro08(bios, init->offset + 6);
+	u16 port = nv_ro16(bios, init->offset + 7);
+	u8 index = nv_ro08(bios, init->offset + 9);
+	u8  mask = nv_ro08(bios, init->offset + 10);
+	u8  data;
+
+	trace("COPY\t0x%04x[0x%02x] &= 0x%02x |= "
+	      "((R[0x%06x] %s 0x%02x) & 0x%02x)\n",
+	      port, index, mask, reg, (shift & 0x80) ? "<<" : ">>",
+	      (shift & 0x80) ? (0x100 - shift) : shift, smask);
+	init->offset += 11;
+
+	data  = init_rdvgai(init, port, index) & mask;
+	data |= init_shift(init_rd32(init, reg), shift) & smask;
+	init_wrvgai(init, port, index, data);
+}
+
+/**
+ * INIT_NOT - opcode 0x38
+ *
+ */
+static void
+init_not(struct nvbios_init *init)
+{
+	trace("NOT\n");
+	init->offset += 1;
+	init_exec_inv(init);
+}
+
+/**
+ * INIT_IO_FLAG_CONDITION - opcode 0x39
+ *
+ */
+static void
+init_io_flag_condition(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 cond = nv_ro08(bios, init->offset + 1);
+
+	trace("IO_FLAG_CONDITION\t0x%02x\n", cond);
+	init->offset += 2;
+
+	if (!init_io_flag_condition_met(init, cond))
+		init_exec_set(init, false);
+}
+
+/**
+ * INIT_DP_CONDITION - opcode 0x3a
+ *
+ */
+static void
+init_dp_condition(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  cond = nv_ro08(bios, init->offset + 1);
+	u8  unkn = nv_ro08(bios, init->offset + 2);
+	u8  ver, len;
+	u16 data;
+
+	trace("DP_CONDITION\t0x%02x 0x%02x\n", cond, unkn);
+	init->offset += 3;
+
+	switch (cond) {
+	case 0:
+		if (init_conn(init) != DCB_CONNECTOR_eDP)
+			init_exec_set(init, false);
+		break;
+	case 1:
+	case 2:
+		if ( init->outp &&
+		    (data = dp_outp_match(bios, init->outp, &ver, &len))) {
+			if (ver <= 0x40 && !(nv_ro08(bios, data + 5) & cond))
+				init_exec_set(init, false);
+			if (ver == 0x40 && !(nv_ro08(bios, data + 4) & cond))
+				init_exec_set(init, false);
+			break;
+		}
+
+		warn("script needs dp output table data\n");
+		break;
+	case 5:
+		if (!(init_rdauxr(init, 0x0d) & 1))
+			init_exec_set(init, false);
+		break;
+	default:
+		warn("unknown dp condition 0x%02x\n", cond);
+		break;
+	}
+}
+
+/**
+ * INIT_IO_MASK_OR - opcode 0x3b
+ *
+ */
+static void
+init_io_mask_or(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8    or = init_or(init);
+	u8  data;
+
+	trace("IO_MASK_OR\t0x03d4[0x%02x] &= ~(1 << 0x%02x)", index, or);
+	init->offset += 2;
+
+	data = init_rdvgai(init, 0x03d4, index);
+	init_wrvgai(init, 0x03d4, index, data &= ~(1 << or));
+}
+
+/**
+ * INIT_IO_OR - opcode 0x3c
+ *
+ */
+static void
+init_io_or(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8    or = init_or(init);
+	u8  data;
+
+	trace("IO_OR\t0x03d4[0x%02x] |= (1 << 0x%02x)", index, or);
+	init->offset += 2;
+
+	data = init_rdvgai(init, 0x03d4, index);
+	init_wrvgai(init, 0x03d4, index, data | (1 << or));
+}
+
+/**
+ * INIT_INDEX_ADDRESS_LATCHED - opcode 0x49
+ *
+ */
+static void
+init_idx_addr_latched(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 creg = nv_ro32(bios, init->offset + 1);
+	u32 dreg = nv_ro32(bios, init->offset + 5);
+	u32 mask = nv_ro32(bios, init->offset + 9);
+	u32 data = nv_ro32(bios, init->offset + 13);
+	u8 count = nv_ro08(bios, init->offset + 17);
+
+	trace("INDEX_ADDRESS_LATCHED\t"
+	      "R[0x%06x] : R[0x%06x]\n\tCTRL &= 0x%08x |= 0x%08x\n",
+	      creg, dreg, mask, data);
+	init->offset += 18;
+
+	while (count--) {
+		u8 iaddr = nv_ro08(bios, init->offset + 0);
+		u8 idata = nv_ro08(bios, init->offset + 1);
+
+		trace("\t[0x%02x] = 0x%02x\n", iaddr, idata);
+		init->offset += 2;
+
+		init_wr32(init, dreg, idata);
+		init_mask(init, creg, ~mask, data | idata);
+	}
+}
+
+/**
+ * INIT_IO_RESTRICT_PLL2 - opcode 0x4a
+ *
+ */
+static void
+init_io_restrict_pll2(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8 index = nv_ro08(bios, init->offset + 3);
+	u8  mask = nv_ro08(bios, init->offset + 4);
+	u8 shift = nv_ro08(bios, init->offset + 5);
+	u8 count = nv_ro08(bios, init->offset + 6);
+	u32  reg = nv_ro32(bios, init->offset + 7);
+	u8  conf, i;
+
+	trace("IO_RESTRICT_PLL2\t"
+	      "R[0x%06x] =PLL= ((0x%04x[0x%02x] & 0x%02x) >> 0x%02x) [{\n",
+	      reg, port, index, mask, shift);
+	init->offset += 11;
+
+	conf = (init_rdvgai(init, port, index) & mask) >> shift;
+	for (i = 0; i < count; i++) {
+		u32 freq = nv_ro32(bios, init->offset);
+		if (i == conf) {
+			trace("\t%dkHz *\n", freq);
+			init_prog_pll(init, reg, freq);
+		} else {
+			trace("\t%dkHz\n", freq);
+		}
+		init->offset += 4;
+	}
+	trace("}]\n");
+}
+
+/**
+ * INIT_PLL2 - opcode 0x4b
+ *
+ */
+static void
+init_pll2(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32  reg = nv_ro32(bios, init->offset + 1);
+	u32 freq = nv_ro32(bios, init->offset + 5);
+
+	trace("PLL2\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
+	init->offset += 9;
+
+	init_prog_pll(init, reg, freq);
+}
+
+/**
+ * INIT_I2C_BYTE - opcode 0x4c
+ *
+ */
+static void
+init_i2c_byte(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8  addr = nv_ro08(bios, init->offset + 2) >> 1;
+	u8 count = nv_ro08(bios, init->offset + 3);
+
+	trace("I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
+	init->offset += 4;
+
+	while (count--) {
+		u8  reg = nv_ro08(bios, init->offset + 0);
+		u8 mask = nv_ro08(bios, init->offset + 1);
+		u8 data = nv_ro08(bios, init->offset + 2);
+		int val;
+
+		trace("\t[0x%02x] &= 0x%02x |= 0x%02x\n", reg, mask, data);
+		init->offset += 3;
+
+		val = init_rdi2cr(init, index, addr, reg);
+		if (val < 0)
+			continue;
+		init_wri2cr(init, index, addr, reg, (val & mask) | data);
+	}
+}
+
+/**
+ * INIT_ZM_I2C_BYTE - opcode 0x4d
+ *
+ */
+static void
+init_zm_i2c_byte(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8  addr = nv_ro08(bios, init->offset + 2) >> 1;
+	u8 count = nv_ro08(bios, init->offset + 3);
+
+	trace("ZM_I2C_BYTE\tI2C[0x%02x][0x%02x]\n", index, addr);
+	init->offset += 4;
+
+	while (count--) {
+		u8  reg = nv_ro08(bios, init->offset + 0);
+		u8 data = nv_ro08(bios, init->offset + 1);
+
+		trace("\t[0x%02x] = 0x%02x\n", reg, data);
+		init->offset += 2;
+
+		init_wri2cr(init, index, addr, reg, data);
+	}
+
+}
+
+/**
+ * INIT_ZM_I2C - opcode 0x4e
+ *
+ */
+static void
+init_zm_i2c(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8  addr = nv_ro08(bios, init->offset + 2) >> 1;
+	u8 count = nv_ro08(bios, init->offset + 3);
+	u8 data[256], i;
+
+	trace("ZM_I2C\tI2C[0x%02x][0x%02x]\n", index, addr);
+	init->offset += 4;
+
+	for (i = 0; i < count; i++) {
+		data[i] = nv_ro08(bios, init->offset);
+		trace("\t0x%02x\n", data[i]);
+		init->offset++;
+	}
+
+	if (init_exec(init)) {
+		struct nouveau_i2c_port *port = init_i2c(init, index);
+		struct i2c_msg msg = {
+			.addr = addr, .flags = 0, .len = count, .buf = data,
+		};
+		int ret;
+
+		if (port && (ret = i2c_transfer(&port->adapter, &msg, 1)) != 1)
+			warn("i2c wr failed, %d\n", ret);
+	}
+}
+
+/**
+ * INIT_TMDS - opcode 0x4f
+ *
+ */
+static void
+init_tmds(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 tmds = nv_ro08(bios, init->offset + 1);
+	u8 addr = nv_ro08(bios, init->offset + 2);
+	u8 mask = nv_ro08(bios, init->offset + 3);
+	u8 data = nv_ro08(bios, init->offset + 4);
+	u32 reg = init_tmds_reg(init, tmds);
+
+	trace("TMDS\tT[0x%02x][0x%02x] &= 0x%02x |= 0x%02x\n",
+	      tmds, addr, mask, data);
+	init->offset += 5;
+
+	if (reg == 0)
+		return;
+
+	init_wr32(init, reg + 0, addr | 0x00010000);
+	init_wr32(init, reg + 4, data | (init_rd32(init, reg + 4) & mask));
+	init_wr32(init, reg + 0, addr);
+}
+
+/**
+ * INIT_ZM_TMDS_GROUP - opcode 0x50
+ *
+ */
+static void
+init_zm_tmds_group(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  tmds = nv_ro08(bios, init->offset + 1);
+	u8 count = nv_ro08(bios, init->offset + 2);
+	u32  reg = init_tmds_reg(init, tmds);
+
+	trace("TMDS_ZM_GROUP\tT[0x%02x]\n", tmds);
+	init->offset += 3;
+
+	while (count--) {
+		u8 addr = nv_ro08(bios, init->offset + 0);
+		u8 data = nv_ro08(bios, init->offset + 1);
+
+		trace("\t[0x%02x] = 0x%02x\n", addr, data);
+		init->offset += 2;
+
+		init_wr32(init, reg + 4, data);
+		init_wr32(init, reg + 0, addr);
+	}
+}
+
+/**
+ * INIT_CR_INDEX_ADDRESS_LATCHED - opcode 0x51
+ *
+ */
+static void
+init_cr_idx_adr_latch(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 addr0 = nv_ro08(bios, init->offset + 1);
+	u8 addr1 = nv_ro08(bios, init->offset + 2);
+	u8  base = nv_ro08(bios, init->offset + 3);
+	u8 count = nv_ro08(bios, init->offset + 4);
+	u8 save0;
+
+	trace("CR_INDEX_ADDR C[%02x] C[%02x]\n", addr0, addr1);
+	init->offset += 5;
+
+	save0 = init_rdvgai(init, 0x03d4, addr0);
+	while (count--) {
+		u8 data = nv_ro08(bios, init->offset);
+
+		trace("\t\t[0x%02x] = 0x%02x\n", base, data);
+		init->offset += 1;
+
+		init_wrvgai(init, 0x03d4, addr0, base++);
+		init_wrvgai(init, 0x03d4, addr1, data);
+	}
+	init_wrvgai(init, 0x03d4, addr0, save0);
+}
+
+/**
+ * INIT_CR - opcode 0x52
+ *
+ */
+static void
+init_cr(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 addr = nv_ro08(bios, init->offset + 1);
+	u8 mask = nv_ro08(bios, init->offset + 2);
+	u8 data = nv_ro08(bios, init->offset + 3);
+	u8 val;
+
+	trace("CR\t\tC[0x%02x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
+	init->offset += 4;
+
+	val = init_rdvgai(init, 0x03d4, addr) & mask;
+	init_wrvgai(init, 0x03d4, addr, val | data);
+}
+
+/**
+ * INIT_ZM_CR - opcode 0x53
+ *
+ */
+static void
+init_zm_cr(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 addr = nv_ro08(bios, init->offset + 1);
+	u8 data = nv_ro08(bios, init->offset + 2);
+
+	trace("ZM_CR\tC[0x%02x] = 0x%02x\n", addr,  data);
+	init->offset += 3;
+
+	init_wrvgai(init, 0x03d4, addr, data);
+}
+
+/**
+ * INIT_ZM_CR_GROUP - opcode 0x54
+ *
+ */
+static void
+init_zm_cr_group(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 count = nv_ro08(bios, init->offset + 1);
+
+	trace("ZM_CR_GROUP\n");
+	init->offset += 2;
+
+	while (count--) {
+		u8 addr = nv_ro08(bios, init->offset + 0);
+		u8 data = nv_ro08(bios, init->offset + 1);
+
+		trace("\t\tC[0x%02x] = 0x%02x\n", addr, data);
+		init->offset += 2;
+
+		init_wrvgai(init, 0x03d4, addr, data);
+	}
+}
+
+/**
+ * INIT_CONDITION_TIME - opcode 0x56
+ *
+ */
+static void
+init_condition_time(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  cond = nv_ro08(bios, init->offset + 1);
+	u8 retry = nv_ro08(bios, init->offset + 2);
+	u8  wait = min((u16)retry * 50, 100);
+
+	trace("CONDITION_TIME\t0x%02x 0x%02x\n", cond, retry);
+	init->offset += 3;
+
+	if (!init_exec(init))
+		return;
+
+	while (wait--) {
+		if (init_condition_met(init, cond))
+			return;
+		mdelay(20);
+	}
+
+	init_exec_set(init, false);
+}
+
+/**
+ * INIT_LTIME - opcode 0x57
+ *
+ */
+static void
+init_ltime(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 msec = nv_ro16(bios, init->offset + 1);
+
+	trace("LTIME\t0x%04x\n", msec);
+	init->offset += 3;
+
+	if (init_exec(init))
+		mdelay(msec);
+}
+
+/**
+ * INIT_ZM_REG_SEQUENCE - opcode 0x58
+ *
+ */
+static void
+init_zm_reg_sequence(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 base = nv_ro32(bios, init->offset + 1);
+	u8 count = nv_ro08(bios, init->offset + 5);
+
+	trace("ZM_REG_SEQUENCE\t0x%02x\n", count);
+	init->offset += 6;
+
+	while (count--) {
+		u32 data = nv_ro32(bios, init->offset);
+
+		trace("\t\tR[0x%06x] = 0x%08x\n", base, data);
+		init->offset += 4;
+
+		init_wr32(init, base, data);
+		base += 4;
+	}
+}
+
+/**
+ * INIT_SUB_DIRECT - opcode 0x5b
+ *
+ */
+static void
+init_sub_direct(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 addr = nv_ro16(bios, init->offset + 1);
+	u16 save;
+
+	trace("SUB_DIRECT\t0x%04x\n", addr);
+
+	if (init_exec(init)) {
+		save = init->offset;
+		init->offset = addr;
+		if (nvbios_exec(init)) {
+			error("error parsing sub-table\n");
+			return;
+		}
+		init->offset = save;
+	}
+
+	init->offset += 3;
+}
+
+/**
+ * INIT_JUMP - opcode 0x5c
+ *
+ */
+static void
+init_jump(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 offset = nv_ro16(bios, init->offset + 1);
+
+	trace("JUMP\t0x%04x\n", offset);
+	init->offset = offset;
+}
+
+/**
+ * INIT_I2C_IF - opcode 0x5e
+ *
+ */
+static void
+init_i2c_if(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8  addr = nv_ro08(bios, init->offset + 2);
+	u8   reg = nv_ro08(bios, init->offset + 3);
+	u8  mask = nv_ro08(bios, init->offset + 4);
+	u8  data = nv_ro08(bios, init->offset + 5);
+	u8 value;
+
+	trace("I2C_IF\tI2C[0x%02x][0x%02x][0x%02x] & 0x%02x == 0x%02x\n",
+	      index, addr, reg, mask, data);
+	init->offset += 6;
+	init_exec_force(init, true);
+
+	value = init_rdi2cr(init, index, addr, reg);
+	if ((value & mask) != data)
+		init_exec_set(init, false);
+
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_COPY_NV_REG - opcode 0x5f
+ *
+ */
+static void
+init_copy_nv_reg(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32  sreg = nv_ro32(bios, init->offset + 1);
+	u8  shift = nv_ro08(bios, init->offset + 5);
+	u32 smask = nv_ro32(bios, init->offset + 6);
+	u32  sxor = nv_ro32(bios, init->offset + 10);
+	u32  dreg = nv_ro32(bios, init->offset + 14);
+	u32 dmask = nv_ro32(bios, init->offset + 18);
+	u32 data;
+
+	trace("COPY_NV_REG\tR[0x%06x] &= 0x%08x |= "
+	      "((R[0x%06x] %s 0x%02x) & 0x%08x ^ 0x%08x)\n",
+	      dreg, dmask, sreg, (shift & 0x80) ? "<<" : ">>",
+	      (shift & 0x80) ? (0x100 - shift) : shift, smask, sxor);
+	init->offset += 22;
+
+	data = init_shift(init_rd32(init, sreg), shift);
+	init_mask(init, dreg, ~dmask, (data & smask) ^ sxor);
+}
+
+/**
+ * INIT_ZM_INDEX_IO - opcode 0x62
+ *
+ */
+static void
+init_zm_index_io(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8 index = nv_ro08(bios, init->offset + 3);
+	u8  data = nv_ro08(bios, init->offset + 4);
+
+	trace("ZM_INDEX_IO\tI[0x%04x][0x%02x] = 0x%02x\n", port, index, data);
+	init->offset += 5;
+
+	init_wrvgai(init, port, index, data);
+}
+
+/**
+ * INIT_COMPUTE_MEM - opcode 0x63
+ *
+ */
+static void
+init_compute_mem(struct nvbios_init *init)
+{
+	struct nouveau_devinit *devinit = nouveau_devinit(init->bios);
+
+	trace("COMPUTE_MEM\n");
+	init->offset += 1;
+
+	init_exec_force(init, true);
+	if (init_exec(init) && devinit->meminit)
+		devinit->meminit(devinit);
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_RESET - opcode 0x65
+ *
+ */
+static void
+init_reset(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32   reg = nv_ro32(bios, init->offset + 1);
+	u32 data1 = nv_ro32(bios, init->offset + 5);
+	u32 data2 = nv_ro32(bios, init->offset + 9);
+	u32 savepci19;
+
+	trace("RESET\tR[0x%08x] = 0x%08x, 0x%08x", reg, data1, data2);
+	init->offset += 13;
+	init_exec_force(init, true);
+
+	savepci19 = init_mask(init, 0x00184c, 0x00000f00, 0x00000000);
+	init_wr32(init, reg, data1);
+	udelay(10);
+	init_wr32(init, reg, data2);
+	init_wr32(init, 0x00184c, savepci19);
+	init_mask(init, 0x001850, 0x00000001, 0x00000000);
+
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_CONFIGURE_MEM - opcode 0x66
+ *
+ */
+static u16
+init_configure_mem_clk(struct nvbios_init *init)
+{
+	u16 mdata = bmp_mem_init_table(init->bios);
+	if (mdata)
+		mdata += (init_rdvgai(init, 0x03d4, 0x3c) >> 4) * 66;
+	return mdata;
+}
+
+static void
+init_configure_mem(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 mdata, sdata;
+	u32 addr, data;
+
+	trace("CONFIGURE_MEM\n");
+	init->offset += 1;
+
+	if (bios->version.major > 2) {
+		init_done(init);
+		return;
+	}
+	init_exec_force(init, true);
+
+	mdata = init_configure_mem_clk(init);
+	sdata = bmp_sdr_seq_table(bios);
+	if (nv_ro08(bios, mdata) & 0x01)
+		sdata = bmp_ddr_seq_table(bios);
+	mdata += 6; /* skip to data */
+
+	data = init_rdvgai(init, 0x03c4, 0x01);
+	init_wrvgai(init, 0x03c4, 0x01, data | 0x20);
+
+	while ((addr = nv_ro32(bios, sdata)) != 0xffffffff) {
+		switch (addr) {
+		case 0x10021c: /* CKE_NORMAL */
+		case 0x1002d0: /* CMD_REFRESH */
+		case 0x1002d4: /* CMD_PRECHARGE */
+			data = 0x00000001;
+			break;
+		default:
+			data = nv_ro32(bios, mdata);
+			mdata += 4;
+			if (data == 0xffffffff)
+				continue;
+			break;
+		}
+
+		init_wr32(init, addr, data);
+	}
+
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_CONFIGURE_CLK - opcode 0x67
+ *
+ */
+static void
+init_configure_clk(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 mdata, clock;
+
+	trace("CONFIGURE_CLK\n");
+	init->offset += 1;
+
+	if (bios->version.major > 2) {
+		init_done(init);
+		return;
+	}
+	init_exec_force(init, true);
+
+	mdata = init_configure_mem_clk(init);
+
+	/* NVPLL */
+	clock = nv_ro16(bios, mdata + 4) * 10;
+	init_prog_pll(init, 0x680500, clock);
+
+	/* MPLL */
+	clock = nv_ro16(bios, mdata + 2) * 10;
+	if (nv_ro08(bios, mdata) & 0x01)
+		clock *= 2;
+	init_prog_pll(init, 0x680504, clock);
+
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_CONFIGURE_PREINIT - opcode 0x68
+ *
+ */
+static void
+init_configure_preinit(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 strap;
+
+	trace("CONFIGURE_PREINIT\n");
+	init->offset += 1;
+
+	if (bios->version.major > 2) {
+		init_done(init);
+		return;
+	}
+	init_exec_force(init, true);
+
+	strap = init_rd32(init, 0x101000);
+	strap = ((strap << 2) & 0xf0) | ((strap & 0x40) >> 6);
+	init_wrvgai(init, 0x03d4, 0x3c, strap);
+
+	init_exec_force(init, false);
+}
+
+/**
+ * INIT_IO - opcode 0x69
+ *
+ */
+static void
+init_io(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8  mask = nv_ro16(bios, init->offset + 3);
+	u8  data = nv_ro16(bios, init->offset + 4);
+	u8 value;
+
+	trace("IO\t\tI[0x%04x] &= 0x%02x |= 0x%02x\n", port, mask, data);
+	init->offset += 5;
+
+	/* ummm.. yes.. should really figure out wtf this is and why it's
+	 * needed some day..  it's almost certainly wrong, but, it also
+	 * somehow makes things work...
+	 */
+	if (nv_device(init->bios)->card_type >= NV_50 &&
+	    port == 0x03c3 && data == 0x01) {
+		init_mask(init, 0x614100, 0xf0800000, 0x00800000);
+		init_mask(init, 0x00e18c, 0x00020000, 0x00020000);
+		init_mask(init, 0x614900, 0xf0800000, 0x00800000);
+		init_mask(init, 0x000200, 0x40000000, 0x00000000);
+		mdelay(10);
+		init_mask(init, 0x00e18c, 0x00020000, 0x00000000);
+		init_mask(init, 0x000200, 0x40000000, 0x40000000);
+		init_wr32(init, 0x614100, 0x00800018);
+		init_wr32(init, 0x614900, 0x00800018);
+		mdelay(10);
+		init_wr32(init, 0x614100, 0x10000018);
+		init_wr32(init, 0x614900, 0x10000018);
+		return;
+	}
+
+	value = init_rdport(init, port) & mask;
+	init_wrport(init, port, data | value);
+}
+
+/**
+ * INIT_SUB - opcode 0x6b
+ *
+ */
+static void
+init_sub(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u16 addr, save;
+
+	trace("SUB\t0x%02x\n", index);
+
+	addr = init_script(bios, index);
+	if (addr && init_exec(init)) {
+		save = init->offset;
+		init->offset = addr;
+		if (nvbios_exec(init)) {
+			error("error parsing sub-table\n");
+			return;
+		}
+		init->offset = save;
+	}
+
+	init->offset += 2;
+}
+
+/**
+ * INIT_RAM_CONDITION - opcode 0x6d
+ *
+ */
+static void
+init_ram_condition(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  mask = nv_ro08(bios, init->offset + 1);
+	u8 value = nv_ro08(bios, init->offset + 2);
+
+	trace("RAM_CONDITION\t"
+	      "(R[0x100000] & 0x%02x) == 0x%02x\n", mask, value);
+	init->offset += 3;
+
+	if ((init_rd32(init, 0x100000) & mask) != value)
+		init_exec_set(init, false);
+}
+
+/**
+ * INIT_NV_REG - opcode 0x6e
+ *
+ */
+static void
+init_nv_reg(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32  reg = nv_ro32(bios, init->offset + 1);
+	u32 mask = nv_ro32(bios, init->offset + 5);
+	u32 data = nv_ro32(bios, init->offset + 9);
+
+	trace("NV_REG\tR[0x%06x] &= 0x%08x |= 0x%08x\n", reg, mask, data);
+	init->offset += 13;
+
+	init_mask(init, reg, ~mask, data);
+}
+
+/**
+ * INIT_MACRO - opcode 0x6f
+ *
+ */
+static void
+init_macro(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  macro = nv_ro08(bios, init->offset + 1);
+	u16 table;
+
+	trace("MACRO\t0x%02x\n", macro);
+
+	table = init_macro_table(init);
+	if (table) {
+		u32 addr = nv_ro32(bios, table + (macro * 8) + 0);
+		u32 data = nv_ro32(bios, table + (macro * 8) + 4);
+		trace("\t\tR[0x%06x] = 0x%08x\n", addr, data);
+		init_wr32(init, addr, data);
+	}
+
+	init->offset += 2;
+}
+
+/**
+ * INIT_RESUME - opcode 0x72
+ *
+ */
+static void
+init_resume(struct nvbios_init *init)
+{
+	trace("RESUME\n");
+	init->offset += 1;
+	init_exec_set(init, true);
+}
+
+/**
+ * INIT_TIME - opcode 0x74
+ *
+ */
+static void
+init_time(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 usec = nv_ro16(bios, init->offset + 1);
+
+	trace("TIME\t0x%04x\n", usec);
+	init->offset += 3;
+
+	if (init_exec(init)) {
+		if (usec < 1000)
+			udelay(usec);
+		else
+			mdelay((usec + 900) / 1000);
+	}
+}
+
+/**
+ * INIT_CONDITION - opcode 0x75
+ *
+ */
+static void
+init_condition(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 cond = nv_ro08(bios, init->offset + 1);
+
+	trace("CONDITION\t0x%02x\n", cond);
+	init->offset += 2;
+
+	if (!init_condition_met(init, cond))
+		init_exec_set(init, false);
+}
+
+/**
+ * INIT_IO_CONDITION - opcode 0x76
+ *
+ */
+static void
+init_io_condition(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 cond = nv_ro08(bios, init->offset + 1);
+
+	trace("IO_CONDITION\t0x%02x\n", cond);
+	init->offset += 2;
+
+	if (!init_io_condition_met(init, cond))
+		init_exec_set(init, false);
+}
+
+/**
+ * INIT_INDEX_IO - opcode 0x78
+ *
+ */
+static void
+init_index_io(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u16 port = nv_ro16(bios, init->offset + 1);
+	u8 index = nv_ro16(bios, init->offset + 3);
+	u8  mask = nv_ro08(bios, init->offset + 4);
+	u8  data = nv_ro08(bios, init->offset + 5);
+	u8 value;
+
+	trace("INDEX_IO\tI[0x%04x][0x%02x] &= 0x%02x |= 0x%02x\n",
+	      port, index, mask, data);
+	init->offset += 6;
+
+	value = init_rdvgai(init, port, index) & mask;
+	init_wrvgai(init, port, index, data | value);
+}
+
+/**
+ * INIT_PLL - opcode 0x79
+ *
+ */
+static void
+init_pll(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32  reg = nv_ro32(bios, init->offset + 1);
+	u32 freq = nv_ro16(bios, init->offset + 5) * 10;
+
+	trace("PLL\tR[0x%06x] =PLL= %dkHz\n", reg, freq);
+	init->offset += 7;
+
+	init_prog_pll(init, reg, freq);
+}
+
+/**
+ * INIT_ZM_REG - opcode 0x7a
+ *
+ */
+static void
+init_zm_reg(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u32 data = nv_ro32(bios, init->offset + 5);
+
+	trace("ZM_REG\tR[0x%06x] = 0x%08x\n", addr, data);
+	init->offset += 9;
+
+	if (addr == 0x000200)
+		data |= 0x00000001;
+
+	init_wr32(init, addr, data);
+}
+
+/**
+ * INIT_RAM_RESTRICT_PLL - opcde 0x87
+ *
+ */
+static void
+init_ram_restrict_pll(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8  type = nv_ro08(bios, init->offset + 1);
+	u8 count = init_ram_restrict_group_count(init);
+	u8 strap = init_ram_restrict(init);
+	u8 cconf;
+
+	trace("RAM_RESTRICT_PLL\t0x%02x\n", type);
+	init->offset += 2;
+
+	for (cconf = 0; cconf < count; cconf++) {
+		u32 freq = nv_ro32(bios, init->offset);
+
+		if (cconf == strap) {
+			trace("%dkHz *\n", freq);
+			init_prog_pll(init, type, freq);
+		} else {
+			trace("%dkHz\n", freq);
+		}
+
+		init->offset += 4;
+	}
+}
+
+/**
+ * INIT_GPIO - opcode 0x8e
+ *
+ */
+static void
+init_gpio(struct nvbios_init *init)
+{
+	struct nouveau_gpio *gpio = nouveau_gpio(init->bios);
+
+	trace("GPIO\n");
+	init->offset += 1;
+
+	if (init_exec(init) && gpio && gpio->reset)
+		gpio->reset(gpio);
+}
+
+/**
+ * INIT_RAM_RESTRICT_ZM_GROUP - opcode 0x8f
+ *
+ */
+static void
+init_ram_restrict_zm_reg_group(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u8  incr = nv_ro08(bios, init->offset + 5);
+	u8   num = nv_ro08(bios, init->offset + 6);
+	u8 count = init_ram_restrict_group_count(init);
+	u8 index = init_ram_restrict(init);
+	u8 i, j;
+
+	trace("RAM_RESTRICT_ZM_REG_GROUP\t"
+	      "R[%08x] 0x%02x 0x%02x\n", addr, incr, num);
+	init->offset += 7;
+
+	for (i = 0; i < num; i++) {
+		trace("\tR[0x%06x] = {\n", addr);
+		for (j = 0; j < count; j++) {
+			u32 data = nv_ro32(bios, init->offset);
+
+			if (j == index) {
+				trace("\t\t0x%08x *\n", data);
+				init_wr32(init, addr, data);
+			} else {
+				trace("\t\t0x%08x\n", data);
+			}
+
+			init->offset += 4;
+		}
+		trace("\t}\n");
+		addr += incr;
+	}
+}
+
+/**
+ * INIT_COPY_ZM_REG - opcode 0x90
+ *
+ */
+static void
+init_copy_zm_reg(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 sreg = nv_ro32(bios, init->offset + 1);
+	u32 dreg = nv_ro32(bios, init->offset + 5);
+
+	trace("COPY_ZM_REG\tR[0x%06x] = R[0x%06x]\n", sreg, dreg);
+	init->offset += 9;
+
+	init_wr32(init, dreg, init_rd32(init, sreg));
+}
+
+/**
+ * INIT_ZM_REG_GROUP - opcode 0x91
+ *
+ */
+static void
+init_zm_reg_group(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u8 count = nv_ro08(bios, init->offset + 5);
+
+	trace("ZM_REG_GROUP\tR[0x%06x] =\n");
+	init->offset += 6;
+
+	while (count--) {
+		u32 data = nv_ro32(bios, init->offset);
+		trace("\t0x%08x\n", data);
+		init_wr32(init, addr, data);
+		init->offset += 4;
+	}
+}
+
+/**
+ * INIT_XLAT - opcode 0x96
+ *
+ */
+static void
+init_xlat(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 saddr = nv_ro32(bios, init->offset + 1);
+	u8 sshift = nv_ro08(bios, init->offset + 5);
+	u8  smask = nv_ro08(bios, init->offset + 6);
+	u8  index = nv_ro08(bios, init->offset + 7);
+	u32 daddr = nv_ro32(bios, init->offset + 8);
+	u32 dmask = nv_ro32(bios, init->offset + 12);
+	u8  shift = nv_ro08(bios, init->offset + 16);
+	u32 data;
+
+	trace("INIT_XLAT\tR[0x%06x] &= 0x%08x |= "
+	      "(X%02x((R[0x%06x] %s 0x%02x) & 0x%02x) << 0x%02x)\n",
+	      daddr, dmask, index, saddr, (sshift & 0x80) ? "<<" : ">>",
+	      (sshift & 0x80) ? (0x100 - sshift) : sshift, smask, shift);
+	init->offset += 17;
+
+	data = init_shift(init_rd32(init, saddr), sshift) & smask;
+	data = init_xlat_(init, index, data) << shift;
+	init_mask(init, daddr, ~dmask, data);
+}
+
+/**
+ * INIT_ZM_MASK_ADD - opcode 0x97
+ *
+ */
+static void
+init_zm_mask_add(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u32 mask = nv_ro32(bios, init->offset + 5);
+	u32  add = nv_ro32(bios, init->offset + 9);
+	u32 data;
+
+	trace("ZM_MASK_ADD\tR[0x%06x] &= 0x%08x += 0x%08x\n", addr, mask, add);
+	init->offset += 13;
+
+	data  =  init_rd32(init, addr) & mask;
+	data |= ((data + add) & ~mask);
+	init_wr32(init, addr, data);
+}
+
+/**
+ * INIT_AUXCH - opcode 0x98
+ *
+ */
+static void
+init_auxch(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u8 count = nv_ro08(bios, init->offset + 5);
+
+	trace("AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
+	init->offset += 6;
+
+	while (count--) {
+		u8 mask = nv_ro08(bios, init->offset + 0);
+		u8 data = nv_ro08(bios, init->offset + 1);
+		trace("\tAUX[0x%08x] &= 0x%02x |= 0x%02x\n", addr, mask, data);
+		mask = init_rdauxr(init, addr) & mask;
+		init_wrauxr(init, addr, mask | data);
+		init->offset += 2;
+	}
+}
+
+/**
+ * INIT_AUXCH - opcode 0x99
+ *
+ */
+static void
+init_zm_auxch(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u32 addr = nv_ro32(bios, init->offset + 1);
+	u8 count = nv_ro08(bios, init->offset + 5);
+
+	trace("ZM_AUXCH\tAUX[0x%08x] 0x%02x\n", addr, count);
+	init->offset += 6;
+
+	while (count--) {
+		u8 data = nv_ro08(bios, init->offset + 0);
+		trace("\tAUX[0x%08x] = 0x%02x\n", addr, data);
+		init_wrauxr(init, addr, data);
+		init->offset += 1;
+	}
+}
+
+/**
+ * INIT_I2C_LONG_IF - opcode 0x9a
+ *
+ */
+static void
+init_i2c_long_if(struct nvbios_init *init)
+{
+	struct nouveau_bios *bios = init->bios;
+	u8 index = nv_ro08(bios, init->offset + 1);
+	u8  addr = nv_ro08(bios, init->offset + 2) >> 1;
+	u8 reglo = nv_ro08(bios, init->offset + 3);
+	u8 reghi = nv_ro08(bios, init->offset + 4);
+	u8  mask = nv_ro08(bios, init->offset + 5);
+	u8  data = nv_ro08(bios, init->offset + 6);
+	struct nouveau_i2c_port *port;
+
+	trace("I2C_LONG_IF\t"
+	      "I2C[0x%02x][0x%02x][0x%02x%02x] & 0x%02x == 0x%02x\n",
+	      index, addr, reglo, reghi, mask, data);
+	init->offset += 7;
+
+	port = init_i2c(init, index);
+	if (port) {
+		u8 i[2] = { reghi, reglo };
+		u8 o[1] = {};
+		struct i2c_msg msg[] = {
+			{ .addr = addr, .flags = 0, .len = 2, .buf = i },
+			{ .addr = addr, .flags = I2C_M_RD, .len = 1, .buf = o }
+		};
+		int ret;
+
+		ret = i2c_transfer(&port->adapter, msg, 2);
+		if (ret == 2 && ((o[0] & mask) == data))
+			return;
+	}
+
+	init_exec_set(init, false);
+}
+
+static struct nvbios_init_opcode {
+	void (*exec)(struct nvbios_init *);
+} init_opcode[] = {
+	[0x32] = { init_io_restrict_prog },
+	[0x33] = { init_repeat },
+	[0x34] = { init_io_restrict_pll },
+	[0x36] = { init_end_repeat },
+	[0x37] = { init_copy },
+	[0x38] = { init_not },
+	[0x39] = { init_io_flag_condition },
+	[0x3a] = { init_dp_condition },
+	[0x3b] = { init_io_mask_or },
+	[0x3c] = { init_io_or },
+	[0x49] = { init_idx_addr_latched },
+	[0x4a] = { init_io_restrict_pll2 },
+	[0x4b] = { init_pll2 },
+	[0x4c] = { init_i2c_byte },
+	[0x4d] = { init_zm_i2c_byte },
+	[0x4e] = { init_zm_i2c },
+	[0x4f] = { init_tmds },
+	[0x50] = { init_zm_tmds_group },
+	[0x51] = { init_cr_idx_adr_latch },
+	[0x52] = { init_cr },
+	[0x53] = { init_zm_cr },
+	[0x54] = { init_zm_cr_group },
+	[0x56] = { init_condition_time },
+	[0x57] = { init_ltime },
+	[0x58] = { init_zm_reg_sequence },
+	[0x5b] = { init_sub_direct },
+	[0x5c] = { init_jump },
+	[0x5e] = { init_i2c_if },
+	[0x5f] = { init_copy_nv_reg },
+	[0x62] = { init_zm_index_io },
+	[0x63] = { init_compute_mem },
+	[0x65] = { init_reset },
+	[0x66] = { init_configure_mem },
+	[0x67] = { init_configure_clk },
+	[0x68] = { init_configure_preinit },
+	[0x69] = { init_io },
+	[0x6b] = { init_sub },
+	[0x6d] = { init_ram_condition },
+	[0x6e] = { init_nv_reg },
+	[0x6f] = { init_macro },
+	[0x71] = { init_done },
+	[0x72] = { init_resume },
+	[0x74] = { init_time },
+	[0x75] = { init_condition },
+	[0x76] = { init_io_condition },
+	[0x78] = { init_index_io },
+	[0x79] = { init_pll },
+	[0x7a] = { init_zm_reg },
+	[0x87] = { init_ram_restrict_pll },
+	[0x8c] = { init_reserved },
+	[0x8d] = { init_reserved },
+	[0x8e] = { init_gpio },
+	[0x8f] = { init_ram_restrict_zm_reg_group },
+	[0x90] = { init_copy_zm_reg },
+	[0x91] = { init_zm_reg_group },
+	[0x92] = { init_reserved },
+	[0x96] = { init_xlat },
+	[0x97] = { init_zm_mask_add },
+	[0x98] = { init_auxch },
+	[0x99] = { init_zm_auxch },
+	[0x9a] = { init_i2c_long_if },
+};
+
+#define init_opcode_nr (sizeof(init_opcode) / sizeof(init_opcode[0]))
+
+int
+nvbios_exec(struct nvbios_init *init)
+{
+	init->nested++;
+	while (init->offset) {
+		u8 opcode = nv_ro08(init->bios, init->offset);
+		if (opcode >= init_opcode_nr || !init_opcode[opcode].exec) {
+			error("unknown opcode 0x%02x\n", opcode);
+			return -EINVAL;
+		}
+
+		init_opcode[opcode].exec(init);
+	}
+	init->nested--;
+	return 0;
+}
+
+int
+nvbios_init(struct nouveau_subdev *subdev, bool execute)
+{
+	struct nouveau_bios *bios = nouveau_bios(subdev);
+	int ret = 0;
+	int i = -1;
+	u16 data;
+
+	if (execute)
+		nv_info(bios, "running init tables\n");
+	while (!ret && (data = (init_script(bios, ++i)))) {
+		struct nvbios_init init = {
+			.subdev = subdev,
+			.bios = bios,
+			.offset = data,
+			.outp = NULL,
+			.crtc = -1,
+			.execute = execute ? 1 : 0,
+		};
+
+		ret = nvbios_exec(&init);
+	}
+
+	/* the vbios parser will run this right after the normal init
+	 * tables, whereas the binary driver appears to run it later.
+	 */
+	if (!ret && (data = init_unknown_script(bios))) {
+		struct nvbios_init init = {
+			.subdev = subdev,
+			.bios = bios,
+			.offset = data,
+			.outp = NULL,
+			.crtc = -1,
+			.execute = execute ? 1 : 0,
+		};
+
+		ret = nvbios_exec(&init);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/mxm.c b/drivers/gpu/drm/nouveau/core/subdev/bios/mxm.c
new file mode 100644
index 00000000000..2610b11a99b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/mxm.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/mxm.h>
+
+u16
+mxm_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr)
+{
+	struct bit_entry x;
+
+	if (bit_entry(bios, 'x', &x)) {
+		nv_debug(bios, "BIT 'x' table not present\n");
+		return 0x0000;
+	}
+
+	*ver = x.version;
+	*hdr = x.length;
+	if (*ver != 1 || *hdr < 3) {
+		nv_warn(bios, "BIT 'x' table %d/%d unknown\n", *ver, *hdr);
+		return 0x0000;
+	}
+
+	return x.offset;
+}
+
+/* These map MXM v2.x digital connection values to the appropriate SOR/link,
+ * hopefully they're correct for all boards within the same chipset...
+ *
+ * MXM v3.x VBIOS are nicer and provide pointers to these tables.
+ */
+static u8 nv84_sor_map[16] = {
+	0x00, 0x12, 0x22, 0x11, 0x32, 0x31, 0x11, 0x31,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static u8 nv92_sor_map[16] = {
+	0x00, 0x12, 0x22, 0x11, 0x32, 0x31, 0x11, 0x31,
+	0x11, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static u8 nv94_sor_map[16] = {
+	0x00, 0x14, 0x24, 0x11, 0x34, 0x31, 0x11, 0x31,
+	0x11, 0x31, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+static u8 nv98_sor_map[16] = {
+	0x00, 0x14, 0x12, 0x11, 0x00, 0x31, 0x11, 0x31,
+	0x11, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+u8
+mxm_sor_map(struct nouveau_bios *bios, u8 conn)
+{
+	u8  ver, hdr;
+	u16 mxm = mxm_table(bios, &ver, &hdr);
+	if (mxm && hdr >= 6) {
+		u16 map = nv_ro16(bios, mxm + 4);
+		if (map) {
+			ver = nv_ro08(bios, map);
+			if (ver == 0x10) {
+				if (conn < nv_ro08(bios, map + 3)) {
+					map += nv_ro08(bios, map + 1);
+					map += conn;
+					return nv_ro08(bios, map);
+				}
+
+				return 0x00;
+			}
+
+			nv_warn(bios, "unknown sor map v%02x\n", ver);
+		}
+	}
+
+	if (bios->version.chip == 0x84 || bios->version.chip == 0x86)
+		return nv84_sor_map[conn];
+	if (bios->version.chip == 0x92)
+		return nv92_sor_map[conn];
+	if (bios->version.chip == 0x94 || bios->version.chip == 0x96)
+		return nv94_sor_map[conn];
+	if (bios->version.chip == 0x98)
+		return nv98_sor_map[conn];
+
+	nv_warn(bios, "missing sor map\n");
+	return 0x00;
+}
+
+u8
+mxm_ddc_map(struct nouveau_bios *bios, u8 port)
+{
+	u8  ver, hdr;
+	u16 mxm = mxm_table(bios, &ver, &hdr);
+	if (mxm && hdr >= 8) {
+		u16 map = nv_ro16(bios, mxm + 6);
+		if (map) {
+			ver = nv_ro08(bios, map);
+			if (ver == 0x10) {
+				if (port < nv_ro08(bios, map + 3)) {
+					map += nv_ro08(bios, map + 1);
+					map += port;
+					return nv_ro08(bios, map);
+				}
+
+				return 0x00;
+			}
+
+			nv_warn(bios, "unknown ddc map v%02x\n", ver);
+		}
+	}
+
+	/* v2.x: directly write port as dcb i2cidx */
+	return (port << 4) | port;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c b/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c
new file mode 100644
index 00000000000..bcbb056c288
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/perf.h>
+
+static u16
+perf_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	struct bit_entry bit_P;
+	u16 perf = 0x0000;
+
+	if (!bit_entry(bios, 'P', &bit_P)) {
+		if (bit_P.version <= 2) {
+			perf = nv_ro16(bios, bit_P.offset + 0);
+			if (perf) {
+				*ver = nv_ro08(bios, perf + 0);
+				*hdr = nv_ro08(bios, perf + 1);
+			}
+		} else
+			nv_error(bios, "unknown offset for perf in BIT P %d\n",
+				bit_P.version);
+	}
+
+	if (bios->bmp_offset) {
+		if (nv_ro08(bios, bios->bmp_offset + 6) >= 0x25) {
+			perf = nv_ro16(bios, bios->bmp_offset + 0x94);
+			if (perf) {
+				*hdr = nv_ro08(bios, perf + 0);
+				*ver = nv_ro08(bios, perf + 1);
+			}
+		}
+	}
+
+	return perf;
+}
+
+int
+nvbios_perf_fan_parse(struct nouveau_bios *bios,
+		      struct nvbios_perf_fan *fan)
+{
+	u8 ver = 0, hdr = 0, cnt = 0, len = 0;
+	u16 perf = perf_table(bios, &ver, &hdr, &cnt, &len);
+	if (!perf)
+		return -ENODEV;
+
+	if (ver >= 0x20 && ver < 0x40 && hdr > 6)
+		fan->pwm_divisor = nv_ro16(bios, perf + 6);
+	else
+		fan->pwm_divisor = 0;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c
new file mode 100644
index 00000000000..5e5f4cddae3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c
@@ -0,0 +1,417 @@
+/*
+ * Copyright 2005-2006 Erik Waling
+ * Copyright 2006 Stephane Marchesin
+ * Copyright 2007-2009 Stuart Bennett
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <subdev/vga.h>
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/bmp.h>
+#include <subdev/bios/pll.h>
+
+struct pll_mapping {
+	u8  type;
+	u32 reg;
+};
+
+static struct pll_mapping
+nv04_pll_mapping[] = {
+	{ PLL_CORE  , 0x680500 },
+	{ PLL_MEMORY, 0x680504 },
+	{ PLL_VPLL0 , 0x680508 },
+	{ PLL_VPLL1 , 0x680520 },
+	{}
+};
+
+static struct pll_mapping
+nv40_pll_mapping[] = {
+	{ PLL_CORE  , 0x004000 },
+	{ PLL_MEMORY, 0x004020 },
+	{ PLL_VPLL0 , 0x680508 },
+	{ PLL_VPLL1 , 0x680520 },
+	{}
+};
+
+static struct pll_mapping
+nv50_pll_mapping[] = {
+	{ PLL_CORE  , 0x004028 },
+	{ PLL_SHADER, 0x004020 },
+	{ PLL_UNK03 , 0x004000 },
+	{ PLL_MEMORY, 0x004008 },
+	{ PLL_UNK40 , 0x00e810 },
+	{ PLL_UNK41 , 0x00e818 },
+	{ PLL_UNK42 , 0x00e824 },
+	{ PLL_VPLL0 , 0x614100 },
+	{ PLL_VPLL1 , 0x614900 },
+	{}
+};
+
+static struct pll_mapping
+nv84_pll_mapping[] = {
+	{ PLL_CORE  , 0x004028 },
+	{ PLL_SHADER, 0x004020 },
+	{ PLL_MEMORY, 0x004008 },
+	{ PLL_VDEC  , 0x004030 },
+	{ PLL_UNK41 , 0x00e818 },
+	{ PLL_VPLL0 , 0x614100 },
+	{ PLL_VPLL1 , 0x614900 },
+	{}
+};
+
+static u16
+pll_limits_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+	struct bit_entry bit_C;
+
+	if (!bit_entry(bios, 'C', &bit_C) && bit_C.length >= 10) {
+		u16 data = nv_ro16(bios, bit_C.offset + 8);
+		if (data) {
+			*ver = nv_ro08(bios, data + 0);
+			*hdr = nv_ro08(bios, data + 1);
+			*len = nv_ro08(bios, data + 2);
+			*cnt = nv_ro08(bios, data + 3);
+			return data;
+		}
+	}
+
+	if (bmp_version(bios) >= 0x0524) {
+		u16 data = nv_ro16(bios, bios->bmp_offset + 142);
+		if (data) {
+			*ver = nv_ro08(bios, data + 0);
+			*hdr = 1;
+			*cnt = 1;
+			*len = 0x18;
+			return data;
+		}
+	}
+
+	*ver = 0x00;
+	return 0x0000;
+}
+
+static struct pll_mapping *
+pll_map(struct nouveau_bios *bios)
+{
+	switch (nv_device(bios)->card_type) {
+	case NV_04:
+	case NV_10:
+	case NV_20:
+	case NV_30:
+		return nv04_pll_mapping;
+		break;
+	case NV_40:
+		return nv40_pll_mapping;
+	case NV_50:
+		if (nv_device(bios)->chipset == 0x50)
+			return nv50_pll_mapping;
+		else
+		if (nv_device(bios)->chipset <  0xa3 ||
+		    nv_device(bios)->chipset == 0xaa ||
+		    nv_device(bios)->chipset == 0xac)
+			return nv84_pll_mapping;
+	default:
+		return NULL;
+	}
+}
+
+static u16
+pll_map_reg(struct nouveau_bios *bios, u32 reg, u32 *type, u8 *ver, u8 *len)
+{
+	struct pll_mapping *map;
+	u8  hdr, cnt;
+	u16 data;
+
+	data = pll_limits_table(bios, ver, &hdr, &cnt, len);
+	if (data && *ver >= 0x30) {
+		data += hdr;
+		while (cnt--) {
+			if (nv_ro32(bios, data + 3) == reg) {
+				*type = nv_ro08(bios, data + 0);
+				return data;
+			}
+			data += *len;
+		}
+		return 0x0000;
+	}
+
+	map = pll_map(bios);
+	while (map->reg) {
+		if (map->reg == reg && *ver >= 0x20) {
+			u16 addr = (data += hdr);
+			while (cnt--) {
+				if (nv_ro32(bios, data) == map->reg) {
+					*type = map->type;
+					return data;
+				}
+				data += *len;
+			}
+			return addr;
+		} else
+		if (map->reg == reg) {
+			*type = map->type;
+			return data + 1;
+		}
+		map++;
+	}
+
+	return 0x0000;
+}
+
+static u16
+pll_map_type(struct nouveau_bios *bios, u8 type, u32 *reg, u8 *ver, u8 *len)
+{
+	struct pll_mapping *map;
+	u8  hdr, cnt;
+	u16 data;
+
+	data = pll_limits_table(bios, ver, &hdr, &cnt, len);
+	if (data && *ver >= 0x30) {
+		data += hdr;
+		while (cnt--) {
+			if (nv_ro08(bios, data + 0) == type) {
+				*reg = nv_ro32(bios, data + 3);
+				return data;
+			}
+			data += *len;
+		}
+		return 0x0000;
+	}
+
+	map = pll_map(bios);
+	while (map->reg) {
+		if (map->type == type && *ver >= 0x20) {
+			u16 addr = (data += hdr);
+			while (cnt--) {
+				if (nv_ro32(bios, data) == map->reg) {
+					*reg = map->reg;
+					return data;
+				}
+				data += *len;
+			}
+			return addr;
+		} else
+		if (map->type == type) {
+			*reg = map->reg;
+			return data + 1;
+		}
+		map++;
+	}
+
+	return 0x0000;
+}
+
+int
+nvbios_pll_parse(struct nouveau_bios *bios, u32 type, struct nvbios_pll *info)
+{
+	u8  ver, len;
+	u32 reg = type;
+	u16 data;
+
+	if (type > PLL_MAX) {
+		reg  = type;
+		data = pll_map_reg(bios, reg, &type, &ver, &len);
+	} else {
+		data = pll_map_type(bios, type, &reg, &ver, &len);
+	}
+
+	if (ver && !data)
+		return -ENOENT;
+
+	memset(info, 0, sizeof(*info));
+	info->type = type;
+	info->reg = reg;
+
+	switch (ver) {
+	case 0x00:
+		break;
+	case 0x10:
+	case 0x11:
+		info->vco1.min_freq = nv_ro32(bios, data + 0);
+		info->vco1.max_freq = nv_ro32(bios, data + 4);
+		info->vco2.min_freq = nv_ro32(bios, data + 8);
+		info->vco2.max_freq = nv_ro32(bios, data + 12);
+		info->vco1.min_inputfreq = nv_ro32(bios, data + 16);
+		info->vco2.min_inputfreq = nv_ro32(bios, data + 20);
+		info->vco1.max_inputfreq = INT_MAX;
+		info->vco2.max_inputfreq = INT_MAX;
+
+		info->max_p = 0x7;
+		info->max_p_usable = 0x6;
+
+		/* these values taken from nv30/31/36 */
+		switch (bios->version.chip) {
+		case 0x36:
+			info->vco1.min_n = 0x5;
+			break;
+		default:
+			info->vco1.min_n = 0x1;
+			break;
+		}
+		info->vco1.max_n = 0xff;
+		info->vco1.min_m = 0x1;
+		info->vco1.max_m = 0xd;
+
+		/*
+		 * On nv30, 31, 36 (i.e. all cards with two stage PLLs with this
+		 * table version (apart from nv35)), N2 is compared to
+		 * maxN2 (0x46) and 10 * maxM2 (0x4), so set maxN2 to 0x28 and
+		 * save a comparison
+		 */
+		info->vco2.min_n = 0x4;
+		switch (bios->version.chip) {
+		case 0x30:
+		case 0x35:
+			info->vco2.max_n = 0x1f;
+			break;
+		default:
+			info->vco2.max_n = 0x28;
+			break;
+		}
+		info->vco2.min_m = 0x1;
+		info->vco2.max_m = 0x4;
+		break;
+	case 0x20:
+	case 0x21:
+		info->vco1.min_freq = nv_ro16(bios, data + 4) * 1000;
+		info->vco1.max_freq = nv_ro16(bios, data + 6) * 1000;
+		info->vco2.min_freq = nv_ro16(bios, data + 8) * 1000;
+		info->vco2.max_freq = nv_ro16(bios, data + 10) * 1000;
+		info->vco1.min_inputfreq = nv_ro16(bios, data + 12) * 1000;
+		info->vco2.min_inputfreq = nv_ro16(bios, data + 14) * 1000;
+		info->vco1.max_inputfreq = nv_ro16(bios, data + 16) * 1000;
+		info->vco2.max_inputfreq = nv_ro16(bios, data + 18) * 1000;
+		info->vco1.min_n = nv_ro08(bios, data + 20);
+		info->vco1.max_n = nv_ro08(bios, data + 21);
+		info->vco1.min_m = nv_ro08(bios, data + 22);
+		info->vco1.max_m = nv_ro08(bios, data + 23);
+		info->vco2.min_n = nv_ro08(bios, data + 24);
+		info->vco2.max_n = nv_ro08(bios, data + 25);
+		info->vco2.min_m = nv_ro08(bios, data + 26);
+		info->vco2.max_m = nv_ro08(bios, data + 27);
+
+		info->max_p = nv_ro08(bios, data + 29);
+		info->max_p_usable = info->max_p;
+		if (bios->version.chip < 0x60)
+			info->max_p_usable = 0x6;
+		info->bias_p = nv_ro08(bios, data + 30);
+
+		if (len > 0x22)
+			info->refclk = nv_ro32(bios, data + 31);
+		break;
+	case 0x30:
+		data = nv_ro16(bios, data + 1);
+
+		info->vco1.min_freq = nv_ro16(bios, data + 0) * 1000;
+		info->vco1.max_freq = nv_ro16(bios, data + 2) * 1000;
+		info->vco2.min_freq = nv_ro16(bios, data + 4) * 1000;
+		info->vco2.max_freq = nv_ro16(bios, data + 6) * 1000;
+		info->vco1.min_inputfreq = nv_ro16(bios, data + 8) * 1000;
+		info->vco2.min_inputfreq = nv_ro16(bios, data + 10) * 1000;
+		info->vco1.max_inputfreq = nv_ro16(bios, data + 12) * 1000;
+		info->vco2.max_inputfreq = nv_ro16(bios, data + 14) * 1000;
+		info->vco1.min_n = nv_ro08(bios, data + 16);
+		info->vco1.max_n = nv_ro08(bios, data + 17);
+		info->vco1.min_m = nv_ro08(bios, data + 18);
+		info->vco1.max_m = nv_ro08(bios, data + 19);
+		info->vco2.min_n = nv_ro08(bios, data + 20);
+		info->vco2.max_n = nv_ro08(bios, data + 21);
+		info->vco2.min_m = nv_ro08(bios, data + 22);
+		info->vco2.max_m = nv_ro08(bios, data + 23);
+		info->max_p_usable = info->max_p = nv_ro08(bios, data + 25);
+		info->bias_p = nv_ro08(bios, data + 27);
+		info->refclk = nv_ro32(bios, data + 28);
+		break;
+	case 0x40:
+		info->refclk = nv_ro16(bios, data + 9) * 1000;
+		data = nv_ro16(bios, data + 1);
+
+		info->vco1.min_freq = nv_ro16(bios, data + 0) * 1000;
+		info->vco1.max_freq = nv_ro16(bios, data + 2) * 1000;
+		info->vco1.min_inputfreq = nv_ro16(bios, data + 4) * 1000;
+		info->vco1.max_inputfreq = nv_ro16(bios, data + 6) * 1000;
+		info->vco1.min_m = nv_ro08(bios, data + 8);
+		info->vco1.max_m = nv_ro08(bios, data + 9);
+		info->vco1.min_n = nv_ro08(bios, data + 10);
+		info->vco1.max_n = nv_ro08(bios, data + 11);
+		info->min_p = nv_ro08(bios, data + 12);
+		info->max_p = nv_ro08(bios, data + 13);
+		break;
+	default:
+		nv_error(bios, "unknown pll limits version 0x%02x\n", ver);
+		return -EINVAL;
+	}
+
+	if (!info->refclk) {
+		info->refclk = nv_device(bios)->crystal;
+		if (bios->version.chip == 0x51) {
+			u32 sel_clk = nv_rd32(bios, 0x680524);
+			if ((info->reg == 0x680508 && sel_clk & 0x20) ||
+			    (info->reg == 0x680520 && sel_clk & 0x80)) {
+				if (nv_rdvgac(bios, 0, 0x27) < 0xa3)
+					info->refclk = 200000;
+				else
+					info->refclk = 25000;
+			}
+		}
+	}
+
+	/*
+	 * By now any valid limit table ought to have set a max frequency for
+	 * vco1, so if it's zero it's either a pre limit table bios, or one
+	 * with an empty limit table (seen on nv18)
+	 */
+	if (!info->vco1.max_freq) {
+		info->vco1.max_freq = nv_ro32(bios, bios->bmp_offset + 67);
+		info->vco1.min_freq = nv_ro32(bios, bios->bmp_offset + 71);
+		if (bmp_version(bios) < 0x0506) {
+			info->vco1.max_freq = 256000;
+			info->vco1.min_freq = 128000;
+		}
+
+		info->vco1.min_inputfreq = 0;
+		info->vco1.max_inputfreq = INT_MAX;
+		info->vco1.min_n = 0x1;
+		info->vco1.max_n = 0xff;
+		info->vco1.min_m = 0x1;
+
+		if (nv_device(bios)->crystal == 13500) {
+			/* nv05 does this, nv11 doesn't, nv10 unknown */
+			if (bios->version.chip < 0x11)
+				info->vco1.min_m = 0x7;
+			info->vco1.max_m = 0xd;
+		} else {
+			if (bios->version.chip < 0x11)
+				info->vco1.min_m = 0x8;
+			info->vco1.max_m = 0xe;
+		}
+
+		if (bios->version.chip <  0x17 ||
+		    bios->version.chip == 0x1a ||
+		    bios->version.chip == 0x20)
+			info->max_p = 4;
+		else
+			info->max_p = 5;
+		info->max_p_usable = info->max_p;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c b/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c
new file mode 100644
index 00000000000..862a08a2ae2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/therm.h>
+
+static u16
+therm_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *len, u8 *cnt)
+{
+	struct bit_entry bit_P;
+	u16 therm = 0;
+
+	if (!bit_entry(bios, 'P', &bit_P)) {
+		if (bit_P.version == 1)
+			therm = nv_ro16(bios, bit_P.offset + 12);
+		else if (bit_P.version == 2)
+			therm = nv_ro16(bios, bit_P.offset + 16);
+		else
+			nv_error(bios,
+				"unknown offset for thermal in BIT P %d\n",
+				bit_P.version);
+	}
+
+	/* exit now if we haven't found the thermal table */
+	if (!therm)
+		return 0x0000;
+
+	*ver = nv_ro08(bios, therm + 0);
+	*hdr = nv_ro08(bios, therm + 1);
+	*len = nv_ro08(bios, therm + 2);
+	*cnt = nv_ro08(bios, therm + 3);
+
+	return therm + nv_ro08(bios, therm + 1);
+}
+
+u16
+nvbios_therm_entry(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+	u8 hdr, cnt;
+	u16 therm = therm_table(bios, ver, &hdr, len, &cnt);
+	if (therm && idx < cnt)
+		return therm + idx * *len;
+	return 0x0000;
+}
+
+int
+nvbios_therm_sensor_parse(struct nouveau_bios *bios,
+			  enum nvbios_therm_domain domain,
+			  struct nvbios_therm_sensor *sensor)
+{
+	s8 thrs_section, sensor_section, offset;
+	u8 ver, len, i;
+	u16 entry;
+
+	/* we only support the core domain for now */
+	if (domain != NVBIOS_THERM_DOMAIN_CORE)
+		return -EINVAL;
+
+	/* Read the entries from the table */
+	thrs_section = 0;
+	sensor_section = -1;
+	i = 0;
+	while ((entry = nvbios_therm_entry(bios, i++, &ver, &len))) {
+		s16 value = nv_ro16(bios, entry + 1);
+
+		switch (nv_ro08(bios, entry + 0)) {
+		case 0x0:
+			thrs_section = value;
+			if (value > 0)
+				return 0; /* we do not try to support ambient */
+			break;
+		case 0x01:
+			sensor_section++;
+			if (sensor_section == 0) {
+				offset = ((s8) nv_ro08(bios, entry + 2)) / 2;
+				sensor->offset_constant = offset;
+			}
+			break;
+
+		case 0x04:
+			if (thrs_section == 0) {
+				sensor->thrs_critical.temp = (value & 0xff0) >> 4;
+				sensor->thrs_critical.hysteresis = value & 0xf;
+			}
+			break;
+
+		case 0x07:
+			if (thrs_section == 0) {
+				sensor->thrs_down_clock.temp = (value & 0xff0) >> 4;
+				sensor->thrs_down_clock.hysteresis = value & 0xf;
+			}
+			break;
+
+		case 0x08:
+			if (thrs_section == 0) {
+				sensor->thrs_fan_boost.temp = (value & 0xff0) >> 4;
+				sensor->thrs_fan_boost.hysteresis = value & 0xf;
+			}
+			break;
+
+		case 0x10:
+			if (sensor_section == 0)
+				sensor->offset_num = value;
+			break;
+
+		case 0x11:
+			if (sensor_section == 0)
+				sensor->offset_den = value;
+			break;
+
+		case 0x12:
+			if (sensor_section == 0)
+				sensor->slope_mult = value;
+			break;
+
+		case 0x13:
+			if (sensor_section == 0)
+				sensor->slope_div = value;
+			break;
+		case 0x32:
+			if (thrs_section == 0) {
+				sensor->thrs_shutdown.temp = (value & 0xff0) >> 4;
+				sensor->thrs_shutdown.hysteresis = value & 0xf;
+			}
+			break;
+		}
+	}
+
+	return 0;
+}
+
+int
+nvbios_therm_fan_parse(struct nouveau_bios *bios,
+			  struct nvbios_therm_fan *fan)
+{
+	u8 ver, len, i;
+	u16 entry;
+
+	i = 0;
+	while ((entry = nvbios_therm_entry(bios, i++, &ver, &len))) {
+		s16 value = nv_ro16(bios, entry + 1);
+
+		switch (nv_ro08(bios, entry + 0)) {
+		case 0x22:
+			fan->min_duty = value & 0xff;
+			fan->max_duty = (value & 0xff00) >> 8;
+			break;
+		case 0x26:
+			fan->pwm_freq = value;
+			break;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
new file mode 100644
index 00000000000..b7fd1151166
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
@@ -0,0 +1,359 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+struct nv04_clock_priv {
+	struct nouveau_clock base;
+};
+
+static int
+powerctrl_1_shift(int chip_version, int reg)
+{
+	int shift = -4;
+
+	if (chip_version < 0x17 || chip_version == 0x1a || chip_version == 0x20)
+		return shift;
+
+	switch (reg) {
+	case 0x680520:
+		shift += 4;
+	case 0x680508:
+		shift += 4;
+	case 0x680504:
+		shift += 4;
+	case 0x680500:
+		shift += 4;
+	}
+
+	/*
+	 * the shift for vpll regs is only used for nv3x chips with a single
+	 * stage pll
+	 */
+	if (shift > 4 && (chip_version < 0x32 || chip_version == 0x35 ||
+			  chip_version == 0x36 || chip_version >= 0x40))
+		shift = -4;
+
+	return shift;
+}
+
+static void
+setPLL_single(struct nv04_clock_priv *priv, u32 reg,
+	      struct nouveau_pll_vals *pv)
+{
+	int chip_version = nouveau_bios(priv)->version.chip;
+	uint32_t oldpll = nv_rd32(priv, reg);
+	int oldN = (oldpll >> 8) & 0xff, oldM = oldpll & 0xff;
+	uint32_t pll = (oldpll & 0xfff80000) | pv->log2P << 16 | pv->NM1;
+	uint32_t saved_powerctrl_1 = 0;
+	int shift_powerctrl_1 = powerctrl_1_shift(chip_version, reg);
+
+	if (oldpll == pll)
+		return;	/* already set */
+
+	if (shift_powerctrl_1 >= 0) {
+		saved_powerctrl_1 = nv_rd32(priv, 0x001584);
+		nv_wr32(priv, 0x001584,
+			(saved_powerctrl_1 & ~(0xf << shift_powerctrl_1)) |
+			1 << shift_powerctrl_1);
+	}
+
+	if (oldM && pv->M1 && (oldN / oldM < pv->N1 / pv->M1))
+		/* upclock -- write new post divider first */
+		nv_wr32(priv, reg, pv->log2P << 16 | (oldpll & 0xffff));
+	else
+		/* downclock -- write new NM first */
+		nv_wr32(priv, reg, (oldpll & 0xffff0000) | pv->NM1);
+
+	if (chip_version < 0x17 && chip_version != 0x11)
+		/* wait a bit on older chips */
+		msleep(64);
+	nv_rd32(priv, reg);
+
+	/* then write the other half as well */
+	nv_wr32(priv, reg, pll);
+
+	if (shift_powerctrl_1 >= 0)
+		nv_wr32(priv, 0x001584, saved_powerctrl_1);
+}
+
+static uint32_t
+new_ramdac580(uint32_t reg1, bool ss, uint32_t ramdac580)
+{
+	bool head_a = (reg1 == 0x680508);
+
+	if (ss)	/* single stage pll mode */
+		ramdac580 |= head_a ? 0x00000100 : 0x10000000;
+	else
+		ramdac580 &= head_a ? 0xfffffeff : 0xefffffff;
+
+	return ramdac580;
+}
+
+static void
+setPLL_double_highregs(struct nv04_clock_priv *priv, u32 reg1,
+		       struct nouveau_pll_vals *pv)
+{
+	int chip_version = nouveau_bios(priv)->version.chip;
+	bool nv3035 = chip_version == 0x30 || chip_version == 0x35;
+	uint32_t reg2 = reg1 + ((reg1 == 0x680520) ? 0x5c : 0x70);
+	uint32_t oldpll1 = nv_rd32(priv, reg1);
+	uint32_t oldpll2 = !nv3035 ? nv_rd32(priv, reg2) : 0;
+	uint32_t pll1 = (oldpll1 & 0xfff80000) | pv->log2P << 16 | pv->NM1;
+	uint32_t pll2 = (oldpll2 & 0x7fff0000) | 1 << 31 | pv->NM2;
+	uint32_t oldramdac580 = 0, ramdac580 = 0;
+	bool single_stage = !pv->NM2 || pv->N2 == pv->M2;	/* nv41+ only */
+	uint32_t saved_powerctrl_1 = 0, savedc040 = 0;
+	int shift_powerctrl_1 = powerctrl_1_shift(chip_version, reg1);
+
+	/* model specific additions to generic pll1 and pll2 set up above */
+	if (nv3035) {
+		pll1 = (pll1 & 0xfcc7ffff) | (pv->N2 & 0x18) << 21 |
+		       (pv->N2 & 0x7) << 19 | 8 << 4 | (pv->M2 & 7) << 4;
+		pll2 = 0;
+	}
+	if (chip_version > 0x40 && reg1 >= 0x680508) { /* !nv40 */
+		oldramdac580 = nv_rd32(priv, 0x680580);
+		ramdac580 = new_ramdac580(reg1, single_stage, oldramdac580);
+		if (oldramdac580 != ramdac580)
+			oldpll1 = ~0;	/* force mismatch */
+		if (single_stage)
+			/* magic value used by nvidia in single stage mode */
+			pll2 |= 0x011f;
+	}
+	if (chip_version > 0x70)
+		/* magic bits set by the blob (but not the bios) on g71-73 */
+		pll1 = (pll1 & 0x7fffffff) | (single_stage ? 0x4 : 0xc) << 28;
+
+	if (oldpll1 == pll1 && oldpll2 == pll2)
+		return;	/* already set */
+
+	if (shift_powerctrl_1 >= 0) {
+		saved_powerctrl_1 = nv_rd32(priv, 0x001584);
+		nv_wr32(priv, 0x001584,
+			(saved_powerctrl_1 & ~(0xf << shift_powerctrl_1)) |
+			1 << shift_powerctrl_1);
+	}
+
+	if (chip_version >= 0x40) {
+		int shift_c040 = 14;
+
+		switch (reg1) {
+		case 0x680504:
+			shift_c040 += 2;
+		case 0x680500:
+			shift_c040 += 2;
+		case 0x680520:
+			shift_c040 += 2;
+		case 0x680508:
+			shift_c040 += 2;
+		}
+
+		savedc040 = nv_rd32(priv, 0xc040);
+		if (shift_c040 != 14)
+			nv_wr32(priv, 0xc040, savedc040 & ~(3 << shift_c040));
+	}
+
+	if (oldramdac580 != ramdac580)
+		nv_wr32(priv, 0x680580, ramdac580);
+
+	if (!nv3035)
+		nv_wr32(priv, reg2, pll2);
+	nv_wr32(priv, reg1, pll1);
+
+	if (shift_powerctrl_1 >= 0)
+		nv_wr32(priv, 0x001584, saved_powerctrl_1);
+	if (chip_version >= 0x40)
+		nv_wr32(priv, 0xc040, savedc040);
+}
+
+static void
+setPLL_double_lowregs(struct nv04_clock_priv *priv, u32 NMNMreg,
+		      struct nouveau_pll_vals *pv)
+{
+	/* When setting PLLs, there is a merry game of disabling and enabling
+	 * various bits of hardware during the process. This function is a
+	 * synthesis of six nv4x traces, nearly each card doing a subtly
+	 * different thing. With luck all the necessary bits for each card are
+	 * combined herein. Without luck it deviates from each card's formula
+	 * so as to not work on any :)
+	 */
+
+	uint32_t Preg = NMNMreg - 4;
+	bool mpll = Preg == 0x4020;
+	uint32_t oldPval = nv_rd32(priv, Preg);
+	uint32_t NMNM = pv->NM2 << 16 | pv->NM1;
+	uint32_t Pval = (oldPval & (mpll ? ~(0x77 << 16) : ~(7 << 16))) |
+			0xc << 28 | pv->log2P << 16;
+	uint32_t saved4600 = 0;
+	/* some cards have different maskc040s */
+	uint32_t maskc040 = ~(3 << 14), savedc040;
+	bool single_stage = !pv->NM2 || pv->N2 == pv->M2;
+
+	if (nv_rd32(priv, NMNMreg) == NMNM && (oldPval & 0xc0070000) == Pval)
+		return;
+
+	if (Preg == 0x4000)
+		maskc040 = ~0x333;
+	if (Preg == 0x4058)
+		maskc040 = ~(0xc << 24);
+
+	if (mpll) {
+		struct nvbios_pll info;
+		uint8_t Pval2;
+
+		if (nvbios_pll_parse(nouveau_bios(priv), Preg, &info))
+			return;
+
+		Pval2 = pv->log2P + info.bias_p;
+		if (Pval2 > info.max_p)
+			Pval2 = info.max_p;
+		Pval |= 1 << 28 | Pval2 << 20;
+
+		saved4600 = nv_rd32(priv, 0x4600);
+		nv_wr32(priv, 0x4600, saved4600 | 8 << 28);
+	}
+	if (single_stage)
+		Pval |= mpll ? 1 << 12 : 1 << 8;
+
+	nv_wr32(priv, Preg, oldPval | 1 << 28);
+	nv_wr32(priv, Preg, Pval & ~(4 << 28));
+	if (mpll) {
+		Pval |= 8 << 20;
+		nv_wr32(priv, 0x4020, Pval & ~(0xc << 28));
+		nv_wr32(priv, 0x4038, Pval & ~(0xc << 28));
+	}
+
+	savedc040 = nv_rd32(priv, 0xc040);
+	nv_wr32(priv, 0xc040, savedc040 & maskc040);
+
+	nv_wr32(priv, NMNMreg, NMNM);
+	if (NMNMreg == 0x4024)
+		nv_wr32(priv, 0x403c, NMNM);
+
+	nv_wr32(priv, Preg, Pval);
+	if (mpll) {
+		Pval &= ~(8 << 20);
+		nv_wr32(priv, 0x4020, Pval);
+		nv_wr32(priv, 0x4038, Pval);
+		nv_wr32(priv, 0x4600, saved4600);
+	}
+
+	nv_wr32(priv, 0xc040, savedc040);
+
+	if (mpll) {
+		nv_wr32(priv, 0x4020, Pval & ~(1 << 28));
+		nv_wr32(priv, 0x4038, Pval & ~(1 << 28));
+	}
+}
+
+int
+nv04_clock_pll_set(struct nouveau_clock *clk, u32 type, u32 freq)
+{
+	struct nv04_clock_priv *priv = (void *)clk;
+	struct nouveau_pll_vals pv;
+	struct nvbios_pll info;
+	int ret;
+
+	ret = nvbios_pll_parse(nouveau_bios(priv), type > 0x405c ?
+			       type : type - 4, &info);
+	if (ret)
+		return ret;
+
+	ret = clk->pll_calc(clk, &info, freq, &pv);
+	if (!ret)
+		return ret;
+
+	return clk->pll_prog(clk, type, &pv);
+}
+
+int
+nv04_clock_pll_calc(struct nouveau_clock *clock, struct nvbios_pll *info,
+		    int clk, struct nouveau_pll_vals *pv)
+{
+	int N1, M1, N2, M2, P;
+	int ret = nv04_pll_calc(clock, info, clk, &N1, &M1, &N2, &M2, &P);
+	if (ret) {
+		pv->refclk = info->refclk;
+		pv->N1 = N1;
+		pv->M1 = M1;
+		pv->N2 = N2;
+		pv->M2 = M2;
+		pv->log2P = P;
+	}
+	return ret;
+}
+
+int
+nv04_clock_pll_prog(struct nouveau_clock *clk, u32 reg1,
+		    struct nouveau_pll_vals *pv)
+{
+	struct nv04_clock_priv *priv = (void *)clk;
+	int cv = nouveau_bios(clk)->version.chip;
+
+	if (cv == 0x30 || cv == 0x31 || cv == 0x35 || cv == 0x36 ||
+	    cv >= 0x40) {
+		if (reg1 > 0x405c)
+			setPLL_double_highregs(priv, reg1, pv);
+		else
+			setPLL_double_lowregs(priv, reg1, pv);
+	} else
+		setPLL_single(priv, reg1, pv);
+
+	return 0;
+}
+
+static int
+nv04_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv04_clock_priv *priv;
+	int ret;
+
+	ret = nouveau_clock_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.pll_set = nv04_clock_pll_set;
+	priv->base.pll_calc = nv04_clock_pll_calc;
+	priv->base.pll_prog = nv04_clock_pll_prog;
+	return 0;
+}
+
+struct nouveau_oclass
+nv04_clock_oclass = {
+	.handle = NV_SUBDEV(CLOCK, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_clock_ctor,
+		.dtor = _nouveau_clock_dtor,
+		.init = _nouveau_clock_init,
+		.fini = _nouveau_clock_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.h b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
index c82de98fee0..a4b2b7ebf9a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ramht.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2010 Red Hat Inc.
+ * Copyright 2012 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -22,34 +22,38 @@
  * Authors: Ben Skeggs
  */
 
-#ifndef __NOUVEAU_RAMHT_H__
-#define __NOUVEAU_RAMHT_H__
+#include <subdev/clock.h>
 
-struct nouveau_ramht_entry {
-	struct list_head head;
-	struct nouveau_channel *channel;
-	struct nouveau_gpuobj *gpuobj;
-	u32 handle;
+struct nv40_clock_priv {
+	struct nouveau_clock base;
 };
 
-struct nouveau_ramht {
-	struct drm_device *dev;
-	struct kref refcount;
-	spinlock_t lock;
-	struct nouveau_gpuobj *gpuobj;
-	struct list_head entries;
-	int bits;
-};
+static int
+nv40_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv40_clock_priv *priv;
+	int ret;
 
-extern int  nouveau_ramht_new(struct drm_device *, struct nouveau_gpuobj *,
-			      struct nouveau_ramht **);
-extern void nouveau_ramht_ref(struct nouveau_ramht *, struct nouveau_ramht **,
-			      struct nouveau_channel *unref_channel);
+	ret = nouveau_clock_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
 
-extern int  nouveau_ramht_insert(struct nouveau_channel *, u32 handle,
-				 struct nouveau_gpuobj *);
-extern int  nouveau_ramht_remove(struct nouveau_channel *, u32 handle);
-extern struct nouveau_gpuobj *
-nouveau_ramht_find(struct nouveau_channel *chan, u32 handle);
+	priv->base.pll_set = nv04_clock_pll_set;
+	priv->base.pll_calc = nv04_clock_pll_calc;
+	priv->base.pll_prog = nv04_clock_pll_prog;
+	return 0;
+}
 
-#endif
+struct nouveau_oclass
+nv40_clock_oclass = {
+	.handle = NV_SUBDEV(CLOCK, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_clock_ctor,
+		.dtor = _nouveau_clock_dtor,
+		.init = _nouveau_clock_init,
+		.fini = _nouveau_clock_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
new file mode 100644
index 00000000000..fd181fbcedd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+struct nv50_clock_priv {
+	struct nouveau_clock base;
+};
+
+static int
+nv50_clock_pll_set(struct nouveau_clock *clk, u32 type, u32 freq)
+{
+	struct nv50_clock_priv *priv = (void *)clk;
+	struct nouveau_bios *bios = nouveau_bios(priv);
+	struct nvbios_pll info;
+	int N1, M1, N2, M2, P;
+	int ret;
+
+	ret = nvbios_pll_parse(bios, type, &info);
+	if (ret) {
+		nv_error(clk, "failed to retrieve pll data, %d\n", ret);
+		return ret;
+	}
+
+	ret = nv04_pll_calc(clk, &info, freq, &N1, &M1, &N2, &M2, &P);
+	if (!ret) {
+		nv_error(clk, "failed pll calculation\n");
+		return ret;
+	}
+
+	switch (info.type) {
+	case PLL_VPLL0:
+	case PLL_VPLL1:
+		nv_wr32(priv, info.reg + 0, 0x10000611);
+		nv_mask(priv, info.reg + 4, 0x00ff00ff, (M1 << 16) | N1);
+		nv_mask(priv, info.reg + 8, 0x7fff00ff, (P  << 28) |
+							(M2 << 16) | N2);
+		break;
+	case PLL_MEMORY:
+		nv_mask(priv, info.reg + 0, 0x01ff0000, (P << 22) |
+						        (info.bias_p << 19) |
+							(P << 16));
+		nv_wr32(priv, info.reg + 4, (N1 << 8) | M1);
+		break;
+	default:
+		nv_mask(priv, info.reg + 0, 0x00070000, (P << 16));
+		nv_wr32(priv, info.reg + 4, (N1 << 8) | M1);
+		break;
+	}
+
+	return 0;
+}
+
+static int
+nv50_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv50_clock_priv *priv;
+	int ret;
+
+	ret = nouveau_clock_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.pll_set = nv50_clock_pll_set;
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_clock_oclass = {
+	.handle = NV_SUBDEV(CLOCK, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_clock_ctor,
+		.dtor = _nouveau_clock_dtor,
+		.init = _nouveau_clock_init,
+		.fini = _nouveau_clock_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
new file mode 100644
index 00000000000..cc8d7d162d7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+struct nva3_clock_priv {
+	struct nouveau_clock base;
+};
+
+static int
+nva3_clock_pll_set(struct nouveau_clock *clk, u32 type, u32 freq)
+{
+	struct nva3_clock_priv *priv = (void *)clk;
+	struct nouveau_bios *bios = nouveau_bios(priv);
+	struct nvbios_pll info;
+	int N, fN, M, P;
+	int ret;
+
+	ret = nvbios_pll_parse(bios, type, &info);
+	if (ret)
+		return ret;
+
+	ret = nva3_pll_calc(clk, &info, freq, &N, &fN, &M, &P);
+	if (ret < 0)
+		return ret;
+
+	switch (info.type) {
+	case PLL_VPLL0:
+	case PLL_VPLL1:
+		nv_wr32(priv, info.reg + 0, 0x50000610);
+		nv_mask(priv, info.reg + 4, 0x003fffff,
+					    (P << 16) | (M << 8) | N);
+		nv_wr32(priv, info.reg + 8, fN);
+		break;
+	default:
+		nv_warn(priv, "0x%08x/%dKhz unimplemented\n", type, freq);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nva3_clock_priv *priv;
+	int ret;
+
+	ret = nouveau_clock_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.pll_set = nva3_clock_pll_set;
+	return 0;
+}
+
+struct nouveau_oclass
+nva3_clock_oclass = {
+	.handle = NV_SUBDEV(CLOCK, 0xa3),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nva3_clock_ctor,
+		.dtor = _nouveau_clock_dtor,
+		.init = _nouveau_clock_init,
+		.fini = _nouveau_clock_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
new file mode 100644
index 00000000000..5ccce0b17bf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+struct nvc0_clock_priv {
+	struct nouveau_clock base;
+};
+
+static int
+nvc0_clock_pll_set(struct nouveau_clock *clk, u32 type, u32 freq)
+{
+	struct nvc0_clock_priv *priv = (void *)clk;
+	struct nouveau_bios *bios = nouveau_bios(priv);
+	struct nvbios_pll info;
+	int N, fN, M, P;
+	int ret;
+
+	ret = nvbios_pll_parse(bios, type, &info);
+	if (ret)
+		return ret;
+
+	ret = nva3_pll_calc(clk, &info, freq, &N, &fN, &M, &P);
+	if (ret < 0)
+		return ret;
+
+	switch (info.type) {
+	case PLL_VPLL0:
+	case PLL_VPLL1:
+		nv_mask(priv, info.reg + 0x0c, 0x00000000, 0x00000100);
+		nv_wr32(priv, info.reg + 0x04, (P << 16) | (N << 8) | M);
+		nv_wr32(priv, info.reg + 0x10, fN << 16);
+		break;
+	default:
+		nv_warn(priv, "0x%08x/%dKhz unimplemented\n", type, freq);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+nvc0_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nvc0_clock_priv *priv;
+	int ret;
+
+	ret = nouveau_clock_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.pll_set = nvc0_clock_pll_set;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_clock_oclass = {
+	.handle = NV_SUBDEV(CLOCK, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_clock_ctor,
+		.dtor = _nouveau_clock_dtor,
+		.init = _nouveau_clock_init,
+		.fini = _nouveau_clock_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/pll.h b/drivers/gpu/drm/nouveau/core/subdev/clock/pll.h
new file mode 100644
index 00000000000..ef2c0078f33
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/pll.h
@@ -0,0 +1,9 @@
+#ifndef __NOUVEAU_PLL_H__
+#define __NOUVEAU_PLL_H__
+
+int nv04_pll_calc(struct nouveau_clock *, struct nvbios_pll *, u32 freq,
+		  int *N1, int *M1, int *N2, int *M2, int *P);
+int nva3_pll_calc(struct nouveau_clock *, struct nvbios_pll *, u32 freq,
+		  int *N, int *fN, int *M, int *P);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c
new file mode 100644
index 00000000000..a2ab6d051ba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright 1993-2003 NVIDIA, Corporation
+ * Copyright 2007-2009 Stuart Bennett
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+static int
+getMNP_single(struct nouveau_clock *clock, struct nvbios_pll *info, int clk,
+	      int *pN, int *pM, int *pP)
+{
+	/* Find M, N and P for a single stage PLL
+	 *
+	 * Note that some bioses (NV3x) have lookup tables of precomputed MNP
+	 * values, but we're too lazy to use those atm
+	 *
+	 * "clk" parameter in kHz
+	 * returns calculated clock
+	 */
+	int cv = nouveau_bios(clock)->version.chip;
+	int minvco = info->vco1.min_freq, maxvco = info->vco1.max_freq;
+	int minM = info->vco1.min_m, maxM = info->vco1.max_m;
+	int minN = info->vco1.min_n, maxN = info->vco1.max_n;
+	int minU = info->vco1.min_inputfreq;
+	int maxU = info->vco1.max_inputfreq;
+	int minP = info->min_p;
+	int maxP = info->max_p_usable;
+	int crystal = info->refclk;
+	int M, N, thisP, P;
+	int clkP, calcclk;
+	int delta, bestdelta = INT_MAX;
+	int bestclk = 0;
+
+	/* this division verified for nv20, nv18, nv28 (Haiku), and nv34 */
+	/* possibly correlated with introduction of 27MHz crystal */
+	if (cv < 0x17 || cv == 0x1a || cv == 0x20) {
+		if (clk > 250000)
+			maxM = 6;
+		if (clk > 340000)
+			maxM = 2;
+	} else if (cv < 0x40) {
+		if (clk > 150000)
+			maxM = 6;
+		if (clk > 200000)
+			maxM = 4;
+		if (clk > 340000)
+			maxM = 2;
+	}
+
+	P = 1 << maxP;
+	if ((clk * P) < minvco) {
+		minvco = clk * maxP;
+		maxvco = minvco * 2;
+	}
+
+	if (clk + clk/200 > maxvco)	/* +0.5% */
+		maxvco = clk + clk/200;
+
+	/* NV34 goes maxlog2P->0, NV20 goes 0->maxlog2P */
+	for (thisP = minP; thisP <= maxP; thisP++) {
+		P = 1 << thisP;
+		clkP = clk * P;
+
+		if (clkP < minvco)
+			continue;
+		if (clkP > maxvco)
+			return bestclk;
+
+		for (M = minM; M <= maxM; M++) {
+			if (crystal/M < minU)
+				return bestclk;
+			if (crystal/M > maxU)
+				continue;
+
+			/* add crystal/2 to round better */
+			N = (clkP * M + crystal/2) / crystal;
+
+			if (N < minN)
+				continue;
+			if (N > maxN)
+				break;
+
+			/* more rounding additions */
+			calcclk = ((N * crystal + P/2) / P + M/2) / M;
+			delta = abs(calcclk - clk);
+			/* we do an exhaustive search rather than terminating
+			 * on an optimality condition...
+			 */
+			if (delta < bestdelta) {
+				bestdelta = delta;
+				bestclk = calcclk;
+				*pN = N;
+				*pM = M;
+				*pP = thisP;
+				if (delta == 0)	/* except this one */
+					return bestclk;
+			}
+		}
+	}
+
+	return bestclk;
+}
+
+static int
+getMNP_double(struct nouveau_clock *clock, struct nvbios_pll *info, int clk,
+	      int *pN1, int *pM1, int *pN2, int *pM2, int *pP)
+{
+	/* Find M, N and P for a two stage PLL
+	 *
+	 * Note that some bioses (NV30+) have lookup tables of precomputed MNP
+	 * values, but we're too lazy to use those atm
+	 *
+	 * "clk" parameter in kHz
+	 * returns calculated clock
+	 */
+	int chip_version = nouveau_bios(clock)->version.chip;
+	int minvco1 = info->vco1.min_freq, maxvco1 = info->vco1.max_freq;
+	int minvco2 = info->vco2.min_freq, maxvco2 = info->vco2.max_freq;
+	int minU1 = info->vco1.min_inputfreq, minU2 = info->vco2.min_inputfreq;
+	int maxU1 = info->vco1.max_inputfreq, maxU2 = info->vco2.max_inputfreq;
+	int minM1 = info->vco1.min_m, maxM1 = info->vco1.max_m;
+	int minN1 = info->vco1.min_n, maxN1 = info->vco1.max_n;
+	int minM2 = info->vco2.min_m, maxM2 = info->vco2.max_m;
+	int minN2 = info->vco2.min_n, maxN2 = info->vco2.max_n;
+	int maxlog2P = info->max_p_usable;
+	int crystal = info->refclk;
+	bool fixedgain2 = (minM2 == maxM2 && minN2 == maxN2);
+	int M1, N1, M2, N2, log2P;
+	int clkP, calcclk1, calcclk2, calcclkout;
+	int delta, bestdelta = INT_MAX;
+	int bestclk = 0;
+
+	int vco2 = (maxvco2 - maxvco2/200) / 2;
+	for (log2P = 0; clk && log2P < maxlog2P && clk <= (vco2 >> log2P); log2P++)
+		;
+	clkP = clk << log2P;
+
+	if (maxvco2 < clk + clk/200)	/* +0.5% */
+		maxvco2 = clk + clk/200;
+
+	for (M1 = minM1; M1 <= maxM1; M1++) {
+		if (crystal/M1 < minU1)
+			return bestclk;
+		if (crystal/M1 > maxU1)
+			continue;
+
+		for (N1 = minN1; N1 <= maxN1; N1++) {
+			calcclk1 = crystal * N1 / M1;
+			if (calcclk1 < minvco1)
+				continue;
+			if (calcclk1 > maxvco1)
+				break;
+
+			for (M2 = minM2; M2 <= maxM2; M2++) {
+				if (calcclk1/M2 < minU2)
+					break;
+				if (calcclk1/M2 > maxU2)
+					continue;
+
+				/* add calcclk1/2 to round better */
+				N2 = (clkP * M2 + calcclk1/2) / calcclk1;
+				if (N2 < minN2)
+					continue;
+				if (N2 > maxN2)
+					break;
+
+				if (!fixedgain2) {
+					if (chip_version < 0x60)
+						if (N2/M2 < 4 || N2/M2 > 10)
+							continue;
+
+					calcclk2 = calcclk1 * N2 / M2;
+					if (calcclk2 < minvco2)
+						break;
+					if (calcclk2 > maxvco2)
+						continue;
+				} else
+					calcclk2 = calcclk1;
+
+				calcclkout = calcclk2 >> log2P;
+				delta = abs(calcclkout - clk);
+				/* we do an exhaustive search rather than terminating
+				 * on an optimality condition...
+				 */
+				if (delta < bestdelta) {
+					bestdelta = delta;
+					bestclk = calcclkout;
+					*pN1 = N1;
+					*pM1 = M1;
+					*pN2 = N2;
+					*pM2 = M2;
+					*pP = log2P;
+					if (delta == 0)	/* except this one */
+						return bestclk;
+				}
+			}
+		}
+	}
+
+	return bestclk;
+}
+
+int
+nv04_pll_calc(struct nouveau_clock *clk, struct nvbios_pll *info, u32 freq,
+	      int *N1, int *M1, int *N2, int *M2, int *P)
+{
+	int ret;
+
+	if (!info->vco2.max_freq) {
+		ret = getMNP_single(clk, info, freq, N1, M1, P);
+		*N2 = 1;
+		*M2 = 1;
+	} else {
+		ret = getMNP_double(clk, info, freq, N1, M1, N2, M2, P);
+	}
+
+	if (!ret)
+		nv_error(clk, "unable to compute acceptable pll values\n");
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_calc.c b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c
index 8cf63a8b30c..eed5c16cf61 100644
--- a/drivers/gpu/drm/nouveau/nv50_calc.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c
@@ -22,60 +22,43 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_hw.h"
+#include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
 
-int
-nv50_calc_pll(struct drm_device *dev, struct pll_lims *pll, int clk,
-	      int *N1, int *M1, int *N2, int *M2, int *P)
-{
-	struct nouveau_pll_vals pll_vals;
-	int ret;
-
-	ret = nouveau_calc_pll_mnp(dev, pll, clk, &pll_vals);
-	if (ret <= 0)
-		return ret;
-
-	*N1 = pll_vals.N1;
-	*M1 = pll_vals.M1;
-	*N2 = pll_vals.N2;
-	*M2 = pll_vals.M2;
-	*P = pll_vals.log2P;
-	return ret;
-}
+#include "pll.h"
 
 int
-nva3_calc_pll(struct drm_device *dev, struct pll_lims *pll, int clk,
-	      int *pN, int *pfN, int *pM, int *P)
+nva3_pll_calc(struct nouveau_clock *clock, struct nvbios_pll *info,
+	      u32 freq, int *pN, int *pfN, int *pM, int *P)
 {
 	u32 best_err = ~0, err;
 	int M, lM, hM, N, fN;
 
-	*P = pll->vco1.maxfreq / clk;
-	if (*P > pll->max_p)
-		*P = pll->max_p;
-	if (*P < pll->min_p)
-		*P = pll->min_p;
+	*P = info->vco1.max_freq / freq;
+	if (*P > info->max_p)
+		*P = info->max_p;
+	if (*P < info->min_p)
+		*P = info->min_p;
 
-	lM = (pll->refclk + pll->vco1.max_inputfreq) / pll->vco1.max_inputfreq;
-	lM = max(lM, (int)pll->vco1.min_m);
-	hM = (pll->refclk + pll->vco1.min_inputfreq) / pll->vco1.min_inputfreq;
-	hM = min(hM, (int)pll->vco1.max_m);
+	lM = (info->refclk + info->vco1.max_inputfreq) / info->vco1.max_inputfreq;
+	lM = max(lM, (int)info->vco1.min_m);
+	hM = (info->refclk + info->vco1.min_inputfreq) / info->vco1.min_inputfreq;
+	hM = min(hM, (int)info->vco1.max_m);
 
 	for (M = lM; M <= hM; M++) {
-		u32 tmp = clk * *P * M;
-		N  = tmp / pll->refclk;
-		fN = tmp % pll->refclk;
-		if (!pfN && fN >= pll->refclk / 2)
+		u32 tmp = freq * *P * M;
+		N  = tmp / info->refclk;
+		fN = tmp % info->refclk;
+		if (!pfN && fN >= info->refclk / 2)
 			N++;
 
-		if (N < pll->vco1.min_n)
+		if (N < info->vco1.min_n)
 			continue;
-		if (N > pll->vco1.max_n)
+		if (N > info->vco1.max_n)
 			break;
 
-		err = abs(clk - (pll->refclk * N / M / *P));
+		err = abs(freq - (info->refclk * N / M / *P));
 		if (err < best_err) {
 			best_err = err;
 			*pN = N;
@@ -83,15 +66,15 @@ nva3_calc_pll(struct drm_device *dev, struct pll_lims *pll, int clk,
 		}
 
 		if (pfN) {
-			*pfN = (((fN << 13) / pll->refclk) - 4096) & 0xffff;
-			return clk;
+			*pfN = (((fN << 13) / info->refclk) - 4096) & 0xffff;
+			return freq;
 		}
 	}
 
 	if (unlikely(best_err == ~0)) {
-		NV_ERROR(dev, "unable to find matching pll values\n");
+		nv_error(clock, "unable to find matching pll values\n");
 		return -EINVAL;
 	}
 
-	return pll->refclk * *pN / *pM / *P;
+	return info->refclk * *pN / *pM / *P;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/base.c b/drivers/gpu/drm/nouveau/core/subdev/device/base.c
new file mode 100644
index 00000000000..ca9a4648bd8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/base.c
@@ -0,0 +1,472 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/device.h>
+#include <core/client.h>
+#include <core/device.h>
+#include <core/option.h>
+
+#include <core/class.h>
+
+#include <subdev/device.h>
+
+static DEFINE_MUTEX(nv_devices_mutex);
+static LIST_HEAD(nv_devices);
+
+struct nouveau_device *
+nouveau_device_find(u64 name)
+{
+	struct nouveau_device *device, *match = NULL;
+	mutex_lock(&nv_devices_mutex);
+	list_for_each_entry(device, &nv_devices, head) {
+		if (device->handle == name) {
+			match = device;
+			break;
+		}
+	}
+	mutex_unlock(&nv_devices_mutex);
+	return match;
+}
+
+/******************************************************************************
+ * nouveau_devobj (0x0080): class implementation
+ *****************************************************************************/
+struct nouveau_devobj {
+	struct nouveau_parent base;
+	struct nouveau_object *subdev[NVDEV_SUBDEV_NR];
+	bool created;
+};
+
+static const u64 disable_map[] = {
+	[NVDEV_SUBDEV_VBIOS]	= NV_DEVICE_DISABLE_VBIOS,
+	[NVDEV_SUBDEV_GPIO]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_I2C]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_DEVINIT]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_MC]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_TIMER]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_FB]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_VM]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_INSTMEM]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_BAR]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_VOLT]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_CLOCK]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_SUBDEV_THERM]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_ENGINE_DMAOBJ]	= NV_DEVICE_DISABLE_CORE,
+	[NVDEV_ENGINE_GR]	= NV_DEVICE_DISABLE_GRAPH,
+	[NVDEV_ENGINE_MPEG]	= NV_DEVICE_DISABLE_MPEG,
+	[NVDEV_ENGINE_ME]	= NV_DEVICE_DISABLE_ME,
+	[NVDEV_ENGINE_VP]	= NV_DEVICE_DISABLE_VP,
+	[NVDEV_ENGINE_CRYPT]	= NV_DEVICE_DISABLE_CRYPT,
+	[NVDEV_ENGINE_BSP]	= NV_DEVICE_DISABLE_BSP,
+	[NVDEV_ENGINE_PPP]	= NV_DEVICE_DISABLE_PPP,
+	[NVDEV_ENGINE_COPY0]	= NV_DEVICE_DISABLE_COPY0,
+	[NVDEV_ENGINE_COPY1]	= NV_DEVICE_DISABLE_COPY1,
+	[NVDEV_ENGINE_UNK1C1]	= NV_DEVICE_DISABLE_UNK1C1,
+	[NVDEV_ENGINE_FIFO]	= NV_DEVICE_DISABLE_FIFO,
+	[NVDEV_ENGINE_DISP]	= NV_DEVICE_DISABLE_DISP,
+	[NVDEV_SUBDEV_NR]	= 0,
+};
+
+static int
+nouveau_devobj_ctor(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, void *data, u32 size,
+		    struct nouveau_object **pobject)
+{
+	struct nouveau_client *client = nv_client(parent);
+	struct nouveau_device *device;
+	struct nouveau_devobj *devobj;
+	struct nv_device_class *args = data;
+	u64 disable, boot0, strap;
+	u64 mmio_base, mmio_size;
+	void __iomem *map;
+	int ret, i, c;
+
+	if (size < sizeof(struct nv_device_class))
+		return -EINVAL;
+
+	/* find the device subdev that matches what the client requested */
+	device = nv_device(client->device);
+	if (args->device != ~0) {
+		device = nouveau_device_find(args->device);
+		if (!device)
+			return -ENODEV;
+	}
+
+	ret = nouveau_parent_create(parent, nv_object(device), oclass, 0, NULL,
+				    (1ULL << NVDEV_ENGINE_DMAOBJ) |
+				    (1ULL << NVDEV_ENGINE_FIFO) |
+				    (1ULL << NVDEV_ENGINE_DISP), &devobj);
+	*pobject = nv_object(devobj);
+	if (ret)
+		return ret;
+
+	mmio_base = pci_resource_start(device->pdev, 0);
+	mmio_size = pci_resource_len(device->pdev, 0);
+
+	/* translate api disable mask into internal mapping */
+	disable = args->debug0;
+	for (i = 0; i < NVDEV_SUBDEV_NR; i++) {
+		if (args->disable & disable_map[i])
+			disable |= (1ULL << i);
+	}
+
+	/* identify the chipset, and determine classes of subdev/engines */
+	if (!(args->disable & NV_DEVICE_DISABLE_IDENTIFY) &&
+	    !device->card_type) {
+		map = ioremap(mmio_base, 0x102000);
+		if (map == NULL)
+			return -ENOMEM;
+
+		/* switch mmio to cpu's native endianness */
+#ifndef __BIG_ENDIAN
+		if (ioread32_native(map + 0x000004) != 0x00000000)
+#else
+		if (ioread32_native(map + 0x000004) == 0x00000000)
+#endif
+			iowrite32_native(0x01000001, map + 0x000004);
+
+		/* read boot0 and strapping information */
+		boot0 = ioread32_native(map + 0x000000);
+		strap = ioread32_native(map + 0x101000);
+		iounmap(map);
+
+		/* determine chipset and derive architecture from it */
+		if ((boot0 & 0x0f000000) > 0) {
+			device->chipset = (boot0 & 0xff00000) >> 20;
+			switch (device->chipset & 0xf0) {
+			case 0x10: device->card_type = NV_10; break;
+			case 0x20: device->card_type = NV_20; break;
+			case 0x30: device->card_type = NV_30; break;
+			case 0x40:
+			case 0x60: device->card_type = NV_40; break;
+			case 0x50:
+			case 0x80:
+			case 0x90:
+			case 0xa0: device->card_type = NV_50; break;
+			case 0xc0: device->card_type = NV_C0; break;
+			case 0xd0: device->card_type = NV_D0; break;
+			case 0xe0: device->card_type = NV_E0; break;
+			default:
+				break;
+			}
+		} else
+		if ((boot0 & 0xff00fff0) == 0x20004000) {
+			if (boot0 & 0x00f00000)
+				device->chipset = 0x05;
+			else
+				device->chipset = 0x04;
+			device->card_type = NV_04;
+		}
+
+		switch (device->card_type) {
+		case NV_04: ret = nv04_identify(device); break;
+		case NV_10: ret = nv10_identify(device); break;
+		case NV_20: ret = nv20_identify(device); break;
+		case NV_30: ret = nv30_identify(device); break;
+		case NV_40: ret = nv40_identify(device); break;
+		case NV_50: ret = nv50_identify(device); break;
+		case NV_C0:
+		case NV_D0: ret = nvc0_identify(device); break;
+		case NV_E0: ret = nve0_identify(device); break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+
+		if (ret) {
+			nv_error(device, "unknown chipset, 0x%08x\n", boot0);
+			return ret;
+		}
+
+		nv_info(device, "BOOT0  : 0x%08x\n", boot0);
+		nv_info(device, "Chipset: %s (NV%02X)\n",
+			device->cname, device->chipset);
+		nv_info(device, "Family : NV%02X\n", device->card_type);
+
+		/* determine frequency of timing crystal */
+		if ( device->chipset < 0x17 ||
+		    (device->chipset >= 0x20 && device->chipset <= 0x25))
+			strap &= 0x00000040;
+		else
+			strap &= 0x00400040;
+
+		switch (strap) {
+		case 0x00000000: device->crystal = 13500; break;
+		case 0x00000040: device->crystal = 14318; break;
+		case 0x00400000: device->crystal = 27000; break;
+		case 0x00400040: device->crystal = 25000; break;
+		}
+
+		nv_debug(device, "crystal freq: %dKHz\n", device->crystal);
+	}
+
+	if (!(args->disable & NV_DEVICE_DISABLE_MMIO) &&
+	    !nv_subdev(device)->mmio) {
+		nv_subdev(device)->mmio  = ioremap(mmio_base, mmio_size);
+		if (!nv_subdev(device)->mmio) {
+			nv_error(device, "unable to map device registers\n");
+			return -ENOMEM;
+		}
+	}
+
+	/* ensure requested subsystems are available for use */
+	for (i = 0, c = 0; i < NVDEV_SUBDEV_NR; i++) {
+		if (!(oclass = device->oclass[i]) || (disable & (1ULL << i)))
+			continue;
+
+		if (!device->subdev[i]) {
+			ret = nouveau_object_ctor(nv_object(device), NULL,
+						  oclass, NULL, i,
+						  &devobj->subdev[i]);
+			if (ret == -ENODEV)
+				continue;
+			if (ret)
+				return ret;
+
+			if (nv_iclass(devobj->subdev[i], NV_ENGINE_CLASS))
+				nouveau_subdev_reset(devobj->subdev[i]);
+		} else {
+			nouveau_object_ref(device->subdev[i],
+					  &devobj->subdev[i]);
+		}
+
+		/* note: can't init *any* subdevs until devinit has been run
+		 * due to not knowing exactly what the vbios init tables will
+		 * mess with.  devinit also can't be run until all of its
+		 * dependencies have been created.
+		 *
+		 * this code delays init of any subdev until all of devinit's
+		 * dependencies have been created, and then initialises each
+		 * subdev in turn as they're created.
+		 */
+		while (i >= NVDEV_SUBDEV_DEVINIT_LAST && c <= i) {
+			struct nouveau_object *subdev = devobj->subdev[c++];
+			if (subdev && !nv_iclass(subdev, NV_ENGINE_CLASS)) {
+				ret = nouveau_object_inc(subdev);
+				if (ret)
+					return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static void
+nouveau_devobj_dtor(struct nouveau_object *object)
+{
+	struct nouveau_devobj *devobj = (void *)object;
+	int i;
+
+	for (i = NVDEV_SUBDEV_NR - 1; i >= 0; i--)
+		nouveau_object_ref(NULL, &devobj->subdev[i]);
+
+	nouveau_parent_destroy(&devobj->base);
+}
+
+static int
+nouveau_devobj_init(struct nouveau_object *object)
+{
+	struct nouveau_devobj *devobj = (void *)object;
+	struct nouveau_object *subdev;
+	int ret, i;
+
+	ret = nouveau_parent_init(&devobj->base);
+	if (ret)
+		return ret;
+
+	for (i = 0; devobj->created && i < NVDEV_SUBDEV_NR; i++) {
+		if ((subdev = devobj->subdev[i])) {
+			if (!nv_iclass(subdev, NV_ENGINE_CLASS)) {
+				ret = nouveau_object_inc(subdev);
+				if (ret)
+					goto fail;
+			}
+		}
+	}
+
+	devobj->created = true;
+	return 0;
+
+fail:
+	for (--i; i >= 0; i--) {
+		if ((subdev = devobj->subdev[i])) {
+			if (!nv_iclass(subdev, NV_ENGINE_CLASS))
+				nouveau_object_dec(subdev, false);
+		}
+	}
+
+	return ret;
+}
+
+static int
+nouveau_devobj_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nouveau_devobj *devobj = (void *)object;
+	struct nouveau_object *subdev;
+	int ret, i;
+
+	for (i = NVDEV_SUBDEV_NR - 1; i >= 0; i--) {
+		if ((subdev = devobj->subdev[i])) {
+			if (!nv_iclass(subdev, NV_ENGINE_CLASS)) {
+				ret = nouveau_object_dec(subdev, suspend);
+				if (ret && suspend)
+					goto fail;
+			}
+		}
+	}
+
+	ret = nouveau_parent_fini(&devobj->base, suspend);
+fail:
+	for (; ret && suspend && i < NVDEV_SUBDEV_NR; i++) {
+		if ((subdev = devobj->subdev[i])) {
+			if (!nv_iclass(subdev, NV_ENGINE_CLASS)) {
+				ret = nouveau_object_inc(subdev);
+				if (ret) {
+					/* XXX */
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+static u8
+nouveau_devobj_rd08(struct nouveau_object *object, u32 addr)
+{
+	return nv_rd08(object->engine, addr);
+}
+
+static u16
+nouveau_devobj_rd16(struct nouveau_object *object, u32 addr)
+{
+	return nv_rd16(object->engine, addr);
+}
+
+static u32
+nouveau_devobj_rd32(struct nouveau_object *object, u32 addr)
+{
+	return nv_rd32(object->engine, addr);
+}
+
+static void
+nouveau_devobj_wr08(struct nouveau_object *object, u32 addr, u8 data)
+{
+	nv_wr08(object->engine, addr, data);
+}
+
+static void
+nouveau_devobj_wr16(struct nouveau_object *object, u32 addr, u16 data)
+{
+	nv_wr16(object->engine, addr, data);
+}
+
+static void
+nouveau_devobj_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	nv_wr32(object->engine, addr, data);
+}
+
+static struct nouveau_ofuncs
+nouveau_devobj_ofuncs = {
+	.ctor = nouveau_devobj_ctor,
+	.dtor = nouveau_devobj_dtor,
+	.init = nouveau_devobj_init,
+	.fini = nouveau_devobj_fini,
+	.rd08 = nouveau_devobj_rd08,
+	.rd16 = nouveau_devobj_rd16,
+	.rd32 = nouveau_devobj_rd32,
+	.wr08 = nouveau_devobj_wr08,
+	.wr16 = nouveau_devobj_wr16,
+	.wr32 = nouveau_devobj_wr32,
+};
+
+/******************************************************************************
+ * nouveau_device: engine functions
+ *****************************************************************************/
+struct nouveau_oclass
+nouveau_device_sclass[] = {
+	{ 0x0080, &nouveau_devobj_ofuncs },
+	{}
+};
+
+static void
+nouveau_device_dtor(struct nouveau_object *object)
+{
+	struct nouveau_device *device = (void *)object;
+
+	mutex_lock(&nv_devices_mutex);
+	list_del(&device->head);
+	mutex_unlock(&nv_devices_mutex);
+
+	if (device->base.mmio)
+		iounmap(device->base.mmio);
+
+	nouveau_subdev_destroy(&device->base);
+}
+
+static struct nouveau_oclass
+nouveau_device_oclass = {
+	.handle = NV_SUBDEV(DEVICE, 0x00),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.dtor = nouveau_device_dtor,
+	},
+};
+
+int
+nouveau_device_create_(struct pci_dev *pdev, u64 name, const char *sname,
+		       const char *cfg, const char *dbg,
+		       int length, void **pobject)
+{
+	struct nouveau_device *device;
+	int ret = -EEXIST;
+
+	mutex_lock(&nv_devices_mutex);
+	list_for_each_entry(device, &nv_devices, head) {
+		if (device->handle == name)
+			goto done;
+	}
+
+	ret = nouveau_subdev_create_(NULL, NULL, &nouveau_device_oclass, 0,
+				     "DEVICE", "device", length, pobject);
+	device = *pobject;
+	if (ret)
+		goto done;
+
+	atomic_set(&nv_object(device)->usecount, 2);
+	device->pdev = pdev;
+	device->handle = name;
+	device->cfgopt = cfg;
+	device->dbgopt = dbg;
+	device->name = sname;
+
+	nv_subdev(device)->debug = nouveau_dbgopt(device->dbgopt, "DEVICE");
+	list_add(&device->head, &nv_devices);
+done:
+	mutex_unlock(&nv_devices_mutex);
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv04.c
new file mode 100644
index 00000000000..8626d0d6cbb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv04.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/disp.h>
+
+int
+nv04_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x04:
+		device->cname = "NV04";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv04_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv04_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv04_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv04_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x05:
+		device->cname = "NV05";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv05_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv04_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv04_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv04_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown RIVA chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv10.c
new file mode 100644
index 00000000000..f09accfd0e3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv10.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/disp.h>
+
+int
+nv10_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x10:
+		device->cname = "NV10";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x15:
+		device->cname = "NV15";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x16:
+		device->cname = "NV16";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x1a:
+		device->cname = "nForce";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x11:
+		device->cname = "NV11";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x17:
+		device->cname = "NV17";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x1f:
+		device->cname = "nForce2";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x18:
+		device->cname = "NV18";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Celsius chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv20.c
new file mode 100644
index 00000000000..5fa58b7369b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv20.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/disp.h>
+
+int
+nv20_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x20:
+		device->cname = "NV20";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv20_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv20_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x25:
+		device->cname = "NV25";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv20_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x28:
+		device->cname = "NV28";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv20_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x2a:
+		device->cname = "NV2A";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv20_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv2a_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Kelvin chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv30.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv30.c
new file mode 100644
index 00000000000..7f4b8fe6ccc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv30.c
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/mpeg.h>
+#include <engine/disp.h>
+
+int
+nv30_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x30:
+		device->cname = "NV30";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x35:
+		device->cname = "NV35";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x31:
+		device->cname = "NV31";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x36:
+		device->cname = "NV36";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x34:
+		device->cname = "NV34";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv34_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Rankine chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv40.c
new file mode 100644
index 00000000000..42deadca0f0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv40.c
@@ -0,0 +1,375 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/mpeg.h>
+#include <engine/disp.h>
+
+int
+nv40_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x40:
+		device->cname = "NV40";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x41:
+		device->cname = "NV41";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x42:
+		device->cname = "NV42";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x43:
+		device->cname = "NV43";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x45:
+		device->cname = "NV45";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x47:
+		device->cname = "G70";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x49:
+		device->cname = "G71";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x4b:
+		device->cname = "G73";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x44:
+		device->cname = "NV44";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x46:
+		device->cname = "G72";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x4a:
+		device->cname = "NV44A";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x4c:
+		device->cname = "C61";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x4e:
+		device->cname = "C51";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x63:
+		device->cname = "C73";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x67:
+		device->cname = "C67";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	case 0x68:
+		device->cname = "C68";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv10_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Curie chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/device/nv50.c
new file mode 100644
index 00000000000..fec3bcc9a6f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nv50.c
@@ -0,0 +1,410 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/mxm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/mpeg.h>
+#include <engine/vp.h>
+#include <engine/crypt.h>
+#include <engine/bsp.h>
+#include <engine/ppp.h>
+#include <engine/copy.h>
+#include <engine/disp.h>
+
+int
+nv50_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0x50:
+		device->cname = "G80";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv50_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv50_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x84:
+		device->cname = "G84";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x86:
+		device->cname = "G86";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x92:
+		device->cname = "G92";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x94:
+		device->cname = "G94";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x96:
+		device->cname = "G96";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0x98:
+		device->cname = "G98";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xa0:
+		device->cname = "G200";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xaa:
+		device->cname = "MCP77/MCP78";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xac:
+		device->cname = "MCP79/MCP7A";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xa3:
+		device->cname = "GT215";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nva3_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xa5:
+		device->cname = "GT216";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nva3_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xa8:
+		device->cname = "GT218";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nva3_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xaf:
+		device->cname = "MCP89";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nva3_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Tesla chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/device/nvc0.c
new file mode 100644
index 00000000000..6697f0f9c29
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nvc0.c
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/mxm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/ltcg.h>
+#include <subdev/ibus.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/vp.h>
+#include <engine/bsp.h>
+#include <engine/ppp.h>
+#include <engine/copy.h>
+#include <engine/disp.h>
+
+int
+nvc0_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0xc0:
+		device->cname = "GF100";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xc4:
+		device->cname = "GF104";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xc3:
+		device->cname = "GF106";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xce:
+		device->cname = "GF114";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xcf:
+		device->cname = "GF116";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xc1:
+		device->cname = "GF108";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xc8:
+		device->cname = "GF110";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+		break;
+	case 0xd9:
+		device->cname = "GF119";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nvd0_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nvc0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
+		device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
+		device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Fermi chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/device/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/device/nve0.c
new file mode 100644
index 00000000000..4a280b7ab85
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/device/nve0.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/device.h>
+#include <subdev/bios.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/mxm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/ltcg.h>
+#include <subdev/ibus.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/disp.h>
+#include <engine/copy.h>
+
+int
+nve0_identify(struct nouveau_device *device)
+{
+	switch (device->chipset) {
+	case 0xe4:
+		device->cname = "GK104";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nvd0_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nve0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
+		break;
+	case 0xe7:
+		device->cname = "GK107";
+		device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+		device->oclass[NVDEV_SUBDEV_GPIO   ] = &nvd0_gpio_oclass;
+		device->oclass[NVDEV_SUBDEV_I2C    ] = &nouveau_i2c_oclass;
+		device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+		device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
+		device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+		device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
+		device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
+		device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+		device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+		device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+		device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+		device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+		device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+		device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+		device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
+		device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
+		device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+		device->oclass[NVDEV_ENGINE_GR     ] = &nve0_graph_oclass;
+		device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+		device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
+		device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
+		break;
+	default:
+		nv_fatal(device, "unknown Kepler chipset\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nv98_ppp.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
index a987dd6e003..5a07a39c173 100644
--- a/drivers/gpu/drm/nouveau/nv98_ppp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2011 Red Hat Inc.
+ * Copyright 2012 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -22,57 +22,48 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
+#include <core/option.h>
 
-struct nv98_ppp_engine {
-	struct nouveau_exec_engine base;
-};
+#include <subdev/devinit.h>
+#include <subdev/bios.h>
+#include <subdev/bios/init.h>
 
-static int
-nv98_ppp_fini(struct drm_device *dev, int engine, bool suspend)
+int
+nouveau_devinit_init(struct nouveau_devinit *devinit)
 {
-	if (!(nv_rd32(dev, 0x000200) & 0x00000002))
-		return 0;
+	int ret = nouveau_subdev_init(&devinit->base);
+	if (ret)
+		return ret;
 
-	nv_mask(dev, 0x000200, 0x00000002, 0x00000000);
-	return 0;
+	return nvbios_init(&devinit->base, devinit->post);
 }
 
-static int
-nv98_ppp_init(struct drm_device *dev, int engine)
-{
-	nv_mask(dev, 0x000200, 0x00000002, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000002, 0x00000002);
-	return 0;
-}
-
-static void
-nv98_ppp_destroy(struct drm_device *dev, int engine)
+int
+nouveau_devinit_fini(struct nouveau_devinit *devinit, bool suspend)
 {
-	struct nv98_ppp_engine *pppp = nv_engine(dev, engine);
+	/* force full reinit on resume */
+	if (suspend)
+		devinit->post = true;
 
-	NVOBJ_ENGINE_DEL(dev, PPP);
-
-	kfree(pppp);
+	return nouveau_subdev_fini(&devinit->base, suspend);
 }
 
 int
-nv98_ppp_create(struct drm_device *dev)
+nouveau_devinit_create_(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass,
+			int size, void **pobject)
 {
-	struct nv98_ppp_engine *pppp;
-
-	pppp = kzalloc(sizeof(*pppp), GFP_KERNEL);
-	if (!pppp)
-		return -ENOMEM;
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_devinit *devinit;
+	int ret;
 
-	pppp->base.destroy = nv98_ppp_destroy;
-	pppp->base.init = nv98_ppp_init;
-	pppp->base.fini = nv98_ppp_fini;
+	ret = nouveau_subdev_create_(parent, engine, oclass, 0, "DEVINIT",
+				     "init", size, pobject);
+	devinit = *pobject;
+	if (ret)
+		return ret;
 
-	NVOBJ_ENGINE_ADD(dev, PPP, &pppp->base);
+	devinit->post = nouveau_boolopt(device->cfgopt, "NvForcePost", false);
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h
new file mode 100644
index 00000000000..6b56a0f4cb4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#define NV04_PFB_BOOT_0						0x00100000
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT			0x00000003
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_32MB			0x00000000
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_4MB			0x00000001
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_8MB			0x00000002
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_16MB			0x00000003
+#	define NV04_PFB_BOOT_0_RAM_WIDTH_128			0x00000004
+#	define NV04_PFB_BOOT_0_RAM_TYPE				0x00000028
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_8MBIT		0x00000000
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT		0x00000008
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT_4BANK	0x00000010
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_16MBIT		0x00000018
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_64MBIT		0x00000020
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_64MBITX16		0x00000028
+#	define NV04_PFB_BOOT_0_UMA_ENABLE			0x00000100
+#	define NV04_PFB_BOOT_0_UMA_SIZE				0x0000f000
+#define NV04_PFB_DEBUG_0					0x00100080
+#	define NV04_PFB_DEBUG_0_PAGE_MODE			0x00000001
+#	define NV04_PFB_DEBUG_0_REFRESH_OFF			0x00000010
+#	define NV04_PFB_DEBUG_0_REFRESH_COUNTX64		0x00003f00
+#	define NV04_PFB_DEBUG_0_REFRESH_SLOW_CLK		0x00004000
+#	define NV04_PFB_DEBUG_0_SAFE_MODE			0x00008000
+#	define NV04_PFB_DEBUG_0_ALOM_ENABLE			0x00010000
+#	define NV04_PFB_DEBUG_0_CASOE				0x00100000
+#	define NV04_PFB_DEBUG_0_CKE_INVERT			0x10000000
+#	define NV04_PFB_DEBUG_0_REFINC				0x20000000
+#	define NV04_PFB_DEBUG_0_SAVE_POWER_OFF			0x40000000
+#define NV04_PFB_CFG0						0x00100200
+#	define NV04_PFB_CFG0_SCRAMBLE				0x20000000
+#define NV04_PFB_CFG1						0x00100204
+#define NV04_PFB_SCRAMBLE(i)                         (0x00100400 + 4 * (i))
+
+#define NV10_PFB_REFCTRL					0x00100210
+#	define NV10_PFB_REFCTRL_VALID_1				(1 << 31)
+
+static inline struct io_mapping *
+fbmem_init(struct pci_dev *pdev)
+{
+	return io_mapping_create_wc(pci_resource_start(pdev, 1),
+				    pci_resource_len(pdev, 1));
+}
+
+static inline void
+fbmem_fini(struct io_mapping *fb)
+{
+	io_mapping_free(fb);
+}
+
+static inline u32
+fbmem_peek(struct io_mapping *fb, u32 off)
+{
+	u8 __iomem *p = io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
+	u32 val = ioread32(p + (off & ~PAGE_MASK));
+	io_mapping_unmap_atomic(p);
+	return val;
+}
+
+static inline void
+fbmem_poke(struct io_mapping *fb, u32 off, u32 val)
+{
+	u8 __iomem *p = io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
+	iowrite32(val, p + (off & ~PAGE_MASK));
+	wmb();
+	io_mapping_unmap_atomic(p);
+}
+
+static inline bool
+fbmem_readback(struct io_mapping *fb, u32 off, u32 val)
+{
+	fbmem_poke(fb, off, val);
+	return val == fbmem_peek(fb, off);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
new file mode 100644
index 00000000000..7a72d939434
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/devinit.h>
+#include <subdev/vga.h>
+
+#include "fbmem.h"
+
+struct nv04_devinit_priv {
+	struct nouveau_devinit base;
+	int owner;
+};
+
+static void
+nv04_devinit_meminit(struct nouveau_devinit *devinit)
+{
+	struct nv04_devinit_priv *priv = (void *)devinit;
+	u32 patt = 0xdeadbeef;
+	struct io_mapping *fb;
+	int i;
+
+	/* Map the framebuffer aperture */
+	fb = fbmem_init(nv_device(priv)->pdev);
+	if (!fb) {
+		nv_error(priv, "failed to map fb\n");
+		return;
+	}
+
+	/* Sequencer and refresh off */
+	nv_wrvgas(priv, 0, 1, nv_rdvgas(priv, 0, 1) | 0x20);
+	nv_mask(priv, NV04_PFB_DEBUG_0, 0, NV04_PFB_DEBUG_0_REFRESH_OFF);
+
+	nv_mask(priv, NV04_PFB_BOOT_0, ~0,
+		      NV04_PFB_BOOT_0_RAM_AMOUNT_16MB |
+		      NV04_PFB_BOOT_0_RAM_WIDTH_128 |
+		      NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT);
+
+	for (i = 0; i < 4; i++)
+		fbmem_poke(fb, 4 * i, patt);
+
+	fbmem_poke(fb, 0x400000, patt + 1);
+
+	if (fbmem_peek(fb, 0) == patt + 1) {
+		nv_mask(priv, NV04_PFB_BOOT_0,
+			      NV04_PFB_BOOT_0_RAM_TYPE,
+			      NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_16MBIT);
+		nv_mask(priv, NV04_PFB_DEBUG_0,
+			      NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
+
+		for (i = 0; i < 4; i++)
+			fbmem_poke(fb, 4 * i, patt);
+
+		if ((fbmem_peek(fb, 0xc) & 0xffff) != (patt & 0xffff))
+			nv_mask(priv, NV04_PFB_BOOT_0,
+				      NV04_PFB_BOOT_0_RAM_WIDTH_128 |
+				      NV04_PFB_BOOT_0_RAM_AMOUNT,
+				      NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
+	} else
+	if ((fbmem_peek(fb, 0xc) & 0xffff0000) != (patt & 0xffff0000)) {
+		nv_mask(priv, NV04_PFB_BOOT_0,
+			      NV04_PFB_BOOT_0_RAM_WIDTH_128 |
+			      NV04_PFB_BOOT_0_RAM_AMOUNT,
+			      NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
+	} else
+	if (fbmem_peek(fb, 0) != patt) {
+		if (fbmem_readback(fb, 0x800000, patt))
+			nv_mask(priv, NV04_PFB_BOOT_0,
+				      NV04_PFB_BOOT_0_RAM_AMOUNT,
+				      NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
+		else
+			nv_mask(priv, NV04_PFB_BOOT_0,
+				      NV04_PFB_BOOT_0_RAM_AMOUNT,
+				      NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
+
+		nv_mask(priv, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_TYPE,
+			      NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_8MBIT);
+	} else
+	if (!fbmem_readback(fb, 0x800000, patt)) {
+		nv_mask(priv, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
+			      NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
+
+	}
+
+	/* Refresh on, sequencer on */
+	nv_mask(priv, NV04_PFB_DEBUG_0, NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
+	nv_wrvgas(priv, 0, 1, nv_rdvgas(priv, 0, 1) & ~0x20);
+	fbmem_fini(fb);
+}
+
+static int
+nv04_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv04_devinit_priv *priv;
+	int ret;
+
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.meminit = nv04_devinit_meminit;
+	priv->owner = -1;
+	return 0;
+}
+
+void
+nv04_devinit_dtor(struct nouveau_object *object)
+{
+	struct nv04_devinit_priv *priv = (void *)object;
+
+	/* restore vga owner saved at first init, and lock crtc regs  */
+	nv_wrvgaowner(priv, priv->owner);
+	nv_lockvgac(priv, true);
+
+	nouveau_devinit_destroy(&priv->base);
+}
+
+int
+nv04_devinit_init(struct nouveau_object *object)
+{
+	struct nv04_devinit_priv *priv = (void *)object;
+
+	if (!priv->base.post) {
+		u32 htotal = nv_rdvgac(priv, 0, 0x06);
+		htotal |= (nv_rdvgac(priv, 0, 0x07) & 0x01) << 8;
+		htotal |= (nv_rdvgac(priv, 0, 0x07) & 0x20) << 4;
+		htotal |= (nv_rdvgac(priv, 0, 0x25) & 0x01) << 10;
+		htotal |= (nv_rdvgac(priv, 0, 0x41) & 0x01) << 11;
+		if (!htotal) {
+			nv_info(priv, "adaptor not initialised\n");
+			priv->base.post = true;
+		}
+	}
+
+	return nouveau_devinit_init(&priv->base);
+}
+
+int
+nv04_devinit_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv04_devinit_priv *priv = (void *)object;
+
+	/* make i2c busses accessible */
+	nv_mask(priv, 0x000200, 0x00000001, 0x00000001);
+
+	/* unlock extended vga crtc regs, and unslave crtcs */
+	nv_lockvgac(priv, false);
+	if (priv->owner < 0)
+		priv->owner = nv_rdvgaowner(priv);
+	nv_wrvgaowner(priv, 0);
+
+	return nouveau_devinit_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv04_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_devinit_ctor,
+		.dtor = nv04_devinit_dtor,
+		.init = nv04_devinit_init,
+		.fini = nv04_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
new file mode 100644
index 00000000000..191447d0d25
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/devinit.h>
+#include <subdev/bios.h>
+#include <subdev/bios/bmp.h>
+#include <subdev/vga.h>
+
+#include "fbmem.h"
+
+struct nv05_devinit_priv {
+	struct nouveau_devinit base;
+	u8 owner;
+};
+
+static void
+nv05_devinit_meminit(struct nouveau_devinit *devinit)
+{
+	static const u8 default_config_tab[][2] = {
+		{ 0x24, 0x00 },
+		{ 0x28, 0x00 },
+		{ 0x24, 0x01 },
+		{ 0x1f, 0x00 },
+		{ 0x0f, 0x00 },
+		{ 0x17, 0x00 },
+		{ 0x06, 0x00 },
+		{ 0x00, 0x00 }
+	};
+	struct nv05_devinit_priv *priv = (void *)devinit;
+	struct nouveau_bios *bios = nouveau_bios(priv);
+	struct io_mapping *fb;
+	u32 patt = 0xdeadbeef;
+	u16 data;
+	u8 strap, ramcfg[2];
+	int i, v;
+
+	/* Map the framebuffer aperture */
+	fb = fbmem_init(nv_device(priv)->pdev);
+	if (!fb) {
+		nv_error(priv, "failed to map fb\n");
+		return;
+	}
+
+	strap = (nv_rd32(priv, 0x101000) & 0x0000003c) >> 2;
+	if ((data = bmp_mem_init_table(bios))) {
+		ramcfg[0] = nv_ro08(bios, data + 2 * strap + 0);
+		ramcfg[1] = nv_ro08(bios, data + 2 * strap + 1);
+	} else {
+		ramcfg[0] = default_config_tab[strap][0];
+		ramcfg[1] = default_config_tab[strap][1];
+	}
+
+	/* Sequencer off */
+	nv_wrvgas(priv, 0, 1, nv_rdvgas(priv, 0, 1) | 0x20);
+
+	if (nv_rd32(priv, NV04_PFB_BOOT_0) & NV04_PFB_BOOT_0_UMA_ENABLE)
+		goto out;
+
+	nv_mask(priv, NV04_PFB_DEBUG_0, NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
+
+	/* If present load the hardcoded scrambling table */
+	if (data) {
+		for (i = 0, data += 0x10; i < 8; i++, data += 4) {
+			u32 scramble = nv_ro32(bios, data);
+			nv_wr32(priv, NV04_PFB_SCRAMBLE(i), scramble);
+		}
+	}
+
+	/* Set memory type/width/length defaults depending on the straps */
+	nv_mask(priv, NV04_PFB_BOOT_0, 0x3f, ramcfg[0]);
+
+	if (ramcfg[1] & 0x80)
+		nv_mask(priv, NV04_PFB_CFG0, 0, NV04_PFB_CFG0_SCRAMBLE);
+
+	nv_mask(priv, NV04_PFB_CFG1, 0x700001, (ramcfg[1] & 1) << 20);
+	nv_mask(priv, NV04_PFB_CFG1, 0, 1);
+
+	/* Probe memory bus width */
+	for (i = 0; i < 4; i++)
+		fbmem_poke(fb, 4 * i, patt);
+
+	if (fbmem_peek(fb, 0xc) != patt)
+		nv_mask(priv, NV04_PFB_BOOT_0,
+			  NV04_PFB_BOOT_0_RAM_WIDTH_128, 0);
+
+	/* Probe memory length */
+	v = nv_rd32(priv, NV04_PFB_BOOT_0) & NV04_PFB_BOOT_0_RAM_AMOUNT;
+
+	if (v == NV04_PFB_BOOT_0_RAM_AMOUNT_32MB &&
+	    (!fbmem_readback(fb, 0x1000000, ++patt) ||
+	     !fbmem_readback(fb, 0, ++patt)))
+		nv_mask(priv, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
+			  NV04_PFB_BOOT_0_RAM_AMOUNT_16MB);
+
+	if (v == NV04_PFB_BOOT_0_RAM_AMOUNT_16MB &&
+	    !fbmem_readback(fb, 0x800000, ++patt))
+		nv_mask(priv, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
+			  NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
+
+	if (!fbmem_readback(fb, 0x400000, ++patt))
+		nv_mask(priv, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
+			  NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
+
+out:
+	/* Sequencer on */
+	nv_wrvgas(priv, 0, 1, nv_rdvgas(priv, 0, 1) & ~0x20);
+	fbmem_fini(fb);
+}
+
+static int
+nv05_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv05_devinit_priv *priv;
+	int ret;
+
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.meminit = nv05_devinit_meminit;
+	return 0;
+}
+
+struct nouveau_oclass
+nv05_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x05),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv05_devinit_ctor,
+		.dtor = nv04_devinit_dtor,
+		.init = nv04_devinit_init,
+		.fini = nv04_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
new file mode 100644
index 00000000000..eb76ffab6b0
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/devinit.h>
+#include <subdev/vga.h>
+
+#include "fbmem.h"
+
+struct nv10_devinit_priv {
+	struct nouveau_devinit base;
+	u8 owner;
+};
+
+static void
+nv10_devinit_meminit(struct nouveau_devinit *devinit)
+{
+	struct nv10_devinit_priv *priv = (void *)devinit;
+	const int mem_width[] = { 0x10, 0x00, 0x20 };
+	const int mem_width_count = nv_device(priv)->chipset >= 0x17 ? 3 : 2;
+	uint32_t patt = 0xdeadbeef;
+	struct io_mapping *fb;
+	int i, j, k;
+
+	/* Map the framebuffer aperture */
+	fb = fbmem_init(nv_device(priv)->pdev);
+	if (!fb) {
+		nv_error(priv, "failed to map fb\n");
+		return;
+	}
+
+	nv_wr32(priv, NV10_PFB_REFCTRL, NV10_PFB_REFCTRL_VALID_1);
+
+	/* Probe memory bus width */
+	for (i = 0; i < mem_width_count; i++) {
+		nv_mask(priv, NV04_PFB_CFG0, 0x30, mem_width[i]);
+
+		for (j = 0; j < 4; j++) {
+			for (k = 0; k < 4; k++)
+				fbmem_poke(fb, 0x1c, 0);
+
+			fbmem_poke(fb, 0x1c, patt);
+			fbmem_poke(fb, 0x3c, 0);
+
+			if (fbmem_peek(fb, 0x1c) == patt)
+				goto mem_width_found;
+		}
+	}
+
+mem_width_found:
+	patt <<= 1;
+
+	/* Probe amount of installed memory */
+	for (i = 0; i < 4; i++) {
+		int off = nv_rd32(priv, 0x10020c) - 0x100000;
+
+		fbmem_poke(fb, off, patt);
+		fbmem_poke(fb, 0, 0);
+
+		fbmem_peek(fb, 0);
+		fbmem_peek(fb, 0);
+		fbmem_peek(fb, 0);
+		fbmem_peek(fb, 0);
+
+		if (fbmem_peek(fb, off) == patt)
+			goto amount_found;
+	}
+
+	/* IC missing - disable the upper half memory space. */
+	nv_mask(priv, NV04_PFB_CFG0, 0x1000, 0);
+
+amount_found:
+	fbmem_fini(fb);
+}
+
+static int
+nv10_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv10_devinit_priv *priv;
+	int ret;
+
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.meminit = nv10_devinit_meminit;
+	return 0;
+}
+
+struct nouveau_oclass
+nv10_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_devinit_ctor,
+		.dtor = nv04_devinit_dtor,
+		.init = nv04_devinit_init,
+		.fini = nv04_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_i2c.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
index 1d083893a4d..5b2ba630d91 100644
--- a/drivers/gpu/drm/nouveau/nouveau_i2c.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv1a.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2009 Red Hat Inc.
+ * Copyright 2012 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -18,42 +18,41 @@
  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
  */
 
-#ifndef __NOUVEAU_I2C_H__
-#define __NOUVEAU_I2C_H__
-
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include "drm_dp_helper.h"
-
-#define NV_I2C_PORT(n)    (0x00 + (n))
-#define NV_I2C_PORT_NUM    0x10
-#define NV_I2C_DEFAULT(n) (0x80 + (n))
+#include <subdev/devinit.h>
+#include <subdev/vga.h>
 
-struct nouveau_i2c_chan {
-	struct i2c_adapter adapter;
-	struct drm_device *dev;
-	struct i2c_algo_bit_data bit;
-	struct list_head head;
-	u8  index;
-	u8  type;
-	u32 dcb;
-	u32 drive;
-	u32 sense;
-	u32 state;
+struct nv1a_devinit_priv {
+	struct nouveau_devinit base;
+	u8 owner;
 };
 
-int  nouveau_i2c_init(struct drm_device *);
-void nouveau_i2c_fini(struct drm_device *);
-struct nouveau_i2c_chan *nouveau_i2c_find(struct drm_device *, u8 index);
-bool nouveau_probe_i2c_addr(struct nouveau_i2c_chan *i2c, int addr);
-int nouveau_i2c_identify(struct drm_device *dev, const char *what,
-			 struct i2c_board_info *info,
-			 bool (*match)(struct nouveau_i2c_chan *,
-				       struct i2c_board_info *),
-			 int index);
+static int
+nv1a_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv1a_devinit_priv *priv;
+	int ret;
 
-extern const struct i2c_algorithm nouveau_dp_i2c_algo;
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
 
-#endif /* __NOUVEAU_I2C_H__ */
+	return 0;
+}
+
+struct nouveau_oclass
+nv1a_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x1a),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv1a_devinit_ctor,
+		.dtor = nv04_devinit_dtor,
+		.init = nv04_devinit_init,
+		.fini = nv04_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
new file mode 100644
index 00000000000..eb32e99005e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/devinit.h>
+#include <subdev/vga.h>
+
+#include "fbmem.h"
+
+struct nv20_devinit_priv {
+	struct nouveau_devinit base;
+	u8 owner;
+};
+
+static void
+nv20_devinit_meminit(struct nouveau_devinit *devinit)
+{
+	struct nv20_devinit_priv *priv = (void *)devinit;
+	struct nouveau_device *device = nv_device(priv);
+	uint32_t mask = (device->chipset >= 0x25 ? 0x300 : 0x900);
+	uint32_t amount, off;
+	struct io_mapping *fb;
+
+	/* Map the framebuffer aperture */
+	fb = fbmem_init(nv_device(priv)->pdev);
+	if (!fb) {
+		nv_error(priv, "failed to map fb\n");
+		return;
+	}
+
+	nv_wr32(priv, NV10_PFB_REFCTRL, NV10_PFB_REFCTRL_VALID_1);
+
+	/* Allow full addressing */
+	nv_mask(priv, NV04_PFB_CFG0, 0, mask);
+
+	amount = nv_rd32(priv, 0x10020c);
+	for (off = amount; off > 0x2000000; off -= 0x2000000)
+		fbmem_poke(fb, off - 4, off);
+
+	amount = nv_rd32(priv, 0x10020c);
+	if (amount != fbmem_peek(fb, amount - 4))
+		/* IC missing - disable the upper half memory space. */
+		nv_mask(priv, NV04_PFB_CFG0, mask, 0);
+
+	fbmem_fini(fb);
+}
+
+static int
+nv20_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv20_devinit_priv *priv;
+	int ret;
+
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.meminit = nv20_devinit_meminit;
+	return 0;
+}
+
+struct nouveau_oclass
+nv20_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x20),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv20_devinit_ctor,
+		.dtor = nv04_devinit_dtor,
+		.init = nv04_devinit_init,
+		.fini = nv04_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
new file mode 100644
index 00000000000..61becfa732e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/devinit.h>
+#include <subdev/vga.h>
+
+struct nv50_devinit_priv {
+	struct nouveau_devinit base;
+};
+
+static int
+nv50_devinit_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv50_devinit_priv *priv;
+	int ret;
+
+	ret = nouveau_devinit_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void
+nv50_devinit_dtor(struct nouveau_object *object)
+{
+	struct nv50_devinit_priv *priv = (void *)object;
+	nouveau_devinit_destroy(&priv->base);
+}
+
+static int
+nv50_devinit_init(struct nouveau_object *object)
+{
+	struct nv50_devinit_priv *priv = (void *)object;
+
+	if (!priv->base.post) {
+		if (!nv_rdvgac(priv, 0, 0x00) &&
+		    !nv_rdvgac(priv, 0, 0x1a)) {
+			nv_info(priv, "adaptor not initialised\n");
+			priv->base.post = true;
+		}
+	}
+
+	return nouveau_devinit_init(&priv->base);
+}
+
+static int
+nv50_devinit_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv50_devinit_priv *priv = (void *)object;
+	return nouveau_devinit_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv50_devinit_oclass = {
+	.handle = NV_SUBDEV(DEVINIT, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_devinit_ctor,
+		.dtor = nv50_devinit_dtor,
+		.init = nv50_devinit_init,
+		.fini = nv50_devinit_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
new file mode 100644
index 00000000000..f0086de8af3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "subdev/fb.h"
+#include "subdev/bios.h"
+#include "subdev/bios/bit.h"
+
+int
+nouveau_fb_bios_memtype(struct nouveau_bios *bios)
+{
+	struct bit_entry M;
+	u8 ramcfg;
+
+	ramcfg = (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
+	if (!bit_entry(bios, 'M', &M) && M.version == 2 && M.length >= 5) {
+		u16 table   = nv_ro16(bios, M.offset + 3);
+		u8  version = nv_ro08(bios, table + 0);
+		u8  header  = nv_ro08(bios, table + 1);
+		u8  record  = nv_ro08(bios, table + 2);
+		u8  entries = nv_ro08(bios, table + 3);
+		if (table && version == 0x10 && ramcfg < entries) {
+			u16 entry = table + header + (ramcfg * record);
+			switch (nv_ro08(bios, entry) & 0x0f) {
+			case 0: return NV_MEM_TYPE_DDR2;
+			case 1: return NV_MEM_TYPE_DDR3;
+			case 2: return NV_MEM_TYPE_GDDR3;
+			case 3: return NV_MEM_TYPE_GDDR5;
+			default:
+				break;
+			}
+
+		}
+	}
+
+	return NV_MEM_TYPE_UNKNOWN;
+}
+
+int
+nouveau_fb_init(struct nouveau_fb *pfb)
+{
+	int ret, i;
+
+	ret = nouveau_subdev_init(&pfb->base);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < pfb->tile.regions; i++)
+		pfb->tile.prog(pfb, i, &pfb->tile.region[i]);
+
+	return 0;
+}
+
+int
+_nouveau_fb_init(struct nouveau_object *object)
+{
+	struct nouveau_fb *pfb = (void *)object;
+	return nouveau_fb_init(pfb);
+}
+
+void
+nouveau_fb_destroy(struct nouveau_fb *pfb)
+{
+	int i;
+
+	for (i = 0; i < pfb->tile.regions; i++)
+		pfb->tile.fini(pfb, i, &pfb->tile.region[i]);
+
+	if (pfb->tags.block_size)
+		nouveau_mm_fini(&pfb->tags);
+
+	if (pfb->vram.block_size)
+		nouveau_mm_fini(&pfb->vram);
+
+	nouveau_subdev_destroy(&pfb->base);
+}
+
+void
+_nouveau_fb_dtor(struct nouveau_object *object)
+{
+	struct nouveau_fb *pfb = (void *)object;
+	nouveau_fb_destroy(pfb);
+}
+
+int
+nouveau_fb_created(struct nouveau_fb *pfb)
+{
+	static const char *name[] = {
+		[NV_MEM_TYPE_UNKNOWN] = "unknown",
+		[NV_MEM_TYPE_STOLEN ] = "stolen system memory",
+		[NV_MEM_TYPE_SGRAM  ] = "SGRAM",
+		[NV_MEM_TYPE_SDRAM  ] = "SDRAM",
+		[NV_MEM_TYPE_DDR1   ] = "DDR1",
+		[NV_MEM_TYPE_DDR2   ] = "DDR2",
+		[NV_MEM_TYPE_DDR3   ] = "DDR3",
+		[NV_MEM_TYPE_GDDR2  ] = "GDDR2",
+		[NV_MEM_TYPE_GDDR3  ] = "GDDR3",
+		[NV_MEM_TYPE_GDDR4  ] = "GDDR4",
+		[NV_MEM_TYPE_GDDR5  ] = "GDDR5",
+	};
+
+	if (pfb->ram.size == 0) {
+		nv_fatal(pfb, "no vram detected!!\n");
+		return -ERANGE;
+	}
+
+	nv_info(pfb, "RAM type: %s\n", name[pfb->ram.type]);
+	nv_info(pfb, "RAM size: %d MiB\n", (int)(pfb->ram.size >> 20));
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c
new file mode 100644
index 00000000000..eb06836b69f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/fb.h>
+
+#define NV04_PFB_BOOT_0						0x00100000
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT			0x00000003
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_32MB			0x00000000
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_4MB			0x00000001
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_8MB			0x00000002
+#	define NV04_PFB_BOOT_0_RAM_AMOUNT_16MB			0x00000003
+#	define NV04_PFB_BOOT_0_RAM_WIDTH_128			0x00000004
+#	define NV04_PFB_BOOT_0_RAM_TYPE				0x00000028
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_8MBIT		0x00000000
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT		0x00000008
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT_4BANK	0x00000010
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_16MBIT		0x00000018
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_64MBIT		0x00000020
+#	define NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_64MBITX16		0x00000028
+#	define NV04_PFB_BOOT_0_UMA_ENABLE			0x00000100
+#	define NV04_PFB_BOOT_0_UMA_SIZE				0x0000f000
+#define NV04_PFB_CFG0						0x00100200
+
+struct nv04_fb_priv {
+	struct nouveau_fb base;
+};
+
+bool
+nv04_fb_memtype_valid(struct nouveau_fb *pfb, u32 tile_flags)
+{
+	if (!(tile_flags & 0xff00))
+		return true;
+
+	return false;
+}
+
+static int
+nv04_fb_init(struct nouveau_object *object)
+{
+	struct nv04_fb_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fb_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* This is what the DDX did for NV_ARCH_04, but a mmio-trace shows
+	 * nvidia reading PFB_CFG_0, then writing back its original value.
+	 * (which was 0x701114 in this case)
+	 */
+	nv_wr32(priv, NV04_PFB_CFG0, 0x1114);
+	return 0;
+}
+
+static int
+nv04_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv04_fb_priv *priv;
+	u32 boot0;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	boot0 = nv_rd32(priv, NV04_PFB_BOOT_0);
+	if (boot0 & 0x00000100) {
+		priv->base.ram.size  = ((boot0 >> 12) & 0xf) * 2 + 2;
+		priv->base.ram.size *= 1024 * 1024;
+	} else {
+		switch (boot0 & NV04_PFB_BOOT_0_RAM_AMOUNT) {
+		case NV04_PFB_BOOT_0_RAM_AMOUNT_32MB:
+			priv->base.ram.size = 32 * 1024 * 1024;
+			break;
+		case NV04_PFB_BOOT_0_RAM_AMOUNT_16MB:
+			priv->base.ram.size = 16 * 1024 * 1024;
+			break;
+		case NV04_PFB_BOOT_0_RAM_AMOUNT_8MB:
+			priv->base.ram.size = 8 * 1024 * 1024;
+			break;
+		case NV04_PFB_BOOT_0_RAM_AMOUNT_4MB:
+			priv->base.ram.size = 4 * 1024 * 1024;
+			break;
+		}
+	}
+
+	if ((boot0 & 0x00000038) <= 0x10)
+		priv->base.ram.type = NV_MEM_TYPE_SGRAM;
+	else
+		priv->base.ram.type = NV_MEM_TYPE_SDRAM;
+
+
+	priv->base.memtype_valid = nv04_fb_memtype_valid;
+	return nouveau_fb_created(&priv->base);
+}
+
+struct nouveau_oclass
+nv04_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_fb_ctor,
+		.dtor = _nouveau_fb_dtor,
+		.init = nv04_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c
new file mode 100644
index 00000000000..f037a422d2f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/fb.h>
+
+struct nv10_fb_priv {
+	struct nouveau_fb base;
+};
+
+static void
+nv10_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
+		  u32 flags, struct nouveau_fb_tile *tile)
+{
+	tile->addr  = 0x80000000 | addr;
+	tile->limit = max(1u, addr + size) - 1;
+	tile->pitch = pitch;
+}
+
+static void
+nv10_fb_tile_fini(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	tile->addr  = 0;
+	tile->limit = 0;
+	tile->pitch = 0;
+	tile->zcomp = 0;
+}
+
+void
+nv10_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	nv_wr32(pfb, 0x100244 + (i * 0x10), tile->limit);
+	nv_wr32(pfb, 0x100248 + (i * 0x10), tile->pitch);
+	nv_wr32(pfb, 0x100240 + (i * 0x10), tile->addr);
+}
+
+static int
+nv10_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nv10_fb_priv *priv;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	if (device->chipset == 0x1a ||  device->chipset == 0x1f) {
+		struct pci_dev *bridge;
+		u32 mem, mib;
+
+		bridge = pci_get_bus_and_slot(0, PCI_DEVFN(0, 1));
+		if (!bridge) {
+			nv_fatal(device, "no bridge device\n");
+			return 0;
+		}
+
+		if (device->chipset == 0x1a) {
+			pci_read_config_dword(bridge, 0x7c, &mem);
+			mib = ((mem >> 6) & 31) + 1;
+		} else {
+			pci_read_config_dword(bridge, 0x84, &mem);
+			mib = ((mem >> 4) & 127) + 1;
+		}
+
+		priv->base.ram.type = NV_MEM_TYPE_STOLEN;
+		priv->base.ram.size = mib * 1024 * 1024;
+	} else {
+		u32 cfg0 = nv_rd32(priv, 0x100200);
+		if (cfg0 & 0x00000001)
+			priv->base.ram.type = NV_MEM_TYPE_DDR1;
+		else
+			priv->base.ram.type = NV_MEM_TYPE_SDRAM;
+
+		priv->base.ram.size = nv_rd32(priv, 0x10020c) & 0xff000000;
+	}
+
+	priv->base.memtype_valid = nv04_fb_memtype_valid;
+	priv->base.tile.regions = 8;
+	priv->base.tile.init = nv10_fb_tile_init;
+	priv->base.tile.fini = nv10_fb_tile_fini;
+	priv->base.tile.prog = nv10_fb_tile_prog;
+	return nouveau_fb_created(&priv->base);
+}
+
+struct nouveau_oclass
+nv10_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_fb_ctor,
+		.dtor = _nouveau_fb_dtor,
+		.init = _nouveau_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c
new file mode 100644
index 00000000000..4b3578fcb7f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/fb.h>
+
+struct nv20_fb_priv {
+	struct nouveau_fb base;
+};
+
+static void
+nv20_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
+		  u32 flags, struct nouveau_fb_tile *tile)
+{
+	struct nouveau_device *device = nv_device(pfb);
+	int bpp = (flags & 2) ? 32 : 16;
+
+	tile->addr  = 0x00000001 | addr;
+	tile->limit = max(1u, addr + size) - 1;
+	tile->pitch = pitch;
+
+	/* Allocate some of the on-die tag memory, used to store Z
+	 * compression meta-data (most likely just a bitmap determining
+	 * if a given tile is compressed or not).
+	 */
+	size /= 256;
+	if (flags & 4) {
+		if (!nouveau_mm_head(&pfb->tags, 1, size, size, 1, &tile->tag)) {
+			/* Enable Z compression */
+			tile->zcomp = tile->tag->offset;
+			if (device->chipset >= 0x25) {
+				if (bpp == 16)
+					tile->zcomp |= 0x00100000;
+				else
+					tile->zcomp |= 0x00200000;
+			} else {
+				tile->zcomp |= 0x80000000;
+				if (bpp != 16)
+					tile->zcomp |= 0x04000000;
+			}
+		}
+
+		tile->addr |= 2;
+	}
+}
+
+static void
+nv20_fb_tile_fini(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	tile->addr  = 0;
+	tile->limit = 0;
+	tile->pitch = 0;
+	tile->zcomp = 0;
+	nouveau_mm_free(&pfb->tags, &tile->tag);
+}
+
+static void
+nv20_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	nv_wr32(pfb, 0x100244 + (i * 0x10), tile->limit);
+	nv_wr32(pfb, 0x100248 + (i * 0x10), tile->pitch);
+	nv_wr32(pfb, 0x100240 + (i * 0x10), tile->addr);
+	nv_wr32(pfb, 0x100300 + (i * 0x04), tile->zcomp);
+}
+
+static int
+nv20_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nv20_fb_priv *priv;
+	u32 pbus1218;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	pbus1218 = nv_rd32(priv, 0x001218);
+	switch (pbus1218 & 0x00000300) {
+	case 0x00000000: priv->base.ram.type = NV_MEM_TYPE_SDRAM; break;
+	case 0x00000100: priv->base.ram.type = NV_MEM_TYPE_DDR1; break;
+	case 0x00000200: priv->base.ram.type = NV_MEM_TYPE_GDDR3; break;
+	case 0x00000300: priv->base.ram.type = NV_MEM_TYPE_GDDR2; break;
+	}
+	priv->base.ram.size = nv_rd32(priv, 0x10020c) & 0xff000000;
+
+	if (device->chipset >= 0x25)
+		ret = nouveau_mm_init(&priv->base.tags, 0, 64 * 1024, 1);
+	else
+		ret = nouveau_mm_init(&priv->base.tags, 0, 32 * 1024, 1);
+	if (ret)
+		return ret;
+
+	priv->base.memtype_valid = nv04_fb_memtype_valid;
+	priv->base.tile.regions = 8;
+	priv->base.tile.init = nv20_fb_tile_init;
+	priv->base.tile.fini = nv20_fb_tile_fini;
+	priv->base.tile.prog = nv20_fb_tile_prog;
+	return nouveau_fb_created(&priv->base);
+}
+
+struct nouveau_oclass
+nv20_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x20),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv20_fb_ctor,
+		.dtor = _nouveau_fb_dtor,
+		.init = _nouveau_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c
new file mode 100644
index 00000000000..cba67bc9139
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/fb.h>
+
+struct nv30_fb_priv {
+	struct nouveau_fb base;
+};
+
+void
+nv30_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
+		  u32 flags, struct nouveau_fb_tile *tile)
+{
+	tile->addr = addr | 1;
+	tile->limit = max(1u, addr + size) - 1;
+	tile->pitch = pitch;
+}
+
+void
+nv30_fb_tile_fini(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	tile->addr  = 0;
+	tile->limit = 0;
+	tile->pitch = 0;
+}
+
+static int
+calc_bias(struct nv30_fb_priv *priv, int k, int i, int j)
+{
+	struct nouveau_device *device = nv_device(priv);
+	int b = (device->chipset > 0x30 ?
+		 nv_rd32(priv, 0x122c + 0x10 * k + 0x4 * j) >> (4 * (i ^ 1)) :
+		 0) & 0xf;
+
+	return 2 * (b & 0x8 ? b - 0x10 : b);
+}
+
+static int
+calc_ref(struct nv30_fb_priv *priv, int l, int k, int i)
+{
+	int j, x = 0;
+
+	for (j = 0; j < 4; j++) {
+		int m = (l >> (8 * i) & 0xff) + calc_bias(priv, k, i, j);
+
+		x |= (0x80 | clamp(m, 0, 0x1f)) << (8 * j);
+	}
+
+	return x;
+}
+
+static int
+nv30_fb_init(struct nouveau_object *object)
+{
+	struct nouveau_device *device = nv_device(object);
+	struct nv30_fb_priv *priv = (void *)object;
+	int ret, i, j;
+
+	ret = nouveau_fb_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* Init the memory timing regs at 0x10037c/0x1003ac */
+	if (device->chipset == 0x30 ||
+	    device->chipset == 0x31 ||
+	    device->chipset == 0x35) {
+		/* Related to ROP count */
+		int n = (device->chipset == 0x31 ? 2 : 4);
+		int l = nv_rd32(priv, 0x1003d0);
+
+		for (i = 0; i < n; i++) {
+			for (j = 0; j < 3; j++)
+				nv_wr32(priv, 0x10037c + 0xc * i + 0x4 * j,
+					calc_ref(priv, l, 0, j));
+
+			for (j = 0; j < 2; j++)
+				nv_wr32(priv, 0x1003ac + 0x8 * i + 0x4 * j,
+					calc_ref(priv, l, 1, j));
+		}
+	}
+
+	return 0;
+}
+
+static int
+nv30_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv30_fb_priv *priv;
+	u32 pbus1218;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	pbus1218 = nv_rd32(priv, 0x001218);
+	switch (pbus1218 & 0x00000300) {
+	case 0x00000000: priv->base.ram.type = NV_MEM_TYPE_SDRAM; break;
+	case 0x00000100: priv->base.ram.type = NV_MEM_TYPE_DDR1; break;
+	case 0x00000200: priv->base.ram.type = NV_MEM_TYPE_GDDR3; break;
+	case 0x00000300: priv->base.ram.type = NV_MEM_TYPE_GDDR2; break;
+	}
+	priv->base.ram.size = nv_rd32(priv, 0x10020c) & 0xff000000;
+
+	priv->base.memtype_valid = nv04_fb_memtype_valid;
+	priv->base.tile.regions = 8;
+	priv->base.tile.init = nv30_fb_tile_init;
+	priv->base.tile.fini = nv30_fb_tile_fini;
+	priv->base.tile.prog = nv10_fb_tile_prog;
+	return nouveau_fb_created(&priv->base);
+}
+
+struct nouveau_oclass
+nv30_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x30),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv30_fb_ctor,
+		.dtor = _nouveau_fb_dtor,
+		.init = nv30_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c
new file mode 100644
index 00000000000..347a496fcad
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) 2010 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/fb.h>
+
+struct nv40_fb_priv {
+	struct nouveau_fb base;
+};
+
+static inline int
+nv44_graph_class(struct nouveau_device *device)
+{
+	if ((device->chipset & 0xf0) == 0x60)
+		return 1;
+
+	return !(0x0baf & (1 << (device->chipset & 0x0f)));
+}
+
+static void
+nv40_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
+{
+	nv_wr32(pfb, 0x100604 + (i * 0x10), tile->limit);
+	nv_wr32(pfb, 0x100608 + (i * 0x10), tile->pitch);
+	nv_wr32(pfb, 0x100600 + (i * 0x10), tile->addr);
+}
+
+static void
+nv40_fb_init_gart(struct nv40_fb_priv *priv)
+{
+	nv_wr32(priv, 0x100800, 0x00000001);
+}
+
+static void
+nv44_fb_init_gart(struct nv40_fb_priv *priv)
+{
+	nv_wr32(priv, 0x100850, 0x80000000);
+	nv_wr32(priv, 0x100800, 0x00000001);
+}
+
+static int
+nv40_fb_init(struct nouveau_object *object)
+{
+	struct nv40_fb_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fb_init(&priv->base);
+	if (ret)
+		return ret;
+
+	switch (nv_device(priv)->chipset) {
+	case 0x40:
+	case 0x45:
+		nv_mask(priv, 0x10033c, 0x00008000, 0x00000000);
+		break;
+	default:
+		if (nv44_graph_class(nv_device(priv)))
+			nv44_fb_init_gart(priv);
+		else
+			nv40_fb_init_gart(priv);
+		break;
+	}
+
+	return 0;
+}
+
+static int
+nv40_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nv40_fb_priv *priv;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	/* 0x001218 is actually present on a few other NV4X I looked at,
+	 * and even contains sane values matching 0x100474.  From looking
+	 * at various vbios images however, this isn't the case everywhere.
+	 * So, I chose to use the same regs I've seen NVIDIA reading around
+	 * the memory detection, hopefully that'll get us the right numbers
+	 */
+	if (device->chipset == 0x40) {
+		u32 pbus1218 = nv_rd32(priv, 0x001218);
+		switch (pbus1218 & 0x00000300) {
+		case 0x00000000: priv->base.ram.type = NV_MEM_TYPE_SDRAM; break;
+		case 0x00000100: priv->base.ram.type = NV_MEM_TYPE_DDR1; break;
+		case 0x00000200: priv->base.ram.type = NV_MEM_TYPE_GDDR3; break;
+		case 0x00000300: priv->base.ram.type = NV_MEM_TYPE_DDR2; break;
+		}
+	} else
+	if (device->chipset == 0x49 || device->chipset == 0x4b) {
+		u32 pfb914 = nv_rd32(priv, 0x100914);
+		switch (pfb914 & 0x00000003) {
+		case 0x00000000: priv->base.ram.type = NV_MEM_TYPE_DDR1; break;
+		case 0x00000001: priv->base.ram.type = NV_MEM_TYPE_DDR2; break;
+		case 0x00000002: priv->base.ram.type = NV_MEM_TYPE_GDDR3; break;
+		case 0x00000003: break;
+		}
+	} else
+	if (device->chipset != 0x4e) {
+		u32 pfb474 = nv_rd32(priv, 0x100474);
+		if (pfb474 & 0x00000004)
+			priv->base.ram.type = NV_MEM_TYPE_GDDR3;
+		if (pfb474 & 0x00000002)
+			priv->base.ram.type = NV_MEM_TYPE_DDR2;
+		if (pfb474 & 0x00000001)
+			priv->base.ram.type = NV_MEM_TYPE_DDR1;
+	} else {
+		priv->base.ram.type = NV_MEM_TYPE_STOLEN;
+	}
+
+	priv->base.ram.size = nv_rd32(priv, 0x10020c) & 0xff000000;
+
+	priv->base.memtype_valid = nv04_fb_memtype_valid;
+	switch (device->chipset) {
+	case 0x40:
+	case 0x45:
+		priv->base.tile.regions = 8;
+		break;
+	case 0x46:
+	case 0x47:
+	case 0x49:
+	case 0x4b:
+	case 0x4c:
+		priv->base.tile.regions = 15;
+		break;
+	default:
+		priv->base.tile.regions = 12;
+		break;
+	}
+	priv->base.tile.init = nv30_fb_tile_init;
+	priv->base.tile.fini = nv30_fb_tile_fini;
+	if (device->chipset == 0x40)
+		priv->base.tile.prog = nv10_fb_tile_prog;
+	else
+		priv->base.tile.prog = nv40_fb_tile_prog;
+
+	return nouveau_fb_created(&priv->base);
+}
+
+
+struct nouveau_oclass
+nv40_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_fb_ctor,
+		.dtor = _nouveau_fb_dtor,
+		.init = nv40_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
new file mode 100644
index 00000000000..436e9efe7ef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
@@ -0,0 +1,498 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/enum.h>
+
+#include <subdev/fb.h>
+#include <subdev/bios.h>
+
+struct nv50_fb_priv {
+	struct nouveau_fb base;
+	struct page *r100c08_page;
+	dma_addr_t r100c08;
+};
+
+static int types[0x80] = {
+	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0,
+	1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0,
+	0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2,
+	1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
+};
+
+static bool
+nv50_fb_memtype_valid(struct nouveau_fb *pfb, u32 memtype)
+{
+	return types[(memtype & 0xff00) >> 8] != 0;
+}
+
+static int
+nv50_fb_vram_new(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
+		 u32 memtype, struct nouveau_mem **pmem)
+{
+	struct nv50_fb_priv *priv = (void *)pfb;
+	struct nouveau_mm *heap = &priv->base.vram;
+	struct nouveau_mm *tags = &priv->base.tags;
+	struct nouveau_mm_node *r;
+	struct nouveau_mem *mem;
+	int comp = (memtype & 0x300) >> 8;
+	int type = (memtype & 0x07f);
+	int back = (memtype & 0x800);
+	int min, max, ret;
+
+	max = (size >> 12);
+	min = ncmin ? (ncmin >> 12) : max;
+	align >>= 12;
+
+	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	mutex_lock(&pfb->base.mutex);
+	if (comp) {
+		if (align == 16) {
+			int n = (max >> 4) * comp;
+
+			ret = nouveau_mm_head(tags, 1, n, n, 1, &mem->tag);
+			if (ret)
+				mem->tag = NULL;
+		}
+
+		if (unlikely(!mem->tag))
+			comp = 0;
+	}
+
+	INIT_LIST_HEAD(&mem->regions);
+	mem->memtype = (comp << 7) | type;
+	mem->size = max;
+
+	type = types[type];
+	do {
+		if (back)
+			ret = nouveau_mm_tail(heap, type, max, min, align, &r);
+		else
+			ret = nouveau_mm_head(heap, type, max, min, align, &r);
+		if (ret) {
+			mutex_unlock(&pfb->base.mutex);
+			pfb->ram.put(pfb, &mem);
+			return ret;
+		}
+
+		list_add_tail(&r->rl_entry, &mem->regions);
+		max -= r->length;
+	} while (max);
+	mutex_unlock(&pfb->base.mutex);
+
+	r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
+	mem->offset = (u64)r->offset << 12;
+	*pmem = mem;
+	return 0;
+}
+
+void
+nv50_fb_vram_del(struct nouveau_fb *pfb, struct nouveau_mem **pmem)
+{
+	struct nv50_fb_priv *priv = (void *)pfb;
+	struct nouveau_mm_node *this;
+	struct nouveau_mem *mem;
+
+	mem = *pmem;
+	*pmem = NULL;
+	if (unlikely(mem == NULL))
+		return;
+
+	mutex_lock(&pfb->base.mutex);
+	while (!list_empty(&mem->regions)) {
+		this = list_first_entry(&mem->regions, typeof(*this), rl_entry);
+
+		list_del(&this->rl_entry);
+		nouveau_mm_free(&priv->base.vram, &this);
+	}
+
+	nouveau_mm_free(&priv->base.tags, &mem->tag);
+	mutex_unlock(&pfb->base.mutex);
+
+	kfree(mem);
+}
+
+static u32
+nv50_vram_rblock(struct nv50_fb_priv *priv)
+{
+	int i, parts, colbits, rowbitsa, rowbitsb, banks;
+	u64 rowsize, predicted;
+	u32 r0, r4, rt, ru, rblock_size;
+
+	r0 = nv_rd32(priv, 0x100200);
+	r4 = nv_rd32(priv, 0x100204);
+	rt = nv_rd32(priv, 0x100250);
+	ru = nv_rd32(priv, 0x001540);
+	nv_debug(priv, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n", r0, r4, rt, ru);
+
+	for (i = 0, parts = 0; i < 8; i++) {
+		if (ru & (0x00010000 << i))
+			parts++;
+	}
+
+	colbits  =  (r4 & 0x0000f000) >> 12;
+	rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
+	rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
+	banks    = 1 << (((r4 & 0x03000000) >> 24) + 2);
+
+	rowsize = parts * banks * (1 << colbits) * 8;
+	predicted = rowsize << rowbitsa;
+	if (r0 & 0x00000004)
+		predicted += rowsize << rowbitsb;
+
+	if (predicted != priv->base.ram.size) {
+		nv_warn(priv, "memory controller reports %d MiB VRAM\n",
+			(u32)(priv->base.ram.size >> 20));
+	}
+
+	rblock_size = rowsize;
+	if (rt & 1)
+		rblock_size *= 3;
+
+	nv_debug(priv, "rblock %d bytes\n", rblock_size);
+	return rblock_size;
+}
+
+static int
+nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+	const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+	struct nv50_fb_priv *priv;
+	u32 tags;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	switch (nv_rd32(priv, 0x100714) & 0x00000007) {
+	case 0: priv->base.ram.type = NV_MEM_TYPE_DDR1; break;
+	case 1:
+		if (nouveau_fb_bios_memtype(bios) == NV_MEM_TYPE_DDR3)
+			priv->base.ram.type = NV_MEM_TYPE_DDR3;
+		else
+			priv->base.ram.type = NV_MEM_TYPE_DDR2;
+		break;
+	case 2: priv->base.ram.type = NV_MEM_TYPE_GDDR3; break;
+	case 3: priv->base.ram.type = NV_MEM_TYPE_GDDR4; break;
+	case 4: priv->base.ram.type = NV_MEM_TYPE_GDDR5; break;
+	default:
+		break;
+	}
+
+	priv->base.ram.size = nv_rd32(priv, 0x10020c);
+	priv->base.ram.size = (priv->base.ram.size & 0xffffff00) |
+			     ((priv->base.ram.size & 0x000000ff) << 32);
+
+	tags = nv_rd32(priv, 0x100320);
+	if (tags) {
+		ret = nouveau_mm_init(&priv->base.tags, 0, tags, 1);
+		if (ret)
+			return ret;
+
+		nv_debug(priv, "%d compression tags\n", tags);
+	}
+
+	size = (priv->base.ram.size >> 12) - rsvd_head - rsvd_tail;
+	switch (device->chipset) {
+	case 0xaa:
+	case 0xac:
+	case 0xaf: /* IGPs, no reordering, no real VRAM */
+		ret = nouveau_mm_init(&priv->base.vram, rsvd_head, size, 1);
+		if (ret)
+			return ret;
+
+		priv->base.ram.stolen = (u64)nv_rd32(priv, 0x100e10) << 12;
+		break;
+	default:
+		ret = nouveau_mm_init(&priv->base.vram, rsvd_head, size,
+				      nv50_vram_rblock(priv) >> 12);
+		if (ret)
+			return ret;
+
+		priv->base.ram.ranks = (nv_rd32(priv, 0x100200) & 0x4) ? 2 : 1;
+		break;
+	}
+
+	priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (priv->r100c08_page) {
+		priv->r100c08 = pci_map_page(device->pdev, priv->r100c08_page,
+					     0, PAGE_SIZE,
+					     PCI_DMA_BIDIRECTIONAL);
+		if (pci_dma_mapping_error(device->pdev, priv->r100c08))
+			nv_warn(priv, "failed 0x100c08 page map\n");
+	} else {
+		nv_warn(priv, "failed 0x100c08 page alloc\n");
+	}
+
+	priv->base.memtype_valid = nv50_fb_memtype_valid;
+	priv->base.ram.get = nv50_fb_vram_new;
+	priv->base.ram.put = nv50_fb_vram_del;
+	return nouveau_fb_created(&priv->base);
+}
+
+static void
+nv50_fb_dtor(struct nouveau_object *object)
+{
+	struct nouveau_device *device = nv_device(object);
+	struct nv50_fb_priv *priv = (void *)object;
+
+	if (priv->r100c08_page) {
+		pci_unmap_page(device->pdev, priv->r100c08, PAGE_SIZE,
+			       PCI_DMA_BIDIRECTIONAL);
+		__free_page(priv->r100c08_page);
+	}
+
+	nouveau_mm_fini(&priv->base.vram);
+	nouveau_fb_destroy(&priv->base);
+}
+
+static int
+nv50_fb_init(struct nouveau_object *object)
+{
+	struct nouveau_device *device = nv_device(object);
+	struct nv50_fb_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fb_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* Not a clue what this is exactly.  Without pointing it at a
+	 * scratch page, VRAM->GART blits with M2MF (as in DDX DFS)
+	 * cause IOMMU "read from address 0" errors (rh#561267)
+	 */
+	nv_wr32(priv, 0x100c08, priv->r100c08 >> 8);
+
+	/* This is needed to get meaningful information from 100c90
+	 * on traps. No idea what these values mean exactly. */
+	switch (device->chipset) {
+	case 0x50:
+		nv_wr32(priv, 0x100c90, 0x000707ff);
+		break;
+	case 0xa3:
+	case 0xa5:
+	case 0xa8:
+		nv_wr32(priv, 0x100c90, 0x000d0fff);
+		break;
+	case 0xaf:
+		nv_wr32(priv, 0x100c90, 0x089d1fff);
+		break;
+	default:
+		nv_wr32(priv, 0x100c90, 0x001d07ff);
+		break;
+	}
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_fb_ctor,
+		.dtor = nv50_fb_dtor,
+		.init = nv50_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
+
+static const struct nouveau_enum vm_dispatch_subclients[] = {
+	{ 0x00000000, "GRCTX", NULL },
+	{ 0x00000001, "NOTIFY", NULL },
+	{ 0x00000002, "QUERY", NULL },
+	{ 0x00000003, "COND", NULL },
+	{ 0x00000004, "M2M_IN", NULL },
+	{ 0x00000005, "M2M_OUT", NULL },
+	{ 0x00000006, "M2M_NOTIFY", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_ccache_subclients[] = {
+	{ 0x00000000, "CB", NULL },
+	{ 0x00000001, "TIC", NULL },
+	{ 0x00000002, "TSC", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_prop_subclients[] = {
+	{ 0x00000000, "RT0", NULL },
+	{ 0x00000001, "RT1", NULL },
+	{ 0x00000002, "RT2", NULL },
+	{ 0x00000003, "RT3", NULL },
+	{ 0x00000004, "RT4", NULL },
+	{ 0x00000005, "RT5", NULL },
+	{ 0x00000006, "RT6", NULL },
+	{ 0x00000007, "RT7", NULL },
+	{ 0x00000008, "ZETA", NULL },
+	{ 0x00000009, "LOCAL", NULL },
+	{ 0x0000000a, "GLOBAL", NULL },
+	{ 0x0000000b, "STACK", NULL },
+	{ 0x0000000c, "DST2D", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_pfifo_subclients[] = {
+	{ 0x00000000, "PUSHBUF", NULL },
+	{ 0x00000001, "SEMAPHORE", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_bar_subclients[] = {
+	{ 0x00000000, "FB", NULL },
+	{ 0x00000001, "IN", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_client[] = {
+	{ 0x00000000, "STRMOUT", NULL },
+	{ 0x00000003, "DISPATCH", vm_dispatch_subclients },
+	{ 0x00000004, "PFIFO_WRITE", NULL },
+	{ 0x00000005, "CCACHE", vm_ccache_subclients },
+	{ 0x00000006, "PPPP", NULL },
+	{ 0x00000007, "CLIPID", NULL },
+	{ 0x00000008, "PFIFO_READ", NULL },
+	{ 0x00000009, "VFETCH", NULL },
+	{ 0x0000000a, "TEXTURE", NULL },
+	{ 0x0000000b, "PROP", vm_prop_subclients },
+	{ 0x0000000c, "PVP", NULL },
+	{ 0x0000000d, "PBSP", NULL },
+	{ 0x0000000e, "PCRYPT", NULL },
+	{ 0x0000000f, "PCOUNTER", NULL },
+	{ 0x00000011, "PDAEMON", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_engine[] = {
+	{ 0x00000000, "PGRAPH", NULL },
+	{ 0x00000001, "PVP", NULL },
+	{ 0x00000004, "PEEPHOLE", NULL },
+	{ 0x00000005, "PFIFO", vm_pfifo_subclients },
+	{ 0x00000006, "BAR", vm_bar_subclients },
+	{ 0x00000008, "PPPP", NULL },
+	{ 0x00000009, "PBSP", NULL },
+	{ 0x0000000a, "PCRYPT", NULL },
+	{ 0x0000000b, "PCOUNTER", NULL },
+	{ 0x0000000c, "SEMAPHORE_BG", NULL },
+	{ 0x0000000d, "PCOPY", NULL },
+	{ 0x0000000e, "PDAEMON", NULL },
+	{}
+};
+
+static const struct nouveau_enum vm_fault[] = {
+	{ 0x00000000, "PT_NOT_PRESENT", NULL },
+	{ 0x00000001, "PT_TOO_SHORT", NULL },
+	{ 0x00000002, "PAGE_NOT_PRESENT", NULL },
+	{ 0x00000003, "PAGE_SYSTEM_ONLY", NULL },
+	{ 0x00000004, "PAGE_READ_ONLY", NULL },
+	{ 0x00000006, "NULL_DMAOBJ", NULL },
+	{ 0x00000007, "WRONG_MEMTYPE", NULL },
+	{ 0x0000000b, "VRAM_LIMIT", NULL },
+	{ 0x0000000f, "DMAOBJ_LIMIT", NULL },
+	{}
+};
+
+void
+nv50_fb_trap(struct nouveau_fb *pfb, int display)
+{
+	struct nouveau_device *device = nv_device(pfb);
+	struct nv50_fb_priv *priv = (void *)pfb;
+	const struct nouveau_enum *en, *cl;
+	u32 trap[6], idx, chan;
+	u8 st0, st1, st2, st3;
+	int i;
+
+	idx = nv_rd32(priv, 0x100c90);
+	if (!(idx & 0x80000000))
+		return;
+	idx &= 0x00ffffff;
+
+	for (i = 0; i < 6; i++) {
+		nv_wr32(priv, 0x100c90, idx | i << 24);
+		trap[i] = nv_rd32(priv, 0x100c94);
+	}
+	nv_wr32(priv, 0x100c90, idx | 0x80000000);
+
+	if (!display)
+		return;
+
+	/* decode status bits into something more useful */
+	if (device->chipset  < 0xa3 ||
+	    device->chipset == 0xaa || device->chipset == 0xac) {
+		st0 = (trap[0] & 0x0000000f) >> 0;
+		st1 = (trap[0] & 0x000000f0) >> 4;
+		st2 = (trap[0] & 0x00000f00) >> 8;
+		st3 = (trap[0] & 0x0000f000) >> 12;
+	} else {
+		st0 = (trap[0] & 0x000000ff) >> 0;
+		st1 = (trap[0] & 0x0000ff00) >> 8;
+		st2 = (trap[0] & 0x00ff0000) >> 16;
+		st3 = (trap[0] & 0xff000000) >> 24;
+	}
+	chan = (trap[2] << 16) | trap[1];
+
+	nv_error(priv, "trapped %s at 0x%02x%04x%04x on channel 0x%08x ",
+		 (trap[5] & 0x00000100) ? "read" : "write",
+		 trap[5] & 0xff, trap[4] & 0xffff, trap[3] & 0xffff, chan);
+
+	en = nouveau_enum_find(vm_engine, st0);
+	if (en)
+		printk("%s/", en->name);
+	else
+		printk("%02x/", st0);
+
+	cl = nouveau_enum_find(vm_client, st2);
+	if (cl)
+		printk("%s/", cl->name);
+	else
+		printk("%02x/", st2);
+
+	if      (cl && cl->data) cl = nouveau_enum_find(cl->data, st3);
+	else if (en && en->data) cl = nouveau_enum_find(en->data, st3);
+	else                     cl = NULL;
+	if (cl)
+		printk("%s", cl->name);
+	else
+		printk("%02x", st3);
+
+	printk(" reason: ");
+	en = nouveau_enum_find(vm_fault, st1);
+	if (en)
+		printk("%s\n", en->name);
+	else
+		printk("0x%08x\n", st1);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
new file mode 100644
index 00000000000..9f59f2bf007
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
@@ -0,0 +1,245 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/fb.h>
+#include <subdev/bios.h>
+
+struct nvc0_fb_priv {
+	struct nouveau_fb base;
+	struct page *r100c10_page;
+	dma_addr_t r100c10;
+};
+
+/* 0 = unsupported
+ * 1 = non-compressed
+ * 3 = compressed
+ */
+static const u8 types[256] = {
+	1, 1, 3, 3, 3, 3, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
+	0, 1, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3,
+	3, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 1, 1, 1, 1, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 3, 3, 3, 3, 1, 1, 1, 1, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
+	3, 3, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3,
+	3, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3, 0, 3,
+	3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 0, 3, 0, 3, 3, 0,
+	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 1, 1, 0
+};
+
+static bool
+nvc0_fb_memtype_valid(struct nouveau_fb *pfb, u32 tile_flags)
+{
+	u8 memtype = (tile_flags & 0x0000ff00) >> 8;
+	return likely((types[memtype] == 1));
+}
+
+static int
+nvc0_fb_vram_new(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
+		 u32 memtype, struct nouveau_mem **pmem)
+{
+	struct nouveau_mm *mm = &pfb->vram;
+	struct nouveau_mm_node *r;
+	struct nouveau_mem *mem;
+	int type = (memtype & 0x0ff);
+	int back = (memtype & 0x800);
+	int ret;
+
+	size  >>= 12;
+	align >>= 12;
+	ncmin >>= 12;
+	if (!ncmin)
+		ncmin = size;
+
+	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+	if (!mem)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&mem->regions);
+	mem->memtype = type;
+	mem->size = size;
+
+	mutex_lock(&mm->mutex);
+	do {
+		if (back)
+			ret = nouveau_mm_tail(mm, 1, size, ncmin, align, &r);
+		else
+			ret = nouveau_mm_head(mm, 1, size, ncmin, align, &r);
+		if (ret) {
+			mutex_unlock(&mm->mutex);
+			pfb->ram.put(pfb, &mem);
+			return ret;
+		}
+
+		list_add_tail(&r->rl_entry, &mem->regions);
+		size -= r->length;
+	} while (size);
+	mutex_unlock(&mm->mutex);
+
+	r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
+	mem->offset = (u64)r->offset << 12;
+	*pmem = mem;
+	return 0;
+}
+
+static int
+nvc0_fb_init(struct nouveau_object *object)
+{
+	struct nvc0_fb_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_fb_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x100c10, priv->r100c10 >> 8);
+	return 0;
+}
+
+static void
+nvc0_fb_dtor(struct nouveau_object *object)
+{
+	struct nouveau_device *device = nv_device(object);
+	struct nvc0_fb_priv *priv = (void *)object;
+
+	if (priv->r100c10_page) {
+		pci_unmap_page(device->pdev, priv->r100c10, PAGE_SIZE,
+			       PCI_DMA_BIDIRECTIONAL);
+		__free_page(priv->r100c10_page);
+	}
+
+	nouveau_fb_destroy(&priv->base);
+}
+
+static int
+nvc0_vram_detect(struct nvc0_fb_priv *priv)
+{
+	struct nouveau_bios *bios = nouveau_bios(priv);
+	struct nouveau_fb *pfb = &priv->base;
+	const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+	const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+	u32 parts = nv_rd32(priv, 0x022438);
+	u32 pmask = nv_rd32(priv, 0x022554);
+	u32 bsize = nv_rd32(priv, 0x10f20c);
+	u32 offset, length;
+	bool uniform = true;
+	int ret, part;
+
+	nv_debug(priv, "0x100800: 0x%08x\n", nv_rd32(priv, 0x100800));
+	nv_debug(priv, "parts 0x%08x mask 0x%08x\n", parts, pmask);
+
+	priv->base.ram.type = nouveau_fb_bios_memtype(bios);
+	priv->base.ram.ranks = (nv_rd32(priv, 0x10f200) & 0x00000004) ? 2 : 1;
+
+	/* read amount of vram attached to each memory controller */
+	for (part = 0; part < parts; part++) {
+		if (!(pmask & (1 << part))) {
+			u32 psize = nv_rd32(priv, 0x11020c + (part * 0x1000));
+			if (psize != bsize) {
+				if (psize < bsize)
+					bsize = psize;
+				uniform = false;
+			}
+
+			nv_debug(priv, "%d: mem_amount 0x%08x\n", part, psize);
+			priv->base.ram.size += (u64)psize << 20;
+		}
+	}
+
+	/* if all controllers have the same amount attached, there's no holes */
+	if (uniform) {
+		offset = rsvd_head;
+		length = (priv->base.ram.size >> 12) - rsvd_head - rsvd_tail;
+		return nouveau_mm_init(&pfb->vram, offset, length, 1);
+	}
+
+	/* otherwise, address lowest common amount from 0GiB */
+	ret = nouveau_mm_init(&pfb->vram, rsvd_head, (bsize << 8) * parts, 1);
+	if (ret)
+		return ret;
+
+	/* and the rest starting from (8GiB + common_size) */
+	offset = (0x0200000000ULL >> 12) + (bsize << 8);
+	length = (priv->base.ram.size >> 12) - (bsize << 8) - rsvd_tail;
+
+	ret = nouveau_mm_init(&pfb->vram, offset, length, 0);
+	if (ret) {
+		nouveau_mm_fini(&pfb->vram);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nvc0_fb_priv *priv;
+	int ret;
+
+	ret = nouveau_fb_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.memtype_valid = nvc0_fb_memtype_valid;
+	priv->base.ram.get = nvc0_fb_vram_new;
+	priv->base.ram.put = nv50_fb_vram_del;
+
+	ret = nvc0_vram_detect(priv);
+	if (ret)
+		return ret;
+
+	priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!priv->r100c10_page)
+		return -ENOMEM;
+
+	priv->r100c10 = pci_map_page(device->pdev, priv->r100c10_page, 0,
+				     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+	if (pci_dma_mapping_error(device->pdev, priv->r100c10))
+		return -EFAULT;
+
+	return nouveau_fb_created(&priv->base);
+}
+
+
+struct nouveau_oclass
+nvc0_fb_oclass = {
+	.handle = NV_SUBDEV(FB, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_fb_ctor,
+		.dtor = nvc0_fb_dtor,
+		.init = nvc0_fb_init,
+		.fini = _nouveau_fb_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
new file mode 100644
index 00000000000..acf818c58bf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/gpio.h>
+#include <subdev/bios.h>
+#include <subdev/bios/gpio.h>
+
+static int
+nouveau_gpio_drive(struct nouveau_gpio *gpio,
+		   int idx, int line, int dir, int out)
+{
+	return gpio->drive ? gpio->drive(gpio, line, dir, out) : -ENODEV;
+}
+
+static int
+nouveau_gpio_sense(struct nouveau_gpio *gpio, int idx, int line)
+{
+	return gpio->sense ? gpio->sense(gpio, line) : -ENODEV;
+}
+
+static int
+nouveau_gpio_find(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line,
+		  struct dcb_gpio_func *func)
+{
+	if (line == 0xff && tag == 0xff)
+		return -EINVAL;
+
+	if (!dcb_gpio_parse(nouveau_bios(gpio), idx, tag, line, func))
+		return 0;
+
+	/* Apple iMac G4 NV18 */
+	if (nv_device_match(nv_object(gpio), 0x0189, 0x10de, 0x0010)) {
+		if (tag == DCB_GPIO_TVDAC0) {
+			*func = (struct dcb_gpio_func) {
+				.func = DCB_GPIO_TVDAC0,
+				.line = 4,
+				.log[0] = 0,
+				.log[1] = 1,
+			};
+			return 0;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static int
+nouveau_gpio_set(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line, int state)
+{
+	struct dcb_gpio_func func;
+	int ret;
+
+	ret = nouveau_gpio_find(gpio, idx, tag, line, &func);
+	if (ret == 0) {
+		int dir = !!(func.log[state] & 0x02);
+		int out = !!(func.log[state] & 0x01);
+		ret = nouveau_gpio_drive(gpio, idx, func.line, dir, out);
+	}
+
+	return ret;
+}
+
+static int
+nouveau_gpio_get(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line)
+{
+	struct dcb_gpio_func func;
+	int ret;
+
+	ret = nouveau_gpio_find(gpio, idx, tag, line, &func);
+	if (ret == 0) {
+		ret = nouveau_gpio_sense(gpio, idx, func.line);
+		if (ret >= 0)
+			ret = (ret == (func.log[1] & 1));
+	}
+
+	return ret;
+}
+
+static int
+nouveau_gpio_irq(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line, bool on)
+{
+	struct dcb_gpio_func func;
+	int ret;
+
+	ret = nouveau_gpio_find(gpio, idx, tag, line, &func);
+	if (ret == 0) {
+		if (idx == 0 && gpio->irq_enable)
+			gpio->irq_enable(gpio, func.line, on);
+		else
+			ret = -ENODEV;
+	}
+
+	return ret;
+}
+
+struct gpio_isr {
+	struct nouveau_gpio *gpio;
+	struct list_head head;
+	struct work_struct work;
+	int idx;
+	struct dcb_gpio_func func;
+	void (*handler)(void *, int);
+	void *data;
+	bool inhibit;
+};
+
+static void
+nouveau_gpio_isr_bh(struct work_struct *work)
+{
+	struct gpio_isr *isr = container_of(work, struct gpio_isr, work);
+	struct nouveau_gpio *gpio = isr->gpio;
+	unsigned long flags;
+	int state;
+
+	state = nouveau_gpio_get(gpio, isr->idx, isr->func.func,
+						 isr->func.line);
+	if (state >= 0)
+		isr->handler(isr->data, state);
+
+	spin_lock_irqsave(&gpio->lock, flags);
+	isr->inhibit = false;
+	spin_unlock_irqrestore(&gpio->lock, flags);
+}
+
+static void
+nouveau_gpio_isr_run(struct nouveau_gpio *gpio, int idx, u32 line_mask)
+{
+	struct gpio_isr *isr;
+
+	if (idx != 0)
+		return;
+
+	spin_lock(&gpio->lock);
+	list_for_each_entry(isr, &gpio->isr, head) {
+		if (line_mask & (1 << isr->func.line)) {
+			if (isr->inhibit)
+				continue;
+			isr->inhibit = true;
+			schedule_work(&isr->work);
+		}
+	}
+	spin_unlock(&gpio->lock);
+}
+
+static int
+nouveau_gpio_isr_add(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line,
+		     void (*handler)(void *, int), void *data)
+{
+	struct gpio_isr *isr;
+	unsigned long flags;
+	int ret;
+
+	isr = kzalloc(sizeof(*isr), GFP_KERNEL);
+	if (!isr)
+		return -ENOMEM;
+
+	ret = nouveau_gpio_find(gpio, idx, tag, line, &isr->func);
+	if (ret) {
+		kfree(isr);
+		return ret;
+	}
+
+	INIT_WORK(&isr->work, nouveau_gpio_isr_bh);
+	isr->gpio = gpio;
+	isr->handler = handler;
+	isr->data = data;
+	isr->idx = idx;
+
+	spin_lock_irqsave(&gpio->lock, flags);
+	list_add(&isr->head, &gpio->isr);
+	spin_unlock_irqrestore(&gpio->lock, flags);
+	return 0;
+}
+
+static void
+nouveau_gpio_isr_del(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line,
+		     void (*handler)(void *, int), void *data)
+{
+	struct gpio_isr *isr, *tmp;
+	struct dcb_gpio_func func;
+	unsigned long flags;
+	LIST_HEAD(tofree);
+	int ret;
+
+	ret = nouveau_gpio_find(gpio, idx, tag, line, &func);
+	if (ret == 0) {
+		spin_lock_irqsave(&gpio->lock, flags);
+		list_for_each_entry_safe(isr, tmp, &gpio->isr, head) {
+			if (memcmp(&isr->func, &func, sizeof(func)) ||
+			    isr->idx != idx ||
+			    isr->handler != handler || isr->data != data)
+				continue;
+			list_move_tail(&isr->head, &tofree);
+		}
+		spin_unlock_irqrestore(&gpio->lock, flags);
+
+		list_for_each_entry_safe(isr, tmp, &tofree, head) {
+			flush_work(&isr->work);
+			kfree(isr);
+		}
+	}
+}
+
+int
+nouveau_gpio_create_(struct nouveau_object *parent,
+		     struct nouveau_object *engine,
+		     struct nouveau_oclass *oclass, int length, void **pobject)
+{
+	struct nouveau_gpio *gpio;
+	int ret;
+
+	ret = nouveau_subdev_create_(parent, engine, oclass, 0, "GPIO", "gpio",
+				     length, pobject);
+	gpio = *pobject;
+	if (ret)
+		return ret;
+
+	gpio->find = nouveau_gpio_find;
+	gpio->set  = nouveau_gpio_set;
+	gpio->get  = nouveau_gpio_get;
+	gpio->irq  = nouveau_gpio_irq;
+	gpio->isr_run = nouveau_gpio_isr_run;
+	gpio->isr_add = nouveau_gpio_isr_add;
+	gpio->isr_del = nouveau_gpio_isr_del;
+	INIT_LIST_HEAD(&gpio->isr);
+	spin_lock_init(&gpio->lock);
+	return 0;
+}
+
+static struct dmi_system_id gpio_reset_ids[] = {
+	{
+		.ident = "Apple Macbook 10,1",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro10,1"),
+		}
+	},
+	{ }
+};
+
+int
+nouveau_gpio_init(struct nouveau_gpio *gpio)
+{
+	int ret = nouveau_subdev_init(&gpio->base);
+	if (ret == 0 && gpio->reset) {
+		if (dmi_check_system(gpio_reset_ids))
+			gpio->reset(gpio);
+	}
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv10.c
new file mode 100644
index 00000000000..168d16a9a8e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv10.c
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2009 Francisco Jerez.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <subdev/gpio.h>
+
+struct nv10_gpio_priv {
+	struct nouveau_gpio base;
+};
+
+static int
+nv10_gpio_sense(struct nouveau_gpio *gpio, int line)
+{
+	if (line < 2) {
+		line = line * 16;
+		line = nv_rd32(gpio, 0x600818) >> line;
+		return !!(line & 0x0100);
+	} else
+	if (line < 10) {
+		line = (line - 2) * 4;
+		line = nv_rd32(gpio, 0x60081c) >> line;
+		return !!(line & 0x04);
+	} else
+	if (line < 14) {
+		line = (line - 10) * 4;
+		line = nv_rd32(gpio, 0x600850) >> line;
+		return !!(line & 0x04);
+	}
+
+	return -EINVAL;
+}
+
+static int
+nv10_gpio_drive(struct nouveau_gpio *gpio, int line, int dir, int out)
+{
+	u32 reg, mask, data;
+
+	if (line < 2) {
+		line = line * 16;
+		reg  = 0x600818;
+		mask = 0x00000011;
+		data = (dir << 4) | out;
+	} else
+	if (line < 10) {
+		line = (line - 2) * 4;
+		reg  = 0x60081c;
+		mask = 0x00000003;
+		data = (dir << 1) | out;
+	} else
+	if (line < 14) {
+		line = (line - 10) * 4;
+		reg  = 0x600850;
+		mask = 0x00000003;
+		data = (dir << 1) | out;
+	} else {
+		return -EINVAL;
+	}
+
+	nv_mask(gpio, reg, mask << line, data << line);
+	return 0;
+}
+
+static void
+nv10_gpio_irq_enable(struct nouveau_gpio *gpio, int line, bool on)
+{
+	u32 mask = 0x00010001 << line;
+
+	nv_wr32(gpio, 0x001104, mask);
+	nv_mask(gpio, 0x001144, mask, on ? mask : 0);
+}
+
+static void
+nv10_gpio_intr(struct nouveau_subdev *subdev)
+{
+	struct nv10_gpio_priv *priv = (void *)subdev;
+	u32 intr = nv_rd32(priv, 0x001104);
+	u32 hi = (intr & 0x0000ffff) >> 0;
+	u32 lo = (intr & 0xffff0000) >> 16;
+
+	priv->base.isr_run(&priv->base, 0, hi | lo);
+
+	nv_wr32(priv, 0x001104, intr);
+}
+
+static int
+nv10_gpio_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv10_gpio_priv *priv;
+	int ret;
+
+	ret = nouveau_gpio_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.drive = nv10_gpio_drive;
+	priv->base.sense = nv10_gpio_sense;
+	priv->base.irq_enable = nv10_gpio_irq_enable;
+	nv_subdev(priv)->intr = nv10_gpio_intr;
+	return 0;
+}
+
+static void
+nv10_gpio_dtor(struct nouveau_object *object)
+{
+	struct nv10_gpio_priv *priv = (void *)object;
+	nouveau_gpio_destroy(&priv->base);
+}
+
+static int
+nv10_gpio_init(struct nouveau_object *object)
+{
+	struct nv10_gpio_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_gpio_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x001140, 0x00000000);
+	nv_wr32(priv, 0x001100, 0xffffffff);
+	nv_wr32(priv, 0x001144, 0x00000000);
+	nv_wr32(priv, 0x001104, 0xffffffff);
+	return 0;
+}
+
+static int
+nv10_gpio_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv10_gpio_priv *priv = (void *)object;
+	nv_wr32(priv, 0x001140, 0x00000000);
+	nv_wr32(priv, 0x001144, 0x00000000);
+	return nouveau_gpio_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv10_gpio_oclass = {
+	.handle = NV_SUBDEV(GPIO, 0x10),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv10_gpio_ctor,
+		.dtor = nv10_gpio_dtor,
+		.init = nv10_gpio_init,
+		.fini = nv10_gpio_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
new file mode 100644
index 00000000000..f3502c961cd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/gpio.h>
+
+struct nv50_gpio_priv {
+	struct nouveau_gpio base;
+};
+
+static void
+nv50_gpio_reset(struct nouveau_gpio *gpio)
+{
+	struct nouveau_bios *bios = nouveau_bios(gpio);
+	struct nv50_gpio_priv *priv = (void *)gpio;
+	u16 entry;
+	u8 ver;
+	int ent = -1;
+
+	while ((entry = dcb_gpio_entry(bios, 0, ++ent, &ver))) {
+		static const u32 regs[] = { 0xe100, 0xe28c };
+		u32 data = nv_ro32(bios, entry);
+		u8  line =   (data & 0x0000001f);
+		u8  func =   (data & 0x0000ff00) >> 8;
+		u8  defs = !!(data & 0x01000000);
+		u8  unk0 = !!(data & 0x02000000);
+		u8  unk1 = !!(data & 0x04000000);
+		u32 val = (unk1 << 16) | unk0;
+		u32 reg = regs[line >> 4]; line &= 0x0f;
+
+		if (func == 0xff)
+			continue;
+
+		gpio->set(gpio, 0, func, line, defs);
+
+		nv_mask(priv, reg, 0x00010001 << line, val << line);
+	}
+}
+
+static int
+nv50_gpio_location(int line, u32 *reg, u32 *shift)
+{
+	const u32 nv50_gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
+
+	if (line >= 32)
+		return -EINVAL;
+
+	*reg = nv50_gpio_reg[line >> 3];
+	*shift = (line & 7) << 2;
+	return 0;
+}
+
+static int
+nv50_gpio_drive(struct nouveau_gpio *gpio, int line, int dir, int out)
+{
+	u32 reg, shift;
+
+	if (nv50_gpio_location(line, &reg, &shift))
+		return -EINVAL;
+
+	nv_mask(gpio, reg, 7 << shift, (((dir ^ 1) << 1) | out) << shift);
+	return 0;
+}
+
+static int
+nv50_gpio_sense(struct nouveau_gpio *gpio, int line)
+{
+	u32 reg, shift;
+
+	if (nv50_gpio_location(line, &reg, &shift))
+		return -EINVAL;
+
+	return !!(nv_rd32(gpio, reg) & (4 << shift));
+}
+
+void
+nv50_gpio_irq_enable(struct nouveau_gpio *gpio, int line, bool on)
+{
+	u32 reg  = line < 16 ? 0xe050 : 0xe070;
+	u32 mask = 0x00010001 << (line & 0xf);
+
+	nv_wr32(gpio, reg + 4, mask);
+	nv_mask(gpio, reg + 0, mask, on ? mask : 0);
+}
+
+void
+nv50_gpio_intr(struct nouveau_subdev *subdev)
+{
+	struct nv50_gpio_priv *priv = (void *)subdev;
+	u32 intr0, intr1 = 0;
+	u32 hi, lo;
+
+	intr0 = nv_rd32(priv, 0xe054) & nv_rd32(priv, 0xe050);
+	if (nv_device(priv)->chipset >= 0x90)
+		intr1 = nv_rd32(priv, 0xe074) & nv_rd32(priv, 0xe070);
+
+	hi = (intr0 & 0x0000ffff) | (intr1 << 16);
+	lo = (intr0 >> 16) | (intr1 & 0xffff0000);
+	priv->base.isr_run(&priv->base, 0, hi | lo);
+
+	nv_wr32(priv, 0xe054, intr0);
+	if (nv_device(priv)->chipset >= 0x90)
+		nv_wr32(priv, 0xe074, intr1);
+}
+
+static int
+nv50_gpio_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nv50_gpio_priv *priv;
+	int ret;
+
+	ret = nouveau_gpio_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.reset = nv50_gpio_reset;
+	priv->base.drive = nv50_gpio_drive;
+	priv->base.sense = nv50_gpio_sense;
+	priv->base.irq_enable = nv50_gpio_irq_enable;
+	nv_subdev(priv)->intr = nv50_gpio_intr;
+	return 0;
+}
+
+void
+nv50_gpio_dtor(struct nouveau_object *object)
+{
+	struct nv50_gpio_priv *priv = (void *)object;
+	nouveau_gpio_destroy(&priv->base);
+}
+
+int
+nv50_gpio_init(struct nouveau_object *object)
+{
+	struct nv50_gpio_priv *priv = (void *)object;
+	int ret;
+
+	ret = nouveau_gpio_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* disable, and ack any pending gpio interrupts */
+	nv_wr32(priv, 0xe050, 0x00000000);
+	nv_wr32(priv, 0xe054, 0xffffffff);
+	if (nv_device(priv)->chipset >= 0x90) {
+		nv_wr32(priv, 0xe070, 0x00000000);
+		nv_wr32(priv, 0xe074, 0xffffffff);
+	}
+
+	return 0;
+}
+
+int
+nv50_gpio_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv50_gpio_priv *priv = (void *)object;
+	nv_wr32(priv, 0xe050, 0x00000000);
+	if (nv_device(priv)->chipset >= 0x90)
+		nv_wr32(priv, 0xe070, 0x00000000);
+	return nouveau_gpio_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv50_gpio_oclass = {
+	.handle = NV_SUBDEV(GPIO, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_gpio_ctor,
+		.dtor = nv50_gpio_dtor,
+		.init = nv50_gpio_init,
+		.fini = nv50_gpio_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nvd0.c
new file mode 100644
index 00000000000..8d18fcad26e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nvd0.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/gpio.h>
+
+struct nvd0_gpio_priv {
+	struct nouveau_gpio base;
+};
+
+static void
+nvd0_gpio_reset(struct nouveau_gpio *gpio)
+{
+	struct nouveau_bios *bios = nouveau_bios(gpio);
+	struct nvd0_gpio_priv *priv = (void *)gpio;
+	u16 entry;
+	u8 ver;
+	int ent = -1;
+
+	while ((entry = dcb_gpio_entry(bios, 0, ++ent, &ver))) {
+		u32 data = nv_ro32(bios, entry);
+		u8  line =   (data & 0x0000003f);
+		u8  defs = !!(data & 0x00000080);
+		u8  func =   (data & 0x0000ff00) >> 8;
+		u8  unk0 =   (data & 0x00ff0000) >> 16;
+		u8  unk1 =   (data & 0x1f000000) >> 24;
+
+		if (func == 0xff)
+			continue;
+
+		gpio->set(gpio, 0, func, line, defs);
+
+		nv_mask(priv, 0x00d610 + (line * 4), 0xff, unk0);
+		if (unk1--)
+			nv_mask(priv, 0x00d740 + (unk1 * 4), 0xff, line);
+	}
+}
+
+static int
+nvd0_gpio_drive(struct nouveau_gpio *gpio, int line, int dir, int out)
+{
+	u32 data = ((dir ^ 1) << 13) | (out << 12);
+	nv_mask(gpio, 0x00d610 + (line * 4), 0x00003000, data);
+	nv_mask(gpio, 0x00d604, 0x00000001, 0x00000001); /* update? */
+	return 0;
+}
+
+static int
+nvd0_gpio_sense(struct nouveau_gpio *gpio, int line)
+{
+	return !!(nv_rd32(gpio, 0x00d610 + (line * 4)) & 0x00004000);
+}
+
+static int
+nvd0_gpio_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nvd0_gpio_priv *priv;
+	int ret;
+
+	ret = nouveau_gpio_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.reset = nvd0_gpio_reset;
+	priv->base.drive = nvd0_gpio_drive;
+	priv->base.sense = nvd0_gpio_sense;
+	priv->base.irq_enable = nv50_gpio_irq_enable;
+	nv_subdev(priv)->intr = nv50_gpio_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nvd0_gpio_oclass = {
+	.handle = NV_SUBDEV(GPIO, 0xd0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvd0_gpio_ctor,
+		.dtor = nv50_gpio_dtor,
+		.init = nv50_gpio_init,
+		.fini = nv50_gpio_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/aux.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/aux.c
new file mode 100644
index 00000000000..fe1ebf199ba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/aux.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2009 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/i2c.h>
+
+/******************************************************************************
+ * aux channel util functions
+ *****************************************************************************/
+#define AUX_DBG(fmt, args...) nv_debug(aux, "AUXCH(%d): " fmt, ch, ##args)
+#define AUX_ERR(fmt, args...) nv_error(aux, "AUXCH(%d): " fmt, ch, ##args)
+
+static void
+auxch_fini(struct nouveau_i2c *aux, int ch)
+{
+	nv_mask(aux, 0x00e4e4 + (ch * 0x50), 0x00310000, 0x00000000);
+}
+
+static int
+auxch_init(struct nouveau_i2c *aux, int ch)
+{
+	const u32 unksel = 1; /* nfi which to use, or if it matters.. */
+	const u32 ureq = unksel ? 0x00100000 : 0x00200000;
+	const u32 urep = unksel ? 0x01000000 : 0x02000000;
+	u32 ctrl, timeout;
+
+	/* wait up to 1ms for any previous transaction to be done... */
+	timeout = 1000;
+	do {
+		ctrl = nv_rd32(aux, 0x00e4e4 + (ch * 0x50));
+		udelay(1);
+		if (!timeout--) {
+			AUX_ERR("begin idle timeout 0x%08x", ctrl);
+			return -EBUSY;
+		}
+	} while (ctrl & 0x03010000);
+
+	/* set some magic, and wait up to 1ms for it to appear */
+	nv_mask(aux, 0x00e4e4 + (ch * 0x50), 0x00300000, ureq);
+	timeout = 1000;
+	do {
+		ctrl = nv_rd32(aux, 0x00e4e4 + (ch * 0x50));
+		udelay(1);
+		if (!timeout--) {
+			AUX_ERR("magic wait 0x%08x\n", ctrl);
+			auxch_fini(aux, ch);
+			return -EBUSY;
+		}
+	} while ((ctrl & 0x03000000) != urep);
+
+	return 0;
+}
+
+static int
+auxch_tx(struct nouveau_i2c *aux, int ch, u8 type, u32 addr, u8 *data, u8 size)
+{
+	u32 ctrl, stat, timeout, retries;
+	u32 xbuf[4] = {};
+	int ret, i;
+
+	AUX_DBG("%d: 0x%08x %d\n", type, addr, size);
+
+	ret = auxch_init(aux, ch);
+	if (ret)
+		goto out;
+
+	stat = nv_rd32(aux, 0x00e4e8 + (ch * 0x50));
+	if (!(stat & 0x10000000)) {
+		AUX_DBG("sink not detected\n");
+		ret = -ENXIO;
+		goto out;
+	}
+
+	if (!(type & 1)) {
+		memcpy(xbuf, data, size);
+		for (i = 0; i < 16; i += 4) {
+			AUX_DBG("wr 0x%08x\n", xbuf[i / 4]);
+			nv_wr32(aux, 0x00e4c0 + (ch * 0x50) + i, xbuf[i / 4]);
+		}
+	}
+
+	ctrl  = nv_rd32(aux, 0x00e4e4 + (ch * 0x50));
+	ctrl &= ~0x0001f0ff;
+	ctrl |= type << 12;
+	ctrl |= size - 1;
+	nv_wr32(aux, 0x00e4e0 + (ch * 0x50), addr);
+
+	/* retry transaction a number of times on failure... */
+	ret = -EREMOTEIO;
+	for (retries = 0; retries < 32; retries++) {
+		/* reset, and delay a while if this is a retry */
+		nv_wr32(aux, 0x00e4e4 + (ch * 0x50), 0x80000000 | ctrl);
+		nv_wr32(aux, 0x00e4e4 + (ch * 0x50), 0x00000000 | ctrl);
+		if (retries)
+			udelay(400);
+
+		/* transaction request, wait up to 1ms for it to complete */
+		nv_wr32(aux, 0x00e4e4 + (ch * 0x50), 0x00010000 | ctrl);
+
+		timeout = 1000;
+		do {
+			ctrl = nv_rd32(aux, 0x00e4e4 + (ch * 0x50));
+			udelay(1);
+			if (!timeout--) {
+				AUX_ERR("tx req timeout 0x%08x\n", ctrl);
+				goto out;
+			}
+		} while (ctrl & 0x00010000);
+
+		/* read status, and check if transaction completed ok */
+		stat = nv_mask(aux, 0x00e4e8 + (ch * 0x50), 0, 0);
+		if (!(stat & 0x000f0f00)) {
+			ret = 0;
+			break;
+		}
+
+		AUX_DBG("%02d 0x%08x 0x%08x\n", retries, ctrl, stat);
+	}
+
+	if (type & 1) {
+		for (i = 0; i < 16; i += 4) {
+			xbuf[i / 4] = nv_rd32(aux, 0x00e4d0 + (ch * 0x50) + i);
+			AUX_DBG("rd 0x%08x\n", xbuf[i / 4]);
+		}
+		memcpy(data, xbuf, size);
+	}
+
+out:
+	auxch_fini(aux, ch);
+	return ret;
+}
+
+int
+nv_rdaux(struct nouveau_i2c_port *auxch, u32 addr, u8 *data, u8 size)
+{
+	return auxch_tx(auxch->i2c, auxch->drive, 9, addr, data, size);
+}
+
+int
+nv_wraux(struct nouveau_i2c_port *auxch, u32 addr, u8 *data, u8 size)
+{
+	return auxch_tx(auxch->i2c, auxch->drive, 8, addr, data, size);
+}
+
+static int
+aux_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	struct nouveau_i2c_port *auxch = (struct nouveau_i2c_port *)adap;
+	struct i2c_msg *msg = msgs;
+	int ret, mcnt = num;
+
+	while (mcnt--) {
+		u8 remaining = msg->len;
+		u8 *ptr = msg->buf;
+
+		while (remaining) {
+			u8 cnt = (remaining > 16) ? 16 : remaining;
+			u8 cmd;
+
+			if (msg->flags & I2C_M_RD)
+				cmd = 1;
+			else
+				cmd = 0;
+
+			if (mcnt || remaining > 16)
+				cmd |= 4; /* MOT */
+
+			ret = auxch_tx(auxch->i2c, auxch->drive, cmd,
+				       msg->addr, ptr, cnt);
+			if (ret < 0)
+				return ret;
+
+			ptr += cnt;
+			remaining -= cnt;
+		}
+
+		msg++;
+	}
+
+	return num;
+}
+
+static u32
+aux_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+const struct i2c_algorithm nouveau_i2c_aux_algo = {
+	.master_xfer = aux_xfer,
+	.functionality = aux_func
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
new file mode 100644
index 00000000000..3d2c88310f9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -0,0 +1,407 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "core/option.h"
+
+#include "subdev/i2c.h"
+#include "subdev/vga.h"
+
+int
+nv_rdi2cr(struct nouveau_i2c_port *port, u8 addr, u8 reg)
+{
+	u8 val;
+	struct i2c_msg msgs[] = {
+		{ .addr = addr, .flags = 0, .len = 1, .buf = &reg },
+		{ .addr = addr, .flags = I2C_M_RD, .len = 1, .buf = &val },
+	};
+
+	int ret = i2c_transfer(&port->adapter, msgs, 2);
+	if (ret != 2)
+		return -EIO;
+
+	return val;
+}
+
+int
+nv_wri2cr(struct nouveau_i2c_port *port, u8 addr, u8 reg, u8 val)
+{
+	struct i2c_msg msgs[] = {
+		{ .addr = addr, .flags = 0, .len = 1, .buf = &reg },
+		{ .addr = addr, .flags = 0, .len = 1, .buf = &val },
+	};
+
+	int ret = i2c_transfer(&port->adapter, msgs, 2);
+	if (ret != 2)
+		return -EIO;
+
+	return 0;
+}
+
+bool
+nv_probe_i2c(struct nouveau_i2c_port *port, u8 addr)
+{
+	u8 buf[] = { 0 };
+	struct i2c_msg msgs[] = {
+		{
+			.addr = addr,
+			.flags = 0,
+			.len = 1,
+			.buf = buf,
+		},
+		{
+			.addr = addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = buf,
+		}
+	};
+
+	return i2c_transfer(&port->adapter, msgs, 2) == 2;
+}
+
+static struct nouveau_i2c_port *
+nouveau_i2c_find(struct nouveau_i2c *i2c, u8 index)
+{
+	struct nouveau_bios *bios = nouveau_bios(i2c);
+	struct nouveau_i2c_port *port;
+
+	if (index == NV_I2C_DEFAULT(0) ||
+	    index == NV_I2C_DEFAULT(1)) {
+		u8  ver, hdr, cnt, len;
+		u16 i2c = dcb_i2c_table(bios, &ver, &hdr, &cnt, &len);
+		if (i2c && ver >= 0x30) {
+			u8 auxidx = nv_ro08(bios, i2c + 4);
+			if (index == NV_I2C_DEFAULT(0))
+				index = (auxidx & 0x0f) >> 0;
+			else
+				index = (auxidx & 0xf0) >> 4;
+		} else {
+			index = 2;
+		}
+	}
+
+	list_for_each_entry(port, &i2c->ports, head) {
+		if (port->index == index)
+			break;
+	}
+
+	if (&port->head == &i2c->ports)
+		return NULL;
+
+	if (nv_device(i2c)->card_type >= NV_50 && (port->dcb & 0x00000100)) {
+		u32 reg = 0x00e500, val;
+		if (port->type == 6) {
+			reg += port->drive * 0x50;
+			val  = 0x2002;
+		} else {
+			reg += ((port->dcb & 0x1e00) >> 9) * 0x50;
+			val  = 0xe001;
+		}
+
+		/* nfi, but neither auxch or i2c work if it's 1 */
+		nv_mask(i2c, reg + 0x0c, 0x00000001, 0x00000000);
+		/* nfi, but switches auxch vs normal i2c */
+		nv_mask(i2c, reg + 0x00, 0x0000f003, val);
+	}
+
+	return port;
+}
+
+static int
+nouveau_i2c_identify(struct nouveau_i2c *i2c, int index, const char *what,
+		     struct i2c_board_info *info,
+		     bool (*match)(struct nouveau_i2c_port *,
+				   struct i2c_board_info *))
+{
+	struct nouveau_i2c_port *port = nouveau_i2c_find(i2c, index);
+	int i;
+
+	if (!port) {
+		nv_debug(i2c, "no bus when probing %s on %d\n", what, index);
+		return -ENODEV;
+	}
+
+	nv_debug(i2c, "probing %ss on bus: %d\n", what, port->index);
+	for (i = 0; info[i].addr; i++) {
+		if (nv_probe_i2c(port, info[i].addr) &&
+		    (!match || match(port, &info[i]))) {
+			nv_info(i2c, "detected %s: %s\n", what, info[i].type);
+			return i;
+		}
+	}
+
+	nv_debug(i2c, "no devices found.\n");
+	return -ENODEV;
+}
+
+void
+nouveau_i2c_drive_scl(void *data, int state)
+{
+	struct nouveau_i2c_port *port = data;
+
+	if (port->type == DCB_I2C_NV04_BIT) {
+		u8 val = nv_rdvgac(port->i2c, 0, port->drive);
+		if (state) val |= 0x20;
+		else	   val &= 0xdf;
+		nv_wrvgac(port->i2c, 0, port->drive, val | 0x01);
+	} else
+	if (port->type == DCB_I2C_NV4E_BIT) {
+		nv_mask(port->i2c, port->drive, 0x2f, state ? 0x21 : 0x01);
+	} else
+	if (port->type == DCB_I2C_NVIO_BIT) {
+		if (state) port->state |= 0x01;
+		else	   port->state &= 0xfe;
+		nv_wr32(port->i2c, port->drive, 4 | port->state);
+	}
+}
+
+void
+nouveau_i2c_drive_sda(void *data, int state)
+{
+	struct nouveau_i2c_port *port = data;
+
+	if (port->type == DCB_I2C_NV04_BIT) {
+		u8 val = nv_rdvgac(port->i2c, 0, port->drive);
+		if (state) val |= 0x10;
+		else	   val &= 0xef;
+		nv_wrvgac(port->i2c, 0, port->drive, val | 0x01);
+	} else
+	if (port->type == DCB_I2C_NV4E_BIT) {
+		nv_mask(port->i2c, port->drive, 0x1f, state ? 0x11 : 0x01);
+	} else
+	if (port->type == DCB_I2C_NVIO_BIT) {
+		if (state) port->state |= 0x02;
+		else	   port->state &= 0xfd;
+		nv_wr32(port->i2c, port->drive, 4 | port->state);
+	}
+}
+
+int
+nouveau_i2c_sense_scl(void *data)
+{
+	struct nouveau_i2c_port *port = data;
+	struct nouveau_device *device = nv_device(port->i2c);
+
+	if (port->type == DCB_I2C_NV04_BIT) {
+		return !!(nv_rdvgac(port->i2c, 0, port->sense) & 0x04);
+	} else
+	if (port->type == DCB_I2C_NV4E_BIT) {
+		return !!(nv_rd32(port->i2c, port->sense) & 0x00040000);
+	} else
+	if (port->type == DCB_I2C_NVIO_BIT) {
+		if (device->card_type < NV_D0)
+			return !!(nv_rd32(port->i2c, port->sense) & 0x01);
+		else
+			return !!(nv_rd32(port->i2c, port->sense) & 0x10);
+	}
+
+	return 0;
+}
+
+int
+nouveau_i2c_sense_sda(void *data)
+{
+	struct nouveau_i2c_port *port = data;
+	struct nouveau_device *device = nv_device(port->i2c);
+
+	if (port->type == DCB_I2C_NV04_BIT) {
+		return !!(nv_rdvgac(port->i2c, 0, port->sense) & 0x08);
+	} else
+	if (port->type == DCB_I2C_NV4E_BIT) {
+		return !!(nv_rd32(port->i2c, port->sense) & 0x00080000);
+	} else
+	if (port->type == DCB_I2C_NVIO_BIT) {
+		if (device->card_type < NV_D0)
+			return !!(nv_rd32(port->i2c, port->sense) & 0x02);
+		else
+			return !!(nv_rd32(port->i2c, port->sense) & 0x20);
+	}
+
+	return 0;
+}
+
+static const u32 nv50_i2c_port[] = {
+	0x00e138, 0x00e150, 0x00e168, 0x00e180,
+	0x00e254, 0x00e274, 0x00e764, 0x00e780,
+	0x00e79c, 0x00e7b8
+};
+
+static int
+nouveau_i2c_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		 struct nouveau_oclass *oclass, void *data, u32 size,
+		 struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_bios *bios = nouveau_bios(parent);
+	struct nouveau_i2c_port *port;
+	struct nouveau_i2c *i2c;
+	struct dcb_i2c_entry info;
+	int ret, i = -1;
+
+	ret = nouveau_subdev_create(parent, engine, oclass, 0,
+				    "I2C", "i2c", &i2c);
+	*pobject = nv_object(i2c);
+	if (ret)
+		return ret;
+
+	i2c->find = nouveau_i2c_find;
+	i2c->identify = nouveau_i2c_identify;
+	INIT_LIST_HEAD(&i2c->ports);
+
+	while (!dcb_i2c_parse(bios, ++i, &info)) {
+		if (info.type == DCB_I2C_UNUSED)
+			continue;
+
+		port = kzalloc(sizeof(*port), GFP_KERNEL);
+		if (!port) {
+			nv_error(i2c, "failed port memory alloc at %d\n", i);
+			break;
+		}
+
+		port->type = info.type;
+		switch (port->type) {
+		case DCB_I2C_NV04_BIT:
+			port->drive = info.drive;
+			port->sense = info.sense;
+			break;
+		case DCB_I2C_NV4E_BIT:
+			port->drive = 0x600800 + info.drive;
+			port->sense = port->drive;
+			break;
+		case DCB_I2C_NVIO_BIT:
+			port->drive = info.drive & 0x0f;
+			if (device->card_type < NV_D0) {
+				if (info.drive >= ARRAY_SIZE(nv50_i2c_port))
+					break;
+				port->drive = nv50_i2c_port[port->drive];
+				port->sense = port->drive;
+			} else {
+				port->drive = 0x00d014 + (port->drive * 0x20);
+				port->sense = port->drive;
+			}
+			break;
+		case DCB_I2C_NVIO_AUX:
+			port->drive = info.drive & 0x0f;
+			port->sense = port->drive;
+			port->adapter.algo = &nouveau_i2c_aux_algo;
+			break;
+		default:
+			break;
+		}
+
+		if (!port->adapter.algo && !port->drive) {
+			nv_error(i2c, "I2C%d: type %d index %x/%x unknown\n",
+				 i, port->type, port->drive, port->sense);
+			kfree(port);
+			continue;
+		}
+
+		snprintf(port->adapter.name, sizeof(port->adapter.name),
+			 "nouveau-%s-%d", device->name, i);
+		port->adapter.owner = THIS_MODULE;
+		port->adapter.dev.parent = &device->pdev->dev;
+		port->i2c = i2c;
+		port->index = i;
+		port->dcb = info.data;
+		i2c_set_adapdata(&port->adapter, i2c);
+
+		if (port->adapter.algo != &nouveau_i2c_aux_algo) {
+			nouveau_i2c_drive_scl(port, 0);
+			nouveau_i2c_drive_sda(port, 1);
+			nouveau_i2c_drive_scl(port, 1);
+
+#ifdef CONFIG_NOUVEAU_I2C_INTERNAL_DEFAULT
+			if (nouveau_boolopt(device->cfgopt, "NvI2C", true)) {
+#else
+			if (nouveau_boolopt(device->cfgopt, "NvI2C", false)) {
+#endif
+				port->adapter.algo = &nouveau_i2c_bit_algo;
+				ret = i2c_add_adapter(&port->adapter);
+			} else {
+				port->adapter.algo_data = &port->bit;
+				port->bit.udelay = 10;
+				port->bit.timeout = usecs_to_jiffies(2200);
+				port->bit.data = port;
+				port->bit.setsda = nouveau_i2c_drive_sda;
+				port->bit.setscl = nouveau_i2c_drive_scl;
+				port->bit.getsda = nouveau_i2c_sense_sda;
+				port->bit.getscl = nouveau_i2c_sense_scl;
+				ret = i2c_bit_add_bus(&port->adapter);
+			}
+		} else {
+			port->adapter.algo = &nouveau_i2c_aux_algo;
+			ret = i2c_add_adapter(&port->adapter);
+		}
+
+		if (ret) {
+			nv_error(i2c, "I2C%d: failed register: %d\n", i, ret);
+			kfree(port);
+			continue;
+		}
+
+		list_add_tail(&port->head, &i2c->ports);
+	}
+
+	return 0;
+}
+
+static void
+nouveau_i2c_dtor(struct nouveau_object *object)
+{
+	struct nouveau_i2c *i2c = (void *)object;
+	struct nouveau_i2c_port *port, *temp;
+
+	list_for_each_entry_safe(port, temp, &i2c->ports, head) {
+		i2c_del_adapter(&port->adapter);
+		list_del(&port->head);
+		kfree(port);
+	}
+
+	nouveau_subdev_destroy(&i2c->base);
+}
+
+static int
+nouveau_i2c_init(struct nouveau_object *object)
+{
+	struct nouveau_i2c *i2c = (void *)object;
+	return nouveau_subdev_init(&i2c->base);
+}
+
+static int
+nouveau_i2c_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nouveau_i2c *i2c = (void *)object;
+	return nouveau_subdev_fini(&i2c->base, suspend);
+}
+
+struct nouveau_oclass
+nouveau_i2c_oclass = {
+	.handle = NV_SUBDEV(I2C, 0x00),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nouveau_i2c_ctor,
+		.dtor = nouveau_i2c_dtor,
+		.init = nouveau_i2c_init,
+		.fini = nouveau_i2c_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/bit.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/bit.c
new file mode 100644
index 00000000000..1c4c9a5c8e2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/bit.c
@@ -0,0 +1,230 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "subdev/i2c.h"
+
+#ifdef CONFIG_NOUVEAU_I2C_INTERNAL
+#define T_TIMEOUT  2200000
+#define T_RISEFALL 1000
+#define T_HOLD     5000
+
+static inline void
+i2c_drive_scl(struct nouveau_i2c_port *port, int state)
+{
+	nouveau_i2c_drive_scl(port, state);
+}
+
+static inline void
+i2c_drive_sda(struct nouveau_i2c_port *port, int state)
+{
+	nouveau_i2c_drive_sda(port, state);
+}
+
+static inline int
+i2c_sense_scl(struct nouveau_i2c_port *port)
+{
+	return nouveau_i2c_sense_scl(port);
+}
+
+static inline int
+i2c_sense_sda(struct nouveau_i2c_port *port)
+{
+	return nouveau_i2c_sense_sda(port);
+}
+
+static void
+i2c_delay(struct nouveau_i2c_port *port, u32 nsec)
+{
+	udelay((nsec + 500) / 1000);
+}
+
+static bool
+i2c_raise_scl(struct nouveau_i2c_port *port)
+{
+	u32 timeout = T_TIMEOUT / T_RISEFALL;
+
+	i2c_drive_scl(port, 1);
+	do {
+		i2c_delay(port, T_RISEFALL);
+	} while (!i2c_sense_scl(port) && --timeout);
+
+	return timeout != 0;
+}
+
+static int
+i2c_start(struct nouveau_i2c_port *port)
+{
+	int ret = 0;
+
+	port->state  = i2c_sense_scl(port);
+	port->state |= i2c_sense_sda(port) << 1;
+	if (port->state != 3) {
+		i2c_drive_scl(port, 0);
+		i2c_drive_sda(port, 1);
+		if (!i2c_raise_scl(port))
+			ret = -EBUSY;
+	}
+
+	i2c_drive_sda(port, 0);
+	i2c_delay(port, T_HOLD);
+	i2c_drive_scl(port, 0);
+	i2c_delay(port, T_HOLD);
+	return ret;
+}
+
+static void
+i2c_stop(struct nouveau_i2c_port *port)
+{
+	i2c_drive_scl(port, 0);
+	i2c_drive_sda(port, 0);
+	i2c_delay(port, T_RISEFALL);
+
+	i2c_drive_scl(port, 1);
+	i2c_delay(port, T_HOLD);
+	i2c_drive_sda(port, 1);
+	i2c_delay(port, T_HOLD);
+}
+
+static int
+i2c_bitw(struct nouveau_i2c_port *port, int sda)
+{
+	i2c_drive_sda(port, sda);
+	i2c_delay(port, T_RISEFALL);
+
+	if (!i2c_raise_scl(port))
+		return -ETIMEDOUT;
+	i2c_delay(port, T_HOLD);
+
+	i2c_drive_scl(port, 0);
+	i2c_delay(port, T_HOLD);
+	return 0;
+}
+
+static int
+i2c_bitr(struct nouveau_i2c_port *port)
+{
+	int sda;
+
+	i2c_drive_sda(port, 1);
+	i2c_delay(port, T_RISEFALL);
+
+	if (!i2c_raise_scl(port))
+		return -ETIMEDOUT;
+	i2c_delay(port, T_HOLD);
+
+	sda = i2c_sense_sda(port);
+
+	i2c_drive_scl(port, 0);
+	i2c_delay(port, T_HOLD);
+	return sda;
+}
+
+static int
+i2c_get_byte(struct nouveau_i2c_port *port, u8 *byte, bool last)
+{
+	int i, bit;
+
+	*byte = 0;
+	for (i = 7; i >= 0; i--) {
+		bit = i2c_bitr(port);
+		if (bit < 0)
+			return bit;
+		*byte |= bit << i;
+	}
+
+	return i2c_bitw(port, last ? 1 : 0);
+}
+
+static int
+i2c_put_byte(struct nouveau_i2c_port *port, u8 byte)
+{
+	int i, ret;
+	for (i = 7; i >= 0; i--) {
+		ret = i2c_bitw(port, !!(byte & (1 << i)));
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = i2c_bitr(port);
+	if (ret == 1) /* nack */
+		ret = -EIO;
+	return ret;
+}
+
+static int
+i2c_addr(struct nouveau_i2c_port *port, struct i2c_msg *msg)
+{
+	u32 addr = msg->addr << 1;
+	if (msg->flags & I2C_M_RD)
+		addr |= 1;
+	return i2c_put_byte(port, addr);
+}
+
+static int
+i2c_bit_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	struct nouveau_i2c_port *port = (struct nouveau_i2c_port *)adap;
+	struct i2c_msg *msg = msgs;
+	int ret = 0, mcnt = num;
+
+	while (!ret && mcnt--) {
+		u8 remaining = msg->len;
+		u8 *ptr = msg->buf;
+
+		ret = i2c_start(port);
+		if (ret == 0)
+			ret = i2c_addr(port, msg);
+
+		if (msg->flags & I2C_M_RD) {
+			while (!ret && remaining--)
+				ret = i2c_get_byte(port, ptr++, !remaining);
+		} else {
+			while (!ret && remaining--)
+				ret = i2c_put_byte(port, *ptr++);
+		}
+
+		msg++;
+	}
+
+	i2c_stop(port);
+	return (ret < 0) ? ret : num;
+}
+#else
+static int
+i2c_bit_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	return -ENODEV;
+}
+#endif
+
+static u32
+i2c_bit_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+const struct i2c_algorithm nouveau_i2c_bit_algo = {
+	.master_xfer = i2c_bit_xfer,
+	.functionality = i2c_bit_func
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ibus/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/ibus/nvc0.c
new file mode 100644
index 00000000000..4e977ff27e4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ibus/nvc0.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/ibus.h>
+
+struct nvc0_ibus_priv {
+	struct nouveau_ibus base;
+};
+
+static void
+nvc0_ibus_intr_hub(struct nvc0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x122120 + (i * 0x0400));
+	u32 data = nv_rd32(priv, 0x122124 + (i * 0x0400));
+	u32 stat = nv_rd32(priv, 0x122128 + (i * 0x0400));
+	nv_error(priv, "HUB%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x122128 + (i * 0x0400), 0x00000200, 0x00000000);
+}
+
+static void
+nvc0_ibus_intr_rop(struct nvc0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x124120 + (i * 0x0400));
+	u32 data = nv_rd32(priv, 0x124124 + (i * 0x0400));
+	u32 stat = nv_rd32(priv, 0x124128 + (i * 0x0400));
+	nv_error(priv, "ROP%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x124128 + (i * 0x0400), 0x00000200, 0x00000000);
+}
+
+static void
+nvc0_ibus_intr_gpc(struct nvc0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x128120 + (i * 0x0400));
+	u32 data = nv_rd32(priv, 0x128124 + (i * 0x0400));
+	u32 stat = nv_rd32(priv, 0x128128 + (i * 0x0400));
+	nv_error(priv, "GPC%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x128128 + (i * 0x0400), 0x00000200, 0x00000000);
+}
+
+static void
+nvc0_ibus_intr(struct nouveau_subdev *subdev)
+{
+	struct nvc0_ibus_priv *priv = (void *)subdev;
+	u32 intr0 = nv_rd32(priv, 0x121c58);
+	u32 intr1 = nv_rd32(priv, 0x121c5c);
+	u32 hubnr = nv_rd32(priv, 0x121c70);
+	u32 ropnr = nv_rd32(priv, 0x121c74);
+	u32 gpcnr = nv_rd32(priv, 0x121c78);
+	u32 i;
+
+	for (i = 0; (intr0 & 0x0000ff00) && i < hubnr; i++) {
+		u32 stat = 0x00000100 << i;
+		if (intr0 & stat) {
+			nvc0_ibus_intr_hub(priv, i);
+			intr0 &= ~stat;
+		}
+	}
+
+	for (i = 0; (intr0 & 0xffff0000) && i < ropnr; i++) {
+		u32 stat = 0x00010000 << i;
+		if (intr0 & stat) {
+			nvc0_ibus_intr_rop(priv, i);
+			intr0 &= ~stat;
+		}
+	}
+
+	for (i = 0; intr1 && i < gpcnr; i++) {
+		u32 stat = 0x00000001 << i;
+		if (intr1 & stat) {
+			nvc0_ibus_intr_gpc(priv, i);
+			intr1 &= ~stat;
+		}
+	}
+}
+
+static int
+nvc0_ibus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nvc0_ibus_priv *priv;
+	int ret;
+
+	ret = nouveau_ibus_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nvc0_ibus_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_ibus_oclass = {
+	.handle = NV_SUBDEV(IBUS, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_ibus_ctor,
+		.dtor = _nouveau_ibus_dtor,
+		.init = _nouveau_ibus_init,
+		.fini = _nouveau_ibus_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ibus/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/ibus/nve0.c
new file mode 100644
index 00000000000..7120124dcea
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ibus/nve0.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/ibus.h>
+
+struct nve0_ibus_priv {
+	struct nouveau_ibus base;
+};
+
+static void
+nve0_ibus_intr_hub(struct nve0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x122120 + (i * 0x0800));
+	u32 data = nv_rd32(priv, 0x122124 + (i * 0x0800));
+	u32 stat = nv_rd32(priv, 0x122128 + (i * 0x0800));
+	nv_error(priv, "HUB%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x122128 + (i * 0x0800), 0x00000200, 0x00000000);
+}
+
+static void
+nve0_ibus_intr_rop(struct nve0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x124120 + (i * 0x0800));
+	u32 data = nv_rd32(priv, 0x124124 + (i * 0x0800));
+	u32 stat = nv_rd32(priv, 0x124128 + (i * 0x0800));
+	nv_error(priv, "ROP%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x124128 + (i * 0x0800), 0x00000200, 0x00000000);
+}
+
+static void
+nve0_ibus_intr_gpc(struct nve0_ibus_priv *priv, int i)
+{
+	u32 addr = nv_rd32(priv, 0x128120 + (i * 0x0800));
+	u32 data = nv_rd32(priv, 0x128124 + (i * 0x0800));
+	u32 stat = nv_rd32(priv, 0x128128 + (i * 0x0800));
+	nv_error(priv, "GPC%d: 0x%06x 0x%08x (0x%08x)\n", i, addr, data, stat);
+	nv_mask(priv, 0x128128 + (i * 0x0800), 0x00000200, 0x00000000);
+}
+
+static void
+nve0_ibus_intr(struct nouveau_subdev *subdev)
+{
+	struct nve0_ibus_priv *priv = (void *)subdev;
+	u32 intr0 = nv_rd32(priv, 0x120058);
+	u32 intr1 = nv_rd32(priv, 0x12005c);
+	u32 hubnr = nv_rd32(priv, 0x120070);
+	u32 ropnr = nv_rd32(priv, 0x120074);
+	u32 gpcnr = nv_rd32(priv, 0x120078);
+	u32 i;
+
+	for (i = 0; (intr0 & 0x0000ff00) && i < hubnr; i++) {
+		u32 stat = 0x00000100 << i;
+		if (intr0 & stat) {
+			nve0_ibus_intr_hub(priv, i);
+			intr0 &= ~stat;
+		}
+	}
+
+	for (i = 0; (intr0 & 0xffff0000) && i < ropnr; i++) {
+		u32 stat = 0x00010000 << i;
+		if (intr0 & stat) {
+			nve0_ibus_intr_rop(priv, i);
+			intr0 &= ~stat;
+		}
+	}
+
+	for (i = 0; intr1 && i < gpcnr; i++) {
+		u32 stat = 0x00000001 << i;
+		if (intr1 & stat) {
+			nve0_ibus_intr_gpc(priv, i);
+			intr1 &= ~stat;
+		}
+	}
+}
+
+static int
+nve0_ibus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nve0_ibus_priv *priv;
+	int ret;
+
+	ret = nouveau_ibus_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nve0_ibus_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nve0_ibus_oclass = {
+	.handle = NV_SUBDEV(IBUS, 0xe0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nve0_ibus_ctor,
+		.dtor = _nouveau_ibus_dtor,
+		.init = _nouveau_ibus_init,
+		.fini = _nouveau_ibus_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/core/subdev/instmem/base.c
new file mode 100644
index 00000000000..1188227ca6a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/base.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/instmem.h>
+
+int
+nouveau_instobj_create_(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass,
+			int length, void **pobject)
+{
+	struct nouveau_instmem *imem = (void *)engine;
+	struct nouveau_instobj *iobj;
+	int ret;
+
+	ret = nouveau_object_create_(parent, engine, oclass, NV_MEMOBJ_CLASS,
+				     length, pobject);
+	iobj = *pobject;
+	if (ret)
+		return ret;
+
+	list_add(&iobj->head, &imem->list);
+	return 0;
+}
+
+void
+nouveau_instobj_destroy(struct nouveau_instobj *iobj)
+{
+	if (iobj->head.prev)
+		list_del(&iobj->head);
+	return nouveau_object_destroy(&iobj->base);
+}
+
+void
+_nouveau_instobj_dtor(struct nouveau_object *object)
+{
+	struct nouveau_instobj *iobj = (void *)object;
+	return nouveau_instobj_destroy(iobj);
+}
+
+int
+nouveau_instmem_create_(struct nouveau_object *parent,
+			struct nouveau_object *engine,
+			struct nouveau_oclass *oclass,
+			int length, void **pobject)
+{
+	struct nouveau_instmem *imem;
+	int ret;
+
+	ret = nouveau_subdev_create_(parent, engine, oclass, 0,
+				     "INSTMEM", "instmem", length, pobject);
+	imem = *pobject;
+	if (ret)
+		return ret;
+
+	INIT_LIST_HEAD(&imem->list);
+	return 0;
+}
+
+int
+nouveau_instmem_init(struct nouveau_instmem *imem)
+{
+	struct nouveau_instobj *iobj;
+	int ret, i;
+
+	ret = nouveau_subdev_init(&imem->base);
+	if (ret)
+		return ret;
+
+	list_for_each_entry(iobj, &imem->list, head) {
+		if (iobj->suspend) {
+			for (i = 0; i < iobj->size; i += 4)
+				nv_wo32(iobj, i, iobj->suspend[i / 4]);
+			vfree(iobj->suspend);
+			iobj->suspend = NULL;
+		}
+	}
+
+	return 0;
+}
+
+int
+nouveau_instmem_fini(struct nouveau_instmem *imem, bool suspend)
+{
+	struct nouveau_instobj *iobj;
+	int i;
+
+	if (suspend) {
+		list_for_each_entry(iobj, &imem->list, head) {
+			iobj->suspend = vmalloc(iobj->size);
+			if (iobj->suspend) {
+				for (i = 0; i < iobj->size; i += 4)
+					iobj->suspend[i / 4] = nv_ro32(iobj, i);
+			} else
+				return -ENOMEM;
+		}
+	}
+
+	return nouveau_subdev_fini(&imem->base, suspend);
+}
+
+int
+_nouveau_instmem_init(struct nouveau_object *object)
+{
+	struct nouveau_instmem *imem = (void *)object;
+	return nouveau_instmem_init(imem);
+}
+
+int
+_nouveau_instmem_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nouveau_instmem *imem = (void *)object;
+	return nouveau_instmem_fini(imem, suspend);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.c
new file mode 100644
index 00000000000..ba4d28b5036
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.c
@@ -0,0 +1,198 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/fb.h>
+
+#include "nv04.h"
+
+static int
+nv04_instobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *priv = (void *)engine;
+	struct nv04_instobj_priv *node;
+	int ret, align;
+
+	align = (unsigned long)data;
+	if (!align)
+		align = 1;
+
+	ret = nouveau_instobj_create(parent, engine, oclass, &node);
+	*pobject = nv_object(node);
+	if (ret)
+		return ret;
+
+	ret = nouveau_mm_head(&priv->heap, 1, size, size, align, &node->mem);
+	if (ret)
+		return ret;
+
+	node->base.addr = node->mem->offset;
+	node->base.size = node->mem->length;
+	return 0;
+}
+
+static void
+nv04_instobj_dtor(struct nouveau_object *object)
+{
+	struct nv04_instmem_priv *priv = (void *)object->engine;
+	struct nv04_instobj_priv *node = (void *)object;
+	nouveau_mm_free(&priv->heap, &node->mem);
+	nouveau_instobj_destroy(&node->base);
+}
+
+static u32
+nv04_instobj_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nv04_instobj_priv *node = (void *)object;
+	return nv_ro32(object->engine, node->mem->offset + addr);
+}
+
+static void
+nv04_instobj_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nv04_instobj_priv *node = (void *)object;
+	nv_wo32(object->engine, node->mem->offset + addr, data);
+}
+
+static struct nouveau_oclass
+nv04_instobj_oclass = {
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_instobj_ctor,
+		.dtor = nv04_instobj_dtor,
+		.init = _nouveau_instobj_init,
+		.fini = _nouveau_instobj_fini,
+		.rd32 = nv04_instobj_rd32,
+		.wr32 = nv04_instobj_wr32,
+	},
+};
+
+int
+nv04_instmem_alloc(struct nouveau_instmem *imem, struct nouveau_object *parent,
+		   u32 size, u32 align, struct nouveau_object **pobject)
+{
+	struct nouveau_object *engine = nv_object(imem);
+	struct nv04_instmem_priv *priv = (void *)(imem);
+	int ret;
+
+	ret = nouveau_object_ctor(parent, engine, &nv04_instobj_oclass,
+				  (void *)(unsigned long)align, size, pobject);
+	if (ret)
+		return ret;
+
+	/* INSTMEM itself creates objects to reserve (and preserve across
+	 * suspend/resume) various fixed data locations, each one of these
+	 * takes a reference on INSTMEM itself, causing it to never be
+	 * freed.  We drop all the self-references here to avoid this.
+	 */
+	if (unlikely(!priv->created))
+		atomic_dec(&engine->refcount);
+
+	return 0;
+}
+
+static int
+nv04_instmem_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv04_instmem_priv *priv;
+	int ret;
+
+	ret = nouveau_instmem_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	/* PRAMIN aperture maps over the end of VRAM, reserve it */
+	priv->base.reserved = 512 * 1024;
+	priv->base.alloc    = nv04_instmem_alloc;
+
+	ret = nouveau_mm_init(&priv->heap, 0, priv->base.reserved, 1);
+	if (ret)
+		return ret;
+
+	/* 0x00000-0x10000: reserve for probable vbios image */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x10000, 0, 0, &priv->vbios);
+	if (ret)
+		return ret;
+
+	/* 0x10000-0x18000: reserve for RAMHT */
+	ret = nouveau_ramht_new(parent, NULL, 0x08000, 0, &priv->ramht);
+	if (ret)
+		return ret;
+
+	/* 0x18000-0x18800: reserve for RAMFC (enough for 32 nv30 channels) */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x00800, 0,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ramfc);
+	if (ret)
+		return ret;
+
+	/* 0x18800-0x18a00: reserve for RAMRO */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x00200, 0, 0, &priv->ramro);
+	if (ret)
+		return ret;
+
+	priv->created = true;
+	return 0;
+}
+
+void
+nv04_instmem_dtor(struct nouveau_object *object)
+{
+	struct nv04_instmem_priv *priv = (void *)object;
+	nouveau_gpuobj_ref(NULL, &priv->ramfc);
+	nouveau_gpuobj_ref(NULL, &priv->ramro);
+	nouveau_ramht_ref(NULL, &priv->ramht);
+	nouveau_gpuobj_ref(NULL, &priv->vbios);
+	nouveau_mm_fini(&priv->heap);
+	if (priv->iomem)
+		iounmap(priv->iomem);
+	nouveau_instmem_destroy(&priv->base);
+}
+
+static u32
+nv04_instmem_rd32(struct nouveau_object *object, u32 addr)
+{
+	return nv_rd32(object, 0x700000 + addr);
+}
+
+static void
+nv04_instmem_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	return nv_wr32(object, 0x700000 + addr, data);
+}
+
+struct nouveau_oclass
+nv04_instmem_oclass = {
+	.handle = NV_SUBDEV(INSTMEM, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_instmem_ctor,
+		.dtor = nv04_instmem_dtor,
+		.init = _nouveau_instmem_init,
+		.fini = _nouveau_instmem_fini,
+		.rd32 = nv04_instmem_rd32,
+		.wr32 = nv04_instmem_wr32,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.h
new file mode 100644
index 00000000000..7983d8d9b35
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv04.h
@@ -0,0 +1,39 @@
+#ifndef __NV04_INSTMEM_H__
+#define __NV04_INSTMEM_H__
+
+#include <core/gpuobj.h>
+#include <core/ramht.h>
+#include <core/mm.h>
+
+#include <subdev/instmem.h>
+
+struct nv04_instmem_priv {
+	struct nouveau_instmem base;
+	bool created;
+
+	void __iomem *iomem;
+	struct nouveau_mm heap;
+
+	struct nouveau_gpuobj *vbios;
+	struct nouveau_ramht  *ramht;
+	struct nouveau_gpuobj *ramro;
+	struct nouveau_gpuobj *ramfc;
+};
+
+static inline struct nv04_instmem_priv *
+nv04_instmem(void *obj)
+{
+	return (void *)nouveau_instmem(obj);
+}
+
+struct nv04_instobj_priv {
+	struct nouveau_instobj base;
+	struct nouveau_mm_node *mem;
+};
+
+void nv04_instmem_dtor(struct nouveau_object *);
+
+int nv04_instmem_alloc(struct nouveau_instmem *, struct nouveau_object *,
+		       u32 size, u32 align, struct nouveau_object **pobject);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c
new file mode 100644
index 00000000000..73c52ebd593
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+static inline int
+nv44_graph_class(struct nv04_instmem_priv *priv)
+{
+	if ((nv_device(priv)->chipset & 0xf0) == 0x60)
+		return 1;
+	return !(0x0baf & (1 << (nv_device(priv)->chipset & 0x0f)));
+}
+
+static int
+nv40_instmem_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct pci_dev *pdev = device->pdev;
+	struct nv04_instmem_priv *priv;
+	int ret, bar, vs;
+
+	ret = nouveau_instmem_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	/* map bar */
+	if (pci_resource_len(pdev, 2))
+		bar = 2;
+	else
+		bar = 3;
+
+	priv->iomem = ioremap(pci_resource_start(pdev, bar),
+			      pci_resource_len(pdev, bar));
+	if (!priv->iomem) {
+		nv_error(priv, "unable to map PRAMIN BAR\n");
+		return -EFAULT;
+	}
+
+	/* PRAMIN aperture maps over the end of vram, reserve enough space
+	 * to fit graphics contexts for every channel, the magics come
+	 * from engine/graph/nv40.c
+	 */
+	vs = hweight8((nv_rd32(priv, 0x001540) & 0x0000ff00) >> 8);
+	if      (device->chipset == 0x40) priv->base.reserved = 0x6aa0 * vs;
+	else if (device->chipset  < 0x43) priv->base.reserved = 0x4f00 * vs;
+	else if (nv44_graph_class(priv))  priv->base.reserved = 0x4980 * vs;
+	else				  priv->base.reserved = 0x4a40 * vs;
+	priv->base.reserved += 16 * 1024;
+	priv->base.reserved *= 32;		/* per-channel */
+	priv->base.reserved += 512 * 1024;	/* pci(e)gart table */
+	priv->base.reserved += 512 * 1024;	/* object storage */
+
+	priv->base.reserved = round_up(priv->base.reserved, 4096);
+	priv->base.alloc    = nv04_instmem_alloc;
+
+	ret = nouveau_mm_init(&priv->heap, 0, priv->base.reserved, 1);
+	if (ret)
+		return ret;
+
+	/* 0x00000-0x10000: reserve for probable vbios image */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x10000, 0, 0, &priv->vbios);
+	if (ret)
+		return ret;
+
+	/* 0x10000-0x18000: reserve for RAMHT */
+	ret = nouveau_ramht_new(parent, NULL, 0x08000, 0, &priv->ramht);
+	if (ret)
+		return ret;
+
+	/* 0x18000-0x18200: reserve for RAMRO
+	 * 0x18200-0x20000: padding
+	 */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x08000, 0, 0, &priv->ramro);
+	if (ret)
+		return ret;
+
+	/* 0x20000-0x21000: reserve for RAMFC
+	 * 0x21000-0x40000: padding and some unknown crap
+	 */
+	ret = nouveau_gpuobj_new(parent, NULL, 0x20000, 0,
+				 NVOBJ_FLAG_ZERO_ALLOC, &priv->ramfc);
+	if (ret)
+		return ret;
+
+	priv->created = true;
+	return 0;
+}
+
+static u32
+nv40_instmem_rd32(struct nouveau_object *object, u32 addr)
+{
+	struct nv04_instmem_priv *priv = (void *)object;
+	return ioread32_native(priv->iomem + addr);
+}
+
+static void
+nv40_instmem_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	struct nv04_instmem_priv *priv = (void *)object;
+	iowrite32_native(data, priv->iomem + addr);
+}
+
+struct nouveau_oclass
+nv40_instmem_oclass = {
+	.handle = NV_SUBDEV(INSTMEM, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_instmem_ctor,
+		.dtor = nv04_instmem_dtor,
+		.init = _nouveau_instmem_init,
+		.fini = _nouveau_instmem_fini,
+		.rd32 = nv40_instmem_rd32,
+		.wr32 = nv40_instmem_wr32,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv50.c
new file mode 100644
index 00000000000..27ef0891d10
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv50.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/instmem.h>
+#include <subdev/fb.h>
+
+#include <core/mm.h>
+
+struct nv50_instmem_priv {
+	struct nouveau_instmem base;
+	spinlock_t lock;
+	u64 addr;
+};
+
+struct nv50_instobj_priv {
+	struct nouveau_instobj base;
+	struct nouveau_mem *mem;
+};
+
+static int
+nv50_instobj_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nouveau_fb *pfb = nouveau_fb(parent);
+	struct nv50_instobj_priv *node;
+	u32 align = (unsigned long)data;
+	int ret;
+
+	size  = max((size  + 4095) & ~4095, (u32)4096);
+	align = max((align + 4095) & ~4095, (u32)4096);
+
+	ret = nouveau_instobj_create(parent, engine, oclass, &node);
+	*pobject = nv_object(node);
+	if (ret)
+		return ret;
+
+	ret = pfb->ram.get(pfb, size, align, 0, 0x800, &node->mem);
+	if (ret)
+		return ret;
+
+	node->base.addr = node->mem->offset;
+	node->base.size = node->mem->size << 12;
+	node->mem->page_shift = 12;
+	return 0;
+}
+
+static void
+nv50_instobj_dtor(struct nouveau_object *object)
+{
+	struct nv50_instobj_priv *node = (void *)object;
+	struct nouveau_fb *pfb = nouveau_fb(object);
+	pfb->ram.put(pfb, &node->mem);
+	nouveau_instobj_destroy(&node->base);
+}
+
+static u32
+nv50_instobj_rd32(struct nouveau_object *object, u32 offset)
+{
+	struct nv50_instmem_priv *priv = (void *)object->engine;
+	struct nv50_instobj_priv *node = (void *)object;
+	unsigned long flags;
+	u64 base = (node->mem->offset + offset) & 0xffffff00000ULL;
+	u64 addr = (node->mem->offset + offset) & 0x000000fffffULL;
+	u32 data;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	if (unlikely(priv->addr != base)) {
+		nv_wr32(priv, 0x001700, base >> 16);
+		priv->addr = base;
+	}
+	data = nv_rd32(priv, 0x700000 + addr);
+	spin_unlock_irqrestore(&priv->lock, flags);
+	return data;
+}
+
+static void
+nv50_instobj_wr32(struct nouveau_object *object, u32 offset, u32 data)
+{
+	struct nv50_instmem_priv *priv = (void *)object->engine;
+	struct nv50_instobj_priv *node = (void *)object;
+	unsigned long flags;
+	u64 base = (node->mem->offset + offset) & 0xffffff00000ULL;
+	u64 addr = (node->mem->offset + offset) & 0x000000fffffULL;
+
+	spin_lock_irqsave(&priv->lock, flags);
+	if (unlikely(priv->addr != base)) {
+		nv_wr32(priv, 0x001700, base >> 16);
+		priv->addr = base;
+	}
+	nv_wr32(priv, 0x700000 + addr, data);
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static struct nouveau_oclass
+nv50_instobj_oclass = {
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_instobj_ctor,
+		.dtor = nv50_instobj_dtor,
+		.init = _nouveau_instobj_init,
+		.fini = _nouveau_instobj_fini,
+		.rd32 = nv50_instobj_rd32,
+		.wr32 = nv50_instobj_wr32,
+	},
+};
+
+static int
+nv50_instmem_alloc(struct nouveau_instmem *imem, struct nouveau_object *parent,
+		   u32 size, u32 align, struct nouveau_object **pobject)
+{
+	struct nouveau_object *engine = nv_object(imem);
+	return nouveau_object_ctor(parent, engine, &nv50_instobj_oclass,
+				   (void *)(unsigned long)align, size, pobject);
+}
+
+static int
+nv50_instmem_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		  struct nouveau_oclass *oclass, void *data, u32 size,
+		  struct nouveau_object **pobject)
+{
+	struct nv50_instmem_priv *priv;
+	int ret;
+
+	ret = nouveau_instmem_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	spin_lock_init(&priv->lock);
+	priv->base.alloc = nv50_instmem_alloc;
+	return 0;
+}
+
+static int
+nv50_instmem_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv50_instmem_priv *priv = (void *)object;
+	priv->addr = ~0ULL;
+	return nouveau_instmem_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv50_instmem_oclass = {
+	.handle = NV_SUBDEV(INSTMEM, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_instmem_ctor,
+		.dtor = _nouveau_instmem_dtor,
+		.init = _nouveau_instmem_init,
+		.fini = nv50_instmem_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/ltcg/nvc0.c
new file mode 100644
index 00000000000..078a2b9d6bd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltcg/nvc0.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/ltcg.h>
+
+struct nvc0_ltcg_priv {
+	struct nouveau_ltcg base;
+	u32 subp_nr;
+};
+
+static void
+nvc0_ltcg_subp_isr(struct nvc0_ltcg_priv *priv, int unit, int subp)
+{
+	u32 subp_base = 0x141000 + (unit * 0x2000) + (subp * 0x400);
+	u32 stat = nv_rd32(priv, subp_base + 0x020);
+
+	if (stat) {
+		nv_info(priv, "LTC%d_LTS%d: 0x%08x\n", unit, subp, stat);
+		nv_wr32(priv, subp_base + 0x020, stat);
+	}
+}
+
+static void
+nvc0_ltcg_intr(struct nouveau_subdev *subdev)
+{
+	struct nvc0_ltcg_priv *priv = (void *)subdev;
+	u32 units;
+
+	units = nv_rd32(priv, 0x00017c);
+	while (units) {
+		u32 subp, unit = ffs(units) - 1;
+		for (subp = 0; subp < priv->subp_nr; subp++)
+			nvc0_ltcg_subp_isr(priv, unit, subp);
+		units &= ~(1 << unit);
+	}
+
+	/* we do something horribly wrong and upset PMFB a lot, so mask off
+	 * interrupts from it after the first one until it's fixed
+	 */
+	nv_mask(priv, 0x000640, 0x02000000, 0x00000000);
+}
+
+static int
+nvc0_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	       struct nouveau_oclass *oclass, void *data, u32 size,
+	       struct nouveau_object **pobject)
+{
+	struct nvc0_ltcg_priv *priv;
+	int ret;
+
+	ret = nouveau_ltcg_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->subp_nr = nv_rd32(priv, 0x17e8dc) >> 24;
+	nv_mask(priv, 0x17e820, 0x00100000, 0x00000000); /* INTR_EN &= ~0x10 */
+
+	nv_subdev(priv)->intr = nvc0_ltcg_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_ltcg_oclass = {
+	.handle = NV_SUBDEV(LTCG, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_ltcg_ctor,
+		.dtor = _nouveau_ltcg_dtor,
+		.init = _nouveau_ltcg_init,
+		.fini = _nouveau_ltcg_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
new file mode 100644
index 00000000000..de5721cfc4c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+void
+nouveau_mc_intr(struct nouveau_subdev *subdev)
+{
+	struct nouveau_mc *pmc = nouveau_mc(subdev);
+	const struct nouveau_mc_intr *map = pmc->intr_map;
+	struct nouveau_subdev *unit;
+	u32 stat;
+
+	stat = nv_rd32(pmc, 0x000100);
+	while (stat && map->stat) {
+		if (stat & map->stat) {
+			unit = nouveau_subdev(subdev, map->unit);
+			if (unit && unit->intr)
+				unit->intr(unit);
+			stat &= ~map->stat;
+		}
+		map++;
+	}
+
+	if (stat) {
+		nv_error(pmc, "unknown intr 0x%08x\n", stat);
+	}
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c
new file mode 100644
index 00000000000..23ebe477a6f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+struct nv04_mc_priv {
+	struct nouveau_mc base;
+};
+
+const struct nouveau_mc_intr
+nv04_mc_intr[] = {
+	{ 0x00000001, NVDEV_ENGINE_MPEG },	/* NV17- MPEG/ME */
+	{ 0x00000100, NVDEV_ENGINE_FIFO },
+	{ 0x00001000, NVDEV_ENGINE_GR },
+	{ 0x00020000, NVDEV_ENGINE_VP },	/* NV40- */
+	{ 0x00100000, NVDEV_SUBDEV_TIMER },
+	{ 0x01000000, NVDEV_ENGINE_DISP },	/* NV04- PCRTC0 */
+	{ 0x02000000, NVDEV_ENGINE_DISP },	/* NV11- PCRTC1 */
+	{ 0x10000000, NVDEV_SUBDEV_GPIO },	/* PBUS */
+	{ 0x80000000, NVDEV_ENGINE_SW },
+	{}
+};
+
+static int
+nv04_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv04_mc_priv *priv;
+	int ret;
+
+	ret = nouveau_mc_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nouveau_mc_intr;
+	priv->base.intr_map = nv04_mc_intr;
+	return 0;
+}
+
+int
+nv04_mc_init(struct nouveau_object *object)
+{
+	struct nv04_mc_priv *priv = (void *)object;
+
+	nv_wr32(priv, 0x000200, 0xffffffff); /* everything enabled */
+	nv_wr32(priv, 0x001850, 0x00000001); /* disable rom access */
+
+	return nouveau_mc_init(&priv->base);
+}
+
+struct nouveau_oclass
+nv04_mc_oclass = {
+	.handle = NV_SUBDEV(MC, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_mc_ctor,
+		.dtor = _nouveau_mc_dtor,
+		.init = nv04_mc_init,
+		.fini = _nouveau_mc_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c
new file mode 100644
index 00000000000..397d868359a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+struct nv44_mc_priv {
+	struct nouveau_mc base;
+};
+
+static int
+nv44_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv44_mc_priv *priv;
+	int ret;
+
+	ret = nouveau_mc_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nouveau_mc_intr;
+	priv->base.intr_map = nv04_mc_intr;
+	return 0;
+}
+
+static int
+nv44_mc_init(struct nouveau_object *object)
+{
+	struct nv44_mc_priv *priv = (void *)object;
+	u32 tmp = nv_rd32(priv, 0x10020c);
+
+	nv_wr32(priv, 0x000200, 0xffffffff); /* everything enabled */
+
+	nv_wr32(priv, 0x001700, tmp);
+	nv_wr32(priv, 0x001704, 0);
+	nv_wr32(priv, 0x001708, 0);
+	nv_wr32(priv, 0x00170c, tmp);
+
+	return nouveau_mc_init(&priv->base);
+}
+
+struct nouveau_oclass
+nv44_mc_oclass = {
+	.handle = NV_SUBDEV(MC, 0x44),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv44_mc_ctor,
+		.dtor = _nouveau_mc_dtor,
+		.init = nv44_mc_init,
+		.fini = _nouveau_mc_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c
new file mode 100644
index 00000000000..cedf33b0297
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+struct nv50_mc_priv {
+	struct nouveau_mc base;
+};
+
+static const struct nouveau_mc_intr
+nv50_mc_intr[] = {
+	{ 0x00000001, NVDEV_ENGINE_MPEG },
+	{ 0x00000100, NVDEV_ENGINE_FIFO },
+	{ 0x00001000, NVDEV_ENGINE_GR },
+	{ 0x00004000, NVDEV_ENGINE_CRYPT },	/* NV84- */
+	{ 0x00008000, NVDEV_ENGINE_BSP },	/* NV84- */
+	{ 0x00100000, NVDEV_SUBDEV_TIMER },
+	{ 0x00200000, NVDEV_SUBDEV_GPIO },
+	{ 0x04000000, NVDEV_ENGINE_DISP },
+	{ 0x80000000, NVDEV_ENGINE_SW },
+	{},
+};
+
+static int
+nv50_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv50_mc_priv *priv;
+	int ret;
+
+	ret = nouveau_mc_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nouveau_mc_intr;
+	priv->base.intr_map = nv50_mc_intr;
+	return 0;
+}
+
+int
+nv50_mc_init(struct nouveau_object *object)
+{
+	struct nv50_mc_priv *priv = (void *)object;
+	nv_wr32(priv, 0x000200, 0xffffffff); /* everything on */
+	return nouveau_mc_init(&priv->base);
+}
+
+struct nouveau_oclass
+nv50_mc_oclass = {
+	.handle = NV_SUBDEV(MC, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_mc_ctor,
+		.dtor = _nouveau_mc_dtor,
+		.init = nv50_mc_init,
+		.fini = _nouveau_mc_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c
new file mode 100644
index 00000000000..a001e4c4d38
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+struct nv98_mc_priv {
+	struct nouveau_mc base;
+};
+
+static const struct nouveau_mc_intr
+nv98_mc_intr[] = {
+	{ 0x00000001, NVDEV_ENGINE_PPP },
+	{ 0x00000100, NVDEV_ENGINE_FIFO },
+	{ 0x00001000, NVDEV_ENGINE_GR },
+	{ 0x00004000, NVDEV_ENGINE_CRYPT },	/* NV84:NVA3 */
+	{ 0x00008000, NVDEV_ENGINE_BSP },
+	{ 0x00100000, NVDEV_SUBDEV_TIMER },
+	{ 0x00200000, NVDEV_SUBDEV_GPIO },
+	{ 0x00400000, NVDEV_ENGINE_COPY0 },	/* NVA3-     */
+	{ 0x04000000, NVDEV_ENGINE_DISP },
+	{ 0x80000000, NVDEV_ENGINE_SW },
+	{},
+};
+
+static int
+nv98_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nv98_mc_priv *priv;
+	int ret;
+
+	ret = nouveau_mc_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nouveau_mc_intr;
+	priv->base.intr_map = nv98_mc_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nv98_mc_oclass = {
+	.handle = NV_SUBDEV(MC, 0x98),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv98_mc_ctor,
+		.dtor = _nouveau_mc_dtor,
+		.init = nv50_mc_init,
+		.fini = _nouveau_mc_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
new file mode 100644
index 00000000000..c2b81e30a17
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mc.h>
+
+struct nvc0_mc_priv {
+	struct nouveau_mc base;
+};
+
+static const struct nouveau_mc_intr
+nvc0_mc_intr[] = {
+	{ 0x00000001, NVDEV_ENGINE_PPP },
+	{ 0x00000020, NVDEV_ENGINE_COPY0 },
+	{ 0x00000040, NVDEV_ENGINE_COPY1 },
+	{ 0x00000100, NVDEV_ENGINE_FIFO },
+	{ 0x00001000, NVDEV_ENGINE_GR },
+	{ 0x00008000, NVDEV_ENGINE_BSP },
+	{ 0x00100000, NVDEV_SUBDEV_TIMER },
+	{ 0x00200000, NVDEV_SUBDEV_GPIO },
+	{ 0x02000000, NVDEV_SUBDEV_LTCG },
+	{ 0x04000000, NVDEV_ENGINE_DISP },
+	{ 0x40000000, NVDEV_SUBDEV_IBUS },
+	{ 0x80000000, NVDEV_ENGINE_SW },
+	{},
+};
+
+static int
+nvc0_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	     struct nouveau_oclass *oclass, void *data, u32 size,
+	     struct nouveau_object **pobject)
+{
+	struct nvc0_mc_priv *priv;
+	int ret;
+
+	ret = nouveau_mc_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	nv_subdev(priv)->intr = nouveau_mc_intr;
+	priv->base.intr_map = nvc0_mc_intr;
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_mc_oclass = {
+	.handle = NV_SUBDEV(MC, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_mc_ctor,
+		.dtor = _nouveau_mc_dtor,
+		.init = nv50_mc_init,
+		.fini = _nouveau_mc_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c b/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c
new file mode 100644
index 00000000000..93e3ddf7303
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c
@@ -0,0 +1,290 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/option.h>
+
+#include <subdev/i2c.h>
+#include <subdev/mxm.h>
+#include <subdev/bios.h>
+#include <subdev/bios/mxm.h>
+
+#include "mxms.h"
+
+static bool
+mxm_shadow_rom_fetch(struct nouveau_i2c_port *i2c, u8 addr,
+		     u8 offset, u8 size, u8 *data)
+{
+	struct i2c_msg msgs[] = {
+		{ .addr = addr, .flags = 0, .len = 1, .buf = &offset },
+		{ .addr = addr, .flags = I2C_M_RD, .len = size, .buf = data, },
+	};
+
+	return i2c_transfer(&i2c->adapter, msgs, 2) == 2;
+}
+
+static bool
+mxm_shadow_rom(struct nouveau_mxm *mxm, u8 version)
+{
+	struct nouveau_bios *bios = nouveau_bios(mxm);
+	struct nouveau_i2c *i2c = nouveau_i2c(mxm);
+	struct nouveau_i2c_port *port = NULL;
+	u8 i2cidx, mxms[6], addr, size;
+
+	i2cidx = mxm_ddc_map(bios, 1 /* LVDS_DDC */) & 0x0f;
+	if (i2cidx < 0x0f)
+		port = i2c->find(i2c, i2cidx);
+	if (!port)
+		return false;
+
+	addr = 0x54;
+	if (!mxm_shadow_rom_fetch(port, addr, 0, 6, mxms)) {
+		addr = 0x56;
+		if (!mxm_shadow_rom_fetch(port, addr, 0, 6, mxms))
+			return false;
+	}
+
+	mxm->mxms = mxms;
+	size = mxms_headerlen(mxm) + mxms_structlen(mxm);
+	mxm->mxms = kmalloc(size, GFP_KERNEL);
+
+	if (mxm->mxms &&
+	    mxm_shadow_rom_fetch(port, addr, 0, size, mxm->mxms))
+		return true;
+
+	kfree(mxm->mxms);
+	mxm->mxms = NULL;
+	return false;
+}
+
+#if defined(CONFIG_ACPI)
+static bool
+mxm_shadow_dsm(struct nouveau_mxm *mxm, u8 version)
+{
+	struct nouveau_device *device = nv_device(mxm);
+	static char muid[] = {
+		0x00, 0xA4, 0x04, 0x40, 0x7D, 0x91, 0xF2, 0x4C,
+		0xB8, 0x9C, 0x79, 0xB6, 0x2F, 0xD5, 0x56, 0x65
+	};
+	u32 mxms_args[] = { 0x00000000 };
+	union acpi_object args[4] = {
+		/* _DSM MUID */
+		{ .buffer.type = 3,
+		  .buffer.length = sizeof(muid),
+		  .buffer.pointer = muid,
+		},
+		/* spec says this can be zero to mean "highest revision", but
+		 * of course there's at least one bios out there which fails
+		 * unless you pass in exactly the version it supports..
+		 */
+		{ .integer.type = ACPI_TYPE_INTEGER,
+		  .integer.value = (version & 0xf0) << 4 | (version & 0x0f),
+		},
+		/* MXMS function */
+		{ .integer.type = ACPI_TYPE_INTEGER,
+		  .integer.value = 0x00000010,
+		},
+		/* Pointer to MXMS arguments */
+		{ .buffer.type = ACPI_TYPE_BUFFER,
+		  .buffer.length = sizeof(mxms_args),
+		  .buffer.pointer = (char *)mxms_args,
+		},
+	};
+	struct acpi_object_list list = { ARRAY_SIZE(args), args };
+	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_handle handle;
+	int ret;
+
+	handle = DEVICE_ACPI_HANDLE(&device->pdev->dev);
+	if (!handle)
+		return false;
+
+	ret = acpi_evaluate_object(handle, "_DSM", &list, &retn);
+	if (ret) {
+		nv_debug(mxm, "DSM MXMS failed: %d\n", ret);
+		return false;
+	}
+
+	obj = retn.pointer;
+	if (obj->type == ACPI_TYPE_BUFFER) {
+		mxm->mxms = kmemdup(obj->buffer.pointer,
+					 obj->buffer.length, GFP_KERNEL);
+	} else
+	if (obj->type == ACPI_TYPE_INTEGER) {
+		nv_debug(mxm, "DSM MXMS returned 0x%llx\n", obj->integer.value);
+	}
+
+	kfree(obj);
+	return mxm->mxms != NULL;
+}
+#endif
+
+#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
+
+#define WMI_WMMX_GUID "F6CB5C3C-9CAE-4EBD-B577-931EA32A2CC0"
+
+static u8
+wmi_wmmx_mxmi(struct nouveau_mxm *mxm, u8 version)
+{
+	u32 mxmi_args[] = { 0x494D584D /* MXMI */, version, 0 };
+	struct acpi_buffer args = { sizeof(mxmi_args), mxmi_args };
+	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_status status;
+
+	status = wmi_evaluate_method(WMI_WMMX_GUID, 0, 0, &args, &retn);
+	if (ACPI_FAILURE(status)) {
+		nv_debug(mxm, "WMMX MXMI returned %d\n", status);
+		return 0x00;
+	}
+
+	obj = retn.pointer;
+	if (obj->type == ACPI_TYPE_INTEGER) {
+		version = obj->integer.value;
+		nv_debug(mxm, "WMMX MXMI version %d.%d\n",
+			     (version >> 4), version & 0x0f);
+	} else {
+		version = 0;
+		nv_debug(mxm, "WMMX MXMI returned non-integer\n");
+	}
+
+	kfree(obj);
+	return version;
+}
+
+static bool
+mxm_shadow_wmi(struct nouveau_mxm *mxm, u8 version)
+{
+	u32 mxms_args[] = { 0x534D584D /* MXMS */, version, 0 };
+	struct acpi_buffer args = { sizeof(mxms_args), mxms_args };
+	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
+	union acpi_object *obj;
+	acpi_status status;
+
+	if (!wmi_has_guid(WMI_WMMX_GUID)) {
+		nv_debug(mxm, "WMMX GUID not found\n");
+		return false;
+	}
+
+	mxms_args[1] = wmi_wmmx_mxmi(mxm, 0x00);
+	if (!mxms_args[1])
+		mxms_args[1] = wmi_wmmx_mxmi(mxm, version);
+	if (!mxms_args[1])
+		return false;
+
+	status = wmi_evaluate_method(WMI_WMMX_GUID, 0, 0, &args, &retn);
+	if (ACPI_FAILURE(status)) {
+		nv_debug(mxm, "WMMX MXMS returned %d\n", status);
+		return false;
+	}
+
+	obj = retn.pointer;
+	if (obj->type == ACPI_TYPE_BUFFER) {
+		mxm->mxms = kmemdup(obj->buffer.pointer,
+					 obj->buffer.length, GFP_KERNEL);
+	}
+
+	kfree(obj);
+	return mxm->mxms != NULL;
+}
+#endif
+
+static struct mxm_shadow_h {
+	const char *name;
+	bool (*exec)(struct nouveau_mxm *, u8 version);
+} _mxm_shadow[] = {
+	{ "ROM", mxm_shadow_rom },
+#if defined(CONFIG_ACPI)
+	{ "DSM", mxm_shadow_dsm },
+#endif
+#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
+	{ "WMI", mxm_shadow_wmi },
+#endif
+	{}
+};
+
+static int
+mxm_shadow(struct nouveau_mxm *mxm, u8 version)
+{
+	struct mxm_shadow_h *shadow = _mxm_shadow;
+	do {
+		nv_debug(mxm, "checking %s\n", shadow->name);
+		if (shadow->exec(mxm, version)) {
+			if (mxms_valid(mxm))
+				return 0;
+			kfree(mxm->mxms);
+			mxm->mxms = NULL;
+		}
+	} while ((++shadow)->name);
+	return -ENOENT;
+}
+
+int
+nouveau_mxm_create_(struct nouveau_object *parent,
+		    struct nouveau_object *engine,
+		    struct nouveau_oclass *oclass, int length, void **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nouveau_mxm *mxm;
+	u8  ver, len;
+	u16 data;
+	int ret;
+
+	ret = nouveau_subdev_create_(parent, engine, oclass, 0, "MXM", "mxm",
+				     length, pobject);
+	mxm = *pobject;
+	if (ret)
+		return ret;
+
+	data = mxm_table(bios, &ver, &len);
+	if (!data || !(ver = nv_ro08(bios, data))) {
+		nv_info(mxm, "no VBIOS data, nothing to do\n");
+		return 0;
+	}
+
+	nv_info(mxm, "BIOS version %d.%d\n", ver >> 4, ver & 0x0f);
+
+	if (mxm_shadow(mxm, ver)) {
+		nv_info(mxm, "failed to locate valid SIS\n");
+#if 0
+		/* we should, perhaps, fall back to some kind of limited
+		 * mode here if the x86 vbios hasn't already done the
+		 * work for us (so we prevent loading with completely
+		 * whacked vbios tables).
+		 */
+		return -EINVAL;
+#else
+		return 0;
+#endif
+	}
+
+	nv_info(mxm, "MXMS Version %d.%d\n",
+		mxms_version(mxm) >> 8, mxms_version(mxm) & 0xff);
+	mxms_foreach(mxm, 0, NULL, NULL);
+
+	if (nouveau_boolopt(device->cfgopt, "NvMXMDCB", true))
+		mxm->action |= MXM_SANITISE_DCB;
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.c b/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.c
new file mode 100644
index 00000000000..839ca1edc13
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mxm.h>
+#include "mxms.h"
+
+#define ROM16(x) le16_to_cpu(*(u16 *)&(x))
+#define ROM32(x) le32_to_cpu(*(u32 *)&(x))
+
+static u8 *
+mxms_data(struct nouveau_mxm *mxm)
+{
+	return mxm->mxms;
+
+}
+
+u16
+mxms_version(struct nouveau_mxm *mxm)
+{
+	u8 *mxms = mxms_data(mxm);
+	u16 version = (mxms[4] << 8) | mxms[5];
+	switch (version ) {
+	case 0x0200:
+	case 0x0201:
+	case 0x0300:
+		return version;
+	default:
+		break;
+	}
+
+	nv_debug(mxm, "unknown version %d.%d\n", mxms[4], mxms[5]);
+	return 0x0000;
+}
+
+u16
+mxms_headerlen(struct nouveau_mxm *mxm)
+{
+	return 8;
+}
+
+u16
+mxms_structlen(struct nouveau_mxm *mxm)
+{
+	return *(u16 *)&mxms_data(mxm)[6];
+}
+
+bool
+mxms_checksum(struct nouveau_mxm *mxm)
+{
+	u16 size = mxms_headerlen(mxm) + mxms_structlen(mxm);
+	u8 *mxms = mxms_data(mxm), sum = 0;
+	while (size--)
+		sum += *mxms++;
+	if (sum) {
+		nv_debug(mxm, "checksum invalid\n");
+		return false;
+	}
+	return true;
+}
+
+bool
+mxms_valid(struct nouveau_mxm *mxm)
+{
+	u8 *mxms = mxms_data(mxm);
+	if (*(u32 *)mxms != 0x5f4d584d) {
+		nv_debug(mxm, "signature invalid\n");
+		return false;
+	}
+
+	if (!mxms_version(mxm) || !mxms_checksum(mxm))
+		return false;
+
+	return true;
+}
+
+bool
+mxms_foreach(struct nouveau_mxm *mxm, u8 types,
+	     bool (*exec)(struct nouveau_mxm *, u8 *, void *), void *info)
+{
+	u8 *mxms = mxms_data(mxm);
+	u8 *desc = mxms + mxms_headerlen(mxm);
+	u8 *fini = desc + mxms_structlen(mxm) - 1;
+	while (desc < fini) {
+		u8 type = desc[0] & 0x0f;
+		u8 headerlen = 0;
+		u8 recordlen = 0;
+		u8 entries = 0;
+
+		switch (type) {
+		case 0: /* Output Device Structure */
+			if (mxms_version(mxm) >= 0x0300)
+				headerlen = 8;
+			else
+				headerlen = 6;
+			break;
+		case 1: /* System Cooling Capability Structure */
+		case 2: /* Thermal Structure */
+		case 3: /* Input Power Structure */
+			headerlen = 4;
+			break;
+		case 4: /* GPIO Device Structure */
+			headerlen = 4;
+			recordlen = 2;
+			entries   = (ROM32(desc[0]) & 0x01f00000) >> 20;
+			break;
+		case 5: /* Vendor Specific Structure */
+			headerlen = 8;
+			break;
+		case 6: /* Backlight Control Structure */
+			if (mxms_version(mxm) >= 0x0300) {
+				headerlen = 4;
+				recordlen = 8;
+				entries   = (desc[1] & 0xf0) >> 4;
+			} else {
+				headerlen = 8;
+			}
+			break;
+		case 7: /* Fan Control Structure */
+			headerlen = 8;
+			recordlen = 4;
+			entries   = desc[1] & 0x07;
+			break;
+		default:
+			nv_debug(mxm, "unknown descriptor type %d\n", type);
+			return false;
+		}
+
+		if (nv_subdev(mxm)->debug >= NV_DBG_DEBUG && (exec == NULL)) {
+			static const char * mxms_desc_name[] = {
+				"ODS", "SCCS", "TS", "IPS",
+				"GSD", "VSS", "BCS", "FCS",
+			};
+			u8 *dump = desc;
+			int i, j;
+
+			nv_debug(mxm, "%4s: ", mxms_desc_name[type]);
+			for (j = headerlen - 1; j >= 0; j--)
+				printk("%02x", dump[j]);
+			printk("\n");
+			dump += headerlen;
+
+			for (i = 0; i < entries; i++, dump += recordlen) {
+				nv_debug(mxm, "      ");
+				for (j = recordlen - 1; j >= 0; j--)
+					printk("%02x", dump[j]);
+				printk("\n");
+			}
+		}
+
+		if (types & (1 << type)) {
+			if (!exec(mxm, desc, info))
+				return false;
+		}
+
+		desc += headerlen + (entries * recordlen);
+	}
+
+	return true;
+}
+
+void
+mxms_output_device(struct nouveau_mxm *mxm, u8 *pdata, struct mxms_odev *desc)
+{
+	u64 data = ROM32(pdata[0]);
+	if (mxms_version(mxm) >= 0x0300)
+		data |= (u64)ROM16(pdata[4]) << 32;
+
+	desc->outp_type = (data & 0x00000000000000f0ULL) >> 4;
+	desc->ddc_port  = (data & 0x0000000000000f00ULL) >> 8;
+	desc->conn_type = (data & 0x000000000001f000ULL) >> 12;
+	desc->dig_conn  = (data & 0x0000000000780000ULL) >> 19;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.h b/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.h
new file mode 100644
index 00000000000..5e0be0c591c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mxm/mxms.h
@@ -0,0 +1,22 @@
+#ifndef __NVMXM_MXMS_H__
+#define __NVMXM_MXMS_H__
+
+struct mxms_odev {
+	u8 outp_type;
+	u8 conn_type;
+	u8 ddc_port;
+	u8 dig_conn;
+};
+
+void mxms_output_device(struct nouveau_mxm *, u8 *, struct mxms_odev *);
+
+u16  mxms_version(struct nouveau_mxm *);
+u16  mxms_headerlen(struct nouveau_mxm *);
+u16  mxms_structlen(struct nouveau_mxm *);
+bool mxms_checksum(struct nouveau_mxm *);
+bool mxms_valid(struct nouveau_mxm *);
+
+bool mxms_foreach(struct nouveau_mxm *, u8,
+		  bool (*)(struct nouveau_mxm *, u8 *, void *), void *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mxm/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/mxm/nv50.c
new file mode 100644
index 00000000000..af129c2e811
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mxm/nv50.c
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2011 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/mxm.h>
+#include <subdev/bios.h>
+#include <subdev/bios/conn.h>
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/mxm.h>
+
+#include "mxms.h"
+
+struct nv50_mxm_priv {
+	struct nouveau_mxm base;
+};
+
+struct context {
+	u32 *outp;
+	struct mxms_odev desc;
+};
+
+static bool
+mxm_match_tmds_partner(struct nouveau_mxm *mxm, u8 *data, void *info)
+{
+	struct context *ctx = info;
+	struct mxms_odev desc;
+
+	mxms_output_device(mxm, data, &desc);
+	if (desc.outp_type == 2 &&
+	    desc.dig_conn == ctx->desc.dig_conn)
+		return false;
+	return true;
+}
+
+static bool
+mxm_match_dcb(struct nouveau_mxm *mxm, u8 *data, void *info)
+{
+	struct nouveau_bios *bios = nouveau_bios(mxm);
+	struct context *ctx = info;
+	u64 desc = *(u64 *)data;
+
+	mxms_output_device(mxm, data, &ctx->desc);
+
+	/* match dcb encoder type to mxm-ods device type */
+	if ((ctx->outp[0] & 0x0000000f) != ctx->desc.outp_type)
+		return true;
+
+	/* digital output, have some extra stuff to match here, there's a
+	 * table in the vbios that provides a mapping from the mxm digital
+	 * connection enum values to SOR/link
+	 */
+	if ((desc & 0x00000000000000f0) >= 0x20) {
+		/* check against sor index */
+		u8 link = mxm_sor_map(bios, ctx->desc.dig_conn);
+		if ((ctx->outp[0] & 0x0f000000) != (link & 0x0f) << 24)
+			return true;
+
+		/* check dcb entry has a compatible link field */
+		link = (link & 0x30) >> 4;
+		if ((link & ((ctx->outp[1] & 0x00000030) >> 4)) != link)
+			return true;
+	}
+
+	/* mark this descriptor accounted for by setting invalid device type,
+	 * except of course some manufactures don't follow specs properly and
+	 * we need to avoid killing off the TMDS function on DP connectors
+	 * if MXM-SIS is missing an entry for it.
+	 */
+	data[0] &= ~0xf0;
+	if (ctx->desc.outp_type == 6 && ctx->desc.conn_type == 6 &&
+	    mxms_foreach(mxm, 0x01, mxm_match_tmds_partner, ctx)) {
+		data[0] |= 0x20; /* modify descriptor to match TMDS now */
+	} else {
+		data[0] |= 0xf0;
+	}
+
+	return false;
+}
+
+static int
+mxm_dcb_sanitise_entry(struct nouveau_bios *bios, void *data, int idx, u16 pdcb)
+{
+	struct nouveau_mxm *mxm = nouveau_mxm(bios);
+	struct context ctx = { .outp = (u32 *)(bios->data + pdcb) };
+	u8 type, i2cidx, link, ver, len;
+	u8 *conn;
+
+	/* look for an output device structure that matches this dcb entry.
+	 * if one isn't found, disable it.
+	 */
+	if (mxms_foreach(mxm, 0x01, mxm_match_dcb, &ctx)) {
+		nv_debug(mxm, "disable %d: 0x%08x 0x%08x\n",
+			idx, ctx.outp[0], ctx.outp[1]);
+		ctx.outp[0] |= 0x0000000f;
+		return 0;
+	}
+
+	/* modify the output's ddc/aux port, there's a pointer to a table
+	 * with the mapping from mxm ddc/aux port to dcb i2c_index in the
+	 * vbios mxm table
+	 */
+	i2cidx = mxm_ddc_map(bios, ctx.desc.ddc_port);
+	if ((ctx.outp[0] & 0x0000000f) != DCB_OUTPUT_DP)
+		i2cidx = (i2cidx & 0x0f) << 4;
+	else
+		i2cidx = (i2cidx & 0xf0);
+
+	if (i2cidx != 0xf0) {
+		ctx.outp[0] &= ~0x000000f0;
+		ctx.outp[0] |= i2cidx;
+	}
+
+	/* override dcb sorconf.link, based on what mxm data says */
+	switch (ctx.desc.outp_type) {
+	case 0x00: /* Analog CRT */
+	case 0x01: /* Analog TV/HDTV */
+		break;
+	default:
+		link = mxm_sor_map(bios, ctx.desc.dig_conn) & 0x30;
+		ctx.outp[1] &= ~0x00000030;
+		ctx.outp[1] |= link;
+		break;
+	}
+
+	/* we may need to fixup various other vbios tables based on what
+	 * the descriptor says the connector type should be.
+	 *
+	 * in a lot of cases, the vbios tables will claim DVI-I is possible,
+	 * and the mxm data says the connector is really HDMI.  another
+	 * common example is DP->eDP.
+	 */
+	conn  = bios->data;
+	conn += dcb_conn(bios, (ctx.outp[0] & 0x0000f000) >> 12, &ver, &len);
+	type  = conn[0];
+	switch (ctx.desc.conn_type) {
+	case 0x01: /* LVDS */
+		ctx.outp[1] |= 0x00000004; /* use_power_scripts */
+		/* XXX: modify default link width in LVDS table */
+		break;
+	case 0x02: /* HDMI */
+		type = DCB_CONNECTOR_HDMI_1;
+		break;
+	case 0x03: /* DVI-D */
+		type = DCB_CONNECTOR_DVI_D;
+		break;
+	case 0x0e: /* eDP, falls through to DPint */
+		ctx.outp[1] |= 0x00010000;
+	case 0x07: /* DP internal, wtf is this?? HP8670w */
+		ctx.outp[1] |= 0x00000004; /* use_power_scripts? */
+		type = DCB_CONNECTOR_eDP;
+		break;
+	default:
+		break;
+	}
+
+	if (mxms_version(mxm) >= 0x0300)
+		conn[0] = type;
+
+	return 0;
+}
+
+static bool
+mxm_show_unmatched(struct nouveau_mxm *mxm, u8 *data, void *info)
+{
+	u64 desc = *(u64 *)data;
+	if ((desc & 0xf0) != 0xf0)
+	nv_info(mxm, "unmatched output device 0x%016llx\n", desc);
+	return true;
+}
+
+static void
+mxm_dcb_sanitise(struct nouveau_mxm *mxm)
+{
+	struct nouveau_bios *bios = nouveau_bios(mxm);
+	u8  ver, hdr, cnt, len;
+	u16 dcb = dcb_table(bios, &ver, &hdr, &cnt, &len);
+	if (dcb == 0x0000 || ver != 0x40) {
+		nv_debug(mxm, "unsupported DCB version\n");
+		return;
+	}
+
+	dcb_outp_foreach(bios, NULL, mxm_dcb_sanitise_entry);
+	mxms_foreach(mxm, 0x01, mxm_show_unmatched, NULL);
+}
+
+static int
+nv50_mxm_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+	      struct nouveau_oclass *oclass, void *data, u32 size,
+	      struct nouveau_object **pobject)
+{
+	struct nv50_mxm_priv *priv;
+	int ret;
+
+	ret = nouveau_mxm_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	if (priv->base.action & MXM_SANITISE_DCB)
+		mxm_dcb_sanitise(&priv->base);
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_mxm_oclass = {
+	.handle = NV_SUBDEV(MXM, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_mxm_ctor,
+		.dtor = _nouveau_mxm_dtor,
+		.init = _nouveau_mxm_init,
+		.fini = _nouveau_mxm_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/base.c b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
new file mode 100644
index 00000000000..1674c74a76c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2012 The Nouveau community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <core/object.h>
+#include <core/device.h>
+
+#include <subdev/bios.h>
+
+#include "priv.h"
+
+int
+nouveau_therm_attr_get(struct nouveau_therm *therm,
+		       enum nouveau_therm_attr_type type)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	switch (type) {
+	case NOUVEAU_THERM_ATTR_FAN_MIN_DUTY:
+		return priv->bios_fan.min_duty;
+	case NOUVEAU_THERM_ATTR_FAN_MAX_DUTY:
+		return priv->bios_fan.max_duty;
+	case NOUVEAU_THERM_ATTR_FAN_MODE:
+		return priv->fan.mode;
+	case NOUVEAU_THERM_ATTR_THRS_FAN_BOOST:
+		return priv->bios_sensor.thrs_fan_boost.temp;
+	case NOUVEAU_THERM_ATTR_THRS_FAN_BOOST_HYST:
+		return priv->bios_sensor.thrs_fan_boost.hysteresis;
+	case NOUVEAU_THERM_ATTR_THRS_DOWN_CLK:
+		return priv->bios_sensor.thrs_down_clock.temp;
+	case NOUVEAU_THERM_ATTR_THRS_DOWN_CLK_HYST:
+		return priv->bios_sensor.thrs_down_clock.hysteresis;
+	case NOUVEAU_THERM_ATTR_THRS_CRITICAL:
+		return priv->bios_sensor.thrs_critical.temp;
+	case NOUVEAU_THERM_ATTR_THRS_CRITICAL_HYST:
+		return priv->bios_sensor.thrs_critical.hysteresis;
+	case NOUVEAU_THERM_ATTR_THRS_SHUTDOWN:
+		return priv->bios_sensor.thrs_shutdown.temp;
+	case NOUVEAU_THERM_ATTR_THRS_SHUTDOWN_HYST:
+		return priv->bios_sensor.thrs_shutdown.hysteresis;
+	}
+
+	return -EINVAL;
+}
+
+int
+nouveau_therm_attr_set(struct nouveau_therm *therm,
+		       enum nouveau_therm_attr_type type, int value)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	switch (type) {
+	case NOUVEAU_THERM_ATTR_FAN_MIN_DUTY:
+		if (value < 0)
+			value = 0;
+		if (value > priv->bios_fan.max_duty)
+			value = priv->bios_fan.max_duty;
+		priv->bios_fan.min_duty = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_FAN_MAX_DUTY:
+		if (value < 0)
+			value = 0;
+		if (value < priv->bios_fan.min_duty)
+			value = priv->bios_fan.min_duty;
+		priv->bios_fan.max_duty = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_FAN_MODE:
+		return nouveau_therm_fan_set_mode(therm, value);
+	case NOUVEAU_THERM_ATTR_THRS_FAN_BOOST:
+		priv->bios_sensor.thrs_fan_boost.temp = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_FAN_BOOST_HYST:
+		priv->bios_sensor.thrs_fan_boost.hysteresis = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_DOWN_CLK:
+		priv->bios_sensor.thrs_down_clock.temp = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_DOWN_CLK_HYST:
+		priv->bios_sensor.thrs_down_clock.hysteresis = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_CRITICAL:
+		priv->bios_sensor.thrs_critical.temp = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_CRITICAL_HYST:
+		priv->bios_sensor.thrs_critical.hysteresis = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_SHUTDOWN:
+		priv->bios_sensor.thrs_shutdown.temp = value;
+		return 0;
+	case NOUVEAU_THERM_ATTR_THRS_SHUTDOWN_HYST:
+		priv->bios_sensor.thrs_shutdown.hysteresis = value;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+int
+nouveau_therm_init(struct nouveau_object *object)
+{
+	struct nouveau_therm *therm = (void *)object;
+	struct nouveau_therm_priv *priv = (void *)therm;
+	int ret;
+
+	ret = nouveau_subdev_init(&therm->base);
+	if (ret)
+		return ret;
+
+	if (priv->fan.percent >= 0)
+		therm->fan_set(therm, priv->fan.percent);
+
+	return 0;
+}
+
+int
+nouveau_therm_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nouveau_therm *therm = (void *)object;
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	priv->fan.percent = therm->fan_get(therm);
+
+	return nouveau_subdev_fini(&therm->base, suspend);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
new file mode 100644
index 00000000000..b29237970fa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ * 	    Martin Peres
+ */
+
+#include "priv.h"
+
+#include <core/object.h>
+#include <core/device.h>
+#include <subdev/gpio.h>
+#include <subdev/timer.h>
+
+int
+nouveau_therm_fan_get(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_gpio *gpio = nouveau_gpio(therm);
+	struct dcb_gpio_func func;
+	int card_type = nv_device(therm)->card_type;
+	u32 divs, duty;
+	int ret;
+
+	if (!priv->fan.pwm_get)
+		return -ENODEV;
+
+	ret = gpio->find(gpio, 0, DCB_GPIO_PWM_FAN, 0xff, &func);
+	if (ret == 0) {
+		ret = priv->fan.pwm_get(therm, func.line, &divs, &duty);
+		if (ret == 0 && divs) {
+			divs = max(divs, duty);
+			if (card_type <= NV_40 || (func.log[0] & 1))
+				duty = divs - duty;
+			return (duty * 100) / divs;
+		}
+
+		return gpio->get(gpio, 0, func.func, func.line) * 100;
+	}
+
+	return -ENODEV;
+}
+
+int
+nouveau_therm_fan_set(struct nouveau_therm *therm, int percent)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_gpio *gpio = nouveau_gpio(therm);
+	struct dcb_gpio_func func;
+	int card_type = nv_device(therm)->card_type;
+	u32 divs, duty;
+	int ret;
+
+	if (priv->fan.mode == FAN_CONTROL_NONE)
+		return -EINVAL;
+
+	if (!priv->fan.pwm_set)
+		return -ENODEV;
+
+	if (percent < priv->bios_fan.min_duty)
+		percent = priv->bios_fan.min_duty;
+	if (percent > priv->bios_fan.max_duty)
+		percent = priv->bios_fan.max_duty;
+
+	ret = gpio->find(gpio, 0, DCB_GPIO_PWM_FAN, 0xff, &func);
+	if (ret == 0) {
+		divs = priv->bios_perf_fan.pwm_divisor;
+		if (priv->bios_fan.pwm_freq) {
+			divs = 1;
+			if (priv->fan.pwm_clock)
+				divs = priv->fan.pwm_clock(therm);
+			divs /= priv->bios_fan.pwm_freq;
+		}
+
+		duty = ((divs * percent) + 99) / 100;
+		if (card_type <= NV_40 || (func.log[0] & 1))
+			duty = divs - duty;
+
+		ret = priv->fan.pwm_set(therm, func.line, divs, duty);
+		return ret;
+	}
+
+	return -ENODEV;
+}
+
+int
+nouveau_therm_fan_sense(struct nouveau_therm *therm)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(therm);
+	struct nouveau_gpio *gpio = nouveau_gpio(therm);
+	struct dcb_gpio_func func;
+	u32 cycles, cur, prev;
+	u64 start, end, tach;
+
+	if (gpio->find(gpio, 0, DCB_GPIO_FAN_SENSE, 0xff, &func))
+		return -ENODEV;
+
+	/* Time a complete rotation and extrapolate to RPM:
+	 * When the fan spins, it changes the value of GPIO FAN_SENSE.
+	 * We get 4 changes (0 -> 1 -> 0 -> 1) per complete rotation.
+	 */
+	start = ptimer->read(ptimer);
+	prev = gpio->get(gpio, 0, func.func, func.line);
+	cycles = 0;
+	do {
+		usleep_range(500, 1000); /* supports 0 < rpm < 7500 */
+
+		cur = gpio->get(gpio, 0, func.func, func.line);
+		if (prev != cur) {
+			if (!start)
+				start = ptimer->read(ptimer);
+			cycles++;
+			prev = cur;
+		}
+	} while (cycles < 5 && ptimer->read(ptimer) - start < 250000000);
+	end = ptimer->read(ptimer);
+
+	if (cycles == 5) {
+		tach = (u64)60000000000;
+		do_div(tach, (end - start));
+		return tach;
+	} else
+		return 0;
+}
+
+int
+nouveau_therm_fan_set_mode(struct nouveau_therm *therm,
+			   enum nouveau_therm_fan_mode mode)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	if (priv->fan.mode == mode)
+		return 0;
+
+	if (mode < FAN_CONTROL_NONE || mode >= FAN_CONTROL_NR)
+		return -EINVAL;
+
+	switch (mode)
+	{
+	case FAN_CONTROL_NONE:
+		nv_info(therm, "switch fan to no-control mode\n");
+		break;
+	case FAN_CONTROL_MANUAL:
+		nv_info(therm, "switch fan to manual mode\n");
+		break;
+	case FAN_CONTROL_NR:
+		break;
+	}
+
+	priv->fan.mode = mode;
+	return 0;
+}
+
+int
+nouveau_therm_fan_user_get(struct nouveau_therm *therm)
+{
+	return nouveau_therm_fan_get(therm);
+}
+
+int
+nouveau_therm_fan_user_set(struct nouveau_therm *therm, int percent)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	if (priv->fan.mode != FAN_CONTROL_MANUAL)
+		return -EINVAL;
+
+	return nouveau_therm_fan_set(therm, percent);
+}
+
+void
+nouveau_therm_fan_set_defaults(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	priv->bios_fan.pwm_freq = 0;
+	priv->bios_fan.min_duty = 0;
+	priv->bios_fan.max_duty = 100;
+}
+
+
+static void
+nouveau_therm_fan_safety_checks(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	if (priv->bios_fan.min_duty > 100)
+		priv->bios_fan.min_duty = 100;
+	if (priv->bios_fan.max_duty > 100)
+		priv->bios_fan.max_duty = 100;
+
+	if (priv->bios_fan.min_duty > priv->bios_fan.max_duty)
+		priv->bios_fan.min_duty = priv->bios_fan.max_duty;
+}
+
+int nouveau_fan_pwm_clock_dummy(struct nouveau_therm *therm)
+{
+	return 1;
+}
+
+int
+nouveau_therm_fan_ctor(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_bios *bios = nouveau_bios(therm);
+
+	nouveau_therm_fan_set_defaults(therm);
+	nvbios_perf_fan_parse(bios, &priv->bios_perf_fan);
+	if (nvbios_therm_fan_parse(bios, &priv->bios_fan))
+		nv_error(therm, "parsing the thermal table failed\n");
+	nouveau_therm_fan_safety_checks(therm);
+
+	nouveau_therm_fan_set_mode(therm, FAN_CONTROL_NONE);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c b/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c
new file mode 100644
index 00000000000..e512ff0aae6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2012 Nouveau community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include "priv.h"
+
+#include <subdev/i2c.h>
+#include <subdev/bios/extdev.h>
+
+static bool
+probe_monitoring_device(struct nouveau_i2c_port *i2c,
+			struct i2c_board_info *info)
+{
+	struct nouveau_therm_priv *priv = (void *)nouveau_therm(i2c->i2c);
+	struct i2c_client *client;
+
+	request_module("%s%s", I2C_MODULE_PREFIX, info->type);
+
+	client = i2c_new_device(&i2c->adapter, info);
+	if (!client)
+		return false;
+
+	if (!client->driver || client->driver->detect(client, info)) {
+		i2c_unregister_device(client);
+		return false;
+	}
+
+	nv_info(priv,
+		"Found an %s at address 0x%x (controlled by lm_sensors)\n",
+		info->type, info->addr);
+	priv->ic = client;
+
+	return true;
+}
+
+void
+nouveau_therm_ic_ctor(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_bios *bios = nouveau_bios(therm);
+	struct nouveau_i2c *i2c = nouveau_i2c(therm);
+	struct nvbios_extdev_func extdev_entry;
+	struct i2c_board_info info[] = {
+		{ I2C_BOARD_INFO("w83l785ts", 0x2d) },
+		{ I2C_BOARD_INFO("w83781d", 0x2d) },
+		{ I2C_BOARD_INFO("adt7473", 0x2e) },
+		{ I2C_BOARD_INFO("adt7473", 0x2d) },
+		{ I2C_BOARD_INFO("adt7473", 0x2c) },
+		{ I2C_BOARD_INFO("f75375", 0x2e) },
+		{ I2C_BOARD_INFO("lm99", 0x4c) },
+		{ I2C_BOARD_INFO("lm90", 0x4c) },
+		{ I2C_BOARD_INFO("lm90", 0x4d) },
+		{ I2C_BOARD_INFO("adm1021", 0x18) },
+		{ I2C_BOARD_INFO("adm1021", 0x19) },
+		{ I2C_BOARD_INFO("adm1021", 0x1a) },
+		{ I2C_BOARD_INFO("adm1021", 0x29) },
+		{ I2C_BOARD_INFO("adm1021", 0x2a) },
+		{ I2C_BOARD_INFO("adm1021", 0x2b) },
+		{ I2C_BOARD_INFO("adm1021", 0x4c) },
+		{ I2C_BOARD_INFO("adm1021", 0x4d) },
+		{ I2C_BOARD_INFO("adm1021", 0x4e) },
+		{ I2C_BOARD_INFO("lm63", 0x18) },
+		{ I2C_BOARD_INFO("lm63", 0x4e) },
+		{ }
+	};
+
+	if (!nvbios_extdev_find(bios, NVBIOS_EXTDEV_LM89, &extdev_entry)) {
+		struct i2c_board_info board[] = {
+			{ I2C_BOARD_INFO("lm90", extdev_entry.addr >> 1) },
+			{ }
+		};
+
+		i2c->identify(i2c, NV_I2C_DEFAULT(0), "monitoring device",
+				  board, probe_monitoring_device);
+		if (priv->ic)
+			return;
+	}
+
+	if (!nvbios_extdev_find(bios, NVBIOS_EXTDEV_ADT7473, &extdev_entry)) {
+		struct i2c_board_info board[] = {
+			{ I2C_BOARD_INFO("adt7473", extdev_entry.addr >> 1) },
+			{ }
+		};
+
+		i2c->identify(i2c, NV_I2C_DEFAULT(0), "monitoring device",
+				  board, probe_monitoring_device);
+		if (priv->ic)
+			return;
+	}
+
+	/* The vbios doesn't provide the address of an exisiting monitoring
+	   device. Let's try our static list.
+	 */
+	i2c->identify(i2c, NV_I2C_DEFAULT(0), "monitoring device", info,
+		      probe_monitoring_device);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nv40.c
new file mode 100644
index 00000000000..fcf2cfe731d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nv40.c
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ * 	    Martin Peres
+ */
+
+#include "priv.h"
+
+static int
+nv40_sensor_setup(struct nouveau_therm *therm)
+{
+	struct nouveau_device *device = nv_device(therm);
+
+	/* enable ADC readout and disable the ALARM threshold */
+	if (device->chipset >= 0x46) {
+		nv_mask(therm, 0x15b8, 0x80000000, 0);
+		nv_wr32(therm, 0x15b0, 0x80003fff);
+		return nv_rd32(therm, 0x15b4) & 0x3fff;
+	} else {
+		nv_wr32(therm, 0x15b0, 0xff);
+		return nv_rd32(therm, 0x15b4) & 0xff;
+	}
+}
+
+static int
+nv40_temp_get(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_device *device = nv_device(therm);
+	struct nvbios_therm_sensor *sensor = &priv->bios_sensor;
+	int core_temp;
+
+	if (device->chipset >= 0x46) {
+		nv_wr32(therm, 0x15b0, 0x80003fff);
+		core_temp = nv_rd32(therm, 0x15b4) & 0x3fff;
+	} else {
+		nv_wr32(therm, 0x15b0, 0xff);
+		core_temp = nv_rd32(therm, 0x15b4) & 0xff;
+	}
+
+	/* Setup the sensor if the temperature is 0 */
+	if (core_temp == 0)
+		core_temp = nv40_sensor_setup(therm);
+
+	if (sensor->slope_div == 0)
+		sensor->slope_div = 1;
+	if (sensor->offset_den == 0)
+		sensor->offset_den = 1;
+	if (sensor->slope_mult < 1)
+		sensor->slope_mult = 1;
+
+	core_temp = core_temp * sensor->slope_mult / sensor->slope_div;
+	core_temp = core_temp + sensor->offset_num / sensor->offset_den;
+	core_temp = core_temp + sensor->offset_constant - 8;
+
+	return core_temp;
+}
+
+int
+nv40_fan_pwm_get(struct nouveau_therm *therm, int line, u32 *divs, u32 *duty)
+{
+	if (line == 2) {
+		u32 reg = nv_rd32(therm, 0x0010f0);
+		if (reg & 0x80000000) {
+			*duty = (reg & 0x7fff0000) >> 16;
+			*divs = (reg & 0x00007fff);
+			return 0;
+		}
+	} else
+	if (line == 9) {
+		u32 reg = nv_rd32(therm, 0x0015f4);
+		if (reg & 0x80000000) {
+			*divs = nv_rd32(therm, 0x0015f8);
+			*duty = (reg & 0x7fffffff);
+			return 0;
+		}
+	} else {
+		nv_error(therm, "unknown pwm ctrl for gpio %d\n", line);
+		return -ENODEV;
+	}
+
+	return -EINVAL;
+}
+
+int
+nv40_fan_pwm_set(struct nouveau_therm *therm, int line, u32 divs, u32 duty)
+{
+	if (line == 2) {
+		nv_wr32(therm, 0x0010f0, 0x80000000 | (duty << 16) | divs);
+	} else
+	if (line == 9) {
+		nv_wr32(therm, 0x0015f8, divs);
+		nv_wr32(therm, 0x0015f4, duty | 0x80000000);
+	} else {
+		nv_error(therm, "unknown pwm ctrl for gpio %d\n", line);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int
+nv40_therm_ctor(struct nouveau_object *parent,
+		   struct nouveau_object *engine,
+		   struct nouveau_oclass *oclass, void *data, u32 size,
+		   struct nouveau_object **pobject)
+{
+	struct nouveau_therm_priv *priv;
+	struct nouveau_therm *therm;
+	int ret;
+
+	ret = nouveau_therm_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	therm = (void *) priv;
+	if (ret)
+		return ret;
+
+	nouveau_therm_ic_ctor(therm);
+	nouveau_therm_sensor_ctor(therm);
+	nouveau_therm_fan_ctor(therm);
+
+	priv->fan.pwm_get = nv40_fan_pwm_get;
+	priv->fan.pwm_set = nv40_fan_pwm_set;
+
+	therm->temp_get = nv40_temp_get;
+	therm->fan_get = nouveau_therm_fan_user_get;
+	therm->fan_set = nouveau_therm_fan_user_set;
+	therm->fan_sense = nouveau_therm_fan_sense;
+	therm->attr_get = nouveau_therm_attr_get;
+	therm->attr_set = nouveau_therm_attr_set;
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv40_therm_oclass = {
+	.handle = NV_SUBDEV(THERM, 0x40),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv40_therm_ctor,
+		.dtor = _nouveau_therm_dtor,
+		.init = nouveau_therm_init,
+		.fini = nouveau_therm_fini,
+	},
+};
\ No newline at end of file
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c
new file mode 100644
index 00000000000..f87a7a3eb4e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ * 	    Martin Peres
+ */
+
+#include "priv.h"
+
+static int
+pwm_info(struct nouveau_therm *therm, int *line, int *ctrl, int *indx)
+{
+	if (*line == 0x04) {
+		*ctrl = 0x00e100;
+		*line = 4;
+		*indx = 0;
+	} else
+	if (*line == 0x09) {
+		*ctrl = 0x00e100;
+		*line = 9;
+		*indx = 1;
+	} else
+	if (*line == 0x10) {
+		*ctrl = 0x00e28c;
+		*line = 0;
+		*indx = 0;
+	} else {
+		nv_error(therm, "unknown pwm ctrl for gpio %d\n", *line);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+int
+nv50_fan_pwm_get(struct nouveau_therm *therm, int line, u32 *divs, u32 *duty)
+{
+	int ctrl, id, ret = pwm_info(therm, &line, &ctrl, &id);
+	if (ret)
+		return ret;
+
+	if (nv_rd32(therm, ctrl) & (1 << line)) {
+		*divs = nv_rd32(therm, 0x00e114 + (id * 8));
+		*duty = nv_rd32(therm, 0x00e118 + (id * 8));
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+int
+nv50_fan_pwm_set(struct nouveau_therm *therm, int line, u32 divs, u32 duty)
+{
+	int ctrl, id, ret = pwm_info(therm, &line, &ctrl, &id);
+	if (ret)
+		return ret;
+
+	nv_mask(therm, ctrl, 0x00010001 << line, 0x00000001 << line);
+	nv_wr32(therm, 0x00e114 + (id * 8), divs);
+	nv_wr32(therm, 0x00e118 + (id * 8), duty | 0x80000000);
+	return 0;
+}
+
+int
+nv50_fan_pwm_clock(struct nouveau_therm *therm)
+{
+	int chipset = nv_device(therm)->chipset;
+	int crystal = nv_device(therm)->crystal;
+	int pwm_clock;
+
+	/* determine the PWM source clock */
+	if (chipset > 0x50 && chipset < 0x94) {
+		u8 pwm_div = nv_rd32(therm, 0x410c);
+		if (nv_rd32(therm, 0xc040) & 0x800000) {
+			/* Use the HOST clock (100 MHz)
+			* Where does this constant(2.4) comes from? */
+			pwm_clock = (100000000 >> pwm_div) / 10 / 24;
+		} else {
+			/* Where does this constant(20) comes from? */
+			pwm_clock = (crystal * 1000) >> pwm_div;
+			pwm_clock /= 20;
+		}
+	} else {
+		pwm_clock = (crystal * 1000) / 20;
+	}
+
+	return pwm_clock;
+}
+
+int
+nv50_temp_get(struct nouveau_therm *therm)
+{
+	return nv_rd32(therm, 0x20400);
+}
+
+static int
+nv50_therm_ctor(struct nouveau_object *parent,
+		   struct nouveau_object *engine,
+		   struct nouveau_oclass *oclass, void *data, u32 size,
+		   struct nouveau_object **pobject)
+{
+	struct nouveau_therm_priv *priv;
+	struct nouveau_therm *therm;
+	int ret;
+
+	ret = nouveau_therm_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	therm = (void *) priv;
+	if (ret)
+		return ret;
+
+	nouveau_therm_ic_ctor(therm);
+	nouveau_therm_sensor_ctor(therm);
+	nouveau_therm_fan_ctor(therm);
+
+	priv->fan.pwm_get = nv50_fan_pwm_get;
+	priv->fan.pwm_set = nv50_fan_pwm_set;
+	priv->fan.pwm_clock = nv50_fan_pwm_clock;
+
+	therm->temp_get = nv50_temp_get;
+	therm->fan_get = nouveau_therm_fan_user_get;
+	therm->fan_set = nouveau_therm_fan_user_set;
+	therm->fan_sense = nouveau_therm_fan_sense;
+	therm->attr_get = nouveau_therm_attr_get;
+	therm->attr_set = nouveau_therm_attr_set;
+
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_therm_oclass = {
+	.handle = NV_SUBDEV(THERM, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_therm_ctor,
+		.dtor = _nouveau_therm_dtor,
+		.init = nouveau_therm_init,
+		.fini = nouveau_therm_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
new file mode 100644
index 00000000000..1c3cd6abc36
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2012 The Nouveau community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/therm.h>
+
+#include <subdev/bios/extdev.h>
+#include <subdev/bios/perf.h>
+#include <subdev/bios/therm.h>
+
+struct nouveau_therm_priv {
+	struct nouveau_therm base;
+
+	/* bios */
+	struct nvbios_therm_sensor bios_sensor;
+	struct nvbios_therm_fan bios_fan;
+	struct nvbios_perf_fan bios_perf_fan;
+
+	/* fan priv */
+	struct {
+		enum nouveau_therm_fan_mode mode;
+		int percent;
+
+		int (*pwm_get)(struct nouveau_therm *, int line, u32*, u32*);
+		int (*pwm_set)(struct nouveau_therm *, int line, u32, u32);
+		int (*pwm_clock)(struct nouveau_therm *);
+	} fan;
+
+	/* ic */
+	struct i2c_client *ic;
+};
+
+int nouveau_therm_init(struct nouveau_object *object);
+int nouveau_therm_fini(struct nouveau_object *object, bool suspend);
+int nouveau_therm_attr_get(struct nouveau_therm *therm,
+		       enum nouveau_therm_attr_type type);
+int nouveau_therm_attr_set(struct nouveau_therm *therm,
+		       enum nouveau_therm_attr_type type, int value);
+
+void nouveau_therm_ic_ctor(struct nouveau_therm *therm);
+
+int nouveau_therm_sensor_ctor(struct nouveau_therm *therm);
+
+int nouveau_therm_fan_ctor(struct nouveau_therm *therm);
+int nouveau_therm_fan_get(struct nouveau_therm *therm);
+int nouveau_therm_fan_set(struct nouveau_therm *therm, int percent);
+int nouveau_therm_fan_user_get(struct nouveau_therm *therm);
+int nouveau_therm_fan_user_set(struct nouveau_therm *therm, int percent);
+int nouveau_therm_fan_set_mode(struct nouveau_therm *therm,
+			   enum nouveau_therm_fan_mode mode);
+
+
+int nouveau_therm_fan_sense(struct nouveau_therm *therm);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c b/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c
new file mode 100644
index 00000000000..204282301fb
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2012 The Nouveau community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include "priv.h"
+
+#include <core/object.h>
+#include <core/device.h>
+
+#include <subdev/bios.h>
+
+static void
+nouveau_therm_temp_set_defaults(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	priv->bios_sensor.slope_mult = 1;
+	priv->bios_sensor.slope_div = 1;
+	priv->bios_sensor.offset_num = 0;
+	priv->bios_sensor.offset_den = 1;
+	priv->bios_sensor.offset_constant = 0;
+
+	priv->bios_sensor.thrs_fan_boost.temp = 90;
+	priv->bios_sensor.thrs_fan_boost.hysteresis = 3;
+
+	priv->bios_sensor.thrs_down_clock.temp = 95;
+	priv->bios_sensor.thrs_down_clock.hysteresis = 3;
+
+	priv->bios_sensor.thrs_critical.temp = 105;
+	priv->bios_sensor.thrs_critical.hysteresis = 5;
+
+	priv->bios_sensor.thrs_shutdown.temp = 135;
+	priv->bios_sensor.thrs_shutdown.hysteresis = 5; /*not that it matters */
+}
+
+
+static void
+nouveau_therm_temp_safety_checks(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+
+	if (!priv->bios_sensor.slope_div)
+		priv->bios_sensor.slope_div = 1;
+	if (!priv->bios_sensor.offset_den)
+		priv->bios_sensor.offset_den = 1;
+}
+
+int
+nouveau_therm_sensor_ctor(struct nouveau_therm *therm)
+{
+	struct nouveau_therm_priv *priv = (void *)therm;
+	struct nouveau_bios *bios = nouveau_bios(therm);
+
+	nouveau_therm_temp_set_defaults(therm);
+	if (nvbios_therm_sensor_parse(bios, NVBIOS_THERM_DOMAIN_CORE,
+				      &priv->bios_sensor))
+		nv_error(therm, "nvbios_therm_sensor_parse failed\n");
+	nouveau_therm_temp_safety_checks(therm);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/base.c b/drivers/gpu/drm/nouveau/core/subdev/timer/base.c
new file mode 100644
index 00000000000..5d417cc9949
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/base.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "subdev/timer.h"
+
+bool
+nouveau_timer_wait_eq(void *obj, u64 nsec, u32 addr, u32 mask, u32 data)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(obj);
+	u64 time0;
+
+	time0 = ptimer->read(ptimer);
+	do {
+		if (nv_iclass(obj, NV_SUBDEV_CLASS)) {
+			if ((nv_rd32(obj, addr) & mask) == data)
+				return true;
+		} else {
+			if ((nv_ro32(obj, addr) & mask) == data)
+				return true;
+		}
+	} while (ptimer->read(ptimer) - time0 < nsec);
+
+	return false;
+}
+
+bool
+nouveau_timer_wait_ne(void *obj, u64 nsec, u32 addr, u32 mask, u32 data)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(obj);
+	u64 time0;
+
+	time0 = ptimer->read(ptimer);
+	do {
+		if (nv_iclass(obj, NV_SUBDEV_CLASS)) {
+			if ((nv_rd32(obj, addr) & mask) != data)
+				return true;
+		} else {
+			if ((nv_ro32(obj, addr) & mask) != data)
+				return true;
+		}
+	} while (ptimer->read(ptimer) - time0 < nsec);
+
+	return false;
+}
+
+bool
+nouveau_timer_wait_cb(void *obj, u64 nsec, bool (*func)(void *), void *data)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(obj);
+	u64 time0;
+
+	time0 = ptimer->read(ptimer);
+	do {
+		if (func(data) == true)
+			return true;
+	} while (ptimer->read(ptimer) - time0 < nsec);
+
+	return false;
+}
+
+void
+nouveau_timer_alarm(void *obj, u32 nsec, struct nouveau_alarm *alarm)
+{
+	struct nouveau_timer *ptimer = nouveau_timer(obj);
+	ptimer->alarm(ptimer, nsec, alarm);
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
new file mode 100644
index 00000000000..49976be4d73
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/timer.h>
+
+#define NV04_PTIMER_INTR_0      0x009100
+#define NV04_PTIMER_INTR_EN_0   0x009140
+#define NV04_PTIMER_NUMERATOR   0x009200
+#define NV04_PTIMER_DENOMINATOR 0x009210
+#define NV04_PTIMER_TIME_0      0x009400
+#define NV04_PTIMER_TIME_1      0x009410
+#define NV04_PTIMER_ALARM_0     0x009420
+
+struct nv04_timer_priv {
+	struct nouveau_timer base;
+	struct list_head alarms;
+	spinlock_t lock;
+};
+
+static u64
+nv04_timer_read(struct nouveau_timer *ptimer)
+{
+	struct nv04_timer_priv *priv = (void *)ptimer;
+	u32 hi, lo;
+
+	do {
+		hi = nv_rd32(priv, NV04_PTIMER_TIME_1);
+		lo = nv_rd32(priv, NV04_PTIMER_TIME_0);
+	} while (hi != nv_rd32(priv, NV04_PTIMER_TIME_1));
+
+	return ((u64)hi << 32 | lo);
+}
+
+static void
+nv04_timer_alarm_trigger(struct nouveau_timer *ptimer)
+{
+	struct nv04_timer_priv *priv = (void *)ptimer;
+	struct nouveau_alarm *alarm, *atemp;
+	unsigned long flags;
+	LIST_HEAD(exec);
+
+	/* move any due alarms off the pending list */
+	spin_lock_irqsave(&priv->lock, flags);
+	list_for_each_entry_safe(alarm, atemp, &priv->alarms, head) {
+		if (alarm->timestamp <= ptimer->read(ptimer))
+			list_move_tail(&alarm->head, &exec);
+	}
+
+	/* reschedule interrupt for next alarm time */
+	if (!list_empty(&priv->alarms)) {
+		alarm = list_first_entry(&priv->alarms, typeof(*alarm), head);
+		nv_wr32(priv, NV04_PTIMER_ALARM_0, alarm->timestamp);
+		nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000001);
+	} else {
+		nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000000);
+	}
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	/* execute any pending alarm handlers */
+	list_for_each_entry_safe(alarm, atemp, &exec, head) {
+		list_del(&alarm->head);
+		alarm->func(alarm);
+	}
+}
+
+static void
+nv04_timer_alarm(struct nouveau_timer *ptimer, u32 time,
+		 struct nouveau_alarm *alarm)
+{
+	struct nv04_timer_priv *priv = (void *)ptimer;
+	struct nouveau_alarm *list;
+	unsigned long flags;
+
+	alarm->timestamp = ptimer->read(ptimer) + time;
+
+	/* append new alarm to list, in soonest-alarm-first order */
+	spin_lock_irqsave(&priv->lock, flags);
+	list_for_each_entry(list, &priv->alarms, head) {
+		if (list->timestamp > alarm->timestamp)
+			break;
+	}
+	list_add_tail(&alarm->head, &list->head);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	/* process pending alarms */
+	nv04_timer_alarm_trigger(ptimer);
+}
+
+static void
+nv04_timer_intr(struct nouveau_subdev *subdev)
+{
+	struct nv04_timer_priv *priv = (void *)subdev;
+	u32 stat = nv_rd32(priv, NV04_PTIMER_INTR_0);
+
+	if (stat & 0x00000001) {
+		nv04_timer_alarm_trigger(&priv->base);
+		nv_wr32(priv, NV04_PTIMER_INTR_0, 0x00000001);
+		stat &= ~0x00000001;
+	}
+
+	if (stat) {
+		nv_error(priv, "unknown stat 0x%08x\n", stat);
+		nv_wr32(priv, NV04_PTIMER_INTR_0, stat);
+	}
+}
+
+static int
+nv04_timer_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv04_timer_priv *priv;
+	int ret;
+
+	ret = nouveau_timer_create(parent, engine, oclass, &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.base.intr = nv04_timer_intr;
+	priv->base.read = nv04_timer_read;
+	priv->base.alarm = nv04_timer_alarm;
+
+	INIT_LIST_HEAD(&priv->alarms);
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+static void
+nv04_timer_dtor(struct nouveau_object *object)
+{
+	struct nv04_timer_priv *priv = (void *)object;
+	return nouveau_timer_destroy(&priv->base);
+}
+
+static int
+nv04_timer_init(struct nouveau_object *object)
+{
+	struct nouveau_device *device = nv_device(object);
+	struct nv04_timer_priv *priv = (void *)object;
+	u32 m = 1, f, n, d;
+	int ret;
+
+	ret = nouveau_timer_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* aim for 31.25MHz, which gives us nanosecond timestamps */
+	d = 1000000 / 32;
+
+	/* determine base clock for timer source */
+#if 0 /*XXX*/
+	if (device->chipset < 0x40) {
+		n = nouveau_hw_get_clock(device, PLL_CORE);
+	} else
+#endif
+	if (device->chipset <= 0x40) {
+		/*XXX: figure this out */
+		f = -1;
+		n = 0;
+	} else {
+		f = device->crystal;
+		n = f;
+		while (n < (d * 2)) {
+			n += (n / m);
+			m++;
+		}
+
+		nv_wr32(priv, 0x009220, m - 1);
+	}
+
+	if (!n) {
+		nv_warn(priv, "unknown input clock freq\n");
+		if (!nv_rd32(priv, NV04_PTIMER_NUMERATOR) ||
+		    !nv_rd32(priv, NV04_PTIMER_DENOMINATOR)) {
+			nv_wr32(priv, NV04_PTIMER_NUMERATOR, 1);
+			nv_wr32(priv, NV04_PTIMER_DENOMINATOR, 1);
+		}
+		return 0;
+	}
+
+	/* reduce ratio to acceptable values */
+	while (((n % 5) == 0) && ((d % 5) == 0)) {
+		n /= 5;
+		d /= 5;
+	}
+
+	while (((n % 2) == 0) && ((d % 2) == 0)) {
+		n /= 2;
+		d /= 2;
+	}
+
+	while (n > 0xffff || d > 0xffff) {
+		n >>= 1;
+		d >>= 1;
+	}
+
+	nv_debug(priv, "input frequency : %dHz\n", f);
+	nv_debug(priv, "input multiplier: %d\n", m);
+	nv_debug(priv, "numerator       : 0x%08x\n", n);
+	nv_debug(priv, "denominator     : 0x%08x\n", d);
+	nv_debug(priv, "timer frequency : %dHz\n", (f * m) * d / n);
+
+	nv_wr32(priv, NV04_PTIMER_NUMERATOR, n);
+	nv_wr32(priv, NV04_PTIMER_DENOMINATOR, d);
+	nv_wr32(priv, NV04_PTIMER_INTR_0, 0xffffffff);
+	nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000000);
+	return 0;
+}
+
+static int
+nv04_timer_fini(struct nouveau_object *object, bool suspend)
+{
+	struct nv04_timer_priv *priv = (void *)object;
+	nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000000);
+	return nouveau_timer_fini(&priv->base, suspend);
+}
+
+struct nouveau_oclass
+nv04_timer_oclass = {
+	.handle = NV_SUBDEV(TIMER, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_timer_ctor,
+		.dtor = nv04_timer_dtor,
+		.init = nv04_timer_init,
+		.fini = nv04_timer_fini,
+	}
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_vm.c b/drivers/gpu/drm/nouveau/core/subdev/vm/base.c
index 11edd5e91a0..082c11b75ac 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vm.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/base.c
@@ -22,22 +22,24 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nouveau_vm.h"
+#include <core/gpuobj.h>
+#include <core/mm.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
 
 void
 nouveau_vm_map_at(struct nouveau_vma *vma, u64 delta, struct nouveau_mem *node)
 {
 	struct nouveau_vm *vm = vma->vm;
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	struct nouveau_mm_node *r;
-	int big = vma->node->type != vm->spg_shift;
+	int big = vma->node->type != vmm->spg_shift;
 	u32 offset = vma->node->offset + (delta >> 12);
 	u32 bits = vma->node->type - 12;
-	u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
-	u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
-	u32 max  = 1 << (vm->pgt_bits - bits);
+	u32 pde  = (offset >> vmm->pgt_bits) - vm->fpde;
+	u32 pte  = (offset & ((1 << vmm->pgt_bits) - 1)) >> bits;
+	u32 max  = 1 << (vmm->pgt_bits - bits);
 	u32 end, len;
 
 	delta = 0;
@@ -53,7 +55,7 @@ nouveau_vm_map_at(struct nouveau_vma *vma, u64 delta, struct nouveau_mem *node)
 				end = max;
 			len = end - pte;
 
-			vm->map(vma, pgt, node, pte, len, phys, delta);
+			vmm->map(vma, pgt, node, pte, len, phys, delta);
 
 			num -= len;
 			pte += len;
@@ -67,7 +69,7 @@ nouveau_vm_map_at(struct nouveau_vma *vma, u64 delta, struct nouveau_mem *node)
 		}
 	}
 
-	vm->flush(vm);
+	vmm->flush(vm);
 }
 
 void
@@ -81,13 +83,14 @@ nouveau_vm_map_sg_table(struct nouveau_vma *vma, u64 delta, u64 length,
 			struct nouveau_mem *mem)
 {
 	struct nouveau_vm *vm = vma->vm;
-	int big = vma->node->type != vm->spg_shift;
+	struct nouveau_vmmgr *vmm = vm->vmm;
+	int big = vma->node->type != vmm->spg_shift;
 	u32 offset = vma->node->offset + (delta >> 12);
 	u32 bits = vma->node->type - 12;
 	u32 num  = length >> vma->node->type;
-	u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
-	u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
-	u32 max  = 1 << (vm->pgt_bits - bits);
+	u32 pde  = (offset >> vmm->pgt_bits) - vm->fpde;
+	u32 pte  = (offset & ((1 << vmm->pgt_bits) - 1)) >> bits;
+	u32 max  = 1 << (vmm->pgt_bits - bits);
 	unsigned m, sglen;
 	u32 end, len;
 	int i;
@@ -105,7 +108,7 @@ nouveau_vm_map_sg_table(struct nouveau_vma *vma, u64 delta, u64 length,
 		for (m = 0; m < len; m++) {
 			dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 
-			vm->map_sg(vma, pgt, mem, pte, 1, &addr);
+			vmm->map_sg(vma, pgt, mem, pte, 1, &addr);
 			num--;
 			pte++;
 
@@ -120,7 +123,7 @@ nouveau_vm_map_sg_table(struct nouveau_vma *vma, u64 delta, u64 length,
 			for (; m < sglen; m++) {
 				dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 
-				vm->map_sg(vma, pgt, mem, pte, 1, &addr);
+				vmm->map_sg(vma, pgt, mem, pte, 1, &addr);
 				num--;
 				pte++;
 				if (num == 0)
@@ -130,7 +133,7 @@ nouveau_vm_map_sg_table(struct nouveau_vma *vma, u64 delta, u64 length,
 
 	}
 finish:
-	vm->flush(vm);
+	vmm->flush(vm);
 }
 
 void
@@ -138,14 +141,15 @@ nouveau_vm_map_sg(struct nouveau_vma *vma, u64 delta, u64 length,
 		  struct nouveau_mem *mem)
 {
 	struct nouveau_vm *vm = vma->vm;
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	dma_addr_t *list = mem->pages;
-	int big = vma->node->type != vm->spg_shift;
+	int big = vma->node->type != vmm->spg_shift;
 	u32 offset = vma->node->offset + (delta >> 12);
 	u32 bits = vma->node->type - 12;
 	u32 num  = length >> vma->node->type;
-	u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
-	u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
-	u32 max  = 1 << (vm->pgt_bits - bits);
+	u32 pde  = (offset >> vmm->pgt_bits) - vm->fpde;
+	u32 pte  = (offset & ((1 << vmm->pgt_bits) - 1)) >> bits;
+	u32 max  = 1 << (vmm->pgt_bits - bits);
 	u32 end, len;
 
 	while (num) {
@@ -156,7 +160,7 @@ nouveau_vm_map_sg(struct nouveau_vma *vma, u64 delta, u64 length,
 			end = max;
 		len = end - pte;
 
-		vm->map_sg(vma, pgt, mem, pte, len, list);
+		vmm->map_sg(vma, pgt, mem, pte, len, list);
 
 		num  -= len;
 		pte  += len;
@@ -167,20 +171,21 @@ nouveau_vm_map_sg(struct nouveau_vma *vma, u64 delta, u64 length,
 		}
 	}
 
-	vm->flush(vm);
+	vmm->flush(vm);
 }
 
 void
 nouveau_vm_unmap_at(struct nouveau_vma *vma, u64 delta, u64 length)
 {
 	struct nouveau_vm *vm = vma->vm;
-	int big = vma->node->type != vm->spg_shift;
+	struct nouveau_vmmgr *vmm = vm->vmm;
+	int big = vma->node->type != vmm->spg_shift;
 	u32 offset = vma->node->offset + (delta >> 12);
 	u32 bits = vma->node->type - 12;
 	u32 num  = length >> vma->node->type;
-	u32 pde  = (offset >> vm->pgt_bits) - vm->fpde;
-	u32 pte  = (offset & ((1 << vm->pgt_bits) - 1)) >> bits;
-	u32 max  = 1 << (vm->pgt_bits - bits);
+	u32 pde  = (offset >> vmm->pgt_bits) - vm->fpde;
+	u32 pte  = (offset & ((1 << vmm->pgt_bits) - 1)) >> bits;
+	u32 max  = 1 << (vmm->pgt_bits - bits);
 	u32 end, len;
 
 	while (num) {
@@ -191,7 +196,7 @@ nouveau_vm_unmap_at(struct nouveau_vma *vma, u64 delta, u64 length)
 			end = max;
 		len = end - pte;
 
-		vm->unmap(pgt, pte, len);
+		vmm->unmap(pgt, pte, len);
 
 		num -= len;
 		pte += len;
@@ -201,7 +206,7 @@ nouveau_vm_unmap_at(struct nouveau_vma *vma, u64 delta, u64 length)
 		}
 	}
 
-	vm->flush(vm);
+	vmm->flush(vm);
 }
 
 void
@@ -213,6 +218,7 @@ nouveau_vm_unmap(struct nouveau_vma *vma)
 static void
 nouveau_vm_unmap_pgt(struct nouveau_vm *vm, int big, u32 fpde, u32 lpde)
 {
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	struct nouveau_vm_pgd *vpgd;
 	struct nouveau_vm_pgt *vpgt;
 	struct nouveau_gpuobj *pgt;
@@ -227,7 +233,7 @@ nouveau_vm_unmap_pgt(struct nouveau_vm *vm, int big, u32 fpde, u32 lpde)
 		vpgt->obj[big] = NULL;
 
 		list_for_each_entry(vpgd, &vm->pgd_list, head) {
-			vm->map_pgt(vpgd->obj, pde, vpgt->obj);
+			vmm->map_pgt(vpgd->obj, pde, vpgt->obj);
 		}
 
 		mutex_unlock(&vm->mm.mutex);
@@ -239,18 +245,19 @@ nouveau_vm_unmap_pgt(struct nouveau_vm *vm, int big, u32 fpde, u32 lpde)
 static int
 nouveau_vm_map_pgt(struct nouveau_vm *vm, u32 pde, u32 type)
 {
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
 	struct nouveau_vm_pgd *vpgd;
 	struct nouveau_gpuobj *pgt;
-	int big = (type != vm->spg_shift);
+	int big = (type != vmm->spg_shift);
 	u32 pgt_size;
 	int ret;
 
-	pgt_size  = (1 << (vm->pgt_bits + 12)) >> type;
+	pgt_size  = (1 << (vmm->pgt_bits + 12)) >> type;
 	pgt_size *= 8;
 
 	mutex_unlock(&vm->mm.mutex);
-	ret = nouveau_gpuobj_new(vm->dev, NULL, pgt_size, 0x1000,
+	ret = nouveau_gpuobj_new(nv_object(vm->vmm), NULL, pgt_size, 0x1000,
 				 NVOBJ_FLAG_ZERO_ALLOC, &pgt);
 	mutex_lock(&vm->mm.mutex);
 	if (unlikely(ret))
@@ -266,7 +273,7 @@ nouveau_vm_map_pgt(struct nouveau_vm *vm, u32 pde, u32 type)
 
 	vpgt->obj[big] = pgt;
 	list_for_each_entry(vpgd, &vm->pgd_list, head) {
-		vm->map_pgt(vpgd->obj, pde, vpgt->obj);
+		vmm->map_pgt(vpgd->obj, pde, vpgt->obj);
 	}
 
 	return 0;
@@ -276,23 +283,26 @@ int
 nouveau_vm_get(struct nouveau_vm *vm, u64 size, u32 page_shift,
 	       u32 access, struct nouveau_vma *vma)
 {
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	u32 align = (1 << page_shift) >> 12;
 	u32 msize = size >> 12;
 	u32 fpde, lpde, pde;
 	int ret;
 
 	mutex_lock(&vm->mm.mutex);
-	ret = nouveau_mm_get(&vm->mm, page_shift, msize, 0, align, &vma->node);
+	ret = nouveau_mm_head(&vm->mm, page_shift, msize, msize, align,
+			     &vma->node);
 	if (unlikely(ret != 0)) {
 		mutex_unlock(&vm->mm.mutex);
 		return ret;
 	}
 
-	fpde = (vma->node->offset >> vm->pgt_bits);
-	lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
+	fpde = (vma->node->offset >> vmm->pgt_bits);
+	lpde = (vma->node->offset + vma->node->length - 1) >> vmm->pgt_bits;
+
 	for (pde = fpde; pde <= lpde; pde++) {
 		struct nouveau_vm_pgt *vpgt = &vm->pgt[pde - vm->fpde];
-		int big = (vma->node->type != vm->spg_shift);
+		int big = (vma->node->type != vmm->spg_shift);
 
 		if (likely(vpgt->refcount[big])) {
 			vpgt->refcount[big]++;
@@ -303,9 +313,8 @@ nouveau_vm_get(struct nouveau_vm *vm, u64 size, u32 page_shift,
 		if (ret) {
 			if (pde != fpde)
 				nouveau_vm_unmap_pgt(vm, big, fpde, pde - 1);
-			nouveau_mm_put(&vm->mm, vma->node);
+			nouveau_mm_free(&vm->mm, &vma->node);
 			mutex_unlock(&vm->mm.mutex);
-			vma->node = NULL;
 			return ret;
 		}
 	}
@@ -321,91 +330,67 @@ void
 nouveau_vm_put(struct nouveau_vma *vma)
 {
 	struct nouveau_vm *vm = vma->vm;
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	u32 fpde, lpde;
 
 	if (unlikely(vma->node == NULL))
 		return;
-	fpde = (vma->node->offset >> vm->pgt_bits);
-	lpde = (vma->node->offset + vma->node->length - 1) >> vm->pgt_bits;
+	fpde = (vma->node->offset >> vmm->pgt_bits);
+	lpde = (vma->node->offset + vma->node->length - 1) >> vmm->pgt_bits;
 
 	mutex_lock(&vm->mm.mutex);
-	nouveau_vm_unmap_pgt(vm, vma->node->type != vm->spg_shift, fpde, lpde);
-	nouveau_mm_put(&vm->mm, vma->node);
-	vma->node = NULL;
+	nouveau_vm_unmap_pgt(vm, vma->node->type != vmm->spg_shift, fpde, lpde);
+	nouveau_mm_free(&vm->mm, &vma->node);
 	mutex_unlock(&vm->mm.mutex);
 }
 
 int
-nouveau_vm_new(struct drm_device *dev, u64 offset, u64 length, u64 mm_offset,
-	       struct nouveau_vm **pvm)
+nouveau_vm_create(struct nouveau_vmmgr *vmm, u64 offset, u64 length,
+		  u64 mm_offset, u32 block, struct nouveau_vm **pvm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_vm *vm;
 	u64 mm_length = (offset + length) - mm_offset;
-	u32 block, pgt_bits;
 	int ret;
 
-	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	vm = *pvm = kzalloc(sizeof(*vm), GFP_KERNEL);
 	if (!vm)
 		return -ENOMEM;
 
-	if (dev_priv->card_type == NV_50) {
-		vm->map_pgt = nv50_vm_map_pgt;
-		vm->map = nv50_vm_map;
-		vm->map_sg = nv50_vm_map_sg;
-		vm->unmap = nv50_vm_unmap;
-		vm->flush = nv50_vm_flush;
-		vm->spg_shift = 12;
-		vm->lpg_shift = 16;
-
-		pgt_bits = 29;
-		block = (1 << pgt_bits);
-		if (length < block)
-			block = length;
-
-	} else
-	if (dev_priv->card_type >= NV_C0) {
-		vm->map_pgt = nvc0_vm_map_pgt;
-		vm->map = nvc0_vm_map;
-		vm->map_sg = nvc0_vm_map_sg;
-		vm->unmap = nvc0_vm_unmap;
-		vm->flush = nvc0_vm_flush;
-		vm->spg_shift = 12;
-		vm->lpg_shift = 17;
-		pgt_bits = 27;
-		block = 4096;
-	} else {
-		kfree(vm);
-		return -ENOSYS;
-	}
+	INIT_LIST_HEAD(&vm->pgd_list);
+	vm->vmm = vmm;
+	vm->refcount = 1;
+	vm->fpde = offset >> (vmm->pgt_bits + 12);
+	vm->lpde = (offset + length - 1) >> (vmm->pgt_bits + 12);
 
-	vm->fpde   = offset >> pgt_bits;
-	vm->lpde   = (offset + length - 1) >> pgt_bits;
-	vm->pgt = kcalloc(vm->lpde - vm->fpde + 1, sizeof(*vm->pgt), GFP_KERNEL);
+	vm->pgt  = kcalloc(vm->lpde - vm->fpde + 1, sizeof(*vm->pgt), GFP_KERNEL);
 	if (!vm->pgt) {
 		kfree(vm);
 		return -ENOMEM;
 	}
 
-	INIT_LIST_HEAD(&vm->pgd_list);
-	vm->dev = dev;
-	vm->refcount = 1;
-	vm->pgt_bits = pgt_bits - 12;
-
 	ret = nouveau_mm_init(&vm->mm, mm_offset >> 12, mm_length >> 12,
 			      block >> 12);
 	if (ret) {
+		kfree(vm->pgt);
 		kfree(vm);
 		return ret;
 	}
 
-	*pvm = vm;
 	return 0;
 }
 
+int
+nouveau_vm_new(struct nouveau_device *device, u64 offset, u64 length,
+	       u64 mm_offset, struct nouveau_vm **pvm)
+{
+	struct nouveau_vmmgr *vmm = nouveau_vmmgr(device);
+	return vmm->create(vmm, offset, length, mm_offset, pvm);
+}
+
 static int
 nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
 {
+	struct nouveau_vmmgr *vmm = vm->vmm;
 	struct nouveau_vm_pgd *vpgd;
 	int i;
 
@@ -420,7 +405,7 @@ nouveau_vm_link(struct nouveau_vm *vm, struct nouveau_gpuobj *pgd)
 
 	mutex_lock(&vm->mm.mutex);
 	for (i = vm->fpde; i <= vm->lpde; i++)
-		vm->map_pgt(pgd, i, vm->pgt[i - vm->fpde].obj);
+		vmm->map_pgt(pgd, i, vm->pgt[i - vm->fpde].obj);
 	list_add(&vpgd->head, &vm->pgd_list);
 	mutex_unlock(&vm->mm.mutex);
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.c
new file mode 100644
index 00000000000..6adbbc9cc36
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+
+#include "nv04.h"
+
+#define NV04_PDMA_SIZE (128 * 1024 * 1024)
+#define NV04_PDMA_PAGE (  4 * 1024)
+
+/*******************************************************************************
+ * VM map/unmap callbacks
+ ******************************************************************************/
+
+static void
+nv04_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+	       struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
+{
+	pte = 0x00008 + (pte * 4);
+	while (cnt) {
+		u32 page = PAGE_SIZE / NV04_PDMA_PAGE;
+		u32 phys = (u32)*list++;
+		while (cnt && page--) {
+			nv_wo32(pgt, pte, phys | 3);
+			phys += NV04_PDMA_PAGE;
+			pte += 4;
+			cnt -= 1;
+		}
+	}
+}
+
+static void
+nv04_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
+{
+	pte = 0x00008 + (pte * 4);
+	while (cnt--) {
+		nv_wo32(pgt, pte, 0x00000000);
+		pte += 4;
+	}
+}
+
+static void
+nv04_vm_flush(struct nouveau_vm *vm)
+{
+}
+
+/*******************************************************************************
+ * VM object
+ ******************************************************************************/
+
+int
+nv04_vm_create(struct nouveau_vmmgr *vmm, u64 offset, u64 length, u64 mmstart,
+	       struct nouveau_vm **pvm)
+{
+	return -EINVAL;
+}
+
+/*******************************************************************************
+ * VMMGR subdev
+ ******************************************************************************/
+
+static int
+nv04_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv04_vmmgr_priv *priv;
+	struct nouveau_gpuobj *dma;
+	int ret;
+
+	ret = nouveau_vmmgr_create(parent, engine, oclass, "PCIGART",
+				   "pcigart", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.create = nv04_vm_create;
+	priv->base.limit = NV04_PDMA_SIZE;
+	priv->base.dma_bits = 32;
+	priv->base.pgt_bits = 32 - 12;
+	priv->base.spg_shift = 12;
+	priv->base.lpg_shift = 12;
+	priv->base.map_sg = nv04_vm_map_sg;
+	priv->base.unmap = nv04_vm_unmap;
+	priv->base.flush = nv04_vm_flush;
+
+	ret = nouveau_vm_create(&priv->base, 0, NV04_PDMA_SIZE, 0, 4096,
+				&priv->vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL,
+				 (NV04_PDMA_SIZE / NV04_PDMA_PAGE) * 4 +
+				 8, 16, NVOBJ_FLAG_ZERO_ALLOC,
+				 &priv->vm->pgt[0].obj[0]);
+	dma = priv->vm->pgt[0].obj[0];
+	priv->vm->pgt[0].refcount[0] = 1;
+	if (ret)
+		return ret;
+
+	nv_wo32(dma, 0x00000, 0x0002103d); /* PCI, RW, PT, !LN */
+	nv_wo32(dma, 0x00004, NV04_PDMA_SIZE - 1);
+	return 0;
+}
+
+void
+nv04_vmmgr_dtor(struct nouveau_object *object)
+{
+	struct nv04_vmmgr_priv *priv = (void *)object;
+	if (priv->vm) {
+		nouveau_gpuobj_ref(NULL, &priv->vm->pgt[0].obj[0]);
+		nouveau_vm_ref(NULL, &priv->vm, NULL);
+	}
+	if (priv->nullp) {
+		pci_free_consistent(nv_device(priv)->pdev, 16 * 1024,
+				    priv->nullp, priv->null);
+	}
+	nouveau_vmmgr_destroy(&priv->base);
+}
+
+struct nouveau_oclass
+nv04_vmmgr_oclass = {
+	.handle = NV_SUBDEV(VM, 0x04),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv04_vmmgr_ctor,
+		.dtor = nv04_vmmgr_dtor,
+		.init = _nouveau_vmmgr_init,
+		.fini = _nouveau_vmmgr_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.h
new file mode 100644
index 00000000000..ec42d4bc86a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nv04.h
@@ -0,0 +1,19 @@
+#ifndef __NV04_VMMGR_PRIV__
+#define __NV04_VMMGR_PRIV__
+
+#include <subdev/vm.h>
+
+struct nv04_vmmgr_priv {
+	struct nouveau_vmmgr base;
+	struct nouveau_vm *vm;
+	dma_addr_t null;
+	void *nullp;
+};
+
+static inline struct nv04_vmmgr_priv *
+nv04_vmmgr(void *obj)
+{
+	return (void *)nouveau_vmmgr(obj);
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/vm/nv41.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nv41.c
new file mode 100644
index 00000000000..0203e1e12ca
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nv41.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+#include <core/option.h>
+
+#include <subdev/timer.h>
+#include <subdev/vm.h>
+
+#include "nv04.h"
+
+#define NV41_GART_SIZE (512 * 1024 * 1024)
+#define NV41_GART_PAGE (  4 * 1024)
+
+/*******************************************************************************
+ * VM map/unmap callbacks
+ ******************************************************************************/
+
+static void
+nv41_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+	       struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
+{
+	pte = pte * 4;
+	while (cnt) {
+		u32 page = PAGE_SIZE / NV41_GART_PAGE;
+		u64 phys = (u64)*list++;
+		while (cnt && page--) {
+			nv_wo32(pgt, pte, (phys >> 7) | 1);
+			phys += NV41_GART_PAGE;
+			pte += 4;
+			cnt -= 1;
+		}
+	}
+}
+
+static void
+nv41_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
+{
+	pte = pte * 4;
+	while (cnt--) {
+		nv_wo32(pgt, pte, 0x00000000);
+		pte += 4;
+	}
+}
+
+static void
+nv41_vm_flush(struct nouveau_vm *vm)
+{
+	struct nv04_vm_priv *priv = (void *)vm->vmm;
+
+	mutex_lock(&nv_subdev(priv)->mutex);
+	nv_wr32(priv, 0x100810, 0x00000022);
+	if (!nv_wait(priv, 0x100810, 0x00000020, 0x00000020)) {
+		nv_warn(priv, "flush timeout, 0x%08x\n",
+			nv_rd32(priv, 0x100810));
+	}
+	nv_wr32(priv, 0x100810, 0x00000000);
+	mutex_unlock(&nv_subdev(priv)->mutex);
+}
+
+/*******************************************************************************
+ * VMMGR subdev
+ ******************************************************************************/
+
+static int
+nv41_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nv04_vmmgr_priv *priv;
+	int ret;
+
+	if (!nouveau_boolopt(device->cfgopt, "NvPCIE", true)) {
+		return nouveau_object_ctor(parent, engine, &nv04_vmmgr_oclass,
+					   data, size, pobject);
+	}
+
+	ret = nouveau_vmmgr_create(parent, engine, oclass, "PCIEGART",
+				   "pciegart", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.create = nv04_vm_create;
+	priv->base.limit = NV41_GART_SIZE;
+	priv->base.dma_bits = 39;
+	priv->base.pgt_bits = 32 - 12;
+	priv->base.spg_shift = 12;
+	priv->base.lpg_shift = 12;
+	priv->base.map_sg = nv41_vm_map_sg;
+	priv->base.unmap = nv41_vm_unmap;
+	priv->base.flush = nv41_vm_flush;
+
+	ret = nouveau_vm_create(&priv->base, 0, NV41_GART_SIZE, 0, 4096,
+				&priv->vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL,
+				(NV41_GART_SIZE / NV41_GART_PAGE) * 4,
+				 16, NVOBJ_FLAG_ZERO_ALLOC,
+				 &priv->vm->pgt[0].obj[0]);
+	priv->vm->pgt[0].refcount[0] = 1;
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv41_vmmgr_init(struct nouveau_object *object)
+{
+	struct nv04_vmmgr_priv *priv = (void *)object;
+	struct nouveau_gpuobj *dma = priv->vm->pgt[0].obj[0];
+	int ret;
+
+	ret = nouveau_vmmgr_init(&priv->base);
+	if (ret)
+		return ret;
+
+	nv_wr32(priv, 0x100800, dma->addr | 0x00000002);
+	nv_mask(priv, 0x10008c, 0x00000100, 0x00000100);
+	nv_wr32(priv, 0x100820, 0x00000000);
+	return 0;
+}
+
+struct nouveau_oclass
+nv41_vmmgr_oclass = {
+	.handle = NV_SUBDEV(VM, 0x41),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv41_vmmgr_ctor,
+		.dtor = nv04_vmmgr_dtor,
+		.init = nv41_vmmgr_init,
+		.fini = _nouveau_vmmgr_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/vm/nv44.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nv44.c
new file mode 100644
index 00000000000..0ac18d05a14
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nv44.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/gpuobj.h>
+#include <core/option.h>
+
+#include <subdev/timer.h>
+#include <subdev/vm.h>
+
+#include "nv04.h"
+
+#define NV44_GART_SIZE (512 * 1024 * 1024)
+#define NV44_GART_PAGE (  4 * 1024)
+
+/*******************************************************************************
+ * VM map/unmap callbacks
+ ******************************************************************************/
+
+static void
+nv44_vm_fill(struct nouveau_gpuobj *pgt, dma_addr_t null,
+	     dma_addr_t *list, u32 pte, u32 cnt)
+{
+	u32 base = (pte << 2) & ~0x0000000f;
+	u32 tmp[4];
+
+	tmp[0] = nv_ro32(pgt, base + 0x0);
+	tmp[1] = nv_ro32(pgt, base + 0x4);
+	tmp[2] = nv_ro32(pgt, base + 0x8);
+	tmp[3] = nv_ro32(pgt, base + 0xc);
+
+	while (cnt--) {
+		u32 addr = list ? (*list++ >> 12) : (null >> 12);
+		switch (pte++ & 0x3) {
+		case 0:
+			tmp[0] &= ~0x07ffffff;
+			tmp[0] |= addr;
+			break;
+		case 1:
+			tmp[0] &= ~0xf8000000;
+			tmp[0] |= addr << 27;
+			tmp[1] &= ~0x003fffff;
+			tmp[1] |= addr >> 5;
+			break;
+		case 2:
+			tmp[1] &= ~0xffc00000;
+			tmp[1] |= addr << 22;
+			tmp[2] &= ~0x0001ffff;
+			tmp[2] |= addr >> 10;
+			break;
+		case 3:
+			tmp[2] &= ~0xfffe0000;
+			tmp[2] |= addr << 17;
+			tmp[3] &= ~0x00000fff;
+			tmp[3] |= addr >> 15;
+			break;
+		}
+	}
+
+	nv_wo32(pgt, base + 0x0, tmp[0]);
+	nv_wo32(pgt, base + 0x4, tmp[1]);
+	nv_wo32(pgt, base + 0x8, tmp[2]);
+	nv_wo32(pgt, base + 0xc, tmp[3] | 0x40000000);
+}
+
+static void
+nv44_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
+	       struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
+{
+	struct nv04_vmmgr_priv *priv = (void *)vma->vm->vmm;
+	u32 tmp[4];
+	int i;
+
+	if (pte & 3) {
+		u32  max = 4 - (pte & 3);
+		u32 part = (cnt > max) ? max : cnt;
+		nv44_vm_fill(pgt, priv->null, list, pte, part);
+		pte  += part;
+		list += part;
+		cnt  -= part;
+	}
+
+	while (cnt >= 4) {
+		for (i = 0; i < 4; i++)
+			tmp[i] = *list++ >> 12;
+		nv_wo32(pgt, pte++ * 4, tmp[0] >>  0 | tmp[1] << 27);
+		nv_wo32(pgt, pte++ * 4, tmp[1] >>  5 | tmp[2] << 22);
+		nv_wo32(pgt, pte++ * 4, tmp[2] >> 10 | tmp[3] << 17);
+		nv_wo32(pgt, pte++ * 4, tmp[3] >> 15 | 0x40000000);
+		cnt -= 4;
+	}
+
+	if (cnt)
+		nv44_vm_fill(pgt, priv->null, list, pte, cnt);
+}
+
+static void
+nv44_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
+{
+	struct nv04_vmmgr_priv *priv = (void *)nouveau_vmmgr(pgt);
+
+	if (pte & 3) {
+		u32  max = 4 - (pte & 3);
+		u32 part = (cnt > max) ? max : cnt;
+		nv44_vm_fill(pgt, priv->null, NULL, pte, part);
+		pte  += part;
+		cnt  -= part;
+	}
+
+	while (cnt >= 4) {
+		nv_wo32(pgt, pte++ * 4, 0x00000000);
+		nv_wo32(pgt, pte++ * 4, 0x00000000);
+		nv_wo32(pgt, pte++ * 4, 0x00000000);
+		nv_wo32(pgt, pte++ * 4, 0x00000000);
+		cnt -= 4;
+	}
+
+	if (cnt)
+		nv44_vm_fill(pgt, priv->null, NULL, pte, cnt);
+}
+
+static void
+nv44_vm_flush(struct nouveau_vm *vm)
+{
+	struct nv04_vmmgr_priv *priv = (void *)vm->vmm;
+	nv_wr32(priv, 0x100814, priv->base.limit - NV44_GART_PAGE);
+	nv_wr32(priv, 0x100808, 0x00000020);
+	if (!nv_wait(priv, 0x100808, 0x00000001, 0x00000001))
+		nv_error(priv, "timeout: 0x%08x\n", nv_rd32(priv, 0x100808));
+	nv_wr32(priv, 0x100808, 0x00000000);
+}
+
+/*******************************************************************************
+ * VMMGR subdev
+ ******************************************************************************/
+
+static int
+nv44_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nouveau_device *device = nv_device(parent);
+	struct nv04_vmmgr_priv *priv;
+	int ret;
+
+	if (!nouveau_boolopt(device->cfgopt, "NvPCIE", true)) {
+		return nouveau_object_ctor(parent, engine, &nv04_vmmgr_oclass,
+					   data, size, pobject);
+	}
+
+	ret = nouveau_vmmgr_create(parent, engine, oclass, "PCIEGART",
+				   "pciegart", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.create = nv04_vm_create;
+	priv->base.limit = NV44_GART_SIZE;
+	priv->base.dma_bits = 39;
+	priv->base.pgt_bits = 32 - 12;
+	priv->base.spg_shift = 12;
+	priv->base.lpg_shift = 12;
+	priv->base.map_sg = nv44_vm_map_sg;
+	priv->base.unmap = nv44_vm_unmap;
+	priv->base.flush = nv44_vm_flush;
+
+	priv->nullp = pci_alloc_consistent(device->pdev, 16 * 1024, &priv->null);
+	if (!priv->nullp) {
+		nv_error(priv, "unable to allocate dummy pages\n");
+		return -ENOMEM;
+	}
+
+	ret = nouveau_vm_create(&priv->base, 0, NV44_GART_SIZE, 0, 4096,
+				&priv->vm);
+	if (ret)
+		return ret;
+
+	ret = nouveau_gpuobj_new(parent, NULL,
+				(NV44_GART_SIZE / NV44_GART_PAGE) * 4,
+				 512 * 1024, NVOBJ_FLAG_ZERO_ALLOC,
+				 &priv->vm->pgt[0].obj[0]);
+	priv->vm->pgt[0].refcount[0] = 1;
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int
+nv44_vmmgr_init(struct nouveau_object *object)
+{
+	struct nv04_vmmgr_priv *priv = (void *)object;
+	struct nouveau_gpuobj *gart = priv->vm->pgt[0].obj[0];
+	u32 addr;
+	int ret;
+
+	ret = nouveau_vmmgr_init(&priv->base);
+	if (ret)
+		return ret;
+
+	/* calculate vram address of this PRAMIN block, object must be
+	 * allocated on 512KiB alignment, and not exceed a total size
+	 * of 512KiB for this to work correctly
+	 */
+	addr  = nv_rd32(priv, 0x10020c);
+	addr -= ((gart->addr >> 19) + 1) << 19;
+
+	nv_wr32(priv, 0x100850, 0x80000000);
+	nv_wr32(priv, 0x100818, priv->null);
+	nv_wr32(priv, 0x100804, NV44_GART_SIZE);
+	nv_wr32(priv, 0x100850, 0x00008000);
+	nv_mask(priv, 0x10008c, 0x00000200, 0x00000200);
+	nv_wr32(priv, 0x100820, 0x00000000);
+	nv_wr32(priv, 0x10082c, 0x00000001);
+	nv_wr32(priv, 0x100800, addr | 0x00000010);
+	return 0;
+}
+
+struct nouveau_oclass
+nv44_vmmgr_oclass = {
+	.handle = NV_SUBDEV(VM, 0x44),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv44_vmmgr_ctor,
+		.dtor = nv04_vmmgr_dtor,
+		.init = nv44_vmmgr_init,
+		.fini = _nouveau_vmmgr_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nv50_vm.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nv50.c
index 179bb42a635..e067f81c97b 100644
--- a/drivers/gpu/drm/nouveau/nv50_vm.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nv50.c
@@ -22,12 +22,19 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <core/device.h>
+#include <core/gpuobj.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_vm.h"
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/vm.h>
 
-void
+struct nv50_vmmgr_priv {
+	struct nouveau_vmmgr base;
+	spinlock_t lock;
+};
+
+static void
 nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
 		struct nouveau_gpuobj *pgt[2])
 {
@@ -35,11 +42,11 @@ nv50_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 pde,
 	u32 coverage = 0;
 
 	if (pgt[0]) {
-		phys = 0x00000003 | pgt[0]->vinst; /* present, 4KiB pages */
+		phys = 0x00000003 | pgt[0]->addr; /* present, 4KiB pages */
 		coverage = (pgt[0]->size >> 3) << 12;
 	} else
 	if (pgt[1]) {
-		phys = 0x00000001 | pgt[1]->vinst; /* present */
+		phys = 0x00000001 | pgt[1]->addr; /* present */
 		coverage = (pgt[1]->size >> 3) << 16;
 	}
 
@@ -69,19 +76,18 @@ vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
 	return phys;
 }
 
-void
+static void
 nv50_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	    struct nouveau_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
 {
-	struct drm_nouveau_private *dev_priv = vma->vm->dev->dev_private;
 	u32 comp = (mem->memtype & 0x180) >> 7;
 	u32 block, target;
 	int i;
 
 	/* IGPs don't have real VRAM, re-target to stolen system memory */
 	target = 0;
-	if (dev_priv->vram_sys_base) {
-		phys += dev_priv->vram_sys_base;
+	if (nouveau_fb(vma->vm->vmm)->ram.stolen) {
+		phys += nouveau_fb(vma->vm->vmm)->ram.stolen;
 		target = 3;
 	}
 
@@ -103,7 +109,7 @@ nv50_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 		phys += block << (vma->node->type - 3);
 		cnt  -= block;
 		if (comp) {
-			u32 tag = mem->tag->start + ((delta >> 16) * comp);
+			u32 tag = mem->tag->offset + ((delta >> 16) * comp);
 			offset_h |= (tag << 17);
 			delta    += block << (vma->node->type - 3);
 		}
@@ -117,7 +123,7 @@ nv50_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	}
 }
 
-void
+static void
 nv50_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	       struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
 {
@@ -131,7 +137,7 @@ nv50_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	}
 }
 
-void
+static void
 nv50_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
 {
 	pte <<= 3;
@@ -142,36 +148,80 @@ nv50_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
 	}
 }
 
-void
+static void
 nv50_vm_flush(struct nouveau_vm *vm)
 {
-	struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
+	struct nouveau_engine *engine;
 	int i;
 
-	pinstmem->flush(vm->dev);
-
-	/* BAR */
-	if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm) {
-		nv50_vm_flush_engine(vm->dev, 6);
-		return;
-	}
-
-	for (i = 0; i < NVOBJ_ENGINE_NR; i++) {
-		if (atomic_read(&vm->engref[i]))
-			dev_priv->eng[i]->tlb_flush(vm->dev, i);
+	for (i = 0; i < NVDEV_SUBDEV_NR; i++) {
+		if (atomic_read(&vm->engref[i])) {
+			engine = nouveau_engine(vm->vmm, i);
+			if (engine && engine->tlb_flush)
+				engine->tlb_flush(engine);
+		}
 	}
 }
 
 void
-nv50_vm_flush_engine(struct drm_device *dev, int engine)
+nv50_vm_flush_engine(struct nouveau_subdev *subdev, int engine)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nv50_vmmgr_priv *priv = (void *)nouveau_vmmgr(subdev);
 	unsigned long flags;
 
-	spin_lock_irqsave(&dev_priv->vm_lock, flags);
-	nv_wr32(dev, 0x100c80, (engine << 16) | 1);
-	if (!nv_wait(dev, 0x100c80, 0x00000001, 0x00000000))
-		NV_ERROR(dev, "vm flush timeout: engine %d\n", engine);
-	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
+	spin_lock_irqsave(&priv->lock, flags);
+	nv_wr32(subdev, 0x100c80, (engine << 16) | 1);
+	if (!nv_wait(subdev, 0x100c80, 0x00000001, 0x00000000))
+		nv_error(subdev, "vm flush timeout: engine %d\n", engine);
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
+
+static int
+nv50_vm_create(struct nouveau_vmmgr *vmm, u64 offset, u64 length,
+	       u64 mm_offset, struct nouveau_vm **pvm)
+{
+	u32 block = (1 << (vmm->pgt_bits + 12));
+	if (block > length)
+		block = length;
+
+	return nouveau_vm_create(vmm, offset, length, mm_offset, block, pvm);
 }
+
+static int
+nv50_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nv50_vmmgr_priv *priv;
+	int ret;
+
+	ret = nouveau_vmmgr_create(parent, engine, oclass, "VM", "vm", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.limit = 1ULL << 40;
+	priv->base.dma_bits = 40;
+	priv->base.pgt_bits  = 29 - 12;
+	priv->base.spg_shift = 12;
+	priv->base.lpg_shift = 16;
+	priv->base.create = nv50_vm_create;
+	priv->base.map_pgt = nv50_vm_map_pgt;
+	priv->base.map = nv50_vm_map;
+	priv->base.map_sg = nv50_vm_map_sg;
+	priv->base.unmap = nv50_vm_unmap;
+	priv->base.flush = nv50_vm_flush;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+struct nouveau_oclass
+nv50_vmmgr_oclass = {
+	.handle = NV_SUBDEV(VM, 0x50),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nv50_vmmgr_ctor,
+		.dtor = _nouveau_vmmgr_dtor,
+		.init = _nouveau_vmmgr_init,
+		.fini = _nouveau_vmmgr_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nvc0_vm.c b/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c
index 30d2bd58828..30c61e6c201 100644
--- a/drivers/gpu/drm/nouveau/nvc0_vm.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/vm/nvc0.c
@@ -22,21 +22,28 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <core/device.h>
+#include <core/gpuobj.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_vm.h"
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/vm.h>
 
-void
+struct nvc0_vmmgr_priv {
+	struct nouveau_vmmgr base;
+	spinlock_t lock;
+};
+
+static void
 nvc0_vm_map_pgt(struct nouveau_gpuobj *pgd, u32 index,
 		struct nouveau_gpuobj *pgt[2])
 {
 	u32 pde[2] = { 0, 0 };
 
 	if (pgt[0])
-		pde[1] = 0x00000001 | (pgt[0]->vinst >> 8);
+		pde[1] = 0x00000001 | (pgt[0]->addr >> 8);
 	if (pgt[1])
-		pde[0] = 0x00000001 | (pgt[1]->vinst >> 8);
+		pde[0] = 0x00000001 | (pgt[1]->addr >> 8);
 
 	nv_wo32(pgd, (index * 8) + 0, pde[0]);
 	nv_wo32(pgd, (index * 8) + 4, pde[1]);
@@ -57,7 +64,7 @@ nvc0_vm_addr(struct nouveau_vma *vma, u64 phys, u32 memtype, u32 target)
 	return phys;
 }
 
-void
+static void
 nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	    struct nouveau_mem *mem, u32 pte, u32 cnt, u64 phys, u64 delta)
 {
@@ -73,7 +80,7 @@ nvc0_vm_map(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	}
 }
 
-void
+static void
 nvc0_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	       struct nouveau_mem *mem, u32 pte, u32 cnt, dma_addr_t *list)
 {
@@ -88,7 +95,7 @@ nvc0_vm_map_sg(struct nouveau_vma *vma, struct nouveau_gpuobj *pgt,
 	}
 }
 
-void
+static void
 nvc0_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
 {
 	pte <<= 3;
@@ -100,37 +107,83 @@ nvc0_vm_unmap(struct nouveau_gpuobj *pgt, u32 pte, u32 cnt)
 }
 
 void
-nvc0_vm_flush(struct nouveau_vm *vm)
+nvc0_vm_flush_engine(struct nouveau_subdev *subdev, u64 addr, int type)
 {
-	struct drm_nouveau_private *dev_priv = vm->dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct drm_device *dev = vm->dev;
-	struct nouveau_vm_pgd *vpgd;
+	struct nvc0_vmmgr_priv *priv = (void *)nouveau_vmmgr(subdev);
 	unsigned long flags;
-	u32 engine;
 
-	engine = 1;
-	if (vm == dev_priv->bar1_vm || vm == dev_priv->bar3_vm)
-		engine |= 4;
+	/* looks like maybe a "free flush slots" counter, the
+	 * faster you write to 0x100cbc to more it decreases
+	 */
+	spin_lock_irqsave(&priv->lock, flags);
+	if (!nv_wait_ne(subdev, 0x100c80, 0x00ff0000, 0x00000000)) {
+		nv_error(subdev, "vm timeout 0: 0x%08x %d\n",
+			 nv_rd32(subdev, 0x100c80), type);
+	}
+
+	nv_wr32(subdev, 0x100cb8, addr >> 8);
+	nv_wr32(subdev, 0x100cbc, 0x80000000 | type);
+
+	/* wait for flush to be queued? */
+	if (!nv_wait(subdev, 0x100c80, 0x00008000, 0x00008000)) {
+		nv_error(subdev, "vm timeout 1: 0x%08x %d\n",
+			 nv_rd32(subdev, 0x100c80), type);
+	}
+	spin_unlock_irqrestore(&priv->lock, flags);
+}
 
-	pinstmem->flush(vm->dev);
+static void
+nvc0_vm_flush(struct nouveau_vm *vm)
+{
+	struct nouveau_vm_pgd *vpgd;
 
-	spin_lock_irqsave(&dev_priv->vm_lock, flags);
 	list_for_each_entry(vpgd, &vm->pgd_list, head) {
-		/* looks like maybe a "free flush slots" counter, the
-		 * faster you write to 0x100cbc to more it decreases
-		 */
-		if (!nv_wait_ne(dev, 0x100c80, 0x00ff0000, 0x00000000)) {
-			NV_ERROR(dev, "vm timeout 0: 0x%08x %d\n",
-				 nv_rd32(dev, 0x100c80), engine);
-		}
-		nv_wr32(dev, 0x100cb8, vpgd->obj->vinst >> 8);
-		nv_wr32(dev, 0x100cbc, 0x80000000 | engine);
-		/* wait for flush to be queued? */
-		if (!nv_wait(dev, 0x100c80, 0x00008000, 0x00008000)) {
-			NV_ERROR(dev, "vm timeout 1: 0x%08x %d\n",
-				 nv_rd32(dev, 0x100c80), engine);
-		}
+		nvc0_vm_flush_engine(nv_subdev(vm->vmm), vpgd->obj->addr, 1);
 	}
-	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
 }
+
+static int
+nvc0_vm_create(struct nouveau_vmmgr *vmm, u64 offset, u64 length,
+	       u64 mm_offset, struct nouveau_vm **pvm)
+{
+	return nouveau_vm_create(vmm, offset, length, mm_offset, 4096, pvm);
+}
+
+static int
+nvc0_vmmgr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+		struct nouveau_oclass *oclass, void *data, u32 size,
+		struct nouveau_object **pobject)
+{
+	struct nvc0_vmmgr_priv *priv;
+	int ret;
+
+	ret = nouveau_vmmgr_create(parent, engine, oclass, "VM", "vm", &priv);
+	*pobject = nv_object(priv);
+	if (ret)
+		return ret;
+
+	priv->base.limit = 1ULL << 40;
+	priv->base.dma_bits = 40;
+	priv->base.pgt_bits  = 27 - 12;
+	priv->base.spg_shift = 12;
+	priv->base.lpg_shift = 17;
+	priv->base.create = nvc0_vm_create;
+	priv->base.map_pgt = nvc0_vm_map_pgt;
+	priv->base.map = nvc0_vm_map;
+	priv->base.map_sg = nvc0_vm_map_sg;
+	priv->base.unmap = nvc0_vm_unmap;
+	priv->base.flush = nvc0_vm_flush;
+	spin_lock_init(&priv->lock);
+	return 0;
+}
+
+struct nouveau_oclass
+nvc0_vmmgr_oclass = {
+	.handle = NV_SUBDEV(VM, 0xc0),
+	.ofuncs = &(struct nouveau_ofuncs) {
+		.ctor = nvc0_vmmgr_ctor,
+		.dtor = _nouveau_vmmgr_dtor,
+		.init = _nouveau_vmmgr_init,
+		.fini = _nouveau_vmmgr_fini,
+	},
+};
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 3ca240b4413..cc79c796afe 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -21,23 +21,153 @@
  *
  */
 
-#include "drmP.h"
+#include <core/object.h>
+#include <core/client.h>
+#include <core/device.h>
+#include <core/class.h>
+#include <core/mm.h>
 
-#include "nouveau_drv.h"
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+#include <subdev/instmem.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
+#include "nouveau_gem.h"
+#include "nouveau_chan.h"
 #include "nouveau_abi16.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
+
+struct nouveau_abi16 *
+nouveau_abi16_get(struct drm_file *file_priv, struct drm_device *dev)
+{
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
+	mutex_lock(&cli->mutex);
+	if (!cli->abi16) {
+		struct nouveau_abi16 *abi16;
+		cli->abi16 = abi16 = kzalloc(sizeof(*abi16), GFP_KERNEL);
+		if (cli->abi16) {
+			INIT_LIST_HEAD(&abi16->channels);
+			abi16->client = nv_object(cli);
+
+			/* allocate device object targeting client's default
+			 * device (ie. the one that belongs to the fd it
+			 * opened)
+			 */
+			if (nouveau_object_new(abi16->client, NVDRM_CLIENT,
+					       NVDRM_DEVICE, 0x0080,
+					       &(struct nv_device_class) {
+						.device = ~0ULL,
+					       },
+					       sizeof(struct nv_device_class),
+					       &abi16->device) == 0)
+				return cli->abi16;
+
+			kfree(cli->abi16);
+			cli->abi16 = NULL;
+		}
+
+		mutex_unlock(&cli->mutex);
+	}
+	return cli->abi16;
+}
+
+int
+nouveau_abi16_put(struct nouveau_abi16 *abi16, int ret)
+{
+	struct nouveau_cli *cli = (void *)abi16->client;
+	mutex_unlock(&cli->mutex);
+	return ret;
+}
+
+u16
+nouveau_abi16_swclass(struct nouveau_drm *drm)
+{
+	switch (nv_device(drm->device)->card_type) {
+	case NV_04:
+		return 0x006e;
+	case NV_10:
+	case NV_20:
+	case NV_30:
+	case NV_40:
+		return 0x016e;
+	case NV_50:
+		return 0x506e;
+	case NV_C0:
+	case NV_D0:
+	case NV_E0:
+		return 0x906e;
+	}
+
+	return 0x0000;
+}
+
+static void
+nouveau_abi16_ntfy_fini(struct nouveau_abi16_chan *chan,
+			struct nouveau_abi16_ntfy *ntfy)
+{
+	nouveau_mm_free(&chan->heap, &ntfy->node);
+	list_del(&ntfy->head);
+	kfree(ntfy);
+}
+
+static void
+nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
+			struct nouveau_abi16_chan *chan)
+{
+	struct nouveau_abi16_ntfy *ntfy, *temp;
+
+	/* cleanup notifier state */
+	list_for_each_entry_safe(ntfy, temp, &chan->notifiers, head) {
+		nouveau_abi16_ntfy_fini(chan, ntfy);
+	}
+
+	if (chan->ntfy) {
+		nouveau_bo_vma_del(chan->ntfy, &chan->ntfy_vma);
+		drm_gem_object_unreference_unlocked(chan->ntfy->gem);
+	}
+
+	if (chan->heap.block_size)
+		nouveau_mm_fini(&chan->heap);
+
+	/* destroy channel object, all children will be killed too */
+	if (chan->chan) {
+		abi16->handles &= ~(1 << (chan->chan->handle & 0xffff));
+		nouveau_channel_del(&chan->chan);
+	}
+
+	list_del(&chan->head);
+	kfree(chan);
+}
+
+void
+nouveau_abi16_fini(struct nouveau_abi16 *abi16)
+{
+	struct nouveau_cli *cli = (void *)abi16->client;
+	struct nouveau_abi16_chan *chan, *temp;
+
+	/* cleanup channels */
+	list_for_each_entry_safe(chan, temp, &abi16->channels, head) {
+		nouveau_abi16_chan_fini(abi16, chan);
+	}
+
+	/* destroy the device object */
+	nouveau_object_del(abi16->client, NVDRM_CLIENT, NVDRM_DEVICE);
+
+	kfree(cli->abi16);
+	cli->abi16 = NULL;
+}
 
 int
 nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_timer *ptimer = nouveau_timer(device);
 	struct drm_nouveau_getparam *getparam = data;
 
 	switch (getparam->param) {
 	case NOUVEAU_GETPARAM_CHIPSET_ID:
-		getparam->value = dev_priv->chipset;
+		getparam->value = device->chipset;
 		break;
 	case NOUVEAU_GETPARAM_PCI_VENDOR:
 		getparam->value = dev->pci_vendor;
@@ -55,16 +185,16 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 			getparam->value = 2;
 		break;
 	case NOUVEAU_GETPARAM_FB_SIZE:
-		getparam->value = dev_priv->fb_available_size;
+		getparam->value = drm->gem.vram_available;
 		break;
 	case NOUVEAU_GETPARAM_AGP_SIZE:
-		getparam->value = dev_priv->gart_info.aper_size;
+		getparam->value = drm->gem.gart_available;
 		break;
 	case NOUVEAU_GETPARAM_VM_VRAM_BASE:
 		getparam->value = 0; /* deprecated */
 		break;
 	case NOUVEAU_GETPARAM_PTIMER_TIME:
-		getparam->value = dev_priv->engine.timer.read(dev);
+		getparam->value = ptimer->read(ptimer);
 		break;
 	case NOUVEAU_GETPARAM_HAS_BO_USAGE:
 		getparam->value = 1;
@@ -76,13 +206,13 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
 		/* NV40 and NV50 versions are quite different, but register
 		 * address is the same. User is supposed to know the card
 		 * family anyway... */
-		if (dev_priv->chipset >= 0x40) {
-			getparam->value = nv_rd32(dev, NV40_PMC_GRAPH_UNITS);
+		if (device->chipset >= 0x40) {
+			getparam->value = nv_rd32(device, 0x001540);
 			break;
 		}
 		/* FALLTHRU */
 	default:
-		NV_DEBUG(dev, "unknown parameter %lld\n", getparam->param);
+		nv_debug(device, "unknown parameter %lld\n", getparam->param);
 		return -EINVAL;
 	}
 
@@ -98,148 +228,252 @@ nouveau_abi16_ioctl_setparam(ABI16_IOCTL_ARGS)
 int
 nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct drm_nouveau_channel_alloc *init = data;
-	struct nouveau_channel *chan;
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_abi16_chan *chan;
+	struct nouveau_client *client;
+	struct nouveau_device *device;
+	struct nouveau_instmem *imem;
+	struct nouveau_fb *pfb;
 	int ret;
 
-	if (!dev_priv->eng[NVOBJ_ENGINE_GR])
-		return -ENODEV;
+	if (unlikely(!abi16))
+		return -ENOMEM;
+	client = nv_client(abi16->client);
 
 	if (init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0)
-		return -EINVAL;
+		return nouveau_abi16_put(abi16, -EINVAL);
+
+	device = nv_device(abi16->device);
+	imem   = nouveau_instmem(device);
+	pfb    = nouveau_fb(device);
+
+	/* allocate "abi16 channel" data and make up a handle for it */
+	init->channel = ffsll(~abi16->handles);
+	if (!init->channel--)
+		return nouveau_abi16_put(abi16, -ENOSPC);
+
+	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return nouveau_abi16_put(abi16, -ENOMEM);
+
+	INIT_LIST_HEAD(&chan->notifiers);
+	list_add(&chan->head, &abi16->channels);
+	abi16->handles |= (1 << init->channel);
+
+	/* create channel object and initialise dma and fence management */
+	if (device->card_type >= NV_E0) {
+		init->fb_ctxdma_handle = NVE0_CHANNEL_IND_ENGINE_GR;
+		init->tt_ctxdma_handle = 0;
+	}
 
-	ret = nouveau_channel_alloc(dev, &chan, file_priv,
-				    init->fb_ctxdma_handle,
-				    init->tt_ctxdma_handle);
+	ret = nouveau_channel_new(drm, cli, NVDRM_DEVICE, NVDRM_CHAN |
+				  init->channel, init->fb_ctxdma_handle,
+				  init->tt_ctxdma_handle, &chan->chan);
 	if (ret)
-		return ret;
-	init->channel  = chan->id;
-
-	if (nouveau_vram_pushbuf == 0) {
-		if (chan->dma.ib_max)
-			init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM |
-						NOUVEAU_GEM_DOMAIN_GART;
-		else if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_VRAM)
-			init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM;
-		else
-			init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
-	} else {
+		goto done;
+
+	if (device->card_type >= NV_50)
+		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM |
+					NOUVEAU_GEM_DOMAIN_GART;
+	else
+	if (chan->chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM)
 		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_VRAM;
-	}
+	else
+		init->pushbuf_domains = NOUVEAU_GEM_DOMAIN_GART;
 
-	if (dev_priv->card_type < NV_C0) {
+	if (device->card_type < NV_C0) {
 		init->subchan[0].handle = 0x00000000;
 		init->subchan[0].grclass = 0x0000;
 		init->subchan[1].handle = NvSw;
-		init->subchan[1].grclass = NV_SW;
+		init->subchan[1].grclass = 0x506e;
 		init->nr_subchan = 2;
 	}
 
 	/* Named memory object area */
-	ret = drm_gem_handle_create(file_priv, chan->notifier_bo->gem,
+	ret = nouveau_gem_new(dev, PAGE_SIZE, 0, NOUVEAU_GEM_DOMAIN_GART,
+			      0, 0, &chan->ntfy);
+	if (ret == 0)
+		ret = nouveau_bo_pin(chan->ntfy, TTM_PL_FLAG_TT);
+	if (ret)
+		goto done;
+
+	if (device->card_type >= NV_50) {
+		ret = nouveau_bo_vma_add(chan->ntfy, client->vm,
+					&chan->ntfy_vma);
+		if (ret)
+			goto done;
+	}
+
+	ret = drm_gem_handle_create(file_priv, chan->ntfy->gem,
 				    &init->notifier_handle);
+	if (ret)
+		goto done;
 
-	if (ret == 0)
-		atomic_inc(&chan->users); /* userspace reference */
-	nouveau_channel_put(&chan);
-	return ret;
+	ret = nouveau_mm_init(&chan->heap, 0, PAGE_SIZE, 1);
+done:
+	if (ret)
+		nouveau_abi16_chan_fini(abi16, chan);
+	return nouveau_abi16_put(abi16, ret);
 }
 
+
 int
 nouveau_abi16_ioctl_channel_free(ABI16_IOCTL_ARGS)
 {
 	struct drm_nouveau_channel_free *req = data;
-	struct nouveau_channel *chan;
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_abi16_chan *chan;
+	int ret = -ENOENT;
 
-	chan = nouveau_channel_get(file_priv, req->channel);
-	if (IS_ERR(chan))
-		return PTR_ERR(chan);
+	if (unlikely(!abi16))
+		return -ENOMEM;
 
-	list_del(&chan->list);
-	atomic_dec(&chan->users);
-	nouveau_channel_put(&chan);
-	return 0;
+	list_for_each_entry(chan, &abi16->channels, head) {
+		if (chan->chan->handle == (NVDRM_CHAN | req->channel)) {
+			nouveau_abi16_chan_fini(abi16, chan);
+			return nouveau_abi16_put(abi16, 0);
+		}
+	}
+
+	return nouveau_abi16_put(abi16, ret);
 }
 
 int
 nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS)
 {
 	struct drm_nouveau_grobj_alloc *init = data;
-	struct nouveau_channel *chan;
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_object *object;
 	int ret;
 
+	if (unlikely(!abi16))
+		return -ENOMEM;
+
 	if (init->handle == ~0)
-		return -EINVAL;
+		return nouveau_abi16_put(abi16, -EINVAL);
 
 	/* compatibility with userspace that assumes 506e for all chipsets */
 	if (init->class == 0x506e) {
-		init->class = nouveau_software_class(dev);
+		init->class = nouveau_abi16_swclass(drm);
 		if (init->class == 0x906e)
-			return 0;
-	} else
-	if (init->class == 0x906e) {
-		NV_DEBUG(dev, "906e not supported yet\n");
-		return -EINVAL;
-	}
-
-	chan = nouveau_channel_get(file_priv, init->channel);
-	if (IS_ERR(chan))
-		return PTR_ERR(chan);
-
-	if (nouveau_ramht_find(chan, init->handle)) {
-		ret = -EEXIST;
-		goto out;
-	}
-
-	ret = nouveau_gpuobj_gr_new(chan, init->handle, init->class);
-	if (ret) {
-		NV_ERROR(dev, "Error creating object: %d (%d/0x%08x)\n",
-			 ret, init->channel, init->handle);
+			return nouveau_abi16_put(abi16, 0);
 	}
 
-out:
-	nouveau_channel_put(&chan);
-	return ret;
+	ret = nouveau_object_new(abi16->client, NVDRM_CHAN | init->channel,
+				  init->handle, init->class, NULL, 0, &object);
+	return nouveau_abi16_put(abi16, ret);
 }
 
 int
 nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct drm_nouveau_notifierobj_alloc *na = data;
-	struct nouveau_channel *chan;
+	struct drm_nouveau_notifierobj_alloc *info = data;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_abi16_chan *chan, *temp;
+	struct nouveau_abi16_ntfy *ntfy;
+	struct nouveau_object *object;
+	struct nv_dma_class args;
 	int ret;
 
+	if (unlikely(!abi16))
+		return -ENOMEM;
+
 	/* completely unnecessary for these chipsets... */
-	if (unlikely(dev_priv->card_type >= NV_C0))
-		return -EINVAL;
+	if (unlikely(nv_device(abi16->device)->card_type >= NV_C0))
+		return nouveau_abi16_put(abi16, -EINVAL);
 
-	chan = nouveau_channel_get(file_priv, na->channel);
-	if (IS_ERR(chan))
-		return PTR_ERR(chan);
+	list_for_each_entry_safe(chan, temp, &abi16->channels, head) {
+		if (chan->chan->handle == (NVDRM_CHAN | info->channel))
+			break;
+		chan = NULL;
+	}
 
-	ret = nouveau_notifier_alloc(chan, na->handle, na->size, 0, 0x1000,
-				     &na->offset);
-	nouveau_channel_put(&chan);
-	return ret;
+	if (!chan)
+		return nouveau_abi16_put(abi16, -ENOENT);
+
+	ntfy = kzalloc(sizeof(*ntfy), GFP_KERNEL);
+	if (!ntfy)
+		return nouveau_abi16_put(abi16, -ENOMEM);
+
+	list_add(&ntfy->head, &chan->notifiers);
+	ntfy->handle = info->handle;
+
+	ret = nouveau_mm_head(&chan->heap, 1, info->size, info->size, 1,
+			      &ntfy->node);
+	if (ret)
+		goto done;
+
+	args.start = ntfy->node->offset;
+	args.limit = ntfy->node->offset + ntfy->node->length - 1;
+	if (device->card_type >= NV_50) {
+		args.flags  = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+		args.start += chan->ntfy_vma.offset;
+		args.limit += chan->ntfy_vma.offset;
+	} else
+	if (drm->agp.stat == ENABLED) {
+		args.flags  = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
+		args.start += drm->agp.base + chan->ntfy->bo.offset;
+		args.limit += drm->agp.base + chan->ntfy->bo.offset;
+	} else {
+		args.flags  = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
+		args.start += chan->ntfy->bo.offset;
+		args.limit += chan->ntfy->bo.offset;
+	}
+
+	ret = nouveau_object_new(abi16->client, chan->chan->handle,
+				 ntfy->handle, 0x003d, &args,
+				 sizeof(args), &object);
+	if (ret)
+		goto done;
+
+done:
+	if (ret)
+		nouveau_abi16_ntfy_fini(chan, ntfy);
+	return nouveau_abi16_put(abi16, ret);
 }
 
 int
 nouveau_abi16_ioctl_gpuobj_free(ABI16_IOCTL_ARGS)
 {
-	struct drm_nouveau_gpuobj_free *objfree = data;
-	struct nouveau_channel *chan;
+	struct drm_nouveau_gpuobj_free *fini = data;
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_abi16_chan *chan, *temp;
+	struct nouveau_abi16_ntfy *ntfy;
 	int ret;
 
-	chan = nouveau_channel_get(file_priv, objfree->channel);
-	if (IS_ERR(chan))
-		return PTR_ERR(chan);
+	if (unlikely(!abi16))
+		return -ENOMEM;
+
+	list_for_each_entry_safe(chan, temp, &abi16->channels, head) {
+		if (chan->chan->handle == (NVDRM_CHAN | fini->channel))
+			break;
+		chan = NULL;
+	}
+
+	if (!chan)
+		return nouveau_abi16_put(abi16, -ENOENT);
 
-	/* Synchronize with the user channel */
-	nouveau_channel_idle(chan);
+	/* synchronize with the user channel and destroy the gpu object */
+	nouveau_channel_idle(chan->chan);
 
-	ret = nouveau_ramht_remove(chan, objfree->handle);
-	nouveau_channel_put(&chan);
-	return ret;
+	ret = nouveau_object_del(abi16->client, chan->chan->handle, fini->handle);
+	if (ret)
+		return nouveau_abi16_put(abi16, ret);
+
+	/* cleanup extra state if this object was a notifier */
+	list_for_each_entry(ntfy, &chan->notifiers, head) {
+		if (ntfy->handle == fini->handle) {
+			nouveau_mm_free(&chan->heap, &ntfy->node);
+			list_del(&ntfy->head);
+			break;
+		}
+	}
+
+	return nouveau_abi16_put(abi16, 0);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.h b/drivers/gpu/drm/nouveau/nouveau_abi16.h
index e6328b008a8..90004081a50 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.h
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.h
@@ -3,6 +3,7 @@
 
 #define ABI16_IOCTL_ARGS                                                       \
 	struct drm_device *dev, void *data, struct drm_file *file_priv
+
 int nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_setparam(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS);
@@ -11,6 +12,37 @@ int nouveau_abi16_ioctl_grobj_alloc(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_notifierobj_alloc(ABI16_IOCTL_ARGS);
 int nouveau_abi16_ioctl_gpuobj_free(ABI16_IOCTL_ARGS);
 
+struct nouveau_abi16_ntfy {
+	struct list_head head;
+	struct nouveau_mm_node *node;
+	u32 handle;
+};
+
+struct nouveau_abi16_chan {
+	struct list_head head;
+	struct nouveau_channel *chan;
+	struct list_head notifiers;
+	struct nouveau_bo *ntfy;
+	struct nouveau_vma ntfy_vma;
+	struct nouveau_mm  heap;
+};
+
+struct nouveau_abi16 {
+	struct nouveau_object *client;
+	struct nouveau_object *device;
+	struct list_head channels;
+	u64 handles;
+};
+
+struct nouveau_drm;
+struct nouveau_abi16 *nouveau_abi16_get(struct drm_file *, struct drm_device *);
+int  nouveau_abi16_put(struct nouveau_abi16 *, int);
+void nouveau_abi16_fini(struct nouveau_abi16 *);
+u16  nouveau_abi16_swclass(struct nouveau_drm *);
+
+#define NOUVEAU_GEM_DOMAIN_VRAM      (1 << 1)
+#define NOUVEAU_GEM_DOMAIN_GART      (1 << 2)
+
 struct drm_nouveau_channel_alloc {
 	uint32_t     fb_ctxdma_handle;
 	uint32_t     tt_ctxdma_handle;
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 26ebffebe71..48783e14114 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -7,17 +7,13 @@
 #include <acpi/acpi.h>
 #include <linux/mxm-wmi.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-#include "drm_crtc_helper.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nv50_display.h"
-#include "nouveau_connector.h"
-
 #include <linux/vga_switcheroo.h>
 
+#include <drm/drm_edid.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_acpi.h"
+
 #define NOUVEAU_DSM_LED 0x02
 #define NOUVEAU_DSM_LED_STATE 0x00
 #define NOUVEAU_DSM_LED_OFF 0x10
@@ -390,10 +386,9 @@ int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len)
 	return nouveau_rom_call(nouveau_dsm_priv.rom_handle, bios, offset, len);
 }
 
-int
+void *
 nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
 {
-	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct acpi_device *acpidev;
 	acpi_handle handle;
 	int type, ret;
@@ -405,21 +400,20 @@ nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
 		type = ACPI_VIDEO_DISPLAY_LCD;
 		break;
 	default:
-		return -EINVAL;
+		return NULL;
 	}
 
 	handle = DEVICE_ACPI_HANDLE(&dev->pdev->dev);
 	if (!handle)
-		return -ENODEV;
+		return NULL;
 
 	ret = acpi_bus_get_device(handle, &acpidev);
 	if (ret)
-		return -ENODEV;
+		return NULL;
 
 	ret = acpi_video_get_edid(acpidev, type, -1, &edid);
 	if (ret < 0)
-		return ret;
+		return NULL;
 
-	nv_connector->edid = kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
-	return 0;
+	return kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.h b/drivers/gpu/drm/nouveau/nouveau_acpi.h
new file mode 100644
index 00000000000..08af67722b5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.h
@@ -0,0 +1,22 @@
+#ifndef __NOUVEAU_ACPI_H__
+#define __NOUVEAU_ACPI_H__
+
+#define ROM_BIOS_PAGE 4096
+
+#if defined(CONFIG_ACPI)
+void nouveau_register_dsm_handler(void);
+void nouveau_unregister_dsm_handler(void);
+void nouveau_switcheroo_optimus_dsm(void);
+int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
+bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
+void *nouveau_acpi_edid(struct drm_device *, struct drm_connector *);
+#else
+static inline void nouveau_register_dsm_handler(void) {}
+static inline void nouveau_unregister_dsm_handler(void) {}
+static inline void nouveau_switcheroo_optimus_dsm(void) {}
+static inline bool nouveau_acpi_rom_supported(struct pci_dev *pdev) { return false; }
+static inline int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len) { return -EINVAL; }
+static inline void *nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector) { return NULL; }
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_agp.c b/drivers/gpu/drm/nouveau/nouveau_agp.c
new file mode 100644
index 00000000000..d28430cd2ba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_agp.c
@@ -0,0 +1,152 @@
+#include <linux/module.h>
+
+#include <core/device.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_agp.h"
+#include "nouveau_reg.h"
+
+#if __OS_HAS_AGP
+MODULE_PARM_DESC(agpmode, "AGP mode (0 to disable AGP)");
+static int nouveau_agpmode = -1;
+module_param_named(agpmode, nouveau_agpmode, int, 0400);
+
+static unsigned long
+get_agp_mode(struct nouveau_drm *drm, unsigned long mode)
+{
+	struct nouveau_device *device = nv_device(drm->device);
+
+	/*
+	 * FW seems to be broken on nv18, it makes the card lock up
+	 * randomly.
+	 */
+	if (device->chipset == 0x18)
+		mode &= ~PCI_AGP_COMMAND_FW;
+
+	/*
+	 * AGP mode set in the command line.
+	 */
+	if (nouveau_agpmode > 0) {
+		bool agpv3 = mode & 0x8;
+		int rate = agpv3 ? nouveau_agpmode / 4 : nouveau_agpmode;
+
+		mode = (mode & ~0x7) | (rate & 0x7);
+	}
+
+	return mode;
+}
+
+static bool
+nouveau_agp_enabled(struct nouveau_drm *drm)
+{
+	struct drm_device *dev = drm->dev;
+
+	if (!drm_pci_device_is_agp(dev) || !dev->agp)
+		return false;
+
+	if (drm->agp.stat == UNKNOWN) {
+		if (!nouveau_agpmode)
+			return false;
+		return true;
+	}
+
+	return (drm->agp.stat == ENABLED);
+}
+#endif
+
+void
+nouveau_agp_reset(struct nouveau_drm *drm)
+{
+#if __OS_HAS_AGP
+	struct nouveau_device *device = nv_device(drm->device);
+	struct drm_device *dev = drm->dev;
+	u32 save[2];
+	int ret;
+
+	if (!nouveau_agp_enabled(drm))
+		return;
+
+	/* First of all, disable fast writes, otherwise if it's
+	 * already enabled in the AGP bridge and we disable the card's
+	 * AGP controller we might be locking ourselves out of it. */
+	if ((nv_rd32(device, NV04_PBUS_PCI_NV_19) |
+	     dev->agp->mode) & PCI_AGP_COMMAND_FW) {
+		struct drm_agp_info info;
+		struct drm_agp_mode mode;
+
+		ret = drm_agp_info(dev, &info);
+		if (ret)
+			return;
+
+		mode.mode  = get_agp_mode(drm, info.mode);
+		mode.mode &= ~PCI_AGP_COMMAND_FW;
+
+		ret = drm_agp_enable(dev, mode);
+		if (ret)
+			return;
+	}
+
+
+	/* clear busmaster bit, and disable AGP */
+	save[0] = nv_mask(device, NV04_PBUS_PCI_NV_1, 0x00000004, 0x00000000);
+	nv_wr32(device, NV04_PBUS_PCI_NV_19, 0);
+
+	/* reset PGRAPH, PFIFO and PTIMER */
+	save[1] = nv_mask(device, 0x000200, 0x00011100, 0x00000000);
+	nv_mask(device, 0x000200, 0x00011100, save[1]);
+
+	/* and restore bustmaster bit (gives effect of resetting AGP) */
+	nv_wr32(device, NV04_PBUS_PCI_NV_1, save[0]);
+#endif
+}
+
+void
+nouveau_agp_init(struct nouveau_drm *drm)
+{
+#if __OS_HAS_AGP
+	struct nouveau_device *device = nv_device(drm->device);
+	struct drm_device *dev = drm->dev;
+	struct drm_agp_info info;
+	struct drm_agp_mode mode;
+	int ret;
+
+	if (!nouveau_agp_enabled(drm))
+		return;
+	drm->agp.stat = DISABLE;
+
+	ret = drm_agp_acquire(dev);
+	if (ret) {
+		nv_error(device, "unable to acquire AGP: %d\n", ret);
+		return;
+	}
+
+	ret = drm_agp_info(dev, &info);
+	if (ret) {
+		nv_error(device, "unable to get AGP info: %d\n", ret);
+		return;
+	}
+
+	/* see agp.h for the AGPSTAT_* modes available */
+	mode.mode = get_agp_mode(drm, info.mode);
+
+	ret = drm_agp_enable(dev, mode);
+	if (ret) {
+		nv_error(device, "unable to enable AGP: %d\n", ret);
+		return;
+	}
+
+	drm->agp.stat = ENABLED;
+	drm->agp.base = info.aperture_base;
+	drm->agp.size = info.aperture_size;
+#endif
+}
+
+void
+nouveau_agp_fini(struct nouveau_drm *drm)
+{
+#if __OS_HAS_AGP
+	struct drm_device *dev = drm->dev;
+	if (dev->agp && dev->agp->acquired)
+		drm_agp_release(dev);
+#endif
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_agp.h b/drivers/gpu/drm/nouveau/nouveau_agp.h
new file mode 100644
index 00000000000..b55c0865296
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_agp.h
@@ -0,0 +1,10 @@
+#ifndef __NOUVEAU_AGP_H__
+#define __NOUVEAU_AGP_H__
+
+struct nouveau_drm;
+
+void nouveau_agp_reset(struct nouveau_drm *);
+void nouveau_agp_init(struct nouveau_drm *);
+void nouveau_agp_fini(struct nouveau_drm *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index fa22b28e877..f65b20a375f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -33,8 +33,6 @@
 #include <linux/backlight.h>
 #include <linux/acpi.h>
 
-#include "drmP.h"
-#include "nouveau_drv.h"
 #include "nouveau_drm.h"
 #include "nouveau_reg.h"
 #include "nouveau_encoder.h"
@@ -42,9 +40,10 @@
 static int
 nv40_get_intensity(struct backlight_device *bd)
 {
-	struct drm_device *dev = bl_get_data(bd);
-	int val = (nv_rd32(dev, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK)
-									>> 16;
+	struct nouveau_drm *drm = bl_get_data(bd);
+	struct nouveau_device *device = nv_device(drm->device);
+	int val = (nv_rd32(device, NV40_PMC_BACKLIGHT) &
+				   NV40_PMC_BACKLIGHT_MASK) >> 16;
 
 	return val;
 }
@@ -52,11 +51,12 @@ nv40_get_intensity(struct backlight_device *bd)
 static int
 nv40_set_intensity(struct backlight_device *bd)
 {
-	struct drm_device *dev = bl_get_data(bd);
+	struct nouveau_drm *drm = bl_get_data(bd);
+	struct nouveau_device *device = nv_device(drm->device);
 	int val = bd->props.brightness;
-	int reg = nv_rd32(dev, NV40_PMC_BACKLIGHT);
+	int reg = nv_rd32(device, NV40_PMC_BACKLIGHT);
 
-	nv_wr32(dev, NV40_PMC_BACKLIGHT,
+	nv_wr32(device, NV40_PMC_BACKLIGHT,
 		 (val << 16) | (reg & ~NV40_PMC_BACKLIGHT_MASK));
 
 	return 0;
@@ -71,23 +71,20 @@ static const struct backlight_ops nv40_bl_ops = {
 static int
 nv40_backlight_init(struct drm_connector *connector)
 {
-	struct drm_device *dev = connector->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	struct backlight_properties props;
 	struct backlight_device *bd;
 
-	if (!(nv_rd32(dev, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
+	if (!(nv_rd32(device, NV40_PMC_BACKLIGHT) & NV40_PMC_BACKLIGHT_MASK))
 		return 0;
 
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_RAW;
 	props.max_brightness = 31;
-	bd = backlight_device_register("nv_backlight", &connector->kdev, dev,
+	bd = backlight_device_register("nv_backlight", &connector->kdev, drm,
 				       &nv40_bl_ops, &props);
-	if (IS_ERR(bd))
-		return PTR_ERR(bd);
-
-	dev_priv->backlight = bd;
+	drm->backlight = bd;
 	bd->props.brightness = nv40_get_intensity(bd);
 	backlight_update_status(bd);
 
@@ -98,12 +95,13 @@ static int
 nv50_get_intensity(struct backlight_device *bd)
 {
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
-	struct drm_device *dev = nv_encoder->base.base.dev;
+	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int or = nv_encoder->or;
 	u32 div = 1025;
 	u32 val;
 
-	val  = nv_rd32(dev, NV50_PDISP_SOR_PWM_CTL(or));
+	val  = nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
 	val &= NV50_PDISP_SOR_PWM_CTL_VAL;
 	return ((val * 100) + (div / 2)) / div;
 }
@@ -112,13 +110,14 @@ static int
 nv50_set_intensity(struct backlight_device *bd)
 {
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
-	struct drm_device *dev = nv_encoder->base.base.dev;
+	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int or = nv_encoder->or;
 	u32 div = 1025;
 	u32 val = (bd->props.brightness * div) / 100;
 
-	nv_wr32(dev, NV50_PDISP_SOR_PWM_CTL(or),
-		     NV50_PDISP_SOR_PWM_CTL_NEW | val);
+	nv_wr32(device, NV50_PDISP_SOR_PWM_CTL(or),
+			NV50_PDISP_SOR_PWM_CTL_NEW | val);
 	return 0;
 }
 
@@ -132,12 +131,13 @@ static int
 nva3_get_intensity(struct backlight_device *bd)
 {
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
-	struct drm_device *dev = nv_encoder->base.base.dev;
+	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int or = nv_encoder->or;
 	u32 div, val;
 
-	div  = nv_rd32(dev, NV50_PDISP_SOR_PWM_DIV(or));
-	val  = nv_rd32(dev, NV50_PDISP_SOR_PWM_CTL(or));
+	div  = nv_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
+	val  = nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(or));
 	val &= NVA3_PDISP_SOR_PWM_CTL_VAL;
 	if (div && div >= val)
 		return ((val * 100) + (div / 2)) / div;
@@ -149,16 +149,17 @@ static int
 nva3_set_intensity(struct backlight_device *bd)
 {
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
-	struct drm_device *dev = nv_encoder->base.base.dev;
+	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int or = nv_encoder->or;
 	u32 div, val;
 
-	div = nv_rd32(dev, NV50_PDISP_SOR_PWM_DIV(or));
+	div = nv_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
 	val = (bd->props.brightness * div) / 100;
 	if (div) {
-		nv_wr32(dev, NV50_PDISP_SOR_PWM_CTL(or), val |
-			     NV50_PDISP_SOR_PWM_CTL_NEW |
-			     NVA3_PDISP_SOR_PWM_CTL_UNK);
+		nv_wr32(device, NV50_PDISP_SOR_PWM_CTL(or), val |
+				NV50_PDISP_SOR_PWM_CTL_NEW |
+				NVA3_PDISP_SOR_PWM_CTL_UNK);
 		return 0;
 	}
 
@@ -174,26 +175,26 @@ static const struct backlight_ops nva3_bl_ops = {
 static int
 nv50_backlight_init(struct drm_connector *connector)
 {
-	struct drm_device *dev = connector->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	struct nouveau_encoder *nv_encoder;
 	struct backlight_properties props;
 	struct backlight_device *bd;
 	const struct backlight_ops *ops;
 
-	nv_encoder = find_encoder(connector, OUTPUT_LVDS);
+	nv_encoder = find_encoder(connector, DCB_OUTPUT_LVDS);
 	if (!nv_encoder) {
-		nv_encoder = find_encoder(connector, OUTPUT_DP);
+		nv_encoder = find_encoder(connector, DCB_OUTPUT_DP);
 		if (!nv_encoder)
 			return -ENODEV;
 	}
 
-	if (!nv_rd32(dev, NV50_PDISP_SOR_PWM_CTL(nv_encoder->or)))
+	if (!nv_rd32(device, NV50_PDISP_SOR_PWM_CTL(nv_encoder->or)))
 		return 0;
 
-	if (dev_priv->chipset <= 0xa0 ||
-	    dev_priv->chipset == 0xaa ||
-	    dev_priv->chipset == 0xac)
+	if (device->chipset <= 0xa0 ||
+	    device->chipset == 0xaa ||
+	    device->chipset == 0xac)
 		ops = &nv50_bl_ops;
 	else
 		ops = &nva3_bl_ops;
@@ -206,7 +207,7 @@ nv50_backlight_init(struct drm_connector *connector)
 	if (IS_ERR(bd))
 		return PTR_ERR(bd);
 
-	dev_priv->backlight = bd;
+	drm->backlight = bd;
 	bd->props.brightness = bd->ops->get_brightness(bd);
 	backlight_update_status(bd);
 	return 0;
@@ -215,12 +216,13 @@ nv50_backlight_init(struct drm_connector *connector)
 int
 nouveau_backlight_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	struct drm_connector *connector;
 
 #ifdef CONFIG_ACPI
 	if (acpi_video_backlight_support()) {
-		NV_INFO(dev, "ACPI backlight interface available, "
+		NV_INFO(drm, "ACPI backlight interface available, "
 			     "not registering our own\n");
 		return 0;
 	}
@@ -231,7 +233,7 @@ nouveau_backlight_init(struct drm_device *dev)
 		    connector->connector_type != DRM_MODE_CONNECTOR_eDP)
 			continue;
 
-		switch (dev_priv->card_type) {
+		switch (device->card_type) {
 		case NV_40:
 			return nv40_backlight_init(connector);
 		case NV_50:
@@ -248,10 +250,10 @@ nouveau_backlight_init(struct drm_device *dev)
 void
 nouveau_backlight_exit(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	if (dev_priv->backlight) {
-		backlight_device_unregister(dev_priv->backlight);
-		dev_priv->backlight = NULL;
+	if (drm->backlight) {
+		backlight_device_unregister(drm->backlight);
+		drm->backlight = NULL;
 	}
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index a0a3fe3c016..09fdef23588 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -22,12 +22,14 @@
  * SOFTWARE.
  */
 
-#include "drmP.h"
-#define NV_DEBUG_NOTRACE
-#include "nouveau_drv.h"
+#include <subdev/bios.h>
+
+#include <drm/drmP.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_hw.h"
 #include "nouveau_encoder.h"
-#include "nouveau_gpio.h"
 
 #include <linux/io-mapping.h>
 #include <linux/firmware.h>
@@ -65,3677 +67,6 @@ static bool nv_cksum(const uint8_t *data, unsigned int length)
 	return false;
 }
 
-static int
-score_vbios(struct nvbios *bios, const bool writeable)
-{
-	if (!bios->data || bios->data[0] != 0x55 || bios->data[1] != 0xAA) {
-		NV_TRACEWARN(bios->dev, "... BIOS signature not found\n");
-		return 0;
-	}
-
-	if (nv_cksum(bios->data, bios->data[2] * 512)) {
-		NV_TRACEWARN(bios->dev, "... BIOS checksum invalid\n");
-		/* if a ro image is somewhat bad, it's probably all rubbish */
-		return writeable ? 2 : 1;
-	}
-
-	NV_TRACE(bios->dev, "... appears to be valid\n");
-	return 3;
-}
-
-static void
-bios_shadow_prom(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 pcireg, access;
-	u16 pcir;
-	int i;
-
-	/* enable access to rom */
-	if (dev_priv->card_type >= NV_50)
-		pcireg = 0x088050;
-	else
-		pcireg = NV_PBUS_PCI_NV_20;
-	access = nv_mask(dev, pcireg, 0x00000001, 0x00000000);
-
-	/* bail if no rom signature, with a workaround for a PROM reading
-	 * issue on some chipsets.  the first read after a period of
-	 * inactivity returns the wrong result, so retry the first header
-	 * byte a few times before giving up as a workaround
-	 */
-	i = 16;
-	do {
-		if (nv_rd08(dev, NV_PROM_OFFSET + 0) == 0x55)
-			break;
-	} while (i--);
-
-	if (!i || nv_rd08(dev, NV_PROM_OFFSET + 1) != 0xaa)
-		goto out;
-
-	/* additional check (see note below) - read PCI record header */
-	pcir = nv_rd08(dev, NV_PROM_OFFSET + 0x18) |
-	       nv_rd08(dev, NV_PROM_OFFSET + 0x19) << 8;
-	if (nv_rd08(dev, NV_PROM_OFFSET + pcir + 0) != 'P' ||
-	    nv_rd08(dev, NV_PROM_OFFSET + pcir + 1) != 'C' ||
-	    nv_rd08(dev, NV_PROM_OFFSET + pcir + 2) != 'I' ||
-	    nv_rd08(dev, NV_PROM_OFFSET + pcir + 3) != 'R')
-		goto out;
-
-	/* read entire bios image to system memory */
-	bios->length = nv_rd08(dev, NV_PROM_OFFSET + 2) * 512;
-	bios->data = kmalloc(bios->length, GFP_KERNEL);
-	if (bios->data) {
-		for (i = 0; i < bios->length; i++)
-			bios->data[i] = nv_rd08(dev, NV_PROM_OFFSET + i);
-	}
-
-out:
-	/* disable access to rom */
-	nv_wr32(dev, pcireg, access);
-}
-
-static void
-bios_shadow_pramin(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 bar0 = 0;
-	int i;
-
-	if (dev_priv->card_type >= NV_50) {
-		u64 addr = (u64)(nv_rd32(dev, 0x619f04) & 0xffffff00) << 8;
-		if (!addr) {
-			addr  = (u64)nv_rd32(dev, 0x001700) << 16;
-			addr += 0xf0000;
-		}
-
-		bar0 = nv_mask(dev, 0x001700, 0xffffffff, addr >> 16);
-	}
-
-	/* bail if no rom signature */
-	if (nv_rd08(dev, NV_PRAMIN_OFFSET + 0) != 0x55 ||
-	    nv_rd08(dev, NV_PRAMIN_OFFSET + 1) != 0xaa)
-		goto out;
-
-	bios->length = nv_rd08(dev, NV_PRAMIN_OFFSET + 2) * 512;
-	bios->data = kmalloc(bios->length, GFP_KERNEL);
-	if (bios->data) {
-		for (i = 0; i < bios->length; i++)
-			bios->data[i] = nv_rd08(dev, NV_PRAMIN_OFFSET + i);
-	}
-
-out:
-	if (dev_priv->card_type >= NV_50)
-		nv_wr32(dev, 0x001700, bar0);
-}
-
-static void
-bios_shadow_pci(struct nvbios *bios)
-{
-	struct pci_dev *pdev = bios->dev->pdev;
-	size_t length;
-
-	if (!pci_enable_rom(pdev)) {
-		void __iomem *rom = pci_map_rom(pdev, &length);
-		if (rom && length) {
-			bios->data = kmalloc(length, GFP_KERNEL);
-			if (bios->data) {
-				memcpy_fromio(bios->data, rom, length);
-				bios->length = length;
-			}
-		}
-		if (rom)
-			pci_unmap_rom(pdev, rom);
-
-		pci_disable_rom(pdev);
-	}
-}
-
-static void
-bios_shadow_acpi(struct nvbios *bios)
-{
-	struct pci_dev *pdev = bios->dev->pdev;
-	int cnt = 65536 / ROM_BIOS_PAGE;
-	int ret;
-
-	if (!nouveau_acpi_rom_supported(pdev))
-		return;
-
-	bios->data = kmalloc(cnt * ROM_BIOS_PAGE, GFP_KERNEL);
-	if (!bios->data)
-		return;
-
-	bios->length = 0;
-	while (cnt--) {
-		ret = nouveau_acpi_get_bios_chunk(bios->data, bios->length,
-						  ROM_BIOS_PAGE);
-		if (ret != ROM_BIOS_PAGE)
-			return;
-
-		bios->length += ROM_BIOS_PAGE;
-	}
-}
-
-struct methods {
-	const char desc[8];
-	void (*shadow)(struct nvbios *);
-	const bool rw;
-	int score;
-	u32 size;
-	u8 *data;
-};
-
-static bool
-bios_shadow(struct drm_device *dev)
-{
-	struct methods shadow_methods[] = {
-		{ "PRAMIN", bios_shadow_pramin, true, 0, 0, NULL },
-		{ "PROM", bios_shadow_prom, false, 0, 0, NULL },
-		{ "ACPI", bios_shadow_acpi, true, 0, 0, NULL },
-		{ "PCIROM", bios_shadow_pci, true, 0, 0, NULL },
-		{}
-	};
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct methods *mthd, *best;
-	const struct firmware *fw;
-	char fname[32];
-	int ret;
-
-	if (nouveau_vbios) {
-		/* try to match one of the built-in methods */
-		mthd = shadow_methods;
-		do {
-			if (strcasecmp(nouveau_vbios, mthd->desc))
-				continue;
-			NV_INFO(dev, "VBIOS source: %s\n", mthd->desc);
-
-			mthd->shadow(bios);
-			mthd->score = score_vbios(bios, mthd->rw);
-			if (mthd->score)
-				return true;
-		} while ((++mthd)->shadow);
-
-		/* attempt to load firmware image */
-		snprintf(fname, sizeof(fname), "nouveau/%s", nouveau_vbios);
-		ret = request_firmware(&fw, fname, &dev->pdev->dev);
-		if (ret == 0) {
-			bios->length = fw->size;
-			bios->data   = kmemdup(fw->data, fw->size, GFP_KERNEL);
-			release_firmware(fw);
-
-			NV_INFO(dev, "VBIOS image: %s\n", nouveau_vbios);
-			if (score_vbios(bios, 1))
-				return true;
-
-			kfree(bios->data);
-			bios->data = NULL;
-		}
-
-		NV_ERROR(dev, "VBIOS source \'%s\' invalid\n", nouveau_vbios);
-	}
-
-	mthd = shadow_methods;
-	do {
-		NV_TRACE(dev, "Checking %s for VBIOS\n", mthd->desc);
-		mthd->shadow(bios);
-		mthd->score = score_vbios(bios, mthd->rw);
-		mthd->size = bios->length;
-		mthd->data = bios->data;
-		bios->data = NULL;
-	} while (mthd->score != 3 && (++mthd)->shadow);
-
-	mthd = shadow_methods;
-	best = mthd;
-	do {
-		if (mthd->score > best->score) {
-			kfree(best->data);
-			best = mthd;
-		}
-	} while ((++mthd)->shadow);
-
-	if (best->score) {
-		NV_TRACE(dev, "Using VBIOS from %s\n", best->desc);
-		bios->length = best->size;
-		bios->data = best->data;
-		return true;
-	}
-
-	NV_ERROR(dev, "No valid VBIOS image found\n");
-	return false;
-}
-
-struct init_tbl_entry {
-	char *name;
-	uint8_t id;
-	/* Return:
-	 *  > 0: success, length of opcode
-	 *    0: success, but abort further parsing of table (INIT_DONE etc)
-	 *  < 0: failure, table parsing will be aborted
-	 */
-	int (*handler)(struct nvbios *, uint16_t, struct init_exec *);
-};
-
-static int parse_init_table(struct nvbios *, uint16_t, struct init_exec *);
-
-#define MACRO_INDEX_SIZE	2
-#define MACRO_SIZE		8
-#define CONDITION_SIZE		12
-#define IO_FLAG_CONDITION_SIZE	9
-#define IO_CONDITION_SIZE	5
-#define MEM_INIT_SIZE		66
-
-static void still_alive(void)
-{
-#if 0
-	sync();
-	mdelay(2);
-#endif
-}
-
-static uint32_t
-munge_reg(struct nvbios *bios, uint32_t reg)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	struct dcb_entry *dcbent = bios->display.output;
-
-	if (dev_priv->card_type < NV_50)
-		return reg;
-
-	if (reg & 0x80000000) {
-		BUG_ON(bios->display.crtc < 0);
-		reg += bios->display.crtc * 0x800;
-	}
-
-	if (reg & 0x40000000) {
-		BUG_ON(!dcbent);
-
-		reg += (ffs(dcbent->or) - 1) * 0x800;
-		if ((reg & 0x20000000) && !(dcbent->sorconf.link & 1))
-			reg += 0x00000080;
-	}
-
-	reg &= ~0xe0000000;
-	return reg;
-}
-
-static int
-valid_reg(struct nvbios *bios, uint32_t reg)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	struct drm_device *dev = bios->dev;
-
-	/* C51 has misaligned regs on purpose. Marvellous */
-	if (reg & 0x2 ||
-	    (reg & 0x1 && dev_priv->vbios.chip_version != 0x51))
-		NV_ERROR(dev, "======= misaligned reg 0x%08X =======\n", reg);
-
-	/* warn on C51 regs that haven't been verified accessible in tracing */
-	if (reg & 0x1 && dev_priv->vbios.chip_version == 0x51 &&
-	    reg != 0x130d && reg != 0x1311 && reg != 0x60081d)
-		NV_WARN(dev, "=== C51 misaligned reg 0x%08X not verified ===\n",
-			reg);
-
-	if (reg >= (8*1024*1024)) {
-		NV_ERROR(dev, "=== reg 0x%08x out of mapped bounds ===\n", reg);
-		return 0;
-	}
-
-	return 1;
-}
-
-static bool
-valid_idx_port(struct nvbios *bios, uint16_t port)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	struct drm_device *dev = bios->dev;
-
-	/*
-	 * If adding more ports here, the read/write functions below will need
-	 * updating so that the correct mmio range (PRMCIO, PRMDIO, PRMVIO) is
-	 * used for the port in question
-	 */
-	if (dev_priv->card_type < NV_50) {
-		if (port == NV_CIO_CRX__COLOR)
-			return true;
-		if (port == NV_VIO_SRX)
-			return true;
-	} else {
-		if (port == NV_CIO_CRX__COLOR)
-			return true;
-	}
-
-	NV_ERROR(dev, "========== unknown indexed io port 0x%04X ==========\n",
-		 port);
-
-	return false;
-}
-
-static bool
-valid_port(struct nvbios *bios, uint16_t port)
-{
-	struct drm_device *dev = bios->dev;
-
-	/*
-	 * If adding more ports here, the read/write functions below will need
-	 * updating so that the correct mmio range (PRMCIO, PRMDIO, PRMVIO) is
-	 * used for the port in question
-	 */
-	if (port == NV_VIO_VSE2)
-		return true;
-
-	NV_ERROR(dev, "========== unknown io port 0x%04X ==========\n", port);
-
-	return false;
-}
-
-static uint32_t
-bios_rd32(struct nvbios *bios, uint32_t reg)
-{
-	uint32_t data;
-
-	reg = munge_reg(bios, reg);
-	if (!valid_reg(bios, reg))
-		return 0;
-
-	/*
-	 * C51 sometimes uses regs with bit0 set in the address. For these
-	 * cases there should exist a translation in a BIOS table to an IO
-	 * port address which the BIOS uses for accessing the reg
-	 *
-	 * These only seem to appear for the power control regs to a flat panel,
-	 * and the GPIO regs at 0x60081*.  In C51 mmio traces the normal regs
-	 * for 0x1308 and 0x1310 are used - hence the mask below.  An S3
-	 * suspend-resume mmio trace from a C51 will be required to see if this
-	 * is true for the power microcode in 0x14.., or whether the direct IO
-	 * port access method is needed
-	 */
-	if (reg & 0x1)
-		reg &= ~0x1;
-
-	data = nv_rd32(bios->dev, reg);
-
-	BIOSLOG(bios, "	Read:  Reg: 0x%08X, Data: 0x%08X\n", reg, data);
-
-	return data;
-}
-
-static void
-bios_wr32(struct nvbios *bios, uint32_t reg, uint32_t data)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-
-	reg = munge_reg(bios, reg);
-	if (!valid_reg(bios, reg))
-		return;
-
-	/* see note in bios_rd32 */
-	if (reg & 0x1)
-		reg &= 0xfffffffe;
-
-	LOG_OLD_VALUE(bios_rd32(bios, reg));
-	BIOSLOG(bios, "	Write: Reg: 0x%08X, Data: 0x%08X\n", reg, data);
-
-	if (dev_priv->vbios.execute) {
-		still_alive();
-		nv_wr32(bios->dev, reg, data);
-	}
-}
-
-static uint8_t
-bios_idxprt_rd(struct nvbios *bios, uint16_t port, uint8_t index)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	struct drm_device *dev = bios->dev;
-	uint8_t data;
-
-	if (!valid_idx_port(bios, port))
-		return 0;
-
-	if (dev_priv->card_type < NV_50) {
-		if (port == NV_VIO_SRX)
-			data = NVReadVgaSeq(dev, bios->state.crtchead, index);
-		else	/* assume NV_CIO_CRX__COLOR */
-			data = NVReadVgaCrtc(dev, bios->state.crtchead, index);
-	} else {
-		uint32_t data32;
-
-		data32 = bios_rd32(bios, NV50_PDISPLAY_VGACRTC(index & ~3));
-		data = (data32 >> ((index & 3) << 3)) & 0xff;
-	}
-
-	BIOSLOG(bios, "	Indexed IO read:  Port: 0x%04X, Index: 0x%02X, "
-		      "Head: 0x%02X, Data: 0x%02X\n",
-		port, index, bios->state.crtchead, data);
-	return data;
-}
-
-static void
-bios_idxprt_wr(struct nvbios *bios, uint16_t port, uint8_t index, uint8_t data)
-{
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	struct drm_device *dev = bios->dev;
-
-	if (!valid_idx_port(bios, port))
-		return;
-
-	/*
-	 * The current head is maintained in the nvbios member  state.crtchead.
-	 * We trap changes to CR44 and update the head variable and hence the
-	 * register set written.
-	 * As CR44 only exists on CRTC0, we update crtchead to head0 in advance
-	 * of the write, and to head1 after the write
-	 */
-	if (port == NV_CIO_CRX__COLOR && index == NV_CIO_CRE_44 &&
-	    data != NV_CIO_CRE_44_HEADB)
-		bios->state.crtchead = 0;
-
-	LOG_OLD_VALUE(bios_idxprt_rd(bios, port, index));
-	BIOSLOG(bios, "	Indexed IO write: Port: 0x%04X, Index: 0x%02X, "
-		      "Head: 0x%02X, Data: 0x%02X\n",
-		port, index, bios->state.crtchead, data);
-
-	if (bios->execute && dev_priv->card_type < NV_50) {
-		still_alive();
-		if (port == NV_VIO_SRX)
-			NVWriteVgaSeq(dev, bios->state.crtchead, index, data);
-		else	/* assume NV_CIO_CRX__COLOR */
-			NVWriteVgaCrtc(dev, bios->state.crtchead, index, data);
-	} else
-	if (bios->execute) {
-		uint32_t data32, shift = (index & 3) << 3;
-
-		still_alive();
-
-		data32  = bios_rd32(bios, NV50_PDISPLAY_VGACRTC(index & ~3));
-		data32 &= ~(0xff << shift);
-		data32 |= (data << shift);
-		bios_wr32(bios, NV50_PDISPLAY_VGACRTC(index & ~3), data32);
-	}
-
-	if (port == NV_CIO_CRX__COLOR &&
-	    index == NV_CIO_CRE_44 && data == NV_CIO_CRE_44_HEADB)
-		bios->state.crtchead = 1;
-}
-
-static uint8_t
-bios_port_rd(struct nvbios *bios, uint16_t port)
-{
-	uint8_t data, head = bios->state.crtchead;
-
-	if (!valid_port(bios, port))
-		return 0;
-
-	data = NVReadPRMVIO(bios->dev, head, NV_PRMVIO0_OFFSET + port);
-
-	BIOSLOG(bios, "	IO read:  Port: 0x%04X, Head: 0x%02X, Data: 0x%02X\n",
-		port, head, data);
-
-	return data;
-}
-
-static void
-bios_port_wr(struct nvbios *bios, uint16_t port, uint8_t data)
-{
-	int head = bios->state.crtchead;
-
-	if (!valid_port(bios, port))
-		return;
-
-	LOG_OLD_VALUE(bios_port_rd(bios, port));
-	BIOSLOG(bios, "	IO write: Port: 0x%04X, Head: 0x%02X, Data: 0x%02X\n",
-		port, head, data);
-
-	if (!bios->execute)
-		return;
-
-	still_alive();
-	NVWritePRMVIO(bios->dev, head, NV_PRMVIO0_OFFSET + port, data);
-}
-
-static bool
-io_flag_condition_met(struct nvbios *bios, uint16_t offset, uint8_t cond)
-{
-	/*
-	 * The IO flag condition entry has 2 bytes for the CRTC port; 1 byte
-	 * for the CRTC index; 1 byte for the mask to apply to the value
-	 * retrieved from the CRTC; 1 byte for the shift right to apply to the
-	 * masked CRTC value; 2 bytes for the offset to the flag array, to
-	 * which the shifted value is added; 1 byte for the mask applied to the
-	 * value read from the flag array; and 1 byte for the value to compare
-	 * against the masked byte from the flag table.
-	 */
-
-	uint16_t condptr = bios->io_flag_condition_tbl_ptr + cond * IO_FLAG_CONDITION_SIZE;
-	uint16_t crtcport = ROM16(bios->data[condptr]);
-	uint8_t crtcindex = bios->data[condptr + 2];
-	uint8_t mask = bios->data[condptr + 3];
-	uint8_t shift = bios->data[condptr + 4];
-	uint16_t flagarray = ROM16(bios->data[condptr + 5]);
-	uint8_t flagarraymask = bios->data[condptr + 7];
-	uint8_t cmpval = bios->data[condptr + 8];
-	uint8_t data;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X, "
-		      "Shift: 0x%02X, FlagArray: 0x%04X, FAMask: 0x%02X, "
-		      "Cmpval: 0x%02X\n",
-		offset, crtcport, crtcindex, mask, shift, flagarray, flagarraymask, cmpval);
-
-	data = bios_idxprt_rd(bios, crtcport, crtcindex);
-
-	data = bios->data[flagarray + ((data & mask) >> shift)];
-	data &= flagarraymask;
-
-	BIOSLOG(bios, "0x%04X: Checking if 0x%02X equals 0x%02X\n",
-		offset, data, cmpval);
-
-	return (data == cmpval);
-}
-
-static bool
-bios_condition_met(struct nvbios *bios, uint16_t offset, uint8_t cond)
-{
-	/*
-	 * The condition table entry has 4 bytes for the address of the
-	 * register to check, 4 bytes for a mask to apply to the register and
-	 * 4 for a test comparison value
-	 */
-
-	uint16_t condptr = bios->condition_tbl_ptr + cond * CONDITION_SIZE;
-	uint32_t reg = ROM32(bios->data[condptr]);
-	uint32_t mask = ROM32(bios->data[condptr + 4]);
-	uint32_t cmpval = ROM32(bios->data[condptr + 8]);
-	uint32_t data;
-
-	BIOSLOG(bios, "0x%04X: Cond: 0x%02X, Reg: 0x%08X, Mask: 0x%08X\n",
-		offset, cond, reg, mask);
-
-	data = bios_rd32(bios, reg) & mask;
-
-	BIOSLOG(bios, "0x%04X: Checking if 0x%08X equals 0x%08X\n",
-		offset, data, cmpval);
-
-	return (data == cmpval);
-}
-
-static bool
-io_condition_met(struct nvbios *bios, uint16_t offset, uint8_t cond)
-{
-	/*
-	 * The IO condition entry has 2 bytes for the IO port address; 1 byte
-	 * for the index to write to io_port; 1 byte for the mask to apply to
-	 * the byte read from io_port+1; and 1 byte for the value to compare
-	 * against the masked byte.
-	 */
-
-	uint16_t condptr = bios->io_condition_tbl_ptr + cond * IO_CONDITION_SIZE;
-	uint16_t io_port = ROM16(bios->data[condptr]);
-	uint8_t port_index = bios->data[condptr + 2];
-	uint8_t mask = bios->data[condptr + 3];
-	uint8_t cmpval = bios->data[condptr + 4];
-
-	uint8_t data = bios_idxprt_rd(bios, io_port, port_index) & mask;
-
-	BIOSLOG(bios, "0x%04X: Checking if 0x%02X equals 0x%02X\n",
-		offset, data, cmpval);
-
-	return (data == cmpval);
-}
-
-static int
-nv50_pll_set(struct drm_device *dev, uint32_t reg, uint32_t clk)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pll_vals pll;
-	struct pll_lims pll_limits;
-	u32 ctrl, mask, coef;
-	int ret;
-
-	ret = get_pll_limits(dev, reg, &pll_limits);
-	if (ret)
-		return ret;
-
-	clk = nouveau_calc_pll_mnp(dev, &pll_limits, clk, &pll);
-	if (!clk)
-		return -ERANGE;
-
-	coef = pll.N1 << 8 | pll.M1;
-	ctrl = pll.log2P << 16;
-	mask = 0x00070000;
-	if (reg == 0x004008) {
-		mask |= 0x01f80000;
-		ctrl |= (pll_limits.log2p_bias << 19);
-		ctrl |= (pll.log2P << 22);
-	}
-
-	if (!dev_priv->vbios.execute)
-		return 0;
-
-	nv_mask(dev, reg + 0, mask, ctrl);
-	nv_wr32(dev, reg + 4, coef);
-	return 0;
-}
-
-static int
-setPLL(struct nvbios *bios, uint32_t reg, uint32_t clk)
-{
-	struct drm_device *dev = bios->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	/* clk in kHz */
-	struct pll_lims pll_lim;
-	struct nouveau_pll_vals pllvals;
-	int ret;
-
-	if (dev_priv->card_type >= NV_50)
-		return nv50_pll_set(dev, reg, clk);
-
-	/* high regs (such as in the mac g5 table) are not -= 4 */
-	ret = get_pll_limits(dev, reg > 0x405c ? reg : reg - 4, &pll_lim);
-	if (ret)
-		return ret;
-
-	clk = nouveau_calc_pll_mnp(dev, &pll_lim, clk, &pllvals);
-	if (!clk)
-		return -ERANGE;
-
-	if (bios->execute) {
-		still_alive();
-		nouveau_hw_setpll(dev, reg, &pllvals);
-	}
-
-	return 0;
-}
-
-static int dcb_entry_idx_from_crtchead(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-
-	/*
-	 * For the results of this function to be correct, CR44 must have been
-	 * set (using bios_idxprt_wr to set crtchead), CR58 set for CR57 = 0,
-	 * and the DCB table parsed, before the script calling the function is
-	 * run.  run_digital_op_script is example of how to do such setup
-	 */
-
-	uint8_t dcb_entry = NVReadVgaCrtc5758(dev, bios->state.crtchead, 0);
-
-	if (dcb_entry > bios->dcb.entries) {
-		NV_ERROR(dev, "CR58 doesn't have a valid DCB entry currently "
-				"(%02X)\n", dcb_entry);
-		dcb_entry = 0x7f;	/* unused / invalid marker */
-	}
-
-	return dcb_entry;
-}
-
-static struct nouveau_i2c_chan *
-init_i2c_device_find(struct drm_device *dev, int i2c_index)
-{
-	if (i2c_index == 0xff) {
-		struct drm_nouveau_private *dev_priv = dev->dev_private;
-		struct dcb_table *dcb = &dev_priv->vbios.dcb;
-		/* note: dcb_entry_idx_from_crtchead needs pre-script set-up */
-		int idx = dcb_entry_idx_from_crtchead(dev);
-
-		i2c_index = NV_I2C_DEFAULT(0);
-		if (idx != 0x7f && dcb->entry[idx].i2c_upper_default)
-			i2c_index = NV_I2C_DEFAULT(1);
-	}
-
-	return nouveau_i2c_find(dev, i2c_index);
-}
-
-static uint32_t
-get_tmds_index_reg(struct drm_device *dev, uint8_t mlv)
-{
-	/*
-	 * For mlv < 0x80, it is an index into a table of TMDS base addresses.
-	 * For mlv == 0x80 use the "or" value of the dcb_entry indexed by
-	 * CR58 for CR57 = 0 to index a table of offsets to the basic
-	 * 0x6808b0 address.
-	 * For mlv == 0x81 use the "or" value of the dcb_entry indexed by
-	 * CR58 for CR57 = 0 to index a table of offsets to the basic
-	 * 0x6808b0 address, and then flip the offset by 8.
-	 */
-
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	const int pramdac_offset[13] = {
-		0, 0, 0x8, 0, 0x2000, 0, 0, 0, 0x2008, 0, 0, 0, 0x2000 };
-	const uint32_t pramdac_table[4] = {
-		0x6808b0, 0x6808b8, 0x6828b0, 0x6828b8 };
-
-	if (mlv >= 0x80) {
-		int dcb_entry, dacoffset;
-
-		/* note: dcb_entry_idx_from_crtchead needs pre-script set-up */
-		dcb_entry = dcb_entry_idx_from_crtchead(dev);
-		if (dcb_entry == 0x7f)
-			return 0;
-		dacoffset = pramdac_offset[bios->dcb.entry[dcb_entry].or];
-		if (mlv == 0x81)
-			dacoffset ^= 8;
-		return 0x6808b0 + dacoffset;
-	} else {
-		if (mlv >= ARRAY_SIZE(pramdac_table)) {
-			NV_ERROR(dev, "Magic Lookup Value too big (%02X)\n",
-									mlv);
-			return 0;
-		}
-		return pramdac_table[mlv];
-	}
-}
-
-static int
-init_io_restrict_prog(struct nvbios *bios, uint16_t offset,
-		      struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO_RESTRICT_PROG   opcode: 0x32 ('2')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): CRTC index
-	 * offset + 4  (8  bit): mask
-	 * offset + 5  (8  bit): shift
-	 * offset + 6  (8  bit): count
-	 * offset + 7  (32 bit): register
-	 * offset + 11 (32 bit): configuration 1
-	 * ...
-	 *
-	 * Starting at offset + 11 there are "count" 32 bit values.
-	 * To find out which value to use read index "CRTC index" on "CRTC
-	 * port", AND this value with "mask" and then bit shift right "shift"
-	 * bits.  Read the appropriate value using this index and write to
-	 * "register"
-	 */
-
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t crtcindex = bios->data[offset + 3];
-	uint8_t mask = bios->data[offset + 4];
-	uint8_t shift = bios->data[offset + 5];
-	uint8_t count = bios->data[offset + 6];
-	uint32_t reg = ROM32(bios->data[offset + 7]);
-	uint8_t config;
-	uint32_t configval;
-	int len = 11 + count * 4;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X, "
-		      "Shift: 0x%02X, Count: 0x%02X, Reg: 0x%08X\n",
-		offset, crtcport, crtcindex, mask, shift, count, reg);
-
-	config = (bios_idxprt_rd(bios, crtcport, crtcindex) & mask) >> shift;
-	if (config > count) {
-		NV_ERROR(bios->dev,
-			 "0x%04X: Config 0x%02X exceeds maximal bound 0x%02X\n",
-			 offset, config, count);
-		return len;
-	}
-
-	configval = ROM32(bios->data[offset + 11 + config * 4]);
-
-	BIOSLOG(bios, "0x%04X: Writing config %02X\n", offset, config);
-
-	bios_wr32(bios, reg, configval);
-
-	return len;
-}
-
-static int
-init_repeat(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_REPEAT   opcode: 0x33 ('3')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): count
-	 *
-	 * Execute script following this opcode up to INIT_REPEAT_END
-	 * "count" times
-	 */
-
-	uint8_t count = bios->data[offset + 1];
-	uint8_t i;
-
-	/* no iexec->execute check by design */
-
-	BIOSLOG(bios, "0x%04X: Repeating following segment %d times\n",
-		offset, count);
-
-	iexec->repeat = true;
-
-	/*
-	 * count - 1, as the script block will execute once when we leave this
-	 * opcode -- this is compatible with bios behaviour as:
-	 * a) the block is always executed at least once, even if count == 0
-	 * b) the bios interpreter skips to the op following INIT_END_REPEAT,
-	 * while we don't
-	 */
-	for (i = 0; i < count - 1; i++)
-		parse_init_table(bios, offset + 2, iexec);
-
-	iexec->repeat = false;
-
-	return 2;
-}
-
-static int
-init_io_restrict_pll(struct nvbios *bios, uint16_t offset,
-		     struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO_RESTRICT_PLL   opcode: 0x34 ('4')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): CRTC index
-	 * offset + 4  (8  bit): mask
-	 * offset + 5  (8  bit): shift
-	 * offset + 6  (8  bit): IO flag condition index
-	 * offset + 7  (8  bit): count
-	 * offset + 8  (32 bit): register
-	 * offset + 12 (16 bit): frequency 1
-	 * ...
-	 *
-	 * Starting at offset + 12 there are "count" 16 bit frequencies (10kHz).
-	 * Set PLL register "register" to coefficients for frequency n,
-	 * selected by reading index "CRTC index" of "CRTC port" ANDed with
-	 * "mask" and shifted right by "shift".
-	 *
-	 * If "IO flag condition index" > 0, and condition met, double
-	 * frequency before setting it.
-	 */
-
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t crtcindex = bios->data[offset + 3];
-	uint8_t mask = bios->data[offset + 4];
-	uint8_t shift = bios->data[offset + 5];
-	int8_t io_flag_condition_idx = bios->data[offset + 6];
-	uint8_t count = bios->data[offset + 7];
-	uint32_t reg = ROM32(bios->data[offset + 8]);
-	uint8_t config;
-	uint16_t freq;
-	int len = 12 + count * 2;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X, "
-		      "Shift: 0x%02X, IO Flag Condition: 0x%02X, "
-		      "Count: 0x%02X, Reg: 0x%08X\n",
-		offset, crtcport, crtcindex, mask, shift,
-		io_flag_condition_idx, count, reg);
-
-	config = (bios_idxprt_rd(bios, crtcport, crtcindex) & mask) >> shift;
-	if (config > count) {
-		NV_ERROR(bios->dev,
-			 "0x%04X: Config 0x%02X exceeds maximal bound 0x%02X\n",
-			 offset, config, count);
-		return len;
-	}
-
-	freq = ROM16(bios->data[offset + 12 + config * 2]);
-
-	if (io_flag_condition_idx > 0) {
-		if (io_flag_condition_met(bios, offset, io_flag_condition_idx)) {
-			BIOSLOG(bios, "0x%04X: Condition fulfilled -- "
-				      "frequency doubled\n", offset);
-			freq *= 2;
-		} else
-			BIOSLOG(bios, "0x%04X: Condition not fulfilled -- "
-				      "frequency unchanged\n", offset);
-	}
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, Config: 0x%02X, Freq: %d0kHz\n",
-		offset, reg, config, freq);
-
-	setPLL(bios, reg, freq * 10);
-
-	return len;
-}
-
-static int
-init_end_repeat(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_END_REPEAT   opcode: 0x36 ('6')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Marks the end of the block for INIT_REPEAT to repeat
-	 */
-
-	/* no iexec->execute check by design */
-
-	/*
-	 * iexec->repeat flag necessary to go past INIT_END_REPEAT opcode when
-	 * we're not in repeat mode
-	 */
-	if (iexec->repeat)
-		return 0;
-
-	return 1;
-}
-
-static int
-init_copy(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_COPY   opcode: 0x37 ('7')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (8  bit): shift
-	 * offset + 6  (8  bit): srcmask
-	 * offset + 7  (16 bit): CRTC port
-	 * offset + 9  (8 bit): CRTC index
-	 * offset + 10  (8 bit): mask
-	 *
-	 * Read index "CRTC index" on "CRTC port", AND with "mask", OR with
-	 * (REGVAL("register") >> "shift" & "srcmask") and write-back to CRTC
-	 * port
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint8_t shift = bios->data[offset + 5];
-	uint8_t srcmask = bios->data[offset + 6];
-	uint16_t crtcport = ROM16(bios->data[offset + 7]);
-	uint8_t crtcindex = bios->data[offset + 9];
-	uint8_t mask = bios->data[offset + 10];
-	uint32_t data;
-	uint8_t crtcdata;
-
-	if (!iexec->execute)
-		return 11;
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, Shift: 0x%02X, SrcMask: 0x%02X, "
-		      "Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X\n",
-		offset, reg, shift, srcmask, crtcport, crtcindex, mask);
-
-	data = bios_rd32(bios, reg);
-
-	if (shift < 0x80)
-		data >>= shift;
-	else
-		data <<= (0x100 - shift);
-
-	data &= srcmask;
-
-	crtcdata  = bios_idxprt_rd(bios, crtcport, crtcindex) & mask;
-	crtcdata |= (uint8_t)data;
-	bios_idxprt_wr(bios, crtcport, crtcindex, crtcdata);
-
-	return 11;
-}
-
-static int
-init_not(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_NOT   opcode: 0x38 ('8')
-	 *
-	 * offset      (8  bit): opcode
-	 *
-	 * Invert the current execute / no-execute condition (i.e. "else")
-	 */
-	if (iexec->execute)
-		BIOSLOG(bios, "0x%04X: ------ Skipping following commands  ------\n", offset);
-	else
-		BIOSLOG(bios, "0x%04X: ------ Executing following commands ------\n", offset);
-
-	iexec->execute = !iexec->execute;
-	return 1;
-}
-
-static int
-init_io_flag_condition(struct nvbios *bios, uint16_t offset,
-		       struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO_FLAG_CONDITION   opcode: 0x39 ('9')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): condition number
-	 *
-	 * Check condition "condition number" in the IO flag condition table.
-	 * If condition not met skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t cond = bios->data[offset + 1];
-
-	if (!iexec->execute)
-		return 2;
-
-	if (io_flag_condition_met(bios, offset, cond))
-		BIOSLOG(bios, "0x%04X: Condition fulfilled -- continuing to execute\n", offset);
-	else {
-		BIOSLOG(bios, "0x%04X: Condition not fulfilled -- skipping following commands\n", offset);
-		iexec->execute = false;
-	}
-
-	return 2;
-}
-
-static int
-init_dp_condition(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_DP_CONDITION   opcode: 0x3A ('')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): "sub" opcode
-	 * offset + 2  (8 bit): unknown
-	 *
-	 */
-
-	struct dcb_entry *dcb = bios->display.output;
-	struct drm_device *dev = bios->dev;
-	uint8_t cond = bios->data[offset + 1];
-	uint8_t *table, *entry;
-
-	BIOSLOG(bios, "0x%04X: subop 0x%02X\n", offset, cond);
-
-	if (!iexec->execute)
-		return 3;
-
-	table = nouveau_dp_bios_data(dev, dcb, &entry);
-	if (!table)
-		return 3;
-
-	switch (cond) {
-	case 0:
-		entry = dcb_conn(dev, dcb->connector);
-		if (!entry || entry[0] != DCB_CONNECTOR_eDP)
-			iexec->execute = false;
-		break;
-	case 1:
-	case 2:
-		if ((table[0]  < 0x40 && !(entry[5] & cond)) ||
-		    (table[0] == 0x40 && !(entry[4] & cond)))
-			iexec->execute = false;
-		break;
-	case 5:
-	{
-		struct nouveau_i2c_chan *auxch;
-		int ret;
-
-		auxch = nouveau_i2c_find(dev, bios->display.output->i2c_index);
-		if (!auxch) {
-			NV_ERROR(dev, "0x%04X: couldn't get auxch\n", offset);
-			return 3;
-		}
-
-		ret = nouveau_dp_auxch(auxch, 9, 0xd, &cond, 1);
-		if (ret) {
-			NV_ERROR(dev, "0x%04X: auxch rd fail: %d\n", offset, ret);
-			return 3;
-		}
-
-		if (!(cond & 1))
-			iexec->execute = false;
-	}
-		break;
-	default:
-		NV_WARN(dev, "0x%04X: unknown INIT_3A op: %d\n", offset, cond);
-		break;
-	}
-
-	if (iexec->execute)
-		BIOSLOG(bios, "0x%04X: continuing to execute\n", offset);
-	else
-		BIOSLOG(bios, "0x%04X: skipping following commands\n", offset);
-
-	return 3;
-}
-
-static int
-init_op_3b(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_3B   opcode: 0x3B ('')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): crtc index
-	 *
-	 */
-
-	uint8_t or = ffs(bios->display.output->or) - 1;
-	uint8_t index = bios->data[offset + 1];
-	uint8_t data;
-
-	if (!iexec->execute)
-		return 2;
-
-	data = bios_idxprt_rd(bios, 0x3d4, index);
-	bios_idxprt_wr(bios, 0x3d4, index, data & ~(1 << or));
-	return 2;
-}
-
-static int
-init_op_3c(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_3C   opcode: 0x3C ('')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): crtc index
-	 *
-	 */
-
-	uint8_t or = ffs(bios->display.output->or) - 1;
-	uint8_t index = bios->data[offset + 1];
-	uint8_t data;
-
-	if (!iexec->execute)
-		return 2;
-
-	data = bios_idxprt_rd(bios, 0x3d4, index);
-	bios_idxprt_wr(bios, 0x3d4, index, data | (1 << or));
-	return 2;
-}
-
-static int
-init_idx_addr_latched(struct nvbios *bios, uint16_t offset,
-		      struct init_exec *iexec)
-{
-	/*
-	 * INIT_INDEX_ADDRESS_LATCHED   opcode: 0x49 ('I')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): control register
-	 * offset + 5  (32 bit): data register
-	 * offset + 9  (32 bit): mask
-	 * offset + 13 (32 bit): data
-	 * offset + 17 (8  bit): count
-	 * offset + 18 (8  bit): address 1
-	 * offset + 19 (8  bit): data 1
-	 * ...
-	 *
-	 * For each of "count" address and data pairs, write "data n" to
-	 * "data register", read the current value of "control register",
-	 * and write it back once ANDed with "mask", ORed with "data",
-	 * and ORed with "address n"
-	 */
-
-	uint32_t controlreg = ROM32(bios->data[offset + 1]);
-	uint32_t datareg = ROM32(bios->data[offset + 5]);
-	uint32_t mask = ROM32(bios->data[offset + 9]);
-	uint32_t data = ROM32(bios->data[offset + 13]);
-	uint8_t count = bios->data[offset + 17];
-	int len = 18 + count * 2;
-	uint32_t value;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: ControlReg: 0x%08X, DataReg: 0x%08X, "
-		      "Mask: 0x%08X, Data: 0x%08X, Count: 0x%02X\n",
-		offset, controlreg, datareg, mask, data, count);
-
-	for (i = 0; i < count; i++) {
-		uint8_t instaddress = bios->data[offset + 18 + i * 2];
-		uint8_t instdata = bios->data[offset + 19 + i * 2];
-
-		BIOSLOG(bios, "0x%04X: Address: 0x%02X, Data: 0x%02X\n",
-			offset, instaddress, instdata);
-
-		bios_wr32(bios, datareg, instdata);
-		value  = bios_rd32(bios, controlreg) & mask;
-		value |= data;
-		value |= instaddress;
-		bios_wr32(bios, controlreg, value);
-	}
-
-	return len;
-}
-
-static int
-init_io_restrict_pll2(struct nvbios *bios, uint16_t offset,
-		      struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO_RESTRICT_PLL2   opcode: 0x4A ('J')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): CRTC index
-	 * offset + 4  (8  bit): mask
-	 * offset + 5  (8  bit): shift
-	 * offset + 6  (8  bit): count
-	 * offset + 7  (32 bit): register
-	 * offset + 11 (32 bit): frequency 1
-	 * ...
-	 *
-	 * Starting at offset + 11 there are "count" 32 bit frequencies (kHz).
-	 * Set PLL register "register" to coefficients for frequency n,
-	 * selected by reading index "CRTC index" of "CRTC port" ANDed with
-	 * "mask" and shifted right by "shift".
-	 */
-
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t crtcindex = bios->data[offset + 3];
-	uint8_t mask = bios->data[offset + 4];
-	uint8_t shift = bios->data[offset + 5];
-	uint8_t count = bios->data[offset + 6];
-	uint32_t reg = ROM32(bios->data[offset + 7]);
-	int len = 11 + count * 4;
-	uint8_t config;
-	uint32_t freq;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X, "
-		      "Shift: 0x%02X, Count: 0x%02X, Reg: 0x%08X\n",
-		offset, crtcport, crtcindex, mask, shift, count, reg);
-
-	if (!reg)
-		return len;
-
-	config = (bios_idxprt_rd(bios, crtcport, crtcindex) & mask) >> shift;
-	if (config > count) {
-		NV_ERROR(bios->dev,
-			 "0x%04X: Config 0x%02X exceeds maximal bound 0x%02X\n",
-			 offset, config, count);
-		return len;
-	}
-
-	freq = ROM32(bios->data[offset + 11 + config * 4]);
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, Config: 0x%02X, Freq: %dkHz\n",
-		offset, reg, config, freq);
-
-	setPLL(bios, reg, freq);
-
-	return len;
-}
-
-static int
-init_pll2(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_PLL2   opcode: 0x4B ('K')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (32 bit): freq
-	 *
-	 * Set PLL register "register" to coefficients for frequency "freq"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint32_t freq = ROM32(bios->data[offset + 5]);
-
-	if (!iexec->execute)
-		return 9;
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%04X, Freq: %dkHz\n",
-		offset, reg, freq);
-
-	setPLL(bios, reg, freq);
-	return 9;
-}
-
-static int
-init_i2c_byte(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_I2C_BYTE   opcode: 0x4C ('L')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): DCB I2C table entry index
-	 * offset + 2  (8 bit): I2C slave address
-	 * offset + 3  (8 bit): count
-	 * offset + 4  (8 bit): I2C register 1
-	 * offset + 5  (8 bit): mask 1
-	 * offset + 6  (8 bit): data 1
-	 * ...
-	 *
-	 * For each of "count" registers given by "I2C register n" on the device
-	 * addressed by "I2C slave address" on the I2C bus given by
-	 * "DCB I2C table entry index", read the register, AND the result with
-	 * "mask n" and OR it with "data n" before writing it back to the device
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint8_t i2c_index = bios->data[offset + 1];
-	uint8_t i2c_address = bios->data[offset + 2] >> 1;
-	uint8_t count = bios->data[offset + 3];
-	struct nouveau_i2c_chan *chan;
-	int len = 4 + count * 3;
-	int ret, i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: DCBI2CIndex: 0x%02X, I2CAddress: 0x%02X, "
-		      "Count: 0x%02X\n",
-		offset, i2c_index, i2c_address, count);
-
-	chan = init_i2c_device_find(dev, i2c_index);
-	if (!chan) {
-		NV_ERROR(dev, "0x%04X: i2c bus not found\n", offset);
-		return len;
-	}
-
-	for (i = 0; i < count; i++) {
-		uint8_t reg = bios->data[offset + 4 + i * 3];
-		uint8_t mask = bios->data[offset + 5 + i * 3];
-		uint8_t data = bios->data[offset + 6 + i * 3];
-		union i2c_smbus_data val;
-
-		ret = i2c_smbus_xfer(&chan->adapter, i2c_address, 0,
-				     I2C_SMBUS_READ, reg,
-				     I2C_SMBUS_BYTE_DATA, &val);
-		if (ret < 0) {
-			NV_ERROR(dev, "0x%04X: i2c rd fail: %d\n", offset, ret);
-			return len;
-		}
-
-		BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X, Value: 0x%02X, "
-			      "Mask: 0x%02X, Data: 0x%02X\n",
-			offset, reg, val.byte, mask, data);
-
-		if (!bios->execute)
-			continue;
-
-		val.byte &= mask;
-		val.byte |= data;
-		ret = i2c_smbus_xfer(&chan->adapter, i2c_address, 0,
-				     I2C_SMBUS_WRITE, reg,
-				     I2C_SMBUS_BYTE_DATA, &val);
-		if (ret < 0) {
-			NV_ERROR(dev, "0x%04X: i2c wr fail: %d\n", offset, ret);
-			return len;
-		}
-	}
-
-	return len;
-}
-
-static int
-init_zm_i2c_byte(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_I2C_BYTE   opcode: 0x4D ('M')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): DCB I2C table entry index
-	 * offset + 2  (8 bit): I2C slave address
-	 * offset + 3  (8 bit): count
-	 * offset + 4  (8 bit): I2C register 1
-	 * offset + 5  (8 bit): data 1
-	 * ...
-	 *
-	 * For each of "count" registers given by "I2C register n" on the device
-	 * addressed by "I2C slave address" on the I2C bus given by
-	 * "DCB I2C table entry index", set the register to "data n"
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint8_t i2c_index = bios->data[offset + 1];
-	uint8_t i2c_address = bios->data[offset + 2] >> 1;
-	uint8_t count = bios->data[offset + 3];
-	struct nouveau_i2c_chan *chan;
-	int len = 4 + count * 2;
-	int ret, i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: DCBI2CIndex: 0x%02X, I2CAddress: 0x%02X, "
-		      "Count: 0x%02X\n",
-		offset, i2c_index, i2c_address, count);
-
-	chan = init_i2c_device_find(dev, i2c_index);
-	if (!chan) {
-		NV_ERROR(dev, "0x%04X: i2c bus not found\n", offset);
-		return len;
-	}
-
-	for (i = 0; i < count; i++) {
-		uint8_t reg = bios->data[offset + 4 + i * 2];
-		union i2c_smbus_data val;
-
-		val.byte = bios->data[offset + 5 + i * 2];
-
-		BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X, Data: 0x%02X\n",
-			offset, reg, val.byte);
-
-		if (!bios->execute)
-			continue;
-
-		ret = i2c_smbus_xfer(&chan->adapter, i2c_address, 0,
-				     I2C_SMBUS_WRITE, reg,
-				     I2C_SMBUS_BYTE_DATA, &val);
-		if (ret < 0) {
-			NV_ERROR(dev, "0x%04X: i2c wr fail: %d\n", offset, ret);
-			return len;
-		}
-	}
-
-	return len;
-}
-
-static int
-init_zm_i2c(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_I2C   opcode: 0x4E ('N')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): DCB I2C table entry index
-	 * offset + 2  (8 bit): I2C slave address
-	 * offset + 3  (8 bit): count
-	 * offset + 4  (8 bit): data 1
-	 * ...
-	 *
-	 * Send "count" bytes ("data n") to the device addressed by "I2C slave
-	 * address" on the I2C bus given by "DCB I2C table entry index"
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint8_t i2c_index = bios->data[offset + 1];
-	uint8_t i2c_address = bios->data[offset + 2] >> 1;
-	uint8_t count = bios->data[offset + 3];
-	int len = 4 + count;
-	struct nouveau_i2c_chan *chan;
-	struct i2c_msg msg;
-	uint8_t data[256];
-	int ret, i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: DCBI2CIndex: 0x%02X, I2CAddress: 0x%02X, "
-		      "Count: 0x%02X\n",
-		offset, i2c_index, i2c_address, count);
-
-	chan = init_i2c_device_find(dev, i2c_index);
-	if (!chan) {
-		NV_ERROR(dev, "0x%04X: i2c bus not found\n", offset);
-		return len;
-	}
-
-	for (i = 0; i < count; i++) {
-		data[i] = bios->data[offset + 4 + i];
-
-		BIOSLOG(bios, "0x%04X: Data: 0x%02X\n", offset, data[i]);
-	}
-
-	if (bios->execute) {
-		msg.addr = i2c_address;
-		msg.flags = 0;
-		msg.len = count;
-		msg.buf = data;
-		ret = i2c_transfer(&chan->adapter, &msg, 1);
-		if (ret != 1) {
-			NV_ERROR(dev, "0x%04X: i2c wr fail: %d\n", offset, ret);
-			return len;
-		}
-	}
-
-	return len;
-}
-
-static int
-init_tmds(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_TMDS   opcode: 0x4F ('O')	(non-canon name)
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): magic lookup value
-	 * offset + 2  (8 bit): TMDS address
-	 * offset + 3  (8 bit): mask
-	 * offset + 4  (8 bit): data
-	 *
-	 * Read the data reg for TMDS address "TMDS address", AND it with mask
-	 * and OR it with data, then write it back
-	 * "magic lookup value" determines which TMDS base address register is
-	 * used -- see get_tmds_index_reg()
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint8_t mlv = bios->data[offset + 1];
-	uint32_t tmdsaddr = bios->data[offset + 2];
-	uint8_t mask = bios->data[offset + 3];
-	uint8_t data = bios->data[offset + 4];
-	uint32_t reg, value;
-
-	if (!iexec->execute)
-		return 5;
-
-	BIOSLOG(bios, "0x%04X: MagicLookupValue: 0x%02X, TMDSAddr: 0x%02X, "
-		      "Mask: 0x%02X, Data: 0x%02X\n",
-		offset, mlv, tmdsaddr, mask, data);
-
-	reg = get_tmds_index_reg(bios->dev, mlv);
-	if (!reg) {
-		NV_ERROR(dev, "0x%04X: no tmds_index_reg\n", offset);
-		return 5;
-	}
-
-	bios_wr32(bios, reg,
-		  tmdsaddr | NV_PRAMDAC_FP_TMDS_CONTROL_WRITE_DISABLE);
-	value = (bios_rd32(bios, reg + 4) & mask) | data;
-	bios_wr32(bios, reg + 4, value);
-	bios_wr32(bios, reg, tmdsaddr);
-
-	return 5;
-}
-
-static int
-init_zm_tmds_group(struct nvbios *bios, uint16_t offset,
-		   struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_TMDS_GROUP   opcode: 0x50 ('P')	(non-canon name)
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): magic lookup value
-	 * offset + 2  (8 bit): count
-	 * offset + 3  (8 bit): addr 1
-	 * offset + 4  (8 bit): data 1
-	 * ...
-	 *
-	 * For each of "count" TMDS address and data pairs write "data n" to
-	 * "addr n".  "magic lookup value" determines which TMDS base address
-	 * register is used -- see get_tmds_index_reg()
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint8_t mlv = bios->data[offset + 1];
-	uint8_t count = bios->data[offset + 2];
-	int len = 3 + count * 2;
-	uint32_t reg;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: MagicLookupValue: 0x%02X, Count: 0x%02X\n",
-		offset, mlv, count);
-
-	reg = get_tmds_index_reg(bios->dev, mlv);
-	if (!reg) {
-		NV_ERROR(dev, "0x%04X: no tmds_index_reg\n", offset);
-		return len;
-	}
-
-	for (i = 0; i < count; i++) {
-		uint8_t tmdsaddr = bios->data[offset + 3 + i * 2];
-		uint8_t tmdsdata = bios->data[offset + 4 + i * 2];
-
-		bios_wr32(bios, reg + 4, tmdsdata);
-		bios_wr32(bios, reg, tmdsaddr);
-	}
-
-	return len;
-}
-
-static int
-init_cr_idx_adr_latch(struct nvbios *bios, uint16_t offset,
-		      struct init_exec *iexec)
-{
-	/*
-	 * INIT_CR_INDEX_ADDRESS_LATCHED   opcode: 0x51 ('Q')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): CRTC index1
-	 * offset + 2  (8 bit): CRTC index2
-	 * offset + 3  (8 bit): baseaddr
-	 * offset + 4  (8 bit): count
-	 * offset + 5  (8 bit): data 1
-	 * ...
-	 *
-	 * For each of "count" address and data pairs, write "baseaddr + n" to
-	 * "CRTC index1" and "data n" to "CRTC index2"
-	 * Once complete, restore initial value read from "CRTC index1"
-	 */
-	uint8_t crtcindex1 = bios->data[offset + 1];
-	uint8_t crtcindex2 = bios->data[offset + 2];
-	uint8_t baseaddr = bios->data[offset + 3];
-	uint8_t count = bios->data[offset + 4];
-	int len = 5 + count;
-	uint8_t oldaddr, data;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: Index1: 0x%02X, Index2: 0x%02X, "
-		      "BaseAddr: 0x%02X, Count: 0x%02X\n",
-		offset, crtcindex1, crtcindex2, baseaddr, count);
-
-	oldaddr = bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, crtcindex1);
-
-	for (i = 0; i < count; i++) {
-		bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, crtcindex1,
-				     baseaddr + i);
-		data = bios->data[offset + 5 + i];
-		bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, crtcindex2, data);
-	}
-
-	bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, crtcindex1, oldaddr);
-
-	return len;
-}
-
-static int
-init_cr(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_CR   opcode: 0x52 ('R')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (8  bit): CRTC index
-	 * offset + 2  (8  bit): mask
-	 * offset + 3  (8  bit): data
-	 *
-	 * Assign the value of at "CRTC index" ANDed with mask and ORed with
-	 * data back to "CRTC index"
-	 */
-
-	uint8_t crtcindex = bios->data[offset + 1];
-	uint8_t mask = bios->data[offset + 2];
-	uint8_t data = bios->data[offset + 3];
-	uint8_t value;
-
-	if (!iexec->execute)
-		return 4;
-
-	BIOSLOG(bios, "0x%04X: Index: 0x%02X, Mask: 0x%02X, Data: 0x%02X\n",
-		offset, crtcindex, mask, data);
-
-	value  = bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, crtcindex) & mask;
-	value |= data;
-	bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, crtcindex, value);
-
-	return 4;
-}
-
-static int
-init_zm_cr(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_CR   opcode: 0x53 ('S')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): CRTC index
-	 * offset + 2  (8 bit): value
-	 *
-	 * Assign "value" to CRTC register with index "CRTC index".
-	 */
-
-	uint8_t crtcindex = ROM32(bios->data[offset + 1]);
-	uint8_t data = bios->data[offset + 2];
-
-	if (!iexec->execute)
-		return 3;
-
-	bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, crtcindex, data);
-
-	return 3;
-}
-
-static int
-init_zm_cr_group(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_CR_GROUP   opcode: 0x54 ('T')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): count
-	 * offset + 2  (8 bit): CRTC index 1
-	 * offset + 3  (8 bit): value 1
-	 * ...
-	 *
-	 * For "count", assign "value n" to CRTC register with index
-	 * "CRTC index n".
-	 */
-
-	uint8_t count = bios->data[offset + 1];
-	int len = 2 + count * 2;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	for (i = 0; i < count; i++)
-		init_zm_cr(bios, offset + 2 + 2 * i - 1, iexec);
-
-	return len;
-}
-
-static int
-init_condition_time(struct nvbios *bios, uint16_t offset,
-		    struct init_exec *iexec)
-{
-	/*
-	 * INIT_CONDITION_TIME   opcode: 0x56 ('V')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): condition number
-	 * offset + 2  (8 bit): retries / 50
-	 *
-	 * Check condition "condition number" in the condition table.
-	 * Bios code then sleeps for 2ms if the condition is not met, and
-	 * repeats up to "retries" times, but on one C51 this has proved
-	 * insufficient.  In mmiotraces the driver sleeps for 20ms, so we do
-	 * this, and bail after "retries" times, or 2s, whichever is less.
-	 * If still not met after retries, clear execution flag for this table.
-	 */
-
-	uint8_t cond = bios->data[offset + 1];
-	uint16_t retries = bios->data[offset + 2] * 50;
-	unsigned cnt;
-
-	if (!iexec->execute)
-		return 3;
-
-	if (retries > 100)
-		retries = 100;
-
-	BIOSLOG(bios, "0x%04X: Condition: 0x%02X, Retries: 0x%02X\n",
-		offset, cond, retries);
-
-	if (!bios->execute) /* avoid 2s delays when "faking" execution */
-		retries = 1;
-
-	for (cnt = 0; cnt < retries; cnt++) {
-		if (bios_condition_met(bios, offset, cond)) {
-			BIOSLOG(bios, "0x%04X: Condition met, continuing\n",
-								offset);
-			break;
-		} else {
-			BIOSLOG(bios, "0x%04X: "
-				"Condition not met, sleeping for 20ms\n",
-								offset);
-			mdelay(20);
-		}
-	}
-
-	if (!bios_condition_met(bios, offset, cond)) {
-		NV_WARN(bios->dev,
-			"0x%04X: Condition still not met after %dms, "
-			"skipping following opcodes\n", offset, 20 * retries);
-		iexec->execute = false;
-	}
-
-	return 3;
-}
-
-static int
-init_ltime(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_LTIME   opcode: 0x57 ('V')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): time
-	 *
-	 * Sleep for "time" milliseconds.
-	 */
-
-	unsigned time = ROM16(bios->data[offset + 1]);
-
-	if (!iexec->execute)
-		return 3;
-
-	BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X milliseconds\n",
-		offset, time);
-
-	mdelay(time);
-
-	return 3;
-}
-
-static int
-init_zm_reg_sequence(struct nvbios *bios, uint16_t offset,
-		     struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_REG_SEQUENCE   opcode: 0x58 ('X')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): base register
-	 * offset + 5  (8  bit): count
-	 * offset + 6  (32 bit): value 1
-	 * ...
-	 *
-	 * Starting at offset + 6 there are "count" 32 bit values.
-	 * For "count" iterations set "base register" + 4 * current_iteration
-	 * to "value current_iteration"
-	 */
-
-	uint32_t basereg = ROM32(bios->data[offset + 1]);
-	uint32_t count = bios->data[offset + 5];
-	int len = 6 + count * 4;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	BIOSLOG(bios, "0x%04X: BaseReg: 0x%08X, Count: 0x%02X\n",
-		offset, basereg, count);
-
-	for (i = 0; i < count; i++) {
-		uint32_t reg = basereg + i * 4;
-		uint32_t data = ROM32(bios->data[offset + 6 + i * 4]);
-
-		bios_wr32(bios, reg, data);
-	}
-
-	return len;
-}
-
-static int
-init_sub_direct(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_SUB_DIRECT   opcode: 0x5B ('[')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): subroutine offset (in bios)
-	 *
-	 * Calls a subroutine that will execute commands until INIT_DONE
-	 * is found.
-	 */
-
-	uint16_t sub_offset = ROM16(bios->data[offset + 1]);
-
-	if (!iexec->execute)
-		return 3;
-
-	BIOSLOG(bios, "0x%04X: Executing subroutine at 0x%04X\n",
-		offset, sub_offset);
-
-	parse_init_table(bios, sub_offset, iexec);
-
-	BIOSLOG(bios, "0x%04X: End of 0x%04X subroutine\n", offset, sub_offset);
-
-	return 3;
-}
-
-static int
-init_jump(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_JUMP   opcode: 0x5C ('\')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): offset (in bios)
-	 *
-	 * Continue execution of init table from 'offset'
-	 */
-
-	uint16_t jmp_offset = ROM16(bios->data[offset + 1]);
-
-	if (!iexec->execute)
-		return 3;
-
-	BIOSLOG(bios, "0x%04X: Jump to 0x%04X\n", offset, jmp_offset);
-	return jmp_offset - offset;
-}
-
-static int
-init_i2c_if(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_I2C_IF   opcode: 0x5E ('^')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): DCB I2C table entry index
-	 * offset + 2  (8 bit): I2C slave address
-	 * offset + 3  (8 bit): I2C register
-	 * offset + 4  (8 bit): mask
-	 * offset + 5  (8 bit): data
-	 *
-	 * Read the register given by "I2C register" on the device addressed
-	 * by "I2C slave address" on the I2C bus given by "DCB I2C table
-	 * entry index". Compare the result AND "mask" to "data".
-	 * If they're not equal, skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t i2c_index = bios->data[offset + 1];
-	uint8_t i2c_address = bios->data[offset + 2] >> 1;
-	uint8_t reg = bios->data[offset + 3];
-	uint8_t mask = bios->data[offset + 4];
-	uint8_t data = bios->data[offset + 5];
-	struct nouveau_i2c_chan *chan;
-	union i2c_smbus_data val;
-	int ret;
-
-	/* no execute check by design */
-
-	BIOSLOG(bios, "0x%04X: DCBI2CIndex: 0x%02X, I2CAddress: 0x%02X\n",
-		offset, i2c_index, i2c_address);
-
-	chan = init_i2c_device_find(bios->dev, i2c_index);
-	if (!chan)
-		return -ENODEV;
-
-	ret = i2c_smbus_xfer(&chan->adapter, i2c_address, 0,
-			     I2C_SMBUS_READ, reg,
-			     I2C_SMBUS_BYTE_DATA, &val);
-	if (ret < 0) {
-		BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X, Value: [no device], "
-			      "Mask: 0x%02X, Data: 0x%02X\n",
-			offset, reg, mask, data);
-		iexec->execute = 0;
-		return 6;
-	}
-
-	BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X, Value: 0x%02X, "
-		      "Mask: 0x%02X, Data: 0x%02X\n",
-		offset, reg, val.byte, mask, data);
-
-	iexec->execute = ((val.byte & mask) == data);
-
-	return 6;
-}
-
-static int
-init_copy_nv_reg(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_COPY_NV_REG   opcode: 0x5F ('_')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): src reg
-	 * offset + 5  (8  bit): shift
-	 * offset + 6  (32 bit): src mask
-	 * offset + 10 (32 bit): xor
-	 * offset + 14 (32 bit): dst reg
-	 * offset + 18 (32 bit): dst mask
-	 *
-	 * Shift REGVAL("src reg") right by (signed) "shift", AND result with
-	 * "src mask", then XOR with "xor". Write this OR'd with
-	 * (REGVAL("dst reg") AND'd with "dst mask") to "dst reg"
-	 */
-
-	uint32_t srcreg = *((uint32_t *)(&bios->data[offset + 1]));
-	uint8_t shift = bios->data[offset + 5];
-	uint32_t srcmask = *((uint32_t *)(&bios->data[offset + 6]));
-	uint32_t xor = *((uint32_t *)(&bios->data[offset + 10]));
-	uint32_t dstreg = *((uint32_t *)(&bios->data[offset + 14]));
-	uint32_t dstmask = *((uint32_t *)(&bios->data[offset + 18]));
-	uint32_t srcvalue, dstvalue;
-
-	if (!iexec->execute)
-		return 22;
-
-	BIOSLOG(bios, "0x%04X: SrcReg: 0x%08X, Shift: 0x%02X, SrcMask: 0x%08X, "
-		      "Xor: 0x%08X, DstReg: 0x%08X, DstMask: 0x%08X\n",
-		offset, srcreg, shift, srcmask, xor, dstreg, dstmask);
-
-	srcvalue = bios_rd32(bios, srcreg);
-
-	if (shift < 0x80)
-		srcvalue >>= shift;
-	else
-		srcvalue <<= (0x100 - shift);
-
-	srcvalue = (srcvalue & srcmask) ^ xor;
-
-	dstvalue = bios_rd32(bios, dstreg) & dstmask;
-
-	bios_wr32(bios, dstreg, dstvalue | srcvalue);
-
-	return 22;
-}
-
-static int
-init_zm_index_io(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_INDEX_IO   opcode: 0x62 ('b')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): CRTC index
-	 * offset + 4  (8  bit): data
-	 *
-	 * Write "data" to index "CRTC index" of "CRTC port"
-	 */
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t crtcindex = bios->data[offset + 3];
-	uint8_t data = bios->data[offset + 4];
-
-	if (!iexec->execute)
-		return 5;
-
-	bios_idxprt_wr(bios, crtcport, crtcindex, data);
-
-	return 5;
-}
-
-static inline void
-bios_md32(struct nvbios *bios, uint32_t reg,
-	  uint32_t mask, uint32_t val)
-{
-	bios_wr32(bios, reg, (bios_rd32(bios, reg) & ~mask) | val);
-}
-
-static uint32_t
-peek_fb(struct drm_device *dev, struct io_mapping *fb,
-	uint32_t off)
-{
-	uint32_t val = 0;
-
-	if (off < pci_resource_len(dev->pdev, 1)) {
-		uint8_t __iomem *p =
-			io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
-
-		val = ioread32(p + (off & ~PAGE_MASK));
-
-		io_mapping_unmap_atomic(p);
-	}
-
-	return val;
-}
-
-static void
-poke_fb(struct drm_device *dev, struct io_mapping *fb,
-	uint32_t off, uint32_t val)
-{
-	if (off < pci_resource_len(dev->pdev, 1)) {
-		uint8_t __iomem *p =
-			io_mapping_map_atomic_wc(fb, off & PAGE_MASK);
-
-		iowrite32(val, p + (off & ~PAGE_MASK));
-		wmb();
-
-		io_mapping_unmap_atomic(p);
-	}
-}
-
-static inline bool
-read_back_fb(struct drm_device *dev, struct io_mapping *fb,
-	     uint32_t off, uint32_t val)
-{
-	poke_fb(dev, fb, off, val);
-	return val == peek_fb(dev, fb, off);
-}
-
-static int
-nv04_init_compute_mem(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	uint32_t patt = 0xdeadbeef;
-	struct io_mapping *fb;
-	int i;
-
-	/* Map the framebuffer aperture */
-	fb = io_mapping_create_wc(pci_resource_start(dev->pdev, 1),
-				  pci_resource_len(dev->pdev, 1));
-	if (!fb)
-		return -ENOMEM;
-
-	/* Sequencer and refresh off */
-	NVWriteVgaSeq(dev, 0, 1, NVReadVgaSeq(dev, 0, 1) | 0x20);
-	bios_md32(bios, NV04_PFB_DEBUG_0, 0, NV04_PFB_DEBUG_0_REFRESH_OFF);
-
-	bios_md32(bios, NV04_PFB_BOOT_0, ~0,
-		  NV04_PFB_BOOT_0_RAM_AMOUNT_16MB |
-		  NV04_PFB_BOOT_0_RAM_WIDTH_128 |
-		  NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_16MBIT);
-
-	for (i = 0; i < 4; i++)
-		poke_fb(dev, fb, 4 * i, patt);
-
-	poke_fb(dev, fb, 0x400000, patt + 1);
-
-	if (peek_fb(dev, fb, 0) == patt + 1) {
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_TYPE,
-			  NV04_PFB_BOOT_0_RAM_TYPE_SDRAM_16MBIT);
-		bios_md32(bios, NV04_PFB_DEBUG_0,
-			  NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
-
-		for (i = 0; i < 4; i++)
-			poke_fb(dev, fb, 4 * i, patt);
-
-		if ((peek_fb(dev, fb, 0xc) & 0xffff) != (patt & 0xffff))
-			bios_md32(bios, NV04_PFB_BOOT_0,
-				  NV04_PFB_BOOT_0_RAM_WIDTH_128 |
-				  NV04_PFB_BOOT_0_RAM_AMOUNT,
-				  NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
-
-	} else if ((peek_fb(dev, fb, 0xc) & 0xffff0000) !=
-		   (patt & 0xffff0000)) {
-		bios_md32(bios, NV04_PFB_BOOT_0,
-			  NV04_PFB_BOOT_0_RAM_WIDTH_128 |
-			  NV04_PFB_BOOT_0_RAM_AMOUNT,
-			  NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
-
-	} else if (peek_fb(dev, fb, 0) != patt) {
-		if (read_back_fb(dev, fb, 0x800000, patt))
-			bios_md32(bios, NV04_PFB_BOOT_0,
-				  NV04_PFB_BOOT_0_RAM_AMOUNT,
-				  NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
-		else
-			bios_md32(bios, NV04_PFB_BOOT_0,
-				  NV04_PFB_BOOT_0_RAM_AMOUNT,
-				  NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
-
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_TYPE,
-			  NV04_PFB_BOOT_0_RAM_TYPE_SGRAM_8MBIT);
-
-	} else if (!read_back_fb(dev, fb, 0x800000, patt)) {
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
-			  NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
-
-	}
-
-	/* Refresh on, sequencer on */
-	bios_md32(bios, NV04_PFB_DEBUG_0, NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
-	NVWriteVgaSeq(dev, 0, 1, NVReadVgaSeq(dev, 0, 1) & ~0x20);
-
-	io_mapping_free(fb);
-	return 0;
-}
-
-static const uint8_t *
-nv05_memory_config(struct nvbios *bios)
-{
-	/* Defaults for BIOSes lacking a memory config table */
-	static const uint8_t default_config_tab[][2] = {
-		{ 0x24, 0x00 },
-		{ 0x28, 0x00 },
-		{ 0x24, 0x01 },
-		{ 0x1f, 0x00 },
-		{ 0x0f, 0x00 },
-		{ 0x17, 0x00 },
-		{ 0x06, 0x00 },
-		{ 0x00, 0x00 }
-	};
-	int i = (bios_rd32(bios, NV_PEXTDEV_BOOT_0) &
-		 NV_PEXTDEV_BOOT_0_RAMCFG) >> 2;
-
-	if (bios->legacy.mem_init_tbl_ptr)
-		return &bios->data[bios->legacy.mem_init_tbl_ptr + 2 * i];
-	else
-		return default_config_tab[i];
-}
-
-static int
-nv05_init_compute_mem(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	const uint8_t *ramcfg = nv05_memory_config(bios);
-	uint32_t patt = 0xdeadbeef;
-	struct io_mapping *fb;
-	int i, v;
-
-	/* Map the framebuffer aperture */
-	fb = io_mapping_create_wc(pci_resource_start(dev->pdev, 1),
-				  pci_resource_len(dev->pdev, 1));
-	if (!fb)
-		return -ENOMEM;
-
-	/* Sequencer off */
-	NVWriteVgaSeq(dev, 0, 1, NVReadVgaSeq(dev, 0, 1) | 0x20);
-
-	if (bios_rd32(bios, NV04_PFB_BOOT_0) & NV04_PFB_BOOT_0_UMA_ENABLE)
-		goto out;
-
-	bios_md32(bios, NV04_PFB_DEBUG_0, NV04_PFB_DEBUG_0_REFRESH_OFF, 0);
-
-	/* If present load the hardcoded scrambling table */
-	if (bios->legacy.mem_init_tbl_ptr) {
-		uint32_t *scramble_tab = (uint32_t *)&bios->data[
-			bios->legacy.mem_init_tbl_ptr + 0x10];
-
-		for (i = 0; i < 8; i++)
-			bios_wr32(bios, NV04_PFB_SCRAMBLE(i),
-				  ROM32(scramble_tab[i]));
-	}
-
-	/* Set memory type/width/length defaults depending on the straps */
-	bios_md32(bios, NV04_PFB_BOOT_0, 0x3f, ramcfg[0]);
-
-	if (ramcfg[1] & 0x80)
-		bios_md32(bios, NV04_PFB_CFG0, 0, NV04_PFB_CFG0_SCRAMBLE);
-
-	bios_md32(bios, NV04_PFB_CFG1, 0x700001, (ramcfg[1] & 1) << 20);
-	bios_md32(bios, NV04_PFB_CFG1, 0, 1);
-
-	/* Probe memory bus width */
-	for (i = 0; i < 4; i++)
-		poke_fb(dev, fb, 4 * i, patt);
-
-	if (peek_fb(dev, fb, 0xc) != patt)
-		bios_md32(bios, NV04_PFB_BOOT_0,
-			  NV04_PFB_BOOT_0_RAM_WIDTH_128, 0);
-
-	/* Probe memory length */
-	v = bios_rd32(bios, NV04_PFB_BOOT_0) & NV04_PFB_BOOT_0_RAM_AMOUNT;
-
-	if (v == NV04_PFB_BOOT_0_RAM_AMOUNT_32MB &&
-	    (!read_back_fb(dev, fb, 0x1000000, ++patt) ||
-	     !read_back_fb(dev, fb, 0, ++patt)))
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
-			  NV04_PFB_BOOT_0_RAM_AMOUNT_16MB);
-
-	if (v == NV04_PFB_BOOT_0_RAM_AMOUNT_16MB &&
-	    !read_back_fb(dev, fb, 0x800000, ++patt))
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
-			  NV04_PFB_BOOT_0_RAM_AMOUNT_8MB);
-
-	if (!read_back_fb(dev, fb, 0x400000, ++patt))
-		bios_md32(bios, NV04_PFB_BOOT_0, NV04_PFB_BOOT_0_RAM_AMOUNT,
-			  NV04_PFB_BOOT_0_RAM_AMOUNT_4MB);
-
-out:
-	/* Sequencer on */
-	NVWriteVgaSeq(dev, 0, 1, NVReadVgaSeq(dev, 0, 1) & ~0x20);
-
-	io_mapping_free(fb);
-	return 0;
-}
-
-static int
-nv10_init_compute_mem(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	const int mem_width[] = { 0x10, 0x00, 0x20 };
-	const int mem_width_count = (dev_priv->chipset >= 0x17 ? 3 : 2);
-	uint32_t patt = 0xdeadbeef;
-	struct io_mapping *fb;
-	int i, j, k;
-
-	/* Map the framebuffer aperture */
-	fb = io_mapping_create_wc(pci_resource_start(dev->pdev, 1),
-				  pci_resource_len(dev->pdev, 1));
-	if (!fb)
-		return -ENOMEM;
-
-	bios_wr32(bios, NV10_PFB_REFCTRL, NV10_PFB_REFCTRL_VALID_1);
-
-	/* Probe memory bus width */
-	for (i = 0; i < mem_width_count; i++) {
-		bios_md32(bios, NV04_PFB_CFG0, 0x30, mem_width[i]);
-
-		for (j = 0; j < 4; j++) {
-			for (k = 0; k < 4; k++)
-				poke_fb(dev, fb, 0x1c, 0);
-
-			poke_fb(dev, fb, 0x1c, patt);
-			poke_fb(dev, fb, 0x3c, 0);
-
-			if (peek_fb(dev, fb, 0x1c) == patt)
-				goto mem_width_found;
-		}
-	}
-
-mem_width_found:
-	patt <<= 1;
-
-	/* Probe amount of installed memory */
-	for (i = 0; i < 4; i++) {
-		int off = bios_rd32(bios, NV04_PFB_FIFO_DATA) - 0x100000;
-
-		poke_fb(dev, fb, off, patt);
-		poke_fb(dev, fb, 0, 0);
-
-		peek_fb(dev, fb, 0);
-		peek_fb(dev, fb, 0);
-		peek_fb(dev, fb, 0);
-		peek_fb(dev, fb, 0);
-
-		if (peek_fb(dev, fb, off) == patt)
-			goto amount_found;
-	}
-
-	/* IC missing - disable the upper half memory space. */
-	bios_md32(bios, NV04_PFB_CFG0, 0x1000, 0);
-
-amount_found:
-	io_mapping_free(fb);
-	return 0;
-}
-
-static int
-nv20_init_compute_mem(struct nvbios *bios)
-{
-	struct drm_device *dev = bios->dev;
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	uint32_t mask = (dev_priv->chipset >= 0x25 ? 0x300 : 0x900);
-	uint32_t amount, off;
-	struct io_mapping *fb;
-
-	/* Map the framebuffer aperture */
-	fb = io_mapping_create_wc(pci_resource_start(dev->pdev, 1),
-				  pci_resource_len(dev->pdev, 1));
-	if (!fb)
-		return -ENOMEM;
-
-	bios_wr32(bios, NV10_PFB_REFCTRL, NV10_PFB_REFCTRL_VALID_1);
-
-	/* Allow full addressing */
-	bios_md32(bios, NV04_PFB_CFG0, 0, mask);
-
-	amount = bios_rd32(bios, NV04_PFB_FIFO_DATA);
-	for (off = amount; off > 0x2000000; off -= 0x2000000)
-		poke_fb(dev, fb, off - 4, off);
-
-	amount = bios_rd32(bios, NV04_PFB_FIFO_DATA);
-	if (amount != peek_fb(dev, fb, amount - 4))
-		/* IC missing - disable the upper half memory space. */
-		bios_md32(bios, NV04_PFB_CFG0, mask, 0);
-
-	io_mapping_free(fb);
-	return 0;
-}
-
-static int
-init_compute_mem(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_COMPUTE_MEM   opcode: 0x63 ('c')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * This opcode is meant to set the PFB memory config registers
-	 * appropriately so that we can correctly calculate how much VRAM it
-	 * has (on nv10 and better chipsets the amount of installed VRAM is
-	 * subsequently reported in NV_PFB_CSTATUS (0x10020C)).
-	 *
-	 * The implementation of this opcode in general consists of several
-	 * parts:
-	 *
-	 * 1) Determination of memory type and density. Only necessary for
-	 *    really old chipsets, the memory type reported by the strap bits
-	 *    (0x101000) is assumed to be accurate on nv05 and newer.
-	 *
-	 * 2) Determination of the memory bus width. Usually done by a cunning
-	 *    combination of writes to offsets 0x1c and 0x3c in the fb, and
-	 *    seeing whether the written values are read back correctly.
-	 *
-	 *    Only necessary on nv0x-nv1x and nv34, on the other cards we can
-	 *    trust the straps.
-	 *
-	 * 3) Determination of how many of the card's RAM pads have ICs
-	 *    attached, usually done by a cunning combination of writes to an
-	 *    offset slightly less than the maximum memory reported by
-	 *    NV_PFB_CSTATUS, then seeing if the test pattern can be read back.
-	 *
-	 * This appears to be a NOP on IGPs and NV4x or newer chipsets, both io
-	 * logs of the VBIOS and kmmio traces of the binary driver POSTing the
-	 * card show nothing being done for this opcode. Why is it still listed
-	 * in the table?!
-	 */
-
-	/* no iexec->execute check by design */
-
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	int ret;
-
-	if (dev_priv->chipset >= 0x40 ||
-	    dev_priv->chipset == 0x1a ||
-	    dev_priv->chipset == 0x1f)
-		ret = 0;
-	else if (dev_priv->chipset >= 0x20 &&
-		 dev_priv->chipset != 0x34)
-		ret = nv20_init_compute_mem(bios);
-	else if (dev_priv->chipset >= 0x10)
-		ret = nv10_init_compute_mem(bios);
-	else if (dev_priv->chipset >= 0x5)
-		ret = nv05_init_compute_mem(bios);
-	else
-		ret = nv04_init_compute_mem(bios);
-
-	if (ret)
-		return ret;
-
-	return 1;
-}
-
-static int
-init_reset(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_RESET   opcode: 0x65 ('e')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (32 bit): value1
-	 * offset + 9  (32 bit): value2
-	 *
-	 * Assign "value1" to "register", then assign "value2" to "register"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint32_t value1 = ROM32(bios->data[offset + 5]);
-	uint32_t value2 = ROM32(bios->data[offset + 9]);
-	uint32_t pci_nv_19, pci_nv_20;
-
-	/* no iexec->execute check by design */
-
-	pci_nv_19 = bios_rd32(bios, NV_PBUS_PCI_NV_19);
-	bios_wr32(bios, NV_PBUS_PCI_NV_19, pci_nv_19 & ~0xf00);
-
-	bios_wr32(bios, reg, value1);
-
-	udelay(10);
-
-	bios_wr32(bios, reg, value2);
-	bios_wr32(bios, NV_PBUS_PCI_NV_19, pci_nv_19);
-
-	pci_nv_20 = bios_rd32(bios, NV_PBUS_PCI_NV_20);
-	pci_nv_20 &= ~NV_PBUS_PCI_NV_20_ROM_SHADOW_ENABLED;	/* 0xfffffffe */
-	bios_wr32(bios, NV_PBUS_PCI_NV_20, pci_nv_20);
-
-	return 13;
-}
-
-static int
-init_configure_mem(struct nvbios *bios, uint16_t offset,
-		   struct init_exec *iexec)
-{
-	/*
-	 * INIT_CONFIGURE_MEM   opcode: 0x66 ('f')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Equivalent to INIT_DONE on bios version 3 or greater.
-	 * For early bios versions, sets up the memory registers, using values
-	 * taken from the memory init table
-	 */
-
-	/* no iexec->execute check by design */
-
-	uint16_t meminitoffs = bios->legacy.mem_init_tbl_ptr + MEM_INIT_SIZE * (bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, NV_CIO_CRE_SCRATCH4__INDEX) >> 4);
-	uint16_t seqtbloffs = bios->legacy.sdr_seq_tbl_ptr, meminitdata = meminitoffs + 6;
-	uint32_t reg, data;
-
-	if (bios->major_version > 2)
-		return 0;
-
-	bios_idxprt_wr(bios, NV_VIO_SRX, NV_VIO_SR_CLOCK_INDEX, bios_idxprt_rd(
-		       bios, NV_VIO_SRX, NV_VIO_SR_CLOCK_INDEX) | 0x20);
-
-	if (bios->data[meminitoffs] & 1)
-		seqtbloffs = bios->legacy.ddr_seq_tbl_ptr;
-
-	for (reg = ROM32(bios->data[seqtbloffs]);
-	     reg != 0xffffffff;
-	     reg = ROM32(bios->data[seqtbloffs += 4])) {
-
-		switch (reg) {
-		case NV04_PFB_PRE:
-			data = NV04_PFB_PRE_CMD_PRECHARGE;
-			break;
-		case NV04_PFB_PAD:
-			data = NV04_PFB_PAD_CKE_NORMAL;
-			break;
-		case NV04_PFB_REF:
-			data = NV04_PFB_REF_CMD_REFRESH;
-			break;
-		default:
-			data = ROM32(bios->data[meminitdata]);
-			meminitdata += 4;
-			if (data == 0xffffffff)
-				continue;
-		}
-
-		bios_wr32(bios, reg, data);
-	}
-
-	return 1;
-}
-
-static int
-init_configure_clk(struct nvbios *bios, uint16_t offset,
-		   struct init_exec *iexec)
-{
-	/*
-	 * INIT_CONFIGURE_CLK   opcode: 0x67 ('g')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Equivalent to INIT_DONE on bios version 3 or greater.
-	 * For early bios versions, sets up the NVClk and MClk PLLs, using
-	 * values taken from the memory init table
-	 */
-
-	/* no iexec->execute check by design */
-
-	uint16_t meminitoffs = bios->legacy.mem_init_tbl_ptr + MEM_INIT_SIZE * (bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, NV_CIO_CRE_SCRATCH4__INDEX) >> 4);
-	int clock;
-
-	if (bios->major_version > 2)
-		return 0;
-
-	clock = ROM16(bios->data[meminitoffs + 4]) * 10;
-	setPLL(bios, NV_PRAMDAC_NVPLL_COEFF, clock);
-
-	clock = ROM16(bios->data[meminitoffs + 2]) * 10;
-	if (bios->data[meminitoffs] & 1) /* DDR */
-		clock *= 2;
-	setPLL(bios, NV_PRAMDAC_MPLL_COEFF, clock);
-
-	return 1;
-}
-
-static int
-init_configure_preinit(struct nvbios *bios, uint16_t offset,
-		       struct init_exec *iexec)
-{
-	/*
-	 * INIT_CONFIGURE_PREINIT   opcode: 0x68 ('h')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Equivalent to INIT_DONE on bios version 3 or greater.
-	 * For early bios versions, does early init, loading ram and crystal
-	 * configuration from straps into CR3C
-	 */
-
-	/* no iexec->execute check by design */
-
-	uint32_t straps = bios_rd32(bios, NV_PEXTDEV_BOOT_0);
-	uint8_t cr3c = ((straps << 2) & 0xf0) | (straps & 0x40) >> 6;
-
-	if (bios->major_version > 2)
-		return 0;
-
-	bios_idxprt_wr(bios, NV_CIO_CRX__COLOR,
-			     NV_CIO_CRE_SCRATCH4__INDEX, cr3c);
-
-	return 1;
-}
-
-static int
-init_io(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO   opcode: 0x69 ('i')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): mask
-	 * offset + 4  (8  bit): data
-	 *
-	 * Assign ((IOVAL("crtc port") & "mask") | "data") to "crtc port"
-	 */
-
-	struct drm_nouveau_private *dev_priv = bios->dev->dev_private;
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t mask = bios->data[offset + 3];
-	uint8_t data = bios->data[offset + 4];
-
-	if (!iexec->execute)
-		return 5;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Mask: 0x%02X, Data: 0x%02X\n",
-		offset, crtcport, mask, data);
-
-	/*
-	 * I have no idea what this does, but NVIDIA do this magic sequence
-	 * in the places where this INIT_IO happens..
-	 */
-	if (dev_priv->card_type >= NV_50 && crtcport == 0x3c3 && data == 1) {
-		int i;
-
-		bios_wr32(bios, 0x614100, (bios_rd32(
-			  bios, 0x614100) & 0x0fffffff) | 0x00800000);
-
-		bios_wr32(bios, 0x00e18c, bios_rd32(
-			  bios, 0x00e18c) | 0x00020000);
-
-		bios_wr32(bios, 0x614900, (bios_rd32(
-			  bios, 0x614900) & 0x0fffffff) | 0x00800000);
-
-		bios_wr32(bios, 0x000200, bios_rd32(
-			  bios, 0x000200) & ~0x40000000);
-
-		mdelay(10);
-
-		bios_wr32(bios, 0x00e18c, bios_rd32(
-			  bios, 0x00e18c) & ~0x00020000);
-
-		bios_wr32(bios, 0x000200, bios_rd32(
-			  bios, 0x000200) | 0x40000000);
-
-		bios_wr32(bios, 0x614100, 0x00800018);
-		bios_wr32(bios, 0x614900, 0x00800018);
-
-		mdelay(10);
-
-		bios_wr32(bios, 0x614100, 0x10000018);
-		bios_wr32(bios, 0x614900, 0x10000018);
-
-		for (i = 0; i < 3; i++)
-			bios_wr32(bios, 0x614280 + (i*0x800), bios_rd32(
-				  bios, 0x614280 + (i*0x800)) & 0xf0f0f0f0);
-
-		for (i = 0; i < 2; i++)
-			bios_wr32(bios, 0x614300 + (i*0x800), bios_rd32(
-				  bios, 0x614300 + (i*0x800)) & 0xfffff0f0);
-
-		for (i = 0; i < 3; i++)
-			bios_wr32(bios, 0x614380 + (i*0x800), bios_rd32(
-				  bios, 0x614380 + (i*0x800)) & 0xfffff0f0);
-
-		for (i = 0; i < 2; i++)
-			bios_wr32(bios, 0x614200 + (i*0x800), bios_rd32(
-				  bios, 0x614200 + (i*0x800)) & 0xfffffff0);
-
-		for (i = 0; i < 2; i++)
-			bios_wr32(bios, 0x614108 + (i*0x800), bios_rd32(
-				  bios, 0x614108 + (i*0x800)) & 0x0fffffff);
-		return 5;
-	}
-
-	bios_port_wr(bios, crtcport, (bios_port_rd(bios, crtcport) & mask) |
-									data);
-	return 5;
-}
-
-static int
-init_sub(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_SUB   opcode: 0x6B ('k')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): script number
-	 *
-	 * Execute script number "script number", as a subroutine
-	 */
-
-	uint8_t sub = bios->data[offset + 1];
-
-	if (!iexec->execute)
-		return 2;
-
-	BIOSLOG(bios, "0x%04X: Calling script %d\n", offset, sub);
-
-	parse_init_table(bios,
-			 ROM16(bios->data[bios->init_script_tbls_ptr + sub * 2]),
-			 iexec);
-
-	BIOSLOG(bios, "0x%04X: End of script %d\n", offset, sub);
-
-	return 2;
-}
-
-static int
-init_ram_condition(struct nvbios *bios, uint16_t offset,
-		   struct init_exec *iexec)
-{
-	/*
-	 * INIT_RAM_CONDITION   opcode: 0x6D ('m')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): mask
-	 * offset + 2  (8 bit): cmpval
-	 *
-	 * Test if (NV04_PFB_BOOT_0 & "mask") equals "cmpval".
-	 * If condition not met skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t mask = bios->data[offset + 1];
-	uint8_t cmpval = bios->data[offset + 2];
-	uint8_t data;
-
-	if (!iexec->execute)
-		return 3;
-
-	data = bios_rd32(bios, NV04_PFB_BOOT_0) & mask;
-
-	BIOSLOG(bios, "0x%04X: Checking if 0x%08X equals 0x%08X\n",
-		offset, data, cmpval);
-
-	if (data == cmpval)
-		BIOSLOG(bios, "0x%04X: Condition fulfilled -- continuing to execute\n", offset);
-	else {
-		BIOSLOG(bios, "0x%04X: Condition not fulfilled -- skipping following commands\n", offset);
-		iexec->execute = false;
-	}
-
-	return 3;
-}
-
-static int
-init_nv_reg(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_NV_REG   opcode: 0x6E ('n')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (32 bit): mask
-	 * offset + 9  (32 bit): data
-	 *
-	 * Assign ((REGVAL("register") & "mask") | "data") to "register"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint32_t mask = ROM32(bios->data[offset + 5]);
-	uint32_t data = ROM32(bios->data[offset + 9]);
-
-	if (!iexec->execute)
-		return 13;
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, Mask: 0x%08X, Data: 0x%08X\n",
-		offset, reg, mask, data);
-
-	bios_wr32(bios, reg, (bios_rd32(bios, reg) & mask) | data);
-
-	return 13;
-}
-
-static int
-init_macro(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_MACRO   opcode: 0x6F ('o')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): macro number
-	 *
-	 * Look up macro index "macro number" in the macro index table.
-	 * The macro index table entry has 1 byte for the index in the macro
-	 * table, and 1 byte for the number of times to repeat the macro.
-	 * The macro table entry has 4 bytes for the register address and
-	 * 4 bytes for the value to write to that register
-	 */
-
-	uint8_t macro_index_tbl_idx = bios->data[offset + 1];
-	uint16_t tmp = bios->macro_index_tbl_ptr + (macro_index_tbl_idx * MACRO_INDEX_SIZE);
-	uint8_t macro_tbl_idx = bios->data[tmp];
-	uint8_t count = bios->data[tmp + 1];
-	uint32_t reg, data;
-	int i;
-
-	if (!iexec->execute)
-		return 2;
-
-	BIOSLOG(bios, "0x%04X: Macro: 0x%02X, MacroTableIndex: 0x%02X, "
-		      "Count: 0x%02X\n",
-		offset, macro_index_tbl_idx, macro_tbl_idx, count);
-
-	for (i = 0; i < count; i++) {
-		uint16_t macroentryptr = bios->macro_tbl_ptr + (macro_tbl_idx + i) * MACRO_SIZE;
-
-		reg = ROM32(bios->data[macroentryptr]);
-		data = ROM32(bios->data[macroentryptr + 4]);
-
-		bios_wr32(bios, reg, data);
-	}
-
-	return 2;
-}
-
-static int
-init_done(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_DONE   opcode: 0x71 ('q')
-	 *
-	 * offset      (8  bit): opcode
-	 *
-	 * End the current script
-	 */
-
-	/* mild retval abuse to stop parsing this table */
-	return 0;
-}
-
-static int
-init_resume(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_RESUME   opcode: 0x72 ('r')
-	 *
-	 * offset      (8  bit): opcode
-	 *
-	 * End the current execute / no-execute condition
-	 */
-
-	if (iexec->execute)
-		return 1;
-
-	iexec->execute = true;
-	BIOSLOG(bios, "0x%04X: ---- Executing following commands ----\n", offset);
-
-	return 1;
-}
-
-static int
-init_time(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_TIME   opcode: 0x74 ('t')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): time
-	 *
-	 * Sleep for "time" microseconds.
-	 */
-
-	unsigned time = ROM16(bios->data[offset + 1]);
-
-	if (!iexec->execute)
-		return 3;
-
-	BIOSLOG(bios, "0x%04X: Sleeping for 0x%04X microseconds\n",
-		offset, time);
-
-	if (time < 1000)
-		udelay(time);
-	else
-		mdelay((time + 900) / 1000);
-
-	return 3;
-}
-
-static int
-init_condition(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_CONDITION   opcode: 0x75 ('u')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): condition number
-	 *
-	 * Check condition "condition number" in the condition table.
-	 * If condition not met skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t cond = bios->data[offset + 1];
-
-	if (!iexec->execute)
-		return 2;
-
-	BIOSLOG(bios, "0x%04X: Condition: 0x%02X\n", offset, cond);
-
-	if (bios_condition_met(bios, offset, cond))
-		BIOSLOG(bios, "0x%04X: Condition fulfilled -- continuing to execute\n", offset);
-	else {
-		BIOSLOG(bios, "0x%04X: Condition not fulfilled -- skipping following commands\n", offset);
-		iexec->execute = false;
-	}
-
-	return 2;
-}
-
-static int
-init_io_condition(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_IO_CONDITION  opcode: 0x76
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): condition number
-	 *
-	 * Check condition "condition number" in the io condition table.
-	 * If condition not met skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t cond = bios->data[offset + 1];
-
-	if (!iexec->execute)
-		return 2;
-
-	BIOSLOG(bios, "0x%04X: IO condition: 0x%02X\n", offset, cond);
-
-	if (io_condition_met(bios, offset, cond))
-		BIOSLOG(bios, "0x%04X: Condition fulfilled -- continuing to execute\n", offset);
-	else {
-		BIOSLOG(bios, "0x%04X: Condition not fulfilled -- skipping following commands\n", offset);
-		iexec->execute = false;
-	}
-
-	return 2;
-}
-
-static int
-init_index_io(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_INDEX_IO   opcode: 0x78 ('x')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (16 bit): CRTC port
-	 * offset + 3  (8  bit): CRTC index
-	 * offset + 4  (8  bit): mask
-	 * offset + 5  (8  bit): data
-	 *
-	 * Read value at index "CRTC index" on "CRTC port", AND with "mask",
-	 * OR with "data", write-back
-	 */
-
-	uint16_t crtcport = ROM16(bios->data[offset + 1]);
-	uint8_t crtcindex = bios->data[offset + 3];
-	uint8_t mask = bios->data[offset + 4];
-	uint8_t data = bios->data[offset + 5];
-	uint8_t value;
-
-	if (!iexec->execute)
-		return 6;
-
-	BIOSLOG(bios, "0x%04X: Port: 0x%04X, Index: 0x%02X, Mask: 0x%02X, "
-		      "Data: 0x%02X\n",
-		offset, crtcport, crtcindex, mask, data);
-
-	value = (bios_idxprt_rd(bios, crtcport, crtcindex) & mask) | data;
-	bios_idxprt_wr(bios, crtcport, crtcindex, value);
-
-	return 6;
-}
-
-static int
-init_pll(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_PLL   opcode: 0x79 ('y')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (16 bit): freq
-	 *
-	 * Set PLL register "register" to coefficients for frequency (10kHz)
-	 * "freq"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint16_t freq = ROM16(bios->data[offset + 5]);
-
-	if (!iexec->execute)
-		return 7;
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, Freq: %d0kHz\n", offset, reg, freq);
-
-	setPLL(bios, reg, freq * 10);
-
-	return 7;
-}
-
-static int
-init_zm_reg(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_REG   opcode: 0x7A ('z')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (32 bit): value
-	 *
-	 * Assign "value" to "register"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint32_t value = ROM32(bios->data[offset + 5]);
-
-	if (!iexec->execute)
-		return 9;
-
-	if (reg == 0x000200)
-		value |= 1;
-
-	bios_wr32(bios, reg, value);
-
-	return 9;
-}
-
-static int
-init_ram_restrict_pll(struct nvbios *bios, uint16_t offset,
-		      struct init_exec *iexec)
-{
-	/*
-	 * INIT_RAM_RESTRICT_PLL   opcode: 0x87 ('')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): PLL type
-	 * offset + 2 (32 bit): frequency 0
-	 *
-	 * Uses the RAMCFG strap of PEXTDEV_BOOT as an index into the table at
-	 * ram_restrict_table_ptr.  The value read from there is used to select
-	 * a frequency from the table starting at 'frequency 0' to be
-	 * programmed into the PLL corresponding to 'type'.
-	 *
-	 * The PLL limits table on cards using this opcode has a mapping of
-	 * 'type' to the relevant registers.
-	 */
-
-	struct drm_device *dev = bios->dev;
-	uint32_t strap = (bios_rd32(bios, NV_PEXTDEV_BOOT_0) & 0x0000003c) >> 2;
-	uint8_t index = bios->data[bios->ram_restrict_tbl_ptr + strap];
-	uint8_t type = bios->data[offset + 1];
-	uint32_t freq = ROM32(bios->data[offset + 2 + (index * 4)]);
-	uint8_t *pll_limits = &bios->data[bios->pll_limit_tbl_ptr], *entry;
-	int len = 2 + bios->ram_restrict_group_count * 4;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	if (!bios->pll_limit_tbl_ptr || (pll_limits[0] & 0xf0) != 0x30) {
-		NV_ERROR(dev, "PLL limits table not version 3.x\n");
-		return len; /* deliberate, allow default clocks to remain */
-	}
-
-	entry = pll_limits + pll_limits[1];
-	for (i = 0; i < pll_limits[3]; i++, entry += pll_limits[2]) {
-		if (entry[0] == type) {
-			uint32_t reg = ROM32(entry[3]);
-
-			BIOSLOG(bios, "0x%04X: "
-				      "Type %02x Reg 0x%08x Freq %dKHz\n",
-				offset, type, reg, freq);
-
-			setPLL(bios, reg, freq);
-			return len;
-		}
-	}
-
-	NV_ERROR(dev, "PLL type 0x%02x not found in PLL limits table", type);
-	return len;
-}
-
-static int
-init_8c(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_8C   opcode: 0x8C ('')
-	 *
-	 * NOP so far....
-	 *
-	 */
-
-	return 1;
-}
-
-static int
-init_8d(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_8D   opcode: 0x8D ('')
-	 *
-	 * NOP so far....
-	 *
-	 */
-
-	return 1;
-}
-
-static int
-init_gpio(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_GPIO   opcode: 0x8E ('')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Loop over all entries in the DCB GPIO table, and initialise
-	 * each GPIO according to various values listed in each entry
-	 */
-
-	if (iexec->execute && bios->execute)
-		nouveau_gpio_reset(bios->dev);
-
-	return 1;
-}
-
-static int
-init_ram_restrict_zm_reg_group(struct nvbios *bios, uint16_t offset,
-			       struct init_exec *iexec)
-{
-	/*
-	 * INIT_RAM_RESTRICT_ZM_REG_GROUP   opcode: 0x8F ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): reg
-	 * offset + 5  (8  bit): regincrement
-	 * offset + 6  (8  bit): count
-	 * offset + 7  (32 bit): value 1,1
-	 * ...
-	 *
-	 * Use the RAMCFG strap of PEXTDEV_BOOT as an index into the table at
-	 * ram_restrict_table_ptr. The value read from here is 'n', and
-	 * "value 1,n" gets written to "reg". This repeats "count" times and on
-	 * each iteration 'm', "reg" increases by "regincrement" and
-	 * "value m,n" is used. The extent of n is limited by a number read
-	 * from the 'M' BIT table, herein called "blocklen"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint8_t regincrement = bios->data[offset + 5];
-	uint8_t count = bios->data[offset + 6];
-	uint32_t strap_ramcfg, data;
-	/* previously set by 'M' BIT table */
-	uint16_t blocklen = bios->ram_restrict_group_count * 4;
-	int len = 7 + count * blocklen;
-	uint8_t index;
-	int i;
-
-	/* critical! to know the length of the opcode */;
-	if (!blocklen) {
-		NV_ERROR(bios->dev,
-			 "0x%04X: Zero block length - has the M table "
-			 "been parsed?\n", offset);
-		return -EINVAL;
-	}
-
-	if (!iexec->execute)
-		return len;
-
-	strap_ramcfg = (bios_rd32(bios, NV_PEXTDEV_BOOT_0) >> 2) & 0xf;
-	index = bios->data[bios->ram_restrict_tbl_ptr + strap_ramcfg];
-
-	BIOSLOG(bios, "0x%04X: Reg: 0x%08X, RegIncrement: 0x%02X, "
-		      "Count: 0x%02X, StrapRamCfg: 0x%02X, Index: 0x%02X\n",
-		offset, reg, regincrement, count, strap_ramcfg, index);
-
-	for (i = 0; i < count; i++) {
-		data = ROM32(bios->data[offset + 7 + index * 4 + blocklen * i]);
-
-		bios_wr32(bios, reg, data);
-
-		reg += regincrement;
-	}
-
-	return len;
-}
-
-static int
-init_copy_zm_reg(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_COPY_ZM_REG   opcode: 0x90 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): src reg
-	 * offset + 5  (32 bit): dst reg
-	 *
-	 * Put contents of "src reg" into "dst reg"
-	 */
-
-	uint32_t srcreg = ROM32(bios->data[offset + 1]);
-	uint32_t dstreg = ROM32(bios->data[offset + 5]);
-
-	if (!iexec->execute)
-		return 9;
-
-	bios_wr32(bios, dstreg, bios_rd32(bios, srcreg));
-
-	return 9;
-}
-
-static int
-init_zm_reg_group_addr_latched(struct nvbios *bios, uint16_t offset,
-			       struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_REG_GROUP_ADDRESS_LATCHED   opcode: 0x91 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): dst reg
-	 * offset + 5  (8  bit): count
-	 * offset + 6  (32 bit): data 1
-	 * ...
-	 *
-	 * For each of "count" values write "data n" to "dst reg"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint8_t count = bios->data[offset + 5];
-	int len = 6 + count * 4;
-	int i;
-
-	if (!iexec->execute)
-		return len;
-
-	for (i = 0; i < count; i++) {
-		uint32_t data = ROM32(bios->data[offset + 6 + 4 * i]);
-		bios_wr32(bios, reg, data);
-	}
-
-	return len;
-}
-
-static int
-init_reserved(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_RESERVED   opcode: 0x92 ('')
-	 *
-	 * offset      (8 bit): opcode
-	 *
-	 * Seemingly does nothing
-	 */
-
-	return 1;
-}
-
-static int
-init_96(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_96   opcode: 0x96 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): sreg
-	 * offset + 5  (8  bit): sshift
-	 * offset + 6  (8  bit): smask
-	 * offset + 7  (8  bit): index
-	 * offset + 8  (32 bit): reg
-	 * offset + 12 (32 bit): mask
-	 * offset + 16 (8  bit): shift
-	 *
-	 */
-
-	uint16_t xlatptr = bios->init96_tbl_ptr + (bios->data[offset + 7] * 2);
-	uint32_t reg = ROM32(bios->data[offset + 8]);
-	uint32_t mask = ROM32(bios->data[offset + 12]);
-	uint32_t val;
-
-	val = bios_rd32(bios, ROM32(bios->data[offset + 1]));
-	if (bios->data[offset + 5] < 0x80)
-		val >>= bios->data[offset + 5];
-	else
-		val <<= (0x100 - bios->data[offset + 5]);
-	val &= bios->data[offset + 6];
-
-	val   = bios->data[ROM16(bios->data[xlatptr]) + val];
-	val <<= bios->data[offset + 16];
-
-	if (!iexec->execute)
-		return 17;
-
-	bios_wr32(bios, reg, (bios_rd32(bios, reg) & mask) | val);
-	return 17;
-}
-
-static int
-init_97(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_97   opcode: 0x97 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): register
-	 * offset + 5  (32 bit): mask
-	 * offset + 9  (32 bit): value
-	 *
-	 * Adds "value" to "register" preserving the fields specified
-	 * by "mask"
-	 */
-
-	uint32_t reg = ROM32(bios->data[offset + 1]);
-	uint32_t mask = ROM32(bios->data[offset + 5]);
-	uint32_t add = ROM32(bios->data[offset + 9]);
-	uint32_t val;
-
-	val = bios_rd32(bios, reg);
-	val = (val & mask) | ((val + add) & ~mask);
-
-	if (!iexec->execute)
-		return 13;
-
-	bios_wr32(bios, reg, val);
-	return 13;
-}
-
-static int
-init_auxch(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_AUXCH   opcode: 0x98 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): address
-	 * offset + 5  (8  bit): count
-	 * offset + 6  (8  bit): mask 0
-	 * offset + 7  (8  bit): data 0
-	 *  ...
-	 *
-	 */
-
-	struct drm_device *dev = bios->dev;
-	struct nouveau_i2c_chan *auxch;
-	uint32_t addr = ROM32(bios->data[offset + 1]);
-	uint8_t count = bios->data[offset + 5];
-	int len = 6 + count * 2;
-	int ret, i;
-
-	if (!bios->display.output) {
-		NV_ERROR(dev, "INIT_AUXCH: no active output\n");
-		return len;
-	}
-
-	auxch = init_i2c_device_find(dev, bios->display.output->i2c_index);
-	if (!auxch) {
-		NV_ERROR(dev, "INIT_AUXCH: couldn't get auxch %d\n",
-			 bios->display.output->i2c_index);
-		return len;
-	}
-
-	if (!iexec->execute)
-		return len;
-
-	offset += 6;
-	for (i = 0; i < count; i++, offset += 2) {
-		uint8_t data;
-
-		ret = nouveau_dp_auxch(auxch, 9, addr, &data, 1);
-		if (ret) {
-			NV_ERROR(dev, "INIT_AUXCH: rd auxch fail %d\n", ret);
-			return len;
-		}
-
-		data &= bios->data[offset + 0];
-		data |= bios->data[offset + 1];
-
-		ret = nouveau_dp_auxch(auxch, 8, addr, &data, 1);
-		if (ret) {
-			NV_ERROR(dev, "INIT_AUXCH: wr auxch fail %d\n", ret);
-			return len;
-		}
-	}
-
-	return len;
-}
-
-static int
-init_zm_auxch(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_ZM_AUXCH   opcode: 0x99 ('')
-	 *
-	 * offset      (8  bit): opcode
-	 * offset + 1  (32 bit): address
-	 * offset + 5  (8  bit): count
-	 * offset + 6  (8  bit): data 0
-	 *  ...
-	 *
-	 */
-
-	struct drm_device *dev = bios->dev;
-	struct nouveau_i2c_chan *auxch;
-	uint32_t addr = ROM32(bios->data[offset + 1]);
-	uint8_t count = bios->data[offset + 5];
-	int len = 6 + count;
-	int ret, i;
-
-	if (!bios->display.output) {
-		NV_ERROR(dev, "INIT_ZM_AUXCH: no active output\n");
-		return len;
-	}
-
-	auxch = init_i2c_device_find(dev, bios->display.output->i2c_index);
-	if (!auxch) {
-		NV_ERROR(dev, "INIT_ZM_AUXCH: couldn't get auxch %d\n",
-			 bios->display.output->i2c_index);
-		return len;
-	}
-
-	if (!iexec->execute)
-		return len;
-
-	offset += 6;
-	for (i = 0; i < count; i++, offset++) {
-		ret = nouveau_dp_auxch(auxch, 8, addr, &bios->data[offset], 1);
-		if (ret) {
-			NV_ERROR(dev, "INIT_ZM_AUXCH: wr auxch fail %d\n", ret);
-			return len;
-		}
-	}
-
-	return len;
-}
-
-static int
-init_i2c_long_if(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * INIT_I2C_LONG_IF   opcode: 0x9A ('')
-	 *
-	 * offset      (8 bit): opcode
-	 * offset + 1  (8 bit): DCB I2C table entry index
-	 * offset + 2  (8 bit): I2C slave address
-	 * offset + 3  (16 bit): I2C register
-	 * offset + 5  (8 bit): mask
-	 * offset + 6  (8 bit): data
-	 *
-	 * Read the register given by "I2C register" on the device addressed
-	 * by "I2C slave address" on the I2C bus given by "DCB I2C table
-	 * entry index". Compare the result AND "mask" to "data".
-	 * If they're not equal, skip subsequent opcodes until condition is
-	 * inverted (INIT_NOT), or we hit INIT_RESUME
-	 */
-
-	uint8_t i2c_index = bios->data[offset + 1];
-	uint8_t i2c_address = bios->data[offset + 2] >> 1;
-	uint8_t reglo = bios->data[offset + 3];
-	uint8_t reghi = bios->data[offset + 4];
-	uint8_t mask = bios->data[offset + 5];
-	uint8_t data = bios->data[offset + 6];
-	struct nouveau_i2c_chan *chan;
-	uint8_t buf0[2] = { reghi, reglo };
-	uint8_t buf1[1];
-	struct i2c_msg msg[2] = {
-		{ i2c_address, 0, 1, buf0 },
-		{ i2c_address, I2C_M_RD, 1, buf1 },
-	};
-	int ret;
-
-	/* no execute check by design */
-
-	BIOSLOG(bios, "0x%04X: DCBI2CIndex: 0x%02X, I2CAddress: 0x%02X\n",
-		offset, i2c_index, i2c_address);
-
-	chan = init_i2c_device_find(bios->dev, i2c_index);
-	if (!chan)
-		return -ENODEV;
-
-
-	ret = i2c_transfer(&chan->adapter, msg, 2);
-	if (ret < 0) {
-		BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X:0x%02X, Value: [no device], "
-			      "Mask: 0x%02X, Data: 0x%02X\n",
-			offset, reghi, reglo, mask, data);
-		iexec->execute = 0;
-		return 7;
-	}
-
-	BIOSLOG(bios, "0x%04X: I2CReg: 0x%02X:0x%02X, Value: 0x%02X, "
-		      "Mask: 0x%02X, Data: 0x%02X\n",
-		offset, reghi, reglo, buf1[0], mask, data);
-
-	iexec->execute = ((buf1[0] & mask) == data);
-
-	return 7;
-}
-
-static struct init_tbl_entry itbl_entry[] = {
-	/* command name                       , id  , length  , offset  , mult    , command handler                 */
-	/* INIT_PROG (0x31, 15, 10, 4) removed due to no example of use */
-	{ "INIT_IO_RESTRICT_PROG"             , 0x32, init_io_restrict_prog           },
-	{ "INIT_REPEAT"                       , 0x33, init_repeat                     },
-	{ "INIT_IO_RESTRICT_PLL"              , 0x34, init_io_restrict_pll            },
-	{ "INIT_END_REPEAT"                   , 0x36, init_end_repeat                 },
-	{ "INIT_COPY"                         , 0x37, init_copy                       },
-	{ "INIT_NOT"                          , 0x38, init_not                        },
-	{ "INIT_IO_FLAG_CONDITION"            , 0x39, init_io_flag_condition          },
-	{ "INIT_DP_CONDITION"                 , 0x3A, init_dp_condition               },
-	{ "INIT_OP_3B"                        , 0x3B, init_op_3b                      },
-	{ "INIT_OP_3C"                        , 0x3C, init_op_3c                      },
-	{ "INIT_INDEX_ADDRESS_LATCHED"        , 0x49, init_idx_addr_latched           },
-	{ "INIT_IO_RESTRICT_PLL2"             , 0x4A, init_io_restrict_pll2           },
-	{ "INIT_PLL2"                         , 0x4B, init_pll2                       },
-	{ "INIT_I2C_BYTE"                     , 0x4C, init_i2c_byte                   },
-	{ "INIT_ZM_I2C_BYTE"                  , 0x4D, init_zm_i2c_byte                },
-	{ "INIT_ZM_I2C"                       , 0x4E, init_zm_i2c                     },
-	{ "INIT_TMDS"                         , 0x4F, init_tmds                       },
-	{ "INIT_ZM_TMDS_GROUP"                , 0x50, init_zm_tmds_group              },
-	{ "INIT_CR_INDEX_ADDRESS_LATCHED"     , 0x51, init_cr_idx_adr_latch           },
-	{ "INIT_CR"                           , 0x52, init_cr                         },
-	{ "INIT_ZM_CR"                        , 0x53, init_zm_cr                      },
-	{ "INIT_ZM_CR_GROUP"                  , 0x54, init_zm_cr_group                },
-	{ "INIT_CONDITION_TIME"               , 0x56, init_condition_time             },
-	{ "INIT_LTIME"                        , 0x57, init_ltime                      },
-	{ "INIT_ZM_REG_SEQUENCE"              , 0x58, init_zm_reg_sequence            },
-	/* INIT_INDIRECT_REG (0x5A, 7, 0, 0) removed due to no example of use */
-	{ "INIT_SUB_DIRECT"                   , 0x5B, init_sub_direct                 },
-	{ "INIT_JUMP"                         , 0x5C, init_jump                       },
-	{ "INIT_I2C_IF"                       , 0x5E, init_i2c_if                     },
-	{ "INIT_COPY_NV_REG"                  , 0x5F, init_copy_nv_reg                },
-	{ "INIT_ZM_INDEX_IO"                  , 0x62, init_zm_index_io                },
-	{ "INIT_COMPUTE_MEM"                  , 0x63, init_compute_mem                },
-	{ "INIT_RESET"                        , 0x65, init_reset                      },
-	{ "INIT_CONFIGURE_MEM"                , 0x66, init_configure_mem              },
-	{ "INIT_CONFIGURE_CLK"                , 0x67, init_configure_clk              },
-	{ "INIT_CONFIGURE_PREINIT"            , 0x68, init_configure_preinit          },
-	{ "INIT_IO"                           , 0x69, init_io                         },
-	{ "INIT_SUB"                          , 0x6B, init_sub                        },
-	{ "INIT_RAM_CONDITION"                , 0x6D, init_ram_condition              },
-	{ "INIT_NV_REG"                       , 0x6E, init_nv_reg                     },
-	{ "INIT_MACRO"                        , 0x6F, init_macro                      },
-	{ "INIT_DONE"                         , 0x71, init_done                       },
-	{ "INIT_RESUME"                       , 0x72, init_resume                     },
-	/* INIT_RAM_CONDITION2 (0x73, 9, 0, 0) removed due to no example of use */
-	{ "INIT_TIME"                         , 0x74, init_time                       },
-	{ "INIT_CONDITION"                    , 0x75, init_condition                  },
-	{ "INIT_IO_CONDITION"                 , 0x76, init_io_condition               },
-	{ "INIT_INDEX_IO"                     , 0x78, init_index_io                   },
-	{ "INIT_PLL"                          , 0x79, init_pll                        },
-	{ "INIT_ZM_REG"                       , 0x7A, init_zm_reg                     },
-	{ "INIT_RAM_RESTRICT_PLL"             , 0x87, init_ram_restrict_pll           },
-	{ "INIT_8C"                           , 0x8C, init_8c                         },
-	{ "INIT_8D"                           , 0x8D, init_8d                         },
-	{ "INIT_GPIO"                         , 0x8E, init_gpio                       },
-	{ "INIT_RAM_RESTRICT_ZM_REG_GROUP"    , 0x8F, init_ram_restrict_zm_reg_group  },
-	{ "INIT_COPY_ZM_REG"                  , 0x90, init_copy_zm_reg                },
-	{ "INIT_ZM_REG_GROUP_ADDRESS_LATCHED" , 0x91, init_zm_reg_group_addr_latched  },
-	{ "INIT_RESERVED"                     , 0x92, init_reserved                   },
-	{ "INIT_96"                           , 0x96, init_96                         },
-	{ "INIT_97"                           , 0x97, init_97                         },
-	{ "INIT_AUXCH"                        , 0x98, init_auxch                      },
-	{ "INIT_ZM_AUXCH"                     , 0x99, init_zm_auxch                   },
-	{ "INIT_I2C_LONG_IF"                  , 0x9A, init_i2c_long_if                },
-	{ NULL                                , 0   , NULL                            }
-};
-
-#define MAX_TABLE_OPS 1000
-
-static int
-parse_init_table(struct nvbios *bios, uint16_t offset, struct init_exec *iexec)
-{
-	/*
-	 * Parses all commands in an init table.
-	 *
-	 * We start out executing all commands found in the init table. Some
-	 * opcodes may change the status of iexec->execute to SKIP, which will
-	 * cause the following opcodes to perform no operation until the value
-	 * is changed back to EXECUTE.
-	 */
-
-	int count = 0, i, ret;
-	uint8_t id;
-
-	/* catch NULL script pointers */
-	if (offset == 0)
-		return 0;
-
-	/*
-	 * Loop until INIT_DONE causes us to break out of the loop
-	 * (or until offset > bios length just in case... )
-	 * (and no more than MAX_TABLE_OPS iterations, just in case... )
-	 */
-	while ((offset < bios->length) && (count++ < MAX_TABLE_OPS)) {
-		id = bios->data[offset];
-
-		/* Find matching id in itbl_entry */
-		for (i = 0; itbl_entry[i].name && (itbl_entry[i].id != id); i++)
-			;
-
-		if (!itbl_entry[i].name) {
-			NV_ERROR(bios->dev,
-				 "0x%04X: Init table command not found: "
-				 "0x%02X\n", offset, id);
-			return -ENOENT;
-		}
-
-		BIOSLOG(bios, "0x%04X: [ (0x%02X) - %s ]\n", offset,
-			itbl_entry[i].id, itbl_entry[i].name);
-
-		/* execute eventual command handler */
-		ret = (*itbl_entry[i].handler)(bios, offset, iexec);
-		if (ret < 0) {
-			NV_ERROR(bios->dev, "0x%04X: Failed parsing init "
-				 "table opcode: %s %d\n", offset,
-				 itbl_entry[i].name, ret);
-		}
-
-		if (ret <= 0)
-			break;
-
-		/*
-		 * Add the offset of the current command including all data
-		 * of that command. The offset will then be pointing on the
-		 * next op code.
-		 */
-		offset += ret;
-	}
-
-	if (offset >= bios->length)
-		NV_WARN(bios->dev,
-			"Offset 0x%04X greater than known bios image length.  "
-			"Corrupt image?\n", offset);
-	if (count >= MAX_TABLE_OPS)
-		NV_WARN(bios->dev,
-			"More than %d opcodes to a table is unlikely, "
-			"is the bios image corrupt?\n", MAX_TABLE_OPS);
-
-	return 0;
-}
-
-static void
-parse_init_tables(struct nvbios *bios)
-{
-	/* Loops and calls parse_init_table() for each present table. */
-
-	int i = 0;
-	uint16_t table;
-	struct init_exec iexec = {true, false};
-
-	if (bios->old_style_init) {
-		if (bios->init_script_tbls_ptr)
-			parse_init_table(bios, bios->init_script_tbls_ptr, &iexec);
-		if (bios->extra_init_script_tbl_ptr)
-			parse_init_table(bios, bios->extra_init_script_tbl_ptr, &iexec);
-
-		return;
-	}
-
-	while ((table = ROM16(bios->data[bios->init_script_tbls_ptr + i]))) {
-		NV_INFO(bios->dev,
-			"Parsing VBIOS init table %d at offset 0x%04X\n",
-			i / 2, table);
-		BIOSLOG(bios, "0x%04X: ------ Executing following commands ------\n", table);
-
-		parse_init_table(bios, table, &iexec);
-		i += 2;
-	}
-}
-
 static uint16_t clkcmptable(struct nvbios *bios, uint16_t clktable, int pxclk)
 {
 	int compare_record_len, i = 0;
@@ -3764,28 +95,24 @@ static uint16_t clkcmptable(struct nvbios *bios, uint16_t clktable, int pxclk)
 
 static void
 run_digital_op_script(struct drm_device *dev, uint16_t scriptptr,
-		      struct dcb_entry *dcbent, int head, bool dl)
+		      struct dcb_output *dcbent, int head, bool dl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct init_exec iexec = {true, false};
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	NV_TRACE(dev, "0x%04X: Parsing digital output script table\n",
+	NV_INFO(drm, "0x%04X: Parsing digital output script table\n",
 		 scriptptr);
-	bios_idxprt_wr(bios, NV_CIO_CRX__COLOR, NV_CIO_CRE_44,
-		       head ? NV_CIO_CRE_44_HEADB : NV_CIO_CRE_44_HEADA);
-	/* note: if dcb entries have been merged, index may be misleading */
-	NVWriteVgaCrtc5758(dev, head, 0, dcbent->index);
-	parse_init_table(bios, scriptptr, &iexec);
+	NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_44, head ? NV_CIO_CRE_44_HEADB :
+					         NV_CIO_CRE_44_HEADA);
+	nouveau_bios_run_init_table(dev, scriptptr, dcbent, head);
 
 	nv04_dfp_bind_head(dev, dcbent, head, dl);
 }
 
-static int call_lvds_manufacturer_script(struct drm_device *dev, struct dcb_entry *dcbent, int head, enum LVDS_script script)
+static int call_lvds_manufacturer_script(struct drm_device *dev, struct dcb_output *dcbent, int head, enum LVDS_script script)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	uint8_t sub = bios->data[bios->fp.xlated_entry + script] + (bios->fp.link_c_increment && dcbent->or & OUTPUT_C ? 1 : 0);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
+	uint8_t sub = bios->data[bios->fp.xlated_entry + script] + (bios->fp.link_c_increment && dcbent->or & DCB_OUTPUT_C ? 1 : 0);
 	uint16_t scriptofs = ROM16(bios->data[bios->init_script_tbls_ptr + sub * 2]);
 
 	if (!bios->fp.xlated_entry || !sub || !scriptofs)
@@ -3808,7 +135,7 @@ static int call_lvds_manufacturer_script(struct drm_device *dev, struct dcb_entr
 	return 0;
 }
 
-static int run_lvds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, enum LVDS_script script, int pxclk)
+static int run_lvds_table(struct drm_device *dev, struct dcb_output *dcbent, int head, enum LVDS_script script, int pxclk)
 {
 	/*
 	 * The BIT LVDS table's header has the information to setup the
@@ -3820,8 +147,8 @@ static int run_lvds_table(struct drm_device *dev, struct dcb_entry *dcbent, int
 	 * conf byte. These tables are similar to the TMDS tables, consisting
 	 * of a list of pxclks and script pointers.
 	 */
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	unsigned int outputset = (dcbent->or == 4) ? 1 : 0;
 	uint16_t scriptptr = 0, clktable;
 
@@ -3866,14 +193,14 @@ static int run_lvds_table(struct drm_device *dev, struct dcb_entry *dcbent, int
 
 		clktable = ROM16(bios->data[clktable]);
 		if (!clktable) {
-			NV_ERROR(dev, "Pixel clock comparison table not found\n");
+			NV_ERROR(drm, "Pixel clock comparison table not found\n");
 			return -ENOENT;
 		}
 		scriptptr = clkcmptable(bios, clktable, pxclk);
 	}
 
 	if (!scriptptr) {
-		NV_ERROR(dev, "LVDS output init script not found\n");
+		NV_ERROR(drm, "LVDS output init script not found\n");
 		return -ENOENT;
 	}
 	run_digital_op_script(dev, scriptptr, dcbent, head, bios->fp.dual_link);
@@ -3881,7 +208,7 @@ static int run_lvds_table(struct drm_device *dev, struct dcb_entry *dcbent, int
 	return 0;
 }
 
-int call_lvds_script(struct drm_device *dev, struct dcb_entry *dcbent, int head, enum LVDS_script script, int pxclk)
+int call_lvds_script(struct drm_device *dev, struct dcb_output *dcbent, int head, enum LVDS_script script, int pxclk)
 {
 	/*
 	 * LVDS operations are multiplexed in an effort to present a single API
@@ -3889,8 +216,9 @@ int call_lvds_script(struct drm_device *dev, struct dcb_entry *dcbent, int head,
 	 * This acts as the demux
 	 */
 
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nvbios *bios = &drm->vbios;
 	uint8_t lvds_ver = bios->data[bios->fp.lvdsmanufacturerpointer];
 	uint32_t sel_clk_binding, sel_clk;
 	int ret;
@@ -3909,10 +237,10 @@ int call_lvds_script(struct drm_device *dev, struct dcb_entry *dcbent, int head,
 	if (script == LVDS_RESET && bios->fp.power_off_for_reset)
 		call_lvds_script(dev, dcbent, head, LVDS_PANEL_OFF, pxclk);
 
-	NV_TRACE(dev, "Calling LVDS script %d:\n", script);
+	NV_INFO(drm, "Calling LVDS script %d:\n", script);
 
 	/* don't let script change pll->head binding */
-	sel_clk_binding = bios_rd32(bios, NV_PRAMDAC_SEL_CLK) & 0x50000;
+	sel_clk_binding = nv_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
 
 	if (lvds_ver < 0x30)
 		ret = call_lvds_manufacturer_script(dev, dcbent, head, script);
@@ -3924,7 +252,7 @@ int call_lvds_script(struct drm_device *dev, struct dcb_entry *dcbent, int head,
 	sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK) & ~0x50000;
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, sel_clk | sel_clk_binding);
 	/* some scripts set a value in NV_PBUS_POWERCTRL_2 and break video overlay */
-	nvWriteMC(dev, NV_PBUS_POWERCTRL_2, 0);
+	nv_wr32(device, NV_PBUS_POWERCTRL_2, 0);
 
 	return ret;
 }
@@ -3942,12 +270,13 @@ static int parse_lvds_manufacturer_table_header(struct drm_device *dev, struct n
 	 * the maximum number of records that can be held in the table.
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t lvds_ver, headerlen, recordlen;
 
 	memset(lth, 0, sizeof(struct lvdstableheader));
 
 	if (bios->fp.lvdsmanufacturerpointer == 0x0) {
-		NV_ERROR(dev, "Pointer to LVDS manufacturer table invalid\n");
+		NV_ERROR(drm, "Pointer to LVDS manufacturer table invalid\n");
 		return -EINVAL;
 	}
 
@@ -3961,7 +290,7 @@ static int parse_lvds_manufacturer_table_header(struct drm_device *dev, struct n
 	case 0x30:	/* NV4x */
 		headerlen = bios->data[bios->fp.lvdsmanufacturerpointer + 1];
 		if (headerlen < 0x1f) {
-			NV_ERROR(dev, "LVDS table header not understood\n");
+			NV_ERROR(drm, "LVDS table header not understood\n");
 			return -EINVAL;
 		}
 		recordlen = bios->data[bios->fp.lvdsmanufacturerpointer + 2];
@@ -3969,13 +298,13 @@ static int parse_lvds_manufacturer_table_header(struct drm_device *dev, struct n
 	case 0x40:	/* G80/G90 */
 		headerlen = bios->data[bios->fp.lvdsmanufacturerpointer + 1];
 		if (headerlen < 0x7) {
-			NV_ERROR(dev, "LVDS table header not understood\n");
+			NV_ERROR(drm, "LVDS table header not understood\n");
 			return -EINVAL;
 		}
 		recordlen = bios->data[bios->fp.lvdsmanufacturerpointer + 2];
 		break;
 	default:
-		NV_ERROR(dev,
+		NV_ERROR(drm,
 			 "LVDS table revision %d.%d not currently supported\n",
 			 lvds_ver >> 4, lvds_ver & 0xf);
 		return -ENOSYS;
@@ -3991,7 +320,7 @@ static int parse_lvds_manufacturer_table_header(struct drm_device *dev, struct n
 static int
 get_fp_strap(struct drm_device *dev, struct nvbios *bios)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 
 	/*
 	 * The fp strap is normally dictated by the "User Strap" in
@@ -4005,14 +334,15 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios)
 	if (bios->major_version < 5 && bios->data[0x48] & 0x4)
 		return NVReadVgaCrtc5758(dev, 0, 0xf) & 0xf;
 
-	if (dev_priv->card_type >= NV_50)
-		return (bios_rd32(bios, NV_PEXTDEV_BOOT_0) >> 24) & 0xf;
+	if (device->card_type >= NV_50)
+		return (nv_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf;
 	else
-		return (bios_rd32(bios, NV_PEXTDEV_BOOT_0) >> 16) & 0xf;
+		return (nv_rd32(device, NV_PEXTDEV_BOOT_0) >> 16) & 0xf;
 }
 
 static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t *fptable;
 	uint8_t fptable_ver, headerlen = 0, recordlen, fpentries = 0xf, fpindex;
 	int ret, ofs, fpstrapping;
@@ -4022,7 +352,7 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 		/* Apple cards don't have the fp table; the laptops use DDC */
 		/* The table is also missing on some x86 IGPs */
 #ifndef __powerpc__
-		NV_ERROR(dev, "Pointer to flat panel table invalid\n");
+		NV_ERROR(drm, "Pointer to flat panel table invalid\n");
 #endif
 		bios->digital_min_front_porch = 0x4b;
 		return 0;
@@ -4061,7 +391,7 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 		ofs = -7;
 		break;
 	default:
-		NV_ERROR(dev,
+		NV_ERROR(drm,
 			 "FP table revision %d.%d not currently supported\n",
 			 fptable_ver >> 4, fptable_ver & 0xf);
 		return -ENOSYS;
@@ -4080,7 +410,7 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 		bios->fp.xlatwidth = lth.recordlen;
 	}
 	if (bios->fp.fpxlatetableptr == 0x0) {
-		NV_ERROR(dev, "Pointer to flat panel xlat table invalid\n");
+		NV_ERROR(drm, "Pointer to flat panel xlat table invalid\n");
 		return -EINVAL;
 	}
 
@@ -4090,7 +420,7 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 					fpstrapping * bios->fp.xlatwidth];
 
 	if (fpindex > fpentries) {
-		NV_ERROR(dev, "Bad flat panel table index\n");
+		NV_ERROR(drm, "Bad flat panel table index\n");
 		return -ENOENT;
 	}
 
@@ -4109,7 +439,7 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 	bios->fp.mode_ptr = bios->fp.fptablepointer + headerlen +
 			    recordlen * fpindex + ofs;
 
-	NV_TRACE(dev, "BIOS FP mode: %dx%d (%dkHz pixel clock)\n",
+	NV_INFO(drm, "BIOS FP mode: %dx%d (%dkHz pixel clock)\n",
 		 ROM16(bios->data[bios->fp.mode_ptr + 11]) + 1,
 		 ROM16(bios->data[bios->fp.mode_ptr + 25]) + 1,
 		 ROM16(bios->data[bios->fp.mode_ptr + 7]) * 10);
@@ -4119,8 +449,8 @@ static int parse_fp_mode_table(struct drm_device *dev, struct nvbios *bios)
 
 bool nouveau_bios_fp_mode(struct drm_device *dev, struct drm_display_mode *mode)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	uint8_t *mode_entry = &bios->data[bios->fp.mode_ptr];
 
 	if (!mode)	/* just checking whether we can produce a mode */
@@ -4190,8 +520,8 @@ int nouveau_bios_parse_lvds_table(struct drm_device *dev, int pxclk, bool *dl, b
 	 * requiring tests against the native-mode pixel clock, cannot be done
 	 * until later, when this function should be called with non-zero pxclk
 	 */
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	int fpstrapping = get_fp_strap(dev, bios), lvdsmanufacturerindex = 0;
 	struct lvdstableheader lth;
 	uint16_t lvdsofs;
@@ -4252,7 +582,7 @@ int nouveau_bios_parse_lvds_table(struct drm_device *dev, int pxclk, bool *dl, b
 		lvdsmanufacturerindex = fpstrapping;
 		break;
 	default:
-		NV_ERROR(dev, "LVDS table revision not currently supported\n");
+		NV_ERROR(drm, "LVDS table revision not currently supported\n");
 		return -ENOSYS;
 	}
 
@@ -4300,7 +630,7 @@ int nouveau_bios_parse_lvds_table(struct drm_device *dev, int pxclk, bool *dl, b
  * This function returns true if a particular DCB entry matches.
  */
 bool
-bios_encoder_match(struct dcb_entry *dcb, u32 hash)
+bios_encoder_match(struct dcb_output *dcb, u32 hash)
 {
 	if ((hash & 0x000000f0) != (dcb->location << 4))
 		return false;
@@ -4310,9 +640,9 @@ bios_encoder_match(struct dcb_entry *dcb, u32 hash)
 		return false;
 
 	switch (dcb->type) {
-	case OUTPUT_TMDS:
-	case OUTPUT_LVDS:
-	case OUTPUT_DP:
+	case DCB_OUTPUT_TMDS:
+	case DCB_OUTPUT_LVDS:
+	case DCB_OUTPUT_DP:
 		if (hash & 0x00c00000) {
 			if (!(hash & (dcb->sorconf.link << 22)))
 				return false;
@@ -4324,7 +654,7 @@ bios_encoder_match(struct dcb_entry *dcb, u32 hash)
 
 int
 nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
-			       struct dcb_entry *dcbent, int crtc)
+			       struct dcb_output *dcbent, int crtc)
 {
 	/*
 	 * The display script table is located by the BIT 'U' table.
@@ -4349,15 +679,15 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 	 * offset + 5   (16 bits): pointer to first output script table
 	 */
 
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	uint8_t *table = &bios->data[bios->display.script_table_ptr];
 	uint8_t *otable = NULL;
 	uint16_t script;
 	int i;
 
 	if (!bios->display.script_table_ptr) {
-		NV_ERROR(dev, "No pointer to output script table\n");
+		NV_ERROR(drm, "No pointer to output script table\n");
 		return 1;
 	}
 
@@ -4369,7 +699,7 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 		return 1;
 
 	if (table[0] != 0x20 && table[0] != 0x21) {
-		NV_ERROR(dev, "Output script table version 0x%02x unknown\n",
+		NV_ERROR(drm, "Output script table version 0x%02x unknown\n",
 			 table[0]);
 		return 1;
 	}
@@ -4404,7 +734,7 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 	 * script tables is a pointer to the script to execute.
 	 */
 
-	NV_DEBUG_KMS(dev, "Searching for output entry for %d %d %d\n",
+	NV_DEBUG(drm, "Searching for output entry for %d %d %d\n",
 			dcbent->type, dcbent->location, dcbent->or);
 	for (i = 0; i < table[3]; i++) {
 		otable = ROMPTR(dev, table[table[1] + (i * table[2])]);
@@ -4413,7 +743,7 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 	}
 
 	if (!otable) {
-		NV_DEBUG_KMS(dev, "failed to match any output table\n");
+		NV_DEBUG(drm, "failed to match any output table\n");
 		return 1;
 	}
 
@@ -4425,7 +755,7 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 		}
 
 		if (i == otable[5]) {
-			NV_ERROR(dev, "Table 0x%04x not found for %d/%d, "
+			NV_ERROR(drm, "Table 0x%04x not found for %d/%d, "
 				      "using first\n",
 				 type, dcbent->type, dcbent->or);
 			i = 0;
@@ -4435,21 +765,21 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 	if (pclk == 0) {
 		script = ROM16(otable[6]);
 		if (!script) {
-			NV_DEBUG_KMS(dev, "output script 0 not found\n");
+			NV_DEBUG(drm, "output script 0 not found\n");
 			return 1;
 		}
 
-		NV_DEBUG_KMS(dev, "0x%04X: parsing output script 0\n", script);
+		NV_DEBUG(drm, "0x%04X: parsing output script 0\n", script);
 		nouveau_bios_run_init_table(dev, script, dcbent, crtc);
 	} else
 	if (pclk == -1) {
 		script = ROM16(otable[8]);
 		if (!script) {
-			NV_DEBUG_KMS(dev, "output script 1 not found\n");
+			NV_DEBUG(drm, "output script 1 not found\n");
 			return 1;
 		}
 
-		NV_DEBUG_KMS(dev, "0x%04X: parsing output script 1\n", script);
+		NV_DEBUG(drm, "0x%04X: parsing output script 1\n", script);
 		nouveau_bios_run_init_table(dev, script, dcbent, crtc);
 	} else
 	if (pclk == -2) {
@@ -4458,11 +788,11 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 		else
 			script = 0;
 		if (!script) {
-			NV_DEBUG_KMS(dev, "output script 2 not found\n");
+			NV_DEBUG(drm, "output script 2 not found\n");
 			return 1;
 		}
 
-		NV_DEBUG_KMS(dev, "0x%04X: parsing output script 2\n", script);
+		NV_DEBUG(drm, "0x%04X: parsing output script 2\n", script);
 		nouveau_bios_run_init_table(dev, script, dcbent, crtc);
 	} else
 	if (pclk > 0) {
@@ -4470,11 +800,11 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 		if (script)
 			script = clkcmptable(bios, script, pclk);
 		if (!script) {
-			NV_DEBUG_KMS(dev, "clock script 0 not found\n");
+			NV_DEBUG(drm, "clock script 0 not found\n");
 			return 1;
 		}
 
-		NV_DEBUG_KMS(dev, "0x%04X: parsing clock script 0\n", script);
+		NV_DEBUG(drm, "0x%04X: parsing clock script 0\n", script);
 		nouveau_bios_run_init_table(dev, script, dcbent, crtc);
 	} else
 	if (pclk < 0) {
@@ -4482,11 +812,11 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 		if (script)
 			script = clkcmptable(bios, script, -pclk);
 		if (!script) {
-			NV_DEBUG_KMS(dev, "clock script 1 not found\n");
+			NV_DEBUG(drm, "clock script 1 not found\n");
 			return 1;
 		}
 
-		NV_DEBUG_KMS(dev, "0x%04X: parsing clock script 1\n", script);
+		NV_DEBUG(drm, "0x%04X: parsing clock script 1\n", script);
 		nouveau_bios_run_init_table(dev, script, dcbent, crtc);
 	}
 
@@ -4494,7 +824,7 @@ nouveau_bios_run_display_table(struct drm_device *dev, u16 type, int pclk,
 }
 
 
-int run_tmds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, int pxclk)
+int run_tmds_table(struct drm_device *dev, struct dcb_output *dcbent, int head, int pxclk)
 {
 	/*
 	 * the pxclk parameter is in kHz
@@ -4505,8 +835,9 @@ int run_tmds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, i
 	 * ffs(or) == 3, use the second.
 	 */
 
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nvbios *bios = &drm->vbios;
 	int cv = bios->chip_version;
 	uint16_t clktable = 0, scriptptr;
 	uint32_t sel_clk_binding, sel_clk;
@@ -4527,19 +858,19 @@ int run_tmds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, i
 	}
 
 	if (!clktable) {
-		NV_ERROR(dev, "Pixel clock comparison table not found\n");
+		NV_ERROR(drm, "Pixel clock comparison table not found\n");
 		return -EINVAL;
 	}
 
 	scriptptr = clkcmptable(bios, clktable, pxclk);
 
 	if (!scriptptr) {
-		NV_ERROR(dev, "TMDS output init script not found\n");
+		NV_ERROR(drm, "TMDS output init script not found\n");
 		return -ENOENT;
 	}
 
 	/* don't let script change pll->head binding */
-	sel_clk_binding = bios_rd32(bios, NV_PRAMDAC_SEL_CLK) & 0x50000;
+	sel_clk_binding = nv_rd32(device, NV_PRAMDAC_SEL_CLK) & 0x50000;
 	run_digital_op_script(dev, scriptptr, dcbent, head, pxclk >= 165000);
 	sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK) & ~0x50000;
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, sel_clk | sel_clk_binding);
@@ -4547,447 +878,6 @@ int run_tmds_table(struct drm_device *dev, struct dcb_entry *dcbent, int head, i
 	return 0;
 }
 
-struct pll_mapping {
-	u8  type;
-	u32 reg;
-};
-
-static struct pll_mapping nv04_pll_mapping[] = {
-	{ PLL_CORE  , NV_PRAMDAC_NVPLL_COEFF },
-	{ PLL_MEMORY, NV_PRAMDAC_MPLL_COEFF },
-	{ PLL_VPLL0 , NV_PRAMDAC_VPLL_COEFF },
-	{ PLL_VPLL1 , NV_RAMDAC_VPLL2 },
-	{}
-};
-
-static struct pll_mapping nv40_pll_mapping[] = {
-	{ PLL_CORE  , 0x004000 },
-	{ PLL_MEMORY, 0x004020 },
-	{ PLL_VPLL0 , NV_PRAMDAC_VPLL_COEFF },
-	{ PLL_VPLL1 , NV_RAMDAC_VPLL2 },
-	{}
-};
-
-static struct pll_mapping nv50_pll_mapping[] = {
-	{ PLL_CORE  , 0x004028 },
-	{ PLL_SHADER, 0x004020 },
-	{ PLL_UNK03 , 0x004000 },
-	{ PLL_MEMORY, 0x004008 },
-	{ PLL_UNK40 , 0x00e810 },
-	{ PLL_UNK41 , 0x00e818 },
-	{ PLL_UNK42 , 0x00e824 },
-	{ PLL_VPLL0 , 0x614100 },
-	{ PLL_VPLL1 , 0x614900 },
-	{}
-};
-
-static struct pll_mapping nv84_pll_mapping[] = {
-	{ PLL_CORE  , 0x004028 },
-	{ PLL_SHADER, 0x004020 },
-	{ PLL_MEMORY, 0x004008 },
-	{ PLL_VDEC  , 0x004030 },
-	{ PLL_UNK41 , 0x00e818 },
-	{ PLL_VPLL0 , 0x614100 },
-	{ PLL_VPLL1 , 0x614900 },
-	{}
-};
-
-u32
-get_pll_register(struct drm_device *dev, enum pll_types type)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct pll_mapping *map;
-	int i;
-
-	if (dev_priv->card_type < NV_40)
-		map = nv04_pll_mapping;
-	else
-	if (dev_priv->card_type < NV_50)
-		map = nv40_pll_mapping;
-	else {
-		u8 *plim = &bios->data[bios->pll_limit_tbl_ptr];
-
-		if (plim[0] >= 0x30) {
-			u8 *entry = plim + plim[1];
-			for (i = 0; i < plim[3]; i++, entry += plim[2]) {
-				if (entry[0] == type)
-					return ROM32(entry[3]);
-			}
-
-			return 0;
-		}
-
-		if (dev_priv->chipset == 0x50)
-			map = nv50_pll_mapping;
-		else
-			map = nv84_pll_mapping;
-	}
-
-	while (map->reg) {
-		if (map->type == type)
-			return map->reg;
-		map++;
-	}
-
-	return 0;
-}
-
-int get_pll_limits(struct drm_device *dev, uint32_t limit_match, struct pll_lims *pll_lim)
-{
-	/*
-	 * PLL limits table
-	 *
-	 * Version 0x10: NV30, NV31
-	 * One byte header (version), one record of 24 bytes
-	 * Version 0x11: NV36 - Not implemented
-	 * Seems to have same record style as 0x10, but 3 records rather than 1
-	 * Version 0x20: Found on Geforce 6 cards
-	 * Trivial 4 byte BIT header. 31 (0x1f) byte record length
-	 * Version 0x21: Found on Geforce 7, 8 and some Geforce 6 cards
-	 * 5 byte header, fifth byte of unknown purpose. 35 (0x23) byte record
-	 * length in general, some (integrated) have an extra configuration byte
-	 * Version 0x30: Found on Geforce 8, separates the register mapping
-	 * from the limits tables.
-	 */
-
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	int cv = bios->chip_version, pllindex = 0;
-	uint8_t pll_lim_ver = 0, headerlen = 0, recordlen = 0, entries = 0;
-	uint32_t crystal_strap_mask, crystal_straps;
-
-	if (!bios->pll_limit_tbl_ptr) {
-		if (cv == 0x30 || cv == 0x31 || cv == 0x35 || cv == 0x36 ||
-		    cv >= 0x40) {
-			NV_ERROR(dev, "Pointer to PLL limits table invalid\n");
-			return -EINVAL;
-		}
-	} else
-		pll_lim_ver = bios->data[bios->pll_limit_tbl_ptr];
-
-	crystal_strap_mask = 1 << 6;
-	/* open coded dev->twoHeads test */
-	if (cv > 0x10 && cv != 0x15 && cv != 0x1a && cv != 0x20)
-		crystal_strap_mask |= 1 << 22;
-	crystal_straps = nvReadEXTDEV(dev, NV_PEXTDEV_BOOT_0) &
-							crystal_strap_mask;
-
-	switch (pll_lim_ver) {
-	/*
-	 * We use version 0 to indicate a pre limit table bios (single stage
-	 * pll) and load the hard coded limits instead.
-	 */
-	case 0:
-		break;
-	case 0x10:
-	case 0x11:
-		/*
-		 * Strictly v0x11 has 3 entries, but the last two don't seem
-		 * to get used.
-		 */
-		headerlen = 1;
-		recordlen = 0x18;
-		entries = 1;
-		pllindex = 0;
-		break;
-	case 0x20:
-	case 0x21:
-	case 0x30:
-	case 0x40:
-		headerlen = bios->data[bios->pll_limit_tbl_ptr + 1];
-		recordlen = bios->data[bios->pll_limit_tbl_ptr + 2];
-		entries = bios->data[bios->pll_limit_tbl_ptr + 3];
-		break;
-	default:
-		NV_ERROR(dev, "PLL limits table revision 0x%X not currently "
-				"supported\n", pll_lim_ver);
-		return -ENOSYS;
-	}
-
-	/* initialize all members to zero */
-	memset(pll_lim, 0, sizeof(struct pll_lims));
-
-	/* if we were passed a type rather than a register, figure
-	 * out the register and store it
-	 */
-	if (limit_match > PLL_MAX)
-		pll_lim->reg = limit_match;
-	else {
-		pll_lim->reg = get_pll_register(dev, limit_match);
-		if (!pll_lim->reg)
-			return -ENOENT;
-	}
-
-	if (pll_lim_ver == 0x10 || pll_lim_ver == 0x11) {
-		uint8_t *pll_rec = &bios->data[bios->pll_limit_tbl_ptr + headerlen + recordlen * pllindex];
-
-		pll_lim->vco1.minfreq = ROM32(pll_rec[0]);
-		pll_lim->vco1.maxfreq = ROM32(pll_rec[4]);
-		pll_lim->vco2.minfreq = ROM32(pll_rec[8]);
-		pll_lim->vco2.maxfreq = ROM32(pll_rec[12]);
-		pll_lim->vco1.min_inputfreq = ROM32(pll_rec[16]);
-		pll_lim->vco2.min_inputfreq = ROM32(pll_rec[20]);
-		pll_lim->vco1.max_inputfreq = pll_lim->vco2.max_inputfreq = INT_MAX;
-
-		/* these values taken from nv30/31/36 */
-		pll_lim->vco1.min_n = 0x1;
-		if (cv == 0x36)
-			pll_lim->vco1.min_n = 0x5;
-		pll_lim->vco1.max_n = 0xff;
-		pll_lim->vco1.min_m = 0x1;
-		pll_lim->vco1.max_m = 0xd;
-		pll_lim->vco2.min_n = 0x4;
-		/*
-		 * On nv30, 31, 36 (i.e. all cards with two stage PLLs with this
-		 * table version (apart from nv35)), N2 is compared to
-		 * maxN2 (0x46) and 10 * maxM2 (0x4), so set maxN2 to 0x28 and
-		 * save a comparison
-		 */
-		pll_lim->vco2.max_n = 0x28;
-		if (cv == 0x30 || cv == 0x35)
-			/* only 5 bits available for N2 on nv30/35 */
-			pll_lim->vco2.max_n = 0x1f;
-		pll_lim->vco2.min_m = 0x1;
-		pll_lim->vco2.max_m = 0x4;
-		pll_lim->max_log2p = 0x7;
-		pll_lim->max_usable_log2p = 0x6;
-	} else if (pll_lim_ver == 0x20 || pll_lim_ver == 0x21) {
-		uint16_t plloffs = bios->pll_limit_tbl_ptr + headerlen;
-		uint8_t *pll_rec;
-		int i;
-
-		/*
-		 * First entry is default match, if nothing better. warn if
-		 * reg field nonzero
-		 */
-		if (ROM32(bios->data[plloffs]))
-			NV_WARN(dev, "Default PLL limit entry has non-zero "
-				       "register field\n");
-
-		for (i = 1; i < entries; i++)
-			if (ROM32(bios->data[plloffs + recordlen * i]) == pll_lim->reg) {
-				pllindex = i;
-				break;
-			}
-
-		if ((dev_priv->card_type >= NV_50) && (pllindex == 0)) {
-			NV_ERROR(dev, "Register 0x%08x not found in PLL "
-				 "limits table", pll_lim->reg);
-			return -ENOENT;
-		}
-
-		pll_rec = &bios->data[plloffs + recordlen * pllindex];
-
-		BIOSLOG(bios, "Loading PLL limits for reg 0x%08x\n",
-			pllindex ? pll_lim->reg : 0);
-
-		/*
-		 * Frequencies are stored in tables in MHz, kHz are more
-		 * useful, so we convert.
-		 */
-
-		/* What output frequencies can each VCO generate? */
-		pll_lim->vco1.minfreq = ROM16(pll_rec[4]) * 1000;
-		pll_lim->vco1.maxfreq = ROM16(pll_rec[6]) * 1000;
-		pll_lim->vco2.minfreq = ROM16(pll_rec[8]) * 1000;
-		pll_lim->vco2.maxfreq = ROM16(pll_rec[10]) * 1000;
-
-		/* What input frequencies they accept (past the m-divider)? */
-		pll_lim->vco1.min_inputfreq = ROM16(pll_rec[12]) * 1000;
-		pll_lim->vco2.min_inputfreq = ROM16(pll_rec[14]) * 1000;
-		pll_lim->vco1.max_inputfreq = ROM16(pll_rec[16]) * 1000;
-		pll_lim->vco2.max_inputfreq = ROM16(pll_rec[18]) * 1000;
-
-		/* What values are accepted as multiplier and divider? */
-		pll_lim->vco1.min_n = pll_rec[20];
-		pll_lim->vco1.max_n = pll_rec[21];
-		pll_lim->vco1.min_m = pll_rec[22];
-		pll_lim->vco1.max_m = pll_rec[23];
-		pll_lim->vco2.min_n = pll_rec[24];
-		pll_lim->vco2.max_n = pll_rec[25];
-		pll_lim->vco2.min_m = pll_rec[26];
-		pll_lim->vco2.max_m = pll_rec[27];
-
-		pll_lim->max_usable_log2p = pll_lim->max_log2p = pll_rec[29];
-		if (pll_lim->max_log2p > 0x7)
-			/* pll decoding in nv_hw.c assumes never > 7 */
-			NV_WARN(dev, "Max log2 P value greater than 7 (%d)\n",
-				pll_lim->max_log2p);
-		if (cv < 0x60)
-			pll_lim->max_usable_log2p = 0x6;
-		pll_lim->log2p_bias = pll_rec[30];
-
-		if (recordlen > 0x22)
-			pll_lim->refclk = ROM32(pll_rec[31]);
-
-		if (recordlen > 0x23 && pll_rec[35])
-			NV_WARN(dev,
-				"Bits set in PLL configuration byte (%x)\n",
-				pll_rec[35]);
-
-		/* C51 special not seen elsewhere */
-		if (cv == 0x51 && !pll_lim->refclk) {
-			uint32_t sel_clk = bios_rd32(bios, NV_PRAMDAC_SEL_CLK);
-
-			if ((pll_lim->reg == NV_PRAMDAC_VPLL_COEFF && sel_clk & 0x20) ||
-			    (pll_lim->reg == NV_RAMDAC_VPLL2 && sel_clk & 0x80)) {
-				if (bios_idxprt_rd(bios, NV_CIO_CRX__COLOR, NV_CIO_CRE_CHIP_ID_INDEX) < 0xa3)
-					pll_lim->refclk = 200000;
-				else
-					pll_lim->refclk = 25000;
-			}
-		}
-	} else if (pll_lim_ver == 0x30) { /* ver 0x30 */
-		uint8_t *entry = &bios->data[bios->pll_limit_tbl_ptr + headerlen];
-		uint8_t *record = NULL;
-		int i;
-
-		BIOSLOG(bios, "Loading PLL limits for register 0x%08x\n",
-			pll_lim->reg);
-
-		for (i = 0; i < entries; i++, entry += recordlen) {
-			if (ROM32(entry[3]) == pll_lim->reg) {
-				record = &bios->data[ROM16(entry[1])];
-				break;
-			}
-		}
-
-		if (!record) {
-			NV_ERROR(dev, "Register 0x%08x not found in PLL "
-				 "limits table", pll_lim->reg);
-			return -ENOENT;
-		}
-
-		pll_lim->vco1.minfreq = ROM16(record[0]) * 1000;
-		pll_lim->vco1.maxfreq = ROM16(record[2]) * 1000;
-		pll_lim->vco2.minfreq = ROM16(record[4]) * 1000;
-		pll_lim->vco2.maxfreq = ROM16(record[6]) * 1000;
-		pll_lim->vco1.min_inputfreq = ROM16(record[8]) * 1000;
-		pll_lim->vco2.min_inputfreq = ROM16(record[10]) * 1000;
-		pll_lim->vco1.max_inputfreq = ROM16(record[12]) * 1000;
-		pll_lim->vco2.max_inputfreq = ROM16(record[14]) * 1000;
-		pll_lim->vco1.min_n = record[16];
-		pll_lim->vco1.max_n = record[17];
-		pll_lim->vco1.min_m = record[18];
-		pll_lim->vco1.max_m = record[19];
-		pll_lim->vco2.min_n = record[20];
-		pll_lim->vco2.max_n = record[21];
-		pll_lim->vco2.min_m = record[22];
-		pll_lim->vco2.max_m = record[23];
-		pll_lim->max_usable_log2p = pll_lim->max_log2p = record[25];
-		pll_lim->log2p_bias = record[27];
-		pll_lim->refclk = ROM32(record[28]);
-	} else if (pll_lim_ver) { /* ver 0x40 */
-		uint8_t *entry = &bios->data[bios->pll_limit_tbl_ptr + headerlen];
-		uint8_t *record = NULL;
-		int i;
-
-		BIOSLOG(bios, "Loading PLL limits for register 0x%08x\n",
-			pll_lim->reg);
-
-		for (i = 0; i < entries; i++, entry += recordlen) {
-			if (ROM32(entry[3]) == pll_lim->reg) {
-				record = &bios->data[ROM16(entry[1])];
-				break;
-			}
-		}
-
-		if (!record) {
-			NV_ERROR(dev, "Register 0x%08x not found in PLL "
-				 "limits table", pll_lim->reg);
-			return -ENOENT;
-		}
-
-		pll_lim->vco1.minfreq = ROM16(record[0]) * 1000;
-		pll_lim->vco1.maxfreq = ROM16(record[2]) * 1000;
-		pll_lim->vco1.min_inputfreq = ROM16(record[4]) * 1000;
-		pll_lim->vco1.max_inputfreq = ROM16(record[6]) * 1000;
-		pll_lim->vco1.min_m = record[8];
-		pll_lim->vco1.max_m = record[9];
-		pll_lim->vco1.min_n = record[10];
-		pll_lim->vco1.max_n = record[11];
-		pll_lim->min_p = record[12];
-		pll_lim->max_p = record[13];
-		pll_lim->refclk = ROM16(entry[9]) * 1000;
-	}
-
-	/*
-	 * By now any valid limit table ought to have set a max frequency for
-	 * vco1, so if it's zero it's either a pre limit table bios, or one
-	 * with an empty limit table (seen on nv18)
-	 */
-	if (!pll_lim->vco1.maxfreq) {
-		pll_lim->vco1.minfreq = bios->fminvco;
-		pll_lim->vco1.maxfreq = bios->fmaxvco;
-		pll_lim->vco1.min_inputfreq = 0;
-		pll_lim->vco1.max_inputfreq = INT_MAX;
-		pll_lim->vco1.min_n = 0x1;
-		pll_lim->vco1.max_n = 0xff;
-		pll_lim->vco1.min_m = 0x1;
-		if (crystal_straps == 0) {
-			/* nv05 does this, nv11 doesn't, nv10 unknown */
-			if (cv < 0x11)
-				pll_lim->vco1.min_m = 0x7;
-			pll_lim->vco1.max_m = 0xd;
-		} else {
-			if (cv < 0x11)
-				pll_lim->vco1.min_m = 0x8;
-			pll_lim->vco1.max_m = 0xe;
-		}
-		if (cv < 0x17 || cv == 0x1a || cv == 0x20)
-			pll_lim->max_log2p = 4;
-		else
-			pll_lim->max_log2p = 5;
-		pll_lim->max_usable_log2p = pll_lim->max_log2p;
-	}
-
-	if (!pll_lim->refclk)
-		switch (crystal_straps) {
-		case 0:
-			pll_lim->refclk = 13500;
-			break;
-		case (1 << 6):
-			pll_lim->refclk = 14318;
-			break;
-		case (1 << 22):
-			pll_lim->refclk = 27000;
-			break;
-		case (1 << 22 | 1 << 6):
-			pll_lim->refclk = 25000;
-			break;
-		}
-
-	NV_DEBUG(dev, "pll.vco1.minfreq: %d\n", pll_lim->vco1.minfreq);
-	NV_DEBUG(dev, "pll.vco1.maxfreq: %d\n", pll_lim->vco1.maxfreq);
-	NV_DEBUG(dev, "pll.vco1.min_inputfreq: %d\n", pll_lim->vco1.min_inputfreq);
-	NV_DEBUG(dev, "pll.vco1.max_inputfreq: %d\n", pll_lim->vco1.max_inputfreq);
-	NV_DEBUG(dev, "pll.vco1.min_n: %d\n", pll_lim->vco1.min_n);
-	NV_DEBUG(dev, "pll.vco1.max_n: %d\n", pll_lim->vco1.max_n);
-	NV_DEBUG(dev, "pll.vco1.min_m: %d\n", pll_lim->vco1.min_m);
-	NV_DEBUG(dev, "pll.vco1.max_m: %d\n", pll_lim->vco1.max_m);
-	if (pll_lim->vco2.maxfreq) {
-		NV_DEBUG(dev, "pll.vco2.minfreq: %d\n", pll_lim->vco2.minfreq);
-		NV_DEBUG(dev, "pll.vco2.maxfreq: %d\n", pll_lim->vco2.maxfreq);
-		NV_DEBUG(dev, "pll.vco2.min_inputfreq: %d\n", pll_lim->vco2.min_inputfreq);
-		NV_DEBUG(dev, "pll.vco2.max_inputfreq: %d\n", pll_lim->vco2.max_inputfreq);
-		NV_DEBUG(dev, "pll.vco2.min_n: %d\n", pll_lim->vco2.min_n);
-		NV_DEBUG(dev, "pll.vco2.max_n: %d\n", pll_lim->vco2.max_n);
-		NV_DEBUG(dev, "pll.vco2.min_m: %d\n", pll_lim->vco2.min_m);
-		NV_DEBUG(dev, "pll.vco2.max_m: %d\n", pll_lim->vco2.max_m);
-	}
-	if (!pll_lim->max_p) {
-		NV_DEBUG(dev, "pll.max_log2p: %d\n", pll_lim->max_log2p);
-		NV_DEBUG(dev, "pll.log2p_bias: %d\n", pll_lim->log2p_bias);
-	} else {
-		NV_DEBUG(dev, "pll.min_p: %d\n", pll_lim->min_p);
-		NV_DEBUG(dev, "pll.max_p: %d\n", pll_lim->max_p);
-	}
-	NV_DEBUG(dev, "pll.refclk: %d\n", pll_lim->refclk);
-
-	return 0;
-}
-
 static void parse_bios_version(struct drm_device *dev, struct nvbios *bios, uint16_t offset)
 {
 	/*
@@ -4996,10 +886,11 @@ static void parse_bios_version(struct drm_device *dev, struct nvbios *bios, uint
 	 * offset + 2  (8 bits): Chip version
 	 * offset + 3  (8 bits): Major version
 	 */
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	bios->major_version = bios->data[offset + 3];
 	bios->chip_version = bios->data[offset + 2];
-	NV_TRACE(dev, "Bios version %02x.%02x.%02x.%02x\n",
+	NV_INFO(drm, "Bios version %02x.%02x.%02x.%02x\n",
 		 bios->data[offset + 3], bios->data[offset + 2],
 		 bios->data[offset + 1], bios->data[offset]);
 }
@@ -5035,25 +926,26 @@ static int parse_bit_A_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	 * offset + 0 (16 bits): loadval table pointer
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint16_t load_table_ptr;
 	uint8_t version, headerlen, entrylen, num_entries;
 
 	if (bitentry->length != 3) {
-		NV_ERROR(dev, "Do not understand BIT A table\n");
+		NV_ERROR(drm, "Do not understand BIT A table\n");
 		return -EINVAL;
 	}
 
 	load_table_ptr = ROM16(bios->data[bitentry->offset]);
 
 	if (load_table_ptr == 0x0) {
-		NV_DEBUG(dev, "Pointer to BIT loadval table invalid\n");
+		NV_DEBUG(drm, "Pointer to BIT loadval table invalid\n");
 		return -EINVAL;
 	}
 
 	version = bios->data[load_table_ptr];
 
 	if (version != 0x10) {
-		NV_ERROR(dev, "BIT loadval table version %d.%d not supported\n",
+		NV_ERROR(drm, "BIT loadval table version %d.%d not supported\n",
 			 version >> 4, version & 0xF);
 		return -ENOSYS;
 	}
@@ -5063,7 +955,7 @@ static int parse_bit_A_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	num_entries = bios->data[load_table_ptr + 3];
 
 	if (headerlen != 4 || entrylen != 4 || num_entries != 2) {
-		NV_ERROR(dev, "Do not understand BIT loadval table\n");
+		NV_ERROR(drm, "Do not understand BIT loadval table\n");
 		return -EINVAL;
 	}
 
@@ -5080,9 +972,10 @@ static int parse_bit_C_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	 *
 	 * There's more in here, but that's unknown.
 	 */
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	if (bitentry->length < 10) {
-		NV_ERROR(dev, "Do not understand BIT C table\n");
+		NV_ERROR(drm, "Do not understand BIT C table\n");
 		return -EINVAL;
 	}
 
@@ -5101,9 +994,10 @@ static int parse_bit_display_tbl_entry(struct drm_device *dev, struct nvbios *bi
 	 * records beginning with a freq.
 	 * offset + 2  (16 bits): mode table pointer
 	 */
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	if (bitentry->length != 4) {
-		NV_ERROR(dev, "Do not understand BIT display table\n");
+		NV_ERROR(drm, "Do not understand BIT display table\n");
 		return -EINVAL;
 	}
 
@@ -5119,9 +1013,10 @@ static int parse_bit_init_tbl_entry(struct drm_device *dev, struct nvbios *bios,
 	 *
 	 * See parse_script_table_pointers for layout
 	 */
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	if (bitentry->length < 14) {
-		NV_ERROR(dev, "Do not understand init table\n");
+		NV_ERROR(drm, "Do not understand init table\n");
 		return -EINVAL;
 	}
 
@@ -5148,11 +1043,12 @@ static int parse_bit_i_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	 * There's other things in the table, purpose unknown
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint16_t daccmpoffset;
 	uint8_t dacver, dacheaderlen;
 
 	if (bitentry->length < 6) {
-		NV_ERROR(dev, "BIT i table too short for needed information\n");
+		NV_ERROR(drm, "BIT i table too short for needed information\n");
 		return -EINVAL;
 	}
 
@@ -5166,7 +1062,7 @@ static int parse_bit_i_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	bios->is_mobile = bios->feature_byte & FEATURE_MOBILE;
 
 	if (bitentry->length < 15) {
-		NV_WARN(dev, "BIT i table not long enough for DAC load "
+		NV_WARN(drm, "BIT i table not long enough for DAC load "
 			       "detection comparison table\n");
 		return -EINVAL;
 	}
@@ -5187,7 +1083,7 @@ static int parse_bit_i_tbl_entry(struct drm_device *dev, struct nvbios *bios, st
 	dacheaderlen = bios->data[daccmpoffset + 1];
 
 	if (dacver != 0x00 && dacver != 0x10) {
-		NV_WARN(dev, "DAC load detection comparison table version "
+		NV_WARN(drm, "DAC load detection comparison table version "
 			       "%d.%d not known\n", dacver >> 4, dacver & 0xf);
 		return -ENOSYS;
 	}
@@ -5207,8 +1103,10 @@ static int parse_bit_lvds_tbl_entry(struct drm_device *dev, struct nvbios *bios,
 	 * offset + 0  (16 bits): LVDS strap xlate table pointer
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	if (bitentry->length != 2) {
-		NV_ERROR(dev, "Do not understand BIT LVDS table\n");
+		NV_ERROR(drm, "Do not understand BIT LVDS table\n");
 		return -EINVAL;
 	}
 
@@ -5278,20 +1176,21 @@ static int parse_bit_tmds_tbl_entry(struct drm_device *dev, struct nvbios *bios,
 	 * "or" from the DCB.
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint16_t tmdstableptr, script1, script2;
 
 	if (bitentry->length != 2) {
-		NV_ERROR(dev, "Do not understand BIT TMDS table\n");
+		NV_ERROR(drm, "Do not understand BIT TMDS table\n");
 		return -EINVAL;
 	}
 
 	tmdstableptr = ROM16(bios->data[bitentry->offset]);
 	if (!tmdstableptr) {
-		NV_ERROR(dev, "Pointer to TMDS table invalid\n");
+		NV_ERROR(drm, "Pointer to TMDS table invalid\n");
 		return -EINVAL;
 	}
 
-	NV_INFO(dev, "TMDS table version %d.%d\n",
+	NV_INFO(drm, "TMDS table version %d.%d\n",
 		bios->data[tmdstableptr] >> 4, bios->data[tmdstableptr] & 0xf);
 
 	/* nv50+ has v2.0, but we don't parse it atm */
@@ -5305,7 +1204,7 @@ static int parse_bit_tmds_tbl_entry(struct drm_device *dev, struct nvbios *bios,
 	script1 = ROM16(bios->data[tmdstableptr + 7]);
 	script2 = ROM16(bios->data[tmdstableptr + 9]);
 	if (bios->data[script1] != 'q' || bios->data[script2] != 'q')
-		NV_WARN(dev, "TMDS table script pointers not stubbed\n");
+		NV_WARN(drm, "TMDS table script pointers not stubbed\n");
 
 	bios->tmds.output0_script_ptr = ROM16(bios->data[tmdstableptr + 11]);
 	bios->tmds.output1_script_ptr = ROM16(bios->data[tmdstableptr + 13]);
@@ -5325,10 +1224,11 @@ parse_bit_U_tbl_entry(struct drm_device *dev, struct nvbios *bios,
 	 * offset + 0  (16 bits): output script table pointer
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint16_t outputscripttableptr;
 
 	if (bitentry->length != 3) {
-		NV_ERROR(dev, "Do not understand BIT U table\n");
+		NV_ERROR(drm, "Do not understand BIT U table\n");
 		return -EINVAL;
 	}
 
@@ -5347,8 +1247,8 @@ struct bit_table {
 int
 bit_table(struct drm_device *dev, u8 id, struct bit_entry *bit)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	u8 entries, *entry;
 
 	if (bios->type != NVBIOS_BIT)
@@ -5377,12 +1277,13 @@ parse_bit_table(struct nvbios *bios, const uint16_t bitoffset,
 		struct bit_table *table)
 {
 	struct drm_device *dev = bios->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct bit_entry bitentry;
 
 	if (bit_table(dev, table->id, &bitentry) == 0)
 		return table->parse_fn(dev, bios, &bitentry);
 
-	NV_INFO(dev, "BIT table '%c' not found\n", table->id);
+	NV_INFO(drm, "BIT table '%c' not found\n", table->id);
 	return -ENOSYS;
 }
 
@@ -5462,6 +1363,7 @@ static int parse_bmp_structure(struct drm_device *dev, struct nvbios *bios, unsi
 	 * offset + 156: minimum pixel clock for LVDS dual link
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t *bmp = &bios->data[offset], bmp_version_major, bmp_version_minor;
 	uint16_t bmplength;
 	uint16_t legacy_scripts_offset, legacy_i2c_offset;
@@ -5475,7 +1377,7 @@ static int parse_bmp_structure(struct drm_device *dev, struct nvbios *bios, unsi
 	bmp_version_major = bmp[5];
 	bmp_version_minor = bmp[6];
 
-	NV_TRACE(dev, "BMP version %d.%d\n",
+	NV_INFO(drm, "BMP version %d.%d\n",
 		 bmp_version_major, bmp_version_minor);
 
 	/*
@@ -5491,7 +1393,7 @@ static int parse_bmp_structure(struct drm_device *dev, struct nvbios *bios, unsi
 	 * happened instead.
 	 */
 	if ((bmp_version_major < 5 && bmp_version_minor != 1) || bmp_version_major > 5) {
-		NV_ERROR(dev, "You have an unsupported BMP version. "
+		NV_ERROR(drm, "You have an unsupported BMP version. "
 				"Please send in your bios\n");
 		return -ENOSYS;
 	}
@@ -5540,7 +1442,7 @@ static int parse_bmp_structure(struct drm_device *dev, struct nvbios *bios, unsi
 
 	/* checksum */
 	if (nv_cksum(bmp, 8)) {
-		NV_ERROR(dev, "Bad BMP checksum\n");
+		NV_ERROR(drm, "Bad BMP checksum\n");
 		return -EINVAL;
 	}
 
@@ -5625,20 +1527,20 @@ static uint16_t findstr(uint8_t *data, int n, const uint8_t *str, int len)
 }
 
 void *
-dcb_table(struct drm_device *dev)
+olddcb_table(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u8 *dcb = NULL;
 
-	if (dev_priv->card_type > NV_04)
-		dcb = ROMPTR(dev, dev_priv->vbios.data[0x36]);
+	if (nv_device(drm->device)->card_type > NV_04)
+		dcb = ROMPTR(dev, drm->vbios.data[0x36]);
 	if (!dcb) {
-		NV_WARNONCE(dev, "No DCB data found in VBIOS\n");
+		NV_WARN(drm, "No DCB data found in VBIOS\n");
 		return NULL;
 	}
 
 	if (dcb[0] >= 0x41) {
-		NV_WARNONCE(dev, "DCB version 0x%02x unknown\n", dcb[0]);
+		NV_WARN(drm, "DCB version 0x%02x unknown\n", dcb[0]);
 		return NULL;
 	} else
 	if (dcb[0] >= 0x30) {
@@ -5670,18 +1572,18 @@ dcb_table(struct drm_device *dev)
 		 *
 		 * v1.1 (NV5+, maybe some NV4) is entirely unhelpful
 		 */
-		NV_WARNONCE(dev, "No useful DCB data in VBIOS\n");
+		NV_WARN(drm, "No useful DCB data in VBIOS\n");
 		return NULL;
 	}
 
-	NV_WARNONCE(dev, "DCB header validation failed\n");
+	NV_WARN(drm, "DCB header validation failed\n");
 	return NULL;
 }
 
 void *
-dcb_outp(struct drm_device *dev, u8 idx)
+olddcb_outp(struct drm_device *dev, u8 idx)
 {
-	u8 *dcb = dcb_table(dev);
+	u8 *dcb = olddcb_table(dev);
 	if (dcb && dcb[0] >= 0x30) {
 		if (idx < dcb[2])
 			return dcb + dcb[1] + (idx * dcb[3]);
@@ -5703,20 +1605,20 @@ dcb_outp(struct drm_device *dev, u8 idx)
 }
 
 int
-dcb_outp_foreach(struct drm_device *dev, void *data,
+olddcb_outp_foreach(struct drm_device *dev, void *data,
 		 int (*exec)(struct drm_device *, void *, int idx, u8 *outp))
 {
 	int ret, idx = -1;
 	u8 *outp = NULL;
-	while ((outp = dcb_outp(dev, ++idx))) {
+	while ((outp = olddcb_outp(dev, ++idx))) {
 		if (ROM32(outp[0]) == 0x00000000)
 			break; /* seen on an NV11 with DCB v1.5 */
 		if (ROM32(outp[0]) == 0xffffffff)
 			break; /* seen on an NV17 with DCB v2.0 */
 
-		if ((outp[0] & 0x0f) == OUTPUT_UNUSED)
+		if ((outp[0] & 0x0f) == DCB_OUTPUT_UNUSED)
 			continue;
-		if ((outp[0] & 0x0f) == OUTPUT_EOL)
+		if ((outp[0] & 0x0f) == DCB_OUTPUT_EOL)
 			break;
 
 		ret = exec(dev, data, idx, outp);
@@ -5728,9 +1630,9 @@ dcb_outp_foreach(struct drm_device *dev, void *data,
 }
 
 u8 *
-dcb_conntab(struct drm_device *dev)
+olddcb_conntab(struct drm_device *dev)
 {
-	u8 *dcb = dcb_table(dev);
+	u8 *dcb = olddcb_table(dev);
 	if (dcb && dcb[0] >= 0x30 && dcb[1] >= 0x16) {
 		u8 *conntab = ROMPTR(dev, dcb[0x14]);
 		if (conntab && conntab[0] >= 0x30 && conntab[0] <= 0x40)
@@ -5740,19 +1642,19 @@ dcb_conntab(struct drm_device *dev)
 }
 
 u8 *
-dcb_conn(struct drm_device *dev, u8 idx)
+olddcb_conn(struct drm_device *dev, u8 idx)
 {
-	u8 *conntab = dcb_conntab(dev);
+	u8 *conntab = olddcb_conntab(dev);
 	if (conntab && idx < conntab[2])
 		return conntab + conntab[1] + (idx * conntab[3]);
 	return NULL;
 }
 
-static struct dcb_entry *new_dcb_entry(struct dcb_table *dcb)
+static struct dcb_output *new_dcb_entry(struct dcb_table *dcb)
 {
-	struct dcb_entry *entry = &dcb->entry[dcb->entries];
+	struct dcb_output *entry = &dcb->entry[dcb->entries];
 
-	memset(entry, 0, sizeof(struct dcb_entry));
+	memset(entry, 0, sizeof(struct dcb_output));
 	entry->index = dcb->entries++;
 
 	return entry;
@@ -5761,20 +1663,22 @@ static struct dcb_entry *new_dcb_entry(struct dcb_table *dcb)
 static void fabricate_dcb_output(struct dcb_table *dcb, int type, int i2c,
 				 int heads, int or)
 {
-	struct dcb_entry *entry = new_dcb_entry(dcb);
+	struct dcb_output *entry = new_dcb_entry(dcb);
 
 	entry->type = type;
 	entry->i2c_index = i2c;
 	entry->heads = heads;
-	if (type != OUTPUT_ANALOG)
+	if (type != DCB_OUTPUT_ANALOG)
 		entry->location = !DCB_LOC_ON_CHIP; /* ie OFF CHIP */
 	entry->or = or;
 }
 
 static bool
 parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
-		  uint32_t conn, uint32_t conf, struct dcb_entry *entry)
+		  uint32_t conn, uint32_t conf, struct dcb_output *entry)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	entry->type = conn & 0xf;
 	entry->i2c_index = (conn >> 4) & 0xf;
 	entry->heads = (conn >> 8) & 0xf;
@@ -5784,7 +1688,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 	entry->or = (conn >> 24) & 0xf;
 
 	switch (entry->type) {
-	case OUTPUT_ANALOG:
+	case DCB_OUTPUT_ANALOG:
 		/*
 		 * Although the rest of a CRT conf dword is usually
 		 * zeros, mac biosen have stuff there so we must mask
@@ -5793,7 +1697,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 					 (conf & 0xffff) * 10 :
 					 (conf & 0xff) * 10000;
 		break;
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		{
 		uint32_t mask;
 		if (conf & 0x1)
@@ -5828,12 +1732,12 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 			if (dcb->version >= 0x40)
 				break;
 
-			NV_ERROR(dev, "Unknown LVDS configuration bits, "
+			NV_ERROR(drm, "Unknown LVDS configuration bits, "
 				      "please report\n");
 		}
 		break;
 		}
-	case OUTPUT_TV:
+	case DCB_OUTPUT_TV:
 	{
 		if (dcb->version >= 0x30)
 			entry->tvconf.has_component_output = conf & (0x8 << 4);
@@ -5842,7 +1746,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 
 		break;
 	}
-	case OUTPUT_DP:
+	case DCB_OUTPUT_DP:
 		entry->dpconf.sor.link = (conf & 0x00000030) >> 4;
 		switch ((conf & 0x00e00000) >> 21) {
 		case 0:
@@ -5864,7 +1768,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 			break;
 		}
 		break;
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		if (dcb->version >= 0x40)
 			entry->tmdsconf.sor.link = (conf & 0x00000030) >> 4;
 		else if (dcb->version >= 0x30)
@@ -5873,7 +1777,7 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 			entry->tmdsconf.slave_addr = (conf & 0x00000070) >> 4;
 
 		break;
-	case OUTPUT_EOL:
+	case DCB_OUTPUT_EOL:
 		/* weird g80 mobile type that "nv" treats as a terminator */
 		dcb->entries--;
 		return false;
@@ -5900,27 +1804,29 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
 
 static bool
 parse_dcb15_entry(struct drm_device *dev, struct dcb_table *dcb,
-		  uint32_t conn, uint32_t conf, struct dcb_entry *entry)
+		  uint32_t conn, uint32_t conf, struct dcb_output *entry)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	switch (conn & 0x0000000f) {
 	case 0:
-		entry->type = OUTPUT_ANALOG;
+		entry->type = DCB_OUTPUT_ANALOG;
 		break;
 	case 1:
-		entry->type = OUTPUT_TV;
+		entry->type = DCB_OUTPUT_TV;
 		break;
 	case 2:
 	case 4:
 		if (conn & 0x10)
-			entry->type = OUTPUT_LVDS;
+			entry->type = DCB_OUTPUT_LVDS;
 		else
-			entry->type = OUTPUT_TMDS;
+			entry->type = DCB_OUTPUT_TMDS;
 		break;
 	case 3:
-		entry->type = OUTPUT_LVDS;
+		entry->type = DCB_OUTPUT_LVDS;
 		break;
 	default:
-		NV_ERROR(dev, "Unknown DCB type %d\n", conn & 0x0000000f);
+		NV_ERROR(drm, "Unknown DCB type %d\n", conn & 0x0000000f);
 		return false;
 	}
 
@@ -5932,13 +1838,13 @@ parse_dcb15_entry(struct drm_device *dev, struct dcb_table *dcb,
 	entry->duallink_possible = false;
 
 	switch (entry->type) {
-	case OUTPUT_ANALOG:
+	case DCB_OUTPUT_ANALOG:
 		entry->crtconf.maxfreq = (conf & 0xffff) * 10;
 		break;
-	case OUTPUT_TV:
+	case DCB_OUTPUT_TV:
 		entry->tvconf.has_component_output = false;
 		break;
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		if ((conn & 0x00003f00) >> 8 != 0x10)
 			entry->lvdsconf.use_straps_for_mode = true;
 		entry->lvdsconf.use_power_scripts = true;
@@ -5959,14 +1865,15 @@ void merge_like_dcb_entries(struct drm_device *dev, struct dcb_table *dcb)
 	 * more options
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int i, newentries = 0;
 
 	for (i = 0; i < dcb->entries; i++) {
-		struct dcb_entry *ient = &dcb->entry[i];
+		struct dcb_output *ient = &dcb->entry[i];
 		int j;
 
 		for (j = i + 1; j < dcb->entries; j++) {
-			struct dcb_entry *jent = &dcb->entry[j];
+			struct dcb_output *jent = &dcb->entry[j];
 
 			if (jent->type == 100) /* already merged entry */
 				continue;
@@ -5976,7 +1883,7 @@ void merge_like_dcb_entries(struct drm_device *dev, struct dcb_table *dcb)
 			    jent->type == ient->type &&
 			    jent->location == ient->location &&
 			    jent->or == ient->or) {
-				NV_TRACE(dev, "Merging DCB entries %d and %d\n",
+				NV_INFO(drm, "Merging DCB entries %d and %d\n",
 					 i, j);
 				ient->heads |= jent->heads;
 				jent->type = 100; /* dummy value */
@@ -6002,8 +1909,8 @@ void merge_like_dcb_entries(struct drm_device *dev, struct dcb_table *dcb)
 static bool
 apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_table *dcb = &dev_priv->vbios.dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_table *dcb = &drm->vbios.dcb;
 
 	/* Dell Precision M6300
 	 *   DCB entry 2: 02025312 00000010
@@ -6029,7 +1936,7 @@ apply_dcb_encoder_quirks(struct drm_device *dev, int idx, u32 *conn, u32 *conf)
 	 */
 	if (nv_match_device(dev, 0x0201, 0x1462, 0x8851)) {
 		if (*conn == 0xf2005014 && *conf == 0xffffffff) {
-			fabricate_dcb_output(dcb, OUTPUT_TMDS, 1, 1, 1);
+			fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 1, 1, 1);
 			return false;
 		}
 	}
@@ -6115,24 +2022,24 @@ fabricate_dcb_encoder_table(struct drm_device *dev, struct nvbios *bios)
 #ifdef __powerpc__
 	/* Apple iMac G4 NV17 */
 	if (of_machine_is_compatible("PowerMac4,5")) {
-		fabricate_dcb_output(dcb, OUTPUT_TMDS, 0, all_heads, 1);
-		fabricate_dcb_output(dcb, OUTPUT_ANALOG, 1, all_heads, 2);
+		fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS, 0, all_heads, 1);
+		fabricate_dcb_output(dcb, DCB_OUTPUT_ANALOG, 1, all_heads, 2);
 		return;
 	}
 #endif
 
 	/* Make up some sane defaults */
-	fabricate_dcb_output(dcb, OUTPUT_ANALOG,
+	fabricate_dcb_output(dcb, DCB_OUTPUT_ANALOG,
 			     bios->legacy.i2c_indices.crt, 1, 1);
 
 	if (nv04_tv_identify(dev, bios->legacy.i2c_indices.tv) >= 0)
-		fabricate_dcb_output(dcb, OUTPUT_TV,
+		fabricate_dcb_output(dcb, DCB_OUTPUT_TV,
 				     bios->legacy.i2c_indices.tv,
 				     all_heads, 0);
 
 	else if (bios->tmds.output0_script_ptr ||
 		 bios->tmds.output1_script_ptr)
-		fabricate_dcb_output(dcb, OUTPUT_TMDS,
+		fabricate_dcb_output(dcb, DCB_OUTPUT_TMDS,
 				     bios->legacy.i2c_indices.panel,
 				     all_heads, 1);
 }
@@ -6140,16 +2047,16 @@ fabricate_dcb_encoder_table(struct drm_device *dev, struct nvbios *bios)
 static int
 parse_dcb_entry(struct drm_device *dev, void *data, int idx, u8 *outp)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_table *dcb = &dev_priv->vbios.dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_table *dcb = &drm->vbios.dcb;
 	u32 conf = (dcb->version >= 0x20) ? ROM32(outp[4]) : ROM32(outp[6]);
 	u32 conn = ROM32(outp[0]);
 	bool ret;
 
 	if (apply_dcb_encoder_quirks(dev, idx, &conn, &conf)) {
-		struct dcb_entry *entry = new_dcb_entry(dcb);
+		struct dcb_output *entry = new_dcb_entry(dcb);
 
-		NV_TRACEWARN(dev, "DCB outp %02d: %08x %08x\n", idx, conn, conf);
+		NV_INFO(drm, "DCB outp %02d: %08x %08x\n", idx, conn, conf);
 
 		if (dcb->version >= 0x20)
 			ret = parse_dcb20_entry(dev, dcb, conn, conf, entry);
@@ -6162,7 +2069,7 @@ parse_dcb_entry(struct drm_device *dev, void *data, int idx, u8 *outp)
 		 * are cards with bogus values (nv31m in bug 23212),
 		 * and it's otherwise useless.
 		 */
-		if (entry->type == OUTPUT_TV &&
+		if (entry->type == DCB_OUTPUT_TV &&
 		    entry->location == DCB_LOC_ON_CHIP)
 			entry->i2c_index = 0x0f;
 	}
@@ -6210,7 +2117,7 @@ dcb_fake_connectors(struct nvbios *bios)
 	 * table - just in case it has random, rather than stub, entries.
 	 */
 	if (i > 1) {
-		u8 *conntab = dcb_conntab(bios->dev);
+		u8 *conntab = olddcb_conntab(bios->dev);
 		if (conntab)
 			conntab[0] = 0x00;
 	}
@@ -6219,11 +2126,12 @@ dcb_fake_connectors(struct nvbios *bios)
 static int
 parse_dcb_table(struct drm_device *dev, struct nvbios *bios)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct dcb_table *dcb = &bios->dcb;
 	u8 *dcbt, *conn;
 	int idx;
 
-	dcbt = dcb_table(dev);
+	dcbt = olddcb_table(dev);
 	if (!dcbt) {
 		/* handle pre-DCB boards */
 		if (bios->type == NVBIOS_BMP) {
@@ -6234,10 +2142,10 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios)
 		return -EINVAL;
 	}
 
-	NV_TRACE(dev, "DCB version %d.%d\n", dcbt[0] >> 4, dcbt[0] & 0xf);
+	NV_INFO(drm, "DCB version %d.%d\n", dcbt[0] >> 4, dcbt[0] & 0xf);
 
 	dcb->version = dcbt[0];
-	dcb_outp_foreach(dev, NULL, parse_dcb_entry);
+	olddcb_outp_foreach(dev, NULL, parse_dcb_entry);
 
 	/*
 	 * apart for v2.1+ not being known for requiring merging, this
@@ -6251,10 +2159,10 @@ parse_dcb_table(struct drm_device *dev, struct nvbios *bios)
 
 	/* dump connector table entries to log, if any exist */
 	idx = -1;
-	while ((conn = dcb_conn(dev, ++idx))) {
+	while ((conn = olddcb_conn(dev, ++idx))) {
 		if (conn[0] != 0xff) {
-			NV_TRACE(dev, "DCB conn %02d: ", idx);
-			if (dcb_conntab(dev)[3] < 4)
+			NV_INFO(drm, "DCB conn %02d: ", idx);
+			if (olddcb_conntab(dev)[3] < 4)
 				printk("%04x\n", ROM16(conn[0]));
 			else
 				printk("%08x\n", ROM32(conn[0]));
@@ -6275,12 +2183,14 @@ static int load_nv17_hwsq_ucode_entry(struct drm_device *dev, struct nvbios *bio
 	 * starting at reg 0x00001400
 	 */
 
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	uint8_t bytes_to_write;
 	uint16_t hwsq_entry_offset;
 	int i;
 
 	if (bios->data[hwsq_offset] <= entry) {
-		NV_ERROR(dev, "Too few entries in HW sequencer table for "
+		NV_ERROR(drm, "Too few entries in HW sequencer table for "
 				"requested entry\n");
 		return -ENOENT;
 	}
@@ -6288,24 +2198,24 @@ static int load_nv17_hwsq_ucode_entry(struct drm_device *dev, struct nvbios *bio
 	bytes_to_write = bios->data[hwsq_offset + 1];
 
 	if (bytes_to_write != 36) {
-		NV_ERROR(dev, "Unknown HW sequencer entry size\n");
+		NV_ERROR(drm, "Unknown HW sequencer entry size\n");
 		return -EINVAL;
 	}
 
-	NV_TRACE(dev, "Loading NV17 power sequencing microcode\n");
+	NV_INFO(drm, "Loading NV17 power sequencing microcode\n");
 
 	hwsq_entry_offset = hwsq_offset + 2 + entry * bytes_to_write;
 
 	/* set sequencer control */
-	bios_wr32(bios, 0x00001304, ROM32(bios->data[hwsq_entry_offset]));
+	nv_wr32(device, 0x00001304, ROM32(bios->data[hwsq_entry_offset]));
 	bytes_to_write -= 4;
 
 	/* write ucode */
 	for (i = 0; i < bytes_to_write; i += 4)
-		bios_wr32(bios, 0x00001400 + i, ROM32(bios->data[hwsq_entry_offset + i + 4]));
+		nv_wr32(device, 0x00001400 + i, ROM32(bios->data[hwsq_entry_offset + i + 4]));
 
 	/* twiddle NV_PBUS_DEBUG_4 */
-	bios_wr32(bios, NV_PBUS_DEBUG_4, bios_rd32(bios, NV_PBUS_DEBUG_4) | 0x18);
+	nv_wr32(device, NV_PBUS_DEBUG_4, nv_rd32(device, NV_PBUS_DEBUG_4) | 0x18);
 
 	return 0;
 }
@@ -6336,8 +2246,8 @@ static int load_nv17_hw_sequencer_ucode(struct drm_device *dev,
 
 uint8_t *nouveau_bios_embedded_edid(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	const uint8_t edid_sig[] = {
 			0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
 	uint16_t offset = 0;
@@ -6360,53 +2270,29 @@ uint8_t *nouveau_bios_embedded_edid(struct drm_device *dev)
 		offset++;
 	}
 
-	NV_TRACE(dev, "Found EDID in BIOS\n");
+	NV_INFO(drm, "Found EDID in BIOS\n");
 
 	return bios->fp.edid = &bios->data[offset];
 }
 
-void
-nouveau_bios_run_init_table(struct drm_device *dev, uint16_t table,
-			    struct dcb_entry *dcbent, int crtc)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct init_exec iexec = { true, false };
-
-	spin_lock_bh(&bios->lock);
-	bios->display.output = dcbent;
-	bios->display.crtc = crtc;
-	parse_init_table(bios, table, &iexec);
-	bios->display.output = NULL;
-	spin_unlock_bh(&bios->lock);
-}
-
-void
-nouveau_bios_init_exec(struct drm_device *dev, uint16_t table)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct init_exec iexec = { true, false };
-
-	parse_init_table(bios, table, &iexec);
-}
-
 static bool NVInitVBIOS(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 
 	memset(bios, 0, sizeof(struct nvbios));
 	spin_lock_init(&bios->lock);
 	bios->dev = dev;
 
-	return bios_shadow(dev);
+	bios->data = nouveau_bios(drm->device)->data;
+	bios->length = nouveau_bios(drm->device)->size;
+	return true;
 }
 
 static int nouveau_parse_vbios_struct(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	const uint8_t bit_signature[] = { 0xff, 0xb8, 'B', 'I', 'T' };
 	const uint8_t bmp_signature[] = { 0xff, 0x7f, 'N', 'V', 0x0 };
 	int offset;
@@ -6414,7 +2300,7 @@ static int nouveau_parse_vbios_struct(struct drm_device *dev)
 	offset = findstr(bios->data, bios->length,
 					bit_signature, sizeof(bit_signature));
 	if (offset) {
-		NV_TRACE(dev, "BIT BIOS found\n");
+		NV_INFO(drm, "BIT BIOS found\n");
 		bios->type = NVBIOS_BIT;
 		bios->offset = offset;
 		return parse_bit_structure(bios, offset + 6);
@@ -6423,21 +2309,21 @@ static int nouveau_parse_vbios_struct(struct drm_device *dev)
 	offset = findstr(bios->data, bios->length,
 					bmp_signature, sizeof(bmp_signature));
 	if (offset) {
-		NV_TRACE(dev, "BMP BIOS found\n");
+		NV_INFO(drm, "BMP BIOS found\n");
 		bios->type = NVBIOS_BMP;
 		bios->offset = offset;
 		return parse_bmp_structure(dev, bios, offset);
 	}
 
-	NV_ERROR(dev, "No known BIOS signature found\n");
+	NV_ERROR(drm, "No known BIOS signature found\n");
 	return -ENODEV;
 }
 
 int
 nouveau_run_vbios_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	int i, ret = 0;
 
 	/* Reset the BIOS head to 0. */
@@ -6451,23 +2337,8 @@ nouveau_run_vbios_init(struct drm_device *dev)
 		bios->fp.lvds_init_run = false;
 	}
 
-	parse_init_tables(bios);
-
-	/*
-	 * Runs some additional script seen on G8x VBIOSen.  The VBIOS'
-	 * parser will run this right after the init tables, the binary
-	 * driver appears to run it at some point later.
-	 */
-	if (bios->some_script_ptr) {
-		struct init_exec iexec = {true, false};
-
-		NV_INFO(dev, "Parsing VBIOS init table at offset 0x%04X\n",
-			bios->some_script_ptr);
-		parse_init_table(bios, bios->some_script_ptr, &iexec);
-	}
-
-	if (dev_priv->card_type >= NV_50) {
-		for (i = 0; i < bios->dcb.entries; i++) {
+	if (nv_device(drm->device)->card_type >= NV_50) {
+		for (i = 0; bios->execute && i < bios->dcb.entries; i++) {
 			nouveau_bios_run_display_table(dev, 0, 0,
 						       &bios->dcb.entry[i], -1);
 		}
@@ -6479,10 +2350,10 @@ nouveau_run_vbios_init(struct drm_device *dev)
 static bool
 nouveau_bios_posted(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	unsigned htotal;
 
-	if (dev_priv->card_type >= NV_50) {
+	if (nv_device(drm->device)->card_type >= NV_50) {
 		if (NVReadVgaCrtc(dev, 0, 0x00) == 0 &&
 		    NVReadVgaCrtc(dev, 0, 0x1a) == 0)
 			return false;
@@ -6501,8 +2372,8 @@ nouveau_bios_posted(struct drm_device *dev)
 int
 nouveau_bios_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	int ret;
 
 	if (!NVInitVBIOS(dev))
@@ -6512,14 +2383,6 @@ nouveau_bios_init(struct drm_device *dev)
 	if (ret)
 		return ret;
 
-	ret = nouveau_i2c_init(dev);
-	if (ret)
-		return ret;
-
-	ret = nouveau_mxm_init(dev);
-	if (ret)
-		return ret;
-
 	ret = parse_dcb_table(dev, bios);
 	if (ret)
 		return ret;
@@ -6532,12 +2395,10 @@ nouveau_bios_init(struct drm_device *dev)
 
 	/* ... unless card isn't POSTed already */
 	if (!nouveau_bios_posted(dev)) {
-		NV_INFO(dev, "Adaptor not initialised, "
+		NV_INFO(drm, "Adaptor not initialised, "
 			"running VBIOS init tables.\n");
 		bios->execute = true;
 	}
-	if (nouveau_force_post)
-		bios->execute = true;
 
 	ret = nouveau_run_vbios_init(dev);
 	if (ret)
@@ -6560,10 +2421,4 @@ nouveau_bios_init(struct drm_device *dev)
 void
 nouveau_bios_takedown(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nouveau_mxm_fini(dev);
-	nouveau_i2c_fini(dev);
-
-	kfree(dev_priv->vbios.data);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.h b/drivers/gpu/drm/nouveau/nouveau_bios.h
index 298a3af48d1..3befbb821a5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.h
@@ -21,11 +21,10 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef __NOUVEAU_BIOS_H__
-#define __NOUVEAU_BIOS_H__
+#ifndef __NOUVEAU_DISPBIOS_H__
+#define __NOUVEAU_DISPBIOS_H__
 
 #include "nvreg.h"
-#include "nouveau_i2c.h"
 
 #define DCB_MAX_NUM_ENTRIES 16
 #define DCB_MAX_NUM_I2C_ENTRIES 16
@@ -39,8 +38,8 @@
 #define ROM48(x) ({ u8 *p = &(x); (u64)ROM16(p[4]) << 32 | ROM32(p[0]); })
 #define ROM64(x) le64_to_cpu(*(u64 *)&(x))
 #define ROMPTR(d,x) ({            \
-	struct drm_nouveau_private *dev_priv = (d)->dev_private; \
-	ROM16(x) ? &dev_priv->vbios.data[ROM16(x)] : NULL; \
+	struct nouveau_drm *drm = nouveau_drm((d)); \
+	ROM16(x) ? &drm->vbios.data[ROM16(x)] : NULL; \
 })
 
 struct bit_entry {
@@ -53,95 +52,19 @@ struct bit_entry {
 
 int bit_table(struct drm_device *, u8 id, struct bit_entry *);
 
-enum dcb_gpio_tag {
-	DCB_GPIO_PANEL_POWER = 0x01,
-	DCB_GPIO_TVDAC0 = 0x0c,
-	DCB_GPIO_TVDAC1 = 0x2d,
-	DCB_GPIO_PWM_FAN = 0x09,
-	DCB_GPIO_FAN_SENSE = 0x3d,
-	DCB_GPIO_UNUSED = 0xff
-};
-
-enum dcb_connector_type {
-	DCB_CONNECTOR_VGA = 0x00,
-	DCB_CONNECTOR_TV_0 = 0x10,
-	DCB_CONNECTOR_TV_1 = 0x11,
-	DCB_CONNECTOR_TV_3 = 0x13,
-	DCB_CONNECTOR_DVI_I = 0x30,
-	DCB_CONNECTOR_DVI_D = 0x31,
-	DCB_CONNECTOR_DMS59_0 = 0x38,
-	DCB_CONNECTOR_DMS59_1 = 0x39,
-	DCB_CONNECTOR_LVDS = 0x40,
-	DCB_CONNECTOR_LVDS_SPWG = 0x41,
-	DCB_CONNECTOR_DP = 0x46,
-	DCB_CONNECTOR_eDP = 0x47,
-	DCB_CONNECTOR_HDMI_0 = 0x60,
-	DCB_CONNECTOR_HDMI_1 = 0x61,
-	DCB_CONNECTOR_DMS59_DP0 = 0x64,
-	DCB_CONNECTOR_DMS59_DP1 = 0x65,
-	DCB_CONNECTOR_NONE = 0xff
-};
-
-enum dcb_type {
-	OUTPUT_ANALOG = 0,
-	OUTPUT_TV = 1,
-	OUTPUT_TMDS = 2,
-	OUTPUT_LVDS = 3,
-	OUTPUT_DP = 6,
-	OUTPUT_EOL = 14, /* DCB 4.0+, appears to be end-of-list */
-	OUTPUT_UNUSED = 15,
-	OUTPUT_ANY = -1
-};
-
-struct dcb_entry {
-	int index;	/* may not be raw dcb index if merging has happened */
-	enum dcb_type type;
-	uint8_t i2c_index;
-	uint8_t heads;
-	uint8_t connector;
-	uint8_t bus;
-	uint8_t location;
-	uint8_t or;
-	bool duallink_possible;
-	union {
-		struct sor_conf {
-			int link;
-		} sorconf;
-		struct {
-			int maxfreq;
-		} crtconf;
-		struct {
-			struct sor_conf sor;
-			bool use_straps_for_mode;
-			bool use_acpi_for_edid;
-			bool use_power_scripts;
-		} lvdsconf;
-		struct {
-			bool has_component_output;
-		} tvconf;
-		struct {
-			struct sor_conf sor;
-			int link_nr;
-			int link_bw;
-		} dpconf;
-		struct {
-			struct sor_conf sor;
-			int slave_addr;
-		} tmdsconf;
-	};
-	bool i2c_upper_default;
-};
+#include <subdev/bios/dcb.h>
+#include <subdev/bios/conn.h>
 
 struct dcb_table {
 	uint8_t version;
 	int entries;
-	struct dcb_entry entry[DCB_MAX_NUM_ENTRIES];
+	struct dcb_output entry[DCB_MAX_NUM_ENTRIES];
 };
 
 enum nouveau_or {
-	OUTPUT_A = (1 << 0),
-	OUTPUT_B = (1 << 1),
-	OUTPUT_C = (1 << 2)
+	DCB_OUTPUT_A = (1 << 0),
+	DCB_OUTPUT_B = (1 << 1),
+	DCB_OUTPUT_C = (1 << 2)
 };
 
 enum LVDS_script {
@@ -154,58 +77,6 @@ enum LVDS_script {
 	LVDS_PANEL_OFF
 };
 
-/* these match types in pll limits table version 0x40,
- * nouveau uses them on all chipsets internally where a
- * specific pll needs to be referenced, but the exact
- * register isn't known.
- */
-enum pll_types {
-	PLL_CORE   = 0x01,
-	PLL_SHADER = 0x02,
-	PLL_UNK03  = 0x03,
-	PLL_MEMORY = 0x04,
-	PLL_VDEC   = 0x05,
-	PLL_UNK40  = 0x40,
-	PLL_UNK41  = 0x41,
-	PLL_UNK42  = 0x42,
-	PLL_VPLL0  = 0x80,
-	PLL_VPLL1  = 0x81,
-	PLL_MAX    = 0xff
-};
-
-struct pll_lims {
-	u32 reg;
-
-	struct {
-		int minfreq;
-		int maxfreq;
-		int min_inputfreq;
-		int max_inputfreq;
-
-		uint8_t min_m;
-		uint8_t max_m;
-		uint8_t min_n;
-		uint8_t max_n;
-	} vco1, vco2;
-
-	uint8_t max_log2p;
-	/*
-	 * for most pre nv50 cards setting a log2P of 7 (the common max_log2p
-	 * value) is no different to 6 (at least for vplls) so allowing the MNP
-	 * calc to use 7 causes the generated clock to be out by a factor of 2.
-	 * however, max_log2p cannot be fixed-up during parsing as the
-	 * unmodified max_log2p value is still needed for setting mplls, hence
-	 * an additional max_usable_log2p member
-	 */
-	uint8_t max_usable_log2p;
-	uint8_t log2p_bias;
-
-	uint8_t min_p;
-	uint8_t max_p;
-
-	int refclk;
-};
-
 struct nvbios {
 	struct drm_device *dev;
 	enum {
@@ -257,7 +128,7 @@ struct nvbios {
 	} state;
 
 	struct {
-		struct dcb_entry *output;
+		struct dcb_output *output;
 		int crtc;
 		uint16_t script_table_ptr;
 	} display;
@@ -302,11 +173,28 @@ struct nvbios {
 	} legacy;
 };
 
-void *dcb_table(struct drm_device *);
-void *dcb_outp(struct drm_device *, u8 idx);
-int dcb_outp_foreach(struct drm_device *, void *data,
+void *olddcb_table(struct drm_device *);
+void *olddcb_outp(struct drm_device *, u8 idx);
+int olddcb_outp_foreach(struct drm_device *, void *data,
 		     int (*)(struct drm_device *, void *, int idx, u8 *outp));
-u8 *dcb_conntab(struct drm_device *);
-u8 *dcb_conn(struct drm_device *, u8 idx);
+u8 *olddcb_conntab(struct drm_device *);
+u8 *olddcb_conn(struct drm_device *, u8 idx);
+
+int nouveau_bios_init(struct drm_device *);
+void nouveau_bios_takedown(struct drm_device *dev);
+int nouveau_run_vbios_init(struct drm_device *);
+struct dcb_connector_table_entry *
+nouveau_bios_connector_entry(struct drm_device *, int index);
+int nouveau_bios_run_display_table(struct drm_device *, u16 id, int clk,
+					  struct dcb_output *, int crtc);
+bool nouveau_bios_fp_mode(struct drm_device *, struct drm_display_mode *);
+uint8_t *nouveau_bios_embedded_edid(struct drm_device *);
+int nouveau_bios_parse_lvds_table(struct drm_device *, int pxclk,
+					 bool *dl, bool *if_is_24bit);
+int run_tmds_table(struct drm_device *, struct dcb_output *,
+			  int head, int pxclk);
+int call_lvds_script(struct drm_device *, struct dcb_output *, int head,
+			    enum LVDS_script, int pxclk);
+bool bios_encoder_match(struct dcb_output *, u32 hash);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 7f80ed52356..259e5f1adf4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -27,31 +27,127 @@
  *	    Jeremy Kolb  <jkolb@brandeis.edu>
  */
 
-#include "drmP.h"
-#include "ttm/ttm_page_alloc.h"
+#include <core/engine.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
 
 #include "nouveau_drm.h"
-#include "nouveau_drv.h"
 #include "nouveau_dma.h"
-#include "nouveau_mm.h"
-#include "nouveau_vm.h"
 #include "nouveau_fence.h"
-#include "nouveau_ramht.h"
 
-#include <linux/log2.h>
-#include <linux/slab.h>
+#include "nouveau_bo.h"
+#include "nouveau_ttm.h"
+#include "nouveau_gem.h"
+
+/*
+ * NV10-NV40 tiling helpers
+ */
+
+static void
+nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
+			   u32 addr, u32 size, u32 pitch, u32 flags)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	int i = reg - drm->tile.reg;
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	struct nouveau_fb_tile *tile = &pfb->tile.region[i];
+	struct nouveau_engine *engine;
+
+	nouveau_fence_unref(&reg->fence);
+
+	if (tile->pitch)
+		pfb->tile.fini(pfb, i, tile);
+
+	if (pitch)
+		pfb->tile.init(pfb, i, addr, size, pitch, flags, tile);
+
+	pfb->tile.prog(pfb, i, tile);
+
+	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_GR)))
+		engine->tile_prog(engine, i);
+	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_MPEG)))
+		engine->tile_prog(engine, i);
+}
+
+static struct nouveau_drm_tile *
+nv10_bo_get_tile_region(struct drm_device *dev, int i)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
+
+	spin_lock(&drm->tile.lock);
+
+	if (!tile->used &&
+	    (!tile->fence || nouveau_fence_done(tile->fence)))
+		tile->used = true;
+	else
+		tile = NULL;
+
+	spin_unlock(&drm->tile.lock);
+	return tile;
+}
+
+static void
+nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
+			struct nouveau_fence *fence)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	if (tile) {
+		spin_lock(&drm->tile.lock);
+		if (fence) {
+			/* Mark it as pending. */
+			tile->fence = fence;
+			nouveau_fence_ref(fence);
+		}
+
+		tile->used = false;
+		spin_unlock(&drm->tile.lock);
+	}
+}
+
+static struct nouveau_drm_tile *
+nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
+		   u32 size, u32 pitch, u32 flags)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	struct nouveau_drm_tile *tile, *found = NULL;
+	int i;
+
+	for (i = 0; i < pfb->tile.regions; i++) {
+		tile = nv10_bo_get_tile_region(dev, i);
+
+		if (pitch && !found) {
+			found = tile;
+			continue;
+
+		} else if (tile && pfb->tile.region[i].pitch) {
+			/* Kill an unused tile region. */
+			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
+		}
+
+		nv10_bo_put_tile_region(dev, tile, NULL);
+	}
+
+	if (found)
+		nv10_bo_update_tile_region(dev, found, addr, size,
+					    pitch, flags);
+	return found;
+}
 
 static void
 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
+	struct drm_device *dev = drm->dev;
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
 
 	if (unlikely(nvbo->gem))
 		DRM_ERROR("bo %p still attached to GEM object\n", bo);
-
-	nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
+	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
 	kfree(nvbo);
 }
 
@@ -59,23 +155,24 @@ static void
 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
 		       int *align, int *size)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
+	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
+	struct nouveau_device *device = nv_device(drm->device);
 
-	if (dev_priv->card_type < NV_50) {
+	if (device->card_type < NV_50) {
 		if (nvbo->tile_mode) {
-			if (dev_priv->chipset >= 0x40) {
+			if (device->chipset >= 0x40) {
 				*align = 65536;
 				*size = roundup(*size, 64 * nvbo->tile_mode);
 
-			} else if (dev_priv->chipset >= 0x30) {
+			} else if (device->chipset >= 0x30) {
 				*align = 32768;
 				*size = roundup(*size, 64 * nvbo->tile_mode);
 
-			} else if (dev_priv->chipset >= 0x20) {
+			} else if (device->chipset >= 0x20) {
 				*align = 16384;
 				*size = roundup(*size, 64 * nvbo->tile_mode);
 
-			} else if (dev_priv->chipset >= 0x10) {
+			} else if (device->chipset >= 0x10) {
 				*align = 16384;
 				*size = roundup(*size, 32 * nvbo->tile_mode);
 			}
@@ -94,7 +191,7 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
 	       struct sg_table *sg,
 	       struct nouveau_bo **pnvbo)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_bo *nvbo;
 	size_t acc_size;
 	int ret;
@@ -111,22 +208,22 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
 	INIT_LIST_HEAD(&nvbo->vma_list);
 	nvbo->tile_mode = tile_mode;
 	nvbo->tile_flags = tile_flags;
-	nvbo->bo.bdev = &dev_priv->ttm.bdev;
+	nvbo->bo.bdev = &drm->ttm.bdev;
 
 	nvbo->page_shift = 12;
-	if (dev_priv->bar1_vm) {
+	if (drm->client.base.vm) {
 		if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
-			nvbo->page_shift = dev_priv->bar1_vm->lpg_shift;
+			nvbo->page_shift = drm->client.base.vm->vmm->lpg_shift;
 	}
 
 	nouveau_bo_fixup_align(nvbo, flags, &align, &size);
 	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
 	nouveau_bo_placement_set(nvbo, flags, 0);
 
-	acc_size = ttm_bo_dma_acc_size(&dev_priv->ttm.bdev, size,
+	acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
 				       sizeof(struct nouveau_bo));
 
-	ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
+	ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
 			  type, &nvbo->placement,
 			  align >> PAGE_SHIFT, 0, false, NULL, acc_size, sg,
 			  nouveau_bo_del_ttm);
@@ -155,10 +252,11 @@ set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
 static void
 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
-	int vram_pages = dev_priv->vram_size >> PAGE_SHIFT;
+	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	u32 vram_pages = pfb->ram.size >> PAGE_SHIFT;
 
-	if (dev_priv->card_type == NV_10 &&
+	if (nv_device(drm->device)->card_type == NV_10 &&
 	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
 	    nvbo->bo.mem.num_pages < vram_pages / 4) {
 		/*
@@ -198,13 +296,12 @@ nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
 int
 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
+	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 	struct ttm_buffer_object *bo = &nvbo->bo;
 	int ret;
 
 	if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
-		NV_ERROR(nouveau_bdev(bo->bdev)->dev,
-			 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
+		NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
 			 1 << bo->mem.mem_type, memtype);
 		return -EINVAL;
 	}
@@ -222,10 +319,10 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
 	if (ret == 0) {
 		switch (bo->mem.mem_type) {
 		case TTM_PL_VRAM:
-			dev_priv->fb_aper_free -= bo->mem.size;
+			drm->gem.vram_available -= bo->mem.size;
 			break;
 		case TTM_PL_TT:
-			dev_priv->gart_info.aper_free -= bo->mem.size;
+			drm->gem.gart_available -= bo->mem.size;
 			break;
 		default:
 			break;
@@ -241,7 +338,7 @@ out:
 int
 nouveau_bo_unpin(struct nouveau_bo *nvbo)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
+	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
 	struct ttm_buffer_object *bo = &nvbo->bo;
 	int ret;
 
@@ -258,10 +355,10 @@ nouveau_bo_unpin(struct nouveau_bo *nvbo)
 	if (ret == 0) {
 		switch (bo->mem.mem_type) {
 		case TTM_PL_VRAM:
-			dev_priv->fb_aper_free += bo->mem.size;
+			drm->gem.vram_available += bo->mem.size;
 			break;
 		case TTM_PL_TT:
-			dev_priv->gart_info.aper_free += bo->mem.size;
+			drm->gem.gart_available += bo->mem.size;
 			break;
 		default:
 			break;
@@ -356,30 +453,18 @@ nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
 }
 
 static struct ttm_tt *
-nouveau_ttm_tt_create(struct ttm_bo_device *bdev,
-		      unsigned long size, uint32_t page_flags,
-		      struct page *dummy_read_page)
+nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
+		      uint32_t page_flags, struct page *dummy_read)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bdev);
+	struct drm_device *dev = drm->dev;
 
-	switch (dev_priv->gart_info.type) {
-#if __OS_HAS_AGP
-	case NOUVEAU_GART_AGP:
-		return ttm_agp_tt_create(bdev, dev->agp->bridge,
-					 size, page_flags, dummy_read_page);
-#endif
-	case NOUVEAU_GART_PDMA:
-	case NOUVEAU_GART_HW:
-		return nouveau_sgdma_create_ttm(bdev, size, page_flags,
-						dummy_read_page);
-	default:
-		NV_ERROR(dev, "Unknown GART type %d\n",
-			 dev_priv->gart_info.type);
-		break;
+	if (drm->agp.stat == ENABLED) {
+		return ttm_agp_tt_create(bdev, dev->agp->bridge, size,
+					 page_flags, dummy_read);
 	}
 
-	return NULL;
+	return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
 }
 
 static int
@@ -393,8 +478,7 @@ static int
 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 			 struct ttm_mem_type_manager *man)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bdev);
 
 	switch (type) {
 	case TTM_PL_SYSTEM:
@@ -403,7 +487,7 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		man->default_caching = TTM_PL_FLAG_CACHED;
 		break;
 	case TTM_PL_VRAM:
-		if (dev_priv->card_type >= NV_50) {
+		if (nv_device(drm->device)->card_type >= NV_50) {
 			man->func = &nouveau_vram_manager;
 			man->io_reserve_fastpath = false;
 			man->use_io_reserve_lru = true;
@@ -417,32 +501,28 @@ nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 		man->default_caching = TTM_PL_FLAG_WC;
 		break;
 	case TTM_PL_TT:
-		if (dev_priv->card_type >= NV_50)
+		if (nv_device(drm->device)->card_type >= NV_50)
 			man->func = &nouveau_gart_manager;
 		else
+		if (drm->agp.stat != ENABLED)
+			man->func = &nv04_gart_manager;
+		else
 			man->func = &ttm_bo_manager_func;
-		switch (dev_priv->gart_info.type) {
-		case NOUVEAU_GART_AGP:
+
+		if (drm->agp.stat == ENABLED) {
 			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
 			man->available_caching = TTM_PL_FLAG_UNCACHED |
 				TTM_PL_FLAG_WC;
 			man->default_caching = TTM_PL_FLAG_WC;
-			break;
-		case NOUVEAU_GART_PDMA:
-		case NOUVEAU_GART_HW:
+		} else {
 			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
 				     TTM_MEMTYPE_FLAG_CMA;
 			man->available_caching = TTM_PL_MASK_CACHING;
 			man->default_caching = TTM_PL_FLAG_CACHED;
-			break;
-		default:
-			NV_ERROR(dev, "Unknown GART type: %d\n",
-				 dev_priv->gart_info.type);
-			return -EINVAL;
 		}
+
 		break;
 	default:
-		NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
 		return -EINVAL;
 	}
 	return 0;
@@ -491,6 +571,18 @@ nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
 }
 
 static int
+nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
+{
+	int ret = RING_SPACE(chan, 2);
+	if (ret == 0) {
+		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
+		OUT_RING  (chan, handle);
+		FIRE_RING (chan);
+	}
+	return ret;
+}
+
+static int
 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
 {
@@ -676,20 +768,14 @@ nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 static int
 nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
 {
-	int ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
-					 &chan->m2mf_ntfy);
+	int ret = RING_SPACE(chan, 6);
 	if (ret == 0) {
-		ret = RING_SPACE(chan, 6);
-		if (ret == 0) {
-			BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
-			OUT_RING  (chan, handle);
-			BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
-			OUT_RING  (chan, NvNotify0);
-			OUT_RING  (chan, NvDmaFB);
-			OUT_RING  (chan, NvDmaFB);
-		} else {
-			nouveau_ramht_remove(chan, NvNotify0);
-		}
+		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
+		OUT_RING  (chan, handle);
+		BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
+		OUT_RING  (chan, NvNotify0);
+		OUT_RING  (chan, NvDmaFB);
+		OUT_RING  (chan, NvDmaFB);
 	}
 
 	return ret;
@@ -788,16 +874,12 @@ nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
 static int
 nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
 {
-	int ret = nouveau_notifier_alloc(chan, NvNotify0, 32, 0xfe0, 0x1000,
-					 &chan->m2mf_ntfy);
+	int ret = RING_SPACE(chan, 4);
 	if (ret == 0) {
-		ret = RING_SPACE(chan, 4);
-		if (ret == 0) {
-			BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
-			OUT_RING  (chan, handle);
-			BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
-			OUT_RING  (chan, NvNotify0);
-		}
+		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
+		OUT_RING  (chan, handle);
+		BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
+		OUT_RING  (chan, NvNotify0);
 	}
 
 	return ret;
@@ -808,8 +890,8 @@ nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
 		      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
 {
 	if (mem->mem_type == TTM_PL_TT)
-		return chan->gart_handle;
-	return chan->vram_handle;
+		return NvDmaTT;
+	return NvDmaFB;
 }
 
 static int
@@ -865,8 +947,9 @@ nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo,
 	struct nouveau_mem *node = mem->mm_node;
 	int ret;
 
-	ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT,
-			     node->page_shift, NV_MEM_ACCESS_RO, vma);
+	ret = nouveau_vm_get(nv_client(chan->cli)->vm, mem->num_pages <<
+			     PAGE_SHIFT, node->page_shift,
+			     NV_MEM_ACCESS_RW, vma);
 	if (ret)
 		return ret;
 
@@ -883,19 +966,19 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 		     bool no_wait_reserve, bool no_wait_gpu,
 		     struct ttm_mem_reg *new_mem)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-	struct nouveau_channel *chan = chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
+	struct nouveau_channel *chan = chan = drm->channel;
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	int ret;
 
-	mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
+	mutex_lock(&chan->cli->mutex);
 
 	/* create temporary vmas for the transfer and attach them to the
 	 * old nouveau_mem node, these will get cleaned up after ttm has
 	 * destroyed the ttm_mem_reg
 	 */
-	if (dev_priv->card_type >= NV_50) {
+	if (nv_device(drm->device)->card_type >= NV_50) {
 		struct nouveau_mem *node = old_mem->mm_node;
 
 		ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]);
@@ -907,7 +990,7 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 			goto out;
 	}
 
-	ret = dev_priv->ttm.move(chan, bo, &bo->mem, new_mem);
+	ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
 	if (ret == 0) {
 		ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
 						    no_wait_reserve,
@@ -915,14 +998,13 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
 	}
 
 out:
-	mutex_unlock(&chan->mutex);
+	mutex_unlock(&chan->cli->mutex);
 	return ret;
 }
 
 void
-nouveau_bo_move_init(struct nouveau_channel *chan)
+nouveau_bo_move_init(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
 	static const struct {
 		const char *name;
 		int engine;
@@ -932,7 +1014,8 @@ nouveau_bo_move_init(struct nouveau_channel *chan)
 			    struct ttm_mem_reg *, struct ttm_mem_reg *);
 		int (*init)(struct nouveau_channel *, u32 handle);
 	} _methods[] = {
-		{  "COPY", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
+		{  "COPY", 0, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
+		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
 		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
 		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
 		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
@@ -947,19 +1030,34 @@ nouveau_bo_move_init(struct nouveau_channel *chan)
 	int ret;
 
 	do {
+		struct nouveau_object *object;
+		struct nouveau_channel *chan;
 		u32 handle = (mthd->engine << 16) | mthd->oclass;
-		ret = nouveau_gpuobj_gr_new(chan, handle, mthd->oclass);
+
+		if (mthd->init == nve0_bo_move_init)
+			chan = drm->cechan;
+		else
+			chan = drm->channel;
+		if (chan == NULL)
+			continue;
+
+		ret = nouveau_object_new(nv_object(drm), chan->handle, handle,
+					 mthd->oclass, NULL, 0, &object);
 		if (ret == 0) {
 			ret = mthd->init(chan, handle);
-			if (ret == 0) {
-				dev_priv->ttm.move = mthd->exec;
-				name = mthd->name;
-				break;
+			if (ret) {
+				nouveau_object_del(nv_object(drm),
+						   chan->handle, handle);
+				continue;
 			}
+
+			drm->ttm.move = mthd->exec;
+			name = mthd->name;
+			break;
 		}
 	} while ((++mthd)->exec);
 
-	NV_INFO(chan->dev, "MM: using %s for buffer copies\n", name);
+	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
 }
 
 static int
@@ -1044,7 +1142,7 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 			nouveau_vm_map(vma, new_mem->mm_node);
 		} else
 		if (new_mem && new_mem->mem_type == TTM_PL_TT &&
-		    nvbo->page_shift == vma->vm->spg_shift) {
+		    nvbo->page_shift == vma->vm->vmm->spg_shift) {
 			if (((struct nouveau_mem *)new_mem->mm_node)->sg)
 				nouveau_vm_map_sg_table(vma, 0, new_mem->
 						  num_pages << PAGE_SHIFT,
@@ -1061,10 +1159,10 @@ nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
 
 static int
 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
-		   struct nouveau_tile_reg **new_tile)
+		   struct nouveau_drm_tile **new_tile)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
+	struct drm_device *dev = drm->dev;
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
 	u64 offset = new_mem->start << PAGE_SHIFT;
 
@@ -1072,8 +1170,8 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
 	if (new_mem->mem_type != TTM_PL_VRAM)
 		return 0;
 
-	if (dev_priv->card_type >= NV_10) {
-		*new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
+	if (nv_device(drm->device)->card_type >= NV_10) {
+		*new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
 						nvbo->tile_mode,
 						nvbo->tile_flags);
 	}
@@ -1083,13 +1181,13 @@ nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
 
 static void
 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
-		      struct nouveau_tile_reg *new_tile,
-		      struct nouveau_tile_reg **old_tile)
+		      struct nouveau_drm_tile *new_tile,
+		      struct nouveau_drm_tile **old_tile)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
+	struct drm_device *dev = drm->dev;
 
-	nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
+	nv10_bo_put_tile_region(dev, *old_tile, bo->sync_obj);
 	*old_tile = new_tile;
 }
 
@@ -1098,13 +1196,13 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 		bool no_wait_reserve, bool no_wait_gpu,
 		struct ttm_mem_reg *new_mem)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
 	struct ttm_mem_reg *old_mem = &bo->mem;
-	struct nouveau_tile_reg *new_tile = NULL;
+	struct nouveau_drm_tile *new_tile = NULL;
 	int ret = 0;
 
-	if (dev_priv->card_type < NV_50) {
+	if (nv_device(drm->device)->card_type < NV_50) {
 		ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
 		if (ret)
 			return ret;
@@ -1119,7 +1217,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 	}
 
 	/* CPU copy if we have no accelerated method available */
-	if (!dev_priv->ttm.move) {
+	if (!drm->ttm.move) {
 		ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 		goto out;
 	}
@@ -1139,7 +1237,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 	ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
 
 out:
-	if (dev_priv->card_type < NV_50) {
+	if (nv_device(drm->device)->card_type < NV_50) {
 		if (ret)
 			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
 		else
@@ -1159,8 +1257,8 @@ static int
 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bdev);
+	struct drm_device *dev = drm->dev;
 	int ret;
 
 	mem->bus.addr = NULL;
@@ -1176,48 +1274,28 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 		return 0;
 	case TTM_PL_TT:
 #if __OS_HAS_AGP
-		if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
+		if (drm->agp.stat == ENABLED) {
 			mem->bus.offset = mem->start << PAGE_SHIFT;
-			mem->bus.base = dev_priv->gart_info.aper_base;
+			mem->bus.base = drm->agp.base;
 			mem->bus.is_iomem = true;
 		}
 #endif
 		break;
 	case TTM_PL_VRAM:
-	{
-		struct nouveau_mem *node = mem->mm_node;
-		u8 page_shift;
-
-		if (!dev_priv->bar1_vm) {
-			mem->bus.offset = mem->start << PAGE_SHIFT;
-			mem->bus.base = pci_resource_start(dev->pdev, 1);
-			mem->bus.is_iomem = true;
-			break;
-		}
-
-		if (dev_priv->card_type >= NV_C0)
-			page_shift = node->page_shift;
-		else
-			page_shift = 12;
+		mem->bus.offset = mem->start << PAGE_SHIFT;
+		mem->bus.base = pci_resource_start(dev->pdev, 1);
+		mem->bus.is_iomem = true;
+		if (nv_device(drm->device)->card_type >= NV_50) {
+			struct nouveau_bar *bar = nouveau_bar(drm->device);
+			struct nouveau_mem *node = mem->mm_node;
 
-		ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
-				     page_shift, NV_MEM_ACCESS_RW,
-				     &node->bar_vma);
-		if (ret)
-			return ret;
+			ret = bar->umap(bar, node, NV_MEM_ACCESS_RW,
+					&node->bar_vma);
+			if (ret)
+				return ret;
 
-		nouveau_vm_map(&node->bar_vma, node);
-		if (ret) {
-			nouveau_vm_put(&node->bar_vma);
-			return ret;
+			mem->bus.offset = node->bar_vma.offset;
 		}
-
-		mem->bus.offset = node->bar_vma.offset;
-		if (dev_priv->card_type == NV_50) /*XXX*/
-			mem->bus.offset -= 0x0020000000ULL;
-		mem->bus.base = pci_resource_start(dev->pdev, 1);
-		mem->bus.is_iomem = true;
-	}
 		break;
 	default:
 		return -EINVAL;
@@ -1228,41 +1306,40 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 static void
 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
+	struct nouveau_drm *drm = nouveau_bdev(bdev);
+	struct nouveau_bar *bar = nouveau_bar(drm->device);
 	struct nouveau_mem *node = mem->mm_node;
 
-	if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
-		return;
-
 	if (!node->bar_vma.node)
 		return;
 
-	nouveau_vm_unmap(&node->bar_vma);
-	nouveau_vm_put(&node->bar_vma);
+	bar->unmap(bar, &node->bar_vma);
 }
 
 static int
 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
+	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
+	struct nouveau_device *device = nv_device(drm->device);
+	u32 mappable = pci_resource_len(device->pdev, 1) >> PAGE_SHIFT;
 
 	/* as long as the bo isn't in vram, and isn't tiled, we've got
 	 * nothing to do here.
 	 */
 	if (bo->mem.mem_type != TTM_PL_VRAM) {
-		if (dev_priv->card_type < NV_50 ||
+		if (nv_device(drm->device)->card_type < NV_50 ||
 		    !nouveau_bo_tile_layout(nvbo))
 			return 0;
 	}
 
 	/* make sure bo is in mappable vram */
-	if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
+	if (bo->mem.start + bo->mem.num_pages < mappable)
 		return 0;
 
 
 	nvbo->placement.fpfn = 0;
-	nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
+	nvbo->placement.lpfn = mappable;
 	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
 	return nouveau_bo_validate(nvbo, false, true, false);
 }
@@ -1271,7 +1348,7 @@ static int
 nouveau_ttm_tt_populate(struct ttm_tt *ttm)
 {
 	struct ttm_dma_tt *ttm_dma = (void *)ttm;
-	struct drm_nouveau_private *dev_priv;
+	struct nouveau_drm *drm;
 	struct drm_device *dev;
 	unsigned i;
 	int r;
@@ -1288,11 +1365,11 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
 		return 0;
 	}
 
-	dev_priv = nouveau_bdev(ttm->bdev);
-	dev = dev_priv->dev;
+	drm = nouveau_bdev(ttm->bdev);
+	dev = drm->dev;
 
 #if __OS_HAS_AGP
-	if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
+	if (drm->agp.stat == ENABLED) {
 		return ttm_agp_tt_populate(ttm);
 	}
 #endif
@@ -1329,7 +1406,7 @@ static void
 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
 {
 	struct ttm_dma_tt *ttm_dma = (void *)ttm;
-	struct drm_nouveau_private *dev_priv;
+	struct nouveau_drm *drm;
 	struct drm_device *dev;
 	unsigned i;
 	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
@@ -1337,11 +1414,11 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
 	if (slave)
 		return;
 
-	dev_priv = nouveau_bdev(ttm->bdev);
-	dev = dev_priv->dev;
+	drm = nouveau_bdev(ttm->bdev);
+	dev = drm->dev;
 
 #if __OS_HAS_AGP
-	if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
+	if (drm->agp.stat == ENABLED) {
 		ttm_agp_tt_unpopulate(ttm);
 		return;
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h
new file mode 100644
index 00000000000..dec51b1098f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.h
@@ -0,0 +1,99 @@
+#ifndef __NOUVEAU_BO_H__
+#define __NOUVEAU_BO_H__
+
+struct nouveau_channel;
+struct nouveau_fence;
+struct nouveau_vma;
+
+struct nouveau_bo {
+	struct ttm_buffer_object bo;
+	struct ttm_placement placement;
+	u32 valid_domains;
+	u32 placements[3];
+	u32 busy_placements[3];
+	struct ttm_bo_kmap_obj kmap;
+	struct list_head head;
+
+	/* protected by ttm_bo_reserve() */
+	struct drm_file *reserved_by;
+	struct list_head entry;
+	int pbbo_index;
+	bool validate_mapped;
+
+	struct list_head vma_list;
+	unsigned page_shift;
+
+	u32 tile_mode;
+	u32 tile_flags;
+	struct nouveau_drm_tile *tile;
+
+	struct drm_gem_object *gem;
+	int pin_refcnt;
+
+	struct ttm_bo_kmap_obj dma_buf_vmap;
+	int vmapping_count;
+};
+
+static inline struct nouveau_bo *
+nouveau_bo(struct ttm_buffer_object *bo)
+{
+	return container_of(bo, struct nouveau_bo, bo);
+}
+
+static inline int
+nouveau_bo_ref(struct nouveau_bo *ref, struct nouveau_bo **pnvbo)
+{
+	struct nouveau_bo *prev;
+
+	if (!pnvbo)
+		return -EINVAL;
+	prev = *pnvbo;
+
+	*pnvbo = ref ? nouveau_bo(ttm_bo_reference(&ref->bo)) : NULL;
+	if (prev) {
+		struct ttm_buffer_object *bo = &prev->bo;
+
+		ttm_bo_unref(&bo);
+	}
+
+	return 0;
+}
+
+extern struct ttm_bo_driver nouveau_bo_driver;
+
+void nouveau_bo_move_init(struct nouveau_drm *);
+int  nouveau_bo_new(struct drm_device *, int size, int align, u32 flags,
+		    u32 tile_mode, u32 tile_flags, struct sg_table *sg,
+		    struct nouveau_bo **);
+int  nouveau_bo_pin(struct nouveau_bo *, u32 flags);
+int  nouveau_bo_unpin(struct nouveau_bo *);
+int  nouveau_bo_map(struct nouveau_bo *);
+void nouveau_bo_unmap(struct nouveau_bo *);
+void nouveau_bo_placement_set(struct nouveau_bo *, u32 type, u32 busy);
+u16  nouveau_bo_rd16(struct nouveau_bo *, unsigned index);
+void nouveau_bo_wr16(struct nouveau_bo *, unsigned index, u16 val);
+u32  nouveau_bo_rd32(struct nouveau_bo *, unsigned index);
+void nouveau_bo_wr32(struct nouveau_bo *, unsigned index, u32 val);
+void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
+int  nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
+			 bool no_wait_reserve, bool no_wait_gpu);
+
+struct nouveau_vma *
+nouveau_bo_vma_find(struct nouveau_bo *, struct nouveau_vm *);
+
+int  nouveau_bo_vma_add(struct nouveau_bo *, struct nouveau_vm *,
+			struct nouveau_vma *);
+void nouveau_bo_vma_del(struct nouveau_bo *, struct nouveau_vma *);
+
+/* TODO: submit equivalent to TTM generic API upstream? */
+static inline void __iomem *
+nvbo_kmap_obj_iovirtual(struct nouveau_bo *nvbo)
+{
+	bool is_iomem;
+	void __iomem *ioptr = (void __force __iomem *)ttm_kmap_obj_virtual(
+						&nvbo->kmap, &is_iomem);
+	WARN_ON_ONCE(ioptr && !is_iomem);
+	return ioptr;
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_calc.c b/drivers/gpu/drm/nouveau/nouveau_calc.c
index dad96cce5e3..6da576445b3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_calc.c
+++ b/drivers/gpu/drm/nouveau/nouveau_calc.c
@@ -21,8 +21,10 @@
  * SOFTWARE.
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_hw.h"
 
 /****************************************************************************\
@@ -195,12 +197,13 @@ static void
 nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
 		int *burst, int *lwm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nv_fifo_info fifo_data;
 	struct nv_sim_state sim_data;
 	int MClk = nouveau_hw_get_clock(dev, PLL_MEMORY);
 	int NVClk = nouveau_hw_get_clock(dev, PLL_CORE);
-	uint32_t cfg1 = nvReadFB(dev, NV04_PFB_CFG1);
+	uint32_t cfg1 = nv_rd32(device, NV04_PFB_CFG1);
 
 	sim_data.pclk_khz = VClk;
 	sim_data.mclk_khz = MClk;
@@ -218,13 +221,13 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
 		sim_data.mem_latency = 3;
 		sim_data.mem_page_miss = 10;
 	} else {
-		sim_data.memory_type = nvReadFB(dev, NV04_PFB_CFG0) & 0x1;
-		sim_data.memory_width = (nvReadEXTDEV(dev, NV_PEXTDEV_BOOT_0) & 0x10) ? 128 : 64;
+		sim_data.memory_type = nv_rd32(device, NV04_PFB_CFG0) & 0x1;
+		sim_data.memory_width = (nv_rd32(device, NV_PEXTDEV_BOOT_0) & 0x10) ? 128 : 64;
 		sim_data.mem_latency = cfg1 & 0xf;
 		sim_data.mem_page_miss = ((cfg1 >> 4) & 0xf) + ((cfg1 >> 31) & 0x1);
 	}
 
-	if (dev_priv->card_type == NV_04)
+	if (nv_device(drm->device)->card_type == NV_04)
 		nv04_calc_arb(&fifo_data, &sim_data);
 	else
 		nv10_calc_arb(&fifo_data, &sim_data);
@@ -249,9 +252,9 @@ nv20_update_arb(int *burst, int *lwm)
 void
 nouveau_calc_arb(struct drm_device *dev, int vclk, int bpp, int *burst, int *lwm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	if (dev_priv->card_type < NV_20)
+	if (nv_device(drm->device)->card_type < NV_20)
 		nv04_update_arb(dev, vclk, bpp, burst, lwm);
 	else if ((dev->pci_device & 0xfff0) == 0x0240 /*CHIPSET_C51*/ ||
 		 (dev->pci_device & 0xfff0) == 0x03d0 /*CHIPSET_C512*/) {
@@ -260,219 +263,3 @@ nouveau_calc_arb(struct drm_device *dev, int vclk, int bpp, int *burst, int *lwm
 	} else
 		nv20_update_arb(burst, lwm);
 }
-
-static int
-getMNP_single(struct drm_device *dev, struct pll_lims *pll_lim, int clk,
-	      struct nouveau_pll_vals *bestpv)
-{
-	/* Find M, N and P for a single stage PLL
-	 *
-	 * Note that some bioses (NV3x) have lookup tables of precomputed MNP
-	 * values, but we're too lazy to use those atm
-	 *
-	 * "clk" parameter in kHz
-	 * returns calculated clock
-	 */
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int cv = dev_priv->vbios.chip_version;
-	int minvco = pll_lim->vco1.minfreq, maxvco = pll_lim->vco1.maxfreq;
-	int minM = pll_lim->vco1.min_m, maxM = pll_lim->vco1.max_m;
-	int minN = pll_lim->vco1.min_n, maxN = pll_lim->vco1.max_n;
-	int minU = pll_lim->vco1.min_inputfreq;
-	int maxU = pll_lim->vco1.max_inputfreq;
-	int minP = pll_lim->max_p ? pll_lim->min_p : 0;
-	int maxP = pll_lim->max_p ? pll_lim->max_p : pll_lim->max_usable_log2p;
-	int crystal = pll_lim->refclk;
-	int M, N, thisP, P;
-	int clkP, calcclk;
-	int delta, bestdelta = INT_MAX;
-	int bestclk = 0;
-
-	/* this division verified for nv20, nv18, nv28 (Haiku), and nv34 */
-	/* possibly correlated with introduction of 27MHz crystal */
-	if (dev_priv->card_type < NV_50) {
-		if (cv < 0x17 || cv == 0x1a || cv == 0x20) {
-			if (clk > 250000)
-				maxM = 6;
-			if (clk > 340000)
-				maxM = 2;
-		} else if (cv < 0x40) {
-			if (clk > 150000)
-				maxM = 6;
-			if (clk > 200000)
-				maxM = 4;
-			if (clk > 340000)
-				maxM = 2;
-		}
-	}
-
-	P = pll_lim->max_p ? maxP : (1 << maxP);
-	if ((clk * P) < minvco) {
-		minvco = clk * maxP;
-		maxvco = minvco * 2;
-	}
-
-	if (clk + clk/200 > maxvco)	/* +0.5% */
-		maxvco = clk + clk/200;
-
-	/* NV34 goes maxlog2P->0, NV20 goes 0->maxlog2P */
-	for (thisP = minP; thisP <= maxP; thisP++) {
-		P = pll_lim->max_p ? thisP : (1 << thisP);
-		clkP = clk * P;
-
-		if (clkP < minvco)
-			continue;
-		if (clkP > maxvco)
-			return bestclk;
-
-		for (M = minM; M <= maxM; M++) {
-			if (crystal/M < minU)
-				return bestclk;
-			if (crystal/M > maxU)
-				continue;
-
-			/* add crystal/2 to round better */
-			N = (clkP * M + crystal/2) / crystal;
-
-			if (N < minN)
-				continue;
-			if (N > maxN)
-				break;
-
-			/* more rounding additions */
-			calcclk = ((N * crystal + P/2) / P + M/2) / M;
-			delta = abs(calcclk - clk);
-			/* we do an exhaustive search rather than terminating
-			 * on an optimality condition...
-			 */
-			if (delta < bestdelta) {
-				bestdelta = delta;
-				bestclk = calcclk;
-				bestpv->N1 = N;
-				bestpv->M1 = M;
-				bestpv->log2P = thisP;
-				if (delta == 0)	/* except this one */
-					return bestclk;
-			}
-		}
-	}
-
-	return bestclk;
-}
-
-static int
-getMNP_double(struct drm_device *dev, struct pll_lims *pll_lim, int clk,
-	      struct nouveau_pll_vals *bestpv)
-{
-	/* Find M, N and P for a two stage PLL
-	 *
-	 * Note that some bioses (NV30+) have lookup tables of precomputed MNP
-	 * values, but we're too lazy to use those atm
-	 *
-	 * "clk" parameter in kHz
-	 * returns calculated clock
-	 */
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chip_version = dev_priv->vbios.chip_version;
-	int minvco1 = pll_lim->vco1.minfreq, maxvco1 = pll_lim->vco1.maxfreq;
-	int minvco2 = pll_lim->vco2.minfreq, maxvco2 = pll_lim->vco2.maxfreq;
-	int minU1 = pll_lim->vco1.min_inputfreq, minU2 = pll_lim->vco2.min_inputfreq;
-	int maxU1 = pll_lim->vco1.max_inputfreq, maxU2 = pll_lim->vco2.max_inputfreq;
-	int minM1 = pll_lim->vco1.min_m, maxM1 = pll_lim->vco1.max_m;
-	int minN1 = pll_lim->vco1.min_n, maxN1 = pll_lim->vco1.max_n;
-	int minM2 = pll_lim->vco2.min_m, maxM2 = pll_lim->vco2.max_m;
-	int minN2 = pll_lim->vco2.min_n, maxN2 = pll_lim->vco2.max_n;
-	int maxlog2P = pll_lim->max_usable_log2p;
-	int crystal = pll_lim->refclk;
-	bool fixedgain2 = (minM2 == maxM2 && minN2 == maxN2);
-	int M1, N1, M2, N2, log2P;
-	int clkP, calcclk1, calcclk2, calcclkout;
-	int delta, bestdelta = INT_MAX;
-	int bestclk = 0;
-
-	int vco2 = (maxvco2 - maxvco2/200) / 2;
-	for (log2P = 0; clk && log2P < maxlog2P && clk <= (vco2 >> log2P); log2P++)
-		;
-	clkP = clk << log2P;
-
-	if (maxvco2 < clk + clk/200)	/* +0.5% */
-		maxvco2 = clk + clk/200;
-
-	for (M1 = minM1; M1 <= maxM1; M1++) {
-		if (crystal/M1 < minU1)
-			return bestclk;
-		if (crystal/M1 > maxU1)
-			continue;
-
-		for (N1 = minN1; N1 <= maxN1; N1++) {
-			calcclk1 = crystal * N1 / M1;
-			if (calcclk1 < minvco1)
-				continue;
-			if (calcclk1 > maxvco1)
-				break;
-
-			for (M2 = minM2; M2 <= maxM2; M2++) {
-				if (calcclk1/M2 < minU2)
-					break;
-				if (calcclk1/M2 > maxU2)
-					continue;
-
-				/* add calcclk1/2 to round better */
-				N2 = (clkP * M2 + calcclk1/2) / calcclk1;
-				if (N2 < minN2)
-					continue;
-				if (N2 > maxN2)
-					break;
-
-				if (!fixedgain2) {
-					if (chip_version < 0x60)
-						if (N2/M2 < 4 || N2/M2 > 10)
-							continue;
-
-					calcclk2 = calcclk1 * N2 / M2;
-					if (calcclk2 < minvco2)
-						break;
-					if (calcclk2 > maxvco2)
-						continue;
-				} else
-					calcclk2 = calcclk1;
-
-				calcclkout = calcclk2 >> log2P;
-				delta = abs(calcclkout - clk);
-				/* we do an exhaustive search rather than terminating
-				 * on an optimality condition...
-				 */
-				if (delta < bestdelta) {
-					bestdelta = delta;
-					bestclk = calcclkout;
-					bestpv->N1 = N1;
-					bestpv->M1 = M1;
-					bestpv->N2 = N2;
-					bestpv->M2 = M2;
-					bestpv->log2P = log2P;
-					if (delta == 0)	/* except this one */
-						return bestclk;
-				}
-			}
-		}
-	}
-
-	return bestclk;
-}
-
-int
-nouveau_calc_pll_mnp(struct drm_device *dev, struct pll_lims *pll_lim, int clk,
-		     struct nouveau_pll_vals *pv)
-{
-	int outclk;
-
-	if (!pll_lim->vco2.maxfreq)
-		outclk = getMNP_single(dev, pll_lim, clk, pv);
-	else
-		outclk = getMNP_double(dev, pll_lim, clk, pv);
-
-	if (!outclk)
-		NV_ERROR(dev, "Could not find a compatible set of PLL values\n");
-
-	return outclk;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
new file mode 100644
index 00000000000..c1d7301c0e9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -0,0 +1,400 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/object.h>
+#include <core/client.h>
+#include <core/device.h>
+#include <core/class.h>
+
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/instmem.h>
+
+#include <engine/software.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
+#include "nouveau_bo.h"
+#include "nouveau_chan.h"
+#include "nouveau_fence.h"
+#include "nouveau_abi16.h"
+
+MODULE_PARM_DESC(vram_pushbuf, "Create DMA push buffers in VRAM");
+static int nouveau_vram_pushbuf;
+module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400);
+
+int
+nouveau_channel_idle(struct nouveau_channel *chan)
+{
+	struct nouveau_cli *cli = chan->cli;
+	struct nouveau_fence *fence = NULL;
+	int ret;
+
+	ret = nouveau_fence_new(chan, &fence);
+	if (!ret) {
+		ret = nouveau_fence_wait(fence, false, false);
+		nouveau_fence_unref(&fence);
+	}
+
+	if (ret)
+		NV_ERROR(cli, "failed to idle channel 0x%08x\n", chan->handle);
+	return ret;
+}
+
+void
+nouveau_channel_del(struct nouveau_channel **pchan)
+{
+	struct nouveau_channel *chan = *pchan;
+	if (chan) {
+		struct nouveau_object *client = nv_object(chan->cli);
+		if (chan->fence) {
+			nouveau_channel_idle(chan);
+			nouveau_fence(chan->drm)->context_del(chan);
+		}
+		nouveau_object_del(client, NVDRM_DEVICE, chan->handle);
+		nouveau_object_del(client, NVDRM_DEVICE, chan->push.handle);
+		nouveau_bo_vma_del(chan->push.buffer, &chan->push.vma);
+		nouveau_bo_unmap(chan->push.buffer);
+		nouveau_bo_ref(NULL, &chan->push.buffer);
+		kfree(chan);
+	}
+	*pchan = NULL;
+}
+
+static int
+nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
+		     u32 parent, u32 handle, u32 size,
+		     struct nouveau_channel **pchan)
+{
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_instmem *imem = nouveau_instmem(device);
+	struct nouveau_vmmgr *vmm = nouveau_vmmgr(device);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	struct nouveau_client *client = &cli->base;
+	struct nv_dma_class args = {};
+	struct nouveau_channel *chan;
+	struct nouveau_object *push;
+	u32 target;
+	int ret;
+
+	chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return -ENOMEM;
+
+	chan->cli = cli;
+	chan->drm = drm;
+	chan->handle = handle;
+
+	/* allocate memory for dma push buffer */
+	target = TTM_PL_FLAG_TT;
+	if (nouveau_vram_pushbuf)
+		target = TTM_PL_FLAG_VRAM;
+
+	ret = nouveau_bo_new(drm->dev, size, 0, target, 0, 0, NULL,
+			    &chan->push.buffer);
+	if (ret == 0) {
+		ret = nouveau_bo_pin(chan->push.buffer, target);
+		if (ret == 0)
+			ret = nouveau_bo_map(chan->push.buffer);
+	}
+
+	if (ret) {
+		nouveau_channel_del(pchan);
+		return ret;
+	}
+
+	/* create dma object covering the *entire* memory space that the
+	 * pushbuf lives in, this is because the GEM code requires that
+	 * we be able to call out to other (indirect) push buffers
+	 */
+	chan->push.vma.offset = chan->push.buffer->bo.offset;
+	chan->push.handle = NVDRM_PUSH | (handle & 0xffff);
+
+	if (device->card_type >= NV_50) {
+		ret = nouveau_bo_vma_add(chan->push.buffer, client->vm,
+					&chan->push.vma);
+		if (ret) {
+			nouveau_channel_del(pchan);
+			return ret;
+		}
+
+		args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+		args.start = 0;
+		args.limit = client->vm->vmm->limit - 1;
+	} else
+	if (chan->push.buffer->bo.mem.mem_type == TTM_PL_VRAM) {
+		u64 limit = pfb->ram.size - imem->reserved - 1;
+		if (device->card_type == NV_04) {
+			/* nv04 vram pushbuf hack, retarget to its location in
+			 * the framebuffer bar rather than direct vram access..
+			 * nfi why this exists, it came from the -nv ddx.
+			 */
+			args.flags = NV_DMA_TARGET_PCI | NV_DMA_ACCESS_RDWR;
+			args.start = pci_resource_start(device->pdev, 1);
+			args.limit = args.start + limit;
+		} else {
+			args.flags = NV_DMA_TARGET_VRAM | NV_DMA_ACCESS_RDWR;
+			args.start = 0;
+			args.limit = limit;
+		}
+	} else {
+		if (chan->drm->agp.stat == ENABLED) {
+			args.flags = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
+			args.start = chan->drm->agp.base;
+			args.limit = chan->drm->agp.base +
+				     chan->drm->agp.size - 1;
+		} else {
+			args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
+			args.start = 0;
+			args.limit = vmm->limit - 1;
+		}
+	}
+
+	ret = nouveau_object_new(nv_object(chan->cli), parent,
+				 chan->push.handle, 0x0002,
+				 &args, sizeof(args), &push);
+	if (ret) {
+		nouveau_channel_del(pchan);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+nouveau_channel_ind(struct nouveau_drm *drm, struct nouveau_cli *cli,
+		    u32 parent, u32 handle, u32 engine,
+		    struct nouveau_channel **pchan)
+{
+	static const u16 oclasses[] = { NVE0_CHANNEL_IND_CLASS,
+					NVC0_CHANNEL_IND_CLASS,
+					NV84_CHANNEL_IND_CLASS,
+					NV50_CHANNEL_IND_CLASS,
+					0 };
+	const u16 *oclass = oclasses;
+	struct nve0_channel_ind_class args;
+	struct nouveau_channel *chan;
+	int ret;
+
+	/* allocate dma push buffer */
+	ret = nouveau_channel_prep(drm, cli, parent, handle, 0x12000, &chan);
+	*pchan = chan;
+	if (ret)
+		return ret;
+
+	/* create channel object */
+	args.pushbuf = chan->push.handle;
+	args.ioffset = 0x10000 + chan->push.vma.offset;
+	args.ilength = 0x02000;
+	args.engine  = engine;
+
+	do {
+		ret = nouveau_object_new(nv_object(cli), parent, handle,
+					 *oclass++, &args, sizeof(args),
+					 &chan->object);
+		if (ret == 0)
+			return ret;
+	} while (*oclass);
+
+	nouveau_channel_del(pchan);
+	return ret;
+}
+
+static int
+nouveau_channel_dma(struct nouveau_drm *drm, struct nouveau_cli *cli,
+		    u32 parent, u32 handle, struct nouveau_channel **pchan)
+{
+	static const u16 oclasses[] = { NV40_CHANNEL_DMA_CLASS,
+					NV17_CHANNEL_DMA_CLASS,
+					NV10_CHANNEL_DMA_CLASS,
+					NV03_CHANNEL_DMA_CLASS,
+					0 };
+	const u16 *oclass = oclasses;
+	struct nv03_channel_dma_class args;
+	struct nouveau_channel *chan;
+	int ret;
+
+	/* allocate dma push buffer */
+	ret = nouveau_channel_prep(drm, cli, parent, handle, 0x10000, &chan);
+	*pchan = chan;
+	if (ret)
+		return ret;
+
+	/* create channel object */
+	args.pushbuf = chan->push.handle;
+	args.offset = chan->push.vma.offset;
+
+	do {
+		ret = nouveau_object_new(nv_object(cli), parent, handle,
+					 *oclass++, &args, sizeof(args),
+					 &chan->object);
+		if (ret == 0)
+			return ret;
+	} while (ret && *oclass);
+
+	nouveau_channel_del(pchan);
+	return ret;
+}
+
+static int
+nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart)
+{
+	struct nouveau_client *client = nv_client(chan->cli);
+	struct nouveau_device *device = nv_device(chan->drm->device);
+	struct nouveau_instmem *imem = nouveau_instmem(device);
+	struct nouveau_vmmgr *vmm = nouveau_vmmgr(device);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	struct nouveau_software_chan *swch;
+	struct nouveau_object *object;
+	struct nv_dma_class args;
+	int ret, i;
+
+	/* allocate dma objects to cover all allowed vram, and gart */
+	if (device->card_type < NV_C0) {
+		if (device->card_type >= NV_50) {
+			args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+			args.start = 0;
+			args.limit = client->vm->vmm->limit - 1;
+		} else {
+			args.flags = NV_DMA_TARGET_VRAM | NV_DMA_ACCESS_RDWR;
+			args.start = 0;
+			args.limit = pfb->ram.size - imem->reserved - 1;
+		}
+
+		ret = nouveau_object_new(nv_object(client), chan->handle, vram,
+					 0x003d, &args, sizeof(args), &object);
+		if (ret)
+			return ret;
+
+		if (device->card_type >= NV_50) {
+			args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_VM;
+			args.start = 0;
+			args.limit = client->vm->vmm->limit - 1;
+		} else
+		if (chan->drm->agp.stat == ENABLED) {
+			args.flags = NV_DMA_TARGET_AGP | NV_DMA_ACCESS_RDWR;
+			args.start = chan->drm->agp.base;
+			args.limit = chan->drm->agp.base +
+				     chan->drm->agp.size - 1;
+		} else {
+			args.flags = NV_DMA_TARGET_VM | NV_DMA_ACCESS_RDWR;
+			args.start = 0;
+			args.limit = vmm->limit - 1;
+		}
+
+		ret = nouveau_object_new(nv_object(client), chan->handle, gart,
+					 0x003d, &args, sizeof(args), &object);
+		if (ret)
+			return ret;
+
+		chan->vram = vram;
+		chan->gart = gart;
+	}
+
+	/* initialise dma tracking parameters */
+	switch (nv_hclass(chan->object) & 0x00ff) {
+	case 0x006b:
+	case 0x006e:
+		chan->user_put = 0x40;
+		chan->user_get = 0x44;
+		chan->dma.max = (0x10000 / 4) - 2;
+		break;
+	default:
+		chan->user_put = 0x40;
+		chan->user_get = 0x44;
+		chan->user_get_hi = 0x60;
+		chan->dma.ib_base =  0x10000 / 4;
+		chan->dma.ib_max  = (0x02000 / 8) - 1;
+		chan->dma.ib_put  = 0;
+		chan->dma.ib_free = chan->dma.ib_max - chan->dma.ib_put;
+		chan->dma.max = chan->dma.ib_base;
+		break;
+	}
+
+	chan->dma.put = 0;
+	chan->dma.cur = chan->dma.put;
+	chan->dma.free = chan->dma.max - chan->dma.cur;
+
+	ret = RING_SPACE(chan, NOUVEAU_DMA_SKIPS);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
+		OUT_RING(chan, 0x00000000);
+
+	/* allocate software object class (used for fences on <= nv05, and
+	 * to signal flip completion), bind it to a subchannel.
+	 */
+	if (chan != chan->drm->cechan) {
+		ret = nouveau_object_new(nv_object(client), chan->handle,
+					 NvSw, nouveau_abi16_swclass(chan->drm),
+					 NULL, 0, &object);
+		if (ret)
+			return ret;
+
+		swch = (void *)object->parent;
+		swch->flip = nouveau_flip_complete;
+		swch->flip_data = chan;
+	}
+
+	if (device->card_type < NV_C0) {
+		ret = RING_SPACE(chan, 2);
+		if (ret)
+			return ret;
+
+		BEGIN_NV04(chan, NvSubSw, 0x0000, 1);
+		OUT_RING  (chan, NvSw);
+		FIRE_RING (chan);
+	}
+
+	/* initialise synchronisation */
+	return nouveau_fence(chan->drm)->context_new(chan);
+}
+
+int
+nouveau_channel_new(struct nouveau_drm *drm, struct nouveau_cli *cli,
+		    u32 parent, u32 handle, u32 arg0, u32 arg1,
+		    struct nouveau_channel **pchan)
+{
+	int ret;
+
+	ret = nouveau_channel_ind(drm, cli, parent, handle, arg0, pchan);
+	if (ret) {
+		NV_DEBUG(cli, "ib channel create, %d\n", ret);
+		ret = nouveau_channel_dma(drm, cli, parent, handle, pchan);
+		if (ret) {
+			NV_DEBUG(cli, "dma channel create, %d\n", ret);
+			return ret;
+		}
+	}
+
+	ret = nouveau_channel_init(*pchan, arg0, arg1);
+	if (ret) {
+		NV_ERROR(cli, "channel failed to initialise, %d\n", ret);
+		nouveau_channel_del(pchan);
+		return ret;
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h
new file mode 100644
index 00000000000..40f97e2c47b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.h
@@ -0,0 +1,47 @@
+#ifndef __NOUVEAU_CHAN_H__
+#define __NOUVEAU_CHAN_H__
+
+struct nouveau_cli;
+
+struct nouveau_channel {
+	struct nouveau_cli *cli;
+	struct nouveau_drm *drm;
+
+	u32 handle;
+	u32 vram;
+	u32 gart;
+
+	struct {
+		struct nouveau_bo *buffer;
+		struct nouveau_vma vma;
+		u32 handle;
+	} push;
+
+	/* TODO: this will be reworked in the near future */
+	bool accel_done;
+	void *fence;
+	struct {
+		int max;
+		int free;
+		int cur;
+		int put;
+		int ib_base;
+		int ib_max;
+		int ib_free;
+		int ib_put;
+	} dma;
+	u32 user_get_hi;
+	u32 user_get;
+	u32 user_put;
+
+	struct nouveau_object *object;
+};
+
+
+int  nouveau_channel_new(struct nouveau_drm *, struct nouveau_cli *,
+			 u32 parent, u32 handle, u32 arg0, u32 arg1,
+			 struct nouveau_channel **);
+void nouveau_channel_del(struct nouveau_channel **);
+int  nouveau_channel_idle(struct nouveau_channel *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_channel.c b/drivers/gpu/drm/nouveau/nouveau_channel.c
deleted file mode 100644
index debd90225a8..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_channel.c
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Copyright 2005-2006 Stephane Marchesin
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_dma.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-#include "nouveau_fence.h"
-#include "nouveau_software.h"
-
-static int
-nouveau_channel_pushbuf_init(struct nouveau_channel *chan)
-{
-	u32 mem = nouveau_vram_pushbuf ? TTM_PL_FLAG_VRAM : TTM_PL_FLAG_TT;
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int ret;
-
-	/* allocate buffer object */
-	ret = nouveau_bo_new(dev, 65536, 0, mem, 0, 0, NULL, &chan->pushbuf_bo);
-	if (ret)
-		goto out;
-
-	ret = nouveau_bo_pin(chan->pushbuf_bo, mem);
-	if (ret)
-		goto out;
-
-	ret = nouveau_bo_map(chan->pushbuf_bo);
-	if (ret)
-		goto out;
-
-	/* create DMA object covering the entire memtype where the push
-	 * buffer resides, userspace can submit its own push buffers from
-	 * anywhere within the same memtype.
-	 */
-	chan->pushbuf_base = chan->pushbuf_bo->bo.offset;
-	if (dev_priv->card_type >= NV_50) {
-		ret = nouveau_bo_vma_add(chan->pushbuf_bo, chan->vm,
-					 &chan->pushbuf_vma);
-		if (ret)
-			goto out;
-
-		if (dev_priv->card_type < NV_C0) {
-			ret = nouveau_gpuobj_dma_new(chan,
-						     NV_CLASS_DMA_IN_MEMORY, 0,
-						     (1ULL << 40),
-						     NV_MEM_ACCESS_RO,
-						     NV_MEM_TARGET_VM,
-						     &chan->pushbuf);
-		}
-		chan->pushbuf_base = chan->pushbuf_vma.offset;
-	} else
-	if (chan->pushbuf_bo->bo.mem.mem_type == TTM_PL_TT) {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
-					     dev_priv->gart_info.aper_size,
-					     NV_MEM_ACCESS_RO,
-					     NV_MEM_TARGET_GART,
-					     &chan->pushbuf);
-	} else
-	if (dev_priv->card_type != NV_04) {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, 0,
-					     dev_priv->fb_available_size,
-					     NV_MEM_ACCESS_RO,
-					     NV_MEM_TARGET_VRAM,
-					     &chan->pushbuf);
-	} else {
-		/* NV04 cmdbuf hack, from original ddx.. not sure of it's
-		 * exact reason for existing :)  PCI access to cmdbuf in
-		 * VRAM.
-		 */
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     pci_resource_start(dev->pdev, 1),
-					     dev_priv->fb_available_size,
-					     NV_MEM_ACCESS_RO,
-					     NV_MEM_TARGET_PCI,
-					     &chan->pushbuf);
-	}
-
-out:
-	if (ret) {
-		NV_ERROR(dev, "error initialising pushbuf: %d\n", ret);
-		nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
-		nouveau_gpuobj_ref(NULL, &chan->pushbuf);
-		if (chan->pushbuf_bo) {
-			nouveau_bo_unmap(chan->pushbuf_bo);
-			nouveau_bo_ref(NULL, &chan->pushbuf_bo);
-		}
-	}
-
-	return 0;
-}
-
-/* allocates and initializes a fifo for user space consumption */
-int
-nouveau_channel_alloc(struct drm_device *dev, struct nouveau_channel **chan_ret,
-		      struct drm_file *file_priv,
-		      uint32_t vram_handle, uint32_t gart_handle)
-{
-	struct nouveau_exec_engine *fence = nv_engine(dev, NVOBJ_ENGINE_FENCE);
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
-	struct nouveau_channel *chan;
-	unsigned long flags;
-	int ret, i;
-
-	/* allocate and lock channel structure */
-	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
-	if (!chan)
-		return -ENOMEM;
-	chan->dev = dev;
-	chan->file_priv = file_priv;
-	chan->vram_handle = vram_handle;
-	chan->gart_handle = gart_handle;
-
-	kref_init(&chan->ref);
-	atomic_set(&chan->users, 1);
-	mutex_init(&chan->mutex);
-	mutex_lock(&chan->mutex);
-
-	/* allocate hw channel id */
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (chan->id = 0; chan->id < pfifo->channels; chan->id++) {
-		if (!dev_priv->channels.ptr[chan->id]) {
-			nouveau_channel_ref(chan, &dev_priv->channels.ptr[chan->id]);
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-
-	if (chan->id == pfifo->channels) {
-		mutex_unlock(&chan->mutex);
-		kfree(chan);
-		return -ENODEV;
-	}
-
-	NV_DEBUG(dev, "initialising channel %d\n", chan->id);
-
-	/* setup channel's memory and vm */
-	ret = nouveau_gpuobj_channel_init(chan, vram_handle, gart_handle);
-	if (ret) {
-		NV_ERROR(dev, "gpuobj %d\n", ret);
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	/* Allocate space for per-channel fixed notifier memory */
-	ret = nouveau_notifier_init_channel(chan);
-	if (ret) {
-		NV_ERROR(dev, "ntfy %d\n", ret);
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	/* Allocate DMA push buffer */
-	ret = nouveau_channel_pushbuf_init(chan);
-	if (ret) {
-		NV_ERROR(dev, "pushbuf %d\n", ret);
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	nouveau_dma_init(chan);
-	chan->user_put = 0x40;
-	chan->user_get = 0x44;
-	if (dev_priv->card_type >= NV_50)
-		chan->user_get_hi = 0x60;
-
-	/* create fifo context */
-	ret = pfifo->base.context_new(chan, NVOBJ_ENGINE_FIFO);
-	if (ret) {
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	/* Insert NOPs for NOUVEAU_DMA_SKIPS */
-	ret = RING_SPACE(chan, NOUVEAU_DMA_SKIPS);
-	if (ret) {
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	for (i = 0; i < NOUVEAU_DMA_SKIPS; i++)
-		OUT_RING  (chan, 0x00000000);
-
-	ret = nouveau_gpuobj_gr_new(chan, NvSw, nouveau_software_class(dev));
-	if (ret) {
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	if (dev_priv->card_type < NV_C0) {
-		ret = RING_SPACE(chan, 2);
-		if (ret) {
-			nouveau_channel_put(&chan);
-			return ret;
-		}
-
-		BEGIN_NV04(chan, NvSubSw, NV01_SUBCHAN_OBJECT, 1);
-		OUT_RING  (chan, NvSw);
-		FIRE_RING (chan);
-	}
-
-	FIRE_RING(chan);
-
-	ret = fence->context_new(chan, NVOBJ_ENGINE_FENCE);
-	if (ret) {
-		nouveau_channel_put(&chan);
-		return ret;
-	}
-
-	nouveau_debugfs_channel_init(chan);
-
-	NV_DEBUG(dev, "channel %d initialised\n", chan->id);
-	if (fpriv) {
-		spin_lock(&fpriv->lock);
-		list_add(&chan->list, &fpriv->channels);
-		spin_unlock(&fpriv->lock);
-	}
-	*chan_ret = chan;
-	return 0;
-}
-
-struct nouveau_channel *
-nouveau_channel_get_unlocked(struct nouveau_channel *ref)
-{
-	struct nouveau_channel *chan = NULL;
-
-	if (likely(ref && atomic_inc_not_zero(&ref->users)))
-		nouveau_channel_ref(ref, &chan);
-
-	return chan;
-}
-
-struct nouveau_channel *
-nouveau_channel_get(struct drm_file *file_priv, int id)
-{
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
-	struct nouveau_channel *chan;
-
-	spin_lock(&fpriv->lock);
-	list_for_each_entry(chan, &fpriv->channels, list) {
-		if (chan->id == id) {
-			chan = nouveau_channel_get_unlocked(chan);
-			spin_unlock(&fpriv->lock);
-			mutex_lock(&chan->mutex);
-			return chan;
-		}
-	}
-	spin_unlock(&fpriv->lock);
-
-	return ERR_PTR(-EINVAL);
-}
-
-void
-nouveau_channel_put_unlocked(struct nouveau_channel **pchan)
-{
-	struct nouveau_channel *chan = *pchan;
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-	int i;
-
-	/* decrement the refcount, and we're done if there's still refs */
-	if (likely(!atomic_dec_and_test(&chan->users))) {
-		nouveau_channel_ref(NULL, pchan);
-		return;
-	}
-
-	/* no one wants the channel anymore */
-	NV_DEBUG(dev, "freeing channel %d\n", chan->id);
-	nouveau_debugfs_channel_fini(chan);
-
-	/* give it chance to idle */
-	nouveau_channel_idle(chan);
-
-	/* destroy the engine specific contexts */
-	for (i = NVOBJ_ENGINE_NR - 1; i >= 0; i--) {
-		if (chan->engctx[i])
-			dev_priv->eng[i]->context_del(chan, i);
-	}
-
-	/* aside from its resources, the channel should now be dead,
-	 * remove it from the channel list
-	 */
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	nouveau_channel_ref(NULL, &dev_priv->channels.ptr[chan->id]);
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-
-	/* destroy any resources the channel owned */
-	nouveau_gpuobj_ref(NULL, &chan->pushbuf);
-	if (chan->pushbuf_bo) {
-		nouveau_bo_vma_del(chan->pushbuf_bo, &chan->pushbuf_vma);
-		nouveau_bo_unmap(chan->pushbuf_bo);
-		nouveau_bo_unpin(chan->pushbuf_bo);
-		nouveau_bo_ref(NULL, &chan->pushbuf_bo);
-	}
-	nouveau_ramht_ref(NULL, &chan->ramht, chan);
-	nouveau_notifier_takedown_channel(chan);
-	nouveau_gpuobj_channel_takedown(chan);
-
-	nouveau_channel_ref(NULL, pchan);
-}
-
-void
-nouveau_channel_put(struct nouveau_channel **pchan)
-{
-	mutex_unlock(&(*pchan)->mutex);
-	nouveau_channel_put_unlocked(pchan);
-}
-
-static void
-nouveau_channel_del(struct kref *ref)
-{
-	struct nouveau_channel *chan =
-		container_of(ref, struct nouveau_channel, ref);
-
-	kfree(chan);
-}
-
-void
-nouveau_channel_ref(struct nouveau_channel *chan,
-		    struct nouveau_channel **pchan)
-{
-	if (chan)
-		kref_get(&chan->ref);
-
-	if (*pchan)
-		kref_put(&(*pchan)->ref, nouveau_channel_del);
-
-	*pchan = chan;
-}
-
-int
-nouveau_channel_idle(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_fence *fence = NULL;
-	int ret;
-
-	ret = nouveau_fence_new(chan, &fence);
-	if (!ret) {
-		ret = nouveau_fence_wait(fence, false, false);
-		nouveau_fence_unref(&fence);
-	}
-
-	if (ret)
-		NV_ERROR(dev, "Failed to idle channel %d.\n", chan->id);
-	return ret;
-}
-
-/* cleans up all the fifos from file_priv */
-void
-nouveau_channel_cleanup(struct drm_device *dev, struct drm_file *file_priv)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct nouveau_channel *chan;
-	int i;
-
-	if (!pfifo)
-		return;
-
-	NV_DEBUG(dev, "clearing FIFO enables from file_priv\n");
-	for (i = 0; i < pfifo->channels; i++) {
-		chan = nouveau_channel_get(file_priv, i);
-		if (IS_ERR(chan))
-			continue;
-
-		list_del(&chan->list);
-		atomic_dec(&chan->users);
-		nouveau_channel_put(&chan);
-	}
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 7b11edb077d..9a6e2cb282d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -26,17 +26,34 @@
 
 #include <acpi/button.h>
 
-#include "drmP.h"
-#include "drm_edid.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
 
 #include "nouveau_reg.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_hw.h"
+#include "nouveau_acpi.h"
+
+#include "nouveau_display.h"
+#include "nouveau_connector.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
-#include "nouveau_connector.h"
-#include "nouveau_gpio.h"
-#include "nouveau_hw.h"
+
+#include <subdev/i2c.h>
+#include <subdev/gpio.h>
+
+MODULE_PARM_DESC(tv_disable, "Disable TV-out detection");
+static int nouveau_tv_disable = 0;
+module_param_named(tv_disable, nouveau_tv_disable, int, 0400);
+
+MODULE_PARM_DESC(ignorelid, "Ignore ACPI lid status");
+static int nouveau_ignorelid = 0;
+module_param_named(ignorelid, nouveau_ignorelid, int, 0400);
+
+MODULE_PARM_DESC(duallink, "Allow dual-link TMDS (default: enabled)");
+static int nouveau_duallink = 1;
+module_param_named(duallink, nouveau_duallink, int, 0400);
 
 static void nouveau_connector_hotplug(void *, int);
 
@@ -58,7 +75,7 @@ find_encoder(struct drm_connector *connector, int type)
 			continue;
 		nv_encoder = nouveau_encoder(obj_to_encoder(obj));
 
-		if (type == OUTPUT_ANY || nv_encoder->dcb->type == type)
+		if (type == DCB_OUTPUT_ANY || nv_encoder->dcb->type == type)
 			return nv_encoder;
 	}
 
@@ -83,19 +100,21 @@ static void
 nouveau_connector_destroy(struct drm_connector *connector)
 {
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct drm_nouveau_private *dev_priv;
+	struct nouveau_gpio *gpio;
+	struct nouveau_drm *drm;
 	struct drm_device *dev;
 
 	if (!nv_connector)
 		return;
 
-	dev = nv_connector->base.dev;
-	dev_priv = dev->dev_private;
-	NV_DEBUG_KMS(dev, "\n");
+	dev  = nv_connector->base.dev;
+	drm  = nouveau_drm(dev);
+	gpio = nouveau_gpio(drm->device);
+	NV_DEBUG(drm, "\n");
 
-	if (nv_connector->hpd != DCB_GPIO_UNUSED) {
-		nouveau_gpio_isr_del(dev, 0, nv_connector->hpd, 0xff,
-				     nouveau_connector_hotplug, connector);
+	if (gpio && nv_connector->hpd != DCB_GPIO_UNUSED) {
+		gpio->isr_del(gpio, 0, nv_connector->hpd, 0xff,
+			      nouveau_connector_hotplug, connector);
 	}
 
 	kfree(nv_connector->edid);
@@ -104,15 +123,17 @@ nouveau_connector_destroy(struct drm_connector *connector)
 	kfree(connector);
 }
 
-static struct nouveau_i2c_chan *
+static struct nouveau_i2c_port *
 nouveau_connector_ddc_detect(struct drm_connector *connector,
 			     struct nouveau_encoder **pnv_encoder)
 {
 	struct drm_device *dev = connector->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
 	int i;
 
 	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-		struct nouveau_i2c_chan *i2c = NULL;
+		struct nouveau_i2c_port *port = NULL;
 		struct nouveau_encoder *nv_encoder;
 		struct drm_mode_object *obj;
 		int id;
@@ -127,11 +148,10 @@ nouveau_connector_ddc_detect(struct drm_connector *connector,
 		nv_encoder = nouveau_encoder(obj_to_encoder(obj));
 
 		if (nv_encoder->dcb->i2c_index < 0xf)
-			i2c = nouveau_i2c_find(dev, nv_encoder->dcb->i2c_index);
-
-		if (i2c && nouveau_probe_i2c_addr(i2c, 0x50)) {
+			port = i2c->find(i2c, nv_encoder->dcb->i2c_index);
+		if (port && nv_probe_i2c(port, 0x50)) {
 			*pnv_encoder = nv_encoder;
-			return i2c;
+			return port;
 		}
 	}
 
@@ -148,8 +168,8 @@ nouveau_connector_of_detect(struct drm_connector *connector)
 	struct device_node *cn, *dn = pci_device_to_OF_node(dev->pdev);
 
 	if (!dn ||
-	    !((nv_encoder = find_encoder(connector, OUTPUT_TMDS)) ||
-	      (nv_encoder = find_encoder(connector, OUTPUT_ANALOG))))
+	    !((nv_encoder = find_encoder(connector, DCB_OUTPUT_TMDS)) ||
+	      (nv_encoder = find_encoder(connector, DCB_OUTPUT_ANALOG))))
 		return NULL;
 
 	for_each_child_of_node(dn, cn) {
@@ -173,25 +193,25 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
 			      struct nouveau_encoder *nv_encoder)
 {
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct drm_device *dev = connector->dev;
 
 	if (nv_connector->detected_encoder == nv_encoder)
 		return;
 	nv_connector->detected_encoder = nv_encoder;
 
-	if (dev_priv->card_type >= NV_50) {
+	if (nv_device(drm->device)->card_type >= NV_50) {
 		connector->interlace_allowed = true;
 		connector->doublescan_allowed = true;
 	} else
-	if (nv_encoder->dcb->type == OUTPUT_LVDS ||
-	    nv_encoder->dcb->type == OUTPUT_TMDS) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS ||
+	    nv_encoder->dcb->type == DCB_OUTPUT_TMDS) {
 		connector->doublescan_allowed = false;
 		connector->interlace_allowed = false;
 	} else {
 		connector->doublescan_allowed = true;
-		if (dev_priv->card_type == NV_20 ||
-		   (dev_priv->card_type == NV_10 &&
+		if (nv_device(drm->device)->card_type == NV_20 ||
+		   (nv_device(drm->device)->card_type == NV_10 &&
 		    (dev->pci_device & 0x0ff0) != 0x0100 &&
 		    (dev->pci_device & 0x0ff0) != 0x0150))
 			/* HW is broken */
@@ -203,7 +223,7 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
 	if (nv_connector->type == DCB_CONNECTOR_DVI_I) {
 		drm_connector_property_set_value(connector,
 			dev->mode_config.dvi_i_subconnector_property,
-			nv_encoder->dcb->type == OUTPUT_TMDS ?
+			nv_encoder->dcb->type == DCB_OUTPUT_TMDS ?
 			DRM_MODE_SUBCONNECTOR_DVID :
 			DRM_MODE_SUBCONNECTOR_DVIA);
 	}
@@ -213,10 +233,11 @@ static enum drm_connector_status
 nouveau_connector_detect(struct drm_connector *connector, bool force)
 {
 	struct drm_device *dev = connector->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = NULL;
 	struct nouveau_encoder *nv_partner;
-	struct nouveau_i2c_chan *i2c;
+	struct nouveau_i2c_port *i2c;
 	int type;
 
 	/* Cleanup the previous EDID block. */
@@ -232,14 +253,14 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 		drm_mode_connector_update_edid_property(connector,
 							nv_connector->edid);
 		if (!nv_connector->edid) {
-			NV_ERROR(dev, "DDC responded, but no EDID for %s\n",
+			NV_ERROR(drm, "DDC responded, but no EDID for %s\n",
 				 drm_get_connector_name(connector));
 			goto detect_analog;
 		}
 
-		if (nv_encoder->dcb->type == OUTPUT_DP &&
+		if (nv_encoder->dcb->type == DCB_OUTPUT_DP &&
 		    !nouveau_dp_detect(to_drm_encoder(nv_encoder))) {
-			NV_ERROR(dev, "Detected %s, but failed init\n",
+			NV_ERROR(drm, "Detected %s, but failed init\n",
 				 drm_get_connector_name(connector));
 			return connector_status_disconnected;
 		}
@@ -250,19 +271,19 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 		 * isn't necessarily correct.
 		 */
 		nv_partner = NULL;
-		if (nv_encoder->dcb->type == OUTPUT_TMDS)
-			nv_partner = find_encoder(connector, OUTPUT_ANALOG);
-		if (nv_encoder->dcb->type == OUTPUT_ANALOG)
-			nv_partner = find_encoder(connector, OUTPUT_TMDS);
-
-		if (nv_partner && ((nv_encoder->dcb->type == OUTPUT_ANALOG &&
-				    nv_partner->dcb->type == OUTPUT_TMDS) ||
-				   (nv_encoder->dcb->type == OUTPUT_TMDS &&
-				    nv_partner->dcb->type == OUTPUT_ANALOG))) {
+		if (nv_encoder->dcb->type == DCB_OUTPUT_TMDS)
+			nv_partner = find_encoder(connector, DCB_OUTPUT_ANALOG);
+		if (nv_encoder->dcb->type == DCB_OUTPUT_ANALOG)
+			nv_partner = find_encoder(connector, DCB_OUTPUT_TMDS);
+
+		if (nv_partner && ((nv_encoder->dcb->type == DCB_OUTPUT_ANALOG &&
+				    nv_partner->dcb->type == DCB_OUTPUT_TMDS) ||
+				   (nv_encoder->dcb->type == DCB_OUTPUT_TMDS &&
+				    nv_partner->dcb->type == DCB_OUTPUT_ANALOG))) {
 			if (nv_connector->edid->input & DRM_EDID_INPUT_DIGITAL)
-				type = OUTPUT_TMDS;
+				type = DCB_OUTPUT_TMDS;
 			else
-				type = OUTPUT_ANALOG;
+				type = DCB_OUTPUT_ANALOG;
 
 			nv_encoder = find_encoder(connector, type);
 		}
@@ -278,9 +299,9 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 	}
 
 detect_analog:
-	nv_encoder = find_encoder(connector, OUTPUT_ANALOG);
+	nv_encoder = find_encoder(connector, DCB_OUTPUT_ANALOG);
 	if (!nv_encoder && !nouveau_tv_disable)
-		nv_encoder = find_encoder(connector, OUTPUT_TV);
+		nv_encoder = find_encoder(connector, DCB_OUTPUT_TV);
 	if (nv_encoder && force) {
 		struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
 		struct drm_encoder_helper_funcs *helper =
@@ -301,7 +322,7 @@ static enum drm_connector_status
 nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 {
 	struct drm_device *dev = connector->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = NULL;
 	enum drm_connector_status status = connector_status_disconnected;
@@ -313,12 +334,12 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 		nv_connector->edid = NULL;
 	}
 
-	nv_encoder = find_encoder(connector, OUTPUT_LVDS);
+	nv_encoder = find_encoder(connector, DCB_OUTPUT_LVDS);
 	if (!nv_encoder)
 		return connector_status_disconnected;
 
 	/* Try retrieving EDID via DDC */
-	if (!dev_priv->vbios.fp_no_ddc) {
+	if (!drm->vbios.fp_no_ddc) {
 		status = nouveau_connector_detect(connector, force);
 		if (status == connector_status_connected)
 			goto out;
@@ -334,7 +355,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	 * valid - it's not (rh#613284)
 	 */
 	if (nv_encoder->dcb->lvdsconf.use_acpi_for_edid) {
-		if (!nouveau_acpi_edid(dev, connector)) {
+		if (!(nv_connector->edid = nouveau_acpi_edid(dev, connector))) {
 			status = connector_status_connected;
 			goto out;
 		}
@@ -344,7 +365,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	 * modeline is avalilable for the panel, set it as the panel's
 	 * native mode and exit.
 	 */
-	if (nouveau_bios_fp_mode(dev, NULL) && (dev_priv->vbios.fp_no_ddc ||
+	if (nouveau_bios_fp_mode(dev, NULL) && (drm->vbios.fp_no_ddc ||
 	    nv_encoder->dcb->lvdsconf.use_straps_for_mode)) {
 		status = connector_status_connected;
 		goto out;
@@ -353,7 +374,7 @@ nouveau_connector_detect_lvds(struct drm_connector *connector, bool force)
 	/* Still nothing, some VBIOS images have a hardcoded EDID block
 	 * stored for the panel stored in them.
 	 */
-	if (!dev_priv->vbios.fp_no_ddc) {
+	if (!drm->vbios.fp_no_ddc) {
 		struct edid *edid =
 			(struct edid *)nouveau_bios_embedded_edid(dev);
 		if (edid) {
@@ -379,21 +400,22 @@ out:
 static void
 nouveau_connector_force(struct drm_connector *connector)
 {
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder;
 	int type;
 
 	if (nv_connector->type == DCB_CONNECTOR_DVI_I) {
 		if (connector->force == DRM_FORCE_ON_DIGITAL)
-			type = OUTPUT_TMDS;
+			type = DCB_OUTPUT_TMDS;
 		else
-			type = OUTPUT_ANALOG;
+			type = DCB_OUTPUT_ANALOG;
 	} else
-		type = OUTPUT_ANY;
+		type = DCB_OUTPUT_ANY;
 
 	nv_encoder = find_encoder(connector, type);
 	if (!nv_encoder) {
-		NV_ERROR(connector->dev, "can't find encoder to force %s on!\n",
+		NV_ERROR(drm, "can't find encoder to force %s on!\n",
 			 drm_get_connector_name(connector));
 		connector->status = connector_status_disconnected;
 		return;
@@ -406,8 +428,7 @@ static int
 nouveau_connector_set_property(struct drm_connector *connector,
 			       struct drm_property *property, uint64_t value)
 {
-	struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_display *disp = nouveau_display(connector->dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder;
 	struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
@@ -532,7 +553,7 @@ nouveau_connector_set_property(struct drm_connector *connector,
 		}
 	}
 
-	if (nv_encoder && nv_encoder->dcb->type == OUTPUT_TV)
+	if (nv_encoder && nv_encoder->dcb->type == DCB_OUTPUT_TV)
 		return get_slave_funcs(encoder)->set_property(
 			encoder, connector, property, value);
 
@@ -543,6 +564,7 @@ static struct drm_display_mode *
 nouveau_connector_native_mode(struct drm_connector *connector)
 {
 	struct drm_connector_helper_funcs *helper = connector->helper_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct drm_device *dev = connector->dev;
 	struct drm_display_mode *mode, *largest = NULL;
@@ -556,7 +578,7 @@ nouveau_connector_native_mode(struct drm_connector *connector)
 
 		/* Use preferred mode if there is one.. */
 		if (mode->type & DRM_MODE_TYPE_PREFERRED) {
-			NV_DEBUG_KMS(dev, "native mode from preferred\n");
+			NV_DEBUG(drm, "native mode from preferred\n");
 			return drm_mode_duplicate(dev, mode);
 		}
 
@@ -579,7 +601,7 @@ nouveau_connector_native_mode(struct drm_connector *connector)
 		largest = mode;
 	}
 
-	NV_DEBUG_KMS(dev, "native mode from largest: %dx%d@%d\n",
+	NV_DEBUG(drm, "native mode from largest: %dx%d@%d\n",
 		      high_w, high_h, high_v);
 	return largest ? drm_mode_duplicate(dev, largest) : NULL;
 }
@@ -643,10 +665,10 @@ nouveau_connector_scaler_modes_add(struct drm_connector *connector)
 static void
 nouveau_connector_detect_depth(struct drm_connector *connector)
 {
-	struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nvbios *bios = &drm->vbios;
 	struct drm_display_mode *mode = nv_connector->native_mode;
 	bool duallink;
 
@@ -661,7 +683,7 @@ nouveau_connector_detect_depth(struct drm_connector *connector)
 	}
 
 	/* we're out of options unless we're LVDS, default to 8bpc */
-	if (nv_encoder->dcb->type != OUTPUT_LVDS) {
+	if (nv_encoder->dcb->type != DCB_OUTPUT_LVDS) {
 		connector->display_info.bpc = 8;
 		return;
 	}
@@ -693,7 +715,7 @@ static int
 nouveau_connector_get_modes(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
 	struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder;
 	struct drm_encoder *encoder = to_drm_encoder(nv_encoder);
@@ -709,9 +731,9 @@ nouveau_connector_get_modes(struct drm_connector *connector)
 	if (nv_connector->edid)
 		ret = drm_add_edid_modes(connector, nv_connector->edid);
 	else
-	if (nv_encoder->dcb->type == OUTPUT_LVDS &&
+	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS &&
 	    (nv_encoder->dcb->lvdsconf.use_straps_for_mode ||
-	     dev_priv->vbios.fp_no_ddc) && nouveau_bios_fp_mode(dev, NULL)) {
+	     drm->vbios.fp_no_ddc) && nouveau_bios_fp_mode(dev, NULL)) {
 		struct drm_display_mode mode;
 
 		nouveau_bios_fp_mode(dev, &mode);
@@ -746,7 +768,7 @@ nouveau_connector_get_modes(struct drm_connector *connector)
 	if (connector->connector_type == DRM_MODE_CONNECTOR_LVDS)
 		nouveau_connector_detect_depth(connector);
 
-	if (nv_encoder->dcb->type == OUTPUT_TV)
+	if (nv_encoder->dcb->type == DCB_OUTPUT_TV)
 		ret = get_slave_funcs(encoder)->get_modes(encoder, connector);
 
 	if (nv_connector->type == DCB_CONNECTOR_LVDS ||
@@ -761,15 +783,15 @@ static unsigned
 get_tmds_link_bandwidth(struct drm_connector *connector)
 {
 	struct nouveau_connector *nv_connector = nouveau_connector(connector);
-	struct drm_nouveau_private *dev_priv = connector->dev->dev_private;
-	struct dcb_entry *dcb = nv_connector->detected_encoder->dcb;
+	struct nouveau_drm *drm = nouveau_drm(connector->dev);
+	struct dcb_output *dcb = nv_connector->detected_encoder->dcb;
 
 	if (dcb->location != DCB_LOC_ON_CHIP ||
-	    dev_priv->chipset >= 0x46)
+	    nv_device(drm->device)->chipset >= 0x46)
 		return 165000;
-	else if (dev_priv->chipset >= 0x40)
+	else if (nv_device(drm->device)->chipset >= 0x40)
 		return 155000;
-	else if (dev_priv->chipset >= 0x18)
+	else if (nv_device(drm->device)->chipset >= 0x18)
 		return 135000;
 	else
 		return 112000;
@@ -786,7 +808,7 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
 	unsigned clock = mode->clock;
 
 	switch (nv_encoder->dcb->type) {
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		if (nv_connector->native_mode &&
 		    (mode->hdisplay > nv_connector->native_mode->hdisplay ||
 		     mode->vdisplay > nv_connector->native_mode->vdisplay))
@@ -795,19 +817,19 @@ nouveau_connector_mode_valid(struct drm_connector *connector,
 		min_clock = 0;
 		max_clock = 400000;
 		break;
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		max_clock = get_tmds_link_bandwidth(connector);
 		if (nouveau_duallink && nv_encoder->dcb->duallink_possible)
 			max_clock *= 2;
 		break;
-	case OUTPUT_ANALOG:
+	case DCB_OUTPUT_ANALOG:
 		max_clock = nv_encoder->dcb->crtconf.maxfreq;
 		if (!max_clock)
 			max_clock = 350000;
 		break;
-	case OUTPUT_TV:
+	case DCB_OUTPUT_TV:
 		return get_slave_funcs(encoder)->mode_valid(encoder, mode);
-	case OUTPUT_DP:
+	case DCB_OUTPUT_DP:
 		max_clock  = nv_encoder->dp.link_nr;
 		max_clock *= nv_encoder->dp.link_bw;
 		clock = clock * (connector->display_info.bpc * 3) / 10;
@@ -899,14 +921,15 @@ struct drm_connector *
 nouveau_connector_create(struct drm_device *dev, int index)
 {
 	const struct drm_connector_funcs *funcs = &nouveau_connector_funcs;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
+	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_connector *nv_connector = NULL;
 	struct drm_connector *connector;
 	int type, ret = 0;
 	bool dummy;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		nv_connector = nouveau_connector(connector);
@@ -922,7 +945,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	nv_connector->index = index;
 
 	/* attempt to parse vbios connector type and hotplug gpio */
-	nv_connector->dcb = dcb_conn(dev, index);
+	nv_connector->dcb = olddcb_conn(dev, index);
 	if (nv_connector->dcb) {
 		static const u8 hpd[16] = {
 			0xff, 0x07, 0x08, 0xff, 0xff, 0x51, 0x52, 0xff,
@@ -930,7 +953,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 		};
 
 		u32 entry = ROM16(nv_connector->dcb[0]);
-		if (dcb_conntab(dev)[3] >= 4)
+		if (olddcb_conntab(dev)[3] >= 4)
 			entry |= (u32)ROM16(nv_connector->dcb[2]) << 16;
 
 		nv_connector->hpd = ffs((entry & 0x07033000) >> 12);
@@ -939,7 +962,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 		nv_connector->type = nv_connector->dcb[0];
 		if (drm_conntype_from_dcb(nv_connector->type) ==
 					  DRM_MODE_CONNECTOR_Unknown) {
-			NV_WARN(dev, "unknown connector type %02x\n",
+			NV_WARN(drm, "unknown connector type %02x\n",
 				nv_connector->type);
 			nv_connector->type = DCB_CONNECTOR_NONE;
 		}
@@ -964,8 +987,8 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	 * figure out something suitable ourselves
 	 */
 	if (nv_connector->type == DCB_CONNECTOR_NONE) {
-		struct drm_nouveau_private *dev_priv = dev->dev_private;
-		struct dcb_table *dcbt = &dev_priv->vbios.dcb;
+		struct nouveau_drm *drm = nouveau_drm(dev);
+		struct dcb_table *dcbt = &drm->vbios.dcb;
 		u32 encoders = 0;
 		int i;
 
@@ -974,25 +997,25 @@ nouveau_connector_create(struct drm_device *dev, int index)
 				encoders |= (1 << dcbt->entry[i].type);
 		}
 
-		if (encoders & (1 << OUTPUT_DP)) {
-			if (encoders & (1 << OUTPUT_TMDS))
+		if (encoders & (1 << DCB_OUTPUT_DP)) {
+			if (encoders & (1 << DCB_OUTPUT_TMDS))
 				nv_connector->type = DCB_CONNECTOR_DP;
 			else
 				nv_connector->type = DCB_CONNECTOR_eDP;
 		} else
-		if (encoders & (1 << OUTPUT_TMDS)) {
-			if (encoders & (1 << OUTPUT_ANALOG))
+		if (encoders & (1 << DCB_OUTPUT_TMDS)) {
+			if (encoders & (1 << DCB_OUTPUT_ANALOG))
 				nv_connector->type = DCB_CONNECTOR_DVI_I;
 			else
 				nv_connector->type = DCB_CONNECTOR_DVI_D;
 		} else
-		if (encoders & (1 << OUTPUT_ANALOG)) {
+		if (encoders & (1 << DCB_OUTPUT_ANALOG)) {
 			nv_connector->type = DCB_CONNECTOR_VGA;
 		} else
-		if (encoders & (1 << OUTPUT_LVDS)) {
+		if (encoders & (1 << DCB_OUTPUT_LVDS)) {
 			nv_connector->type = DCB_CONNECTOR_LVDS;
 		} else
-		if (encoders & (1 << OUTPUT_TV)) {
+		if (encoders & (1 << DCB_OUTPUT_TV)) {
 			nv_connector->type = DCB_CONNECTOR_TV_0;
 		}
 	}
@@ -1001,7 +1024,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	if (type == DRM_MODE_CONNECTOR_LVDS) {
 		ret = nouveau_bios_parse_lvds_table(dev, 0, &dummy, &dummy);
 		if (ret) {
-			NV_ERROR(dev, "Error parsing LVDS table, disabling\n");
+			NV_ERROR(drm, "Error parsing LVDS table, disabling\n");
 			kfree(nv_connector);
 			return ERR_PTR(ret);
 		}
@@ -1051,7 +1074,7 @@ nouveau_connector_create(struct drm_device *dev, int index)
 
 	switch (nv_connector->type) {
 	case DCB_CONNECTOR_VGA:
-		if (dev_priv->card_type >= NV_50) {
+		if (nv_device(drm->device)->card_type >= NV_50) {
 			drm_connector_attach_property(connector,
 					dev->mode_config.scaling_mode_property,
 					nv_connector->scaling_mode);
@@ -1084,10 +1107,9 @@ nouveau_connector_create(struct drm_device *dev, int index)
 	}
 
 	connector->polled = DRM_CONNECTOR_POLL_CONNECT;
-	if (nv_connector->hpd != DCB_GPIO_UNUSED) {
-		ret = nouveau_gpio_isr_add(dev, 0, nv_connector->hpd, 0xff,
-					   nouveau_connector_hotplug,
-					   connector);
+	if (gpio && nv_connector->hpd != DCB_GPIO_UNUSED) {
+		ret = gpio->isr_add(gpio, 0, nv_connector->hpd, 0xff,
+				    nouveau_connector_hotplug, connector);
 		if (ret == 0)
 			connector->polled = DRM_CONNECTOR_POLL_HPD;
 	}
@@ -1101,8 +1123,9 @@ nouveau_connector_hotplug(void *data, int plugged)
 {
 	struct drm_connector *connector = data;
 	struct drm_device *dev = connector->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	NV_DEBUG(dev, "%splugged %s\n", plugged ? "" : "un",
+	NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un",
 		 drm_get_connector_name(connector));
 
 	if (plugged)
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index e4857021304..ebdb87670a8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -27,8 +27,9 @@
 #ifndef __NOUVEAU_CONNECTOR_H__
 #define __NOUVEAU_CONNECTOR_H__
 
-#include "drm_edid.h"
-#include "nouveau_i2c.h"
+#include <drm/drm_edid.h>
+
+struct nouveau_i2c_port;
 
 enum nouveau_underscan_type {
 	UNDERSCAN_OFF,
diff --git a/drivers/gpu/drm/nouveau/nouveau_debugfs.c b/drivers/gpu/drm/nouveau/nouveau_debugfs.c
deleted file mode 100644
index 188c92b327e..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_debugfs.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Copyright (C) 2009 Red Hat <bskeggs@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-/*
- * Authors:
- *  Ben Skeggs <bskeggs@redhat.com>
- */
-
-#include <linux/debugfs.h>
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-
-#include <ttm/ttm_page_alloc.h>
-
-static int
-nouveau_debugfs_channel_info(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct nouveau_channel *chan = node->info_ent->data;
-
-	seq_printf(m, "channel id    : %d\n", chan->id);
-
-	seq_printf(m, "cpu fifo state:\n");
-	seq_printf(m, "          base: 0x%10llx\n", chan->pushbuf_base);
-	seq_printf(m, "           max: 0x%08x\n", chan->dma.max << 2);
-	seq_printf(m, "           cur: 0x%08x\n", chan->dma.cur << 2);
-	seq_printf(m, "           put: 0x%08x\n", chan->dma.put << 2);
-	seq_printf(m, "          free: 0x%08x\n", chan->dma.free << 2);
-	if (chan->dma.ib_max) {
-		seq_printf(m, "        ib max: 0x%08x\n", chan->dma.ib_max);
-		seq_printf(m, "        ib put: 0x%08x\n", chan->dma.ib_put);
-		seq_printf(m, "       ib free: 0x%08x\n", chan->dma.ib_free);
-	}
-
-	seq_printf(m, "gpu fifo state:\n");
-	seq_printf(m, "           get: 0x%08x\n",
-					nvchan_rd32(chan, chan->user_get));
-	seq_printf(m, "           put: 0x%08x\n",
-					nvchan_rd32(chan, chan->user_put));
-	if (chan->dma.ib_max) {
-		seq_printf(m, "        ib get: 0x%08x\n",
-			   nvchan_rd32(chan, 0x88));
-		seq_printf(m, "        ib put: 0x%08x\n",
-			   nvchan_rd32(chan, 0x8c));
-	}
-
-	return 0;
-}
-
-int
-nouveau_debugfs_channel_init(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct drm_minor *minor = chan->dev->primary;
-	int ret;
-
-	if (!dev_priv->debugfs.channel_root) {
-		dev_priv->debugfs.channel_root =
-			debugfs_create_dir("channel", minor->debugfs_root);
-		if (!dev_priv->debugfs.channel_root)
-			return -ENOENT;
-	}
-
-	snprintf(chan->debugfs.name, 32, "%d", chan->id);
-	chan->debugfs.info.name = chan->debugfs.name;
-	chan->debugfs.info.show = nouveau_debugfs_channel_info;
-	chan->debugfs.info.driver_features = 0;
-	chan->debugfs.info.data = chan;
-
-	ret = drm_debugfs_create_files(&chan->debugfs.info, 1,
-				       dev_priv->debugfs.channel_root,
-				       chan->dev->primary);
-	if (ret == 0)
-		chan->debugfs.active = true;
-	return ret;
-}
-
-void
-nouveau_debugfs_channel_fini(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-
-	if (!chan->debugfs.active)
-		return;
-
-	drm_debugfs_remove_files(&chan->debugfs.info, 1, chan->dev->primary);
-	chan->debugfs.active = false;
-
-	if (chan == dev_priv->channel) {
-		debugfs_remove(dev_priv->debugfs.channel_root);
-		dev_priv->debugfs.channel_root = NULL;
-	}
-}
-
-static int
-nouveau_debugfs_chipset_info(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_minor *minor = node->minor;
-	struct drm_device *dev = minor->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t ppci_0;
-
-	ppci_0 = nv_rd32(dev, dev_priv->chipset >= 0x40 ? 0x88000 : 0x1800);
-
-	seq_printf(m, "PMC_BOOT_0: 0x%08x\n", nv_rd32(dev, NV03_PMC_BOOT_0));
-	seq_printf(m, "PCI ID    : 0x%04x:0x%04x\n",
-		   ppci_0 & 0xffff, ppci_0 >> 16);
-	return 0;
-}
-
-static int
-nouveau_debugfs_memory_info(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_minor *minor = node->minor;
-	struct drm_nouveau_private *dev_priv = minor->dev->dev_private;
-
-	seq_printf(m, "VRAM total: %dKiB\n", (int)(dev_priv->vram_size >> 10));
-	return 0;
-}
-
-static int
-nouveau_debugfs_vbios_image(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_nouveau_private *dev_priv = node->minor->dev->dev_private;
-	int i;
-
-	for (i = 0; i < dev_priv->vbios.length; i++)
-		seq_printf(m, "%c", dev_priv->vbios.data[i]);
-	return 0;
-}
-
-static int
-nouveau_debugfs_evict_vram(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_nouveau_private *dev_priv = node->minor->dev->dev_private;
-	int ret;
-
-	ret = ttm_bo_evict_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
-	if (ret)
-		seq_printf(m, "failed: %d", ret);
-	else
-		seq_printf(m, "succeeded\n");
-	return 0;
-}
-
-static struct drm_info_list nouveau_debugfs_list[] = {
-	{ "evict_vram", nouveau_debugfs_evict_vram, 0, NULL },
-	{ "chipset", nouveau_debugfs_chipset_info, 0, NULL },
-	{ "memory", nouveau_debugfs_memory_info, 0, NULL },
-	{ "vbios.rom", nouveau_debugfs_vbios_image, 0, NULL },
-	{ "ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL },
-	{ "ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL },
-};
-#define NOUVEAU_DEBUGFS_ENTRIES ARRAY_SIZE(nouveau_debugfs_list)
-
-int
-nouveau_debugfs_init(struct drm_minor *minor)
-{
-	drm_debugfs_create_files(nouveau_debugfs_list, NOUVEAU_DEBUGFS_ENTRIES,
-				 minor->debugfs_root, minor);
-	return 0;
-}
-
-void
-nouveau_debugfs_takedown(struct drm_minor *minor)
-{
-	drm_debugfs_remove_files(nouveau_debugfs_list, NOUVEAU_DEBUGFS_ENTRIES,
-				 minor);
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 7e16dc5e646..8f98e5a8c48 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -24,20 +24,23 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "nouveau_drv.h"
-#include "nouveau_fb.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+
 #include "nouveau_fbcon.h"
 #include "nouveau_hw.h"
 #include "nouveau_crtc.h"
 #include "nouveau_dma.h"
+#include "nouveau_gem.h"
 #include "nouveau_connector.h"
-#include "nouveau_software.h"
-#include "nouveau_gpio.h"
-#include "nouveau_fence.h"
 #include "nv50_display.h"
 
+#include "nouveau_fence.h"
+
+#include <subdev/bios/gpio.h>
+#include <subdev/gpio.h>
+#include <engine/disp.h>
+
 static void
 nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
 {
@@ -71,7 +74,7 @@ nouveau_framebuffer_init(struct drm_device *dev,
 			 struct drm_mode_fb_cmd2 *mode_cmd,
 			 struct nouveau_bo *nvbo)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_framebuffer *fb = &nv_fb->base;
 	int ret;
 
@@ -83,7 +86,7 @@ nouveau_framebuffer_init(struct drm_device *dev,
 	drm_helper_mode_fill_fb_struct(fb, mode_cmd);
 	nv_fb->nvbo = nvbo;
 
-	if (dev_priv->card_type >= NV_50) {
+	if (nv_device(drm->device)->card_type >= NV_50) {
 		u32 tile_flags = nouveau_bo_tile_layout(nvbo);
 		if (tile_flags == 0x7a00 ||
 		    tile_flags == 0xfe00)
@@ -102,21 +105,21 @@ nouveau_framebuffer_init(struct drm_device *dev,
 		case 32: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_24; break;
 		case 30: nv_fb->r_format = NV50_EVO_CRTC_FB_DEPTH_30; break;
 		default:
-			 NV_ERROR(dev, "unknown depth %d\n", fb->depth);
+			 NV_ERROR(drm, "unknown depth %d\n", fb->depth);
 			 return -EINVAL;
 		}
 
-		if (dev_priv->chipset == 0x50)
+		if (nv_device(drm->device)->chipset == 0x50)
 			nv_fb->r_format |= (tile_flags << 8);
 
 		if (!tile_flags) {
-			if (dev_priv->card_type < NV_D0)
+			if (nv_device(drm->device)->card_type < NV_D0)
 				nv_fb->r_pitch = 0x00100000 | fb->pitches[0];
 			else
 				nv_fb->r_pitch = 0x01000000 | fb->pitches[0];
 		} else {
 			u32 mode = nvbo->tile_mode;
-			if (dev_priv->card_type >= NV_C0)
+			if (nv_device(drm->device)->card_type >= NV_C0)
 				mode >>= 4;
 			nv_fb->r_pitch = ((fb->pitches[0] / 4) << 4) | mode;
 		}
@@ -212,8 +215,9 @@ static struct nouveau_drm_prop_enum_list dither_depth[] = {
 int
 nouveau_display_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_display *disp = nouveau_display(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
 	struct drm_connector *connector;
 	int ret;
 
@@ -225,8 +229,8 @@ nouveau_display_init(struct drm_device *dev)
 	 * some vbios default this to off for some reason, causing the
 	 * panel to not work after resume
 	 */
-	if (nouveau_gpio_func_get(dev, DCB_GPIO_PANEL_POWER) == 0) {
-		nouveau_gpio_func_set(dev, DCB_GPIO_PANEL_POWER, true);
+	if (gpio && gpio->get(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff) == 0) {
+		gpio->set(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff, 1);
 		msleep(300);
 	}
 
@@ -236,7 +240,8 @@ nouveau_display_init(struct drm_device *dev)
 	/* enable hotplug interrupts */
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct nouveau_connector *conn = nouveau_connector(connector);
-		nouveau_gpio_irq(dev, 0, conn->hpd, 0xff, true);
+		if (gpio)
+			gpio->irq(gpio, 0, conn->hpd, 0xff, true);
 	}
 
 	return ret;
@@ -245,35 +250,65 @@ nouveau_display_init(struct drm_device *dev)
 void
 nouveau_display_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_display *disp = nouveau_display(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
 	struct drm_connector *connector;
 
 	/* disable hotplug interrupts */
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		struct nouveau_connector *conn = nouveau_connector(connector);
-		nouveau_gpio_irq(dev, 0, conn->hpd, 0xff, false);
+		if (gpio)
+			gpio->irq(gpio, 0, conn->hpd, 0xff, false);
 	}
 
 	drm_kms_helper_poll_disable(dev);
 	disp->fini(dev);
 }
 
+static void
+nouveau_display_vblank_notify(void *data, int crtc)
+{
+	drm_handle_vblank(data, crtc);
+}
+
+static void
+nouveau_display_vblank_get(void *data, int crtc)
+{
+	drm_vblank_get(data, crtc);
+}
+
+static void
+nouveau_display_vblank_put(void *data, int crtc)
+{
+	drm_vblank_put(data, crtc);
+}
+
 int
 nouveau_display_create(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_disp *pdisp = nouveau_disp(drm->device);
+	struct nouveau_display *disp;
 	int ret, gen;
 
+	disp = drm->display = kzalloc(sizeof(*disp), GFP_KERNEL);
+	if (!disp)
+		return -ENOMEM;
+
+	pdisp->vblank.data = dev;
+	pdisp->vblank.notify = nouveau_display_vblank_notify;
+	pdisp->vblank.get = nouveau_display_vblank_get;
+	pdisp->vblank.put = nouveau_display_vblank_put;
+
 	drm_mode_config_init(dev);
 	drm_mode_create_scaling_mode_property(dev);
 	drm_mode_create_dvi_i_properties(dev);
 
-	if (dev_priv->card_type < NV_50)
+	if (nv_device(drm->device)->card_type < NV_50)
 		gen = 0;
 	else
-	if (dev_priv->card_type < NV_D0)
+	if (nv_device(drm->device)->card_type < NV_D0)
 		gen = 1;
 	else
 		gen = 2;
@@ -307,11 +342,11 @@ nouveau_display_create(struct drm_device *dev)
 
 	dev->mode_config.min_width = 0;
 	dev->mode_config.min_height = 0;
-	if (dev_priv->card_type < NV_10) {
+	if (nv_device(drm->device)->card_type < NV_10) {
 		dev->mode_config.max_width = 2048;
 		dev->mode_config.max_height = 2048;
 	} else
-	if (dev_priv->card_type < NV_50) {
+	if (nv_device(drm->device)->card_type < NV_50) {
 		dev->mode_config.max_width = 4096;
 		dev->mode_config.max_height = 4096;
 	} else {
@@ -325,7 +360,13 @@ nouveau_display_create(struct drm_device *dev)
 	drm_kms_helper_poll_init(dev);
 	drm_kms_helper_poll_disable(dev);
 
-	ret = disp->create(dev);
+	if (nv_device(drm->device)->card_type < NV_50)
+		ret = nv04_display_create(dev);
+	else
+	if (nv_device(drm->device)->card_type < NV_D0)
+		ret = nv50_display_create(dev);
+	else
+		ret = nvd0_display_create(dev);
 	if (ret)
 		goto disp_create_err;
 
@@ -335,10 +376,11 @@ nouveau_display_create(struct drm_device *dev)
 			goto vblank_err;
 	}
 
+	nouveau_backlight_init(dev);
 	return 0;
 
 vblank_err:
-	disp->destroy(dev);
+	disp->dtor(dev);
 disp_create_err:
 	drm_kms_helper_poll_fini(dev);
 	drm_mode_config_cleanup(dev);
@@ -348,24 +390,109 @@ disp_create_err:
 void
 nouveau_display_destroy(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_display_engine *disp = &dev_priv->engine.display;
+	struct nouveau_display *disp = nouveau_display(dev);
 
+	nouveau_backlight_exit(dev);
 	drm_vblank_cleanup(dev);
 
-	disp->destroy(dev);
+	disp->dtor(dev);
 
 	drm_kms_helper_poll_fini(dev);
 	drm_mode_config_cleanup(dev);
+	nouveau_drm(dev)->display = NULL;
+	kfree(disp);
+}
+
+int
+nouveau_display_suspend(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct drm_crtc *crtc;
+
+	nouveau_display_fini(dev);
+
+	NV_INFO(drm, "unpinning framebuffer(s)...\n");
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_framebuffer *nouveau_fb;
+
+		nouveau_fb = nouveau_framebuffer(crtc->fb);
+		if (!nouveau_fb || !nouveau_fb->nvbo)
+			continue;
+
+		nouveau_bo_unpin(nouveau_fb->nvbo);
+	}
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+
+		nouveau_bo_unmap(nv_crtc->cursor.nvbo);
+		nouveau_bo_unpin(nv_crtc->cursor.nvbo);
+	}
+
+	return 0;
+}
+
+void
+nouveau_display_resume(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct drm_crtc *crtc;
+	int ret;
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_framebuffer *nouveau_fb;
+
+		nouveau_fb = nouveau_framebuffer(crtc->fb);
+		if (!nouveau_fb || !nouveau_fb->nvbo)
+			continue;
+
+		nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM);
+	}
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+
+		ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
+		if (!ret)
+			ret = nouveau_bo_map(nv_crtc->cursor.nvbo);
+		if (ret)
+			NV_ERROR(drm, "Could not pin/map cursor.\n");
+	}
+
+	nouveau_fbcon_set_suspend(dev, 0);
+	nouveau_fbcon_zfill_all(dev);
+
+	nouveau_display_init(dev);
+
+	/* Force CLUT to get re-loaded during modeset */
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+
+		nv_crtc->lut.depth = 0;
+	}
+
+	drm_helper_resume_force_mode(dev);
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+		u32 offset = nv_crtc->cursor.nvbo->bo.offset;
+
+		nv_crtc->cursor.set_offset(nv_crtc, offset);
+		nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
+						 nv_crtc->cursor_saved_y);
+	}
 }
 
 int
 nouveau_vblank_enable(struct drm_device *dev, int crtc)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 
-	if (dev_priv->card_type >= NV_50)
-		nv_mask(dev, NV50_PDISPLAY_INTR_EN_1, 0,
+	if (device->card_type >= NV_D0)
+		nv_mask(device, 0x6100c0 + (crtc * 0x800), 1, 1);
+	else
+	if (device->card_type >= NV_50)
+		nv_mask(device, NV50_PDISPLAY_INTR_EN_1, 0,
 			NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc));
 	else
 		NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0,
@@ -377,10 +504,13 @@ nouveau_vblank_enable(struct drm_device *dev, int crtc)
 void
 nouveau_vblank_disable(struct drm_device *dev, int crtc)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 
-	if (dev_priv->card_type >= NV_50)
-		nv_mask(dev, NV50_PDISPLAY_INTR_EN_1,
+	if (device->card_type >= NV_D0)
+		nv_mask(device, 0x6100c0 + (crtc * 0x800), 1, 0);
+	else
+	if (device->card_type >= NV_50)
+		nv_mask(device, NV50_PDISPLAY_INTR_EN_1,
 			NV50_PDISPLAY_INTR_EN_1_VBLANK_CRTC_(crtc), 0);
 	else
 		NVWriteCRTC(dev, crtc, NV_PCRTC_INTR_EN_0, 0);
@@ -434,15 +564,15 @@ nouveau_page_flip_emit(struct nouveau_channel *chan,
 		       struct nouveau_page_flip_state *s,
 		       struct nouveau_fence **pfence)
 {
-	struct nouveau_software_chan *swch = chan->engctx[NVOBJ_ENGINE_SW];
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct drm_device *dev = chan->dev;
+	struct nouveau_fence_chan *fctx = chan->fence;
+	struct nouveau_drm *drm = chan->drm;
+	struct drm_device *dev = drm->dev;
 	unsigned long flags;
 	int ret;
 
 	/* Queue it to the pending list */
 	spin_lock_irqsave(&dev->event_lock, flags);
-	list_add_tail(&s->head, &swch->flip);
+	list_add_tail(&s->head, &fctx->flip);
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 
 	/* Synchronize with the old framebuffer */
@@ -455,7 +585,7 @@ nouveau_page_flip_emit(struct nouveau_channel *chan,
 	if (ret)
 		goto fail;
 
-	if (dev_priv->card_type < NV_C0) {
+	if (nv_device(drm->device)->card_type < NV_C0) {
 		BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1);
 		OUT_RING  (chan, 0x00000000);
 		OUT_RING  (chan, 0x00000000);
@@ -483,7 +613,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 		       struct drm_pending_vblank_event *event)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->fb)->nvbo;
 	struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
 	struct nouveau_page_flip_state *s;
@@ -491,7 +621,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 	struct nouveau_fence *fence;
 	int ret;
 
-	if (!dev_priv->channel)
+	if (!drm->channel)
 		return -ENODEV;
 
 	s = kzalloc(sizeof(*s), GFP_KERNEL);
@@ -512,25 +642,25 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 	/* Choose the channel the flip will be handled in */
 	fence = new_bo->bo.sync_obj;
 	if (fence)
-		chan = nouveau_channel_get_unlocked(fence->channel);
+		chan = fence->channel;
 	if (!chan)
-		chan = nouveau_channel_get_unlocked(dev_priv->channel);
-	mutex_lock(&chan->mutex);
+		chan = drm->channel;
+	mutex_lock(&chan->cli->mutex);
 
 	/* Emit a page flip */
-	if (dev_priv->card_type >= NV_50) {
-		if (dev_priv->card_type >= NV_D0)
+	if (nv_device(drm->device)->card_type >= NV_50) {
+		if (nv_device(drm->device)->card_type >= NV_D0)
 			ret = nvd0_display_flip_next(crtc, fb, chan, 0);
 		else
 			ret = nv50_display_flip_next(crtc, fb, chan);
 		if (ret) {
-			nouveau_channel_put(&chan);
+			mutex_unlock(&chan->cli->mutex);
 			goto fail_unreserve;
 		}
 	}
 
 	ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence);
-	nouveau_channel_put(&chan);
+	mutex_unlock(&chan->cli->mutex);
 	if (ret)
 		goto fail_unreserve;
 
@@ -552,20 +682,21 @@ int
 nouveau_finish_page_flip(struct nouveau_channel *chan,
 			 struct nouveau_page_flip_state *ps)
 {
-	struct nouveau_software_chan *swch = chan->engctx[NVOBJ_ENGINE_SW];
-	struct drm_device *dev = chan->dev;
+	struct nouveau_fence_chan *fctx = chan->fence;
+	struct nouveau_drm *drm = chan->drm;
+	struct drm_device *dev = drm->dev;
 	struct nouveau_page_flip_state *s;
 	unsigned long flags;
 
 	spin_lock_irqsave(&dev->event_lock, flags);
 
-	if (list_empty(&swch->flip)) {
-		NV_ERROR(dev, "Unexpected pageflip in channel %d.\n", chan->id);
+	if (list_empty(&fctx->flip)) {
+		NV_ERROR(drm, "unexpected pageflip\n");
 		spin_unlock_irqrestore(&dev->event_lock, flags);
 		return -EINVAL;
 	}
 
-	s = list_first_entry(&swch->flip, struct nouveau_page_flip_state, head);
+	s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head);
 	if (s->event) {
 		struct drm_pending_vblank_event *e = s->event;
 		struct timeval now;
@@ -588,6 +719,24 @@ nouveau_finish_page_flip(struct nouveau_channel *chan,
 }
 
 int
+nouveau_flip_complete(void *data)
+{
+	struct nouveau_channel *chan = data;
+	struct nouveau_drm *drm = chan->drm;
+	struct nouveau_page_flip_state state;
+
+	if (!nouveau_finish_page_flip(chan, &state)) {
+		if (nv_device(drm->device)->card_type < NV_50) {
+			nv_set_crtc_base(drm->dev, state.crtc, state.offset +
+					 state.y * state.pitch +
+					 state.x * state.bpp / 8);
+		}
+	}
+
+	return 0;
+}
+
+int
 nouveau_display_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
 			    struct drm_mode_create_dumb *args)
 {
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
new file mode 100644
index 00000000000..722548bb3bd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -0,0 +1,94 @@
+#ifndef __NOUVEAU_DISPLAY_H__
+#define __NOUVEAU_DISPLAY_H__
+
+#include <subdev/vm.h>
+
+#include "nouveau_drm.h"
+
+struct nouveau_framebuffer {
+	struct drm_framebuffer base;
+	struct nouveau_bo *nvbo;
+	struct nouveau_vma vma;
+	u32 r_dma;
+	u32 r_format;
+	u32 r_pitch;
+};
+
+static inline struct nouveau_framebuffer *
+nouveau_framebuffer(struct drm_framebuffer *fb)
+{
+	return container_of(fb, struct nouveau_framebuffer, base);
+}
+
+int nouveau_framebuffer_init(struct drm_device *, struct nouveau_framebuffer *,
+			     struct drm_mode_fb_cmd2 *, struct nouveau_bo *);
+
+struct nouveau_page_flip_state {
+	struct list_head head;
+	struct drm_pending_vblank_event *event;
+	int crtc, bpp, pitch, x, y;
+	u64 offset;
+};
+
+struct nouveau_display {
+	void *priv;
+	void (*dtor)(struct drm_device *);
+	int  (*init)(struct drm_device *);
+	void (*fini)(struct drm_device *);
+
+	struct drm_property *dithering_mode;
+	struct drm_property *dithering_depth;
+	struct drm_property *underscan_property;
+	struct drm_property *underscan_hborder_property;
+	struct drm_property *underscan_vborder_property;
+	/* not really hue and saturation: */
+	struct drm_property *vibrant_hue_property;
+	struct drm_property *color_vibrance_property;
+};
+
+static inline struct nouveau_display *
+nouveau_display(struct drm_device *dev)
+{
+	return nouveau_drm(dev)->display;
+}
+
+int  nouveau_display_create(struct drm_device *dev);
+void nouveau_display_destroy(struct drm_device *dev);
+int  nouveau_display_init(struct drm_device *dev);
+void nouveau_display_fini(struct drm_device *dev);
+int  nouveau_display_suspend(struct drm_device *dev);
+void nouveau_display_resume(struct drm_device *dev);
+
+int  nouveau_vblank_enable(struct drm_device *dev, int crtc);
+void nouveau_vblank_disable(struct drm_device *dev, int crtc);
+
+int  nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+			    struct drm_pending_vblank_event *event);
+int  nouveau_finish_page_flip(struct nouveau_channel *,
+			      struct nouveau_page_flip_state *);
+
+int  nouveau_display_dumb_create(struct drm_file *, struct drm_device *,
+				 struct drm_mode_create_dumb *args);
+int  nouveau_display_dumb_map_offset(struct drm_file *, struct drm_device *,
+				     u32 handle, u64 *offset);
+int  nouveau_display_dumb_destroy(struct drm_file *, struct drm_device *,
+				  u32 handle);
+
+void nouveau_hdmi_mode_set(struct drm_encoder *, struct drm_display_mode *);
+
+#ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
+extern int nouveau_backlight_init(struct drm_device *);
+extern void nouveau_backlight_exit(struct drm_device *);
+#else
+static inline int
+nouveau_backlight_init(struct drm_device *dev)
+{
+	return 0;
+}
+
+static inline void
+nouveau_backlight_exit(struct drm_device *dev) {
+}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c
index 295932e66ac..40f91e1e584 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.c
@@ -24,41 +24,16 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_dma.h"
-#include "nouveau_ramht.h"
-
-void
-nouveau_dma_init(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nouveau_bo *pushbuf = chan->pushbuf_bo;
-
-	if (dev_priv->card_type >= NV_50) {
-		const int ib_size = pushbuf->bo.mem.size / 2;
-
-		chan->dma.ib_base = (pushbuf->bo.mem.size - ib_size) >> 2;
-		chan->dma.ib_max = (ib_size / 8) - 1;
-		chan->dma.ib_put = 0;
-		chan->dma.ib_free = chan->dma.ib_max - chan->dma.ib_put;
+#include <core/client.h>
 
-		chan->dma.max = (pushbuf->bo.mem.size - ib_size) >> 2;
-	} else {
-		chan->dma.max  = (pushbuf->bo.mem.size >> 2) - 2;
-	}
-
-	chan->dma.put  = 0;
-	chan->dma.cur  = chan->dma.put;
-	chan->dma.free = chan->dma.max - chan->dma.cur;
-}
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
 
 void
 OUT_RINGp(struct nouveau_channel *chan, const void *data, unsigned nr_dwords)
 {
 	bool is_iomem;
-	u32 *mem = ttm_kmap_obj_virtual(&chan->pushbuf_bo->kmap, &is_iomem);
+	u32 *mem = ttm_kmap_obj_virtual(&chan->push.buffer->kmap, &is_iomem);
 	mem = &mem[chan->dma.cur];
 	if (is_iomem)
 		memcpy_toio((void __force __iomem *)mem, data, nr_dwords * 4);
@@ -79,9 +54,9 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)
 {
 	uint64_t val;
 
-	val = nvchan_rd32(chan, chan->user_get);
+	val = nv_ro32(chan->object, chan->user_get);
         if (chan->user_get_hi)
-                val |= (uint64_t)nvchan_rd32(chan, chan->user_get_hi) << 32;
+                val |= (uint64_t)nv_ro32(chan->object, chan->user_get_hi) << 32;
 
 	/* reset counter as long as GET is still advancing, this is
 	 * to avoid misdetecting a GPU lockup if the GPU happens to
@@ -93,32 +68,33 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)
 	}
 
 	if ((++*timeout & 0xff) == 0) {
-		DRM_UDELAY(1);
+		udelay(1);
 		if (*timeout > 100000)
 			return -EBUSY;
 	}
 
-	if (val < chan->pushbuf_base ||
-	    val > chan->pushbuf_base + (chan->dma.max << 2))
+	if (val < chan->push.vma.offset ||
+	    val > chan->push.vma.offset + (chan->dma.max << 2))
 		return -EINVAL;
 
-	return (val - chan->pushbuf_base) >> 2;
+	return (val - chan->push.vma.offset) >> 2;
 }
 
 void
 nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
 	      int delta, int length)
 {
-	struct nouveau_bo *pb = chan->pushbuf_bo;
+	struct nouveau_bo *pb = chan->push.buffer;
 	struct nouveau_vma *vma;
 	int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;
 	u64 offset;
 
-	vma = nouveau_bo_vma_find(bo, chan->vm);
+	vma = nouveau_bo_vma_find(bo, nv_client(chan->cli)->vm);
 	BUG_ON(!vma);
 	offset = vma->offset + delta;
 
 	BUG_ON(chan->dma.ib_free < 1);
+
 	nouveau_bo_wr32(pb, ip++, lower_32_bits(offset));
 	nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8);
 
@@ -128,7 +104,7 @@ nv50_dma_push(struct nouveau_channel *chan, struct nouveau_bo *bo,
 	/* Flush writes. */
 	nouveau_bo_rd32(pb, 0);
 
-	nvchan_wr32(chan, 0x8c, chan->dma.ib_put);
+	nv_wo32(chan->object, 0x8c, chan->dma.ib_put);
 	chan->dma.ib_free--;
 }
 
@@ -138,7 +114,7 @@ nv50_dma_push_wait(struct nouveau_channel *chan, int count)
 	uint32_t cnt = 0, prev_get = 0;
 
 	while (chan->dma.ib_free < count) {
-		uint32_t get = nvchan_rd32(chan, 0x88);
+		uint32_t get = nv_ro32(chan->object, 0x88);
 		if (get != prev_get) {
 			prev_get = get;
 			cnt = 0;
@@ -249,7 +225,7 @@ nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size)
 			 * instruct the GPU to jump back to the start right
 			 * after processing the currently pending commands.
 			 */
-			OUT_RING(chan, chan->pushbuf_base | 0x20000000);
+			OUT_RING(chan, chan->push.vma.offset | 0x20000000);
 
 			/* wait for GET to depart from the skips area.
 			 * prevents writing GET==PUT and causing a race
diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.h b/drivers/gpu/drm/nouveau/nouveau_dma.h
index 8db68be9544..5c2e22932d1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dma.h
+++ b/drivers/gpu/drm/nouveau/nouveau_dma.h
@@ -27,10 +27,10 @@
 #ifndef __NOUVEAU_DMA_H__
 #define __NOUVEAU_DMA_H__
 
-#ifndef NOUVEAU_DMA_DEBUG
-#define NOUVEAU_DMA_DEBUG 0
-#endif
+#include "nouveau_bo.h"
+#include "nouveau_chan.h"
 
+int nouveau_dma_wait(struct nouveau_channel *, int slots, int size);
 void nv50_dma_push(struct nouveau_channel *, struct nouveau_bo *,
 		   int delta, int length);
 
@@ -116,12 +116,7 @@ RING_SPACE(struct nouveau_channel *chan, int size)
 static inline void
 OUT_RING(struct nouveau_channel *chan, int data)
 {
-	if (NOUVEAU_DMA_DEBUG) {
-		NV_INFO(chan->dev, "Ch%d/0x%08x: 0x%08x\n",
-			chan->id, chan->dma.cur << 2, data);
-	}
-
-	nouveau_bo_wr32(chan->pushbuf_bo, chan->dma.cur++, data);
+	nouveau_bo_wr32(chan->push.buffer, chan->dma.cur++, data);
 }
 
 extern void
@@ -159,24 +154,19 @@ BEGIN_IMC0(struct nouveau_channel *chan, int subc, int mthd, u16 data)
 
 #define WRITE_PUT(val) do {                                                    \
 	DRM_MEMORYBARRIER();                                                   \
-	nouveau_bo_rd32(chan->pushbuf_bo, 0);                                  \
-	nvchan_wr32(chan, chan->user_put, ((val) << 2) + chan->pushbuf_base);  \
+	nouveau_bo_rd32(chan->push.buffer, 0);                                 \
+	nv_wo32(chan->object, chan->user_put, ((val) << 2) + chan->push.vma.offset);  \
 } while (0)
 
 static inline void
 FIRE_RING(struct nouveau_channel *chan)
 {
-	if (NOUVEAU_DMA_DEBUG) {
-		NV_INFO(chan->dev, "Ch%d/0x%08x: PUSH!\n",
-			chan->id, chan->dma.cur << 2);
-	}
-
 	if (chan->dma.cur == chan->dma.put)
 		return;
 	chan->accel_done = true;
 
 	if (chan->dma.ib_max) {
-		nv50_dma_push(chan, chan->pushbuf_bo, chan->dma.put << 2,
+		nv50_dma_push(chan, chan->push.buffer, chan->dma.put << 2,
 			      (chan->dma.cur - chan->dma.put) << 2);
 	} else {
 		WRITE_PUT(chan->dma.cur);
@@ -191,4 +181,31 @@ WIND_RING(struct nouveau_channel *chan)
 	chan->dma.cur = chan->dma.put;
 }
 
+/* FIFO methods */
+#define NV01_SUBCHAN_OBJECT                                          0x00000000
+#define NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH                          0x00000010
+#define NV84_SUBCHAN_SEMAPHORE_ADDRESS_LOW                           0x00000014
+#define NV84_SUBCHAN_SEMAPHORE_SEQUENCE                              0x00000018
+#define NV84_SUBCHAN_SEMAPHORE_TRIGGER                               0x0000001c
+#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL                 0x00000001
+#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG                    0x00000002
+#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_GEQUAL                0x00000004
+#define NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD                         0x00001000
+#define NV84_SUBCHAN_NOTIFY_INTR                                     0x00000020
+#define NV84_SUBCHAN_WRCACHE_FLUSH                                   0x00000024
+#define NV10_SUBCHAN_REF_CNT                                         0x00000050
+#define NVSW_SUBCHAN_PAGE_FLIP                                       0x00000054
+#define NV11_SUBCHAN_DMA_SEMAPHORE                                   0x00000060
+#define NV11_SUBCHAN_SEMAPHORE_OFFSET                                0x00000064
+#define NV11_SUBCHAN_SEMAPHORE_ACQUIRE                               0x00000068
+#define NV11_SUBCHAN_SEMAPHORE_RELEASE                               0x0000006c
+#define NV40_SUBCHAN_YIELD                                           0x00000080
+
+/* NV_SW object class */
+#define NV_SW_DMA_VBLSEM                                             0x0000018c
+#define NV_SW_VBLSEM_OFFSET                                          0x00000400
+#define NV_SW_VBLSEM_RELEASE_VALUE                                   0x00000404
+#define NV_SW_VBLSEM_RELEASE                                         0x00000408
+#define NV_SW_PAGE_FLIP                                              0x00000500
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c
index 7e289d2ad8e..978a108ba7a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_dp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dp.c
@@ -22,165 +22,38 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
+#include <drm/drm_dp_helper.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_i2c.h"
+#include "nouveau_drm.h"
 #include "nouveau_connector.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
-#include "nouveau_gpio.h"
 
-/******************************************************************************
- * aux channel util functions
- *****************************************************************************/
-#define AUX_DBG(fmt, args...) do {                                             \
-	if (nouveau_reg_debug & NOUVEAU_REG_DEBUG_AUXCH) {                     \
-		NV_PRINTK(KERN_DEBUG, dev, "AUXCH(%d): " fmt, ch, ##args);     \
-	}                                                                      \
-} while (0)
-#define AUX_ERR(fmt, args...) NV_ERROR(dev, "AUXCH(%d): " fmt, ch, ##args)
-
-static void
-auxch_fini(struct drm_device *dev, int ch)
-{
-	nv_mask(dev, 0x00e4e4 + (ch * 0x50), 0x00310000, 0x00000000);
-}
-
-static int
-auxch_init(struct drm_device *dev, int ch)
-{
-	const u32 unksel = 1; /* nfi which to use, or if it matters.. */
-	const u32 ureq = unksel ? 0x00100000 : 0x00200000;
-	const u32 urep = unksel ? 0x01000000 : 0x02000000;
-	u32 ctrl, timeout;
-
-	/* wait up to 1ms for any previous transaction to be done... */
-	timeout = 1000;
-	do {
-		ctrl = nv_rd32(dev, 0x00e4e4 + (ch * 0x50));
-		udelay(1);
-		if (!timeout--) {
-			AUX_ERR("begin idle timeout 0x%08x", ctrl);
-			return -EBUSY;
-		}
-	} while (ctrl & 0x03010000);
-
-	/* set some magic, and wait up to 1ms for it to appear */
-	nv_mask(dev, 0x00e4e4 + (ch * 0x50), 0x00300000, ureq);
-	timeout = 1000;
-	do {
-		ctrl = nv_rd32(dev, 0x00e4e4 + (ch * 0x50));
-		udelay(1);
-		if (!timeout--) {
-			AUX_ERR("magic wait 0x%08x\n", ctrl);
-			auxch_fini(dev, ch);
-			return -EBUSY;
-		}
-	} while ((ctrl & 0x03000000) != urep);
-
-	return 0;
-}
-
-static int
-auxch_tx(struct drm_device *dev, int ch, u8 type, u32 addr, u8 *data, u8 size)
-{
-	u32 ctrl, stat, timeout, retries;
-	u32 xbuf[4] = {};
-	int ret, i;
-
-	AUX_DBG("%d: 0x%08x %d\n", type, addr, size);
-
-	ret = auxch_init(dev, ch);
-	if (ret)
-		goto out;
-
-	stat = nv_rd32(dev, 0x00e4e8 + (ch * 0x50));
-	if (!(stat & 0x10000000)) {
-		AUX_DBG("sink not detected\n");
-		ret = -ENXIO;
-		goto out;
-	}
-
-	if (!(type & 1)) {
-		memcpy(xbuf, data, size);
-		for (i = 0; i < 16; i += 4) {
-			AUX_DBG("wr 0x%08x\n", xbuf[i / 4]);
-			nv_wr32(dev, 0x00e4c0 + (ch * 0x50) + i, xbuf[i / 4]);
-		}
-	}
-
-	ctrl  = nv_rd32(dev, 0x00e4e4 + (ch * 0x50));
-	ctrl &= ~0x0001f0ff;
-	ctrl |= type << 12;
-	ctrl |= size - 1;
-	nv_wr32(dev, 0x00e4e0 + (ch * 0x50), addr);
-
-	/* retry transaction a number of times on failure... */
-	ret = -EREMOTEIO;
-	for (retries = 0; retries < 32; retries++) {
-		/* reset, and delay a while if this is a retry */
-		nv_wr32(dev, 0x00e4e4 + (ch * 0x50), 0x80000000 | ctrl);
-		nv_wr32(dev, 0x00e4e4 + (ch * 0x50), 0x00000000 | ctrl);
-		if (retries)
-			udelay(400);
-
-		/* transaction request, wait up to 1ms for it to complete */
-		nv_wr32(dev, 0x00e4e4 + (ch * 0x50), 0x00010000 | ctrl);
-
-		timeout = 1000;
-		do {
-			ctrl = nv_rd32(dev, 0x00e4e4 + (ch * 0x50));
-			udelay(1);
-			if (!timeout--) {
-				AUX_ERR("tx req timeout 0x%08x\n", ctrl);
-				goto out;
-			}
-		} while (ctrl & 0x00010000);
-
-		/* read status, and check if transaction completed ok */
-		stat = nv_mask(dev, 0x00e4e8 + (ch * 0x50), 0, 0);
-		if (!(stat & 0x000f0f00)) {
-			ret = 0;
-			break;
-		}
-
-		AUX_DBG("%02d 0x%08x 0x%08x\n", retries, ctrl, stat);
-	}
-
-	if (type & 1) {
-		for (i = 0; i < 16; i += 4) {
-			xbuf[i / 4] = nv_rd32(dev, 0x00e4d0 + (ch * 0x50) + i);
-			AUX_DBG("rd 0x%08x\n", xbuf[i / 4]);
-		}
-		memcpy(data, xbuf, size);
-	}
-
-out:
-	auxch_fini(dev, ch);
-	return ret;
-}
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
 
 u8 *
-nouveau_dp_bios_data(struct drm_device *dev, struct dcb_entry *dcb, u8 **entry)
+nouveau_dp_bios_data(struct drm_device *dev, struct dcb_output *dcb, u8 **entry)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct bit_entry d;
 	u8 *table;
 	int i;
 
 	if (bit_table(dev, 'd', &d)) {
-		NV_ERROR(dev, "BIT 'd' table not found\n");
+		NV_ERROR(drm, "BIT 'd' table not found\n");
 		return NULL;
 	}
 
 	if (d.version != 1) {
-		NV_ERROR(dev, "BIT 'd' table version %d unknown\n", d.version);
+		NV_ERROR(drm, "BIT 'd' table version %d unknown\n", d.version);
 		return NULL;
 	}
 
 	table = ROMPTR(dev, d.data[0]);
 	if (!table) {
-		NV_ERROR(dev, "displayport table pointer invalid\n");
+		NV_ERROR(drm, "displayport table pointer invalid\n");
 		return NULL;
 	}
 
@@ -191,7 +64,7 @@ nouveau_dp_bios_data(struct drm_device *dev, struct dcb_entry *dcb, u8 **entry)
 	case 0x40:
 		break;
 	default:
-		NV_ERROR(dev, "displayport table 0x%02x unknown\n", table[0]);
+		NV_ERROR(drm, "displayport table 0x%02x unknown\n", table[0]);
 		return NULL;
 	}
 
@@ -201,7 +74,7 @@ nouveau_dp_bios_data(struct drm_device *dev, struct dcb_entry *dcb, u8 **entry)
 			return table;
 	}
 
-	NV_ERROR(dev, "displayport encoder table not found\n");
+	NV_ERROR(drm, "displayport encoder table not found\n");
 	return NULL;
 }
 
@@ -209,9 +82,9 @@ nouveau_dp_bios_data(struct drm_device *dev, struct dcb_entry *dcb, u8 **entry)
  * link training
  *****************************************************************************/
 struct dp_state {
+	struct nouveau_i2c_port *auxch;
 	struct dp_train_func *func;
-	struct dcb_entry *dcb;
-	int auxch;
+	struct dcb_output *dcb;
 	int crtc;
 	u8 *dpcd;
 	int link_nr;
@@ -223,9 +96,10 @@ struct dp_state {
 static void
 dp_set_link_config(struct drm_device *dev, struct dp_state *dp)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u8 sink[2];
 
-	NV_DEBUG_KMS(dev, "%d lanes at %d KB/s\n", dp->link_nr, dp->link_bw);
+	NV_DEBUG(drm, "%d lanes at %d KB/s\n", dp->link_nr, dp->link_bw);
 
 	/* set desired link configuration on the source */
 	dp->func->link_set(dev, dp->dcb, dp->crtc, dp->link_nr, dp->link_bw,
@@ -237,27 +111,29 @@ dp_set_link_config(struct drm_device *dev, struct dp_state *dp)
 	if (dp->dpcd[2] & DP_ENHANCED_FRAME_CAP)
 		sink[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 
-	auxch_tx(dev, dp->auxch, 8, DP_LINK_BW_SET, sink, 2);
+	nv_wraux(dp->auxch, DP_LINK_BW_SET, sink, 2);
 }
 
 static void
 dp_set_training_pattern(struct drm_device *dev, struct dp_state *dp, u8 pattern)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u8 sink_tp;
 
-	NV_DEBUG_KMS(dev, "training pattern %d\n", pattern);
+	NV_DEBUG(drm, "training pattern %d\n", pattern);
 
 	dp->func->train_set(dev, dp->dcb, pattern);
 
-	auxch_tx(dev, dp->auxch, 9, DP_TRAINING_PATTERN_SET, &sink_tp, 1);
+	nv_rdaux(dp->auxch, DP_TRAINING_PATTERN_SET, &sink_tp, 1);
 	sink_tp &= ~DP_TRAINING_PATTERN_MASK;
 	sink_tp |= pattern;
-	auxch_tx(dev, dp->auxch, 8, DP_TRAINING_PATTERN_SET, &sink_tp, 1);
+	nv_wraux(dp->auxch, DP_TRAINING_PATTERN_SET, &sink_tp, 1);
 }
 
 static int
 dp_link_train_commit(struct drm_device *dev, struct dp_state *dp)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int i;
 
 	for (i = 0; i < dp->link_nr; i++) {
@@ -271,27 +147,26 @@ dp_link_train_commit(struct drm_device *dev, struct dp_state *dp)
 		if ((lpre << 3) == DP_TRAIN_PRE_EMPHASIS_9_5)
 			dp->conf[i] |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
 
-		NV_DEBUG_KMS(dev, "config lane %d %02x\n", i, dp->conf[i]);
+		NV_DEBUG(drm, "config lane %d %02x\n", i, dp->conf[i]);
 		dp->func->train_adj(dev, dp->dcb, i, lvsw, lpre);
 	}
 
-	return auxch_tx(dev, dp->auxch, 8, DP_TRAINING_LANE0_SET, dp->conf, 4);
+	return nv_wraux(dp->auxch, DP_TRAINING_LANE0_SET, dp->conf, 4);
 }
 
 static int
 dp_link_train_update(struct drm_device *dev, struct dp_state *dp, u32 delay)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int ret;
 
 	udelay(delay);
 
-	ret = auxch_tx(dev, dp->auxch, 9, DP_LANE0_1_STATUS, dp->stat, 6);
+	ret = nv_rdaux(dp->auxch, DP_LANE0_1_STATUS, dp->stat, 6);
 	if (ret)
 		return ret;
 
-	NV_DEBUG_KMS(dev, "status %02x %02x %02x %02x %02x %02x\n",
-		     dp->stat[0], dp->stat[1], dp->stat[2], dp->stat[3],
-		     dp->stat[4], dp->stat[5]);
+	NV_DEBUG(drm, "status %*ph\n", 6, dp->stat);
 	return 0;
 }
 
@@ -409,7 +284,7 @@ dp_link_train_fini(struct drm_device *dev, struct dp_state *dp)
 	nouveau_bios_run_init_table(dev, script, dp->dcb, dp->crtc);
 }
 
-bool
+static bool
 nouveau_dp_link_train(struct drm_encoder *encoder, u32 datarate,
 		      struct dp_train_func *func)
 {
@@ -418,19 +293,20 @@ nouveau_dp_link_train(struct drm_encoder *encoder, u32 datarate,
 	struct nouveau_connector *nv_connector =
 		nouveau_encoder_connector_get(nv_encoder);
 	struct drm_device *dev = encoder->dev;
-	struct nouveau_i2c_chan *auxch;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
 	const u32 bw_list[] = { 270000, 162000, 0 };
 	const u32 *link_bw = bw_list;
 	struct dp_state dp;
 
-	auxch = nouveau_i2c_find(dev, nv_encoder->dcb->i2c_index);
-	if (!auxch)
+	dp.auxch = i2c->find(i2c, nv_encoder->dcb->i2c_index);
+	if (!dp.auxch)
 		return false;
 
 	dp.func = func;
 	dp.dcb = nv_encoder->dcb;
 	dp.crtc = nv_crtc->index;
-	dp.auxch = auxch->drive;
 	dp.dpcd = nv_encoder->dp.dpcd;
 
 	/* adjust required bandwidth for 8B/10B coding overhead */
@@ -440,7 +316,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder, u32 datarate,
 	 * we take during link training (DP_SET_POWER is one), we need
 	 * to ignore them for the moment to avoid races.
 	 */
-	nouveau_gpio_irq(dev, 0, nv_connector->hpd, 0xff, false);
+	gpio->irq(gpio, 0, nv_connector->hpd, 0xff, false);
 
 	/* enable down-spreading, if possible */
 	dp_set_downspread(dev, &dp, nv_encoder->dp.dpcd[3] & 1);
@@ -483,7 +359,7 @@ nouveau_dp_link_train(struct drm_encoder *encoder, u32 datarate,
 	dp_link_train_fini(dev, &dp);
 
 	/* re-enable hotplug detect */
-	nouveau_gpio_irq(dev, 0, nv_connector->hpd, 0xff, true);
+	gpio->irq(gpio, 0, nv_connector->hpd, 0xff, true);
 	return true;
 }
 
@@ -492,10 +368,12 @@ nouveau_dp_dpms(struct drm_encoder *encoder, int mode, u32 datarate,
 		struct dp_train_func *func)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct nouveau_i2c_chan *auxch;
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+	struct nouveau_i2c_port *auxch;
 	u8 status;
 
-	auxch = nouveau_i2c_find(encoder->dev, nv_encoder->dcb->i2c_index);
+	auxch = i2c->find(i2c, nv_encoder->dcb->i2c_index);
 	if (!auxch)
 		return;
 
@@ -504,27 +382,28 @@ nouveau_dp_dpms(struct drm_encoder *encoder, int mode, u32 datarate,
 	else
 		status = DP_SET_POWER_D3;
 
-	nouveau_dp_auxch(auxch, 8, DP_SET_POWER, &status, 1);
+	nv_wraux(auxch, DP_SET_POWER, &status, 1);
 
 	if (mode == DRM_MODE_DPMS_ON)
 		nouveau_dp_link_train(encoder, datarate, func);
 }
 
 static void
-nouveau_dp_probe_oui(struct drm_device *dev, struct nouveau_i2c_chan *auxch,
+nouveau_dp_probe_oui(struct drm_device *dev, struct nouveau_i2c_port *auxch,
 		     u8 *dpcd)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u8 buf[3];
 
 	if (!(dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT))
 		return;
 
-	if (!auxch_tx(dev, auxch->drive, 9, DP_SINK_OUI, buf, 3))
-		NV_DEBUG_KMS(dev, "Sink OUI: %02hx%02hx%02hx\n",
+	if (!nv_rdaux(auxch, DP_SINK_OUI, buf, 3))
+		NV_DEBUG(drm, "Sink OUI: %02hx%02hx%02hx\n",
 			     buf[0], buf[1], buf[2]);
 
-	if (!auxch_tx(dev, auxch->drive, 9, DP_BRANCH_OUI, buf, 3))
-		NV_DEBUG_KMS(dev, "Branch OUI: %02hx%02hx%02hx\n",
+	if (!nv_rdaux(auxch, DP_BRANCH_OUI, buf, 3))
+		NV_DEBUG(drm, "Branch OUI: %02hx%02hx%02hx\n",
 			     buf[0], buf[1], buf[2]);
 
 }
@@ -534,24 +413,26 @@ nouveau_dp_detect(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
-	struct nouveau_i2c_chan *auxch;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+	struct nouveau_i2c_port *auxch;
 	u8 *dpcd = nv_encoder->dp.dpcd;
 	int ret;
 
-	auxch = nouveau_i2c_find(dev, nv_encoder->dcb->i2c_index);
+	auxch = i2c->find(i2c, nv_encoder->dcb->i2c_index);
 	if (!auxch)
 		return false;
 
-	ret = auxch_tx(dev, auxch->drive, 9, DP_DPCD_REV, dpcd, 8);
+	ret = nv_rdaux(auxch, DP_DPCD_REV, dpcd, 8);
 	if (ret)
 		return false;
 
 	nv_encoder->dp.link_bw = 27000 * dpcd[1];
 	nv_encoder->dp.link_nr = dpcd[2] & DP_MAX_LANE_COUNT_MASK;
 
-	NV_DEBUG_KMS(dev, "display: %dx%d dpcd 0x%02x\n",
+	NV_DEBUG(drm, "display: %dx%d dpcd 0x%02x\n",
 		     nv_encoder->dp.link_nr, nv_encoder->dp.link_bw, dpcd[0]);
-	NV_DEBUG_KMS(dev, "encoder: %dx%d\n",
+	NV_DEBUG(drm, "encoder: %dx%d\n",
 		     nv_encoder->dcb->dpconf.link_nr,
 		     nv_encoder->dcb->dpconf.link_bw);
 
@@ -560,65 +441,10 @@ nouveau_dp_detect(struct drm_encoder *encoder)
 	if (nv_encoder->dcb->dpconf.link_bw < nv_encoder->dp.link_bw)
 		nv_encoder->dp.link_bw = nv_encoder->dcb->dpconf.link_bw;
 
-	NV_DEBUG_KMS(dev, "maximum: %dx%d\n",
+	NV_DEBUG(drm, "maximum: %dx%d\n",
 		     nv_encoder->dp.link_nr, nv_encoder->dp.link_bw);
 
 	nouveau_dp_probe_oui(dev, auxch, dpcd);
 
 	return true;
 }
-
-int
-nouveau_dp_auxch(struct nouveau_i2c_chan *auxch, int cmd, int addr,
-		 uint8_t *data, int data_nr)
-{
-	return auxch_tx(auxch->dev, auxch->drive, cmd, addr, data, data_nr);
-}
-
-static int
-nouveau_dp_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
-{
-	struct nouveau_i2c_chan *auxch = (struct nouveau_i2c_chan *)adap;
-	struct i2c_msg *msg = msgs;
-	int ret, mcnt = num;
-
-	while (mcnt--) {
-		u8 remaining = msg->len;
-		u8 *ptr = msg->buf;
-
-		while (remaining) {
-			u8 cnt = (remaining > 16) ? 16 : remaining;
-			u8 cmd;
-
-			if (msg->flags & I2C_M_RD)
-				cmd = AUX_I2C_READ;
-			else
-				cmd = AUX_I2C_WRITE;
-
-			if (mcnt || remaining > 16)
-				cmd |= AUX_I2C_MOT;
-
-			ret = nouveau_dp_auxch(auxch, cmd, msg->addr, ptr, cnt);
-			if (ret < 0)
-				return ret;
-
-			ptr += cnt;
-			remaining -= cnt;
-		}
-
-		msg++;
-	}
-
-	return num;
-}
-
-static u32
-nouveau_dp_i2c_func(struct i2c_adapter *adap)
-{
-	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
-}
-
-const struct i2c_algorithm nouveau_dp_i2c_algo = {
-	.master_xfer = nouveau_dp_i2c_xfer,
-	.functionality = nouveau_dp_i2c_func
-};
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
new file mode 100644
index 00000000000..ccae8c26ae2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -0,0 +1,693 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include <core/device.h>
+#include <core/client.h>
+#include <core/gpuobj.h>
+#include <core/class.h>
+
+#include <subdev/device.h>
+#include <subdev/vm.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_irq.h"
+#include "nouveau_dma.h"
+#include "nouveau_ttm.h"
+#include "nouveau_gem.h"
+#include "nouveau_agp.h"
+#include "nouveau_vga.h"
+#include "nouveau_pm.h"
+#include "nouveau_acpi.h"
+#include "nouveau_bios.h"
+#include "nouveau_ioctl.h"
+#include "nouveau_abi16.h"
+#include "nouveau_fbcon.h"
+#include "nouveau_fence.h"
+
+#include "nouveau_ttm.h"
+
+MODULE_PARM_DESC(config, "option string to pass to driver core");
+static char *nouveau_config;
+module_param_named(config, nouveau_config, charp, 0400);
+
+MODULE_PARM_DESC(debug, "debug string to pass to driver core");
+static char *nouveau_debug;
+module_param_named(debug, nouveau_debug, charp, 0400);
+
+MODULE_PARM_DESC(noaccel, "disable kernel/abi16 acceleration");
+static int nouveau_noaccel = 0;
+module_param_named(noaccel, nouveau_noaccel, int, 0400);
+
+MODULE_PARM_DESC(modeset, "enable driver");
+static int nouveau_modeset = -1;
+module_param_named(modeset, nouveau_modeset, int, 0400);
+
+static struct drm_driver driver;
+
+static u64
+nouveau_name(struct pci_dev *pdev)
+{
+	u64 name = (u64)pci_domain_nr(pdev->bus) << 32;
+	name |= pdev->bus->number << 16;
+	name |= PCI_SLOT(pdev->devfn) << 8;
+	return name | PCI_FUNC(pdev->devfn);
+}
+
+static int
+nouveau_cli_create(struct pci_dev *pdev, const char *name,
+		   int size, void **pcli)
+{
+	struct nouveau_cli *cli;
+	int ret;
+
+	ret = nouveau_client_create_(name, nouveau_name(pdev), nouveau_config,
+				     nouveau_debug, size, pcli);
+	cli = *pcli;
+	if (ret)
+		return ret;
+
+	mutex_init(&cli->mutex);
+	return 0;
+}
+
+static void
+nouveau_cli_destroy(struct nouveau_cli *cli)
+{
+	struct nouveau_object *client = nv_object(cli);
+	nouveau_vm_ref(NULL, &cli->base.vm, NULL);
+	nouveau_client_fini(&cli->base, false);
+	atomic_set(&client->refcount, 1);
+	nouveau_object_ref(NULL, &client);
+}
+
+static void
+nouveau_accel_fini(struct nouveau_drm *drm)
+{
+	nouveau_gpuobj_ref(NULL, &drm->notify);
+	nouveau_channel_del(&drm->channel);
+	nouveau_channel_del(&drm->cechan);
+	if (drm->fence)
+		nouveau_fence(drm)->dtor(drm);
+}
+
+static void
+nouveau_accel_init(struct nouveau_drm *drm)
+{
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_object *object;
+	u32 arg0, arg1;
+	int ret;
+
+	if (nouveau_noaccel)
+		return;
+
+	/* initialise synchronisation routines */
+	if      (device->card_type < NV_10) ret = nv04_fence_create(drm);
+	else if (device->chipset   <  0x84) ret = nv10_fence_create(drm);
+	else if (device->card_type < NV_C0) ret = nv84_fence_create(drm);
+	else                                ret = nvc0_fence_create(drm);
+	if (ret) {
+		NV_ERROR(drm, "failed to initialise sync subsystem, %d\n", ret);
+		nouveau_accel_fini(drm);
+		return;
+	}
+
+	if (device->card_type >= NV_E0) {
+		ret = nouveau_channel_new(drm, &drm->client, NVDRM_DEVICE,
+					  NVDRM_CHAN + 1,
+					  NVE0_CHANNEL_IND_ENGINE_CE0 |
+					  NVE0_CHANNEL_IND_ENGINE_CE1, 0,
+					  &drm->cechan);
+		if (ret)
+			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
+
+		arg0 = NVE0_CHANNEL_IND_ENGINE_GR;
+		arg1 = 0;
+	} else {
+		arg0 = NvDmaFB;
+		arg1 = NvDmaTT;
+	}
+
+	ret = nouveau_channel_new(drm, &drm->client, NVDRM_DEVICE, NVDRM_CHAN,
+				  arg0, arg1, &drm->channel);
+	if (ret) {
+		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
+		nouveau_accel_fini(drm);
+		return;
+	}
+
+	if (device->card_type < NV_C0) {
+		ret = nouveau_gpuobj_new(drm->device, NULL, 32, 0, 0,
+					&drm->notify);
+		if (ret) {
+			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
+			nouveau_accel_fini(drm);
+			return;
+		}
+
+		ret = nouveau_object_new(nv_object(drm),
+					 drm->channel->handle, NvNotify0,
+					 0x003d, &(struct nv_dma_class) {
+						.flags = NV_DMA_TARGET_VRAM |
+							 NV_DMA_ACCESS_RDWR,
+						.start = drm->notify->addr,
+						.limit = drm->notify->addr + 31
+						}, sizeof(struct nv_dma_class),
+					 &object);
+		if (ret) {
+			nouveau_accel_fini(drm);
+			return;
+		}
+	}
+
+
+	nouveau_bo_move_init(drm);
+}
+
+static int __devinit
+nouveau_drm_probe(struct pci_dev *pdev, const struct pci_device_id *pent)
+{
+	struct nouveau_device *device;
+	struct apertures_struct *aper;
+	bool boot = false;
+	int ret;
+
+	/* remove conflicting drivers (vesafb, efifb etc) */
+	aper = alloc_apertures(3);
+	if (!aper)
+		return -ENOMEM;
+
+	aper->ranges[0].base = pci_resource_start(pdev, 1);
+	aper->ranges[0].size = pci_resource_len(pdev, 1);
+	aper->count = 1;
+
+	if (pci_resource_len(pdev, 2)) {
+		aper->ranges[aper->count].base = pci_resource_start(pdev, 2);
+		aper->ranges[aper->count].size = pci_resource_len(pdev, 2);
+		aper->count++;
+	}
+
+	if (pci_resource_len(pdev, 3)) {
+		aper->ranges[aper->count].base = pci_resource_start(pdev, 3);
+		aper->ranges[aper->count].size = pci_resource_len(pdev, 3);
+		aper->count++;
+	}
+
+#ifdef CONFIG_X86
+	boot = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+#endif
+	remove_conflicting_framebuffers(aper, "nouveaufb", boot);
+
+	ret = nouveau_device_create(pdev, nouveau_name(pdev), pci_name(pdev),
+				    nouveau_config, nouveau_debug, &device);
+	if (ret)
+		return ret;
+
+	pci_set_master(pdev);
+
+	ret = drm_get_pci_dev(pdev, pent, &driver);
+	if (ret) {
+		nouveau_object_ref(NULL, (struct nouveau_object **)&device);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+nouveau_drm_load(struct drm_device *dev, unsigned long flags)
+{
+	struct pci_dev *pdev = dev->pdev;
+	struct nouveau_device *device;
+	struct nouveau_drm *drm;
+	int ret;
+
+	ret = nouveau_cli_create(pdev, "DRM", sizeof(*drm), (void**)&drm);
+	if (ret)
+		return ret;
+
+	dev->dev_private = drm;
+	drm->dev = dev;
+
+	INIT_LIST_HEAD(&drm->clients);
+	spin_lock_init(&drm->tile.lock);
+
+	/* make sure AGP controller is in a consistent state before we
+	 * (possibly) execute vbios init tables (see nouveau_agp.h)
+	 */
+	if (drm_pci_device_is_agp(dev) && dev->agp) {
+		/* dummy device object, doesn't init anything, but allows
+		 * agp code access to registers
+		 */
+		ret = nouveau_object_new(nv_object(drm), NVDRM_CLIENT,
+					 NVDRM_DEVICE, 0x0080,
+					 &(struct nv_device_class) {
+						.device = ~0,
+						.disable =
+						 ~(NV_DEVICE_DISABLE_MMIO |
+						   NV_DEVICE_DISABLE_IDENTIFY),
+						.debug0 = ~0,
+					 }, sizeof(struct nv_device_class),
+					 &drm->device);
+		if (ret)
+			goto fail_device;
+
+		nouveau_agp_reset(drm);
+		nouveau_object_del(nv_object(drm), NVDRM_CLIENT, NVDRM_DEVICE);
+	}
+
+	ret = nouveau_object_new(nv_object(drm), NVDRM_CLIENT, NVDRM_DEVICE,
+				 0x0080, &(struct nv_device_class) {
+					.device = ~0,
+					.disable = 0,
+					.debug0 = 0,
+				 }, sizeof(struct nv_device_class),
+				 &drm->device);
+	if (ret)
+		goto fail_device;
+
+	/* workaround an odd issue on nvc1 by disabling the device's
+	 * nosnoop capability.  hopefully won't cause issues until a
+	 * better fix is found - assuming there is one...
+	 */
+	device = nv_device(drm->device);
+	if (nv_device(drm->device)->chipset == 0xc1)
+		nv_mask(device, 0x00088080, 0x00000800, 0x00000000);
+
+	nouveau_vga_init(drm);
+	nouveau_agp_init(drm);
+
+	if (device->card_type >= NV_50) {
+		ret = nouveau_vm_new(nv_device(drm->device), 0, (1ULL << 40),
+				     0x1000, &drm->client.base.vm);
+		if (ret)
+			goto fail_device;
+	}
+
+	ret = nouveau_ttm_init(drm);
+	if (ret)
+		goto fail_ttm;
+
+	ret = nouveau_bios_init(dev);
+	if (ret)
+		goto fail_bios;
+
+	ret = nouveau_irq_init(dev);
+	if (ret)
+		goto fail_irq;
+
+	ret = nouveau_display_create(dev);
+	if (ret)
+		goto fail_dispctor;
+
+	if (dev->mode_config.num_crtc) {
+		ret = nouveau_display_init(dev);
+		if (ret)
+			goto fail_dispinit;
+	}
+
+	nouveau_pm_init(dev);
+
+	nouveau_accel_init(drm);
+	nouveau_fbcon_init(dev);
+	return 0;
+
+fail_dispinit:
+	nouveau_display_destroy(dev);
+fail_dispctor:
+	nouveau_irq_fini(dev);
+fail_irq:
+	nouveau_bios_takedown(dev);
+fail_bios:
+	nouveau_ttm_fini(drm);
+fail_ttm:
+	nouveau_agp_fini(drm);
+	nouveau_vga_fini(drm);
+fail_device:
+	nouveau_cli_destroy(&drm->client);
+	return ret;
+}
+
+static int
+nouveau_drm_unload(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	nouveau_fbcon_fini(dev);
+	nouveau_accel_fini(drm);
+
+	nouveau_pm_fini(dev);
+
+	nouveau_display_fini(dev);
+	nouveau_display_destroy(dev);
+
+	nouveau_irq_fini(dev);
+	nouveau_bios_takedown(dev);
+
+	nouveau_ttm_fini(drm);
+	nouveau_agp_fini(drm);
+	nouveau_vga_fini(drm);
+
+	nouveau_cli_destroy(&drm->client);
+	return 0;
+}
+
+static void
+nouveau_drm_remove(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_object *device;
+
+	device = drm->client.base.device;
+	drm_put_dev(dev);
+
+	nouveau_object_ref(NULL, &device);
+	nouveau_object_debug();
+}
+
+int
+nouveau_drm_suspend(struct pci_dev *pdev, pm_message_t pm_state)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_cli *cli;
+	int ret;
+
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF ||
+	    pm_state.event == PM_EVENT_PRETHAW)
+		return 0;
+
+	NV_INFO(drm, "suspending fbcon...\n");
+	nouveau_fbcon_set_suspend(dev, 1);
+
+	NV_INFO(drm, "suspending display...\n");
+	ret = nouveau_display_suspend(dev);
+	if (ret)
+		return ret;
+
+	NV_INFO(drm, "evicting buffers...\n");
+	ttm_bo_evict_mm(&drm->ttm.bdev, TTM_PL_VRAM);
+
+	if (drm->fence && nouveau_fence(drm)->suspend) {
+		if (!nouveau_fence(drm)->suspend(drm))
+			return -ENOMEM;
+	}
+
+	NV_INFO(drm, "suspending client object trees...\n");
+	list_for_each_entry(cli, &drm->clients, head) {
+		ret = nouveau_client_fini(&cli->base, true);
+		if (ret)
+			goto fail_client;
+	}
+
+	ret = nouveau_client_fini(&drm->client.base, true);
+	if (ret)
+		goto fail_client;
+
+	nouveau_agp_fini(drm);
+
+	pci_save_state(pdev);
+	if (pm_state.event == PM_EVENT_SUSPEND) {
+		pci_disable_device(pdev);
+		pci_set_power_state(pdev, PCI_D3hot);
+	}
+
+	return 0;
+
+fail_client:
+	list_for_each_entry_continue_reverse(cli, &drm->clients, head) {
+		nouveau_client_init(&cli->base);
+	}
+
+	NV_INFO(drm, "resuming display...\n");
+	nouveau_display_resume(dev);
+	return ret;
+}
+
+int
+nouveau_drm_resume(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_cli *cli;
+	int ret;
+
+	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+		return 0;
+
+	NV_INFO(drm, "re-enabling device...\n");
+	pci_set_power_state(pdev, PCI_D0);
+	pci_restore_state(pdev);
+	ret = pci_enable_device(pdev);
+	if (ret)
+		return ret;
+	pci_set_master(pdev);
+
+	nouveau_agp_reset(drm);
+
+	NV_INFO(drm, "resuming client object trees...\n");
+	nouveau_client_init(&drm->client.base);
+	nouveau_agp_init(drm);
+
+	list_for_each_entry(cli, &drm->clients, head) {
+		nouveau_client_init(&cli->base);
+	}
+
+	if (drm->fence && nouveau_fence(drm)->resume)
+		nouveau_fence(drm)->resume(drm);
+
+	nouveau_run_vbios_init(dev);
+	nouveau_irq_postinstall(dev);
+	nouveau_pm_resume(dev);
+
+	NV_INFO(drm, "resuming display...\n");
+	nouveau_display_resume(dev);
+	return 0;
+}
+
+static int
+nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
+{
+	struct pci_dev *pdev = dev->pdev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_cli *cli;
+	char name[16];
+	int ret;
+
+	snprintf(name, sizeof(name), "%d", pid_nr(fpriv->pid));
+
+	ret = nouveau_cli_create(pdev, name, sizeof(*cli), (void **)&cli);
+	if (ret)
+		return ret;
+
+	if (nv_device(drm->device)->card_type >= NV_50) {
+		ret = nouveau_vm_new(nv_device(drm->device), 0, (1ULL << 40),
+				     0x1000, &cli->base.vm);
+		if (ret) {
+			nouveau_cli_destroy(cli);
+			return ret;
+		}
+	}
+
+	fpriv->driver_priv = cli;
+
+	mutex_lock(&drm->client.mutex);
+	list_add(&cli->head, &drm->clients);
+	mutex_unlock(&drm->client.mutex);
+	return 0;
+}
+
+static void
+nouveau_drm_preclose(struct drm_device *dev, struct drm_file *fpriv)
+{
+	struct nouveau_cli *cli = nouveau_cli(fpriv);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	if (cli->abi16)
+		nouveau_abi16_fini(cli->abi16);
+
+	mutex_lock(&drm->client.mutex);
+	list_del(&cli->head);
+	mutex_unlock(&drm->client.mutex);
+}
+
+static void
+nouveau_drm_postclose(struct drm_device *dev, struct drm_file *fpriv)
+{
+	struct nouveau_cli *cli = nouveau_cli(fpriv);
+	nouveau_cli_destroy(cli);
+}
+
+static struct drm_ioctl_desc
+nouveau_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH),
+};
+
+static const struct file_operations
+nouveau_driver_fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.mmap = nouveau_ttm_mmap,
+	.poll = drm_poll,
+	.fasync = drm_fasync,
+	.read = drm_read,
+#if defined(CONFIG_COMPAT)
+	.compat_ioctl = nouveau_compat_ioctl,
+#endif
+	.llseek = noop_llseek,
+};
+
+static struct drm_driver
+driver = {
+	.driver_features =
+		DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG |
+		DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM |
+		DRIVER_MODESET | DRIVER_PRIME,
+
+	.load = nouveau_drm_load,
+	.unload = nouveau_drm_unload,
+	.open = nouveau_drm_open,
+	.preclose = nouveau_drm_preclose,
+	.postclose = nouveau_drm_postclose,
+	.lastclose = nouveau_vga_lastclose,
+
+	.irq_preinstall = nouveau_irq_preinstall,
+	.irq_postinstall = nouveau_irq_postinstall,
+	.irq_uninstall = nouveau_irq_uninstall,
+	.irq_handler = nouveau_irq_handler,
+
+	.get_vblank_counter = drm_vblank_count,
+	.enable_vblank = nouveau_vblank_enable,
+	.disable_vblank = nouveau_vblank_disable,
+
+	.ioctls = nouveau_ioctls,
+	.fops = &nouveau_driver_fops,
+
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_export = nouveau_gem_prime_export,
+	.gem_prime_import = nouveau_gem_prime_import,
+
+	.gem_init_object = nouveau_gem_object_new,
+	.gem_free_object = nouveau_gem_object_del,
+	.gem_open_object = nouveau_gem_object_open,
+	.gem_close_object = nouveau_gem_object_close,
+
+	.dumb_create = nouveau_display_dumb_create,
+	.dumb_map_offset = nouveau_display_dumb_map_offset,
+	.dumb_destroy = nouveau_display_dumb_destroy,
+
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+#ifdef GIT_REVISION
+	.date = GIT_REVISION,
+#else
+	.date = DRIVER_DATE,
+#endif
+	.major = DRIVER_MAJOR,
+	.minor = DRIVER_MINOR,
+	.patchlevel = DRIVER_PATCHLEVEL,
+};
+
+static struct pci_device_id
+nouveau_drm_pci_table[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID),
+		.class = PCI_BASE_CLASS_DISPLAY << 16,
+		.class_mask  = 0xff << 16,
+	},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA_SGS, PCI_ANY_ID),
+		.class = PCI_BASE_CLASS_DISPLAY << 16,
+		.class_mask  = 0xff << 16,
+	},
+	{}
+};
+
+static struct pci_driver
+nouveau_drm_pci_driver = {
+	.name = "nouveau",
+	.id_table = nouveau_drm_pci_table,
+	.probe = nouveau_drm_probe,
+	.remove = nouveau_drm_remove,
+	.suspend = nouveau_drm_suspend,
+	.resume = nouveau_drm_resume,
+};
+
+static int __init
+nouveau_drm_init(void)
+{
+	driver.num_ioctls = ARRAY_SIZE(nouveau_ioctls);
+
+	if (nouveau_modeset == -1) {
+#ifdef CONFIG_VGA_CONSOLE
+		if (vgacon_text_force())
+			nouveau_modeset = 0;
+		else
+#endif
+			nouveau_modeset = 1;
+	}
+
+	if (!nouveau_modeset)
+		return 0;
+
+	nouveau_register_dsm_handler();
+	return drm_pci_init(&driver, &nouveau_drm_pci_driver);
+}
+
+static void __exit
+nouveau_drm_exit(void)
+{
+	if (!nouveau_modeset)
+		return;
+
+	drm_pci_exit(&driver, &nouveau_drm_pci_driver);
+	nouveau_unregister_dsm_handler();
+}
+
+module_init(nouveau_drm_init);
+module_exit(nouveau_drm_exit);
+
+MODULE_DEVICE_TABLE(pci, nouveau_drm_pci_table);
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.h b/drivers/gpu/drm/nouveau/nouveau_drm.h
new file mode 100644
index 00000000000..81947121754
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.h
@@ -0,0 +1,144 @@
+#ifndef __NOUVEAU_DRMCLI_H__
+#define __NOUVEAU_DRMCLI_H__
+
+#define DRIVER_AUTHOR		"Nouveau Project"
+#define DRIVER_EMAIL		"nouveau@lists.freedesktop.org"
+
+#define DRIVER_NAME		"nouveau"
+#define DRIVER_DESC		"nVidia Riva/TNT/GeForce/Quadro/Tesla"
+#define DRIVER_DATE		"20120801"
+
+#define DRIVER_MAJOR		1
+#define DRIVER_MINOR		1
+#define DRIVER_PATCHLEVEL	0
+
+#include <core/client.h>
+
+#include <subdev/vm.h>
+
+#include <drmP.h>
+#include <drm/nouveau_drm.h>
+
+#include <drm/ttm/ttm_bo_api.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_memory.h>
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_page_alloc.h>
+
+struct nouveau_channel;
+
+#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
+
+#include "nouveau_fence.h"
+#include "nouveau_bios.h"
+
+struct nouveau_drm_tile {
+	struct nouveau_fence *fence;
+	bool used;
+};
+
+enum nouveau_drm_handle {
+	NVDRM_CLIENT = 0xffffffff,
+	NVDRM_DEVICE = 0xdddddddd,
+	NVDRM_PUSH   = 0xbbbb0000, /* |= client chid */
+	NVDRM_CHAN   = 0xcccc0000, /* |= client chid */
+};
+
+struct nouveau_cli {
+	struct nouveau_client base;
+	struct list_head head;
+	struct mutex mutex;
+	void *abi16;
+};
+
+static inline struct nouveau_cli *
+nouveau_cli(struct drm_file *fpriv)
+{
+	return fpriv ? fpriv->driver_priv : NULL;
+}
+
+struct nouveau_drm {
+	struct nouveau_cli client;
+	struct drm_device *dev;
+
+	struct nouveau_object *device;
+	struct list_head clients;
+
+	struct {
+		enum {
+			UNKNOWN = 0,
+			DISABLE = 1,
+			ENABLED = 2
+		} stat;
+		u32 base;
+		u32 size;
+	} agp;
+
+	/* TTM interface support */
+	struct {
+		struct drm_global_reference mem_global_ref;
+		struct ttm_bo_global_ref bo_global_ref;
+		struct ttm_bo_device bdev;
+		atomic_t validate_sequence;
+		int (*move)(struct nouveau_channel *,
+			    struct ttm_buffer_object *,
+			    struct ttm_mem_reg *, struct ttm_mem_reg *);
+		int mtrr;
+	} ttm;
+
+	/* GEM interface support */
+	struct {
+		u64 vram_available;
+		u64 gart_available;
+	} gem;
+
+	/* synchronisation */
+	void *fence;
+
+	/* context for accelerated drm-internal operations */
+	struct nouveau_channel *cechan;
+	struct nouveau_channel *channel;
+	struct nouveau_gpuobj *notify;
+	struct nouveau_fbdev *fbcon;
+
+	/* nv10-nv40 tiling regions */
+	struct {
+		struct nouveau_drm_tile reg[15];
+		spinlock_t lock;
+	} tile;
+
+	/* modesetting */
+	struct nvbios vbios;
+	struct nouveau_display *display;
+	struct backlight_device *backlight;
+
+	/* power management */
+	struct nouveau_pm *pm;
+};
+
+static inline struct nouveau_drm *
+nouveau_drm(struct drm_device *dev)
+{
+	return dev->dev_private;
+}
+
+static inline struct nouveau_device *
+nouveau_dev(struct drm_device *dev)
+{
+	return nv_device(nouveau_drm(dev)->device);
+}
+
+int nouveau_drm_suspend(struct pci_dev *, pm_message_t);
+int nouveau_drm_resume(struct pci_dev *);
+
+#define NV_FATAL(cli, fmt, args...) nv_fatal((cli), fmt, ##args)
+#define NV_ERROR(cli, fmt, args...) nv_error((cli), fmt, ##args)
+#define NV_WARN(cli, fmt, args...) nv_warn((cli), fmt, ##args)
+#define NV_INFO(cli, fmt, args...) nv_info((cli), fmt, ##args)
+#define NV_DEBUG(cli, fmt, args...) do {                                       \
+	if (drm_debug & DRM_UT_DRIVER)                                         \
+		nv_info((cli), fmt, ##args);                                   \
+} while (0)
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.c b/drivers/gpu/drm/nouveau/nouveau_drv.c
deleted file mode 100644
index 9a36f5f39b0..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_drv.c
+++ /dev/null
@@ -1,513 +0,0 @@
-/*
- * Copyright 2005 Stephane Marchesin.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <linux/console.h>
-#include <linux/module.h>
-
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
-#include "nouveau_drv.h"
-#include "nouveau_abi16.h"
-#include "nouveau_hw.h"
-#include "nouveau_fb.h"
-#include "nouveau_fbcon.h"
-#include "nouveau_pm.h"
-#include "nouveau_fifo.h"
-#include "nv50_display.h"
-
-#include "drm_pciids.h"
-
-MODULE_PARM_DESC(agpmode, "AGP mode (0 to disable AGP)");
-int nouveau_agpmode = -1;
-module_param_named(agpmode, nouveau_agpmode, int, 0400);
-
-MODULE_PARM_DESC(modeset, "Enable kernel modesetting");
-int nouveau_modeset = -1;
-module_param_named(modeset, nouveau_modeset, int, 0400);
-
-MODULE_PARM_DESC(vbios, "Override default VBIOS location");
-char *nouveau_vbios;
-module_param_named(vbios, nouveau_vbios, charp, 0400);
-
-MODULE_PARM_DESC(vram_pushbuf, "Force DMA push buffers to be in VRAM");
-int nouveau_vram_pushbuf;
-module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400);
-
-MODULE_PARM_DESC(vram_notify, "Force DMA notifiers to be in VRAM");
-int nouveau_vram_notify = 0;
-module_param_named(vram_notify, nouveau_vram_notify, int, 0400);
-
-MODULE_PARM_DESC(vram_type, "Override detected VRAM type");
-char *nouveau_vram_type;
-module_param_named(vram_type, nouveau_vram_type, charp, 0400);
-
-MODULE_PARM_DESC(duallink, "Allow dual-link TMDS (>=GeForce 8)");
-int nouveau_duallink = 1;
-module_param_named(duallink, nouveau_duallink, int, 0400);
-
-MODULE_PARM_DESC(uscript_lvds, "LVDS output script table ID (>=GeForce 8)");
-int nouveau_uscript_lvds = -1;
-module_param_named(uscript_lvds, nouveau_uscript_lvds, int, 0400);
-
-MODULE_PARM_DESC(uscript_tmds, "TMDS output script table ID (>=GeForce 8)");
-int nouveau_uscript_tmds = -1;
-module_param_named(uscript_tmds, nouveau_uscript_tmds, int, 0400);
-
-MODULE_PARM_DESC(ignorelid, "Ignore ACPI lid status");
-int nouveau_ignorelid = 0;
-module_param_named(ignorelid, nouveau_ignorelid, int, 0400);
-
-MODULE_PARM_DESC(noaccel, "Disable all acceleration");
-int nouveau_noaccel = -1;
-module_param_named(noaccel, nouveau_noaccel, int, 0400);
-
-MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
-int nouveau_nofbaccel = 0;
-module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400);
-
-MODULE_PARM_DESC(force_post, "Force POST");
-int nouveau_force_post = 0;
-module_param_named(force_post, nouveau_force_post, int, 0400);
-
-MODULE_PARM_DESC(override_conntype, "Ignore DCB connector type");
-int nouveau_override_conntype = 0;
-module_param_named(override_conntype, nouveau_override_conntype, int, 0400);
-
-MODULE_PARM_DESC(tv_disable, "Disable TV-out detection");
-int nouveau_tv_disable = 0;
-module_param_named(tv_disable, nouveau_tv_disable, int, 0400);
-
-MODULE_PARM_DESC(tv_norm, "Default TV norm.\n"
-		 "\t\tSupported: PAL, PAL-M, PAL-N, PAL-Nc, NTSC-M, NTSC-J,\n"
-		 "\t\t\thd480i, hd480p, hd576i, hd576p, hd720p, hd1080i.\n"
-		 "\t\tDefault: PAL\n"
-		 "\t\t*NOTE* Ignored for cards with external TV encoders.");
-char *nouveau_tv_norm;
-module_param_named(tv_norm, nouveau_tv_norm, charp, 0400);
-
-MODULE_PARM_DESC(reg_debug, "Register access debug bitmask:\n"
-		"\t\t0x1 mc, 0x2 video, 0x4 fb, 0x8 extdev,\n"
-		"\t\t0x10 crtc, 0x20 ramdac, 0x40 vgacrtc, 0x80 rmvio,\n"
-		"\t\t0x100 vgaattr, 0x200 EVO (G80+)");
-int nouveau_reg_debug;
-module_param_named(reg_debug, nouveau_reg_debug, int, 0600);
-
-MODULE_PARM_DESC(perflvl, "Performance level (default: boot)");
-char *nouveau_perflvl;
-module_param_named(perflvl, nouveau_perflvl, charp, 0400);
-
-MODULE_PARM_DESC(perflvl_wr, "Allow perflvl changes (warning: dangerous!)");
-int nouveau_perflvl_wr;
-module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400);
-
-MODULE_PARM_DESC(msi, "Enable MSI (default: off)");
-int nouveau_msi;
-module_param_named(msi, nouveau_msi, int, 0400);
-
-MODULE_PARM_DESC(ctxfw, "Use external HUB/GPC ucode (fermi)");
-int nouveau_ctxfw;
-module_param_named(ctxfw, nouveau_ctxfw, int, 0400);
-
-MODULE_PARM_DESC(mxmdcb, "Santise DCB table according to MXM-SIS");
-int nouveau_mxmdcb = 1;
-module_param_named(mxmdcb, nouveau_mxmdcb, int, 0400);
-
-int nouveau_fbpercrtc;
-#if 0
-module_param_named(fbpercrtc, nouveau_fbpercrtc, int, 0400);
-#endif
-
-static struct pci_device_id pciidlist[] = {
-	{
-		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID),
-		.class = PCI_BASE_CLASS_DISPLAY << 16,
-		.class_mask  = 0xff << 16,
-	},
-	{
-		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA_SGS, PCI_ANY_ID),
-		.class = PCI_BASE_CLASS_DISPLAY << 16,
-		.class_mask  = 0xff << 16,
-	},
-	{}
-};
-
-MODULE_DEVICE_TABLE(pci, pciidlist);
-
-static struct drm_driver driver;
-
-static int __devinit
-nouveau_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	return drm_get_pci_dev(pdev, ent, &driver);
-}
-
-static void
-nouveau_pci_remove(struct pci_dev *pdev)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-
-	drm_put_dev(dev);
-}
-
-int
-nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct nouveau_channel *chan;
-	struct drm_crtc *crtc;
-	int ret, i, e;
-
-	if (pm_state.event == PM_EVENT_PRETHAW)
-		return 0;
-
-	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
-		return 0;
-
-	NV_INFO(dev, "Disabling display...\n");
-	nouveau_display_fini(dev);
-
-	NV_INFO(dev, "Disabling fbcon...\n");
-	nouveau_fbcon_set_suspend(dev, 1);
-
-	NV_INFO(dev, "Unpinning framebuffer(s)...\n");
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_framebuffer *nouveau_fb;
-
-		nouveau_fb = nouveau_framebuffer(crtc->fb);
-		if (!nouveau_fb || !nouveau_fb->nvbo)
-			continue;
-
-		nouveau_bo_unpin(nouveau_fb->nvbo);
-	}
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-
-		nouveau_bo_unmap(nv_crtc->cursor.nvbo);
-		nouveau_bo_unpin(nv_crtc->cursor.nvbo);
-	}
-
-	NV_INFO(dev, "Evicting buffers...\n");
-	ttm_bo_evict_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
-
-	NV_INFO(dev, "Idling channels...\n");
-	for (i = 0; i < (pfifo ? pfifo->channels : 0); i++) {
-		chan = dev_priv->channels.ptr[i];
-
-		if (chan && chan->pushbuf_bo)
-			nouveau_channel_idle(chan);
-	}
-
-	for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
-		if (!dev_priv->eng[e])
-			continue;
-
-		ret = dev_priv->eng[e]->fini(dev, e, true);
-		if (ret) {
-			NV_ERROR(dev, "... engine %d failed: %d\n", e, ret);
-			goto out_abort;
-		}
-	}
-
-	ret = pinstmem->suspend(dev);
-	if (ret) {
-		NV_ERROR(dev, "... failed: %d\n", ret);
-		goto out_abort;
-	}
-
-	NV_INFO(dev, "Suspending GPU objects...\n");
-	ret = nouveau_gpuobj_suspend(dev);
-	if (ret) {
-		NV_ERROR(dev, "... failed: %d\n", ret);
-		pinstmem->resume(dev);
-		goto out_abort;
-	}
-
-	NV_INFO(dev, "And we're gone!\n");
-	pci_save_state(pdev);
-	if (pm_state.event == PM_EVENT_SUSPEND) {
-		pci_disable_device(pdev);
-		pci_set_power_state(pdev, PCI_D3hot);
-	}
-
-	return 0;
-
-out_abort:
-	NV_INFO(dev, "Re-enabling acceleration..\n");
-	for (e = e + 1; e < NVOBJ_ENGINE_NR; e++) {
-		if (dev_priv->eng[e])
-			dev_priv->eng[e]->init(dev, e);
-	}
-	return ret;
-}
-
-int
-nouveau_pci_resume(struct pci_dev *pdev)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_engine *engine = &dev_priv->engine;
-	struct drm_crtc *crtc;
-	int ret, i;
-
-	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
-		return 0;
-
-	NV_INFO(dev, "We're back, enabling device...\n");
-	pci_set_power_state(pdev, PCI_D0);
-	pci_restore_state(pdev);
-	if (pci_enable_device(pdev))
-		return -1;
-	pci_set_master(dev->pdev);
-
-	/* Make sure the AGP controller is in a consistent state */
-	if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
-		nouveau_mem_reset_agp(dev);
-
-	/* Make the CRTCs accessible */
-	engine->display.early_init(dev);
-
-	NV_INFO(dev, "POSTing device...\n");
-	ret = nouveau_run_vbios_init(dev);
-	if (ret)
-		return ret;
-
-	if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
-		ret = nouveau_mem_init_agp(dev);
-		if (ret) {
-			NV_ERROR(dev, "error reinitialising AGP: %d\n", ret);
-			return ret;
-		}
-	}
-
-	NV_INFO(dev, "Restoring GPU objects...\n");
-	nouveau_gpuobj_resume(dev);
-
-	NV_INFO(dev, "Reinitialising engines...\n");
-	engine->instmem.resume(dev);
-	engine->mc.init(dev);
-	engine->timer.init(dev);
-	engine->fb.init(dev);
-	for (i = 0; i < NVOBJ_ENGINE_NR; i++) {
-		if (dev_priv->eng[i])
-			dev_priv->eng[i]->init(dev, i);
-	}
-
-	nouveau_irq_postinstall(dev);
-
-	/* Re-write SKIPS, they'll have been lost over the suspend */
-	if (nouveau_vram_pushbuf) {
-		struct nouveau_channel *chan;
-		int j;
-
-		for (i = 0; i < (pfifo ? pfifo->channels : 0); i++) {
-			chan = dev_priv->channels.ptr[i];
-			if (!chan || !chan->pushbuf_bo)
-				continue;
-
-			for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
-				nouveau_bo_wr32(chan->pushbuf_bo, i, 0);
-		}
-	}
-
-	nouveau_pm_resume(dev);
-
-	NV_INFO(dev, "Restoring mode...\n");
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_framebuffer *nouveau_fb;
-
-		nouveau_fb = nouveau_framebuffer(crtc->fb);
-		if (!nouveau_fb || !nouveau_fb->nvbo)
-			continue;
-
-		nouveau_bo_pin(nouveau_fb->nvbo, TTM_PL_FLAG_VRAM);
-	}
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-
-		ret = nouveau_bo_pin(nv_crtc->cursor.nvbo, TTM_PL_FLAG_VRAM);
-		if (!ret)
-			ret = nouveau_bo_map(nv_crtc->cursor.nvbo);
-		if (ret)
-			NV_ERROR(dev, "Could not pin/map cursor.\n");
-	}
-
-	nouveau_fbcon_set_suspend(dev, 0);
-	nouveau_fbcon_zfill_all(dev);
-
-	nouveau_display_init(dev);
-
-	/* Force CLUT to get re-loaded during modeset */
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-
-		nv_crtc->lut.depth = 0;
-	}
-
-	drm_helper_resume_force_mode(dev);
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-		u32 offset = nv_crtc->cursor.nvbo->bo.offset;
-
-		nv_crtc->cursor.set_offset(nv_crtc, offset);
-		nv_crtc->cursor.set_pos(nv_crtc, nv_crtc->cursor_saved_x,
-						 nv_crtc->cursor_saved_y);
-	}
-
-	return 0;
-}
-
-static struct drm_ioctl_desc nouveau_ioctls[] = {
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_UNLOCKED|DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_UNLOCKED|DRM_AUTH),
-};
-
-static const struct file_operations nouveau_driver_fops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.mmap = nouveau_ttm_mmap,
-	.poll = drm_poll,
-	.fasync = drm_fasync,
-	.read = drm_read,
-#if defined(CONFIG_COMPAT)
-	.compat_ioctl = nouveau_compat_ioctl,
-#endif
-	.llseek = noop_llseek,
-};
-
-static struct drm_driver driver = {
-	.driver_features =
-		DRIVER_USE_AGP | DRIVER_PCI_DMA | DRIVER_SG |
-		DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM |
-		DRIVER_MODESET | DRIVER_PRIME,
-	.load = nouveau_load,
-	.firstopen = nouveau_firstopen,
-	.lastclose = nouveau_lastclose,
-	.unload = nouveau_unload,
-	.open = nouveau_open,
-	.preclose = nouveau_preclose,
-	.postclose = nouveau_postclose,
-#if defined(CONFIG_DRM_NOUVEAU_DEBUG)
-	.debugfs_init = nouveau_debugfs_init,
-	.debugfs_cleanup = nouveau_debugfs_takedown,
-#endif
-	.irq_preinstall = nouveau_irq_preinstall,
-	.irq_postinstall = nouveau_irq_postinstall,
-	.irq_uninstall = nouveau_irq_uninstall,
-	.irq_handler = nouveau_irq_handler,
-	.get_vblank_counter = drm_vblank_count,
-	.enable_vblank = nouveau_vblank_enable,
-	.disable_vblank = nouveau_vblank_disable,
-	.ioctls = nouveau_ioctls,
-	.fops = &nouveau_driver_fops,
-
-	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
-	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
-	.gem_prime_export = nouveau_gem_prime_export,
-	.gem_prime_import = nouveau_gem_prime_import,
-
-	.gem_init_object = nouveau_gem_object_new,
-	.gem_free_object = nouveau_gem_object_del,
-	.gem_open_object = nouveau_gem_object_open,
-	.gem_close_object = nouveau_gem_object_close,
-
-	.dumb_create = nouveau_display_dumb_create,
-	.dumb_map_offset = nouveau_display_dumb_map_offset,
-	.dumb_destroy = nouveau_display_dumb_destroy,
-
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-#ifdef GIT_REVISION
-	.date = GIT_REVISION,
-#else
-	.date = DRIVER_DATE,
-#endif
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
-};
-
-static struct pci_driver nouveau_pci_driver = {
-		.name = DRIVER_NAME,
-		.id_table = pciidlist,
-		.probe = nouveau_pci_probe,
-		.remove = nouveau_pci_remove,
-		.suspend = nouveau_pci_suspend,
-		.resume = nouveau_pci_resume
-};
-
-static int __init nouveau_init(void)
-{
-	driver.num_ioctls = ARRAY_SIZE(nouveau_ioctls);
-
-	if (nouveau_modeset == -1) {
-#ifdef CONFIG_VGA_CONSOLE
-		if (vgacon_text_force())
-			nouveau_modeset = 0;
-		else
-#endif
-			nouveau_modeset = 1;
-	}
-
-	if (!nouveau_modeset)
-		return 0;
-
-	nouveau_register_dsm_handler();
-	return drm_pci_init(&driver, &nouveau_pci_driver);
-}
-
-static void __exit nouveau_exit(void)
-{
-	if (!nouveau_modeset)
-		return;
-
-	drm_pci_exit(&driver, &nouveau_pci_driver);
-	nouveau_unregister_dsm_handler();
-}
-
-module_init(nouveau_init);
-module_exit(nouveau_exit);
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
deleted file mode 100644
index 4f2cc95ce26..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ /dev/null
@@ -1,1655 +0,0 @@
-/*
- * Copyright 2005 Stephane Marchesin.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef __NOUVEAU_DRV_H__
-#define __NOUVEAU_DRV_H__
-
-#define DRIVER_AUTHOR		"Stephane Marchesin"
-#define DRIVER_EMAIL		"nouveau@lists.freedesktop.org"
-
-#define DRIVER_NAME		"nouveau"
-#define DRIVER_DESC		"nVidia Riva/TNT/GeForce"
-#define DRIVER_DATE		"20120316"
-
-#define DRIVER_MAJOR		1
-#define DRIVER_MINOR		0
-#define DRIVER_PATCHLEVEL	0
-
-#define NOUVEAU_FAMILY   0x0000FFFF
-#define NOUVEAU_FLAGS    0xFFFF0000
-
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
-
-struct nouveau_fpriv {
-	spinlock_t lock;
-	struct list_head channels;
-	struct nouveau_vm *vm;
-};
-
-static inline struct nouveau_fpriv *
-nouveau_fpriv(struct drm_file *file_priv)
-{
-	return file_priv ? file_priv->driver_priv : NULL;
-}
-
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
-#include "nouveau_drm.h"
-#include "nouveau_reg.h"
-#include "nouveau_bios.h"
-#include "nouveau_util.h"
-
-struct nouveau_grctx;
-struct nouveau_mem;
-#include "nouveau_vm.h"
-
-#define MAX_NUM_DCB_ENTRIES 16
-
-#define NOUVEAU_MAX_CHANNEL_NR 4096
-#define NOUVEAU_MAX_TILE_NR 15
-
-struct nouveau_mem {
-	struct drm_device *dev;
-
-	struct nouveau_vma bar_vma;
-	struct nouveau_vma vma[2];
-	u8  page_shift;
-
-	struct drm_mm_node *tag;
-	struct list_head regions;
-	dma_addr_t *pages;
-	u32 memtype;
-	u64 offset;
-	u64 size;
-	struct sg_table *sg;
-};
-
-struct nouveau_tile_reg {
-	bool used;
-	uint32_t addr;
-	uint32_t limit;
-	uint32_t pitch;
-	uint32_t zcomp;
-	struct drm_mm_node *tag_mem;
-	struct nouveau_fence *fence;
-};
-
-struct nouveau_bo {
-	struct ttm_buffer_object bo;
-	struct ttm_placement placement;
-	u32 valid_domains;
-	u32 placements[3];
-	u32 busy_placements[3];
-	struct ttm_bo_kmap_obj kmap;
-	struct list_head head;
-
-	/* protected by ttm_bo_reserve() */
-	struct drm_file *reserved_by;
-	struct list_head entry;
-	int pbbo_index;
-	bool validate_mapped;
-
-	struct list_head vma_list;
-	unsigned page_shift;
-
-	uint32_t tile_mode;
-	uint32_t tile_flags;
-	struct nouveau_tile_reg *tile;
-
-	struct drm_gem_object *gem;
-	int pin_refcnt;
-
-	struct ttm_bo_kmap_obj dma_buf_vmap;
-	int vmapping_count;
-};
-
-#define nouveau_bo_tile_layout(nvbo)				\
-	((nvbo)->tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK)
-
-static inline struct nouveau_bo *
-nouveau_bo(struct ttm_buffer_object *bo)
-{
-	return container_of(bo, struct nouveau_bo, bo);
-}
-
-static inline struct nouveau_bo *
-nouveau_gem_object(struct drm_gem_object *gem)
-{
-	return gem ? gem->driver_private : NULL;
-}
-
-/* TODO: submit equivalent to TTM generic API upstream? */
-static inline void __iomem *
-nvbo_kmap_obj_iovirtual(struct nouveau_bo *nvbo)
-{
-	bool is_iomem;
-	void __iomem *ioptr = (void __force __iomem *)ttm_kmap_obj_virtual(
-						&nvbo->kmap, &is_iomem);
-	WARN_ON_ONCE(ioptr && !is_iomem);
-	return ioptr;
-}
-
-enum nouveau_flags {
-	NV_NFORCE   = 0x10000000,
-	NV_NFORCE2  = 0x20000000
-};
-
-#define NVOBJ_ENGINE_SW		0
-#define NVOBJ_ENGINE_GR		1
-#define NVOBJ_ENGINE_CRYPT	2
-#define NVOBJ_ENGINE_COPY0	3
-#define NVOBJ_ENGINE_COPY1	4
-#define NVOBJ_ENGINE_MPEG	5
-#define NVOBJ_ENGINE_PPP	NVOBJ_ENGINE_MPEG
-#define NVOBJ_ENGINE_BSP	6
-#define NVOBJ_ENGINE_VP		7
-#define NVOBJ_ENGINE_FIFO	14
-#define NVOBJ_ENGINE_FENCE	15
-#define NVOBJ_ENGINE_NR		16
-#define NVOBJ_ENGINE_DISPLAY	(NVOBJ_ENGINE_NR + 0) /*XXX*/
-
-#define NVOBJ_FLAG_DONT_MAP             (1 << 0)
-#define NVOBJ_FLAG_ZERO_ALLOC		(1 << 1)
-#define NVOBJ_FLAG_ZERO_FREE		(1 << 2)
-#define NVOBJ_FLAG_VM			(1 << 3)
-#define NVOBJ_FLAG_VM_USER		(1 << 4)
-
-#define NVOBJ_CINST_GLOBAL	0xdeadbeef
-
-struct nouveau_gpuobj {
-	struct drm_device *dev;
-	struct kref refcount;
-	struct list_head list;
-
-	void *node;
-	u32 *suspend;
-
-	uint32_t flags;
-
-	u32 size;
-	u32 pinst;	/* PRAMIN BAR offset */
-	u32 cinst;	/* Channel offset */
-	u64 vinst;	/* VRAM address */
-	u64 linst;	/* VM address */
-
-	uint32_t engine;
-	uint32_t class;
-
-	void (*dtor)(struct drm_device *, struct nouveau_gpuobj *);
-	void *priv;
-};
-
-struct nouveau_page_flip_state {
-	struct list_head head;
-	struct drm_pending_vblank_event *event;
-	int crtc, bpp, pitch, x, y;
-	uint64_t offset;
-};
-
-enum nouveau_channel_mutex_class {
-	NOUVEAU_UCHANNEL_MUTEX,
-	NOUVEAU_KCHANNEL_MUTEX
-};
-
-struct nouveau_channel {
-	struct drm_device *dev;
-	struct list_head list;
-	int id;
-
-	/* references to the channel data structure */
-	struct kref ref;
-	/* users of the hardware channel resources, the hardware
-	 * context will be kicked off when it reaches zero. */
-	atomic_t users;
-	struct mutex mutex;
-
-	/* owner of this fifo */
-	struct drm_file *file_priv;
-	/* mapping of the fifo itself */
-	struct drm_local_map *map;
-
-	/* mapping of the regs controlling the fifo */
-	void __iomem *user;
-	uint32_t user_get;
-	uint32_t user_get_hi;
-	uint32_t user_put;
-
-	/* DMA push buffer */
-	struct nouveau_gpuobj *pushbuf;
-	struct nouveau_bo     *pushbuf_bo;
-	struct nouveau_vma     pushbuf_vma;
-	uint64_t               pushbuf_base;
-
-	/* Notifier memory */
-	struct nouveau_bo *notifier_bo;
-	struct nouveau_vma notifier_vma;
-	struct drm_mm notifier_heap;
-
-	/* PFIFO context */
-	struct nouveau_gpuobj *ramfc;
-
-	/* Execution engine contexts */
-	void *engctx[NVOBJ_ENGINE_NR];
-
-	/* NV50 VM */
-	struct nouveau_vm     *vm;
-	struct nouveau_gpuobj *vm_pd;
-
-	/* Objects */
-	struct nouveau_gpuobj *ramin; /* Private instmem */
-	struct drm_mm          ramin_heap; /* Private PRAMIN heap */
-	struct nouveau_ramht  *ramht; /* Hash table */
-
-	/* GPU object info for stuff used in-kernel (mm_enabled) */
-	uint32_t m2mf_ntfy;
-	uint32_t vram_handle;
-	uint32_t gart_handle;
-	bool accel_done;
-
-	/* Push buffer state (only for drm's channel on !mm_enabled) */
-	struct {
-		int max;
-		int free;
-		int cur;
-		int put;
-		/* access via pushbuf_bo */
-
-		int ib_base;
-		int ib_max;
-		int ib_free;
-		int ib_put;
-	} dma;
-
-	struct {
-		bool active;
-		char name[32];
-		struct drm_info_list info;
-	} debugfs;
-};
-
-struct nouveau_exec_engine {
-	void (*destroy)(struct drm_device *, int engine);
-	int  (*init)(struct drm_device *, int engine);
-	int  (*fini)(struct drm_device *, int engine, bool suspend);
-	int  (*context_new)(struct nouveau_channel *, int engine);
-	void (*context_del)(struct nouveau_channel *, int engine);
-	int  (*object_new)(struct nouveau_channel *, int engine,
-			   u32 handle, u16 class);
-	void (*set_tile_region)(struct drm_device *dev, int i);
-	void (*tlb_flush)(struct drm_device *, int engine);
-};
-
-struct nouveau_instmem_engine {
-	void	*priv;
-
-	int	(*init)(struct drm_device *dev);
-	void	(*takedown)(struct drm_device *dev);
-	int	(*suspend)(struct drm_device *dev);
-	void	(*resume)(struct drm_device *dev);
-
-	int	(*get)(struct nouveau_gpuobj *, struct nouveau_channel *,
-		       u32 size, u32 align);
-	void	(*put)(struct nouveau_gpuobj *);
-	int	(*map)(struct nouveau_gpuobj *);
-	void	(*unmap)(struct nouveau_gpuobj *);
-
-	void	(*flush)(struct drm_device *);
-};
-
-struct nouveau_mc_engine {
-	int  (*init)(struct drm_device *dev);
-	void (*takedown)(struct drm_device *dev);
-};
-
-struct nouveau_timer_engine {
-	int      (*init)(struct drm_device *dev);
-	void     (*takedown)(struct drm_device *dev);
-	uint64_t (*read)(struct drm_device *dev);
-};
-
-struct nouveau_fb_engine {
-	int num_tiles;
-	struct drm_mm tag_heap;
-	void *priv;
-
-	int  (*init)(struct drm_device *dev);
-	void (*takedown)(struct drm_device *dev);
-
-	void (*init_tile_region)(struct drm_device *dev, int i,
-				 uint32_t addr, uint32_t size,
-				 uint32_t pitch, uint32_t flags);
-	void (*set_tile_region)(struct drm_device *dev, int i);
-	void (*free_tile_region)(struct drm_device *dev, int i);
-};
-
-struct nouveau_display_engine {
-	void *priv;
-	int (*early_init)(struct drm_device *);
-	void (*late_takedown)(struct drm_device *);
-	int (*create)(struct drm_device *);
-	void (*destroy)(struct drm_device *);
-	int (*init)(struct drm_device *);
-	void (*fini)(struct drm_device *);
-
-	struct drm_property *dithering_mode;
-	struct drm_property *dithering_depth;
-	struct drm_property *underscan_property;
-	struct drm_property *underscan_hborder_property;
-	struct drm_property *underscan_vborder_property;
-	/* not really hue and saturation: */
-	struct drm_property *vibrant_hue_property;
-	struct drm_property *color_vibrance_property;
-};
-
-struct nouveau_gpio_engine {
-	spinlock_t lock;
-	struct list_head isr;
-	int (*init)(struct drm_device *);
-	void (*fini)(struct drm_device *);
-	int (*drive)(struct drm_device *, int line, int dir, int out);
-	int (*sense)(struct drm_device *, int line);
-	void (*irq_enable)(struct drm_device *, int line, bool);
-};
-
-struct nouveau_pm_voltage_level {
-	u32 voltage; /* microvolts */
-	u8  vid;
-};
-
-struct nouveau_pm_voltage {
-	bool supported;
-	u8 version;
-	u8 vid_mask;
-
-	struct nouveau_pm_voltage_level *level;
-	int nr_level;
-};
-
-/* Exclusive upper limits */
-#define NV_MEM_CL_DDR2_MAX 8
-#define NV_MEM_WR_DDR2_MAX 9
-#define NV_MEM_CL_DDR3_MAX 17
-#define NV_MEM_WR_DDR3_MAX 17
-#define NV_MEM_CL_GDDR3_MAX 16
-#define NV_MEM_WR_GDDR3_MAX 18
-#define NV_MEM_CL_GDDR5_MAX 21
-#define NV_MEM_WR_GDDR5_MAX 20
-
-struct nouveau_pm_memtiming {
-	int id;
-
-	u32 reg[9];
-	u32 mr[4];
-
-	u8 tCWL;
-
-	u8 odt;
-	u8 drive_strength;
-};
-
-struct nouveau_pm_tbl_header {
-	u8 version;
-	u8 header_len;
-	u8 entry_cnt;
-	u8 entry_len;
-};
-
-struct nouveau_pm_tbl_entry {
-	u8 tWR;
-	u8 tWTR;
-	u8 tCL;
-	u8 tRC;
-	u8 empty_4;
-	u8 tRFC;	/* Byte 5 */
-	u8 empty_6;
-	u8 tRAS;	/* Byte 7 */
-	u8 empty_8;
-	u8 tRP;		/* Byte 9 */
-	u8 tRCDRD;
-	u8 tRCDWR;
-	u8 tRRD;
-	u8 tUNK_13;
-	u8 RAM_FT1;		/* 14, a bitmask of random RAM features */
-	u8 empty_15;
-	u8 tUNK_16;
-	u8 empty_17;
-	u8 tUNK_18;
-	u8 tCWL;
-	u8 tUNK_20, tUNK_21;
-};
-
-struct nouveau_pm_profile;
-struct nouveau_pm_profile_func {
-	void (*destroy)(struct nouveau_pm_profile *);
-	void (*init)(struct nouveau_pm_profile *);
-	void (*fini)(struct nouveau_pm_profile *);
-	struct nouveau_pm_level *(*select)(struct nouveau_pm_profile *);
-};
-
-struct nouveau_pm_profile {
-	const struct nouveau_pm_profile_func *func;
-	struct list_head head;
-	char name[8];
-};
-
-#define NOUVEAU_PM_MAX_LEVEL 8
-struct nouveau_pm_level {
-	struct nouveau_pm_profile profile;
-	struct device_attribute dev_attr;
-	char name[32];
-	int id;
-
-	struct nouveau_pm_memtiming timing;
-	u32 memory;
-	u16 memscript;
-
-	u32 core;
-	u32 shader;
-	u32 rop;
-	u32 copy;
-	u32 daemon;
-	u32 vdec;
-	u32 dom6;
-	u32 unka0;	/* nva3:nvc0 */
-	u32 hub01;	/* nvc0- */
-	u32 hub06;	/* nvc0- */
-	u32 hub07;	/* nvc0- */
-
-	u32 volt_min; /* microvolts */
-	u32 volt_max;
-	u8  fanspeed;
-};
-
-struct nouveau_pm_temp_sensor_constants {
-	u16 offset_constant;
-	s16 offset_mult;
-	s16 offset_div;
-	s16 slope_mult;
-	s16 slope_div;
-};
-
-struct nouveau_pm_threshold_temp {
-	s16 critical;
-	s16 down_clock;
-	s16 fan_boost;
-};
-
-struct nouveau_pm_fan {
-	u32 percent;
-	u32 min_duty;
-	u32 max_duty;
-	u32 pwm_freq;
-	u32 pwm_divisor;
-};
-
-struct nouveau_pm_engine {
-	struct nouveau_pm_voltage voltage;
-	struct nouveau_pm_level perflvl[NOUVEAU_PM_MAX_LEVEL];
-	int nr_perflvl;
-	struct nouveau_pm_temp_sensor_constants sensor_constants;
-	struct nouveau_pm_threshold_temp threshold_temp;
-	struct nouveau_pm_fan fan;
-
-	struct nouveau_pm_profile *profile_ac;
-	struct nouveau_pm_profile *profile_dc;
-	struct nouveau_pm_profile *profile;
-	struct list_head profiles;
-
-	struct nouveau_pm_level boot;
-	struct nouveau_pm_level *cur;
-
-	struct device *hwmon;
-	struct notifier_block acpi_nb;
-
-	int  (*clocks_get)(struct drm_device *, struct nouveau_pm_level *);
-	void *(*clocks_pre)(struct drm_device *, struct nouveau_pm_level *);
-	int (*clocks_set)(struct drm_device *, void *);
-
-	int (*voltage_get)(struct drm_device *);
-	int (*voltage_set)(struct drm_device *, int voltage);
-	int (*pwm_get)(struct drm_device *, int line, u32*, u32*);
-	int (*pwm_set)(struct drm_device *, int line, u32, u32);
-	int (*temp_get)(struct drm_device *);
-};
-
-struct nouveau_vram_engine {
-	struct nouveau_mm mm;
-
-	int  (*init)(struct drm_device *);
-	void (*takedown)(struct drm_device *dev);
-	int  (*get)(struct drm_device *, u64, u32 align, u32 size_nc,
-		    u32 type, struct nouveau_mem **);
-	void (*put)(struct drm_device *, struct nouveau_mem **);
-
-	bool (*flags_valid)(struct drm_device *, u32 tile_flags);
-};
-
-struct nouveau_engine {
-	struct nouveau_instmem_engine instmem;
-	struct nouveau_mc_engine      mc;
-	struct nouveau_timer_engine   timer;
-	struct nouveau_fb_engine      fb;
-	struct nouveau_display_engine display;
-	struct nouveau_gpio_engine    gpio;
-	struct nouveau_pm_engine      pm;
-	struct nouveau_vram_engine    vram;
-};
-
-struct nouveau_pll_vals {
-	union {
-		struct {
-#ifdef __BIG_ENDIAN
-			uint8_t N1, M1, N2, M2;
-#else
-			uint8_t M1, N1, M2, N2;
-#endif
-		};
-		struct {
-			uint16_t NM1, NM2;
-		} __attribute__((packed));
-	};
-	int log2P;
-
-	int refclk;
-};
-
-enum nv04_fp_display_regs {
-	FP_DISPLAY_END,
-	FP_TOTAL,
-	FP_CRTC,
-	FP_SYNC_START,
-	FP_SYNC_END,
-	FP_VALID_START,
-	FP_VALID_END
-};
-
-struct nv04_crtc_reg {
-	unsigned char MiscOutReg;
-	uint8_t CRTC[0xa0];
-	uint8_t CR58[0x10];
-	uint8_t Sequencer[5];
-	uint8_t Graphics[9];
-	uint8_t Attribute[21];
-	unsigned char DAC[768];
-
-	/* PCRTC regs */
-	uint32_t fb_start;
-	uint32_t crtc_cfg;
-	uint32_t cursor_cfg;
-	uint32_t gpio_ext;
-	uint32_t crtc_830;
-	uint32_t crtc_834;
-	uint32_t crtc_850;
-	uint32_t crtc_eng_ctrl;
-
-	/* PRAMDAC regs */
-	uint32_t nv10_cursync;
-	struct nouveau_pll_vals pllvals;
-	uint32_t ramdac_gen_ctrl;
-	uint32_t ramdac_630;
-	uint32_t ramdac_634;
-	uint32_t tv_setup;
-	uint32_t tv_vtotal;
-	uint32_t tv_vskew;
-	uint32_t tv_vsync_delay;
-	uint32_t tv_htotal;
-	uint32_t tv_hskew;
-	uint32_t tv_hsync_delay;
-	uint32_t tv_hsync_delay2;
-	uint32_t fp_horiz_regs[7];
-	uint32_t fp_vert_regs[7];
-	uint32_t dither;
-	uint32_t fp_control;
-	uint32_t dither_regs[6];
-	uint32_t fp_debug_0;
-	uint32_t fp_debug_1;
-	uint32_t fp_debug_2;
-	uint32_t fp_margin_color;
-	uint32_t ramdac_8c0;
-	uint32_t ramdac_a20;
-	uint32_t ramdac_a24;
-	uint32_t ramdac_a34;
-	uint32_t ctv_regs[38];
-};
-
-struct nv04_output_reg {
-	uint32_t output;
-	int head;
-};
-
-struct nv04_mode_state {
-	struct nv04_crtc_reg crtc_reg[2];
-	uint32_t pllsel;
-	uint32_t sel_clk;
-};
-
-enum nouveau_card_type {
-	NV_04      = 0x04,
-	NV_10      = 0x10,
-	NV_20      = 0x20,
-	NV_30      = 0x30,
-	NV_40      = 0x40,
-	NV_50      = 0x50,
-	NV_C0      = 0xc0,
-	NV_D0      = 0xd0,
-	NV_E0      = 0xe0,
-};
-
-struct drm_nouveau_private {
-	struct drm_device *dev;
-	bool noaccel;
-
-	/* the card type, takes NV_* as values */
-	enum nouveau_card_type card_type;
-	/* exact chipset, derived from NV_PMC_BOOT_0 */
-	int chipset;
-	int flags;
-	u32 crystal;
-
-	void __iomem *mmio;
-
-	spinlock_t ramin_lock;
-	void __iomem *ramin;
-	u32 ramin_size;
-	u32 ramin_base;
-	bool ramin_available;
-	struct drm_mm ramin_heap;
-	struct nouveau_exec_engine *eng[NVOBJ_ENGINE_NR];
-	struct list_head gpuobj_list;
-	struct list_head classes;
-
-	struct nouveau_bo *vga_ram;
-
-	/* interrupt handling */
-	void (*irq_handler[32])(struct drm_device *);
-	bool msi_enabled;
-
-	struct {
-		struct drm_global_reference mem_global_ref;
-		struct ttm_bo_global_ref bo_global_ref;
-		struct ttm_bo_device bdev;
-		atomic_t validate_sequence;
-		int (*move)(struct nouveau_channel *,
-			    struct ttm_buffer_object *,
-			    struct ttm_mem_reg *, struct ttm_mem_reg *);
-	} ttm;
-
-	struct {
-		spinlock_t lock;
-		struct drm_mm heap;
-		struct nouveau_bo *bo;
-	} fence;
-
-	struct {
-		spinlock_t lock;
-		struct nouveau_channel *ptr[NOUVEAU_MAX_CHANNEL_NR];
-	} channels;
-
-	struct nouveau_engine engine;
-	struct nouveau_channel *channel;
-
-	/* For PFIFO and PGRAPH. */
-	spinlock_t context_switch_lock;
-
-	/* VM/PRAMIN flush, legacy PRAMIN aperture */
-	spinlock_t vm_lock;
-
-	/* RAMIN configuration, RAMFC, RAMHT and RAMRO offsets */
-	struct nouveau_ramht  *ramht;
-	struct nouveau_gpuobj *ramfc;
-	struct nouveau_gpuobj *ramro;
-
-	uint32_t ramin_rsvd_vram;
-
-	struct {
-		enum {
-			NOUVEAU_GART_NONE = 0,
-			NOUVEAU_GART_AGP,	/* AGP */
-			NOUVEAU_GART_PDMA,	/* paged dma object */
-			NOUVEAU_GART_HW		/* on-chip gart/vm */
-		} type;
-		uint64_t aper_base;
-		uint64_t aper_size;
-		uint64_t aper_free;
-
-		struct ttm_backend_func *func;
-
-		struct {
-			struct page *page;
-			dma_addr_t   addr;
-		} dummy;
-
-		struct nouveau_gpuobj *sg_ctxdma;
-	} gart_info;
-
-	/* nv10-nv40 tiling regions */
-	struct {
-		struct nouveau_tile_reg reg[NOUVEAU_MAX_TILE_NR];
-		spinlock_t lock;
-	} tile;
-
-	/* VRAM/fb configuration */
-	enum {
-		NV_MEM_TYPE_UNKNOWN = 0,
-		NV_MEM_TYPE_STOLEN,
-		NV_MEM_TYPE_SGRAM,
-		NV_MEM_TYPE_SDRAM,
-		NV_MEM_TYPE_DDR1,
-		NV_MEM_TYPE_DDR2,
-		NV_MEM_TYPE_DDR3,
-		NV_MEM_TYPE_GDDR2,
-		NV_MEM_TYPE_GDDR3,
-		NV_MEM_TYPE_GDDR4,
-		NV_MEM_TYPE_GDDR5
-	} vram_type;
-	uint64_t vram_size;
-	uint64_t vram_sys_base;
-	bool vram_rank_B;
-
-	uint64_t fb_available_size;
-	uint64_t fb_mappable_pages;
-	uint64_t fb_aper_free;
-	int fb_mtrr;
-
-	/* BAR control (NV50-) */
-	struct nouveau_vm *bar1_vm;
-	struct nouveau_vm *bar3_vm;
-
-	/* G8x/G9x virtual address space */
-	struct nouveau_vm *chan_vm;
-
-	struct nvbios vbios;
-	u8 *mxms;
-	struct list_head i2c_ports;
-
-	struct nv04_mode_state mode_reg;
-	struct nv04_mode_state saved_reg;
-	uint32_t saved_vga_font[4][16384];
-	uint32_t crtc_owner;
-	uint32_t dac_users[4];
-
-	struct backlight_device *backlight;
-
-	struct {
-		struct dentry *channel_root;
-	} debugfs;
-
-	struct nouveau_fbdev *nfbdev;
-	struct apertures_struct *apertures;
-};
-
-static inline struct drm_nouveau_private *
-nouveau_private(struct drm_device *dev)
-{
-	return dev->dev_private;
-}
-
-static inline struct drm_nouveau_private *
-nouveau_bdev(struct ttm_bo_device *bd)
-{
-	return container_of(bd, struct drm_nouveau_private, ttm.bdev);
-}
-
-static inline int
-nouveau_bo_ref(struct nouveau_bo *ref, struct nouveau_bo **pnvbo)
-{
-	struct nouveau_bo *prev;
-
-	if (!pnvbo)
-		return -EINVAL;
-	prev = *pnvbo;
-
-	*pnvbo = ref ? nouveau_bo(ttm_bo_reference(&ref->bo)) : NULL;
-	if (prev) {
-		struct ttm_buffer_object *bo = &prev->bo;
-
-		ttm_bo_unref(&bo);
-	}
-
-	return 0;
-}
-
-/* nouveau_drv.c */
-extern int nouveau_modeset;
-extern int nouveau_agpmode;
-extern int nouveau_duallink;
-extern int nouveau_uscript_lvds;
-extern int nouveau_uscript_tmds;
-extern int nouveau_vram_pushbuf;
-extern int nouveau_vram_notify;
-extern char *nouveau_vram_type;
-extern int nouveau_fbpercrtc;
-extern int nouveau_tv_disable;
-extern char *nouveau_tv_norm;
-extern int nouveau_reg_debug;
-extern char *nouveau_vbios;
-extern int nouveau_ignorelid;
-extern int nouveau_nofbaccel;
-extern int nouveau_noaccel;
-extern int nouveau_force_post;
-extern int nouveau_override_conntype;
-extern char *nouveau_perflvl;
-extern int nouveau_perflvl_wr;
-extern int nouveau_msi;
-extern int nouveau_ctxfw;
-extern int nouveau_mxmdcb;
-
-extern int nouveau_pci_suspend(struct pci_dev *pdev, pm_message_t pm_state);
-extern int nouveau_pci_resume(struct pci_dev *pdev);
-
-/* nouveau_state.c */
-extern int  nouveau_open(struct drm_device *, struct drm_file *);
-extern void nouveau_preclose(struct drm_device *dev, struct drm_file *);
-extern void nouveau_postclose(struct drm_device *, struct drm_file *);
-extern int  nouveau_load(struct drm_device *, unsigned long flags);
-extern int  nouveau_firstopen(struct drm_device *);
-extern void nouveau_lastclose(struct drm_device *);
-extern int  nouveau_unload(struct drm_device *);
-extern bool nouveau_wait_eq(struct drm_device *, uint64_t timeout,
-			    uint32_t reg, uint32_t mask, uint32_t val);
-extern bool nouveau_wait_ne(struct drm_device *, uint64_t timeout,
-			    uint32_t reg, uint32_t mask, uint32_t val);
-extern bool nouveau_wait_cb(struct drm_device *, u64 timeout,
-			    bool (*cond)(void *), void *);
-extern bool nouveau_wait_for_idle(struct drm_device *);
-extern int  nouveau_card_init(struct drm_device *);
-
-/* nouveau_mem.c */
-extern int  nouveau_mem_vram_init(struct drm_device *);
-extern void nouveau_mem_vram_fini(struct drm_device *);
-extern int  nouveau_mem_gart_init(struct drm_device *);
-extern void nouveau_mem_gart_fini(struct drm_device *);
-extern int  nouveau_mem_init_agp(struct drm_device *);
-extern int  nouveau_mem_reset_agp(struct drm_device *);
-extern void nouveau_mem_close(struct drm_device *);
-extern bool nouveau_mem_flags_valid(struct drm_device *, u32 tile_flags);
-extern int  nouveau_mem_timing_calc(struct drm_device *, u32 freq,
-				    struct nouveau_pm_memtiming *);
-extern void nouveau_mem_timing_read(struct drm_device *,
-				    struct nouveau_pm_memtiming *);
-extern int nouveau_mem_vbios_type(struct drm_device *);
-extern struct nouveau_tile_reg *nv10_mem_set_tiling(
-	struct drm_device *dev, uint32_t addr, uint32_t size,
-	uint32_t pitch, uint32_t flags);
-extern void nv10_mem_put_tile_region(struct drm_device *dev,
-				     struct nouveau_tile_reg *tile,
-				     struct nouveau_fence *fence);
-extern const struct ttm_mem_type_manager_func nouveau_vram_manager;
-extern const struct ttm_mem_type_manager_func nouveau_gart_manager;
-
-/* nouveau_notifier.c */
-extern int  nouveau_notifier_init_channel(struct nouveau_channel *);
-extern void nouveau_notifier_takedown_channel(struct nouveau_channel *);
-extern int  nouveau_notifier_alloc(struct nouveau_channel *, uint32_t handle,
-				   int cout, uint32_t start, uint32_t end,
-				   uint32_t *offset);
-
-/* nouveau_channel.c */
-extern void nouveau_channel_cleanup(struct drm_device *, struct drm_file *);
-extern int  nouveau_channel_alloc(struct drm_device *dev,
-				  struct nouveau_channel **chan,
-				  struct drm_file *file_priv,
-				  uint32_t fb_ctxdma, uint32_t tt_ctxdma);
-extern struct nouveau_channel *
-nouveau_channel_get_unlocked(struct nouveau_channel *);
-extern struct nouveau_channel *
-nouveau_channel_get(struct drm_file *, int id);
-extern void nouveau_channel_put_unlocked(struct nouveau_channel **);
-extern void nouveau_channel_put(struct nouveau_channel **);
-extern void nouveau_channel_ref(struct nouveau_channel *chan,
-				struct nouveau_channel **pchan);
-extern int  nouveau_channel_idle(struct nouveau_channel *chan);
-
-/* nouveau_gpuobj.c */
-#define NVOBJ_ENGINE_ADD(d, e, p) do {                                         \
-	struct drm_nouveau_private *dev_priv = (d)->dev_private;               \
-	dev_priv->eng[NVOBJ_ENGINE_##e] = (p);                                 \
-} while (0)
-
-#define NVOBJ_ENGINE_DEL(d, e) do {                                            \
-	struct drm_nouveau_private *dev_priv = (d)->dev_private;               \
-	dev_priv->eng[NVOBJ_ENGINE_##e] = NULL;                                \
-} while (0)
-
-#define NVOBJ_CLASS(d, c, e) do {                                              \
-	int ret = nouveau_gpuobj_class_new((d), (c), NVOBJ_ENGINE_##e);        \
-	if (ret)                                                               \
-		return ret;                                                    \
-} while (0)
-
-#define NVOBJ_MTHD(d, c, m, e) do {                                            \
-	int ret = nouveau_gpuobj_mthd_new((d), (c), (m), (e));                 \
-	if (ret)                                                               \
-		return ret;                                                    \
-} while (0)
-
-extern int  nouveau_gpuobj_early_init(struct drm_device *);
-extern int  nouveau_gpuobj_init(struct drm_device *);
-extern void nouveau_gpuobj_takedown(struct drm_device *);
-extern int  nouveau_gpuobj_suspend(struct drm_device *dev);
-extern void nouveau_gpuobj_resume(struct drm_device *dev);
-extern int  nouveau_gpuobj_class_new(struct drm_device *, u32 class, u32 eng);
-extern int  nouveau_gpuobj_mthd_new(struct drm_device *, u32 class, u32 mthd,
-				    int (*exec)(struct nouveau_channel *,
-						u32 class, u32 mthd, u32 data));
-extern int  nouveau_gpuobj_mthd_call(struct nouveau_channel *, u32, u32, u32);
-extern int  nouveau_gpuobj_mthd_call2(struct drm_device *, int, u32, u32, u32);
-extern int nouveau_gpuobj_channel_init(struct nouveau_channel *,
-				       uint32_t vram_h, uint32_t tt_h);
-extern void nouveau_gpuobj_channel_takedown(struct nouveau_channel *);
-extern int nouveau_gpuobj_new(struct drm_device *, struct nouveau_channel *,
-			      uint32_t size, int align, uint32_t flags,
-			      struct nouveau_gpuobj **);
-extern void nouveau_gpuobj_ref(struct nouveau_gpuobj *,
-			       struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_new_fake(struct drm_device *, u32 pinst, u64 vinst,
-				   u32 size, u32 flags,
-				   struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_dma_new(struct nouveau_channel *, int class,
-				  uint64_t offset, uint64_t size, int access,
-				  int target, struct nouveau_gpuobj **);
-extern int nouveau_gpuobj_gr_new(struct nouveau_channel *, u32 handle, int class);
-extern int nv50_gpuobj_dma_new(struct nouveau_channel *, int class, u64 base,
-			       u64 size, int target, int access, u32 type,
-			       u32 comp, struct nouveau_gpuobj **pobj);
-extern void nv50_gpuobj_dma_init(struct nouveau_gpuobj *, u32 offset,
-				 int class, u64 base, u64 size, int target,
-				 int access, u32 type, u32 comp);
-
-/* nouveau_irq.c */
-extern int         nouveau_irq_init(struct drm_device *);
-extern void        nouveau_irq_fini(struct drm_device *);
-extern irqreturn_t nouveau_irq_handler(DRM_IRQ_ARGS);
-extern void        nouveau_irq_register(struct drm_device *, int status_bit,
-					void (*)(struct drm_device *));
-extern void        nouveau_irq_unregister(struct drm_device *, int status_bit);
-extern void        nouveau_irq_preinstall(struct drm_device *);
-extern int         nouveau_irq_postinstall(struct drm_device *);
-extern void        nouveau_irq_uninstall(struct drm_device *);
-
-/* nouveau_sgdma.c */
-extern int nouveau_sgdma_init(struct drm_device *);
-extern void nouveau_sgdma_takedown(struct drm_device *);
-extern uint32_t nouveau_sgdma_get_physical(struct drm_device *,
-					   uint32_t offset);
-extern struct ttm_tt *nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
-					       unsigned long size,
-					       uint32_t page_flags,
-					       struct page *dummy_read_page);
-
-/* nouveau_debugfs.c */
-#if defined(CONFIG_DRM_NOUVEAU_DEBUG)
-extern int  nouveau_debugfs_init(struct drm_minor *);
-extern void nouveau_debugfs_takedown(struct drm_minor *);
-extern int  nouveau_debugfs_channel_init(struct nouveau_channel *);
-extern void nouveau_debugfs_channel_fini(struct nouveau_channel *);
-#else
-static inline int
-nouveau_debugfs_init(struct drm_minor *minor)
-{
-	return 0;
-}
-
-static inline void nouveau_debugfs_takedown(struct drm_minor *minor)
-{
-}
-
-static inline int
-nouveau_debugfs_channel_init(struct nouveau_channel *chan)
-{
-	return 0;
-}
-
-static inline void
-nouveau_debugfs_channel_fini(struct nouveau_channel *chan)
-{
-}
-#endif
-
-/* nouveau_dma.c */
-extern void nouveau_dma_init(struct nouveau_channel *);
-extern int  nouveau_dma_wait(struct nouveau_channel *, int slots, int size);
-
-/* nouveau_acpi.c */
-#define ROM_BIOS_PAGE 4096
-#if defined(CONFIG_ACPI)
-void nouveau_register_dsm_handler(void);
-void nouveau_unregister_dsm_handler(void);
-void nouveau_switcheroo_optimus_dsm(void);
-int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len);
-bool nouveau_acpi_rom_supported(struct pci_dev *pdev);
-int nouveau_acpi_edid(struct drm_device *, struct drm_connector *);
-#else
-static inline void nouveau_register_dsm_handler(void) {}
-static inline void nouveau_unregister_dsm_handler(void) {}
-static inline void nouveau_switcheroo_optimus_dsm(void) {}
-static inline bool nouveau_acpi_rom_supported(struct pci_dev *pdev) { return false; }
-static inline int nouveau_acpi_get_bios_chunk(uint8_t *bios, int offset, int len) { return -EINVAL; }
-static inline int nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector) { return -EINVAL; }
-#endif
-
-/* nouveau_backlight.c */
-#ifdef CONFIG_DRM_NOUVEAU_BACKLIGHT
-extern int nouveau_backlight_init(struct drm_device *);
-extern void nouveau_backlight_exit(struct drm_device *);
-#else
-static inline int nouveau_backlight_init(struct drm_device *dev)
-{
-	return 0;
-}
-
-static inline void nouveau_backlight_exit(struct drm_device *dev) { }
-#endif
-
-/* nouveau_bios.c */
-extern int nouveau_bios_init(struct drm_device *);
-extern void nouveau_bios_takedown(struct drm_device *dev);
-extern int nouveau_run_vbios_init(struct drm_device *);
-extern void nouveau_bios_run_init_table(struct drm_device *, uint16_t table,
-					struct dcb_entry *, int crtc);
-extern void nouveau_bios_init_exec(struct drm_device *, uint16_t table);
-extern struct dcb_connector_table_entry *
-nouveau_bios_connector_entry(struct drm_device *, int index);
-extern u32 get_pll_register(struct drm_device *, enum pll_types);
-extern int get_pll_limits(struct drm_device *, uint32_t limit_match,
-			  struct pll_lims *);
-extern int nouveau_bios_run_display_table(struct drm_device *, u16 id, int clk,
-					  struct dcb_entry *, int crtc);
-extern bool nouveau_bios_fp_mode(struct drm_device *, struct drm_display_mode *);
-extern uint8_t *nouveau_bios_embedded_edid(struct drm_device *);
-extern int nouveau_bios_parse_lvds_table(struct drm_device *, int pxclk,
-					 bool *dl, bool *if_is_24bit);
-extern int run_tmds_table(struct drm_device *, struct dcb_entry *,
-			  int head, int pxclk);
-extern int call_lvds_script(struct drm_device *, struct dcb_entry *, int head,
-			    enum LVDS_script, int pxclk);
-bool bios_encoder_match(struct dcb_entry *, u32 hash);
-
-/* nouveau_mxm.c */
-int  nouveau_mxm_init(struct drm_device *dev);
-void nouveau_mxm_fini(struct drm_device *dev);
-
-/* nouveau_ttm.c */
-int nouveau_ttm_global_init(struct drm_nouveau_private *);
-void nouveau_ttm_global_release(struct drm_nouveau_private *);
-int nouveau_ttm_mmap(struct file *, struct vm_area_struct *);
-
-/* nouveau_hdmi.c */
-void nouveau_hdmi_mode_set(struct drm_encoder *, struct drm_display_mode *);
-
-/* nv04_fb.c */
-extern int  nv04_fb_vram_init(struct drm_device *);
-extern int  nv04_fb_init(struct drm_device *);
-extern void nv04_fb_takedown(struct drm_device *);
-
-/* nv10_fb.c */
-extern int  nv10_fb_vram_init(struct drm_device *dev);
-extern int  nv1a_fb_vram_init(struct drm_device *dev);
-extern int  nv10_fb_init(struct drm_device *);
-extern void nv10_fb_takedown(struct drm_device *);
-extern void nv10_fb_init_tile_region(struct drm_device *dev, int i,
-				     uint32_t addr, uint32_t size,
-				     uint32_t pitch, uint32_t flags);
-extern void nv10_fb_set_tile_region(struct drm_device *dev, int i);
-extern void nv10_fb_free_tile_region(struct drm_device *dev, int i);
-
-/* nv20_fb.c */
-extern int  nv20_fb_vram_init(struct drm_device *dev);
-extern int  nv20_fb_init(struct drm_device *);
-extern void nv20_fb_takedown(struct drm_device *);
-extern void nv20_fb_init_tile_region(struct drm_device *dev, int i,
-				     uint32_t addr, uint32_t size,
-				     uint32_t pitch, uint32_t flags);
-extern void nv20_fb_set_tile_region(struct drm_device *dev, int i);
-extern void nv20_fb_free_tile_region(struct drm_device *dev, int i);
-
-/* nv30_fb.c */
-extern int  nv30_fb_init(struct drm_device *);
-extern void nv30_fb_takedown(struct drm_device *);
-extern void nv30_fb_init_tile_region(struct drm_device *dev, int i,
-				     uint32_t addr, uint32_t size,
-				     uint32_t pitch, uint32_t flags);
-extern void nv30_fb_free_tile_region(struct drm_device *dev, int i);
-
-/* nv40_fb.c */
-extern int  nv40_fb_vram_init(struct drm_device *dev);
-extern int  nv40_fb_init(struct drm_device *);
-extern void nv40_fb_takedown(struct drm_device *);
-extern void nv40_fb_set_tile_region(struct drm_device *dev, int i);
-
-/* nv50_fb.c */
-extern int  nv50_fb_init(struct drm_device *);
-extern void nv50_fb_takedown(struct drm_device *);
-extern void nv50_fb_vm_trap(struct drm_device *, int display);
-
-/* nvc0_fb.c */
-extern int  nvc0_fb_init(struct drm_device *);
-extern void nvc0_fb_takedown(struct drm_device *);
-
-/* nv04_graph.c */
-extern int  nv04_graph_create(struct drm_device *);
-extern int  nv04_graph_object_new(struct nouveau_channel *, int, u32, u16);
-extern int  nv04_graph_mthd_page_flip(struct nouveau_channel *chan,
-				      u32 class, u32 mthd, u32 data);
-extern struct nouveau_bitfield nv04_graph_nsource[];
-
-/* nv10_graph.c */
-extern int  nv10_graph_create(struct drm_device *);
-extern struct nouveau_channel *nv10_graph_channel(struct drm_device *);
-extern struct nouveau_bitfield nv10_graph_intr[];
-extern struct nouveau_bitfield nv10_graph_nstatus[];
-
-/* nv20_graph.c */
-extern int  nv20_graph_create(struct drm_device *);
-
-/* nv40_graph.c */
-extern int  nv40_graph_create(struct drm_device *);
-extern void nv40_grctx_init(struct drm_device *, u32 *size);
-extern void nv40_grctx_fill(struct drm_device *, struct nouveau_gpuobj *);
-
-/* nv50_graph.c */
-extern int  nv50_graph_create(struct drm_device *);
-extern struct nouveau_enum nv50_data_error_names[];
-extern int  nv50_graph_isr_chid(struct drm_device *dev, u64 inst);
-extern int  nv50_grctx_init(struct drm_device *, u32 *, u32, u32 *, u32 *);
-extern void nv50_grctx_fill(struct drm_device *, struct nouveau_gpuobj *);
-
-/* nvc0_graph.c */
-extern int  nvc0_graph_create(struct drm_device *);
-extern int  nvc0_graph_isr_chid(struct drm_device *dev, u64 inst);
-
-/* nve0_graph.c */
-extern int  nve0_graph_create(struct drm_device *);
-
-/* nv84_crypt.c */
-extern int  nv84_crypt_create(struct drm_device *);
-
-/* nv98_crypt.c */
-extern int  nv98_crypt_create(struct drm_device *dev);
-
-/* nva3_copy.c */
-extern int  nva3_copy_create(struct drm_device *dev);
-
-/* nvc0_copy.c */
-extern int  nvc0_copy_create(struct drm_device *dev, int engine);
-
-/* nv31_mpeg.c */
-extern int  nv31_mpeg_create(struct drm_device *dev);
-
-/* nv50_mpeg.c */
-extern int  nv50_mpeg_create(struct drm_device *dev);
-
-/* nv84_bsp.c */
-/* nv98_bsp.c */
-extern int  nv84_bsp_create(struct drm_device *dev);
-
-/* nv84_vp.c */
-/* nv98_vp.c */
-extern int  nv84_vp_create(struct drm_device *dev);
-
-/* nv98_ppp.c */
-extern int  nv98_ppp_create(struct drm_device *dev);
-
-/* nv04_instmem.c */
-extern int  nv04_instmem_init(struct drm_device *);
-extern void nv04_instmem_takedown(struct drm_device *);
-extern int  nv04_instmem_suspend(struct drm_device *);
-extern void nv04_instmem_resume(struct drm_device *);
-extern int  nv04_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
-			     u32 size, u32 align);
-extern void nv04_instmem_put(struct nouveau_gpuobj *);
-extern int  nv04_instmem_map(struct nouveau_gpuobj *);
-extern void nv04_instmem_unmap(struct nouveau_gpuobj *);
-extern void nv04_instmem_flush(struct drm_device *);
-
-/* nv50_instmem.c */
-extern int  nv50_instmem_init(struct drm_device *);
-extern void nv50_instmem_takedown(struct drm_device *);
-extern int  nv50_instmem_suspend(struct drm_device *);
-extern void nv50_instmem_resume(struct drm_device *);
-extern int  nv50_instmem_get(struct nouveau_gpuobj *, struct nouveau_channel *,
-			     u32 size, u32 align);
-extern void nv50_instmem_put(struct nouveau_gpuobj *);
-extern int  nv50_instmem_map(struct nouveau_gpuobj *);
-extern void nv50_instmem_unmap(struct nouveau_gpuobj *);
-extern void nv50_instmem_flush(struct drm_device *);
-extern void nv84_instmem_flush(struct drm_device *);
-
-/* nvc0_instmem.c */
-extern int  nvc0_instmem_init(struct drm_device *);
-extern void nvc0_instmem_takedown(struct drm_device *);
-extern int  nvc0_instmem_suspend(struct drm_device *);
-extern void nvc0_instmem_resume(struct drm_device *);
-
-/* nv04_mc.c */
-extern int  nv04_mc_init(struct drm_device *);
-extern void nv04_mc_takedown(struct drm_device *);
-
-/* nv40_mc.c */
-extern int  nv40_mc_init(struct drm_device *);
-extern void nv40_mc_takedown(struct drm_device *);
-
-/* nv50_mc.c */
-extern int  nv50_mc_init(struct drm_device *);
-extern void nv50_mc_takedown(struct drm_device *);
-
-/* nv04_timer.c */
-extern int  nv04_timer_init(struct drm_device *);
-extern uint64_t nv04_timer_read(struct drm_device *);
-extern void nv04_timer_takedown(struct drm_device *);
-
-extern long nouveau_compat_ioctl(struct file *file, unsigned int cmd,
-				 unsigned long arg);
-
-/* nv04_dac.c */
-extern int nv04_dac_create(struct drm_connector *, struct dcb_entry *);
-extern uint32_t nv17_dac_sample_load(struct drm_encoder *encoder);
-extern int nv04_dac_output_offset(struct drm_encoder *encoder);
-extern void nv04_dac_update_dacclk(struct drm_encoder *encoder, bool enable);
-extern bool nv04_dac_in_use(struct drm_encoder *encoder);
-
-/* nv04_dfp.c */
-extern int nv04_dfp_create(struct drm_connector *, struct dcb_entry *);
-extern int nv04_dfp_get_bound_head(struct drm_device *dev, struct dcb_entry *dcbent);
-extern void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_entry *dcbent,
-			       int head, bool dl);
-extern void nv04_dfp_disable(struct drm_device *dev, int head);
-extern void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode);
-
-/* nv04_tv.c */
-extern int nv04_tv_identify(struct drm_device *dev, int i2c_index);
-extern int nv04_tv_create(struct drm_connector *, struct dcb_entry *);
-
-/* nv17_tv.c */
-extern int nv17_tv_create(struct drm_connector *, struct dcb_entry *);
-
-/* nv04_display.c */
-extern int nv04_display_early_init(struct drm_device *);
-extern void nv04_display_late_takedown(struct drm_device *);
-extern int nv04_display_create(struct drm_device *);
-extern void nv04_display_destroy(struct drm_device *);
-extern int nv04_display_init(struct drm_device *);
-extern void nv04_display_fini(struct drm_device *);
-
-/* nvd0_display.c */
-extern int nvd0_display_create(struct drm_device *);
-extern void nvd0_display_destroy(struct drm_device *);
-extern int nvd0_display_init(struct drm_device *);
-extern void nvd0_display_fini(struct drm_device *);
-struct nouveau_bo *nvd0_display_crtc_sema(struct drm_device *, int crtc);
-void nvd0_display_flip_stop(struct drm_crtc *);
-int nvd0_display_flip_next(struct drm_crtc *, struct drm_framebuffer *,
-			   struct nouveau_channel *, u32 swap_interval);
-
-/* nv04_crtc.c */
-extern int nv04_crtc_create(struct drm_device *, int index);
-
-/* nouveau_bo.c */
-extern struct ttm_bo_driver nouveau_bo_driver;
-extern void nouveau_bo_move_init(struct nouveau_channel *);
-extern int nouveau_bo_new(struct drm_device *, int size, int align,
-			  uint32_t flags, uint32_t tile_mode,
-			  uint32_t tile_flags,
-			  struct sg_table *sg,
-			  struct nouveau_bo **);
-extern int nouveau_bo_pin(struct nouveau_bo *, uint32_t flags);
-extern int nouveau_bo_unpin(struct nouveau_bo *);
-extern int nouveau_bo_map(struct nouveau_bo *);
-extern void nouveau_bo_unmap(struct nouveau_bo *);
-extern void nouveau_bo_placement_set(struct nouveau_bo *, uint32_t type,
-				     uint32_t busy);
-extern u16 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index);
-extern void nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val);
-extern u32 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index);
-extern void nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val);
-extern void nouveau_bo_fence(struct nouveau_bo *, struct nouveau_fence *);
-extern int nouveau_bo_validate(struct nouveau_bo *, bool interruptible,
-			       bool no_wait_reserve, bool no_wait_gpu);
-
-extern struct nouveau_vma *
-nouveau_bo_vma_find(struct nouveau_bo *, struct nouveau_vm *);
-extern int  nouveau_bo_vma_add(struct nouveau_bo *, struct nouveau_vm *,
-			       struct nouveau_vma *);
-extern void nouveau_bo_vma_del(struct nouveau_bo *, struct nouveau_vma *);
-
-/* nouveau_gem.c */
-extern int nouveau_gem_new(struct drm_device *, int size, int align,
-			   uint32_t domain, uint32_t tile_mode,
-			   uint32_t tile_flags, struct nouveau_bo **);
-extern int nouveau_gem_object_new(struct drm_gem_object *);
-extern void nouveau_gem_object_del(struct drm_gem_object *);
-extern int nouveau_gem_object_open(struct drm_gem_object *, struct drm_file *);
-extern void nouveau_gem_object_close(struct drm_gem_object *,
-				     struct drm_file *);
-extern int nouveau_gem_ioctl_new(struct drm_device *, void *,
-				 struct drm_file *);
-extern int nouveau_gem_ioctl_pushbuf(struct drm_device *, void *,
-				     struct drm_file *);
-extern int nouveau_gem_ioctl_cpu_prep(struct drm_device *, void *,
-				      struct drm_file *);
-extern int nouveau_gem_ioctl_cpu_fini(struct drm_device *, void *,
-				      struct drm_file *);
-extern int nouveau_gem_ioctl_info(struct drm_device *, void *,
-				  struct drm_file *);
-
-extern struct dma_buf *nouveau_gem_prime_export(struct drm_device *dev,
-				struct drm_gem_object *obj, int flags);
-extern struct drm_gem_object *nouveau_gem_prime_import(struct drm_device *dev,
-				struct dma_buf *dma_buf);
-
-/* nouveau_display.c */
-int nouveau_display_create(struct drm_device *dev);
-void nouveau_display_destroy(struct drm_device *dev);
-int nouveau_display_init(struct drm_device *dev);
-void nouveau_display_fini(struct drm_device *dev);
-int nouveau_vblank_enable(struct drm_device *dev, int crtc);
-void nouveau_vblank_disable(struct drm_device *dev, int crtc);
-int nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-			   struct drm_pending_vblank_event *event);
-int nouveau_finish_page_flip(struct nouveau_channel *,
-			     struct nouveau_page_flip_state *);
-int nouveau_display_dumb_create(struct drm_file *, struct drm_device *,
-				struct drm_mode_create_dumb *args);
-int nouveau_display_dumb_map_offset(struct drm_file *, struct drm_device *,
-				    uint32_t handle, uint64_t *offset);
-int nouveau_display_dumb_destroy(struct drm_file *, struct drm_device *,
-				 uint32_t handle);
-
-/* nv10_gpio.c */
-int nv10_gpio_init(struct drm_device *dev);
-void nv10_gpio_fini(struct drm_device *dev);
-int nv10_gpio_drive(struct drm_device *dev, int line, int dir, int out);
-int nv10_gpio_sense(struct drm_device *dev, int line);
-void nv10_gpio_irq_enable(struct drm_device *, int line, bool on);
-
-/* nv50_gpio.c */
-int nv50_gpio_init(struct drm_device *dev);
-void nv50_gpio_fini(struct drm_device *dev);
-int nv50_gpio_drive(struct drm_device *dev, int line, int dir, int out);
-int nv50_gpio_sense(struct drm_device *dev, int line);
-void nv50_gpio_irq_enable(struct drm_device *, int line, bool on);
-int nvd0_gpio_drive(struct drm_device *dev, int line, int dir, int out);
-int nvd0_gpio_sense(struct drm_device *dev, int line);
-
-/* nv50_calc.c */
-int nv50_calc_pll(struct drm_device *, struct pll_lims *, int clk,
-		  int *N1, int *M1, int *N2, int *M2, int *P);
-int nva3_calc_pll(struct drm_device *, struct pll_lims *,
-		  int clk, int *N, int *fN, int *M, int *P);
-
-#ifndef ioread32_native
-#ifdef __BIG_ENDIAN
-#define ioread16_native ioread16be
-#define iowrite16_native iowrite16be
-#define ioread32_native  ioread32be
-#define iowrite32_native iowrite32be
-#else /* def __BIG_ENDIAN */
-#define ioread16_native ioread16
-#define iowrite16_native iowrite16
-#define ioread32_native  ioread32
-#define iowrite32_native iowrite32
-#endif /* def __BIG_ENDIAN else */
-#endif /* !ioread32_native */
-
-/* channel control reg access */
-static inline u32 nvchan_rd32(struct nouveau_channel *chan, unsigned reg)
-{
-	return ioread32_native(chan->user + reg);
-}
-
-static inline void nvchan_wr32(struct nouveau_channel *chan,
-							unsigned reg, u32 val)
-{
-	iowrite32_native(val, chan->user + reg);
-}
-
-/* register access */
-static inline u32 nv_rd32(struct drm_device *dev, unsigned reg)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return ioread32_native(dev_priv->mmio + reg);
-}
-
-static inline void nv_wr32(struct drm_device *dev, unsigned reg, u32 val)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	iowrite32_native(val, dev_priv->mmio + reg);
-}
-
-static inline u32 nv_mask(struct drm_device *dev, u32 reg, u32 mask, u32 val)
-{
-	u32 tmp = nv_rd32(dev, reg);
-	nv_wr32(dev, reg, (tmp & ~mask) | val);
-	return tmp;
-}
-
-static inline u8 nv_rd08(struct drm_device *dev, unsigned reg)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return ioread8(dev_priv->mmio + reg);
-}
-
-static inline void nv_wr08(struct drm_device *dev, unsigned reg, u8 val)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	iowrite8(val, dev_priv->mmio + reg);
-}
-
-#define nv_wait(dev, reg, mask, val) \
-	nouveau_wait_eq(dev, 2000000000ULL, (reg), (mask), (val))
-#define nv_wait_ne(dev, reg, mask, val) \
-	nouveau_wait_ne(dev, 2000000000ULL, (reg), (mask), (val))
-#define nv_wait_cb(dev, func, data) \
-	nouveau_wait_cb(dev, 2000000000ULL, (func), (data))
-
-/* PRAMIN access */
-static inline u32 nv_ri32(struct drm_device *dev, unsigned offset)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return ioread32_native(dev_priv->ramin + offset);
-}
-
-static inline void nv_wi32(struct drm_device *dev, unsigned offset, u32 val)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	iowrite32_native(val, dev_priv->ramin + offset);
-}
-
-/* object access */
-extern u32 nv_ro32(struct nouveau_gpuobj *, u32 offset);
-extern void nv_wo32(struct nouveau_gpuobj *, u32 offset, u32 val);
-
-/*
- * Logging
- * Argument d is (struct drm_device *).
- */
-#define NV_PRINTK(level, d, fmt, arg...) \
-	printk(level "[" DRM_NAME "] " DRIVER_NAME " %s: " fmt, \
-					pci_name(d->pdev), ##arg)
-#ifndef NV_DEBUG_NOTRACE
-#define NV_DEBUG(d, fmt, arg...) do {                                          \
-	if (drm_debug & DRM_UT_DRIVER) {                                       \
-		NV_PRINTK(KERN_DEBUG, d, "%s:%d - " fmt, __func__,             \
-			  __LINE__, ##arg);                                    \
-	}                                                                      \
-} while (0)
-#define NV_DEBUG_KMS(d, fmt, arg...) do {                                      \
-	if (drm_debug & DRM_UT_KMS) {                                          \
-		NV_PRINTK(KERN_DEBUG, d, "%s:%d - " fmt, __func__,             \
-			  __LINE__, ##arg);                                    \
-	}                                                                      \
-} while (0)
-#else
-#define NV_DEBUG(d, fmt, arg...) do {                                          \
-	if (drm_debug & DRM_UT_DRIVER)                                         \
-		NV_PRINTK(KERN_DEBUG, d, fmt, ##arg);                          \
-} while (0)
-#define NV_DEBUG_KMS(d, fmt, arg...) do {                                      \
-	if (drm_debug & DRM_UT_KMS)                                            \
-		NV_PRINTK(KERN_DEBUG, d, fmt, ##arg);                          \
-} while (0)
-#endif
-#define NV_ERROR(d, fmt, arg...) NV_PRINTK(KERN_ERR, d, fmt, ##arg)
-#define NV_INFO(d, fmt, arg...) NV_PRINTK(KERN_INFO, d, fmt, ##arg)
-#define NV_TRACEWARN(d, fmt, arg...) NV_PRINTK(KERN_NOTICE, d, fmt, ##arg)
-#define NV_TRACE(d, fmt, arg...) NV_PRINTK(KERN_INFO, d, fmt, ##arg)
-#define NV_WARN(d, fmt, arg...) NV_PRINTK(KERN_WARNING, d, fmt, ##arg)
-#define NV_WARNONCE(d, fmt, arg...) do {                                       \
-	static int _warned = 0;                                                \
-	if (!_warned) {                                                        \
-		NV_WARN(d, fmt, ##arg);                                        \
-		_warned = 1;                                                   \
-	}                                                                      \
-} while(0)
-
-/* nouveau_reg_debug bitmask */
-enum {
-	NOUVEAU_REG_DEBUG_MC             = 0x1,
-	NOUVEAU_REG_DEBUG_VIDEO          = 0x2,
-	NOUVEAU_REG_DEBUG_FB             = 0x4,
-	NOUVEAU_REG_DEBUG_EXTDEV         = 0x8,
-	NOUVEAU_REG_DEBUG_CRTC           = 0x10,
-	NOUVEAU_REG_DEBUG_RAMDAC         = 0x20,
-	NOUVEAU_REG_DEBUG_VGACRTC        = 0x40,
-	NOUVEAU_REG_DEBUG_RMVIO          = 0x80,
-	NOUVEAU_REG_DEBUG_VGAATTR        = 0x100,
-	NOUVEAU_REG_DEBUG_EVO            = 0x200,
-	NOUVEAU_REG_DEBUG_AUXCH          = 0x400
-};
-
-#define NV_REG_DEBUG(type, dev, fmt, arg...) do { \
-	if (nouveau_reg_debug & NOUVEAU_REG_DEBUG_##type) \
-		NV_PRINTK(KERN_DEBUG, dev, "%s: " fmt, __func__, ##arg); \
-} while (0)
-
-static inline bool
-nv_two_heads(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	const int impl = dev->pci_device & 0x0ff0;
-
-	if (dev_priv->card_type >= NV_10 && impl != 0x0100 &&
-	    impl != 0x0150 && impl != 0x01a0 && impl != 0x0200)
-		return true;
-
-	return false;
-}
-
-static inline bool
-nv_gf4_disp_arch(struct drm_device *dev)
-{
-	return nv_two_heads(dev) && (dev->pci_device & 0x0ff0) != 0x0110;
-}
-
-static inline bool
-nv_two_reg_pll(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	const int impl = dev->pci_device & 0x0ff0;
-
-	if (impl == 0x0310 || impl == 0x0340 || dev_priv->card_type >= NV_40)
-		return true;
-	return false;
-}
-
-static inline bool
-nv_match_device(struct drm_device *dev, unsigned device,
-		unsigned sub_vendor, unsigned sub_device)
-{
-	return dev->pdev->device == device &&
-		dev->pdev->subsystem_vendor == sub_vendor &&
-		dev->pdev->subsystem_device == sub_device;
-}
-
-static inline void *
-nv_engine(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return (void *)dev_priv->eng[engine];
-}
-
-/* returns 1 if device is one of the nv4x using the 0x4497 object class,
- * helpful to determine a number of other hardware features
- */
-static inline int
-nv44_graph_class(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if ((dev_priv->chipset & 0xf0) == 0x60)
-		return 1;
-
-	return !(0x0baf & (1 << (dev_priv->chipset & 0x0f)));
-}
-
-/* memory type/access flags, do not match hardware values */
-#define NV_MEM_ACCESS_RO  1
-#define NV_MEM_ACCESS_WO  2
-#define NV_MEM_ACCESS_RW (NV_MEM_ACCESS_RO | NV_MEM_ACCESS_WO)
-#define NV_MEM_ACCESS_SYS 4
-#define NV_MEM_ACCESS_VM  8
-#define NV_MEM_ACCESS_NOSNOOP 16
-
-#define NV_MEM_TARGET_VRAM        0
-#define NV_MEM_TARGET_PCI         1
-#define NV_MEM_TARGET_PCI_NOSNOOP 2
-#define NV_MEM_TARGET_VM          3
-#define NV_MEM_TARGET_GART        4
-
-#define NV_MEM_TYPE_VM 0x7f
-#define NV_MEM_COMP_VM 0x03
-
-/* FIFO methods */
-#define NV01_SUBCHAN_OBJECT                                          0x00000000
-#define NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH                          0x00000010
-#define NV84_SUBCHAN_SEMAPHORE_ADDRESS_LOW                           0x00000014
-#define NV84_SUBCHAN_SEMAPHORE_SEQUENCE                              0x00000018
-#define NV84_SUBCHAN_SEMAPHORE_TRIGGER                               0x0000001c
-#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL                 0x00000001
-#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG                    0x00000002
-#define NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_GEQUAL                0x00000004
-#define NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD                         0x00001000
-#define NV84_SUBCHAN_NOTIFY_INTR                                     0x00000020
-#define NV84_SUBCHAN_WRCACHE_FLUSH                                   0x00000024
-#define NV10_SUBCHAN_REF_CNT                                         0x00000050
-#define NVSW_SUBCHAN_PAGE_FLIP                                       0x00000054
-#define NV11_SUBCHAN_DMA_SEMAPHORE                                   0x00000060
-#define NV11_SUBCHAN_SEMAPHORE_OFFSET                                0x00000064
-#define NV11_SUBCHAN_SEMAPHORE_ACQUIRE                               0x00000068
-#define NV11_SUBCHAN_SEMAPHORE_RELEASE                               0x0000006c
-#define NV40_SUBCHAN_YIELD                                           0x00000080
-
-/* NV_SW object class */
-#define NV_SW                                                        0x0000506e
-#define NV_SW_DMA_VBLSEM                                             0x0000018c
-#define NV_SW_VBLSEM_OFFSET                                          0x00000400
-#define NV_SW_VBLSEM_RELEASE_VALUE                                   0x00000404
-#define NV_SW_VBLSEM_RELEASE                                         0x00000408
-#define NV_SW_PAGE_FLIP                                              0x00000500
-
-#endif /* __NOUVEAU_DRV_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index 3dc14a3dcc4..6a17bf2ba9a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -27,23 +27,27 @@
 #ifndef __NOUVEAU_ENCODER_H__
 #define __NOUVEAU_ENCODER_H__
 
-#include "drm_encoder_slave.h"
-#include "nouveau_drv.h"
+#include <subdev/bios/dcb.h>
+
+#include <drm/drm_encoder_slave.h>
+#include "nv04_display.h"
 
 #define NV_DPMS_CLEARED 0x80
 
+struct nouveau_i2c_port;
+
 struct dp_train_func {
-	void (*link_set)(struct drm_device *, struct dcb_entry *, int crtc,
+	void (*link_set)(struct drm_device *, struct dcb_output *, int crtc,
 			 int nr, u32 bw, bool enhframe);
-	void (*train_set)(struct drm_device *, struct dcb_entry *, u8 pattern);
-	void (*train_adj)(struct drm_device *, struct dcb_entry *,
+	void (*train_set)(struct drm_device *, struct dcb_output *, u8 pattern);
+	void (*train_adj)(struct drm_device *, struct dcb_output *,
 			  u8 lane, u8 swing, u8 preem);
 };
 
 struct nouveau_encoder {
 	struct drm_encoder_slave base;
 
-	struct dcb_entry *dcb;
+	struct dcb_output *dcb;
 	int or;
 
 	/* different to drm_encoder.crtc, this reflects what's
@@ -87,18 +91,16 @@ get_slave_funcs(struct drm_encoder *enc)
 }
 
 /* nouveau_dp.c */
-int nouveau_dp_auxch(struct nouveau_i2c_chan *auxch, int cmd, int addr,
-		     uint8_t *data, int data_nr);
 bool nouveau_dp_detect(struct drm_encoder *);
 void nouveau_dp_dpms(struct drm_encoder *, int mode, u32 datarate,
 		     struct dp_train_func *);
-u8 *nouveau_dp_bios_data(struct drm_device *, struct dcb_entry *, u8 **);
+u8 *nouveau_dp_bios_data(struct drm_device *, struct dcb_output *, u8 **);
 
 struct nouveau_connector *
 nouveau_encoder_connector_get(struct nouveau_encoder *encoder);
-int nv50_sor_create(struct drm_connector *, struct dcb_entry *);
+int nv50_sor_create(struct drm_connector *, struct dcb_output *);
 void nv50_sor_dp_calc_tu(struct drm_device *, int, int, u32, u32);
-int nv50_dac_create(struct drm_connector *, struct dcb_entry *);
+int nv50_dac_create(struct drm_connector *, struct dcb_output *);
 
 
 #endif /* __NOUVEAU_ENCODER_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_fb.h b/drivers/gpu/drm/nouveau/nouveau_fb.h
deleted file mode 100644
index f3fb649fe45..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_fb.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2008 Maarten Maathuis.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef __NOUVEAU_FB_H__
-#define __NOUVEAU_FB_H__
-
-struct nouveau_framebuffer {
-	struct drm_framebuffer base;
-	struct nouveau_bo *nvbo;
-	struct nouveau_vma vma;
-	u32 r_dma;
-	u32 r_format;
-	u32 r_pitch;
-};
-
-static inline struct nouveau_framebuffer *
-nouveau_framebuffer(struct drm_framebuffer *fb)
-{
-	return container_of(fb, struct nouveau_framebuffer, base);
-}
-
-int nouveau_framebuffer_init(struct drm_device *dev, struct nouveau_framebuffer *nouveau_fb,
-			     struct drm_mode_fb_cmd2 *mode_cmd, struct nouveau_bo *nvbo);
-#endif /* __NOUVEAU_FB_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 1074bc5dd41..67a1a069de2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -38,24 +38,34 @@
 #include <linux/vga_switcheroo.h>
 #include <linux/console.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+
 #include "nouveau_drm.h"
-#include "nouveau_crtc.h"
-#include "nouveau_fb.h"
+#include "nouveau_gem.h"
+#include "nouveau_bo.h"
 #include "nouveau_fbcon.h"
-#include "nouveau_dma.h"
+#include "nouveau_chan.h"
+
+#include "nouveau_crtc.h"
+
+#include <core/client.h>
+#include <core/device.h>
+
+#include <subdev/fb.h>
+
+MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
+static int nouveau_nofbaccel = 0;
+module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400);
 
 static void
 nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
-	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_fbdev *fbcon = info->par;
+	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int ret;
 
 	if (info->state != FBINFO_STATE_RUNNING)
@@ -63,15 +73,15 @@ nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 
 	ret = -ENODEV;
 	if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
-	    mutex_trylock(&dev_priv->channel->mutex)) {
-		if (dev_priv->card_type < NV_50)
+	    mutex_trylock(&drm->client.mutex)) {
+		if (device->card_type < NV_50)
 			ret = nv04_fbcon_fillrect(info, rect);
 		else
-		if (dev_priv->card_type < NV_C0)
+		if (device->card_type < NV_C0)
 			ret = nv50_fbcon_fillrect(info, rect);
 		else
 			ret = nvc0_fbcon_fillrect(info, rect);
-		mutex_unlock(&dev_priv->channel->mutex);
+		mutex_unlock(&drm->client.mutex);
 	}
 
 	if (ret == 0)
@@ -85,9 +95,9 @@ nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 static void
 nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
 {
-	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_fbdev *fbcon = info->par;
+	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int ret;
 
 	if (info->state != FBINFO_STATE_RUNNING)
@@ -95,15 +105,15 @@ nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
 
 	ret = -ENODEV;
 	if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
-	    mutex_trylock(&dev_priv->channel->mutex)) {
-		if (dev_priv->card_type < NV_50)
+	    mutex_trylock(&drm->client.mutex)) {
+		if (device->card_type < NV_50)
 			ret = nv04_fbcon_copyarea(info, image);
 		else
-		if (dev_priv->card_type < NV_C0)
+		if (device->card_type < NV_C0)
 			ret = nv50_fbcon_copyarea(info, image);
 		else
 			ret = nvc0_fbcon_copyarea(info, image);
-		mutex_unlock(&dev_priv->channel->mutex);
+		mutex_unlock(&drm->client.mutex);
 	}
 
 	if (ret == 0)
@@ -117,9 +127,9 @@ nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image)
 static void
 nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
-	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_fbdev *fbcon = info->par;
+	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	int ret;
 
 	if (info->state != FBINFO_STATE_RUNNING)
@@ -127,15 +137,15 @@ nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 
 	ret = -ENODEV;
 	if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) &&
-	    mutex_trylock(&dev_priv->channel->mutex)) {
-		if (dev_priv->card_type < NV_50)
+	    mutex_trylock(&drm->client.mutex)) {
+		if (device->card_type < NV_50)
 			ret = nv04_fbcon_imageblit(info, image);
 		else
-		if (dev_priv->card_type < NV_C0)
+		if (device->card_type < NV_C0)
 			ret = nv50_fbcon_imageblit(info, image);
 		else
 			ret = nvc0_fbcon_imageblit(info, image);
-		mutex_unlock(&dev_priv->channel->mutex);
+		mutex_unlock(&drm->client.mutex);
 	}
 
 	if (ret == 0)
@@ -149,10 +159,9 @@ nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 static int
 nouveau_fbcon_sync(struct fb_info *info)
 {
-	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_fbdev *fbcon = info->par;
+	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	if (!chan || !chan->accel_done || in_interrupt() ||
@@ -160,11 +169,11 @@ nouveau_fbcon_sync(struct fb_info *info)
 	    info->flags & FBINFO_HWACCEL_DISABLED)
 		return 0;
 
-	if (!mutex_trylock(&chan->mutex))
+	if (!mutex_trylock(&drm->client.mutex))
 		return 0;
 
 	ret = nouveau_channel_idle(chan);
-	mutex_unlock(&chan->mutex);
+	mutex_unlock(&drm->client.mutex);
 	if (ret) {
 		nouveau_fbcon_gpu_lockup(info);
 		return 0;
@@ -224,9 +233,9 @@ static void nouveau_fbcon_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
 }
 
 static void
-nouveau_fbcon_zfill(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
+nouveau_fbcon_zfill(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 {
-	struct fb_info *info = nfbdev->helper.fbdev;
+	struct fb_info *info = fbcon->helper.fbdev;
 	struct fb_fillrect rect;
 
 	/* Clear the entire fbcon.  The drm will program every connector
@@ -242,11 +251,12 @@ nouveau_fbcon_zfill(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
 }
 
 static int
-nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
+nouveau_fbcon_create(struct nouveau_fbdev *fbcon,
 		     struct drm_fb_helper_surface_size *sizes)
 {
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct drm_device *dev = fbcon->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
 	struct fb_info *info;
 	struct drm_framebuffer *fb;
 	struct nouveau_framebuffer *nouveau_fb;
@@ -254,7 +264,6 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
 	struct nouveau_bo *nvbo;
 	struct drm_mode_fb_cmd2 mode_cmd;
 	struct pci_dev *pdev = dev->pdev;
-	struct device *device = &pdev->dev;
 	int size, ret;
 
 	mode_cmd.width = sizes->surface_width;
@@ -272,37 +281,38 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
 	ret = nouveau_gem_new(dev, size, 0, NOUVEAU_GEM_DOMAIN_VRAM,
 			      0, 0x0000, &nvbo);
 	if (ret) {
-		NV_ERROR(dev, "failed to allocate framebuffer\n");
+		NV_ERROR(drm, "failed to allocate framebuffer\n");
 		goto out;
 	}
 
 	ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM);
 	if (ret) {
-		NV_ERROR(dev, "failed to pin fb: %d\n", ret);
+		NV_ERROR(drm, "failed to pin fb: %d\n", ret);
 		nouveau_bo_ref(NULL, &nvbo);
 		goto out;
 	}
 
 	ret = nouveau_bo_map(nvbo);
 	if (ret) {
-		NV_ERROR(dev, "failed to map fb: %d\n", ret);
+		NV_ERROR(drm, "failed to map fb: %d\n", ret);
 		nouveau_bo_unpin(nvbo);
 		nouveau_bo_ref(NULL, &nvbo);
 		goto out;
 	}
 
-	chan = nouveau_nofbaccel ? NULL : dev_priv->channel;
-	if (chan && dev_priv->card_type >= NV_50) {
-		ret = nouveau_bo_vma_add(nvbo, chan->vm, &nfbdev->nouveau_fb.vma);
+	chan = nouveau_nofbaccel ? NULL : drm->channel;
+	if (chan && device->card_type >= NV_50) {
+		ret = nouveau_bo_vma_add(nvbo, nv_client(chan->cli)->vm,
+					&fbcon->nouveau_fb.vma);
 		if (ret) {
-			NV_ERROR(dev, "failed to map fb into chan: %d\n", ret);
+			NV_ERROR(drm, "failed to map fb into chan: %d\n", ret);
 			chan = NULL;
 		}
 	}
 
 	mutex_lock(&dev->struct_mutex);
 
-	info = framebuffer_alloc(0, device);
+	info = framebuffer_alloc(0, &pdev->dev);
 	if (!info) {
 		ret = -ENOMEM;
 		goto out_unref;
@@ -314,16 +324,16 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
 		goto out_unref;
 	}
 
-	info->par = nfbdev;
+	info->par = fbcon;
 
-	nouveau_framebuffer_init(dev, &nfbdev->nouveau_fb, &mode_cmd, nvbo);
+	nouveau_framebuffer_init(dev, &fbcon->nouveau_fb, &mode_cmd, nvbo);
 
-	nouveau_fb = &nfbdev->nouveau_fb;
+	nouveau_fb = &fbcon->nouveau_fb;
 	fb = &nouveau_fb->base;
 
 	/* setup helper */
-	nfbdev->helper.fb = fb;
-	nfbdev->helper.fbdev = info;
+	fbcon->helper.fb = fb;
+	fbcon->helper.fbdev = info;
 
 	strcpy(info->fix.id, "nouveaufb");
 	if (nouveau_nofbaccel)
@@ -342,25 +352,18 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
 	info->screen_size = size;
 
 	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->depth);
-	drm_fb_helper_fill_var(info, &nfbdev->helper, sizes->fb_width, sizes->fb_height);
-
-	/* Set aperture base/size for vesafb takeover */
-	info->apertures = dev_priv->apertures;
-	if (!info->apertures) {
-		ret = -ENOMEM;
-		goto out_unref;
-	}
+	drm_fb_helper_fill_var(info, &fbcon->helper, sizes->fb_width, sizes->fb_height);
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
 
 	mutex_unlock(&dev->struct_mutex);
 
-	if (dev_priv->channel && !nouveau_nofbaccel) {
+	if (chan) {
 		ret = -ENODEV;
-		if (dev_priv->card_type < NV_50)
+		if (device->card_type < NV_50)
 			ret = nv04_fbcon_accel_init(info);
 		else
-		if (dev_priv->card_type < NV_C0)
+		if (device->card_type < NV_C0)
 			ret = nv50_fbcon_accel_init(info);
 		else
 			ret = nvc0_fbcon_accel_init(info);
@@ -369,13 +372,12 @@ nouveau_fbcon_create(struct nouveau_fbdev *nfbdev,
 			info->fbops = &nouveau_fbcon_ops;
 	}
 
-	nouveau_fbcon_zfill(dev, nfbdev);
+	nouveau_fbcon_zfill(dev, fbcon);
 
 	/* To allow resizeing without swapping buffers */
-	NV_INFO(dev, "allocated %dx%d fb: 0x%lx, bo %p\n",
-						nouveau_fb->base.width,
-						nouveau_fb->base.height,
-						nvbo->bo.offset, nvbo);
+	NV_INFO(drm, "allocated %dx%d fb: 0x%lx, bo %p\n",
+		nouveau_fb->base.width, nouveau_fb->base.height,
+		nvbo->bo.offset, nvbo);
 
 	vga_switcheroo_client_fb_set(dev->pdev, info);
 	return 0;
@@ -390,12 +392,12 @@ static int
 nouveau_fbcon_find_or_create_single(struct drm_fb_helper *helper,
 				    struct drm_fb_helper_surface_size *sizes)
 {
-	struct nouveau_fbdev *nfbdev = (struct nouveau_fbdev *)helper;
+	struct nouveau_fbdev *fbcon = (struct nouveau_fbdev *)helper;
 	int new_fb = 0;
 	int ret;
 
 	if (!helper->fb) {
-		ret = nouveau_fbcon_create(nfbdev, sizes);
+		ret = nouveau_fbcon_create(fbcon, sizes);
 		if (ret)
 			return ret;
 		new_fb = 1;
@@ -406,18 +408,18 @@ nouveau_fbcon_find_or_create_single(struct drm_fb_helper *helper,
 void
 nouveau_fbcon_output_poll_changed(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	drm_fb_helper_hotplug_event(&dev_priv->nfbdev->helper);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	drm_fb_helper_hotplug_event(&drm->fbcon->helper);
 }
 
 static int
-nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
+nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 {
-	struct nouveau_framebuffer *nouveau_fb = &nfbdev->nouveau_fb;
+	struct nouveau_framebuffer *nouveau_fb = &fbcon->nouveau_fb;
 	struct fb_info *info;
 
-	if (nfbdev->helper.fbdev) {
-		info = nfbdev->helper.fbdev;
+	if (fbcon->helper.fbdev) {
+		info = fbcon->helper.fbdev;
 		unregister_framebuffer(info);
 		if (info->cmap.len)
 			fb_dealloc_cmap(&info->cmap);
@@ -430,17 +432,17 @@ nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *nfbdev)
 		drm_gem_object_unreference_unlocked(nouveau_fb->nvbo->gem);
 		nouveau_fb->nvbo = NULL;
 	}
-	drm_fb_helper_fini(&nfbdev->helper);
+	drm_fb_helper_fini(&fbcon->helper);
 	drm_framebuffer_cleanup(&nouveau_fb->base);
 	return 0;
 }
 
 void nouveau_fbcon_gpu_lockup(struct fb_info *info)
 {
-	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
+	struct nouveau_fbdev *fbcon = info->par;
+	struct nouveau_drm *drm = nouveau_drm(fbcon->dev);
 
-	NV_ERROR(dev, "GPU lockup - switching to software fbcon\n");
+	NV_ERROR(drm, "GPU lockup - switching to software fbcon\n");
 	info->flags |= FBINFO_HWACCEL_DISABLED;
 }
 
@@ -451,74 +453,81 @@ static struct drm_fb_helper_funcs nouveau_fbcon_helper_funcs = {
 };
 
 
-int nouveau_fbcon_init(struct drm_device *dev)
+int
+nouveau_fbcon_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fbdev *nfbdev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	struct nouveau_fbdev *fbcon;
 	int preferred_bpp;
 	int ret;
 
-	nfbdev = kzalloc(sizeof(struct nouveau_fbdev), GFP_KERNEL);
-	if (!nfbdev)
+	if (!dev->mode_config.num_crtc)
+		return 0;
+
+	fbcon = kzalloc(sizeof(struct nouveau_fbdev), GFP_KERNEL);
+	if (!fbcon)
 		return -ENOMEM;
 
-	nfbdev->dev = dev;
-	dev_priv->nfbdev = nfbdev;
-	nfbdev->helper.funcs = &nouveau_fbcon_helper_funcs;
+	fbcon->dev = dev;
+	drm->fbcon = fbcon;
+	fbcon->helper.funcs = &nouveau_fbcon_helper_funcs;
 
-	ret = drm_fb_helper_init(dev, &nfbdev->helper,
+	ret = drm_fb_helper_init(dev, &fbcon->helper,
 				 dev->mode_config.num_crtc, 4);
 	if (ret) {
-		kfree(nfbdev);
+		kfree(fbcon);
 		return ret;
 	}
 
-	drm_fb_helper_single_add_all_connectors(&nfbdev->helper);
+	drm_fb_helper_single_add_all_connectors(&fbcon->helper);
 
-	if (dev_priv->vram_size <= 32 * 1024 * 1024)
+	if (pfb->ram.size <= 32 * 1024 * 1024)
 		preferred_bpp = 8;
-	else if (dev_priv->vram_size <= 64 * 1024 * 1024)
+	else
+	if (pfb->ram.size <= 64 * 1024 * 1024)
 		preferred_bpp = 16;
 	else
 		preferred_bpp = 32;
 
-	drm_fb_helper_initial_config(&nfbdev->helper, preferred_bpp);
+	drm_fb_helper_initial_config(&fbcon->helper, preferred_bpp);
 	return 0;
 }
 
-void nouveau_fbcon_fini(struct drm_device *dev)
+void
+nouveau_fbcon_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	if (!dev_priv->nfbdev)
+	if (!drm->fbcon)
 		return;
 
-	nouveau_fbcon_destroy(dev, dev_priv->nfbdev);
-	kfree(dev_priv->nfbdev);
-	dev_priv->nfbdev = NULL;
+	nouveau_fbcon_destroy(dev, drm->fbcon);
+	kfree(drm->fbcon);
+	drm->fbcon = NULL;
 }
 
 void nouveau_fbcon_save_disable_accel(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	dev_priv->nfbdev->saved_flags = dev_priv->nfbdev->helper.fbdev->flags;
-	dev_priv->nfbdev->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
+	drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags;
+	drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
 }
 
 void nouveau_fbcon_restore_accel(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	dev_priv->nfbdev->helper.fbdev->flags = dev_priv->nfbdev->saved_flags;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags;
 }
 
 void nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	console_lock();
 	if (state == 0)
 		nouveau_fbcon_save_disable_accel(dev);
-	fb_set_suspend(dev_priv->nfbdev->helper.fbdev, state);
+	fb_set_suspend(drm->fbcon->helper.fbdev, state);
 	if (state == 1)
 		nouveau_fbcon_restore_accel(dev);
 	console_unlock();
@@ -526,6 +535,6 @@ void nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
 
 void nouveau_fbcon_zfill_all(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	nouveau_fbcon_zfill(dev, dev_priv->nfbdev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	nouveau_fbcon_zfill(dev, drm->fbcon);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index b73c29f87fc..fdfc0c94fbc 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -27,9 +27,10 @@
 #ifndef __NOUVEAU_FBCON_H__
 #define __NOUVEAU_FBCON_H__
 
-#include "drm_fb_helper.h"
+#include <drm/drm_fb_helper.h>
+
+#include "nouveau_display.h"
 
-#include "nouveau_fb.h"
 struct nouveau_fbdev {
 	struct drm_fb_helper helper;
 	struct nouveau_framebuffer nouveau_fb;
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index 3c180493dab..1d049be79f7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -24,17 +24,14 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include <linux/ktime.h>
 #include <linux/hrtimer.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-#include "nouveau_fence.h"
-#include "nouveau_software.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
+#include "nouveau_fence.h"
 
 void
 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
@@ -54,16 +51,16 @@ nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
 void
 nouveau_fence_context_new(struct nouveau_fence_chan *fctx)
 {
+	INIT_LIST_HEAD(&fctx->flip);
 	INIT_LIST_HEAD(&fctx->pending);
 	spin_lock_init(&fctx->lock);
 }
 
-void
+static void
 nouveau_fence_update(struct nouveau_channel *chan)
 {
-	struct drm_device *dev = chan->dev;
-	struct nouveau_fence_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FENCE);
-	struct nouveau_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
+	struct nouveau_fence_priv *priv = chan->drm->fence;
+	struct nouveau_fence_chan *fctx = chan->fence;
 	struct nouveau_fence *fence, *fnext;
 
 	spin_lock(&fctx->lock);
@@ -83,9 +80,8 @@ nouveau_fence_update(struct nouveau_channel *chan)
 int
 nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan)
 {
-	struct drm_device *dev = chan->dev;
-	struct nouveau_fence_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FENCE);
-	struct nouveau_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
+	struct nouveau_fence_priv *priv = chan->drm->fence;
+	struct nouveau_fence_chan *fctx = chan->fence;
 	int ret;
 
 	fence->channel  = chan;
@@ -147,19 +143,17 @@ nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 int
 nouveau_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *chan)
 {
-	struct drm_device *dev = chan->dev;
-	struct nouveau_fence_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FENCE);
+	struct nouveau_fence_priv *priv = chan->drm->fence;
 	struct nouveau_channel *prev;
 	int ret = 0;
 
-	prev = fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
+	prev = fence ? fence->channel : NULL;
 	if (prev) {
 		if (unlikely(prev != chan && !nouveau_fence_done(fence))) {
 			ret = priv->sync(fence, prev, chan);
 			if (unlikely(ret))
 				ret = nouveau_fence_wait(fence, true, false);
 		}
-		nouveau_channel_put_unlocked(&prev);
 	}
 
 	return ret;
@@ -193,7 +187,7 @@ nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence)
 	struct nouveau_fence *fence;
 	int ret = 0;
 
-	if (unlikely(!chan->engctx[NVOBJ_ENGINE_FENCE]))
+	if (unlikely(!chan->fence))
 		return -ENODEV;
 
 	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h
index 82ba733393a..bedafd1c953 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.h
@@ -1,6 +1,8 @@
 #ifndef __NOUVEAU_FENCE_H__
 #define __NOUVEAU_FENCE_H__
 
+struct nouveau_drm;
+
 struct nouveau_fence {
 	struct list_head head;
 	struct kref kref;
@@ -22,31 +24,48 @@ int  nouveau_fence_emit(struct nouveau_fence *, struct nouveau_channel *);
 bool nouveau_fence_done(struct nouveau_fence *);
 int  nouveau_fence_wait(struct nouveau_fence *, bool lazy, bool intr);
 int  nouveau_fence_sync(struct nouveau_fence *, struct nouveau_channel *);
-void nouveau_fence_idle(struct nouveau_channel *);
-void nouveau_fence_update(struct nouveau_channel *);
 
 struct nouveau_fence_chan {
 	struct list_head pending;
+	struct list_head flip;
+
 	spinlock_t lock;
 	u32 sequence;
 };
 
 struct nouveau_fence_priv {
-	struct nouveau_exec_engine engine;
-	int (*emit)(struct nouveau_fence *);
-	int (*sync)(struct nouveau_fence *, struct nouveau_channel *,
-		    struct nouveau_channel *);
-	u32 (*read)(struct nouveau_channel *);
+	void (*dtor)(struct nouveau_drm *);
+	bool (*suspend)(struct nouveau_drm *);
+	void (*resume)(struct nouveau_drm *);
+	int  (*context_new)(struct nouveau_channel *);
+	void (*context_del)(struct nouveau_channel *);
+	int  (*emit)(struct nouveau_fence *);
+	int  (*sync)(struct nouveau_fence *, struct nouveau_channel *,
+		     struct nouveau_channel *);
+	u32  (*read)(struct nouveau_channel *);
 };
 
+#define nouveau_fence(drm) ((struct nouveau_fence_priv *)(drm)->fence)
+
 void nouveau_fence_context_new(struct nouveau_fence_chan *);
 void nouveau_fence_context_del(struct nouveau_fence_chan *);
 
-int nv04_fence_create(struct drm_device *dev);
+int nv04_fence_create(struct nouveau_drm *);
 int nv04_fence_mthd(struct nouveau_channel *, u32, u32, u32);
 
-int nv10_fence_create(struct drm_device *dev);
-int nv84_fence_create(struct drm_device *dev);
-int nvc0_fence_create(struct drm_device *dev);
+int  nv10_fence_emit(struct nouveau_fence *);
+int  nv17_fence_sync(struct nouveau_fence *, struct nouveau_channel *,
+		     struct nouveau_channel *);
+u32  nv10_fence_read(struct nouveau_channel *);
+void nv10_fence_context_del(struct nouveau_channel *);
+void nv10_fence_destroy(struct nouveau_drm *);
+int  nv10_fence_create(struct nouveau_drm *);
+
+int nv50_fence_create(struct nouveau_drm *);
+int nv84_fence_create(struct nouveau_drm *);
+int nvc0_fence_create(struct nouveau_drm *);
+u64 nvc0_fence_crtc(struct nouveau_channel *, int crtc);
+
+int nouveau_flip_complete(void *chan);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_fifo.h b/drivers/gpu/drm/nouveau/nouveau_fifo.h
deleted file mode 100644
index ce99cab2f25..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_fifo.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#ifndef __NOUVEAU_FIFO_H__
-#define __NOUVEAU_FIFO_H__
-
-struct nouveau_fifo_priv {
-	struct nouveau_exec_engine base;
-	u32 channels;
-};
-
-struct nouveau_fifo_chan {
-};
-
-bool nv04_fifo_cache_pull(struct drm_device *, bool);
-void nv04_fifo_context_del(struct nouveau_channel *, int);
-int  nv04_fifo_fini(struct drm_device *, int, bool);
-int  nv04_fifo_init(struct drm_device *, int);
-void nv04_fifo_isr(struct drm_device *);
-void nv04_fifo_destroy(struct drm_device *, int);
-
-void nv50_fifo_playlist_update(struct drm_device *);
-void nv50_fifo_destroy(struct drm_device *, int);
-void nv50_fifo_tlb_flush(struct drm_device *, int);
-
-int  nv04_fifo_create(struct drm_device *);
-int  nv10_fifo_create(struct drm_device *);
-int  nv17_fifo_create(struct drm_device *);
-int  nv40_fifo_create(struct drm_device *);
-int  nv50_fifo_create(struct drm_device *);
-int  nv84_fifo_create(struct drm_device *);
-int  nvc0_fifo_create(struct drm_device *);
-int  nve0_fifo_create(struct drm_device *);
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index af7cfb82571..5e2f52158f1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -23,16 +23,18 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
+
 #include <linux/dma-buf.h>
-#include "drmP.h"
-#include "drm.h"
 
-#include "nouveau_drv.h"
+#include <subdev/fb.h>
+
 #include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_fence.h"
+#include "nouveau_abi16.h"
 
-#define nouveau_gem_pushbuf_sync(chan) 0
+#include "nouveau_ttm.h"
+#include "nouveau_gem.h"
 
 int
 nouveau_gem_object_new(struct drm_gem_object *gem)
@@ -67,19 +69,19 @@ nouveau_gem_object_del(struct drm_gem_object *gem)
 int
 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
 {
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 	struct nouveau_vma *vma;
 	int ret;
 
-	if (!fpriv->vm)
+	if (!cli->base.vm)
 		return 0;
 
 	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
 	if (ret)
 		return ret;
 
-	vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+	vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
 	if (!vma) {
 		vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 		if (!vma) {
@@ -87,7 +89,7 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
 			goto out;
 		}
 
-		ret = nouveau_bo_vma_add(nvbo, fpriv->vm, vma);
+		ret = nouveau_bo_vma_add(nvbo, cli->base.vm, vma);
 		if (ret) {
 			kfree(vma);
 			goto out;
@@ -104,19 +106,19 @@ out:
 void
 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
 {
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 	struct nouveau_vma *vma;
 	int ret;
 
-	if (!fpriv->vm)
+	if (!cli->base.vm)
 		return;
 
 	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
 	if (ret)
 		return;
 
-	vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+	vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
 	if (vma) {
 		if (--vma->refcount == 0) {
 			nouveau_bo_vma_del(nvbo, vma);
@@ -131,7 +133,7 @@ nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
 		uint32_t tile_mode, uint32_t tile_flags,
 		struct nouveau_bo **pnvbo)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_bo *nvbo;
 	u32 flags = 0;
 	int ret;
@@ -155,7 +157,7 @@ nouveau_gem_new(struct drm_device *dev, int size, int align, uint32_t domain,
 	 */
 	nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
 			      NOUVEAU_GEM_DOMAIN_GART;
-	if (dev_priv->card_type >= NV_50)
+	if (nv_device(drm->device)->card_type >= NV_50)
 		nvbo->valid_domains &= domain;
 
 	nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size);
@@ -173,7 +175,7 @@ static int
 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
 		 struct drm_nouveau_gem_info *rep)
 {
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
 	struct nouveau_bo *nvbo = nouveau_gem_object(gem);
 	struct nouveau_vma *vma;
 
@@ -183,8 +185,8 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
 		rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
 
 	rep->offset = nvbo->bo.offset;
-	if (fpriv->vm) {
-		vma = nouveau_bo_vma_find(nvbo, fpriv->vm);
+	if (cli->base.vm) {
+		vma = nouveau_bo_vma_find(nvbo, cli->base.vm);
 		if (!vma)
 			return -EINVAL;
 
@@ -202,15 +204,16 @@ int
 nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
 		      struct drm_file *file_priv)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
 	struct drm_nouveau_gem_new *req = data;
 	struct nouveau_bo *nvbo = NULL;
 	int ret = 0;
 
-	dev_priv->ttm.bdev.dev_mapping = dev->dev_mapping;
+	drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping;
 
-	if (!dev_priv->engine.vram.flags_valid(dev, req->info.tile_flags)) {
-		NV_ERROR(dev, "bad page flags: 0x%08x\n", req->info.tile_flags);
+	if (!pfb->memtype_valid(pfb, req->info.tile_flags)) {
+		NV_ERROR(drm, "bad page flags: 0x%08x\n", req->info.tile_flags);
 		return -EINVAL;
 	}
 
@@ -312,16 +315,16 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
 	      struct drm_nouveau_gem_pushbuf_bo *pbbo,
 	      int nr_buffers, struct validate_op *op)
 {
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct drm_device *dev = chan->drm->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint32_t sequence;
 	int trycnt = 0;
 	int ret, i;
 
-	sequence = atomic_add_return(1, &dev_priv->ttm.validate_sequence);
+	sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
 retry:
 	if (++trycnt > 100000) {
-		NV_ERROR(dev, "%s failed and gave up.\n", __func__);
+		NV_ERROR(drm, "%s failed and gave up.\n", __func__);
 		return -EINVAL;
 	}
 
@@ -332,14 +335,14 @@ retry:
 
 		gem = drm_gem_object_lookup(dev, file_priv, b->handle);
 		if (!gem) {
-			NV_ERROR(dev, "Unknown handle 0x%08x\n", b->handle);
+			NV_ERROR(drm, "Unknown handle 0x%08x\n", b->handle);
 			validate_fini(op, NULL);
 			return -ENOENT;
 		}
 		nvbo = gem->driver_private;
 
 		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
-			NV_ERROR(dev, "multiple instances of buffer %d on "
+			NV_ERROR(drm, "multiple instances of buffer %d on "
 				      "validation list\n", b->handle);
 			drm_gem_object_unreference_unlocked(gem);
 			validate_fini(op, NULL);
@@ -354,7 +357,7 @@ retry:
 			drm_gem_object_unreference_unlocked(gem);
 			if (unlikely(ret)) {
 				if (ret != -ERESTARTSYS)
-					NV_ERROR(dev, "fail reserve\n");
+					NV_ERROR(drm, "fail reserve\n");
 				return ret;
 			}
 			goto retry;
@@ -373,7 +376,7 @@ retry:
 		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
 			list_add_tail(&nvbo->entry, &op->gart_list);
 		else {
-			NV_ERROR(dev, "invalid valid domains: 0x%08x\n",
+			NV_ERROR(drm, "invalid valid domains: 0x%08x\n",
 				 b->valid_domains);
 			list_add_tail(&nvbo->entry, &op->both_list);
 			validate_fini(op, NULL);
@@ -407,10 +410,9 @@ static int
 validate_list(struct nouveau_channel *chan, struct list_head *list,
 	      struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr)
 {
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
+	struct nouveau_drm *drm = chan->drm;
 	struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
 				(void __force __user *)(uintptr_t)user_pbbo_ptr;
-	struct drm_device *dev = chan->dev;
 	struct nouveau_bo *nvbo;
 	int ret, relocs = 0;
 
@@ -419,7 +421,7 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
 
 		ret = validate_sync(chan, nvbo);
 		if (unlikely(ret)) {
-			NV_ERROR(dev, "fail pre-validate sync\n");
+			NV_ERROR(drm, "fail pre-validate sync\n");
 			return ret;
 		}
 
@@ -427,24 +429,24 @@ validate_list(struct nouveau_channel *chan, struct list_head *list,
 					     b->write_domains,
 					     b->valid_domains);
 		if (unlikely(ret)) {
-			NV_ERROR(dev, "fail set_domain\n");
+			NV_ERROR(drm, "fail set_domain\n");
 			return ret;
 		}
 
 		ret = nouveau_bo_validate(nvbo, true, false, false);
 		if (unlikely(ret)) {
 			if (ret != -ERESTARTSYS)
-				NV_ERROR(dev, "fail ttm_validate\n");
+				NV_ERROR(drm, "fail ttm_validate\n");
 			return ret;
 		}
 
 		ret = validate_sync(chan, nvbo);
 		if (unlikely(ret)) {
-			NV_ERROR(dev, "fail post-validate sync\n");
+			NV_ERROR(drm, "fail post-validate sync\n");
 			return ret;
 		}
 
-		if (dev_priv->card_type < NV_50) {
+		if (nv_device(drm->device)->card_type < NV_50) {
 			if (nvbo->bo.offset == b->presumed.offset &&
 			    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
 			      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
@@ -476,7 +478,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 			     uint64_t user_buffers, int nr_buffers,
 			     struct validate_op *op, int *apply_relocs)
 {
-	struct drm_device *dev = chan->dev;
+	struct nouveau_drm *drm = chan->drm;
 	int ret, relocs = 0;
 
 	INIT_LIST_HEAD(&op->vram_list);
@@ -489,14 +491,14 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 	ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
 	if (unlikely(ret)) {
 		if (ret != -ERESTARTSYS)
-			NV_ERROR(dev, "validate_init\n");
+			NV_ERROR(drm, "validate_init\n");
 		return ret;
 	}
 
 	ret = validate_list(chan, &op->vram_list, pbbo, user_buffers);
 	if (unlikely(ret < 0)) {
 		if (ret != -ERESTARTSYS)
-			NV_ERROR(dev, "validate vram_list\n");
+			NV_ERROR(drm, "validate vram_list\n");
 		validate_fini(op, NULL);
 		return ret;
 	}
@@ -505,7 +507,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 	ret = validate_list(chan, &op->gart_list, pbbo, user_buffers);
 	if (unlikely(ret < 0)) {
 		if (ret != -ERESTARTSYS)
-			NV_ERROR(dev, "validate gart_list\n");
+			NV_ERROR(drm, "validate gart_list\n");
 		validate_fini(op, NULL);
 		return ret;
 	}
@@ -514,7 +516,7 @@ nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
 	ret = validate_list(chan, &op->both_list, pbbo, user_buffers);
 	if (unlikely(ret < 0)) {
 		if (ret != -ERESTARTSYS)
-			NV_ERROR(dev, "validate both_list\n");
+			NV_ERROR(drm, "validate both_list\n");
 		validate_fini(op, NULL);
 		return ret;
 	}
@@ -547,6 +549,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 				struct drm_nouveau_gem_pushbuf *req,
 				struct drm_nouveau_gem_pushbuf_bo *bo)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
 	int ret = 0;
 	unsigned i;
@@ -562,7 +565,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 		uint32_t data;
 
 		if (unlikely(r->bo_index > req->nr_buffers)) {
-			NV_ERROR(dev, "reloc bo index invalid\n");
+			NV_ERROR(drm, "reloc bo index invalid\n");
 			ret = -EINVAL;
 			break;
 		}
@@ -572,7 +575,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 			continue;
 
 		if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
-			NV_ERROR(dev, "reloc container bo index invalid\n");
+			NV_ERROR(drm, "reloc container bo index invalid\n");
 			ret = -EINVAL;
 			break;
 		}
@@ -580,7 +583,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 
 		if (unlikely(r->reloc_bo_offset + 4 >
 			     nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
-			NV_ERROR(dev, "reloc outside of bo\n");
+			NV_ERROR(drm, "reloc outside of bo\n");
 			ret = -EINVAL;
 			break;
 		}
@@ -589,7 +592,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 			ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
 					  &nvbo->kmap);
 			if (ret) {
-				NV_ERROR(dev, "failed kmap for reloc\n");
+				NV_ERROR(drm, "failed kmap for reloc\n");
 				break;
 			}
 			nvbo->validate_mapped = true;
@@ -614,7 +617,7 @@ nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
 		ret = ttm_bo_wait(&nvbo->bo, false, false, false);
 		spin_unlock(&nvbo->bo.bdev->fence_lock);
 		if (ret) {
-			NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret);
+			NV_ERROR(drm, "reloc wait_idle failed: %d\n", ret);
 			break;
 		}
 
@@ -629,62 +632,67 @@ int
 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv, dev);
+	struct nouveau_abi16_chan *temp;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_nouveau_gem_pushbuf *req = data;
 	struct drm_nouveau_gem_pushbuf_push *push;
 	struct drm_nouveau_gem_pushbuf_bo *bo;
-	struct nouveau_channel *chan;
+	struct nouveau_channel *chan = NULL;
 	struct validate_op op;
 	struct nouveau_fence *fence = NULL;
 	int i, j, ret = 0, do_reloc = 0;
 
-	chan = nouveau_channel_get(file_priv, req->channel);
-	if (IS_ERR(chan))
-		return PTR_ERR(chan);
+	if (unlikely(!abi16))
+		return -ENOMEM;
+
+	list_for_each_entry(temp, &abi16->channels, head) {
+		if (temp->chan->handle == (NVDRM_CHAN | req->channel)) {
+			chan = temp->chan;
+			break;
+		}
+	}
 
-	req->vram_available = dev_priv->fb_aper_free;
-	req->gart_available = dev_priv->gart_info.aper_free;
+	if (!chan)
+		return nouveau_abi16_put(abi16, -ENOENT);
+
+	req->vram_available = drm->gem.vram_available;
+	req->gart_available = drm->gem.gart_available;
 	if (unlikely(req->nr_push == 0))
 		goto out_next;
 
 	if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
-		NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n",
+		NV_ERROR(drm, "pushbuf push count exceeds limit: %d max %d\n",
 			 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
-		nouveau_channel_put(&chan);
-		return -EINVAL;
+		return nouveau_abi16_put(abi16, -EINVAL);
 	}
 
 	if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
-		NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n",
+		NV_ERROR(drm, "pushbuf bo count exceeds limit: %d max %d\n",
 			 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
-		nouveau_channel_put(&chan);
-		return -EINVAL;
+		return nouveau_abi16_put(abi16, -EINVAL);
 	}
 
 	if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
-		NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n",
+		NV_ERROR(drm, "pushbuf reloc count exceeds limit: %d max %d\n",
 			 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
-		nouveau_channel_put(&chan);
-		return -EINVAL;
+		return nouveau_abi16_put(abi16, -EINVAL);
 	}
 
 	push = u_memcpya(req->push, req->nr_push, sizeof(*push));
-	if (IS_ERR(push)) {
-		nouveau_channel_put(&chan);
-		return PTR_ERR(push);
-	}
+	if (IS_ERR(push))
+		return nouveau_abi16_put(abi16, PTR_ERR(push));
 
 	bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
 	if (IS_ERR(bo)) {
 		kfree(push);
-		nouveau_channel_put(&chan);
-		return PTR_ERR(bo);
+		return nouveau_abi16_put(abi16, PTR_ERR(bo));
 	}
 
 	/* Ensure all push buffers are on validate list */
 	for (i = 0; i < req->nr_push; i++) {
 		if (push[i].bo_index >= req->nr_buffers) {
-			NV_ERROR(dev, "push %d buffer not in list\n", i);
+			NV_ERROR(drm, "push %d buffer not in list\n", i);
 			ret = -EINVAL;
 			goto out_prevalid;
 		}
@@ -695,7 +703,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 					   req->nr_buffers, &op, &do_reloc);
 	if (ret) {
 		if (ret != -ERESTARTSYS)
-			NV_ERROR(dev, "validate: %d\n", ret);
+			NV_ERROR(drm, "validate: %d\n", ret);
 		goto out_prevalid;
 	}
 
@@ -703,7 +711,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 	if (do_reloc) {
 		ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo);
 		if (ret) {
-			NV_ERROR(dev, "reloc apply: %d\n", ret);
+			NV_ERROR(drm, "reloc apply: %d\n", ret);
 			goto out;
 		}
 	}
@@ -711,7 +719,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 	if (chan->dma.ib_max) {
 		ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
 		if (ret) {
-			NV_INFO(dev, "nv50cal_space: %d\n", ret);
+			NV_ERROR(drm, "nv50cal_space: %d\n", ret);
 			goto out;
 		}
 
@@ -723,36 +731,33 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 				      push[i].length);
 		}
 	} else
-	if (dev_priv->chipset >= 0x25) {
+	if (nv_device(drm->device)->chipset >= 0x25) {
 		ret = RING_SPACE(chan, req->nr_push * 2);
 		if (ret) {
-			NV_ERROR(dev, "cal_space: %d\n", ret);
+			NV_ERROR(drm, "cal_space: %d\n", ret);
 			goto out;
 		}
 
 		for (i = 0; i < req->nr_push; i++) {
 			struct nouveau_bo *nvbo = (void *)(unsigned long)
 				bo[push[i].bo_index].user_priv;
-			struct drm_mm_node *mem = nvbo->bo.mem.mm_node;
 
-			OUT_RING(chan, ((mem->start << PAGE_SHIFT) +
-					push[i].offset) | 2);
+			OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2);
 			OUT_RING(chan, 0);
 		}
 	} else {
 		ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
 		if (ret) {
-			NV_ERROR(dev, "jmp_space: %d\n", ret);
+			NV_ERROR(drm, "jmp_space: %d\n", ret);
 			goto out;
 		}
 
 		for (i = 0; i < req->nr_push; i++) {
 			struct nouveau_bo *nvbo = (void *)(unsigned long)
 				bo[push[i].bo_index].user_priv;
-			struct drm_mm_node *mem = nvbo->bo.mem.mm_node;
 			uint32_t cmd;
 
-			cmd = chan->pushbuf_base + ((chan->dma.cur + 2) << 2);
+			cmd = chan->push.vma.offset + ((chan->dma.cur + 2) << 2);
 			cmd |= 0x20000000;
 			if (unlikely(cmd != req->suffix0)) {
 				if (!nvbo->kmap.virtual) {
@@ -771,8 +776,8 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 						push[i].length - 8) / 4, cmd);
 			}
 
-			OUT_RING(chan, ((mem->start << PAGE_SHIFT) +
-					push[i].offset) | 0x20000000);
+			OUT_RING(chan, 0x20000000 |
+				      (nvbo->bo.offset + push[i].offset));
 			OUT_RING(chan, 0);
 			for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
 				OUT_RING(chan, 0);
@@ -781,7 +786,7 @@ nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
 
 	ret = nouveau_fence_new(chan, &fence);
 	if (ret) {
-		NV_ERROR(dev, "error fencing pushbuf: %d\n", ret);
+		NV_ERROR(drm, "error fencing pushbuf: %d\n", ret);
 		WIND_RING(chan);
 		goto out;
 	}
@@ -799,17 +804,16 @@ out_next:
 		req->suffix0 = 0x00000000;
 		req->suffix1 = 0x00000000;
 	} else
-	if (dev_priv->chipset >= 0x25) {
+	if (nv_device(drm->device)->chipset >= 0x25) {
 		req->suffix0 = 0x00020000;
 		req->suffix1 = 0x00000000;
 	} else {
 		req->suffix0 = 0x20000000 |
-			      (chan->pushbuf_base + ((chan->dma.cur + 2) << 2));
+			      (chan->push.vma.offset + ((chan->dma.cur + 2) << 2));
 		req->suffix1 = 0x00000000;
 	}
 
-	nouveau_channel_put(&chan);
-	return ret;
+	return nouveau_abi16_put(abi16, ret);
 }
 
 static inline uint32_t
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.h b/drivers/gpu/drm/nouveau/nouveau_gem.h
new file mode 100644
index 00000000000..5c1049236d2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.h
@@ -0,0 +1,43 @@
+#ifndef __NOUVEAU_GEM_H__
+#define __NOUVEAU_GEM_H__
+
+#include <drm/drmP.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_bo.h"
+
+#define nouveau_bo_tile_layout(nvbo)				\
+	((nvbo)->tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK)
+
+static inline struct nouveau_bo *
+nouveau_gem_object(struct drm_gem_object *gem)
+{
+	return gem ? gem->driver_private : NULL;
+}
+
+/* nouveau_gem.c */
+extern int nouveau_gem_new(struct drm_device *, int size, int align,
+			   uint32_t domain, uint32_t tile_mode,
+			   uint32_t tile_flags, struct nouveau_bo **);
+extern int nouveau_gem_object_new(struct drm_gem_object *);
+extern void nouveau_gem_object_del(struct drm_gem_object *);
+extern int nouveau_gem_object_open(struct drm_gem_object *, struct drm_file *);
+extern void nouveau_gem_object_close(struct drm_gem_object *,
+				     struct drm_file *);
+extern int nouveau_gem_ioctl_new(struct drm_device *, void *,
+				 struct drm_file *);
+extern int nouveau_gem_ioctl_pushbuf(struct drm_device *, void *,
+				     struct drm_file *);
+extern int nouveau_gem_ioctl_cpu_prep(struct drm_device *, void *,
+				      struct drm_file *);
+extern int nouveau_gem_ioctl_cpu_fini(struct drm_device *, void *,
+				      struct drm_file *);
+extern int nouveau_gem_ioctl_info(struct drm_device *, void *,
+				  struct drm_file *);
+
+extern struct dma_buf *nouveau_gem_prime_export(struct drm_device *dev,
+				struct drm_gem_object *obj, int flags);
+extern struct drm_gem_object *nouveau_gem_prime_import(struct drm_device *dev,
+				struct dma_buf *dma_buf);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_gpio.c b/drivers/gpu/drm/nouveau/nouveau_gpio.c
deleted file mode 100644
index 0fe4e17c461..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_gpio.c
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_i2c.h"
-#include "nouveau_gpio.h"
-
-static u8 *
-dcb_gpio_table(struct drm_device *dev)
-{
-	u8 *dcb = dcb_table(dev);
-	if (dcb) {
-		if (dcb[0] >= 0x30 && dcb[1] >= 0x0c)
-			return ROMPTR(dev, dcb[0x0a]);
-		if (dcb[0] >= 0x22 && dcb[-1] >= 0x13)
-			return ROMPTR(dev, dcb[-15]);
-	}
-	return NULL;
-}
-
-static u8 *
-dcb_gpio_entry(struct drm_device *dev, int idx, int ent, u8 *version)
-{
-	u8 *table = dcb_gpio_table(dev);
-	if (table) {
-		*version = table[0];
-		if (*version < 0x30 && ent < table[2])
-			return table + 3 + (ent * table[1]);
-		else if (ent < table[2])
-			return table + table[1] + (ent * table[3]);
-	}
-	return NULL;
-}
-
-int
-nouveau_gpio_drive(struct drm_device *dev, int idx, int line, int dir, int out)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-
-	return pgpio->drive ? pgpio->drive(dev, line, dir, out) : -ENODEV;
-}
-
-int
-nouveau_gpio_sense(struct drm_device *dev, int idx, int line)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-
-	return pgpio->sense ? pgpio->sense(dev, line) : -ENODEV;
-}
-
-int
-nouveau_gpio_find(struct drm_device *dev, int idx, u8 func, u8 line,
-		  struct gpio_func *gpio)
-{
-	u8 *table, *entry, version;
-	int i = -1;
-
-	if (line == 0xff && func == 0xff)
-		return -EINVAL;
-
-	while ((entry = dcb_gpio_entry(dev, idx, ++i, &version))) {
-		if (version < 0x40) {
-			u16 data = ROM16(entry[0]);
-			*gpio = (struct gpio_func) {
-				.line = (data & 0x001f) >> 0,
-				.func = (data & 0x07e0) >> 5,
-				.log[0] = (data & 0x1800) >> 11,
-				.log[1] = (data & 0x6000) >> 13,
-			};
-		} else
-		if (version < 0x41) {
-			*gpio = (struct gpio_func) {
-				.line = entry[0] & 0x1f,
-				.func = entry[1],
-				.log[0] = (entry[3] & 0x18) >> 3,
-				.log[1] = (entry[3] & 0x60) >> 5,
-			};
-		} else {
-			*gpio = (struct gpio_func) {
-				.line = entry[0] & 0x3f,
-				.func = entry[1],
-				.log[0] = (entry[4] & 0x30) >> 4,
-				.log[1] = (entry[4] & 0xc0) >> 6,
-			};
-		}
-
-		if ((line == 0xff || line == gpio->line) &&
-		    (func == 0xff || func == gpio->func))
-			return 0;
-	}
-
-	/* DCB 2.2, fixed TVDAC GPIO data */
-	if ((table = dcb_table(dev)) && table[0] >= 0x22) {
-		if (func == DCB_GPIO_TVDAC0) {
-			*gpio = (struct gpio_func) {
-				.func = DCB_GPIO_TVDAC0,
-				.line = table[-4] >> 4,
-				.log[0] = !!(table[-5] & 2),
-				.log[1] =  !(table[-5] & 2),
-			};
-			return 0;
-		}
-	}
-
-	/* Apple iMac G4 NV18 */
-	if (nv_match_device(dev, 0x0189, 0x10de, 0x0010)) {
-		if (func == DCB_GPIO_TVDAC0) {
-			*gpio = (struct gpio_func) {
-				.func = DCB_GPIO_TVDAC0,
-				.line = 4,
-				.log[0] = 0,
-				.log[1] = 1,
-			};
-			return 0;
-		}
-	}
-
-	return -EINVAL;
-}
-
-int
-nouveau_gpio_set(struct drm_device *dev, int idx, u8 tag, u8 line, int state)
-{
-	struct gpio_func gpio;
-	int ret;
-
-	ret = nouveau_gpio_find(dev, idx, tag, line, &gpio);
-	if (ret == 0) {
-		int dir = !!(gpio.log[state] & 0x02);
-		int out = !!(gpio.log[state] & 0x01);
-		ret = nouveau_gpio_drive(dev, idx, gpio.line, dir, out);
-	}
-
-	return ret;
-}
-
-int
-nouveau_gpio_get(struct drm_device *dev, int idx, u8 tag, u8 line)
-{
-	struct gpio_func gpio;
-	int ret;
-
-	ret = nouveau_gpio_find(dev, idx, tag, line, &gpio);
-	if (ret == 0) {
-		ret = nouveau_gpio_sense(dev, idx, gpio.line);
-		if (ret >= 0)
-			ret = (ret == (gpio.log[1] & 1));
-	}
-
-	return ret;
-}
-
-int
-nouveau_gpio_irq(struct drm_device *dev, int idx, u8 tag, u8 line, bool on)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	struct gpio_func gpio;
-	int ret;
-
-	ret = nouveau_gpio_find(dev, idx, tag, line, &gpio);
-	if (ret == 0) {
-		if (idx == 0 && pgpio->irq_enable)
-			pgpio->irq_enable(dev, gpio.line, on);
-		else
-			ret = -ENODEV;
-	}
-
-	return ret;
-}
-
-struct gpio_isr {
-	struct drm_device *dev;
-	struct list_head head;
-	struct work_struct work;
-	int idx;
-	struct gpio_func func;
-	void (*handler)(void *, int);
-	void *data;
-	bool inhibit;
-};
-
-static void
-nouveau_gpio_isr_bh(struct work_struct *work)
-{
-	struct gpio_isr *isr = container_of(work, struct gpio_isr, work);
-	struct drm_device *dev = isr->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	unsigned long flags;
-	int state;
-
-	state = nouveau_gpio_get(dev, isr->idx, isr->func.func, isr->func.line);
-	if (state >= 0)
-		isr->handler(isr->data, state);
-
-	spin_lock_irqsave(&pgpio->lock, flags);
-	isr->inhibit = false;
-	spin_unlock_irqrestore(&pgpio->lock, flags);
-}
-
-void
-nouveau_gpio_isr(struct drm_device *dev, int idx, u32 line_mask)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	struct gpio_isr *isr;
-
-	if (idx != 0)
-		return;
-
-	spin_lock(&pgpio->lock);
-	list_for_each_entry(isr, &pgpio->isr, head) {
-		if (line_mask & (1 << isr->func.line)) {
-			if (isr->inhibit)
-				continue;
-			isr->inhibit = true;
-			schedule_work(&isr->work);
-		}
-	}
-	spin_unlock(&pgpio->lock);
-}
-
-int
-nouveau_gpio_isr_add(struct drm_device *dev, int idx, u8 tag, u8 line,
-		     void (*handler)(void *, int), void *data)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	struct gpio_isr *isr;
-	unsigned long flags;
-	int ret;
-
-	isr = kzalloc(sizeof(*isr), GFP_KERNEL);
-	if (!isr)
-		return -ENOMEM;
-
-	ret = nouveau_gpio_find(dev, idx, tag, line, &isr->func);
-	if (ret) {
-		kfree(isr);
-		return ret;
-	}
-
-	INIT_WORK(&isr->work, nouveau_gpio_isr_bh);
-	isr->dev = dev;
-	isr->handler = handler;
-	isr->data = data;
-	isr->idx = idx;
-
-	spin_lock_irqsave(&pgpio->lock, flags);
-	list_add(&isr->head, &pgpio->isr);
-	spin_unlock_irqrestore(&pgpio->lock, flags);
-	return 0;
-}
-
-void
-nouveau_gpio_isr_del(struct drm_device *dev, int idx, u8 tag, u8 line,
-		     void (*handler)(void *, int), void *data)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	struct gpio_isr *isr, *tmp;
-	struct gpio_func func;
-	unsigned long flags;
-	LIST_HEAD(tofree);
-	int ret;
-
-	ret = nouveau_gpio_find(dev, idx, tag, line, &func);
-	if (ret == 0) {
-		spin_lock_irqsave(&pgpio->lock, flags);
-		list_for_each_entry_safe(isr, tmp, &pgpio->isr, head) {
-			if (memcmp(&isr->func, &func, sizeof(func)) ||
-			    isr->idx != idx ||
-			    isr->handler != handler || isr->data != data)
-				continue;
-			list_move(&isr->head, &tofree);
-		}
-		spin_unlock_irqrestore(&pgpio->lock, flags);
-
-		list_for_each_entry_safe(isr, tmp, &tofree, head) {
-			flush_work(&isr->work);
-			kfree(isr);
-		}
-	}
-}
-
-int
-nouveau_gpio_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-
-	INIT_LIST_HEAD(&pgpio->isr);
-	spin_lock_init(&pgpio->lock);
-
-	return nouveau_gpio_init(dev);
-}
-
-void
-nouveau_gpio_destroy(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-
-	nouveau_gpio_fini(dev);
-	BUG_ON(!list_empty(&pgpio->isr));
-}
-
-int
-nouveau_gpio_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-	int ret = 0;
-
-	if (pgpio->init)
-		ret = pgpio->init(dev);
-
-	return ret;
-}
-
-void
-nouveau_gpio_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpio_engine *pgpio = &dev_priv->engine.gpio;
-
-	if (pgpio->fini)
-		pgpio->fini(dev);
-}
-
-void
-nouveau_gpio_reset(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u8 *entry, version;
-	int ent = -1;
-
-	while ((entry = dcb_gpio_entry(dev, 0, ++ent, &version))) {
-		u8 func = 0xff, line, defs, unk0, unk1;
-		if (version >= 0x41) {
-			defs = !!(entry[0] & 0x80);
-			line = entry[0] & 0x3f;
-			func = entry[1];
-			unk0 = entry[2];
-			unk1 = entry[3] & 0x1f;
-		} else
-		if (version >= 0x40) {
-			line = entry[0] & 0x1f;
-			func = entry[1];
-			defs = !!(entry[3] & 0x01);
-			unk0 = !!(entry[3] & 0x02);
-			unk1 = !!(entry[3] & 0x04);
-		} else {
-			break;
-		}
-
-		if (func == 0xff)
-			continue;
-
-		nouveau_gpio_func_set(dev, func, defs);
-
-		if (dev_priv->card_type >= NV_D0) {
-			nv_mask(dev, 0x00d610 + (line * 4), 0xff, unk0);
-			if (unk1--)
-				nv_mask(dev, 0x00d740 + (unk1 * 4), 0xff, line);
-		} else
-		if (dev_priv->card_type >= NV_50) {
-			static const u32 regs[] = { 0xe100, 0xe28c };
-			u32 val = (unk1 << 16) | unk0;
-			u32 reg = regs[line >> 4]; line &= 0x0f;
-
-			nv_mask(dev, reg, 0x00010001 << line, val << line);
-		}
-	}
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_gpio.h b/drivers/gpu/drm/nouveau/nouveau_gpio.h
deleted file mode 100644
index 64c5cb077ac..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_gpio.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#ifndef __NOUVEAU_GPIO_H__
-#define __NOUVEAU_GPIO_H__
-
-struct gpio_func {
-	u8 func;
-	u8 line;
-	u8 log[2];
-};
-
-/* nouveau_gpio.c */
-int  nouveau_gpio_create(struct drm_device *);
-void nouveau_gpio_destroy(struct drm_device *);
-int  nouveau_gpio_init(struct drm_device *);
-void nouveau_gpio_fini(struct drm_device *);
-void nouveau_gpio_reset(struct drm_device *);
-int  nouveau_gpio_drive(struct drm_device *, int idx, int line,
-			int dir, int out);
-int  nouveau_gpio_sense(struct drm_device *, int idx, int line);
-int  nouveau_gpio_find(struct drm_device *, int idx, u8 tag, u8 line,
-		       struct gpio_func *);
-int  nouveau_gpio_set(struct drm_device *, int idx, u8 tag, u8 line, int state);
-int  nouveau_gpio_get(struct drm_device *, int idx, u8 tag, u8 line);
-int  nouveau_gpio_irq(struct drm_device *, int idx, u8 tag, u8 line, bool on);
-void nouveau_gpio_isr(struct drm_device *, int idx, u32 mask);
-int  nouveau_gpio_isr_add(struct drm_device *, int idx, u8 tag, u8 line,
-			  void (*)(void *, int state), void *data);
-void nouveau_gpio_isr_del(struct drm_device *, int idx, u8 tag, u8 line,
-			  void (*)(void *, int state), void *data);
-
-static inline bool
-nouveau_gpio_func_valid(struct drm_device *dev, u8 tag)
-{
-	struct gpio_func func;
-	return (nouveau_gpio_find(dev, 0, tag, 0xff, &func)) == 0;
-}
-
-static inline int
-nouveau_gpio_func_set(struct drm_device *dev, u8 tag, int state)
-{
-	return nouveau_gpio_set(dev, 0, tag, 0xff, state);
-}
-
-static inline int
-nouveau_gpio_func_get(struct drm_device *dev, u8 tag)
-{
-	return nouveau_gpio_get(dev, 0, tag, 0xff);
-}
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_gpuobj.c b/drivers/gpu/drm/nouveau/nouveau_gpuobj.c
deleted file mode 100644
index bd79fedb705..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_gpuobj.c
+++ /dev/null
@@ -1,808 +0,0 @@
-/*
- * Copyright (C) 2006 Ben Skeggs.
- *
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-/*
- * Authors:
- *   Ben Skeggs <darktama@iinet.net.au>
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
-#include "nouveau_vm.h"
-
-struct nouveau_gpuobj_method {
-	struct list_head head;
-	u32 mthd;
-	int (*exec)(struct nouveau_channel *, u32 class, u32 mthd, u32 data);
-};
-
-struct nouveau_gpuobj_class {
-	struct list_head head;
-	struct list_head methods;
-	u32 id;
-	u32 engine;
-};
-
-int
-nouveau_gpuobj_class_new(struct drm_device *dev, u32 class, u32 engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj_class *oc;
-
-	oc = kzalloc(sizeof(*oc), GFP_KERNEL);
-	if (!oc)
-		return -ENOMEM;
-
-	INIT_LIST_HEAD(&oc->methods);
-	oc->id = class;
-	oc->engine = engine;
-	list_add(&oc->head, &dev_priv->classes);
-	return 0;
-}
-
-int
-nouveau_gpuobj_mthd_new(struct drm_device *dev, u32 class, u32 mthd,
-			int (*exec)(struct nouveau_channel *, u32, u32, u32))
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj_method *om;
-	struct nouveau_gpuobj_class *oc;
-
-	list_for_each_entry(oc, &dev_priv->classes, head) {
-		if (oc->id == class)
-			goto found;
-	}
-
-	return -EINVAL;
-
-found:
-	om = kzalloc(sizeof(*om), GFP_KERNEL);
-	if (!om)
-		return -ENOMEM;
-
-	om->mthd = mthd;
-	om->exec = exec;
-	list_add(&om->head, &oc->methods);
-	return 0;
-}
-
-int
-nouveau_gpuobj_mthd_call(struct nouveau_channel *chan,
-			 u32 class, u32 mthd, u32 data)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nouveau_gpuobj_method *om;
-	struct nouveau_gpuobj_class *oc;
-
-	list_for_each_entry(oc, &dev_priv->classes, head) {
-		if (oc->id != class)
-			continue;
-
-		list_for_each_entry(om, &oc->methods, head) {
-			if (om->mthd == mthd)
-				return om->exec(chan, class, mthd, data);
-		}
-	}
-
-	return -ENOENT;
-}
-
-int
-nouveau_gpuobj_mthd_call2(struct drm_device *dev, int chid,
-			  u32 class, u32 mthd, u32 data)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct nouveau_channel *chan = NULL;
-	unsigned long flags;
-	int ret = -EINVAL;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	if (chid >= 0 && chid < pfifo->channels)
-		chan = dev_priv->channels.ptr[chid];
-	if (chan)
-		ret = nouveau_gpuobj_mthd_call(chan, class, mthd, data);
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return ret;
-}
-
-int
-nouveau_gpuobj_new(struct drm_device *dev, struct nouveau_channel *chan,
-		   uint32_t size, int align, uint32_t flags,
-		   struct nouveau_gpuobj **gpuobj_ret)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-	struct nouveau_gpuobj *gpuobj;
-	struct drm_mm_node *ramin = NULL;
-	int ret, i;
-
-	NV_DEBUG(dev, "ch%d size=%u align=%d flags=0x%08x\n",
-		 chan ? chan->id : -1, size, align, flags);
-
-	gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
-	if (!gpuobj)
-		return -ENOMEM;
-	NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
-	gpuobj->dev = dev;
-	gpuobj->flags = flags;
-	kref_init(&gpuobj->refcount);
-	gpuobj->size = size;
-
-	spin_lock(&dev_priv->ramin_lock);
-	list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
-	spin_unlock(&dev_priv->ramin_lock);
-
-	if (!(flags & NVOBJ_FLAG_VM) && chan) {
-		ramin = drm_mm_search_free(&chan->ramin_heap, size, align, 0);
-		if (ramin)
-			ramin = drm_mm_get_block(ramin, size, align);
-		if (!ramin) {
-			nouveau_gpuobj_ref(NULL, &gpuobj);
-			return -ENOMEM;
-		}
-
-		gpuobj->pinst = chan->ramin->pinst;
-		if (gpuobj->pinst != ~0)
-			gpuobj->pinst += ramin->start;
-
-		gpuobj->cinst = ramin->start;
-		gpuobj->vinst = ramin->start + chan->ramin->vinst;
-		gpuobj->node  = ramin;
-	} else {
-		ret = instmem->get(gpuobj, chan, size, align);
-		if (ret) {
-			nouveau_gpuobj_ref(NULL, &gpuobj);
-			return ret;
-		}
-
-		ret = -ENOSYS;
-		if (!(flags & NVOBJ_FLAG_DONT_MAP))
-			ret = instmem->map(gpuobj);
-		if (ret)
-			gpuobj->pinst = ~0;
-
-		gpuobj->cinst = NVOBJ_CINST_GLOBAL;
-	}
-
-	if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
-		for (i = 0; i < gpuobj->size; i += 4)
-			nv_wo32(gpuobj, i, 0);
-		instmem->flush(dev);
-	}
-
-
-	*gpuobj_ret = gpuobj;
-	return 0;
-}
-
-int
-nouveau_gpuobj_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	NV_DEBUG(dev, "\n");
-
-	INIT_LIST_HEAD(&dev_priv->gpuobj_list);
-	INIT_LIST_HEAD(&dev_priv->classes);
-	spin_lock_init(&dev_priv->ramin_lock);
-	dev_priv->ramin_base = ~0;
-
-	return 0;
-}
-
-void
-nouveau_gpuobj_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj_method *om, *tm;
-	struct nouveau_gpuobj_class *oc, *tc;
-
-	NV_DEBUG(dev, "\n");
-
-	list_for_each_entry_safe(oc, tc, &dev_priv->classes, head) {
-		list_for_each_entry_safe(om, tm, &oc->methods, head) {
-			list_del(&om->head);
-			kfree(om);
-		}
-		list_del(&oc->head);
-		kfree(oc);
-	}
-
-	WARN_ON(!list_empty(&dev_priv->gpuobj_list));
-}
-
-
-static void
-nouveau_gpuobj_del(struct kref *ref)
-{
-	struct nouveau_gpuobj *gpuobj =
-		container_of(ref, struct nouveau_gpuobj, refcount);
-	struct drm_device *dev = gpuobj->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-	int i;
-
-	NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
-
-	if (gpuobj->node && (gpuobj->flags & NVOBJ_FLAG_ZERO_FREE)) {
-		for (i = 0; i < gpuobj->size; i += 4)
-			nv_wo32(gpuobj, i, 0);
-		instmem->flush(dev);
-	}
-
-	if (gpuobj->dtor)
-		gpuobj->dtor(dev, gpuobj);
-
-	if (gpuobj->cinst == NVOBJ_CINST_GLOBAL) {
-		if (gpuobj->node) {
-			instmem->unmap(gpuobj);
-			instmem->put(gpuobj);
-		}
-	} else {
-		if (gpuobj->node) {
-			spin_lock(&dev_priv->ramin_lock);
-			drm_mm_put_block(gpuobj->node);
-			spin_unlock(&dev_priv->ramin_lock);
-		}
-	}
-
-	spin_lock(&dev_priv->ramin_lock);
-	list_del(&gpuobj->list);
-	spin_unlock(&dev_priv->ramin_lock);
-
-	kfree(gpuobj);
-}
-
-void
-nouveau_gpuobj_ref(struct nouveau_gpuobj *ref, struct nouveau_gpuobj **ptr)
-{
-	if (ref)
-		kref_get(&ref->refcount);
-
-	if (*ptr)
-		kref_put(&(*ptr)->refcount, nouveau_gpuobj_del);
-
-	*ptr = ref;
-}
-
-int
-nouveau_gpuobj_new_fake(struct drm_device *dev, u32 pinst, u64 vinst,
-			u32 size, u32 flags, struct nouveau_gpuobj **pgpuobj)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj = NULL;
-	int i;
-
-	NV_DEBUG(dev,
-		 "pinst=0x%08x vinst=0x%010llx size=0x%08x flags=0x%08x\n",
-		 pinst, vinst, size, flags);
-
-	gpuobj = kzalloc(sizeof(*gpuobj), GFP_KERNEL);
-	if (!gpuobj)
-		return -ENOMEM;
-	NV_DEBUG(dev, "gpuobj %p\n", gpuobj);
-	gpuobj->dev = dev;
-	gpuobj->flags = flags;
-	kref_init(&gpuobj->refcount);
-	gpuobj->size  = size;
-	gpuobj->pinst = pinst;
-	gpuobj->cinst = NVOBJ_CINST_GLOBAL;
-	gpuobj->vinst = vinst;
-
-	if (gpuobj->flags & NVOBJ_FLAG_ZERO_ALLOC) {
-		for (i = 0; i < gpuobj->size; i += 4)
-			nv_wo32(gpuobj, i, 0);
-		dev_priv->engine.instmem.flush(dev);
-	}
-
-	spin_lock(&dev_priv->ramin_lock);
-	list_add_tail(&gpuobj->list, &dev_priv->gpuobj_list);
-	spin_unlock(&dev_priv->ramin_lock);
-	*pgpuobj = gpuobj;
-	return 0;
-}
-
-void
-nv50_gpuobj_dma_init(struct nouveau_gpuobj *obj, u32 offset, int class,
-		     u64 base, u64 size, int target, int access,
-		     u32 type, u32 comp)
-{
-	struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	u32 flags0;
-
-	flags0  = (comp << 29) | (type << 22) | class;
-	flags0 |= 0x00100000;
-
-	switch (access) {
-	case NV_MEM_ACCESS_RO: flags0 |= 0x00040000; break;
-	case NV_MEM_ACCESS_RW:
-	case NV_MEM_ACCESS_WO: flags0 |= 0x00080000; break;
-	default:
-		break;
-	}
-
-	switch (target) {
-	case NV_MEM_TARGET_VRAM:
-		flags0 |= 0x00010000;
-		break;
-	case NV_MEM_TARGET_PCI:
-		flags0 |= 0x00020000;
-		break;
-	case NV_MEM_TARGET_PCI_NOSNOOP:
-		flags0 |= 0x00030000;
-		break;
-	case NV_MEM_TARGET_GART:
-		base += dev_priv->gart_info.aper_base;
-	default:
-		flags0 &= ~0x00100000;
-		break;
-	}
-
-	/* convert to base + limit */
-	size = (base + size) - 1;
-
-	nv_wo32(obj, offset + 0x00, flags0);
-	nv_wo32(obj, offset + 0x04, lower_32_bits(size));
-	nv_wo32(obj, offset + 0x08, lower_32_bits(base));
-	nv_wo32(obj, offset + 0x0c, upper_32_bits(size) << 24 |
-				    upper_32_bits(base));
-	nv_wo32(obj, offset + 0x10, 0x00000000);
-	nv_wo32(obj, offset + 0x14, 0x00000000);
-
-	pinstmem->flush(obj->dev);
-}
-
-int
-nv50_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base, u64 size,
-		    int target, int access, u32 type, u32 comp,
-		    struct nouveau_gpuobj **pobj)
-{
-	struct drm_device *dev = chan->dev;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 24, 16, NVOBJ_FLAG_ZERO_FREE, pobj);
-	if (ret)
-		return ret;
-
-	nv50_gpuobj_dma_init(*pobj, 0, class, base, size, target,
-			     access, type, comp);
-	return 0;
-}
-
-int
-nouveau_gpuobj_dma_new(struct nouveau_channel *chan, int class, u64 base,
-		       u64 size, int access, int target,
-		       struct nouveau_gpuobj **pobj)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj;
-	u32 flags0, flags2;
-	int ret;
-
-	if (dev_priv->card_type >= NV_50) {
-		u32 comp = (target == NV_MEM_TARGET_VM) ? NV_MEM_COMP_VM : 0;
-		u32 type = (target == NV_MEM_TARGET_VM) ? NV_MEM_TYPE_VM : 0;
-
-		return nv50_gpuobj_dma_new(chan, class, base, size,
-					   target, access, type, comp, pobj);
-	}
-
-	if (target == NV_MEM_TARGET_GART) {
-		struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
-
-		if (dev_priv->gart_info.type == NOUVEAU_GART_PDMA) {
-			if (base == 0) {
-				nouveau_gpuobj_ref(gart, pobj);
-				return 0;
-			}
-
-			base   = nouveau_sgdma_get_physical(dev, base);
-			target = NV_MEM_TARGET_PCI;
-		} else {
-			base += dev_priv->gart_info.aper_base;
-			if (dev_priv->gart_info.type == NOUVEAU_GART_AGP)
-				target = NV_MEM_TARGET_PCI_NOSNOOP;
-			else
-				target = NV_MEM_TARGET_PCI;
-		}
-	}
-
-	flags0  = class;
-	flags0 |= 0x00003000; /* PT present, PT linear */
-	flags2  = 0;
-
-	switch (target) {
-	case NV_MEM_TARGET_PCI:
-		flags0 |= 0x00020000;
-		break;
-	case NV_MEM_TARGET_PCI_NOSNOOP:
-		flags0 |= 0x00030000;
-		break;
-	default:
-		break;
-	}
-
-	switch (access) {
-	case NV_MEM_ACCESS_RO:
-		flags0 |= 0x00004000;
-		break;
-	case NV_MEM_ACCESS_WO:
-		flags0 |= 0x00008000;
-	default:
-		flags2 |= 0x00000002;
-		break;
-	}
-
-	flags0 |= (base & 0x00000fff) << 20;
-	flags2 |= (base & 0xfffff000);
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-
-	nv_wo32(obj, 0x00, flags0);
-	nv_wo32(obj, 0x04, size - 1);
-	nv_wo32(obj, 0x08, flags2);
-	nv_wo32(obj, 0x0c, flags2);
-
-	obj->engine = NVOBJ_ENGINE_SW;
-	obj->class  = class;
-	*pobj = obj;
-	return 0;
-}
-
-int
-nouveau_gpuobj_gr_new(struct nouveau_channel *chan, u32 handle, int class)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj_class *oc;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d class=0x%04x\n", chan->id, class);
-
-	list_for_each_entry(oc, &dev_priv->classes, head) {
-		struct nouveau_exec_engine *eng = dev_priv->eng[oc->engine];
-
-		if (oc->id != class)
-			continue;
-
-		if (!chan->engctx[oc->engine]) {
-			ret = eng->context_new(chan, oc->engine);
-			if (ret)
-				return ret;
-		}
-
-		return eng->object_new(chan, oc->engine, handle, class);
-	}
-
-	return -EINVAL;
-}
-
-static int
-nouveau_gpuobj_channel_init_pramin(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t size;
-	uint32_t base;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	/* Base amount for object storage (4KiB enough?) */
-	size = 0x2000;
-	base = 0;
-
-	if (dev_priv->card_type == NV_50) {
-		/* Various fixed table thingos */
-		size += 0x1400; /* mostly unknown stuff */
-		size += 0x4000; /* vm pd */
-		base  = 0x6000;
-		/* RAMHT, not sure about setting size yet, 32KiB to be safe */
-		size += 0x8000;
-		/* RAMFC */
-		size += 0x1000;
-	}
-
-	ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
-	if (ret) {
-		NV_ERROR(dev, "Error allocating channel PRAMIN: %d\n", ret);
-		return ret;
-	}
-
-	ret = drm_mm_init(&chan->ramin_heap, base, size - base);
-	if (ret) {
-		NV_ERROR(dev, "Error creating PRAMIN heap: %d\n", ret);
-		nouveau_gpuobj_ref(NULL, &chan->ramin);
-		return ret;
-	}
-
-	return 0;
-}
-
-static int
-nvc0_gpuobj_channel_init(struct nouveau_channel *chan, struct nouveau_vm *vm)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *pgd = NULL;
-	struct nouveau_vm_pgd *vpgd;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 4096, 0x1000, 0, &chan->ramin);
-	if (ret)
-		return ret;
-
-	/* create page directory for this vm if none currently exists,
-	 * will be destroyed automagically when last reference to the
-	 * vm is removed
-	 */
-	if (list_empty(&vm->pgd_list)) {
-		ret = nouveau_gpuobj_new(dev, NULL, 65536, 0x1000, 0, &pgd);
-		if (ret)
-			return ret;
-	}
-	nouveau_vm_ref(vm, &chan->vm, pgd);
-	nouveau_gpuobj_ref(NULL, &pgd);
-
-	/* point channel at vm's page directory */
-	vpgd = list_first_entry(&vm->pgd_list, struct nouveau_vm_pgd, head);
-	nv_wo32(chan->ramin, 0x0200, lower_32_bits(vpgd->obj->vinst));
-	nv_wo32(chan->ramin, 0x0204, upper_32_bits(vpgd->obj->vinst));
-	nv_wo32(chan->ramin, 0x0208, 0xffffffff);
-	nv_wo32(chan->ramin, 0x020c, 0x000000ff);
-
-	return 0;
-}
-
-int
-nouveau_gpuobj_channel_init(struct nouveau_channel *chan,
-			    uint32_t vram_h, uint32_t tt_h)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(chan->file_priv);
-	struct nouveau_vm *vm = fpriv ? fpriv->vm : dev_priv->chan_vm;
-	struct nouveau_gpuobj *vram = NULL, *tt = NULL;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d vram=0x%08x tt=0x%08x\n", chan->id, vram_h, tt_h);
-	if (dev_priv->card_type >= NV_C0)
-		return nvc0_gpuobj_channel_init(chan, vm);
-
-	/* Allocate a chunk of memory for per-channel object storage */
-	ret = nouveau_gpuobj_channel_init_pramin(chan);
-	if (ret) {
-		NV_ERROR(dev, "init pramin\n");
-		return ret;
-	}
-
-	/* NV50 VM
-	 *  - Allocate per-channel page-directory
-	 *  - Link with shared channel VM
-	 */
-	if (vm) {
-		u32 pgd_offs = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
-		u64 vm_vinst = chan->ramin->vinst + pgd_offs;
-		u32 vm_pinst = chan->ramin->pinst;
-
-		if (vm_pinst != ~0)
-			vm_pinst += pgd_offs;
-
-		ret = nouveau_gpuobj_new_fake(dev, vm_pinst, vm_vinst, 0x4000,
-					      0, &chan->vm_pd);
-		if (ret)
-			return ret;
-
-		nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
-	}
-
-	/* RAMHT */
-	if (dev_priv->card_type < NV_50) {
-		nouveau_ramht_ref(dev_priv->ramht, &chan->ramht, NULL);
-	} else {
-		struct nouveau_gpuobj *ramht = NULL;
-
-		ret = nouveau_gpuobj_new(dev, chan, 0x8000, 16,
-					 NVOBJ_FLAG_ZERO_ALLOC, &ramht);
-		if (ret)
-			return ret;
-
-		ret = nouveau_ramht_new(dev, ramht, &chan->ramht);
-		nouveau_gpuobj_ref(NULL, &ramht);
-		if (ret)
-			return ret;
-	}
-
-	/* VRAM ctxdma */
-	if (dev_priv->card_type >= NV_50) {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     0, (1ULL << 40), NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_VM, &vram);
-		if (ret) {
-			NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
-			return ret;
-		}
-	} else {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     0, dev_priv->fb_available_size,
-					     NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_VRAM, &vram);
-		if (ret) {
-			NV_ERROR(dev, "Error creating VRAM ctxdma: %d\n", ret);
-			return ret;
-		}
-	}
-
-	ret = nouveau_ramht_insert(chan, vram_h, vram);
-	nouveau_gpuobj_ref(NULL, &vram);
-	if (ret) {
-		NV_ERROR(dev, "Error adding VRAM ctxdma to RAMHT: %d\n", ret);
-		return ret;
-	}
-
-	/* TT memory ctxdma */
-	if (dev_priv->card_type >= NV_50) {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     0, (1ULL << 40), NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_VM, &tt);
-	} else {
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     0, dev_priv->gart_info.aper_size,
-					     NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_GART, &tt);
-	}
-
-	if (ret) {
-		NV_ERROR(dev, "Error creating TT ctxdma: %d\n", ret);
-		return ret;
-	}
-
-	ret = nouveau_ramht_insert(chan, tt_h, tt);
-	nouveau_gpuobj_ref(NULL, &tt);
-	if (ret) {
-		NV_ERROR(dev, "Error adding TT ctxdma to RAMHT: %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-void
-nouveau_gpuobj_channel_takedown(struct nouveau_channel *chan)
-{
-	NV_DEBUG(chan->dev, "ch%d\n", chan->id);
-
-	nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
-	nouveau_gpuobj_ref(NULL, &chan->vm_pd);
-
-	if (drm_mm_initialized(&chan->ramin_heap))
-		drm_mm_takedown(&chan->ramin_heap);
-	nouveau_gpuobj_ref(NULL, &chan->ramin);
-}
-
-int
-nouveau_gpuobj_suspend(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj;
-	int i;
-
-	list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
-		if (gpuobj->cinst != NVOBJ_CINST_GLOBAL)
-			continue;
-
-		gpuobj->suspend = vmalloc(gpuobj->size);
-		if (!gpuobj->suspend) {
-			nouveau_gpuobj_resume(dev);
-			return -ENOMEM;
-		}
-
-		for (i = 0; i < gpuobj->size; i += 4)
-			gpuobj->suspend[i/4] = nv_ro32(gpuobj, i);
-	}
-
-	return 0;
-}
-
-void
-nouveau_gpuobj_resume(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj;
-	int i;
-
-	list_for_each_entry(gpuobj, &dev_priv->gpuobj_list, list) {
-		if (!gpuobj->suspend)
-			continue;
-
-		for (i = 0; i < gpuobj->size; i += 4)
-			nv_wo32(gpuobj, i, gpuobj->suspend[i/4]);
-
-		vfree(gpuobj->suspend);
-		gpuobj->suspend = NULL;
-	}
-
-	dev_priv->engine.instmem.flush(dev);
-}
-
-u32
-nv_ro32(struct nouveau_gpuobj *gpuobj, u32 offset)
-{
-	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
-	struct drm_device *dev = gpuobj->dev;
-	unsigned long flags;
-
-	if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
-		u64  ptr = gpuobj->vinst + offset;
-		u32 base = ptr >> 16;
-		u32  val;
-
-		spin_lock_irqsave(&dev_priv->vm_lock, flags);
-		if (dev_priv->ramin_base != base) {
-			dev_priv->ramin_base = base;
-			nv_wr32(dev, 0x001700, dev_priv->ramin_base);
-		}
-		val = nv_rd32(dev, 0x700000 + (ptr & 0xffff));
-		spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
-		return val;
-	}
-
-	return nv_ri32(dev, gpuobj->pinst + offset);
-}
-
-void
-nv_wo32(struct nouveau_gpuobj *gpuobj, u32 offset, u32 val)
-{
-	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
-	struct drm_device *dev = gpuobj->dev;
-	unsigned long flags;
-
-	if (gpuobj->pinst == ~0 || !dev_priv->ramin_available) {
-		u64  ptr = gpuobj->vinst + offset;
-		u32 base = ptr >> 16;
-
-		spin_lock_irqsave(&dev_priv->vm_lock, flags);
-		if (dev_priv->ramin_base != base) {
-			dev_priv->ramin_base = base;
-			nv_wr32(dev, 0x001700, dev_priv->ramin_base);
-		}
-		nv_wr32(dev, 0x700000 + (ptr & 0xffff), val);
-		spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
-		return;
-	}
-
-	nv_wi32(dev, gpuobj->pinst + offset, val);
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_hdmi.c b/drivers/gpu/drm/nouveau/nouveau_hdmi.c
index c3de3638452..2c672cebc88 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hdmi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hdmi.c
@@ -22,8 +22,8 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
 #include "nouveau_connector.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
@@ -31,10 +31,10 @@
 static bool
 hdmi_sor(struct drm_encoder *encoder)
 {
-	struct drm_nouveau_private *dev_priv = encoder->dev->dev_private;
-	if (dev_priv->chipset <  0xa3 ||
-	    dev_priv->chipset == 0xaa ||
-	    dev_priv->chipset == 0xac)
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	if (nv_device(drm->device)->chipset <  0xa3 ||
+	    nv_device(drm->device)->chipset == 0xaa ||
+	    nv_device(drm->device)->chipset == 0xac)
 		return false;
 	return true;
 }
@@ -52,13 +52,15 @@ hdmi_base(struct drm_encoder *encoder)
 static void
 hdmi_wr32(struct drm_encoder *encoder, u32 reg, u32 val)
 {
-	nv_wr32(encoder->dev, hdmi_base(encoder) + reg, val);
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
+	nv_wr32(device, hdmi_base(encoder) + reg, val);
 }
 
 static u32
 hdmi_rd32(struct drm_encoder *encoder, u32 reg)
 {
-	return nv_rd32(encoder->dev, hdmi_base(encoder) + reg);
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
+	return nv_rd32(device, hdmi_base(encoder) + reg);
 }
 
 static u32
@@ -73,12 +75,11 @@ static void
 nouveau_audio_disconnect(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
 	u32 or = nv_encoder->or * 0x800;
 
-	if (hdmi_sor(encoder)) {
-		nv_mask(dev, 0x61c448 + or, 0x00000003, 0x00000000);
-	}
+	if (hdmi_sor(encoder))
+		nv_mask(device, 0x61c448 + or, 0x00000003, 0x00000000);
 }
 
 static void
@@ -86,8 +87,8 @@ nouveau_audio_mode_set(struct drm_encoder *encoder,
 		       struct drm_display_mode *mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
 	struct nouveau_connector *nv_connector;
-	struct drm_device *dev = encoder->dev;
 	u32 or = nv_encoder->or * 0x800;
 	int i;
 
@@ -98,16 +99,16 @@ nouveau_audio_mode_set(struct drm_encoder *encoder,
 	}
 
 	if (hdmi_sor(encoder)) {
-		nv_mask(dev, 0x61c448 + or, 0x00000001, 0x00000001);
+		nv_mask(device, 0x61c448 + or, 0x00000001, 0x00000001);
 
 		drm_edid_to_eld(&nv_connector->base, nv_connector->edid);
 		if (nv_connector->base.eld[0]) {
 			u8 *eld = nv_connector->base.eld;
 			for (i = 0; i < eld[2] * 4; i++)
-				nv_wr32(dev, 0x61c440 + or, (i << 8) | eld[i]);
+				nv_wr32(device, 0x61c440 + or, (i << 8) | eld[i]);
 			for (i = eld[2] * 4; i < 0x60; i++)
-				nv_wr32(dev, 0x61c440 + or, (i << 8) | 0x00);
-			nv_mask(dev, 0x61c448 + or, 0x00000002, 0x00000002);
+				nv_wr32(device, 0x61c440 + or, (i << 8) | 0x00);
+			nv_mask(device, 0x61c448 + or, 0x00000002, 0x00000002);
 		}
 	}
 }
@@ -219,9 +220,9 @@ void
 nouveau_hdmi_mode_set(struct drm_encoder *encoder,
 		      struct drm_display_mode *mode)
 {
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_connector *nv_connector;
-	struct drm_device *dev = encoder->dev;
 	u32 max_ac_packet, rekey;
 
 	nv_connector = nouveau_encoder_connector_get(nv_encoder);
@@ -238,9 +239,9 @@ nouveau_hdmi_mode_set(struct drm_encoder *encoder,
 	hdmi_mask(encoder, 0x068, 0x00010101, 0x00000000); /* ACR_CTRL, ?? */
 	hdmi_mask(encoder, 0x078, 0x80000000, 0x80000000); /* ACR_0441_ENABLE */
 
-	nv_mask(dev, 0x61733c, 0x00100000, 0x00100000); /* RESETF */
-	nv_mask(dev, 0x61733c, 0x10000000, 0x10000000); /* LOOKUP_EN */
-	nv_mask(dev, 0x61733c, 0x00100000, 0x00000000); /* !RESETF */
+	nv_mask(device, 0x61733c, 0x00100000, 0x00100000); /* RESETF */
+	nv_mask(device, 0x61733c, 0x10000000, 0x10000000); /* LOOKUP_EN */
+	nv_mask(device, 0x61733c, 0x00100000, 0x00000000); /* !RESETF */
 
 	/* value matches nvidia binary driver, and tegra constant */
 	rekey = 56;
diff --git a/drivers/gpu/drm/nouveau/nouveau_hw.c b/drivers/gpu/drm/nouveau/nouveau_hw.c
index b87ad3bd773..617a06ffdb4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hw.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hw.c
@@ -22,10 +22,14 @@
  * SOFTWARE.
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
 #include "nouveau_hw.h"
 
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+
 #define CHIPSET_NFORCE 0x01a0
 #define CHIPSET_NFORCE2 0x01f0
 
@@ -82,12 +86,12 @@ NVReadVgaGr(struct drm_device *dev, int head, uint8_t index)
 void
 NVSetOwner(struct drm_device *dev, int owner)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	if (owner == 1)
 		owner *= 3;
 
-	if (dev_priv->chipset == 0x11) {
+	if (nv_device(drm->device)->chipset == 0x11) {
 		/* This might seem stupid, but the blob does it and
 		 * omitting it often locks the system up.
 		 */
@@ -98,7 +102,7 @@ NVSetOwner(struct drm_device *dev, int owner)
 	/* CR44 is always changed on CRTC0 */
 	NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_44, owner);
 
-	if (dev_priv->chipset == 0x11) {	/* set me harder */
+	if (nv_device(drm->device)->chipset == 0x11) {	/* set me harder */
 		NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_2E, owner);
 		NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_2E, owner);
 	}
@@ -123,270 +127,6 @@ NVBlankScreen(struct drm_device *dev, int head, bool blank)
 }
 
 /*
- * PLL setting
- */
-
-static int
-powerctrl_1_shift(int chip_version, int reg)
-{
-	int shift = -4;
-
-	if (chip_version < 0x17 || chip_version == 0x1a || chip_version == 0x20)
-		return shift;
-
-	switch (reg) {
-	case NV_RAMDAC_VPLL2:
-		shift += 4;
-	case NV_PRAMDAC_VPLL_COEFF:
-		shift += 4;
-	case NV_PRAMDAC_MPLL_COEFF:
-		shift += 4;
-	case NV_PRAMDAC_NVPLL_COEFF:
-		shift += 4;
-	}
-
-	/*
-	 * the shift for vpll regs is only used for nv3x chips with a single
-	 * stage pll
-	 */
-	if (shift > 4 && (chip_version < 0x32 || chip_version == 0x35 ||
-			  chip_version == 0x36 || chip_version >= 0x40))
-		shift = -4;
-
-	return shift;
-}
-
-static void
-setPLL_single(struct drm_device *dev, uint32_t reg, struct nouveau_pll_vals *pv)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chip_version = dev_priv->vbios.chip_version;
-	uint32_t oldpll = NVReadRAMDAC(dev, 0, reg);
-	int oldN = (oldpll >> 8) & 0xff, oldM = oldpll & 0xff;
-	uint32_t pll = (oldpll & 0xfff80000) | pv->log2P << 16 | pv->NM1;
-	uint32_t saved_powerctrl_1 = 0;
-	int shift_powerctrl_1 = powerctrl_1_shift(chip_version, reg);
-
-	if (oldpll == pll)
-		return;	/* already set */
-
-	if (shift_powerctrl_1 >= 0) {
-		saved_powerctrl_1 = nvReadMC(dev, NV_PBUS_POWERCTRL_1);
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_1,
-			(saved_powerctrl_1 & ~(0xf << shift_powerctrl_1)) |
-			1 << shift_powerctrl_1);
-	}
-
-	if (oldM && pv->M1 && (oldN / oldM < pv->N1 / pv->M1))
-		/* upclock -- write new post divider first */
-		NVWriteRAMDAC(dev, 0, reg, pv->log2P << 16 | (oldpll & 0xffff));
-	else
-		/* downclock -- write new NM first */
-		NVWriteRAMDAC(dev, 0, reg, (oldpll & 0xffff0000) | pv->NM1);
-
-	if (chip_version < 0x17 && chip_version != 0x11)
-		/* wait a bit on older chips */
-		msleep(64);
-	NVReadRAMDAC(dev, 0, reg);
-
-	/* then write the other half as well */
-	NVWriteRAMDAC(dev, 0, reg, pll);
-
-	if (shift_powerctrl_1 >= 0)
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_1, saved_powerctrl_1);
-}
-
-static uint32_t
-new_ramdac580(uint32_t reg1, bool ss, uint32_t ramdac580)
-{
-	bool head_a = (reg1 == NV_PRAMDAC_VPLL_COEFF);
-
-	if (ss)	/* single stage pll mode */
-		ramdac580 |= head_a ? NV_RAMDAC_580_VPLL1_ACTIVE :
-				      NV_RAMDAC_580_VPLL2_ACTIVE;
-	else
-		ramdac580 &= head_a ? ~NV_RAMDAC_580_VPLL1_ACTIVE :
-				      ~NV_RAMDAC_580_VPLL2_ACTIVE;
-
-	return ramdac580;
-}
-
-static void
-setPLL_double_highregs(struct drm_device *dev, uint32_t reg1,
-		       struct nouveau_pll_vals *pv)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chip_version = dev_priv->vbios.chip_version;
-	bool nv3035 = chip_version == 0x30 || chip_version == 0x35;
-	uint32_t reg2 = reg1 + ((reg1 == NV_RAMDAC_VPLL2) ? 0x5c : 0x70);
-	uint32_t oldpll1 = NVReadRAMDAC(dev, 0, reg1);
-	uint32_t oldpll2 = !nv3035 ? NVReadRAMDAC(dev, 0, reg2) : 0;
-	uint32_t pll1 = (oldpll1 & 0xfff80000) | pv->log2P << 16 | pv->NM1;
-	uint32_t pll2 = (oldpll2 & 0x7fff0000) | 1 << 31 | pv->NM2;
-	uint32_t oldramdac580 = 0, ramdac580 = 0;
-	bool single_stage = !pv->NM2 || pv->N2 == pv->M2;	/* nv41+ only */
-	uint32_t saved_powerctrl_1 = 0, savedc040 = 0;
-	int shift_powerctrl_1 = powerctrl_1_shift(chip_version, reg1);
-
-	/* model specific additions to generic pll1 and pll2 set up above */
-	if (nv3035) {
-		pll1 = (pll1 & 0xfcc7ffff) | (pv->N2 & 0x18) << 21 |
-		       (pv->N2 & 0x7) << 19 | 8 << 4 | (pv->M2 & 7) << 4;
-		pll2 = 0;
-	}
-	if (chip_version > 0x40 && reg1 >= NV_PRAMDAC_VPLL_COEFF) { /* !nv40 */
-		oldramdac580 = NVReadRAMDAC(dev, 0, NV_PRAMDAC_580);
-		ramdac580 = new_ramdac580(reg1, single_stage, oldramdac580);
-		if (oldramdac580 != ramdac580)
-			oldpll1 = ~0;	/* force mismatch */
-		if (single_stage)
-			/* magic value used by nvidia in single stage mode */
-			pll2 |= 0x011f;
-	}
-	if (chip_version > 0x70)
-		/* magic bits set by the blob (but not the bios) on g71-73 */
-		pll1 = (pll1 & 0x7fffffff) | (single_stage ? 0x4 : 0xc) << 28;
-
-	if (oldpll1 == pll1 && oldpll2 == pll2)
-		return;	/* already set */
-
-	if (shift_powerctrl_1 >= 0) {
-		saved_powerctrl_1 = nvReadMC(dev, NV_PBUS_POWERCTRL_1);
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_1,
-			(saved_powerctrl_1 & ~(0xf << shift_powerctrl_1)) |
-			1 << shift_powerctrl_1);
-	}
-
-	if (chip_version >= 0x40) {
-		int shift_c040 = 14;
-
-		switch (reg1) {
-		case NV_PRAMDAC_MPLL_COEFF:
-			shift_c040 += 2;
-		case NV_PRAMDAC_NVPLL_COEFF:
-			shift_c040 += 2;
-		case NV_RAMDAC_VPLL2:
-			shift_c040 += 2;
-		case NV_PRAMDAC_VPLL_COEFF:
-			shift_c040 += 2;
-		}
-
-		savedc040 = nvReadMC(dev, 0xc040);
-		if (shift_c040 != 14)
-			nvWriteMC(dev, 0xc040, savedc040 & ~(3 << shift_c040));
-	}
-
-	if (oldramdac580 != ramdac580)
-		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_580, ramdac580);
-
-	if (!nv3035)
-		NVWriteRAMDAC(dev, 0, reg2, pll2);
-	NVWriteRAMDAC(dev, 0, reg1, pll1);
-
-	if (shift_powerctrl_1 >= 0)
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_1, saved_powerctrl_1);
-	if (chip_version >= 0x40)
-		nvWriteMC(dev, 0xc040, savedc040);
-}
-
-static void
-setPLL_double_lowregs(struct drm_device *dev, uint32_t NMNMreg,
-		      struct nouveau_pll_vals *pv)
-{
-	/* When setting PLLs, there is a merry game of disabling and enabling
-	 * various bits of hardware during the process. This function is a
-	 * synthesis of six nv4x traces, nearly each card doing a subtly
-	 * different thing. With luck all the necessary bits for each card are
-	 * combined herein. Without luck it deviates from each card's formula
-	 * so as to not work on any :)
-	 */
-
-	uint32_t Preg = NMNMreg - 4;
-	bool mpll = Preg == 0x4020;
-	uint32_t oldPval = nvReadMC(dev, Preg);
-	uint32_t NMNM = pv->NM2 << 16 | pv->NM1;
-	uint32_t Pval = (oldPval & (mpll ? ~(0x77 << 16) : ~(7 << 16))) |
-			0xc << 28 | pv->log2P << 16;
-	uint32_t saved4600 = 0;
-	/* some cards have different maskc040s */
-	uint32_t maskc040 = ~(3 << 14), savedc040;
-	bool single_stage = !pv->NM2 || pv->N2 == pv->M2;
-
-	if (nvReadMC(dev, NMNMreg) == NMNM && (oldPval & 0xc0070000) == Pval)
-		return;
-
-	if (Preg == 0x4000)
-		maskc040 = ~0x333;
-	if (Preg == 0x4058)
-		maskc040 = ~(0xc << 24);
-
-	if (mpll) {
-		struct pll_lims pll_lim;
-		uint8_t Pval2;
-
-		if (get_pll_limits(dev, Preg, &pll_lim))
-			return;
-
-		Pval2 = pv->log2P + pll_lim.log2p_bias;
-		if (Pval2 > pll_lim.max_log2p)
-			Pval2 = pll_lim.max_log2p;
-		Pval |= 1 << 28 | Pval2 << 20;
-
-		saved4600 = nvReadMC(dev, 0x4600);
-		nvWriteMC(dev, 0x4600, saved4600 | 8 << 28);
-	}
-	if (single_stage)
-		Pval |= mpll ? 1 << 12 : 1 << 8;
-
-	nvWriteMC(dev, Preg, oldPval | 1 << 28);
-	nvWriteMC(dev, Preg, Pval & ~(4 << 28));
-	if (mpll) {
-		Pval |= 8 << 20;
-		nvWriteMC(dev, 0x4020, Pval & ~(0xc << 28));
-		nvWriteMC(dev, 0x4038, Pval & ~(0xc << 28));
-	}
-
-	savedc040 = nvReadMC(dev, 0xc040);
-	nvWriteMC(dev, 0xc040, savedc040 & maskc040);
-
-	nvWriteMC(dev, NMNMreg, NMNM);
-	if (NMNMreg == 0x4024)
-		nvWriteMC(dev, 0x403c, NMNM);
-
-	nvWriteMC(dev, Preg, Pval);
-	if (mpll) {
-		Pval &= ~(8 << 20);
-		nvWriteMC(dev, 0x4020, Pval);
-		nvWriteMC(dev, 0x4038, Pval);
-		nvWriteMC(dev, 0x4600, saved4600);
-	}
-
-	nvWriteMC(dev, 0xc040, savedc040);
-
-	if (mpll) {
-		nvWriteMC(dev, 0x4020, Pval & ~(1 << 28));
-		nvWriteMC(dev, 0x4038, Pval & ~(1 << 28));
-	}
-}
-
-void
-nouveau_hw_setpll(struct drm_device *dev, uint32_t reg1,
-		  struct nouveau_pll_vals *pv)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int cv = dev_priv->vbios.chip_version;
-
-	if (cv == 0x30 || cv == 0x31 || cv == 0x35 || cv == 0x36 ||
-	    cv >= 0x40) {
-		if (reg1 > 0x405c)
-			setPLL_double_highregs(dev, reg1, pv);
-		else
-			setPLL_double_lowregs(dev, reg1, pv);
-	} else
-		setPLL_single(dev, reg1, pv);
-}
-
-/*
  * PLL getting
  */
 
@@ -394,7 +134,7 @@ static void
 nouveau_hw_decode_pll(struct drm_device *dev, uint32_t reg1, uint32_t pll1,
 		      uint32_t pll2, struct nouveau_pll_vals *pllvals)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	/* to force parsing as single stage (i.e. nv40 vplls) pass pll2 as 0 */
 
@@ -411,7 +151,7 @@ nouveau_hw_decode_pll(struct drm_device *dev, uint32_t reg1, uint32_t pll1,
 		pllvals->NM1 = pll1 & 0xffff;
 		if (nv_two_reg_pll(dev) && pll2 & NV31_RAMDAC_ENABLE_VCO2)
 			pllvals->NM2 = pll2 & 0xffff;
-		else if (dev_priv->chipset == 0x30 || dev_priv->chipset == 0x35) {
+		else if (nv_device(drm->device)->chipset == 0x30 || nv_device(drm->device)->chipset == 0x35) {
 			pllvals->M1 &= 0xf; /* only 4 bits */
 			if (pll1 & NV30_RAMDAC_ENABLE_VCO2) {
 				pllvals->M2 = (pll1 >> 4) & 0x7;
@@ -423,28 +163,30 @@ nouveau_hw_decode_pll(struct drm_device *dev, uint32_t reg1, uint32_t pll1,
 }
 
 int
-nouveau_hw_get_pllvals(struct drm_device *dev, enum pll_types plltype,
+nouveau_hw_get_pllvals(struct drm_device *dev, enum nvbios_pll_type plltype,
 		       struct nouveau_pll_vals *pllvals)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t reg1 = get_pll_register(dev, plltype), pll1, pll2 = 0;
-	struct pll_lims pll_lim;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	uint32_t reg1, pll1, pll2 = 0;
+	struct nvbios_pll pll_lim;
 	int ret;
 
-	if (reg1 == 0)
+	ret = nvbios_pll_parse(bios, plltype, &pll_lim);
+	if (ret || !(reg1 = pll_lim.reg))
 		return -ENOENT;
 
-	pll1 = nvReadMC(dev, reg1);
-
+	pll1 = nv_rd32(device, reg1);
 	if (reg1 <= 0x405c)
-		pll2 = nvReadMC(dev, reg1 + 4);
+		pll2 = nv_rd32(device, reg1 + 4);
 	else if (nv_two_reg_pll(dev)) {
 		uint32_t reg2 = reg1 + (reg1 == NV_RAMDAC_VPLL2 ? 0x5c : 0x70);
 
-		pll2 = nvReadMC(dev, reg2);
+		pll2 = nv_rd32(device, reg2);
 	}
 
-	if (dev_priv->card_type == 0x40 && reg1 >= NV_PRAMDAC_VPLL_COEFF) {
+	if (nv_device(drm->device)->card_type == 0x40 && reg1 >= NV_PRAMDAC_VPLL_COEFF) {
 		uint32_t ramdac580 = NVReadRAMDAC(dev, 0, NV_PRAMDAC_580);
 
 		/* check whether vpll has been forced into single stage mode */
@@ -457,13 +199,7 @@ nouveau_hw_get_pllvals(struct drm_device *dev, enum pll_types plltype,
 	}
 
 	nouveau_hw_decode_pll(dev, reg1, pll1, pll2, pllvals);
-
-	ret = get_pll_limits(dev, plltype, &pll_lim);
-	if (ret)
-		return ret;
-
 	pllvals->refclk = pll_lim.refclk;
-
 	return 0;
 }
 
@@ -478,7 +214,7 @@ nouveau_hw_pllvals_to_clk(struct nouveau_pll_vals *pv)
 }
 
 int
-nouveau_hw_get_clock(struct drm_device *dev, enum pll_types plltype)
+nouveau_hw_get_clock(struct drm_device *dev, enum nvbios_pll_type plltype)
 {
 	struct nouveau_pll_vals pllvals;
 	int ret;
@@ -517,26 +253,30 @@ nouveau_hw_fix_bad_vpll(struct drm_device *dev, int head)
 	 * when such a condition detected.  only seen on nv11 to date
 	 */
 
-	struct pll_lims pll_lim;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_clock *clk = nouveau_clock(device);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nvbios_pll pll_lim;
 	struct nouveau_pll_vals pv;
-	enum pll_types pll = head ? PLL_VPLL1 : PLL_VPLL0;
+	enum nvbios_pll_type pll = head ? PLL_VPLL1 : PLL_VPLL0;
 
-	if (get_pll_limits(dev, pll, &pll_lim))
+	if (nvbios_pll_parse(bios, pll, &pll_lim))
 		return;
 	nouveau_hw_get_pllvals(dev, pll, &pv);
 
 	if (pv.M1 >= pll_lim.vco1.min_m && pv.M1 <= pll_lim.vco1.max_m &&
 	    pv.N1 >= pll_lim.vco1.min_n && pv.N1 <= pll_lim.vco1.max_n &&
-	    pv.log2P <= pll_lim.max_log2p)
+	    pv.log2P <= pll_lim.max_p)
 		return;
 
-	NV_WARN(dev, "VPLL %d outwith limits, attempting to fix\n", head + 1);
+	NV_WARN(drm, "VPLL %d outwith limits, attempting to fix\n", head + 1);
 
 	/* set lowest clock within static limits */
 	pv.M1 = pll_lim.vco1.max_m;
 	pv.N1 = pll_lim.vco1.min_n;
-	pv.log2P = pll_lim.max_usable_log2p;
-	nouveau_hw_setpll(dev, pll_lim.reg, &pv);
+	pv.log2P = pll_lim.max_p_usable;
+	clk->pll_prog(clk, pll_lim.reg, &pv);
 }
 
 /*
@@ -547,17 +287,16 @@ static void nouveau_vga_font_io(struct drm_device *dev,
 				void __iomem *iovram,
 				bool save, unsigned plane)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	unsigned i;
 
 	NVWriteVgaSeq(dev, 0, NV_VIO_SR_PLANE_MASK_INDEX, 1 << plane);
 	NVWriteVgaGr(dev, 0, NV_VIO_GX_READ_MAP_INDEX, plane);
 	for (i = 0; i < 16384; i++) {
 		if (save) {
-			dev_priv->saved_vga_font[plane][i] =
+			nv04_display(dev)->saved_vga_font[plane][i] =
 					ioread32_native(iovram + i * 4);
 		} else {
-			iowrite32_native(dev_priv->saved_vga_font[plane][i],
+			iowrite32_native(nv04_display(dev)->saved_vga_font[plane][i],
 							iovram + i * 4);
 		}
 	}
@@ -566,6 +305,7 @@ static void nouveau_vga_font_io(struct drm_device *dev,
 void
 nouveau_hw_save_vga_fonts(struct drm_device *dev, bool save)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t misc, gr4, gr5, gr6, seq2, seq4;
 	bool graphicsmode;
 	unsigned plane;
@@ -581,12 +321,12 @@ nouveau_hw_save_vga_fonts(struct drm_device *dev, bool save)
 	if (graphicsmode) /* graphics mode => framebuffer => no need to save */
 		return;
 
-	NV_INFO(dev, "%sing VGA fonts\n", save ? "Sav" : "Restor");
+	NV_INFO(drm, "%sing VGA fonts\n", save ? "Sav" : "Restor");
 
 	/* map first 64KiB of VRAM, holds VGA fonts etc */
 	iovram = ioremap(pci_resource_start(dev->pdev, 1), 65536);
 	if (!iovram) {
-		NV_ERROR(dev, "Failed to map VRAM, "
+		NV_ERROR(drm, "Failed to map VRAM, "
 					"cannot save/restore VGA fonts.\n");
 		return;
 	}
@@ -649,25 +389,25 @@ static void
 nv_save_state_ramdac(struct drm_device *dev, int head,
 		     struct nv04_mode_state *state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv04_crtc_reg *regp = &state->crtc_reg[head];
 	int i;
 
-	if (dev_priv->card_type >= NV_10)
+	if (nv_device(drm->device)->card_type >= NV_10)
 		regp->nv10_cursync = NVReadRAMDAC(dev, head, NV_RAMDAC_NV10_CURSYNC);
 
 	nouveau_hw_get_pllvals(dev, head ? PLL_VPLL1 : PLL_VPLL0, &regp->pllvals);
 	state->pllsel = NVReadRAMDAC(dev, 0, NV_PRAMDAC_PLL_COEFF_SELECT);
 	if (nv_two_heads(dev))
 		state->sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK);
-	if (dev_priv->chipset == 0x11)
+	if (nv_device(drm->device)->chipset == 0x11)
 		regp->dither = NVReadRAMDAC(dev, head, NV_RAMDAC_DITHER_NV11);
 
 	regp->ramdac_gen_ctrl = NVReadRAMDAC(dev, head, NV_PRAMDAC_GENERAL_CONTROL);
 
 	if (nv_gf4_disp_arch(dev))
 		regp->ramdac_630 = NVReadRAMDAC(dev, head, NV_PRAMDAC_630);
-	if (dev_priv->chipset >= 0x30)
+	if (nv_device(drm->device)->chipset >= 0x30)
 		regp->ramdac_634 = NVReadRAMDAC(dev, head, NV_PRAMDAC_634);
 
 	regp->tv_setup = NVReadRAMDAC(dev, head, NV_PRAMDAC_TV_SETUP);
@@ -709,7 +449,7 @@ nv_save_state_ramdac(struct drm_device *dev, int head,
 	if (nv_gf4_disp_arch(dev))
 		regp->ramdac_8c0 = NVReadRAMDAC(dev, head, NV_PRAMDAC_8C0);
 
-	if (dev_priv->card_type == NV_40) {
+	if (nv_device(drm->device)->card_type == NV_40) {
 		regp->ramdac_a20 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A20);
 		regp->ramdac_a24 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A24);
 		regp->ramdac_a34 = NVReadRAMDAC(dev, head, NV_PRAMDAC_A34);
@@ -724,26 +464,27 @@ static void
 nv_load_state_ramdac(struct drm_device *dev, int head,
 		     struct nv04_mode_state *state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_clock *clk = nouveau_clock(drm->device);
 	struct nv04_crtc_reg *regp = &state->crtc_reg[head];
 	uint32_t pllreg = head ? NV_RAMDAC_VPLL2 : NV_PRAMDAC_VPLL_COEFF;
 	int i;
 
-	if (dev_priv->card_type >= NV_10)
+	if (nv_device(drm->device)->card_type >= NV_10)
 		NVWriteRAMDAC(dev, head, NV_RAMDAC_NV10_CURSYNC, regp->nv10_cursync);
 
-	nouveau_hw_setpll(dev, pllreg, &regp->pllvals);
+	clk->pll_prog(clk, pllreg, &regp->pllvals);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_PLL_COEFF_SELECT, state->pllsel);
 	if (nv_two_heads(dev))
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, state->sel_clk);
-	if (dev_priv->chipset == 0x11)
+	if (nv_device(drm->device)->chipset == 0x11)
 		NVWriteRAMDAC(dev, head, NV_RAMDAC_DITHER_NV11, regp->dither);
 
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_GENERAL_CONTROL, regp->ramdac_gen_ctrl);
 
 	if (nv_gf4_disp_arch(dev))
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_630, regp->ramdac_630);
-	if (dev_priv->chipset >= 0x30)
+	if (nv_device(drm->device)->chipset >= 0x30)
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_634, regp->ramdac_634);
 
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_TV_SETUP, regp->tv_setup);
@@ -780,7 +521,7 @@ nv_load_state_ramdac(struct drm_device *dev, int head,
 	if (nv_gf4_disp_arch(dev))
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_8C0, regp->ramdac_8c0);
 
-	if (dev_priv->card_type == NV_40) {
+	if (nv_device(drm->device)->card_type == NV_40) {
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_A20, regp->ramdac_a20);
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_A24, regp->ramdac_a24);
 		NVWriteRAMDAC(dev, head, NV_PRAMDAC_A34, regp->ramdac_a34);
@@ -845,7 +586,7 @@ static void
 nv_save_state_ext(struct drm_device *dev, int head,
 		  struct nv04_mode_state *state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv04_crtc_reg *regp = &state->crtc_reg[head];
 	int i;
 
@@ -861,10 +602,10 @@ nv_save_state_ext(struct drm_device *dev, int head,
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_21);
 
-	if (dev_priv->card_type >= NV_20)
+	if (nv_device(drm->device)->card_type >= NV_20)
 		rd_cio_state(dev, head, regp, NV_CIO_CRE_47);
 
-	if (dev_priv->card_type >= NV_30)
+	if (nv_device(drm->device)->card_type >= NV_30)
 		rd_cio_state(dev, head, regp, 0x9f);
 
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_49);
@@ -873,14 +614,14 @@ nv_save_state_ext(struct drm_device *dev, int head,
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_ILACE__INDEX);
 
-	if (dev_priv->card_type >= NV_10) {
+	if (nv_device(drm->device)->card_type >= NV_10) {
 		regp->crtc_830 = NVReadCRTC(dev, head, NV_PCRTC_830);
 		regp->crtc_834 = NVReadCRTC(dev, head, NV_PCRTC_834);
 
-		if (dev_priv->card_type >= NV_30)
+		if (nv_device(drm->device)->card_type >= NV_30)
 			regp->gpio_ext = NVReadCRTC(dev, head, NV_PCRTC_GPIO_EXT);
 
-		if (dev_priv->card_type == NV_40)
+		if (nv_device(drm->device)->card_type == NV_40)
 			regp->crtc_850 = NVReadCRTC(dev, head, NV_PCRTC_850);
 
 		if (nv_two_heads(dev))
@@ -892,7 +633,7 @@ nv_save_state_ext(struct drm_device *dev, int head,
 
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH3__INDEX);
 	rd_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH4__INDEX);
-	if (dev_priv->card_type >= NV_10) {
+	if (nv_device(drm->device)->card_type >= NV_10) {
 		rd_cio_state(dev, head, regp, NV_CIO_CRE_EBR_INDEX);
 		rd_cio_state(dev, head, regp, NV_CIO_CRE_CSB);
 		rd_cio_state(dev, head, regp, NV_CIO_CRE_4B);
@@ -920,12 +661,14 @@ static void
 nv_load_state_ext(struct drm_device *dev, int head,
 		  struct nv04_mode_state *state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_timer *ptimer = nouveau_timer(device);
 	struct nv04_crtc_reg *regp = &state->crtc_reg[head];
 	uint32_t reg900;
 	int i;
 
-	if (dev_priv->card_type >= NV_10) {
+	if (nv_device(drm->device)->card_type >= NV_10) {
 		if (nv_two_heads(dev))
 			/* setting ENGINE_CTRL (EC) *must* come before
 			 * CIO_CRE_LCD, as writing CRE_LCD sets bits 16 & 17 in
@@ -933,24 +676,24 @@ nv_load_state_ext(struct drm_device *dev, int head,
 			 */
 			NVWriteCRTC(dev, head, NV_PCRTC_ENGINE_CTRL, regp->crtc_eng_ctrl);
 
-		nvWriteVIDEO(dev, NV_PVIDEO_STOP, 1);
-		nvWriteVIDEO(dev, NV_PVIDEO_INTR_EN, 0);
-		nvWriteVIDEO(dev, NV_PVIDEO_OFFSET_BUFF(0), 0);
-		nvWriteVIDEO(dev, NV_PVIDEO_OFFSET_BUFF(1), 0);
-		nvWriteVIDEO(dev, NV_PVIDEO_LIMIT(0), dev_priv->fb_available_size - 1);
-		nvWriteVIDEO(dev, NV_PVIDEO_LIMIT(1), dev_priv->fb_available_size - 1);
-		nvWriteVIDEO(dev, NV_PVIDEO_UVPLANE_LIMIT(0), dev_priv->fb_available_size - 1);
-		nvWriteVIDEO(dev, NV_PVIDEO_UVPLANE_LIMIT(1), dev_priv->fb_available_size - 1);
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_2, 0);
+		nv_wr32(device, NV_PVIDEO_STOP, 1);
+		nv_wr32(device, NV_PVIDEO_INTR_EN, 0);
+		nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(0), 0);
+		nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(1), 0);
+		nv_wr32(device, NV_PVIDEO_LIMIT(0), 0); //drm->fb_available_size - 1);
+		nv_wr32(device, NV_PVIDEO_LIMIT(1), 0); //drm->fb_available_size - 1);
+		nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(0), 0); //drm->fb_available_size - 1);
+		nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(1), 0); //drm->fb_available_size - 1);
+		nv_wr32(device, NV_PBUS_POWERCTRL_2, 0);
 
 		NVWriteCRTC(dev, head, NV_PCRTC_CURSOR_CONFIG, regp->cursor_cfg);
 		NVWriteCRTC(dev, head, NV_PCRTC_830, regp->crtc_830);
 		NVWriteCRTC(dev, head, NV_PCRTC_834, regp->crtc_834);
 
-		if (dev_priv->card_type >= NV_30)
+		if (nv_device(drm->device)->card_type >= NV_30)
 			NVWriteCRTC(dev, head, NV_PCRTC_GPIO_EXT, regp->gpio_ext);
 
-		if (dev_priv->card_type == NV_40) {
+		if (nv_device(drm->device)->card_type == NV_40) {
 			NVWriteCRTC(dev, head, NV_PCRTC_850, regp->crtc_850);
 
 			reg900 = NVReadRAMDAC(dev, head, NV_PRAMDAC_900);
@@ -973,23 +716,23 @@ nv_load_state_ext(struct drm_device *dev, int head,
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_FF_INDEX);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-	if (dev_priv->card_type >= NV_20)
+	if (nv_device(drm->device)->card_type >= NV_20)
 		wr_cio_state(dev, head, regp, NV_CIO_CRE_47);
 
-	if (dev_priv->card_type >= NV_30)
+	if (nv_device(drm->device)->card_type >= NV_30)
 		wr_cio_state(dev, head, regp, 0x9f);
 
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_49);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR1_INDEX);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		nv_fix_nv40_hw_cursor(dev, head);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_ILACE__INDEX);
 
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH3__INDEX);
 	wr_cio_state(dev, head, regp, NV_CIO_CRE_SCRATCH4__INDEX);
-	if (dev_priv->card_type >= NV_10) {
+	if (nv_device(drm->device)->card_type >= NV_10) {
 		wr_cio_state(dev, head, regp, NV_CIO_CRE_EBR_INDEX);
 		wr_cio_state(dev, head, regp, NV_CIO_CRE_CSB);
 		wr_cio_state(dev, head, regp, NV_CIO_CRE_4B);
@@ -997,11 +740,11 @@ nv_load_state_ext(struct drm_device *dev, int head,
 	}
 	/* NV11 and NV20 stop at 0x52. */
 	if (nv_gf4_disp_arch(dev)) {
-		if (dev_priv->card_type == NV_10) {
+		if (nv_device(drm->device)->card_type == NV_10) {
 			/* Not waiting for vertical retrace before modifying
 			   CRE_53/CRE_54 causes lockups. */
-			nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
-			nouveau_wait_eq(dev, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0);
+			nouveau_timer_wait_eq(ptimer, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
+			nouveau_timer_wait_eq(ptimer, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x0);
 		}
 
 		wr_cio_state(dev, head, regp, NV_CIO_CRE_42);
@@ -1024,14 +767,15 @@ static void
 nv_save_state_palette(struct drm_device *dev, int head,
 		      struct nv04_mode_state *state)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	int head_offset = head * NV_PRMDIO_SIZE, i;
 
-	nv_wr08(dev, NV_PRMDIO_PIXEL_MASK + head_offset,
+	nv_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
 				NV_PRMDIO_PIXEL_MASK_MASK);
-	nv_wr08(dev, NV_PRMDIO_READ_MODE_ADDRESS + head_offset, 0x0);
+	nv_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS + head_offset, 0x0);
 
 	for (i = 0; i < 768; i++) {
-		state->crtc_reg[head].DAC[i] = nv_rd08(dev,
+		state->crtc_reg[head].DAC[i] = nv_rd08(device,
 				NV_PRMDIO_PALETTE_DATA + head_offset);
 	}
 
@@ -1042,14 +786,15 @@ void
 nouveau_hw_load_state_palette(struct drm_device *dev, int head,
 			      struct nv04_mode_state *state)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	int head_offset = head * NV_PRMDIO_SIZE, i;
 
-	nv_wr08(dev, NV_PRMDIO_PIXEL_MASK + head_offset,
+	nv_wr08(device, NV_PRMDIO_PIXEL_MASK + head_offset,
 				NV_PRMDIO_PIXEL_MASK_MASK);
-	nv_wr08(dev, NV_PRMDIO_WRITE_MODE_ADDRESS + head_offset, 0x0);
+	nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS + head_offset, 0x0);
 
 	for (i = 0; i < 768; i++) {
-		nv_wr08(dev, NV_PRMDIO_PALETTE_DATA + head_offset,
+		nv_wr08(device, NV_PRMDIO_PALETTE_DATA + head_offset,
 				state->crtc_reg[head].DAC[i]);
 	}
 
@@ -1059,9 +804,9 @@ nouveau_hw_load_state_palette(struct drm_device *dev, int head,
 void nouveau_hw_save_state(struct drm_device *dev, int head,
 			   struct nv04_mode_state *state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	if (dev_priv->chipset == 0x11)
+	if (nv_device(drm->device)->chipset == 0x11)
 		/* NB: no attempt is made to restore the bad pll later on */
 		nouveau_hw_fix_bad_vpll(dev, head);
 	nv_save_state_ramdac(dev, head, state);
diff --git a/drivers/gpu/drm/nouveau/nouveau_hw.h b/drivers/gpu/drm/nouveau/nouveau_hw.h
index 2989090b943..7dff1021fab 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hw.h
+++ b/drivers/gpu/drm/nouveau/nouveau_hw.h
@@ -23,8 +23,10 @@
 #ifndef __NOUVEAU_HW_H__
 #define __NOUVEAU_HW_H__
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nv04_display.h"
+
+#include <subdev/bios/pll.h>
 
 #define MASK(field) ( \
 	(0xffffffff >> (31 - ((1 ? field) - (0 ? field)))) << (0 ? field))
@@ -38,12 +40,10 @@ void NVWriteVgaGr(struct drm_device *, int head, uint8_t index, uint8_t value);
 uint8_t NVReadVgaGr(struct drm_device *, int head, uint8_t index);
 void NVSetOwner(struct drm_device *, int owner);
 void NVBlankScreen(struct drm_device *, int head, bool blank);
-void nouveau_hw_setpll(struct drm_device *, uint32_t reg1,
-		       struct nouveau_pll_vals *pv);
-int nouveau_hw_get_pllvals(struct drm_device *, enum pll_types plltype,
+int nouveau_hw_get_pllvals(struct drm_device *, enum nvbios_pll_type plltype,
 			   struct nouveau_pll_vals *pllvals);
 int nouveau_hw_pllvals_to_clk(struct nouveau_pll_vals *pllvals);
-int nouveau_hw_get_clock(struct drm_device *, enum pll_types plltype);
+int nouveau_hw_get_clock(struct drm_device *, enum nvbios_pll_type plltype);
 void nouveau_hw_save_vga_fonts(struct drm_device *, bool save);
 void nouveau_hw_save_state(struct drm_device *, int head,
 			   struct nv04_mode_state *state);
@@ -55,115 +55,51 @@ void nouveau_hw_load_state_palette(struct drm_device *, int head,
 /* nouveau_calc.c */
 extern void nouveau_calc_arb(struct drm_device *, int vclk, int bpp,
 			     int *burst, int *lwm);
-extern int nouveau_calc_pll_mnp(struct drm_device *, struct pll_lims *pll_lim,
-				int clk, struct nouveau_pll_vals *pv);
-
-static inline uint32_t
-nvReadMC(struct drm_device *dev, uint32_t reg)
-{
-	uint32_t val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(MC, dev, "reg %08x val %08x\n", reg, val);
-	return val;
-}
-
-static inline void
-nvWriteMC(struct drm_device *dev, uint32_t reg, uint32_t val)
-{
-	NV_REG_DEBUG(MC, dev, "reg %08x val %08x\n", reg, val);
-	nv_wr32(dev, reg, val);
-}
-
-static inline uint32_t
-nvReadVIDEO(struct drm_device *dev, uint32_t reg)
-{
-	uint32_t val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(VIDEO, dev, "reg %08x val %08x\n", reg, val);
-	return val;
-}
-
-static inline void
-nvWriteVIDEO(struct drm_device *dev, uint32_t reg, uint32_t val)
-{
-	NV_REG_DEBUG(VIDEO, dev, "reg %08x val %08x\n", reg, val);
-	nv_wr32(dev, reg, val);
-}
-
-static inline uint32_t
-nvReadFB(struct drm_device *dev, uint32_t reg)
-{
-	uint32_t val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(FB, dev, "reg %08x val %08x\n", reg, val);
-	return val;
-}
-
-static inline void
-nvWriteFB(struct drm_device *dev, uint32_t reg, uint32_t val)
-{
-	NV_REG_DEBUG(FB, dev, "reg %08x val %08x\n", reg, val);
-	nv_wr32(dev, reg, val);
-}
-
-static inline uint32_t
-nvReadEXTDEV(struct drm_device *dev, uint32_t reg)
-{
-	uint32_t val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(EXTDEV, dev, "reg %08x val %08x\n", reg, val);
-	return val;
-}
-
-static inline void
-nvWriteEXTDEV(struct drm_device *dev, uint32_t reg, uint32_t val)
-{
-	NV_REG_DEBUG(EXTDEV, dev, "reg %08x val %08x\n", reg, val);
-	nv_wr32(dev, reg, val);
-}
 
 static inline uint32_t NVReadCRTC(struct drm_device *dev,
 					int head, uint32_t reg)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	uint32_t val;
 	if (head)
 		reg += NV_PCRTC0_SIZE;
-	val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(CRTC, dev, "head %d reg %08x val %08x\n", head, reg, val);
+	val = nv_rd32(device, reg);
 	return val;
 }
 
 static inline void NVWriteCRTC(struct drm_device *dev,
 					int head, uint32_t reg, uint32_t val)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	if (head)
 		reg += NV_PCRTC0_SIZE;
-	NV_REG_DEBUG(CRTC, dev, "head %d reg %08x val %08x\n", head, reg, val);
-	nv_wr32(dev, reg, val);
+	nv_wr32(device, reg, val);
 }
 
 static inline uint32_t NVReadRAMDAC(struct drm_device *dev,
 					int head, uint32_t reg)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	uint32_t val;
 	if (head)
 		reg += NV_PRAMDAC0_SIZE;
-	val = nv_rd32(dev, reg);
-	NV_REG_DEBUG(RAMDAC, dev, "head %d reg %08x val %08x\n",
-							head, reg, val);
+	val = nv_rd32(device, reg);
 	return val;
 }
 
 static inline void NVWriteRAMDAC(struct drm_device *dev,
 					int head, uint32_t reg, uint32_t val)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	if (head)
 		reg += NV_PRAMDAC0_SIZE;
-	NV_REG_DEBUG(RAMDAC, dev, "head %d reg %08x val %08x\n",
-							head, reg, val);
-	nv_wr32(dev, reg, val);
+	nv_wr32(device, reg, val);
 }
 
 static inline uint8_t nv_read_tmds(struct drm_device *dev,
 					int or, int dl, uint8_t address)
 {
-	int ramdac = (or & OUTPUT_C) >> 2;
+	int ramdac = (or & DCB_OUTPUT_C) >> 2;
 
 	NVWriteRAMDAC(dev, ramdac, NV_PRAMDAC_FP_TMDS_CONTROL + dl * 8,
 	NV_PRAMDAC_FP_TMDS_CONTROL_WRITE_DISABLE | address);
@@ -174,7 +110,7 @@ static inline void nv_write_tmds(struct drm_device *dev,
 					int or, int dl, uint8_t address,
 					uint8_t data)
 {
-	int ramdac = (or & OUTPUT_C) >> 2;
+	int ramdac = (or & DCB_OUTPUT_C) >> 2;
 
 	NVWriteRAMDAC(dev, ramdac, NV_PRAMDAC_FP_TMDS_DATA + dl * 8, data);
 	NVWriteRAMDAC(dev, ramdac, NV_PRAMDAC_FP_TMDS_CONTROL + dl * 8, address);
@@ -183,20 +119,18 @@ static inline void nv_write_tmds(struct drm_device *dev,
 static inline void NVWriteVgaCrtc(struct drm_device *dev,
 					int head, uint8_t index, uint8_t value)
 {
-	NV_REG_DEBUG(VGACRTC, dev, "head %d index 0x%02x data 0x%02x\n",
-							head, index, value);
-	nv_wr08(dev, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
-	nv_wr08(dev, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE, value);
+	struct nouveau_device *device = nouveau_dev(dev);
+	nv_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
+	nv_wr08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE, value);
 }
 
 static inline uint8_t NVReadVgaCrtc(struct drm_device *dev,
 					int head, uint8_t index)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	uint8_t val;
-	nv_wr08(dev, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
-	val = nv_rd08(dev, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE);
-	NV_REG_DEBUG(VGACRTC, dev, "head %d index 0x%02x data 0x%02x\n",
-							head, index, val);
+	nv_wr08(device, NV_PRMCIO_CRX__COLOR + head * NV_PRMCIO_SIZE, index);
+	val = nv_rd08(device, NV_PRMCIO_CR__COLOR + head * NV_PRMCIO_SIZE);
 	return val;
 }
 
@@ -230,75 +164,74 @@ static inline uint8_t NVReadVgaCrtc5758(struct drm_device *dev, int head, uint8_
 static inline uint8_t NVReadPRMVIO(struct drm_device *dev,
 					int head, uint32_t reg)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t val;
 
 	/* Only NV4x have two pvio ranges; other twoHeads cards MUST call
 	 * NVSetOwner for the relevant head to be programmed */
-	if (head && dev_priv->card_type == NV_40)
+	if (head && nv_device(drm->device)->card_type == NV_40)
 		reg += NV_PRMVIO_SIZE;
 
-	val = nv_rd08(dev, reg);
-	NV_REG_DEBUG(RMVIO, dev, "head %d reg %08x val %02x\n", head, reg, val);
+	val = nv_rd08(device, reg);
 	return val;
 }
 
 static inline void NVWritePRMVIO(struct drm_device *dev,
 					int head, uint32_t reg, uint8_t value)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	/* Only NV4x have two pvio ranges; other twoHeads cards MUST call
 	 * NVSetOwner for the relevant head to be programmed */
-	if (head && dev_priv->card_type == NV_40)
+	if (head && nv_device(drm->device)->card_type == NV_40)
 		reg += NV_PRMVIO_SIZE;
 
-	NV_REG_DEBUG(RMVIO, dev, "head %d reg %08x val %02x\n",
-						head, reg, value);
-	nv_wr08(dev, reg, value);
+	nv_wr08(device, reg, value);
 }
 
 static inline void NVSetEnablePalette(struct drm_device *dev, int head, bool enable)
 {
-	nv_rd08(dev, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-	nv_wr08(dev, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, enable ? 0 : 0x20);
+	struct nouveau_device *device = nouveau_dev(dev);
+	nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+	nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, enable ? 0 : 0x20);
 }
 
 static inline bool NVGetEnablePalette(struct drm_device *dev, int head)
 {
-	nv_rd08(dev, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-	return !(nv_rd08(dev, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE) & 0x20);
+	struct nouveau_device *device = nouveau_dev(dev);
+	nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+	return !(nv_rd08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE) & 0x20);
 }
 
 static inline void NVWriteVgaAttr(struct drm_device *dev,
 					int head, uint8_t index, uint8_t value)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	if (NVGetEnablePalette(dev, head))
 		index &= ~0x20;
 	else
 		index |= 0x20;
 
-	nv_rd08(dev, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-	NV_REG_DEBUG(VGAATTR, dev, "head %d index 0x%02x data 0x%02x\n",
-							head, index, value);
-	nv_wr08(dev, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
-	nv_wr08(dev, NV_PRMCIO_AR__WRITE + head * NV_PRMCIO_SIZE, value);
+	nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+	nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
+	nv_wr08(device, NV_PRMCIO_AR__WRITE + head * NV_PRMCIO_SIZE, value);
 }
 
 static inline uint8_t NVReadVgaAttr(struct drm_device *dev,
 					int head, uint8_t index)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	uint8_t val;
 	if (NVGetEnablePalette(dev, head))
 		index &= ~0x20;
 	else
 		index |= 0x20;
 
-	nv_rd08(dev, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
-	nv_wr08(dev, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
-	val = nv_rd08(dev, NV_PRMCIO_AR__READ + head * NV_PRMCIO_SIZE);
-	NV_REG_DEBUG(VGAATTR, dev, "head %d index 0x%02x data 0x%02x\n",
-							head, index, val);
+	nv_rd08(device, NV_PRMCIO_INP0__COLOR + head * NV_PRMCIO_SIZE);
+	nv_wr08(device, NV_PRMCIO_ARX + head * NV_PRMCIO_SIZE, index);
+	val = nv_rd08(device, NV_PRMCIO_AR__READ + head * NV_PRMCIO_SIZE);
 	return val;
 }
 
@@ -325,10 +258,11 @@ static inline void NVVgaProtect(struct drm_device *dev, int head, bool protect)
 static inline bool
 nv_heads_tied(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	if (dev_priv->chipset == 0x11)
-		return !!(nvReadMC(dev, NV_PBUS_DEBUG_1) & (1 << 28));
+	if (nv_device(drm->device)->chipset == 0x11)
+		return !!(nv_rd32(device, NV_PBUS_DEBUG_1) & (1 << 28));
 
 	return NVReadVgaCrtc(dev, 0, NV_CIO_CRE_44) & 0x4;
 }
@@ -377,13 +311,13 @@ nv_lock_vga_crtc_shadow(struct drm_device *dev, int head, int lock)
 static inline bool
 NVLockVgaCrtcs(struct drm_device *dev, bool lock)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	bool waslocked = !NVReadVgaCrtc(dev, 0, NV_CIO_SR_LOCK_INDEX);
 
 	NVWriteVgaCrtc(dev, 0, NV_CIO_SR_LOCK_INDEX,
 		       lock ? NV_CIO_SR_LOCK_VALUE : NV_CIO_SR_UNLOCK_RW_VALUE);
 	/* NV11 has independently lockable extended crtcs, except when tied */
-	if (dev_priv->chipset == 0x11 && !nv_heads_tied(dev))
+	if (nv_device(drm->device)->chipset == 0x11 && !nv_heads_tied(dev))
 		NVWriteVgaCrtc(dev, 1, NV_CIO_SR_LOCK_INDEX,
 			       lock ? NV_CIO_SR_LOCK_VALUE :
 				      NV_CIO_SR_UNLOCK_RW_VALUE);
@@ -398,9 +332,9 @@ NVLockVgaCrtcs(struct drm_device *dev, bool lock)
 
 static inline int nv_cursor_width(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	return dev_priv->card_type >= NV_10 ? NV10_CURSOR_SIZE : NV04_CURSOR_SIZE;
+	return nv_device(drm->device)->card_type >= NV_10 ? NV10_CURSOR_SIZE : NV04_CURSOR_SIZE;
 }
 
 static inline void
@@ -418,11 +352,11 @@ nv_fix_nv40_hw_cursor(struct drm_device *dev, int head)
 static inline void
 nv_set_crtc_base(struct drm_device *dev, int head, uint32_t offset)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	NVWriteCRTC(dev, head, NV_PCRTC_START, offset);
 
-	if (dev_priv->card_type == NV_04) {
+	if (nv_device(drm->device)->card_type == NV_04) {
 		/*
 		 * Hilarious, the 24th bit doesn't want to stick to
 		 * PCRTC_START...
@@ -437,9 +371,9 @@ nv_set_crtc_base(struct drm_device *dev, int head, uint32_t offset)
 static inline void
 nv_show_cursor(struct drm_device *dev, int head, bool show)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t *curctl1 =
-		&dev_priv->mode_reg.crtc_reg[head].CRTC[NV_CIO_CRE_HCUR_ADDR1_INDEX];
+		&nv04_display(dev)->mode_reg.crtc_reg[head].CRTC[NV_CIO_CRE_HCUR_ADDR1_INDEX];
 
 	if (show)
 		*curctl1 |= MASK(NV_CIO_CRE_HCUR_ADDR1_ENABLE);
@@ -447,14 +381,14 @@ nv_show_cursor(struct drm_device *dev, int head, bool show)
 		*curctl1 &= ~MASK(NV_CIO_CRE_HCUR_ADDR1_ENABLE);
 	NVWriteVgaCrtc(dev, head, NV_CIO_CRE_HCUR_ADDR1_INDEX, *curctl1);
 
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		nv_fix_nv40_hw_cursor(dev, head);
 }
 
 static inline uint32_t
 nv_pitch_align(struct drm_device *dev, uint32_t width, int bpp)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int mask;
 
 	if (bpp == 15)
@@ -463,7 +397,7 @@ nv_pitch_align(struct drm_device *dev, uint32_t width, int bpp)
 		bpp = 8;
 
 	/* Alignment requirements taken from the Haiku driver */
-	if (dev_priv->card_type == NV_04)
+	if (nv_device(drm->device)->card_type == NV_04)
 		mask = 128 / bpp - 1;
 	else
 		mask = 512 / bpp - 1;
diff --git a/drivers/gpu/drm/nouveau/nouveau_i2c.c b/drivers/gpu/drm/nouveau/nouveau_i2c.c
deleted file mode 100644
index 240cf962c99..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_i2c.c
+++ /dev/null
@@ -1,394 +0,0 @@
-/*
- * Copyright 2009 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/module.h>
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_i2c.h"
-#include "nouveau_hw.h"
-
-static void
-i2c_drive_scl(void *data, int state)
-{
-	struct nouveau_i2c_chan *port = data;
-	if (port->type == 0) {
-		u8 val = NVReadVgaCrtc(port->dev, 0, port->drive);
-		if (state) val |= 0x20;
-		else	   val &= 0xdf;
-		NVWriteVgaCrtc(port->dev, 0, port->drive, val | 0x01);
-	} else
-	if (port->type == 4) {
-		nv_mask(port->dev, port->drive, 0x2f, state ? 0x21 : 0x01);
-	} else
-	if (port->type == 5) {
-		if (state) port->state |= 0x01;
-		else	   port->state &= 0xfe;
-		nv_wr32(port->dev, port->drive, 4 | port->state);
-	}
-}
-
-static void
-i2c_drive_sda(void *data, int state)
-{
-	struct nouveau_i2c_chan *port = data;
-	if (port->type == 0) {
-		u8 val = NVReadVgaCrtc(port->dev, 0, port->drive);
-		if (state) val |= 0x10;
-		else	   val &= 0xef;
-		NVWriteVgaCrtc(port->dev, 0, port->drive, val | 0x01);
-	} else
-	if (port->type == 4) {
-		nv_mask(port->dev, port->drive, 0x1f, state ? 0x11 : 0x01);
-	} else
-	if (port->type == 5) {
-		if (state) port->state |= 0x02;
-		else	   port->state &= 0xfd;
-		nv_wr32(port->dev, port->drive, 4 | port->state);
-	}
-}
-
-static int
-i2c_sense_scl(void *data)
-{
-	struct nouveau_i2c_chan *port = data;
-	struct drm_nouveau_private *dev_priv = port->dev->dev_private;
-	if (port->type == 0) {
-		return !!(NVReadVgaCrtc(port->dev, 0, port->sense) & 0x04);
-	} else
-	if (port->type == 4) {
-		return !!(nv_rd32(port->dev, port->sense) & 0x00040000);
-	} else
-	if (port->type == 5) {
-		if (dev_priv->card_type < NV_D0)
-			return !!(nv_rd32(port->dev, port->sense) & 0x01);
-		else
-			return !!(nv_rd32(port->dev, port->sense) & 0x10);
-	}
-	return 0;
-}
-
-static int
-i2c_sense_sda(void *data)
-{
-	struct nouveau_i2c_chan *port = data;
-	struct drm_nouveau_private *dev_priv = port->dev->dev_private;
-	if (port->type == 0) {
-		return !!(NVReadVgaCrtc(port->dev, 0, port->sense) & 0x08);
-	} else
-	if (port->type == 4) {
-		return !!(nv_rd32(port->dev, port->sense) & 0x00080000);
-	} else
-	if (port->type == 5) {
-		if (dev_priv->card_type < NV_D0)
-			return !!(nv_rd32(port->dev, port->sense) & 0x02);
-		else
-			return !!(nv_rd32(port->dev, port->sense) & 0x20);
-	}
-	return 0;
-}
-
-static const uint32_t nv50_i2c_port[] = {
-	0x00e138, 0x00e150, 0x00e168, 0x00e180,
-	0x00e254, 0x00e274, 0x00e764, 0x00e780,
-	0x00e79c, 0x00e7b8
-};
-
-static u8 *
-i2c_table(struct drm_device *dev, u8 *version)
-{
-	u8 *dcb = dcb_table(dev), *i2c = NULL;
-	if (dcb) {
-		if (dcb[0] >= 0x15)
-			i2c = ROMPTR(dev, dcb[2]);
-		if (dcb[0] >= 0x30)
-			i2c = ROMPTR(dev, dcb[4]);
-	}
-
-	/* early revisions had no version number, use dcb version */
-	if (i2c) {
-		*version = dcb[0];
-		if (*version >= 0x30)
-			*version = i2c[0];
-	}
-
-	return i2c;
-}
-
-int
-nouveau_i2c_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct nouveau_i2c_chan *port;
-	u8 version = 0x00, entries, recordlen;
-	u8 *i2c, *entry, legacy[2][4] = {};
-	int ret, i;
-
-	INIT_LIST_HEAD(&dev_priv->i2c_ports);
-
-	i2c = i2c_table(dev, &version);
-	if (!i2c) {
-		u8 *bmp = &bios->data[bios->offset];
-		if (bios->type != NVBIOS_BMP)
-			return -ENODEV;
-
-		legacy[0][0] = NV_CIO_CRE_DDC_WR__INDEX;
-		legacy[0][1] = NV_CIO_CRE_DDC_STATUS__INDEX;
-		legacy[1][0] = NV_CIO_CRE_DDC0_WR__INDEX;
-		legacy[1][1] = NV_CIO_CRE_DDC0_STATUS__INDEX;
-
-		/* BMP (from v4.0) has i2c info in the structure, it's in a
-		 * fixed location on earlier VBIOS
-		 */
-		if (bmp[5] < 4)
-			i2c = &bios->data[0x48];
-		else
-			i2c = &bmp[0x36];
-
-		if (i2c[4]) legacy[0][0] = i2c[4];
-		if (i2c[5]) legacy[0][1] = i2c[5];
-		if (i2c[6]) legacy[1][0] = i2c[6];
-		if (i2c[7]) legacy[1][1] = i2c[7];
-	}
-
-	if (version >= 0x30) {
-		entry     = i2c[1] + i2c;
-		entries   = i2c[2];
-		recordlen = i2c[3];
-	} else
-	if (version) {
-		entry     = i2c;
-		entries   = 16;
-		recordlen = 4;
-	} else {
-		entry     = legacy[0];
-		entries   = 2;
-		recordlen = 4;
-	}
-
-	for (i = 0; i < entries; i++, entry += recordlen) {
-		port = kzalloc(sizeof(*port), GFP_KERNEL);
-		if (port == NULL) {
-			nouveau_i2c_fini(dev);
-			return -ENOMEM;
-		}
-
-		port->type = entry[3];
-		if (version < 0x30) {
-			port->type &= 0x07;
-			if (port->type == 0x07)
-				port->type = 0xff;
-		}
-
-		if (port->type == 0xff) {
-			kfree(port);
-			continue;
-		}
-
-		switch (port->type) {
-		case 0: /* NV04:NV50 */
-			port->drive = entry[0];
-			port->sense = entry[1];
-			break;
-		case 4: /* NV4E */
-			port->drive = 0x600800 + entry[1];
-			port->sense = port->drive;
-			break;
-		case 5: /* NV50- */
-			port->drive = entry[0] & 0x0f;
-			if (dev_priv->card_type < NV_D0) {
-				if (port->drive >= ARRAY_SIZE(nv50_i2c_port))
-					break;
-				port->drive = nv50_i2c_port[port->drive];
-				port->sense = port->drive;
-			} else {
-				port->drive = 0x00d014 + (port->drive * 0x20);
-				port->sense = port->drive;
-			}
-			break;
-		case 6: /* NV50- DP AUX */
-			port->drive = entry[0] & 0x0f;
-			port->sense = port->drive;
-			port->adapter.algo = &nouveau_dp_i2c_algo;
-			break;
-		default:
-			break;
-		}
-
-		if (!port->adapter.algo && !port->drive) {
-			NV_ERROR(dev, "I2C%d: type %d index %x/%x unknown\n",
-				 i, port->type, port->drive, port->sense);
-			kfree(port);
-			continue;
-		}
-
-		snprintf(port->adapter.name, sizeof(port->adapter.name),
-			 "nouveau-%s-%d", pci_name(dev->pdev), i);
-		port->adapter.owner = THIS_MODULE;
-		port->adapter.dev.parent = &dev->pdev->dev;
-		port->dev = dev;
-		port->index = i;
-		port->dcb = ROM32(entry[0]);
-		i2c_set_adapdata(&port->adapter, i2c);
-
-		if (port->adapter.algo != &nouveau_dp_i2c_algo) {
-			port->adapter.algo_data = &port->bit;
-			port->bit.udelay = 10;
-			port->bit.timeout = usecs_to_jiffies(2200);
-			port->bit.data = port;
-			port->bit.setsda = i2c_drive_sda;
-			port->bit.setscl = i2c_drive_scl;
-			port->bit.getsda = i2c_sense_sda;
-			port->bit.getscl = i2c_sense_scl;
-
-			i2c_drive_scl(port, 0);
-			i2c_drive_sda(port, 1);
-			i2c_drive_scl(port, 1);
-
-			ret = i2c_bit_add_bus(&port->adapter);
-		} else {
-			port->adapter.algo = &nouveau_dp_i2c_algo;
-			ret = i2c_add_adapter(&port->adapter);
-		}
-
-		if (ret) {
-			NV_ERROR(dev, "I2C%d: failed register: %d\n", i, ret);
-			kfree(port);
-			continue;
-		}
-
-		list_add_tail(&port->head, &dev_priv->i2c_ports);
-	}
-
-	return 0;
-}
-
-void
-nouveau_i2c_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_i2c_chan *port, *tmp;
-
-	list_for_each_entry_safe(port, tmp, &dev_priv->i2c_ports, head) {
-		i2c_del_adapter(&port->adapter);
-		kfree(port);
-	}
-}
-
-struct nouveau_i2c_chan *
-nouveau_i2c_find(struct drm_device *dev, u8 index)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_i2c_chan *port;
-
-	if (index == NV_I2C_DEFAULT(0) ||
-	    index == NV_I2C_DEFAULT(1)) {
-		u8 version, *i2c = i2c_table(dev, &version);
-		if (i2c && version >= 0x30) {
-			if (index == NV_I2C_DEFAULT(0))
-				index = (i2c[4] & 0x0f);
-			else
-				index = (i2c[4] & 0xf0) >> 4;
-		} else {
-			index = 2;
-		}
-	}
-
-	list_for_each_entry(port, &dev_priv->i2c_ports, head) {
-		if (port->index == index)
-			break;
-	}
-
-	if (&port->head == &dev_priv->i2c_ports)
-		return NULL;
-
-	if (dev_priv->card_type >= NV_50 && (port->dcb & 0x00000100)) {
-		u32 reg = 0x00e500, val;
-		if (port->type == 6) {
-			reg += port->drive * 0x50;
-			val  = 0x2002;
-		} else {
-			reg += ((port->dcb & 0x1e00) >> 9) * 0x50;
-			val  = 0xe001;
-		}
-
-		/* nfi, but neither auxch or i2c work if it's 1 */
-		nv_mask(dev, reg + 0x0c, 0x00000001, 0x00000000);
-		/* nfi, but switches auxch vs normal i2c */
-		nv_mask(dev, reg + 0x00, 0x0000f003, val);
-	}
-
-	return port;
-}
-
-bool
-nouveau_probe_i2c_addr(struct nouveau_i2c_chan *i2c, int addr)
-{
-	uint8_t buf[] = { 0 };
-	struct i2c_msg msgs[] = {
-		{
-			.addr = addr,
-			.flags = 0,
-			.len = 1,
-			.buf = buf,
-		},
-		{
-			.addr = addr,
-			.flags = I2C_M_RD,
-			.len = 1,
-			.buf = buf,
-		}
-	};
-
-	return i2c_transfer(&i2c->adapter, msgs, 2) == 2;
-}
-
-int
-nouveau_i2c_identify(struct drm_device *dev, const char *what,
-		     struct i2c_board_info *info,
-		     bool (*match)(struct nouveau_i2c_chan *,
-				   struct i2c_board_info *),
-		     int index)
-{
-	struct nouveau_i2c_chan *i2c = nouveau_i2c_find(dev, index);
-	int i;
-
-	if (!i2c) {
-		NV_DEBUG(dev, "No bus when probing %s on %d\n", what, index);
-		return -ENODEV;
-	}
-
-	NV_DEBUG(dev, "Probing %ss on I2C bus: %d\n", what, i2c->index);
-	for (i = 0; info[i].addr; i++) {
-		if (nouveau_probe_i2c_addr(i2c, info[i].addr) &&
-		    (!match || match(i2c, &info[i]))) {
-			NV_INFO(dev, "Detected %s: %s\n", what, info[i].type);
-			return i;
-		}
-	}
-
-	NV_DEBUG(dev, "No devices found.\n");
-	return -ENODEV;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ioc32.c b/drivers/gpu/drm/nouveau/nouveau_ioc32.c
index 475ba810bba..08214bcdcb1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ioc32.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ioc32.c
@@ -33,10 +33,9 @@
 
 #include <linux/compat.h>
 
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_ioctl.h"
 
 /**
  * Called whenever a 32-bit process running under a 64-bit kernel
diff --git a/drivers/gpu/drm/nouveau/nouveau_ioctl.h b/drivers/gpu/drm/nouveau/nouveau_ioctl.h
new file mode 100644
index 00000000000..ef2b2906d9e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_ioctl.h
@@ -0,0 +1,6 @@
+#ifndef __NOUVEAU_IOCTL_H__
+#define __NOUVEAU_IOCTL_H__
+
+long nouveau_compat_ioctl(struct file *, unsigned int cmd, unsigned long arg);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_irq.c b/drivers/gpu/drm/nouveau/nouveau_irq.c
index b2c2937531a..9ca8afdb554 100644
--- a/drivers/gpu/drm/nouveau/nouveau_irq.c
+++ b/drivers/gpu/drm/nouveau/nouveau_irq.c
@@ -1,147 +1,86 @@
 /*
- * Copyright (C) 2006 Ben Skeggs.
+ * Copyright 2012 Red Hat Inc.
  *
- * All Rights Reserved.
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
  *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
  *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
  *
+ * Authors: Ben Skeggs
  */
 
-/*
- * Authors:
- *   Ben Skeggs <darktama@iinet.net.au>
- */
+#include <subdev/mc.h>
 
-#include "drmP.h"
-#include "drm.h"
 #include "nouveau_drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_reg.h"
-#include "nouveau_ramht.h"
-#include "nouveau_util.h"
+#include "nouveau_irq.h"
+#include "nv50_display.h"
 
 void
 nouveau_irq_preinstall(struct drm_device *dev)
 {
-	/* Master disable */
-	nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
+	nv_wr32(nouveau_dev(dev), 0x000140, 0x00000000);
 }
 
 int
 nouveau_irq_postinstall(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	/* Master enable */
-	nv_wr32(dev, NV03_PMC_INTR_EN_0, NV_PMC_INTR_EN_0_MASTER_ENABLE);
-	if (dev_priv->msi_enabled)
-		nv_wr08(dev, 0x00088068, 0xff);
-
+	nv_wr32(nouveau_dev(dev), 0x000140, 0x00000001);
 	return 0;
 }
 
 void
 nouveau_irq_uninstall(struct drm_device *dev)
 {
-	/* Master disable */
-	nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
+	nv_wr32(nouveau_dev(dev), 0x000140, 0x00000000);
 }
 
 irqreturn_t
 nouveau_irq_handler(DRM_IRQ_ARGS)
 {
-	struct drm_device *dev = (struct drm_device *)arg;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
+	struct drm_device *dev = arg;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_mc *pmc = nouveau_mc(device);
 	u32 stat;
-	int i;
 
-	stat = nv_rd32(dev, NV03_PMC_INTR_0);
+	stat = nv_rd32(device, 0x000100);
 	if (stat == 0 || stat == ~0)
 		return IRQ_NONE;
 
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	for (i = 0; i < 32 && stat; i++) {
-		if (!(stat & (1 << i)) || !dev_priv->irq_handler[i])
-			continue;
+	nv_subdev(pmc)->intr(nv_subdev(pmc));
 
-		dev_priv->irq_handler[i](dev);
-		stat &= ~(1 << i);
+	if (device->card_type >= NV_D0) {
+		if (nv_rd32(device, 0x000100) & 0x04000000)
+			nvd0_display_intr(dev);
+	} else
+	if (device->card_type >= NV_50) {
+		if (nv_rd32(device, 0x000100) & 0x04000000)
+			nv50_display_intr(dev);
 	}
 
-	if (dev_priv->msi_enabled)
-		nv_wr08(dev, 0x00088068, 0xff);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	if (stat && nouveau_ratelimit())
-		NV_ERROR(dev, "PMC - unhandled INTR 0x%08x\n", stat);
 	return IRQ_HANDLED;
 }
 
 int
 nouveau_irq_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int ret;
-
-	if (nouveau_msi != 0 && dev_priv->card_type >= NV_50) {
-		ret = pci_enable_msi(dev->pdev);
-		if (ret == 0) {
-			NV_INFO(dev, "enabled MSI\n");
-			dev_priv->msi_enabled = true;
-		}
-	}
-
 	return drm_irq_install(dev);
 }
 
 void
 nouveau_irq_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
 	drm_irq_uninstall(dev);
-	if (dev_priv->msi_enabled)
-		pci_disable_msi(dev->pdev);
-}
-
-void
-nouveau_irq_register(struct drm_device *dev, int status_bit,
-		     void (*handler)(struct drm_device *))
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	dev_priv->irq_handler[status_bit] = handler;
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-}
-
-void
-nouveau_irq_unregister(struct drm_device *dev, int status_bit)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	dev_priv->irq_handler[status_bit] = NULL;
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_irq.h b/drivers/gpu/drm/nouveau/nouveau_irq.h
new file mode 100644
index 00000000000..06714ad857b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_irq.h
@@ -0,0 +1,11 @@
+#ifndef __NOUVEAU_IRQ_H__
+#define __NOUVEAU_IRQ_H__
+
+extern int         nouveau_irq_init(struct drm_device *);
+extern void        nouveau_irq_fini(struct drm_device *);
+extern irqreturn_t nouveau_irq_handler(DRM_IRQ_ARGS);
+extern void        nouveau_irq_preinstall(struct drm_device *);
+extern int         nouveau_irq_postinstall(struct drm_device *);
+extern void        nouveau_irq_uninstall(struct drm_device *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 5b498ea32e1..7e0ff10a275 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -30,448 +30,10 @@
  *    Roy Spliet <r.spliet@student.tudelft.nl>
  */
 
-
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_pm.h"
-#include "nouveau_mm.h"
-#include "nouveau_vm.h"
-#include "nouveau_fifo.h"
-#include "nouveau_fence.h"
-
-/*
- * NV10-NV40 tiling helpers
- */
-
-static void
-nv10_mem_update_tile_region(struct drm_device *dev,
-			    struct nouveau_tile_reg *tile, uint32_t addr,
-			    uint32_t size, uint32_t pitch, uint32_t flags)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i = tile - dev_priv->tile.reg, j;
-	unsigned long save;
-
-	nouveau_fence_unref(&tile->fence);
-
-	if (tile->pitch)
-		pfb->free_tile_region(dev, i);
-
-	if (pitch)
-		pfb->init_tile_region(dev, i, addr, size, pitch, flags);
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, save);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 0);
-	nv04_fifo_cache_pull(dev, false);
-
-	nouveau_wait_for_idle(dev);
-
-	pfb->set_tile_region(dev, i);
-	for (j = 0; j < NVOBJ_ENGINE_NR; j++) {
-		if (dev_priv->eng[j] && dev_priv->eng[j]->set_tile_region)
-			dev_priv->eng[j]->set_tile_region(dev, i);
-	}
-
-	nv04_fifo_cache_pull(dev, true);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 1);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, save);
-}
-
-static struct nouveau_tile_reg *
-nv10_mem_get_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	spin_lock(&dev_priv->tile.lock);
-
-	if (!tile->used &&
-	    (!tile->fence || nouveau_fence_done(tile->fence)))
-		tile->used = true;
-	else
-		tile = NULL;
-
-	spin_unlock(&dev_priv->tile.lock);
-	return tile;
-}
-
-void
-nv10_mem_put_tile_region(struct drm_device *dev, struct nouveau_tile_reg *tile,
-			 struct nouveau_fence *fence)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (tile) {
-		spin_lock(&dev_priv->tile.lock);
-		if (fence) {
-			/* Mark it as pending. */
-			tile->fence = fence;
-			nouveau_fence_ref(fence);
-		}
-
-		tile->used = false;
-		spin_unlock(&dev_priv->tile.lock);
-	}
-}
-
-struct nouveau_tile_reg *
-nv10_mem_set_tiling(struct drm_device *dev, uint32_t addr, uint32_t size,
-		    uint32_t pitch, uint32_t flags)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct nouveau_tile_reg *tile, *found = NULL;
-	int i;
-
-	for (i = 0; i < pfb->num_tiles; i++) {
-		tile = nv10_mem_get_tile_region(dev, i);
-
-		if (pitch && !found) {
-			found = tile;
-			continue;
-
-		} else if (tile && tile->pitch) {
-			/* Kill an unused tile region. */
-			nv10_mem_update_tile_region(dev, tile, 0, 0, 0, 0);
-		}
-
-		nv10_mem_put_tile_region(dev, tile, NULL);
-	}
-
-	if (found)
-		nv10_mem_update_tile_region(dev, found, addr, size,
-					    pitch, flags);
-	return found;
-}
-
-/*
- * Cleanup everything
- */
-void
-nouveau_mem_vram_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	ttm_bo_device_release(&dev_priv->ttm.bdev);
-
-	nouveau_ttm_global_release(dev_priv);
-
-	if (dev_priv->fb_mtrr >= 0) {
-		drm_mtrr_del(dev_priv->fb_mtrr,
-			     pci_resource_start(dev->pdev, 1),
-			     pci_resource_len(dev->pdev, 1), DRM_MTRR_WC);
-		dev_priv->fb_mtrr = -1;
-	}
-}
-
-void
-nouveau_mem_gart_fini(struct drm_device *dev)
-{
-	nouveau_sgdma_takedown(dev);
-
-	if (drm_core_has_AGP(dev) && dev->agp) {
-		struct drm_agp_mem *entry, *tempe;
-
-		/* Remove AGP resources, but leave dev->agp
-		   intact until drv_cleanup is called. */
-		list_for_each_entry_safe(entry, tempe, &dev->agp->memory, head) {
-			if (entry->bound)
-				drm_unbind_agp(entry->memory);
-			drm_free_agp(entry->memory, entry->pages);
-			kfree(entry);
-		}
-		INIT_LIST_HEAD(&dev->agp->memory);
 
-		if (dev->agp->acquired)
-			drm_agp_release(dev);
-
-		dev->agp->acquired = 0;
-		dev->agp->enabled = 0;
-	}
-}
-
-bool
-nouveau_mem_flags_valid(struct drm_device *dev, u32 tile_flags)
-{
-	if (!(tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK))
-		return true;
-
-	return false;
-}
-
-#if __OS_HAS_AGP
-static unsigned long
-get_agp_mode(struct drm_device *dev, unsigned long mode)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	/*
-	 * FW seems to be broken on nv18, it makes the card lock up
-	 * randomly.
-	 */
-	if (dev_priv->chipset == 0x18)
-		mode &= ~PCI_AGP_COMMAND_FW;
-
-	/*
-	 * AGP mode set in the command line.
-	 */
-	if (nouveau_agpmode > 0) {
-		bool agpv3 = mode & 0x8;
-		int rate = agpv3 ? nouveau_agpmode / 4 : nouveau_agpmode;
-
-		mode = (mode & ~0x7) | (rate & 0x7);
-	}
-
-	return mode;
-}
-#endif
-
-int
-nouveau_mem_reset_agp(struct drm_device *dev)
-{
-#if __OS_HAS_AGP
-	uint32_t saved_pci_nv_1, pmc_enable;
-	int ret;
-
-	/* First of all, disable fast writes, otherwise if it's
-	 * already enabled in the AGP bridge and we disable the card's
-	 * AGP controller we might be locking ourselves out of it. */
-	if ((nv_rd32(dev, NV04_PBUS_PCI_NV_19) |
-	     dev->agp->mode) & PCI_AGP_COMMAND_FW) {
-		struct drm_agp_info info;
-		struct drm_agp_mode mode;
-
-		ret = drm_agp_info(dev, &info);
-		if (ret)
-			return ret;
-
-		mode.mode = get_agp_mode(dev, info.mode) & ~PCI_AGP_COMMAND_FW;
-		ret = drm_agp_enable(dev, mode);
-		if (ret)
-			return ret;
-	}
-
-	saved_pci_nv_1 = nv_rd32(dev, NV04_PBUS_PCI_NV_1);
-
-	/* clear busmaster bit */
-	nv_wr32(dev, NV04_PBUS_PCI_NV_1, saved_pci_nv_1 & ~0x4);
-	/* disable AGP */
-	nv_wr32(dev, NV04_PBUS_PCI_NV_19, 0);
-
-	/* power cycle pgraph, if enabled */
-	pmc_enable = nv_rd32(dev, NV03_PMC_ENABLE);
-	if (pmc_enable & NV_PMC_ENABLE_PGRAPH) {
-		nv_wr32(dev, NV03_PMC_ENABLE,
-				pmc_enable & ~NV_PMC_ENABLE_PGRAPH);
-		nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
-				NV_PMC_ENABLE_PGRAPH);
-	}
-
-	/* and restore (gives effect of resetting AGP) */
-	nv_wr32(dev, NV04_PBUS_PCI_NV_1, saved_pci_nv_1);
-#endif
-
-	return 0;
-}
-
-int
-nouveau_mem_init_agp(struct drm_device *dev)
-{
-#if __OS_HAS_AGP
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct drm_agp_info info;
-	struct drm_agp_mode mode;
-	int ret;
-
-	if (!dev->agp->acquired) {
-		ret = drm_agp_acquire(dev);
-		if (ret) {
-			NV_ERROR(dev, "Unable to acquire AGP: %d\n", ret);
-			return ret;
-		}
-	}
-
-	nouveau_mem_reset_agp(dev);
-
-	ret = drm_agp_info(dev, &info);
-	if (ret) {
-		NV_ERROR(dev, "Unable to get AGP info: %d\n", ret);
-		return ret;
-	}
-
-	/* see agp.h for the AGPSTAT_* modes available */
-	mode.mode = get_agp_mode(dev, info.mode);
-	ret = drm_agp_enable(dev, mode);
-	if (ret) {
-		NV_ERROR(dev, "Unable to enable AGP: %d\n", ret);
-		return ret;
-	}
-
-	dev_priv->gart_info.type	= NOUVEAU_GART_AGP;
-	dev_priv->gart_info.aper_base	= info.aperture_base;
-	dev_priv->gart_info.aper_size	= info.aperture_size;
-#endif
-	return 0;
-}
-
-static const struct vram_types {
-	int value;
-	const char *name;
-} vram_type_map[] = {
-	{ NV_MEM_TYPE_STOLEN , "stolen system memory" },
-	{ NV_MEM_TYPE_SGRAM  , "SGRAM" },
-	{ NV_MEM_TYPE_SDRAM  , "SDRAM" },
-	{ NV_MEM_TYPE_DDR1   , "DDR1" },
-	{ NV_MEM_TYPE_DDR2   , "DDR2" },
-	{ NV_MEM_TYPE_DDR3   , "DDR3" },
-	{ NV_MEM_TYPE_GDDR2  , "GDDR2" },
-	{ NV_MEM_TYPE_GDDR3  , "GDDR3" },
-	{ NV_MEM_TYPE_GDDR4  , "GDDR4" },
-	{ NV_MEM_TYPE_GDDR5  , "GDDR5" },
-	{ NV_MEM_TYPE_UNKNOWN, "unknown type" }
-};
-
-int
-nouveau_mem_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-	const struct vram_types *vram_type;
-	int ret, dma_bits;
-
-	dma_bits = 32;
-	if (dev_priv->card_type >= NV_50) {
-		if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(40)))
-			dma_bits = 40;
-	} else
-	if (0 && pci_is_pcie(dev->pdev) &&
-	    dev_priv->chipset  > 0x40 &&
-	    dev_priv->chipset != 0x45) {
-		if (pci_dma_supported(dev->pdev, DMA_BIT_MASK(39)))
-			dma_bits = 39;
-	}
-
-	ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(dma_bits));
-	if (ret)
-		return ret;
-	ret = pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(dma_bits));
-	if (ret) {
-		/* Reset to default value. */
-		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(32));
-	}
-
-
-	ret = nouveau_ttm_global_init(dev_priv);
-	if (ret)
-		return ret;
-
-	ret = ttm_bo_device_init(&dev_priv->ttm.bdev,
-				 dev_priv->ttm.bo_global_ref.ref.object,
-				 &nouveau_bo_driver, DRM_FILE_PAGE_OFFSET,
-				 dma_bits <= 32 ? true : false);
-	if (ret) {
-		NV_ERROR(dev, "Error initialising bo driver: %d\n", ret);
-		return ret;
-	}
-
-	vram_type = vram_type_map;
-	while (vram_type->value != NV_MEM_TYPE_UNKNOWN) {
-		if (nouveau_vram_type) {
-			if (!strcasecmp(nouveau_vram_type, vram_type->name))
-				break;
-			dev_priv->vram_type = vram_type->value;
-		} else {
-			if (vram_type->value == dev_priv->vram_type)
-				break;
-		}
-		vram_type++;
-	}
-
-	NV_INFO(dev, "Detected %dMiB VRAM (%s)\n",
-		(int)(dev_priv->vram_size >> 20), vram_type->name);
-	if (dev_priv->vram_sys_base) {
-		NV_INFO(dev, "Stolen system memory at: 0x%010llx\n",
-			dev_priv->vram_sys_base);
-	}
-
-	dev_priv->fb_available_size = dev_priv->vram_size;
-	dev_priv->fb_mappable_pages = dev_priv->fb_available_size;
-	if (dev_priv->fb_mappable_pages > pci_resource_len(dev->pdev, 1))
-		dev_priv->fb_mappable_pages = pci_resource_len(dev->pdev, 1);
-	dev_priv->fb_mappable_pages >>= PAGE_SHIFT;
-
-	dev_priv->fb_available_size -= dev_priv->ramin_rsvd_vram;
-	dev_priv->fb_aper_free = dev_priv->fb_available_size;
-
-	/* mappable vram */
-	ret = ttm_bo_init_mm(bdev, TTM_PL_VRAM,
-			     dev_priv->fb_available_size >> PAGE_SHIFT);
-	if (ret) {
-		NV_ERROR(dev, "Failed VRAM mm init: %d\n", ret);
-		return ret;
-	}
-
-	if (dev_priv->card_type < NV_50) {
-		ret = nouveau_bo_new(dev, 256*1024, 0, TTM_PL_FLAG_VRAM,
-				     0, 0, NULL, &dev_priv->vga_ram);
-		if (ret == 0)
-			ret = nouveau_bo_pin(dev_priv->vga_ram,
-					     TTM_PL_FLAG_VRAM);
-
-		if (ret) {
-			NV_WARN(dev, "failed to reserve VGA memory\n");
-			nouveau_bo_ref(NULL, &dev_priv->vga_ram);
-		}
-	}
-
-	dev_priv->fb_mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 1),
-					 pci_resource_len(dev->pdev, 1),
-					 DRM_MTRR_WC);
-	return 0;
-}
-
-int
-nouveau_mem_gart_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct ttm_bo_device *bdev = &dev_priv->ttm.bdev;
-	int ret;
-
-	dev_priv->gart_info.type = NOUVEAU_GART_NONE;
-
-#if !defined(__powerpc__) && !defined(__ia64__)
-	if (drm_pci_device_is_agp(dev) && dev->agp && nouveau_agpmode) {
-		ret = nouveau_mem_init_agp(dev);
-		if (ret)
-			NV_ERROR(dev, "Error initialising AGP: %d\n", ret);
-	}
-#endif
-
-	if (dev_priv->gart_info.type == NOUVEAU_GART_NONE) {
-		ret = nouveau_sgdma_init(dev);
-		if (ret) {
-			NV_ERROR(dev, "Error initialising PCI(E): %d\n", ret);
-			return ret;
-		}
-	}
-
-	NV_INFO(dev, "%d MiB GART (aperture)\n",
-		(int)(dev_priv->gart_info.aper_size >> 20));
-	dev_priv->gart_info.aper_free = dev_priv->gart_info.aper_size;
-
-	ret = ttm_bo_init_mm(bdev, TTM_PL_TT,
-			     dev_priv->gart_info.aper_size >> PAGE_SHIFT);
-	if (ret) {
-		NV_ERROR(dev, "Failed TT mm init: %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
+#include <subdev/fb.h>
 
 static int
 nv40_mem_timing_calc(struct drm_device *dev, u32 freq,
@@ -479,6 +41,8 @@ nv40_mem_timing_calc(struct drm_device *dev, u32 freq,
 		     struct nouveau_pm_memtiming *boot,
 		     struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	t->reg[0] = (e->tRP << 24 | e->tRAS << 16 | e->tRFC << 8 | e->tRC);
 
 	/* XXX: I don't trust the -1's and +1's... they must come
@@ -494,7 +58,7 @@ nv40_mem_timing_calc(struct drm_device *dev, u32 freq,
 		     e->tRCDWR << 8 |
 		     e->tRCDRD);
 
-	NV_DEBUG(dev, "Entry %d: 220: %08x %08x %08x\n", t->id,
+	NV_DEBUG(drm, "Entry %d: 220: %08x %08x %08x\n", t->id,
 		 t->reg[0], t->reg[1], t->reg[2]);
 	return 0;
 }
@@ -505,7 +69,9 @@ nv50_mem_timing_calc(struct drm_device *dev, u32 freq,
 		     struct nouveau_pm_memtiming *boot,
 		     struct nouveau_pm_memtiming *t)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct bit_entry P;
 	uint8_t unk18 = 1, unk20 = 0, unk21 = 0, tmp7_3;
 
@@ -559,7 +125,7 @@ nv50_mem_timing_calc(struct drm_device *dev, u32 freq,
 		t->reg[7] = 0x4000202 | (e->tCL - 1) << 16;
 
 		/* XXX: P.version == 1 only has DDR2 and GDDR3? */
-		if (dev_priv->vram_type == NV_MEM_TYPE_DDR2) {
+		if (pfb->ram.type == NV_MEM_TYPE_DDR2) {
 			t->reg[5] |= (e->tCL + 3) << 8;
 			t->reg[6] |= (t->tCWL - 2) << 8;
 			t->reg[8] |= (e->tCL - 4);
@@ -592,11 +158,11 @@ nv50_mem_timing_calc(struct drm_device *dev, u32 freq,
 			    0x202;
 	}
 
-	NV_DEBUG(dev, "Entry %d: 220: %08x %08x %08x %08x\n", t->id,
+	NV_DEBUG(drm, "Entry %d: 220: %08x %08x %08x %08x\n", t->id,
 		 t->reg[0], t->reg[1], t->reg[2], t->reg[3]);
-	NV_DEBUG(dev, "         230: %08x %08x %08x %08x\n",
+	NV_DEBUG(drm, "         230: %08x %08x %08x %08x\n",
 		 t->reg[4], t->reg[5], t->reg[6], t->reg[7]);
-	NV_DEBUG(dev, "         240: %08x\n", t->reg[8]);
+	NV_DEBUG(drm, "         240: %08x\n", t->reg[8]);
 	return 0;
 }
 
@@ -606,6 +172,8 @@ nvc0_mem_timing_calc(struct drm_device *dev, u32 freq,
 		     struct nouveau_pm_memtiming *boot,
 		     struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	if (e->tCWL > 0)
 		t->tCWL = e->tCWL;
 
@@ -628,9 +196,9 @@ nvc0_mem_timing_calc(struct drm_device *dev, u32 freq,
 	t->reg[4] = (boot->reg[4] & 0xfff00fff) |
 		    (e->tRRD&0x1f) << 15;
 
-	NV_DEBUG(dev, "Entry %d: 290: %08x %08x %08x %08x\n", t->id,
+	NV_DEBUG(drm, "Entry %d: 290: %08x %08x %08x %08x\n", t->id,
 		 t->reg[0], t->reg[1], t->reg[2], t->reg[3]);
-	NV_DEBUG(dev, "         2a0: %08x\n", t->reg[4]);
+	NV_DEBUG(drm, "         2a0: %08x\n", t->reg[4]);
 	return 0;
 }
 
@@ -644,6 +212,8 @@ nouveau_mem_ddr2_mr(struct drm_device *dev, u32 freq,
 		    struct nouveau_pm_memtiming *boot,
 		    struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	t->drive_strength = 0;
 	if (len < 15) {
 		t->odt = boot->odt;
@@ -652,17 +222,17 @@ nouveau_mem_ddr2_mr(struct drm_device *dev, u32 freq,
 	}
 
 	if (e->tCL >= NV_MEM_CL_DDR2_MAX) {
-		NV_WARN(dev, "(%u) Invalid tCL: %u", t->id, e->tCL);
+		NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
 		return -ERANGE;
 	}
 
 	if (e->tWR >= NV_MEM_WR_DDR2_MAX) {
-		NV_WARN(dev, "(%u) Invalid tWR: %u", t->id, e->tWR);
+		NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
 		return -ERANGE;
 	}
 
 	if (t->odt > 3) {
-		NV_WARN(dev, "(%u) Invalid odt value, assuming disabled: %x",
+		NV_WARN(drm, "(%u) Invalid odt value, assuming disabled: %x",
 			t->id, t->odt);
 		t->odt = 0;
 	}
@@ -674,11 +244,11 @@ nouveau_mem_ddr2_mr(struct drm_device *dev, u32 freq,
 		   (t->odt & 0x1) << 2 |
 		   (t->odt & 0x2) << 5;
 
-	NV_DEBUG(dev, "(%u) MR: %08x", t->id, t->mr[0]);
+	NV_DEBUG(drm, "(%u) MR: %08x", t->id, t->mr[0]);
 	return 0;
 }
 
-uint8_t nv_mem_wr_lut_ddr3[NV_MEM_WR_DDR3_MAX] = {
+static const uint8_t nv_mem_wr_lut_ddr3[NV_MEM_WR_DDR3_MAX] = {
 	0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 0, 0};
 
 static int
@@ -687,6 +257,7 @@ nouveau_mem_ddr3_mr(struct drm_device *dev, u32 freq,
 		    struct nouveau_pm_memtiming *boot,
 		    struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u8 cl = e->tCL - 4;
 
 	t->drive_strength = 0;
@@ -697,17 +268,17 @@ nouveau_mem_ddr3_mr(struct drm_device *dev, u32 freq,
 	}
 
 	if (e->tCL >= NV_MEM_CL_DDR3_MAX || e->tCL < 4) {
-		NV_WARN(dev, "(%u) Invalid tCL: %u", t->id, e->tCL);
+		NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
 		return -ERANGE;
 	}
 
 	if (e->tWR >= NV_MEM_WR_DDR3_MAX || e->tWR < 4) {
-		NV_WARN(dev, "(%u) Invalid tWR: %u", t->id, e->tWR);
+		NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
 		return -ERANGE;
 	}
 
 	if (e->tCWL < 5) {
-		NV_WARN(dev, "(%u) Invalid tCWL: %u", t->id, e->tCWL);
+		NV_WARN(drm, "(%u) Invalid tCWL: %u", t->id, e->tCWL);
 		return -ERANGE;
 	}
 
@@ -722,13 +293,13 @@ nouveau_mem_ddr3_mr(struct drm_device *dev, u32 freq,
 		   (t->odt & 0x4) << 7;
 	t->mr[2] = (boot->mr[2] & 0x20ffb7) | (e->tCWL - 5) << 3;
 
-	NV_DEBUG(dev, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[2]);
+	NV_DEBUG(drm, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[2]);
 	return 0;
 }
 
-uint8_t nv_mem_cl_lut_gddr3[NV_MEM_CL_GDDR3_MAX] = {
+static const uint8_t nv_mem_cl_lut_gddr3[NV_MEM_CL_GDDR3_MAX] = {
 	0, 0, 0, 0, 4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11};
-uint8_t nv_mem_wr_lut_gddr3[NV_MEM_WR_GDDR3_MAX] = {
+static const uint8_t nv_mem_wr_lut_gddr3[NV_MEM_WR_GDDR3_MAX] = {
 	0, 0, 0, 0, 0, 2, 3, 8, 9, 10, 11, 0, 0, 1, 1, 0, 3};
 
 static int
@@ -737,6 +308,8 @@ nouveau_mem_gddr3_mr(struct drm_device *dev, u32 freq,
 		     struct nouveau_pm_memtiming *boot,
 		     struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	if (len < 15) {
 		t->drive_strength = boot->drive_strength;
 		t->odt = boot->odt;
@@ -746,17 +319,17 @@ nouveau_mem_gddr3_mr(struct drm_device *dev, u32 freq,
 	}
 
 	if (e->tCL >= NV_MEM_CL_GDDR3_MAX) {
-		NV_WARN(dev, "(%u) Invalid tCL: %u", t->id, e->tCL);
+		NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
 		return -ERANGE;
 	}
 
 	if (e->tWR >= NV_MEM_WR_GDDR3_MAX) {
-		NV_WARN(dev, "(%u) Invalid tWR: %u", t->id, e->tWR);
+		NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
 		return -ERANGE;
 	}
 
 	if (t->odt > 3) {
-		NV_WARN(dev, "(%u) Invalid odt value, assuming autocal: %x",
+		NV_WARN(drm, "(%u) Invalid odt value, assuming autocal: %x",
 			t->id, t->odt);
 		t->odt = 0;
 	}
@@ -770,7 +343,7 @@ nouveau_mem_gddr3_mr(struct drm_device *dev, u32 freq,
 		   (nv_mem_wr_lut_gddr3[e->tWR] & 0xf) << 4;
 	t->mr[2] = boot->mr[2];
 
-	NV_DEBUG(dev, "(%u) MR: %08x %08x %08x", t->id,
+	NV_DEBUG(drm, "(%u) MR: %08x %08x %08x", t->id,
 		      t->mr[0], t->mr[1], t->mr[2]);
 	return 0;
 }
@@ -781,6 +354,8 @@ nouveau_mem_gddr5_mr(struct drm_device *dev, u32 freq,
 		     struct nouveau_pm_memtiming *boot,
 		     struct nouveau_pm_memtiming *t)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	if (len < 15) {
 		t->drive_strength = boot->drive_strength;
 		t->odt = boot->odt;
@@ -790,17 +365,17 @@ nouveau_mem_gddr5_mr(struct drm_device *dev, u32 freq,
 	}
 
 	if (e->tCL >= NV_MEM_CL_GDDR5_MAX) {
-		NV_WARN(dev, "(%u) Invalid tCL: %u", t->id, e->tCL);
+		NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
 		return -ERANGE;
 	}
 
 	if (e->tWR >= NV_MEM_WR_GDDR5_MAX) {
-		NV_WARN(dev, "(%u) Invalid tWR: %u", t->id, e->tWR);
+		NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
 		return -ERANGE;
 	}
 
 	if (t->odt > 3) {
-		NV_WARN(dev, "(%u) Invalid odt value, assuming autocal: %x",
+		NV_WARN(drm, "(%u) Invalid odt value, assuming autocal: %x",
 			t->id, t->odt);
 		t->odt = 0;
 	}
@@ -812,7 +387,7 @@ nouveau_mem_gddr5_mr(struct drm_device *dev, u32 freq,
 		   t->drive_strength |
 		   (t->odt << 2);
 
-	NV_DEBUG(dev, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[1]);
+	NV_DEBUG(drm, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[1]);
 	return 0;
 }
 
@@ -820,8 +395,9 @@ int
 nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
 			struct nouveau_pm_memtiming *t)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_memtiming *boot = &pm->boot.timing;
 	struct nouveau_pm_tbl_entry *e;
 	u8 ver, len, *ptr, *ramcfg;
@@ -836,7 +412,7 @@ nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
 
 	t->tCWL = boot->tCWL;
 
-	switch (dev_priv->card_type) {
+	switch (device->card_type) {
 	case NV_40:
 		ret = nv40_mem_timing_calc(dev, freq, e, len, boot, t);
 		break;
@@ -852,7 +428,7 @@ nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
 		break;
 	}
 
-	switch (dev_priv->vram_type * !ret) {
+	switch (pfb->ram.type * !ret) {
 	case NV_MEM_TYPE_GDDR3:
 		ret = nouveau_mem_gddr3_mr(dev, freq, e, len, boot, t);
 		break;
@@ -879,7 +455,7 @@ nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
 		else
 			dll_off = !!(ramcfg[2] & 0x40);
 
-		switch (dev_priv->vram_type) {
+		switch (pfb->ram.type) {
 		case NV_MEM_TYPE_GDDR3:
 			t->mr[1] &= ~0x00000040;
 			t->mr[1] |=  0x00000040 * dll_off;
@@ -897,11 +473,12 @@ nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
 void
 nouveau_mem_timing_read(struct drm_device *dev, struct nouveau_pm_memtiming *t)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
 	u32 timing_base, timing_regs, mr_base;
 	int i;
 
-	if (dev_priv->card_type >= 0xC0) {
+	if (device->card_type >= 0xC0) {
 		timing_base = 0x10f290;
 		mr_base = 0x10f300;
 	} else {
@@ -911,7 +488,7 @@ nouveau_mem_timing_read(struct drm_device *dev, struct nouveau_pm_memtiming *t)
 
 	t->id = -1;
 
-	switch (dev_priv->card_type) {
+	switch (device->card_type) {
 	case NV_50:
 		timing_regs = 9;
 		break;
@@ -928,24 +505,24 @@ nouveau_mem_timing_read(struct drm_device *dev, struct nouveau_pm_memtiming *t)
 		return;
 	}
 	for(i = 0; i < timing_regs; i++)
-		t->reg[i] = nv_rd32(dev, timing_base + (0x04 * i));
+		t->reg[i] = nv_rd32(device, timing_base + (0x04 * i));
 
 	t->tCWL = 0;
-	if (dev_priv->card_type < NV_C0) {
-		t->tCWL = ((nv_rd32(dev, 0x100228) & 0x0f000000) >> 24) + 1;
-	} else if (dev_priv->card_type <= NV_D0) {
-		t->tCWL = ((nv_rd32(dev, 0x10f294) & 0x00000f80) >> 7);
+	if (device->card_type < NV_C0) {
+		t->tCWL = ((nv_rd32(device, 0x100228) & 0x0f000000) >> 24) + 1;
+	} else if (device->card_type <= NV_D0) {
+		t->tCWL = ((nv_rd32(device, 0x10f294) & 0x00000f80) >> 7);
 	}
 
-	t->mr[0] = nv_rd32(dev, mr_base);
-	t->mr[1] = nv_rd32(dev, mr_base + 0x04);
-	t->mr[2] = nv_rd32(dev, mr_base + 0x20);
-	t->mr[3] = nv_rd32(dev, mr_base + 0x24);
+	t->mr[0] = nv_rd32(device, mr_base);
+	t->mr[1] = nv_rd32(device, mr_base + 0x04);
+	t->mr[2] = nv_rd32(device, mr_base + 0x20);
+	t->mr[3] = nv_rd32(device, mr_base + 0x24);
 
 	t->odt = 0;
 	t->drive_strength = 0;
 
-	switch (dev_priv->vram_type) {
+	switch (pfb->ram.type) {
 	case NV_MEM_TYPE_DDR3:
 		t->odt |= (t->mr[1] & 0x200) >> 7;
 	case NV_MEM_TYPE_DDR2:
@@ -966,13 +543,15 @@ int
 nouveau_mem_exec(struct nouveau_mem_exec_func *exec,
 		 struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = exec->dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(exec->dev);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
 	struct nouveau_pm_memtiming *info = &perflvl->timing;
 	u32 tMRD = 1000, tCKSRE = 0, tCKSRX = 0, tXS = 0, tDLLK = 0;
 	u32 mr[3] = { info->mr[0], info->mr[1], info->mr[2] };
 	u32 mr1_dlloff;
 
-	switch (dev_priv->vram_type) {
+	switch (pfb->ram.type) {
 	case NV_MEM_TYPE_DDR2:
 		tDLLK = 2000;
 		mr1_dlloff = 0x00000001;
@@ -988,12 +567,12 @@ nouveau_mem_exec(struct nouveau_mem_exec_func *exec,
 		mr1_dlloff = 0x00000040;
 		break;
 	default:
-		NV_ERROR(exec->dev, "cannot reclock unsupported memtype\n");
+		NV_ERROR(drm, "cannot reclock unsupported memtype\n");
 		return -ENODEV;
 	}
 
 	/* fetch current MRs */
-	switch (dev_priv->vram_type) {
+	switch (pfb->ram.type) {
 	case NV_MEM_TYPE_GDDR3:
 	case NV_MEM_TYPE_DDR3:
 		mr[2] = exec->mrg(exec, 2);
@@ -1060,194 +639,9 @@ nouveau_mem_exec(struct nouveau_mem_exec_func *exec,
 		exec->mrs (exec, 0, info->mr[0] | 0x00000000);
 		exec->wait(exec, tMRD);
 		exec->wait(exec, tDLLK);
-		if (dev_priv->vram_type == NV_MEM_TYPE_GDDR3)
+		if (pfb->ram.type == NV_MEM_TYPE_GDDR3)
 			exec->precharge(exec);
 	}
 
 	return 0;
 }
-
-int
-nouveau_mem_vbios_type(struct drm_device *dev)
-{
-	struct bit_entry M;
-	u8 ramcfg = (nv_rd32(dev, 0x101000) & 0x0000003c) >> 2;
-	if (!bit_table(dev, 'M', &M) || M.version != 2 || M.length < 5) {
-		u8 *table = ROMPTR(dev, M.data[3]);
-		if (table && table[0] == 0x10 && ramcfg < table[3]) {
-			u8 *entry = table + table[1] + (ramcfg * table[2]);
-			switch (entry[0] & 0x0f) {
-			case 0: return NV_MEM_TYPE_DDR2;
-			case 1: return NV_MEM_TYPE_DDR3;
-			case 2: return NV_MEM_TYPE_GDDR3;
-			case 3: return NV_MEM_TYPE_GDDR5;
-			default:
-				break;
-			}
-
-		}
-	}
-	return NV_MEM_TYPE_UNKNOWN;
-}
-
-static int
-nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
-{
-	/* nothing to do */
-	return 0;
-}
-
-static int
-nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
-{
-	/* nothing to do */
-	return 0;
-}
-
-static inline void
-nouveau_mem_node_cleanup(struct nouveau_mem *node)
-{
-	if (node->vma[0].node) {
-		nouveau_vm_unmap(&node->vma[0]);
-		nouveau_vm_put(&node->vma[0]);
-	}
-
-	if (node->vma[1].node) {
-		nouveau_vm_unmap(&node->vma[1]);
-		nouveau_vm_put(&node->vma[1]);
-	}
-}
-
-static void
-nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
-			 struct ttm_mem_reg *mem)
-{
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	struct drm_device *dev = dev_priv->dev;
-
-	nouveau_mem_node_cleanup(mem->mm_node);
-	vram->put(dev, (struct nouveau_mem **)&mem->mm_node);
-}
-
-static int
-nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
-			 struct ttm_buffer_object *bo,
-			 struct ttm_placement *placement,
-			 struct ttm_mem_reg *mem)
-{
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(man->bdev);
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	struct drm_device *dev = dev_priv->dev;
-	struct nouveau_bo *nvbo = nouveau_bo(bo);
-	struct nouveau_mem *node;
-	u32 size_nc = 0;
-	int ret;
-
-	if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
-		size_nc = 1 << nvbo->page_shift;
-
-	ret = vram->get(dev, mem->num_pages << PAGE_SHIFT,
-			mem->page_alignment << PAGE_SHIFT, size_nc,
-			(nvbo->tile_flags >> 8) & 0x3ff, &node);
-	if (ret) {
-		mem->mm_node = NULL;
-		return (ret == -ENOSPC) ? 0 : ret;
-	}
-
-	node->page_shift = nvbo->page_shift;
-
-	mem->mm_node = node;
-	mem->start   = node->offset >> PAGE_SHIFT;
-	return 0;
-}
-
-void
-nouveau_vram_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
-{
-	struct nouveau_mm *mm = man->priv;
-	struct nouveau_mm_node *r;
-	u32 total = 0, free = 0;
-
-	mutex_lock(&mm->mutex);
-	list_for_each_entry(r, &mm->nodes, nl_entry) {
-		printk(KERN_DEBUG "%s %d: 0x%010llx 0x%010llx\n",
-		       prefix, r->type, ((u64)r->offset << 12),
-		       (((u64)r->offset + r->length) << 12));
-
-		total += r->length;
-		if (!r->type)
-			free += r->length;
-	}
-	mutex_unlock(&mm->mutex);
-
-	printk(KERN_DEBUG "%s  total: 0x%010llx free: 0x%010llx\n",
-	       prefix, (u64)total << 12, (u64)free << 12);
-	printk(KERN_DEBUG "%s  block: 0x%08x\n",
-	       prefix, mm->block_size << 12);
-}
-
-const struct ttm_mem_type_manager_func nouveau_vram_manager = {
-	nouveau_vram_manager_init,
-	nouveau_vram_manager_fini,
-	nouveau_vram_manager_new,
-	nouveau_vram_manager_del,
-	nouveau_vram_manager_debug
-};
-
-static int
-nouveau_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
-{
-	return 0;
-}
-
-static int
-nouveau_gart_manager_fini(struct ttm_mem_type_manager *man)
-{
-	return 0;
-}
-
-static void
-nouveau_gart_manager_del(struct ttm_mem_type_manager *man,
-			 struct ttm_mem_reg *mem)
-{
-	nouveau_mem_node_cleanup(mem->mm_node);
-	kfree(mem->mm_node);
-	mem->mm_node = NULL;
-}
-
-static int
-nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
-			 struct ttm_buffer_object *bo,
-			 struct ttm_placement *placement,
-			 struct ttm_mem_reg *mem)
-{
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
-	struct nouveau_mem *node;
-
-	if (unlikely((mem->num_pages << PAGE_SHIFT) >=
-		     dev_priv->gart_info.aper_size))
-		return -ENOMEM;
-
-	node = kzalloc(sizeof(*node), GFP_KERNEL);
-	if (!node)
-		return -ENOMEM;
-	node->page_shift = 12;
-
-	mem->mm_node = node;
-	mem->start   = 0;
-	return 0;
-}
-
-void
-nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
-{
-}
-
-const struct ttm_mem_type_manager_func nouveau_gart_manager = {
-	nouveau_gart_manager_init,
-	nouveau_gart_manager_fini,
-	nouveau_gart_manager_new,
-	nouveau_gart_manager_del,
-	nouveau_gart_manager_debug
-};
diff --git a/drivers/gpu/drm/nouveau/nouveau_mm.h b/drivers/gpu/drm/nouveau/nouveau_mm.h
deleted file mode 100644
index 57a600c35c9..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_mm.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#ifndef __NOUVEAU_REGION_H__
-#define __NOUVEAU_REGION_H__
-
-struct nouveau_mm_node {
-	struct list_head nl_entry;
-	struct list_head fl_entry;
-	struct list_head rl_entry;
-
-	u8  type;
-	u32 offset;
-	u32 length;
-};
-
-struct nouveau_mm {
-	struct list_head nodes;
-	struct list_head free;
-
-	struct mutex mutex;
-
-	u32 block_size;
-	int heap_nodes;
-};
-
-int  nouveau_mm_init(struct nouveau_mm *, u32 offset, u32 length, u32 block);
-int  nouveau_mm_fini(struct nouveau_mm *);
-int  nouveau_mm_pre(struct nouveau_mm *);
-int  nouveau_mm_get(struct nouveau_mm *, int type, u32 size, u32 size_nc,
-		    u32 align, struct nouveau_mm_node **);
-void nouveau_mm_put(struct nouveau_mm *, struct nouveau_mm_node *);
-
-int  nv50_vram_init(struct drm_device *);
-void nv50_vram_fini(struct drm_device *);
-int  nv50_vram_new(struct drm_device *, u64 size, u32 align, u32 size_nc,
-		    u32 memtype, struct nouveau_mem **);
-void nv50_vram_del(struct drm_device *, struct nouveau_mem **);
-bool nv50_vram_flags_valid(struct drm_device *, u32 tile_flags);
-
-int  nvc0_vram_init(struct drm_device *);
-int  nvc0_vram_new(struct drm_device *, u64 size, u32 align, u32 ncmin,
-		    u32 memtype, struct nouveau_mem **);
-bool nvc0_vram_flags_valid(struct drm_device *, u32 tile_flags);
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_mxm.c b/drivers/gpu/drm/nouveau/nouveau_mxm.c
deleted file mode 100644
index 07d0d1e0369..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_mxm.c
+++ /dev/null
@@ -1,723 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/acpi.h>
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-
-#define MXM_DBG(dev, fmt, args...) NV_DEBUG((dev), "MXM: " fmt, ##args)
-#define MXM_MSG(dev, fmt, args...) NV_INFO((dev), "MXM: " fmt, ##args)
-
-static u8 *
-mxms_data(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return dev_priv->mxms;
-
-}
-
-static u16
-mxms_version(struct drm_device *dev)
-{
-	u8 *mxms = mxms_data(dev);
-	u16 version = (mxms[4] << 8) | mxms[5];
-	switch (version ) {
-	case 0x0200:
-	case 0x0201:
-	case 0x0300:
-		return version;
-	default:
-		break;
-	}
-
-	MXM_DBG(dev, "unknown version %d.%d\n", mxms[4], mxms[5]);
-	return 0x0000;
-}
-
-static u16
-mxms_headerlen(struct drm_device *dev)
-{
-	return 8;
-}
-
-static u16
-mxms_structlen(struct drm_device *dev)
-{
-	return *(u16 *)&mxms_data(dev)[6];
-}
-
-static bool
-mxms_checksum(struct drm_device *dev)
-{
-	u16 size = mxms_headerlen(dev) + mxms_structlen(dev);
-	u8 *mxms = mxms_data(dev), sum = 0;
-	while (size--)
-		sum += *mxms++;
-	if (sum) {
-		MXM_DBG(dev, "checksum invalid\n");
-		return false;
-	}
-	return true;
-}
-
-static bool
-mxms_valid(struct drm_device *dev)
-{
-	u8 *mxms = mxms_data(dev);
-	if (*(u32 *)mxms != 0x5f4d584d) {
-		MXM_DBG(dev, "signature invalid\n");
-		return false;
-	}
-
-	if (!mxms_version(dev) || !mxms_checksum(dev))
-		return false;
-
-	return true;
-}
-
-static bool
-mxms_foreach(struct drm_device *dev, u8 types,
-	     bool (*exec)(struct drm_device *, u8 *, void *), void *info)
-{
-	u8 *mxms = mxms_data(dev);
-	u8 *desc = mxms + mxms_headerlen(dev);
-	u8 *fini = desc + mxms_structlen(dev) - 1;
-	while (desc < fini) {
-		u8 type = desc[0] & 0x0f;
-		u8 headerlen = 0;
-		u8 recordlen = 0;
-		u8 entries = 0;
-
-		switch (type) {
-		case 0: /* Output Device Structure */
-			if (mxms_version(dev) >= 0x0300)
-				headerlen = 8;
-			else
-				headerlen = 6;
-			break;
-		case 1: /* System Cooling Capability Structure */
-		case 2: /* Thermal Structure */
-		case 3: /* Input Power Structure */
-			headerlen = 4;
-			break;
-		case 4: /* GPIO Device Structure */
-			headerlen = 4;
-			recordlen = 2;
-			entries   = (ROM32(desc[0]) & 0x01f00000) >> 20;
-			break;
-		case 5: /* Vendor Specific Structure */
-			headerlen = 8;
-			break;
-		case 6: /* Backlight Control Structure */
-			if (mxms_version(dev) >= 0x0300) {
-				headerlen = 4;
-				recordlen = 8;
-				entries   = (desc[1] & 0xf0) >> 4;
-			} else {
-				headerlen = 8;
-			}
-			break;
-		case 7: /* Fan Control Structure */
-			headerlen = 8;
-			recordlen = 4;
-			entries   = desc[1] & 0x07;
-			break;
-		default:
-			MXM_DBG(dev, "unknown descriptor type %d\n", type);
-			return false;
-		}
-
-		if ((drm_debug & DRM_UT_DRIVER) && (exec == NULL)) {
-			static const char * mxms_desc_name[] = {
-				"ODS", "SCCS", "TS", "IPS",
-				"GSD", "VSS", "BCS", "FCS",
-			};
-			u8 *dump = desc;
-			int i, j;
-
-			MXM_DBG(dev, "%4s: ", mxms_desc_name[type]);
-			for (j = headerlen - 1; j >= 0; j--)
-				printk("%02x", dump[j]);
-			printk("\n");
-			dump += headerlen;
-
-			for (i = 0; i < entries; i++, dump += recordlen) {
-				MXM_DBG(dev, "      ");
-				for (j = recordlen - 1; j >= 0; j--)
-					printk("%02x", dump[j]);
-				printk("\n");
-			}
-		}
-
-		if (types & (1 << type)) {
-			if (!exec(dev, desc, info))
-				return false;
-		}
-
-		desc += headerlen + (entries * recordlen);
-	}
-
-	return true;
-}
-
-static u8 *
-mxm_table(struct drm_device *dev, u8 *size)
-{
-	struct bit_entry x;
-
-	if (bit_table(dev, 'x', &x)) {
-		MXM_DBG(dev, "BIT 'x' table not present\n");
-		return NULL;
-	}
-
-	if (x.version != 1 || x.length < 3) {
-		MXM_MSG(dev, "BIT x table %d/%d unknown\n",
-			x.version, x.length);
-		return NULL;
-	}
-
-	*size = x.length;
-	return x.data;
-}
-
-/* These map MXM v2.x digital connection values to the appropriate SOR/link,
- * hopefully they're correct for all boards within the same chipset...
- *
- * MXM v3.x VBIOS are nicer and provide pointers to these tables.
- */
-static u8 nv84_sor_map[16] = {
-	0x00, 0x12, 0x22, 0x11, 0x32, 0x31, 0x11, 0x31,
-	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
-};
-
-static u8 nv92_sor_map[16] = {
-	0x00, 0x12, 0x22, 0x11, 0x32, 0x31, 0x11, 0x31,
-	0x11, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
-};
-
-static u8 nv94_sor_map[16] = {
-	0x00, 0x14, 0x24, 0x11, 0x34, 0x31, 0x11, 0x31,
-	0x11, 0x31, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00
-};
-
-static u8 nv96_sor_map[16] = {
-	0x00, 0x14, 0x24, 0x00, 0x34, 0x00, 0x11, 0x31,
-	0x11, 0x31, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00
-};
-
-static u8 nv98_sor_map[16] = {
-	0x00, 0x14, 0x12, 0x11, 0x00, 0x31, 0x11, 0x31,
-	0x11, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
-};
-
-static u8
-mxm_sor_map(struct drm_device *dev, u8 conn)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u8 len, *mxm = mxm_table(dev, &len);
-	if (mxm && len >= 6) {
-		u8 *map = ROMPTR(dev, mxm[4]);
-		if (map) {
-			if (map[0] == 0x10) {
-				if (conn < map[3])
-					return map[map[1] + conn];
-				return 0x00;
-			}
-
-			MXM_MSG(dev, "unknown sor map 0x%02x\n", map[0]);
-		}
-	}
-
-	if (dev_priv->chipset == 0x84 || dev_priv->chipset == 0x86)
-		return nv84_sor_map[conn];
-	if (dev_priv->chipset == 0x92)
-		return nv92_sor_map[conn];
-	if (dev_priv->chipset == 0x94)
-		return nv94_sor_map[conn];
-	if (dev_priv->chipset == 0x96)
-		return nv96_sor_map[conn];
-	if (dev_priv->chipset == 0x98)
-		return nv98_sor_map[conn];
-
-	MXM_MSG(dev, "missing sor map\n");
-	return 0x00;
-}
-
-static u8
-mxm_ddc_map(struct drm_device *dev, u8 port)
-{
-	u8 len, *mxm = mxm_table(dev, &len);
-	if (mxm && len >= 8) {
-		u8 *map = ROMPTR(dev, mxm[6]);
-		if (map) {
-			if (map[0] == 0x10) {
-				if (port < map[3])
-					return map[map[1] + port];
-				return 0x00;
-			}
-
-			MXM_MSG(dev, "unknown ddc map 0x%02x\n", map[0]);
-		}
-	}
-
-	/* v2.x: directly write port as dcb i2cidx */
-	return (port << 4) | port;
-}
-
-struct mxms_odev {
-	u8 outp_type;
-	u8 conn_type;
-	u8 ddc_port;
-	u8 dig_conn;
-};
-
-static void
-mxms_output_device(struct drm_device *dev, u8 *pdata, struct mxms_odev *desc)
-{
-	u64 data = ROM32(pdata[0]);
-	if (mxms_version(dev) >= 0x0300)
-		data |= (u64)ROM16(pdata[4]) << 32;
-
-	desc->outp_type = (data & 0x00000000000000f0ULL) >> 4;
-	desc->ddc_port  = (data & 0x0000000000000f00ULL) >> 8;
-	desc->conn_type = (data & 0x000000000001f000ULL) >> 12;
-	desc->dig_conn  = (data & 0x0000000000780000ULL) >> 19;
-}
-
-struct context {
-	u32 *outp;
-	struct mxms_odev desc;
-};
-
-static bool
-mxm_match_tmds_partner(struct drm_device *dev, u8 *data, void *info)
-{
-	struct context *ctx = info;
-	struct mxms_odev desc;
-
-	mxms_output_device(dev, data, &desc);
-	if (desc.outp_type == 2 &&
-	    desc.dig_conn == ctx->desc.dig_conn)
-		return false;
-	return true;
-}
-
-static bool
-mxm_match_dcb(struct drm_device *dev, u8 *data, void *info)
-{
-	struct context *ctx = info;
-	u64 desc = *(u64 *)data;
-
-	mxms_output_device(dev, data, &ctx->desc);
-
-	/* match dcb encoder type to mxm-ods device type */
-	if ((ctx->outp[0] & 0x0000000f) != ctx->desc.outp_type)
-		return true;
-
-	/* digital output, have some extra stuff to match here, there's a
-	 * table in the vbios that provides a mapping from the mxm digital
-	 * connection enum values to SOR/link
-	 */
-	if ((desc & 0x00000000000000f0) >= 0x20) {
-		/* check against sor index */
-		u8 link = mxm_sor_map(dev, ctx->desc.dig_conn);
-		if ((ctx->outp[0] & 0x0f000000) != (link & 0x0f) << 24)
-			return true;
-
-		/* check dcb entry has a compatible link field */
-		link = (link & 0x30) >> 4;
-		if ((link & ((ctx->outp[1] & 0x00000030) >> 4)) != link)
-			return true;
-	}
-
-	/* mark this descriptor accounted for by setting invalid device type,
-	 * except of course some manufactures don't follow specs properly and
-	 * we need to avoid killing off the TMDS function on DP connectors
-	 * if MXM-SIS is missing an entry for it.
-	 */
-	data[0] &= ~0xf0;
-	if (ctx->desc.outp_type == 6 && ctx->desc.conn_type == 6 &&
-	    mxms_foreach(dev, 0x01, mxm_match_tmds_partner, ctx)) {
-		data[0] |= 0x20; /* modify descriptor to match TMDS now */
-	} else {
-		data[0] |= 0xf0;
-	}
-
-	return false;
-}
-
-static int
-mxm_dcb_sanitise_entry(struct drm_device *dev, void *data, int idx, u8 *dcbe)
-{
-	struct context ctx = { .outp = (u32 *)dcbe };
-	u8 type, i2cidx, link;
-	u8 *conn;
-
-	/* look for an output device structure that matches this dcb entry.
-	 * if one isn't found, disable it.
-	 */
-	if (mxms_foreach(dev, 0x01, mxm_match_dcb, &ctx)) {
-		MXM_DBG(dev, "disable %d: 0x%08x 0x%08x\n",
-			idx, ctx.outp[0], ctx.outp[1]);
-		ctx.outp[0] |= 0x0000000f;
-		return 0;
-	}
-
-	/* modify the output's ddc/aux port, there's a pointer to a table
-	 * with the mapping from mxm ddc/aux port to dcb i2c_index in the
-	 * vbios mxm table
-	 */
-	i2cidx = mxm_ddc_map(dev, ctx.desc.ddc_port);
-	if ((ctx.outp[0] & 0x0000000f) != OUTPUT_DP)
-		i2cidx = (i2cidx & 0x0f) << 4;
-	else
-		i2cidx = (i2cidx & 0xf0);
-
-	if (i2cidx != 0xf0) {
-		ctx.outp[0] &= ~0x000000f0;
-		ctx.outp[0] |= i2cidx;
-	}
-
-	/* override dcb sorconf.link, based on what mxm data says */
-	switch (ctx.desc.outp_type) {
-	case 0x00: /* Analog CRT */
-	case 0x01: /* Analog TV/HDTV */
-		break;
-	default:
-		link = mxm_sor_map(dev, ctx.desc.dig_conn) & 0x30;
-		ctx.outp[1] &= ~0x00000030;
-		ctx.outp[1] |= link;
-		break;
-	}
-
-	/* we may need to fixup various other vbios tables based on what
-	 * the descriptor says the connector type should be.
-	 *
-	 * in a lot of cases, the vbios tables will claim DVI-I is possible,
-	 * and the mxm data says the connector is really HDMI.  another
-	 * common example is DP->eDP.
-	 */
-	conn = dcb_conn(dev, (ctx.outp[0] & 0x0000f000) >> 12);
-	type = conn[0];
-	switch (ctx.desc.conn_type) {
-	case 0x01: /* LVDS */
-		ctx.outp[1] |= 0x00000004; /* use_power_scripts */
-		/* XXX: modify default link width in LVDS table */
-		break;
-	case 0x02: /* HDMI */
-		type = DCB_CONNECTOR_HDMI_1;
-		break;
-	case 0x03: /* DVI-D */
-		type = DCB_CONNECTOR_DVI_D;
-		break;
-	case 0x0e: /* eDP, falls through to DPint */
-		ctx.outp[1] |= 0x00010000;
-	case 0x07: /* DP internal, wtf is this?? HP8670w */
-		ctx.outp[1] |= 0x00000004; /* use_power_scripts? */
-		type = DCB_CONNECTOR_eDP;
-		break;
-	default:
-		break;
-	}
-
-	if (mxms_version(dev) >= 0x0300)
-		conn[0] = type;
-
-	return 0;
-}
-
-static bool
-mxm_show_unmatched(struct drm_device *dev, u8 *data, void *info)
-{
-	u64 desc = *(u64 *)data;
-	if ((desc & 0xf0) != 0xf0)
-		MXM_MSG(dev, "unmatched output device 0x%016llx\n", desc);
-	return true;
-}
-
-static void
-mxm_dcb_sanitise(struct drm_device *dev)
-{
-	u8 *dcb = dcb_table(dev);
-	if (!dcb || dcb[0] != 0x40) {
-		MXM_DBG(dev, "unsupported DCB version\n");
-		return;
-	}
-
-	dcb_outp_foreach(dev, NULL, mxm_dcb_sanitise_entry);
-	mxms_foreach(dev, 0x01, mxm_show_unmatched, NULL);
-}
-
-static bool
-mxm_shadow_rom_fetch(struct nouveau_i2c_chan *i2c, u8 addr,
-		     u8 offset, u8 size, u8 *data)
-{
-	struct i2c_msg msgs[] = {
-		{ .addr = addr, .flags = 0, .len = 1, .buf = &offset },
-		{ .addr = addr, .flags = I2C_M_RD, .len = size, .buf = data, },
-	};
-
-	return i2c_transfer(&i2c->adapter, msgs, 2) == 2;
-}
-
-static bool
-mxm_shadow_rom(struct drm_device *dev, u8 version)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_i2c_chan *i2c = NULL;
-	u8 i2cidx, mxms[6], addr, size;
-
-	i2cidx = mxm_ddc_map(dev, 1 /* LVDS_DDC */) & 0x0f;
-	if (i2cidx < 0x0f)
-		i2c = nouveau_i2c_find(dev, i2cidx);
-	if (!i2c)
-		return false;
-
-	addr = 0x54;
-	if (!mxm_shadow_rom_fetch(i2c, addr, 0, 6, mxms)) {
-		addr = 0x56;
-		if (!mxm_shadow_rom_fetch(i2c, addr, 0, 6, mxms))
-			return false;
-	}
-
-	dev_priv->mxms = mxms;
-	size = mxms_headerlen(dev) + mxms_structlen(dev);
-	dev_priv->mxms = kmalloc(size, GFP_KERNEL);
-
-	if (dev_priv->mxms &&
-	    mxm_shadow_rom_fetch(i2c, addr, 0, size, dev_priv->mxms))
-		return true;
-
-	kfree(dev_priv->mxms);
-	dev_priv->mxms = NULL;
-	return false;
-}
-
-#if defined(CONFIG_ACPI)
-static bool
-mxm_shadow_dsm(struct drm_device *dev, u8 version)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	static char muid[] = {
-		0x00, 0xA4, 0x04, 0x40, 0x7D, 0x91, 0xF2, 0x4C,
-		0xB8, 0x9C, 0x79, 0xB6, 0x2F, 0xD5, 0x56, 0x65
-	};
-	u32 mxms_args[] = { 0x00000000 };
-	union acpi_object args[4] = {
-		/* _DSM MUID */
-		{ .buffer.type = 3,
-		  .buffer.length = sizeof(muid),
-		  .buffer.pointer = muid,
-		},
-		/* spec says this can be zero to mean "highest revision", but
-		 * of course there's at least one bios out there which fails
-		 * unless you pass in exactly the version it supports..
-		 */
-		{ .integer.type = ACPI_TYPE_INTEGER,
-		  .integer.value = (version & 0xf0) << 4 | (version & 0x0f),
-		},
-		/* MXMS function */
-		{ .integer.type = ACPI_TYPE_INTEGER,
-		  .integer.value = 0x00000010,
-		},
-		/* Pointer to MXMS arguments */
-		{ .buffer.type = ACPI_TYPE_BUFFER,
-		  .buffer.length = sizeof(mxms_args),
-		  .buffer.pointer = (char *)mxms_args,
-		},
-	};
-	struct acpi_object_list list = { ARRAY_SIZE(args), args };
-	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
-	acpi_handle handle;
-	int ret;
-
-	handle = DEVICE_ACPI_HANDLE(&dev->pdev->dev);
-	if (!handle)
-		return false;
-
-	ret = acpi_evaluate_object(handle, "_DSM", &list, &retn);
-	if (ret) {
-		MXM_DBG(dev, "DSM MXMS failed: %d\n", ret);
-		return false;
-	}
-
-	obj = retn.pointer;
-	if (obj->type == ACPI_TYPE_BUFFER) {
-		dev_priv->mxms = kmemdup(obj->buffer.pointer,
-					 obj->buffer.length, GFP_KERNEL);
-	} else
-	if (obj->type == ACPI_TYPE_INTEGER) {
-		MXM_DBG(dev, "DSM MXMS returned 0x%llx\n", obj->integer.value);
-	}
-
-	kfree(obj);
-	return dev_priv->mxms != NULL;
-}
-#endif
-
-#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
-
-#define WMI_WMMX_GUID "F6CB5C3C-9CAE-4EBD-B577-931EA32A2CC0"
-
-static u8
-wmi_wmmx_mxmi(struct drm_device *dev, u8 version)
-{
-	u32 mxmi_args[] = { 0x494D584D /* MXMI */, version, 0 };
-	struct acpi_buffer args = { sizeof(mxmi_args), mxmi_args };
-	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
-	acpi_status status;
-
-	status = wmi_evaluate_method(WMI_WMMX_GUID, 0, 0, &args, &retn);
-	if (ACPI_FAILURE(status)) {
-		MXM_DBG(dev, "WMMX MXMI returned %d\n", status);
-		return 0x00;
-	}
-
-	obj = retn.pointer;
-	if (obj->type == ACPI_TYPE_INTEGER) {
-		version = obj->integer.value;
-		MXM_DBG(dev, "WMMX MXMI version %d.%d\n",
-			     (version >> 4), version & 0x0f);
-	} else {
-		version = 0;
-		MXM_DBG(dev, "WMMX MXMI returned non-integer\n");
-	}
-
-	kfree(obj);
-	return version;
-}
-
-static bool
-mxm_shadow_wmi(struct drm_device *dev, u8 version)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 mxms_args[] = { 0x534D584D /* MXMS */, version, 0 };
-	struct acpi_buffer args = { sizeof(mxms_args), mxms_args };
-	struct acpi_buffer retn = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
-	acpi_status status;
-
-	if (!wmi_has_guid(WMI_WMMX_GUID)) {
-		MXM_DBG(dev, "WMMX GUID not found\n");
-		return false;
-	}
-
-	mxms_args[1] = wmi_wmmx_mxmi(dev, 0x00);
-	if (!mxms_args[1])
-		mxms_args[1] = wmi_wmmx_mxmi(dev, version);
-	if (!mxms_args[1])
-		return false;
-
-	status = wmi_evaluate_method(WMI_WMMX_GUID, 0, 0, &args, &retn);
-	if (ACPI_FAILURE(status)) {
-		MXM_DBG(dev, "WMMX MXMS returned %d\n", status);
-		return false;
-	}
-
-	obj = retn.pointer;
-	if (obj->type == ACPI_TYPE_BUFFER) {
-		dev_priv->mxms = kmemdup(obj->buffer.pointer,
-					 obj->buffer.length, GFP_KERNEL);
-	}
-
-	kfree(obj);
-	return dev_priv->mxms != NULL;
-}
-#endif
-
-struct mxm_shadow_h {
-	const char *name;
-	bool (*exec)(struct drm_device *, u8 version);
-} _mxm_shadow[] = {
-	{ "ROM", mxm_shadow_rom },
-#if defined(CONFIG_ACPI)
-	{ "DSM", mxm_shadow_dsm },
-#endif
-#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE)
-	{ "WMI", mxm_shadow_wmi },
-#endif
-	{}
-};
-
-static int
-mxm_shadow(struct drm_device *dev, u8 version)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct mxm_shadow_h *shadow = _mxm_shadow;
-	do {
-		MXM_DBG(dev, "checking %s\n", shadow->name);
-		if (shadow->exec(dev, version)) {
-			if (mxms_valid(dev))
-				return 0;
-			kfree(dev_priv->mxms);
-			dev_priv->mxms = NULL;
-		}
-	} while ((++shadow)->name);
-	return -ENOENT;
-}
-
-int
-nouveau_mxm_init(struct drm_device *dev)
-{
-	u8 mxm_size, *mxm = mxm_table(dev, &mxm_size);
-	if (!mxm || !mxm[0]) {
-		MXM_MSG(dev, "no VBIOS data, nothing to do\n");
-		return 0;
-	}
-
-	MXM_MSG(dev, "BIOS version %d.%d\n", mxm[0] >> 4, mxm[0] & 0x0f);
-
-	if (mxm_shadow(dev, mxm[0])) {
-		MXM_MSG(dev, "failed to locate valid SIS\n");
-#if 0
-		/* we should, perhaps, fall back to some kind of limited
-		 * mode here if the x86 vbios hasn't already done the
-		 * work for us (so we prevent loading with completely
-		 * whacked vbios tables).
-		 */
-		return -EINVAL;
-#else
-		return 0;
-#endif
-	}
-
-	MXM_MSG(dev, "MXMS Version %d.%d\n",
-		mxms_version(dev) >> 8, mxms_version(dev) & 0xff);
-	mxms_foreach(dev, 0, NULL, NULL);
-
-	if (nouveau_mxmdcb)
-		mxm_dcb_sanitise(dev);
-	return 0;
-}
-
-void
-nouveau_mxm_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	kfree(dev_priv->mxms);
-	dev_priv->mxms = NULL;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_notifier.c b/drivers/gpu/drm/nouveau/nouveau_notifier.c
deleted file mode 100644
index 69c93b86451..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_notifier.c
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Copyright (C) 2007 Ben Skeggs.
- *
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-
-int
-nouveau_notifier_init_channel(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_bo *ntfy = NULL;
-	uint32_t flags, ttmpl;
-	int ret;
-
-	if (nouveau_vram_notify) {
-		flags = NOUVEAU_GEM_DOMAIN_VRAM;
-		ttmpl = TTM_PL_FLAG_VRAM;
-	} else {
-		flags = NOUVEAU_GEM_DOMAIN_GART;
-		ttmpl = TTM_PL_FLAG_TT;
-	}
-
-	ret = nouveau_gem_new(dev, PAGE_SIZE, 0, flags, 0, 0, &ntfy);
-	if (ret)
-		return ret;
-
-	ret = nouveau_bo_pin(ntfy, ttmpl);
-	if (ret)
-		goto out_err;
-
-	ret = nouveau_bo_map(ntfy);
-	if (ret)
-		goto out_err;
-
-	if (dev_priv->card_type >= NV_50) {
-		ret = nouveau_bo_vma_add(ntfy, chan->vm, &chan->notifier_vma);
-		if (ret)
-			goto out_err;
-	}
-
-	ret = drm_mm_init(&chan->notifier_heap, 0, ntfy->bo.mem.size);
-	if (ret)
-		goto out_err;
-
-	chan->notifier_bo = ntfy;
-out_err:
-	if (ret) {
-		nouveau_bo_vma_del(ntfy, &chan->notifier_vma);
-		drm_gem_object_unreference_unlocked(ntfy->gem);
-	}
-
-	return ret;
-}
-
-void
-nouveau_notifier_takedown_channel(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-
-	if (!chan->notifier_bo)
-		return;
-
-	nouveau_bo_vma_del(chan->notifier_bo, &chan->notifier_vma);
-	nouveau_bo_unmap(chan->notifier_bo);
-	mutex_lock(&dev->struct_mutex);
-	nouveau_bo_unpin(chan->notifier_bo);
-	mutex_unlock(&dev->struct_mutex);
-	drm_gem_object_unreference_unlocked(chan->notifier_bo->gem);
-	drm_mm_takedown(&chan->notifier_heap);
-}
-
-static void
-nouveau_notifier_gpuobj_dtor(struct drm_device *dev,
-			     struct nouveau_gpuobj *gpuobj)
-{
-	NV_DEBUG(dev, "\n");
-
-	if (gpuobj->priv)
-		drm_mm_put_block(gpuobj->priv);
-}
-
-int
-nouveau_notifier_alloc(struct nouveau_channel *chan, uint32_t handle,
-		       int size, uint32_t start, uint32_t end,
-		       uint32_t *b_offset)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *nobj = NULL;
-	struct drm_mm_node *mem;
-	uint64_t offset;
-	int target, ret;
-
-	mem = drm_mm_search_free_in_range(&chan->notifier_heap, size, 0,
-					  start, end, 0);
-	if (mem)
-		mem = drm_mm_get_block_range(mem, size, 0, start, end);
-	if (!mem) {
-		NV_ERROR(dev, "Channel %d notifier block full\n", chan->id);
-		return -ENOMEM;
-	}
-
-	if (dev_priv->card_type < NV_50) {
-		if (chan->notifier_bo->bo.mem.mem_type == TTM_PL_VRAM)
-			target = NV_MEM_TARGET_VRAM;
-		else
-			target = NV_MEM_TARGET_GART;
-		offset  = chan->notifier_bo->bo.offset;
-	} else {
-		target = NV_MEM_TARGET_VM;
-		offset = chan->notifier_vma.offset;
-	}
-	offset += mem->start;
-
-	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY, offset,
-				     mem->size, NV_MEM_ACCESS_RW, target,
-				     &nobj);
-	if (ret) {
-		drm_mm_put_block(mem);
-		NV_ERROR(dev, "Error creating notifier ctxdma: %d\n", ret);
-		return ret;
-	}
-	nobj->dtor = nouveau_notifier_gpuobj_dtor;
-	nobj->priv = mem;
-
-	ret = nouveau_ramht_insert(chan, handle, nobj);
-	nouveau_gpuobj_ref(NULL, &nobj);
-	if (ret) {
-		drm_mm_put_block(mem);
-		NV_ERROR(dev, "Error adding notifier to ramht: %d\n", ret);
-		return ret;
-	}
-
-	*b_offset = mem->start;
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_perf.c b/drivers/gpu/drm/nouveau/nouveau_perf.c
index ea6acf1c4a7..4fe883c5491 100644
--- a/drivers/gpu/drm/nouveau/nouveau_perf.c
+++ b/drivers/gpu/drm/nouveau/nouveau_perf.c
@@ -22,16 +22,17 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_pm.h"
 
 static u8 *
 nouveau_perf_table(struct drm_device *dev, u8 *ver)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	struct bit_entry P;
 
 	if (!bit_table(dev, 'P', &P) && P.version && P.version <= 2) {
@@ -87,7 +88,7 @@ u8 *
 nouveau_perf_rammap(struct drm_device *dev, u32 freq,
 		    u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct bit_entry P;
 	u8 *perf, i = 0;
 
@@ -114,8 +115,8 @@ nouveau_perf_rammap(struct drm_device *dev, u32 freq,
 		return NULL;
 	}
 
-	if (dev_priv->chipset == 0x49 ||
-	    dev_priv->chipset == 0x4b)
+	if (nv_device(drm->device)->chipset == 0x49 ||
+	    nv_device(drm->device)->chipset == 0x4b)
 		freq /= 2;
 
 	while ((perf = nouveau_perf_entry(dev, i++, ver, hdr, cnt, len))) {
@@ -142,12 +143,13 @@ nouveau_perf_rammap(struct drm_device *dev, u32 freq,
 u8 *
 nouveau_perf_ramcfg(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	u8 strap, hdr, cnt;
 	u8 *rammap;
 
-	strap = (nv_rd32(dev, 0x101000) & 0x0000003c) >> 2;
+	strap = (nv_rd32(device, 0x101000) & 0x0000003c) >> 2;
 	if (bios->ram_restrict_tbl_ptr)
 		strap = bios->data[bios->ram_restrict_tbl_ptr + strap];
 
@@ -161,8 +163,8 @@ nouveau_perf_ramcfg(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
 u8 *
 nouveau_perf_timing(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
 	struct bit_entry P;
 	u8 *perf, *timing = NULL;
 	u8 i = 0, hdr, cnt;
@@ -202,20 +204,21 @@ nouveau_perf_timing(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
 static void
 legacy_perf_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nvbios *bios = &drm->vbios;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	char *perf, *entry, *bmp = &bios->data[bios->offset];
 	int headerlen, use_straps;
 
 	if (bmp[5] < 0x5 || bmp[6] < 0x14) {
-		NV_DEBUG(dev, "BMP version too old for perf\n");
+		NV_DEBUG(drm, "BMP version too old for perf\n");
 		return;
 	}
 
 	perf = ROMPTR(dev, bmp[0x73]);
 	if (!perf) {
-		NV_DEBUG(dev, "No memclock table pointer found.\n");
+		NV_DEBUG(drm, "No memclock table pointer found.\n");
 		return;
 	}
 
@@ -231,13 +234,13 @@ legacy_perf_init(struct drm_device *dev)
 		headerlen = (use_straps ? 8 : 2);
 		break;
 	default:
-		NV_WARN(dev, "Unknown memclock table version %x.\n", perf[0]);
+		NV_WARN(drm, "Unknown memclock table version %x.\n", perf[0]);
 		return;
 	}
 
 	entry = perf + headerlen;
 	if (use_straps)
-		entry += (nv_rd32(dev, NV_PEXTDEV_BOOT_0) & 0x3c) >> 1;
+		entry += (nv_rd32(device, NV_PEXTDEV_BOOT_0) & 0x3c) >> 1;
 
 	sprintf(pm->perflvl[0].name, "performance_level_0");
 	pm->perflvl[0].memory = ROM16(entry[0]) * 20;
@@ -247,7 +250,7 @@ legacy_perf_init(struct drm_device *dev)
 static void
 nouveau_perf_voltage(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct bit_entry P;
 	u8 *vmap;
 	int id;
@@ -258,7 +261,7 @@ nouveau_perf_voltage(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	/* boards using voltage table version <0x40 store the voltage
 	 * level directly in the perflvl entry as a multiple of 10mV
 	 */
-	if (dev_priv->engine.pm.voltage.version < 0x40) {
+	if (drm->pm->voltage.version < 0x40) {
 		perflvl->volt_min = id * 10000;
 		perflvl->volt_max = perflvl->volt_min;
 		return;
@@ -268,14 +271,14 @@ nouveau_perf_voltage(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	 * vbios table containing a min/max voltage value for the perflvl
 	 */
 	if (bit_table(dev, 'P', &P) || P.version != 2 || P.length < 34) {
-		NV_DEBUG(dev, "where's our volt map table ptr? %d %d\n",
+		NV_DEBUG(drm, "where's our volt map table ptr? %d %d\n",
 			 P.version, P.length);
 		return;
 	}
 
 	vmap = ROMPTR(dev, P.data[32]);
 	if (!vmap) {
-		NV_DEBUG(dev, "volt map table pointer invalid\n");
+		NV_DEBUG(drm, "volt map table pointer invalid\n");
 		return;
 	}
 
@@ -289,9 +292,9 @@ nouveau_perf_voltage(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 void
 nouveau_perf_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_pm *pm = nouveau_pm(dev);
+	struct nvbios *bios = &drm->vbios;
 	u8 *perf, ver, hdr, cnt, len;
 	int ret, vid, i = -1;
 
@@ -301,8 +304,6 @@ nouveau_perf_init(struct drm_device *dev)
 	}
 
 	perf = nouveau_perf_table(dev, &ver);
-	if (ver >= 0x20 && ver < 0x40)
-		pm->fan.pwm_divisor = ROM16(perf[6]);
 
 	while ((perf = nouveau_perf_entry(dev, ++i, &ver, &hdr, &cnt, &len))) {
 		struct nouveau_pm_level *perflvl = &pm->perflvl[pm->nr_perflvl];
@@ -328,8 +329,8 @@ nouveau_perf_init(struct drm_device *dev)
 			perflvl->shader = ROM16(perf[6]) * 1000;
 			perflvl->core = perflvl->shader;
 			perflvl->core += (signed char)perf[8] * 1000;
-			if (dev_priv->chipset == 0x49 ||
-			    dev_priv->chipset == 0x4b)
+			if (nv_device(drm->device)->chipset == 0x49 ||
+			    nv_device(drm->device)->chipset == 0x4b)
 				perflvl->memory = ROM16(perf[11]) * 1000;
 			else
 				perflvl->memory = ROM16(perf[11]) * 2000;
@@ -356,7 +357,7 @@ nouveau_perf_init(struct drm_device *dev)
 #define subent(n) ((ROM16(perf[hdr + (n) * len]) & 0xfff) * 1000)
 			perflvl->fanspeed = 0; /*XXX*/
 			perflvl->volt_min = perf[2];
-			if (dev_priv->card_type == NV_50) {
+			if (nv_device(drm->device)->card_type == NV_50) {
 				perflvl->core   = subent(0);
 				perflvl->shader = subent(1);
 				perflvl->memory = subent(2);
@@ -382,7 +383,7 @@ nouveau_perf_init(struct drm_device *dev)
 		if (pm->voltage.supported && perflvl->volt_min) {
 			vid = nouveau_volt_vid_lookup(dev, perflvl->volt_min);
 			if (vid < 0) {
-				NV_DEBUG(dev, "perflvl %d, bad vid\n", i);
+				NV_DEBUG(drm, "perflvl %d, bad vid\n", i);
 				continue;
 			}
 		}
@@ -391,7 +392,7 @@ nouveau_perf_init(struct drm_device *dev)
 		ret = nouveau_mem_timing_calc(dev, perflvl->memory,
 					          &perflvl->timing);
 		if (ret) {
-			NV_DEBUG(dev, "perflvl %d, bad timing: %d\n", i, ret);
+			NV_DEBUG(drm, "perflvl %d, bad timing: %d\n", i, ret);
 			continue;
 		}
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.c b/drivers/gpu/drm/nouveau/nouveau_pm.c
index da3e7c3abab..0bf64c90aa2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_pm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_pm.c
@@ -22,12 +22,6 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_pm.h"
-#include "nouveau_gpio.h"
-
 #ifdef CONFIG_ACPI
 #include <linux/acpi.h>
 #endif
@@ -35,85 +29,41 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 
-static int
-nouveau_pwmfan_get(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct gpio_func gpio;
-	u32 divs, duty;
-	int ret;
-
-	if (!pm->pwm_get)
-		return -ENODEV;
-
-	ret = nouveau_gpio_find(dev, 0, DCB_GPIO_PWM_FAN, 0xff, &gpio);
-	if (ret == 0) {
-		ret = pm->pwm_get(dev, gpio.line, &divs, &duty);
-		if (ret == 0 && divs) {
-			divs = max(divs, duty);
-			if (dev_priv->card_type <= NV_40 || (gpio.log[0] & 1))
-				duty = divs - duty;
-			return (duty * 100) / divs;
-		}
-
-		return nouveau_gpio_func_get(dev, gpio.func) * 100;
-	}
-
-	return -ENODEV;
-}
-
-static int
-nouveau_pwmfan_set(struct drm_device *dev, int percent)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct gpio_func gpio;
-	u32 divs, duty;
-	int ret;
-
-	if (!pm->pwm_set)
-		return -ENODEV;
+#include <drm/drmP.h>
 
-	ret = nouveau_gpio_find(dev, 0, DCB_GPIO_PWM_FAN, 0xff, &gpio);
-	if (ret == 0) {
-		divs = pm->fan.pwm_divisor;
-		if (pm->fan.pwm_freq) {
-			/*XXX: PNVIO clock more than likely... */
-			divs = 135000 / pm->fan.pwm_freq;
-			if (dev_priv->chipset < 0xa3)
-				divs /= 4;
-		}
+#include "nouveau_drm.h"
+#include "nouveau_pm.h"
 
-		duty = ((divs * percent) + 99) / 100;
-		if (dev_priv->card_type <= NV_40 || (gpio.log[0] & 1))
-			duty = divs - duty;
+#include <subdev/gpio.h>
+#include <subdev/timer.h>
+#include <subdev/therm.h>
 
-		ret = pm->pwm_set(dev, gpio.line, divs, duty);
-		if (!ret)
-			pm->fan.percent = percent;
-		return ret;
-	}
+MODULE_PARM_DESC(perflvl, "Performance level (default: boot)");
+static char *nouveau_perflvl;
+module_param_named(perflvl, nouveau_perflvl, charp, 0400);
 
-	return -ENODEV;
-}
+MODULE_PARM_DESC(perflvl_wr, "Allow perflvl changes (warning: dangerous!)");
+static int nouveau_perflvl_wr;
+module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400);
 
 static int
 nouveau_pm_perflvl_aux(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 		       struct nouveau_pm_level *a, struct nouveau_pm_level *b)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_pm *pm = nouveau_pm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm);
 	int ret;
 
 	/*XXX: not on all boards, we should control based on temperature
 	 *     on recent boards..  or maybe on some other factor we don't
 	 *     know about?
 	 */
-	if (a->fanspeed && b->fanspeed && b->fanspeed > a->fanspeed) {
-		ret = nouveau_pwmfan_set(dev, perflvl->fanspeed);
+	if (therm && therm->fan_set &&
+		a->fanspeed && b->fanspeed && b->fanspeed > a->fanspeed) {
+		ret = therm->fan_set(therm, perflvl->fanspeed);
 		if (ret && ret != -ENODEV) {
-			NV_ERROR(dev, "fanspeed set failed: %d\n", ret);
+			NV_ERROR(drm, "fanspeed set failed: %d\n", ret);
 			return ret;
 		}
 	}
@@ -122,7 +72,7 @@ nouveau_pm_perflvl_aux(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 		if (perflvl->volt_min && b->volt_min > a->volt_min) {
 			ret = pm->voltage_set(dev, perflvl->volt_min);
 			if (ret) {
-				NV_ERROR(dev, "voltage set failed: %d\n", ret);
+				NV_ERROR(drm, "voltage set failed: %d\n", ret);
 				return ret;
 			}
 		}
@@ -134,8 +84,7 @@ nouveau_pm_perflvl_aux(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 static int
 nouveau_pm_perflvl_set(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	void *state;
 	int ret;
 
@@ -171,8 +120,9 @@ error:
 void
 nouveau_pm_trigger(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_timer *ptimer = nouveau_timer(drm->device);
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_profile *profile = NULL;
 	struct nouveau_pm_level *perflvl = NULL;
 	int ret;
@@ -194,24 +144,22 @@ nouveau_pm_trigger(struct drm_device *dev)
 
 	/* change perflvl, if necessary */
 	if (perflvl != pm->cur) {
-		struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-		u64 time0 = ptimer->read(dev);
+		u64 time0 = ptimer->read(ptimer);
 
-		NV_INFO(dev, "setting performance level: %d", perflvl->id);
+		NV_INFO(drm, "setting performance level: %d", perflvl->id);
 		ret = nouveau_pm_perflvl_set(dev, perflvl);
 		if (ret)
-			NV_INFO(dev, "> reclocking failed: %d\n\n", ret);
+			NV_INFO(drm, "> reclocking failed: %d\n\n", ret);
 
-		NV_INFO(dev, "> reclocking took %lluns\n\n",
-			     ptimer->read(dev) - time0);
+		NV_INFO(drm, "> reclocking took %lluns\n\n",
+			     ptimer->read(ptimer) - time0);
 	}
 }
 
 static struct nouveau_pm_profile *
 profile_find(struct drm_device *dev, const char *string)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_profile *profile;
 
 	list_for_each_entry(profile, &pm->profiles, head) {
@@ -225,8 +173,7 @@ profile_find(struct drm_device *dev, const char *string)
 static int
 nouveau_pm_profile_set(struct drm_device *dev, const char *profile)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_profile *ac = NULL, *dc = NULL;
 	char string[16], *cur = string, *ptr;
 
@@ -279,8 +226,9 @@ const struct nouveau_pm_profile_func nouveau_pm_static_profile_func = {
 static int
 nouveau_pm_perflvl_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_pm *pm = nouveau_pm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	int ret;
 
 	memset(perflvl, 0, sizeof(*perflvl));
@@ -299,9 +247,11 @@ nouveau_pm_perflvl_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 		}
 	}
 
-	ret = nouveau_pwmfan_get(dev);
-	if (ret > 0)
-		perflvl->fanspeed = ret;
+	if (therm && therm->fan_get) {
+		ret = therm->fan_get(therm);
+		if (ret >= 0)
+			perflvl->fanspeed = ret;
+	}
 
 	nouveau_mem_timing_read(dev, &perflvl->timing);
 	return 0;
@@ -362,8 +312,7 @@ static ssize_t
 nouveau_pm_get_perflvl(struct device *d, struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = pci_get_drvdata(to_pci_dev(d));
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_level cur;
 	int len = PAGE_SIZE, ret;
 	char *ptr = buf;
@@ -398,8 +347,8 @@ static DEVICE_ATTR(performance_level, S_IRUGO | S_IWUSR,
 static int
 nouveau_sysfs_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct device *d = &dev->pdev->dev;
 	int ret, i;
 
@@ -418,7 +367,7 @@ nouveau_sysfs_init(struct drm_device *dev)
 
 		ret = device_create_file(d, &perflvl->dev_attr);
 		if (ret) {
-			NV_ERROR(dev, "failed pervlvl %d sysfs: %d\n",
+			NV_ERROR(drm, "failed pervlvl %d sysfs: %d\n",
 				 perflvl->id, i);
 			perflvl->dev_attr.attr.name = NULL;
 			nouveau_pm_fini(dev);
@@ -432,8 +381,7 @@ nouveau_sysfs_init(struct drm_device *dev)
 static void
 nouveau_sysfs_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct device *d = &dev->pdev->dev;
 	int i;
 
@@ -453,10 +401,10 @@ static ssize_t
 nouveau_hwmon_show_temp(struct device *d, struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", pm->temp_get(dev)*1000);
+	return snprintf(buf, PAGE_SIZE, "%d\n", therm->temp_get(therm) * 1000);
 }
 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, nouveau_hwmon_show_temp,
 						  NULL, 0);
@@ -465,28 +413,25 @@ static ssize_t
 nouveau_hwmon_max_temp(struct device *d, struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", temp->down_clock*1000);
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+	       therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_DOWN_CLK) * 1000);
 }
 static ssize_t
 nouveau_hwmon_set_max_temp(struct device *d, struct device_attribute *a,
 						const char *buf, size_t count)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	long value;
 
 	if (kstrtol(buf, 10, &value) == -EINVAL)
 		return count;
 
-	temp->down_clock = value/1000;
-
-	nouveau_temp_safety_checks(dev);
+	therm->attr_set(therm, NOUVEAU_THERM_ATTR_THRS_DOWN_CLK, value / 1000);
 
 	return count;
 }
@@ -499,11 +444,11 @@ nouveau_hwmon_critical_temp(struct device *d, struct device_attribute *a,
 							char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", temp->critical*1000);
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+	       therm->attr_get(therm, NOUVEAU_THERM_ATTR_THRS_CRITICAL) * 1000);
 }
 static ssize_t
 nouveau_hwmon_set_critical_temp(struct device *d, struct device_attribute *a,
@@ -511,17 +456,14 @@ nouveau_hwmon_set_critical_temp(struct device *d, struct device_attribute *a,
 								size_t count)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_threshold_temp *temp = &pm->threshold_temp;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	long value;
 
 	if (kstrtol(buf, 10, &value) == -EINVAL)
 		return count;
 
-	temp->critical = value/1000;
-
-	nouveau_temp_safety_checks(dev);
+	therm->attr_set(therm, NOUVEAU_THERM_ATTR_THRS_CRITICAL, value / 1000);
 
 	return count;
 }
@@ -553,47 +495,62 @@ nouveau_hwmon_show_fan0_input(struct device *d, struct device_attribute *attr,
 			      char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-	struct gpio_func gpio;
-	u32 cycles, cur, prev;
-	u64 start;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", therm->fan_sense(therm));
+}
+static SENSOR_DEVICE_ATTR(fan0_input, S_IRUGO, nouveau_hwmon_show_fan0_input,
+			  NULL, 0);
+
+ static ssize_t
+nouveau_hwmon_get_pwm1_enable(struct device *d,
+			   struct device_attribute *a, char *buf)
+{
+	struct drm_device *dev = dev_get_drvdata(d);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	int ret;
 
-	ret = nouveau_gpio_find(dev, 0, DCB_GPIO_FAN_SENSE, 0xff, &gpio);
-	if (ret)
+	ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MODE);
+	if (ret < 0)
 		return ret;
 
-	/* Monitor the GPIO input 0x3b for 250ms.
-	 * When the fan spins, it changes the value of GPIO FAN_SENSE.
-	 * We get 4 changes (0 -> 1 -> 0 -> 1 -> [...]) per complete rotation.
-	 */
-	start = ptimer->read(dev);
-	prev = nouveau_gpio_sense(dev, 0, gpio.line);
-	cycles = 0;
-	do {
-		cur = nouveau_gpio_sense(dev, 0, gpio.line);
-		if (prev != cur) {
-			cycles++;
-			prev = cur;
-		}
+	return sprintf(buf, "%i\n", ret);
+}
 
-		usleep_range(500, 1000); /* supports 0 < rpm < 7500 */
-	} while (ptimer->read(dev) - start < 250000000);
+static ssize_t
+nouveau_hwmon_set_pwm1_enable(struct device *d, struct device_attribute *a,
+			   const char *buf, size_t count)
+{
+	struct drm_device *dev = dev_get_drvdata(d);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
+	long value;
+	int ret;
 
-	/* interpolate to get rpm */
-	return sprintf(buf, "%i\n", cycles / 4 * 4 * 60);
+	if (strict_strtol(buf, 10, &value) == -EINVAL)
+		return -EINVAL;
+
+	ret = therm->attr_set(therm, NOUVEAU_THERM_ATTR_FAN_MODE, value);
+	if (ret)
+		return ret;
+	else
+		return count;
 }
-static SENSOR_DEVICE_ATTR(fan0_input, S_IRUGO, nouveau_hwmon_show_fan0_input,
-			  NULL, 0);
+static SENSOR_DEVICE_ATTR(pwm1_enable, S_IRUGO | S_IWUSR,
+			  nouveau_hwmon_get_pwm1_enable,
+			  nouveau_hwmon_set_pwm1_enable, 0);
 
 static ssize_t
-nouveau_hwmon_get_pwm0(struct device *d, struct device_attribute *a, char *buf)
+nouveau_hwmon_get_pwm1(struct device *d, struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	int ret;
 
-	ret = nouveau_pwmfan_get(dev);
+	ret = therm->fan_get(therm);
 	if (ret < 0)
 		return ret;
 
@@ -601,12 +558,12 @@ nouveau_hwmon_get_pwm0(struct device *d, struct device_attribute *a, char *buf)
 }
 
 static ssize_t
-nouveau_hwmon_set_pwm0(struct device *d, struct device_attribute *a,
+nouveau_hwmon_set_pwm1(struct device *d, struct device_attribute *a,
 		       const char *buf, size_t count)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	int ret = -ENODEV;
 	long value;
 
@@ -616,103 +573,96 @@ nouveau_hwmon_set_pwm0(struct device *d, struct device_attribute *a,
 	if (kstrtol(buf, 10, &value) == -EINVAL)
 		return -EINVAL;
 
-	if (value < pm->fan.min_duty)
-		value = pm->fan.min_duty;
-	if (value > pm->fan.max_duty)
-		value = pm->fan.max_duty;
-
-	ret = nouveau_pwmfan_set(dev, value);
+	ret = therm->fan_set(therm, value);
 	if (ret)
 		return ret;
 
 	return count;
 }
 
-static SENSOR_DEVICE_ATTR(pwm0, S_IRUGO | S_IWUSR,
-			  nouveau_hwmon_get_pwm0,
-			  nouveau_hwmon_set_pwm0, 0);
+static SENSOR_DEVICE_ATTR(pwm1, S_IRUGO | S_IWUSR,
+			  nouveau_hwmon_get_pwm1,
+			  nouveau_hwmon_set_pwm1, 0);
 
 static ssize_t
-nouveau_hwmon_get_pwm0_min(struct device *d,
+nouveau_hwmon_get_pwm1_min(struct device *d,
 			   struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
+	int ret;
+
+	ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MIN_DUTY);
+	if (ret < 0)
+		return ret;
 
-	return sprintf(buf, "%i\n", pm->fan.min_duty);
+	return sprintf(buf, "%i\n", ret);
 }
 
 static ssize_t
-nouveau_hwmon_set_pwm0_min(struct device *d, struct device_attribute *a,
+nouveau_hwmon_set_pwm1_min(struct device *d, struct device_attribute *a,
 			   const char *buf, size_t count)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	long value;
+	int ret;
 
 	if (kstrtol(buf, 10, &value) == -EINVAL)
 		return -EINVAL;
 
-	if (value < 0)
-		value = 0;
-
-	if (pm->fan.max_duty - value < 10)
-		value = pm->fan.max_duty - 10;
-
-	if (value < 10)
-		pm->fan.min_duty = 10;
-	else
-		pm->fan.min_duty = value;
+	ret = therm->attr_set(therm, NOUVEAU_THERM_ATTR_FAN_MIN_DUTY, value);
+	if (ret < 0)
+		return ret;
 
 	return count;
 }
 
-static SENSOR_DEVICE_ATTR(pwm0_min, S_IRUGO | S_IWUSR,
-			  nouveau_hwmon_get_pwm0_min,
-			  nouveau_hwmon_set_pwm0_min, 0);
+static SENSOR_DEVICE_ATTR(pwm1_min, S_IRUGO | S_IWUSR,
+			  nouveau_hwmon_get_pwm1_min,
+			  nouveau_hwmon_set_pwm1_min, 0);
 
 static ssize_t
-nouveau_hwmon_get_pwm0_max(struct device *d,
+nouveau_hwmon_get_pwm1_max(struct device *d,
 			   struct device_attribute *a, char *buf)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
+	int ret;
+
+	ret = therm->attr_get(therm, NOUVEAU_THERM_ATTR_FAN_MAX_DUTY);
+	if (ret < 0)
+		return ret;
 
-	return sprintf(buf, "%i\n", pm->fan.max_duty);
+	return sprintf(buf, "%i\n", ret);
 }
 
 static ssize_t
-nouveau_hwmon_set_pwm0_max(struct device *d, struct device_attribute *a,
+nouveau_hwmon_set_pwm1_max(struct device *d, struct device_attribute *a,
 			   const char *buf, size_t count)
 {
 	struct drm_device *dev = dev_get_drvdata(d);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
 	long value;
+	int ret;
 
 	if (kstrtol(buf, 10, &value) == -EINVAL)
 		return -EINVAL;
 
-	if (value < 0)
-		value = 0;
-
-	if (value - pm->fan.min_duty < 10)
-		value = pm->fan.min_duty + 10;
-
-	if (value > 100)
-		pm->fan.max_duty = 100;
-	else
-		pm->fan.max_duty = value;
+	ret = therm->attr_set(therm, NOUVEAU_THERM_ATTR_FAN_MAX_DUTY, value);
+	if (ret < 0)
+		return ret;
 
 	return count;
 }
 
-static SENSOR_DEVICE_ATTR(pwm0_max, S_IRUGO | S_IWUSR,
-			  nouveau_hwmon_get_pwm0_max,
-			  nouveau_hwmon_set_pwm0_max, 0);
+static SENSOR_DEVICE_ATTR(pwm1_max, S_IRUGO | S_IWUSR,
+			  nouveau_hwmon_get_pwm1_max,
+			  nouveau_hwmon_set_pwm1_max, 0);
 
 static struct attribute *hwmon_attributes[] = {
 	&sensor_dev_attr_temp1_input.dev_attr.attr,
@@ -727,9 +677,10 @@ static struct attribute *hwmon_fan_rpm_attributes[] = {
 	NULL
 };
 static struct attribute *hwmon_pwm_fan_attributes[] = {
-	&sensor_dev_attr_pwm0.dev_attr.attr,
-	&sensor_dev_attr_pwm0_min.dev_attr.attr,
-	&sensor_dev_attr_pwm0_max.dev_attr.attr,
+	&sensor_dev_attr_pwm1_enable.dev_attr.attr,
+	&sensor_dev_attr_pwm1.dev_attr.attr,
+	&sensor_dev_attr_pwm1_min.dev_attr.attr,
+	&sensor_dev_attr_pwm1_max.dev_attr.attr,
 	NULL
 };
 
@@ -747,20 +698,22 @@ static const struct attribute_group hwmon_pwm_fan_attrgroup = {
 static int
 nouveau_hwmon_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_therm *therm = nouveau_therm(drm->device);
+
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 	struct device *hwmon_dev;
 	int ret = 0;
 
-	if (!pm->temp_get)
+	if (!therm || !therm->temp_get || !therm->attr_get ||
+		!therm->attr_set || therm->temp_get(therm) < 0)
 		return -ENODEV;
 
 	hwmon_dev = hwmon_device_register(&dev->pdev->dev);
 	if (IS_ERR(hwmon_dev)) {
 		ret = PTR_ERR(hwmon_dev);
-		NV_ERROR(dev,
-			"Unable to register hwmon device: %d\n", ret);
+		NV_ERROR(drm, "Unable to register hwmon device: %d\n", ret);
 		return ret;
 	}
 	dev_set_drvdata(hwmon_dev, dev);
@@ -776,7 +729,7 @@ nouveau_hwmon_init(struct drm_device *dev)
 	/*XXX: incorrect, need better detection for this, some boards have
 	 *     the gpio entries for pwm fan control even when there's no
 	 *     actual fan connected to it... therm table? */
-	if (nouveau_pwmfan_get(dev) >= 0) {
+	if (therm->fan_get && therm->fan_get(therm) >= 0) {
 		ret = sysfs_create_group(&dev->pdev->dev.kobj,
 					 &hwmon_pwm_fan_attrgroup);
 		if (ret)
@@ -784,7 +737,7 @@ nouveau_hwmon_init(struct drm_device *dev)
 	}
 
 	/* if the card can read the fan rpm */
-	if (nouveau_gpio_func_valid(dev, DCB_GPIO_FAN_SENSE)) {
+	if (therm->fan_sense(therm) >= 0) {
 		ret = sysfs_create_group(&dev->pdev->dev.kobj,
 					 &hwmon_fan_rpm_attrgroup);
 		if (ret)
@@ -796,7 +749,7 @@ nouveau_hwmon_init(struct drm_device *dev)
 	return 0;
 
 error:
-	NV_ERROR(dev, "Unable to create some hwmon sysfs files: %d\n", ret);
+	NV_ERROR(drm, "Unable to create some hwmon sysfs files: %d\n", ret);
 	hwmon_device_unregister(hwmon_dev);
 	pm->hwmon = NULL;
 	return ret;
@@ -810,8 +763,7 @@ static void
 nouveau_hwmon_fini(struct drm_device *dev)
 {
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 
 	if (pm->hwmon) {
 		sysfs_remove_group(&dev->pdev->dev.kobj, &hwmon_attrgroup);
@@ -829,16 +781,15 @@ nouveau_hwmon_fini(struct drm_device *dev)
 static int
 nouveau_pm_acpi_event(struct notifier_block *nb, unsigned long val, void *data)
 {
-	struct drm_nouveau_private *dev_priv =
-		container_of(nb, struct drm_nouveau_private, engine.pm.acpi_nb);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_pm *pm = container_of(nb, struct nouveau_pm, acpi_nb);
+	struct nouveau_drm *drm = nouveau_drm(pm->dev);
 	struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
 
 	if (strcmp(entry->device_class, "ac_adapter") == 0) {
 		bool ac = power_supply_is_system_supplied();
 
-		NV_DEBUG(dev, "power supply changed: %s\n", ac ? "AC" : "DC");
-		nouveau_pm_trigger(dev);
+		NV_DEBUG(drm, "power supply changed: %s\n", ac ? "AC" : "DC");
+		nouveau_pm_trigger(pm->dev);
 	}
 
 	return NOTIFY_OK;
@@ -848,19 +799,67 @@ nouveau_pm_acpi_event(struct notifier_block *nb, unsigned long val, void *data)
 int
 nouveau_pm_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_pm *pm;
 	char info[256];
 	int ret, i;
 
+	pm = drm->pm = kzalloc(sizeof(*pm), GFP_KERNEL);
+	if (!pm)
+		return -ENOMEM;
+
+	pm->dev = dev;
+
+	if (device->card_type < NV_40) {
+		pm->clocks_get = nv04_pm_clocks_get;
+		pm->clocks_pre = nv04_pm_clocks_pre;
+		pm->clocks_set = nv04_pm_clocks_set;
+		if (nouveau_gpio(drm->device)) {
+			pm->voltage_get = nouveau_voltage_gpio_get;
+			pm->voltage_set = nouveau_voltage_gpio_set;
+		}
+	} else
+	if (device->card_type < NV_50) {
+		pm->clocks_get = nv40_pm_clocks_get;
+		pm->clocks_pre = nv40_pm_clocks_pre;
+		pm->clocks_set = nv40_pm_clocks_set;
+		pm->voltage_get = nouveau_voltage_gpio_get;
+		pm->voltage_set = nouveau_voltage_gpio_set;
+	} else
+	if (device->card_type < NV_C0) {
+		if (device->chipset <  0xa3 ||
+		    device->chipset == 0xaa ||
+		    device->chipset == 0xac) {
+			pm->clocks_get = nv50_pm_clocks_get;
+			pm->clocks_pre = nv50_pm_clocks_pre;
+			pm->clocks_set = nv50_pm_clocks_set;
+		} else {
+			pm->clocks_get = nva3_pm_clocks_get;
+			pm->clocks_pre = nva3_pm_clocks_pre;
+			pm->clocks_set = nva3_pm_clocks_set;
+		}
+		pm->voltage_get = nouveau_voltage_gpio_get;
+		pm->voltage_set = nouveau_voltage_gpio_set;
+	} else
+	if (device->card_type < NV_E0) {
+		pm->clocks_get = nvc0_pm_clocks_get;
+		pm->clocks_pre = nvc0_pm_clocks_pre;
+		pm->clocks_set = nvc0_pm_clocks_set;
+		pm->voltage_get = nouveau_voltage_gpio_get;
+		pm->voltage_set = nouveau_voltage_gpio_set;
+	}
+
+
 	/* parse aux tables from vbios */
 	nouveau_volt_init(dev);
-	nouveau_temp_init(dev);
+
+	INIT_LIST_HEAD(&pm->profiles);
 
 	/* determine current ("boot") performance level */
 	ret = nouveau_pm_perflvl_get(dev, &pm->boot);
 	if (ret) {
-		NV_ERROR(dev, "failed to determine boot perflvl\n");
+		NV_ERROR(drm, "failed to determine boot perflvl\n");
 		return ret;
 	}
 
@@ -868,7 +867,6 @@ nouveau_pm_init(struct drm_device *dev)
 	strncpy(pm->boot.profile.name, "boot", 4);
 	pm->boot.profile.func = &nouveau_pm_static_profile_func;
 
-	INIT_LIST_HEAD(&pm->profiles);
 	list_add(&pm->boot.profile.head, &pm->profiles);
 
 	pm->profile_ac = &pm->boot.profile;
@@ -880,22 +878,19 @@ nouveau_pm_init(struct drm_device *dev)
 	nouveau_perf_init(dev);
 
 	/* display available performance levels */
-	NV_INFO(dev, "%d available performance level(s)\n", pm->nr_perflvl);
+	NV_INFO(drm, "%d available performance level(s)\n", pm->nr_perflvl);
 	for (i = 0; i < pm->nr_perflvl; i++) {
 		nouveau_pm_perflvl_info(&pm->perflvl[i], info, sizeof(info));
-		NV_INFO(dev, "%d:%s", pm->perflvl[i].id, info);
+		NV_INFO(drm, "%d:%s", pm->perflvl[i].id, info);
 	}
 
 	nouveau_pm_perflvl_info(&pm->boot, info, sizeof(info));
-	NV_INFO(dev, "c:%s", info);
+	NV_INFO(drm, "c:%s", info);
 
 	/* switch performance levels now if requested */
 	if (nouveau_perflvl != NULL)
 		nouveau_pm_profile_set(dev, nouveau_perflvl);
 
-	/* determine the current fan speed */
-	pm->fan.percent = nouveau_pwmfan_get(dev);
-
 	nouveau_sysfs_init(dev);
 	nouveau_hwmon_init(dev);
 #if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
@@ -909,8 +904,7 @@ nouveau_pm_init(struct drm_device *dev)
 void
 nouveau_pm_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_profile *profile, *tmp;
 
 	list_for_each_entry_safe(profile, tmp, &pm->profiles, head) {
@@ -921,7 +915,6 @@ nouveau_pm_fini(struct drm_device *dev)
 	if (pm->cur != &pm->boot)
 		nouveau_pm_perflvl_set(dev, &pm->boot);
 
-	nouveau_temp_fini(dev);
 	nouveau_perf_fini(dev);
 	nouveau_volt_fini(dev);
 
@@ -930,13 +923,15 @@ nouveau_pm_fini(struct drm_device *dev)
 #endif
 	nouveau_hwmon_fini(dev);
 	nouveau_sysfs_fini(dev);
+
+	nouveau_drm(dev)->pm = NULL;
+	kfree(pm);
 }
 
 void
 nouveau_pm_resume(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_level *perflvl;
 
 	if (!pm->cur || pm->cur == &pm->boot)
@@ -945,5 +940,4 @@ nouveau_pm_resume(struct drm_device *dev)
 	perflvl = pm->cur;
 	pm->cur = &pm->boot;
 	nouveau_pm_perflvl_set(dev, perflvl);
-	nouveau_pwmfan_set(dev, pm->fan.percent);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.h b/drivers/gpu/drm/nouveau/nouveau_pm.h
index 07cac72c72b..73b789c230a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_pm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_pm.h
@@ -25,6 +25,165 @@
 #ifndef __NOUVEAU_PM_H__
 #define __NOUVEAU_PM_H__
 
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
+
+struct nouveau_pm_voltage_level {
+	u32 voltage; /* microvolts */
+	u8  vid;
+};
+
+struct nouveau_pm_voltage {
+	bool supported;
+	u8 version;
+	u8 vid_mask;
+
+	struct nouveau_pm_voltage_level *level;
+	int nr_level;
+};
+
+/* Exclusive upper limits */
+#define NV_MEM_CL_DDR2_MAX 8
+#define NV_MEM_WR_DDR2_MAX 9
+#define NV_MEM_CL_DDR3_MAX 17
+#define NV_MEM_WR_DDR3_MAX 17
+#define NV_MEM_CL_GDDR3_MAX 16
+#define NV_MEM_WR_GDDR3_MAX 18
+#define NV_MEM_CL_GDDR5_MAX 21
+#define NV_MEM_WR_GDDR5_MAX 20
+
+struct nouveau_pm_memtiming {
+	int id;
+
+	u32 reg[9];
+	u32 mr[4];
+
+	u8 tCWL;
+
+	u8 odt;
+	u8 drive_strength;
+};
+
+struct nouveau_pm_tbl_header {
+	u8 version;
+	u8 header_len;
+	u8 entry_cnt;
+	u8 entry_len;
+};
+
+struct nouveau_pm_tbl_entry {
+	u8 tWR;
+	u8 tWTR;
+	u8 tCL;
+	u8 tRC;
+	u8 empty_4;
+	u8 tRFC;	/* Byte 5 */
+	u8 empty_6;
+	u8 tRAS;	/* Byte 7 */
+	u8 empty_8;
+	u8 tRP;		/* Byte 9 */
+	u8 tRCDRD;
+	u8 tRCDWR;
+	u8 tRRD;
+	u8 tUNK_13;
+	u8 RAM_FT1;		/* 14, a bitmask of random RAM features */
+	u8 empty_15;
+	u8 tUNK_16;
+	u8 empty_17;
+	u8 tUNK_18;
+	u8 tCWL;
+	u8 tUNK_20, tUNK_21;
+};
+
+struct nouveau_pm_profile;
+struct nouveau_pm_profile_func {
+	void (*destroy)(struct nouveau_pm_profile *);
+	void (*init)(struct nouveau_pm_profile *);
+	void (*fini)(struct nouveau_pm_profile *);
+	struct nouveau_pm_level *(*select)(struct nouveau_pm_profile *);
+};
+
+struct nouveau_pm_profile {
+	const struct nouveau_pm_profile_func *func;
+	struct list_head head;
+	char name[8];
+};
+
+#define NOUVEAU_PM_MAX_LEVEL 8
+struct nouveau_pm_level {
+	struct nouveau_pm_profile profile;
+	struct device_attribute dev_attr;
+	char name[32];
+	int id;
+
+	struct nouveau_pm_memtiming timing;
+	u32 memory;
+	u16 memscript;
+
+	u32 core;
+	u32 shader;
+	u32 rop;
+	u32 copy;
+	u32 daemon;
+	u32 vdec;
+	u32 dom6;
+	u32 unka0;	/* nva3:nvc0 */
+	u32 hub01;	/* nvc0- */
+	u32 hub06;	/* nvc0- */
+	u32 hub07;	/* nvc0- */
+
+	u32 volt_min; /* microvolts */
+	u32 volt_max;
+	u8  fanspeed;
+};
+
+struct nouveau_pm_temp_sensor_constants {
+	u16 offset_constant;
+	s16 offset_mult;
+	s16 offset_div;
+	s16 slope_mult;
+	s16 slope_div;
+};
+
+struct nouveau_pm_threshold_temp {
+	s16 critical;
+	s16 down_clock;
+};
+
+struct nouveau_pm {
+	struct drm_device *dev;
+
+	struct nouveau_pm_voltage voltage;
+	struct nouveau_pm_level perflvl[NOUVEAU_PM_MAX_LEVEL];
+	int nr_perflvl;
+	struct nouveau_pm_temp_sensor_constants sensor_constants;
+	struct nouveau_pm_threshold_temp threshold_temp;
+
+	struct nouveau_pm_profile *profile_ac;
+	struct nouveau_pm_profile *profile_dc;
+	struct nouveau_pm_profile *profile;
+	struct list_head profiles;
+
+	struct nouveau_pm_level boot;
+	struct nouveau_pm_level *cur;
+
+	struct device *hwmon;
+	struct notifier_block acpi_nb;
+
+	int  (*clocks_get)(struct drm_device *, struct nouveau_pm_level *);
+	void *(*clocks_pre)(struct drm_device *, struct nouveau_pm_level *);
+	int (*clocks_set)(struct drm_device *, void *);
+
+	int (*voltage_get)(struct drm_device *);
+	int (*voltage_set)(struct drm_device *, int voltage);
+};
+
+static inline struct nouveau_pm *
+nouveau_pm(struct drm_device *dev)
+{
+	return nouveau_drm(dev)->pm;
+}
+
 struct nouveau_mem_exec_func {
 	struct drm_device *dev;
 	void (*precharge)(struct nouveau_mem_exec_func *);
@@ -99,11 +258,26 @@ int nvc0_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
 void *nvc0_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
 int nvc0_pm_clocks_set(struct drm_device *, void *);
 
-/* nouveau_temp.c */
-void nouveau_temp_init(struct drm_device *dev);
-void nouveau_temp_fini(struct drm_device *dev);
-void nouveau_temp_safety_checks(struct drm_device *dev);
-int nv40_temp_get(struct drm_device *dev);
-int nv84_temp_get(struct drm_device *dev);
+/* nouveau_mem.c */
+int  nouveau_mem_timing_calc(struct drm_device *, u32 freq,
+			     struct nouveau_pm_memtiming *);
+void nouveau_mem_timing_read(struct drm_device *,
+			     struct nouveau_pm_memtiming *);
+
+static inline int
+nva3_calc_pll(struct drm_device *dev, struct nvbios_pll *pll, u32 freq,
+	      int *N, int *fN, int *M, int *P)
+{
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_clock *clk = nouveau_clock(device);
+	struct nouveau_pll_vals pv;
+	int ret;
+
+	ret = clk->pll_calc(clk, pll, freq, &pv);
+	*N = pv.N1;
+	*M = pv.M1;
+	*P = pv.log2P;
+	return ret;
+}
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_prime.c b/drivers/gpu/drm/nouveau/nouveau_prime.c
index a25cf2cb931..366462cf8a2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_prime.c
+++ b/drivers/gpu/drm/nouveau/nouveau_prime.c
@@ -22,14 +22,12 @@
  * Authors: Dave Airlie
  */
 
-#include "drmP.h"
-#include "drm.h"
+#include <linux/dma-buf.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_dma.h"
+#include <drm/drmP.h>
 
-#include <linux/dma-buf.h>
+#include "nouveau_drm.h"
+#include "nouveau_gem.h"
 
 static struct sg_table *nouveau_gem_map_dma_buf(struct dma_buf_attachment *attachment,
 					  enum dma_data_direction dir)
diff --git a/drivers/gpu/drm/nouveau/nouveau_ramht.c b/drivers/gpu/drm/nouveau/nouveau_ramht.c
deleted file mode 100644
index a24a81f5a89..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_ramht.c
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-
-static u32
-nouveau_ramht_hash_handle(struct nouveau_channel *chan, u32 handle)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_ramht *ramht = chan->ramht;
-	u32 hash = 0;
-	int i;
-
-	NV_DEBUG(dev, "ch%d handle=0x%08x\n", chan->id, handle);
-
-	for (i = 32; i > 0; i -= ramht->bits) {
-		hash ^= (handle & ((1 << ramht->bits) - 1));
-		handle >>= ramht->bits;
-	}
-
-	if (dev_priv->card_type < NV_50)
-		hash ^= chan->id << (ramht->bits - 4);
-	hash <<= 3;
-
-	NV_DEBUG(dev, "hash=0x%08x\n", hash);
-	return hash;
-}
-
-static int
-nouveau_ramht_entry_valid(struct drm_device *dev, struct nouveau_gpuobj *ramht,
-			  u32 offset)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 ctx = nv_ro32(ramht, offset + 4);
-
-	if (dev_priv->card_type < NV_40)
-		return ((ctx & NV_RAMHT_CONTEXT_VALID) != 0);
-	return (ctx != 0);
-}
-
-static int
-nouveau_ramht_entry_same_channel(struct nouveau_channel *chan,
-				 struct nouveau_gpuobj *ramht, u32 offset)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	u32 ctx = nv_ro32(ramht, offset + 4);
-
-	if (dev_priv->card_type >= NV_50)
-		return true;
-	else if (dev_priv->card_type >= NV_40)
-		return chan->id ==
-			((ctx >> NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) & 0x1f);
-	else
-		return chan->id ==
-			((ctx >> NV_RAMHT_CONTEXT_CHANNEL_SHIFT) & 0x1f);
-}
-
-int
-nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
-		     struct nouveau_gpuobj *gpuobj)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-	struct nouveau_ramht_entry *entry;
-	struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
-	unsigned long flags;
-	u32 ctx, co, ho;
-
-	if (nouveau_ramht_find(chan, handle))
-		return -EEXIST;
-
-	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
-	if (!entry)
-		return -ENOMEM;
-	entry->channel = chan;
-	entry->gpuobj = NULL;
-	entry->handle = handle;
-	nouveau_gpuobj_ref(gpuobj, &entry->gpuobj);
-
-	if (dev_priv->card_type < NV_40) {
-		ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->pinst >> 4) |
-		      (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
-		      (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
-	} else
-	if (dev_priv->card_type < NV_50) {
-		ctx = (gpuobj->pinst >> 4) |
-		      (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
-		      (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
-	} else {
-		if (gpuobj->engine == NVOBJ_ENGINE_DISPLAY) {
-			ctx = (gpuobj->cinst << 10) |
-			      (chan->id << 28) |
-			      chan->id; /* HASH_TAG */
-		} else {
-			ctx = (gpuobj->cinst >> 4) |
-			      ((gpuobj->engine <<
-				NV40_RAMHT_CONTEXT_ENGINE_SHIFT));
-		}
-	}
-
-	spin_lock_irqsave(&chan->ramht->lock, flags);
-	list_add(&entry->head, &chan->ramht->entries);
-
-	co = ho = nouveau_ramht_hash_handle(chan, handle);
-	do {
-		if (!nouveau_ramht_entry_valid(dev, ramht, co)) {
-			NV_DEBUG(dev,
-				 "insert ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
-				 chan->id, co, handle, ctx);
-			nv_wo32(ramht, co + 0, handle);
-			nv_wo32(ramht, co + 4, ctx);
-
-			spin_unlock_irqrestore(&chan->ramht->lock, flags);
-			instmem->flush(dev);
-			return 0;
-		}
-		NV_DEBUG(dev, "collision ch%d 0x%08x: h=0x%08x\n",
-			 chan->id, co, nv_ro32(ramht, co));
-
-		co += 8;
-		if (co >= ramht->size)
-			co = 0;
-	} while (co != ho);
-
-	NV_ERROR(dev, "RAMHT space exhausted. ch=%d\n", chan->id);
-	list_del(&entry->head);
-	spin_unlock_irqrestore(&chan->ramht->lock, flags);
-	kfree(entry);
-	return -ENOMEM;
-}
-
-static struct nouveau_ramht_entry *
-nouveau_ramht_remove_entry(struct nouveau_channel *chan, u32 handle)
-{
-	struct nouveau_ramht *ramht = chan ? chan->ramht : NULL;
-	struct nouveau_ramht_entry *entry;
-	unsigned long flags;
-
-	if (!ramht)
-		return NULL;
-
-	spin_lock_irqsave(&ramht->lock, flags);
-	list_for_each_entry(entry, &ramht->entries, head) {
-		if (entry->channel == chan &&
-		    (!handle || entry->handle == handle)) {
-			list_del(&entry->head);
-			spin_unlock_irqrestore(&ramht->lock, flags);
-
-			return entry;
-		}
-	}
-	spin_unlock_irqrestore(&ramht->lock, flags);
-
-	return NULL;
-}
-
-static void
-nouveau_ramht_remove_hash(struct nouveau_channel *chan, u32 handle)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
-	struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
-	unsigned long flags;
-	u32 co, ho;
-
-	spin_lock_irqsave(&chan->ramht->lock, flags);
-	co = ho = nouveau_ramht_hash_handle(chan, handle);
-	do {
-		if (nouveau_ramht_entry_valid(dev, ramht, co) &&
-		    nouveau_ramht_entry_same_channel(chan, ramht, co) &&
-		    (handle == nv_ro32(ramht, co))) {
-			NV_DEBUG(dev,
-				 "remove ch%d 0x%08x: h=0x%08x, c=0x%08x\n",
-				 chan->id, co, handle, nv_ro32(ramht, co + 4));
-			nv_wo32(ramht, co + 0, 0x00000000);
-			nv_wo32(ramht, co + 4, 0x00000000);
-			instmem->flush(dev);
-			goto out;
-		}
-
-		co += 8;
-		if (co >= ramht->size)
-			co = 0;
-	} while (co != ho);
-
-	NV_ERROR(dev, "RAMHT entry not found. ch=%d, handle=0x%08x\n",
-		 chan->id, handle);
-out:
-	spin_unlock_irqrestore(&chan->ramht->lock, flags);
-}
-
-int
-nouveau_ramht_remove(struct nouveau_channel *chan, u32 handle)
-{
-	struct nouveau_ramht_entry *entry;
-
-	entry = nouveau_ramht_remove_entry(chan, handle);
-	if (!entry)
-		return -ENOENT;
-
-	nouveau_ramht_remove_hash(chan, entry->handle);
-	nouveau_gpuobj_ref(NULL, &entry->gpuobj);
-	kfree(entry);
-	return 0;
-}
-
-struct nouveau_gpuobj *
-nouveau_ramht_find(struct nouveau_channel *chan, u32 handle)
-{
-	struct nouveau_ramht *ramht = chan->ramht;
-	struct nouveau_ramht_entry *entry;
-	struct nouveau_gpuobj *gpuobj = NULL;
-	unsigned long flags;
-
-	if (unlikely(!chan->ramht))
-		return NULL;
-
-	spin_lock_irqsave(&ramht->lock, flags);
-	list_for_each_entry(entry, &chan->ramht->entries, head) {
-		if (entry->channel == chan && entry->handle == handle) {
-			gpuobj = entry->gpuobj;
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&ramht->lock, flags);
-
-	return gpuobj;
-}
-
-int
-nouveau_ramht_new(struct drm_device *dev, struct nouveau_gpuobj *gpuobj,
-		  struct nouveau_ramht **pramht)
-{
-	struct nouveau_ramht *ramht;
-
-	ramht = kzalloc(sizeof(*ramht), GFP_KERNEL);
-	if (!ramht)
-		return -ENOMEM;
-
-	ramht->dev = dev;
-	kref_init(&ramht->refcount);
-	ramht->bits = drm_order(gpuobj->size / 8);
-	INIT_LIST_HEAD(&ramht->entries);
-	spin_lock_init(&ramht->lock);
-	nouveau_gpuobj_ref(gpuobj, &ramht->gpuobj);
-
-	*pramht = ramht;
-	return 0;
-}
-
-static void
-nouveau_ramht_del(struct kref *ref)
-{
-	struct nouveau_ramht *ramht =
-		container_of(ref, struct nouveau_ramht, refcount);
-
-	nouveau_gpuobj_ref(NULL, &ramht->gpuobj);
-	kfree(ramht);
-}
-
-void
-nouveau_ramht_ref(struct nouveau_ramht *ref, struct nouveau_ramht **ptr,
-		  struct nouveau_channel *chan)
-{
-	struct nouveau_ramht_entry *entry;
-	struct nouveau_ramht *ramht;
-
-	if (ref)
-		kref_get(&ref->refcount);
-
-	ramht = *ptr;
-	if (ramht) {
-		while ((entry = nouveau_ramht_remove_entry(chan, 0))) {
-			nouveau_ramht_remove_hash(chan, entry->handle);
-			nouveau_gpuobj_ref(NULL, &entry->gpuobj);
-			kfree(entry);
-		}
-
-		kref_put(&ramht->refcount, nouveau_ramht_del);
-	}
-	*ptr = ref;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
index 38483a042bc..ca5492ac2da 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c
@@ -1,11 +1,10 @@
-#include "drmP.h"
-#include "nouveau_drv.h"
 #include <linux/pagemap.h>
 #include <linux/slab.h>
 
-#define NV_CTXDMA_PAGE_SHIFT 12
-#define NV_CTXDMA_PAGE_SIZE  (1 << NV_CTXDMA_PAGE_SHIFT)
-#define NV_CTXDMA_PAGE_MASK  (NV_CTXDMA_PAGE_SIZE - 1)
+#include <subdev/fb.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_ttm.h"
 
 struct nouveau_sgdma_be {
 	/* this has to be the first field so populate/unpopulated in
@@ -13,7 +12,7 @@ struct nouveau_sgdma_be {
 	 */
 	struct ttm_dma_tt ttm;
 	struct drm_device *dev;
-	u64 offset;
+	struct nouveau_mem *node;
 };
 
 static void
@@ -22,7 +21,6 @@ nouveau_sgdma_destroy(struct ttm_tt *ttm)
 	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
 
 	if (ttm) {
-		NV_DEBUG(nvbe->dev, "\n");
 		ttm_dma_tt_fini(&nvbe->ttm);
 		kfree(nvbe);
 	}
@@ -32,25 +30,18 @@ static int
 nv04_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
 {
 	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_device *dev = nvbe->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
-	unsigned i, j, pte;
-
-	NV_DEBUG(dev, "pg=0x%lx\n", mem->start);
-
-	nvbe->offset = mem->start << PAGE_SHIFT;
-	pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
-	for (i = 0; i < ttm->num_pages; i++) {
-		dma_addr_t dma_offset = nvbe->ttm.dma_address[i];
-		uint32_t offset_l = lower_32_bits(dma_offset);
+	struct nouveau_mem *node = mem->mm_node;
+	u64 size = mem->num_pages << 12;
 
-		for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++) {
-			nv_wo32(gpuobj, (pte * 4) + 0, offset_l | 3);
-			offset_l += NV_CTXDMA_PAGE_SIZE;
-		}
+	if (ttm->sg) {
+		node->sg = ttm->sg;
+		nouveau_vm_map_sg_table(&node->vma[0], 0, size, node);
+	} else {
+		node->pages = nvbe->ttm.dma_address;
+		nouveau_vm_map_sg(&node->vma[0], 0, size, node);
 	}
 
+	nvbe->node = node;
 	return 0;
 }
 
@@ -58,22 +49,7 @@ static int
 nv04_sgdma_unbind(struct ttm_tt *ttm)
 {
 	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_device *dev = nvbe->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
-	unsigned i, j, pte;
-
-	NV_DEBUG(dev, "\n");
-
-	if (ttm->state != tt_bound)
-		return 0;
-
-	pte = (nvbe->offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
-	for (i = 0; i < ttm->num_pages; i++) {
-		for (j = 0; j < PAGE_SIZE / NV_CTXDMA_PAGE_SIZE; j++, pte++)
-			nv_wo32(gpuobj, (pte * 4) + 0, 0x00000000);
-	}
-
+	nouveau_vm_unmap(&nvbe->node->vma[0]);
 	return 0;
 }
 
@@ -83,206 +59,6 @@ static struct ttm_backend_func nv04_sgdma_backend = {
 	.destroy		= nouveau_sgdma_destroy
 };
 
-static void
-nv41_sgdma_flush(struct nouveau_sgdma_be *nvbe)
-{
-	struct drm_device *dev = nvbe->dev;
-
-	nv_wr32(dev, 0x100810, 0x00000022);
-	if (!nv_wait(dev, 0x100810, 0x00000100, 0x00000100))
-		NV_ERROR(dev, "vm flush timeout: 0x%08x\n",
-			 nv_rd32(dev, 0x100810));
-	nv_wr32(dev, 0x100810, 0x00000000);
-}
-
-static int
-nv41_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
-{
-	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
-	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
-	dma_addr_t *list = nvbe->ttm.dma_address;
-	u32 pte = mem->start << 2;
-	u32 cnt = ttm->num_pages;
-
-	nvbe->offset = mem->start << PAGE_SHIFT;
-
-	while (cnt--) {
-		nv_wo32(pgt, pte, (*list++ >> 7) | 1);
-		pte += 4;
-	}
-
-	nv41_sgdma_flush(nvbe);
-	return 0;
-}
-
-static int
-nv41_sgdma_unbind(struct ttm_tt *ttm)
-{
-	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
-	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
-	u32 pte = (nvbe->offset >> 12) << 2;
-	u32 cnt = ttm->num_pages;
-
-	while (cnt--) {
-		nv_wo32(pgt, pte, 0x00000000);
-		pte += 4;
-	}
-
-	nv41_sgdma_flush(nvbe);
-	return 0;
-}
-
-static struct ttm_backend_func nv41_sgdma_backend = {
-	.bind			= nv41_sgdma_bind,
-	.unbind			= nv41_sgdma_unbind,
-	.destroy		= nouveau_sgdma_destroy
-};
-
-static void
-nv44_sgdma_flush(struct ttm_tt *ttm)
-{
-	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_device *dev = nvbe->dev;
-
-	nv_wr32(dev, 0x100814, (ttm->num_pages - 1) << 12);
-	nv_wr32(dev, 0x100808, nvbe->offset | 0x20);
-	if (!nv_wait(dev, 0x100808, 0x00000001, 0x00000001))
-		NV_ERROR(dev, "gart flush timeout: 0x%08x\n",
-			 nv_rd32(dev, 0x100808));
-	nv_wr32(dev, 0x100808, 0x00000000);
-}
-
-static void
-nv44_sgdma_fill(struct nouveau_gpuobj *pgt, dma_addr_t *list, u32 base, u32 cnt)
-{
-	struct drm_nouveau_private *dev_priv = pgt->dev->dev_private;
-	dma_addr_t dummy = dev_priv->gart_info.dummy.addr;
-	u32 pte, tmp[4];
-
-	pte   = base >> 2;
-	base &= ~0x0000000f;
-
-	tmp[0] = nv_ro32(pgt, base + 0x0);
-	tmp[1] = nv_ro32(pgt, base + 0x4);
-	tmp[2] = nv_ro32(pgt, base + 0x8);
-	tmp[3] = nv_ro32(pgt, base + 0xc);
-	while (cnt--) {
-		u32 addr = list ? (*list++ >> 12) : (dummy >> 12);
-		switch (pte++ & 0x3) {
-		case 0:
-			tmp[0] &= ~0x07ffffff;
-			tmp[0] |= addr;
-			break;
-		case 1:
-			tmp[0] &= ~0xf8000000;
-			tmp[0] |= addr << 27;
-			tmp[1] &= ~0x003fffff;
-			tmp[1] |= addr >> 5;
-			break;
-		case 2:
-			tmp[1] &= ~0xffc00000;
-			tmp[1] |= addr << 22;
-			tmp[2] &= ~0x0001ffff;
-			tmp[2] |= addr >> 10;
-			break;
-		case 3:
-			tmp[2] &= ~0xfffe0000;
-			tmp[2] |= addr << 17;
-			tmp[3] &= ~0x00000fff;
-			tmp[3] |= addr >> 15;
-			break;
-		}
-	}
-
-	tmp[3] |= 0x40000000;
-
-	nv_wo32(pgt, base + 0x0, tmp[0]);
-	nv_wo32(pgt, base + 0x4, tmp[1]);
-	nv_wo32(pgt, base + 0x8, tmp[2]);
-	nv_wo32(pgt, base + 0xc, tmp[3]);
-}
-
-static int
-nv44_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
-{
-	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
-	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
-	dma_addr_t *list = nvbe->ttm.dma_address;
-	u32 pte = mem->start << 2, tmp[4];
-	u32 cnt = ttm->num_pages;
-	int i;
-
-	nvbe->offset = mem->start << PAGE_SHIFT;
-
-	if (pte & 0x0000000c) {
-		u32  max = 4 - ((pte >> 2) & 0x3);
-		u32 part = (cnt > max) ? max : cnt;
-		nv44_sgdma_fill(pgt, list, pte, part);
-		pte  += (part << 2);
-		list += part;
-		cnt  -= part;
-	}
-
-	while (cnt >= 4) {
-		for (i = 0; i < 4; i++)
-			tmp[i] = *list++ >> 12;
-		nv_wo32(pgt, pte + 0x0, tmp[0] >>  0 | tmp[1] << 27);
-		nv_wo32(pgt, pte + 0x4, tmp[1] >>  5 | tmp[2] << 22);
-		nv_wo32(pgt, pte + 0x8, tmp[2] >> 10 | tmp[3] << 17);
-		nv_wo32(pgt, pte + 0xc, tmp[3] >> 15 | 0x40000000);
-		pte  += 0x10;
-		cnt  -= 4;
-	}
-
-	if (cnt)
-		nv44_sgdma_fill(pgt, list, pte, cnt);
-
-	nv44_sgdma_flush(ttm);
-	return 0;
-}
-
-static int
-nv44_sgdma_unbind(struct ttm_tt *ttm)
-{
-	struct nouveau_sgdma_be *nvbe = (struct nouveau_sgdma_be *)ttm;
-	struct drm_nouveau_private *dev_priv = nvbe->dev->dev_private;
-	struct nouveau_gpuobj *pgt = dev_priv->gart_info.sg_ctxdma;
-	u32 pte = (nvbe->offset >> 12) << 2;
-	u32 cnt = ttm->num_pages;
-
-	if (pte & 0x0000000c) {
-		u32  max = 4 - ((pte >> 2) & 0x3);
-		u32 part = (cnt > max) ? max : cnt;
-		nv44_sgdma_fill(pgt, NULL, pte, part);
-		pte  += (part << 2);
-		cnt  -= part;
-	}
-
-	while (cnt >= 4) {
-		nv_wo32(pgt, pte + 0x0, 0x00000000);
-		nv_wo32(pgt, pte + 0x4, 0x00000000);
-		nv_wo32(pgt, pte + 0x8, 0x00000000);
-		nv_wo32(pgt, pte + 0xc, 0x00000000);
-		pte  += 0x10;
-		cnt  -= 4;
-	}
-
-	if (cnt)
-		nv44_sgdma_fill(pgt, NULL, pte, cnt);
-
-	nv44_sgdma_flush(ttm);
-	return 0;
-}
-
-static struct ttm_backend_func nv44_sgdma_backend = {
-	.bind			= nv44_sgdma_bind,
-	.unbind			= nv44_sgdma_unbind,
-	.destroy		= nouveau_sgdma_destroy
-};
-
 static int
 nv50_sgdma_bind(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
 {
@@ -315,16 +91,18 @@ nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
 			 unsigned long size, uint32_t page_flags,
 			 struct page *dummy_read_page)
 {
-	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_bdev(bdev);
 	struct nouveau_sgdma_be *nvbe;
 
 	nvbe = kzalloc(sizeof(*nvbe), GFP_KERNEL);
 	if (!nvbe)
 		return NULL;
 
-	nvbe->dev = dev;
-	nvbe->ttm.ttm.func = dev_priv->gart_info.func;
+	nvbe->dev = drm->dev;
+	if (nv_device(drm->device)->card_type < NV_50)
+		nvbe->ttm.ttm.func = &nv04_sgdma_backend;
+	else
+		nvbe->ttm.ttm.func = &nv50_sgdma_backend;
 
 	if (ttm_dma_tt_init(&nvbe->ttm, bdev, size, page_flags, dummy_read_page)) {
 		kfree(nvbe);
@@ -332,116 +110,3 @@ nouveau_sgdma_create_ttm(struct ttm_bo_device *bdev,
 	}
 	return &nvbe->ttm.ttm;
 }
-
-int
-nouveau_sgdma_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj = NULL;
-	u32 aper_size, align;
-	int ret;
-
-	if (dev_priv->card_type >= NV_40)
-		aper_size = 512 * 1024 * 1024;
-	else
-		aper_size = 128 * 1024 * 1024;
-
-	/* Dear NVIDIA, NV44+ would like proper present bits in PTEs for
-	 * christmas.  The cards before it have them, the cards after
-	 * it have them, why is NV44 so unloved?
-	 */
-	dev_priv->gart_info.dummy.page = alloc_page(GFP_DMA32 | GFP_KERNEL);
-	if (!dev_priv->gart_info.dummy.page)
-		return -ENOMEM;
-
-	dev_priv->gart_info.dummy.addr =
-		pci_map_page(dev->pdev, dev_priv->gart_info.dummy.page,
-			     0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(dev->pdev, dev_priv->gart_info.dummy.addr)) {
-		NV_ERROR(dev, "error mapping dummy page\n");
-		__free_page(dev_priv->gart_info.dummy.page);
-		dev_priv->gart_info.dummy.page = NULL;
-		return -ENOMEM;
-	}
-
-	if (dev_priv->card_type >= NV_50) {
-		dev_priv->gart_info.aper_base = 0;
-		dev_priv->gart_info.aper_size = aper_size;
-		dev_priv->gart_info.type = NOUVEAU_GART_HW;
-		dev_priv->gart_info.func = &nv50_sgdma_backend;
-	} else
-	if (0 && pci_is_pcie(dev->pdev) &&
-	    dev_priv->chipset > 0x40 && dev_priv->chipset != 0x45) {
-		if (nv44_graph_class(dev)) {
-			dev_priv->gart_info.func = &nv44_sgdma_backend;
-			align = 512 * 1024;
-		} else {
-			dev_priv->gart_info.func = &nv41_sgdma_backend;
-			align = 16;
-		}
-
-		ret = nouveau_gpuobj_new(dev, NULL, aper_size / 1024, align,
-					 NVOBJ_FLAG_ZERO_ALLOC |
-					 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
-		if (ret) {
-			NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
-			return ret;
-		}
-
-		dev_priv->gart_info.sg_ctxdma = gpuobj;
-		dev_priv->gart_info.aper_base = 0;
-		dev_priv->gart_info.aper_size = aper_size;
-		dev_priv->gart_info.type = NOUVEAU_GART_HW;
-	} else {
-		ret = nouveau_gpuobj_new(dev, NULL, (aper_size / 1024) + 8, 16,
-					 NVOBJ_FLAG_ZERO_ALLOC |
-					 NVOBJ_FLAG_ZERO_FREE, &gpuobj);
-		if (ret) {
-			NV_ERROR(dev, "Error creating sgdma object: %d\n", ret);
-			return ret;
-		}
-
-		nv_wo32(gpuobj, 0, NV_CLASS_DMA_IN_MEMORY |
-				   (1 << 12) /* PT present */ |
-				   (0 << 13) /* PT *not* linear */ |
-				   (0 << 14) /* RW */ |
-				   (2 << 16) /* PCI */);
-		nv_wo32(gpuobj, 4, aper_size - 1);
-
-		dev_priv->gart_info.sg_ctxdma = gpuobj;
-		dev_priv->gart_info.aper_base = 0;
-		dev_priv->gart_info.aper_size = aper_size;
-		dev_priv->gart_info.type = NOUVEAU_GART_PDMA;
-		dev_priv->gart_info.func = &nv04_sgdma_backend;
-	}
-
-	return 0;
-}
-
-void
-nouveau_sgdma_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nouveau_gpuobj_ref(NULL, &dev_priv->gart_info.sg_ctxdma);
-
-	if (dev_priv->gart_info.dummy.page) {
-		pci_unmap_page(dev->pdev, dev_priv->gart_info.dummy.addr,
-			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-		__free_page(dev_priv->gart_info.dummy.page);
-		dev_priv->gart_info.dummy.page = NULL;
-	}
-}
-
-uint32_t
-nouveau_sgdma_get_physical(struct drm_device *dev, uint32_t offset)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gpuobj = dev_priv->gart_info.sg_ctxdma;
-	int pte = (offset >> NV_CTXDMA_PAGE_SHIFT) + 2;
-
-	BUG_ON(dev_priv->card_type >= NV_50);
-
-	return (nv_ro32(gpuobj, 4 * pte) & ~NV_CTXDMA_PAGE_MASK) |
-		(offset & NV_CTXDMA_PAGE_MASK);
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_software.h b/drivers/gpu/drm/nouveau/nouveau_software.h
deleted file mode 100644
index 709e5ac680e..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_software.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef __NOUVEAU_SOFTWARE_H__
-#define __NOUVEAU_SOFTWARE_H__
-
-struct nouveau_software_priv {
-	struct nouveau_exec_engine base;
-	struct list_head vblank;
-	spinlock_t peephole_lock;
-};
-
-struct nouveau_software_chan {
-	struct list_head flip;
-	struct {
-		struct list_head list;
-		u32 channel;
-		u32 ctxdma;
-		u32 offset;
-		u32 value;
-		u32 head;
-	} vblank;
-};
-
-static inline void
-nouveau_software_context_new(struct nouveau_software_chan *pch)
-{
-	INIT_LIST_HEAD(&pch->flip);
-	INIT_LIST_HEAD(&pch->vblank.list);
-}
-
-static inline void
-nouveau_software_create(struct nouveau_software_priv *psw)
-{
-	INIT_LIST_HEAD(&psw->vblank);
-	spin_lock_init(&psw->peephole_lock);
-}
-
-static inline u16
-nouveau_software_class(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	if (dev_priv->card_type <= NV_04)
-		return 0x006e;
-	if (dev_priv->card_type <= NV_40)
-		return 0x016e;
-	if (dev_priv->card_type <= NV_50)
-		return 0x506e;
-	if (dev_priv->card_type <= NV_E0)
-		return 0x906e;
-	return 0x0000;
-}
-
-int nv04_software_create(struct drm_device *);
-int nv50_software_create(struct drm_device *);
-int nvc0_software_create(struct drm_device *);
-u64 nvc0_software_crtc(struct nouveau_channel *, int crtc);
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_state.c b/drivers/gpu/drm/nouveau/nouveau_state.c
deleted file mode 100644
index c61014442aa..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_state.c
+++ /dev/null
@@ -1,1306 +0,0 @@
-/*
- * Copyright 2005 Stephane Marchesin
- * Copyright 2008 Stuart Bennett
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include <linux/swab.h>
-#include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-#include "drm_crtc_helper.h"
-#include <linux/vgaarb.h>
-#include <linux/vga_switcheroo.h>
-
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_fbcon.h"
-#include "nouveau_ramht.h"
-#include "nouveau_gpio.h"
-#include "nouveau_pm.h"
-#include "nv50_display.h"
-#include "nouveau_fifo.h"
-#include "nouveau_fence.h"
-#include "nouveau_software.h"
-
-static void nouveau_stub_takedown(struct drm_device *dev) {}
-static int nouveau_stub_init(struct drm_device *dev) { return 0; }
-
-static int nouveau_init_engine_ptrs(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_engine *engine = &dev_priv->engine;
-
-	switch (dev_priv->chipset & 0xf0) {
-	case 0x00:
-		engine->instmem.init		= nv04_instmem_init;
-		engine->instmem.takedown	= nv04_instmem_takedown;
-		engine->instmem.suspend		= nv04_instmem_suspend;
-		engine->instmem.resume		= nv04_instmem_resume;
-		engine->instmem.get		= nv04_instmem_get;
-		engine->instmem.put		= nv04_instmem_put;
-		engine->instmem.map		= nv04_instmem_map;
-		engine->instmem.unmap		= nv04_instmem_unmap;
-		engine->instmem.flush		= nv04_instmem_flush;
-		engine->mc.init			= nv04_mc_init;
-		engine->mc.takedown		= nv04_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv04_fb_init;
-		engine->fb.takedown		= nv04_fb_takedown;
-		engine->display.early_init	= nv04_display_early_init;
-		engine->display.late_takedown	= nv04_display_late_takedown;
-		engine->display.create		= nv04_display_create;
-		engine->display.destroy		= nv04_display_destroy;
-		engine->display.init		= nv04_display_init;
-		engine->display.fini		= nv04_display_fini;
-		engine->pm.clocks_get		= nv04_pm_clocks_get;
-		engine->pm.clocks_pre		= nv04_pm_clocks_pre;
-		engine->pm.clocks_set		= nv04_pm_clocks_set;
-		engine->vram.init		= nv04_fb_vram_init;
-		engine->vram.takedown		= nouveau_stub_takedown;
-		engine->vram.flags_valid	= nouveau_mem_flags_valid;
-		break;
-	case 0x10:
-		engine->instmem.init		= nv04_instmem_init;
-		engine->instmem.takedown	= nv04_instmem_takedown;
-		engine->instmem.suspend		= nv04_instmem_suspend;
-		engine->instmem.resume		= nv04_instmem_resume;
-		engine->instmem.get		= nv04_instmem_get;
-		engine->instmem.put		= nv04_instmem_put;
-		engine->instmem.map		= nv04_instmem_map;
-		engine->instmem.unmap		= nv04_instmem_unmap;
-		engine->instmem.flush		= nv04_instmem_flush;
-		engine->mc.init			= nv04_mc_init;
-		engine->mc.takedown		= nv04_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv10_fb_init;
-		engine->fb.takedown		= nv10_fb_takedown;
-		engine->fb.init_tile_region	= nv10_fb_init_tile_region;
-		engine->fb.set_tile_region	= nv10_fb_set_tile_region;
-		engine->fb.free_tile_region	= nv10_fb_free_tile_region;
-		engine->display.early_init	= nv04_display_early_init;
-		engine->display.late_takedown	= nv04_display_late_takedown;
-		engine->display.create		= nv04_display_create;
-		engine->display.destroy		= nv04_display_destroy;
-		engine->display.init		= nv04_display_init;
-		engine->display.fini		= nv04_display_fini;
-		engine->gpio.drive		= nv10_gpio_drive;
-		engine->gpio.sense		= nv10_gpio_sense;
-		engine->pm.clocks_get		= nv04_pm_clocks_get;
-		engine->pm.clocks_pre		= nv04_pm_clocks_pre;
-		engine->pm.clocks_set		= nv04_pm_clocks_set;
-		if (dev_priv->chipset == 0x1a ||
-		    dev_priv->chipset == 0x1f)
-			engine->vram.init	= nv1a_fb_vram_init;
-		else
-			engine->vram.init	= nv10_fb_vram_init;
-		engine->vram.takedown		= nouveau_stub_takedown;
-		engine->vram.flags_valid	= nouveau_mem_flags_valid;
-		break;
-	case 0x20:
-		engine->instmem.init		= nv04_instmem_init;
-		engine->instmem.takedown	= nv04_instmem_takedown;
-		engine->instmem.suspend		= nv04_instmem_suspend;
-		engine->instmem.resume		= nv04_instmem_resume;
-		engine->instmem.get		= nv04_instmem_get;
-		engine->instmem.put		= nv04_instmem_put;
-		engine->instmem.map		= nv04_instmem_map;
-		engine->instmem.unmap		= nv04_instmem_unmap;
-		engine->instmem.flush		= nv04_instmem_flush;
-		engine->mc.init			= nv04_mc_init;
-		engine->mc.takedown		= nv04_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv20_fb_init;
-		engine->fb.takedown		= nv20_fb_takedown;
-		engine->fb.init_tile_region	= nv20_fb_init_tile_region;
-		engine->fb.set_tile_region	= nv20_fb_set_tile_region;
-		engine->fb.free_tile_region	= nv20_fb_free_tile_region;
-		engine->display.early_init	= nv04_display_early_init;
-		engine->display.late_takedown	= nv04_display_late_takedown;
-		engine->display.create		= nv04_display_create;
-		engine->display.destroy		= nv04_display_destroy;
-		engine->display.init		= nv04_display_init;
-		engine->display.fini		= nv04_display_fini;
-		engine->gpio.drive		= nv10_gpio_drive;
-		engine->gpio.sense		= nv10_gpio_sense;
-		engine->pm.clocks_get		= nv04_pm_clocks_get;
-		engine->pm.clocks_pre		= nv04_pm_clocks_pre;
-		engine->pm.clocks_set		= nv04_pm_clocks_set;
-		engine->vram.init		= nv20_fb_vram_init;
-		engine->vram.takedown		= nouveau_stub_takedown;
-		engine->vram.flags_valid	= nouveau_mem_flags_valid;
-		break;
-	case 0x30:
-		engine->instmem.init		= nv04_instmem_init;
-		engine->instmem.takedown	= nv04_instmem_takedown;
-		engine->instmem.suspend		= nv04_instmem_suspend;
-		engine->instmem.resume		= nv04_instmem_resume;
-		engine->instmem.get		= nv04_instmem_get;
-		engine->instmem.put		= nv04_instmem_put;
-		engine->instmem.map		= nv04_instmem_map;
-		engine->instmem.unmap		= nv04_instmem_unmap;
-		engine->instmem.flush		= nv04_instmem_flush;
-		engine->mc.init			= nv04_mc_init;
-		engine->mc.takedown		= nv04_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv30_fb_init;
-		engine->fb.takedown		= nv30_fb_takedown;
-		engine->fb.init_tile_region	= nv30_fb_init_tile_region;
-		engine->fb.set_tile_region	= nv10_fb_set_tile_region;
-		engine->fb.free_tile_region	= nv30_fb_free_tile_region;
-		engine->display.early_init	= nv04_display_early_init;
-		engine->display.late_takedown	= nv04_display_late_takedown;
-		engine->display.create		= nv04_display_create;
-		engine->display.destroy		= nv04_display_destroy;
-		engine->display.init		= nv04_display_init;
-		engine->display.fini		= nv04_display_fini;
-		engine->gpio.drive		= nv10_gpio_drive;
-		engine->gpio.sense		= nv10_gpio_sense;
-		engine->pm.clocks_get		= nv04_pm_clocks_get;
-		engine->pm.clocks_pre		= nv04_pm_clocks_pre;
-		engine->pm.clocks_set		= nv04_pm_clocks_set;
-		engine->pm.voltage_get		= nouveau_voltage_gpio_get;
-		engine->pm.voltage_set		= nouveau_voltage_gpio_set;
-		engine->vram.init		= nv20_fb_vram_init;
-		engine->vram.takedown		= nouveau_stub_takedown;
-		engine->vram.flags_valid	= nouveau_mem_flags_valid;
-		break;
-	case 0x40:
-	case 0x60:
-		engine->instmem.init		= nv04_instmem_init;
-		engine->instmem.takedown	= nv04_instmem_takedown;
-		engine->instmem.suspend		= nv04_instmem_suspend;
-		engine->instmem.resume		= nv04_instmem_resume;
-		engine->instmem.get		= nv04_instmem_get;
-		engine->instmem.put		= nv04_instmem_put;
-		engine->instmem.map		= nv04_instmem_map;
-		engine->instmem.unmap		= nv04_instmem_unmap;
-		engine->instmem.flush		= nv04_instmem_flush;
-		engine->mc.init			= nv40_mc_init;
-		engine->mc.takedown		= nv40_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv40_fb_init;
-		engine->fb.takedown		= nv40_fb_takedown;
-		engine->fb.init_tile_region	= nv30_fb_init_tile_region;
-		engine->fb.set_tile_region	= nv40_fb_set_tile_region;
-		engine->fb.free_tile_region	= nv30_fb_free_tile_region;
-		engine->display.early_init	= nv04_display_early_init;
-		engine->display.late_takedown	= nv04_display_late_takedown;
-		engine->display.create		= nv04_display_create;
-		engine->display.destroy		= nv04_display_destroy;
-		engine->display.init		= nv04_display_init;
-		engine->display.fini		= nv04_display_fini;
-		engine->gpio.init		= nv10_gpio_init;
-		engine->gpio.fini		= nv10_gpio_fini;
-		engine->gpio.drive		= nv10_gpio_drive;
-		engine->gpio.sense		= nv10_gpio_sense;
-		engine->gpio.irq_enable		= nv10_gpio_irq_enable;
-		engine->pm.clocks_get		= nv40_pm_clocks_get;
-		engine->pm.clocks_pre		= nv40_pm_clocks_pre;
-		engine->pm.clocks_set		= nv40_pm_clocks_set;
-		engine->pm.voltage_get		= nouveau_voltage_gpio_get;
-		engine->pm.voltage_set		= nouveau_voltage_gpio_set;
-		engine->pm.temp_get		= nv40_temp_get;
-		engine->pm.pwm_get		= nv40_pm_pwm_get;
-		engine->pm.pwm_set		= nv40_pm_pwm_set;
-		engine->vram.init		= nv40_fb_vram_init;
-		engine->vram.takedown		= nouveau_stub_takedown;
-		engine->vram.flags_valid	= nouveau_mem_flags_valid;
-		break;
-	case 0x50:
-	case 0x80: /* gotta love NVIDIA's consistency.. */
-	case 0x90:
-	case 0xa0:
-		engine->instmem.init		= nv50_instmem_init;
-		engine->instmem.takedown	= nv50_instmem_takedown;
-		engine->instmem.suspend		= nv50_instmem_suspend;
-		engine->instmem.resume		= nv50_instmem_resume;
-		engine->instmem.get		= nv50_instmem_get;
-		engine->instmem.put		= nv50_instmem_put;
-		engine->instmem.map		= nv50_instmem_map;
-		engine->instmem.unmap		= nv50_instmem_unmap;
-		if (dev_priv->chipset == 0x50)
-			engine->instmem.flush	= nv50_instmem_flush;
-		else
-			engine->instmem.flush	= nv84_instmem_flush;
-		engine->mc.init			= nv50_mc_init;
-		engine->mc.takedown		= nv50_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nv50_fb_init;
-		engine->fb.takedown		= nv50_fb_takedown;
-		engine->display.early_init	= nv50_display_early_init;
-		engine->display.late_takedown	= nv50_display_late_takedown;
-		engine->display.create		= nv50_display_create;
-		engine->display.destroy		= nv50_display_destroy;
-		engine->display.init		= nv50_display_init;
-		engine->display.fini		= nv50_display_fini;
-		engine->gpio.init		= nv50_gpio_init;
-		engine->gpio.fini		= nv50_gpio_fini;
-		engine->gpio.drive		= nv50_gpio_drive;
-		engine->gpio.sense		= nv50_gpio_sense;
-		engine->gpio.irq_enable		= nv50_gpio_irq_enable;
-		switch (dev_priv->chipset) {
-		case 0x84:
-		case 0x86:
-		case 0x92:
-		case 0x94:
-		case 0x96:
-		case 0x98:
-		case 0xa0:
-		case 0xaa:
-		case 0xac:
-		case 0x50:
-			engine->pm.clocks_get	= nv50_pm_clocks_get;
-			engine->pm.clocks_pre	= nv50_pm_clocks_pre;
-			engine->pm.clocks_set	= nv50_pm_clocks_set;
-			break;
-		default:
-			engine->pm.clocks_get	= nva3_pm_clocks_get;
-			engine->pm.clocks_pre	= nva3_pm_clocks_pre;
-			engine->pm.clocks_set	= nva3_pm_clocks_set;
-			break;
-		}
-		engine->pm.voltage_get		= nouveau_voltage_gpio_get;
-		engine->pm.voltage_set		= nouveau_voltage_gpio_set;
-		if (dev_priv->chipset >= 0x84)
-			engine->pm.temp_get	= nv84_temp_get;
-		else
-			engine->pm.temp_get	= nv40_temp_get;
-		engine->pm.pwm_get		= nv50_pm_pwm_get;
-		engine->pm.pwm_set		= nv50_pm_pwm_set;
-		engine->vram.init		= nv50_vram_init;
-		engine->vram.takedown		= nv50_vram_fini;
-		engine->vram.get		= nv50_vram_new;
-		engine->vram.put		= nv50_vram_del;
-		engine->vram.flags_valid	= nv50_vram_flags_valid;
-		break;
-	case 0xc0:
-		engine->instmem.init		= nvc0_instmem_init;
-		engine->instmem.takedown	= nvc0_instmem_takedown;
-		engine->instmem.suspend		= nvc0_instmem_suspend;
-		engine->instmem.resume		= nvc0_instmem_resume;
-		engine->instmem.get		= nv50_instmem_get;
-		engine->instmem.put		= nv50_instmem_put;
-		engine->instmem.map		= nv50_instmem_map;
-		engine->instmem.unmap		= nv50_instmem_unmap;
-		engine->instmem.flush		= nv84_instmem_flush;
-		engine->mc.init			= nv50_mc_init;
-		engine->mc.takedown		= nv50_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nvc0_fb_init;
-		engine->fb.takedown		= nvc0_fb_takedown;
-		engine->display.early_init	= nv50_display_early_init;
-		engine->display.late_takedown	= nv50_display_late_takedown;
-		engine->display.create		= nv50_display_create;
-		engine->display.destroy		= nv50_display_destroy;
-		engine->display.init		= nv50_display_init;
-		engine->display.fini		= nv50_display_fini;
-		engine->gpio.init		= nv50_gpio_init;
-		engine->gpio.fini		= nv50_gpio_fini;
-		engine->gpio.drive		= nv50_gpio_drive;
-		engine->gpio.sense		= nv50_gpio_sense;
-		engine->gpio.irq_enable		= nv50_gpio_irq_enable;
-		engine->vram.init		= nvc0_vram_init;
-		engine->vram.takedown		= nv50_vram_fini;
-		engine->vram.get		= nvc0_vram_new;
-		engine->vram.put		= nv50_vram_del;
-		engine->vram.flags_valid	= nvc0_vram_flags_valid;
-		engine->pm.temp_get		= nv84_temp_get;
-		engine->pm.clocks_get		= nvc0_pm_clocks_get;
-		engine->pm.clocks_pre		= nvc0_pm_clocks_pre;
-		engine->pm.clocks_set		= nvc0_pm_clocks_set;
-		engine->pm.voltage_get		= nouveau_voltage_gpio_get;
-		engine->pm.voltage_set		= nouveau_voltage_gpio_set;
-		engine->pm.pwm_get		= nv50_pm_pwm_get;
-		engine->pm.pwm_set		= nv50_pm_pwm_set;
-		break;
-	case 0xd0:
-		engine->instmem.init		= nvc0_instmem_init;
-		engine->instmem.takedown	= nvc0_instmem_takedown;
-		engine->instmem.suspend		= nvc0_instmem_suspend;
-		engine->instmem.resume		= nvc0_instmem_resume;
-		engine->instmem.get		= nv50_instmem_get;
-		engine->instmem.put		= nv50_instmem_put;
-		engine->instmem.map		= nv50_instmem_map;
-		engine->instmem.unmap		= nv50_instmem_unmap;
-		engine->instmem.flush		= nv84_instmem_flush;
-		engine->mc.init			= nv50_mc_init;
-		engine->mc.takedown		= nv50_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nvc0_fb_init;
-		engine->fb.takedown		= nvc0_fb_takedown;
-		engine->display.early_init	= nouveau_stub_init;
-		engine->display.late_takedown	= nouveau_stub_takedown;
-		engine->display.create		= nvd0_display_create;
-		engine->display.destroy		= nvd0_display_destroy;
-		engine->display.init		= nvd0_display_init;
-		engine->display.fini		= nvd0_display_fini;
-		engine->gpio.init		= nv50_gpio_init;
-		engine->gpio.fini		= nv50_gpio_fini;
-		engine->gpio.drive		= nvd0_gpio_drive;
-		engine->gpio.sense		= nvd0_gpio_sense;
-		engine->gpio.irq_enable		= nv50_gpio_irq_enable;
-		engine->vram.init		= nvc0_vram_init;
-		engine->vram.takedown		= nv50_vram_fini;
-		engine->vram.get		= nvc0_vram_new;
-		engine->vram.put		= nv50_vram_del;
-		engine->vram.flags_valid	= nvc0_vram_flags_valid;
-		engine->pm.temp_get		= nv84_temp_get;
-		engine->pm.clocks_get		= nvc0_pm_clocks_get;
-		engine->pm.clocks_pre		= nvc0_pm_clocks_pre;
-		engine->pm.clocks_set		= nvc0_pm_clocks_set;
-		engine->pm.voltage_get		= nouveau_voltage_gpio_get;
-		engine->pm.voltage_set		= nouveau_voltage_gpio_set;
-		break;
-	case 0xe0:
-		engine->instmem.init		= nvc0_instmem_init;
-		engine->instmem.takedown	= nvc0_instmem_takedown;
-		engine->instmem.suspend		= nvc0_instmem_suspend;
-		engine->instmem.resume		= nvc0_instmem_resume;
-		engine->instmem.get		= nv50_instmem_get;
-		engine->instmem.put		= nv50_instmem_put;
-		engine->instmem.map		= nv50_instmem_map;
-		engine->instmem.unmap		= nv50_instmem_unmap;
-		engine->instmem.flush		= nv84_instmem_flush;
-		engine->mc.init			= nv50_mc_init;
-		engine->mc.takedown		= nv50_mc_takedown;
-		engine->timer.init		= nv04_timer_init;
-		engine->timer.read		= nv04_timer_read;
-		engine->timer.takedown		= nv04_timer_takedown;
-		engine->fb.init			= nvc0_fb_init;
-		engine->fb.takedown		= nvc0_fb_takedown;
-		engine->display.early_init	= nouveau_stub_init;
-		engine->display.late_takedown	= nouveau_stub_takedown;
-		engine->display.create		= nvd0_display_create;
-		engine->display.destroy		= nvd0_display_destroy;
-		engine->display.init		= nvd0_display_init;
-		engine->display.fini		= nvd0_display_fini;
-		engine->gpio.init		= nv50_gpio_init;
-		engine->gpio.fini		= nv50_gpio_fini;
-		engine->gpio.drive		= nvd0_gpio_drive;
-		engine->gpio.sense		= nvd0_gpio_sense;
-		engine->gpio.irq_enable		= nv50_gpio_irq_enable;
-		engine->vram.init		= nvc0_vram_init;
-		engine->vram.takedown		= nv50_vram_fini;
-		engine->vram.get		= nvc0_vram_new;
-		engine->vram.put		= nv50_vram_del;
-		engine->vram.flags_valid	= nvc0_vram_flags_valid;
-		break;
-	default:
-		NV_ERROR(dev, "NV%02x unsupported\n", dev_priv->chipset);
-		return 1;
-	}
-
-	/* headless mode */
-	if (nouveau_modeset == 2) {
-		engine->display.early_init = nouveau_stub_init;
-		engine->display.late_takedown = nouveau_stub_takedown;
-		engine->display.create = nouveau_stub_init;
-		engine->display.init = nouveau_stub_init;
-		engine->display.destroy = nouveau_stub_takedown;
-	}
-
-	return 0;
-}
-
-static unsigned int
-nouveau_vga_set_decode(void *priv, bool state)
-{
-	struct drm_device *dev = priv;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->chipset >= 0x40)
-		nv_wr32(dev, 0x88054, state);
-	else
-		nv_wr32(dev, 0x1854, state);
-
-	if (state)
-		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
-		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-	else
-		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-}
-
-static void nouveau_switcheroo_set_state(struct pci_dev *pdev,
-					 enum vga_switcheroo_state state)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-	pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
-	if (state == VGA_SWITCHEROO_ON) {
-		printk(KERN_ERR "VGA switcheroo: switched nouveau on\n");
-		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-		nouveau_pci_resume(pdev);
-		drm_kms_helper_poll_enable(dev);
-		dev->switch_power_state = DRM_SWITCH_POWER_ON;
-	} else {
-		printk(KERN_ERR "VGA switcheroo: switched nouveau off\n");
-		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-		drm_kms_helper_poll_disable(dev);
-		nouveau_switcheroo_optimus_dsm();
-		nouveau_pci_suspend(pdev, pmm);
-		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
-	}
-}
-
-static void nouveau_switcheroo_reprobe(struct pci_dev *pdev)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-	nouveau_fbcon_output_poll_changed(dev);
-}
-
-static bool nouveau_switcheroo_can_switch(struct pci_dev *pdev)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-	bool can_switch;
-
-	spin_lock(&dev->count_lock);
-	can_switch = (dev->open_count == 0);
-	spin_unlock(&dev->count_lock);
-	return can_switch;
-}
-
-static void
-nouveau_card_channel_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->channel)
-		nouveau_channel_put_unlocked(&dev_priv->channel);
-}
-
-static int
-nouveau_card_channel_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan;
-	int ret;
-
-	ret = nouveau_channel_alloc(dev, &chan, NULL, NvDmaFB, NvDmaTT);
-	dev_priv->channel = chan;
-	if (ret)
-		return ret;
-	mutex_unlock(&dev_priv->channel->mutex);
-
-	nouveau_bo_move_init(chan);
-	return 0;
-}
-
-static const struct vga_switcheroo_client_ops nouveau_switcheroo_ops = {
-	.set_gpu_state = nouveau_switcheroo_set_state,
-	.reprobe = nouveau_switcheroo_reprobe,
-	.can_switch = nouveau_switcheroo_can_switch,
-};
-
-int
-nouveau_card_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_engine *engine;
-	int ret, e = 0;
-
-	vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
-	vga_switcheroo_register_client(dev->pdev, &nouveau_switcheroo_ops);
-
-	/* Initialise internal driver API hooks */
-	ret = nouveau_init_engine_ptrs(dev);
-	if (ret)
-		goto out;
-	engine = &dev_priv->engine;
-	spin_lock_init(&dev_priv->channels.lock);
-	spin_lock_init(&dev_priv->tile.lock);
-	spin_lock_init(&dev_priv->context_switch_lock);
-	spin_lock_init(&dev_priv->vm_lock);
-
-	/* Make the CRTCs and I2C buses accessible */
-	ret = engine->display.early_init(dev);
-	if (ret)
-		goto out;
-
-	/* Parse BIOS tables / Run init tables if card not POSTed */
-	ret = nouveau_bios_init(dev);
-	if (ret)
-		goto out_display_early;
-
-	/* workaround an odd issue on nvc1 by disabling the device's
-	 * nosnoop capability.  hopefully won't cause issues until a
-	 * better fix is found - assuming there is one...
-	 */
-	if (dev_priv->chipset == 0xc1) {
-		nv_mask(dev, 0x00088080, 0x00000800, 0x00000000);
-	}
-
-	/* PMC */
-	ret = engine->mc.init(dev);
-	if (ret)
-		goto out_bios;
-
-	/* PTIMER */
-	ret = engine->timer.init(dev);
-	if (ret)
-		goto out_mc;
-
-	/* PFB */
-	ret = engine->fb.init(dev);
-	if (ret)
-		goto out_timer;
-
-	ret = engine->vram.init(dev);
-	if (ret)
-		goto out_fb;
-
-	/* PGPIO */
-	ret = nouveau_gpio_create(dev);
-	if (ret)
-		goto out_vram;
-
-	ret = nouveau_gpuobj_init(dev);
-	if (ret)
-		goto out_gpio;
-
-	ret = engine->instmem.init(dev);
-	if (ret)
-		goto out_gpuobj;
-
-	ret = nouveau_mem_vram_init(dev);
-	if (ret)
-		goto out_instmem;
-
-	ret = nouveau_mem_gart_init(dev);
-	if (ret)
-		goto out_ttmvram;
-
-	if (!dev_priv->noaccel) {
-		switch (dev_priv->card_type) {
-		case NV_04:
-			nv04_fifo_create(dev);
-			break;
-		case NV_10:
-		case NV_20:
-		case NV_30:
-			if (dev_priv->chipset < 0x17)
-				nv10_fifo_create(dev);
-			else
-				nv17_fifo_create(dev);
-			break;
-		case NV_40:
-			nv40_fifo_create(dev);
-			break;
-		case NV_50:
-			if (dev_priv->chipset == 0x50)
-				nv50_fifo_create(dev);
-			else
-				nv84_fifo_create(dev);
-			break;
-		case NV_C0:
-		case NV_D0:
-			nvc0_fifo_create(dev);
-			break;
-		case NV_E0:
-			nve0_fifo_create(dev);
-			break;
-		default:
-			break;
-		}
-
-		switch (dev_priv->card_type) {
-		case NV_04:
-			nv04_fence_create(dev);
-			break;
-		case NV_10:
-		case NV_20:
-		case NV_30:
-		case NV_40:
-		case NV_50:
-			if (dev_priv->chipset < 0x84)
-				nv10_fence_create(dev);
-			else
-				nv84_fence_create(dev);
-			break;
-		case NV_C0:
-		case NV_D0:
-		case NV_E0:
-			nvc0_fence_create(dev);
-			break;
-		default:
-			break;
-		}
-
-		switch (dev_priv->card_type) {
-		case NV_04:
-		case NV_10:
-		case NV_20:
-		case NV_30:
-		case NV_40:
-			nv04_software_create(dev);
-			break;
-		case NV_50:
-			nv50_software_create(dev);
-			break;
-		case NV_C0:
-		case NV_D0:
-		case NV_E0:
-			nvc0_software_create(dev);
-			break;
-		default:
-			break;
-		}
-
-		switch (dev_priv->card_type) {
-		case NV_04:
-			nv04_graph_create(dev);
-			break;
-		case NV_10:
-			nv10_graph_create(dev);
-			break;
-		case NV_20:
-		case NV_30:
-			nv20_graph_create(dev);
-			break;
-		case NV_40:
-			nv40_graph_create(dev);
-			break;
-		case NV_50:
-			nv50_graph_create(dev);
-			break;
-		case NV_C0:
-		case NV_D0:
-			nvc0_graph_create(dev);
-			break;
-		case NV_E0:
-			nve0_graph_create(dev);
-			break;
-		default:
-			break;
-		}
-
-		switch (dev_priv->chipset) {
-		case 0x84:
-		case 0x86:
-		case 0x92:
-		case 0x94:
-		case 0x96:
-		case 0xa0:
-			nv84_crypt_create(dev);
-			break;
-		case 0x98:
-		case 0xaa:
-		case 0xac:
-			nv98_crypt_create(dev);
-			break;
-		}
-
-		switch (dev_priv->card_type) {
-		case NV_50:
-			switch (dev_priv->chipset) {
-			case 0xa3:
-			case 0xa5:
-			case 0xa8:
-				nva3_copy_create(dev);
-				break;
-			}
-			break;
-		case NV_C0:
-			if (!(nv_rd32(dev, 0x022500) & 0x00000200))
-				nvc0_copy_create(dev, 1);
-		case NV_D0:
-			if (!(nv_rd32(dev, 0x022500) & 0x00000100))
-				nvc0_copy_create(dev, 0);
-			break;
-		default:
-			break;
-		}
-
-		if (dev_priv->chipset >= 0xa3 || dev_priv->chipset == 0x98) {
-			nv84_bsp_create(dev);
-			nv84_vp_create(dev);
-			nv98_ppp_create(dev);
-		} else
-		if (dev_priv->chipset >= 0x84) {
-			nv50_mpeg_create(dev);
-			nv84_bsp_create(dev);
-			nv84_vp_create(dev);
-		} else
-		if (dev_priv->chipset >= 0x50) {
-			nv50_mpeg_create(dev);
-		} else
-		if (dev_priv->card_type == NV_40 ||
-		    dev_priv->chipset == 0x31 ||
-		    dev_priv->chipset == 0x34 ||
-		    dev_priv->chipset == 0x36) {
-			nv31_mpeg_create(dev);
-		}
-
-		for (e = 0; e < NVOBJ_ENGINE_NR; e++) {
-			if (dev_priv->eng[e]) {
-				ret = dev_priv->eng[e]->init(dev, e);
-				if (ret)
-					goto out_engine;
-			}
-		}
-	}
-
-	ret = nouveau_irq_init(dev);
-	if (ret)
-		goto out_engine;
-
-	ret = nouveau_display_create(dev);
-	if (ret)
-		goto out_irq;
-
-	nouveau_backlight_init(dev);
-	nouveau_pm_init(dev);
-
-	if (dev_priv->eng[NVOBJ_ENGINE_GR]) {
-		ret = nouveau_card_channel_init(dev);
-		if (ret)
-			goto out_pm;
-	}
-
-	if (dev->mode_config.num_crtc) {
-		ret = nouveau_display_init(dev);
-		if (ret)
-			goto out_chan;
-
-		nouveau_fbcon_init(dev);
-	}
-
-	return 0;
-
-out_chan:
-	nouveau_card_channel_fini(dev);
-out_pm:
-	nouveau_pm_fini(dev);
-	nouveau_backlight_exit(dev);
-	nouveau_display_destroy(dev);
-out_irq:
-	nouveau_irq_fini(dev);
-out_engine:
-	if (!dev_priv->noaccel) {
-		for (e = e - 1; e >= 0; e--) {
-			if (!dev_priv->eng[e])
-				continue;
-			dev_priv->eng[e]->fini(dev, e, false);
-			dev_priv->eng[e]->destroy(dev,e );
-		}
-	}
-	nouveau_mem_gart_fini(dev);
-out_ttmvram:
-	nouveau_mem_vram_fini(dev);
-out_instmem:
-	engine->instmem.takedown(dev);
-out_gpuobj:
-	nouveau_gpuobj_takedown(dev);
-out_gpio:
-	nouveau_gpio_destroy(dev);
-out_vram:
-	engine->vram.takedown(dev);
-out_fb:
-	engine->fb.takedown(dev);
-out_timer:
-	engine->timer.takedown(dev);
-out_mc:
-	engine->mc.takedown(dev);
-out_bios:
-	nouveau_bios_takedown(dev);
-out_display_early:
-	engine->display.late_takedown(dev);
-out:
-	vga_switcheroo_unregister_client(dev->pdev);
-	vga_client_register(dev->pdev, NULL, NULL, NULL);
-	return ret;
-}
-
-static void nouveau_card_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_engine *engine = &dev_priv->engine;
-	int e;
-
-	if (dev->mode_config.num_crtc) {
-		nouveau_fbcon_fini(dev);
-		nouveau_display_fini(dev);
-	}
-
-	nouveau_card_channel_fini(dev);
-	nouveau_pm_fini(dev);
-	nouveau_backlight_exit(dev);
-	nouveau_display_destroy(dev);
-
-	if (!dev_priv->noaccel) {
-		for (e = NVOBJ_ENGINE_NR - 1; e >= 0; e--) {
-			if (dev_priv->eng[e]) {
-				dev_priv->eng[e]->fini(dev, e, false);
-				dev_priv->eng[e]->destroy(dev,e );
-			}
-		}
-	}
-
-	if (dev_priv->vga_ram) {
-		nouveau_bo_unpin(dev_priv->vga_ram);
-		nouveau_bo_ref(NULL, &dev_priv->vga_ram);
-	}
-
-	mutex_lock(&dev->struct_mutex);
-	ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_VRAM);
-	ttm_bo_clean_mm(&dev_priv->ttm.bdev, TTM_PL_TT);
-	mutex_unlock(&dev->struct_mutex);
-	nouveau_mem_gart_fini(dev);
-	nouveau_mem_vram_fini(dev);
-
-	engine->instmem.takedown(dev);
-	nouveau_gpuobj_takedown(dev);
-
-	nouveau_gpio_destroy(dev);
-	engine->vram.takedown(dev);
-	engine->fb.takedown(dev);
-	engine->timer.takedown(dev);
-	engine->mc.takedown(dev);
-
-	nouveau_bios_takedown(dev);
-	engine->display.late_takedown(dev);
-
-	nouveau_irq_fini(dev);
-
-	vga_switcheroo_unregister_client(dev->pdev);
-	vga_client_register(dev->pdev, NULL, NULL, NULL);
-}
-
-int
-nouveau_open(struct drm_device *dev, struct drm_file *file_priv)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fpriv *fpriv;
-	int ret;
-
-	fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
-	if (unlikely(!fpriv))
-		return -ENOMEM;
-
-	spin_lock_init(&fpriv->lock);
-	INIT_LIST_HEAD(&fpriv->channels);
-
-	if (dev_priv->card_type == NV_50) {
-		ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
-				     &fpriv->vm);
-		if (ret) {
-			kfree(fpriv);
-			return ret;
-		}
-	} else
-	if (dev_priv->card_type >= NV_C0) {
-		ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
-				     &fpriv->vm);
-		if (ret) {
-			kfree(fpriv);
-			return ret;
-		}
-	}
-
-	file_priv->driver_priv = fpriv;
-	return 0;
-}
-
-/* here a client dies, release the stuff that was allocated for its
- * file_priv */
-void nouveau_preclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-	nouveau_channel_cleanup(dev, file_priv);
-}
-
-void
-nouveau_postclose(struct drm_device *dev, struct drm_file *file_priv)
-{
-	struct nouveau_fpriv *fpriv = nouveau_fpriv(file_priv);
-	nouveau_vm_ref(NULL, &fpriv->vm, NULL);
-	kfree(fpriv);
-}
-
-/* first module load, setup the mmio/fb mapping */
-/* KMS: we need mmio at load time, not when the first drm client opens. */
-int nouveau_firstopen(struct drm_device *dev)
-{
-	return 0;
-}
-
-/* if we have an OF card, copy vbios to RAMIN */
-static void nouveau_OF_copy_vbios_to_ramin(struct drm_device *dev)
-{
-#if defined(__powerpc__)
-	int size, i;
-	const uint32_t *bios;
-	struct device_node *dn = pci_device_to_OF_node(dev->pdev);
-	if (!dn) {
-		NV_INFO(dev, "Unable to get the OF node\n");
-		return;
-	}
-
-	bios = of_get_property(dn, "NVDA,BMP", &size);
-	if (bios) {
-		for (i = 0; i < size; i += 4)
-			nv_wi32(dev, i, bios[i/4]);
-		NV_INFO(dev, "OF bios successfully copied (%d bytes)\n", size);
-	} else {
-		NV_INFO(dev, "Unable to get the OF bios\n");
-	}
-#endif
-}
-
-static struct apertures_struct *nouveau_get_apertures(struct drm_device *dev)
-{
-	struct pci_dev *pdev = dev->pdev;
-	struct apertures_struct *aper = alloc_apertures(3);
-	if (!aper)
-		return NULL;
-
-	aper->ranges[0].base = pci_resource_start(pdev, 1);
-	aper->ranges[0].size = pci_resource_len(pdev, 1);
-	aper->count = 1;
-
-	if (pci_resource_len(pdev, 2)) {
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 2);
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 2);
-		aper->count++;
-	}
-
-	if (pci_resource_len(pdev, 3)) {
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 3);
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 3);
-		aper->count++;
-	}
-
-	return aper;
-}
-
-static int nouveau_remove_conflicting_drivers(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	bool primary = false;
-	dev_priv->apertures = nouveau_get_apertures(dev);
-	if (!dev_priv->apertures)
-		return -ENOMEM;
-
-#ifdef CONFIG_X86
-	primary = dev->pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
-#endif
-
-	remove_conflicting_framebuffers(dev_priv->apertures, "nouveaufb", primary);
-	return 0;
-}
-
-int nouveau_load(struct drm_device *dev, unsigned long flags)
-{
-	struct drm_nouveau_private *dev_priv;
-	unsigned long long offset, length;
-	uint32_t reg0 = ~0, strap;
-	int ret;
-
-	dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
-	if (!dev_priv) {
-		ret = -ENOMEM;
-		goto err_out;
-	}
-	dev->dev_private = dev_priv;
-	dev_priv->dev = dev;
-
-	pci_set_master(dev->pdev);
-
-	dev_priv->flags = flags & NOUVEAU_FLAGS;
-
-	NV_DEBUG(dev, "vendor: 0x%X device: 0x%X class: 0x%X\n",
-		 dev->pci_vendor, dev->pci_device, dev->pdev->class);
-
-	/* first up, map the start of mmio and determine the chipset */
-	dev_priv->mmio = ioremap(pci_resource_start(dev->pdev, 0), PAGE_SIZE);
-	if (dev_priv->mmio) {
-#ifdef __BIG_ENDIAN
-		/* put the card into big-endian mode if it's not */
-		if (nv_rd32(dev, NV03_PMC_BOOT_1) != 0x01000001)
-			nv_wr32(dev, NV03_PMC_BOOT_1, 0x01000001);
-		DRM_MEMORYBARRIER();
-#endif
-
-		/* determine chipset and derive architecture from it */
-		reg0 = nv_rd32(dev, NV03_PMC_BOOT_0);
-		if ((reg0 & 0x0f000000) > 0) {
-			dev_priv->chipset = (reg0 & 0xff00000) >> 20;
-			switch (dev_priv->chipset & 0xf0) {
-			case 0x10:
-			case 0x20:
-			case 0x30:
-				dev_priv->card_type = dev_priv->chipset & 0xf0;
-				break;
-			case 0x40:
-			case 0x60:
-				dev_priv->card_type = NV_40;
-				break;
-			case 0x50:
-			case 0x80:
-			case 0x90:
-			case 0xa0:
-				dev_priv->card_type = NV_50;
-				break;
-			case 0xc0:
-				dev_priv->card_type = NV_C0;
-				break;
-			case 0xd0:
-				dev_priv->card_type = NV_D0;
-				break;
-			case 0xe0:
-				dev_priv->card_type = NV_E0;
-				break;
-			default:
-				break;
-			}
-		} else
-		if ((reg0 & 0xff00fff0) == 0x20004000) {
-			if (reg0 & 0x00f00000)
-				dev_priv->chipset = 0x05;
-			else
-				dev_priv->chipset = 0x04;
-			dev_priv->card_type = NV_04;
-		}
-
-		iounmap(dev_priv->mmio);
-	}
-
-	if (!dev_priv->card_type) {
-		NV_ERROR(dev, "unsupported chipset 0x%08x\n", reg0);
-		ret = -EINVAL;
-		goto err_priv;
-	}
-
-	NV_INFO(dev, "Detected an NV%02x generation card (0x%08x)\n",
-		     dev_priv->card_type, reg0);
-
-	/* map the mmio regs, limiting the amount to preserve vmap space */
-	offset = pci_resource_start(dev->pdev, 0);
-	length = pci_resource_len(dev->pdev, 0);
-	if (dev_priv->card_type < NV_E0)
-		length = min(length, (unsigned long long)0x00800000);
-
-	dev_priv->mmio = ioremap(offset, length);
-	if (!dev_priv->mmio) {
-		NV_ERROR(dev, "Unable to initialize the mmio mapping. "
-			 "Please report your setup to " DRIVER_EMAIL "\n");
-		ret = -EINVAL;
-		goto err_priv;
-	}
-	NV_DEBUG(dev, "regs mapped ok at 0x%llx\n", offset);
-
-	/* determine frequency of timing crystal */
-	strap = nv_rd32(dev, 0x101000);
-	if ( dev_priv->chipset < 0x17 ||
-	    (dev_priv->chipset >= 0x20 && dev_priv->chipset <= 0x25))
-		strap &= 0x00000040;
-	else
-		strap &= 0x00400040;
-
-	switch (strap) {
-	case 0x00000000: dev_priv->crystal = 13500; break;
-	case 0x00000040: dev_priv->crystal = 14318; break;
-	case 0x00400000: dev_priv->crystal = 27000; break;
-	case 0x00400040: dev_priv->crystal = 25000; break;
-	}
-
-	NV_DEBUG(dev, "crystal freq: %dKHz\n", dev_priv->crystal);
-
-	/* Determine whether we'll attempt acceleration or not, some
-	 * cards are disabled by default here due to them being known
-	 * non-functional, or never been tested due to lack of hw.
-	 */
-	dev_priv->noaccel = !!nouveau_noaccel;
-	if (nouveau_noaccel == -1) {
-		switch (dev_priv->chipset) {
-		case 0xd9: /* known broken */
-		case 0xe4: /* needs binary driver firmware */
-		case 0xe7: /* needs binary driver firmware */
-			NV_INFO(dev, "acceleration disabled by default, pass "
-				     "noaccel=0 to force enable\n");
-			dev_priv->noaccel = true;
-			break;
-		default:
-			dev_priv->noaccel = false;
-			break;
-		}
-	}
-
-	ret = nouveau_remove_conflicting_drivers(dev);
-	if (ret)
-		goto err_mmio;
-
-	/* Map PRAMIN BAR, or on older cards, the aperture within BAR0 */
-	if (dev_priv->card_type >= NV_40) {
-		int ramin_bar = 2;
-		if (pci_resource_len(dev->pdev, ramin_bar) == 0)
-			ramin_bar = 3;
-
-		dev_priv->ramin_size = pci_resource_len(dev->pdev, ramin_bar);
-		dev_priv->ramin =
-			ioremap(pci_resource_start(dev->pdev, ramin_bar),
-				dev_priv->ramin_size);
-		if (!dev_priv->ramin) {
-			NV_ERROR(dev, "Failed to map PRAMIN BAR\n");
-			ret = -ENOMEM;
-			goto err_mmio;
-		}
-	} else {
-		dev_priv->ramin_size = 1 * 1024 * 1024;
-		dev_priv->ramin = ioremap(offset + NV_RAMIN,
-					  dev_priv->ramin_size);
-		if (!dev_priv->ramin) {
-			NV_ERROR(dev, "Failed to map BAR0 PRAMIN.\n");
-			ret = -ENOMEM;
-			goto err_mmio;
-		}
-	}
-
-	nouveau_OF_copy_vbios_to_ramin(dev);
-
-	/* Special flags */
-	if (dev->pci_device == 0x01a0)
-		dev_priv->flags |= NV_NFORCE;
-	else if (dev->pci_device == 0x01f0)
-		dev_priv->flags |= NV_NFORCE2;
-
-	/* For kernel modesetting, init card now and bring up fbcon */
-	ret = nouveau_card_init(dev);
-	if (ret)
-		goto err_ramin;
-
-	return 0;
-
-err_ramin:
-	iounmap(dev_priv->ramin);
-err_mmio:
-	iounmap(dev_priv->mmio);
-err_priv:
-	kfree(dev_priv);
-	dev->dev_private = NULL;
-err_out:
-	return ret;
-}
-
-void nouveau_lastclose(struct drm_device *dev)
-{
-	vga_switcheroo_process_delayed_switch();
-}
-
-int nouveau_unload(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nouveau_card_takedown(dev);
-
-	iounmap(dev_priv->mmio);
-	iounmap(dev_priv->ramin);
-
-	kfree(dev_priv);
-	dev->dev_private = NULL;
-	return 0;
-}
-
-/* Wait until (value(reg) & mask) == val, up until timeout has hit */
-bool
-nouveau_wait_eq(struct drm_device *dev, uint64_t timeout,
-		uint32_t reg, uint32_t mask, uint32_t val)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-	uint64_t start = ptimer->read(dev);
-
-	do {
-		if ((nv_rd32(dev, reg) & mask) == val)
-			return true;
-	} while (ptimer->read(dev) - start < timeout);
-
-	return false;
-}
-
-/* Wait until (value(reg) & mask) != val, up until timeout has hit */
-bool
-nouveau_wait_ne(struct drm_device *dev, uint64_t timeout,
-		uint32_t reg, uint32_t mask, uint32_t val)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-	uint64_t start = ptimer->read(dev);
-
-	do {
-		if ((nv_rd32(dev, reg) & mask) != val)
-			return true;
-	} while (ptimer->read(dev) - start < timeout);
-
-	return false;
-}
-
-/* Wait until cond(data) == true, up until timeout has hit */
-bool
-nouveau_wait_cb(struct drm_device *dev, u64 timeout,
-		bool (*cond)(void *), void *data)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-	u64 start = ptimer->read(dev);
-
-	do {
-		if (cond(data) == true)
-			return true;
-	} while (ptimer->read(dev) - start < timeout);
-
-	return false;
-}
-
-/* Waits for PGRAPH to go completely idle */
-bool nouveau_wait_for_idle(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t mask = ~0;
-
-	if (dev_priv->card_type == NV_40)
-		mask &= ~NV40_PGRAPH_STATUS_SYNC_STALL;
-
-	if (!nv_wait(dev, NV04_PGRAPH_STATUS, mask, 0)) {
-		NV_ERROR(dev, "PGRAPH idle timed out with status 0x%08x\n",
-			 nv_rd32(dev, NV04_PGRAPH_STATUS));
-		return false;
-	}
-
-	return true;
-}
-
diff --git a/drivers/gpu/drm/nouveau/nouveau_temp.c b/drivers/gpu/drm/nouveau/nouveau_temp.c
deleted file mode 100644
index 0f5a3016055..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_temp.c
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * Copyright 2010 PathScale inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Martin Peres
- */
-
-#include <linux/module.h>
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_pm.h"
-
-static void
-nouveau_temp_vbios_parse(struct drm_device *dev, u8 *temp)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
-	struct nouveau_pm_threshold_temp *temps = &pm->threshold_temp;
-	int i, headerlen, recordlen, entries;
-
-	if (!temp) {
-		NV_DEBUG(dev, "temperature table pointer invalid\n");
-		return;
-	}
-
-	/* Set the default sensor's contants */
-	sensor->offset_constant = 0;
-	sensor->offset_mult = 0;
-	sensor->offset_div = 1;
-	sensor->slope_mult = 1;
-	sensor->slope_div = 1;
-
-	/* Set the default temperature thresholds */
-	temps->critical = 110;
-	temps->down_clock = 100;
-	temps->fan_boost = 90;
-
-	/* Set the default range for the pwm fan */
-	pm->fan.min_duty = 30;
-	pm->fan.max_duty = 100;
-
-	/* Set the known default values to setup the temperature sensor */
-	if (dev_priv->card_type >= NV_40) {
-		switch (dev_priv->chipset) {
-		case 0x43:
-			sensor->offset_mult = 32060;
-			sensor->offset_div = 1000;
-			sensor->slope_mult = 792;
-			sensor->slope_div = 1000;
-			break;
-
-		case 0x44:
-		case 0x47:
-		case 0x4a:
-			sensor->offset_mult = 27839;
-			sensor->offset_div = 1000;
-			sensor->slope_mult = 780;
-			sensor->slope_div = 1000;
-			break;
-
-		case 0x46:
-			sensor->offset_mult = -24775;
-			sensor->offset_div = 100;
-			sensor->slope_mult = 467;
-			sensor->slope_div = 10000;
-			break;
-
-		case 0x49:
-			sensor->offset_mult = -25051;
-			sensor->offset_div = 100;
-			sensor->slope_mult = 458;
-			sensor->slope_div = 10000;
-			break;
-
-		case 0x4b:
-			sensor->offset_mult = -24088;
-			sensor->offset_div = 100;
-			sensor->slope_mult = 442;
-			sensor->slope_div = 10000;
-			break;
-
-		case 0x50:
-			sensor->offset_mult = -22749;
-			sensor->offset_div = 100;
-			sensor->slope_mult = 431;
-			sensor->slope_div = 10000;
-			break;
-
-		case 0x67:
-			sensor->offset_mult = -26149;
-			sensor->offset_div = 100;
-			sensor->slope_mult = 484;
-			sensor->slope_div = 10000;
-			break;
-		}
-	}
-
-	headerlen = temp[1];
-	recordlen = temp[2];
-	entries = temp[3];
-	temp = temp + headerlen;
-
-	/* Read the entries from the table */
-	for (i = 0; i < entries; i++) {
-		s16 value = ROM16(temp[1]);
-
-		switch (temp[0]) {
-		case 0x01:
-			if ((value & 0x8f) == 0)
-				sensor->offset_constant = (value >> 9) & 0x7f;
-			break;
-
-		case 0x04:
-			if ((value & 0xf00f) == 0xa000) /* core */
-				temps->critical = (value&0x0ff0) >> 4;
-			break;
-
-		case 0x07:
-			if ((value & 0xf00f) == 0xa000) /* core */
-				temps->down_clock = (value&0x0ff0) >> 4;
-			break;
-
-		case 0x08:
-			if ((value & 0xf00f) == 0xa000) /* core */
-				temps->fan_boost = (value&0x0ff0) >> 4;
-			break;
-
-		case 0x10:
-			sensor->offset_mult = value;
-			break;
-
-		case 0x11:
-			sensor->offset_div = value;
-			break;
-
-		case 0x12:
-			sensor->slope_mult = value;
-			break;
-
-		case 0x13:
-			sensor->slope_div = value;
-			break;
-		case 0x22:
-			pm->fan.min_duty = value & 0xff;
-			pm->fan.max_duty = (value & 0xff00) >> 8;
-			break;
-		case 0x26:
-			pm->fan.pwm_freq = value;
-			break;
-		}
-		temp += recordlen;
-	}
-
-	nouveau_temp_safety_checks(dev);
-
-	/* check the fan min/max settings */
-	if (pm->fan.min_duty < 10)
-		pm->fan.min_duty = 10;
-	if (pm->fan.max_duty > 100)
-		pm->fan.max_duty = 100;
-	if (pm->fan.max_duty < pm->fan.min_duty)
-		pm->fan.max_duty = pm->fan.min_duty;
-}
-
-static int
-nv40_sensor_setup(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
-	s32 offset = sensor->offset_mult / sensor->offset_div;
-	s32 sensor_calibration;
-
-	/* set up the sensors */
-	sensor_calibration = 120 - offset - sensor->offset_constant;
-	sensor_calibration = sensor_calibration * sensor->slope_div /
-				sensor->slope_mult;
-
-	if (dev_priv->chipset >= 0x46)
-		sensor_calibration |= 0x80000000;
-	else
-		sensor_calibration |= 0x10000000;
-
-	nv_wr32(dev, 0x0015b0, sensor_calibration);
-
-	/* Wait for the sensor to update */
-	msleep(5);
-
-	/* read */
-	return nv_rd32(dev, 0x0015b4) & 0x1fff;
-}
-
-int
-nv40_temp_get(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_temp_sensor_constants *sensor = &pm->sensor_constants;
-	int offset = sensor->offset_mult / sensor->offset_div;
-	int core_temp;
-
-	if (dev_priv->card_type >= NV_50) {
-		core_temp = nv_rd32(dev, 0x20008);
-	} else {
-		core_temp = nv_rd32(dev, 0x0015b4) & 0x1fff;
-		/* Setup the sensor if the temperature is 0 */
-		if (core_temp == 0)
-			core_temp = nv40_sensor_setup(dev);
-	}
-
-	core_temp = core_temp * sensor->slope_mult / sensor->slope_div;
-	core_temp = core_temp + offset + sensor->offset_constant;
-
-	return core_temp;
-}
-
-int
-nv84_temp_get(struct drm_device *dev)
-{
-	return nv_rd32(dev, 0x20400);
-}
-
-void
-nouveau_temp_safety_checks(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
-	struct nouveau_pm_threshold_temp *temps = &pm->threshold_temp;
-
-	if (temps->critical > 120)
-		temps->critical = 120;
-	else if (temps->critical < 80)
-		temps->critical = 80;
-
-	if (temps->down_clock > 110)
-		temps->down_clock = 110;
-	else if (temps->down_clock < 60)
-		temps->down_clock = 60;
-
-	if (temps->fan_boost > 100)
-		temps->fan_boost = 100;
-	else if (temps->fan_boost < 40)
-		temps->fan_boost = 40;
-}
-
-static bool
-probe_monitoring_device(struct nouveau_i2c_chan *i2c,
-			struct i2c_board_info *info)
-{
-	struct i2c_client *client;
-
-	request_module("%s%s", I2C_MODULE_PREFIX, info->type);
-
-	client = i2c_new_device(&i2c->adapter, info);
-	if (!client)
-		return false;
-
-	if (!client->driver || client->driver->detect(client, info)) {
-		i2c_unregister_device(client);
-		return false;
-	}
-
-	return true;
-}
-
-static void
-nouveau_temp_probe_i2c(struct drm_device *dev)
-{
-	struct i2c_board_info info[] = {
-		{ I2C_BOARD_INFO("w83l785ts", 0x2d) },
-		{ I2C_BOARD_INFO("w83781d", 0x2d) },
-		{ I2C_BOARD_INFO("adt7473", 0x2e) },
-		{ I2C_BOARD_INFO("f75375", 0x2e) },
-		{ I2C_BOARD_INFO("lm99", 0x4c) },
-		{ }
-	};
-
-	nouveau_i2c_identify(dev, "monitoring device", info,
-			     probe_monitoring_device, NV_I2C_DEFAULT(0));
-}
-
-void
-nouveau_temp_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvbios *bios = &dev_priv->vbios;
-	struct bit_entry P;
-	u8 *temp = NULL;
-
-	if (bios->type == NVBIOS_BIT) {
-		if (bit_table(dev, 'P', &P))
-			return;
-
-		if (P.version == 1)
-			temp = ROMPTR(dev, P.data[12]);
-		else if (P.version == 2)
-			temp = ROMPTR(dev, P.data[16]);
-		else
-			NV_WARN(dev, "unknown temp for BIT P %d\n", P.version);
-
-		nouveau_temp_vbios_parse(dev, temp);
-	}
-
-	nouveau_temp_probe_i2c(dev);
-}
-
-void
-nouveau_temp_fini(struct drm_device *dev)
-{
-
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index bd35f930568..9be9cb58e19 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -24,21 +24,253 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
+#include <subdev/fb.h>
+#include <subdev/vm.h>
+#include <subdev/instmem.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_ttm.h"
+#include "nouveau_gem.h"
+
+static int
+nouveau_vram_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
+{
+	/* nothing to do */
+	return 0;
+}
+
+static int
+nouveau_vram_manager_fini(struct ttm_mem_type_manager *man)
+{
+	/* nothing to do */
+	return 0;
+}
+
+static inline void
+nouveau_mem_node_cleanup(struct nouveau_mem *node)
+{
+	if (node->vma[0].node) {
+		nouveau_vm_unmap(&node->vma[0]);
+		nouveau_vm_put(&node->vma[0]);
+	}
+
+	if (node->vma[1].node) {
+		nouveau_vm_unmap(&node->vma[1]);
+		nouveau_vm_put(&node->vma[1]);
+	}
+}
+
+static void
+nouveau_vram_manager_del(struct ttm_mem_type_manager *man,
+			 struct ttm_mem_reg *mem)
+{
+	struct nouveau_drm *drm = nouveau_bdev(man->bdev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	nouveau_mem_node_cleanup(mem->mm_node);
+	pfb->ram.put(pfb, (struct nouveau_mem **)&mem->mm_node);
+}
+
+static int
+nouveau_vram_manager_new(struct ttm_mem_type_manager *man,
+			 struct ttm_buffer_object *bo,
+			 struct ttm_placement *placement,
+			 struct ttm_mem_reg *mem)
+{
+	struct nouveau_drm *drm = nouveau_bdev(man->bdev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
+	struct nouveau_bo *nvbo = nouveau_bo(bo);
+	struct nouveau_mem *node;
+	u32 size_nc = 0;
+	int ret;
+
+	if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)
+		size_nc = 1 << nvbo->page_shift;
+
+	ret = pfb->ram.get(pfb, mem->num_pages << PAGE_SHIFT,
+			   mem->page_alignment << PAGE_SHIFT, size_nc,
+			   (nvbo->tile_flags >> 8) & 0x3ff, &node);
+	if (ret) {
+		mem->mm_node = NULL;
+		return (ret == -ENOSPC) ? 0 : ret;
+	}
+
+	node->page_shift = nvbo->page_shift;
+
+	mem->mm_node = node;
+	mem->start   = node->offset >> PAGE_SHIFT;
+	return 0;
+}
+
+static void
+nouveau_vram_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+{
+	struct nouveau_mm *mm = man->priv;
+	struct nouveau_mm_node *r;
+	u32 total = 0, free = 0;
+
+	mutex_lock(&mm->mutex);
+	list_for_each_entry(r, &mm->nodes, nl_entry) {
+		printk(KERN_DEBUG "%s %d: 0x%010llx 0x%010llx\n",
+		       prefix, r->type, ((u64)r->offset << 12),
+		       (((u64)r->offset + r->length) << 12));
+
+		total += r->length;
+		if (!r->type)
+			free += r->length;
+	}
+	mutex_unlock(&mm->mutex);
+
+	printk(KERN_DEBUG "%s  total: 0x%010llx free: 0x%010llx\n",
+	       prefix, (u64)total << 12, (u64)free << 12);
+	printk(KERN_DEBUG "%s  block: 0x%08x\n",
+	       prefix, mm->block_size << 12);
+}
+
+const struct ttm_mem_type_manager_func nouveau_vram_manager = {
+	nouveau_vram_manager_init,
+	nouveau_vram_manager_fini,
+	nouveau_vram_manager_new,
+	nouveau_vram_manager_del,
+	nouveau_vram_manager_debug
+};
+
+static int
+nouveau_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
+{
+	return 0;
+}
+
+static int
+nouveau_gart_manager_fini(struct ttm_mem_type_manager *man)
+{
+	return 0;
+}
+
+static void
+nouveau_gart_manager_del(struct ttm_mem_type_manager *man,
+			 struct ttm_mem_reg *mem)
+{
+	nouveau_mem_node_cleanup(mem->mm_node);
+	kfree(mem->mm_node);
+	mem->mm_node = NULL;
+}
+
+static int
+nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
+			 struct ttm_buffer_object *bo,
+			 struct ttm_placement *placement,
+			 struct ttm_mem_reg *mem)
+{
+	struct nouveau_mem *node;
+
+	if (unlikely((mem->num_pages << PAGE_SHIFT) >= 512 * 1024 * 1024))
+		return -ENOMEM;
+
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+	node->page_shift = 12;
+
+	mem->mm_node = node;
+	mem->start   = 0;
+	return 0;
+}
+
+static void
+nouveau_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+{
+}
+
+const struct ttm_mem_type_manager_func nouveau_gart_manager = {
+	nouveau_gart_manager_init,
+	nouveau_gart_manager_fini,
+	nouveau_gart_manager_new,
+	nouveau_gart_manager_del,
+	nouveau_gart_manager_debug
+};
+
+#include <core/subdev/vm/nv04.h>
+static int
+nv04_gart_manager_init(struct ttm_mem_type_manager *man, unsigned long psize)
+{
+	struct nouveau_drm *drm = nouveau_bdev(man->bdev);
+	struct nouveau_vmmgr *vmm = nouveau_vmmgr(drm->device);
+	struct nv04_vmmgr_priv *priv = (void *)vmm;
+	struct nouveau_vm *vm = NULL;
+	nouveau_vm_ref(priv->vm, &vm, NULL);
+	man->priv = vm;
+	return 0;
+}
+
+static int
+nv04_gart_manager_fini(struct ttm_mem_type_manager *man)
+{
+	struct nouveau_vm *vm = man->priv;
+	nouveau_vm_ref(NULL, &vm, NULL);
+	man->priv = NULL;
+	return 0;
+}
+
+static void
+nv04_gart_manager_del(struct ttm_mem_type_manager *man, struct ttm_mem_reg *mem)
+{
+	struct nouveau_mem *node = mem->mm_node;
+	if (node->vma[0].node)
+		nouveau_vm_put(&node->vma[0]);
+	kfree(mem->mm_node);
+	mem->mm_node = NULL;
+}
+
+static int
+nv04_gart_manager_new(struct ttm_mem_type_manager *man,
+		      struct ttm_buffer_object *bo,
+		      struct ttm_placement *placement,
+		      struct ttm_mem_reg *mem)
+{
+	struct nouveau_mem *node;
+	int ret;
+
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	node->page_shift = 12;
+
+	ret = nouveau_vm_get(man->priv, mem->num_pages << 12, node->page_shift,
+			     NV_MEM_ACCESS_RW, &node->vma[0]);
+	if (ret) {
+		kfree(node);
+		return ret;
+	}
+
+	mem->mm_node = node;
+	mem->start   = node->vma[0].offset >> PAGE_SHIFT;
+	return 0;
+}
+
+static void
+nv04_gart_manager_debug(struct ttm_mem_type_manager *man, const char *prefix)
+{
+}
+
+const struct ttm_mem_type_manager_func nv04_gart_manager = {
+	nv04_gart_manager_init,
+	nv04_gart_manager_fini,
+	nv04_gart_manager_new,
+	nv04_gart_manager_del,
+	nv04_gart_manager_debug
+};
 
 int
 nouveau_ttm_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct drm_file *file_priv = filp->private_data;
-	struct drm_nouveau_private *dev_priv =
-		file_priv->minor->dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(file_priv->minor->dev);
 
 	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
 		return drm_mmap(filp, vma);
 
-	return ttm_bo_mmap(filp, vma, &dev_priv->ttm.bdev);
+	return ttm_bo_mmap(filp, vma, &drm->ttm.bdev);
 }
 
 static int
@@ -54,12 +286,12 @@ nouveau_ttm_mem_global_release(struct drm_global_reference *ref)
 }
 
 int
-nouveau_ttm_global_init(struct drm_nouveau_private *dev_priv)
+nouveau_ttm_global_init(struct nouveau_drm *drm)
 {
 	struct drm_global_reference *global_ref;
 	int ret;
 
-	global_ref = &dev_priv->ttm.mem_global_ref;
+	global_ref = &drm->ttm.mem_global_ref;
 	global_ref->global_type = DRM_GLOBAL_TTM_MEM;
 	global_ref->size = sizeof(struct ttm_mem_global);
 	global_ref->init = &nouveau_ttm_mem_global_init;
@@ -68,12 +300,12 @@ nouveau_ttm_global_init(struct drm_nouveau_private *dev_priv)
 	ret = drm_global_item_ref(global_ref);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Failed setting up TTM memory accounting\n");
-		dev_priv->ttm.mem_global_ref.release = NULL;
+		drm->ttm.mem_global_ref.release = NULL;
 		return ret;
 	}
 
-	dev_priv->ttm.bo_global_ref.mem_glob = global_ref->object;
-	global_ref = &dev_priv->ttm.bo_global_ref.ref;
+	drm->ttm.bo_global_ref.mem_glob = global_ref->object;
+	global_ref = &drm->ttm.bo_global_ref.ref;
 	global_ref->global_type = DRM_GLOBAL_TTM_BO;
 	global_ref->size = sizeof(struct ttm_bo_global);
 	global_ref->init = &ttm_bo_global_init;
@@ -82,8 +314,8 @@ nouveau_ttm_global_init(struct drm_nouveau_private *dev_priv)
 	ret = drm_global_item_ref(global_ref);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Failed setting up TTM BO subsystem\n");
-		drm_global_item_unref(&dev_priv->ttm.mem_global_ref);
-		dev_priv->ttm.mem_global_ref.release = NULL;
+		drm_global_item_unref(&drm->ttm.mem_global_ref);
+		drm->ttm.mem_global_ref.release = NULL;
 		return ret;
 	}
 
@@ -91,13 +323,101 @@ nouveau_ttm_global_init(struct drm_nouveau_private *dev_priv)
 }
 
 void
-nouveau_ttm_global_release(struct drm_nouveau_private *dev_priv)
+nouveau_ttm_global_release(struct nouveau_drm *drm)
 {
-	if (dev_priv->ttm.mem_global_ref.release == NULL)
+	if (drm->ttm.mem_global_ref.release == NULL)
 		return;
 
-	drm_global_item_unref(&dev_priv->ttm.bo_global_ref.ref);
-	drm_global_item_unref(&dev_priv->ttm.mem_global_ref);
-	dev_priv->ttm.mem_global_ref.release = NULL;
+	drm_global_item_unref(&drm->ttm.bo_global_ref.ref);
+	drm_global_item_unref(&drm->ttm.mem_global_ref);
+	drm->ttm.mem_global_ref.release = NULL;
 }
 
+int
+nouveau_ttm_init(struct nouveau_drm *drm)
+{
+	struct drm_device *dev = drm->dev;
+	u32 bits;
+	int ret;
+
+	bits = nouveau_vmmgr(drm->device)->dma_bits;
+	if ( drm->agp.stat == ENABLED ||
+	    !pci_dma_supported(dev->pdev, DMA_BIT_MASK(bits)))
+		bits = 32;
+
+	ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
+	if (ret)
+		return ret;
+
+	ret = pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
+	if (ret)
+		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(32));
+
+	ret = nouveau_ttm_global_init(drm);
+	if (ret)
+		return ret;
+
+	ret = ttm_bo_device_init(&drm->ttm.bdev,
+				  drm->ttm.bo_global_ref.ref.object,
+				  &nouveau_bo_driver, DRM_FILE_PAGE_OFFSET,
+				  bits <= 32 ? true : false);
+	if (ret) {
+		NV_ERROR(drm, "error initialising bo driver, %d\n", ret);
+		return ret;
+	}
+
+	/* VRAM init */
+	drm->gem.vram_available  = nouveau_fb(drm->device)->ram.size;
+	drm->gem.vram_available -= nouveau_instmem(drm->device)->reserved;
+
+	ret = ttm_bo_init_mm(&drm->ttm.bdev, TTM_PL_VRAM,
+			      drm->gem.vram_available >> PAGE_SHIFT);
+	if (ret) {
+		NV_ERROR(drm, "VRAM mm init failed, %d\n", ret);
+		return ret;
+	}
+
+	drm->ttm.mtrr = drm_mtrr_add(pci_resource_start(dev->pdev, 1),
+				     pci_resource_len(dev->pdev, 1),
+				     DRM_MTRR_WC);
+
+	/* GART init */
+	if (drm->agp.stat != ENABLED) {
+		drm->gem.gart_available = nouveau_vmmgr(drm->device)->limit;
+		if (drm->gem.gart_available > 512 * 1024 * 1024)
+			drm->gem.gart_available = 512 * 1024 * 1024;
+	} else {
+		drm->gem.gart_available = drm->agp.size;
+	}
+
+	ret = ttm_bo_init_mm(&drm->ttm.bdev, TTM_PL_TT,
+			      drm->gem.gart_available >> PAGE_SHIFT);
+	if (ret) {
+		NV_ERROR(drm, "GART mm init failed, %d\n", ret);
+		return ret;
+	}
+
+	NV_INFO(drm, "VRAM: %d MiB\n", (u32)(drm->gem.vram_available >> 20));
+	NV_INFO(drm, "GART: %d MiB\n", (u32)(drm->gem.gart_available >> 20));
+	return 0;
+}
+
+void
+nouveau_ttm_fini(struct nouveau_drm *drm)
+{
+	mutex_lock(&drm->dev->struct_mutex);
+	ttm_bo_clean_mm(&drm->ttm.bdev, TTM_PL_VRAM);
+	ttm_bo_clean_mm(&drm->ttm.bdev, TTM_PL_TT);
+	mutex_unlock(&drm->dev->struct_mutex);
+
+	ttm_bo_device_release(&drm->ttm.bdev);
+
+	nouveau_ttm_global_release(drm);
+
+	if (drm->ttm.mtrr >= 0) {
+		drm_mtrr_del(drm->ttm.mtrr,
+			     pci_resource_start(drm->dev->pdev, 1),
+			     pci_resource_len(drm->dev->pdev, 1), DRM_MTRR_WC);
+		drm->ttm.mtrr = -1;
+	}
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.h b/drivers/gpu/drm/nouveau/nouveau_ttm.h
new file mode 100644
index 00000000000..25b0de41335
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.h
@@ -0,0 +1,25 @@
+#ifndef __NOUVEAU_TTM_H__
+#define __NOUVEAU_TTM_H__
+
+static inline struct nouveau_drm *
+nouveau_bdev(struct ttm_bo_device *bd)
+{
+	return container_of(bd, struct nouveau_drm, ttm.bdev);
+}
+
+extern const struct ttm_mem_type_manager_func nouveau_vram_manager;
+extern const struct ttm_mem_type_manager_func nouveau_gart_manager;
+extern const struct ttm_mem_type_manager_func nv04_gart_manager;
+
+struct ttm_tt *nouveau_sgdma_create_ttm(struct ttm_bo_device *,
+					unsigned long size, u32 page_flags,
+					struct page *dummy_read_page);
+
+int  nouveau_ttm_init(struct nouveau_drm *drm);
+void nouveau_ttm_fini(struct nouveau_drm *drm);
+int  nouveau_ttm_mmap(struct file *, struct vm_area_struct *);
+
+int  nouveau_ttm_global_init(struct nouveau_drm *);
+void nouveau_ttm_global_release(struct nouveau_drm *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_util.h b/drivers/gpu/drm/nouveau/nouveau_util.h
deleted file mode 100644
index b97719fbb73..00000000000
--- a/drivers/gpu/drm/nouveau/nouveau_util.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright (C) 2010 Nouveau Project
- *
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef __NOUVEAU_UTIL_H__
-#define __NOUVEAU_UTIL_H__
-
-struct nouveau_bitfield {
-	u32 mask;
-	const char *name;
-};
-
-struct nouveau_enum {
-	u32 value;
-	const char *name;
-	void *data;
-};
-
-void nouveau_bitfield_print(const struct nouveau_bitfield *, u32 value);
-void nouveau_enum_print(const struct nouveau_enum *, u32 value);
-const struct nouveau_enum *
-nouveau_enum_find(const struct nouveau_enum *, u32 value);
-
-int nouveau_ratelimit(void);
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c
new file mode 100644
index 00000000000..6f0ac64873d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -0,0 +1,99 @@
+#include <linux/vgaarb.h>
+#include <linux/vga_switcheroo.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_acpi.h"
+#include "nouveau_fbcon.h"
+#include "nouveau_vga.h"
+
+static unsigned int
+nouveau_vga_set_decode(void *priv, bool state)
+{
+	struct nouveau_device *device = nouveau_dev(priv);
+
+	if (device->chipset >= 0x40)
+		nv_wr32(device, 0x088054, state);
+	else
+		nv_wr32(device, 0x001854, state);
+
+	if (state)
+		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
+		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+	else
+		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+}
+
+static void
+nouveau_switcheroo_set_state(struct pci_dev *pdev,
+			     enum vga_switcheroo_state state)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
+
+	if (state == VGA_SWITCHEROO_ON) {
+		printk(KERN_ERR "VGA switcheroo: switched nouveau on\n");
+		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+		nouveau_drm_resume(pdev);
+		drm_kms_helper_poll_enable(dev);
+		dev->switch_power_state = DRM_SWITCH_POWER_ON;
+	} else {
+		printk(KERN_ERR "VGA switcheroo: switched nouveau off\n");
+		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+		drm_kms_helper_poll_disable(dev);
+		nouveau_switcheroo_optimus_dsm();
+		nouveau_drm_suspend(pdev, pmm);
+		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
+	}
+}
+
+static void
+nouveau_switcheroo_reprobe(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	nouveau_fbcon_output_poll_changed(dev);
+}
+
+static bool
+nouveau_switcheroo_can_switch(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	bool can_switch;
+
+	spin_lock(&dev->count_lock);
+	can_switch = (dev->open_count == 0);
+	spin_unlock(&dev->count_lock);
+	return can_switch;
+}
+
+static const struct vga_switcheroo_client_ops
+nouveau_switcheroo_ops = {
+	.set_gpu_state = nouveau_switcheroo_set_state,
+	.reprobe = nouveau_switcheroo_reprobe,
+	.can_switch = nouveau_switcheroo_can_switch,
+};
+
+void
+nouveau_vga_init(struct nouveau_drm *drm)
+{
+	struct drm_device *dev = drm->dev;
+	vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
+	vga_switcheroo_register_client(dev->pdev, &nouveau_switcheroo_ops);
+}
+
+void
+nouveau_vga_fini(struct nouveau_drm *drm)
+{
+	struct drm_device *dev = drm->dev;
+	vga_switcheroo_unregister_client(dev->pdev);
+	vga_client_register(dev->pdev, NULL, NULL, NULL);
+}
+
+
+void
+nouveau_vga_lastclose(struct drm_device *dev)
+{
+	vga_switcheroo_process_delayed_switch();
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.h b/drivers/gpu/drm/nouveau/nouveau_vga.h
new file mode 100644
index 00000000000..ea3ad6974c6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.h
@@ -0,0 +1,8 @@
+#ifndef __NOUVEAU_VGA_H__
+#define __NOUVEAU_VGA_H__
+
+void nouveau_vga_init(struct nouveau_drm *);
+void nouveau_vga_fini(struct nouveau_drm *);
+void nouveau_vga_lastclose(struct drm_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_volt.c b/drivers/gpu/drm/nouveau/nouveau_volt.c
index b010cb997b3..9976414cbe5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_volt.c
+++ b/drivers/gpu/drm/nouveau/nouveau_volt.c
@@ -22,20 +22,23 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_pm.h"
-#include "nouveau_gpio.h"
 
-static const enum dcb_gpio_tag vidtag[] = { 0x04, 0x05, 0x06, 0x1a, 0x73 };
+#include <subdev/bios/gpio.h>
+#include <subdev/gpio.h>
+
+static const enum dcb_gpio_func_name vidtag[] = { 0x04, 0x05, 0x06, 0x1a, 0x73 };
 static int nr_vidtag = sizeof(vidtag) / sizeof(vidtag[0]);
 
 int
 nouveau_voltage_gpio_get(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+	struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(device);
 	u8 vid = 0;
 	int i;
 
@@ -43,7 +46,7 @@ nouveau_voltage_gpio_get(struct drm_device *dev)
 		if (!(volt->vid_mask & (1 << i)))
 			continue;
 
-		vid |= nouveau_gpio_func_get(dev, vidtag[i]) << i;
+		vid |= gpio->get(gpio, 0, vidtag[i], 0xff) << i;
 	}
 
 	return nouveau_volt_lvl_lookup(dev, vid);
@@ -52,8 +55,9 @@ nouveau_voltage_gpio_get(struct drm_device *dev)
 int
 nouveau_voltage_gpio_set(struct drm_device *dev, int voltage)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(device);
+	struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
 	int vid, i;
 
 	vid = nouveau_volt_vid_lookup(dev, voltage);
@@ -64,7 +68,7 @@ nouveau_voltage_gpio_set(struct drm_device *dev, int voltage)
 		if (!(volt->vid_mask & (1 << i)))
 			continue;
 
-		nouveau_gpio_func_set(dev, vidtag[i], !!(vid & (1 << i)));
+		gpio->set(gpio, 0, vidtag[i], 0xff, !!(vid & (1 << i)));
 	}
 
 	return 0;
@@ -73,8 +77,7 @@ nouveau_voltage_gpio_set(struct drm_device *dev, int voltage)
 int
 nouveau_volt_vid_lookup(struct drm_device *dev, int voltage)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+	struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
 	int i;
 
 	for (i = 0; i < volt->nr_level; i++) {
@@ -88,8 +91,7 @@ nouveau_volt_vid_lookup(struct drm_device *dev, int voltage)
 int
 nouveau_volt_lvl_lookup(struct drm_device *dev, int vid)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+	struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
 	int i;
 
 	for (i = 0; i < volt->nr_level; i++) {
@@ -103,10 +105,12 @@ nouveau_volt_lvl_lookup(struct drm_device *dev, int vid)
 void
 nouveau_volt_init(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_engine *pm = &dev_priv->engine.pm;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
+	struct nouveau_pm *pm = nouveau_pm(dev);
 	struct nouveau_pm_voltage *voltage = &pm->voltage;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nvbios *bios = &drm->vbios;
+	struct dcb_gpio_func func;
 	struct bit_entry P;
 	u8 *volt = NULL, *entry;
 	int i, headerlen, recordlen, entries, vidmask, vidshift;
@@ -121,11 +125,11 @@ nouveau_volt_init(struct drm_device *dev)
 		if (P.version == 2)
 			volt = ROMPTR(dev, P.data[12]);
 		else {
-			NV_WARN(dev, "unknown volt for BIT P %d\n", P.version);
+			NV_WARN(drm, "unknown volt for BIT P %d\n", P.version);
 		}
 	} else {
 		if (bios->data[bios->offset + 6] < 0x27) {
-			NV_DEBUG(dev, "BMP version too old for voltage\n");
+			NV_DEBUG(drm, "BMP version too old for voltage\n");
 			return;
 		}
 
@@ -133,7 +137,7 @@ nouveau_volt_init(struct drm_device *dev)
 	}
 
 	if (!volt) {
-		NV_DEBUG(dev, "voltage table pointer invalid\n");
+		NV_DEBUG(drm, "voltage table pointer invalid\n");
 		return;
 	}
 
@@ -177,7 +181,7 @@ nouveau_volt_init(struct drm_device *dev)
 		vidshift  = 0;
 		break;
 	default:
-		NV_WARN(dev, "voltage table 0x%02x unknown\n", volt[0]);
+		NV_WARN(drm, "voltage table 0x%02x unknown\n", volt[0]);
 		return;
 	}
 
@@ -189,12 +193,12 @@ nouveau_volt_init(struct drm_device *dev)
 	i = 0;
 	while (vidmask) {
 		if (i > nr_vidtag) {
-			NV_DEBUG(dev, "vid bit %d unknown\n", i);
+			NV_DEBUG(drm, "vid bit %d unknown\n", i);
 			return;
 		}
 
-		if (!nouveau_gpio_func_valid(dev, vidtag[i])) {
-			NV_DEBUG(dev, "vid bit %d has no gpio tag\n", i);
+		if (gpio && gpio->find(gpio, 0, vidtag[i], 0xff, &func)) {
+			NV_DEBUG(drm, "vid bit %d has no gpio tag\n", i);
 			return;
 		}
 
@@ -240,8 +244,7 @@ nouveau_volt_init(struct drm_device *dev)
 void
 nouveau_volt_fini(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_pm_voltage *volt = &dev_priv->engine.pm.voltage;
+	struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
 
 	kfree(volt->level);
 }
diff --git a/drivers/gpu/drm/nouveau/nv04_crtc.c b/drivers/gpu/drm/nouveau/nv04_crtc.c
index 43accc11102..82a0d9c6cda 100644
--- a/drivers/gpu/drm/nouveau/nv04_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv04_crtc.c
@@ -23,17 +23,23 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
+#include "nouveau_bo.h"
+#include "nouveau_gem.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
-#include "nouveau_fb.h"
 #include "nouveau_hw.h"
 #include "nvreg.h"
 #include "nouveau_fbcon.h"
+#include "nv04_display.h"
+
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
 
 static int
 nv04_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
@@ -49,8 +55,8 @@ crtc_wr_cio_state(struct drm_crtc *crtc, struct nv04_crtc_reg *crtcstate, int in
 static void nv_crtc_set_digital_vibrance(struct drm_crtc *crtc, int level)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct drm_device *dev = crtc->dev;
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 
 	regp->CRTC[NV_CIO_CRE_CSB] = nv_crtc->saturation = level;
 	if (nv_crtc->saturation && nv_gf4_disp_arch(crtc->dev)) {
@@ -64,8 +70,8 @@ static void nv_crtc_set_digital_vibrance(struct drm_crtc *crtc, int level)
 static void nv_crtc_set_image_sharpening(struct drm_crtc *crtc, int level)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct drm_device *dev = crtc->dev;
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 
 	nv_crtc->sharpness = level;
 	if (level < 0)	/* blur is in hw range 0x3f -> 0x20 */
@@ -103,14 +109,17 @@ static void nv_crtc_set_image_sharpening(struct drm_crtc *crtc, int level)
 static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mode * mode, int dot_clock)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_bios *bios = nouveau_bios(drm->device);
+	struct nouveau_clock *clk = nouveau_clock(drm->device);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv04_mode_state *state = &dev_priv->mode_reg;
+	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
 	struct nv04_crtc_reg *regp = &state->crtc_reg[nv_crtc->index];
 	struct nouveau_pll_vals *pv = &regp->pllvals;
-	struct pll_lims pll_lim;
+	struct nvbios_pll pll_lim;
 
-	if (get_pll_limits(dev, nv_crtc->index ? PLL_VPLL1 : PLL_VPLL0, &pll_lim))
+	if (nvbios_pll_parse(bios, nv_crtc->index ? PLL_VPLL1 : PLL_VPLL0,
+			    &pll_lim))
 		return;
 
 	/* NM2 == 0 is used to determine single stage mode on two stage plls */
@@ -126,28 +135,29 @@ static void nv_crtc_calc_state_ext(struct drm_crtc *crtc, struct drm_display_mod
 	 * has yet been observed in allowing the use a single stage pll on all
 	 * nv43 however.  the behaviour of single stage use is untested on nv40
 	 */
-	if (dev_priv->chipset > 0x40 && dot_clock <= (pll_lim.vco1.maxfreq / 2))
+	if (nv_device(drm->device)->chipset > 0x40 && dot_clock <= (pll_lim.vco1.max_freq / 2))
 		memset(&pll_lim.vco2, 0, sizeof(pll_lim.vco2));
 
-	if (!nouveau_calc_pll_mnp(dev, &pll_lim, dot_clock, pv))
+
+	if (!clk->pll_calc(clk, &pll_lim, dot_clock, pv))
 		return;
 
 	state->pllsel &= PLLSEL_VPLL1_MASK | PLLSEL_VPLL2_MASK | PLLSEL_TV_MASK;
 
 	/* The blob uses this always, so let's do the same */
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_USE_VPLL2_TRUE;
 	/* again nv40 and some nv43 act more like nv3x as described above */
-	if (dev_priv->chipset < 0x41)
+	if (nv_device(drm->device)->chipset < 0x41)
 		state->pllsel |= NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_MPLL |
 				 NV_PRAMDAC_PLL_COEFF_SELECT_SOURCE_PROG_NVPLL;
 	state->pllsel |= nv_crtc->index ? PLLSEL_VPLL2_MASK : PLLSEL_VPLL1_MASK;
 
 	if (pv->NM2)
-		NV_DEBUG_KMS(dev, "vpll: n1 %d n2 %d m1 %d m2 %d log2p %d\n",
+		NV_DEBUG(drm, "vpll: n1 %d n2 %d m1 %d m2 %d log2p %d\n",
 			 pv->N1, pv->N2, pv->M1, pv->M2, pv->log2P);
 	else
-		NV_DEBUG_KMS(dev, "vpll: n %d m %d log2p %d\n",
+		NV_DEBUG(drm, "vpll: n %d m %d log2p %d\n",
 			 pv->N1, pv->M1, pv->log2P);
 
 	nv_crtc->cursor.set_offset(nv_crtc, nv_crtc->cursor.offset);
@@ -158,10 +168,11 @@ nv_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	unsigned char seq1 = 0, crtc17 = 0;
 	unsigned char crtc1A;
 
-	NV_DEBUG_KMS(dev, "Setting dpms mode %d on CRTC %d\n", mode,
+	NV_DEBUG(drm, "Setting dpms mode %d on CRTC %d\n", mode,
 							nv_crtc->index);
 
 	if (nv_crtc->last_dpms == mode) /* Don't do unnecessary mode changes. */
@@ -225,9 +236,8 @@ static void
 nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 	struct drm_framebuffer *fb = crtc->fb;
 
 	/* Calculate our timings */
@@ -251,8 +261,8 @@ nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
 		struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 
 		if (encoder->crtc == crtc &&
-		    (nv_encoder->dcb->type == OUTPUT_LVDS ||
-		     nv_encoder->dcb->type == OUTPUT_TMDS))
+		    (nv_encoder->dcb->type == DCB_OUTPUT_LVDS ||
+		     nv_encoder->dcb->type == DCB_OUTPUT_TMDS))
 			fp_output = true;
 	}
 
@@ -264,7 +274,7 @@ nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
 		horizEnd = horizTotal - 2;
 		horizBlankEnd = horizTotal + 4;
 #if 0
-		if (dev->overlayAdaptor && dev_priv->card_type >= NV_10)
+		if (dev->overlayAdaptor && nv_device(drm->device)->card_type >= NV_10)
 			/* This reportedly works around some video overlay bandwidth problems */
 			horizTotal += 2;
 #endif
@@ -452,10 +462,10 @@ static void
 nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
-	struct nv04_crtc_reg *savep = &dev_priv->saved_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *savep = &nv04_display(dev)->saved_reg.crtc_reg[nv_crtc->index];
 	struct drm_encoder *encoder;
 	bool lvds_output = false, tmds_output = false, tv_output = false,
 		off_chip_digital = false;
@@ -467,11 +477,11 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 		if (encoder->crtc != crtc)
 			continue;
 
-		if (nv_encoder->dcb->type == OUTPUT_LVDS)
+		if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS)
 			digital = lvds_output = true;
-		if (nv_encoder->dcb->type == OUTPUT_TV)
+		if (nv_encoder->dcb->type == DCB_OUTPUT_TV)
 			tv_output = true;
-		if (nv_encoder->dcb->type == OUTPUT_TMDS)
+		if (nv_encoder->dcb->type == DCB_OUTPUT_TMDS)
 			digital = tmds_output = true;
 		if (nv_encoder->dcb->location != DCB_LOC_ON_CHIP && digital)
 			off_chip_digital = true;
@@ -500,7 +510,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 	regp->cursor_cfg = NV_PCRTC_CURSOR_CONFIG_CUR_LINES_64 |
 			     NV_PCRTC_CURSOR_CONFIG_CUR_PIXELS_64 |
 			     NV_PCRTC_CURSOR_CONFIG_ADDRESS_SPACE_PNVM;
-	if (dev_priv->chipset >= 0x11)
+	if (nv_device(drm->device)->chipset >= 0x11)
 		regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_CUR_BPP_32;
 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		regp->cursor_cfg |= NV_PCRTC_CURSOR_CONFIG_DOUBLE_SCAN_ENABLE;
@@ -533,7 +543,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 
 	/* The blob seems to take the current value from crtc 0, add 4 to that
 	 * and reuse the old value for crtc 1 */
-	regp->CRTC[NV_CIO_CRE_TVOUT_LATENCY] = dev_priv->saved_reg.crtc_reg[0].CRTC[NV_CIO_CRE_TVOUT_LATENCY];
+	regp->CRTC[NV_CIO_CRE_TVOUT_LATENCY] = nv04_display(dev)->saved_reg.crtc_reg[0].CRTC[NV_CIO_CRE_TVOUT_LATENCY];
 	if (!nv_crtc->index)
 		regp->CRTC[NV_CIO_CRE_TVOUT_LATENCY] += 4;
 
@@ -541,26 +551,26 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 	 * 1 << 30 on 0x60.830), for no apparent reason */
 	regp->CRTC[NV_CIO_CRE_59] = off_chip_digital;
 
-	if (dev_priv->card_type >= NV_30)
+	if (nv_device(drm->device)->card_type >= NV_30)
 		regp->CRTC[0x9f] = off_chip_digital ? 0x11 : 0x1;
 
 	regp->crtc_830 = mode->crtc_vdisplay - 3;
 	regp->crtc_834 = mode->crtc_vdisplay - 1;
 
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		/* This is what the blob does */
 		regp->crtc_850 = NVReadCRTC(dev, 0, NV_PCRTC_850);
 
-	if (dev_priv->card_type >= NV_30)
+	if (nv_device(drm->device)->card_type >= NV_30)
 		regp->gpio_ext = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT);
 
-	if (dev_priv->card_type >= NV_10)
+	if (nv_device(drm->device)->card_type >= NV_10)
 		regp->crtc_cfg = NV10_PCRTC_CONFIG_START_ADDRESS_HSYNC;
 	else
 		regp->crtc_cfg = NV04_PCRTC_CONFIG_START_ADDRESS_HSYNC;
 
 	/* Some misc regs */
-	if (dev_priv->card_type == NV_40) {
+	if (nv_device(drm->device)->card_type == NV_40) {
 		regp->CRTC[NV_CIO_CRE_85] = 0xFF;
 		regp->CRTC[NV_CIO_CRE_86] = 0x1;
 	}
@@ -572,7 +582,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 
 	/* Generic PRAMDAC regs */
 
-	if (dev_priv->card_type >= NV_10)
+	if (nv_device(drm->device)->card_type >= NV_10)
 		/* Only bit that bios and blob set. */
 		regp->nv10_cursync = (1 << 25);
 
@@ -581,7 +591,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
 				NV_PRAMDAC_GENERAL_CONTROL_PIXMIX_ON;
 	if (crtc->fb->depth == 16)
 		regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
-	if (dev_priv->chipset >= 0x11)
+	if (nv_device(drm->device)->chipset >= 0x11)
 		regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_PIPE_LONG;
 
 	regp->ramdac_630 = 0; /* turn off green mode (tv test pattern?) */
@@ -611,9 +621,9 @@ nv_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
 {
 	struct drm_device *dev = crtc->dev;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	NV_DEBUG_KMS(dev, "CTRC mode on CRTC %d:\n", nv_crtc->index);
+	NV_DEBUG(drm, "CTRC mode on CRTC %d:\n", nv_crtc->index);
 	drm_mode_debug_printmodeline(adjusted_mode);
 
 	/* unlock must come after turning off FP_TG_CONTROL in output_prepare */
@@ -621,8 +631,8 @@ nv_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
 
 	nv_crtc_mode_set_vga(crtc, adjusted_mode);
 	/* calculated in nv04_dfp_prepare, nv40 needs it written before calculating PLLs */
-	if (dev_priv->card_type == NV_40)
-		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, dev_priv->mode_reg.sel_clk);
+	if (nv_device(drm->device)->card_type == NV_40)
+		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, nv04_display(dev)->mode_reg.sel_clk);
 	nv_crtc_mode_set_regs(crtc, adjusted_mode);
 	nv_crtc_calc_state_ext(crtc, mode, adjusted_mode->clock);
 	return 0;
@@ -631,10 +641,10 @@ nv_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
 static void nv_crtc_save(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
-	struct nv04_mode_state *state = &dev_priv->mode_reg;
+	struct drm_device *dev = crtc->dev;
+	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
 	struct nv04_crtc_reg *crtc_state = &state->crtc_reg[nv_crtc->index];
-	struct nv04_mode_state *saved = &dev_priv->saved_reg;
+	struct nv04_mode_state *saved = &nv04_display(dev)->saved_reg;
 	struct nv04_crtc_reg *crtc_saved = &saved->crtc_reg[nv_crtc->index];
 
 	if (nv_two_heads(crtc->dev))
@@ -652,14 +662,14 @@ static void nv_crtc_save(struct drm_crtc *crtc)
 static void nv_crtc_restore(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
+	struct drm_device *dev = crtc->dev;
 	int head = nv_crtc->index;
-	uint8_t saved_cr21 = dev_priv->saved_reg.crtc_reg[head].CRTC[NV_CIO_CRE_21];
+	uint8_t saved_cr21 = nv04_display(dev)->saved_reg.crtc_reg[head].CRTC[NV_CIO_CRE_21];
 
 	if (nv_two_heads(crtc->dev))
 		NVSetOwner(crtc->dev, head);
 
-	nouveau_hw_load_state(crtc->dev, head, &dev_priv->saved_reg);
+	nouveau_hw_load_state(crtc->dev, head, &nv04_display(dev)->saved_reg);
 	nv_lock_vga_crtc_shadow(crtc->dev, head, saved_cr21);
 
 	nv_crtc->last_dpms = NV_DPMS_CLEARED;
@@ -668,7 +678,7 @@ static void nv_crtc_restore(struct drm_crtc *crtc)
 static void nv_crtc_prepare(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_crtc_helper_funcs *funcs = crtc->helper_private;
 
@@ -682,7 +692,7 @@ static void nv_crtc_prepare(struct drm_crtc *crtc)
 
 	/* Some more preparation. */
 	NVWriteCRTC(dev, nv_crtc->index, NV_PCRTC_CONFIG, NV_PCRTC_CONFIG_START_ADDRESS_NON_VGA);
-	if (dev_priv->card_type == NV_40) {
+	if (nv_device(drm->device)->card_type == NV_40) {
 		uint32_t reg900 = NVReadRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900);
 		NVWriteRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_900, reg900 & ~0x10000);
 	}
@@ -692,10 +702,9 @@ static void nv_crtc_commit(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_crtc_helper_funcs *funcs = crtc->helper_private;
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 
-	nouveau_hw_load_state(dev, nv_crtc->index, &dev_priv->mode_reg);
+	nouveau_hw_load_state(dev, nv_crtc->index, &nv04_display(dev)->mode_reg);
 	nv04_crtc_mode_set_base(crtc, crtc->x, crtc->y, NULL);
 
 #ifdef __BIG_ENDIAN
@@ -715,8 +724,6 @@ static void nv_crtc_destroy(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 
-	NV_DEBUG_KMS(crtc->dev, "\n");
-
 	if (!nv_crtc)
 		return;
 
@@ -732,18 +739,17 @@ nv_crtc_gamma_load(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct rgb { uint8_t r, g, b; } __attribute__((packed)) *rgbs;
 	int i;
 
-	rgbs = (struct rgb *)dev_priv->mode_reg.crtc_reg[nv_crtc->index].DAC;
+	rgbs = (struct rgb *)nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index].DAC;
 	for (i = 0; i < 256; i++) {
 		rgbs[i].r = nv_crtc->lut.r[i] >> 8;
 		rgbs[i].g = nv_crtc->lut.g[i] >> 8;
 		rgbs[i].b = nv_crtc->lut.b[i] >> 8;
 	}
 
-	nouveau_hw_load_state_palette(dev, nv_crtc->index, &dev_priv->mode_reg);
+	nouveau_hw_load_state_palette(dev, nv_crtc->index, &nv04_display(dev)->mode_reg);
 }
 
 static void
@@ -779,18 +785,18 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 	struct drm_framebuffer *drm_fb;
 	struct nouveau_framebuffer *fb;
 	int arb_burst, arb_lwm;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
+	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
 	/* no fb bound */
 	if (!atomic && !crtc->fb) {
-		NV_DEBUG_KMS(dev, "No FB bound\n");
+		NV_DEBUG(drm, "No FB bound\n");
 		return 0;
 	}
 
@@ -858,7 +864,7 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FF_INDEX);
 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_FFLWM__INDEX);
 
-	if (dev_priv->card_type >= NV_20) {
+	if (nv_device(drm->device)->card_type >= NV_20) {
 		regp->CRTC[NV_CIO_CRE_47] = arb_lwm >> 8;
 		crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_47);
 	}
@@ -878,8 +884,8 @@ nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
 			       struct drm_framebuffer *fb,
 			       int x, int y, enum mode_set_atomic state)
 {
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
+	struct drm_device *dev = drm->dev;
 
 	if (state == ENTER_ATOMIC_MODE_SET)
 		nouveau_fbcon_save_disable_accel(dev);
@@ -934,9 +940,9 @@ static void nv11_cursor_upload(struct drm_device *dev, struct nouveau_bo *src,
 
 #ifdef __BIG_ENDIAN
 		{
-			struct drm_nouveau_private *dev_priv = dev->dev_private;
+			struct nouveau_drm *drm = nouveau_drm(dev);
 
-			if (dev_priv->chipset == 0x11) {
+			if (nv_device(drm->device)->chipset == 0x11) {
 				pixel = ((pixel & 0x000000ff) << 24) |
 					((pixel & 0x0000ff00) << 8) |
 					((pixel & 0x00ff0000) >> 8) |
@@ -953,8 +959,8 @@ static int
 nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
 		     uint32_t buffer_handle, uint32_t width, uint32_t height)
 {
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
-	struct drm_device *dev = dev_priv->dev;
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
+	struct drm_device *dev = drm->dev;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct nouveau_bo *cursor = NULL;
 	struct drm_gem_object *gem;
@@ -977,7 +983,7 @@ nv04_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
 	if (ret)
 		goto out;
 
-	if (dev_priv->chipset >= 0x11)
+	if (nv_device(drm->device)->chipset >= 0x11)
 		nv11_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
 	else
 		nv04_cursor_upload(dev, cursor, nv_crtc->cursor.nvbo);
diff --git a/drivers/gpu/drm/nouveau/nv04_cursor.c b/drivers/gpu/drm/nouveau/nv04_cursor.c
index aaf3de3bc81..fe86f0de348 100644
--- a/drivers/gpu/drm/nouveau/nv04_cursor.c
+++ b/drivers/gpu/drm/nouveau/nv04_cursor.c
@@ -1,7 +1,7 @@
-#include "drmP.h"
-#include "drm_mode.h"
+#include <drm/drmP.h>
+#include <drm/drm_mode.h>
+#include "nouveau_drm.h"
 #include "nouveau_reg.h"
-#include "nouveau_drv.h"
 #include "nouveau_crtc.h"
 #include "nouveau_hw.h"
 
@@ -38,8 +38,8 @@ static void
 nv04_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 	struct drm_crtc *crtc = &nv_crtc->base;
 
 	regp->CRTC[NV_CIO_CRE_HCUR_ADDR0_INDEX] =
@@ -55,7 +55,7 @@ nv04_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR0_INDEX);
 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR1_INDEX);
 	crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_HCUR_ADDR2_INDEX);
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		nv_fix_nv40_hw_cursor(dev, nv_crtc->index);
 }
 
diff --git a/drivers/gpu/drm/nouveau/nv04_dac.c b/drivers/gpu/drm/nouveau/nv04_dac.c
index 38f19479417..347a3bd78d0 100644
--- a/drivers/gpu/drm/nouveau/nv04_dac.c
+++ b/drivers/gpu/drm/nouveau/nv04_dac.c
@@ -24,25 +24,28 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
 #include "nouveau_hw.h"
-#include "nouveau_gpio.h"
 #include "nvreg.h"
 
+#include <subdev/bios/gpio.h>
+#include <subdev/gpio.h>
+#include <subdev/timer.h>
+
 int nv04_dac_output_offset(struct drm_encoder *encoder)
 {
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 	int offset = 0;
 
-	if (dcb->or & (8 | OUTPUT_C))
+	if (dcb->or & (8 | DCB_OUTPUT_C))
 		offset += 0x68;
-	if (dcb->or & (8 | OUTPUT_B))
+	if (dcb->or & (8 | DCB_OUTPUT_B))
 		offset += 0x2000;
 
 	return offset;
@@ -62,6 +65,8 @@ int nv04_dac_output_offset(struct drm_encoder *encoder)
 
 static int sample_load_twice(struct drm_device *dev, bool sense[2])
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_timer *ptimer = nouveau_timer(device);
 	int i;
 
 	for (i = 0; i < 2; i++) {
@@ -75,27 +80,30 @@ static int sample_load_twice(struct drm_device *dev, bool sense[2])
 		 * use a 10ms timeout (guards against crtc being inactive, in
 		 * which case blank state would never change)
 		 */
-		if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-				     0x00000001, 0x00000000))
+		if (!nouveau_timer_wait_eq(ptimer, 10000000,
+					   NV_PRMCIO_INP0__COLOR,
+					   0x00000001, 0x00000000))
 			return -EBUSY;
-		if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-				     0x00000001, 0x00000001))
+		if (!nouveau_timer_wait_eq(ptimer, 10000000,
+					   NV_PRMCIO_INP0__COLOR,
+					   0x00000001, 0x00000001))
 			return -EBUSY;
-		if (!nouveau_wait_eq(dev, 10000000, NV_PRMCIO_INP0__COLOR,
-				     0x00000001, 0x00000000))
+		if (!nouveau_timer_wait_eq(ptimer, 10000000,
+					   NV_PRMCIO_INP0__COLOR,
+					   0x00000001, 0x00000000))
 			return -EBUSY;
 
 		udelay(100);
 		/* when level triggers, sense is _LO_ */
-		sense_a = nv_rd08(dev, NV_PRMCIO_INP0) & 0x10;
+		sense_a = nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
 
 		/* take another reading until it agrees with sense_a... */
 		do {
 			udelay(100);
-			sense_b = nv_rd08(dev, NV_PRMCIO_INP0) & 0x10;
+			sense_b = nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
 			if (sense_a != sense_b) {
 				sense_b_prime =
-					nv_rd08(dev, NV_PRMCIO_INP0) & 0x10;
+					nv_rd08(device, NV_PRMCIO_INP0) & 0x10;
 				if (sense_b == sense_b_prime) {
 					/* ... unless two consecutive subsequent
 					 * samples agree; sense_a is replaced */
@@ -120,6 +128,8 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
 						 struct drm_connector *connector)
 {
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	uint8_t saved_seq1, saved_pi, saved_rpc1, saved_cr_mode;
 	uint8_t saved_palette0[3], saved_palette_mask;
 	uint32_t saved_rtest_ctrl, saved_rgen_ctrl;
@@ -154,11 +164,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
 	saved_rpc1 = NVReadVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX);
 	NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX, saved_rpc1 & ~0xc0);
 
-	nv_wr08(dev, NV_PRMDIO_READ_MODE_ADDRESS, 0x0);
+	nv_wr08(device, NV_PRMDIO_READ_MODE_ADDRESS, 0x0);
 	for (i = 0; i < 3; i++)
-		saved_palette0[i] = nv_rd08(dev, NV_PRMDIO_PALETTE_DATA);
-	saved_palette_mask = nv_rd08(dev, NV_PRMDIO_PIXEL_MASK);
-	nv_wr08(dev, NV_PRMDIO_PIXEL_MASK, 0);
+		saved_palette0[i] = nv_rd08(device, NV_PRMDIO_PALETTE_DATA);
+	saved_palette_mask = nv_rd08(device, NV_PRMDIO_PIXEL_MASK);
+	nv_wr08(device, NV_PRMDIO_PIXEL_MASK, 0);
 
 	saved_rgen_ctrl = NVReadRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL,
@@ -171,11 +181,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
 	do {
 		bool sense_pair[2];
 
-		nv_wr08(dev, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
-		nv_wr08(dev, NV_PRMDIO_PALETTE_DATA, 0);
-		nv_wr08(dev, NV_PRMDIO_PALETTE_DATA, 0);
+		nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
+		nv_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
+		nv_wr08(device, NV_PRMDIO_PALETTE_DATA, 0);
 		/* testing blue won't find monochrome monitors.  I don't care */
-		nv_wr08(dev, NV_PRMDIO_PALETTE_DATA, blue);
+		nv_wr08(device, NV_PRMDIO_PALETTE_DATA, blue);
 
 		i = 0;
 		/* take sample pairs until both samples in the pair agree */
@@ -198,11 +208,11 @@ static enum drm_connector_status nv04_dac_detect(struct drm_encoder *encoder,
 	} while (++blue < 0x18 && sense);
 
 out:
-	nv_wr08(dev, NV_PRMDIO_PIXEL_MASK, saved_palette_mask);
+	nv_wr08(device, NV_PRMDIO_PIXEL_MASK, saved_palette_mask);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_GENERAL_CONTROL, saved_rgen_ctrl);
-	nv_wr08(dev, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
+	nv_wr08(device, NV_PRMDIO_WRITE_MODE_ADDRESS, 0);
 	for (i = 0; i < 3; i++)
-		nv_wr08(dev, NV_PRMDIO_PALETTE_DATA, saved_palette0[i]);
+		nv_wr08(device, NV_PRMDIO_PALETTE_DATA, saved_palette0[i]);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL, saved_rtest_ctrl);
 	NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_PIXEL_INDEX, saved_pi);
 	NVWriteVgaCrtc(dev, 0, NV_CIO_CRE_RPC1_INDEX, saved_rpc1);
@@ -210,7 +220,7 @@ out:
 	NVWriteVgaCrtc(dev, 0, NV_CIO_CR_MODE_INDEX, saved_cr_mode);
 
 	if (blue == 0x18) {
-		NV_INFO(dev, "Load detected on head A\n");
+		NV_INFO(drm, "Load detected on head A\n");
 		return connector_status_connected;
 	}
 
@@ -220,43 +230,46 @@ out:
 uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(device);
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 	uint32_t sample, testval, regoffset = nv04_dac_output_offset(encoder);
 	uint32_t saved_powerctrl_2 = 0, saved_powerctrl_4 = 0, saved_routput,
-		saved_rtest_ctrl, saved_gpio0, saved_gpio1, temp, routput;
+		saved_rtest_ctrl, saved_gpio0 = 0, saved_gpio1 = 0, temp, routput;
 	int head;
 
 #define RGB_TEST_DATA(r, g, b) (r << 0 | g << 10 | b << 20)
-	if (dcb->type == OUTPUT_TV) {
+	if (dcb->type == DCB_OUTPUT_TV) {
 		testval = RGB_TEST_DATA(0xa0, 0xa0, 0xa0);
 
-		if (dev_priv->vbios.tvdactestval)
-			testval = dev_priv->vbios.tvdactestval;
+		if (drm->vbios.tvdactestval)
+			testval = drm->vbios.tvdactestval;
 	} else {
 		testval = RGB_TEST_DATA(0x140, 0x140, 0x140); /* 0x94050140 */
 
-		if (dev_priv->vbios.dactestval)
-			testval = dev_priv->vbios.dactestval;
+		if (drm->vbios.dactestval)
+			testval = drm->vbios.dactestval;
 	}
 
 	saved_rtest_ctrl = NVReadRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset,
 		      saved_rtest_ctrl & ~NV_PRAMDAC_TEST_CONTROL_PWRDWN_DAC_OFF);
 
-	saved_powerctrl_2 = nvReadMC(dev, NV_PBUS_POWERCTRL_2);
+	saved_powerctrl_2 = nv_rd32(device, NV_PBUS_POWERCTRL_2);
 
-	nvWriteMC(dev, NV_PBUS_POWERCTRL_2, saved_powerctrl_2 & 0xd7ffffff);
+	nv_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2 & 0xd7ffffff);
 	if (regoffset == 0x68) {
-		saved_powerctrl_4 = nvReadMC(dev, NV_PBUS_POWERCTRL_4);
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_4, saved_powerctrl_4 & 0xffffffcf);
+		saved_powerctrl_4 = nv_rd32(device, NV_PBUS_POWERCTRL_4);
+		nv_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4 & 0xffffffcf);
 	}
 
-	saved_gpio1 = nouveau_gpio_func_get(dev, DCB_GPIO_TVDAC1);
-	saved_gpio0 = nouveau_gpio_func_get(dev, DCB_GPIO_TVDAC0);
-
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC1, dcb->type == OUTPUT_TV);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC0, dcb->type == OUTPUT_TV);
+	if (gpio) {
+		saved_gpio1 = gpio->get(gpio, 0, DCB_GPIO_TVDAC1, 0xff);
+		saved_gpio0 = gpio->get(gpio, 0, DCB_GPIO_TVDAC0, 0xff);
+		gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, dcb->type == DCB_OUTPUT_TV);
+		gpio->set(gpio, 0, DCB_GPIO_TVDAC0, 0xff, dcb->type == DCB_OUTPUT_TV);
+	}
 
 	msleep(4);
 
@@ -270,8 +283,8 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
 	/* nv driver and nv31 use 0xfffffeee, nv34 and 6600 use 0xfffffece */
 	routput = (saved_routput & 0xfffffece) | head << 8;
 
-	if (dev_priv->card_type >= NV_40) {
-		if (dcb->type == OUTPUT_TV)
+	if (nv_device(drm->device)->card_type >= NV_40) {
+		if (dcb->type == DCB_OUTPUT_TV)
 			routput |= 0x1a << 16;
 		else
 			routput &= ~(0x1a << 16);
@@ -303,11 +316,13 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_DACCLK + regoffset, saved_routput);
 	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + regoffset, saved_rtest_ctrl);
 	if (regoffset == 0x68)
-		nvWriteMC(dev, NV_PBUS_POWERCTRL_4, saved_powerctrl_4);
-	nvWriteMC(dev, NV_PBUS_POWERCTRL_2, saved_powerctrl_2);
+		nv_wr32(device, NV_PBUS_POWERCTRL_4, saved_powerctrl_4);
+	nv_wr32(device, NV_PBUS_POWERCTRL_2, saved_powerctrl_2);
 
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC1, saved_gpio1);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC0, saved_gpio0);
+	if (gpio) {
+		gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, saved_gpio1);
+		gpio->set(gpio, 0, DCB_GPIO_TVDAC0, 0xff, saved_gpio0);
+	}
 
 	return sample;
 }
@@ -315,15 +330,15 @@ uint32_t nv17_dac_sample_load(struct drm_encoder *encoder)
 static enum drm_connector_status
 nv17_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 {
-	struct drm_device *dev = encoder->dev;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 
 	if (nv04_dac_in_use(encoder))
 		return connector_status_disconnected;
 
 	if (nv17_dac_sample_load(encoder) &
 	    NV_PRAMDAC_TEST_CONTROL_SENSEB_ALLHI) {
-		NV_INFO(dev, "Load detected on output %c\n",
+		NV_INFO(drm, "Load detected on output %c\n",
 			'@' + ffs(dcb->or));
 		return connector_status_connected;
 	} else {
@@ -357,7 +372,7 @@ static void nv04_dac_mode_set(struct drm_encoder *encoder,
 			      struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int head = nouveau_crtc(encoder->crtc)->index;
 
 	if (nv_gf4_disp_arch(dev)) {
@@ -372,7 +387,7 @@ static void nv04_dac_mode_set(struct drm_encoder *encoder,
 		/* force any other vga encoders to bind to the other crtc */
 		list_for_each_entry(rebind, &dev->mode_config.encoder_list, head) {
 			if (rebind == encoder
-			    || nouveau_encoder(rebind)->dcb->type != OUTPUT_ANALOG)
+			    || nouveau_encoder(rebind)->dcb->type != DCB_OUTPUT_ANALOG)
 				continue;
 
 			dac_offset = nv04_dac_output_offset(rebind);
@@ -383,7 +398,7 @@ static void nv04_dac_mode_set(struct drm_encoder *encoder,
 	}
 
 	/* This could use refinement for flatpanels, but it should work this way */
-	if (dev_priv->chipset < 0x44)
+	if (nv_device(drm->device)->chipset < 0x44)
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0xf0000000);
 	else
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0x00100000);
@@ -392,13 +407,13 @@ static void nv04_dac_mode_set(struct drm_encoder *encoder,
 static void nv04_dac_commit(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
 
 	helper->dpms(encoder, DRM_MODE_DPMS_ON);
 
-	NV_INFO(dev, "Output %s is running on CRTC %d using output %c\n",
+	NV_INFO(drm, "Output %s is running on CRTC %d using output %c\n",
 		drm_get_connector_name(&nouveau_encoder_connector_get(nv_encoder)->base),
 		nv_crtc->index, '@' + ffs(nv_encoder->dcb->or));
 }
@@ -406,11 +421,10 @@ static void nv04_dac_commit(struct drm_encoder *encoder)
 void nv04_dac_update_dacclk(struct drm_encoder *encoder, bool enable)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 
 	if (nv_gf4_disp_arch(dev)) {
-		uint32_t *dac_users = &dev_priv->dac_users[ffs(dcb->or) - 1];
+		uint32_t *dac_users = &nv04_display(dev)->dac_users[ffs(dcb->or) - 1];
 		int dacclk_off = NV_PRAMDAC_DACCLK + nv04_dac_output_offset(encoder);
 		uint32_t dacclk = NVReadRAMDAC(dev, 0, dacclk_off);
 
@@ -431,23 +445,23 @@ void nv04_dac_update_dacclk(struct drm_encoder *encoder, bool enable)
  * someone else. */
 bool nv04_dac_in_use(struct drm_encoder *encoder)
 {
-	struct drm_nouveau_private *dev_priv = encoder->dev->dev_private;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct drm_device *dev = encoder->dev;
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 
 	return nv_gf4_disp_arch(encoder->dev) &&
-		(dev_priv->dac_users[ffs(dcb->or) - 1] & ~(1 << dcb->index));
+		(nv04_display(dev)->dac_users[ffs(dcb->or) - 1] & ~(1 << dcb->index));
 }
 
 static void nv04_dac_dpms(struct drm_encoder *encoder, int mode)
 {
-	struct drm_device *dev = encoder->dev;
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 
 	if (nv_encoder->last_dpms == mode)
 		return;
 	nv_encoder->last_dpms = mode;
 
-	NV_INFO(dev, "Setting dpms mode %d on vga encoder (output %d)\n",
+	NV_INFO(drm, "Setting dpms mode %d on vga encoder (output %d)\n",
 		     mode, nv_encoder->dcb->index);
 
 	nv04_dac_update_dacclk(encoder, mode == DRM_MODE_DPMS_ON);
@@ -479,8 +493,6 @@ static void nv04_dac_destroy(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 
-	NV_DEBUG_KMS(encoder->dev, "\n");
-
 	drm_encoder_cleanup(encoder);
 	kfree(nv_encoder);
 }
@@ -512,7 +524,7 @@ static const struct drm_encoder_funcs nv04_dac_funcs = {
 };
 
 int
-nv04_dac_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv04_dac_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	const struct drm_encoder_helper_funcs *helper;
 	struct nouveau_encoder *nv_encoder = NULL;
diff --git a/drivers/gpu/drm/nouveau/nv04_dfp.c b/drivers/gpu/drm/nouveau/nv04_dfp.c
index c2675623b7c..da55d7642c8 100644
--- a/drivers/gpu/drm/nouveau/nv04_dfp.c
+++ b/drivers/gpu/drm/nouveau/nv04_dfp.c
@@ -24,17 +24,20 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
 #include "nouveau_hw.h"
 #include "nvreg.h"
 
-#include "i2c/sil164.h"
+#include <drm/i2c/sil164.h>
+
+#include <subdev/i2c.h>
 
 #define FP_TG_CONTROL_ON  (NV_PRAMDAC_FP_TG_CONTROL_DISPEN_POS |	\
 			   NV_PRAMDAC_FP_TG_CONTROL_HSYNC_POS |		\
@@ -49,20 +52,20 @@ static inline bool is_fpc_off(uint32_t fpc)
 			FP_TG_CONTROL_OFF);
 }
 
-int nv04_dfp_get_bound_head(struct drm_device *dev, struct dcb_entry *dcbent)
+int nv04_dfp_get_bound_head(struct drm_device *dev, struct dcb_output *dcbent)
 {
 	/* special case of nv_read_tmds to find crtc associated with an output.
 	 * this does not give a correct answer for off-chip dvi, but there's no
 	 * use for such an answer anyway
 	 */
-	int ramdac = (dcbent->or & OUTPUT_C) >> 2;
+	int ramdac = (dcbent->or & DCB_OUTPUT_C) >> 2;
 
 	NVWriteRAMDAC(dev, ramdac, NV_PRAMDAC_FP_TMDS_CONTROL,
 	NV_PRAMDAC_FP_TMDS_CONTROL_WRITE_DISABLE | 0x4);
 	return ((NVReadRAMDAC(dev, ramdac, NV_PRAMDAC_FP_TMDS_DATA) & 0x8) >> 3) ^ ramdac;
 }
 
-void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_entry *dcbent,
+void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_output *dcbent,
 			int head, bool dl)
 {
 	/* The BIOS scripts don't do this for us, sadly
@@ -72,13 +75,13 @@ void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_entry *dcbent,
 	 * (for VT restore etc.)
 	 */
 
-	int ramdac = (dcbent->or & OUTPUT_C) >> 2;
+	int ramdac = (dcbent->or & DCB_OUTPUT_C) >> 2;
 	uint8_t tmds04 = 0x80;
 
 	if (head != ramdac)
 		tmds04 = 0x88;
 
-	if (dcbent->type == OUTPUT_LVDS)
+	if (dcbent->type == DCB_OUTPUT_LVDS)
 		tmds04 |= 0x01;
 
 	nv_write_tmds(dev, dcbent->or, 0, 0x04, tmds04);
@@ -89,8 +92,7 @@ void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_entry *dcbent,
 
 void nv04_dfp_disable(struct drm_device *dev, int head)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_crtc_reg *crtcstate = dev_priv->mode_reg.crtc_reg;
+	struct nv04_crtc_reg *crtcstate = nv04_display(dev)->mode_reg.crtc_reg;
 
 	if (NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_TG_CONTROL) &
 	    FP_TG_CONTROL_ON) {
@@ -111,14 +113,13 @@ void nv04_dfp_disable(struct drm_device *dev, int head)
 void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct drm_crtc *crtc;
 	struct nouveau_crtc *nv_crtc;
 	uint32_t *fpc;
 
 	if (mode == DRM_MODE_DPMS_ON) {
 		nv_crtc = nouveau_crtc(encoder->crtc);
-		fpc = &dev_priv->mode_reg.crtc_reg[nv_crtc->index].fp_control;
+		fpc = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index].fp_control;
 
 		if (is_fpc_off(*fpc)) {
 			/* using saved value is ok, as (is_digital && dpms_on &&
@@ -133,7 +134,7 @@ void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode)
 	} else {
 		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 			nv_crtc = nouveau_crtc(crtc);
-			fpc = &dev_priv->mode_reg.crtc_reg[nv_crtc->index].fp_control;
+			fpc = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index].fp_control;
 
 			nv_crtc->fp_users &= ~(1 << nouveau_encoder(encoder)->dcb->index);
 			if (!is_fpc_off(*fpc) && !nv_crtc->fp_users) {
@@ -151,10 +152,10 @@ void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode)
 static struct drm_encoder *get_tmds_slave(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 	struct drm_encoder *slave;
 
-	if (dcb->type != OUTPUT_TMDS || dcb->location == DCB_LOC_ON_CHIP)
+	if (dcb->type != DCB_OUTPUT_TMDS || dcb->location == DCB_LOC_ON_CHIP)
 		return NULL;
 
 	/* Some BIOSes (e.g. the one in a Quadro FX1000) report several
@@ -168,9 +169,9 @@ static struct drm_encoder *get_tmds_slave(struct drm_encoder *encoder)
 	 * let's do the same.
 	 */
 	list_for_each_entry(slave, &dev->mode_config.encoder_list, head) {
-		struct dcb_entry *slave_dcb = nouveau_encoder(slave)->dcb;
+		struct dcb_output *slave_dcb = nouveau_encoder(slave)->dcb;
 
-		if (slave_dcb->type == OUTPUT_TMDS && get_slave_funcs(slave) &&
+		if (slave_dcb->type == DCB_OUTPUT_TMDS && get_slave_funcs(slave) &&
 		    slave_dcb->tmdsconf.slave_addr == dcb->tmdsconf.slave_addr)
 			return slave;
 	}
@@ -202,9 +203,8 @@ static bool nv04_dfp_mode_fixup(struct drm_encoder *encoder,
 static void nv04_dfp_prepare_sel_clk(struct drm_device *dev,
 				     struct nouveau_encoder *nv_encoder, int head)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_mode_state *state = &dev_priv->mode_reg;
-	uint32_t bits1618 = nv_encoder->dcb->or & OUTPUT_A ? 0x10000 : 0x40000;
+	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
+	uint32_t bits1618 = nv_encoder->dcb->or & DCB_OUTPUT_A ? 0x10000 : 0x40000;
 
 	if (nv_encoder->dcb->location != DCB_LOC_ON_CHIP)
 		return;
@@ -233,8 +233,8 @@ static void nv04_dfp_prepare_sel_clk(struct drm_device *dev,
 	 * 	and which bit-pair to use, is unclear on nv40 (for earlier cards, the fp table
 	 * 	entry has the necessary info)
 	 */
-	if (nv_encoder->dcb->type == OUTPUT_LVDS && dev_priv->saved_reg.sel_clk & 0xf0) {
-		int shift = (dev_priv->saved_reg.sel_clk & 0x50) ? 0 : 1;
+	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS && nv04_display(dev)->saved_reg.sel_clk & 0xf0) {
+		int shift = (nv04_display(dev)->saved_reg.sel_clk & 0x50) ? 0 : 1;
 
 		state->sel_clk &= ~0xf0;
 		state->sel_clk |= (head ? 0x40 : 0x10) << shift;
@@ -246,9 +246,8 @@ static void nv04_dfp_prepare(struct drm_encoder *encoder)
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	int head = nouveau_crtc(encoder->crtc)->index;
-	struct nv04_crtc_reg *crtcstate = dev_priv->mode_reg.crtc_reg;
+	struct nv04_crtc_reg *crtcstate = nv04_display(dev)->mode_reg.crtc_reg;
 	uint8_t *cr_lcd = &crtcstate[head].CRTC[NV_CIO_CRE_LCD__INDEX];
 	uint8_t *cr_lcd_oth = &crtcstate[head ^ 1].CRTC[NV_CIO_CRE_LCD__INDEX];
 
@@ -263,7 +262,7 @@ static void nv04_dfp_prepare(struct drm_encoder *encoder)
 			*cr_lcd |= head ? 0x0 : 0x8;
 		else {
 			*cr_lcd |= (nv_encoder->dcb->or << 4) & 0x30;
-			if (nv_encoder->dcb->type == OUTPUT_LVDS)
+			if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS)
 				*cr_lcd |= 0x30;
 			if ((*cr_lcd & 0x30) == (*cr_lcd_oth & 0x30)) {
 				/* avoid being connected to both crtcs */
@@ -282,17 +281,18 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 			      struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
-	struct nv04_crtc_reg *savep = &dev_priv->saved_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *savep = &nv04_display(dev)->saved_reg.crtc_reg[nv_crtc->index];
 	struct nouveau_connector *nv_connector = nouveau_crtc_connector_get(nv_crtc);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_display_mode *output_mode = &nv_encoder->mode;
 	struct drm_connector *connector = &nv_connector->base;
 	uint32_t mode_ratio, panel_ratio;
 
-	NV_DEBUG_KMS(dev, "Output mode on CRTC %d:\n", nv_crtc->index);
+	NV_DEBUG(drm, "Output mode on CRTC %d:\n", nv_crtc->index);
 	drm_mode_debug_printmodeline(output_mode);
 
 	/* Initialize the FP registers in this CRTC. */
@@ -300,10 +300,10 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 	regp->fp_horiz_regs[FP_TOTAL] = output_mode->htotal - 1;
 	if (!nv_gf4_disp_arch(dev) ||
 	    (output_mode->hsync_start - output_mode->hdisplay) >=
-					dev_priv->vbios.digital_min_front_porch)
+					drm->vbios.digital_min_front_porch)
 		regp->fp_horiz_regs[FP_CRTC] = output_mode->hdisplay;
 	else
-		regp->fp_horiz_regs[FP_CRTC] = output_mode->hsync_start - dev_priv->vbios.digital_min_front_porch - 1;
+		regp->fp_horiz_regs[FP_CRTC] = output_mode->hsync_start - drm->vbios.digital_min_front_porch - 1;
 	regp->fp_horiz_regs[FP_SYNC_START] = output_mode->hsync_start - 1;
 	regp->fp_horiz_regs[FP_SYNC_END] = output_mode->hsync_end - 1;
 	regp->fp_horiz_regs[FP_VALID_START] = output_mode->hskew;
@@ -335,12 +335,12 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 		regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_MODE_NATIVE;
 	else /* gpu needs to scale */
 		regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_MODE_SCALE;
-	if (nvReadEXTDEV(dev, NV_PEXTDEV_BOOT_0) & NV_PEXTDEV_BOOT_0_STRAP_FP_IFACE_12BIT)
+	if (nv_rd32(device, NV_PEXTDEV_BOOT_0) & NV_PEXTDEV_BOOT_0_STRAP_FP_IFACE_12BIT)
 		regp->fp_control |= NV_PRAMDAC_FP_TG_CONTROL_WIDTH_12;
 	if (nv_encoder->dcb->location != DCB_LOC_ON_CHIP &&
 	    output_mode->clock > 165000)
 		regp->fp_control |= (2 << 24);
-	if (nv_encoder->dcb->type == OUTPUT_LVDS) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS) {
 		bool duallink = false, dummy;
 		if (nv_connector->edid &&
 		    nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
@@ -416,7 +416,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 	if ((nv_connector->dithering_mode == DITHERING_MODE_ON) ||
 	    (nv_connector->dithering_mode == DITHERING_MODE_AUTO &&
 	     encoder->crtc->fb->depth > connector->display_info.bpc * 3)) {
-		if (dev_priv->chipset == 0x11)
+		if (nv_device(drm->device)->chipset == 0x11)
 			regp->dither = savep->dither | 0x00010000;
 		else {
 			int i;
@@ -427,7 +427,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 			}
 		}
 	} else {
-		if (dev_priv->chipset != 0x11) {
+		if (nv_device(drm->device)->chipset != 0x11) {
 			/* reset them */
 			int i;
 			for (i = 0; i < 3; i++) {
@@ -444,26 +444,26 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
 static void nv04_dfp_commit(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct dcb_entry *dcbe = nv_encoder->dcb;
+	struct dcb_output *dcbe = nv_encoder->dcb;
 	int head = nouveau_crtc(encoder->crtc)->index;
 	struct drm_encoder *slave_encoder;
 
-	if (dcbe->type == OUTPUT_TMDS)
+	if (dcbe->type == DCB_OUTPUT_TMDS)
 		run_tmds_table(dev, dcbe, head, nv_encoder->mode.clock);
-	else if (dcbe->type == OUTPUT_LVDS)
+	else if (dcbe->type == DCB_OUTPUT_LVDS)
 		call_lvds_script(dev, dcbe, head, LVDS_RESET, nv_encoder->mode.clock);
 
 	/* update fp_control state for any changes made by scripts,
 	 * so correct value is written at DPMS on */
-	dev_priv->mode_reg.crtc_reg[head].fp_control =
+	nv04_display(dev)->mode_reg.crtc_reg[head].fp_control =
 		NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_TG_CONTROL);
 
 	/* This could use refinement for flatpanels, but it should work this way */
-	if (dev_priv->chipset < 0x44)
+	if (nv_device(drm->device)->chipset < 0x44)
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0xf0000000);
 	else
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL + nv04_dac_output_offset(encoder), 0x00100000);
@@ -476,7 +476,7 @@ static void nv04_dfp_commit(struct drm_encoder *encoder)
 
 	helper->dpms(encoder, DRM_MODE_DPMS_ON);
 
-	NV_INFO(dev, "Output %s is running on CRTC %d using output %c\n",
+	NV_INFO(drm, "Output %s is running on CRTC %d using output %c\n",
 		drm_get_connector_name(&nouveau_encoder_connector_get(nv_encoder)->base),
 		nv_crtc->index, '@' + ffs(nv_encoder->dcb->or));
 }
@@ -485,6 +485,7 @@ static void nv04_dfp_update_backlight(struct drm_encoder *encoder, int mode)
 {
 #ifdef __powerpc__
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 
 	/* BIOS scripts usually take care of the backlight, thanks
 	 * Apple for your consistency.
@@ -492,11 +493,11 @@ static void nv04_dfp_update_backlight(struct drm_encoder *encoder, int mode)
 	if (dev->pci_device == 0x0179 || dev->pci_device == 0x0189 ||
 	    dev->pci_device == 0x0329) {
 		if (mode == DRM_MODE_DPMS_ON) {
-			nv_mask(dev, NV_PBUS_DEBUG_DUALHEAD_CTL, 0, 1 << 31);
-			nv_mask(dev, NV_PCRTC_GPIO_EXT, 3, 1);
+			nv_mask(device, NV_PBUS_DEBUG_DUALHEAD_CTL, 0, 1 << 31);
+			nv_mask(device, NV_PCRTC_GPIO_EXT, 3, 1);
 		} else {
-			nv_mask(dev, NV_PBUS_DEBUG_DUALHEAD_CTL, 1 << 31, 0);
-			nv_mask(dev, NV_PCRTC_GPIO_EXT, 3, 0);
+			nv_mask(device, NV_PBUS_DEBUG_DUALHEAD_CTL, 1 << 31, 0);
+			nv_mask(device, NV_PCRTC_GPIO_EXT, 3, 0);
 		}
 	}
 #endif
@@ -511,7 +512,7 @@ static void nv04_lvds_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_crtc *crtc = encoder->crtc;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	bool was_powersaving = is_powersaving_dpms(nv_encoder->last_dpms);
 
@@ -519,7 +520,7 @@ static void nv04_lvds_dpms(struct drm_encoder *encoder, int mode)
 		return;
 	nv_encoder->last_dpms = mode;
 
-	NV_INFO(dev, "Setting dpms mode %d on lvds encoder (output %d)\n",
+	NV_INFO(drm, "Setting dpms mode %d on lvds encoder (output %d)\n",
 		     mode, nv_encoder->dcb->index);
 
 	if (was_powersaving && is_powersaving_dpms(mode))
@@ -549,22 +550,22 @@ static void nv04_lvds_dpms(struct drm_encoder *encoder, int mode)
 	if (mode == DRM_MODE_DPMS_ON)
 		nv04_dfp_prepare_sel_clk(dev, nv_encoder, nouveau_crtc(crtc)->index);
 	else {
-		dev_priv->mode_reg.sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK);
-		dev_priv->mode_reg.sel_clk &= ~0xf0;
+		nv04_display(dev)->mode_reg.sel_clk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK);
+		nv04_display(dev)->mode_reg.sel_clk &= ~0xf0;
 	}
-	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, dev_priv->mode_reg.sel_clk);
+	NVWriteRAMDAC(dev, 0, NV_PRAMDAC_SEL_CLK, nv04_display(dev)->mode_reg.sel_clk);
 }
 
 static void nv04_tmds_dpms(struct drm_encoder *encoder, int mode)
 {
-	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 
 	if (nv_encoder->last_dpms == mode)
 		return;
 	nv_encoder->last_dpms = mode;
 
-	NV_INFO(dev, "Setting dpms mode %d on tmds encoder (output %d)\n",
+	NV_INFO(drm, "Setting dpms mode %d on tmds encoder (output %d)\n",
 		     mode, nv_encoder->dcb->index);
 
 	nv04_dfp_update_backlight(encoder, mode);
@@ -585,10 +586,9 @@ static void nv04_dfp_restore(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	int head = nv_encoder->restore.head;
 
-	if (nv_encoder->dcb->type == OUTPUT_LVDS) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS) {
 		struct nouveau_connector *connector =
 			nouveau_encoder_connector_get(nv_encoder);
 
@@ -597,9 +597,9 @@ static void nv04_dfp_restore(struct drm_encoder *encoder)
 					 LVDS_PANEL_ON,
 					 connector->native_mode->clock);
 
-	} else if (nv_encoder->dcb->type == OUTPUT_TMDS) {
+	} else if (nv_encoder->dcb->type == DCB_OUTPUT_TMDS) {
 		int clock = nouveau_hw_pllvals_to_clk
-					(&dev_priv->saved_reg.crtc_reg[head].pllvals);
+					(&nv04_display(dev)->saved_reg.crtc_reg[head].pllvals);
 
 		run_tmds_table(dev, nv_encoder->dcb, head, clock);
 	}
@@ -611,8 +611,6 @@ static void nv04_dfp_destroy(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 
-	NV_DEBUG_KMS(encoder->dev, "\n");
-
 	if (get_slave_funcs(encoder))
 		get_slave_funcs(encoder)->destroy(encoder);
 
@@ -623,8 +621,10 @@ static void nv04_dfp_destroy(struct drm_encoder *encoder)
 static void nv04_tmds_slave_init(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
-	struct nouveau_i2c_chan *i2c = nouveau_i2c_find(dev, 2);
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+	struct nouveau_i2c_port *port = i2c->find(i2c, 2);
 	struct i2c_board_info info[] = {
 		{
 			.type = "sil164",
@@ -637,16 +637,16 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
 	};
 	int type;
 
-	if (!nv_gf4_disp_arch(dev) || !i2c ||
+	if (!nv_gf4_disp_arch(dev) || !port ||
 	    get_tmds_slave(encoder))
 		return;
 
-	type = nouveau_i2c_identify(dev, "TMDS transmitter", info, NULL, 2);
+	type = i2c->identify(i2c, 2, "TMDS transmitter", info, NULL);
 	if (type < 0)
 		return;
 
 	drm_i2c_encoder_init(dev, to_encoder_slave(encoder),
-			     &i2c->adapter, &info[type]);
+			     &port->adapter, &info[type]);
 }
 
 static const struct drm_encoder_helper_funcs nv04_lvds_helper_funcs = {
@@ -676,7 +676,7 @@ static const struct drm_encoder_funcs nv04_dfp_funcs = {
 };
 
 int
-nv04_dfp_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv04_dfp_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	const struct drm_encoder_helper_funcs *helper;
 	struct nouveau_encoder *nv_encoder = NULL;
@@ -684,11 +684,11 @@ nv04_dfp_create(struct drm_connector *connector, struct dcb_entry *entry)
 	int type;
 
 	switch (entry->type) {
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		type = DRM_MODE_ENCODER_TMDS;
 		helper = &nv04_tmds_helper_funcs;
 		break;
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		type = DRM_MODE_ENCODER_LVDS;
 		helper = &nv04_lvds_helper_funcs;
 		break;
@@ -711,7 +711,7 @@ nv04_dfp_create(struct drm_connector *connector, struct dcb_entry *entry)
 	encoder->possible_crtcs = entry->heads;
 	encoder->possible_clones = 0;
 
-	if (entry->type == OUTPUT_TMDS &&
+	if (entry->type == DCB_OUTPUT_TMDS &&
 	    entry->location != DCB_LOC_ON_CHIP)
 		nv04_tmds_slave_init(encoder);
 
diff --git a/drivers/gpu/drm/nouveau/nv04_display.c b/drivers/gpu/drm/nouveau/nv04_display.c
index 44488e3a257..846050f04c2 100644
--- a/drivers/gpu/drm/nouveau/nv04_display.c
+++ b/drivers/gpu/drm/nouveau/nv04_display.c
@@ -22,82 +22,18 @@
  * Author: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#include "nouveau_drv.h"
-#include "nouveau_fb.h"
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_hw.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 
-static void nv04_vblank_crtc0_isr(struct drm_device *);
-static void nv04_vblank_crtc1_isr(struct drm_device *);
-
-static void
-nv04_display_store_initial_head_owner(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->chipset != 0x11) {
-		dev_priv->crtc_owner = NVReadVgaCrtc(dev, 0, NV_CIO_CRE_44);
-		return;
-	}
-
-	/* reading CR44 is broken on nv11, so we attempt to infer it */
-	if (nvReadMC(dev, NV_PBUS_DEBUG_1) & (1 << 28))	/* heads tied, restore both */
-		dev_priv->crtc_owner = 0x4;
-	else {
-		uint8_t slaved_on_A, slaved_on_B;
-		bool tvA = false;
-		bool tvB = false;
-
-		slaved_on_B = NVReadVgaCrtc(dev, 1, NV_CIO_CRE_PIXEL_INDEX) &
-									0x80;
-		if (slaved_on_B)
-			tvB = !(NVReadVgaCrtc(dev, 1, NV_CIO_CRE_LCD__INDEX) &
-					MASK(NV_CIO_CRE_LCD_LCD_SELECT));
-
-		slaved_on_A = NVReadVgaCrtc(dev, 0, NV_CIO_CRE_PIXEL_INDEX) &
-									0x80;
-		if (slaved_on_A)
-			tvA = !(NVReadVgaCrtc(dev, 0, NV_CIO_CRE_LCD__INDEX) &
-					MASK(NV_CIO_CRE_LCD_LCD_SELECT));
-
-		if (slaved_on_A && !tvA)
-			dev_priv->crtc_owner = 0x0;
-		else if (slaved_on_B && !tvB)
-			dev_priv->crtc_owner = 0x3;
-		else if (slaved_on_A)
-			dev_priv->crtc_owner = 0x0;
-		else if (slaved_on_B)
-			dev_priv->crtc_owner = 0x3;
-		else
-			dev_priv->crtc_owner = 0x0;
-	}
-}
-
 int
 nv04_display_early_init(struct drm_device *dev)
 {
-	/* Make the I2C buses accessible. */
-	if (!nv_gf4_disp_arch(dev)) {
-		uint32_t pmc_enable = nv_rd32(dev, NV03_PMC_ENABLE);
-
-		if (!(pmc_enable & 1))
-			nv_wr32(dev, NV03_PMC_ENABLE, pmc_enable | 1);
-	}
-
-	/* Unlock the VGA CRTCs. */
-	NVLockVgaCrtcs(dev, false);
-
-	/* Make sure the CRTCs aren't in slaved mode. */
-	if (nv_two_heads(dev)) {
-		nv04_display_store_initial_head_owner(dev);
-		NVSetOwner(dev, 0);
-	}
-
 	/* ensure vblank interrupts are off, they can't be enabled until
 	 * drm_vblank has been initialised
 	 */
@@ -111,25 +47,29 @@ nv04_display_early_init(struct drm_device *dev)
 void
 nv04_display_late_takedown(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (nv_two_heads(dev))
-		NVSetOwner(dev, dev_priv->crtc_owner);
-
-	NVLockVgaCrtcs(dev, true);
 }
 
 int
 nv04_display_create(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_table *dcb = &dev_priv->vbios.dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_table *dcb = &drm->vbios.dcb;
 	struct drm_connector *connector, *ct;
 	struct drm_encoder *encoder;
 	struct drm_crtc *crtc;
+	struct nv04_display *disp;
 	int i, ret;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
+
+	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
+	if (!disp)
+		return -ENOMEM;
+
+	nouveau_display(dev)->priv = disp;
+	nouveau_display(dev)->dtor = nv04_display_destroy;
+	nouveau_display(dev)->init = nv04_display_init;
+	nouveau_display(dev)->fini = nv04_display_fini;
 
 	nouveau_hw_save_vga_fonts(dev, 1);
 
@@ -138,28 +78,28 @@ nv04_display_create(struct drm_device *dev)
 		nv04_crtc_create(dev, 1);
 
 	for (i = 0; i < dcb->entries; i++) {
-		struct dcb_entry *dcbent = &dcb->entry[i];
+		struct dcb_output *dcbent = &dcb->entry[i];
 
 		connector = nouveau_connector_create(dev, dcbent->connector);
 		if (IS_ERR(connector))
 			continue;
 
 		switch (dcbent->type) {
-		case OUTPUT_ANALOG:
+		case DCB_OUTPUT_ANALOG:
 			ret = nv04_dac_create(connector, dcbent);
 			break;
-		case OUTPUT_LVDS:
-		case OUTPUT_TMDS:
+		case DCB_OUTPUT_LVDS:
+		case DCB_OUTPUT_TMDS:
 			ret = nv04_dfp_create(connector, dcbent);
 			break;
-		case OUTPUT_TV:
+		case DCB_OUTPUT_TV:
 			if (dcbent->location == DCB_LOC_ON_CHIP)
 				ret = nv17_tv_create(connector, dcbent);
 			else
 				ret = nv04_tv_create(connector, dcbent);
 			break;
 		default:
-			NV_WARN(dev, "DCB type %d not known\n", dcbent->type);
+			NV_WARN(drm, "DCB type %d not known\n", dcbent->type);
 			continue;
 		}
 
@@ -170,7 +110,7 @@ nv04_display_create(struct drm_device *dev)
 	list_for_each_entry_safe(connector, ct,
 				 &dev->mode_config.connector_list, head) {
 		if (!connector->encoder_ids[0]) {
-			NV_WARN(dev, "%s has no encoders, removing\n",
+			NV_WARN(drm, "%s has no encoders, removing\n",
 				drm_get_connector_name(connector));
 			connector->funcs->destroy(connector);
 		}
@@ -186,21 +126,18 @@ nv04_display_create(struct drm_device *dev)
 		func->save(encoder);
 	}
 
-	nouveau_irq_register(dev, 24, nv04_vblank_crtc0_isr);
-	nouveau_irq_register(dev, 25, nv04_vblank_crtc1_isr);
 	return 0;
 }
 
 void
 nv04_display_destroy(struct drm_device *dev)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv04_display *disp = nv04_display(dev);
 	struct drm_encoder *encoder;
 	struct drm_crtc *crtc;
 
-	NV_DEBUG_KMS(dev, "\n");
-
-	nouveau_irq_unregister(dev, 24);
-	nouveau_irq_unregister(dev, 25);
+	NV_DEBUG(drm, "\n");
 
 	/* Turn every CRTC off. */
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
@@ -222,6 +159,9 @@ nv04_display_destroy(struct drm_device *dev)
 		crtc->funcs->restore(crtc);
 
 	nouveau_hw_save_vga_fonts(dev, 0);
+
+	nouveau_display(dev)->priv = NULL;
+	kfree(disp);
 }
 
 int
@@ -258,17 +198,3 @@ nv04_display_fini(struct drm_device *dev)
 	if (nv_two_heads(dev))
 		NVWriteCRTC(dev, 1, NV_PCRTC_INTR_EN_0, 0);
 }
-
-static void
-nv04_vblank_crtc0_isr(struct drm_device *dev)
-{
-	nv_wr32(dev, NV_CRTC0_INTSTAT, NV_CRTC_INTR_VBLANK);
-	drm_handle_vblank(dev, 0);
-}
-
-static void
-nv04_vblank_crtc1_isr(struct drm_device *dev)
-{
-	nv_wr32(dev, NV_CRTC1_INTSTAT, NV_CRTC_INTR_VBLANK);
-	drm_handle_vblank(dev, 1);
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_display.h b/drivers/gpu/drm/nouveau/nv04_display.h
new file mode 100644
index 00000000000..45322802e37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv04_display.h
@@ -0,0 +1,184 @@
+#ifndef __NV04_DISPLAY_H__
+#define __NV04_DISPLAY_H__
+
+#include <subdev/bios/pll.h>
+
+#include "nouveau_display.h"
+
+enum nv04_fp_display_regs {
+	FP_DISPLAY_END,
+	FP_TOTAL,
+	FP_CRTC,
+	FP_SYNC_START,
+	FP_SYNC_END,
+	FP_VALID_START,
+	FP_VALID_END
+};
+
+struct nv04_crtc_reg {
+	unsigned char MiscOutReg;
+	uint8_t CRTC[0xa0];
+	uint8_t CR58[0x10];
+	uint8_t Sequencer[5];
+	uint8_t Graphics[9];
+	uint8_t Attribute[21];
+	unsigned char DAC[768];
+
+	/* PCRTC regs */
+	uint32_t fb_start;
+	uint32_t crtc_cfg;
+	uint32_t cursor_cfg;
+	uint32_t gpio_ext;
+	uint32_t crtc_830;
+	uint32_t crtc_834;
+	uint32_t crtc_850;
+	uint32_t crtc_eng_ctrl;
+
+	/* PRAMDAC regs */
+	uint32_t nv10_cursync;
+	struct nouveau_pll_vals pllvals;
+	uint32_t ramdac_gen_ctrl;
+	uint32_t ramdac_630;
+	uint32_t ramdac_634;
+	uint32_t tv_setup;
+	uint32_t tv_vtotal;
+	uint32_t tv_vskew;
+	uint32_t tv_vsync_delay;
+	uint32_t tv_htotal;
+	uint32_t tv_hskew;
+	uint32_t tv_hsync_delay;
+	uint32_t tv_hsync_delay2;
+	uint32_t fp_horiz_regs[7];
+	uint32_t fp_vert_regs[7];
+	uint32_t dither;
+	uint32_t fp_control;
+	uint32_t dither_regs[6];
+	uint32_t fp_debug_0;
+	uint32_t fp_debug_1;
+	uint32_t fp_debug_2;
+	uint32_t fp_margin_color;
+	uint32_t ramdac_8c0;
+	uint32_t ramdac_a20;
+	uint32_t ramdac_a24;
+	uint32_t ramdac_a34;
+	uint32_t ctv_regs[38];
+};
+
+struct nv04_output_reg {
+	uint32_t output;
+	int head;
+};
+
+struct nv04_mode_state {
+	struct nv04_crtc_reg crtc_reg[2];
+	uint32_t pllsel;
+	uint32_t sel_clk;
+};
+
+struct nv04_display {
+	struct nv04_mode_state mode_reg;
+	struct nv04_mode_state saved_reg;
+	uint32_t saved_vga_font[4][16384];
+	uint32_t dac_users[4];
+};
+
+static inline struct nv04_display *
+nv04_display(struct drm_device *dev)
+{
+	return nouveau_display(dev)->priv;
+}
+
+/* nv04_display.c */
+int nv04_display_early_init(struct drm_device *);
+void nv04_display_late_takedown(struct drm_device *);
+int nv04_display_create(struct drm_device *);
+void nv04_display_destroy(struct drm_device *);
+int nv04_display_init(struct drm_device *);
+void nv04_display_fini(struct drm_device *);
+
+/* nv04_crtc.c */
+int nv04_crtc_create(struct drm_device *, int index);
+
+/* nv04_dac.c */
+int nv04_dac_create(struct drm_connector *, struct dcb_output *);
+uint32_t nv17_dac_sample_load(struct drm_encoder *encoder);
+int nv04_dac_output_offset(struct drm_encoder *encoder);
+void nv04_dac_update_dacclk(struct drm_encoder *encoder, bool enable);
+bool nv04_dac_in_use(struct drm_encoder *encoder);
+
+/* nv04_dfp.c */
+int nv04_dfp_create(struct drm_connector *, struct dcb_output *);
+int nv04_dfp_get_bound_head(struct drm_device *dev, struct dcb_output *dcbent);
+void nv04_dfp_bind_head(struct drm_device *dev, struct dcb_output *dcbent,
+			       int head, bool dl);
+void nv04_dfp_disable(struct drm_device *dev, int head);
+void nv04_dfp_update_fp_control(struct drm_encoder *encoder, int mode);
+
+/* nv04_tv.c */
+int nv04_tv_identify(struct drm_device *dev, int i2c_index);
+int nv04_tv_create(struct drm_connector *, struct dcb_output *);
+
+/* nv17_tv.c */
+int nv17_tv_create(struct drm_connector *, struct dcb_output *);
+
+static inline bool
+nv_two_heads(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	const int impl = dev->pci_device & 0x0ff0;
+
+	if (nv_device(drm->device)->card_type >= NV_10 && impl != 0x0100 &&
+	    impl != 0x0150 && impl != 0x01a0 && impl != 0x0200)
+		return true;
+
+	return false;
+}
+
+static inline bool
+nv_gf4_disp_arch(struct drm_device *dev)
+{
+	return nv_two_heads(dev) && (dev->pci_device & 0x0ff0) != 0x0110;
+}
+
+static inline bool
+nv_two_reg_pll(struct drm_device *dev)
+{
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	const int impl = dev->pci_device & 0x0ff0;
+
+	if (impl == 0x0310 || impl == 0x0340 || nv_device(drm->device)->card_type >= NV_40)
+		return true;
+	return false;
+}
+
+static inline bool
+nv_match_device(struct drm_device *dev, unsigned device,
+		unsigned sub_vendor, unsigned sub_device)
+{
+	return dev->pdev->device == device &&
+		dev->pdev->subsystem_vendor == sub_vendor &&
+		dev->pdev->subsystem_device == sub_device;
+}
+
+#include <subdev/bios.h>
+#include <subdev/bios/init.h>
+
+static inline void
+nouveau_bios_run_init_table(struct drm_device *dev, u16 table,
+			    struct dcb_output *outp, int crtc)
+{
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nvbios_init init = {
+		.subdev = nv_subdev(bios),
+		.bios = bios,
+		.offset = table,
+		.outp = outp,
+		.crtc = crtc,
+		.execute = 1,
+	};
+
+	nvbios_exec(&init);
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nv04_fb.c b/drivers/gpu/drm/nouveau/nv04_fb.c
deleted file mode 100644
index d5eedd67afe..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_fb.c
+++ /dev/null
@@ -1,55 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-int
-nv04_fb_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 boot0 = nv_rd32(dev, NV04_PFB_BOOT_0);
-
-	if (boot0 & 0x00000100) {
-		dev_priv->vram_size  = ((boot0 >> 12) & 0xf) * 2 + 2;
-		dev_priv->vram_size *= 1024 * 1024;
-	} else {
-		switch (boot0 & NV04_PFB_BOOT_0_RAM_AMOUNT) {
-		case NV04_PFB_BOOT_0_RAM_AMOUNT_32MB:
-			dev_priv->vram_size = 32 * 1024 * 1024;
-			break;
-		case NV04_PFB_BOOT_0_RAM_AMOUNT_16MB:
-			dev_priv->vram_size = 16 * 1024 * 1024;
-			break;
-		case NV04_PFB_BOOT_0_RAM_AMOUNT_8MB:
-			dev_priv->vram_size = 8 * 1024 * 1024;
-			break;
-		case NV04_PFB_BOOT_0_RAM_AMOUNT_4MB:
-			dev_priv->vram_size = 4 * 1024 * 1024;
-			break;
-		}
-	}
-
-	if ((boot0 & 0x00000038) <= 0x10)
-		dev_priv->vram_type = NV_MEM_TYPE_SGRAM;
-	else
-		dev_priv->vram_type = NV_MEM_TYPE_SDRAM;
-
-	return 0;
-}
-
-int
-nv04_fb_init(struct drm_device *dev)
-{
-	/* This is what the DDX did for NV_ARCH_04, but a mmio-trace shows
-	 * nvidia reading PFB_CFG_0, then writing back its original value.
-	 * (which was 0x701114 in this case)
-	 */
-
-	nv_wr32(dev, NV04_PFB_CFG0, 0x1114);
-	return 0;
-}
-
-void
-nv04_fb_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index 7cd7857347e..77dcc9c5077 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -22,19 +22,18 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <core/object.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fbcon.h"
 
 int
 nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, 4);
@@ -53,9 +52,8 @@ int
 nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, 7);
@@ -81,9 +79,8 @@ int
 nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	uint32_t fg;
 	uint32_t bg;
 	uint32_t dsize;
@@ -142,9 +139,10 @@ nv04_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
 	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
-	const int sub = NvSubCtxSurf2D;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_channel *chan = drm->channel;
+	struct nouveau_device *device = nv_device(drm->device);
+	struct nouveau_object *object;
 	int surface_fmt, pattern_fmt, rect_fmt;
 	int ret;
 
@@ -176,31 +174,35 @@ nv04_fbcon_accel_init(struct fb_info *info)
 		return -EINVAL;
 	}
 
-	ret = nouveau_gpuobj_gr_new(chan, NvCtxSurf2D,
-				    dev_priv->card_type >= NV_10 ?
-				    0x0062 : 0x0042);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvCtxSurf2D,
+				 device->card_type >= NV_10 ? 0x0062 : 0x0042,
+				 NULL, 0, &object);
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_gr_new(chan, NvClipRect, 0x0019);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvClipRect,
+				 0x0019, NULL, 0, &object);
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_gr_new(chan, NvRop, 0x0043);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvRop,
+				 0x0043, NULL, 0, &object);
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_gr_new(chan, NvImagePatt, 0x0044);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvImagePatt,
+				 0x0044, NULL, 0, &object);
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_gr_new(chan, NvGdiRect, 0x004a);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvGdiRect,
+				 0x004a, NULL, 0, &object);
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_gr_new(chan, NvImageBlit,
-				    dev_priv->chipset >= 0x11 ?
-				    0x009f : 0x005f);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, NvImageBlit,
+				 device->chipset >= 0x11 ? 0x009f : 0x005f,
+				 NULL, 0, &object);
 	if (ret)
 		return ret;
 
@@ -209,25 +211,25 @@ nv04_fbcon_accel_init(struct fb_info *info)
 		return 0;
 	}
 
-	BEGIN_NV04(chan, sub, 0x0000, 1);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
 	OUT_RING(chan, NvCtxSurf2D);
-	BEGIN_NV04(chan, sub, 0x0184, 2);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0184, 2);
 	OUT_RING(chan, NvDmaFB);
 	OUT_RING(chan, NvDmaFB);
-	BEGIN_NV04(chan, sub, 0x0300, 4);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 4);
 	OUT_RING(chan, surface_fmt);
 	OUT_RING(chan, info->fix.line_length | (info->fix.line_length << 16));
 	OUT_RING(chan, info->fix.smem_start - dev->mode_config.fb_base);
 	OUT_RING(chan, info->fix.smem_start - dev->mode_config.fb_base);
 
-	BEGIN_NV04(chan, sub, 0x0000, 1);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
 	OUT_RING(chan, NvRop);
-	BEGIN_NV04(chan, sub, 0x0300, 1);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 1);
 	OUT_RING(chan, 0x55);
 
-	BEGIN_NV04(chan, sub, 0x0000, 1);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
 	OUT_RING(chan, NvImagePatt);
-	BEGIN_NV04(chan, sub, 0x0300, 8);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 8);
 	OUT_RING(chan, pattern_fmt);
 #ifdef __BIG_ENDIAN
 	OUT_RING(chan, 2);
@@ -241,9 +243,9 @@ nv04_fbcon_accel_init(struct fb_info *info)
 	OUT_RING(chan, ~0);
 	OUT_RING(chan, ~0);
 
-	BEGIN_NV04(chan, sub, 0x0000, 1);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0000, 1);
 	OUT_RING(chan, NvClipRect);
-	BEGIN_NV04(chan, sub, 0x0300, 2);
+	BEGIN_NV04(chan, NvSubCtxSurf2D, 0x0300, 2);
 	OUT_RING(chan, 0);
 	OUT_RING(chan, (info->var.yres_virtual << 16) | info->var.xres_virtual);
 
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index abe89db6de2..a220b94ba9f 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -22,15 +22,14 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <engine/fifo.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fence.h"
 
 struct nv04_fence_chan {
 	struct nouveau_fence_chan base;
-	atomic_t sequence;
 };
 
 struct nv04_fence_priv {
@@ -57,84 +56,56 @@ nv04_fence_sync(struct nouveau_fence *fence,
 	return -ENODEV;
 }
 
-int
-nv04_fence_mthd(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct nv04_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
-	atomic_set(&fctx->sequence, data);
-	return 0;
-}
-
 static u32
 nv04_fence_read(struct nouveau_channel *chan)
 {
-	struct nv04_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
-	return atomic_read(&fctx->sequence);
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	return atomic_read(&fifo->refcnt);
 }
 
 static void
-nv04_fence_context_del(struct nouveau_channel *chan, int engine)
+nv04_fence_context_del(struct nouveau_channel *chan)
 {
-	struct nv04_fence_chan *fctx = chan->engctx[engine];
+	struct nv04_fence_chan *fctx = chan->fence;
 	nouveau_fence_context_del(&fctx->base);
-	chan->engctx[engine] = NULL;
+	chan->fence = NULL;
 	kfree(fctx);
 }
 
 static int
-nv04_fence_context_new(struct nouveau_channel *chan, int engine)
+nv04_fence_context_new(struct nouveau_channel *chan)
 {
 	struct nv04_fence_chan *fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (fctx) {
 		nouveau_fence_context_new(&fctx->base);
-		atomic_set(&fctx->sequence, 0);
-		chan->engctx[engine] = fctx;
+		chan->fence = fctx;
 		return 0;
 	}
 	return -ENOMEM;
 }
 
-static int
-nv04_fence_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static int
-nv04_fence_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
 static void
-nv04_fence_destroy(struct drm_device *dev, int engine)
+nv04_fence_destroy(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fence_priv *priv = nv_engine(dev, engine);
-
-	dev_priv->eng[engine] = NULL;
+	struct nv04_fence_priv *priv = drm->fence;
+	drm->fence = NULL;
 	kfree(priv);
 }
 
 int
-nv04_fence_create(struct drm_device *dev)
+nv04_fence_create(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nv04_fence_priv *priv;
-	int ret;
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	priv = drm->fence = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->base.engine.destroy = nv04_fence_destroy;
-	priv->base.engine.init = nv04_fence_init;
-	priv->base.engine.fini = nv04_fence_fini;
-	priv->base.engine.context_new = nv04_fence_context_new;
-	priv->base.engine.context_del = nv04_fence_context_del;
+	priv->base.dtor = nv04_fence_destroy;
+	priv->base.context_new = nv04_fence_context_new;
+	priv->base.context_del = nv04_fence_context_del;
 	priv->base.emit = nv04_fence_emit;
 	priv->base.sync = nv04_fence_sync;
 	priv->base.read = nv04_fence_read;
-	dev_priv->eng[NVOBJ_ENGINE_FENCE] = &priv->base.engine;
-	return ret;
+	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nv04_fifo.c b/drivers/gpu/drm/nouveau/nv04_fifo.c
deleted file mode 100644
index a6295cd00ec..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_fifo.c
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_util.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
-
-static struct ramfc_desc {
-	unsigned bits:6;
-	unsigned ctxs:5;
-	unsigned ctxp:8;
-	unsigned regs:5;
-	unsigned regp;
-} nv04_ramfc[] = {
-	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
-	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
-	{ 16,  0, 0x08,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
-	{ 16, 16, 0x08,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
-	{ 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_STATE },
-	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
-	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_ENGINE },
-	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_PULL1 },
-	{}
-};
-
-struct nv04_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct ramfc_desc *ramfc_desc;
-};
-
-struct nv04_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *ramfc;
-};
-
-bool
-nv04_fifo_cache_pull(struct drm_device *dev, bool enable)
-{
-	int pull = nv_mask(dev, NV04_PFIFO_CACHE1_PULL0, 1, enable);
-
-	if (!enable) {
-		/* In some cases the PFIFO puller may be left in an
-		 * inconsistent state if you try to stop it when it's
-		 * busy translating handles. Sometimes you get a
-		 * PFIFO_CACHE_ERROR, sometimes it just fails silently
-		 * sending incorrect instance offsets to PGRAPH after
-		 * it's started up again. To avoid the latter we
-		 * invalidate the most recently calculated instance.
-		 */
-		if (!nv_wait(dev, NV04_PFIFO_CACHE1_PULL0,
-				  NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0))
-			NV_ERROR(dev, "Timeout idling the PFIFO puller.\n");
-
-		if (nv_rd32(dev, NV04_PFIFO_CACHE1_PULL0) &
-				 NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
-			nv_wr32(dev, NV03_PFIFO_INTR_0,
-				     NV_PFIFO_INTR_CACHE_ERROR);
-
-		nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
-	}
-
-	return pull & 1;
-}
-
-static int
-nv04_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv = nv_engine(dev, engine);
-	struct nv04_fifo_chan *fctx;
-	unsigned long flags;
-	int ret;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	/* map channel control registers */
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV03_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/* initialise default fifo context */
-	ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramfc->pinst +
-				      chan->id * 32, ~0, 32,
-				      NVOBJ_FLAG_ZERO_FREE, &fctx->ramfc);
-	if (ret)
-		goto error;
-
-	nv_wo32(fctx->ramfc, 0x00, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x04, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x08, chan->pushbuf->pinst >> 4);
-	nv_wo32(fctx->ramfc, 0x0c, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x10, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
-				   NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
-#ifdef __BIG_ENDIAN
-				   NV_PFIFO_CACHE1_BIG_ENDIAN |
-#endif
-				   NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
-	nv_wo32(fctx->ramfc, 0x14, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x18, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x1c, 0x00000000);
-
-	/* enable dma mode on the channel */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PFIFO_MODE, (1 << chan->id), (1 << chan->id));
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-void
-nv04_fifo_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv = nv_engine(chan->dev, engine);
-	struct nv04_fifo_chan *fctx = chan->engctx[engine];
-	struct ramfc_desc *c = priv->ramfc_desc;
-	unsigned long flags;
-	int chid;
-
-	/* prevent fifo context switches */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 0);
-
-	/* if this channel is active, replace it with a null context */
-	chid = nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) & priv->base.channels;
-	if (chid == chan->id) {
-		nv_mask(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
-		nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 0);
-		nv_mask(dev, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
-
-		do {
-			u32 mask = ((1ULL << c->bits) - 1) << c->regs;
-			nv_mask(dev, c->regp, mask, 0x00000000);
-		} while ((++c)->bits);
-
-		nv_wr32(dev, NV03_PFIFO_CACHE1_GET, 0);
-		nv_wr32(dev, NV03_PFIFO_CACHE1_PUT, 0);
-		nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, priv->base.channels);
-		nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 1);
-		nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-	}
-
-	/* restore normal operation, after disabling dma mode */
-	nv_mask(dev, NV04_PFIFO_MODE, 1 << chan->id, 0);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 1);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* clean up */
-	nouveau_gpuobj_ref(NULL, &fctx->ramfc);
-	nouveau_gpuobj_ref(NULL, &chan->ramfc); /*XXX: nv40 */
-	if (chan->user) {
-		iounmap(chan->user);
-		chan->user = NULL;
-	}
-}
-
-int
-nv04_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, 0);
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, NV_PMC_ENABLE_PFIFO);
-
-	nv_wr32(dev, NV04_PFIFO_DELAY_0, 0x000000ff);
-	nv_wr32(dev, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
-
-	nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-				       ((dev_priv->ramht->bits - 9) << 16) |
-				       (dev_priv->ramht->gpuobj->pinst >> 8));
-	nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
-	nv_wr32(dev, NV03_PFIFO_RAMFC, dev_priv->ramfc->pinst >> 8);
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, priv->base.channels);
-
-	nv_wr32(dev, NV03_PFIFO_INTR_0, 0xffffffff);
-	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xffffffff);
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 1);
-	nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 1);
-
-	for (i = 0; i < priv->base.channels; i++) {
-		if (dev_priv->channels.ptr[i])
-			nv_mask(dev, NV04_PFIFO_MODE, (1 << i), (1 << i));
-	}
-
-	return 0;
-}
-
-int
-nv04_fifo_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv = nv_engine(dev, engine);
-	struct nouveau_channel *chan;
-	int chid;
-
-	/* prevent context switches and halt fifo operation */
-	nv_wr32(dev, NV03_PFIFO_CACHES, 0);
-	nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 0);
-	nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 0);
-
-	/* store current fifo context in ramfc */
-	chid = nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) & priv->base.channels;
-	chan = dev_priv->channels.ptr[chid];
-	if (suspend && chid != priv->base.channels && chan) {
-		struct nv04_fifo_chan *fctx = chan->engctx[engine];
-		struct nouveau_gpuobj *ctx = fctx->ramfc;
-		struct ramfc_desc *c = priv->ramfc_desc;
-		do {
-			u32 rm = ((1ULL << c->bits) - 1) << c->regs;
-			u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
-			u32 rv = (nv_rd32(dev, c->regp) &  rm) >> c->regs;
-			u32 cv = (nv_ro32(ctx, c->ctxp) & ~cm);
-			nv_wo32(ctx, c->ctxp, cv | (rv << c->ctxs));
-		} while ((++c)->bits);
-	}
-
-	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0x00000000);
-	return 0;
-}
-
-static bool
-nouveau_fifo_swmthd(struct drm_device *dev, u32 chid, u32 addr, u32 data)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = NULL;
-	struct nouveau_gpuobj *obj;
-	unsigned long flags;
-	const int subc = (addr >> 13) & 0x7;
-	const int mthd = addr & 0x1ffc;
-	bool handled = false;
-	u32 engine;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	if (likely(chid >= 0 && chid < pfifo->channels))
-		chan = dev_priv->channels.ptr[chid];
-	if (unlikely(!chan))
-		goto out;
-
-	switch (mthd) {
-	case 0x0000: /* bind object to subchannel */
-		obj = nouveau_ramht_find(chan, data);
-		if (unlikely(!obj || obj->engine != NVOBJ_ENGINE_SW))
-			break;
-
-		engine = 0x0000000f << (subc * 4);
-
-		nv_mask(dev, NV04_PFIFO_CACHE1_ENGINE, engine, 0x00000000);
-		handled = true;
-		break;
-	default:
-		engine = nv_rd32(dev, NV04_PFIFO_CACHE1_ENGINE);
-		if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
-			break;
-
-		if (!nouveau_gpuobj_mthd_call(chan, nouveau_software_class(dev),
-					      mthd, data))
-			handled = true;
-		break;
-	}
-
-out:
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return handled;
-}
-
-static const char *nv_dma_state_err(u32 state)
-{
-	static const char * const desc[] = {
-		"NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
-		"INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
-	};
-	return desc[(state >> 29) & 0x7];
-}
-
-void
-nv04_fifo_isr(struct drm_device *dev)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t status, reassign;
-	int cnt = 0;
-
-	reassign = nv_rd32(dev, NV03_PFIFO_CACHES) & 1;
-	while ((status = nv_rd32(dev, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
-		uint32_t chid, get;
-
-		nv_wr32(dev, NV03_PFIFO_CACHES, 0);
-
-		chid = nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) & pfifo->channels;
-		get  = nv_rd32(dev, NV03_PFIFO_CACHE1_GET);
-
-		if (status & NV_PFIFO_INTR_CACHE_ERROR) {
-			uint32_t mthd, data;
-			int ptr;
-
-			/* NV_PFIFO_CACHE1_GET actually goes to 0xffc before
-			 * wrapping on my G80 chips, but CACHE1 isn't big
-			 * enough for this much data.. Tests show that it
-			 * wraps around to the start at GET=0x800.. No clue
-			 * as to why..
-			 */
-			ptr = (get & 0x7ff) >> 2;
-
-			if (dev_priv->card_type < NV_40) {
-				mthd = nv_rd32(dev,
-					NV04_PFIFO_CACHE1_METHOD(ptr));
-				data = nv_rd32(dev,
-					NV04_PFIFO_CACHE1_DATA(ptr));
-			} else {
-				mthd = nv_rd32(dev,
-					NV40_PFIFO_CACHE1_METHOD(ptr));
-				data = nv_rd32(dev,
-					NV40_PFIFO_CACHE1_DATA(ptr));
-			}
-
-			if (!nouveau_fifo_swmthd(dev, chid, mthd, data)) {
-				NV_INFO(dev, "PFIFO_CACHE_ERROR - Ch %d/%d "
-					     "Mthd 0x%04x Data 0x%08x\n",
-					chid, (mthd >> 13) & 7, mthd & 0x1ffc,
-					data);
-			}
-
-			nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
-			nv_wr32(dev, NV03_PFIFO_INTR_0,
-						NV_PFIFO_INTR_CACHE_ERROR);
-
-			nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
-				nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) & ~1);
-			nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
-			nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0,
-				nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH0) | 1);
-			nv_wr32(dev, NV04_PFIFO_CACHE1_HASH, 0);
-
-			nv_wr32(dev, NV04_PFIFO_CACHE1_DMA_PUSH,
-				nv_rd32(dev, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
-			nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-
-			status &= ~NV_PFIFO_INTR_CACHE_ERROR;
-		}
-
-		if (status & NV_PFIFO_INTR_DMA_PUSHER) {
-			u32 dma_get = nv_rd32(dev, 0x003244);
-			u32 dma_put = nv_rd32(dev, 0x003240);
-			u32 push = nv_rd32(dev, 0x003220);
-			u32 state = nv_rd32(dev, 0x003228);
-
-			if (dev_priv->card_type == NV_50) {
-				u32 ho_get = nv_rd32(dev, 0x003328);
-				u32 ho_put = nv_rd32(dev, 0x003320);
-				u32 ib_get = nv_rd32(dev, 0x003334);
-				u32 ib_put = nv_rd32(dev, 0x003330);
-
-				if (nouveau_ratelimit())
-					NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%02x%08x "
-					     "Put 0x%02x%08x IbGet 0x%08x IbPut 0x%08x "
-					     "State 0x%08x (err: %s) Push 0x%08x\n",
-						chid, ho_get, dma_get, ho_put,
-						dma_put, ib_get, ib_put, state,
-						nv_dma_state_err(state),
-						push);
-
-				/* METHOD_COUNT, in DMA_STATE on earlier chipsets */
-				nv_wr32(dev, 0x003364, 0x00000000);
-				if (dma_get != dma_put || ho_get != ho_put) {
-					nv_wr32(dev, 0x003244, dma_put);
-					nv_wr32(dev, 0x003328, ho_put);
-				} else
-				if (ib_get != ib_put) {
-					nv_wr32(dev, 0x003334, ib_put);
-				}
-			} else {
-				NV_INFO(dev, "PFIFO_DMA_PUSHER - Ch %d Get 0x%08x "
-					     "Put 0x%08x State 0x%08x (err: %s) Push 0x%08x\n",
-					chid, dma_get, dma_put, state,
-					nv_dma_state_err(state), push);
-
-				if (dma_get != dma_put)
-					nv_wr32(dev, 0x003244, dma_put);
-			}
-
-			nv_wr32(dev, 0x003228, 0x00000000);
-			nv_wr32(dev, 0x003220, 0x00000001);
-			nv_wr32(dev, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
-			status &= ~NV_PFIFO_INTR_DMA_PUSHER;
-		}
-
-		if (status & NV_PFIFO_INTR_SEMAPHORE) {
-			uint32_t sem;
-
-			status &= ~NV_PFIFO_INTR_SEMAPHORE;
-			nv_wr32(dev, NV03_PFIFO_INTR_0,
-				NV_PFIFO_INTR_SEMAPHORE);
-
-			sem = nv_rd32(dev, NV10_PFIFO_CACHE1_SEMAPHORE);
-			nv_wr32(dev, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
-
-			nv_wr32(dev, NV03_PFIFO_CACHE1_GET, get + 4);
-			nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-		}
-
-		if (dev_priv->card_type == NV_50) {
-			if (status & 0x00000010) {
-				nv50_fb_vm_trap(dev, nouveau_ratelimit());
-				status &= ~0x00000010;
-				nv_wr32(dev, 0x002100, 0x00000010);
-			}
-		}
-
-		if (status) {
-			if (nouveau_ratelimit())
-				NV_INFO(dev, "PFIFO_INTR 0x%08x - Ch %d\n",
-					status, chid);
-			nv_wr32(dev, NV03_PFIFO_INTR_0, status);
-			status = 0;
-		}
-
-		nv_wr32(dev, NV03_PFIFO_CACHES, reassign);
-	}
-
-	if (status) {
-		NV_INFO(dev, "PFIFO still angry after %d spins, halt\n", cnt);
-		nv_wr32(dev, 0x2140, 0);
-		nv_wr32(dev, 0x140, 0);
-	}
-
-	nv_wr32(dev, NV03_PMC_INTR_0, NV_PMC_INTR_0_PFIFO_PENDING);
-}
-
-void
-nv04_fifo_destroy(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 8);
-
-	dev_priv->eng[engine] = NULL;
-	kfree(priv);
-}
-
-int
-nv04_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_fifo_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv04_fifo_destroy;
-	priv->base.base.init = nv04_fifo_init;
-	priv->base.base.fini = nv04_fifo_fini;
-	priv->base.base.context_new = nv04_fifo_context_new;
-	priv->base.base.context_del = nv04_fifo_context_del;
-	priv->base.channels = 15;
-	priv->ramfc_desc = nv04_ramfc;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_graph.c b/drivers/gpu/drm/nouveau/nv04_graph.c
deleted file mode 100644
index 72f1a62903b..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_graph.c
+++ /dev/null
@@ -1,1326 +0,0 @@
-/*
- * Copyright 2007 Stephane Marchesin
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_hw.h"
-#include "nouveau_util.h"
-#include "nouveau_ramht.h"
-
-struct nv04_graph_engine {
-	struct nouveau_exec_engine base;
-};
-
-static uint32_t nv04_graph_ctx_regs[] = {
-	0x0040053c,
-	0x00400544,
-	0x00400540,
-	0x00400548,
-	NV04_PGRAPH_CTX_SWITCH1,
-	NV04_PGRAPH_CTX_SWITCH2,
-	NV04_PGRAPH_CTX_SWITCH3,
-	NV04_PGRAPH_CTX_SWITCH4,
-	NV04_PGRAPH_CTX_CACHE1,
-	NV04_PGRAPH_CTX_CACHE2,
-	NV04_PGRAPH_CTX_CACHE3,
-	NV04_PGRAPH_CTX_CACHE4,
-	0x00400184,
-	0x004001a4,
-	0x004001c4,
-	0x004001e4,
-	0x00400188,
-	0x004001a8,
-	0x004001c8,
-	0x004001e8,
-	0x0040018c,
-	0x004001ac,
-	0x004001cc,
-	0x004001ec,
-	0x00400190,
-	0x004001b0,
-	0x004001d0,
-	0x004001f0,
-	0x00400194,
-	0x004001b4,
-	0x004001d4,
-	0x004001f4,
-	0x00400198,
-	0x004001b8,
-	0x004001d8,
-	0x004001f8,
-	0x0040019c,
-	0x004001bc,
-	0x004001dc,
-	0x004001fc,
-	0x00400174,
-	NV04_PGRAPH_DMA_START_0,
-	NV04_PGRAPH_DMA_START_1,
-	NV04_PGRAPH_DMA_LENGTH,
-	NV04_PGRAPH_DMA_MISC,
-	NV04_PGRAPH_DMA_PITCH,
-	NV04_PGRAPH_BOFFSET0,
-	NV04_PGRAPH_BBASE0,
-	NV04_PGRAPH_BLIMIT0,
-	NV04_PGRAPH_BOFFSET1,
-	NV04_PGRAPH_BBASE1,
-	NV04_PGRAPH_BLIMIT1,
-	NV04_PGRAPH_BOFFSET2,
-	NV04_PGRAPH_BBASE2,
-	NV04_PGRAPH_BLIMIT2,
-	NV04_PGRAPH_BOFFSET3,
-	NV04_PGRAPH_BBASE3,
-	NV04_PGRAPH_BLIMIT3,
-	NV04_PGRAPH_BOFFSET4,
-	NV04_PGRAPH_BBASE4,
-	NV04_PGRAPH_BLIMIT4,
-	NV04_PGRAPH_BOFFSET5,
-	NV04_PGRAPH_BBASE5,
-	NV04_PGRAPH_BLIMIT5,
-	NV04_PGRAPH_BPITCH0,
-	NV04_PGRAPH_BPITCH1,
-	NV04_PGRAPH_BPITCH2,
-	NV04_PGRAPH_BPITCH3,
-	NV04_PGRAPH_BPITCH4,
-	NV04_PGRAPH_SURFACE,
-	NV04_PGRAPH_STATE,
-	NV04_PGRAPH_BSWIZZLE2,
-	NV04_PGRAPH_BSWIZZLE5,
-	NV04_PGRAPH_BPIXEL,
-	NV04_PGRAPH_NOTIFY,
-	NV04_PGRAPH_PATT_COLOR0,
-	NV04_PGRAPH_PATT_COLOR1,
-	NV04_PGRAPH_PATT_COLORRAM+0x00,
-	NV04_PGRAPH_PATT_COLORRAM+0x04,
-	NV04_PGRAPH_PATT_COLORRAM+0x08,
-	NV04_PGRAPH_PATT_COLORRAM+0x0c,
-	NV04_PGRAPH_PATT_COLORRAM+0x10,
-	NV04_PGRAPH_PATT_COLORRAM+0x14,
-	NV04_PGRAPH_PATT_COLORRAM+0x18,
-	NV04_PGRAPH_PATT_COLORRAM+0x1c,
-	NV04_PGRAPH_PATT_COLORRAM+0x20,
-	NV04_PGRAPH_PATT_COLORRAM+0x24,
-	NV04_PGRAPH_PATT_COLORRAM+0x28,
-	NV04_PGRAPH_PATT_COLORRAM+0x2c,
-	NV04_PGRAPH_PATT_COLORRAM+0x30,
-	NV04_PGRAPH_PATT_COLORRAM+0x34,
-	NV04_PGRAPH_PATT_COLORRAM+0x38,
-	NV04_PGRAPH_PATT_COLORRAM+0x3c,
-	NV04_PGRAPH_PATT_COLORRAM+0x40,
-	NV04_PGRAPH_PATT_COLORRAM+0x44,
-	NV04_PGRAPH_PATT_COLORRAM+0x48,
-	NV04_PGRAPH_PATT_COLORRAM+0x4c,
-	NV04_PGRAPH_PATT_COLORRAM+0x50,
-	NV04_PGRAPH_PATT_COLORRAM+0x54,
-	NV04_PGRAPH_PATT_COLORRAM+0x58,
-	NV04_PGRAPH_PATT_COLORRAM+0x5c,
-	NV04_PGRAPH_PATT_COLORRAM+0x60,
-	NV04_PGRAPH_PATT_COLORRAM+0x64,
-	NV04_PGRAPH_PATT_COLORRAM+0x68,
-	NV04_PGRAPH_PATT_COLORRAM+0x6c,
-	NV04_PGRAPH_PATT_COLORRAM+0x70,
-	NV04_PGRAPH_PATT_COLORRAM+0x74,
-	NV04_PGRAPH_PATT_COLORRAM+0x78,
-	NV04_PGRAPH_PATT_COLORRAM+0x7c,
-	NV04_PGRAPH_PATT_COLORRAM+0x80,
-	NV04_PGRAPH_PATT_COLORRAM+0x84,
-	NV04_PGRAPH_PATT_COLORRAM+0x88,
-	NV04_PGRAPH_PATT_COLORRAM+0x8c,
-	NV04_PGRAPH_PATT_COLORRAM+0x90,
-	NV04_PGRAPH_PATT_COLORRAM+0x94,
-	NV04_PGRAPH_PATT_COLORRAM+0x98,
-	NV04_PGRAPH_PATT_COLORRAM+0x9c,
-	NV04_PGRAPH_PATT_COLORRAM+0xa0,
-	NV04_PGRAPH_PATT_COLORRAM+0xa4,
-	NV04_PGRAPH_PATT_COLORRAM+0xa8,
-	NV04_PGRAPH_PATT_COLORRAM+0xac,
-	NV04_PGRAPH_PATT_COLORRAM+0xb0,
-	NV04_PGRAPH_PATT_COLORRAM+0xb4,
-	NV04_PGRAPH_PATT_COLORRAM+0xb8,
-	NV04_PGRAPH_PATT_COLORRAM+0xbc,
-	NV04_PGRAPH_PATT_COLORRAM+0xc0,
-	NV04_PGRAPH_PATT_COLORRAM+0xc4,
-	NV04_PGRAPH_PATT_COLORRAM+0xc8,
-	NV04_PGRAPH_PATT_COLORRAM+0xcc,
-	NV04_PGRAPH_PATT_COLORRAM+0xd0,
-	NV04_PGRAPH_PATT_COLORRAM+0xd4,
-	NV04_PGRAPH_PATT_COLORRAM+0xd8,
-	NV04_PGRAPH_PATT_COLORRAM+0xdc,
-	NV04_PGRAPH_PATT_COLORRAM+0xe0,
-	NV04_PGRAPH_PATT_COLORRAM+0xe4,
-	NV04_PGRAPH_PATT_COLORRAM+0xe8,
-	NV04_PGRAPH_PATT_COLORRAM+0xec,
-	NV04_PGRAPH_PATT_COLORRAM+0xf0,
-	NV04_PGRAPH_PATT_COLORRAM+0xf4,
-	NV04_PGRAPH_PATT_COLORRAM+0xf8,
-	NV04_PGRAPH_PATT_COLORRAM+0xfc,
-	NV04_PGRAPH_PATTERN,
-	0x0040080c,
-	NV04_PGRAPH_PATTERN_SHAPE,
-	0x00400600,
-	NV04_PGRAPH_ROP3,
-	NV04_PGRAPH_CHROMA,
-	NV04_PGRAPH_BETA_AND,
-	NV04_PGRAPH_BETA_PREMULT,
-	NV04_PGRAPH_CONTROL0,
-	NV04_PGRAPH_CONTROL1,
-	NV04_PGRAPH_CONTROL2,
-	NV04_PGRAPH_BLEND,
-	NV04_PGRAPH_STORED_FMT,
-	NV04_PGRAPH_SOURCE_COLOR,
-	0x00400560,
-	0x00400568,
-	0x00400564,
-	0x0040056c,
-	0x00400400,
-	0x00400480,
-	0x00400404,
-	0x00400484,
-	0x00400408,
-	0x00400488,
-	0x0040040c,
-	0x0040048c,
-	0x00400410,
-	0x00400490,
-	0x00400414,
-	0x00400494,
-	0x00400418,
-	0x00400498,
-	0x0040041c,
-	0x0040049c,
-	0x00400420,
-	0x004004a0,
-	0x00400424,
-	0x004004a4,
-	0x00400428,
-	0x004004a8,
-	0x0040042c,
-	0x004004ac,
-	0x00400430,
-	0x004004b0,
-	0x00400434,
-	0x004004b4,
-	0x00400438,
-	0x004004b8,
-	0x0040043c,
-	0x004004bc,
-	0x00400440,
-	0x004004c0,
-	0x00400444,
-	0x004004c4,
-	0x00400448,
-	0x004004c8,
-	0x0040044c,
-	0x004004cc,
-	0x00400450,
-	0x004004d0,
-	0x00400454,
-	0x004004d4,
-	0x00400458,
-	0x004004d8,
-	0x0040045c,
-	0x004004dc,
-	0x00400460,
-	0x004004e0,
-	0x00400464,
-	0x004004e4,
-	0x00400468,
-	0x004004e8,
-	0x0040046c,
-	0x004004ec,
-	0x00400470,
-	0x004004f0,
-	0x00400474,
-	0x004004f4,
-	0x00400478,
-	0x004004f8,
-	0x0040047c,
-	0x004004fc,
-	0x00400534,
-	0x00400538,
-	0x00400514,
-	0x00400518,
-	0x0040051c,
-	0x00400520,
-	0x00400524,
-	0x00400528,
-	0x0040052c,
-	0x00400530,
-	0x00400d00,
-	0x00400d40,
-	0x00400d80,
-	0x00400d04,
-	0x00400d44,
-	0x00400d84,
-	0x00400d08,
-	0x00400d48,
-	0x00400d88,
-	0x00400d0c,
-	0x00400d4c,
-	0x00400d8c,
-	0x00400d10,
-	0x00400d50,
-	0x00400d90,
-	0x00400d14,
-	0x00400d54,
-	0x00400d94,
-	0x00400d18,
-	0x00400d58,
-	0x00400d98,
-	0x00400d1c,
-	0x00400d5c,
-	0x00400d9c,
-	0x00400d20,
-	0x00400d60,
-	0x00400da0,
-	0x00400d24,
-	0x00400d64,
-	0x00400da4,
-	0x00400d28,
-	0x00400d68,
-	0x00400da8,
-	0x00400d2c,
-	0x00400d6c,
-	0x00400dac,
-	0x00400d30,
-	0x00400d70,
-	0x00400db0,
-	0x00400d34,
-	0x00400d74,
-	0x00400db4,
-	0x00400d38,
-	0x00400d78,
-	0x00400db8,
-	0x00400d3c,
-	0x00400d7c,
-	0x00400dbc,
-	0x00400590,
-	0x00400594,
-	0x00400598,
-	0x0040059c,
-	0x004005a8,
-	0x004005ac,
-	0x004005b0,
-	0x004005b4,
-	0x004005c0,
-	0x004005c4,
-	0x004005c8,
-	0x004005cc,
-	0x004005d0,
-	0x004005d4,
-	0x004005d8,
-	0x004005dc,
-	0x004005e0,
-	NV04_PGRAPH_PASSTHRU_0,
-	NV04_PGRAPH_PASSTHRU_1,
-	NV04_PGRAPH_PASSTHRU_2,
-	NV04_PGRAPH_DVD_COLORFMT,
-	NV04_PGRAPH_SCALED_FORMAT,
-	NV04_PGRAPH_MISC24_0,
-	NV04_PGRAPH_MISC24_1,
-	NV04_PGRAPH_MISC24_2,
-	0x00400500,
-	0x00400504,
-	NV04_PGRAPH_VALID1,
-	NV04_PGRAPH_VALID2,
-	NV04_PGRAPH_DEBUG_3
-};
-
-struct graph_state {
-	uint32_t nv04[ARRAY_SIZE(nv04_graph_ctx_regs)];
-};
-
-static struct nouveau_channel *
-nv04_graph_channel(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chid = 15;
-
-	if (nv_rd32(dev, NV04_PGRAPH_CTX_CONTROL) & 0x00010000)
-		chid = nv_rd32(dev, NV04_PGRAPH_CTX_USER) >> 24;
-
-	if (chid > 15)
-		return NULL;
-
-	return dev_priv->channels.ptr[chid];
-}
-
-static uint32_t *ctx_reg(struct graph_state *ctx, uint32_t reg)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++) {
-		if (nv04_graph_ctx_regs[i] == reg)
-			return &ctx->nv04[i];
-	}
-
-	return NULL;
-}
-
-static int
-nv04_graph_load_context(struct nouveau_channel *chan)
-{
-	struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	uint32_t tmp;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++)
-		nv_wr32(dev, nv04_graph_ctx_regs[i], pgraph_ctx->nv04[i]);
-
-	nv_wr32(dev, NV04_PGRAPH_CTX_CONTROL, 0x10010100);
-
-	tmp  = nv_rd32(dev, NV04_PGRAPH_CTX_USER) & 0x00ffffff;
-	nv_wr32(dev, NV04_PGRAPH_CTX_USER, tmp | chan->id << 24);
-
-	tmp = nv_rd32(dev, NV04_PGRAPH_FFINTFC_ST2);
-	nv_wr32(dev, NV04_PGRAPH_FFINTFC_ST2, tmp & 0x000fffff);
-
-	return 0;
-}
-
-static int
-nv04_graph_unload_context(struct drm_device *dev)
-{
-	struct nouveau_channel *chan = NULL;
-	struct graph_state *ctx;
-	uint32_t tmp;
-	int i;
-
-	chan = nv04_graph_channel(dev);
-	if (!chan)
-		return 0;
-	ctx = chan->engctx[NVOBJ_ENGINE_GR];
-
-	for (i = 0; i < ARRAY_SIZE(nv04_graph_ctx_regs); i++)
-		ctx->nv04[i] = nv_rd32(dev, nv04_graph_ctx_regs[i]);
-
-	nv_wr32(dev, NV04_PGRAPH_CTX_CONTROL, 0x10000000);
-	tmp  = nv_rd32(dev, NV04_PGRAPH_CTX_USER) & 0x00ffffff;
-	tmp |= 15 << 24;
-	nv_wr32(dev, NV04_PGRAPH_CTX_USER, tmp);
-	return 0;
-}
-
-static int
-nv04_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct graph_state *pgraph_ctx;
-	NV_DEBUG(chan->dev, "nv04_graph_context_create %d\n", chan->id);
-
-	pgraph_ctx = kzalloc(sizeof(*pgraph_ctx), GFP_KERNEL);
-	if (pgraph_ctx == NULL)
-		return -ENOMEM;
-
-	*ctx_reg(pgraph_ctx, NV04_PGRAPH_DEBUG_3) = 0xfad4ff31;
-
-	chan->engctx[engine] = pgraph_ctx;
-	return 0;
-}
-
-static void
-nv04_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct graph_state *pgraph_ctx = chan->engctx[engine];
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-
-	/* Unload the context if it's the currently active one */
-	if (nv04_graph_channel(dev) == chan)
-		nv04_graph_unload_context(dev);
-
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* Free the context resources */
-	kfree(pgraph_ctx);
-	chan->engctx[engine] = NULL;
-}
-
-int
-nv04_graph_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 1;
-	obj->class  = class;
-
-#ifdef __BIG_ENDIAN
-	nv_wo32(obj, 0x00, 0x00080000 | class);
-#else
-	nv_wo32(obj, 0x00, class);
-#endif
-	nv_wo32(obj, 0x04, 0x00000000);
-	nv_wo32(obj, 0x08, 0x00000000);
-	nv_wo32(obj, 0x0c, 0x00000000);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static int
-nv04_graph_init(struct drm_device *dev, int engine)
-{
-	uint32_t tmp;
-
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
-			~NV_PMC_ENABLE_PGRAPH);
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
-			 NV_PMC_ENABLE_PGRAPH);
-
-	/* Enable PGRAPH interrupts */
-	nv_wr32(dev, NV03_PGRAPH_INTR, 0xFFFFFFFF);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV04_PGRAPH_VALID1, 0);
-	nv_wr32(dev, NV04_PGRAPH_VALID2, 0);
-	/*nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x000001FF);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x001FFFFF);*/
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x1231c000);
-	/*1231C000 blob, 001 haiku*/
-	/*V_WRITE(NV04_PGRAPH_DEBUG_1, 0xf2d91100);*/
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x72111100);
-	/*0x72111100 blob , 01 haiku*/
-	/*nv_wr32(dev, NV04_PGRAPH_DEBUG_2, 0x11d5f870);*/
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_2, 0x11d5f071);
-	/*haiku same*/
-
-	/*nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0xfad4ff31);*/
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0xf0d4ff31);
-	/*haiku and blob 10d4*/
-
-	nv_wr32(dev, NV04_PGRAPH_STATE        , 0xFFFFFFFF);
-	nv_wr32(dev, NV04_PGRAPH_CTX_CONTROL  , 0x10000100);
-	tmp  = nv_rd32(dev, NV04_PGRAPH_CTX_USER) & 0x00ffffff;
-	tmp |= 15 << 24;
-	nv_wr32(dev, NV04_PGRAPH_CTX_USER, tmp);
-
-	/* These don't belong here, they're part of a per-channel context */
-	nv_wr32(dev, NV04_PGRAPH_PATTERN_SHAPE, 0x00000000);
-	nv_wr32(dev, NV04_PGRAPH_BETA_AND     , 0xFFFFFFFF);
-
-	return 0;
-}
-
-static int
-nv04_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-	if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
-		nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-		return -EBUSY;
-	}
-	nv04_graph_unload_context(dev);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
-	return 0;
-}
-
-/*
- * Software methods, why they are needed, and how they all work:
- *
- * NV04 and NV05 keep most of the state in PGRAPH context itself, but some
- * 2d engine settings are kept inside the grobjs themselves. The grobjs are
- * 3 words long on both. grobj format on NV04 is:
- *
- * word 0:
- *  - bits 0-7: class
- *  - bit 12: color key active
- *  - bit 13: clip rect active
- *  - bit 14: if set, destination surface is swizzled and taken from buffer 5
- *            [set by NV04_SWIZZLED_SURFACE], otherwise it's linear and taken
- *            from buffer 0 [set by NV04_CONTEXT_SURFACES_2D or
- *            NV03_CONTEXT_SURFACE_DST].
- *  - bits 15-17: 2d operation [aka patch config]
- *  - bit 24: patch valid [enables rendering using this object]
- *  - bit 25: surf3d valid [for tex_tri and multitex_tri only]
- * word 1:
- *  - bits 0-1: mono format
- *  - bits 8-13: color format
- *  - bits 16-31: DMA_NOTIFY instance
- * word 2:
- *  - bits 0-15: DMA_A instance
- *  - bits 16-31: DMA_B instance
- *
- * On NV05 it's:
- *
- * word 0:
- *  - bits 0-7: class
- *  - bit 12: color key active
- *  - bit 13: clip rect active
- *  - bit 14: if set, destination surface is swizzled and taken from buffer 5
- *            [set by NV04_SWIZZLED_SURFACE], otherwise it's linear and taken
- *            from buffer 0 [set by NV04_CONTEXT_SURFACES_2D or
- *            NV03_CONTEXT_SURFACE_DST].
- *  - bits 15-17: 2d operation [aka patch config]
- *  - bits 20-22: dither mode
- *  - bit 24: patch valid [enables rendering using this object]
- *  - bit 25: surface_dst/surface_color/surf2d/surf3d valid
- *  - bit 26: surface_src/surface_zeta valid
- *  - bit 27: pattern valid
- *  - bit 28: rop valid
- *  - bit 29: beta1 valid
- *  - bit 30: beta4 valid
- * word 1:
- *  - bits 0-1: mono format
- *  - bits 8-13: color format
- *  - bits 16-31: DMA_NOTIFY instance
- * word 2:
- *  - bits 0-15: DMA_A instance
- *  - bits 16-31: DMA_B instance
- *
- * NV05 will set/unset the relevant valid bits when you poke the relevant
- * object-binding methods with object of the proper type, or with the NULL
- * type. It'll only allow rendering using the grobj if all needed objects
- * are bound. The needed set of objects depends on selected operation: for
- * example rop object is needed by ROP_AND, but not by SRCCOPY_AND.
- *
- * NV04 doesn't have these methods implemented at all, and doesn't have the
- * relevant bits in grobj. Instead, it'll allow rendering whenever bit 24
- * is set. So we have to emulate them in software, internally keeping the
- * same bits as NV05 does. Since grobjs are aligned to 16 bytes on nv04,
- * but the last word isn't actually used for anything, we abuse it for this
- * purpose.
- *
- * Actually, NV05 can optionally check bit 24 too, but we disable this since
- * there's no use for it.
- *
- * For unknown reasons, NV04 implements surf3d binding in hardware as an
- * exception. Also for unknown reasons, NV04 doesn't implement the clipping
- * methods on the surf3d object, so we have to emulate them too.
- */
-
-static void
-nv04_graph_set_ctx1(struct nouveau_channel *chan, u32 mask, u32 value)
-{
-	struct drm_device *dev = chan->dev;
-	u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
-	int subc = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 13) & 0x7;
-	u32 tmp;
-
-	tmp  = nv_ri32(dev, instance);
-	tmp &= ~mask;
-	tmp |= value;
-
-	nv_wi32(dev, instance, tmp);
-	nv_wr32(dev, NV04_PGRAPH_CTX_SWITCH1, tmp);
-	nv_wr32(dev, NV04_PGRAPH_CTX_CACHE1 + (subc<<2), tmp);
-}
-
-static void
-nv04_graph_set_ctx_val(struct nouveau_channel *chan, u32 mask, u32 value)
-{
-	struct drm_device *dev = chan->dev;
-	u32 instance = (nv_rd32(dev, NV04_PGRAPH_CTX_SWITCH4) & 0xffff) << 4;
-	u32 tmp, ctx1;
-	int class, op, valid = 1;
-
-	ctx1 = nv_ri32(dev, instance);
-	class = ctx1 & 0xff;
-	op = (ctx1 >> 15) & 7;
-	tmp  = nv_ri32(dev, instance + 0xc);
-	tmp &= ~mask;
-	tmp |= value;
-	nv_wi32(dev, instance + 0xc, tmp);
-
-	/* check for valid surf2d/surf_dst/surf_color */
-	if (!(tmp & 0x02000000))
-		valid = 0;
-	/* check for valid surf_src/surf_zeta */
-	if ((class == 0x1f || class == 0x48) && !(tmp & 0x04000000))
-		valid = 0;
-
-	switch (op) {
-	/* SRCCOPY_AND, SRCCOPY: no extra objects required */
-	case 0:
-	case 3:
-		break;
-	/* ROP_AND: requires pattern and rop */
-	case 1:
-		if (!(tmp & 0x18000000))
-			valid = 0;
-		break;
-	/* BLEND_AND: requires beta1 */
-	case 2:
-		if (!(tmp & 0x20000000))
-			valid = 0;
-		break;
-	/* SRCCOPY_PREMULT, BLEND_PREMULT: beta4 required */
-	case 4:
-	case 5:
-		if (!(tmp & 0x40000000))
-			valid = 0;
-		break;
-	}
-
-	nv04_graph_set_ctx1(chan, 0x01000000, valid << 24);
-}
-
-static int
-nv04_graph_mthd_set_operation(struct nouveau_channel *chan,
-			      u32 class, u32 mthd, u32 data)
-{
-	if (data > 5)
-		return 1;
-	/* Old versions of the objects only accept first three operations. */
-	if (data > 2 && class < 0x40)
-		return 1;
-	nv04_graph_set_ctx1(chan, 0x00038000, data << 15);
-	/* changing operation changes set of objects needed for validation */
-	nv04_graph_set_ctx_val(chan, 0, 0);
-	return 0;
-}
-
-static int
-nv04_graph_mthd_surf3d_clip_h(struct nouveau_channel *chan,
-			      u32 class, u32 mthd, u32 data)
-{
-	uint32_t min = data & 0xffff, max;
-	uint32_t w = data >> 16;
-	if (min & 0x8000)
-		/* too large */
-		return 1;
-	if (w & 0x8000)
-		/* yes, it accepts negative for some reason. */
-		w |= 0xffff0000;
-	max = min + w;
-	max &= 0x3ffff;
-	nv_wr32(chan->dev, 0x40053c, min);
-	nv_wr32(chan->dev, 0x400544, max);
-	return 0;
-}
-
-static int
-nv04_graph_mthd_surf3d_clip_v(struct nouveau_channel *chan,
-			      u32 class, u32 mthd, u32 data)
-{
-	uint32_t min = data & 0xffff, max;
-	uint32_t w = data >> 16;
-	if (min & 0x8000)
-		/* too large */
-		return 1;
-	if (w & 0x8000)
-		/* yes, it accepts negative for some reason. */
-		w |= 0xffff0000;
-	max = min + w;
-	max &= 0x3ffff;
-	nv_wr32(chan->dev, 0x400540, min);
-	nv_wr32(chan->dev, 0x400548, max);
-	return 0;
-}
-
-static int
-nv04_graph_mthd_bind_surf2d(struct nouveau_channel *chan,
-			    u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx1(chan, 0x00004000, 0);
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0);
-		return 0;
-	case 0x42:
-		nv04_graph_set_ctx1(chan, 0x00004000, 0);
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0x02000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_surf2d_swzsurf(struct nouveau_channel *chan,
-				    u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx1(chan, 0x00004000, 0);
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0);
-		return 0;
-	case 0x42:
-		nv04_graph_set_ctx1(chan, 0x00004000, 0);
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0x02000000);
-		return 0;
-	case 0x52:
-		nv04_graph_set_ctx1(chan, 0x00004000, 0x00004000);
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0x02000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_nv01_patt(struct nouveau_channel *chan,
-			       u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x08000000, 0);
-		return 0;
-	case 0x18:
-		nv04_graph_set_ctx_val(chan, 0x08000000, 0x08000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_nv04_patt(struct nouveau_channel *chan,
-			       u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x08000000, 0);
-		return 0;
-	case 0x44:
-		nv04_graph_set_ctx_val(chan, 0x08000000, 0x08000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_rop(struct nouveau_channel *chan,
-			 u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x10000000, 0);
-		return 0;
-	case 0x43:
-		nv04_graph_set_ctx_val(chan, 0x10000000, 0x10000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_beta1(struct nouveau_channel *chan,
-			   u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x20000000, 0);
-		return 0;
-	case 0x12:
-		nv04_graph_set_ctx_val(chan, 0x20000000, 0x20000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_beta4(struct nouveau_channel *chan,
-			   u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x40000000, 0);
-		return 0;
-	case 0x72:
-		nv04_graph_set_ctx_val(chan, 0x40000000, 0x40000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_surf_dst(struct nouveau_channel *chan,
-			      u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0);
-		return 0;
-	case 0x58:
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0x02000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_surf_src(struct nouveau_channel *chan,
-			      u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x04000000, 0);
-		return 0;
-	case 0x59:
-		nv04_graph_set_ctx_val(chan, 0x04000000, 0x04000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_surf_color(struct nouveau_channel *chan,
-				u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0);
-		return 0;
-	case 0x5a:
-		nv04_graph_set_ctx_val(chan, 0x02000000, 0x02000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_surf_zeta(struct nouveau_channel *chan,
-			       u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx_val(chan, 0x04000000, 0);
-		return 0;
-	case 0x5b:
-		nv04_graph_set_ctx_val(chan, 0x04000000, 0x04000000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_clip(struct nouveau_channel *chan,
-			  u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx1(chan, 0x2000, 0);
-		return 0;
-	case 0x19:
-		nv04_graph_set_ctx1(chan, 0x2000, 0x2000);
-		return 0;
-	}
-	return 1;
-}
-
-static int
-nv04_graph_mthd_bind_chroma(struct nouveau_channel *chan,
-			    u32 class, u32 mthd, u32 data)
-{
-	switch (nv_ri32(chan->dev, data << 4) & 0xff) {
-	case 0x30:
-		nv04_graph_set_ctx1(chan, 0x1000, 0);
-		return 0;
-	/* Yes, for some reason even the old versions of objects
-	 * accept 0x57 and not 0x17. Consistency be damned.
-	 */
-	case 0x57:
-		nv04_graph_set_ctx1(chan, 0x1000, 0x1000);
-		return 0;
-	}
-	return 1;
-}
-
-static struct nouveau_bitfield nv04_graph_intr[] = {
-	{ NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
-	{}
-};
-
-static struct nouveau_bitfield nv04_graph_nstatus[] = {
-	{ NV04_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
-	{ NV04_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
-	{ NV04_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
-	{ NV04_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
-	{}
-};
-
-struct nouveau_bitfield nv04_graph_nsource[] = {
-	{ NV03_PGRAPH_NSOURCE_NOTIFICATION,       "NOTIFICATION" },
-	{ NV03_PGRAPH_NSOURCE_DATA_ERROR,         "DATA_ERROR" },
-	{ NV03_PGRAPH_NSOURCE_PROTECTION_ERROR,   "PROTECTION_ERROR" },
-	{ NV03_PGRAPH_NSOURCE_RANGE_EXCEPTION,    "RANGE_EXCEPTION" },
-	{ NV03_PGRAPH_NSOURCE_LIMIT_COLOR,        "LIMIT_COLOR" },
-	{ NV03_PGRAPH_NSOURCE_LIMIT_ZETA,         "LIMIT_ZETA" },
-	{ NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD,       "ILLEGAL_MTHD" },
-	{ NV03_PGRAPH_NSOURCE_DMA_R_PROTECTION,   "DMA_R_PROTECTION" },
-	{ NV03_PGRAPH_NSOURCE_DMA_W_PROTECTION,   "DMA_W_PROTECTION" },
-	{ NV03_PGRAPH_NSOURCE_FORMAT_EXCEPTION,   "FORMAT_EXCEPTION" },
-	{ NV03_PGRAPH_NSOURCE_PATCH_EXCEPTION,    "PATCH_EXCEPTION" },
-	{ NV03_PGRAPH_NSOURCE_STATE_INVALID,      "STATE_INVALID" },
-	{ NV03_PGRAPH_NSOURCE_DOUBLE_NOTIFY,      "DOUBLE_NOTIFY" },
-	{ NV03_PGRAPH_NSOURCE_NOTIFY_IN_USE,      "NOTIFY_IN_USE" },
-	{ NV03_PGRAPH_NSOURCE_METHOD_CNT,         "METHOD_CNT" },
-	{ NV03_PGRAPH_NSOURCE_BFR_NOTIFICATION,   "BFR_NOTIFICATION" },
-	{ NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION, "DMA_VTX_PROTECTION" },
-	{ NV03_PGRAPH_NSOURCE_DMA_WIDTH_A,        "DMA_WIDTH_A" },
-	{ NV03_PGRAPH_NSOURCE_DMA_WIDTH_B,        "DMA_WIDTH_B" },
-	{}
-};
-
-static void
-nv04_graph_context_switch(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = NULL;
-	int chid;
-
-	nouveau_wait_for_idle(dev);
-
-	/* If previous context is valid, we need to save it */
-	nv04_graph_unload_context(dev);
-
-	/* Load context for next channel */
-	chid = nv_rd32(dev, NV03_PFIFO_CACHE1_PUSH1) &
-			    NV03_PFIFO_CACHE1_PUSH1_CHID_MASK;
-	chan = dev_priv->channels.ptr[chid];
-	if (chan)
-		nv04_graph_load_context(chan);
-}
-
-static void
-nv04_graph_isr(struct drm_device *dev)
-{
-	u32 stat;
-
-	while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-		u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-		u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
-		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-		u32 chid = (addr & 0x0f000000) >> 24;
-		u32 subc = (addr & 0x0000e000) >> 13;
-		u32 mthd = (addr & 0x00001ffc);
-		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-		u32 class = nv_rd32(dev, 0x400180 + subc * 4) & 0xff;
-		u32 show = stat;
-
-		if (stat & NV_PGRAPH_INTR_NOTIFY) {
-			if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-				if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
-					show &= ~NV_PGRAPH_INTR_NOTIFY;
-			}
-		}
-
-		if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
-			nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
-			stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-			show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-			nv04_graph_context_switch(dev);
-		}
-
-		nv_wr32(dev, NV03_PGRAPH_INTR, stat);
-		nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
-
-		if (show && nouveau_ratelimit()) {
-			NV_INFO(dev, "PGRAPH -");
-			nouveau_bitfield_print(nv04_graph_intr, show);
-			printk(" nsource:");
-			nouveau_bitfield_print(nv04_graph_nsource, nsource);
-			printk(" nstatus:");
-			nouveau_bitfield_print(nv04_graph_nstatus, nstatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
-				     "mthd 0x%04x data 0x%08x\n",
-				chid, subc, class, mthd, data);
-		}
-	}
-}
-
-static void
-nv04_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nv04_graph_engine *pgraph = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 12);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-	kfree(pgraph);
-}
-
-int
-nv04_graph_create(struct drm_device *dev)
-{
-	struct nv04_graph_engine *pgraph;
-
-	pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
-	if (!pgraph)
-		return -ENOMEM;
-
-	pgraph->base.destroy = nv04_graph_destroy;
-	pgraph->base.init = nv04_graph_init;
-	pgraph->base.fini = nv04_graph_fini;
-	pgraph->base.context_new = nv04_graph_context_new;
-	pgraph->base.context_del = nv04_graph_context_del;
-	pgraph->base.object_new = nv04_graph_object_new;
-
-	NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
-	nouveau_irq_register(dev, 12, nv04_graph_isr);
-
-	/* dvd subpicture */
-	NVOBJ_CLASS(dev, 0x0038, GR);
-
-	/* m2mf */
-	NVOBJ_CLASS(dev, 0x0039, GR);
-
-	/* nv03 gdirect */
-	NVOBJ_CLASS(dev, 0x004b, GR);
-	NVOBJ_MTHD (dev, 0x004b, 0x0184, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x004b, 0x0188, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x004b, 0x018c, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x004b, 0x0190, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x004b, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 gdirect */
-	NVOBJ_CLASS(dev, 0x004a, GR);
-	NVOBJ_MTHD (dev, 0x004a, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x004a, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x004a, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x004a, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x004a, 0x0198, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x004a, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv01 imageblit */
-	NVOBJ_CLASS(dev, 0x001f, GR);
-	NVOBJ_MTHD (dev, 0x001f, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x001f, 0x0188, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x001f, 0x018c, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x001f, 0x0190, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x001f, 0x0194, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x001f, 0x0198, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x001f, 0x019c, nv04_graph_mthd_bind_surf_src);
-	NVOBJ_MTHD (dev, 0x001f, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 imageblit */
-	NVOBJ_CLASS(dev, 0x005f, GR);
-	NVOBJ_MTHD (dev, 0x005f, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x005f, 0x0188, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x005f, 0x018c, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x005f, 0x0190, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x005f, 0x0194, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x005f, 0x0198, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x005f, 0x019c, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x005f, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 iifc */
-	NVOBJ_CLASS(dev, 0x0060, GR);
-	NVOBJ_MTHD (dev, 0x0060, 0x0188, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x0060, 0x018c, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x0060, 0x0190, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x0060, 0x0194, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0060, 0x0198, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0060, 0x019c, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x0060, 0x01a0, nv04_graph_mthd_bind_surf2d_swzsurf);
-	NVOBJ_MTHD (dev, 0x0060, 0x03e4, nv04_graph_mthd_set_operation);
-
-	/* nv05 iifc */
-	NVOBJ_CLASS(dev, 0x0064, GR);
-
-	/* nv01 ifc */
-	NVOBJ_CLASS(dev, 0x0021, GR);
-	NVOBJ_MTHD (dev, 0x0021, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x0021, 0x0188, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x0021, 0x018c, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x0021, 0x0190, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0021, 0x0194, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0021, 0x0198, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x0021, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 ifc */
-	NVOBJ_CLASS(dev, 0x0061, GR);
-	NVOBJ_MTHD (dev, 0x0061, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x0061, 0x0188, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x0061, 0x018c, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x0061, 0x0190, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0061, 0x0194, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0061, 0x0198, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x0061, 0x019c, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x0061, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv05 ifc */
-	NVOBJ_CLASS(dev, 0x0065, GR);
-
-	/* nv03 sifc */
-	NVOBJ_CLASS(dev, 0x0036, GR);
-	NVOBJ_MTHD (dev, 0x0036, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x0036, 0x0188, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x0036, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0036, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0036, 0x0194, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x0036, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 sifc */
-	NVOBJ_CLASS(dev, 0x0076, GR);
-	NVOBJ_MTHD (dev, 0x0076, 0x0184, nv04_graph_mthd_bind_chroma);
-	NVOBJ_MTHD (dev, 0x0076, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x0076, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0076, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0076, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x0076, 0x0198, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x0076, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv05 sifc */
-	NVOBJ_CLASS(dev, 0x0066, GR);
-
-	/* nv03 sifm */
-	NVOBJ_CLASS(dev, 0x0037, GR);
-	NVOBJ_MTHD (dev, 0x0037, 0x0188, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x0037, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0037, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0037, 0x0194, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x0037, 0x0304, nv04_graph_mthd_set_operation);
-
-	/* nv04 sifm */
-	NVOBJ_CLASS(dev, 0x0077, GR);
-	NVOBJ_MTHD (dev, 0x0077, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x0077, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x0077, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x0077, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x0077, 0x0198, nv04_graph_mthd_bind_surf2d_swzsurf);
-	NVOBJ_MTHD (dev, 0x0077, 0x0304, nv04_graph_mthd_set_operation);
-
-	/* null */
-	NVOBJ_CLASS(dev, 0x0030, GR);
-
-	/* surf2d */
-	NVOBJ_CLASS(dev, 0x0042, GR);
-
-	/* rop */
-	NVOBJ_CLASS(dev, 0x0043, GR);
-
-	/* beta1 */
-	NVOBJ_CLASS(dev, 0x0012, GR);
-
-	/* beta4 */
-	NVOBJ_CLASS(dev, 0x0072, GR);
-
-	/* cliprect */
-	NVOBJ_CLASS(dev, 0x0019, GR);
-
-	/* nv01 pattern */
-	NVOBJ_CLASS(dev, 0x0018, GR);
-
-	/* nv04 pattern */
-	NVOBJ_CLASS(dev, 0x0044, GR);
-
-	/* swzsurf */
-	NVOBJ_CLASS(dev, 0x0052, GR);
-
-	/* surf3d */
-	NVOBJ_CLASS(dev, 0x0053, GR);
-	NVOBJ_MTHD (dev, 0x0053, 0x02f8, nv04_graph_mthd_surf3d_clip_h);
-	NVOBJ_MTHD (dev, 0x0053, 0x02fc, nv04_graph_mthd_surf3d_clip_v);
-
-	/* nv03 tex_tri */
-	NVOBJ_CLASS(dev, 0x0048, GR);
-	NVOBJ_MTHD (dev, 0x0048, 0x0188, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x0048, 0x018c, nv04_graph_mthd_bind_surf_color);
-	NVOBJ_MTHD (dev, 0x0048, 0x0190, nv04_graph_mthd_bind_surf_zeta);
-
-	/* tex_tri */
-	NVOBJ_CLASS(dev, 0x0054, GR);
-
-	/* multitex_tri */
-	NVOBJ_CLASS(dev, 0x0055, GR);
-
-	/* nv01 chroma */
-	NVOBJ_CLASS(dev, 0x0017, GR);
-
-	/* nv04 chroma */
-	NVOBJ_CLASS(dev, 0x0057, GR);
-
-	/* surf_dst */
-	NVOBJ_CLASS(dev, 0x0058, GR);
-
-	/* surf_src */
-	NVOBJ_CLASS(dev, 0x0059, GR);
-
-	/* surf_color */
-	NVOBJ_CLASS(dev, 0x005a, GR);
-
-	/* surf_zeta */
-	NVOBJ_CLASS(dev, 0x005b, GR);
-
-	/* nv01 line */
-	NVOBJ_CLASS(dev, 0x001c, GR);
-	NVOBJ_MTHD (dev, 0x001c, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x001c, 0x0188, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x001c, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x001c, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x001c, 0x0194, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x001c, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 line */
-	NVOBJ_CLASS(dev, 0x005c, GR);
-	NVOBJ_MTHD (dev, 0x005c, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x005c, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x005c, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x005c, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x005c, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x005c, 0x0198, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x005c, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv01 tri */
-	NVOBJ_CLASS(dev, 0x001d, GR);
-	NVOBJ_MTHD (dev, 0x001d, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x001d, 0x0188, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x001d, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x001d, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x001d, 0x0194, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x001d, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 tri */
-	NVOBJ_CLASS(dev, 0x005d, GR);
-	NVOBJ_MTHD (dev, 0x005d, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x005d, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x005d, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x005d, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x005d, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x005d, 0x0198, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x005d, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv01 rect */
-	NVOBJ_CLASS(dev, 0x001e, GR);
-	NVOBJ_MTHD (dev, 0x001e, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x001e, 0x0188, nv04_graph_mthd_bind_nv01_patt);
-	NVOBJ_MTHD (dev, 0x001e, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x001e, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x001e, 0x0194, nv04_graph_mthd_bind_surf_dst);
-	NVOBJ_MTHD (dev, 0x001e, 0x02fc, nv04_graph_mthd_set_operation);
-
-	/* nv04 rect */
-	NVOBJ_CLASS(dev, 0x005e, GR);
-	NVOBJ_MTHD (dev, 0x005e, 0x0184, nv04_graph_mthd_bind_clip);
-	NVOBJ_MTHD (dev, 0x005e, 0x0188, nv04_graph_mthd_bind_nv04_patt);
-	NVOBJ_MTHD (dev, 0x005e, 0x018c, nv04_graph_mthd_bind_rop);
-	NVOBJ_MTHD (dev, 0x005e, 0x0190, nv04_graph_mthd_bind_beta1);
-	NVOBJ_MTHD (dev, 0x005e, 0x0194, nv04_graph_mthd_bind_beta4);
-	NVOBJ_MTHD (dev, 0x005e, 0x0198, nv04_graph_mthd_bind_surf2d);
-	NVOBJ_MTHD (dev, 0x005e, 0x02fc, nv04_graph_mthd_set_operation);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_instmem.c b/drivers/gpu/drm/nouveau/nv04_instmem.c
deleted file mode 100644
index ef7a934a499..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_instmem.c
+++ /dev/null
@@ -1,193 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-
-/* returns the size of fifo context */
-static int
-nouveau_fifo_ctx_size(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->chipset >= 0x40)
-		return 128 * 32;
-	else
-	if (dev_priv->chipset >= 0x17)
-		return 64 * 32;
-	else
-	if (dev_priv->chipset >= 0x10)
-		return 32 * 32;
-
-	return 32 * 16;
-}
-
-int nv04_instmem_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramht = NULL;
-	u32 offset, length;
-	int ret;
-
-	/* RAMIN always available */
-	dev_priv->ramin_available = true;
-
-	/* Reserve space at end of VRAM for PRAMIN */
-	if (dev_priv->card_type >= NV_40) {
-		u32 vs = hweight8((nv_rd32(dev, 0x001540) & 0x0000ff00) >> 8);
-		u32 rsvd;
-
-		/* estimate grctx size, the magics come from nv40_grctx.c */
-		if      (dev_priv->chipset == 0x40) rsvd = 0x6aa0 * vs;
-		else if (dev_priv->chipset  < 0x43) rsvd = 0x4f00 * vs;
-		else if (nv44_graph_class(dev))	    rsvd = 0x4980 * vs;
-		else				    rsvd = 0x4a40 * vs;
-		rsvd += 16 * 1024;
-		rsvd *= 32; /* per-channel */
-
-		rsvd += 512 * 1024; /* pci(e)gart table */
-		rsvd += 512 * 1024; /* object storage */
-
-		dev_priv->ramin_rsvd_vram = round_up(rsvd, 4096);
-	} else {
-		dev_priv->ramin_rsvd_vram = 512 * 1024;
-	}
-
-	/* Setup shared RAMHT */
-	ret = nouveau_gpuobj_new_fake(dev, 0x10000, ~0, 4096,
-				      NVOBJ_FLAG_ZERO_ALLOC, &ramht);
-	if (ret)
-		return ret;
-
-	ret = nouveau_ramht_new(dev, ramht, &dev_priv->ramht);
-	nouveau_gpuobj_ref(NULL, &ramht);
-	if (ret)
-		return ret;
-
-	/* And RAMRO */
-	ret = nouveau_gpuobj_new_fake(dev, 0x11200, ~0, 512,
-				      NVOBJ_FLAG_ZERO_ALLOC, &dev_priv->ramro);
-	if (ret)
-		return ret;
-
-	/* And RAMFC */
-	length = nouveau_fifo_ctx_size(dev);
-	switch (dev_priv->card_type) {
-	case NV_40:
-		offset = 0x20000;
-		break;
-	default:
-		offset = 0x11400;
-		break;
-	}
-
-	ret = nouveau_gpuobj_new_fake(dev, offset, ~0, length,
-				      NVOBJ_FLAG_ZERO_ALLOC, &dev_priv->ramfc);
-	if (ret)
-		return ret;
-
-	/* Only allow space after RAMFC to be used for object allocation */
-	offset += length;
-
-	/* It appears RAMRO (or something?) is controlled by 0x2220/0x2230
-	 * on certain NV4x chipsets as well as RAMFC.  When 0x2230 == 0
-	 * ("new style" control) the upper 16-bits of 0x2220 points at this
-	 * other mysterious table that's clobbering important things.
-	 *
-	 * We're now pointing this at RAMIN+0x30000 to avoid RAMFC getting
-	 * smashed to pieces on us, so reserve 0x30000-0x40000 too..
-	 */
-	if (dev_priv->card_type >= NV_40) {
-		if (offset < 0x40000)
-			offset = 0x40000;
-	}
-
-	ret = drm_mm_init(&dev_priv->ramin_heap, offset,
-			  dev_priv->ramin_rsvd_vram - offset);
-	if (ret) {
-		NV_ERROR(dev, "Failed to init RAMIN heap: %d\n", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-void
-nv04_instmem_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nouveau_ramht_ref(NULL, &dev_priv->ramht, NULL);
-	nouveau_gpuobj_ref(NULL, &dev_priv->ramro);
-	nouveau_gpuobj_ref(NULL, &dev_priv->ramfc);
-
-	if (drm_mm_initialized(&dev_priv->ramin_heap))
-		drm_mm_takedown(&dev_priv->ramin_heap);
-}
-
-int
-nv04_instmem_suspend(struct drm_device *dev)
-{
-	return 0;
-}
-
-void
-nv04_instmem_resume(struct drm_device *dev)
-{
-}
-
-int
-nv04_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
-		 u32 size, u32 align)
-{
-	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
-	struct drm_mm_node *ramin = NULL;
-
-	do {
-		if (drm_mm_pre_get(&dev_priv->ramin_heap))
-			return -ENOMEM;
-
-		spin_lock(&dev_priv->ramin_lock);
-		ramin = drm_mm_search_free(&dev_priv->ramin_heap, size, align, 0);
-		if (ramin == NULL) {
-			spin_unlock(&dev_priv->ramin_lock);
-			return -ENOMEM;
-		}
-
-		ramin = drm_mm_get_block_atomic(ramin, size, align);
-		spin_unlock(&dev_priv->ramin_lock);
-	} while (ramin == NULL);
-
-	gpuobj->node  = ramin;
-	gpuobj->vinst = ramin->start;
-	return 0;
-}
-
-void
-nv04_instmem_put(struct nouveau_gpuobj *gpuobj)
-{
-	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
-
-	spin_lock(&dev_priv->ramin_lock);
-	drm_mm_put_block(gpuobj->node);
-	gpuobj->node = NULL;
-	spin_unlock(&dev_priv->ramin_lock);
-}
-
-int
-nv04_instmem_map(struct nouveau_gpuobj *gpuobj)
-{
-	gpuobj->pinst = gpuobj->vinst;
-	return 0;
-}
-
-void
-nv04_instmem_unmap(struct nouveau_gpuobj *gpuobj)
-{
-}
-
-void
-nv04_instmem_flush(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_mc.c b/drivers/gpu/drm/nouveau/nv04_mc.c
deleted file mode 100644
index 2af43a1cb2e..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_mc.c
+++ /dev/null
@@ -1,24 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-int
-nv04_mc_init(struct drm_device *dev)
-{
-	/* Power up everything, resetting each individual unit will
-	 * be done later if needed.
-	 */
-
-	nv_wr32(dev, NV03_PMC_ENABLE, 0xFFFFFFFF);
-
-	/* Disable PROM access. */
-	nv_wr32(dev, NV_PBUS_PCI_NV_20, NV_PBUS_PCI_NV_20_ROM_SHADOW_ENABLED);
-
-	return 0;
-}
-
-void
-nv04_mc_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_pm.c b/drivers/gpu/drm/nouveau/nv04_pm.c
index 6e7589918fa..2a0cc9d0614 100644
--- a/drivers/gpu/drm/nouveau/nv04_pm.c
+++ b/drivers/gpu/drm/nouveau/nv04_pm.c
@@ -22,11 +22,16 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_hw.h"
 #include "nouveau_pm.h"
 
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+
 int
 nv04_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
@@ -46,7 +51,7 @@ nv04_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 }
 
 struct nv04_pm_clock {
-	struct pll_lims pll;
+	struct nvbios_pll pll;
 	struct nouveau_pll_vals calc;
 };
 
@@ -58,13 +63,16 @@ struct nv04_pm_state {
 static int
 calc_pll(struct drm_device *dev, u32 id, int khz, struct nv04_pm_clock *clk)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nouveau_clock *pclk = nouveau_clock(device);
 	int ret;
 
-	ret = get_pll_limits(dev, id, &clk->pll);
+	ret = nvbios_pll_parse(bios, id, &clk->pll);
 	if (ret)
 		return ret;
 
-	ret = nouveau_calc_pll_mnp(dev, &clk->pll, khz, &clk->calc);
+	ret = pclk->pll_calc(pclk, &clk->pll, khz, &clk->calc);
 	if (!ret)
 		return -EINVAL;
 
@@ -100,37 +108,38 @@ error:
 static void
 prog_pll(struct drm_device *dev, struct nv04_pm_clock *clk)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_clock *pclk = nouveau_clock(device);
 	u32 reg = clk->pll.reg;
 
 	/* thank the insane nouveau_hw_setpll() interface for this */
-	if (dev_priv->card_type >= NV_40)
+	if (device->card_type >= NV_40)
 		reg += 4;
 
-	nouveau_hw_setpll(dev, reg, &clk->calc);
+	pclk->pll_prog(pclk, reg, &clk->calc);
 }
 
 int
 nv04_pm_clocks_set(struct drm_device *dev, void *pre_state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_timer *ptimer = nouveau_timer(device);
 	struct nv04_pm_state *state = pre_state;
 
 	prog_pll(dev, &state->core);
 
 	if (state->memory.pll.reg) {
 		prog_pll(dev, &state->memory);
-		if (dev_priv->card_type < NV_30) {
-			if (dev_priv->card_type == NV_20)
-				nv_mask(dev, 0x1002c4, 0, 1 << 20);
+		if (device->card_type < NV_30) {
+			if (device->card_type == NV_20)
+				nv_mask(device, 0x1002c4, 0, 1 << 20);
 
 			/* Reset the DLLs */
-			nv_mask(dev, 0x1002c0, 0, 1 << 8);
+			nv_mask(device, 0x1002c0, 0, 1 << 8);
 		}
 	}
 
-	ptimer->init(dev);
+	nv_ofuncs(ptimer)->init(nv_object(ptimer));
 
 	kfree(state);
 	return 0;
diff --git a/drivers/gpu/drm/nouveau/nv04_software.c b/drivers/gpu/drm/nouveau/nv04_software.c
deleted file mode 100644
index 0c41abf4877..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_software.c
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-#include "nouveau_fence.h"
-#include "nouveau_software.h"
-#include "nouveau_hw.h"
-
-struct nv04_software_priv {
-	struct nouveau_software_priv base;
-};
-
-struct nv04_software_chan {
-	struct nouveau_software_chan base;
-};
-
-static int
-mthd_flip(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-
-	struct nouveau_page_flip_state state;
-
-	if (!nouveau_finish_page_flip(chan, &state)) {
-		nv_set_crtc_base(chan->dev, state.crtc, state.offset +
-				 state.y * state.pitch +
-				 state.x * state.bpp / 8);
-	}
-
-	return 0;
-}
-
-static int
-nv04_software_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv04_software_chan *pch;
-
-	pch = kzalloc(sizeof(*pch), GFP_KERNEL);
-	if (!pch)
-		return -ENOMEM;
-
-	nouveau_software_context_new(&pch->base);
-	chan->engctx[engine] = pch;
-	return 0;
-}
-
-static void
-nv04_software_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv04_software_chan *pch = chan->engctx[engine];
-	chan->engctx[engine] = NULL;
-	kfree(pch);
-}
-
-static int
-nv04_software_object_new(struct nouveau_channel *chan, int engine,
-			 u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, 0, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 0;
-	obj->class  = class;
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static int
-nv04_software_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
-static int
-nv04_software_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static void
-nv04_software_destroy(struct drm_device *dev, int engine)
-{
-	struct nv04_software_priv *psw = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, SW);
-	kfree(psw);
-}
-
-int
-nv04_software_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_software_priv *psw;
-
-	psw = kzalloc(sizeof(*psw), GFP_KERNEL);
-	if (!psw)
-		return -ENOMEM;
-
-	psw->base.base.destroy = nv04_software_destroy;
-	psw->base.base.init = nv04_software_init;
-	psw->base.base.fini = nv04_software_fini;
-	psw->base.base.context_new = nv04_software_context_new;
-	psw->base.base.context_del = nv04_software_context_del;
-	psw->base.base.object_new = nv04_software_object_new;
-	nouveau_software_create(&psw->base);
-
-	NVOBJ_ENGINE_ADD(dev, SW, &psw->base.base);
-	if (dev_priv->card_type <= NV_04) {
-		NVOBJ_CLASS(dev, 0x006e, SW);
-		NVOBJ_MTHD (dev, 0x006e, 0x0150, nv04_fence_mthd);
-		NVOBJ_MTHD (dev, 0x006e, 0x0500, mthd_flip);
-	} else {
-		NVOBJ_CLASS(dev, 0x016e, SW);
-		NVOBJ_MTHD (dev, 0x016e, 0x0500, mthd_flip);
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_timer.c b/drivers/gpu/drm/nouveau/nv04_timer.c
deleted file mode 100644
index 55c945290e5..00000000000
--- a/drivers/gpu/drm/nouveau/nv04_timer.c
+++ /dev/null
@@ -1,84 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_hw.h"
-
-int
-nv04_timer_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 m, n, d;
-
-	nv_wr32(dev, NV04_PTIMER_INTR_EN_0, 0x00000000);
-	nv_wr32(dev, NV04_PTIMER_INTR_0, 0xFFFFFFFF);
-
-	/* aim for 31.25MHz, which gives us nanosecond timestamps */
-	d = 1000000 / 32;
-
-	/* determine base clock for timer source */
-	if (dev_priv->chipset < 0x40) {
-		n = nouveau_hw_get_clock(dev, PLL_CORE);
-	} else
-	if (dev_priv->chipset == 0x40) {
-		/*XXX: figure this out */
-		n = 0;
-	} else {
-		n = dev_priv->crystal;
-		m = 1;
-		while (n < (d * 2)) {
-			n += (n / m);
-			m++;
-		}
-
-		nv_wr32(dev, 0x009220, m - 1);
-	}
-
-	if (!n) {
-		NV_WARN(dev, "PTIMER: unknown input clock freq\n");
-		if (!nv_rd32(dev, NV04_PTIMER_NUMERATOR) ||
-		    !nv_rd32(dev, NV04_PTIMER_DENOMINATOR)) {
-			nv_wr32(dev, NV04_PTIMER_NUMERATOR, 1);
-			nv_wr32(dev, NV04_PTIMER_DENOMINATOR, 1);
-		}
-		return 0;
-	}
-
-	/* reduce ratio to acceptable values */
-	while (((n % 5) == 0) && ((d % 5) == 0)) {
-		n /= 5;
-		d /= 5;
-	}
-
-	while (((n % 2) == 0) && ((d % 2) == 0)) {
-		n /= 2;
-		d /= 2;
-	}
-
-	while (n > 0xffff || d > 0xffff) {
-		n >>= 1;
-		d >>= 1;
-	}
-
-	nv_wr32(dev, NV04_PTIMER_NUMERATOR, n);
-	nv_wr32(dev, NV04_PTIMER_DENOMINATOR, d);
-	return 0;
-}
-
-u64
-nv04_timer_read(struct drm_device *dev)
-{
-	u32 hi, lo;
-
-	do {
-		hi = nv_rd32(dev, NV04_PTIMER_TIME_1);
-		lo = nv_rd32(dev, NV04_PTIMER_TIME_0);
-	} while (hi != nv_rd32(dev, NV04_PTIMER_TIME_1));
-
-	return ((u64)hi << 32 | lo);
-}
-
-void
-nv04_timer_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_tv.c b/drivers/gpu/drm/nouveau/nv04_tv.c
index 3eb605ddfd0..099fbeda6e2 100644
--- a/drivers/gpu/drm/nouveau/nv04_tv.c
+++ b/drivers/gpu/drm/nouveau/nv04_tv.c
@@ -24,15 +24,18 @@
  *
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
 #include "nouveau_hw.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_crtc_helper.h>
 
-#include "i2c/ch7006.h"
+#include <drm/i2c/ch7006.h>
+
+#include <subdev/i2c.h>
 
 static struct i2c_board_info nv04_tv_encoder_info[] = {
 	{
@@ -49,8 +52,11 @@ static struct i2c_board_info nv04_tv_encoder_info[] = {
 
 int nv04_tv_identify(struct drm_device *dev, int i2c_index)
 {
-	return nouveau_i2c_identify(dev, "TV encoder", nv04_tv_encoder_info,
-				    NULL, i2c_index);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+
+	return i2c->identify(i2c, i2c_index, "TV encoder",
+			     nv04_tv_encoder_info, NULL);
 }
 
 
@@ -64,12 +70,12 @@ int nv04_tv_identify(struct drm_device *dev, int i2c_index)
 static void nv04_tv_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_mode_state *state = &dev_priv->mode_reg;
+	struct nv04_mode_state *state = &nv04_display(dev)->mode_reg;
 	uint8_t crtc1A;
 
-	NV_INFO(dev, "Setting dpms mode %d on TV encoder (output %d)\n",
+	NV_INFO(drm, "Setting dpms mode %d on TV encoder (output %d)\n",
 		mode, nv_encoder->dcb->index);
 
 	state->pllsel &= ~(PLLSEL_TV_CRTC1_MASK | PLLSEL_TV_CRTC2_MASK);
@@ -94,8 +100,7 @@ static void nv04_tv_dpms(struct drm_encoder *encoder, int mode)
 
 static void nv04_tv_bind(struct drm_device *dev, int head, bool bind)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_crtc_reg *state = &dev_priv->mode_reg.crtc_reg[head];
+	struct nv04_crtc_reg *state = &nv04_display(dev)->mode_reg.crtc_reg[head];
 
 	state->tv_setup = 0;
 
@@ -133,9 +138,8 @@ static void nv04_tv_mode_set(struct drm_encoder *encoder,
 			     struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
-	struct nv04_crtc_reg *regp = &dev_priv->mode_reg.crtc_reg[nv_crtc->index];
+	struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
 
 	regp->tv_htotal = adjusted_mode->htotal;
 	regp->tv_vtotal = adjusted_mode->vtotal;
@@ -157,12 +161,13 @@ static void nv04_tv_commit(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
 
 	helper->dpms(encoder, DRM_MODE_DPMS_ON);
 
-	NV_INFO(dev, "Output %s is running on CRTC %d using output %c\n",
+	NV_INFO(drm, "Output %s is running on CRTC %d using output %c\n",
 		      drm_get_connector_name(&nouveau_encoder_connector_get(nv_encoder)->base), nv_crtc->index,
 		      '@' + ffs(nv_encoder->dcb->or));
 }
@@ -181,15 +186,16 @@ static const struct drm_encoder_funcs nv04_tv_funcs = {
 };
 
 int
-nv04_tv_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv04_tv_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	struct nouveau_encoder *nv_encoder;
 	struct drm_encoder *encoder;
 	struct drm_device *dev = connector->dev;
 	struct drm_encoder_helper_funcs *hfuncs;
 	struct drm_encoder_slave_funcs *sfuncs;
-	struct nouveau_i2c_chan *i2c =
-		nouveau_i2c_find(dev, entry->i2c_index);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
+	struct nouveau_i2c_port *port = i2c->find(i2c, entry->i2c_index);
 	int type, ret;
 
 	/* Ensure that we can talk to this encoder */
@@ -221,7 +227,7 @@ nv04_tv_create(struct drm_connector *connector, struct dcb_entry *entry)
 
 	/* Run the slave-specific initialization */
 	ret = drm_i2c_encoder_init(dev, to_encoder_slave(encoder),
-				   &i2c->adapter, &nv04_tv_encoder_info[type]);
+				   &port->adapter, &nv04_tv_encoder_info[type]);
 	if (ret < 0)
 		goto fail_cleanup;
 
diff --git a/drivers/gpu/drm/nouveau/nv10_fb.c b/drivers/gpu/drm/nouveau/nv10_fb.c
deleted file mode 100644
index 420b1608536..00000000000
--- a/drivers/gpu/drm/nouveau/nv10_fb.c
+++ /dev/null
@@ -1,104 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-void
-nv10_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
-			 uint32_t size, uint32_t pitch, uint32_t flags)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	tile->addr  = 0x80000000 | addr;
-	tile->limit = max(1u, addr + size) - 1;
-	tile->pitch = pitch;
-}
-
-void
-nv10_fb_free_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	tile->addr = tile->limit = tile->pitch = tile->zcomp = 0;
-}
-
-void
-nv10_fb_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
-	nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
-	nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
-}
-
-int
-nv1a_fb_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct pci_dev *bridge;
-	uint32_t mem, mib;
-
-	bridge = pci_get_bus_and_slot(0, PCI_DEVFN(0, 1));
-	if (!bridge) {
-		NV_ERROR(dev, "no bridge device\n");
-		return 0;
-	}
-
-	if (dev_priv->chipset == 0x1a) {
-		pci_read_config_dword(bridge, 0x7c, &mem);
-		mib = ((mem >> 6) & 31) + 1;
-	} else {
-		pci_read_config_dword(bridge, 0x84, &mem);
-		mib = ((mem >> 4) & 127) + 1;
-	}
-
-	dev_priv->vram_size = mib * 1024 * 1024;
-	return 0;
-}
-
-int
-nv10_fb_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 fifo_data = nv_rd32(dev, NV04_PFB_FIFO_DATA);
-	u32 cfg0 = nv_rd32(dev, 0x100200);
-
-	dev_priv->vram_size = fifo_data & NV10_PFB_FIFO_DATA_RAM_AMOUNT_MB_MASK;
-
-	if (cfg0 & 0x00000001)
-		dev_priv->vram_type = NV_MEM_TYPE_DDR1;
-	else
-		dev_priv->vram_type = NV_MEM_TYPE_SDRAM;
-
-	return 0;
-}
-
-int
-nv10_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i;
-
-	/* Turn all the tiling regions off. */
-	pfb->num_tiles = NV10_PFB_TILE__SIZE;
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->set_tile_region(dev, i);
-
-	return 0;
-}
-
-void
-nv10_fb_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i;
-
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->free_tile_region(dev, i);
-}
diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c
index 8a1b7500918..ce752bf5cc4 100644
--- a/drivers/gpu/drm/nouveau/nv10_fence.c
+++ b/drivers/gpu/drm/nouveau/nv10_fence.c
@@ -22,10 +22,11 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <core/object.h>
+#include <core/class.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fence.h"
 
 struct nv10_fence_chan {
@@ -39,7 +40,7 @@ struct nv10_fence_priv {
 	u32 sequence;
 };
 
-static int
+int
 nv10_fence_emit(struct nouveau_fence *fence)
 {
 	struct nouveau_channel *chan = fence->channel;
@@ -60,15 +61,15 @@ nv10_fence_sync(struct nouveau_fence *fence,
 	return -ENODEV;
 }
 
-static int
+int
 nv17_fence_sync(struct nouveau_fence *fence,
 		struct nouveau_channel *prev, struct nouveau_channel *chan)
 {
-	struct nv10_fence_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_FENCE);
+	struct nv10_fence_priv *priv = chan->drm->fence;
 	u32 value;
 	int ret;
 
-	if (!mutex_trylock(&prev->mutex))
+	if (!mutex_trylock(&prev->cli->mutex))
 		return -EBUSY;
 
 	spin_lock(&priv->lock);
@@ -95,34 +96,33 @@ nv17_fence_sync(struct nouveau_fence *fence,
 		FIRE_RING (chan);
 	}
 
-	mutex_unlock(&prev->mutex);
+	mutex_unlock(&prev->cli->mutex);
 	return 0;
 }
 
-static u32
+u32
 nv10_fence_read(struct nouveau_channel *chan)
 {
-	return nvchan_rd32(chan, 0x0048);
+	return nv_ro32(chan->object, 0x0048);
 }
 
-static void
-nv10_fence_context_del(struct nouveau_channel *chan, int engine)
+void
+nv10_fence_context_del(struct nouveau_channel *chan)
 {
-	struct nv10_fence_chan *fctx = chan->engctx[engine];
+	struct nv10_fence_chan *fctx = chan->fence;
 	nouveau_fence_context_del(&fctx->base);
-	chan->engctx[engine] = NULL;
+	chan->fence = NULL;
 	kfree(fctx);
 }
 
 static int
-nv10_fence_context_new(struct nouveau_channel *chan, int engine)
+nv10_fence_context_new(struct nouveau_channel *chan)
 {
-	struct nv10_fence_priv *priv = nv_engine(chan->dev, engine);
+	struct nv10_fence_priv *priv = chan->drm->fence;
 	struct nv10_fence_chan *fctx;
-	struct nouveau_gpuobj *obj;
 	int ret = 0;
 
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (!fctx)
 		return -ENOMEM;
 
@@ -130,69 +130,56 @@ nv10_fence_context_new(struct nouveau_channel *chan, int engine)
 
 	if (priv->bo) {
 		struct ttm_mem_reg *mem = &priv->bo->bo.mem;
-
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
-					     mem->start * PAGE_SIZE, mem->size,
-					     NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_VRAM, &obj);
-		if (!ret) {
-			ret = nouveau_ramht_insert(chan, NvSema, obj);
-			nouveau_gpuobj_ref(NULL, &obj);
-		}
+		struct nouveau_object *object;
+		u32 start = mem->start * PAGE_SIZE;
+		u32 limit = mem->start + mem->size - 1;
+
+		ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
+					 NvSema, 0x0002,
+					 &(struct nv_dma_class) {
+						.flags = NV_DMA_TARGET_VRAM |
+							 NV_DMA_ACCESS_RDWR,
+						.start = start,
+						.limit = limit,
+					 }, sizeof(struct nv_dma_class),
+					 &object);
 	}
 
 	if (ret)
-		nv10_fence_context_del(chan, engine);
+		nv10_fence_context_del(chan);
 	return ret;
 }
 
-static int
-nv10_fence_fini(struct drm_device *dev, int engine, bool suspend)
+void
+nv10_fence_destroy(struct nouveau_drm *drm)
 {
-	return 0;
-}
-
-static int
-nv10_fence_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
-static void
-nv10_fence_destroy(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv10_fence_priv *priv = nv_engine(dev, engine);
-
+	struct nv10_fence_priv *priv = drm->fence;
+	nouveau_bo_unmap(priv->bo);
 	nouveau_bo_ref(NULL, &priv->bo);
-	dev_priv->eng[engine] = NULL;
+	drm->fence = NULL;
 	kfree(priv);
 }
 
 int
-nv10_fence_create(struct drm_device *dev)
+nv10_fence_create(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nv10_fence_priv *priv;
 	int ret = 0;
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	priv = drm->fence = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->base.engine.destroy = nv10_fence_destroy;
-	priv->base.engine.init = nv10_fence_init;
-	priv->base.engine.fini = nv10_fence_fini;
-	priv->base.engine.context_new = nv10_fence_context_new;
-	priv->base.engine.context_del = nv10_fence_context_del;
+	priv->base.dtor = nv10_fence_destroy;
+	priv->base.context_new = nv10_fence_context_new;
+	priv->base.context_del = nv10_fence_context_del;
 	priv->base.emit = nv10_fence_emit;
 	priv->base.read = nv10_fence_read;
 	priv->base.sync = nv10_fence_sync;
-	dev_priv->eng[NVOBJ_ENGINE_FENCE] = &priv->base.engine;
 	spin_lock_init(&priv->lock);
 
-	if (dev_priv->chipset >= 0x17) {
-		ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+	if (nv_device(drm->device)->chipset >= 0x17) {
+		ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
 				     0, 0x0000, NULL, &priv->bo);
 		if (!ret) {
 			ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
@@ -209,6 +196,6 @@ nv10_fence_create(struct drm_device *dev)
 	}
 
 	if (ret)
-		nv10_fence_destroy(dev, NVOBJ_ENGINE_FENCE);
+		nv10_fence_destroy(drm);
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nv10_fifo.c b/drivers/gpu/drm/nouveau/nv10_fifo.c
deleted file mode 100644
index f1fe7d75824..00000000000
--- a/drivers/gpu/drm/nouveau/nv10_fifo.c
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_util.h"
-#include "nouveau_ramht.h"
-
-static struct ramfc_desc {
-	unsigned bits:6;
-	unsigned ctxs:5;
-	unsigned ctxp:8;
-	unsigned regs:5;
-	unsigned regp;
-} nv10_ramfc[] = {
-	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
-	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
-	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
-	{ 16,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
-	{ 16, 16, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
-	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_STATE },
-	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
-	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_ENGINE },
-	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_PULL1 },
-	{}
-};
-
-struct nv10_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct ramfc_desc *ramfc_desc;
-};
-
-struct nv10_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *ramfc;
-};
-
-static int
-nv10_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv10_fifo_priv *priv = nv_engine(dev, engine);
-	struct nv10_fifo_chan *fctx;
-	unsigned long flags;
-	int ret;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	/* map channel control registers */
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV03_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/* initialise default fifo context */
-	ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramfc->pinst +
-				      chan->id * 32, ~0, 32,
-				      NVOBJ_FLAG_ZERO_FREE, &fctx->ramfc);
-	if (ret)
-		goto error;
-
-	nv_wo32(fctx->ramfc, 0x00, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x04, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x08, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x0c, chan->pushbuf->pinst >> 4);
-	nv_wo32(fctx->ramfc, 0x10, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x14, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
-				   NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
-#ifdef __BIG_ENDIAN
-				   NV_PFIFO_CACHE1_BIG_ENDIAN |
-#endif
-				   NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
-	nv_wo32(fctx->ramfc, 0x18, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x1c, 0x00000000);
-
-	/* enable dma mode on the channel */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PFIFO_MODE, (1 << chan->id), (1 << chan->id));
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-int
-nv10_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv10_fifo_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv04_fifo_destroy;
-	priv->base.base.init = nv04_fifo_init;
-	priv->base.base.fini = nv04_fifo_fini;
-	priv->base.base.context_new = nv10_fifo_context_new;
-	priv->base.base.context_del = nv04_fifo_context_del;
-	priv->base.channels = 31;
-	priv->ramfc_desc = nv10_ramfc;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv10_gpio.c b/drivers/gpu/drm/nouveau/nv10_gpio.c
deleted file mode 100644
index 9d79180069d..00000000000
--- a/drivers/gpu/drm/nouveau/nv10_gpio.c
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright (C) 2009 Francisco Jerez.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_hw.h"
-#include "nouveau_gpio.h"
-
-int
-nv10_gpio_sense(struct drm_device *dev, int line)
-{
-	if (line < 2) {
-		line = line * 16;
-		line = NVReadCRTC(dev, 0, NV_PCRTC_GPIO) >> line;
-		return !!(line & 0x0100);
-	} else
-	if (line < 10) {
-		line = (line - 2) * 4;
-		line = NVReadCRTC(dev, 0, NV_PCRTC_GPIO_EXT) >> line;
-		return !!(line & 0x04);
-	} else
-	if (line < 14) {
-		line = (line - 10) * 4;
-		line = NVReadCRTC(dev, 0, NV_PCRTC_850) >> line;
-		return !!(line & 0x04);
-	}
-
-	return -EINVAL;
-}
-
-int
-nv10_gpio_drive(struct drm_device *dev, int line, int dir, int out)
-{
-	u32 reg, mask, data;
-
-	if (line < 2) {
-		line = line * 16;
-		reg  = NV_PCRTC_GPIO;
-		mask = 0x00000011;
-		data = (dir << 4) | out;
-	} else
-	if (line < 10) {
-		line = (line - 2) * 4;
-		reg  = NV_PCRTC_GPIO_EXT;
-		mask = 0x00000003;
-		data = (dir << 1) | out;
-	} else
-	if (line < 14) {
-		line = (line - 10) * 4;
-		reg  = NV_PCRTC_850;
-		mask = 0x00000003;
-		data = (dir << 1) | out;
-	} else {
-		return -EINVAL;
-	}
-
-	mask = NVReadCRTC(dev, 0, reg) & ~(mask << line);
-	NVWriteCRTC(dev, 0, reg, mask | (data << line));
-	return 0;
-}
-
-void
-nv10_gpio_irq_enable(struct drm_device *dev, int line, bool on)
-{
-	u32 mask = 0x00010001 << line;
-
-	nv_wr32(dev, 0x001104, mask);
-	nv_mask(dev, 0x001144, mask, on ? mask : 0);
-}
-
-static void
-nv10_gpio_isr(struct drm_device *dev)
-{
-	u32 intr = nv_rd32(dev, 0x1104);
-	u32 hi = (intr & 0x0000ffff) >> 0;
-	u32 lo = (intr & 0xffff0000) >> 16;
-
-	nouveau_gpio_isr(dev, 0, hi | lo);
-
-	nv_wr32(dev, 0x001104, intr);
-}
-
-int
-nv10_gpio_init(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x001140, 0x00000000);
-	nv_wr32(dev, 0x001100, 0xffffffff);
-	nv_wr32(dev, 0x001144, 0x00000000);
-	nv_wr32(dev, 0x001104, 0xffffffff);
-	nouveau_irq_register(dev, 28, nv10_gpio_isr); /* PBUS */
-	return 0;
-}
-
-void
-nv10_gpio_fini(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x001140, 0x00000000);
-	nv_wr32(dev, 0x001144, 0x00000000);
-	nouveau_irq_unregister(dev, 28);
-}
diff --git a/drivers/gpu/drm/nouveau/nv10_graph.c b/drivers/gpu/drm/nouveau/nv10_graph.c
deleted file mode 100644
index fb1d88a951d..00000000000
--- a/drivers/gpu/drm/nouveau/nv10_graph.c
+++ /dev/null
@@ -1,1189 +0,0 @@
-/*
- * Copyright 2007 Matthieu CASTET <castet.matthieu@free.fr>
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-
-struct nv10_graph_engine {
-	struct nouveau_exec_engine base;
-};
-
-struct pipe_state {
-	uint32_t pipe_0x0000[0x040/4];
-	uint32_t pipe_0x0040[0x010/4];
-	uint32_t pipe_0x0200[0x0c0/4];
-	uint32_t pipe_0x4400[0x080/4];
-	uint32_t pipe_0x6400[0x3b0/4];
-	uint32_t pipe_0x6800[0x2f0/4];
-	uint32_t pipe_0x6c00[0x030/4];
-	uint32_t pipe_0x7000[0x130/4];
-	uint32_t pipe_0x7400[0x0c0/4];
-	uint32_t pipe_0x7800[0x0c0/4];
-};
-
-static int nv10_graph_ctx_regs[] = {
-	NV10_PGRAPH_CTX_SWITCH(0),
-	NV10_PGRAPH_CTX_SWITCH(1),
-	NV10_PGRAPH_CTX_SWITCH(2),
-	NV10_PGRAPH_CTX_SWITCH(3),
-	NV10_PGRAPH_CTX_SWITCH(4),
-	NV10_PGRAPH_CTX_CACHE(0, 0),
-	NV10_PGRAPH_CTX_CACHE(0, 1),
-	NV10_PGRAPH_CTX_CACHE(0, 2),
-	NV10_PGRAPH_CTX_CACHE(0, 3),
-	NV10_PGRAPH_CTX_CACHE(0, 4),
-	NV10_PGRAPH_CTX_CACHE(1, 0),
-	NV10_PGRAPH_CTX_CACHE(1, 1),
-	NV10_PGRAPH_CTX_CACHE(1, 2),
-	NV10_PGRAPH_CTX_CACHE(1, 3),
-	NV10_PGRAPH_CTX_CACHE(1, 4),
-	NV10_PGRAPH_CTX_CACHE(2, 0),
-	NV10_PGRAPH_CTX_CACHE(2, 1),
-	NV10_PGRAPH_CTX_CACHE(2, 2),
-	NV10_PGRAPH_CTX_CACHE(2, 3),
-	NV10_PGRAPH_CTX_CACHE(2, 4),
-	NV10_PGRAPH_CTX_CACHE(3, 0),
-	NV10_PGRAPH_CTX_CACHE(3, 1),
-	NV10_PGRAPH_CTX_CACHE(3, 2),
-	NV10_PGRAPH_CTX_CACHE(3, 3),
-	NV10_PGRAPH_CTX_CACHE(3, 4),
-	NV10_PGRAPH_CTX_CACHE(4, 0),
-	NV10_PGRAPH_CTX_CACHE(4, 1),
-	NV10_PGRAPH_CTX_CACHE(4, 2),
-	NV10_PGRAPH_CTX_CACHE(4, 3),
-	NV10_PGRAPH_CTX_CACHE(4, 4),
-	NV10_PGRAPH_CTX_CACHE(5, 0),
-	NV10_PGRAPH_CTX_CACHE(5, 1),
-	NV10_PGRAPH_CTX_CACHE(5, 2),
-	NV10_PGRAPH_CTX_CACHE(5, 3),
-	NV10_PGRAPH_CTX_CACHE(5, 4),
-	NV10_PGRAPH_CTX_CACHE(6, 0),
-	NV10_PGRAPH_CTX_CACHE(6, 1),
-	NV10_PGRAPH_CTX_CACHE(6, 2),
-	NV10_PGRAPH_CTX_CACHE(6, 3),
-	NV10_PGRAPH_CTX_CACHE(6, 4),
-	NV10_PGRAPH_CTX_CACHE(7, 0),
-	NV10_PGRAPH_CTX_CACHE(7, 1),
-	NV10_PGRAPH_CTX_CACHE(7, 2),
-	NV10_PGRAPH_CTX_CACHE(7, 3),
-	NV10_PGRAPH_CTX_CACHE(7, 4),
-	NV10_PGRAPH_CTX_USER,
-	NV04_PGRAPH_DMA_START_0,
-	NV04_PGRAPH_DMA_START_1,
-	NV04_PGRAPH_DMA_LENGTH,
-	NV04_PGRAPH_DMA_MISC,
-	NV10_PGRAPH_DMA_PITCH,
-	NV04_PGRAPH_BOFFSET0,
-	NV04_PGRAPH_BBASE0,
-	NV04_PGRAPH_BLIMIT0,
-	NV04_PGRAPH_BOFFSET1,
-	NV04_PGRAPH_BBASE1,
-	NV04_PGRAPH_BLIMIT1,
-	NV04_PGRAPH_BOFFSET2,
-	NV04_PGRAPH_BBASE2,
-	NV04_PGRAPH_BLIMIT2,
-	NV04_PGRAPH_BOFFSET3,
-	NV04_PGRAPH_BBASE3,
-	NV04_PGRAPH_BLIMIT3,
-	NV04_PGRAPH_BOFFSET4,
-	NV04_PGRAPH_BBASE4,
-	NV04_PGRAPH_BLIMIT4,
-	NV04_PGRAPH_BOFFSET5,
-	NV04_PGRAPH_BBASE5,
-	NV04_PGRAPH_BLIMIT5,
-	NV04_PGRAPH_BPITCH0,
-	NV04_PGRAPH_BPITCH1,
-	NV04_PGRAPH_BPITCH2,
-	NV04_PGRAPH_BPITCH3,
-	NV04_PGRAPH_BPITCH4,
-	NV10_PGRAPH_SURFACE,
-	NV10_PGRAPH_STATE,
-	NV04_PGRAPH_BSWIZZLE2,
-	NV04_PGRAPH_BSWIZZLE5,
-	NV04_PGRAPH_BPIXEL,
-	NV10_PGRAPH_NOTIFY,
-	NV04_PGRAPH_PATT_COLOR0,
-	NV04_PGRAPH_PATT_COLOR1,
-	NV04_PGRAPH_PATT_COLORRAM, /* 64 values from 0x400900 to 0x4009fc */
-	0x00400904,
-	0x00400908,
-	0x0040090c,
-	0x00400910,
-	0x00400914,
-	0x00400918,
-	0x0040091c,
-	0x00400920,
-	0x00400924,
-	0x00400928,
-	0x0040092c,
-	0x00400930,
-	0x00400934,
-	0x00400938,
-	0x0040093c,
-	0x00400940,
-	0x00400944,
-	0x00400948,
-	0x0040094c,
-	0x00400950,
-	0x00400954,
-	0x00400958,
-	0x0040095c,
-	0x00400960,
-	0x00400964,
-	0x00400968,
-	0x0040096c,
-	0x00400970,
-	0x00400974,
-	0x00400978,
-	0x0040097c,
-	0x00400980,
-	0x00400984,
-	0x00400988,
-	0x0040098c,
-	0x00400990,
-	0x00400994,
-	0x00400998,
-	0x0040099c,
-	0x004009a0,
-	0x004009a4,
-	0x004009a8,
-	0x004009ac,
-	0x004009b0,
-	0x004009b4,
-	0x004009b8,
-	0x004009bc,
-	0x004009c0,
-	0x004009c4,
-	0x004009c8,
-	0x004009cc,
-	0x004009d0,
-	0x004009d4,
-	0x004009d8,
-	0x004009dc,
-	0x004009e0,
-	0x004009e4,
-	0x004009e8,
-	0x004009ec,
-	0x004009f0,
-	0x004009f4,
-	0x004009f8,
-	0x004009fc,
-	NV04_PGRAPH_PATTERN,	/* 2 values from 0x400808 to 0x40080c */
-	0x0040080c,
-	NV04_PGRAPH_PATTERN_SHAPE,
-	NV03_PGRAPH_MONO_COLOR0,
-	NV04_PGRAPH_ROP3,
-	NV04_PGRAPH_CHROMA,
-	NV04_PGRAPH_BETA_AND,
-	NV04_PGRAPH_BETA_PREMULT,
-	0x00400e70,
-	0x00400e74,
-	0x00400e78,
-	0x00400e7c,
-	0x00400e80,
-	0x00400e84,
-	0x00400e88,
-	0x00400e8c,
-	0x00400ea0,
-	0x00400ea4,
-	0x00400ea8,
-	0x00400e90,
-	0x00400e94,
-	0x00400e98,
-	0x00400e9c,
-	NV10_PGRAPH_WINDOWCLIP_HORIZONTAL, /* 8 values from 0x400f00-0x400f1c */
-	NV10_PGRAPH_WINDOWCLIP_VERTICAL,   /* 8 values from 0x400f20-0x400f3c */
-	0x00400f04,
-	0x00400f24,
-	0x00400f08,
-	0x00400f28,
-	0x00400f0c,
-	0x00400f2c,
-	0x00400f10,
-	0x00400f30,
-	0x00400f14,
-	0x00400f34,
-	0x00400f18,
-	0x00400f38,
-	0x00400f1c,
-	0x00400f3c,
-	NV10_PGRAPH_XFMODE0,
-	NV10_PGRAPH_XFMODE1,
-	NV10_PGRAPH_GLOBALSTATE0,
-	NV10_PGRAPH_GLOBALSTATE1,
-	NV04_PGRAPH_STORED_FMT,
-	NV04_PGRAPH_SOURCE_COLOR,
-	NV03_PGRAPH_ABS_X_RAM,	/* 32 values from 0x400400 to 0x40047c */
-	NV03_PGRAPH_ABS_Y_RAM,	/* 32 values from 0x400480 to 0x4004fc */
-	0x00400404,
-	0x00400484,
-	0x00400408,
-	0x00400488,
-	0x0040040c,
-	0x0040048c,
-	0x00400410,
-	0x00400490,
-	0x00400414,
-	0x00400494,
-	0x00400418,
-	0x00400498,
-	0x0040041c,
-	0x0040049c,
-	0x00400420,
-	0x004004a0,
-	0x00400424,
-	0x004004a4,
-	0x00400428,
-	0x004004a8,
-	0x0040042c,
-	0x004004ac,
-	0x00400430,
-	0x004004b0,
-	0x00400434,
-	0x004004b4,
-	0x00400438,
-	0x004004b8,
-	0x0040043c,
-	0x004004bc,
-	0x00400440,
-	0x004004c0,
-	0x00400444,
-	0x004004c4,
-	0x00400448,
-	0x004004c8,
-	0x0040044c,
-	0x004004cc,
-	0x00400450,
-	0x004004d0,
-	0x00400454,
-	0x004004d4,
-	0x00400458,
-	0x004004d8,
-	0x0040045c,
-	0x004004dc,
-	0x00400460,
-	0x004004e0,
-	0x00400464,
-	0x004004e4,
-	0x00400468,
-	0x004004e8,
-	0x0040046c,
-	0x004004ec,
-	0x00400470,
-	0x004004f0,
-	0x00400474,
-	0x004004f4,
-	0x00400478,
-	0x004004f8,
-	0x0040047c,
-	0x004004fc,
-	NV03_PGRAPH_ABS_UCLIP_XMIN,
-	NV03_PGRAPH_ABS_UCLIP_XMAX,
-	NV03_PGRAPH_ABS_UCLIP_YMIN,
-	NV03_PGRAPH_ABS_UCLIP_YMAX,
-	0x00400550,
-	0x00400558,
-	0x00400554,
-	0x0040055c,
-	NV03_PGRAPH_ABS_UCLIPA_XMIN,
-	NV03_PGRAPH_ABS_UCLIPA_XMAX,
-	NV03_PGRAPH_ABS_UCLIPA_YMIN,
-	NV03_PGRAPH_ABS_UCLIPA_YMAX,
-	NV03_PGRAPH_ABS_ICLIP_XMAX,
-	NV03_PGRAPH_ABS_ICLIP_YMAX,
-	NV03_PGRAPH_XY_LOGIC_MISC0,
-	NV03_PGRAPH_XY_LOGIC_MISC1,
-	NV03_PGRAPH_XY_LOGIC_MISC2,
-	NV03_PGRAPH_XY_LOGIC_MISC3,
-	NV03_PGRAPH_CLIPX_0,
-	NV03_PGRAPH_CLIPX_1,
-	NV03_PGRAPH_CLIPY_0,
-	NV03_PGRAPH_CLIPY_1,
-	NV10_PGRAPH_COMBINER0_IN_ALPHA,
-	NV10_PGRAPH_COMBINER1_IN_ALPHA,
-	NV10_PGRAPH_COMBINER0_IN_RGB,
-	NV10_PGRAPH_COMBINER1_IN_RGB,
-	NV10_PGRAPH_COMBINER_COLOR0,
-	NV10_PGRAPH_COMBINER_COLOR1,
-	NV10_PGRAPH_COMBINER0_OUT_ALPHA,
-	NV10_PGRAPH_COMBINER1_OUT_ALPHA,
-	NV10_PGRAPH_COMBINER0_OUT_RGB,
-	NV10_PGRAPH_COMBINER1_OUT_RGB,
-	NV10_PGRAPH_COMBINER_FINAL0,
-	NV10_PGRAPH_COMBINER_FINAL1,
-	0x00400e00,
-	0x00400e04,
-	0x00400e08,
-	0x00400e0c,
-	0x00400e10,
-	0x00400e14,
-	0x00400e18,
-	0x00400e1c,
-	0x00400e20,
-	0x00400e24,
-	0x00400e28,
-	0x00400e2c,
-	0x00400e30,
-	0x00400e34,
-	0x00400e38,
-	0x00400e3c,
-	NV04_PGRAPH_PASSTHRU_0,
-	NV04_PGRAPH_PASSTHRU_1,
-	NV04_PGRAPH_PASSTHRU_2,
-	NV10_PGRAPH_DIMX_TEXTURE,
-	NV10_PGRAPH_WDIMX_TEXTURE,
-	NV10_PGRAPH_DVD_COLORFMT,
-	NV10_PGRAPH_SCALED_FORMAT,
-	NV04_PGRAPH_MISC24_0,
-	NV04_PGRAPH_MISC24_1,
-	NV04_PGRAPH_MISC24_2,
-	NV03_PGRAPH_X_MISC,
-	NV03_PGRAPH_Y_MISC,
-	NV04_PGRAPH_VALID1,
-	NV04_PGRAPH_VALID2,
-};
-
-static int nv17_graph_ctx_regs[] = {
-	NV10_PGRAPH_DEBUG_4,
-	0x004006b0,
-	0x00400eac,
-	0x00400eb0,
-	0x00400eb4,
-	0x00400eb8,
-	0x00400ebc,
-	0x00400ec0,
-	0x00400ec4,
-	0x00400ec8,
-	0x00400ecc,
-	0x00400ed0,
-	0x00400ed4,
-	0x00400ed8,
-	0x00400edc,
-	0x00400ee0,
-	0x00400a00,
-	0x00400a04,
-};
-
-struct graph_state {
-	int nv10[ARRAY_SIZE(nv10_graph_ctx_regs)];
-	int nv17[ARRAY_SIZE(nv17_graph_ctx_regs)];
-	struct pipe_state pipe_state;
-	uint32_t lma_window[4];
-};
-
-#define PIPE_SAVE(dev, state, addr)					\
-	do {								\
-		int __i;						\
-		nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
-		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
-			state[__i] = nv_rd32(dev, NV10_PGRAPH_PIPE_DATA); \
-	} while (0)
-
-#define PIPE_RESTORE(dev, state, addr)					\
-	do {								\
-		int __i;						\
-		nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, addr);		\
-		for (__i = 0; __i < ARRAY_SIZE(state); __i++)		\
-			nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, state[__i]); \
-	} while (0)
-
-static void nv10_graph_save_pipe(struct nouveau_channel *chan)
-{
-	struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	struct pipe_state *pipe = &pgraph_ctx->pipe_state;
-	struct drm_device *dev = chan->dev;
-
-	PIPE_SAVE(dev, pipe->pipe_0x4400, 0x4400);
-	PIPE_SAVE(dev, pipe->pipe_0x0200, 0x0200);
-	PIPE_SAVE(dev, pipe->pipe_0x6400, 0x6400);
-	PIPE_SAVE(dev, pipe->pipe_0x6800, 0x6800);
-	PIPE_SAVE(dev, pipe->pipe_0x6c00, 0x6c00);
-	PIPE_SAVE(dev, pipe->pipe_0x7000, 0x7000);
-	PIPE_SAVE(dev, pipe->pipe_0x7400, 0x7400);
-	PIPE_SAVE(dev, pipe->pipe_0x7800, 0x7800);
-	PIPE_SAVE(dev, pipe->pipe_0x0040, 0x0040);
-	PIPE_SAVE(dev, pipe->pipe_0x0000, 0x0000);
-}
-
-static void nv10_graph_load_pipe(struct nouveau_channel *chan)
-{
-	struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	struct pipe_state *pipe = &pgraph_ctx->pipe_state;
-	struct drm_device *dev = chan->dev;
-	uint32_t xfmode0, xfmode1;
-	int i;
-
-	nouveau_wait_for_idle(dev);
-	/* XXX check haiku comments */
-	xfmode0 = nv_rd32(dev, NV10_PGRAPH_XFMODE0);
-	xfmode1 = nv_rd32(dev, NV10_PGRAPH_XFMODE1);
-	nv_wr32(dev, NV10_PGRAPH_XFMODE0, 0x10000000);
-	nv_wr32(dev, NV10_PGRAPH_XFMODE1, 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
-	for (i = 0; i < 3; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
-	for (i = 0; i < 3; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
-	nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000008);
-
-
-	PIPE_RESTORE(dev, pipe->pipe_0x0200, 0x0200);
-	nouveau_wait_for_idle(dev);
-
-	/* restore XFMODE */
-	nv_wr32(dev, NV10_PGRAPH_XFMODE0, xfmode0);
-	nv_wr32(dev, NV10_PGRAPH_XFMODE1, xfmode1);
-	PIPE_RESTORE(dev, pipe->pipe_0x6400, 0x6400);
-	PIPE_RESTORE(dev, pipe->pipe_0x6800, 0x6800);
-	PIPE_RESTORE(dev, pipe->pipe_0x6c00, 0x6c00);
-	PIPE_RESTORE(dev, pipe->pipe_0x7000, 0x7000);
-	PIPE_RESTORE(dev, pipe->pipe_0x7400, 0x7400);
-	PIPE_RESTORE(dev, pipe->pipe_0x7800, 0x7800);
-	PIPE_RESTORE(dev, pipe->pipe_0x4400, 0x4400);
-	PIPE_RESTORE(dev, pipe->pipe_0x0000, 0x0000);
-	PIPE_RESTORE(dev, pipe->pipe_0x0040, 0x0040);
-	nouveau_wait_for_idle(dev);
-}
-
-static void nv10_graph_create_pipe(struct nouveau_channel *chan)
-{
-	struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	struct pipe_state *fifo_pipe_state = &pgraph_ctx->pipe_state;
-	struct drm_device *dev = chan->dev;
-	uint32_t *fifo_pipe_state_addr;
-	int i;
-#define PIPE_INIT(addr) \
-	do { \
-		fifo_pipe_state_addr = fifo_pipe_state->pipe_##addr; \
-	} while (0)
-#define PIPE_INIT_END(addr) \
-	do { \
-		uint32_t *__end_addr = fifo_pipe_state->pipe_##addr + \
-				ARRAY_SIZE(fifo_pipe_state->pipe_##addr); \
-		if (fifo_pipe_state_addr != __end_addr) \
-			NV_ERROR(dev, "incomplete pipe init for 0x%x :  %p/%p\n", \
-				addr, fifo_pipe_state_addr, __end_addr); \
-	} while (0)
-#define NV_WRITE_PIPE_INIT(value) *(fifo_pipe_state_addr++) = value
-
-	PIPE_INIT(0x0200);
-	for (i = 0; i < 48; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x0200);
-
-	PIPE_INIT(0x6400);
-	for (i = 0; i < 211; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x40000000);
-	NV_WRITE_PIPE_INIT(0x40000000);
-	NV_WRITE_PIPE_INIT(0x40000000);
-	NV_WRITE_PIPE_INIT(0x40000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f000000);
-	NV_WRITE_PIPE_INIT(0x3f000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	PIPE_INIT_END(0x6400);
-
-	PIPE_INIT(0x6800);
-	for (i = 0; i < 162; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x3f800000);
-	for (i = 0; i < 25; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x6800);
-
-	PIPE_INIT(0x6c00);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0xbf800000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x6c00);
-
-	PIPE_INIT(0x7000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x00000000);
-	NV_WRITE_PIPE_INIT(0x7149f2ca);
-	for (i = 0; i < 35; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x7000);
-
-	PIPE_INIT(0x7400);
-	for (i = 0; i < 48; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x7400);
-
-	PIPE_INIT(0x7800);
-	for (i = 0; i < 48; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x7800);
-
-	PIPE_INIT(0x4400);
-	for (i = 0; i < 32; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x4400);
-
-	PIPE_INIT(0x0000);
-	for (i = 0; i < 16; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x0000);
-
-	PIPE_INIT(0x0040);
-	for (i = 0; i < 4; i++)
-		NV_WRITE_PIPE_INIT(0x00000000);
-	PIPE_INIT_END(0x0040);
-
-#undef PIPE_INIT
-#undef PIPE_INIT_END
-#undef NV_WRITE_PIPE_INIT
-}
-
-static int nv10_graph_ctx_regs_find_offset(struct drm_device *dev, int reg)
-{
-	int i;
-	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++) {
-		if (nv10_graph_ctx_regs[i] == reg)
-			return i;
-	}
-	NV_ERROR(dev, "unknow offset nv10_ctx_regs %d\n", reg);
-	return -1;
-}
-
-static int nv17_graph_ctx_regs_find_offset(struct drm_device *dev, int reg)
-{
-	int i;
-	for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++) {
-		if (nv17_graph_ctx_regs[i] == reg)
-			return i;
-	}
-	NV_ERROR(dev, "unknow offset nv17_ctx_regs %d\n", reg);
-	return -1;
-}
-
-static void nv10_graph_load_dma_vtxbuf(struct nouveau_channel *chan,
-				       uint32_t inst)
-{
-	struct drm_device *dev = chan->dev;
-	uint32_t st2, st2_dl, st2_dh, fifo_ptr, fifo[0x60/4];
-	uint32_t ctx_user, ctx_switch[5];
-	int i, subchan = -1;
-
-	/* NV10TCL_DMA_VTXBUF (method 0x18c) modifies hidden state
-	 * that cannot be restored via MMIO. Do it through the FIFO
-	 * instead.
-	 */
-
-	/* Look for a celsius object */
-	for (i = 0; i < 8; i++) {
-		int class = nv_rd32(dev, NV10_PGRAPH_CTX_CACHE(i, 0)) & 0xfff;
-
-		if (class == 0x56 || class == 0x96 || class == 0x99) {
-			subchan = i;
-			break;
-		}
-	}
-
-	if (subchan < 0 || !inst)
-		return;
-
-	/* Save the current ctx object */
-	ctx_user = nv_rd32(dev, NV10_PGRAPH_CTX_USER);
-	for (i = 0; i < 5; i++)
-		ctx_switch[i] = nv_rd32(dev, NV10_PGRAPH_CTX_SWITCH(i));
-
-	/* Save the FIFO state */
-	st2 = nv_rd32(dev, NV10_PGRAPH_FFINTFC_ST2);
-	st2_dl = nv_rd32(dev, NV10_PGRAPH_FFINTFC_ST2_DL);
-	st2_dh = nv_rd32(dev, NV10_PGRAPH_FFINTFC_ST2_DH);
-	fifo_ptr = nv_rd32(dev, NV10_PGRAPH_FFINTFC_FIFO_PTR);
-
-	for (i = 0; i < ARRAY_SIZE(fifo); i++)
-		fifo[i] = nv_rd32(dev, 0x4007a0 + 4 * i);
-
-	/* Switch to the celsius subchannel */
-	for (i = 0; i < 5; i++)
-		nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(i),
-			nv_rd32(dev, NV10_PGRAPH_CTX_CACHE(subchan, i)));
-	nv_mask(dev, NV10_PGRAPH_CTX_USER, 0xe000, subchan << 13);
-
-	/* Inject NV10TCL_DMA_VTXBUF */
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_FIFO_PTR, 0);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2,
-		0x2c000000 | chan->id << 20 | subchan << 16 | 0x18c);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2_DL, inst);
-	nv_mask(dev, NV10_PGRAPH_CTX_CONTROL, 0, 0x10000);
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-
-	/* Restore the FIFO state */
-	for (i = 0; i < ARRAY_SIZE(fifo); i++)
-		nv_wr32(dev, 0x4007a0 + 4 * i, fifo[i]);
-
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_FIFO_PTR, fifo_ptr);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2, st2);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2_DL, st2_dl);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2_DH, st2_dh);
-
-	/* Restore the current ctx object */
-	for (i = 0; i < 5; i++)
-		nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(i), ctx_switch[i]);
-	nv_wr32(dev, NV10_PGRAPH_CTX_USER, ctx_user);
-}
-
-static int
-nv10_graph_load_context(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct graph_state *pgraph_ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	uint32_t tmp;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
-		nv_wr32(dev, nv10_graph_ctx_regs[i], pgraph_ctx->nv10[i]);
-	if (dev_priv->chipset >= 0x17) {
-		for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
-			nv_wr32(dev, nv17_graph_ctx_regs[i],
-						pgraph_ctx->nv17[i]);
-	}
-
-	nv10_graph_load_pipe(chan);
-	nv10_graph_load_dma_vtxbuf(chan, (nv_rd32(dev, NV10_PGRAPH_GLOBALSTATE1)
-					  & 0xffff));
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
-	tmp = nv_rd32(dev, NV10_PGRAPH_CTX_USER);
-	nv_wr32(dev, NV10_PGRAPH_CTX_USER, (tmp & 0xffffff) | chan->id << 24);
-	tmp = nv_rd32(dev, NV10_PGRAPH_FFINTFC_ST2);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2, tmp & 0xcfffffff);
-	return 0;
-}
-
-static int
-nv10_graph_unload_context(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan;
-	struct graph_state *ctx;
-	uint32_t tmp;
-	int i;
-
-	chan = nv10_graph_channel(dev);
-	if (!chan)
-		return 0;
-	ctx = chan->engctx[NVOBJ_ENGINE_GR];
-
-	for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
-		ctx->nv10[i] = nv_rd32(dev, nv10_graph_ctx_regs[i]);
-
-	if (dev_priv->chipset >= 0x17) {
-		for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
-			ctx->nv17[i] = nv_rd32(dev, nv17_graph_ctx_regs[i]);
-	}
-
-	nv10_graph_save_pipe(chan);
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
-	tmp  = nv_rd32(dev, NV10_PGRAPH_CTX_USER) & 0x00ffffff;
-	tmp |= 31 << 24;
-	nv_wr32(dev, NV10_PGRAPH_CTX_USER, tmp);
-	return 0;
-}
-
-static void
-nv10_graph_context_switch(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = NULL;
-	int chid;
-
-	nouveau_wait_for_idle(dev);
-
-	/* If previous context is valid, we need to save it */
-	nv10_graph_unload_context(dev);
-
-	/* Load context for next channel */
-	chid = (nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR) >> 20) & 0x1f;
-	chan = dev_priv->channels.ptr[chid];
-	if (chan && chan->engctx[NVOBJ_ENGINE_GR])
-		nv10_graph_load_context(chan);
-}
-
-#define NV_WRITE_CTX(reg, val) do { \
-	int offset = nv10_graph_ctx_regs_find_offset(dev, reg); \
-	if (offset > 0) \
-		pgraph_ctx->nv10[offset] = val; \
-	} while (0)
-
-#define NV17_WRITE_CTX(reg, val) do { \
-	int offset = nv17_graph_ctx_regs_find_offset(dev, reg); \
-	if (offset > 0) \
-		pgraph_ctx->nv17[offset] = val; \
-	} while (0)
-
-struct nouveau_channel *
-nv10_graph_channel(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chid = 31;
-
-	if (nv_rd32(dev, NV10_PGRAPH_CTX_CONTROL) & 0x00010000)
-		chid = nv_rd32(dev, NV10_PGRAPH_CTX_USER) >> 24;
-
-	if (chid >= 31)
-		return NULL;
-
-	return dev_priv->channels.ptr[chid];
-}
-
-static int
-nv10_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct graph_state *pgraph_ctx;
-
-	NV_DEBUG(dev, "nv10_graph_context_create %d\n", chan->id);
-
-	pgraph_ctx = kzalloc(sizeof(*pgraph_ctx), GFP_KERNEL);
-	if (pgraph_ctx == NULL)
-		return -ENOMEM;
-	chan->engctx[engine] = pgraph_ctx;
-
-	NV_WRITE_CTX(0x00400e88, 0x08000000);
-	NV_WRITE_CTX(0x00400e9c, 0x4b7fffff);
-	NV_WRITE_CTX(NV03_PGRAPH_XY_LOGIC_MISC0, 0x0001ffff);
-	NV_WRITE_CTX(0x00400e10, 0x00001000);
-	NV_WRITE_CTX(0x00400e14, 0x00001000);
-	NV_WRITE_CTX(0x00400e30, 0x00080008);
-	NV_WRITE_CTX(0x00400e34, 0x00080008);
-	if (dev_priv->chipset >= 0x17) {
-		/* is it really needed ??? */
-		NV17_WRITE_CTX(NV10_PGRAPH_DEBUG_4,
-					nv_rd32(dev, NV10_PGRAPH_DEBUG_4));
-		NV17_WRITE_CTX(0x004006b0, nv_rd32(dev, 0x004006b0));
-		NV17_WRITE_CTX(0x00400eac, 0x0fff0000);
-		NV17_WRITE_CTX(0x00400eb0, 0x0fff0000);
-		NV17_WRITE_CTX(0x00400ec0, 0x00000080);
-		NV17_WRITE_CTX(0x00400ed0, 0x00000080);
-	}
-	NV_WRITE_CTX(NV10_PGRAPH_CTX_USER, chan->id << 24);
-
-	nv10_graph_create_pipe(chan);
-	return 0;
-}
-
-static void
-nv10_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct graph_state *pgraph_ctx = chan->engctx[engine];
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-
-	/* Unload the context if it's the currently active one */
-	if (nv10_graph_channel(dev) == chan)
-		nv10_graph_unload_context(dev);
-
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* Free the context resources */
-	chan->engctx[engine] = NULL;
-	kfree(pgraph_ctx);
-}
-
-static void
-nv10_graph_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	nv_wr32(dev, NV10_PGRAPH_TLIMIT(i), tile->limit);
-	nv_wr32(dev, NV10_PGRAPH_TSIZE(i), tile->pitch);
-	nv_wr32(dev, NV10_PGRAPH_TILE(i), tile->addr);
-}
-
-static int
-nv10_graph_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 tmp;
-	int i;
-
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
-			~NV_PMC_ENABLE_PGRAPH);
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
-			 NV_PMC_ENABLE_PGRAPH);
-
-	nv_wr32(dev, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x00000000);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x00118700);
-	/* nv_wr32(dev, NV04_PGRAPH_DEBUG_2, 0x24E00810); */ /* 0x25f92ad9 */
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_2, 0x25f92ad9);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0x55DE0830 |
-				      (1<<29) |
-				      (1<<31));
-	if (dev_priv->chipset >= 0x17) {
-		nv_wr32(dev, NV10_PGRAPH_DEBUG_4, 0x1f000000);
-		nv_wr32(dev, 0x400a10, 0x3ff3fb6);
-		nv_wr32(dev, 0x400838, 0x2f8684);
-		nv_wr32(dev, 0x40083c, 0x115f3f);
-		nv_wr32(dev, 0x004006b0, 0x40000020);
-	} else
-		nv_wr32(dev, NV10_PGRAPH_DEBUG_4, 0x00000000);
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-		nv10_graph_set_tile_region(dev, i);
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(0), 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(1), 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(2), 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(3), 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_CTX_SWITCH(4), 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_STATE, 0xFFFFFFFF);
-
-	tmp  = nv_rd32(dev, NV10_PGRAPH_CTX_USER) & 0x00ffffff;
-	tmp |= 31 << 24;
-	nv_wr32(dev, NV10_PGRAPH_CTX_USER, tmp);
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
-	nv_wr32(dev, NV10_PGRAPH_FFINTFC_ST2, 0x08000000);
-
-	return 0;
-}
-
-static int
-nv10_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-	if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
-		nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-		return -EBUSY;
-	}
-	nv10_graph_unload_context(dev);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
-	return 0;
-}
-
-static int
-nv17_graph_mthd_lma_window(struct nouveau_channel *chan,
-			   u32 class, u32 mthd, u32 data)
-{
-	struct graph_state *ctx = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	struct pipe_state *pipe = &ctx->pipe_state;
-	uint32_t pipe_0x0040[1], pipe_0x64c0[8], pipe_0x6a80[3], pipe_0x6ab0[3];
-	uint32_t xfmode0, xfmode1;
-	int i;
-
-	ctx->lma_window[(mthd - 0x1638) / 4] = data;
-
-	if (mthd != 0x1644)
-		return 0;
-
-	nouveau_wait_for_idle(dev);
-
-	PIPE_SAVE(dev, pipe_0x0040, 0x0040);
-	PIPE_SAVE(dev, pipe->pipe_0x0200, 0x0200);
-
-	PIPE_RESTORE(dev, ctx->lma_window, 0x6790);
-
-	nouveau_wait_for_idle(dev);
-
-	xfmode0 = nv_rd32(dev, NV10_PGRAPH_XFMODE0);
-	xfmode1 = nv_rd32(dev, NV10_PGRAPH_XFMODE1);
-
-	PIPE_SAVE(dev, pipe->pipe_0x4400, 0x4400);
-	PIPE_SAVE(dev, pipe_0x64c0, 0x64c0);
-	PIPE_SAVE(dev, pipe_0x6ab0, 0x6ab0);
-	PIPE_SAVE(dev, pipe_0x6a80, 0x6a80);
-
-	nouveau_wait_for_idle(dev);
-
-	nv_wr32(dev, NV10_PGRAPH_XFMODE0, 0x10000000);
-	nv_wr32(dev, NV10_PGRAPH_XFMODE1, 0x00000000);
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x000064c0);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00006ab0);
-	for (i = 0; i < 3; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x3f800000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00006a80);
-	for (i = 0; i < 3; i++)
-		nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000000);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x00000040);
-	nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000008);
-
-	PIPE_RESTORE(dev, pipe->pipe_0x0200, 0x0200);
-
-	nouveau_wait_for_idle(dev);
-
-	PIPE_RESTORE(dev, pipe_0x0040, 0x0040);
-
-	nv_wr32(dev, NV10_PGRAPH_XFMODE0, xfmode0);
-	nv_wr32(dev, NV10_PGRAPH_XFMODE1, xfmode1);
-
-	PIPE_RESTORE(dev, pipe_0x64c0, 0x64c0);
-	PIPE_RESTORE(dev, pipe_0x6ab0, 0x6ab0);
-	PIPE_RESTORE(dev, pipe_0x6a80, 0x6a80);
-	PIPE_RESTORE(dev, pipe->pipe_0x4400, 0x4400);
-
-	nv_wr32(dev, NV10_PGRAPH_PIPE_ADDRESS, 0x000000c0);
-	nv_wr32(dev, NV10_PGRAPH_PIPE_DATA, 0x00000000);
-
-	nouveau_wait_for_idle(dev);
-
-	return 0;
-}
-
-static int
-nv17_graph_mthd_lma_enable(struct nouveau_channel *chan,
-			   u32 class, u32 mthd, u32 data)
-{
-	struct drm_device *dev = chan->dev;
-
-	nouveau_wait_for_idle(dev);
-
-	nv_wr32(dev, NV10_PGRAPH_DEBUG_4,
-		nv_rd32(dev, NV10_PGRAPH_DEBUG_4) | 0x1 << 8);
-	nv_wr32(dev, 0x004006b0,
-		nv_rd32(dev, 0x004006b0) | 0x8 << 24);
-
-	return 0;
-}
-
-struct nouveau_bitfield nv10_graph_intr[] = {
-	{ NV_PGRAPH_INTR_NOTIFY, "NOTIFY" },
-	{ NV_PGRAPH_INTR_ERROR,  "ERROR"  },
-	{}
-};
-
-struct nouveau_bitfield nv10_graph_nstatus[] = {
-	{ NV10_PGRAPH_NSTATUS_STATE_IN_USE,       "STATE_IN_USE" },
-	{ NV10_PGRAPH_NSTATUS_INVALID_STATE,      "INVALID_STATE" },
-	{ NV10_PGRAPH_NSTATUS_BAD_ARGUMENT,       "BAD_ARGUMENT" },
-	{ NV10_PGRAPH_NSTATUS_PROTECTION_FAULT,   "PROTECTION_FAULT" },
-	{}
-};
-
-static void
-nv10_graph_isr(struct drm_device *dev)
-{
-	u32 stat;
-
-	while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-		u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-		u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
-		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-		u32 chid = (addr & 0x01f00000) >> 20;
-		u32 subc = (addr & 0x00070000) >> 16;
-		u32 mthd = (addr & 0x00001ffc);
-		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-		u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
-		u32 show = stat;
-
-		if (stat & NV_PGRAPH_INTR_ERROR) {
-			if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-				if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
-					show &= ~NV_PGRAPH_INTR_ERROR;
-			}
-		}
-
-		if (stat & NV_PGRAPH_INTR_CONTEXT_SWITCH) {
-			nv_wr32(dev, NV03_PGRAPH_INTR, NV_PGRAPH_INTR_CONTEXT_SWITCH);
-			stat &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-			show &= ~NV_PGRAPH_INTR_CONTEXT_SWITCH;
-			nv10_graph_context_switch(dev);
-		}
-
-		nv_wr32(dev, NV03_PGRAPH_INTR, stat);
-		nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
-
-		if (show && nouveau_ratelimit()) {
-			NV_INFO(dev, "PGRAPH -");
-			nouveau_bitfield_print(nv10_graph_intr, show);
-			printk(" nsource:");
-			nouveau_bitfield_print(nv04_graph_nsource, nsource);
-			printk(" nstatus:");
-			nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
-				     "mthd 0x%04x data 0x%08x\n",
-				chid, subc, class, mthd, data);
-		}
-	}
-}
-
-static void
-nv10_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nv10_graph_engine *pgraph = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 12);
-	kfree(pgraph);
-}
-
-int
-nv10_graph_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv10_graph_engine *pgraph;
-
-	pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
-	if (!pgraph)
-		return -ENOMEM;
-
-	pgraph->base.destroy = nv10_graph_destroy;
-	pgraph->base.init = nv10_graph_init;
-	pgraph->base.fini = nv10_graph_fini;
-	pgraph->base.context_new = nv10_graph_context_new;
-	pgraph->base.context_del = nv10_graph_context_del;
-	pgraph->base.object_new = nv04_graph_object_new;
-	pgraph->base.set_tile_region = nv10_graph_set_tile_region;
-
-	NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
-	nouveau_irq_register(dev, 12, nv10_graph_isr);
-
-	NVOBJ_CLASS(dev, 0x0030, GR); /* null */
-	NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
-	NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
-	NVOBJ_CLASS(dev, 0x005f, GR); /* imageblit */
-	NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
-	NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
-	NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
-	NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
-	NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
-	NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
-	NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
-	NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
-	NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
-	NVOBJ_CLASS(dev, 0x0052, GR); /* swzsurf */
-	NVOBJ_CLASS(dev, 0x0093, GR); /* surf3d */
-	NVOBJ_CLASS(dev, 0x0094, GR); /* tex_tri */
-	NVOBJ_CLASS(dev, 0x0095, GR); /* multitex_tri */
-
-	/* celcius */
-	if (dev_priv->chipset <= 0x10) {
-		NVOBJ_CLASS(dev, 0x0056, GR);
-	} else
-	if (dev_priv->chipset < 0x17 || dev_priv->chipset == 0x1a) {
-		NVOBJ_CLASS(dev, 0x0096, GR);
-	} else {
-		NVOBJ_CLASS(dev, 0x0099, GR);
-		NVOBJ_MTHD (dev, 0x0099, 0x1638, nv17_graph_mthd_lma_window);
-		NVOBJ_MTHD (dev, 0x0099, 0x163c, nv17_graph_mthd_lma_window);
-		NVOBJ_MTHD (dev, 0x0099, 0x1640, nv17_graph_mthd_lma_window);
-		NVOBJ_MTHD (dev, 0x0099, 0x1644, nv17_graph_mthd_lma_window);
-		NVOBJ_MTHD (dev, 0x0099, 0x1658, nv17_graph_mthd_lma_enable);
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv17_fifo.c b/drivers/gpu/drm/nouveau/nv17_fifo.c
deleted file mode 100644
index d9e482e4abe..00000000000
--- a/drivers/gpu/drm/nouveau/nv17_fifo.c
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_util.h"
-#include "nouveau_ramht.h"
-
-static struct ramfc_desc {
-	unsigned bits:6;
-	unsigned ctxs:5;
-	unsigned ctxp:8;
-	unsigned regs:5;
-	unsigned regp;
-} nv17_ramfc[] = {
-	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
-	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
-	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
-	{ 16,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
-	{ 16, 16, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
-	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_STATE },
-	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
-	{ 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_ENGINE },
-	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_PULL1 },
-	{ 32,  0, 0x20,  0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE },
-	{ 32,  0, 0x24,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP },
-	{ 32,  0, 0x28,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT },
-	{ 32,  0, 0x2c,  0, NV10_PFIFO_CACHE1_SEMAPHORE },
-	{ 32,  0, 0x30,  0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE },
-	{}
-};
-
-struct nv17_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct ramfc_desc *ramfc_desc;
-};
-
-struct nv17_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *ramfc;
-};
-
-static int
-nv17_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv17_fifo_priv *priv = nv_engine(dev, engine);
-	struct nv17_fifo_chan *fctx;
-	unsigned long flags;
-	int ret;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	/* map channel control registers */
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV03_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/* initialise default fifo context */
-	ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramfc->pinst +
-				      chan->id * 64, ~0, 64,
-				      NVOBJ_FLAG_ZERO_ALLOC |
-				      NVOBJ_FLAG_ZERO_FREE, &fctx->ramfc);
-	if (ret)
-		goto error;
-
-	nv_wo32(fctx->ramfc, 0x00, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x04, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x0c, chan->pushbuf->pinst >> 4);
-	nv_wo32(fctx->ramfc, 0x14, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
-				   NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
-#ifdef __BIG_ENDIAN
-				   NV_PFIFO_CACHE1_BIG_ENDIAN |
-#endif
-				   NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
-
-	/* enable dma mode on the channel */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PFIFO_MODE, (1 << chan->id), (1 << chan->id));
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static int
-nv17_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv17_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, 0);
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, NV_PMC_ENABLE_PFIFO);
-
-	nv_wr32(dev, NV04_PFIFO_DELAY_0, 0x000000ff);
-	nv_wr32(dev, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
-
-	nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-				       ((dev_priv->ramht->bits - 9) << 16) |
-				       (dev_priv->ramht->gpuobj->pinst >> 8));
-	nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
-	nv_wr32(dev, NV03_PFIFO_RAMFC, 0x00010000 |
-				       dev_priv->ramfc->pinst >> 8);
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, priv->base.channels);
-
-	nv_wr32(dev, NV03_PFIFO_INTR_0, 0xffffffff);
-	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xffffffff);
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 1);
-	nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 1);
-
-	for (i = 0; i < priv->base.channels; i++) {
-		if (dev_priv->channels.ptr[i])
-			nv_mask(dev, NV04_PFIFO_MODE, (1 << i), (1 << i));
-	}
-
-	return 0;
-}
-
-int
-nv17_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv17_fifo_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv04_fifo_destroy;
-	priv->base.base.init = nv17_fifo_init;
-	priv->base.base.fini = nv04_fifo_fini;
-	priv->base.base.context_new = nv17_fifo_context_new;
-	priv->base.base.context_del = nv04_fifo_context_del;
-	priv->base.channels = 31;
-	priv->ramfc_desc = nv17_ramfc;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.c b/drivers/gpu/drm/nouveau/nv17_tv.c
index 67be5db021f..897b63621e2 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv.c
@@ -24,20 +24,34 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include "nouveau_drm.h"
+#include "nouveau_reg.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
-#include "nouveau_gpio.h"
 #include "nouveau_hw.h"
 #include "nv17_tv.h"
 
+#include <core/device.h>
+
+#include <subdev/bios/gpio.h>
+#include <subdev/gpio.h>
+
+MODULE_PARM_DESC(tv_norm, "Default TV norm.\n"
+		 "\t\tSupported: PAL, PAL-M, PAL-N, PAL-Nc, NTSC-M, NTSC-J,\n"
+		 "\t\t\thd480i, hd480p, hd576i, hd576p, hd720p, hd1080i.\n"
+		 "\t\tDefault: PAL\n"
+		 "\t\t*NOTE* Ignored for cards with external TV encoders.");
+static char *nouveau_tv_norm;
+module_param_named(tv_norm, nouveau_tv_norm, charp, 0400);
+
 static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
 	uint32_t testval, regoffset = nv04_dac_output_offset(encoder);
 	uint32_t gpio0, gpio1, fp_htotal, fp_hsync_start, fp_hsync_end,
 		fp_control, test_ctrl, dacclk, ctv_14, ctv_1c, ctv_6c;
@@ -46,15 +60,15 @@ static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 
 #define RGB_TEST_DATA(r, g, b) (r << 0 | g << 10 | b << 20)
 	testval = RGB_TEST_DATA(0x82, 0xeb, 0x82);
-	if (dev_priv->vbios.tvdactestval)
-		testval = dev_priv->vbios.tvdactestval;
+	if (drm->vbios.tvdactestval)
+		testval = drm->vbios.tvdactestval;
 
 	dacclk = NVReadRAMDAC(dev, 0, NV_PRAMDAC_DACCLK + regoffset);
 	head = (dacclk & 0x100) >> 8;
 
 	/* Save the previous state. */
-	gpio1 = nouveau_gpio_func_get(dev, DCB_GPIO_TVDAC1);
-	gpio0 = nouveau_gpio_func_get(dev, DCB_GPIO_TVDAC0);
+	gpio1 = gpio->get(gpio, 0, DCB_GPIO_TVDAC1, 0xff);
+	gpio0 = gpio->get(gpio, 0, DCB_GPIO_TVDAC0, 0xff);
 	fp_htotal = NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_HTOTAL);
 	fp_hsync_start = NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_HSYNC_START);
 	fp_hsync_end = NVReadRAMDAC(dev, head, NV_PRAMDAC_FP_HSYNC_END);
@@ -65,8 +79,8 @@ static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 	ctv_6c = NVReadRAMDAC(dev, head, 0x680c6c);
 
 	/* Prepare the DAC for load detection.  */
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC1, true);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC0, true);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, true);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC0, 0xff, true);
 
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_FP_HTOTAL, 1343);
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_FP_HSYNC_START, 1047);
@@ -111,8 +125,8 @@ static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_FP_HSYNC_END, fp_hsync_end);
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_FP_HSYNC_START, fp_hsync_start);
 	NVWriteRAMDAC(dev, head, NV_PRAMDAC_FP_HTOTAL, fp_htotal);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC1, gpio1);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC0, gpio0);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, gpio1);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC0, 0xff, gpio0);
 
 	return sample;
 }
@@ -120,15 +134,18 @@ static uint32_t nv42_tv_sample_load(struct drm_encoder *encoder)
 static bool
 get_tv_detect_quirks(struct drm_device *dev, uint32_t *pin_mask)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_object *device = drm->device;
+
 	/* Zotac FX5200 */
-	if (nv_match_device(dev, 0x0322, 0x19da, 0x1035) ||
-	    nv_match_device(dev, 0x0322, 0x19da, 0x2035)) {
+	if (nv_device_match(device, 0x0322, 0x19da, 0x1035) ||
+	    nv_device_match(device, 0x0322, 0x19da, 0x2035)) {
 		*pin_mask = 0xc;
 		return false;
 	}
 
 	/* MSI nForce2 IGP */
-	if (nv_match_device(dev, 0x01f0, 0x1462, 0x5710)) {
+	if (nv_device_match(device, 0x01f0, 0x1462, 0x5710)) {
 		*pin_mask = 0xc;
 		return false;
 	}
@@ -140,18 +157,18 @@ static enum drm_connector_status
 nv17_tv_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_mode_config *conf = &dev->mode_config;
 	struct nv17_tv_encoder *tv_enc = to_tv_enc(encoder);
-	struct dcb_entry *dcb = tv_enc->base.dcb;
+	struct dcb_output *dcb = tv_enc->base.dcb;
 	bool reliable = get_tv_detect_quirks(dev, &tv_enc->pin_mask);
 
 	if (nv04_dac_in_use(encoder))
 		return connector_status_disconnected;
 
 	if (reliable) {
-		if (dev_priv->chipset == 0x42 ||
-		    dev_priv->chipset == 0x43)
+		if (nv_device(drm->device)->chipset == 0x42 ||
+		    nv_device(drm->device)->chipset == 0x43)
 			tv_enc->pin_mask =
 				nv42_tv_sample_load(encoder) >> 28 & 0xe;
 		else
@@ -185,7 +202,7 @@ nv17_tv_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 	if (!reliable) {
 		return connector_status_unknown;
 	} else if (tv_enc->subconnector) {
-		NV_INFO(dev, "Load detected on output %c\n",
+		NV_INFO(drm, "Load detected on output %c\n",
 			'@' + ffs(dcb->or));
 		return connector_status_connected;
 	} else {
@@ -357,6 +374,8 @@ static bool nv17_tv_mode_fixup(struct drm_encoder *encoder,
 static void  nv17_tv_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
 	struct nv17_tv_state *regs = &to_tv_enc(encoder)->state;
 	struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
 
@@ -364,7 +383,7 @@ static void  nv17_tv_dpms(struct drm_encoder *encoder, int mode)
 		return;
 	nouveau_encoder(encoder)->last_dpms = mode;
 
-	NV_INFO(dev, "Setting dpms mode %d on TV encoder (output %d)\n",
+	NV_INFO(drm, "Setting dpms mode %d on TV encoder (output %d)\n",
 		 mode, nouveau_encoder(encoder)->dcb->index);
 
 	regs->ptv_200 &= ~1;
@@ -381,8 +400,8 @@ static void  nv17_tv_dpms(struct drm_encoder *encoder, int mode)
 
 	nv_load_ptv(dev, regs, 200);
 
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC1, mode == DRM_MODE_DPMS_ON);
-	nouveau_gpio_func_set(dev, DCB_GPIO_TVDAC0, mode == DRM_MODE_DPMS_ON);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC1, 0xff, mode == DRM_MODE_DPMS_ON);
+	gpio->set(gpio, 0, DCB_GPIO_TVDAC0, 0xff, mode == DRM_MODE_DPMS_ON);
 
 	nv04_dac_update_dacclk(encoder, mode == DRM_MODE_DPMS_ON);
 }
@@ -390,11 +409,11 @@ static void  nv17_tv_dpms(struct drm_encoder *encoder, int mode)
 static void nv17_tv_prepare(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
 	struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
 	int head = nouveau_crtc(encoder->crtc)->index;
-	uint8_t *cr_lcd = &dev_priv->mode_reg.crtc_reg[head].CRTC[
+	uint8_t *cr_lcd = &nv04_display(dev)->mode_reg.crtc_reg[head].CRTC[
 							NV_CIO_CRE_LCD__INDEX];
 	uint32_t dacclk_off = NV_PRAMDAC_DACCLK +
 					nv04_dac_output_offset(encoder);
@@ -410,14 +429,14 @@ static void nv17_tv_prepare(struct drm_encoder *encoder)
 		struct drm_encoder *enc;
 
 		list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
-			struct dcb_entry *dcb = nouveau_encoder(enc)->dcb;
+			struct dcb_output *dcb = nouveau_encoder(enc)->dcb;
 
-			if ((dcb->type == OUTPUT_TMDS ||
-			     dcb->type == OUTPUT_LVDS) &&
+			if ((dcb->type == DCB_OUTPUT_TMDS ||
+			     dcb->type == DCB_OUTPUT_LVDS) &&
 			     !enc->crtc &&
 			     nv04_dfp_get_bound_head(dev, dcb) == head) {
 				nv04_dfp_bind_head(dev, dcb, head ^ 1,
-						dev_priv->vbios.fp.dual_link);
+						drm->vbios.fp.dual_link);
 			}
 		}
 
@@ -429,7 +448,7 @@ static void nv17_tv_prepare(struct drm_encoder *encoder)
 	/* Set the DACCLK register */
 	dacclk = (NVReadRAMDAC(dev, 0, dacclk_off) & ~0x30) | 0x1;
 
-	if (dev_priv->card_type == NV_40)
+	if (nv_device(drm->device)->card_type == NV_40)
 		dacclk |= 0x1a << 16;
 
 	if (tv_norm->kind == CTV_ENC_MODE) {
@@ -453,9 +472,9 @@ static void nv17_tv_mode_set(struct drm_encoder *encoder,
 			     struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int head = nouveau_crtc(encoder->crtc)->index;
-	struct nv04_crtc_reg *regs = &dev_priv->mode_reg.crtc_reg[head];
+	struct nv04_crtc_reg *regs = &nv04_display(dev)->mode_reg.crtc_reg[head];
 	struct nv17_tv_state *tv_regs = &to_tv_enc(encoder)->state;
 	struct nv17_tv_norm_params *tv_norm = get_tv_norm(encoder);
 	int i;
@@ -486,7 +505,7 @@ static void nv17_tv_mode_set(struct drm_encoder *encoder,
 			tv_regs->ptv_614 = 0x13;
 		}
 
-		if (dev_priv->card_type >= NV_30) {
+		if (nv_device(drm->device)->card_type >= NV_30) {
 			tv_regs->ptv_500 = 0xe8e0;
 			tv_regs->ptv_504 = 0x1710;
 			tv_regs->ptv_604 = 0x0;
@@ -566,7 +585,7 @@ static void nv17_tv_mode_set(struct drm_encoder *encoder,
 static void nv17_tv_commit(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_encoder_helper_funcs *helper = encoder->helper_private;
@@ -581,7 +600,7 @@ static void nv17_tv_commit(struct drm_encoder *encoder)
 	nv17_tv_state_load(dev, &to_tv_enc(encoder)->state);
 
 	/* This could use refinement for flatpanels, but it should work */
-	if (dev_priv->chipset < 0x44)
+	if (nv_device(drm->device)->chipset < 0x44)
 		NVWriteRAMDAC(dev, 0, NV_PRAMDAC_TEST_CONTROL +
 					nv04_dac_output_offset(encoder),
 					0xf0000000);
@@ -592,7 +611,7 @@ static void nv17_tv_commit(struct drm_encoder *encoder)
 
 	helper->dpms(encoder, DRM_MODE_DPMS_ON);
 
-	NV_INFO(dev, "Output %s is running on CRTC %d using output %c\n",
+	NV_INFO(drm, "Output %s is running on CRTC %d using output %c\n",
 		drm_get_connector_name(
 			&nouveau_encoder_connector_get(nv_encoder)->base),
 		nv_crtc->index, '@' + ffs(nv_encoder->dcb->or));
@@ -630,9 +649,10 @@ static int nv17_tv_create_resources(struct drm_encoder *encoder,
 				    struct drm_connector *connector)
 {
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_mode_config *conf = &dev->mode_config;
 	struct nv17_tv_encoder *tv_enc = to_tv_enc(encoder);
-	struct dcb_entry *dcb = nouveau_encoder(encoder)->dcb;
+	struct dcb_output *dcb = nouveau_encoder(encoder)->dcb;
 	int num_tv_norms = dcb->tvconf.has_component_output ? NUM_TV_NORMS :
 							NUM_LD_TV_NORMS;
 	int i;
@@ -646,7 +666,7 @@ static int nv17_tv_create_resources(struct drm_encoder *encoder,
 		}
 
 		if (i == num_tv_norms)
-			NV_WARN(dev, "Invalid TV norm setting \"%s\"\n",
+			NV_WARN(drm, "Invalid TV norm setting \"%s\"\n",
 				nouveau_tv_norm);
 	}
 
@@ -759,8 +779,6 @@ static void nv17_tv_destroy(struct drm_encoder *encoder)
 {
 	struct nv17_tv_encoder *tv_enc = to_tv_enc(encoder);
 
-	NV_DEBUG_KMS(encoder->dev, "\n");
-
 	drm_encoder_cleanup(encoder);
 	kfree(tv_enc);
 }
@@ -788,7 +806,7 @@ static struct drm_encoder_funcs nv17_tv_funcs = {
 };
 
 int
-nv17_tv_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv17_tv_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_encoder *encoder;
diff --git a/drivers/gpu/drm/nouveau/nv17_tv.h b/drivers/gpu/drm/nouveau/nv17_tv.h
index 622e7222168..7b331543a41 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv.h
+++ b/drivers/gpu/drm/nouveau/nv17_tv.h
@@ -130,12 +130,14 @@ void nv17_ctv_update_rescaler(struct drm_encoder *encoder);
 static inline void nv_write_ptv(struct drm_device *dev, uint32_t reg,
 				uint32_t val)
 {
-	nv_wr32(dev, reg, val);
+	struct nouveau_device *device = nouveau_dev(dev);
+	nv_wr32(device, reg, val);
 }
 
 static inline uint32_t nv_read_ptv(struct drm_device *dev, uint32_t reg)
 {
-	return nv_rd32(dev, reg);
+	struct nouveau_device *device = nouveau_dev(dev);
+	return nv_rd32(device, reg);
 }
 
 static inline void nv_write_tv_enc(struct drm_device *dev, uint8_t reg,
diff --git a/drivers/gpu/drm/nouveau/nv17_tv_modes.c b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
index 4d1d29f6030..1cdfe2a5875 100644
--- a/drivers/gpu/drm/nouveau/nv17_tv_modes.c
+++ b/drivers/gpu/drm/nouveau/nv17_tv_modes.c
@@ -24,9 +24,9 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include "nouveau_drm.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
 #include "nouveau_hw.h"
@@ -543,10 +543,9 @@ void nv17_tv_update_rescaler(struct drm_encoder *encoder)
 void nv17_ctv_update_rescaler(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nv17_tv_encoder *tv_enc = to_tv_enc(encoder);
 	int head = nouveau_crtc(encoder->crtc)->index;
-	struct nv04_crtc_reg *regs = &dev_priv->mode_reg.crtc_reg[head];
+	struct nv04_crtc_reg *regs = &nv04_display(dev)->mode_reg.crtc_reg[head];
 	struct drm_display_mode *crtc_mode = &encoder->crtc->mode;
 	struct drm_display_mode *output_mode =
 		&get_tv_norm(encoder)->ctv_enc_mode.mode;
diff --git a/drivers/gpu/drm/nouveau/nv20_fb.c b/drivers/gpu/drm/nouveau/nv20_fb.c
deleted file mode 100644
index 19bd64059a6..00000000000
--- a/drivers/gpu/drm/nouveau/nv20_fb.c
+++ /dev/null
@@ -1,148 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-static struct drm_mm_node *
-nv20_fb_alloc_tag(struct drm_device *dev, uint32_t size)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct drm_mm_node *mem;
-	int ret;
-
-	ret = drm_mm_pre_get(&pfb->tag_heap);
-	if (ret)
-		return NULL;
-
-	spin_lock(&dev_priv->tile.lock);
-	mem = drm_mm_search_free(&pfb->tag_heap, size, 0, 0);
-	if (mem)
-		mem = drm_mm_get_block_atomic(mem, size, 0);
-	spin_unlock(&dev_priv->tile.lock);
-
-	return mem;
-}
-
-static void
-nv20_fb_free_tag(struct drm_device *dev, struct drm_mm_node **pmem)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct drm_mm_node *mem = *pmem;
-	if (mem) {
-		spin_lock(&dev_priv->tile.lock);
-		drm_mm_put_block(mem);
-		spin_unlock(&dev_priv->tile.lock);
-		*pmem = NULL;
-	}
-}
-
-void
-nv20_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
-			 uint32_t size, uint32_t pitch, uint32_t flags)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-	int bpp = (flags & NOUVEAU_GEM_TILE_32BPP ? 32 : 16);
-
-	tile->addr  = 0x00000001 | addr;
-	tile->limit = max(1u, addr + size) - 1;
-	tile->pitch = pitch;
-
-	/* Allocate some of the on-die tag memory, used to store Z
-	 * compression meta-data (most likely just a bitmap determining
-	 * if a given tile is compressed or not).
-	 */
-	if (flags & NOUVEAU_GEM_TILE_ZETA) {
-		tile->tag_mem = nv20_fb_alloc_tag(dev, size / 256);
-		if (tile->tag_mem) {
-			/* Enable Z compression */
-			tile->zcomp = tile->tag_mem->start;
-			if (dev_priv->chipset >= 0x25) {
-				if (bpp == 16)
-					tile->zcomp |= NV25_PFB_ZCOMP_MODE_16;
-				else
-					tile->zcomp |= NV25_PFB_ZCOMP_MODE_32;
-			} else {
-				tile->zcomp |= NV20_PFB_ZCOMP_EN;
-				if (bpp != 16)
-					tile->zcomp |= NV20_PFB_ZCOMP_MODE_32;
-			}
-		}
-
-		tile->addr |= 2;
-	}
-}
-
-void
-nv20_fb_free_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	tile->addr = tile->limit = tile->pitch = tile->zcomp = 0;
-	nv20_fb_free_tag(dev, &tile->tag_mem);
-}
-
-void
-nv20_fb_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
-	nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
-	nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
-	nv_wr32(dev, NV20_PFB_ZCOMP(i), tile->zcomp);
-}
-
-int
-nv20_fb_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 mem_size = nv_rd32(dev, 0x10020c);
-	u32 pbus1218 = nv_rd32(dev, 0x001218);
-
-	dev_priv->vram_size = mem_size & 0xff000000;
-	switch (pbus1218 & 0x00000300) {
-	case 0x00000000: dev_priv->vram_type = NV_MEM_TYPE_SDRAM; break;
-	case 0x00000100: dev_priv->vram_type = NV_MEM_TYPE_DDR1; break;
-	case 0x00000200: dev_priv->vram_type = NV_MEM_TYPE_GDDR3; break;
-	case 0x00000300: dev_priv->vram_type = NV_MEM_TYPE_GDDR2; break;
-	}
-
-	return 0;
-}
-
-int
-nv20_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i;
-
-	if (dev_priv->chipset >= 0x25)
-		drm_mm_init(&pfb->tag_heap, 0, 64 * 1024);
-	else
-		drm_mm_init(&pfb->tag_heap, 0, 32 * 1024);
-
-	/* Turn all the tiling regions off. */
-	pfb->num_tiles = NV10_PFB_TILE__SIZE;
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->set_tile_region(dev, i);
-
-	return 0;
-}
-
-void
-nv20_fb_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i;
-
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->free_tile_region(dev, i);
-
-	drm_mm_takedown(&pfb->tag_heap);
-}
diff --git a/drivers/gpu/drm/nouveau/nv20_graph.c b/drivers/gpu/drm/nouveau/nv20_graph.c
deleted file mode 100644
index e34ea30758f..00000000000
--- a/drivers/gpu/drm/nouveau/nv20_graph.c
+++ /dev/null
@@ -1,836 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-/*
- * NV20
- * -----
- * There are 3 families :
- * NV20 is 0x10de:0x020*
- * NV25/28 is 0x10de:0x025* / 0x10de:0x028*
- * NV2A is 0x10de:0x02A0
- *
- * NV30
- * -----
- * There are 3 families :
- * NV30/31 is 0x10de:0x030* / 0x10de:0x031*
- * NV34 is 0x10de:0x032*
- * NV35/36 is 0x10de:0x033* / 0x10de:0x034*
- *
- * Not seen in the wild, no dumps (probably NV35) :
- * NV37 is 0x10de:0x00fc, 0x10de:0x00fd
- * NV38 is 0x10de:0x0333, 0x10de:0x00fe
- *
- */
-
-struct nv20_graph_engine {
-	struct nouveau_exec_engine base;
-	struct nouveau_gpuobj *ctxtab;
-	void (*grctx_init)(struct nouveau_gpuobj *);
-	u32 grctx_size;
-	u32 grctx_user;
-};
-
-#define NV20_GRCTX_SIZE (3580*4)
-#define NV25_GRCTX_SIZE (3529*4)
-#define NV2A_GRCTX_SIZE (3500*4)
-
-#define NV30_31_GRCTX_SIZE (24392)
-#define NV34_GRCTX_SIZE    (18140)
-#define NV35_36_GRCTX_SIZE (22396)
-
-int
-nv20_graph_unload_context(struct drm_device *dev)
-{
-	struct nouveau_channel *chan;
-	struct nouveau_gpuobj *grctx;
-	u32 tmp;
-
-	chan = nv10_graph_channel(dev);
-	if (!chan)
-		return 0;
-	grctx = chan->engctx[NVOBJ_ENGINE_GR];
-
-	nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_POINTER, grctx->pinst >> 4);
-	nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_XFER,
-		     NV20_PGRAPH_CHANNEL_CTX_XFER_SAVE);
-
-	nouveau_wait_for_idle(dev);
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000000);
-	tmp  = nv_rd32(dev, NV10_PGRAPH_CTX_USER) & 0x00ffffff;
-	tmp |= 31 << 24;
-	nv_wr32(dev, NV10_PGRAPH_CTX_USER, tmp);
-	return 0;
-}
-
-static void
-nv20_graph_rdi(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i, writecount = 32;
-	uint32_t rdi_index = 0x2c80000;
-
-	if (dev_priv->chipset == 0x20) {
-		rdi_index = 0x3d0000;
-		writecount = 15;
-	}
-
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, rdi_index);
-	for (i = 0; i < writecount; i++)
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA, 0);
-
-	nouveau_wait_for_idle(dev);
-}
-
-static void
-nv20_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x033c, 0xffff0000);
-	nv_wo32(ctx, 0x03a0, 0x0fff0000);
-	nv_wo32(ctx, 0x03a4, 0x0fff0000);
-	nv_wo32(ctx, 0x047c, 0x00000101);
-	nv_wo32(ctx, 0x0490, 0x00000111);
-	nv_wo32(ctx, 0x04a8, 0x44400000);
-	for (i = 0x04d4; i <= 0x04e0; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x04f4; i <= 0x0500; i += 4)
-		nv_wo32(ctx, i, 0x00080000);
-	for (i = 0x050c; i <= 0x0518; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x051c; i <= 0x0528; i += 4)
-		nv_wo32(ctx, i, 0x000105b8);
-	for (i = 0x052c; i <= 0x0538; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	for (i = 0x055c; i <= 0x0598; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x05a4, 0x4b7fffff);
-	nv_wo32(ctx, 0x05fc, 0x00000001);
-	nv_wo32(ctx, 0x0604, 0x00004000);
-	nv_wo32(ctx, 0x0610, 0x00000001);
-	nv_wo32(ctx, 0x0618, 0x00040000);
-	nv_wo32(ctx, 0x061c, 0x00010000);
-	for (i = 0x1c1c; i <= 0x248c; i += 16) {
-		nv_wo32(ctx, (i + 0), 0x10700ff9);
-		nv_wo32(ctx, (i + 4), 0x0436086c);
-		nv_wo32(ctx, (i + 8), 0x000c001b);
-	}
-	nv_wo32(ctx, 0x281c, 0x3f800000);
-	nv_wo32(ctx, 0x2830, 0x3f800000);
-	nv_wo32(ctx, 0x285c, 0x40000000);
-	nv_wo32(ctx, 0x2860, 0x3f800000);
-	nv_wo32(ctx, 0x2864, 0x3f000000);
-	nv_wo32(ctx, 0x286c, 0x40000000);
-	nv_wo32(ctx, 0x2870, 0x3f800000);
-	nv_wo32(ctx, 0x2878, 0xbf800000);
-	nv_wo32(ctx, 0x2880, 0xbf800000);
-	nv_wo32(ctx, 0x34a4, 0x000fe000);
-	nv_wo32(ctx, 0x3530, 0x000003f8);
-	nv_wo32(ctx, 0x3540, 0x002fe000);
-	for (i = 0x355c; i <= 0x3578; i += 4)
-		nv_wo32(ctx, i, 0x001c527c);
-}
-
-static void
-nv25_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x035c, 0xffff0000);
-	nv_wo32(ctx, 0x03c0, 0x0fff0000);
-	nv_wo32(ctx, 0x03c4, 0x0fff0000);
-	nv_wo32(ctx, 0x049c, 0x00000101);
-	nv_wo32(ctx, 0x04b0, 0x00000111);
-	nv_wo32(ctx, 0x04c8, 0x00000080);
-	nv_wo32(ctx, 0x04cc, 0xffff0000);
-	nv_wo32(ctx, 0x04d0, 0x00000001);
-	nv_wo32(ctx, 0x04e4, 0x44400000);
-	nv_wo32(ctx, 0x04fc, 0x4b800000);
-	for (i = 0x0510; i <= 0x051c; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x0530; i <= 0x053c; i += 4)
-		nv_wo32(ctx, i, 0x00080000);
-	for (i = 0x0548; i <= 0x0554; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x0558; i <= 0x0564; i += 4)
-		nv_wo32(ctx, i, 0x000105b8);
-	for (i = 0x0568; i <= 0x0574; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	for (i = 0x0598; i <= 0x05d4; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x05e0, 0x4b7fffff);
-	nv_wo32(ctx, 0x0620, 0x00000080);
-	nv_wo32(ctx, 0x0624, 0x30201000);
-	nv_wo32(ctx, 0x0628, 0x70605040);
-	nv_wo32(ctx, 0x062c, 0xb0a09080);
-	nv_wo32(ctx, 0x0630, 0xf0e0d0c0);
-	nv_wo32(ctx, 0x0664, 0x00000001);
-	nv_wo32(ctx, 0x066c, 0x00004000);
-	nv_wo32(ctx, 0x0678, 0x00000001);
-	nv_wo32(ctx, 0x0680, 0x00040000);
-	nv_wo32(ctx, 0x0684, 0x00010000);
-	for (i = 0x1b04; i <= 0x2374; i += 16) {
-		nv_wo32(ctx, (i + 0), 0x10700ff9);
-		nv_wo32(ctx, (i + 4), 0x0436086c);
-		nv_wo32(ctx, (i + 8), 0x000c001b);
-	}
-	nv_wo32(ctx, 0x2704, 0x3f800000);
-	nv_wo32(ctx, 0x2718, 0x3f800000);
-	nv_wo32(ctx, 0x2744, 0x40000000);
-	nv_wo32(ctx, 0x2748, 0x3f800000);
-	nv_wo32(ctx, 0x274c, 0x3f000000);
-	nv_wo32(ctx, 0x2754, 0x40000000);
-	nv_wo32(ctx, 0x2758, 0x3f800000);
-	nv_wo32(ctx, 0x2760, 0xbf800000);
-	nv_wo32(ctx, 0x2768, 0xbf800000);
-	nv_wo32(ctx, 0x308c, 0x000fe000);
-	nv_wo32(ctx, 0x3108, 0x000003f8);
-	nv_wo32(ctx, 0x3468, 0x002fe000);
-	for (i = 0x3484; i <= 0x34a0; i += 4)
-		nv_wo32(ctx, i, 0x001c527c);
-}
-
-static void
-nv2a_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x033c, 0xffff0000);
-	nv_wo32(ctx, 0x03a0, 0x0fff0000);
-	nv_wo32(ctx, 0x03a4, 0x0fff0000);
-	nv_wo32(ctx, 0x047c, 0x00000101);
-	nv_wo32(ctx, 0x0490, 0x00000111);
-	nv_wo32(ctx, 0x04a8, 0x44400000);
-	for (i = 0x04d4; i <= 0x04e0; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x04f4; i <= 0x0500; i += 4)
-		nv_wo32(ctx, i, 0x00080000);
-	for (i = 0x050c; i <= 0x0518; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x051c; i <= 0x0528; i += 4)
-		nv_wo32(ctx, i, 0x000105b8);
-	for (i = 0x052c; i <= 0x0538; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	for (i = 0x055c; i <= 0x0598; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x05a4, 0x4b7fffff);
-	nv_wo32(ctx, 0x05fc, 0x00000001);
-	nv_wo32(ctx, 0x0604, 0x00004000);
-	nv_wo32(ctx, 0x0610, 0x00000001);
-	nv_wo32(ctx, 0x0618, 0x00040000);
-	nv_wo32(ctx, 0x061c, 0x00010000);
-	for (i = 0x1a9c; i <= 0x22fc; i += 16) { /*XXX: check!! */
-		nv_wo32(ctx, (i + 0), 0x10700ff9);
-		nv_wo32(ctx, (i + 4), 0x0436086c);
-		nv_wo32(ctx, (i + 8), 0x000c001b);
-	}
-	nv_wo32(ctx, 0x269c, 0x3f800000);
-	nv_wo32(ctx, 0x26b0, 0x3f800000);
-	nv_wo32(ctx, 0x26dc, 0x40000000);
-	nv_wo32(ctx, 0x26e0, 0x3f800000);
-	nv_wo32(ctx, 0x26e4, 0x3f000000);
-	nv_wo32(ctx, 0x26ec, 0x40000000);
-	nv_wo32(ctx, 0x26f0, 0x3f800000);
-	nv_wo32(ctx, 0x26f8, 0xbf800000);
-	nv_wo32(ctx, 0x2700, 0xbf800000);
-	nv_wo32(ctx, 0x3024, 0x000fe000);
-	nv_wo32(ctx, 0x30a0, 0x000003f8);
-	nv_wo32(ctx, 0x33fc, 0x002fe000);
-	for (i = 0x341c; i <= 0x3438; i += 4)
-		nv_wo32(ctx, i, 0x001c527c);
-}
-
-static void
-nv30_31_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x0410, 0x00000101);
-	nv_wo32(ctx, 0x0424, 0x00000111);
-	nv_wo32(ctx, 0x0428, 0x00000060);
-	nv_wo32(ctx, 0x0444, 0x00000080);
-	nv_wo32(ctx, 0x0448, 0xffff0000);
-	nv_wo32(ctx, 0x044c, 0x00000001);
-	nv_wo32(ctx, 0x0460, 0x44400000);
-	nv_wo32(ctx, 0x048c, 0xffff0000);
-	for (i = 0x04e0; i < 0x04e8; i += 4)
-		nv_wo32(ctx, i, 0x0fff0000);
-	nv_wo32(ctx, 0x04ec, 0x00011100);
-	for (i = 0x0508; i < 0x0548; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x0550, 0x4b7fffff);
-	nv_wo32(ctx, 0x058c, 0x00000080);
-	nv_wo32(ctx, 0x0590, 0x30201000);
-	nv_wo32(ctx, 0x0594, 0x70605040);
-	nv_wo32(ctx, 0x0598, 0xb8a89888);
-	nv_wo32(ctx, 0x059c, 0xf8e8d8c8);
-	nv_wo32(ctx, 0x05b0, 0xb0000000);
-	for (i = 0x0600; i < 0x0640; i += 4)
-		nv_wo32(ctx, i, 0x00010588);
-	for (i = 0x0640; i < 0x0680; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x06c0; i < 0x0700; i += 4)
-		nv_wo32(ctx, i, 0x0008aae4);
-	for (i = 0x0700; i < 0x0740; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x0740; i < 0x0780; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	nv_wo32(ctx, 0x085c, 0x00040000);
-	nv_wo32(ctx, 0x0860, 0x00010000);
-	for (i = 0x0864; i < 0x0874; i += 4)
-		nv_wo32(ctx, i, 0x00040004);
-	for (i = 0x1f18; i <= 0x3088 ; i += 16) {
-		nv_wo32(ctx, i + 0, 0x10700ff9);
-		nv_wo32(ctx, i + 1, 0x0436086c);
-		nv_wo32(ctx, i + 2, 0x000c001b);
-	}
-	for (i = 0x30b8; i < 0x30c8; i += 4)
-		nv_wo32(ctx, i, 0x0000ffff);
-	nv_wo32(ctx, 0x344c, 0x3f800000);
-	nv_wo32(ctx, 0x3808, 0x3f800000);
-	nv_wo32(ctx, 0x381c, 0x3f800000);
-	nv_wo32(ctx, 0x3848, 0x40000000);
-	nv_wo32(ctx, 0x384c, 0x3f800000);
-	nv_wo32(ctx, 0x3850, 0x3f000000);
-	nv_wo32(ctx, 0x3858, 0x40000000);
-	nv_wo32(ctx, 0x385c, 0x3f800000);
-	nv_wo32(ctx, 0x3864, 0xbf800000);
-	nv_wo32(ctx, 0x386c, 0xbf800000);
-}
-
-static void
-nv34_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x040c, 0x01000101);
-	nv_wo32(ctx, 0x0420, 0x00000111);
-	nv_wo32(ctx, 0x0424, 0x00000060);
-	nv_wo32(ctx, 0x0440, 0x00000080);
-	nv_wo32(ctx, 0x0444, 0xffff0000);
-	nv_wo32(ctx, 0x0448, 0x00000001);
-	nv_wo32(ctx, 0x045c, 0x44400000);
-	nv_wo32(ctx, 0x0480, 0xffff0000);
-	for (i = 0x04d4; i < 0x04dc; i += 4)
-		nv_wo32(ctx, i, 0x0fff0000);
-	nv_wo32(ctx, 0x04e0, 0x00011100);
-	for (i = 0x04fc; i < 0x053c; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x0544, 0x4b7fffff);
-	nv_wo32(ctx, 0x057c, 0x00000080);
-	nv_wo32(ctx, 0x0580, 0x30201000);
-	nv_wo32(ctx, 0x0584, 0x70605040);
-	nv_wo32(ctx, 0x0588, 0xb8a89888);
-	nv_wo32(ctx, 0x058c, 0xf8e8d8c8);
-	nv_wo32(ctx, 0x05a0, 0xb0000000);
-	for (i = 0x05f0; i < 0x0630; i += 4)
-		nv_wo32(ctx, i, 0x00010588);
-	for (i = 0x0630; i < 0x0670; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x06b0; i < 0x06f0; i += 4)
-		nv_wo32(ctx, i, 0x0008aae4);
-	for (i = 0x06f0; i < 0x0730; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x0730; i < 0x0770; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	nv_wo32(ctx, 0x0850, 0x00040000);
-	nv_wo32(ctx, 0x0854, 0x00010000);
-	for (i = 0x0858; i < 0x0868; i += 4)
-		nv_wo32(ctx, i, 0x00040004);
-	for (i = 0x15ac; i <= 0x271c ; i += 16) {
-		nv_wo32(ctx, i + 0, 0x10700ff9);
-		nv_wo32(ctx, i + 1, 0x0436086c);
-		nv_wo32(ctx, i + 2, 0x000c001b);
-	}
-	for (i = 0x274c; i < 0x275c; i += 4)
-		nv_wo32(ctx, i, 0x0000ffff);
-	nv_wo32(ctx, 0x2ae0, 0x3f800000);
-	nv_wo32(ctx, 0x2e9c, 0x3f800000);
-	nv_wo32(ctx, 0x2eb0, 0x3f800000);
-	nv_wo32(ctx, 0x2edc, 0x40000000);
-	nv_wo32(ctx, 0x2ee0, 0x3f800000);
-	nv_wo32(ctx, 0x2ee4, 0x3f000000);
-	nv_wo32(ctx, 0x2eec, 0x40000000);
-	nv_wo32(ctx, 0x2ef0, 0x3f800000);
-	nv_wo32(ctx, 0x2ef8, 0xbf800000);
-	nv_wo32(ctx, 0x2f00, 0xbf800000);
-}
-
-static void
-nv35_36_graph_context_init(struct nouveau_gpuobj *ctx)
-{
-	int i;
-
-	nv_wo32(ctx, 0x040c, 0x00000101);
-	nv_wo32(ctx, 0x0420, 0x00000111);
-	nv_wo32(ctx, 0x0424, 0x00000060);
-	nv_wo32(ctx, 0x0440, 0x00000080);
-	nv_wo32(ctx, 0x0444, 0xffff0000);
-	nv_wo32(ctx, 0x0448, 0x00000001);
-	nv_wo32(ctx, 0x045c, 0x44400000);
-	nv_wo32(ctx, 0x0488, 0xffff0000);
-	for (i = 0x04dc; i < 0x04e4; i += 4)
-		nv_wo32(ctx, i, 0x0fff0000);
-	nv_wo32(ctx, 0x04e8, 0x00011100);
-	for (i = 0x0504; i < 0x0544; i += 4)
-		nv_wo32(ctx, i, 0x07ff0000);
-	nv_wo32(ctx, 0x054c, 0x4b7fffff);
-	nv_wo32(ctx, 0x0588, 0x00000080);
-	nv_wo32(ctx, 0x058c, 0x30201000);
-	nv_wo32(ctx, 0x0590, 0x70605040);
-	nv_wo32(ctx, 0x0594, 0xb8a89888);
-	nv_wo32(ctx, 0x0598, 0xf8e8d8c8);
-	nv_wo32(ctx, 0x05ac, 0xb0000000);
-	for (i = 0x0604; i < 0x0644; i += 4)
-		nv_wo32(ctx, i, 0x00010588);
-	for (i = 0x0644; i < 0x0684; i += 4)
-		nv_wo32(ctx, i, 0x00030303);
-	for (i = 0x06c4; i < 0x0704; i += 4)
-		nv_wo32(ctx, i, 0x0008aae4);
-	for (i = 0x0704; i < 0x0744; i += 4)
-		nv_wo32(ctx, i, 0x01012000);
-	for (i = 0x0744; i < 0x0784; i += 4)
-		nv_wo32(ctx, i, 0x00080008);
-	nv_wo32(ctx, 0x0860, 0x00040000);
-	nv_wo32(ctx, 0x0864, 0x00010000);
-	for (i = 0x0868; i < 0x0878; i += 4)
-		nv_wo32(ctx, i, 0x00040004);
-	for (i = 0x1f1c; i <= 0x308c ; i += 16) {
-		nv_wo32(ctx, i + 0, 0x10700ff9);
-		nv_wo32(ctx, i + 4, 0x0436086c);
-		nv_wo32(ctx, i + 8, 0x000c001b);
-	}
-	for (i = 0x30bc; i < 0x30cc; i += 4)
-		nv_wo32(ctx, i, 0x0000ffff);
-	nv_wo32(ctx, 0x3450, 0x3f800000);
-	nv_wo32(ctx, 0x380c, 0x3f800000);
-	nv_wo32(ctx, 0x3820, 0x3f800000);
-	nv_wo32(ctx, 0x384c, 0x40000000);
-	nv_wo32(ctx, 0x3850, 0x3f800000);
-	nv_wo32(ctx, 0x3854, 0x3f000000);
-	nv_wo32(ctx, 0x385c, 0x40000000);
-	nv_wo32(ctx, 0x3860, 0x3f800000);
-	nv_wo32(ctx, 0x3868, 0xbf800000);
-	nv_wo32(ctx, 0x3870, 0xbf800000);
-}
-
-int
-nv20_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv20_graph_engine *pgraph = nv_engine(chan->dev, engine);
-	struct nouveau_gpuobj *grctx = NULL;
-	struct drm_device *dev = chan->dev;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 16,
-				 NVOBJ_FLAG_ZERO_ALLOC, &grctx);
-	if (ret)
-		return ret;
-
-	/* Initialise default context values */
-	pgraph->grctx_init(grctx);
-
-	/* nv20: nv_wo32(dev, chan->ramin_grctx->gpuobj, 10, chan->id<<24); */
-	/* CTX_USER */
-	nv_wo32(grctx, pgraph->grctx_user, (chan->id << 24) | 0x1);
-
-	nv_wo32(pgraph->ctxtab, chan->id * 4, grctx->pinst >> 4);
-	chan->engctx[engine] = grctx;
-	return 0;
-}
-
-void
-nv20_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv20_graph_engine *pgraph = nv_engine(chan->dev, engine);
-	struct nouveau_gpuobj *grctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-
-	/* Unload the context if it's the currently active one */
-	if (nv10_graph_channel(dev) == chan)
-		nv20_graph_unload_context(dev);
-
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* Free the context resources */
-	nv_wo32(pgraph->ctxtab, chan->id * 4, 0);
-
-	nouveau_gpuobj_ref(NULL, &grctx);
-	chan->engctx[engine] = NULL;
-}
-
-static void
-nv20_graph_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
-	nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
-	nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
-
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0030 + 4 * i);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->limit);
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0050 + 4 * i);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->pitch);
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0010 + 4 * i);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->addr);
-
-	if (dev_priv->card_type == NV_20) {
-		nv_wr32(dev, NV20_PGRAPH_ZCOMP(i), tile->zcomp);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00ea0090 + 4 * i);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA, tile->zcomp);
-	}
-}
-
-int
-nv20_graph_init(struct drm_device *dev, int engine)
-{
-	struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t tmp, vramsz;
-	int i;
-
-	nv_wr32(dev, NV03_PMC_ENABLE,
-		nv_rd32(dev, NV03_PMC_ENABLE) & ~NV_PMC_ENABLE_PGRAPH);
-	nv_wr32(dev, NV03_PMC_ENABLE,
-		nv_rd32(dev, NV03_PMC_ENABLE) |  NV_PMC_ENABLE_PGRAPH);
-
-	nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE, pgraph->ctxtab->pinst >> 4);
-
-	nv20_graph_rdi(dev);
-
-	nv_wr32(dev, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x00000000);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x00118700);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0xF3CE0475); /* 0x4 = auto ctx switch */
-	nv_wr32(dev, NV10_PGRAPH_DEBUG_4, 0x00000000);
-	nv_wr32(dev, 0x40009C           , 0x00000040);
-
-	if (dev_priv->chipset >= 0x25) {
-		nv_wr32(dev, 0x400890, 0x00a8cfff);
-		nv_wr32(dev, 0x400610, 0x304B1FB6);
-		nv_wr32(dev, 0x400B80, 0x1cbd3883);
-		nv_wr32(dev, 0x400B84, 0x44000000);
-		nv_wr32(dev, 0x400098, 0x40000080);
-		nv_wr32(dev, 0x400B88, 0x000000ff);
-
-	} else {
-		nv_wr32(dev, 0x400880, 0x0008c7df);
-		nv_wr32(dev, 0x400094, 0x00000005);
-		nv_wr32(dev, 0x400B80, 0x45eae20e);
-		nv_wr32(dev, 0x400B84, 0x24000000);
-		nv_wr32(dev, 0x400098, 0x00000040);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E00038);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00000030);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E10038);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00000030);
-	}
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-		nv20_graph_set_tile_region(dev, i);
-
-	nv_wr32(dev, 0x4009a0, nv_rd32(dev, 0x100324));
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA000C);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA, nv_rd32(dev, 0x100324));
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
-	nv_wr32(dev, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
-
-	tmp = nv_rd32(dev, NV10_PGRAPH_SURFACE) & 0x0007ff00;
-	nv_wr32(dev, NV10_PGRAPH_SURFACE, tmp);
-	tmp = nv_rd32(dev, NV10_PGRAPH_SURFACE) | 0x00020100;
-	nv_wr32(dev, NV10_PGRAPH_SURFACE, tmp);
-
-	/* begin RAM config */
-	vramsz = pci_resource_len(dev->pdev, 0) - 1;
-	nv_wr32(dev, 0x4009A4, nv_rd32(dev, NV04_PFB_CFG0));
-	nv_wr32(dev, 0x4009A8, nv_rd32(dev, NV04_PFB_CFG1));
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0000);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA , nv_rd32(dev, NV04_PFB_CFG0));
-	nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0004);
-	nv_wr32(dev, NV10_PGRAPH_RDI_DATA , nv_rd32(dev, NV04_PFB_CFG1));
-	nv_wr32(dev, 0x400820, 0);
-	nv_wr32(dev, 0x400824, 0);
-	nv_wr32(dev, 0x400864, vramsz - 1);
-	nv_wr32(dev, 0x400868, vramsz - 1);
-
-	/* interesting.. the below overwrites some of the tile setup above.. */
-	nv_wr32(dev, 0x400B20, 0x00000000);
-	nv_wr32(dev, 0x400B04, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV03_PGRAPH_ABS_UCLIP_XMIN, 0);
-	nv_wr32(dev, NV03_PGRAPH_ABS_UCLIP_YMIN, 0);
-	nv_wr32(dev, NV03_PGRAPH_ABS_UCLIP_XMAX, 0x7fff);
-	nv_wr32(dev, NV03_PGRAPH_ABS_UCLIP_YMAX, 0x7fff);
-
-	return 0;
-}
-
-int
-nv30_graph_init(struct drm_device *dev, int engine)
-{
-	struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i;
-
-	nv_wr32(dev, NV03_PMC_ENABLE,
-		nv_rd32(dev, NV03_PMC_ENABLE) & ~NV_PMC_ENABLE_PGRAPH);
-	nv_wr32(dev, NV03_PMC_ENABLE,
-		nv_rd32(dev, NV03_PMC_ENABLE) |  NV_PMC_ENABLE_PGRAPH);
-
-	nv_wr32(dev, NV20_PGRAPH_CHANNEL_CTX_TABLE, pgraph->ctxtab->pinst >> 4);
-
-	nv_wr32(dev, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x00000000);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x401287c0);
-	nv_wr32(dev, 0x400890, 0x01b463ff);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0xf2de0475);
-	nv_wr32(dev, NV10_PGRAPH_DEBUG_4, 0x00008000);
-	nv_wr32(dev, NV04_PGRAPH_LIMIT_VIOL_PIX, 0xf04bdff6);
-	nv_wr32(dev, 0x400B80, 0x1003d888);
-	nv_wr32(dev, 0x400B84, 0x0c000000);
-	nv_wr32(dev, 0x400098, 0x00000000);
-	nv_wr32(dev, 0x40009C, 0x0005ad00);
-	nv_wr32(dev, 0x400B88, 0x62ff00ff); /* suspiciously like PGRAPH_DEBUG_2 */
-	nv_wr32(dev, 0x4000a0, 0x00000000);
-	nv_wr32(dev, 0x4000a4, 0x00000008);
-	nv_wr32(dev, 0x4008a8, 0xb784a400);
-	nv_wr32(dev, 0x400ba0, 0x002f8685);
-	nv_wr32(dev, 0x400ba4, 0x00231f3f);
-	nv_wr32(dev, 0x4008a4, 0x40000020);
-
-	if (dev_priv->chipset == 0x34) {
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0004);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00200201);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0008);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00000008);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00EA0000);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00000032);
-		nv_wr32(dev, NV10_PGRAPH_RDI_INDEX, 0x00E00004);
-		nv_wr32(dev, NV10_PGRAPH_RDI_DATA , 0x00000002);
-	}
-
-	nv_wr32(dev, 0x4000c0, 0x00000016);
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < NV10_PFB_TILE__SIZE; i++)
-		nv20_graph_set_tile_region(dev, i);
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10000100);
-	nv_wr32(dev, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
-	nv_wr32(dev, 0x0040075c             , 0x00000001);
-
-	/* begin RAM config */
-	/* vramsz = pci_resource_len(dev->pdev, 0) - 1; */
-	nv_wr32(dev, 0x4009A4, nv_rd32(dev, NV04_PFB_CFG0));
-	nv_wr32(dev, 0x4009A8, nv_rd32(dev, NV04_PFB_CFG1));
-	if (dev_priv->chipset != 0x34) {
-		nv_wr32(dev, 0x400750, 0x00EA0000);
-		nv_wr32(dev, 0x400754, nv_rd32(dev, NV04_PFB_CFG0));
-		nv_wr32(dev, 0x400750, 0x00EA0004);
-		nv_wr32(dev, 0x400754, nv_rd32(dev, NV04_PFB_CFG1));
-	}
-
-	return 0;
-}
-
-int
-nv20_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000000);
-	if (!nv_wait(dev, NV04_PGRAPH_STATUS, ~0, 0) && suspend) {
-		nv_mask(dev, NV04_PGRAPH_FIFO, 0x00000001, 0x00000001);
-		return -EBUSY;
-	}
-	nv20_graph_unload_context(dev);
-	nv_wr32(dev, NV03_PGRAPH_INTR_EN, 0x00000000);
-	return 0;
-}
-
-static void
-nv20_graph_isr(struct drm_device *dev)
-{
-	u32 stat;
-
-	while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-		u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-		u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
-		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-		u32 chid = (addr & 0x01f00000) >> 20;
-		u32 subc = (addr & 0x00070000) >> 16;
-		u32 mthd = (addr & 0x00001ffc);
-		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-		u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xfff;
-		u32 show = stat;
-
-		if (stat & NV_PGRAPH_INTR_ERROR) {
-			if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-				if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
-					show &= ~NV_PGRAPH_INTR_ERROR;
-			}
-		}
-
-		nv_wr32(dev, NV03_PGRAPH_INTR, stat);
-		nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
-
-		if (show && nouveau_ratelimit()) {
-			NV_INFO(dev, "PGRAPH -");
-			nouveau_bitfield_print(nv10_graph_intr, show);
-			printk(" nsource:");
-			nouveau_bitfield_print(nv04_graph_nsource, nsource);
-			printk(" nstatus:");
-			nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - ch %d/%d class 0x%04x "
-				     "mthd 0x%04x data 0x%08x\n",
-				chid, subc, class, mthd, data);
-		}
-	}
-}
-
-static void
-nv20_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nv20_graph_engine *pgraph = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 12);
-	nouveau_gpuobj_ref(NULL, &pgraph->ctxtab);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-	kfree(pgraph);
-}
-
-int
-nv20_graph_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv20_graph_engine *pgraph;
-	int ret;
-
-	pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
-	if (!pgraph)
-		return -ENOMEM;
-
-	pgraph->base.destroy = nv20_graph_destroy;
-	pgraph->base.fini = nv20_graph_fini;
-	pgraph->base.context_new = nv20_graph_context_new;
-	pgraph->base.context_del = nv20_graph_context_del;
-	pgraph->base.object_new = nv04_graph_object_new;
-	pgraph->base.set_tile_region = nv20_graph_set_tile_region;
-
-	pgraph->grctx_user = 0x0028;
-	if (dev_priv->card_type == NV_20) {
-		pgraph->base.init = nv20_graph_init;
-		switch (dev_priv->chipset) {
-		case 0x20:
-			pgraph->grctx_init = nv20_graph_context_init;
-			pgraph->grctx_size = NV20_GRCTX_SIZE;
-			pgraph->grctx_user = 0x0000;
-			break;
-		case 0x25:
-		case 0x28:
-			pgraph->grctx_init = nv25_graph_context_init;
-			pgraph->grctx_size = NV25_GRCTX_SIZE;
-			break;
-		case 0x2a:
-			pgraph->grctx_init = nv2a_graph_context_init;
-			pgraph->grctx_size = NV2A_GRCTX_SIZE;
-			pgraph->grctx_user = 0x0000;
-			break;
-		default:
-			NV_ERROR(dev, "PGRAPH: unknown chipset\n");
-			kfree(pgraph);
-			return 0;
-		}
-	} else {
-		pgraph->base.init = nv30_graph_init;
-		switch (dev_priv->chipset) {
-		case 0x30:
-		case 0x31:
-			pgraph->grctx_init = nv30_31_graph_context_init;
-			pgraph->grctx_size = NV30_31_GRCTX_SIZE;
-			break;
-		case 0x34:
-			pgraph->grctx_init = nv34_graph_context_init;
-			pgraph->grctx_size = NV34_GRCTX_SIZE;
-			break;
-		case 0x35:
-		case 0x36:
-			pgraph->grctx_init = nv35_36_graph_context_init;
-			pgraph->grctx_size = NV35_36_GRCTX_SIZE;
-			break;
-		default:
-			NV_ERROR(dev, "PGRAPH: unknown chipset\n");
-			kfree(pgraph);
-			return 0;
-		}
-	}
-
-	/* Create Context Pointer Table */
-	ret = nouveau_gpuobj_new(dev, NULL, 32 * 4, 16, NVOBJ_FLAG_ZERO_ALLOC,
-				 &pgraph->ctxtab);
-	if (ret) {
-		kfree(pgraph);
-		return ret;
-	}
-
-	NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
-	nouveau_irq_register(dev, 12, nv20_graph_isr);
-
-	NVOBJ_CLASS(dev, 0x0030, GR); /* null */
-	NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
-	NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
-	NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
-	NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
-	NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
-	NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
-	NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
-	NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
-	NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
-	NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
-	NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
-	if (dev_priv->card_type == NV_20) {
-		NVOBJ_CLASS(dev, 0x009e, GR); /* swzsurf */
-		NVOBJ_CLASS(dev, 0x0096, GR); /* celcius */
-
-		/* kelvin */
-		if (dev_priv->chipset < 0x25)
-			NVOBJ_CLASS(dev, 0x0097, GR);
-		else
-			NVOBJ_CLASS(dev, 0x0597, GR);
-	} else {
-		NVOBJ_CLASS(dev, 0x038a, GR); /* ifc (nv30) */
-		NVOBJ_CLASS(dev, 0x0389, GR); /* sifm (nv30) */
-		NVOBJ_CLASS(dev, 0x0362, GR); /* surf2d (nv30) */
-		NVOBJ_CLASS(dev, 0x039e, GR); /* swzsurf */
-
-		/* rankine */
-		if (0x00000003 & (1 << (dev_priv->chipset & 0x0f)))
-			NVOBJ_CLASS(dev, 0x0397, GR);
-		else
-		if (0x00000010 & (1 << (dev_priv->chipset & 0x0f)))
-			NVOBJ_CLASS(dev, 0x0697, GR);
-		else
-		if (0x000001e0 & (1 << (dev_priv->chipset & 0x0f)))
-			NVOBJ_CLASS(dev, 0x0497, GR);
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv30_fb.c b/drivers/gpu/drm/nouveau/nv30_fb.c
deleted file mode 100644
index e0135f0e214..00000000000
--- a/drivers/gpu/drm/nouveau/nv30_fb.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright (C) 2010 Francisco Jerez.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-void
-nv30_fb_init_tile_region(struct drm_device *dev, int i, uint32_t addr,
-			 uint32_t size, uint32_t pitch, uint32_t flags)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	tile->addr = addr | 1;
-	tile->limit = max(1u, addr + size) - 1;
-	tile->pitch = pitch;
-}
-
-void
-nv30_fb_free_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	tile->addr = tile->limit = tile->pitch = 0;
-}
-
-static int
-calc_bias(struct drm_device *dev, int k, int i, int j)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int b = (dev_priv->chipset > 0x30 ?
-		 nv_rd32(dev, 0x122c + 0x10 * k + 0x4 * j) >> (4 * (i ^ 1)) :
-		 0) & 0xf;
-
-	return 2 * (b & 0x8 ? b - 0x10 : b);
-}
-
-static int
-calc_ref(struct drm_device *dev, int l, int k, int i)
-{
-	int j, x = 0;
-
-	for (j = 0; j < 4; j++) {
-		int m = (l >> (8 * i) & 0xff) + calc_bias(dev, k, i, j);
-
-		x |= (0x80 | clamp(m, 0, 0x1f)) << (8 * j);
-	}
-
-	return x;
-}
-
-int
-nv30_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	int i, j;
-
-	pfb->num_tiles = NV10_PFB_TILE__SIZE;
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->set_tile_region(dev, i);
-
-	/* Init the memory timing regs at 0x10037c/0x1003ac */
-	if (dev_priv->chipset == 0x30 ||
-	    dev_priv->chipset == 0x31 ||
-	    dev_priv->chipset == 0x35) {
-		/* Related to ROP count */
-		int n = (dev_priv->chipset == 0x31 ? 2 : 4);
-		int l = nv_rd32(dev, 0x1003d0);
-
-		for (i = 0; i < n; i++) {
-			for (j = 0; j < 3; j++)
-				nv_wr32(dev, 0x10037c + 0xc * i + 0x4 * j,
-					calc_ref(dev, l, 0, j));
-
-			for (j = 0; j < 2; j++)
-				nv_wr32(dev, 0x1003ac + 0x8 * i + 0x4 * j,
-					calc_ref(dev, l, 1, j));
-		}
-	}
-
-	return 0;
-}
-
-void
-nv30_fb_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv31_mpeg.c b/drivers/gpu/drm/nouveau/nv31_mpeg.c
deleted file mode 100644
index 5f239bf658c..00000000000
--- a/drivers/gpu/drm/nouveau/nv31_mpeg.c
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-
-struct nv31_mpeg_engine {
-	struct nouveau_exec_engine base;
-	atomic_t refcount;
-};
-
-
-static int
-nv31_mpeg_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv31_mpeg_engine *pmpeg = nv_engine(chan->dev, engine);
-
-	if (!atomic_add_unless(&pmpeg->refcount, 1, 1))
-		return -EBUSY;
-
-	chan->engctx[engine] = (void *)0xdeadcafe;
-	return 0;
-}
-
-static void
-nv31_mpeg_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv31_mpeg_engine *pmpeg = nv_engine(chan->dev, engine);
-	atomic_dec(&pmpeg->refcount);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nv40_mpeg_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ctx = NULL;
-	unsigned long flags;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	ret = nouveau_gpuobj_new(dev, NULL, 264 * 4, 16, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &ctx);
-	if (ret)
-		return ret;
-
-	nv_wo32(ctx, 0x78, 0x02001ec1);
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
-	if ((nv_rd32(dev, 0x003204) & 0x1f) == chan->id)
-		nv_wr32(dev, 0x00330c, ctx->pinst >> 4);
-	nv_wo32(chan->ramfc, 0x54, ctx->pinst >> 4);
-	nv_mask(dev, 0x002500, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	chan->engctx[engine] = ctx;
-	return 0;
-}
-
-static void
-nv40_mpeg_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	unsigned long flags;
-	u32 inst = 0x80000000 | (ctx->pinst >> 4);
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
-	if (nv_rd32(dev, 0x00b318) == inst)
-		nv_mask(dev, 0x00b318, 0x80000000, 0x00000000);
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	nouveau_gpuobj_ref(NULL, &ctx);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nv31_mpeg_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 20, 16, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 2;
-	obj->class  = class;
-
-	nv_wo32(obj, 0x00, class);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static int
-nv31_mpeg_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv31_mpeg_engine *pmpeg = nv_engine(dev, engine);
-	int i;
-
-	/* VPE init */
-	nv_mask(dev, 0x000200, 0x00000002, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000002, 0x00000002);
-	nv_wr32(dev, 0x00b0e0, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
-	nv_wr32(dev, 0x00b0e8, 0x00000020); /* nvidia: rd 0x01, wr 0x20 */
-
-	for (i = 0; i < dev_priv->engine.fb.num_tiles; i++)
-		pmpeg->base.set_tile_region(dev, i);
-
-	/* PMPEG init */
-	nv_wr32(dev, 0x00b32c, 0x00000000);
-	nv_wr32(dev, 0x00b314, 0x00000100);
-	nv_wr32(dev, 0x00b220, nv44_graph_class(dev) ? 0x00000044 : 0x00000031);
-	nv_wr32(dev, 0x00b300, 0x02001ec1);
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
-
-	nv_wr32(dev, 0x00b100, 0xffffffff);
-	nv_wr32(dev, 0x00b140, 0xffffffff);
-
-	if (!nv_wait(dev, 0x00b200, 0x00000001, 0x00000000)) {
-		NV_ERROR(dev, "PMPEG init: 0x%08x\n", nv_rd32(dev, 0x00b200));
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-static int
-nv31_mpeg_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	/*XXX: context save? */
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x00b140, 0x00000000);
-	return 0;
-}
-
-static int
-nv31_mpeg_mthd_dma(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct drm_device *dev = chan->dev;
-	u32 inst = data << 4;
-	u32 dma0 = nv_ri32(dev, inst + 0);
-	u32 dma1 = nv_ri32(dev, inst + 4);
-	u32 dma2 = nv_ri32(dev, inst + 8);
-	u32 base = (dma2 & 0xfffff000) | (dma0 >> 20);
-	u32 size = dma1 + 1;
-
-	/* only allow linear DMA objects */
-	if (!(dma0 & 0x00002000))
-		return -EINVAL;
-
-	if (mthd == 0x0190) {
-		/* DMA_CMD */
-		nv_mask(dev, 0x00b300, 0x00030000, (dma0 & 0x00030000));
-		nv_wr32(dev, 0x00b334, base);
-		nv_wr32(dev, 0x00b324, size);
-	} else
-	if (mthd == 0x01a0) {
-		/* DMA_DATA */
-		nv_mask(dev, 0x00b300, 0x000c0000, (dma0 & 0x00030000) << 2);
-		nv_wr32(dev, 0x00b360, base);
-		nv_wr32(dev, 0x00b364, size);
-	} else {
-		/* DMA_IMAGE, VRAM only */
-		if (dma0 & 0x000c0000)
-			return -EINVAL;
-
-		nv_wr32(dev, 0x00b370, base);
-		nv_wr32(dev, 0x00b374, size);
-	}
-
-	return 0;
-}
-
-static int
-nv31_mpeg_isr_chid(struct drm_device *dev, u32 inst)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ctx;
-	unsigned long flags;
-	int i;
-
-	/* hardcode drm channel id on nv3x, so swmthd lookup works */
-	if (dev_priv->card_type < NV_40)
-		return 0;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (i = 0; i < pfifo->channels; i++) {
-		if (!dev_priv->channels.ptr[i])
-			continue;
-
-		ctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_MPEG];
-		if (ctx && ctx->pinst == inst)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return i;
-}
-
-static void
-nv31_vpe_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	nv_wr32(dev, 0x00b008 + (i * 0x10), tile->pitch);
-	nv_wr32(dev, 0x00b004 + (i * 0x10), tile->limit);
-	nv_wr32(dev, 0x00b000 + (i * 0x10), tile->addr);
-}
-
-static void
-nv31_mpeg_isr(struct drm_device *dev)
-{
-	u32 inst = (nv_rd32(dev, 0x00b318) & 0x000fffff) << 4;
-	u32 chid = nv31_mpeg_isr_chid(dev, inst);
-	u32 stat = nv_rd32(dev, 0x00b100);
-	u32 type = nv_rd32(dev, 0x00b230);
-	u32 mthd = nv_rd32(dev, 0x00b234);
-	u32 data = nv_rd32(dev, 0x00b238);
-	u32 show = stat;
-
-	if (stat & 0x01000000) {
-		/* happens on initial binding of the object */
-		if (type == 0x00000020 && mthd == 0x0000) {
-			nv_mask(dev, 0x00b308, 0x00000000, 0x00000000);
-			show &= ~0x01000000;
-		}
-
-		if (type == 0x00000010) {
-			if (!nouveau_gpuobj_mthd_call2(dev, chid, 0x3174, mthd, data))
-				show &= ~0x01000000;
-		}
-	}
-
-	nv_wr32(dev, 0x00b100, stat);
-	nv_wr32(dev, 0x00b230, 0x00000001);
-
-	if (show && nouveau_ratelimit()) {
-		NV_INFO(dev, "PMPEG: Ch %d [0x%08x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
-			chid, inst, stat, type, mthd, data);
-	}
-}
-
-static void
-nv31_vpe_isr(struct drm_device *dev)
-{
-	if (nv_rd32(dev, 0x00b100))
-		nv31_mpeg_isr(dev);
-
-	if (nv_rd32(dev, 0x00b800)) {
-		u32 stat = nv_rd32(dev, 0x00b800);
-		NV_INFO(dev, "PMSRCH: 0x%08x\n", stat);
-		nv_wr32(dev, 0xb800, stat);
-	}
-}
-
-static void
-nv31_mpeg_destroy(struct drm_device *dev, int engine)
-{
-	struct nv31_mpeg_engine *pmpeg = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 0);
-
-	NVOBJ_ENGINE_DEL(dev, MPEG);
-	kfree(pmpeg);
-}
-
-int
-nv31_mpeg_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv31_mpeg_engine *pmpeg;
-
-	pmpeg = kzalloc(sizeof(*pmpeg), GFP_KERNEL);
-	if (!pmpeg)
-		return -ENOMEM;
-	atomic_set(&pmpeg->refcount, 0);
-
-	pmpeg->base.destroy = nv31_mpeg_destroy;
-	pmpeg->base.init = nv31_mpeg_init;
-	pmpeg->base.fini = nv31_mpeg_fini;
-	if (dev_priv->card_type < NV_40) {
-		pmpeg->base.context_new = nv31_mpeg_context_new;
-		pmpeg->base.context_del = nv31_mpeg_context_del;
-	} else {
-		pmpeg->base.context_new = nv40_mpeg_context_new;
-		pmpeg->base.context_del = nv40_mpeg_context_del;
-	}
-	pmpeg->base.object_new = nv31_mpeg_object_new;
-
-	/* ISR vector, PMC_ENABLE bit,  and TILE regs are shared between
-	 * all VPE engines, for this driver's purposes the PMPEG engine
-	 * will be treated as the "master" and handle the global VPE
-	 * bits too
-	 */
-	pmpeg->base.set_tile_region = nv31_vpe_set_tile_region;
-	nouveau_irq_register(dev, 0, nv31_vpe_isr);
-
-	NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
-	NVOBJ_CLASS(dev, 0x3174, MPEG);
-	NVOBJ_MTHD (dev, 0x3174, 0x0190, nv31_mpeg_mthd_dma);
-	NVOBJ_MTHD (dev, 0x3174, 0x01a0, nv31_mpeg_mthd_dma);
-	NVOBJ_MTHD (dev, 0x3174, 0x01b0, nv31_mpeg_mthd_dma);
-
-#if 0
-	NVOBJ_ENGINE_ADD(dev, ME, &pme->base);
-	NVOBJ_CLASS(dev, 0x4075, ME);
-#endif
-	return 0;
-
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_fb.c b/drivers/gpu/drm/nouveau/nv40_fb.c
deleted file mode 100644
index 7fbcb334c09..00000000000
--- a/drivers/gpu/drm/nouveau/nv40_fb.c
+++ /dev/null
@@ -1,163 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-void
-nv40_fb_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	switch (dev_priv->chipset) {
-	case 0x40:
-		nv_wr32(dev, NV10_PFB_TLIMIT(i), tile->limit);
-		nv_wr32(dev, NV10_PFB_TSIZE(i), tile->pitch);
-		nv_wr32(dev, NV10_PFB_TILE(i), tile->addr);
-		break;
-
-	default:
-		nv_wr32(dev, NV40_PFB_TLIMIT(i), tile->limit);
-		nv_wr32(dev, NV40_PFB_TSIZE(i), tile->pitch);
-		nv_wr32(dev, NV40_PFB_TILE(i), tile->addr);
-		break;
-	}
-}
-
-static void
-nv40_fb_init_gart(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
-
-	if (dev_priv->gart_info.type != NOUVEAU_GART_HW) {
-		nv_wr32(dev, 0x100800, 0x00000001);
-		return;
-	}
-
-	nv_wr32(dev, 0x100800, gart->pinst | 0x00000002);
-	nv_mask(dev, 0x10008c, 0x00000100, 0x00000100);
-	nv_wr32(dev, 0x100820, 0x00000000);
-}
-
-static void
-nv44_fb_init_gart(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *gart = dev_priv->gart_info.sg_ctxdma;
-	u32 vinst;
-
-	if (dev_priv->gart_info.type != NOUVEAU_GART_HW) {
-		nv_wr32(dev, 0x100850, 0x80000000);
-		nv_wr32(dev, 0x100800, 0x00000001);
-		return;
-	}
-
-	/* calculate vram address of this PRAMIN block, object
-	 * must be allocated on 512KiB alignment, and not exceed
-	 * a total size of 512KiB for this to work correctly
-	 */
-	vinst  = nv_rd32(dev, 0x10020c);
-	vinst -= ((gart->pinst >> 19) + 1) << 19;
-
-	nv_wr32(dev, 0x100850, 0x80000000);
-	nv_wr32(dev, 0x100818, dev_priv->gart_info.dummy.addr);
-
-	nv_wr32(dev, 0x100804, dev_priv->gart_info.aper_size);
-	nv_wr32(dev, 0x100850, 0x00008000);
-	nv_mask(dev, 0x10008c, 0x00000200, 0x00000200);
-	nv_wr32(dev, 0x100820, 0x00000000);
-	nv_wr32(dev, 0x10082c, 0x00000001);
-	nv_wr32(dev, 0x100800, vinst | 0x00000010);
-}
-
-int
-nv40_fb_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	/* 0x001218 is actually present on a few other NV4X I looked at,
-	 * and even contains sane values matching 0x100474.  From looking
-	 * at various vbios images however, this isn't the case everywhere.
-	 * So, I chose to use the same regs I've seen NVIDIA reading around
-	 * the memory detection, hopefully that'll get us the right numbers
-	 */
-	if (dev_priv->chipset == 0x40) {
-		u32 pbus1218 = nv_rd32(dev, 0x001218);
-		switch (pbus1218 & 0x00000300) {
-		case 0x00000000: dev_priv->vram_type = NV_MEM_TYPE_SDRAM; break;
-		case 0x00000100: dev_priv->vram_type = NV_MEM_TYPE_DDR1; break;
-		case 0x00000200: dev_priv->vram_type = NV_MEM_TYPE_GDDR3; break;
-		case 0x00000300: dev_priv->vram_type = NV_MEM_TYPE_DDR2; break;
-		}
-	} else
-	if (dev_priv->chipset == 0x49 || dev_priv->chipset == 0x4b) {
-		u32 pfb914 = nv_rd32(dev, 0x100914);
-		switch (pfb914 & 0x00000003) {
-		case 0x00000000: dev_priv->vram_type = NV_MEM_TYPE_DDR1; break;
-		case 0x00000001: dev_priv->vram_type = NV_MEM_TYPE_DDR2; break;
-		case 0x00000002: dev_priv->vram_type = NV_MEM_TYPE_GDDR3; break;
-		case 0x00000003: break;
-		}
-	} else
-	if (dev_priv->chipset != 0x4e) {
-		u32 pfb474 = nv_rd32(dev, 0x100474);
-		if (pfb474 & 0x00000004)
-			dev_priv->vram_type = NV_MEM_TYPE_GDDR3;
-		if (pfb474 & 0x00000002)
-			dev_priv->vram_type = NV_MEM_TYPE_DDR2;
-		if (pfb474 & 0x00000001)
-			dev_priv->vram_type = NV_MEM_TYPE_DDR1;
-	} else {
-		dev_priv->vram_type = NV_MEM_TYPE_STOLEN;
-	}
-
-	dev_priv->vram_size = nv_rd32(dev, 0x10020c) & 0xff000000;
-	return 0;
-}
-
-int
-nv40_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	uint32_t tmp;
-	int i;
-
-	if (dev_priv->chipset != 0x40 && dev_priv->chipset != 0x45) {
-		if (nv44_graph_class(dev))
-			nv44_fb_init_gart(dev);
-		else
-			nv40_fb_init_gart(dev);
-	}
-
-	switch (dev_priv->chipset) {
-	case 0x40:
-	case 0x45:
-		tmp = nv_rd32(dev, NV10_PFB_CLOSE_PAGE2);
-		nv_wr32(dev, NV10_PFB_CLOSE_PAGE2, tmp & ~(1 << 15));
-		pfb->num_tiles = NV10_PFB_TILE__SIZE;
-		break;
-	case 0x46: /* G72 */
-	case 0x47: /* G70 */
-	case 0x49: /* G71 */
-	case 0x4b: /* G73 */
-	case 0x4c: /* C51 (G7X version) */
-		pfb->num_tiles = NV40_PFB_TILE__SIZE_1;
-		break;
-	default:
-		pfb->num_tiles = NV40_PFB_TILE__SIZE_0;
-		break;
-	}
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < pfb->num_tiles; i++)
-		pfb->set_tile_region(dev, i);
-
-	return 0;
-}
-
-void
-nv40_fb_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_fifo.c b/drivers/gpu/drm/nouveau/nv40_fifo.c
deleted file mode 100644
index cdc818479b0..00000000000
--- a/drivers/gpu/drm/nouveau/nv40_fifo.c
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_util.h"
-#include "nouveau_ramht.h"
-
-static struct ramfc_desc {
-	unsigned bits:6;
-	unsigned ctxs:5;
-	unsigned ctxp:8;
-	unsigned regs:5;
-	unsigned regp;
-} nv40_ramfc[] = {
-	{ 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
-	{ 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
-	{ 32,  0, 0x08,  0, NV10_PFIFO_CACHE1_REF_CNT },
-	{ 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
-	{ 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
-	{ 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_DMA_STATE },
-	{ 28,  0, 0x18,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
-	{  2, 28, 0x18, 28, 0x002058 },
-	{ 32,  0, 0x1c,  0, NV04_PFIFO_CACHE1_ENGINE },
-	{ 32,  0, 0x20,  0, NV04_PFIFO_CACHE1_PULL1 },
-	{ 32,  0, 0x24,  0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE },
-	{ 32,  0, 0x28,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP },
-	{ 32,  0, 0x2c,  0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT },
-	{ 32,  0, 0x30,  0, NV10_PFIFO_CACHE1_SEMAPHORE },
-	{ 32,  0, 0x34,  0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE },
-	{ 32,  0, 0x38,  0, NV40_PFIFO_GRCTX_INSTANCE },
-	{ 17,  0, 0x3c,  0, NV04_PFIFO_DMA_TIMESLICE },
-	{ 32,  0, 0x40,  0, 0x0032e4 },
-	{ 32,  0, 0x44,  0, 0x0032e8 },
-	{ 32,  0, 0x4c,  0, 0x002088 },
-	{ 32,  0, 0x50,  0, 0x003300 },
-	{ 32,  0, 0x54,  0, 0x00330c },
-	{}
-};
-
-struct nv40_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct ramfc_desc *ramfc_desc;
-};
-
-struct nv40_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *ramfc;
-};
-
-static int
-nv40_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv40_fifo_priv *priv = nv_engine(dev, engine);
-	struct nv40_fifo_chan *fctx;
-	unsigned long flags;
-	int ret;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	/* map channel control registers */
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV03_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/* initialise default fifo context */
-	ret = nouveau_gpuobj_new_fake(dev, dev_priv->ramfc->pinst +
-				      chan->id * 128, ~0, 128,
-				      NVOBJ_FLAG_ZERO_ALLOC |
-				      NVOBJ_FLAG_ZERO_FREE, &fctx->ramfc);
-	if (ret)
-		goto error;
-
-	nv_wo32(fctx->ramfc, 0x00, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x04, chan->pushbuf_base);
-	nv_wo32(fctx->ramfc, 0x0c, chan->pushbuf->pinst >> 4);
-	nv_wo32(fctx->ramfc, 0x18, 0x30000000 |
-				   NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
-				   NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
-#ifdef __BIG_ENDIAN
-				   NV_PFIFO_CACHE1_BIG_ENDIAN |
-#endif
-				   NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
-	nv_wo32(fctx->ramfc, 0x3c, 0x0001ffff);
-
-	/* enable dma mode on the channel */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, NV04_PFIFO_MODE, (1 << chan->id), (1 << chan->id));
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/*XXX: remove this later, need fifo engine context commit hook */
-	nouveau_gpuobj_ref(fctx->ramfc, &chan->ramfc);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static int
-nv40_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv40_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, 0);
-	nv_mask(dev, NV03_PMC_ENABLE, NV_PMC_ENABLE_PFIFO, NV_PMC_ENABLE_PFIFO);
-
-	nv_wr32(dev, 0x002040, 0x000000ff);
-	nv_wr32(dev, 0x002044, 0x2101ffff);
-	nv_wr32(dev, 0x002058, 0x00000001);
-
-	nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
-				       ((dev_priv->ramht->bits - 9) << 16) |
-				       (dev_priv->ramht->gpuobj->pinst >> 8));
-	nv_wr32(dev, NV03_PFIFO_RAMRO, dev_priv->ramro->pinst >> 8);
-
-	switch (dev_priv->chipset) {
-	case 0x47:
-	case 0x49:
-	case 0x4b:
-		nv_wr32(dev, 0x002230, 0x00000001);
-	case 0x40:
-	case 0x41:
-	case 0x42:
-	case 0x43:
-	case 0x45:
-	case 0x48:
-		nv_wr32(dev, 0x002220, 0x00030002);
-		break;
-	default:
-		nv_wr32(dev, 0x002230, 0x00000000);
-		nv_wr32(dev, 0x002220, ((dev_priv->vram_size - 512 * 1024 +
-					 dev_priv->ramfc->pinst) >> 16) |
-				       0x00030000);
-		break;
-	}
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, priv->base.channels);
-
-	nv_wr32(dev, NV03_PFIFO_INTR_0, 0xffffffff);
-	nv_wr32(dev, NV03_PFIFO_INTR_EN_0, 0xffffffff);
-
-	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH0, 1);
-	nv_wr32(dev, NV04_PFIFO_CACHE1_PULL0, 1);
-	nv_wr32(dev, NV03_PFIFO_CACHES, 1);
-
-	for (i = 0; i < priv->base.channels; i++) {
-		if (dev_priv->channels.ptr[i])
-			nv_mask(dev, NV04_PFIFO_MODE, (1 << i), (1 << i));
-	}
-
-	return 0;
-}
-
-int
-nv40_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv40_fifo_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv04_fifo_destroy;
-	priv->base.base.init = nv40_fifo_init;
-	priv->base.base.fini = nv04_fifo_fini;
-	priv->base.base.context_new = nv40_fifo_context_new;
-	priv->base.base.context_del = nv04_fifo_context_del;
-	priv->base.channels = 31;
-	priv->ramfc_desc = nv40_ramfc;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_graph.c b/drivers/gpu/drm/nouveau/nv40_graph.c
deleted file mode 100644
index aa9e2df64a2..00000000000
--- a/drivers/gpu/drm/nouveau/nv40_graph.c
+++ /dev/null
@@ -1,467 +0,0 @@
-/*
- * Copyright (C) 2007 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-
-struct nv40_graph_engine {
-	struct nouveau_exec_engine base;
-	u32 grctx_size;
-};
-
-static int
-nv40_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv40_graph_engine *pgraph = nv_engine(chan->dev, engine);
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *grctx = NULL;
-	unsigned long flags;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 16,
-				 NVOBJ_FLAG_ZERO_ALLOC, &grctx);
-	if (ret)
-		return ret;
-
-	/* Initialise default context values */
-	nv40_grctx_fill(dev, grctx);
-	nv_wo32(grctx, 0, grctx->vinst);
-
-	/* init grctx pointer in ramfc, and on PFIFO if channel is
-	 * already active there
-	 */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_wo32(chan->ramfc, 0x38, grctx->vinst >> 4);
-	nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
-	if ((nv_rd32(dev, 0x003204) & 0x0000001f) == chan->id)
-		nv_wr32(dev, 0x0032e0, grctx->vinst >> 4);
-	nv_mask(dev, 0x002500, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	chan->engctx[engine] = grctx;
-	return 0;
-}
-
-static void
-nv40_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nouveau_gpuobj *grctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 inst = 0x01000000 | (grctx->pinst >> 4);
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x400720, 0x00000000, 0x00000001);
-	if (nv_rd32(dev, 0x40032c) == inst)
-		nv_mask(dev, 0x40032c, 0x01000000, 0x00000000);
-	if (nv_rd32(dev, 0x400330) == inst)
-		nv_mask(dev, 0x400330, 0x01000000, 0x00000000);
-	nv_mask(dev, 0x400720, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* Free the context resources */
-	nouveau_gpuobj_ref(NULL, &grctx);
-	chan->engctx[engine] = NULL;
-}
-
-int
-nv40_graph_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 20, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 1;
-	obj->class  = class;
-
-	nv_wo32(obj, 0x00, class);
-	nv_wo32(obj, 0x04, 0x00000000);
-#ifndef __BIG_ENDIAN
-	nv_wo32(obj, 0x08, 0x00000000);
-#else
-	nv_wo32(obj, 0x08, 0x01000000);
-#endif
-	nv_wo32(obj, 0x0c, 0x00000000);
-	nv_wo32(obj, 0x10, 0x00000000);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static void
-nv40_graph_set_tile_region(struct drm_device *dev, int i)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_tile_reg *tile = &dev_priv->tile.reg[i];
-
-	switch (dev_priv->chipset) {
-	case 0x40:
-	case 0x41: /* guess */
-	case 0x42:
-	case 0x43:
-	case 0x45: /* guess */
-	case 0x4e:
-		nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
-		nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
-		nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
-		nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
-		nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
-		nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
-		break;
-	case 0x44:
-	case 0x4a:
-		nv_wr32(dev, NV20_PGRAPH_TSIZE(i), tile->pitch);
-		nv_wr32(dev, NV20_PGRAPH_TLIMIT(i), tile->limit);
-		nv_wr32(dev, NV20_PGRAPH_TILE(i), tile->addr);
-		break;
-	case 0x46:
-	case 0x47:
-	case 0x49:
-	case 0x4b:
-	case 0x4c:
-	case 0x67:
-	default:
-		nv_wr32(dev, NV47_PGRAPH_TSIZE(i), tile->pitch);
-		nv_wr32(dev, NV47_PGRAPH_TLIMIT(i), tile->limit);
-		nv_wr32(dev, NV47_PGRAPH_TILE(i), tile->addr);
-		nv_wr32(dev, NV40_PGRAPH_TSIZE1(i), tile->pitch);
-		nv_wr32(dev, NV40_PGRAPH_TLIMIT1(i), tile->limit);
-		nv_wr32(dev, NV40_PGRAPH_TILE1(i), tile->addr);
-		break;
-	}
-}
-
-/*
- * G70		0x47
- * G71		0x49
- * NV45		0x48
- * G72[M]	0x46
- * G73		0x4b
- * C51_G7X	0x4c
- * C51		0x4e
- */
-int
-nv40_graph_init(struct drm_device *dev, int engine)
-{
-	struct nv40_graph_engine *pgraph = nv_engine(dev, engine);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	uint32_t vramsz;
-	int i, j;
-
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) &
-			~NV_PMC_ENABLE_PGRAPH);
-	nv_wr32(dev, NV03_PMC_ENABLE, nv_rd32(dev, NV03_PMC_ENABLE) |
-			 NV_PMC_ENABLE_PGRAPH);
-
-	/* generate and upload context program */
-	nv40_grctx_init(dev, &pgraph->grctx_size);
-
-	/* No context present currently */
-	nv_wr32(dev, NV40_PGRAPH_CTXCTL_CUR, 0x00000000);
-
-	nv_wr32(dev, NV03_PGRAPH_INTR   , 0xFFFFFFFF);
-	nv_wr32(dev, NV40_PGRAPH_INTR_EN, 0xFFFFFFFF);
-
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0xFFFFFFFF);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_0, 0x00000000);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_1, 0x401287c0);
-	nv_wr32(dev, NV04_PGRAPH_DEBUG_3, 0xe0de8055);
-	nv_wr32(dev, NV10_PGRAPH_DEBUG_4, 0x00008000);
-	nv_wr32(dev, NV04_PGRAPH_LIMIT_VIOL_PIX, 0x00be3c5f);
-
-	nv_wr32(dev, NV10_PGRAPH_CTX_CONTROL, 0x10010100);
-	nv_wr32(dev, NV10_PGRAPH_STATE      , 0xFFFFFFFF);
-
-	j = nv_rd32(dev, 0x1540) & 0xff;
-	if (j) {
-		for (i = 0; !(j & 1); j >>= 1, i++)
-			;
-		nv_wr32(dev, 0x405000, i);
-	}
-
-	if (dev_priv->chipset == 0x40) {
-		nv_wr32(dev, 0x4009b0, 0x83280fff);
-		nv_wr32(dev, 0x4009b4, 0x000000a0);
-	} else {
-		nv_wr32(dev, 0x400820, 0x83280eff);
-		nv_wr32(dev, 0x400824, 0x000000a0);
-	}
-
-	switch (dev_priv->chipset) {
-	case 0x40:
-	case 0x45:
-		nv_wr32(dev, 0x4009b8, 0x0078e366);
-		nv_wr32(dev, 0x4009bc, 0x0000014c);
-		break;
-	case 0x41:
-	case 0x42: /* pciid also 0x00Cx */
-	/* case 0x0120: XXX (pciid) */
-		nv_wr32(dev, 0x400828, 0x007596ff);
-		nv_wr32(dev, 0x40082c, 0x00000108);
-		break;
-	case 0x43:
-		nv_wr32(dev, 0x400828, 0x0072cb77);
-		nv_wr32(dev, 0x40082c, 0x00000108);
-		break;
-	case 0x44:
-	case 0x46: /* G72 */
-	case 0x4a:
-	case 0x4c: /* G7x-based C51 */
-	case 0x4e:
-		nv_wr32(dev, 0x400860, 0);
-		nv_wr32(dev, 0x400864, 0);
-		break;
-	case 0x47: /* G70 */
-	case 0x49: /* G71 */
-	case 0x4b: /* G73 */
-		nv_wr32(dev, 0x400828, 0x07830610);
-		nv_wr32(dev, 0x40082c, 0x0000016A);
-		break;
-	default:
-		break;
-	}
-
-	nv_wr32(dev, 0x400b38, 0x2ffff800);
-	nv_wr32(dev, 0x400b3c, 0x00006000);
-
-	/* Tiling related stuff. */
-	switch (dev_priv->chipset) {
-	case 0x44:
-	case 0x4a:
-		nv_wr32(dev, 0x400bc4, 0x1003d888);
-		nv_wr32(dev, 0x400bbc, 0xb7a7b500);
-		break;
-	case 0x46:
-		nv_wr32(dev, 0x400bc4, 0x0000e024);
-		nv_wr32(dev, 0x400bbc, 0xb7a7b520);
-		break;
-	case 0x4c:
-	case 0x4e:
-	case 0x67:
-		nv_wr32(dev, 0x400bc4, 0x1003d888);
-		nv_wr32(dev, 0x400bbc, 0xb7a7b540);
-		break;
-	default:
-		break;
-	}
-
-	/* Turn all the tiling regions off. */
-	for (i = 0; i < pfb->num_tiles; i++)
-		nv40_graph_set_tile_region(dev, i);
-
-	/* begin RAM config */
-	vramsz = pci_resource_len(dev->pdev, 0) - 1;
-	switch (dev_priv->chipset) {
-	case 0x40:
-		nv_wr32(dev, 0x4009A4, nv_rd32(dev, NV04_PFB_CFG0));
-		nv_wr32(dev, 0x4009A8, nv_rd32(dev, NV04_PFB_CFG1));
-		nv_wr32(dev, 0x4069A4, nv_rd32(dev, NV04_PFB_CFG0));
-		nv_wr32(dev, 0x4069A8, nv_rd32(dev, NV04_PFB_CFG1));
-		nv_wr32(dev, 0x400820, 0);
-		nv_wr32(dev, 0x400824, 0);
-		nv_wr32(dev, 0x400864, vramsz);
-		nv_wr32(dev, 0x400868, vramsz);
-		break;
-	default:
-		switch (dev_priv->chipset) {
-		case 0x41:
-		case 0x42:
-		case 0x43:
-		case 0x45:
-		case 0x4e:
-		case 0x44:
-		case 0x4a:
-			nv_wr32(dev, 0x4009F0, nv_rd32(dev, NV04_PFB_CFG0));
-			nv_wr32(dev, 0x4009F4, nv_rd32(dev, NV04_PFB_CFG1));
-			break;
-		default:
-			nv_wr32(dev, 0x400DF0, nv_rd32(dev, NV04_PFB_CFG0));
-			nv_wr32(dev, 0x400DF4, nv_rd32(dev, NV04_PFB_CFG1));
-			break;
-		}
-		nv_wr32(dev, 0x4069F0, nv_rd32(dev, NV04_PFB_CFG0));
-		nv_wr32(dev, 0x4069F4, nv_rd32(dev, NV04_PFB_CFG1));
-		nv_wr32(dev, 0x400840, 0);
-		nv_wr32(dev, 0x400844, 0);
-		nv_wr32(dev, 0x4008A0, vramsz);
-		nv_wr32(dev, 0x4008A4, vramsz);
-		break;
-	}
-
-	return 0;
-}
-
-static int
-nv40_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	u32 inst = nv_rd32(dev, 0x40032c);
-	if (inst & 0x01000000) {
-		nv_wr32(dev, 0x400720, 0x00000000);
-		nv_wr32(dev, 0x400784, inst);
-		nv_mask(dev, 0x400310, 0x00000020, 0x00000020);
-		nv_mask(dev, 0x400304, 0x00000001, 0x00000001);
-		if (!nv_wait(dev, 0x400300, 0x00000001, 0x00000000)) {
-			u32 insn = nv_rd32(dev, 0x400308);
-			NV_ERROR(dev, "PGRAPH: ctxprog timeout 0x%08x\n", insn);
-		}
-		nv_mask(dev, 0x40032c, 0x01000000, 0x00000000);
-	}
-	return 0;
-}
-
-static int
-nv40_graph_isr_chid(struct drm_device *dev, u32 inst)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *grctx;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (i = 0; i < pfifo->channels; i++) {
-		if (!dev_priv->channels.ptr[i])
-			continue;
-		grctx = dev_priv->channels.ptr[i]->engctx[NVOBJ_ENGINE_GR];
-
-		if (grctx && grctx->pinst == inst)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return i;
-}
-
-static void
-nv40_graph_isr(struct drm_device *dev)
-{
-	u32 stat;
-
-	while ((stat = nv_rd32(dev, NV03_PGRAPH_INTR))) {
-		u32 nsource = nv_rd32(dev, NV03_PGRAPH_NSOURCE);
-		u32 nstatus = nv_rd32(dev, NV03_PGRAPH_NSTATUS);
-		u32 inst = (nv_rd32(dev, 0x40032c) & 0x000fffff) << 4;
-		u32 chid = nv40_graph_isr_chid(dev, inst);
-		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-		u32 subc = (addr & 0x00070000) >> 16;
-		u32 mthd = (addr & 0x00001ffc);
-		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-		u32 class = nv_rd32(dev, 0x400160 + subc * 4) & 0xffff;
-		u32 show = stat;
-
-		if (stat & NV_PGRAPH_INTR_ERROR) {
-			if (nsource & NV03_PGRAPH_NSOURCE_ILLEGAL_MTHD) {
-				if (!nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data))
-					show &= ~NV_PGRAPH_INTR_ERROR;
-			} else
-			if (nsource & NV03_PGRAPH_NSOURCE_DMA_VTX_PROTECTION) {
-				nv_mask(dev, 0x402000, 0, 0);
-			}
-		}
-
-		nv_wr32(dev, NV03_PGRAPH_INTR, stat);
-		nv_wr32(dev, NV04_PGRAPH_FIFO, 0x00000001);
-
-		if (show && nouveau_ratelimit()) {
-			NV_INFO(dev, "PGRAPH -");
-			nouveau_bitfield_print(nv10_graph_intr, show);
-			printk(" nsource:");
-			nouveau_bitfield_print(nv04_graph_nsource, nsource);
-			printk(" nstatus:");
-			nouveau_bitfield_print(nv10_graph_nstatus, nstatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - ch %d (0x%08x) subc %d "
-				     "class 0x%04x mthd 0x%04x data 0x%08x\n",
-				chid, inst, subc, class, mthd, data);
-		}
-	}
-}
-
-static void
-nv40_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nv40_graph_engine *pgraph = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 12);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-	kfree(pgraph);
-}
-
-int
-nv40_graph_create(struct drm_device *dev)
-{
-	struct nv40_graph_engine *pgraph;
-
-	pgraph = kzalloc(sizeof(*pgraph), GFP_KERNEL);
-	if (!pgraph)
-		return -ENOMEM;
-
-	pgraph->base.destroy = nv40_graph_destroy;
-	pgraph->base.init = nv40_graph_init;
-	pgraph->base.fini = nv40_graph_fini;
-	pgraph->base.context_new = nv40_graph_context_new;
-	pgraph->base.context_del = nv40_graph_context_del;
-	pgraph->base.object_new = nv40_graph_object_new;
-	pgraph->base.set_tile_region = nv40_graph_set_tile_region;
-
-	NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
-	nouveau_irq_register(dev, 12, nv40_graph_isr);
-
-	NVOBJ_CLASS(dev, 0x0030, GR); /* null */
-	NVOBJ_CLASS(dev, 0x0039, GR); /* m2mf */
-	NVOBJ_CLASS(dev, 0x004a, GR); /* gdirect */
-	NVOBJ_CLASS(dev, 0x009f, GR); /* imageblit (nv12) */
-	NVOBJ_CLASS(dev, 0x008a, GR); /* ifc */
-	NVOBJ_CLASS(dev, 0x0089, GR); /* sifm */
-	NVOBJ_CLASS(dev, 0x3089, GR); /* sifm (nv40) */
-	NVOBJ_CLASS(dev, 0x0062, GR); /* surf2d */
-	NVOBJ_CLASS(dev, 0x3062, GR); /* surf2d (nv40) */
-	NVOBJ_CLASS(dev, 0x0043, GR); /* rop */
-	NVOBJ_CLASS(dev, 0x0012, GR); /* beta1 */
-	NVOBJ_CLASS(dev, 0x0072, GR); /* beta4 */
-	NVOBJ_CLASS(dev, 0x0019, GR); /* cliprect */
-	NVOBJ_CLASS(dev, 0x0044, GR); /* pattern */
-	NVOBJ_CLASS(dev, 0x309e, GR); /* swzsurf */
-
-	/* curie */
-	if (nv44_graph_class(dev))
-		NVOBJ_CLASS(dev, 0x4497, GR);
-	else
-		NVOBJ_CLASS(dev, 0x4097, GR);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_mc.c b/drivers/gpu/drm/nouveau/nv40_mc.c
deleted file mode 100644
index 03c0d4c3f35..00000000000
--- a/drivers/gpu/drm/nouveau/nv40_mc.c
+++ /dev/null
@@ -1,28 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-int
-nv40_mc_init(struct drm_device *dev)
-{
-	/* Power up everything, resetting each individual unit will
-	 * be done later if needed.
-	 */
-	nv_wr32(dev, NV03_PMC_ENABLE, 0xFFFFFFFF);
-
-	if (nv44_graph_class(dev)) {
-		u32 tmp = nv_rd32(dev, NV04_PFB_FIFO_DATA);
-		nv_wr32(dev, NV40_PMC_1700, tmp);
-		nv_wr32(dev, NV40_PMC_1704, 0);
-		nv_wr32(dev, NV40_PMC_1708, 0);
-		nv_wr32(dev, NV40_PMC_170C, tmp);
-	}
-
-	return 0;
-}
-
-void
-nv40_mc_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_pm.c b/drivers/gpu/drm/nouveau/nv40_pm.c
index e66273aff49..3382064c7f3 100644
--- a/drivers/gpu/drm/nouveau/nv40_pm.c
+++ b/drivers/gpu/drm/nouveau/nv40_pm.c
@@ -22,19 +22,25 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
 #include "nouveau_bios.h"
 #include "nouveau_pm.h"
 #include "nouveau_hw.h"
-#include "nouveau_fifo.h"
+
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
 
 #define min2(a,b) ((a) < (b) ? (a) : (b))
 
 static u32
 read_pll_1(struct drm_device *dev, u32 reg)
 {
-	u32 ctrl = nv_rd32(dev, reg + 0x00);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ctrl = nv_rd32(device, reg + 0x00);
 	int P = (ctrl & 0x00070000) >> 16;
 	int N = (ctrl & 0x0000ff00) >> 8;
 	int M = (ctrl & 0x000000ff) >> 0;
@@ -49,8 +55,9 @@ read_pll_1(struct drm_device *dev, u32 reg)
 static u32
 read_pll_2(struct drm_device *dev, u32 reg)
 {
-	u32 ctrl = nv_rd32(dev, reg + 0x00);
-	u32 coef = nv_rd32(dev, reg + 0x04);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ctrl = nv_rd32(device, reg + 0x00);
+	u32 coef = nv_rd32(device, reg + 0x04);
 	int N2 = (coef & 0xff000000) >> 24;
 	int M2 = (coef & 0x00ff0000) >> 16;
 	int N1 = (coef & 0x0000ff00) >> 8;
@@ -89,7 +96,8 @@ read_clk(struct drm_device *dev, u32 src)
 int
 nv40_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	u32 ctrl = nv_rd32(dev, 0x00c040);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ctrl = nv_rd32(device, 0x00c040);
 
 	perflvl->core   = read_clk(dev, (ctrl & 0x00000003) >> 0);
 	perflvl->shader = read_clk(dev, (ctrl & 0x00000030) >> 4);
@@ -107,27 +115,30 @@ struct nv40_pm_state {
 };
 
 static int
-nv40_calc_pll(struct drm_device *dev, u32 reg, struct pll_lims *pll,
+nv40_calc_pll(struct drm_device *dev, u32 reg, struct nvbios_pll *pll,
 	      u32 clk, int *N1, int *M1, int *N2, int *M2, int *log2P)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nouveau_clock *pclk = nouveau_clock(device);
 	struct nouveau_pll_vals coef;
 	int ret;
 
-	ret = get_pll_limits(dev, reg, pll);
+	ret = nvbios_pll_parse(bios, reg, pll);
 	if (ret)
 		return ret;
 
-	if (clk < pll->vco1.maxfreq)
-		pll->vco2.maxfreq = 0;
+	if (clk < pll->vco1.max_freq)
+		pll->vco2.max_freq = 0;
 
-	ret = nouveau_calc_pll_mnp(dev, pll, clk, &coef);
+	pclk->pll_calc(pclk, pll, clk, &coef);
 	if (ret == 0)
 		return -ERANGE;
 
 	*N1 = coef.N1;
 	*M1 = coef.M1;
 	if (N2 && M2) {
-		if (pll->vco2.maxfreq) {
+		if (pll->vco2.max_freq) {
 			*N2 = coef.N2;
 			*M2 = coef.M2;
 		} else {
@@ -143,7 +154,7 @@ void *
 nv40_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
 	struct nv40_pm_state *info;
-	struct pll_lims pll;
+	struct nvbios_pll pll;
 	int N1, N2, M1, M2, log2P;
 	int ret;
 
@@ -191,7 +202,7 @@ nv40_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 		goto out;
 
 	info->mpll_ctrl  = 0x80000000 | (log2P << 16);
-	info->mpll_ctrl |= min2(pll.log2p_bias + log2P, pll.max_log2p) << 20;
+	info->mpll_ctrl |= min2(pll.bias_p + log2P, pll.max_p) << 20;
 	if (N2 == M2) {
 		info->mpll_ctrl |= 0x00000100;
 		info->mpll_coef  = (N1 << 8) | M1;
@@ -212,12 +223,13 @@ static bool
 nv40_pm_gr_idle(void *data)
 {
 	struct drm_device *dev = data;
+	struct nouveau_device *device = nouveau_dev(dev);
 
-	if ((nv_rd32(dev, 0x400760) & 0x000000f0) >> 4 !=
-	    (nv_rd32(dev, 0x400760) & 0x0000000f))
+	if ((nv_rd32(device, 0x400760) & 0x000000f0) >> 4 !=
+	    (nv_rd32(device, 0x400760) & 0x0000000f))
 		return false;
 
-	if (nv_rd32(dev, 0x400700))
+	if (nv_rd32(device, 0x400700))
 		return false;
 
 	return true;
@@ -226,7 +238,9 @@ nv40_pm_gr_idle(void *data)
 int
 nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_fifo *pfifo = nouveau_fifo(device);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv40_pm_state *info = pre_state;
 	unsigned long flags;
 	struct bit_entry M;
@@ -236,12 +250,12 @@ nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
 
 	/* determine which CRTCs are active, fetch VGA_SR1 for each */
 	for (i = 0; i < 2; i++) {
-		u32 vbl = nv_rd32(dev, 0x600808 + (i * 0x2000));
+		u32 vbl = nv_rd32(device, 0x600808 + (i * 0x2000));
 		u32 cnt = 0;
 		do {
-			if (vbl != nv_rd32(dev, 0x600808 + (i * 0x2000))) {
-				nv_wr08(dev, 0x0c03c4 + (i * 0x2000), 0x01);
-				sr1[i] = nv_rd08(dev, 0x0c03c5 + (i * 0x2000));
+			if (vbl != nv_rd32(device, 0x600808 + (i * 0x2000))) {
+				nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
+				sr1[i] = nv_rd08(device, 0x0c03c5 + (i * 0x2000));
 				if (!(sr1[i] & 0x20))
 					crtc_mask |= (1 << i);
 				break;
@@ -251,28 +265,20 @@ nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
 	}
 
 	/* halt and idle engines */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x002500, 0x00000001, 0x00000000);
-	if (!nv_wait(dev, 0x002500, 0x00000010, 0x00000000))
-		goto resume;
-	nv_mask(dev, 0x003220, 0x00000001, 0x00000000);
-	if (!nv_wait(dev, 0x003220, 0x00000010, 0x00000000))
-		goto resume;
-	nv_mask(dev, 0x003200, 0x00000001, 0x00000000);
-	nv04_fifo_cache_pull(dev, false);
+	pfifo->pause(pfifo, &flags);
 
-	if (!nv_wait_cb(dev, nv40_pm_gr_idle, dev))
+	if (!nv_wait_cb(device, nv40_pm_gr_idle, dev))
 		goto resume;
 
 	ret = 0;
 
 	/* set engine clocks */
-	nv_mask(dev, 0x00c040, 0x00000333, 0x00000000);
-	nv_wr32(dev, 0x004004, info->npll_coef);
-	nv_mask(dev, 0x004000, 0xc0070100, info->npll_ctrl);
-	nv_mask(dev, 0x004008, 0xc007ffff, info->spll);
+	nv_mask(device, 0x00c040, 0x00000333, 0x00000000);
+	nv_wr32(device, 0x004004, info->npll_coef);
+	nv_mask(device, 0x004000, 0xc0070100, info->npll_ctrl);
+	nv_mask(device, 0x004008, 0xc007ffff, info->spll);
 	mdelay(5);
-	nv_mask(dev, 0x00c040, 0x00000333, info->ctrl);
+	nv_mask(device, 0x00c040, 0x00000333, info->ctrl);
 
 	if (!info->mpll_ctrl)
 		goto resume;
@@ -281,52 +287,52 @@ nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
 	for (i = 0; i < 2; i++) {
 		if (!(crtc_mask & (1 << i)))
 			continue;
-		nv_wait(dev, 0x600808 + (i * 0x2000), 0x00010000, 0x00000000);
-		nv_wait(dev, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
-		nv_wr08(dev, 0x0c03c4 + (i * 0x2000), 0x01);
-		nv_wr08(dev, 0x0c03c5 + (i * 0x2000), sr1[i] | 0x20);
+		nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00000000);
+		nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
+		nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
+		nv_wr08(device, 0x0c03c5 + (i * 0x2000), sr1[i] | 0x20);
 	}
 
 	/* prepare ram for reclocking */
-	nv_wr32(dev, 0x1002d4, 0x00000001); /* precharge */
-	nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-	nv_wr32(dev, 0x1002d0, 0x00000001); /* refresh */
-	nv_mask(dev, 0x100210, 0x80000000, 0x00000000); /* no auto refresh */
-	nv_wr32(dev, 0x1002dc, 0x00000001); /* enable self-refresh */
+	nv_wr32(device, 0x1002d4, 0x00000001); /* precharge */
+	nv_wr32(device, 0x1002d0, 0x00000001); /* refresh */
+	nv_wr32(device, 0x1002d0, 0x00000001); /* refresh */
+	nv_mask(device, 0x100210, 0x80000000, 0x00000000); /* no auto refresh */
+	nv_wr32(device, 0x1002dc, 0x00000001); /* enable self-refresh */
 
 	/* change the PLL of each memory partition */
-	nv_mask(dev, 0x00c040, 0x0000c000, 0x00000000);
-	switch (dev_priv->chipset) {
+	nv_mask(device, 0x00c040, 0x0000c000, 0x00000000);
+	switch (nv_device(drm->device)->chipset) {
 	case 0x40:
 	case 0x45:
 	case 0x41:
 	case 0x42:
 	case 0x47:
-		nv_mask(dev, 0x004044, 0xc0771100, info->mpll_ctrl);
-		nv_mask(dev, 0x00402c, 0xc0771100, info->mpll_ctrl);
-		nv_wr32(dev, 0x004048, info->mpll_coef);
-		nv_wr32(dev, 0x004030, info->mpll_coef);
+		nv_mask(device, 0x004044, 0xc0771100, info->mpll_ctrl);
+		nv_mask(device, 0x00402c, 0xc0771100, info->mpll_ctrl);
+		nv_wr32(device, 0x004048, info->mpll_coef);
+		nv_wr32(device, 0x004030, info->mpll_coef);
 	case 0x43:
 	case 0x49:
 	case 0x4b:
-		nv_mask(dev, 0x004038, 0xc0771100, info->mpll_ctrl);
-		nv_wr32(dev, 0x00403c, info->mpll_coef);
+		nv_mask(device, 0x004038, 0xc0771100, info->mpll_ctrl);
+		nv_wr32(device, 0x00403c, info->mpll_coef);
 	default:
-		nv_mask(dev, 0x004020, 0xc0771100, info->mpll_ctrl);
-		nv_wr32(dev, 0x004024, info->mpll_coef);
+		nv_mask(device, 0x004020, 0xc0771100, info->mpll_ctrl);
+		nv_wr32(device, 0x004024, info->mpll_coef);
 		break;
 	}
 	udelay(100);
-	nv_mask(dev, 0x00c040, 0x0000c000, 0x0000c000);
+	nv_mask(device, 0x00c040, 0x0000c000, 0x0000c000);
 
 	/* re-enable normal operation of memory controller */
-	nv_wr32(dev, 0x1002dc, 0x00000000);
-	nv_mask(dev, 0x100210, 0x80000000, 0x80000000);
+	nv_wr32(device, 0x1002dc, 0x00000000);
+	nv_mask(device, 0x100210, 0x80000000, 0x80000000);
 	udelay(100);
 
 	/* execute memory reset script from vbios */
 	if (!bit_table(dev, 'M', &M))
-		nouveau_bios_init_exec(dev, ROM16(M.data[0]));
+		nouveau_bios_run_init_table(dev, ROM16(M.data[0]), NULL, 0);
 
 	/* make sure we're in vblank (hopefully the same one as before), and
 	 * then re-enable crtc memory access
@@ -334,62 +340,14 @@ nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
 	for (i = 0; i < 2; i++) {
 		if (!(crtc_mask & (1 << i)))
 			continue;
-		nv_wait(dev, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
-		nv_wr08(dev, 0x0c03c4 + (i * 0x2000), 0x01);
-		nv_wr08(dev, 0x0c03c5 + (i * 0x2000), sr1[i]);
+		nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
+		nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
+		nv_wr08(device, 0x0c03c5 + (i * 0x2000), sr1[i]);
 	}
 
 	/* resume engines */
 resume:
-	nv_wr32(dev, 0x003250, 0x00000001);
-	nv_mask(dev, 0x003220, 0x00000001, 0x00000001);
-	nv_wr32(dev, 0x003200, 0x00000001);
-	nv_wr32(dev, 0x002500, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
+	pfifo->start(pfifo, &flags);
 	kfree(info);
 	return ret;
 }
-
-int
-nv40_pm_pwm_get(struct drm_device *dev, int line, u32 *divs, u32 *duty)
-{
-	if (line == 2) {
-		u32 reg = nv_rd32(dev, 0x0010f0);
-		if (reg & 0x80000000) {
-			*duty = (reg & 0x7fff0000) >> 16;
-			*divs = (reg & 0x00007fff);
-			return 0;
-		}
-	} else
-	if (line == 9) {
-		u32 reg = nv_rd32(dev, 0x0015f4);
-		if (reg & 0x80000000) {
-			*divs = nv_rd32(dev, 0x0015f8);
-			*duty = (reg & 0x7fffffff);
-			return 0;
-		}
-	} else {
-		NV_ERROR(dev, "unknown pwm ctrl for gpio %d\n", line);
-		return -ENODEV;
-	}
-
-	return -EINVAL;
-}
-
-int
-nv40_pm_pwm_set(struct drm_device *dev, int line, u32 divs, u32 duty)
-{
-	if (line == 2) {
-		nv_wr32(dev, 0x0010f0, 0x80000000 | (duty << 16) | divs);
-	} else
-	if (line == 9) {
-		nv_wr32(dev, 0x0015f8, divs);
-		nv_wr32(dev, 0x0015f4, duty | 0x80000000);
-	} else {
-		NV_ERROR(dev, "unknown pwm ctrl for gpio %d\n", line);
-		return -ENODEV;
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_crtc.c b/drivers/gpu/drm/nouveau/nv50_crtc.c
index 22cebd5dd69..222de77d626 100644
--- a/drivers/gpu/drm/nouveau/nv50_crtc.c
+++ b/drivers/gpu/drm/nouveau/nv50_crtc.c
@@ -24,28 +24,30 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_mode.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
 #include "nouveau_reg.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
+#include "nouveau_gem.h"
 #include "nouveau_hw.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
-#include "nouveau_fb.h"
 #include "nouveau_connector.h"
 #include "nv50_display.h"
 
+#include <subdev/clock.h>
+
 static void
 nv50_crtc_lut_load(struct drm_crtc *crtc)
 {
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	void __iomem *lut = nvbo_kmap_obj_iovirtual(nv_crtc->lut.nvbo);
 	int i;
 
-	NV_DEBUG_KMS(crtc->dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	for (i = 0; i < 256; i++) {
 		writew(nv_crtc->lut.r[i] >> 2, lut + 8*i + 0);
@@ -64,25 +66,25 @@ int
 nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int index = nv_crtc->index, ret;
 
-	NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
-	NV_DEBUG_KMS(dev, "%s\n", blanked ? "blanked" : "unblanked");
+	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
+	NV_DEBUG(drm, "%s\n", blanked ? "blanked" : "unblanked");
 
 	if (blanked) {
 		nv_crtc->cursor.hide(nv_crtc, false);
 
-		ret = RING_SPACE(evo, dev_priv->chipset != 0x50 ? 7 : 5);
+		ret = RING_SPACE(evo, nv_device(drm->device)->chipset != 0x50 ? 7 : 5);
 		if (ret) {
-			NV_ERROR(dev, "no space while blanking crtc\n");
+			NV_ERROR(drm, "no space while blanking crtc\n");
 			return ret;
 		}
 		BEGIN_NV04(evo, 0, NV50_EVO_CRTC(index, CLUT_MODE), 2);
 		OUT_RING(evo, NV50_EVO_CRTC_CLUT_MODE_BLANK);
 		OUT_RING(evo, 0);
-		if (dev_priv->chipset != 0x50) {
+		if (nv_device(drm->device)->chipset != 0x50) {
 			BEGIN_NV04(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1);
 			OUT_RING(evo, NV84_EVO_CRTC_CLUT_DMA_HANDLE_NONE);
 		}
@@ -95,9 +97,9 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
 		else
 			nv_crtc->cursor.hide(nv_crtc, false);
 
-		ret = RING_SPACE(evo, dev_priv->chipset != 0x50 ? 10 : 8);
+		ret = RING_SPACE(evo, nv_device(drm->device)->chipset != 0x50 ? 10 : 8);
 		if (ret) {
-			NV_ERROR(dev, "no space while unblanking crtc\n");
+			NV_ERROR(drm, "no space while unblanking crtc\n");
 			return ret;
 		}
 		BEGIN_NV04(evo, 0, NV50_EVO_CRTC(index, CLUT_MODE), 2);
@@ -105,7 +107,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
 				NV50_EVO_CRTC_CLUT_MODE_OFF :
 				NV50_EVO_CRTC_CLUT_MODE_ON);
 		OUT_RING(evo, nv_crtc->lut.nvbo->bo.offset >> 8);
-		if (dev_priv->chipset != 0x50) {
+		if (nv_device(drm->device)->chipset != 0x50) {
 			BEGIN_NV04(evo, 0, NV84_EVO_CRTC(index, CLUT_DMA), 1);
 			OUT_RING(evo, NvEvoVRAM);
 		}
@@ -114,7 +116,7 @@ nv50_crtc_blank(struct nouveau_crtc *nv_crtc, bool blanked)
 		OUT_RING(evo, nv_crtc->fb.offset >> 8);
 		OUT_RING(evo, 0);
 		BEGIN_NV04(evo, 0, NV50_EVO_CRTC(index, FB_DMA), 1);
-		if (dev_priv->chipset != 0x50)
+		if (nv_device(drm->device)->chipset != 0x50)
 			if (nv_crtc->fb.tile_flags == 0x7a00 ||
 			    nv_crtc->fb.tile_flags == 0xfe00)
 				OUT_RING(evo, NvEvoFB32);
@@ -174,17 +176,18 @@ static int
 nv50_crtc_set_color_vibrance(struct nouveau_crtc *nv_crtc, bool update)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int ret;
 	int adj;
 	u32 hue, vib;
 
-	NV_DEBUG_KMS(dev, "vibrance = %i, hue = %i\n",
+	NV_DEBUG(drm, "vibrance = %i, hue = %i\n",
 		     nv_crtc->color_vibrance, nv_crtc->vibrant_hue);
 
 	ret = RING_SPACE(evo, 2 + (update ? 2 : 0));
 	if (ret) {
-		NV_ERROR(dev, "no space while setting color vibrance\n");
+		NV_ERROR(drm, "no space while setting color vibrance\n");
 		return ret;
 	}
 
@@ -229,17 +232,18 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
 	struct nouveau_connector *nv_connector;
 	struct drm_crtc *crtc = &nv_crtc->base;
 	struct drm_device *dev = crtc->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	struct drm_display_mode *umode = &crtc->mode;
 	struct drm_display_mode *omode;
 	int scaling_mode, ret;
 	u32 ctrl = 0, oX, oY;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	nv_connector = nouveau_crtc_connector_get(nv_crtc);
 	if (!nv_connector || !nv_connector->native_mode) {
-		NV_ERROR(dev, "no native mode, forcing panel scaling\n");
+		NV_ERROR(drm, "no native mode, forcing panel scaling\n");
 		scaling_mode = DRM_MODE_SCALE_NONE;
 	} else {
 		scaling_mode = nv_connector->scaling_mode;
@@ -329,63 +333,19 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
 int
 nv50_crtc_set_clock(struct drm_device *dev, int head, int pclk)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct pll_lims pll;
-	uint32_t reg1, reg2;
-	int ret, N1, M1, N2, M2, P;
-
-	ret = get_pll_limits(dev, PLL_VPLL0 + head, &pll);
-	if (ret)
-		return ret;
-
-	if (pll.vco2.maxfreq) {
-		ret = nv50_calc_pll(dev, &pll, pclk, &N1, &M1, &N2, &M2, &P);
-		if (ret <= 0)
-			return 0;
-
-		NV_DEBUG(dev, "pclk %d out %d NM1 %d %d NM2 %d %d P %d\n",
-			 pclk, ret, N1, M1, N2, M2, P);
-
-		reg1 = nv_rd32(dev, pll.reg + 4) & 0xff00ff00;
-		reg2 = nv_rd32(dev, pll.reg + 8) & 0x8000ff00;
-		nv_wr32(dev, pll.reg + 0, 0x10000611);
-		nv_wr32(dev, pll.reg + 4, reg1 | (M1 << 16) | N1);
-		nv_wr32(dev, pll.reg + 8, reg2 | (P << 28) | (M2 << 16) | N2);
-	} else
-	if (dev_priv->chipset < NV_C0) {
-		ret = nva3_calc_pll(dev, &pll, pclk, &N1, &N2, &M1, &P);
-		if (ret <= 0)
-			return 0;
-
-		NV_DEBUG(dev, "pclk %d out %d N %d fN 0x%04x M %d P %d\n",
-			 pclk, ret, N1, N2, M1, P);
-
-		reg1 = nv_rd32(dev, pll.reg + 4) & 0xffc00000;
-		nv_wr32(dev, pll.reg + 0, 0x50000610);
-		nv_wr32(dev, pll.reg + 4, reg1 | (P << 16) | (M1 << 8) | N1);
-		nv_wr32(dev, pll.reg + 8, N2);
-	} else {
-		ret = nva3_calc_pll(dev, &pll, pclk, &N1, &N2, &M1, &P);
-		if (ret <= 0)
-			return 0;
-
-		NV_DEBUG(dev, "pclk %d out %d N %d fN 0x%04x M %d P %d\n",
-			 pclk, ret, N1, N2, M1, P);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_clock *clk = nouveau_clock(device);
 
-		nv_mask(dev, pll.reg + 0x0c, 0x00000000, 0x00000100);
-		nv_wr32(dev, pll.reg + 0x04, (P << 16) | (N1 << 8) | M1);
-		nv_wr32(dev, pll.reg + 0x10, N2 << 16);
-	}
-
-	return 0;
+	return clk->pll_set(clk, PLL_VPLL0 + head, pclk);
 }
 
 static void
 nv50_crtc_destroy(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
 
-	NV_DEBUG_KMS(crtc->dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	nouveau_bo_unmap(nv_crtc->lut.nvbo);
 	nouveau_bo_ref(NULL, &nv_crtc->lut.nvbo);
@@ -474,13 +434,15 @@ nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
 static void
 nv50_crtc_save(struct drm_crtc *crtc)
 {
-	NV_ERROR(crtc->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static void
 nv50_crtc_restore(struct drm_crtc *crtc)
 {
-	NV_ERROR(crtc->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static const struct drm_crtc_funcs nv50_crtc_funcs = {
@@ -504,8 +466,9 @@ nv50_crtc_prepare(struct drm_crtc *crtc)
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
+	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
 	nv50_display_flip_stop(crtc);
 	drm_vblank_pre_modeset(dev, nv_crtc->index);
@@ -516,9 +479,10 @@ static void
 nv50_crtc_commit(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 
-	NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
+	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
 	nv50_crtc_blank(nv_crtc, false);
 	drm_vblank_post_modeset(dev, nv_crtc->index);
@@ -540,17 +504,17 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
 {
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	struct drm_framebuffer *drm_fb;
 	struct nouveau_framebuffer *fb;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "index %d\n", nv_crtc->index);
+	NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
 	/* no fb bound */
 	if (!atomic && !crtc->fb) {
-		NV_DEBUG_KMS(dev, "No FB bound\n");
+		NV_DEBUG(drm, "No FB bound\n");
 		return 0;
 	}
 
@@ -580,7 +544,7 @@ nv50_crtc_do_mode_set_base(struct drm_crtc *crtc,
 	nv_crtc->fb.offset = fb->nvbo->bo.offset;
 	nv_crtc->fb.tile_flags = nouveau_bo_tile_layout(fb->nvbo);
 	nv_crtc->fb.cpp = drm_fb->bits_per_pixel / 8;
-	if (!nv_crtc->fb.blanked && dev_priv->chipset != 0x50) {
+	if (!nv_crtc->fb.blanked && nv_device(drm->device)->chipset != 0x50) {
 		ret = RING_SPACE(evo, 2);
 		if (ret)
 			return ret;
@@ -738,10 +702,11 @@ static const struct drm_crtc_helper_funcs nv50_crtc_helper_funcs = {
 int
 nv50_crtc_create(struct drm_device *dev, int index)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_crtc *nv_crtc = NULL;
 	int ret, i;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	nv_crtc = kzalloc(sizeof(*nv_crtc), GFP_KERNEL);
 	if (!nv_crtc)
diff --git a/drivers/gpu/drm/nouveau/nv50_cursor.c b/drivers/gpu/drm/nouveau/nv50_cursor.c
index af4ec7bf367..223da113cee 100644
--- a/drivers/gpu/drm/nouveau/nv50_cursor.c
+++ b/drivers/gpu/drm/nouveau/nv50_cursor.c
@@ -24,12 +24,10 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_mode.h"
+#include <drm/drmP.h>
 
-#define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
-#include "nouveau_reg.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
 #include "nouveau_crtc.h"
 #include "nv50_display.h"
 
@@ -37,22 +35,22 @@ static void
 nv50_cursor_show(struct nouveau_crtc *nv_crtc, bool update)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	if (update && nv_crtc->cursor.visible)
 		return;
 
-	ret = RING_SPACE(evo, (dev_priv->chipset != 0x50 ? 5 : 3) + update * 2);
+	ret = RING_SPACE(evo, (nv_device(drm->device)->chipset != 0x50 ? 5 : 3) + update * 2);
 	if (ret) {
-		NV_ERROR(dev, "no space while unhiding cursor\n");
+		NV_ERROR(drm, "no space while unhiding cursor\n");
 		return;
 	}
 
-	if (dev_priv->chipset != 0x50) {
+	if (nv_device(drm->device)->chipset != 0x50) {
 		BEGIN_NV04(evo, 0, NV84_EVO_CRTC(nv_crtc->index, CURSOR_DMA), 1);
 		OUT_RING(evo, NvEvoVRAM);
 	}
@@ -72,24 +70,24 @@ static void
 nv50_cursor_hide(struct nouveau_crtc *nv_crtc, bool update)
 {
 	struct drm_device *dev = nv_crtc->base.dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	if (update && !nv_crtc->cursor.visible)
 		return;
 
-	ret = RING_SPACE(evo, (dev_priv->chipset != 0x50 ? 5 : 3) + update * 2);
+	ret = RING_SPACE(evo, (nv_device(drm->device)->chipset != 0x50 ? 5 : 3) + update * 2);
 	if (ret) {
-		NV_ERROR(dev, "no space while hiding cursor\n");
+		NV_ERROR(drm, "no space while hiding cursor\n");
 		return;
 	}
 	BEGIN_NV04(evo, 0, NV50_EVO_CRTC(nv_crtc->index, CURSOR_CTRL), 2);
 	OUT_RING(evo, NV50_EVO_CRTC_CURSOR_CTRL_HIDE);
 	OUT_RING(evo, 0);
-	if (dev_priv->chipset != 0x50) {
+	if (nv_device(drm->device)->chipset != 0x50) {
 		BEGIN_NV04(evo, 0, NV84_EVO_CRTC(nv_crtc->index, CURSOR_DMA), 1);
 		OUT_RING(evo, NV84_EVO_CRTC_CURSOR_DMA_HANDLE_NONE);
 	}
@@ -105,19 +103,18 @@ nv50_cursor_hide(struct nouveau_crtc *nv_crtc, bool update)
 static void
 nv50_cursor_set_pos(struct nouveau_crtc *nv_crtc, int x, int y)
 {
-	struct drm_device *dev = nv_crtc->base.dev;
+	struct nouveau_device *device = nouveau_dev(nv_crtc->base.dev);
 
 	nv_crtc->cursor_saved_x = x; nv_crtc->cursor_saved_y = y;
-	nv_wr32(dev, NV50_PDISPLAY_CURSOR_USER_POS(nv_crtc->index),
+	nv_wr32(device, NV50_PDISPLAY_CURSOR_USER_POS(nv_crtc->index),
 		((y & 0xFFFF) << 16) | (x & 0xFFFF));
 	/* Needed to make the cursor move. */
-	nv_wr32(dev, NV50_PDISPLAY_CURSOR_USER_POS_CTRL(nv_crtc->index), 0);
+	nv_wr32(device, NV50_PDISPLAY_CURSOR_USER_POS_CTRL(nv_crtc->index), 0);
 }
 
 static void
 nv50_cursor_set_offset(struct nouveau_crtc *nv_crtc, uint32_t offset)
 {
-	NV_DEBUG_KMS(nv_crtc->base.dev, "\n");
 	if (offset == nv_crtc->cursor.offset)
 		return;
 
diff --git a/drivers/gpu/drm/nouveau/nv50_dac.c b/drivers/gpu/drm/nouveau/nv50_dac.c
index 2c36a6b92c5..6a30a174857 100644
--- a/drivers/gpu/drm/nouveau/nv50_dac.c
+++ b/drivers/gpu/drm/nouveau/nv50_dac.c
@@ -24,23 +24,26 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
 #include "nouveau_reg.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
 #include "nv50_display.h"
 
+#include <subdev/timer.h>
+
 static void
 nv50_dac_disconnect(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int ret;
 
@@ -48,11 +51,11 @@ nv50_dac_disconnect(struct drm_encoder *encoder)
 		return;
 	nv50_crtc_blank(nouveau_crtc(nv_encoder->crtc), true);
 
-	NV_DEBUG_KMS(dev, "Disconnecting DAC %d\n", nv_encoder->or);
+	NV_DEBUG(drm, "Disconnecting DAC %d\n", nv_encoder->or);
 
 	ret = RING_SPACE(evo, 4);
 	if (ret) {
-		NV_ERROR(dev, "no space while disconnecting DAC\n");
+		NV_ERROR(drm, "no space while disconnecting DAC\n");
 		return;
 	}
 	BEGIN_NV04(evo, 0, NV50_EVO_DAC(nv_encoder->or, MODE_CTRL), 1);
@@ -67,43 +70,43 @@ static enum drm_connector_status
 nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	enum drm_connector_status status = connector_status_disconnected;
 	uint32_t dpms_state, load_pattern, load_state;
 	int or = nv_encoder->or;
 
-	nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL1(or), 0x00000001);
-	dpms_state = nv_rd32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or));
+	nv_wr32(device, NV50_PDISPLAY_DAC_CLK_CTRL1(or), 0x00000001);
+	dpms_state = nv_rd32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or));
 
-	nv_wr32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
+	nv_wr32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
 		0x00150000 | NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING);
-	if (!nv_wait(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
+	if (!nv_wait(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
 		     NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING, 0)) {
-		NV_ERROR(dev, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
-		NV_ERROR(dev, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
-			  nv_rd32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or)));
+		NV_ERROR(drm, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
+		NV_ERROR(drm, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
+			  nv_rd32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or)));
 		return status;
 	}
 
 	/* Use bios provided value if possible. */
-	if (dev_priv->vbios.dactestval) {
-		load_pattern = dev_priv->vbios.dactestval;
-		NV_DEBUG_KMS(dev, "Using bios provided load_pattern of %d\n",
+	if (drm->vbios.dactestval) {
+		load_pattern = drm->vbios.dactestval;
+		NV_DEBUG(drm, "Using bios provided load_pattern of %d\n",
 			  load_pattern);
 	} else {
 		load_pattern = 340;
-		NV_DEBUG_KMS(dev, "Using default load_pattern of %d\n",
+		NV_DEBUG(drm, "Using default load_pattern of %d\n",
 			 load_pattern);
 	}
 
-	nv_wr32(dev, NV50_PDISPLAY_DAC_LOAD_CTRL(or),
+	nv_wr32(device, NV50_PDISPLAY_DAC_LOAD_CTRL(or),
 		NV50_PDISPLAY_DAC_LOAD_CTRL_ACTIVE | load_pattern);
 	mdelay(45); /* give it some time to process */
-	load_state = nv_rd32(dev, NV50_PDISPLAY_DAC_LOAD_CTRL(or));
+	load_state = nv_rd32(device, NV50_PDISPLAY_DAC_LOAD_CTRL(or));
 
-	nv_wr32(dev, NV50_PDISPLAY_DAC_LOAD_CTRL(or), 0);
-	nv_wr32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or), dpms_state |
+	nv_wr32(device, NV50_PDISPLAY_DAC_LOAD_CTRL(or), 0);
+	nv_wr32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or), dpms_state |
 		NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING);
 
 	if ((load_state & NV50_PDISPLAY_DAC_LOAD_CTRL_PRESENT) ==
@@ -111,9 +114,9 @@ nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 		status = connector_status_connected;
 
 	if (status == connector_status_connected)
-		NV_DEBUG_KMS(dev, "Load was detected on output with or %d\n", or);
+		NV_DEBUG(drm, "Load was detected on output with or %d\n", or);
 	else
-		NV_DEBUG_KMS(dev, "Load was not detected on output with or %d\n", or);
+		NV_DEBUG(drm, "Load was not detected on output with or %d\n", or);
 
 	return status;
 }
@@ -121,23 +124,24 @@ nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 static void
 nv50_dac_dpms(struct drm_encoder *encoder, int mode)
 {
-	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	uint32_t val;
 	int or = nv_encoder->or;
 
-	NV_DEBUG_KMS(dev, "or %d mode %d\n", or, mode);
+	NV_DEBUG(drm, "or %d mode %d\n", or, mode);
 
 	/* wait for it to be done */
-	if (!nv_wait(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
+	if (!nv_wait(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or),
 		     NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING, 0)) {
-		NV_ERROR(dev, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
-		NV_ERROR(dev, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
-			 nv_rd32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or)));
+		NV_ERROR(drm, "timeout: DAC_DPMS_CTRL_PENDING(%d) == 0\n", or);
+		NV_ERROR(drm, "DAC_DPMS_CTRL(%d) = 0x%08x\n", or,
+			 nv_rd32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or)));
 		return;
 	}
 
-	val = nv_rd32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or)) & ~0x7F;
+	val = nv_rd32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or)) & ~0x7F;
 
 	if (mode != DRM_MODE_DPMS_ON)
 		val |= NV50_PDISPLAY_DAC_DPMS_CTRL_BLANKED;
@@ -158,20 +162,22 @@ nv50_dac_dpms(struct drm_encoder *encoder, int mode)
 		break;
 	}
 
-	nv_wr32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(or), val |
+	nv_wr32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(or), val |
 		NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING);
 }
 
 static void
 nv50_dac_save(struct drm_encoder *encoder)
 {
-	NV_ERROR(encoder->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static void
 nv50_dac_restore(struct drm_encoder *encoder)
 {
-	NV_ERROR(encoder->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static bool
@@ -179,14 +185,15 @@ nv50_dac_mode_fixup(struct drm_encoder *encoder,
 		    const struct drm_display_mode *mode,
 		    struct drm_display_mode *adjusted_mode)
 {
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_connector *connector;
 
-	NV_DEBUG_KMS(encoder->dev, "or %d\n", nv_encoder->or);
+	NV_DEBUG(drm, "or %d\n", nv_encoder->or);
 
 	connector = nouveau_encoder_connector_get(nv_encoder);
 	if (!connector) {
-		NV_ERROR(encoder->dev, "Encoder has no connector\n");
+		NV_ERROR(drm, "Encoder has no connector\n");
 		return false;
 	}
 
@@ -207,13 +214,14 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		  struct drm_display_mode *adjusted_mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct drm_device *dev = encoder->dev;
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	struct nouveau_crtc *crtc = nouveau_crtc(encoder->crtc);
 	uint32_t mode_ctl = 0, mode_ctl2 = 0;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "or %d type %d crtc %d\n",
+	NV_DEBUG(drm, "or %d type %d crtc %d\n",
 		     nv_encoder->or, nv_encoder->dcb->type, crtc->index);
 
 	nv50_dac_dpms(encoder, DRM_MODE_DPMS_ON);
@@ -224,10 +232,10 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		mode_ctl |= NV50_EVO_DAC_MODE_CTRL_CRTC0;
 
 	/* Lacking a working tv-out, this is not a 100% sure. */
-	if (nv_encoder->dcb->type == OUTPUT_ANALOG)
+	if (nv_encoder->dcb->type == DCB_OUTPUT_ANALOG)
 		mode_ctl |= 0x40;
 	else
-	if (nv_encoder->dcb->type == OUTPUT_TV)
+	if (nv_encoder->dcb->type == DCB_OUTPUT_TV)
 		mode_ctl |= 0x100;
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
@@ -238,7 +246,7 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
 
 	ret = RING_SPACE(evo, 3);
 	if (ret) {
-		NV_ERROR(dev, "no space while connecting DAC\n");
+		NV_ERROR(drm, "no space while connecting DAC\n");
 		return;
 	}
 	BEGIN_NV04(evo, 0, NV50_EVO_DAC(nv_encoder->or, MODE_CTRL), 2);
@@ -271,11 +279,12 @@ static void
 nv50_dac_destroy(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 
 	if (!encoder)
 		return;
 
-	NV_DEBUG_KMS(encoder->dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	drm_encoder_cleanup(encoder);
 	kfree(nv_encoder);
@@ -286,7 +295,7 @@ static const struct drm_encoder_funcs nv50_dac_encoder_funcs = {
 };
 
 int
-nv50_dac_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv50_dac_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	struct nouveau_encoder *nv_encoder;
 	struct drm_encoder *encoder;
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index b244d9968c5..f97b42cbb6b 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -24,28 +24,30 @@
  *
  */
 
-#define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
+
 #include "nv50_display.h"
 #include "nouveau_crtc.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
-#include "nouveau_fb.h"
 #include "nouveau_fbcon.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_crtc_helper.h>
+#include "nouveau_fence.h"
+
+#include <core/gpuobj.h>
+#include <subdev/timer.h>
 
-static void nv50_display_isr(struct drm_device *);
 static void nv50_display_bh(unsigned long);
 
 static inline int
 nv50_sor_nr(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 
-	if (dev_priv->chipset  < 0x90 ||
-	    dev_priv->chipset == 0x92 ||
-	    dev_priv->chipset == 0xa0)
+	if (device->chipset  < 0x90 ||
+	    device->chipset == 0x92 ||
+	    device->chipset == 0xa0)
 		return 2;
 
 	return 4;
@@ -54,73 +56,29 @@ nv50_sor_nr(struct drm_device *dev)
 u32
 nv50_display_active_crtcs(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 	u32 mask = 0;
 	int i;
 
-	if (dev_priv->chipset  < 0x90 ||
-	    dev_priv->chipset == 0x92 ||
-	    dev_priv->chipset == 0xa0) {
+	if (device->chipset  < 0x90 ||
+	    device->chipset == 0x92 ||
+	    device->chipset == 0xa0) {
 		for (i = 0; i < 2; i++)
-			mask |= nv_rd32(dev, NV50_PDISPLAY_SOR_MODE_CTRL_C(i));
+			mask |= nv_rd32(device, NV50_PDISPLAY_SOR_MODE_CTRL_C(i));
 	} else {
 		for (i = 0; i < 4; i++)
-			mask |= nv_rd32(dev, NV90_PDISPLAY_SOR_MODE_CTRL_C(i));
+			mask |= nv_rd32(device, NV90_PDISPLAY_SOR_MODE_CTRL_C(i));
 	}
 
 	for (i = 0; i < 3; i++)
-		mask |= nv_rd32(dev, NV50_PDISPLAY_DAC_MODE_CTRL_C(i));
+		mask |= nv_rd32(device, NV50_PDISPLAY_DAC_MODE_CTRL_C(i));
 
 	return mask & 3;
 }
 
-static int
-evo_icmd(struct drm_device *dev, int ch, u32 mthd, u32 data)
-{
-	int ret = 0;
-	nv_mask(dev, 0x610300 + (ch * 0x08), 0x00000001, 0x00000001);
-	nv_wr32(dev, 0x610304 + (ch * 0x08), data);
-	nv_wr32(dev, 0x610300 + (ch * 0x08), 0x80000001 | mthd);
-	if (!nv_wait(dev, 0x610300 + (ch * 0x08), 0x80000000, 0x00000000))
-		ret = -EBUSY;
-	if (ret || (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO))
-		NV_INFO(dev, "EvoPIO: %d 0x%04x 0x%08x\n", ch, mthd, data);
-	nv_mask(dev, 0x610300 + (ch * 0x08), 0x00000001, 0x00000000);
-	return ret;
-}
-
 int
 nv50_display_early_init(struct drm_device *dev)
 {
-	u32 ctrl = nv_rd32(dev, 0x610200);
-	int i;
-
-	/* check if master evo channel is already active, a good a sign as any
-	 * that the display engine is in a weird state (hibernate/kexec), if
-	 * it is, do our best to reset the display engine...
-	 */
-	if ((ctrl & 0x00000003) == 0x00000003) {
-		NV_INFO(dev, "PDISP: EVO(0) 0x%08x, resetting...\n", ctrl);
-
-		/* deactivate both heads first, PDISP will disappear forever
-		 * (well, until you power cycle) on some boards as soon as
-		 * PMC_ENABLE is hit unless they are..
-		 */
-		for (i = 0; i < 2; i++) {
-			evo_icmd(dev, 0, 0x0880 + (i * 0x400), 0x05000000);
-			evo_icmd(dev, 0, 0x089c + (i * 0x400), 0);
-			evo_icmd(dev, 0, 0x0840 + (i * 0x400), 0);
-			evo_icmd(dev, 0, 0x0844 + (i * 0x400), 0);
-			evo_icmd(dev, 0, 0x085c + (i * 0x400), 0);
-			evo_icmd(dev, 0, 0x0874 + (i * 0x400), 0);
-		}
-		evo_icmd(dev, 0, 0x0080, 0);
-
-		/* reset PDISP */
-		nv_mask(dev, 0x000200, 0x40000000, 0x00000000);
-		nv_mask(dev, 0x000200, 0x40000000, 0x40000000);
-	}
-
 	return 0;
 }
 
@@ -132,11 +90,8 @@ nv50_display_late_takedown(struct drm_device *dev)
 int
 nv50_display_sync(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
 	struct nv50_display *disp = nv50_display(dev);
 	struct nouveau_channel *evo = disp->master;
-	u64 start;
 	int ret;
 
 	ret = RING_SPACE(evo, 6);
@@ -148,29 +103,28 @@ nv50_display_sync(struct drm_device *dev)
 		BEGIN_NV04(evo, 0, 0x0084, 1);
 		OUT_RING  (evo, 0x00000000);
 
-		nv_wo32(disp->ntfy, 0x000, 0x00000000);
+		nv_wo32(disp->ramin, 0x2000, 0x00000000);
 		FIRE_RING (evo);
 
-		start = ptimer->read(dev);
-		do {
-			if (nv_ro32(disp->ntfy, 0x000))
-				return 0;
-		} while (ptimer->read(dev) - start < 2000000000ULL);
+		if (nv_wait_ne(disp->ramin, 0x2000, 0xffffffff, 0x00000000))
+			return 0;
 	}
 
-	return -EBUSY;
+	return 0;
 }
 
 int
 nv50_display_init(struct drm_device *dev)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nouveau_channel *evo;
 	int ret, i;
 	u32 val;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
-	nv_wr32(dev, 0x00610184, nv_rd32(dev, 0x00614004));
+	nv_wr32(device, 0x00610184, nv_rd32(device, 0x00614004));
 
 	/*
 	 * I think the 0x006101XX range is some kind of main control area
@@ -178,82 +132,82 @@ nv50_display_init(struct drm_device *dev)
 	 */
 	/* CRTC? */
 	for (i = 0; i < 2; i++) {
-		val = nv_rd32(dev, 0x00616100 + (i * 0x800));
-		nv_wr32(dev, 0x00610190 + (i * 0x10), val);
-		val = nv_rd32(dev, 0x00616104 + (i * 0x800));
-		nv_wr32(dev, 0x00610194 + (i * 0x10), val);
-		val = nv_rd32(dev, 0x00616108 + (i * 0x800));
-		nv_wr32(dev, 0x00610198 + (i * 0x10), val);
-		val = nv_rd32(dev, 0x0061610c + (i * 0x800));
-		nv_wr32(dev, 0x0061019c + (i * 0x10), val);
+		val = nv_rd32(device, 0x00616100 + (i * 0x800));
+		nv_wr32(device, 0x00610190 + (i * 0x10), val);
+		val = nv_rd32(device, 0x00616104 + (i * 0x800));
+		nv_wr32(device, 0x00610194 + (i * 0x10), val);
+		val = nv_rd32(device, 0x00616108 + (i * 0x800));
+		nv_wr32(device, 0x00610198 + (i * 0x10), val);
+		val = nv_rd32(device, 0x0061610c + (i * 0x800));
+		nv_wr32(device, 0x0061019c + (i * 0x10), val);
 	}
 
 	/* DAC */
 	for (i = 0; i < 3; i++) {
-		val = nv_rd32(dev, 0x0061a000 + (i * 0x800));
-		nv_wr32(dev, 0x006101d0 + (i * 0x04), val);
+		val = nv_rd32(device, 0x0061a000 + (i * 0x800));
+		nv_wr32(device, 0x006101d0 + (i * 0x04), val);
 	}
 
 	/* SOR */
 	for (i = 0; i < nv50_sor_nr(dev); i++) {
-		val = nv_rd32(dev, 0x0061c000 + (i * 0x800));
-		nv_wr32(dev, 0x006101e0 + (i * 0x04), val);
+		val = nv_rd32(device, 0x0061c000 + (i * 0x800));
+		nv_wr32(device, 0x006101e0 + (i * 0x04), val);
 	}
 
 	/* EXT */
 	for (i = 0; i < 3; i++) {
-		val = nv_rd32(dev, 0x0061e000 + (i * 0x800));
-		nv_wr32(dev, 0x006101f0 + (i * 0x04), val);
+		val = nv_rd32(device, 0x0061e000 + (i * 0x800));
+		nv_wr32(device, 0x006101f0 + (i * 0x04), val);
 	}
 
 	for (i = 0; i < 3; i++) {
-		nv_wr32(dev, NV50_PDISPLAY_DAC_DPMS_CTRL(i), 0x00550000 |
+		nv_wr32(device, NV50_PDISPLAY_DAC_DPMS_CTRL(i), 0x00550000 |
 			NV50_PDISPLAY_DAC_DPMS_CTRL_PENDING);
-		nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL1(i), 0x00000001);
+		nv_wr32(device, NV50_PDISPLAY_DAC_CLK_CTRL1(i), 0x00000001);
 	}
 
 	/* The precise purpose is unknown, i suspect it has something to do
 	 * with text mode.
 	 */
-	if (nv_rd32(dev, NV50_PDISPLAY_INTR_1) & 0x100) {
-		nv_wr32(dev, NV50_PDISPLAY_INTR_1, 0x100);
-		nv_wr32(dev, 0x006194e8, nv_rd32(dev, 0x006194e8) & ~1);
-		if (!nv_wait(dev, 0x006194e8, 2, 0)) {
-			NV_ERROR(dev, "timeout: (0x6194e8 & 2) != 0\n");
-			NV_ERROR(dev, "0x6194e8 = 0x%08x\n",
-						nv_rd32(dev, 0x6194e8));
+	if (nv_rd32(device, NV50_PDISPLAY_INTR_1) & 0x100) {
+		nv_wr32(device, NV50_PDISPLAY_INTR_1, 0x100);
+		nv_wr32(device, 0x006194e8, nv_rd32(device, 0x006194e8) & ~1);
+		if (!nv_wait(device, 0x006194e8, 2, 0)) {
+			NV_ERROR(drm, "timeout: (0x6194e8 & 2) != 0\n");
+			NV_ERROR(drm, "0x6194e8 = 0x%08x\n",
+						nv_rd32(device, 0x6194e8));
 			return -EBUSY;
 		}
 	}
 
 	for (i = 0; i < 2; i++) {
-		nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0x2000);
-		if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+		nv_wr32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0x2000);
+		if (!nv_wait(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
 			     NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS, 0)) {
-			NV_ERROR(dev, "timeout: CURSOR_CTRL2_STATUS == 0\n");
-			NV_ERROR(dev, "CURSOR_CTRL2 = 0x%08x\n",
-				 nv_rd32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
+			NV_ERROR(drm, "timeout: CURSOR_CTRL2_STATUS == 0\n");
+			NV_ERROR(drm, "CURSOR_CTRL2 = 0x%08x\n",
+				 nv_rd32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
 			return -EBUSY;
 		}
 
-		nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+		nv_wr32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
 			NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_ON);
-		if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+		if (!nv_wait(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
 			     NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS,
 			     NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS_ACTIVE)) {
-			NV_ERROR(dev, "timeout: "
+			NV_ERROR(drm, "timeout: "
 				      "CURSOR_CTRL2_STATUS_ACTIVE(%d)\n", i);
-			NV_ERROR(dev, "CURSOR_CTRL2(%d) = 0x%08x\n", i,
-				 nv_rd32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
+			NV_ERROR(drm, "CURSOR_CTRL2(%d) = 0x%08x\n", i,
+				 nv_rd32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
 			return -EBUSY;
 		}
 	}
 
-	nv_wr32(dev, NV50_PDISPLAY_PIO_CTRL, 0x00000000);
-	nv_mask(dev, NV50_PDISPLAY_INTR_0, 0x00000000, 0x00000000);
-	nv_wr32(dev, NV50_PDISPLAY_INTR_EN_0, 0x00000000);
-	nv_mask(dev, NV50_PDISPLAY_INTR_1, 0x00000000, 0x00000000);
-	nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1,
+	nv_wr32(device, NV50_PDISPLAY_PIO_CTRL, 0x00000000);
+	nv_mask(device, NV50_PDISPLAY_INTR_0, 0x00000000, 0x00000000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_EN_0, 0x00000000);
+	nv_mask(device, NV50_PDISPLAY_INTR_1, 0x00000000, 0x00000000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_EN_1,
 		     NV50_PDISPLAY_INTR_EN_1_CLK_UNK10 |
 		     NV50_PDISPLAY_INTR_EN_1_CLK_UNK20 |
 		     NV50_PDISPLAY_INTR_EN_1_CLK_UNK40);
@@ -263,7 +217,7 @@ nv50_display_init(struct drm_device *dev)
 		return ret;
 	evo = nv50_display(dev)->master;
 
-	nv_wr32(dev, NV50_PDISPLAY_OBJECTS, (evo->ramin->vinst >> 8) | 9);
+	nv_wr32(device, NV50_PDISPLAY_OBJECTS, (nv50_display(dev)->ramin->addr >> 8) | 9);
 
 	ret = RING_SPACE(evo, 3);
 	if (ret)
@@ -278,12 +232,14 @@ nv50_display_init(struct drm_device *dev)
 void
 nv50_display_fini(struct drm_device *dev)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nv50_display *disp = nv50_display(dev);
 	struct nouveau_channel *evo = disp->master;
 	struct drm_crtc *drm_crtc;
 	int ret, i;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	list_for_each_entry(drm_crtc, &dev->mode_config.crtc_list, head) {
 		struct nouveau_crtc *crtc = nouveau_crtc(drm_crtc);
@@ -308,55 +264,59 @@ nv50_display_fini(struct drm_device *dev)
 		if (!crtc->base.enabled)
 			continue;
 
-		nv_wr32(dev, NV50_PDISPLAY_INTR_1, mask);
-		if (!nv_wait(dev, NV50_PDISPLAY_INTR_1, mask, mask)) {
-			NV_ERROR(dev, "timeout: (0x610024 & 0x%08x) == "
+		nv_wr32(device, NV50_PDISPLAY_INTR_1, mask);
+		if (!nv_wait(device, NV50_PDISPLAY_INTR_1, mask, mask)) {
+			NV_ERROR(drm, "timeout: (0x610024 & 0x%08x) == "
 				      "0x%08x\n", mask, mask);
-			NV_ERROR(dev, "0x610024 = 0x%08x\n",
-				 nv_rd32(dev, NV50_PDISPLAY_INTR_1));
+			NV_ERROR(drm, "0x610024 = 0x%08x\n",
+				 nv_rd32(device, NV50_PDISPLAY_INTR_1));
 		}
 	}
 
 	for (i = 0; i < 2; i++) {
-		nv_wr32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0);
-		if (!nv_wait(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
+		nv_wr32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i), 0);
+		if (!nv_wait(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i),
 			     NV50_PDISPLAY_CURSOR_CURSOR_CTRL2_STATUS, 0)) {
-			NV_ERROR(dev, "timeout: CURSOR_CTRL2_STATUS == 0\n");
-			NV_ERROR(dev, "CURSOR_CTRL2 = 0x%08x\n",
-				 nv_rd32(dev, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
+			NV_ERROR(drm, "timeout: CURSOR_CTRL2_STATUS == 0\n");
+			NV_ERROR(drm, "CURSOR_CTRL2 = 0x%08x\n",
+				 nv_rd32(device, NV50_PDISPLAY_CURSOR_CURSOR_CTRL2(i)));
 		}
 	}
 
 	nv50_evo_fini(dev);
 
 	for (i = 0; i < 3; i++) {
-		if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(i),
+		if (!nv_wait(device, NV50_PDISPLAY_SOR_DPMS_STATE(i),
 			     NV50_PDISPLAY_SOR_DPMS_STATE_WAIT, 0)) {
-			NV_ERROR(dev, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", i);
-			NV_ERROR(dev, "SOR_DPMS_STATE(%d) = 0x%08x\n", i,
-				  nv_rd32(dev, NV50_PDISPLAY_SOR_DPMS_STATE(i)));
+			NV_ERROR(drm, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", i);
+			NV_ERROR(drm, "SOR_DPMS_STATE(%d) = 0x%08x\n", i,
+				  nv_rd32(device, NV50_PDISPLAY_SOR_DPMS_STATE(i)));
 		}
 	}
 
 	/* disable interrupts. */
-	nv_wr32(dev, NV50_PDISPLAY_INTR_EN_1, 0x00000000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_EN_1, 0x00000000);
 }
 
 int
 nv50_display_create(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct dcb_table *dcb = &dev_priv->vbios.dcb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_table *dcb = &drm->vbios.dcb;
 	struct drm_connector *connector, *ct;
 	struct nv50_display *priv;
 	int ret, i;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
-	dev_priv->engine.display.priv = priv;
+
+	nouveau_display(dev)->priv = priv;
+	nouveau_display(dev)->dtor = nv50_display_destroy;
+	nouveau_display(dev)->init = nv50_display_init;
+	nouveau_display(dev)->fini = nv50_display_fini;
 
 	/* Create CRTC objects */
 	for (i = 0; i < 2; i++) {
@@ -367,10 +327,10 @@ nv50_display_create(struct drm_device *dev)
 
 	/* We setup the encoders from the BIOS table */
 	for (i = 0 ; i < dcb->entries; i++) {
-		struct dcb_entry *entry = &dcb->entry[i];
+		struct dcb_output *entry = &dcb->entry[i];
 
 		if (entry->location != DCB_LOC_ON_CHIP) {
-			NV_WARN(dev, "Off-chip encoder %d/%d unsupported\n",
+			NV_WARN(drm, "Off-chip encoder %d/%d unsupported\n",
 				entry->type, ffs(entry->or) - 1);
 			continue;
 		}
@@ -380,16 +340,16 @@ nv50_display_create(struct drm_device *dev)
 			continue;
 
 		switch (entry->type) {
-		case OUTPUT_TMDS:
-		case OUTPUT_LVDS:
-		case OUTPUT_DP:
+		case DCB_OUTPUT_TMDS:
+		case DCB_OUTPUT_LVDS:
+		case DCB_OUTPUT_DP:
 			nv50_sor_create(connector, entry);
 			break;
-		case OUTPUT_ANALOG:
+		case DCB_OUTPUT_ANALOG:
 			nv50_dac_create(connector, entry);
 			break;
 		default:
-			NV_WARN(dev, "DCB encoder %d unknown\n", entry->type);
+			NV_WARN(drm, "DCB encoder %d unknown\n", entry->type);
 			continue;
 		}
 	}
@@ -397,14 +357,13 @@ nv50_display_create(struct drm_device *dev)
 	list_for_each_entry_safe(connector, ct,
 				 &dev->mode_config.connector_list, head) {
 		if (!connector->encoder_ids[0]) {
-			NV_WARN(dev, "%s has no encoders, removing\n",
+			NV_WARN(drm, "%s has no encoders, removing\n",
 				drm_get_connector_name(connector));
 			connector->funcs->destroy(connector);
 		}
 	}
 
 	tasklet_init(&priv->tasklet, nv50_display_bh, (unsigned long)dev);
-	nouveau_irq_register(dev, 26, nv50_display_isr);
 
 	ret = nv50_evo_create(dev);
 	if (ret) {
@@ -420,13 +379,16 @@ nv50_display_destroy(struct drm_device *dev)
 {
 	struct nv50_display *disp = nv50_display(dev);
 
-	NV_DEBUG_KMS(dev, "\n");
-
 	nv50_evo_destroy(dev);
-	nouveau_irq_unregister(dev, 26);
 	kfree(disp);
 }
 
+struct nouveau_bo *
+nv50_display_crtc_sema(struct drm_device *dev, int crtc)
+{
+	return nv50_display(dev)->crtc[crtc].sem.bo;
+}
+
 void
 nv50_display_flip_stop(struct drm_crtc *crtc)
 {
@@ -457,7 +419,7 @@ int
 nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 		       struct nouveau_channel *chan)
 {
-	struct drm_nouveau_private *dev_priv = crtc->dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
 	struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
 	struct nv50_display *disp = nv50_display(crtc->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
@@ -477,7 +439,7 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 			return ret;
 		}
 
-		if (dev_priv->chipset < 0xc0) {
+		if (nv_device(drm->device)->chipset < 0xc0) {
 			BEGIN_NV04(chan, 0, 0x0060, 2);
 			OUT_RING  (chan, NvEvoSema0 + nv_crtc->index);
 			OUT_RING  (chan, dispc->sem.offset);
@@ -487,12 +449,12 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 			OUT_RING  (chan, dispc->sem.offset ^ 0x10);
 			OUT_RING  (chan, 0x74b1e000);
 			BEGIN_NV04(chan, 0, 0x0060, 1);
-			if (dev_priv->chipset < 0x84)
+			if (nv_device(drm->device)->chipset < 0x84)
 				OUT_RING  (chan, NvSema);
 			else
-				OUT_RING  (chan, chan->vram_handle);
+				OUT_RING  (chan, chan->vram);
 		} else {
-			u64 offset = nvc0_software_crtc(chan, nv_crtc->index);
+			u64 offset = nvc0_fence_crtc(chan, nv_crtc->index);
 			offset += dispc->sem.offset;
 			BEGIN_NVC0(chan, 0, 0x0010, 4);
 			OUT_RING  (chan, upper_32_bits(offset));
@@ -555,13 +517,13 @@ nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 }
 
 static u16
-nv50_display_script_select(struct drm_device *dev, struct dcb_entry *dcb,
+nv50_display_script_select(struct drm_device *dev, struct dcb_output *dcb,
 			   u32 mc, int pxclk)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_connector *nv_connector = NULL;
 	struct drm_encoder *encoder;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nvbios *bios = &drm->vbios;
 	u32 script = 0, or;
 
 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
@@ -576,7 +538,7 @@ nv50_display_script_select(struct drm_device *dev, struct dcb_entry *dcb,
 
 	or = ffs(dcb->or) - 1;
 	switch (dcb->type) {
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		script = (mc >> 8) & 0xf;
 		if (bios->fp_no_ddc) {
 			if (bios->fp.dual_link)
@@ -609,34 +571,20 @@ nv50_display_script_select(struct drm_device *dev, struct dcb_entry *dcb,
 			    (nv_connector->edid->input & 0x70) >= 0x20)
 				script |= 0x0200;
 		}
-
-		if (nouveau_uscript_lvds >= 0) {
-			NV_INFO(dev, "override script 0x%04x with 0x%04x "
-				     "for output LVDS-%d\n", script,
-				     nouveau_uscript_lvds, or);
-			script = nouveau_uscript_lvds;
-		}
 		break;
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		script = (mc >> 8) & 0xf;
 		if (pxclk >= 165000)
 			script |= 0x0100;
-
-		if (nouveau_uscript_tmds >= 0) {
-			NV_INFO(dev, "override script 0x%04x with 0x%04x "
-				     "for output TMDS-%d\n", script,
-				     nouveau_uscript_tmds, or);
-			script = nouveau_uscript_tmds;
-		}
 		break;
-	case OUTPUT_DP:
+	case DCB_OUTPUT_DP:
 		script = (mc >> 8) & 0xf;
 		break;
-	case OUTPUT_ANALOG:
+	case DCB_OUTPUT_ANALOG:
 		script = 0xff;
 		break;
 	default:
-		NV_ERROR(dev, "modeset on unsupported output type!\n");
+		NV_ERROR(drm, "modeset on unsupported output type!\n");
 		break;
 	}
 
@@ -644,59 +592,18 @@ nv50_display_script_select(struct drm_device *dev, struct dcb_entry *dcb,
 }
 
 static void
-nv50_display_vblank_crtc_handler(struct drm_device *dev, int crtc)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_software_priv *psw = nv_engine(dev, NVOBJ_ENGINE_SW);
-	struct nouveau_software_chan *pch, *tmp;
-
-	list_for_each_entry_safe(pch, tmp, &psw->vblank, vblank.list) {
-		if (pch->vblank.head != crtc)
-			continue;
-
-		spin_lock(&psw->peephole_lock);
-		nv_wr32(dev, 0x001704, pch->vblank.channel);
-		nv_wr32(dev, 0x001710, 0x80000000 | pch->vblank.ctxdma);
-		if (dev_priv->chipset == 0x50) {
-			nv_wr32(dev, 0x001570, pch->vblank.offset);
-			nv_wr32(dev, 0x001574, pch->vblank.value);
-		} else {
-			nv_wr32(dev, 0x060010, pch->vblank.offset);
-			nv_wr32(dev, 0x060014, pch->vblank.value);
-		}
-		spin_unlock(&psw->peephole_lock);
-
-		list_del(&pch->vblank.list);
-		drm_vblank_put(dev, crtc);
-	}
-
-	drm_handle_vblank(dev, crtc);
-}
-
-static void
-nv50_display_vblank_handler(struct drm_device *dev, uint32_t intr)
-{
-	if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_0)
-		nv50_display_vblank_crtc_handler(dev, 0);
-
-	if (intr & NV50_PDISPLAY_INTR_1_VBLANK_CRTC_1)
-		nv50_display_vblank_crtc_handler(dev, 1);
-
-	nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_VBLANK_CRTC);
-}
-
-static void
 nv50_display_unk10_handler(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_display *disp = nv50_display(dev);
-	u32 unk30 = nv_rd32(dev, 0x610030), mc;
-	int i, crtc, or = 0, type = OUTPUT_ANY;
+	u32 unk30 = nv_rd32(device, 0x610030), mc;
+	int i, crtc, or = 0, type = DCB_OUTPUT_ANY;
 
-	NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
+	NV_DEBUG(drm, "0x610030: 0x%08x\n", unk30);
 	disp->irq.dcb = NULL;
 
-	nv_wr32(dev, 0x619494, nv_rd32(dev, 0x619494) & ~8);
+	nv_wr32(device, 0x619494, nv_rd32(device, 0x619494) & ~8);
 
 	/* Determine which CRTC we're dealing with, only 1 ever will be
 	 * signalled at the same time with the current nouveau code.
@@ -711,44 +618,44 @@ nv50_display_unk10_handler(struct drm_device *dev)
 		goto ack;
 
 	/* Find which encoder was connected to the CRTC */
-	for (i = 0; type == OUTPUT_ANY && i < 3; i++) {
-		mc = nv_rd32(dev, NV50_PDISPLAY_DAC_MODE_CTRL_C(i));
-		NV_DEBUG_KMS(dev, "DAC-%d mc: 0x%08x\n", i, mc);
+	for (i = 0; type == DCB_OUTPUT_ANY && i < 3; i++) {
+		mc = nv_rd32(device, NV50_PDISPLAY_DAC_MODE_CTRL_C(i));
+		NV_DEBUG(drm, "DAC-%d mc: 0x%08x\n", i, mc);
 		if (!(mc & (1 << crtc)))
 			continue;
 
 		switch ((mc & 0x00000f00) >> 8) {
-		case 0: type = OUTPUT_ANALOG; break;
-		case 1: type = OUTPUT_TV; break;
+		case 0: type = DCB_OUTPUT_ANALOG; break;
+		case 1: type = DCB_OUTPUT_TV; break;
 		default:
-			NV_ERROR(dev, "invalid mc, DAC-%d: 0x%08x\n", i, mc);
+			NV_ERROR(drm, "invalid mc, DAC-%d: 0x%08x\n", i, mc);
 			goto ack;
 		}
 
 		or = i;
 	}
 
-	for (i = 0; type == OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
-		if (dev_priv->chipset  < 0x90 ||
-		    dev_priv->chipset == 0x92 ||
-		    dev_priv->chipset == 0xa0)
-			mc = nv_rd32(dev, NV50_PDISPLAY_SOR_MODE_CTRL_C(i));
+	for (i = 0; type == DCB_OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
+		if (nv_device(drm->device)->chipset  < 0x90 ||
+		    nv_device(drm->device)->chipset == 0x92 ||
+		    nv_device(drm->device)->chipset == 0xa0)
+			mc = nv_rd32(device, NV50_PDISPLAY_SOR_MODE_CTRL_C(i));
 		else
-			mc = nv_rd32(dev, NV90_PDISPLAY_SOR_MODE_CTRL_C(i));
+			mc = nv_rd32(device, NV90_PDISPLAY_SOR_MODE_CTRL_C(i));
 
-		NV_DEBUG_KMS(dev, "SOR-%d mc: 0x%08x\n", i, mc);
+		NV_DEBUG(drm, "SOR-%d mc: 0x%08x\n", i, mc);
 		if (!(mc & (1 << crtc)))
 			continue;
 
 		switch ((mc & 0x00000f00) >> 8) {
-		case 0: type = OUTPUT_LVDS; break;
-		case 1: type = OUTPUT_TMDS; break;
-		case 2: type = OUTPUT_TMDS; break;
-		case 5: type = OUTPUT_TMDS; break;
-		case 8: type = OUTPUT_DP; break;
-		case 9: type = OUTPUT_DP; break;
+		case 0: type = DCB_OUTPUT_LVDS; break;
+		case 1: type = DCB_OUTPUT_TMDS; break;
+		case 2: type = DCB_OUTPUT_TMDS; break;
+		case 5: type = DCB_OUTPUT_TMDS; break;
+		case 8: type = DCB_OUTPUT_DP; break;
+		case 9: type = DCB_OUTPUT_DP; break;
 		default:
-			NV_ERROR(dev, "invalid mc, SOR-%d: 0x%08x\n", i, mc);
+			NV_ERROR(drm, "invalid mc, SOR-%d: 0x%08x\n", i, mc);
 			goto ack;
 		}
 
@@ -756,12 +663,12 @@ nv50_display_unk10_handler(struct drm_device *dev)
 	}
 
 	/* There was no encoder to disable */
-	if (type == OUTPUT_ANY)
+	if (type == DCB_OUTPUT_ANY)
 		goto ack;
 
 	/* Disable the encoder */
-	for (i = 0; i < dev_priv->vbios.dcb.entries; i++) {
-		struct dcb_entry *dcb = &dev_priv->vbios.dcb.entry[i];
+	for (i = 0; i < drm->vbios.dcb.entries; i++) {
+		struct dcb_output *dcb = &drm->vbios.dcb.entry[i];
 
 		if (dcb->type == type && (dcb->or & (1 << or))) {
 			nouveau_bios_run_display_table(dev, 0, -1, dcb, -1);
@@ -770,22 +677,23 @@ nv50_display_unk10_handler(struct drm_device *dev)
 		}
 	}
 
-	NV_ERROR(dev, "no dcb for %d %d 0x%08x\n", or, type, mc);
+	NV_ERROR(drm, "no dcb for %d %d 0x%08x\n", or, type, mc);
 ack:
-	nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK10);
-	nv_wr32(dev, 0x610030, 0x80000000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK10);
+	nv_wr32(device, 0x610030, 0x80000000);
 }
 
 static void
 nv50_display_unk20_handler(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_display *disp = nv50_display(dev);
-	u32 unk30 = nv_rd32(dev, 0x610030), tmp, pclk, script, mc = 0;
-	struct dcb_entry *dcb;
-	int i, crtc, or = 0, type = OUTPUT_ANY;
+	u32 unk30 = nv_rd32(device, 0x610030), tmp, pclk, script, mc = 0;
+	struct dcb_output *dcb;
+	int i, crtc, or = 0, type = DCB_OUTPUT_ANY;
 
-	NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
+	NV_DEBUG(drm, "0x610030: 0x%08x\n", unk30);
 	dcb = disp->irq.dcb;
 	if (dcb) {
 		nouveau_bios_run_display_table(dev, 0, -2, dcb, -1);
@@ -795,86 +703,86 @@ nv50_display_unk20_handler(struct drm_device *dev)
 	/* CRTC clock change requested? */
 	crtc = ffs((unk30 & 0x00000600) >> 9) - 1;
 	if (crtc >= 0) {
-		pclk  = nv_rd32(dev, NV50_PDISPLAY_CRTC_P(crtc, CLOCK));
+		pclk  = nv_rd32(device, NV50_PDISPLAY_CRTC_P(crtc, CLOCK));
 		pclk &= 0x003fffff;
 		if (pclk)
 			nv50_crtc_set_clock(dev, crtc, pclk);
 
-		tmp = nv_rd32(dev, NV50_PDISPLAY_CRTC_CLK_CTRL2(crtc));
+		tmp = nv_rd32(device, NV50_PDISPLAY_CRTC_CLK_CTRL2(crtc));
 		tmp &= ~0x000000f;
-		nv_wr32(dev, NV50_PDISPLAY_CRTC_CLK_CTRL2(crtc), tmp);
+		nv_wr32(device, NV50_PDISPLAY_CRTC_CLK_CTRL2(crtc), tmp);
 	}
 
 	/* Nothing needs to be done for the encoder */
 	crtc = ffs((unk30 & 0x00000180) >> 7) - 1;
 	if (crtc < 0)
 		goto ack;
-	pclk  = nv_rd32(dev, NV50_PDISPLAY_CRTC_P(crtc, CLOCK)) & 0x003fffff;
+	pclk  = nv_rd32(device, NV50_PDISPLAY_CRTC_P(crtc, CLOCK)) & 0x003fffff;
 
 	/* Find which encoder is connected to the CRTC */
-	for (i = 0; type == OUTPUT_ANY && i < 3; i++) {
-		mc = nv_rd32(dev, NV50_PDISPLAY_DAC_MODE_CTRL_P(i));
-		NV_DEBUG_KMS(dev, "DAC-%d mc: 0x%08x\n", i, mc);
+	for (i = 0; type == DCB_OUTPUT_ANY && i < 3; i++) {
+		mc = nv_rd32(device, NV50_PDISPLAY_DAC_MODE_CTRL_P(i));
+		NV_DEBUG(drm, "DAC-%d mc: 0x%08x\n", i, mc);
 		if (!(mc & (1 << crtc)))
 			continue;
 
 		switch ((mc & 0x00000f00) >> 8) {
-		case 0: type = OUTPUT_ANALOG; break;
-		case 1: type = OUTPUT_TV; break;
+		case 0: type = DCB_OUTPUT_ANALOG; break;
+		case 1: type = DCB_OUTPUT_TV; break;
 		default:
-			NV_ERROR(dev, "invalid mc, DAC-%d: 0x%08x\n", i, mc);
+			NV_ERROR(drm, "invalid mc, DAC-%d: 0x%08x\n", i, mc);
 			goto ack;
 		}
 
 		or = i;
 	}
 
-	for (i = 0; type == OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
-		if (dev_priv->chipset  < 0x90 ||
-		    dev_priv->chipset == 0x92 ||
-		    dev_priv->chipset == 0xa0)
-			mc = nv_rd32(dev, NV50_PDISPLAY_SOR_MODE_CTRL_P(i));
+	for (i = 0; type == DCB_OUTPUT_ANY && i < nv50_sor_nr(dev); i++) {
+		if (nv_device(drm->device)->chipset  < 0x90 ||
+		    nv_device(drm->device)->chipset == 0x92 ||
+		    nv_device(drm->device)->chipset == 0xa0)
+			mc = nv_rd32(device, NV50_PDISPLAY_SOR_MODE_CTRL_P(i));
 		else
-			mc = nv_rd32(dev, NV90_PDISPLAY_SOR_MODE_CTRL_P(i));
+			mc = nv_rd32(device, NV90_PDISPLAY_SOR_MODE_CTRL_P(i));
 
-		NV_DEBUG_KMS(dev, "SOR-%d mc: 0x%08x\n", i, mc);
+		NV_DEBUG(drm, "SOR-%d mc: 0x%08x\n", i, mc);
 		if (!(mc & (1 << crtc)))
 			continue;
 
 		switch ((mc & 0x00000f00) >> 8) {
-		case 0: type = OUTPUT_LVDS; break;
-		case 1: type = OUTPUT_TMDS; break;
-		case 2: type = OUTPUT_TMDS; break;
-		case 5: type = OUTPUT_TMDS; break;
-		case 8: type = OUTPUT_DP; break;
-		case 9: type = OUTPUT_DP; break;
+		case 0: type = DCB_OUTPUT_LVDS; break;
+		case 1: type = DCB_OUTPUT_TMDS; break;
+		case 2: type = DCB_OUTPUT_TMDS; break;
+		case 5: type = DCB_OUTPUT_TMDS; break;
+		case 8: type = DCB_OUTPUT_DP; break;
+		case 9: type = DCB_OUTPUT_DP; break;
 		default:
-			NV_ERROR(dev, "invalid mc, SOR-%d: 0x%08x\n", i, mc);
+			NV_ERROR(drm, "invalid mc, SOR-%d: 0x%08x\n", i, mc);
 			goto ack;
 		}
 
 		or = i;
 	}
 
-	if (type == OUTPUT_ANY)
+	if (type == DCB_OUTPUT_ANY)
 		goto ack;
 
 	/* Enable the encoder */
-	for (i = 0; i < dev_priv->vbios.dcb.entries; i++) {
-		dcb = &dev_priv->vbios.dcb.entry[i];
+	for (i = 0; i < drm->vbios.dcb.entries; i++) {
+		dcb = &drm->vbios.dcb.entry[i];
 		if (dcb->type == type && (dcb->or & (1 << or)))
 			break;
 	}
 
-	if (i == dev_priv->vbios.dcb.entries) {
-		NV_ERROR(dev, "no dcb for %d %d 0x%08x\n", or, type, mc);
+	if (i == drm->vbios.dcb.entries) {
+		NV_ERROR(drm, "no dcb for %d %d 0x%08x\n", or, type, mc);
 		goto ack;
 	}
 
 	script = nv50_display_script_select(dev, dcb, mc, pclk);
 	nouveau_bios_run_display_table(dev, script, pclk, dcb, -1);
 
-	if (type == OUTPUT_DP) {
+	if (type == DCB_OUTPUT_DP) {
 		int link = !(dcb->dpconf.sor.link & 1);
 		if ((mc & 0x000f0000) == 0x00020000)
 			nv50_sor_dp_calc_tu(dev, or, link, pclk, 18);
@@ -882,14 +790,14 @@ nv50_display_unk20_handler(struct drm_device *dev)
 			nv50_sor_dp_calc_tu(dev, or, link, pclk, 24);
 	}
 
-	if (dcb->type != OUTPUT_ANALOG) {
-		tmp = nv_rd32(dev, NV50_PDISPLAY_SOR_CLK_CTRL2(or));
+	if (dcb->type != DCB_OUTPUT_ANALOG) {
+		tmp = nv_rd32(device, NV50_PDISPLAY_SOR_CLK_CTRL2(or));
 		tmp &= ~0x00000f0f;
 		if (script & 0x0100)
 			tmp |= 0x00000101;
-		nv_wr32(dev, NV50_PDISPLAY_SOR_CLK_CTRL2(or), tmp);
+		nv_wr32(device, NV50_PDISPLAY_SOR_CLK_CTRL2(or), tmp);
 	} else {
-		nv_wr32(dev, NV50_PDISPLAY_DAC_CLK_CTRL2(or), 0);
+		nv_wr32(device, NV50_PDISPLAY_DAC_CLK_CTRL2(or), 0);
 	}
 
 	disp->irq.dcb = dcb;
@@ -897,8 +805,8 @@ nv50_display_unk20_handler(struct drm_device *dev)
 	disp->irq.script = script;
 
 ack:
-	nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK20);
-	nv_wr32(dev, 0x610030, 0x80000000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK20);
+	nv_wr32(device, 0x610030, 0x80000000);
 }
 
 /* If programming a TMDS output on a SOR that can also be configured for
@@ -910,23 +818,24 @@ ack:
  * programmed for DisplayPort.
  */
 static void
-nv50_display_unk40_dp_set_tmds(struct drm_device *dev, struct dcb_entry *dcb)
+nv50_display_unk40_dp_set_tmds(struct drm_device *dev, struct dcb_output *dcb)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	int or = ffs(dcb->or) - 1, link = !(dcb->dpconf.sor.link & 1);
 	struct drm_encoder *encoder;
 	u32 tmp;
 
-	if (dcb->type != OUTPUT_TMDS)
+	if (dcb->type != DCB_OUTPUT_TMDS)
 		return;
 
 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 		struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 
-		if (nv_encoder->dcb->type == OUTPUT_DP &&
+		if (nv_encoder->dcb->type == DCB_OUTPUT_DP &&
 		    nv_encoder->dcb->or & (1 << or)) {
-			tmp  = nv_rd32(dev, NV50_SOR_DP_CTRL(or, link));
+			tmp  = nv_rd32(device, NV50_SOR_DP_CTRL(or, link));
 			tmp &= ~NV50_SOR_DP_CTRL_ENABLED;
-			nv_wr32(dev, NV50_SOR_DP_CTRL(or, link), tmp);
+			nv_wr32(device, NV50_SOR_DP_CTRL(or, link), tmp);
 			break;
 		}
 	}
@@ -935,12 +844,14 @@ nv50_display_unk40_dp_set_tmds(struct drm_device *dev, struct dcb_entry *dcb)
 static void
 nv50_display_unk40_handler(struct drm_device *dev)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_display *disp = nv50_display(dev);
-	struct dcb_entry *dcb = disp->irq.dcb;
+	struct dcb_output *dcb = disp->irq.dcb;
 	u16 script = disp->irq.script;
-	u32 unk30 = nv_rd32(dev, 0x610030), pclk = disp->irq.pclk;
+	u32 unk30 = nv_rd32(device, 0x610030), pclk = disp->irq.pclk;
 
-	NV_DEBUG_KMS(dev, "0x610030: 0x%08x\n", unk30);
+	NV_DEBUG(drm, "0x610030: 0x%08x\n", unk30);
 	disp->irq.dcb = NULL;
 	if (!dcb)
 		goto ack;
@@ -949,21 +860,23 @@ nv50_display_unk40_handler(struct drm_device *dev)
 	nv50_display_unk40_dp_set_tmds(dev, dcb);
 
 ack:
-	nv_wr32(dev, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK40);
-	nv_wr32(dev, 0x610030, 0x80000000);
-	nv_wr32(dev, 0x619494, nv_rd32(dev, 0x619494) | 8);
+	nv_wr32(device, NV50_PDISPLAY_INTR_1, NV50_PDISPLAY_INTR_1_CLK_UNK40);
+	nv_wr32(device, 0x610030, 0x80000000);
+	nv_wr32(device, 0x619494, nv_rd32(device, 0x619494) | 8);
 }
 
 static void
 nv50_display_bh(unsigned long data)
 {
 	struct drm_device *dev = (struct drm_device *)data;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	for (;;) {
-		uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0);
-		uint32_t intr1 = nv_rd32(dev, NV50_PDISPLAY_INTR_1);
+		uint32_t intr0 = nv_rd32(device, NV50_PDISPLAY_INTR_0);
+		uint32_t intr1 = nv_rd32(device, NV50_PDISPLAY_INTR_1);
 
-		NV_DEBUG_KMS(dev, "PDISPLAY_INTR_BH 0x%08x 0x%08x\n", intr0, intr1);
+		NV_DEBUG(drm, "PDISPLAY_INTR_BH 0x%08x 0x%08x\n", intr0, intr1);
 
 		if (intr1 & NV50_PDISPLAY_INTR_1_CLK_UNK10)
 			nv50_display_unk10_handler(dev);
@@ -977,13 +890,15 @@ nv50_display_bh(unsigned long data)
 			break;
 	}
 
-	nv_wr32(dev, NV03_PMC_INTR_EN_0, 1);
+	nv_wr32(device, NV03_PMC_INTR_EN_0, 1);
 }
 
 static void
 nv50_display_error_handler(struct drm_device *dev)
 {
-	u32 channels = (nv_rd32(dev, NV50_PDISPLAY_INTR_0) & 0x001f0000) >> 16;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	u32 channels = (nv_rd32(device, NV50_PDISPLAY_INTR_0) & 0x001f0000) >> 16;
 	u32 addr, data;
 	int chid;
 
@@ -991,29 +906,31 @@ nv50_display_error_handler(struct drm_device *dev)
 		if (!(channels & (1 << chid)))
 			continue;
 
-		nv_wr32(dev, NV50_PDISPLAY_INTR_0, 0x00010000 << chid);
-		addr = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid));
-		data = nv_rd32(dev, NV50_PDISPLAY_TRAPPED_DATA(chid));
-		NV_ERROR(dev, "EvoCh %d Mthd 0x%04x Data 0x%08x "
+		nv_wr32(device, NV50_PDISPLAY_INTR_0, 0x00010000 << chid);
+		addr = nv_rd32(device, NV50_PDISPLAY_TRAPPED_ADDR(chid));
+		data = nv_rd32(device, NV50_PDISPLAY_TRAPPED_DATA(chid));
+		NV_ERROR(drm, "EvoCh %d Mthd 0x%04x Data 0x%08x "
 			      "(0x%04x 0x%02x)\n", chid,
 			 addr & 0xffc, data, addr >> 16, (addr >> 12) & 0xf);
 
-		nv_wr32(dev, NV50_PDISPLAY_TRAPPED_ADDR(chid), 0x90000000);
+		nv_wr32(device, NV50_PDISPLAY_TRAPPED_ADDR(chid), 0x90000000);
 	}
 }
 
-static void
-nv50_display_isr(struct drm_device *dev)
+void
+nv50_display_intr(struct drm_device *dev)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_display *disp = nv50_display(dev);
 	uint32_t delayed = 0;
 
-	while (nv_rd32(dev, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_DISPLAY) {
-		uint32_t intr0 = nv_rd32(dev, NV50_PDISPLAY_INTR_0);
-		uint32_t intr1 = nv_rd32(dev, NV50_PDISPLAY_INTR_1);
+	while (nv_rd32(device, NV50_PMC_INTR_0) & NV50_PMC_INTR_0_DISPLAY) {
+		uint32_t intr0 = nv_rd32(device, NV50_PDISPLAY_INTR_0);
+		uint32_t intr1 = nv_rd32(device, NV50_PDISPLAY_INTR_1);
 		uint32_t clock;
 
-		NV_DEBUG_KMS(dev, "PDISPLAY_INTR 0x%08x 0x%08x\n", intr0, intr1);
+		NV_DEBUG(drm, "PDISPLAY_INTR 0x%08x 0x%08x\n", intr0, intr1);
 
 		if (!intr0 && !(intr1 & ~delayed))
 			break;
@@ -1024,29 +941,29 @@ nv50_display_isr(struct drm_device *dev)
 		}
 
 		if (intr1 & NV50_PDISPLAY_INTR_1_VBLANK_CRTC) {
-			nv50_display_vblank_handler(dev, intr1);
 			intr1 &= ~NV50_PDISPLAY_INTR_1_VBLANK_CRTC;
+			delayed |= NV50_PDISPLAY_INTR_1_VBLANK_CRTC;
 		}
 
 		clock = (intr1 & (NV50_PDISPLAY_INTR_1_CLK_UNK10 |
 				  NV50_PDISPLAY_INTR_1_CLK_UNK20 |
 				  NV50_PDISPLAY_INTR_1_CLK_UNK40));
 		if (clock) {
-			nv_wr32(dev, NV03_PMC_INTR_EN_0, 0);
+			nv_wr32(device, NV03_PMC_INTR_EN_0, 0);
 			tasklet_schedule(&disp->tasklet);
 			delayed |= clock;
 			intr1 &= ~clock;
 		}
 
 		if (intr0) {
-			NV_ERROR(dev, "unknown PDISPLAY_INTR_0: 0x%08x\n", intr0);
-			nv_wr32(dev, NV50_PDISPLAY_INTR_0, intr0);
+			NV_ERROR(drm, "unknown PDISPLAY_INTR_0: 0x%08x\n", intr0);
+			nv_wr32(device, NV50_PDISPLAY_INTR_0, intr0);
 		}
 
 		if (intr1) {
-			NV_ERROR(dev,
+			NV_ERROR(drm,
 				 "unknown PDISPLAY_INTR_1: 0x%08x\n", intr1);
-			nv_wr32(dev, NV50_PDISPLAY_INTR_1, intr1);
+			nv_wr32(device, NV50_PDISPLAY_INTR_1, intr1);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h
index e9db9b97f04..973554d8a7a 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.h
+++ b/drivers/gpu/drm/nouveau/nv50_display.h
@@ -27,13 +27,9 @@
 #ifndef __NV50_DISPLAY_H__
 #define __NV50_DISPLAY_H__
 
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_dma.h"
-#include "nouveau_reg.h"
+#include "nouveau_display.h"
 #include "nouveau_crtc.h"
-#include "nouveau_software.h"
+#include "nouveau_reg.h"
 #include "nv50_evo.h"
 
 struct nv50_display_crtc {
@@ -47,13 +43,16 @@ struct nv50_display_crtc {
 
 struct nv50_display {
 	struct nouveau_channel *master;
-	struct nouveau_gpuobj *ntfy;
+
+	struct nouveau_gpuobj *ramin;
+	u32 dmao;
+	u32 hash;
 
 	struct nv50_display_crtc crtc[2];
 
 	struct tasklet_struct tasklet;
 	struct {
-		struct dcb_entry *dcb;
+		struct dcb_output *dcb;
 		u16 script;
 		u32 pclk;
 	} irq;
@@ -62,8 +61,7 @@ struct nv50_display {
 static inline struct nv50_display *
 nv50_display(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return dev_priv->engine.display.priv;
+	return nouveau_display(dev)->priv;
 }
 
 int nv50_display_early_init(struct drm_device *dev);
@@ -72,6 +70,7 @@ int nv50_display_create(struct drm_device *dev);
 int nv50_display_init(struct drm_device *dev);
 void nv50_display_fini(struct drm_device *dev);
 void nv50_display_destroy(struct drm_device *dev);
+void nv50_display_intr(struct drm_device *);
 int nv50_crtc_blank(struct nouveau_crtc *, bool blank);
 int nv50_crtc_set_clock(struct drm_device *, int head, int pclk);
 
@@ -91,4 +90,17 @@ void nv50_evo_dmaobj_init(struct nouveau_gpuobj *, u32 memtype, u64 base,
 int  nv50_evo_dmaobj_new(struct nouveau_channel *, u32 handle, u32 memtype,
 			 u64 base, u64 size, struct nouveau_gpuobj **);
 
+int  nvd0_display_create(struct drm_device *);
+void nvd0_display_destroy(struct drm_device *);
+int  nvd0_display_init(struct drm_device *);
+void nvd0_display_fini(struct drm_device *);
+void nvd0_display_intr(struct drm_device *);
+
+void nvd0_display_flip_stop(struct drm_crtc *);
+int  nvd0_display_flip_next(struct drm_crtc *, struct drm_framebuffer *,
+			    struct nouveau_channel *, u32 swap_interval);
+
+struct nouveau_bo *nv50_display_crtc_sema(struct drm_device *, int head);
+struct nouveau_bo *nvd0_display_crtc_sema(struct drm_device *, int head);
+
 #endif /* __NV50_DISPLAY_H__ */
diff --git a/drivers/gpu/drm/nouveau/nv50_evo.c b/drivers/gpu/drm/nouveau/nv50_evo.c
index ddcd5559582..9f6f55cdfa7 100644
--- a/drivers/gpu/drm/nouveau/nv50_evo.c
+++ b/drivers/gpu/drm/nouveau/nv50_evo.c
@@ -22,13 +22,31 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nv50_display.h"
 
+#include <core/gpuobj.h>
+
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
+static u32
+nv50_evo_rd32(struct nouveau_object *object, u32 addr)
+{
+	void __iomem *iomem = object->oclass->ofuncs->rd08;
+	return ioread32_native(iomem + addr);
+}
+
+static void
+nv50_evo_wr32(struct nouveau_object *object, u32 addr, u32 data)
+{
+	void __iomem *iomem = object->oclass->ofuncs->rd08;
+	iowrite32_native(data, iomem + addr);
+}
+
 static void
 nv50_evo_channel_del(struct nouveau_channel **pevo)
 {
@@ -38,26 +56,29 @@ nv50_evo_channel_del(struct nouveau_channel **pevo)
 		return;
 	*pevo = NULL;
 
-	nouveau_ramht_ref(NULL, &evo->ramht, evo);
-	nouveau_gpuobj_channel_takedown(evo);
-	nouveau_bo_unmap(evo->pushbuf_bo);
-	nouveau_bo_ref(NULL, &evo->pushbuf_bo);
+	nouveau_bo_unmap(evo->push.buffer);
+	nouveau_bo_ref(NULL, &evo->push.buffer);
 
-	if (evo->user)
-		iounmap(evo->user);
+	if (evo->object)
+		iounmap(evo->object->oclass->ofuncs);
 
 	kfree(evo);
 }
 
-void
-nv50_evo_dmaobj_init(struct nouveau_gpuobj *obj, u32 memtype, u64 base, u64 size)
+int
+nv50_evo_dmaobj_new(struct nouveau_channel *evo, u32 handle, u32 memtype,
+		    u64 base, u64 size, struct nouveau_gpuobj **pobj)
 {
-	struct drm_nouveau_private *dev_priv = obj->dev->dev_private;
+	struct drm_device *dev = evo->fence;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nv50_display *disp = nv50_display(dev);
+	u32 dmao = disp->dmao;
+	u32 hash = disp->hash;
 	u32 flags5;
 
-	if (dev_priv->chipset < 0xc0) {
+	if (nv_device(drm->device)->chipset < 0xc0) {
 		/* not supported on 0x50, specified in format mthd */
-		if (dev_priv->chipset == 0x50)
+		if (nv_device(drm->device)->chipset == 0x50)
 			memtype = 0;
 		flags5 = 0x00010000;
 	} else {
@@ -67,42 +88,28 @@ nv50_evo_dmaobj_init(struct nouveau_gpuobj *obj, u32 memtype, u64 base, u64 size
 			flags5 = 0x00020000;
 	}
 
-	nv50_gpuobj_dma_init(obj, 0, 0x3d, base, size, NV_MEM_TARGET_VRAM,
-			     NV_MEM_ACCESS_RW, (memtype >> 8) & 0xff, 0);
-	nv_wo32(obj, 0x14, flags5);
-	dev_priv->engine.instmem.flush(obj->dev);
-}
+	nv_wo32(disp->ramin, dmao + 0x00, 0x0019003d | (memtype << 22));
+	nv_wo32(disp->ramin, dmao + 0x04, lower_32_bits(base + size - 1));
+	nv_wo32(disp->ramin, dmao + 0x08, lower_32_bits(base));
+	nv_wo32(disp->ramin, dmao + 0x0c, upper_32_bits(base + size - 1) << 24 |
+					  upper_32_bits(base));
+	nv_wo32(disp->ramin, dmao + 0x10, 0x00000000);
+	nv_wo32(disp->ramin, dmao + 0x14, flags5);
 
-int
-nv50_evo_dmaobj_new(struct nouveau_channel *evo, u32 handle, u32 memtype,
-		    u64 base, u64 size, struct nouveau_gpuobj **pobj)
-{
-	struct nv50_display *disp = nv50_display(evo->dev);
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(evo->dev, disp->master, 6*4, 32, 0, &obj);
-	if (ret)
-		return ret;
-	obj->engine = NVOBJ_ENGINE_DISPLAY;
-
-	nv50_evo_dmaobj_init(obj, memtype, base, size);
-
-	ret = nouveau_ramht_insert(evo, handle, obj);
-	if (ret)
-		goto out;
+	nv_wo32(disp->ramin, hash + 0x00, handle);
+	nv_wo32(disp->ramin, hash + 0x04, (evo->handle << 28) | (dmao << 10) |
+					   evo->handle);
 
-	if (pobj)
-		nouveau_gpuobj_ref(obj, pobj);
-out:
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
+	disp->dmao += 0x20;
+	disp->hash += 0x08;
+	return 0;
 }
 
 static int
 nv50_evo_channel_new(struct drm_device *dev, int chid,
 		     struct nouveau_channel **pevo)
 {
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_display *disp = nv50_display(dev);
 	struct nouveau_channel *evo;
 	int ret;
@@ -112,79 +119,84 @@ nv50_evo_channel_new(struct drm_device *dev, int chid,
 		return -ENOMEM;
 	*pevo = evo;
 
-	evo->id = chid;
-	evo->dev = dev;
+	evo->drm = drm;
+	evo->handle = chid;
+	evo->fence = dev;
 	evo->user_get = 4;
 	evo->user_put = 0;
 
 	ret = nouveau_bo_new(dev, 4096, 0, TTM_PL_FLAG_VRAM, 0, 0, NULL,
-			     &evo->pushbuf_bo);
+			     &evo->push.buffer);
 	if (ret == 0)
-		ret = nouveau_bo_pin(evo->pushbuf_bo, TTM_PL_FLAG_VRAM);
+		ret = nouveau_bo_pin(evo->push.buffer, TTM_PL_FLAG_VRAM);
 	if (ret) {
-		NV_ERROR(dev, "Error creating EVO DMA push buffer: %d\n", ret);
+		NV_ERROR(drm, "Error creating EVO DMA push buffer: %d\n", ret);
 		nv50_evo_channel_del(pevo);
 		return ret;
 	}
 
-	ret = nouveau_bo_map(evo->pushbuf_bo);
+	ret = nouveau_bo_map(evo->push.buffer);
 	if (ret) {
-		NV_ERROR(dev, "Error mapping EVO DMA push buffer: %d\n", ret);
+		NV_ERROR(drm, "Error mapping EVO DMA push buffer: %d\n", ret);
 		nv50_evo_channel_del(pevo);
 		return ret;
 	}
 
-	evo->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			    NV50_PDISPLAY_USER(evo->id), PAGE_SIZE);
-	if (!evo->user) {
-		NV_ERROR(dev, "Error mapping EVO control regs.\n");
-		nv50_evo_channel_del(pevo);
-		return -ENOMEM;
-	}
-
-	/* bind primary evo channel's ramht to the channel */
-	if (disp->master && evo != disp->master)
-		nouveau_ramht_ref(disp->master->ramht, &evo->ramht, NULL);
-
+	evo->object = kzalloc(sizeof(*evo->object), GFP_KERNEL);
+#ifdef NOUVEAU_OBJECT_MAGIC
+	evo->object->_magic = NOUVEAU_OBJECT_MAGIC;
+#endif
+	evo->object->parent = nv_object(disp->ramin)->parent;
+	evo->object->engine = nv_object(disp->ramin)->engine;
+	evo->object->oclass =
+		kzalloc(sizeof(*evo->object->oclass), GFP_KERNEL);
+	evo->object->oclass->ofuncs =
+		kzalloc(sizeof(*evo->object->oclass->ofuncs), GFP_KERNEL);
+	evo->object->oclass->ofuncs->rd32 = nv50_evo_rd32;
+	evo->object->oclass->ofuncs->wr32 = nv50_evo_wr32;
+	evo->object->oclass->ofuncs->rd08 =
+		ioremap(pci_resource_start(dev->pdev, 0) +
+			NV50_PDISPLAY_USER(evo->handle), PAGE_SIZE);
 	return 0;
 }
 
 static int
 nv50_evo_channel_init(struct nouveau_channel *evo)
 {
-	struct drm_device *dev = evo->dev;
-	int id = evo->id, ret, i;
-	u64 pushbuf = evo->pushbuf_bo->bo.offset;
+	struct nouveau_drm *drm = evo->drm;
+	struct nouveau_device *device = nv_device(drm->device);
+	int id = evo->handle, ret, i;
+	u64 pushbuf = evo->push.buffer->bo.offset;
 	u32 tmp;
 
-	tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
+	tmp = nv_rd32(device, NV50_PDISPLAY_EVO_CTRL(id));
 	if ((tmp & 0x009f0000) == 0x00020000)
-		nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00800000);
+		nv_wr32(device, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00800000);
 
-	tmp = nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id));
+	tmp = nv_rd32(device, NV50_PDISPLAY_EVO_CTRL(id));
 	if ((tmp & 0x003f0000) == 0x00030000)
-		nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00600000);
+		nv_wr32(device, NV50_PDISPLAY_EVO_CTRL(id), tmp | 0x00600000);
 
 	/* initialise fifo */
-	nv_wr32(dev, NV50_PDISPLAY_EVO_DMA_CB(id), pushbuf >> 8 |
+	nv_wr32(device, NV50_PDISPLAY_EVO_DMA_CB(id), pushbuf >> 8 |
 		     NV50_PDISPLAY_EVO_DMA_CB_LOCATION_VRAM |
 		     NV50_PDISPLAY_EVO_DMA_CB_VALID);
-	nv_wr32(dev, NV50_PDISPLAY_EVO_UNK2(id), 0x00010000);
-	nv_wr32(dev, NV50_PDISPLAY_EVO_HASH_TAG(id), id);
-	nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), NV50_PDISPLAY_EVO_CTRL_DMA,
+	nv_wr32(device, NV50_PDISPLAY_EVO_UNK2(id), 0x00010000);
+	nv_wr32(device, NV50_PDISPLAY_EVO_HASH_TAG(id), id);
+	nv_mask(device, NV50_PDISPLAY_EVO_CTRL(id), NV50_PDISPLAY_EVO_CTRL_DMA,
 		     NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
 
-	nv_wr32(dev, NV50_PDISPLAY_USER_PUT(id), 0x00000000);
-	nv_wr32(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x01000003 |
+	nv_wr32(device, NV50_PDISPLAY_USER_PUT(id), 0x00000000);
+	nv_wr32(device, NV50_PDISPLAY_EVO_CTRL(id), 0x01000003 |
 		     NV50_PDISPLAY_EVO_CTRL_DMA_ENABLED);
-	if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x80000000, 0x00000000)) {
-		NV_ERROR(dev, "EvoCh %d init timeout: 0x%08x\n", id,
-			 nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
+	if (!nv_wait(device, NV50_PDISPLAY_EVO_CTRL(id), 0x80000000, 0x00000000)) {
+		NV_ERROR(drm, "EvoCh %d init timeout: 0x%08x\n", id,
+			 nv_rd32(device, NV50_PDISPLAY_EVO_CTRL(id)));
 		return -EBUSY;
 	}
 
 	/* enable error reporting on the channel */
-	nv_mask(dev, 0x610028, 0x00000000, 0x00010001 << id);
+	nv_mask(device, 0x610028, 0x00000000, 0x00010001 << id);
 
 	evo->dma.max = (4096/4) - 2;
 	evo->dma.max &= ~7;
@@ -205,16 +217,17 @@ nv50_evo_channel_init(struct nouveau_channel *evo)
 static void
 nv50_evo_channel_fini(struct nouveau_channel *evo)
 {
-	struct drm_device *dev = evo->dev;
-	int id = evo->id;
-
-	nv_mask(dev, 0x610028, 0x00010001 << id, 0x00000000);
-	nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00001010, 0x00001000);
-	nv_wr32(dev, NV50_PDISPLAY_INTR_0, (1 << id));
-	nv_mask(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x00000003, 0x00000000);
-	if (!nv_wait(dev, NV50_PDISPLAY_EVO_CTRL(id), 0x001e0000, 0x00000000)) {
-		NV_ERROR(dev, "EvoCh %d takedown timeout: 0x%08x\n", id,
-			 nv_rd32(dev, NV50_PDISPLAY_EVO_CTRL(id)));
+	struct nouveau_drm *drm = evo->drm;
+	struct nouveau_device *device = nv_device(drm->device);
+	int id = evo->handle;
+
+	nv_mask(device, 0x610028, 0x00010001 << id, 0x00000000);
+	nv_mask(device, NV50_PDISPLAY_EVO_CTRL(id), 0x00001010, 0x00001000);
+	nv_wr32(device, NV50_PDISPLAY_INTR_0, (1 << id));
+	nv_mask(device, NV50_PDISPLAY_EVO_CTRL(id), 0x00000003, 0x00000000);
+	if (!nv_wait(device, NV50_PDISPLAY_EVO_CTRL(id), 0x001e0000, 0x00000000)) {
+		NV_ERROR(drm, "EvoCh %d takedown timeout: 0x%08x\n", id,
+			 nv_rd32(device, NV50_PDISPLAY_EVO_CTRL(id)));
 	}
 }
 
@@ -231,93 +244,66 @@ nv50_evo_destroy(struct drm_device *dev)
 		}
 		nv50_evo_channel_del(&disp->crtc[i].sync);
 	}
-	nouveau_gpuobj_ref(NULL, &disp->ntfy);
 	nv50_evo_channel_del(&disp->master);
+	nouveau_gpuobj_ref(NULL, &disp->ramin);
 }
 
 int
 nv50_evo_create(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_fb *pfb = nouveau_fb(drm->device);
 	struct nv50_display *disp = nv50_display(dev);
-	struct nouveau_gpuobj *ramht = NULL;
 	struct nouveau_channel *evo;
 	int ret, i, j;
 
-	/* create primary evo channel, the one we use for modesetting
-	 * purporses
-	 */
-	ret = nv50_evo_channel_new(dev, 0, &disp->master);
-	if (ret)
-		return ret;
-	evo = disp->master;
-
 	/* setup object management on it, any other evo channel will
 	 * use this also as there's no per-channel support on the
 	 * hardware
 	 */
-	ret = nouveau_gpuobj_new(dev, NULL, 32768, 65536,
-				 NVOBJ_FLAG_ZERO_ALLOC, &evo->ramin);
-	if (ret) {
-		NV_ERROR(dev, "Error allocating EVO channel memory: %d\n", ret);
-		goto err;
-	}
-
-	ret = drm_mm_init(&evo->ramin_heap, 0, 32768);
+	ret = nouveau_gpuobj_new(drm->device, NULL, 32768, 65536,
+				 NVOBJ_FLAG_ZERO_ALLOC, &disp->ramin);
 	if (ret) {
-		NV_ERROR(dev, "Error initialising EVO PRAMIN heap: %d\n", ret);
+		NV_ERROR(drm, "Error allocating EVO channel memory: %d\n", ret);
 		goto err;
 	}
 
-	ret = nouveau_gpuobj_new(dev, evo, 4096, 16, 0, &ramht);
-	if (ret) {
-		NV_ERROR(dev, "Unable to allocate EVO RAMHT: %d\n", ret);
-		goto err;
-	}
-
-	ret = nouveau_ramht_new(dev, ramht, &evo->ramht);
-	nouveau_gpuobj_ref(NULL, &ramht);
-	if (ret)
-		goto err;
+	disp->hash = 0x0000;
+	disp->dmao = 0x1000;
 
-	/* not sure exactly what this is..
-	 *
-	 * the first dword of the structure is used by nvidia to wait on
-	 * full completion of an EVO "update" command.
-	 *
-	 * method 0x8c on the master evo channel will fill a lot more of
-	 * this structure with some undefined info
+	/* create primary evo channel, the one we use for modesetting
+	 * purporses
 	 */
-	ret = nouveau_gpuobj_new(dev, disp->master, 0x1000, 0,
-				 NVOBJ_FLAG_ZERO_ALLOC, &disp->ntfy);
+	ret = nv50_evo_channel_new(dev, 0, &disp->master);
 	if (ret)
-		goto err;
+		return ret;
+	evo = disp->master;
 
 	ret = nv50_evo_dmaobj_new(disp->master, NvEvoSync, 0x0000,
-				  disp->ntfy->vinst, disp->ntfy->size, NULL);
+				  disp->ramin->addr + 0x2000, 0x1000, NULL);
 	if (ret)
 		goto err;
 
 	/* create some default objects for the scanout memtypes we support */
 	ret = nv50_evo_dmaobj_new(disp->master, NvEvoVRAM, 0x0000,
-				  0, dev_priv->vram_size, NULL);
+				  0, pfb->ram.size, NULL);
 	if (ret)
 		goto err;
 
 	ret = nv50_evo_dmaobj_new(disp->master, NvEvoVRAM_LP, 0x80000000,
-				  0, dev_priv->vram_size, NULL);
+				  0, pfb->ram.size, NULL);
 	if (ret)
 		goto err;
 
 	ret = nv50_evo_dmaobj_new(disp->master, NvEvoFB32, 0x80000000 |
-				  (dev_priv->chipset < 0xc0 ? 0x7a00 : 0xfe00),
-				  0, dev_priv->vram_size, NULL);
+				  (nv_device(drm->device)->chipset < 0xc0 ? 0x7a : 0xfe),
+				  0, pfb->ram.size, NULL);
 	if (ret)
 		goto err;
 
 	ret = nv50_evo_dmaobj_new(disp->master, NvEvoFB16, 0x80000000 |
-				  (dev_priv->chipset < 0xc0 ? 0x7000 : 0xfe00),
-				  0, dev_priv->vram_size, NULL);
+				  (nv_device(drm->device)->chipset < 0xc0 ? 0x70 : 0xfe),
+				  0, pfb->ram.size, NULL);
 	if (ret)
 		goto err;
 
@@ -352,21 +338,21 @@ nv50_evo_create(struct drm_device *dev)
 			goto err;
 
 		ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoVRAM_LP, 0x80000000,
-					  0, dev_priv->vram_size, NULL);
+					  0, pfb->ram.size, NULL);
 		if (ret)
 			goto err;
 
 		ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoFB32, 0x80000000 |
-					  (dev_priv->chipset < 0xc0 ?
-					  0x7a00 : 0xfe00),
-					  0, dev_priv->vram_size, NULL);
+					  (nv_device(drm->device)->chipset < 0xc0 ?
+					  0x7a : 0xfe),
+					  0, pfb->ram.size, NULL);
 		if (ret)
 			goto err;
 
 		ret = nv50_evo_dmaobj_new(dispc->sync, NvEvoFB16, 0x80000000 |
-					  (dev_priv->chipset < 0xc0 ?
-					  0x7000 : 0xfe00),
-					  0, dev_priv->vram_size, NULL);
+					  (nv_device(drm->device)->chipset < 0xc0 ?
+					  0x70 : 0xfe),
+					  0, pfb->ram.size, NULL);
 		if (ret)
 			goto err;
 
diff --git a/drivers/gpu/drm/nouveau/nv50_fb.c b/drivers/gpu/drm/nouveau/nv50_fb.c
deleted file mode 100644
index f1e4b9e07d1..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_fb.c
+++ /dev/null
@@ -1,296 +0,0 @@
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-#include "nouveau_fifo.h"
-
-struct nv50_fb_priv {
-	struct page *r100c08_page;
-	dma_addr_t r100c08;
-};
-
-static void
-nv50_fb_destroy(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct nv50_fb_priv *priv = pfb->priv;
-
-	if (drm_mm_initialized(&pfb->tag_heap))
-		drm_mm_takedown(&pfb->tag_heap);
-
-	if (priv->r100c08_page) {
-		pci_unmap_page(dev->pdev, priv->r100c08, PAGE_SIZE,
-			       PCI_DMA_BIDIRECTIONAL);
-		__free_page(priv->r100c08_page);
-	}
-
-	kfree(priv);
-	pfb->priv = NULL;
-}
-
-static int
-nv50_fb_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct nv50_fb_priv *priv;
-	u32 tagmem;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	pfb->priv = priv;
-
-	priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (!priv->r100c08_page) {
-		nv50_fb_destroy(dev);
-		return -ENOMEM;
-	}
-
-	priv->r100c08 = pci_map_page(dev->pdev, priv->r100c08_page, 0,
-				     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(dev->pdev, priv->r100c08)) {
-		nv50_fb_destroy(dev);
-		return -EFAULT;
-	}
-
-	tagmem = nv_rd32(dev, 0x100320);
-	NV_DEBUG(dev, "%d tags available\n", tagmem);
-	ret = drm_mm_init(&pfb->tag_heap, 0, tagmem);
-	if (ret) {
-		nv50_fb_destroy(dev);
-		return ret;
-	}
-
-	return 0;
-}
-
-int
-nv50_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_fb_priv *priv;
-	int ret;
-
-	if (!dev_priv->engine.fb.priv) {
-		ret = nv50_fb_create(dev);
-		if (ret)
-			return ret;
-	}
-	priv = dev_priv->engine.fb.priv;
-
-	/* Not a clue what this is exactly.  Without pointing it at a
-	 * scratch page, VRAM->GART blits with M2MF (as in DDX DFS)
-	 * cause IOMMU "read from address 0" errors (rh#561267)
-	 */
-	nv_wr32(dev, 0x100c08, priv->r100c08 >> 8);
-
-	/* This is needed to get meaningful information from 100c90
-	 * on traps. No idea what these values mean exactly. */
-	switch (dev_priv->chipset) {
-	case 0x50:
-		nv_wr32(dev, 0x100c90, 0x000707ff);
-		break;
-	case 0xa3:
-	case 0xa5:
-	case 0xa8:
-		nv_wr32(dev, 0x100c90, 0x000d0fff);
-		break;
-	case 0xaf:
-		nv_wr32(dev, 0x100c90, 0x089d1fff);
-		break;
-	default:
-		nv_wr32(dev, 0x100c90, 0x001d07ff);
-		break;
-	}
-
-	return 0;
-}
-
-void
-nv50_fb_takedown(struct drm_device *dev)
-{
-	nv50_fb_destroy(dev);
-}
-
-static struct nouveau_enum vm_dispatch_subclients[] = {
-	{ 0x00000000, "GRCTX", NULL },
-	{ 0x00000001, "NOTIFY", NULL },
-	{ 0x00000002, "QUERY", NULL },
-	{ 0x00000003, "COND", NULL },
-	{ 0x00000004, "M2M_IN", NULL },
-	{ 0x00000005, "M2M_OUT", NULL },
-	{ 0x00000006, "M2M_NOTIFY", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_ccache_subclients[] = {
-	{ 0x00000000, "CB", NULL },
-	{ 0x00000001, "TIC", NULL },
-	{ 0x00000002, "TSC", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_prop_subclients[] = {
-	{ 0x00000000, "RT0", NULL },
-	{ 0x00000001, "RT1", NULL },
-	{ 0x00000002, "RT2", NULL },
-	{ 0x00000003, "RT3", NULL },
-	{ 0x00000004, "RT4", NULL },
-	{ 0x00000005, "RT5", NULL },
-	{ 0x00000006, "RT6", NULL },
-	{ 0x00000007, "RT7", NULL },
-	{ 0x00000008, "ZETA", NULL },
-	{ 0x00000009, "LOCAL", NULL },
-	{ 0x0000000a, "GLOBAL", NULL },
-	{ 0x0000000b, "STACK", NULL },
-	{ 0x0000000c, "DST2D", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_pfifo_subclients[] = {
-	{ 0x00000000, "PUSHBUF", NULL },
-	{ 0x00000001, "SEMAPHORE", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_bar_subclients[] = {
-	{ 0x00000000, "FB", NULL },
-	{ 0x00000001, "IN", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_client[] = {
-	{ 0x00000000, "STRMOUT", NULL },
-	{ 0x00000003, "DISPATCH", vm_dispatch_subclients },
-	{ 0x00000004, "PFIFO_WRITE", NULL },
-	{ 0x00000005, "CCACHE", vm_ccache_subclients },
-	{ 0x00000006, "PPPP", NULL },
-	{ 0x00000007, "CLIPID", NULL },
-	{ 0x00000008, "PFIFO_READ", NULL },
-	{ 0x00000009, "VFETCH", NULL },
-	{ 0x0000000a, "TEXTURE", NULL },
-	{ 0x0000000b, "PROP", vm_prop_subclients },
-	{ 0x0000000c, "PVP", NULL },
-	{ 0x0000000d, "PBSP", NULL },
-	{ 0x0000000e, "PCRYPT", NULL },
-	{ 0x0000000f, "PCOUNTER", NULL },
-	{ 0x00000011, "PDAEMON", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_engine[] = {
-	{ 0x00000000, "PGRAPH", NULL },
-	{ 0x00000001, "PVP", NULL },
-	{ 0x00000004, "PEEPHOLE", NULL },
-	{ 0x00000005, "PFIFO", vm_pfifo_subclients },
-	{ 0x00000006, "BAR", vm_bar_subclients },
-	{ 0x00000008, "PPPP", NULL },
-	{ 0x00000009, "PBSP", NULL },
-	{ 0x0000000a, "PCRYPT", NULL },
-	{ 0x0000000b, "PCOUNTER", NULL },
-	{ 0x0000000c, "SEMAPHORE_BG", NULL },
-	{ 0x0000000d, "PCOPY", NULL },
-	{ 0x0000000e, "PDAEMON", NULL },
-	{}
-};
-
-static struct nouveau_enum vm_fault[] = {
-	{ 0x00000000, "PT_NOT_PRESENT", NULL },
-	{ 0x00000001, "PT_TOO_SHORT", NULL },
-	{ 0x00000002, "PAGE_NOT_PRESENT", NULL },
-	{ 0x00000003, "PAGE_SYSTEM_ONLY", NULL },
-	{ 0x00000004, "PAGE_READ_ONLY", NULL },
-	{ 0x00000006, "NULL_DMAOBJ", NULL },
-	{ 0x00000007, "WRONG_MEMTYPE", NULL },
-	{ 0x0000000b, "VRAM_LIMIT", NULL },
-	{ 0x0000000f, "DMAOBJ_LIMIT", NULL },
-	{}
-};
-
-void
-nv50_fb_vm_trap(struct drm_device *dev, int display)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	const struct nouveau_enum *en, *cl;
-	unsigned long flags;
-	u32 trap[6], idx, chinst;
-	u8 st0, st1, st2, st3;
-	int i, ch;
-
-	idx = nv_rd32(dev, 0x100c90);
-	if (!(idx & 0x80000000))
-		return;
-	idx &= 0x00ffffff;
-
-	for (i = 0; i < 6; i++) {
-		nv_wr32(dev, 0x100c90, idx | i << 24);
-		trap[i] = nv_rd32(dev, 0x100c94);
-	}
-	nv_wr32(dev, 0x100c90, idx | 0x80000000);
-
-	if (!display)
-		return;
-
-	/* lookup channel id */
-	chinst = (trap[2] << 16) | trap[1];
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (ch = 0; ch < pfifo->channels; ch++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[ch];
-
-		if (!chan || !chan->ramin)
-			continue;
-
-		if (chinst == chan->ramin->vinst >> 12)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-
-	/* decode status bits into something more useful */
-	if (dev_priv->chipset  < 0xa3 ||
-	    dev_priv->chipset == 0xaa || dev_priv->chipset == 0xac) {
-		st0 = (trap[0] & 0x0000000f) >> 0;
-		st1 = (trap[0] & 0x000000f0) >> 4;
-		st2 = (trap[0] & 0x00000f00) >> 8;
-		st3 = (trap[0] & 0x0000f000) >> 12;
-	} else {
-		st0 = (trap[0] & 0x000000ff) >> 0;
-		st1 = (trap[0] & 0x0000ff00) >> 8;
-		st2 = (trap[0] & 0x00ff0000) >> 16;
-		st3 = (trap[0] & 0xff000000) >> 24;
-	}
-
-	NV_INFO(dev, "VM: trapped %s at 0x%02x%04x%04x on ch %d [0x%08x] ",
-		(trap[5] & 0x00000100) ? "read" : "write",
-		trap[5] & 0xff, trap[4] & 0xffff, trap[3] & 0xffff, ch, chinst);
-
-	en = nouveau_enum_find(vm_engine, st0);
-	if (en)
-		printk("%s/", en->name);
-	else
-		printk("%02x/", st0);
-
-	cl = nouveau_enum_find(vm_client, st2);
-	if (cl)
-		printk("%s/", cl->name);
-	else
-		printk("%02x/", st2);
-
-	if      (cl && cl->data) cl = nouveau_enum_find(cl->data, st3);
-	else if (en && en->data) cl = nouveau_enum_find(en->data, st3);
-	else                     cl = NULL;
-	if (cl)
-		printk("%s", cl->name);
-	else
-		printk("%02x", st3);
-
-	printk(" reason: ");
-	en = nouveau_enum_find(vm_fault, st1);
-	if (en)
-		printk("%s\n", en->name);
-	else
-		printk("0x%08x\n", st1);
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index e3c8b05dcae..52068a0910d 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -22,20 +22,16 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fbcon.h"
-#include "nouveau_mm.h"
 
 int
 nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
@@ -69,9 +65,8 @@ int
 nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, 12);
@@ -98,9 +93,8 @@ int
 nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	uint32_t width, dwords, *data = (uint32_t *)image->data;
 	uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
 	uint32_t *palette = info->pseudo_palette;
@@ -156,10 +150,11 @@ int
 nv50_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
 	struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
+	struct drm_device *dev = nfbdev->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_channel *chan = drm->channel;
+	struct nouveau_object *object;
 	int ret, format;
 
 	switch (info->var.bits_per_pixel) {
@@ -189,7 +184,8 @@ nv50_fbcon_accel_init(struct fb_info *info)
 		return -EINVAL;
 	}
 
-	ret = nouveau_gpuobj_gr_new(dev_priv->channel, Nv2D, 0x502d);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, Nv2D,
+				 0x502d, NULL, 0, &object);
 	if (ret)
 		return ret;
 
@@ -202,9 +198,9 @@ nv50_fbcon_accel_init(struct fb_info *info)
 	BEGIN_NV04(chan, NvSub2D, 0x0000, 1);
 	OUT_RING(chan, Nv2D);
 	BEGIN_NV04(chan, NvSub2D, 0x0184, 3);
-	OUT_RING(chan, chan->vram_handle);
-	OUT_RING(chan, chan->vram_handle);
-	OUT_RING(chan, chan->vram_handle);
+	OUT_RING(chan, NvDmaFB);
+	OUT_RING(chan, NvDmaFB);
+	OUT_RING(chan, NvDmaFB);
 	BEGIN_NV04(chan, NvSub2D, 0x0290, 1);
 	OUT_RING(chan, 0);
 	BEGIN_NV04(chan, NvSub2D, 0x0888, 1);
diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c
new file mode 100644
index 00000000000..e0763ea88ee
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nv50_fence.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <core/object.h>
+#include <core/class.h>
+
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
+#include "nouveau_fence.h"
+
+#include "nv50_display.h"
+
+struct nv50_fence_chan {
+	struct nouveau_fence_chan base;
+};
+
+struct nv50_fence_priv {
+	struct nouveau_fence_priv base;
+	struct nouveau_bo *bo;
+	spinlock_t lock;
+	u32 sequence;
+};
+
+static int
+nv50_fence_context_new(struct nouveau_channel *chan)
+{
+	struct drm_device *dev = chan->drm->dev;
+	struct nv50_fence_priv *priv = chan->drm->fence;
+	struct nv50_fence_chan *fctx;
+	struct ttm_mem_reg *mem = &priv->bo->bo.mem;
+	struct nouveau_object *object;
+	int ret, i;
+
+	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	if (!fctx)
+		return -ENOMEM;
+
+	nouveau_fence_context_new(&fctx->base);
+
+	ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
+				 NvSema, 0x0002,
+				 &(struct nv_dma_class) {
+					.flags = NV_DMA_TARGET_VRAM |
+						 NV_DMA_ACCESS_RDWR,
+					.start = mem->start * PAGE_SIZE,
+					.limit = mem->size - 1,
+				 }, sizeof(struct nv_dma_class),
+				 &object);
+
+	/* dma objects for display sync channel semaphore blocks */
+	for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) {
+		struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i);
+
+		ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
+					 NvEvoSema0 + i, 0x003d,
+					 &(struct nv_dma_class) {
+						.flags = NV_DMA_TARGET_VRAM |
+							 NV_DMA_ACCESS_RDWR,
+						.start = bo->bo.offset,
+						.limit = bo->bo.offset + 0xfff,
+					 }, sizeof(struct nv_dma_class),
+					 &object);
+	}
+
+	if (ret)
+		nv10_fence_context_del(chan);
+	return ret;
+}
+
+int
+nv50_fence_create(struct nouveau_drm *drm)
+{
+	struct nv50_fence_priv *priv;
+	int ret = 0;
+
+	priv = drm->fence = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->base.dtor = nv10_fence_destroy;
+	priv->base.context_new = nv50_fence_context_new;
+	priv->base.context_del = nv10_fence_context_del;
+	priv->base.emit = nv10_fence_emit;
+	priv->base.read = nv10_fence_read;
+	priv->base.sync = nv17_fence_sync;
+	spin_lock_init(&priv->lock);
+
+	ret = nouveau_bo_new(drm->dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
+			     0, 0x0000, NULL, &priv->bo);
+	if (!ret) {
+		ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
+		if (!ret)
+			ret = nouveau_bo_map(priv->bo);
+		if (ret)
+			nouveau_bo_ref(NULL, &priv->bo);
+	}
+
+	if (ret == 0) {
+		nouveau_bo_wr32(priv->bo, 0x000, 0x00000000);
+		priv->base.sync = nv17_fence_sync;
+	}
+
+	if (ret)
+		nv10_fence_destroy(drm);
+	return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/nv50_fifo.c b/drivers/gpu/drm/nouveau/nv50_fifo.c
deleted file mode 100644
index 55383b85db0..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_fifo.c
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-#include "nouveau_vm.h"
-
-struct nv50_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct nouveau_gpuobj *playlist[2];
-	int cur_playlist;
-};
-
-struct nv50_fifo_chan {
-	struct nouveau_fifo_chan base;
-};
-
-void
-nv50_fifo_playlist_update(struct drm_device *dev)
-{
-	struct nv50_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *cur;
-	int i, p;
-
-	cur = priv->playlist[priv->cur_playlist];
-	priv->cur_playlist = !priv->cur_playlist;
-
-	for (i = 0, p = 0; i < priv->base.channels; i++) {
-		if (nv_rd32(dev, 0x002600 + (i * 4)) & 0x80000000)
-			nv_wo32(cur, p++ * 4, i);
-	}
-
-	dev_priv->engine.instmem.flush(dev);
-
-	nv_wr32(dev, 0x0032f4, cur->vinst >> 12);
-	nv_wr32(dev, 0x0032ec, p);
-	nv_wr32(dev, 0x002500, 0x00000101);
-}
-
-static int
-nv50_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv50_fifo_priv *priv = nv_engine(chan->dev, engine);
-	struct nv50_fifo_chan *fctx;
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u64 ib_offset = chan->pushbuf_base + chan->dma.ib_base * 4;
-	u64 instance = chan->ramin->vinst >> 12;
-	unsigned long flags;
-	int ret = 0, i;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-	atomic_inc(&chan->vm->engref[engine]);
-
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV50_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	for (i = 0; i < 0x100; i += 4)
-		nv_wo32(chan->ramin, i, 0x00000000);
-	nv_wo32(chan->ramin, 0x3c, 0x403f6078);
-	nv_wo32(chan->ramin, 0x40, 0x00000000);
-	nv_wo32(chan->ramin, 0x44, 0x01003fff);
-	nv_wo32(chan->ramin, 0x48, chan->pushbuf->cinst >> 4);
-	nv_wo32(chan->ramin, 0x50, lower_32_bits(ib_offset));
-	nv_wo32(chan->ramin, 0x54, upper_32_bits(ib_offset) |
-				   drm_order(chan->dma.ib_max + 1) << 16);
-	nv_wo32(chan->ramin, 0x60, 0x7fffffff);
-	nv_wo32(chan->ramin, 0x78, 0x00000000);
-	nv_wo32(chan->ramin, 0x7c, 0x30000001);
-	nv_wo32(chan->ramin, 0x80, ((chan->ramht->bits - 9) << 27) |
-				   (4 << 24) /* SEARCH_FULL */ |
-				   (chan->ramht->gpuobj->cinst >> 4));
-
-	dev_priv->engine.instmem.flush(dev);
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_wr32(dev, 0x002600 + (chan->id * 4), 0x80000000 | instance);
-	nv50_fifo_playlist_update(dev);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static bool
-nv50_fifo_kickoff(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-	bool done = true;
-	u32 me;
-
-	/* HW bug workaround:
-	 *
-	 * PFIFO will hang forever if the connected engines don't report
-	 * that they've processed the context switch request.
-	 *
-	 * In order for the kickoff to work, we need to ensure all the
-	 * connected engines are in a state where they can answer.
-	 *
-	 * Newer chipsets don't seem to suffer from this issue, and well,
-	 * there's also a "ignore these engines" bitmask reg we can use
-	 * if we hit the issue there..
-	 */
-
-	/* PME: make sure engine is enabled */
-	me = nv_mask(dev, 0x00b860, 0x00000001, 0x00000001);
-
-	/* do the kickoff... */
-	nv_wr32(dev, 0x0032fc, chan->ramin->vinst >> 12);
-	if (!nv_wait_ne(dev, 0x0032fc, 0xffffffff, 0xffffffff)) {
-		NV_INFO(dev, "PFIFO: channel %d unload timeout\n", chan->id);
-		done = false;
-	}
-
-	/* restore any engine states we changed, and exit */
-	nv_wr32(dev, 0x00b860, me);
-	return done;
-}
-
-static void
-nv50_fifo_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv50_fifo_chan *fctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	/* remove channel from playlist, will context switch if active */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000);
-	nv50_fifo_playlist_update(dev);
-
-	/* tell any engines on this channel to unload their contexts */
-	nv50_fifo_kickoff(chan);
-
-	nv_wr32(dev, 0x002600 + (chan->id * 4), 0x00000000);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* clean up */
-	if (chan->user) {
-		iounmap(chan->user);
-		chan->user = NULL;
-	}
-
-	atomic_dec(&chan->vm->engref[engine]);
-	chan->engctx[engine] = NULL;
-	kfree(fctx);
-}
-
-static int
-nv50_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 instance;
-	int i;
-
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
-	nv_wr32(dev, 0x00250c, 0x6f3cfc34);
-	nv_wr32(dev, 0x002044, 0x01003fff);
-
-	nv_wr32(dev, 0x002100, 0xffffffff);
-	nv_wr32(dev, 0x002140, 0xffffffff);
-
-	for (i = 0; i < 128; i++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
-		if (chan && chan->engctx[engine])
-			instance = 0x80000000 | chan->ramin->vinst >> 12;
-		else
-			instance = 0x00000000;
-		nv_wr32(dev, 0x002600 + (i * 4), instance);
-	}
-
-	nv50_fifo_playlist_update(dev);
-
-	nv_wr32(dev, 0x003200, 1);
-	nv_wr32(dev, 0x003250, 1);
-	nv_wr32(dev, 0x002500, 1);
-	return 0;
-}
-
-static int
-nv50_fifo_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	/* set playlist length to zero, fifo will unload context */
-	nv_wr32(dev, 0x0032ec, 0);
-
-	/* tell all connected engines to unload their contexts */
-	for (i = 0; i < priv->base.channels; i++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
-		if (chan && !nv50_fifo_kickoff(chan))
-			return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x002140, 0);
-	return 0;
-}
-
-void
-nv50_fifo_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 5);
-}
-
-void
-nv50_fifo_destroy(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_fifo_priv *priv = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 8);
-
-	nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
-	nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
-
-	dev_priv->eng[engine] = NULL;
-	kfree(priv);
-}
-
-int
-nv50_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_fifo_priv *priv;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv50_fifo_destroy;
-	priv->base.base.init = nv50_fifo_init;
-	priv->base.base.fini = nv50_fifo_fini;
-	priv->base.base.context_new = nv50_fifo_context_new;
-	priv->base.base.context_del = nv50_fifo_context_del;
-	priv->base.base.tlb_flush = nv50_fifo_tlb_flush;
-	priv->base.channels = 127;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	ret = nouveau_gpuobj_new(dev, NULL, priv->base.channels * 4, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->playlist[0]);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, priv->base.channels * 4, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->playlist[1]);
-	if (ret)
-		goto error;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-error:
-	if (ret)
-		priv->base.base.destroy(dev, NVOBJ_ENGINE_FIFO);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_gpio.c b/drivers/gpu/drm/nouveau/nv50_gpio.c
deleted file mode 100644
index c399d510b27..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_gpio.c
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/dmi.h>
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_hw.h"
-#include "nouveau_gpio.h"
-
-#include "nv50_display.h"
-
-static int
-nv50_gpio_location(int line, u32 *reg, u32 *shift)
-{
-	const uint32_t nv50_gpio_reg[4] = { 0xe104, 0xe108, 0xe280, 0xe284 };
-
-	if (line >= 32)
-		return -EINVAL;
-
-	*reg = nv50_gpio_reg[line >> 3];
-	*shift = (line & 7) << 2;
-	return 0;
-}
-
-int
-nv50_gpio_drive(struct drm_device *dev, int line, int dir, int out)
-{
-	u32 reg, shift;
-
-	if (nv50_gpio_location(line, &reg, &shift))
-		return -EINVAL;
-
-	nv_mask(dev, reg, 7 << shift, (((dir ^ 1) << 1) | out) << shift);
-	return 0;
-}
-
-int
-nv50_gpio_sense(struct drm_device *dev, int line)
-{
-	u32 reg, shift;
-
-	if (nv50_gpio_location(line, &reg, &shift))
-		return -EINVAL;
-
-	return !!(nv_rd32(dev, reg) & (4 << shift));
-}
-
-void
-nv50_gpio_irq_enable(struct drm_device *dev, int line, bool on)
-{
-	u32 reg  = line < 16 ? 0xe050 : 0xe070;
-	u32 mask = 0x00010001 << (line & 0xf);
-
-	nv_wr32(dev, reg + 4, mask);
-	nv_mask(dev, reg + 0, mask, on ? mask : 0);
-}
-
-int
-nvd0_gpio_drive(struct drm_device *dev, int line, int dir, int out)
-{
-	u32 data = ((dir ^ 1) << 13) | (out << 12);
-	nv_mask(dev, 0x00d610 + (line * 4), 0x00003000, data);
-	nv_mask(dev, 0x00d604, 0x00000001, 0x00000001); /* update? */
-	return 0;
-}
-
-int
-nvd0_gpio_sense(struct drm_device *dev, int line)
-{
-	return !!(nv_rd32(dev, 0x00d610 + (line * 4)) & 0x00004000);
-}
-
-static void
-nv50_gpio_isr(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 intr0, intr1 = 0;
-	u32 hi, lo;
-
-	intr0 = nv_rd32(dev, 0xe054) & nv_rd32(dev, 0xe050);
-	if (dev_priv->chipset >= 0x90)
-		intr1 = nv_rd32(dev, 0xe074) & nv_rd32(dev, 0xe070);
-
-	hi = (intr0 & 0x0000ffff) | (intr1 << 16);
-	lo = (intr0 >> 16) | (intr1 & 0xffff0000);
-	nouveau_gpio_isr(dev, 0, hi | lo);
-
-	nv_wr32(dev, 0xe054, intr0);
-	if (dev_priv->chipset >= 0x90)
-		nv_wr32(dev, 0xe074, intr1);
-}
-
-static struct dmi_system_id gpio_reset_ids[] = {
-	{
-		.ident = "Apple Macbook 10,1",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro10,1"),
-		}
-	},
-	{ }
-};
-
-int
-nv50_gpio_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	/* initialise gpios and routing to vbios defaults */
-	if (dmi_check_system(gpio_reset_ids))
-		nouveau_gpio_reset(dev);
-
-	/* disable, and ack any pending gpio interrupts */
-	nv_wr32(dev, 0xe050, 0x00000000);
-	nv_wr32(dev, 0xe054, 0xffffffff);
-	if (dev_priv->chipset >= 0x90) {
-		nv_wr32(dev, 0xe070, 0x00000000);
-		nv_wr32(dev, 0xe074, 0xffffffff);
-	}
-
-	nouveau_irq_register(dev, 21, nv50_gpio_isr);
-	return 0;
-}
-
-void
-nv50_gpio_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nv_wr32(dev, 0xe050, 0x00000000);
-	if (dev_priv->chipset >= 0x90)
-		nv_wr32(dev, 0xe070, 0x00000000);
-	nouveau_irq_unregister(dev, 21);
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_graph.c b/drivers/gpu/drm/nouveau/nv50_graph.c
deleted file mode 100644
index 437608d1dfe..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_graph.c
+++ /dev/null
@@ -1,868 +0,0 @@
-/*
- * Copyright (C) 2007 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-#include "nouveau_dma.h"
-#include "nouveau_vm.h"
-#include "nv50_evo.h"
-
-struct nv50_graph_engine {
-	struct nouveau_exec_engine base;
-	u32 ctxprog[512];
-	u32 ctxprog_size;
-	u32 grctx_size;
-};
-
-static int
-nv50_graph_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_graph_engine *pgraph = nv_engine(dev, engine);
-	u32 units = nv_rd32(dev, 0x001540);
-	int i;
-
-	NV_DEBUG(dev, "\n");
-
-	/* master reset */
-	nv_mask(dev, 0x000200, 0x00201000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00201000, 0x00201000);
-	nv_wr32(dev, 0x40008c, 0x00000004); /* HW_CTX_SWITCH_ENABLED */
-
-	/* reset/enable traps and interrupts */
-	nv_wr32(dev, 0x400804, 0xc0000000);
-	nv_wr32(dev, 0x406800, 0xc0000000);
-	nv_wr32(dev, 0x400c04, 0xc0000000);
-	nv_wr32(dev, 0x401800, 0xc0000000);
-	nv_wr32(dev, 0x405018, 0xc0000000);
-	nv_wr32(dev, 0x402000, 0xc0000000);
-	for (i = 0; i < 16; i++) {
-		if (!(units & (1 << i)))
-			continue;
-
-		if (dev_priv->chipset < 0xa0) {
-			nv_wr32(dev, 0x408900 + (i << 12), 0xc0000000);
-			nv_wr32(dev, 0x408e08 + (i << 12), 0xc0000000);
-			nv_wr32(dev, 0x408314 + (i << 12), 0xc0000000);
-		} else {
-			nv_wr32(dev, 0x408600 + (i << 11), 0xc0000000);
-			nv_wr32(dev, 0x408708 + (i << 11), 0xc0000000);
-			nv_wr32(dev, 0x40831c + (i << 11), 0xc0000000);
-		}
-	}
-
-	nv_wr32(dev, 0x400108, 0xffffffff);
-	nv_wr32(dev, 0x400138, 0xffffffff);
-	nv_wr32(dev, 0x400100, 0xffffffff);
-	nv_wr32(dev, 0x40013c, 0xffffffff);
-	nv_wr32(dev, 0x400500, 0x00010001);
-
-	/* upload context program, initialise ctxctl defaults */
-	nv_wr32(dev, 0x400324, 0x00000000);
-	for (i = 0; i < pgraph->ctxprog_size; i++)
-		nv_wr32(dev, 0x400328, pgraph->ctxprog[i]);
-	nv_wr32(dev, 0x400824, 0x00000000);
-	nv_wr32(dev, 0x400828, 0x00000000);
-	nv_wr32(dev, 0x40082c, 0x00000000);
-	nv_wr32(dev, 0x400830, 0x00000000);
-	nv_wr32(dev, 0x400724, 0x00000000);
-	nv_wr32(dev, 0x40032c, 0x00000000);
-	nv_wr32(dev, 0x400320, 4);	/* CTXCTL_CMD = NEWCTXDMA */
-
-	/* some unknown zcull magic */
-	switch (dev_priv->chipset & 0xf0) {
-	case 0x50:
-	case 0x80:
-	case 0x90:
-		nv_wr32(dev, 0x402ca8, 0x00000800);
-		break;
-	case 0xa0:
-	default:
-		nv_wr32(dev, 0x402cc0, 0x00000000);
-		if (dev_priv->chipset == 0xa0 ||
-		    dev_priv->chipset == 0xaa ||
-		    dev_priv->chipset == 0xac) {
-			nv_wr32(dev, 0x402ca8, 0x00000802);
-		} else {
-			nv_wr32(dev, 0x402cc0, 0x00000000);
-			nv_wr32(dev, 0x402ca8, 0x00000002);
-		}
-
-		break;
-	}
-
-	/* zero out zcull regions */
-	for (i = 0; i < 8; i++) {
-		nv_wr32(dev, 0x402c20 + (i * 8), 0x00000000);
-		nv_wr32(dev, 0x402c24 + (i * 8), 0x00000000);
-		nv_wr32(dev, 0x402c28 + (i * 8), 0x00000000);
-		nv_wr32(dev, 0x402c2c + (i * 8), 0x00000000);
-	}
-
-	return 0;
-}
-
-static int
-nv50_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_wr32(dev, 0x40013c, 0x00000000);
-	return 0;
-}
-
-static int
-nv50_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramin = chan->ramin;
-	struct nouveau_gpuobj *grctx = NULL;
-	struct nv50_graph_engine *pgraph = nv_engine(dev, engine);
-	int hdr, ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	ret = nouveau_gpuobj_new(dev, NULL, pgraph->grctx_size, 0,
-				 NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &grctx);
-	if (ret)
-		return ret;
-
-	hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
-	nv_wo32(ramin, hdr + 0x00, 0x00190002);
-	nv_wo32(ramin, hdr + 0x04, grctx->vinst + grctx->size - 1);
-	nv_wo32(ramin, hdr + 0x08, grctx->vinst);
-	nv_wo32(ramin, hdr + 0x0c, 0);
-	nv_wo32(ramin, hdr + 0x10, 0);
-	nv_wo32(ramin, hdr + 0x14, 0x00010000);
-
-	nv50_grctx_fill(dev, grctx);
-	nv_wo32(grctx, 0x00000, chan->ramin->vinst >> 12);
-
-	dev_priv->engine.instmem.flush(dev);
-
-	atomic_inc(&chan->vm->engref[NVOBJ_ENGINE_GR]);
-	chan->engctx[NVOBJ_ENGINE_GR] = grctx;
-	return 0;
-}
-
-static void
-nv50_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nouveau_gpuobj *grctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i, hdr = (dev_priv->chipset == 0x50) ? 0x200 : 0x20;
-
-	for (i = hdr; i < hdr + 24; i += 4)
-		nv_wo32(chan->ramin, i, 0);
-	dev_priv->engine.instmem.flush(dev);
-
-	atomic_dec(&chan->vm->engref[engine]);
-	nouveau_gpuobj_ref(NULL, &grctx);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nv50_graph_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 1;
-	obj->class  = class;
-
-	nv_wo32(obj, 0x00, class);
-	nv_wo32(obj, 0x04, 0x00000000);
-	nv_wo32(obj, 0x08, 0x00000000);
-	nv_wo32(obj, 0x0c, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static void
-nv50_graph_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 0);
-}
-
-static void
-nv84_graph_tlb_flush(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_timer_engine *ptimer = &dev_priv->engine.timer;
-	bool idle, timeout = false;
-	unsigned long flags;
-	u64 start;
-	u32 tmp;
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x400500, 0x00000001, 0x00000000);
-
-	start = ptimer->read(dev);
-	do {
-		idle = true;
-
-		for (tmp = nv_rd32(dev, 0x400380); tmp && idle; tmp >>= 3) {
-			if ((tmp & 7) == 1)
-				idle = false;
-		}
-
-		for (tmp = nv_rd32(dev, 0x400384); tmp && idle; tmp >>= 3) {
-			if ((tmp & 7) == 1)
-				idle = false;
-		}
-
-		for (tmp = nv_rd32(dev, 0x400388); tmp && idle; tmp >>= 3) {
-			if ((tmp & 7) == 1)
-				idle = false;
-		}
-	} while (!idle && !(timeout = ptimer->read(dev) - start > 2000000000));
-
-	if (timeout) {
-		NV_ERROR(dev, "PGRAPH TLB flush idle timeout fail: "
-			      "0x%08x 0x%08x 0x%08x 0x%08x\n",
-			 nv_rd32(dev, 0x400700), nv_rd32(dev, 0x400380),
-			 nv_rd32(dev, 0x400384), nv_rd32(dev, 0x400388));
-	}
-
-	nv50_vm_flush_engine(dev, 0);
-
-	nv_mask(dev, 0x400500, 0x00000001, 0x00000001);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-}
-
-static struct nouveau_enum nv50_mp_exec_error_names[] = {
-	{ 3, "STACK_UNDERFLOW", NULL },
-	{ 4, "QUADON_ACTIVE", NULL },
-	{ 8, "TIMEOUT", NULL },
-	{ 0x10, "INVALID_OPCODE", NULL },
-	{ 0x40, "BREAKPOINT", NULL },
-	{}
-};
-
-static struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
-	{ 0x00000001, "NOTIFY" },
-	{ 0x00000002, "IN" },
-	{ 0x00000004, "OUT" },
-	{}
-};
-
-static struct nouveau_bitfield nv50_graph_trap_vfetch[] = {
-	{ 0x00000001, "FAULT" },
-	{}
-};
-
-static struct nouveau_bitfield nv50_graph_trap_strmout[] = {
-	{ 0x00000001, "FAULT" },
-	{}
-};
-
-static struct nouveau_bitfield nv50_graph_trap_ccache[] = {
-	{ 0x00000001, "FAULT" },
-	{}
-};
-
-/* There must be a *lot* of these. Will take some time to gather them up. */
-struct nouveau_enum nv50_data_error_names[] = {
-	{ 0x00000003, "INVALID_OPERATION", NULL },
-	{ 0x00000004, "INVALID_VALUE", NULL },
-	{ 0x00000005, "INVALID_ENUM", NULL },
-	{ 0x00000008, "INVALID_OBJECT", NULL },
-	{ 0x00000009, "READ_ONLY_OBJECT", NULL },
-	{ 0x0000000a, "SUPERVISOR_OBJECT", NULL },
-	{ 0x0000000b, "INVALID_ADDRESS_ALIGNMENT", NULL },
-	{ 0x0000000c, "INVALID_BITFIELD", NULL },
-	{ 0x0000000d, "BEGIN_END_ACTIVE", NULL },
-	{ 0x0000000e, "SEMANTIC_COLOR_BACK_OVER_LIMIT", NULL },
-	{ 0x0000000f, "VIEWPORT_ID_NEEDS_GP", NULL },
-	{ 0x00000010, "RT_DOUBLE_BIND", NULL },
-	{ 0x00000011, "RT_TYPES_MISMATCH", NULL },
-	{ 0x00000012, "RT_LINEAR_WITH_ZETA", NULL },
-	{ 0x00000015, "FP_TOO_FEW_REGS", NULL },
-	{ 0x00000016, "ZETA_FORMAT_CSAA_MISMATCH", NULL },
-	{ 0x00000017, "RT_LINEAR_WITH_MSAA", NULL },
-	{ 0x00000018, "FP_INTERPOLANT_START_OVER_LIMIT", NULL },
-	{ 0x00000019, "SEMANTIC_LAYER_OVER_LIMIT", NULL },
-	{ 0x0000001a, "RT_INVALID_ALIGNMENT", NULL },
-	{ 0x0000001b, "SAMPLER_OVER_LIMIT", NULL },
-	{ 0x0000001c, "TEXTURE_OVER_LIMIT", NULL },
-	{ 0x0000001e, "GP_TOO_MANY_OUTPUTS", NULL },
-	{ 0x0000001f, "RT_BPP128_WITH_MS8", NULL },
-	{ 0x00000021, "Z_OUT_OF_BOUNDS", NULL },
-	{ 0x00000023, "XY_OUT_OF_BOUNDS", NULL },
-	{ 0x00000024, "VP_ZERO_INPUTS", NULL },
-	{ 0x00000027, "CP_MORE_PARAMS_THAN_SHARED", NULL },
-	{ 0x00000028, "CP_NO_REG_SPACE_STRIPED", NULL },
-	{ 0x00000029, "CP_NO_REG_SPACE_PACKED", NULL },
-	{ 0x0000002a, "CP_NOT_ENOUGH_WARPS", NULL },
-	{ 0x0000002b, "CP_BLOCK_SIZE_MISMATCH", NULL },
-	{ 0x0000002c, "CP_NOT_ENOUGH_LOCAL_WARPS", NULL },
-	{ 0x0000002d, "CP_NOT_ENOUGH_STACK_WARPS", NULL },
-	{ 0x0000002e, "CP_NO_BLOCKDIM_LATCH", NULL },
-	{ 0x00000031, "ENG2D_FORMAT_MISMATCH", NULL },
-	{ 0x0000003f, "PRIMITIVE_ID_NEEDS_GP", NULL },
-	{ 0x00000044, "SEMANTIC_VIEWPORT_OVER_LIMIT", NULL },
-	{ 0x00000045, "SEMANTIC_COLOR_FRONT_OVER_LIMIT", NULL },
-	{ 0x00000046, "LAYER_ID_NEEDS_GP", NULL },
-	{ 0x00000047, "SEMANTIC_CLIP_OVER_LIMIT", NULL },
-	{ 0x00000048, "SEMANTIC_PTSZ_OVER_LIMIT", NULL },
-	{}
-};
-
-static struct nouveau_bitfield nv50_graph_intr[] = {
-	{ 0x00000001, "NOTIFY" },
-	{ 0x00000002, "COMPUTE_QUERY" },
-	{ 0x00000010, "ILLEGAL_MTHD" },
-	{ 0x00000020, "ILLEGAL_CLASS" },
-	{ 0x00000040, "DOUBLE_NOTIFY" },
-	{ 0x00001000, "CONTEXT_SWITCH" },
-	{ 0x00010000, "BUFFER_NOTIFY" },
-	{ 0x00100000, "DATA_ERROR" },
-	{ 0x00200000, "TRAP" },
-	{ 0x01000000, "SINGLE_STEP" },
-	{}
-};
-
-static void
-nv50_pgraph_mp_trap(struct drm_device *dev, int tpid, int display)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	uint32_t units = nv_rd32(dev, 0x1540);
-	uint32_t addr, mp10, status, pc, oplow, ophigh;
-	int i;
-	int mps = 0;
-	for (i = 0; i < 4; i++) {
-		if (!(units & 1 << (i+24)))
-			continue;
-		if (dev_priv->chipset < 0xa0)
-			addr = 0x408200 + (tpid << 12) + (i << 7);
-		else
-			addr = 0x408100 + (tpid << 11) + (i << 7);
-		mp10 = nv_rd32(dev, addr + 0x10);
-		status = nv_rd32(dev, addr + 0x14);
-		if (!status)
-			continue;
-		if (display) {
-			nv_rd32(dev, addr + 0x20);
-			pc = nv_rd32(dev, addr + 0x24);
-			oplow = nv_rd32(dev, addr + 0x70);
-			ophigh = nv_rd32(dev, addr + 0x74);
-			NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - "
-					"TP %d MP %d: ", tpid, i);
-			nouveau_enum_print(nv50_mp_exec_error_names, status);
-			printk(" at %06x warp %d, opcode %08x %08x\n",
-					pc&0xffffff, pc >> 24,
-					oplow, ophigh);
-		}
-		nv_wr32(dev, addr + 0x10, mp10);
-		nv_wr32(dev, addr + 0x14, 0);
-		mps++;
-	}
-	if (!mps && display)
-		NV_INFO(dev, "PGRAPH_TRAP_MP_EXEC - TP %d: "
-				"No MPs claiming errors?\n", tpid);
-}
-
-static void
-nv50_pgraph_tp_trap(struct drm_device *dev, int type, uint32_t ustatus_old,
-		uint32_t ustatus_new, int display, const char *name)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int tps = 0;
-	uint32_t units = nv_rd32(dev, 0x1540);
-	int i, r;
-	uint32_t ustatus_addr, ustatus;
-	for (i = 0; i < 16; i++) {
-		if (!(units & (1 << i)))
-			continue;
-		if (dev_priv->chipset < 0xa0)
-			ustatus_addr = ustatus_old + (i << 12);
-		else
-			ustatus_addr = ustatus_new + (i << 11);
-		ustatus = nv_rd32(dev, ustatus_addr) & 0x7fffffff;
-		if (!ustatus)
-			continue;
-		tps++;
-		switch (type) {
-		case 6: /* texture error... unknown for now */
-			if (display) {
-				NV_ERROR(dev, "magic set %d:\n", i);
-				for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
-					NV_ERROR(dev, "\t0x%08x: 0x%08x\n", r,
-						nv_rd32(dev, r));
-			}
-			break;
-		case 7: /* MP error */
-			if (ustatus & 0x04030000) {
-				nv50_pgraph_mp_trap(dev, i, display);
-				ustatus &= ~0x04030000;
-			}
-			break;
-		case 8: /* TPDMA error */
-			{
-			uint32_t e0c = nv_rd32(dev, ustatus_addr + 4);
-			uint32_t e10 = nv_rd32(dev, ustatus_addr + 8);
-			uint32_t e14 = nv_rd32(dev, ustatus_addr + 0xc);
-			uint32_t e18 = nv_rd32(dev, ustatus_addr + 0x10);
-			uint32_t e1c = nv_rd32(dev, ustatus_addr + 0x14);
-			uint32_t e20 = nv_rd32(dev, ustatus_addr + 0x18);
-			uint32_t e24 = nv_rd32(dev, ustatus_addr + 0x1c);
-			/* 2d engine destination */
-			if (ustatus & 0x00000010) {
-				if (display) {
-					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - Unknown fault at address %02x%08x\n",
-							i, e14, e10);
-					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_2D - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-							i, e0c, e18, e1c, e20, e24);
-				}
-				ustatus &= ~0x00000010;
-			}
-			/* Render target */
-			if (ustatus & 0x00000040) {
-				if (display) {
-					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - Unknown fault at address %02x%08x\n",
-							i, e14, e10);
-					NV_INFO(dev, "PGRAPH_TRAP_TPDMA_RT - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-							i, e0c, e18, e1c, e20, e24);
-				}
-				ustatus &= ~0x00000040;
-			}
-			/* CUDA memory: l[], g[] or stack. */
-			if (ustatus & 0x00000080) {
-				if (display) {
-					if (e18 & 0x80000000) {
-						/* g[] read fault? */
-						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global read fault at address %02x%08x\n",
-								i, e14, e10 | ((e18 >> 24) & 0x1f));
-						e18 &= ~0x1f000000;
-					} else if (e18 & 0xc) {
-						/* g[] write fault? */
-						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Global write fault at address %02x%08x\n",
-								i, e14, e10 | ((e18 >> 7) & 0x1f));
-						e18 &= ~0x00000f80;
-					} else {
-						NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - Unknown CUDA fault at address %02x%08x\n",
-								i, e14, e10);
-					}
-					NV_INFO(dev, "PGRAPH_TRAP_TPDMA - TP %d - e0c: %08x, e18: %08x, e1c: %08x, e20: %08x, e24: %08x\n",
-							i, e0c, e18, e1c, e20, e24);
-				}
-				ustatus &= ~0x00000080;
-			}
-			}
-			break;
-		}
-		if (ustatus) {
-			if (display)
-				NV_INFO(dev, "%s - TP%d: Unhandled ustatus 0x%08x\n", name, i, ustatus);
-		}
-		nv_wr32(dev, ustatus_addr, 0xc0000000);
-	}
-
-	if (!tps && display)
-		NV_INFO(dev, "%s - No TPs claiming errors?\n", name);
-}
-
-static int
-nv50_pgraph_trap_handler(struct drm_device *dev, u32 display, u64 inst, u32 chid)
-{
-	u32 status = nv_rd32(dev, 0x400108);
-	u32 ustatus;
-
-	if (!status && display) {
-		NV_INFO(dev, "PGRAPH - TRAP: no units reporting traps?\n");
-		return 1;
-	}
-
-	/* DISPATCH: Relays commands to other units and handles NOTIFY,
-	 * COND, QUERY. If you get a trap from it, the command is still stuck
-	 * in DISPATCH and you need to do something about it. */
-	if (status & 0x001) {
-		ustatus = nv_rd32(dev, 0x400804) & 0x7fffffff;
-		if (!ustatus && display) {
-			NV_INFO(dev, "PGRAPH_TRAP_DISPATCH - no ustatus?\n");
-		}
-
-		nv_wr32(dev, 0x400500, 0x00000000);
-
-		/* Known to be triggered by screwed up NOTIFY and COND... */
-		if (ustatus & 0x00000001) {
-			u32 addr = nv_rd32(dev, 0x400808);
-			u32 subc = (addr & 0x00070000) >> 16;
-			u32 mthd = (addr & 0x00001ffc);
-			u32 datal = nv_rd32(dev, 0x40080c);
-			u32 datah = nv_rd32(dev, 0x400810);
-			u32 class = nv_rd32(dev, 0x400814);
-			u32 r848 = nv_rd32(dev, 0x400848);
-
-			NV_INFO(dev, "PGRAPH - TRAP DISPATCH_FAULT\n");
-			if (display && (addr & 0x80000000)) {
-				NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
-					     "subc %d class 0x%04x mthd 0x%04x "
-					     "data 0x%08x%08x "
-					     "400808 0x%08x 400848 0x%08x\n",
-					chid, inst, subc, class, mthd, datah,
-					datal, addr, r848);
-			} else
-			if (display) {
-				NV_INFO(dev, "PGRAPH - no stuck command?\n");
-			}
-
-			nv_wr32(dev, 0x400808, 0);
-			nv_wr32(dev, 0x4008e8, nv_rd32(dev, 0x4008e8) & 3);
-			nv_wr32(dev, 0x400848, 0);
-			ustatus &= ~0x00000001;
-		}
-
-		if (ustatus & 0x00000002) {
-			u32 addr = nv_rd32(dev, 0x40084c);
-			u32 subc = (addr & 0x00070000) >> 16;
-			u32 mthd = (addr & 0x00001ffc);
-			u32 data = nv_rd32(dev, 0x40085c);
-			u32 class = nv_rd32(dev, 0x400814);
-
-			NV_INFO(dev, "PGRAPH - TRAP DISPATCH_QUERY\n");
-			if (display && (addr & 0x80000000)) {
-				NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) "
-					     "subc %d class 0x%04x mthd 0x%04x "
-					     "data 0x%08x 40084c 0x%08x\n",
-					chid, inst, subc, class, mthd,
-					data, addr);
-			} else
-			if (display) {
-				NV_INFO(dev, "PGRAPH - no stuck command?\n");
-			}
-
-			nv_wr32(dev, 0x40084c, 0);
-			ustatus &= ~0x00000002;
-		}
-
-		if (ustatus && display) {
-			NV_INFO(dev, "PGRAPH - TRAP_DISPATCH (unknown "
-				      "0x%08x)\n", ustatus);
-		}
-
-		nv_wr32(dev, 0x400804, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x001);
-		status &= ~0x001;
-		if (!status)
-			return 0;
-	}
-
-	/* M2MF: Memory to memory copy engine. */
-	if (status & 0x002) {
-		u32 ustatus = nv_rd32(dev, 0x406800) & 0x7fffffff;
-		if (display) {
-			NV_INFO(dev, "PGRAPH - TRAP_M2MF");
-			nouveau_bitfield_print(nv50_graph_trap_m2mf, ustatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - TRAP_M2MF %08x %08x %08x %08x\n",
-				nv_rd32(dev, 0x406804), nv_rd32(dev, 0x406808),
-				nv_rd32(dev, 0x40680c), nv_rd32(dev, 0x406810));
-
-		}
-
-		/* No sane way found yet -- just reset the bugger. */
-		nv_wr32(dev, 0x400040, 2);
-		nv_wr32(dev, 0x400040, 0);
-		nv_wr32(dev, 0x406800, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x002);
-		status &= ~0x002;
-	}
-
-	/* VFETCH: Fetches data from vertex buffers. */
-	if (status & 0x004) {
-		u32 ustatus = nv_rd32(dev, 0x400c04) & 0x7fffffff;
-		if (display) {
-			NV_INFO(dev, "PGRAPH - TRAP_VFETCH");
-			nouveau_bitfield_print(nv50_graph_trap_vfetch, ustatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - TRAP_VFETCH %08x %08x %08x %08x\n",
-				nv_rd32(dev, 0x400c00), nv_rd32(dev, 0x400c08),
-				nv_rd32(dev, 0x400c0c), nv_rd32(dev, 0x400c10));
-		}
-
-		nv_wr32(dev, 0x400c04, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x004);
-		status &= ~0x004;
-	}
-
-	/* STRMOUT: DirectX streamout / OpenGL transform feedback. */
-	if (status & 0x008) {
-		ustatus = nv_rd32(dev, 0x401800) & 0x7fffffff;
-		if (display) {
-			NV_INFO(dev, "PGRAPH - TRAP_STRMOUT");
-			nouveau_bitfield_print(nv50_graph_trap_strmout, ustatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - TRAP_STRMOUT %08x %08x %08x %08x\n",
-				nv_rd32(dev, 0x401804), nv_rd32(dev, 0x401808),
-				nv_rd32(dev, 0x40180c), nv_rd32(dev, 0x401810));
-
-		}
-
-		/* No sane way found yet -- just reset the bugger. */
-		nv_wr32(dev, 0x400040, 0x80);
-		nv_wr32(dev, 0x400040, 0);
-		nv_wr32(dev, 0x401800, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x008);
-		status &= ~0x008;
-	}
-
-	/* CCACHE: Handles code and c[] caches and fills them. */
-	if (status & 0x010) {
-		ustatus = nv_rd32(dev, 0x405018) & 0x7fffffff;
-		if (display) {
-			NV_INFO(dev, "PGRAPH - TRAP_CCACHE");
-			nouveau_bitfield_print(nv50_graph_trap_ccache, ustatus);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - TRAP_CCACHE %08x %08x %08x %08x"
-				     " %08x %08x %08x\n",
-				nv_rd32(dev, 0x405000), nv_rd32(dev, 0x405004),
-				nv_rd32(dev, 0x405008), nv_rd32(dev, 0x40500c),
-				nv_rd32(dev, 0x405010), nv_rd32(dev, 0x405014),
-				nv_rd32(dev, 0x40501c));
-
-		}
-
-		nv_wr32(dev, 0x405018, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x010);
-		status &= ~0x010;
-	}
-
-	/* Unknown, not seen yet... 0x402000 is the only trap status reg
-	 * remaining, so try to handle it anyway. Perhaps related to that
-	 * unknown DMA slot on tesla? */
-	if (status & 0x20) {
-		ustatus = nv_rd32(dev, 0x402000) & 0x7fffffff;
-		if (display)
-			NV_INFO(dev, "PGRAPH - TRAP_UNKC04 0x%08x\n", ustatus);
-		nv_wr32(dev, 0x402000, 0xc0000000);
-		/* no status modifiction on purpose */
-	}
-
-	/* TEXTURE: CUDA texturing units */
-	if (status & 0x040) {
-		nv50_pgraph_tp_trap(dev, 6, 0x408900, 0x408600, display,
-				    "PGRAPH - TRAP_TEXTURE");
-		nv_wr32(dev, 0x400108, 0x040);
-		status &= ~0x040;
-	}
-
-	/* MP: CUDA execution engines. */
-	if (status & 0x080) {
-		nv50_pgraph_tp_trap(dev, 7, 0x408314, 0x40831c, display,
-				    "PGRAPH - TRAP_MP");
-		nv_wr32(dev, 0x400108, 0x080);
-		status &= ~0x080;
-	}
-
-	/* TPDMA:  Handles TP-initiated uncached memory accesses:
-	 * l[], g[], stack, 2d surfaces, render targets. */
-	if (status & 0x100) {
-		nv50_pgraph_tp_trap(dev, 8, 0x408e08, 0x408708, display,
-				    "PGRAPH - TRAP_TPDMA");
-		nv_wr32(dev, 0x400108, 0x100);
-		status &= ~0x100;
-	}
-
-	if (status) {
-		if (display)
-			NV_INFO(dev, "PGRAPH - TRAP: unknown 0x%08x\n", status);
-		nv_wr32(dev, 0x400108, status);
-	}
-
-	return 1;
-}
-
-int
-nv50_graph_isr_chid(struct drm_device *dev, u64 inst)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (i = 0; i < pfifo->channels; i++) {
-		chan = dev_priv->channels.ptr[i];
-		if (!chan || !chan->ramin)
-			continue;
-
-		if (inst == chan->ramin->vinst)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return i;
-}
-
-static void
-nv50_graph_isr(struct drm_device *dev)
-{
-	u32 stat;
-
-	while ((stat = nv_rd32(dev, 0x400100))) {
-		u64 inst = (u64)(nv_rd32(dev, 0x40032c) & 0x0fffffff) << 12;
-		u32 chid = nv50_graph_isr_chid(dev, inst);
-		u32 addr = nv_rd32(dev, NV04_PGRAPH_TRAPPED_ADDR);
-		u32 subc = (addr & 0x00070000) >> 16;
-		u32 mthd = (addr & 0x00001ffc);
-		u32 data = nv_rd32(dev, NV04_PGRAPH_TRAPPED_DATA);
-		u32 class = nv_rd32(dev, 0x400814);
-		u32 show = stat;
-
-		if (stat & 0x00000010) {
-			if (!nouveau_gpuobj_mthd_call2(dev, chid, class,
-						       mthd, data))
-				show &= ~0x00000010;
-		}
-
-		show = (show && nouveau_ratelimit()) ? show : 0;
-
-		if (show & 0x00100000) {
-			u32 ecode = nv_rd32(dev, 0x400110);
-			NV_INFO(dev, "PGRAPH - DATA_ERROR ");
-			nouveau_enum_print(nv50_data_error_names, ecode);
-			printk("\n");
-		}
-
-		if (stat & 0x00200000) {
-			if (!nv50_pgraph_trap_handler(dev, show, inst, chid))
-				show &= ~0x00200000;
-		}
-
-		nv_wr32(dev, 0x400100, stat);
-		nv_wr32(dev, 0x400500, 0x00010001);
-
-		if (show) {
-			NV_INFO(dev, "PGRAPH -");
-			nouveau_bitfield_print(nv50_graph_intr, show);
-			printk("\n");
-			NV_INFO(dev, "PGRAPH - ch %d (0x%010llx) subc %d "
-				     "class 0x%04x mthd 0x%04x data 0x%08x\n",
-				chid, inst, subc, class, mthd, data);
-			nv50_fb_vm_trap(dev, 1);
-		}
-	}
-
-	if (nv_rd32(dev, 0x400824) & (1 << 31))
-		nv_wr32(dev, 0x400824, nv_rd32(dev, 0x400824) & ~(1 << 31));
-}
-
-static void
-nv50_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nv50_graph_engine *pgraph = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-
-	nouveau_irq_unregister(dev, 12);
-	kfree(pgraph);
-}
-
-int
-nv50_graph_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_graph_engine *pgraph;
-	int ret;
-
-	pgraph = kzalloc(sizeof(*pgraph),GFP_KERNEL);
-	if (!pgraph)
-		return -ENOMEM;
-
-	ret = nv50_grctx_init(dev, pgraph->ctxprog, ARRAY_SIZE(pgraph->ctxprog),
-				  &pgraph->ctxprog_size,
-				  &pgraph->grctx_size);
-	if (ret) {
-		NV_ERROR(dev, "PGRAPH: ctxprog build failed\n");
-		kfree(pgraph);
-		return 0;
-	}
-
-	pgraph->base.destroy = nv50_graph_destroy;
-	pgraph->base.init = nv50_graph_init;
-	pgraph->base.fini = nv50_graph_fini;
-	pgraph->base.context_new = nv50_graph_context_new;
-	pgraph->base.context_del = nv50_graph_context_del;
-	pgraph->base.object_new = nv50_graph_object_new;
-	if (dev_priv->chipset == 0x50 || dev_priv->chipset == 0xac)
-		pgraph->base.tlb_flush = nv50_graph_tlb_flush;
-	else
-		pgraph->base.tlb_flush = nv84_graph_tlb_flush;
-
-	nouveau_irq_register(dev, 12, nv50_graph_isr);
-
-	NVOBJ_ENGINE_ADD(dev, GR, &pgraph->base);
-	NVOBJ_CLASS(dev, 0x0030, GR); /* null */
-	NVOBJ_CLASS(dev, 0x5039, GR); /* m2mf */
-	NVOBJ_CLASS(dev, 0x502d, GR); /* 2d */
-
-	/* tesla */
-	if (dev_priv->chipset == 0x50)
-		NVOBJ_CLASS(dev, 0x5097, GR); /* tesla (nv50) */
-	else
-	if (dev_priv->chipset < 0xa0)
-		NVOBJ_CLASS(dev, 0x8297, GR); /* tesla (nv8x/nv9x) */
-	else {
-		switch (dev_priv->chipset) {
-		case 0xa0:
-		case 0xaa:
-		case 0xac:
-			NVOBJ_CLASS(dev, 0x8397, GR);
-			break;
-		case 0xa3:
-		case 0xa5:
-		case 0xa8:
-			NVOBJ_CLASS(dev, 0x8597, GR);
-			break;
-		case 0xaf:
-			NVOBJ_CLASS(dev, 0x8697, GR);
-			break;
-		}
-	}
-
-	/* compute */
-	NVOBJ_CLASS(dev, 0x50c0, GR);
-	if (dev_priv->chipset  > 0xa0 &&
-	    dev_priv->chipset != 0xaa &&
-	    dev_priv->chipset != 0xac)
-		NVOBJ_CLASS(dev, 0x85c0, GR);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_instmem.c b/drivers/gpu/drm/nouveau/nv50_instmem.c
deleted file mode 100644
index 0bba54f1180..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_instmem.c
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Copyright (C) 2007 Ben Skeggs.
- *
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_vm.h"
-
-#define BAR1_VM_BASE 0x0020000000ULL
-#define BAR1_VM_SIZE pci_resource_len(dev->pdev, 1)
-#define BAR3_VM_BASE 0x0000000000ULL
-#define BAR3_VM_SIZE pci_resource_len(dev->pdev, 3)
-
-struct nv50_instmem_priv {
-	uint32_t save1700[5]; /* 0x1700->0x1710 */
-
-	struct nouveau_gpuobj *bar1_dmaobj;
-	struct nouveau_gpuobj *bar3_dmaobj;
-};
-
-static void
-nv50_channel_del(struct nouveau_channel **pchan)
-{
-	struct nouveau_channel *chan;
-
-	chan = *pchan;
-	*pchan = NULL;
-	if (!chan)
-		return;
-
-	nouveau_gpuobj_ref(NULL, &chan->ramfc);
-	nouveau_vm_ref(NULL, &chan->vm, chan->vm_pd);
-	nouveau_gpuobj_ref(NULL, &chan->vm_pd);
-	if (drm_mm_initialized(&chan->ramin_heap))
-		drm_mm_takedown(&chan->ramin_heap);
-	nouveau_gpuobj_ref(NULL, &chan->ramin);
-	kfree(chan);
-}
-
-static int
-nv50_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
-		 struct nouveau_channel **pchan)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 pgd = (dev_priv->chipset == 0x50) ? 0x1400 : 0x0200;
-	u32  fc = (dev_priv->chipset == 0x50) ? 0x0000 : 0x4200;
-	struct nouveau_channel *chan;
-	int ret, i;
-
-	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
-	if (!chan)
-		return -ENOMEM;
-	chan->dev = dev;
-
-	ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
-	if (ret) {
-		nv50_channel_del(&chan);
-		return ret;
-	}
-
-	ret = drm_mm_init(&chan->ramin_heap, 0x6000, chan->ramin->size - 0x6000);
-	if (ret) {
-		nv50_channel_del(&chan);
-		return ret;
-	}
-
-	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
-				      chan->ramin->pinst + pgd,
-				      chan->ramin->vinst + pgd,
-				      0x4000, NVOBJ_FLAG_ZERO_ALLOC,
-				      &chan->vm_pd);
-	if (ret) {
-		nv50_channel_del(&chan);
-		return ret;
-	}
-
-	for (i = 0; i < 0x4000; i += 8) {
-		nv_wo32(chan->vm_pd, i + 0, 0x00000000);
-		nv_wo32(chan->vm_pd, i + 4, 0xdeadcafe);
-	}
-
-	ret = nouveau_vm_ref(vm, &chan->vm, chan->vm_pd);
-	if (ret) {
-		nv50_channel_del(&chan);
-		return ret;
-	}
-
-	ret = nouveau_gpuobj_new_fake(dev, chan->ramin->pinst == ~0 ? ~0 :
-				      chan->ramin->pinst + fc,
-				      chan->ramin->vinst + fc, 0x100,
-				      NVOBJ_FLAG_ZERO_ALLOC, &chan->ramfc);
-	if (ret) {
-		nv50_channel_del(&chan);
-		return ret;
-	}
-
-	*pchan = chan;
-	return 0;
-}
-
-int
-nv50_instmem_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_instmem_priv *priv;
-	struct nouveau_channel *chan;
-	struct nouveau_vm *vm;
-	int ret, i;
-	u32 tmp;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	dev_priv->engine.instmem.priv = priv;
-
-	/* Save state, will restore at takedown. */
-	for (i = 0x1700; i <= 0x1710; i += 4)
-		priv->save1700[(i-0x1700)/4] = nv_rd32(dev, i);
-
-	/* Global PRAMIN heap */
-	ret = drm_mm_init(&dev_priv->ramin_heap, 0, dev_priv->ramin_size);
-	if (ret) {
-		NV_ERROR(dev, "Failed to init RAMIN heap\n");
-		goto error;
-	}
-
-	/* BAR3 */
-	ret = nouveau_vm_new(dev, BAR3_VM_BASE, BAR3_VM_SIZE, BAR3_VM_BASE,
-			     &dev_priv->bar3_vm);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, (BAR3_VM_SIZE >> 12) * 8,
-				 0x1000, NVOBJ_FLAG_DONT_MAP |
-				 NVOBJ_FLAG_ZERO_ALLOC,
-				 &dev_priv->bar3_vm->pgt[0].obj[0]);
-	if (ret)
-		goto error;
-	dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
-
-	nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
-
-	ret = nv50_channel_new(dev, 128 * 1024, dev_priv->bar3_vm, &chan);
-	if (ret)
-		goto error;
-	dev_priv->channels.ptr[0] = dev_priv->channels.ptr[127] = chan;
-
-	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR3_VM_BASE, BAR3_VM_SIZE,
-				  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
-				  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
-				  &priv->bar3_dmaobj);
-	if (ret)
-		goto error;
-
-	nv_wr32(dev, 0x001704, 0x00000000 | (chan->ramin->vinst >> 12));
-	nv_wr32(dev, 0x001704, 0x40000000 | (chan->ramin->vinst >> 12));
-	nv_wr32(dev, 0x00170c, 0x80000000 | (priv->bar3_dmaobj->cinst >> 4));
-
-	dev_priv->engine.instmem.flush(dev);
-	dev_priv->ramin_available = true;
-
-	tmp = nv_ro32(chan->ramin, 0);
-	nv_wo32(chan->ramin, 0, ~tmp);
-	if (nv_ro32(chan->ramin, 0) != ~tmp) {
-		NV_ERROR(dev, "PRAMIN readback failed\n");
-		ret = -EIO;
-		goto error;
-	}
-	nv_wo32(chan->ramin, 0, tmp);
-
-	/* BAR1 */
-	ret = nouveau_vm_new(dev, BAR1_VM_BASE, BAR1_VM_SIZE, BAR1_VM_BASE, &vm);
-	if (ret)
-		goto error;
-
-	ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, chan->vm_pd);
-	if (ret)
-		goto error;
-	nouveau_vm_ref(NULL, &vm, NULL);
-
-	ret = nv50_gpuobj_dma_new(chan, 0x0000, BAR1_VM_BASE, BAR1_VM_SIZE,
-				  NV_MEM_TARGET_VM, NV_MEM_ACCESS_VM,
-				  NV_MEM_TYPE_VM, NV_MEM_COMP_VM,
-				  &priv->bar1_dmaobj);
-	if (ret)
-		goto error;
-
-	nv_wr32(dev, 0x001708, 0x80000000 | (priv->bar1_dmaobj->cinst >> 4));
-	for (i = 0; i < 8; i++)
-		nv_wr32(dev, 0x1900 + (i*4), 0);
-
-	/* Create shared channel VM, space is reserved at the beginning
-	 * to catch "NULL pointer" references
-	 */
-	ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0020000000ULL,
-			     &dev_priv->chan_vm);
-	if (ret)
-		return ret;
-
-	return 0;
-
-error:
-	nv50_instmem_takedown(dev);
-	return ret;
-}
-
-void
-nv50_instmem_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
-	int i;
-
-	NV_DEBUG(dev, "\n");
-
-	if (!priv)
-		return;
-
-	dev_priv->ramin_available = false;
-
-	nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
-
-	for (i = 0x1700; i <= 0x1710; i += 4)
-		nv_wr32(dev, i, priv->save1700[(i - 0x1700) / 4]);
-
-	nouveau_gpuobj_ref(NULL, &priv->bar3_dmaobj);
-	nouveau_gpuobj_ref(NULL, &priv->bar1_dmaobj);
-
-	nouveau_vm_ref(NULL, &dev_priv->bar1_vm, chan->vm_pd);
-	dev_priv->channels.ptr[127] = 0;
-	nv50_channel_del(&dev_priv->channels.ptr[0]);
-
-	nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
-	nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
-
-	if (drm_mm_initialized(&dev_priv->ramin_heap))
-		drm_mm_takedown(&dev_priv->ramin_heap);
-
-	dev_priv->engine.instmem.priv = NULL;
-	kfree(priv);
-}
-
-int
-nv50_instmem_suspend(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	dev_priv->ramin_available = false;
-	return 0;
-}
-
-void
-nv50_instmem_resume(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_instmem_priv *priv = dev_priv->engine.instmem.priv;
-	struct nouveau_channel *chan = dev_priv->channels.ptr[0];
-	int i;
-
-	/* Poke the relevant regs, and pray it works :) */
-	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12));
-	nv_wr32(dev, NV50_PUNK_UNK1710, 0);
-	nv_wr32(dev, NV50_PUNK_BAR_CFG_BASE, (chan->ramin->vinst >> 12) |
-					 NV50_PUNK_BAR_CFG_BASE_VALID);
-	nv_wr32(dev, NV50_PUNK_BAR1_CTXDMA, (priv->bar1_dmaobj->cinst >> 4) |
-					NV50_PUNK_BAR1_CTXDMA_VALID);
-	nv_wr32(dev, NV50_PUNK_BAR3_CTXDMA, (priv->bar3_dmaobj->cinst >> 4) |
-					NV50_PUNK_BAR3_CTXDMA_VALID);
-
-	for (i = 0; i < 8; i++)
-		nv_wr32(dev, 0x1900 + (i*4), 0);
-
-	dev_priv->ramin_available = true;
-}
-
-struct nv50_gpuobj_node {
-	struct nouveau_mem *vram;
-	struct nouveau_vma chan_vma;
-	u32 align;
-};
-
-int
-nv50_instmem_get(struct nouveau_gpuobj *gpuobj, struct nouveau_channel *chan,
-		 u32 size, u32 align)
-{
-	struct drm_device *dev = gpuobj->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	struct nv50_gpuobj_node *node = NULL;
-	int ret;
-
-	node = kzalloc(sizeof(*node), GFP_KERNEL);
-	if (!node)
-		return -ENOMEM;
-	node->align = align;
-
-	size  = (size + 4095) & ~4095;
-	align = max(align, (u32)4096);
-
-	ret = vram->get(dev, size, align, 0, 0, &node->vram);
-	if (ret) {
-		kfree(node);
-		return ret;
-	}
-
-	gpuobj->vinst = node->vram->offset;
-
-	if (gpuobj->flags & NVOBJ_FLAG_VM) {
-		u32 flags = NV_MEM_ACCESS_RW;
-		if (!(gpuobj->flags & NVOBJ_FLAG_VM_USER))
-			flags |= NV_MEM_ACCESS_SYS;
-
-		ret = nouveau_vm_get(chan->vm, size, 12, flags,
-				     &node->chan_vma);
-		if (ret) {
-			vram->put(dev, &node->vram);
-			kfree(node);
-			return ret;
-		}
-
-		nouveau_vm_map(&node->chan_vma, node->vram);
-		gpuobj->linst = node->chan_vma.offset;
-	}
-
-	gpuobj->size = size;
-	gpuobj->node = node;
-	return 0;
-}
-
-void
-nv50_instmem_put(struct nouveau_gpuobj *gpuobj)
-{
-	struct drm_device *dev = gpuobj->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	struct nv50_gpuobj_node *node;
-
-	node = gpuobj->node;
-	gpuobj->node = NULL;
-
-	if (node->chan_vma.node) {
-		nouveau_vm_unmap(&node->chan_vma);
-		nouveau_vm_put(&node->chan_vma);
-	}
-	vram->put(dev, &node->vram);
-	kfree(node);
-}
-
-int
-nv50_instmem_map(struct nouveau_gpuobj *gpuobj)
-{
-	struct drm_nouveau_private *dev_priv = gpuobj->dev->dev_private;
-	struct nv50_gpuobj_node *node = gpuobj->node;
-	int ret;
-
-	ret = nouveau_vm_get(dev_priv->bar3_vm, gpuobj->size, 12,
-			     NV_MEM_ACCESS_RW, &node->vram->bar_vma);
-	if (ret)
-		return ret;
-
-	nouveau_vm_map(&node->vram->bar_vma, node->vram);
-	gpuobj->pinst = node->vram->bar_vma.offset;
-	return 0;
-}
-
-void
-nv50_instmem_unmap(struct nouveau_gpuobj *gpuobj)
-{
-	struct nv50_gpuobj_node *node = gpuobj->node;
-
-	if (node->vram->bar_vma.node) {
-		nouveau_vm_unmap(&node->vram->bar_vma);
-		nouveau_vm_put(&node->vram->bar_vma);
-	}
-}
-
-void
-nv50_instmem_flush(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->vm_lock, flags);
-	nv_wr32(dev, 0x00330c, 0x00000001);
-	if (!nv_wait(dev, 0x00330c, 0x00000002, 0x00000000))
-		NV_ERROR(dev, "PRAMIN flush timeout\n");
-	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
-}
-
-void
-nv84_instmem_flush(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dev_priv->vm_lock, flags);
-	nv_wr32(dev, 0x070000, 0x00000001);
-	if (!nv_wait(dev, 0x070000, 0x00000002, 0x00000000))
-		NV_ERROR(dev, "PRAMIN flush timeout\n");
-	spin_unlock_irqrestore(&dev_priv->vm_lock, flags);
-}
-
diff --git a/drivers/gpu/drm/nouveau/nv50_mc.c b/drivers/gpu/drm/nouveau/nv50_mc.c
deleted file mode 100644
index e0a9c3faa20..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_mc.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (C) 2007 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-
-int
-nv50_mc_init(struct drm_device *dev)
-{
-	nv_wr32(dev, NV03_PMC_ENABLE, 0xFFFFFFFF);
-	return 0;
-}
-
-void nv50_mc_takedown(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_mpeg.c b/drivers/gpu/drm/nouveau/nv50_mpeg.c
deleted file mode 100644
index 90e8ed22cfc..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_mpeg.c
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-
-struct nv50_mpeg_engine {
-	struct nouveau_exec_engine base;
-};
-
-static inline u32
-CTX_PTR(struct drm_device *dev, u32 offset)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->chipset == 0x50)
-		offset += 0x0260;
-	else
-		offset += 0x0060;
-
-	return offset;
-}
-
-static int
-nv50_mpeg_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramin = chan->ramin;
-	struct nouveau_gpuobj *ctx = NULL;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	ret = nouveau_gpuobj_new(dev, chan, 128 * 4, 0, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &ctx);
-	if (ret)
-		return ret;
-
-	nv_wo32(ramin, CTX_PTR(dev, 0x00), 0x80190002);
-	nv_wo32(ramin, CTX_PTR(dev, 0x04), ctx->vinst + ctx->size - 1);
-	nv_wo32(ramin, CTX_PTR(dev, 0x08), ctx->vinst);
-	nv_wo32(ramin, CTX_PTR(dev, 0x0c), 0);
-	nv_wo32(ramin, CTX_PTR(dev, 0x10), 0);
-	nv_wo32(ramin, CTX_PTR(dev, 0x14), 0x00010000);
-
-	nv_wo32(ctx, 0x70, 0x00801ec1);
-	nv_wo32(ctx, 0x7c, 0x0000037c);
-	dev_priv->engine.instmem.flush(dev);
-
-	chan->engctx[engine] = ctx;
-	return 0;
-}
-
-static void
-nv50_mpeg_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	int i;
-
-	for (i = 0x00; i <= 0x14; i += 4)
-		nv_wo32(chan->ramin, CTX_PTR(dev, i), 0x00000000);
-
-	nouveau_gpuobj_ref(NULL, &ctx);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nv50_mpeg_object_new(struct nouveau_channel *chan, int engine,
-		     u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 2;
-	obj->class  = class;
-
-	nv_wo32(obj, 0x00, class);
-	nv_wo32(obj, 0x04, 0x00000000);
-	nv_wo32(obj, 0x08, 0x00000000);
-	nv_wo32(obj, 0x0c, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static void
-nv50_mpeg_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 0x08);
-}
-
-static int
-nv50_mpeg_init(struct drm_device *dev, int engine)
-{
-	nv_wr32(dev, 0x00b32c, 0x00000000);
-	nv_wr32(dev, 0x00b314, 0x00000100);
-	nv_wr32(dev, 0x00b0e0, 0x0000001a);
-
-	nv_wr32(dev, 0x00b220, 0x00000044);
-	nv_wr32(dev, 0x00b300, 0x00801ec1);
-	nv_wr32(dev, 0x00b390, 0x00000000);
-	nv_wr32(dev, 0x00b394, 0x00000000);
-	nv_wr32(dev, 0x00b398, 0x00000000);
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000001);
-
-	nv_wr32(dev, 0x00b100, 0xffffffff);
-	nv_wr32(dev, 0x00b140, 0xffffffff);
-
-	if (!nv_wait(dev, 0x00b200, 0x00000001, 0x00000000)) {
-		NV_ERROR(dev, "PMPEG init: 0x%08x\n", nv_rd32(dev, 0x00b200));
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-static int
-nv50_mpeg_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, 0x00b32c, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x00b140, 0x00000000);
-	return 0;
-}
-
-static void
-nv50_mpeg_isr(struct drm_device *dev)
-{
-	u32 stat = nv_rd32(dev, 0x00b100);
-	u32 type = nv_rd32(dev, 0x00b230);
-	u32 mthd = nv_rd32(dev, 0x00b234);
-	u32 data = nv_rd32(dev, 0x00b238);
-	u32 show = stat;
-
-	if (stat & 0x01000000) {
-		/* happens on initial binding of the object */
-		if (type == 0x00000020 && mthd == 0x0000) {
-			nv_wr32(dev, 0x00b308, 0x00000100);
-			show &= ~0x01000000;
-		}
-	}
-
-	if (show && nouveau_ratelimit()) {
-		NV_INFO(dev, "PMPEG - 0x%08x 0x%08x 0x%08x 0x%08x\n",
-			stat, type, mthd, data);
-	}
-
-	nv_wr32(dev, 0x00b100, stat);
-	nv_wr32(dev, 0x00b230, 0x00000001);
-	nv50_fb_vm_trap(dev, 1);
-}
-
-static void
-nv50_vpe_isr(struct drm_device *dev)
-{
-	if (nv_rd32(dev, 0x00b100))
-		nv50_mpeg_isr(dev);
-
-	if (nv_rd32(dev, 0x00b800)) {
-		u32 stat = nv_rd32(dev, 0x00b800);
-		NV_INFO(dev, "PMSRCH: 0x%08x\n", stat);
-		nv_wr32(dev, 0xb800, stat);
-	}
-}
-
-static void
-nv50_mpeg_destroy(struct drm_device *dev, int engine)
-{
-	struct nv50_mpeg_engine *pmpeg = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 0);
-
-	NVOBJ_ENGINE_DEL(dev, MPEG);
-	kfree(pmpeg);
-}
-
-int
-nv50_mpeg_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv50_mpeg_engine *pmpeg;
-
-	pmpeg = kzalloc(sizeof(*pmpeg), GFP_KERNEL);
-	if (!pmpeg)
-		return -ENOMEM;
-
-	pmpeg->base.destroy = nv50_mpeg_destroy;
-	pmpeg->base.init = nv50_mpeg_init;
-	pmpeg->base.fini = nv50_mpeg_fini;
-	pmpeg->base.context_new = nv50_mpeg_context_new;
-	pmpeg->base.context_del = nv50_mpeg_context_del;
-	pmpeg->base.object_new = nv50_mpeg_object_new;
-	pmpeg->base.tlb_flush = nv50_mpeg_tlb_flush;
-
-	if (dev_priv->chipset == 0x50) {
-		nouveau_irq_register(dev, 0, nv50_vpe_isr);
-		NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
-		NVOBJ_CLASS(dev, 0x3174, MPEG);
-#if 0
-		NVOBJ_ENGINE_ADD(dev, ME, &pme->base);
-		NVOBJ_CLASS(dev, 0x4075, ME);
-#endif
-	} else {
-		nouveau_irq_register(dev, 0, nv50_mpeg_isr);
-		NVOBJ_ENGINE_ADD(dev, MPEG, &pmpeg->base);
-		NVOBJ_CLASS(dev, 0x8274, MPEG);
-	}
-
-	return 0;
-
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_pm.c b/drivers/gpu/drm/nouveau/nv50_pm.c
index d020ed4979b..c4a65039b1c 100644
--- a/drivers/gpu/drm/nouveau/nv50_pm.c
+++ b/drivers/gpu/drm/nouveau/nv50_pm.c
@@ -22,14 +22,20 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
 #include "nouveau_bios.h"
 #include "nouveau_hw.h"
 #include "nouveau_pm.h"
 #include "nouveau_hwsq.h"
+
 #include "nv50_display.h"
 
+#include <subdev/bios/pll.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
 enum clk_src {
 	clk_src_crystal,
 	clk_src_href,
@@ -49,19 +55,20 @@ static u32 read_clk(struct drm_device *, enum clk_src);
 static u32
 read_div(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 
-	switch (dev_priv->chipset) {
+	switch (nv_device(drm->device)->chipset) {
 	case 0x50: /* it exists, but only has bit 31, not the dividers.. */
 	case 0x84:
 	case 0x86:
 	case 0x98:
 	case 0xa0:
-		return nv_rd32(dev, 0x004700);
+		return nv_rd32(device, 0x004700);
 	case 0x92:
 	case 0x94:
 	case 0x96:
-		return nv_rd32(dev, 0x004800);
+		return nv_rd32(device, 0x004800);
 	default:
 		return 0x00000000;
 	}
@@ -70,12 +77,13 @@ read_div(struct drm_device *dev)
 static u32
 read_pll_src(struct drm_device *dev, u32 base)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u32 coef, ref = read_clk(dev, clk_src_crystal);
-	u32 rsel = nv_rd32(dev, 0x00e18c);
+	u32 rsel = nv_rd32(device, 0x00e18c);
 	int P, N, M, id;
 
-	switch (dev_priv->chipset) {
+	switch (nv_device(drm->device)->chipset) {
 	case 0x50:
 	case 0xa0:
 		switch (base) {
@@ -84,11 +92,11 @@ read_pll_src(struct drm_device *dev, u32 base)
 		case 0x4008: id = !!(rsel & 0x00000008); break;
 		case 0x4030: id = 0; break;
 		default:
-			NV_ERROR(dev, "ref: bad pll 0x%06x\n", base);
+			NV_ERROR(drm, "ref: bad pll 0x%06x\n", base);
 			return 0;
 		}
 
-		coef = nv_rd32(dev, 0x00e81c + (id * 0x0c));
+		coef = nv_rd32(device, 0x00e81c + (id * 0x0c));
 		ref *=  (coef & 0x01000000) ? 2 : 4;
 		P    =  (coef & 0x00070000) >> 16;
 		N    = ((coef & 0x0000ff00) >> 8) + 1;
@@ -97,7 +105,7 @@ read_pll_src(struct drm_device *dev, u32 base)
 	case 0x84:
 	case 0x86:
 	case 0x92:
-		coef = nv_rd32(dev, 0x00e81c);
+		coef = nv_rd32(device, 0x00e81c);
 		P    = (coef & 0x00070000) >> 16;
 		N    = (coef & 0x0000ff00) >> 8;
 		M    = (coef & 0x000000ff) >> 0;
@@ -105,14 +113,14 @@ read_pll_src(struct drm_device *dev, u32 base)
 	case 0x94:
 	case 0x96:
 	case 0x98:
-		rsel = nv_rd32(dev, 0x00c050);
+		rsel = nv_rd32(device, 0x00c050);
 		switch (base) {
 		case 0x4020: rsel = (rsel & 0x00000003) >> 0; break;
 		case 0x4008: rsel = (rsel & 0x0000000c) >> 2; break;
 		case 0x4028: rsel = (rsel & 0x00001800) >> 11; break;
 		case 0x4030: rsel = 3; break;
 		default:
-			NV_ERROR(dev, "ref: bad pll 0x%06x\n", base);
+			NV_ERROR(drm, "ref: bad pll 0x%06x\n", base);
 			return 0;
 		}
 
@@ -123,8 +131,8 @@ read_pll_src(struct drm_device *dev, u32 base)
 		case 3: id = 0; break;
 		}
 
-		coef =  nv_rd32(dev, 0x00e81c + (id * 0x28));
-		P    = (nv_rd32(dev, 0x00e824 + (id * 0x28)) >> 16) & 7;
+		coef =  nv_rd32(device, 0x00e81c + (id * 0x28));
+		P    = (nv_rd32(device, 0x00e824 + (id * 0x28)) >> 16) & 7;
 		P   += (coef & 0x00070000) >> 16;
 		N    = (coef & 0x0000ff00) >> 8;
 		M    = (coef & 0x000000ff) >> 0;
@@ -141,7 +149,9 @@ read_pll_src(struct drm_device *dev, u32 base)
 static u32
 read_pll_ref(struct drm_device *dev, u32 base)
 {
-	u32 src, mast = nv_rd32(dev, 0x00c040);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	u32 src, mast = nv_rd32(device, 0x00c040);
 
 	switch (base) {
 	case 0x004028:
@@ -159,7 +169,7 @@ read_pll_ref(struct drm_device *dev, u32 base)
 	case 0x00e810:
 		return read_clk(dev, clk_src_crystal);
 	default:
-		NV_ERROR(dev, "bad pll 0x%06x\n", base);
+		NV_ERROR(drm, "bad pll 0x%06x\n", base);
 		return 0;
 	}
 
@@ -171,17 +181,18 @@ read_pll_ref(struct drm_device *dev, u32 base)
 static u32
 read_pll(struct drm_device *dev, u32 base)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 mast = nv_rd32(dev, 0x00c040);
-	u32 ctrl = nv_rd32(dev, base + 0);
-	u32 coef = nv_rd32(dev, base + 4);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	u32 mast = nv_rd32(device, 0x00c040);
+	u32 ctrl = nv_rd32(device, base + 0);
+	u32 coef = nv_rd32(device, base + 4);
 	u32 ref = read_pll_ref(dev, base);
 	u32 clk = 0;
 	int N1, N2, M1, M2;
 
 	if (base == 0x004028 && (mast & 0x00100000)) {
 		/* wtf, appears to only disable post-divider on nva0 */
-		if (dev_priv->chipset != 0xa0)
+		if (nv_device(drm->device)->chipset != 0xa0)
 			return read_clk(dev, clk_src_dom6);
 	}
 
@@ -205,13 +216,14 @@ read_pll(struct drm_device *dev, u32 base)
 static u32
 read_clk(struct drm_device *dev, enum clk_src src)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	u32 mast = nv_rd32(dev, 0x00c040);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	u32 mast = nv_rd32(device, 0x00c040);
 	u32 P = 0;
 
 	switch (src) {
 	case clk_src_crystal:
-		return dev_priv->crystal;
+		return device->crystal;
 	case clk_src_href:
 		return 100000; /* PCIE reference clock */
 	case clk_src_hclk:
@@ -230,7 +242,7 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		break;
 	case clk_src_nvclk:
 		if (!(mast & 0x00100000))
-			P = (nv_rd32(dev, 0x004028) & 0x00070000) >> 16;
+			P = (nv_rd32(device, 0x004028) & 0x00070000) >> 16;
 		switch (mast & 0x00000003) {
 		case 0x00000000: return read_clk(dev, clk_src_crystal) >> P;
 		case 0x00000001: return read_clk(dev, clk_src_dom6);
@@ -239,7 +251,7 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		}
 		break;
 	case clk_src_sclk:
-		P = (nv_rd32(dev, 0x004020) & 0x00070000) >> 16;
+		P = (nv_rd32(device, 0x004020) & 0x00070000) >> 16;
 		switch (mast & 0x00000030) {
 		case 0x00000000:
 			if (mast & 0x00000080)
@@ -251,8 +263,8 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		}
 		break;
 	case clk_src_mclk:
-		P = (nv_rd32(dev, 0x004008) & 0x00070000) >> 16;
-		if (nv_rd32(dev, 0x004008) & 0x00000200) {
+		P = (nv_rd32(device, 0x004008) & 0x00070000) >> 16;
+		if (nv_rd32(device, 0x004008) & 0x00000200) {
 			switch (mast & 0x0000c000) {
 			case 0x00000000:
 				return read_clk(dev, clk_src_crystal) >> P;
@@ -266,7 +278,7 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		break;
 	case clk_src_vdec:
 		P = (read_div(dev) & 0x00000700) >> 8;
-		switch (dev_priv->chipset) {
+		switch (nv_device(drm->device)->chipset) {
 		case 0x84:
 		case 0x86:
 		case 0x92:
@@ -275,7 +287,7 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		case 0xa0:
 			switch (mast & 0x00000c00) {
 			case 0x00000000:
-				if (dev_priv->chipset == 0xa0) /* wtf?? */
+				if (nv_device(drm->device)->chipset == 0xa0) /* wtf?? */
 					return read_clk(dev, clk_src_nvclk) >> P;
 				return read_clk(dev, clk_src_crystal) >> P;
 			case 0x00000400:
@@ -303,7 +315,7 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		}
 		break;
 	case clk_src_dom6:
-		switch (dev_priv->chipset) {
+		switch (nv_device(drm->device)->chipset) {
 		case 0x50:
 		case 0xa0:
 			return read_pll(dev, 0x00e810) >> 2;
@@ -329,22 +341,22 @@ read_clk(struct drm_device *dev, enum clk_src src)
 		break;
 	}
 
-	NV_DEBUG(dev, "unknown clock source %d 0x%08x\n", src, mast);
+	NV_DEBUG(drm, "unknown clock source %d 0x%08x\n", src, mast);
 	return 0;
 }
 
 int
 nv50_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	if (dev_priv->chipset == 0xaa ||
-	    dev_priv->chipset == 0xac)
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	if (nv_device(drm->device)->chipset == 0xaa ||
+	    nv_device(drm->device)->chipset == 0xac)
 		return 0;
 
 	perflvl->core   = read_clk(dev, clk_src_nvclk);
 	perflvl->shader = read_clk(dev, clk_src_sclk);
 	perflvl->memory = read_clk(dev, clk_src_mclk);
-	if (dev_priv->chipset != 0x50) {
+	if (nv_device(drm->device)->chipset != 0x50) {
 		perflvl->vdec = read_clk(dev, clk_src_vdec);
 		perflvl->dom6 = read_clk(dev, clk_src_dom6);
 	}
@@ -363,22 +375,25 @@ struct nv50_pm_state {
 };
 
 static u32
-calc_pll(struct drm_device *dev, u32 reg, struct pll_lims *pll,
+calc_pll(struct drm_device *dev, u32 reg, struct nvbios_pll *pll,
 	 u32 clk, int *N1, int *M1, int *log2P)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nouveau_clock *pclk = nouveau_clock(device);
 	struct nouveau_pll_vals coef;
 	int ret;
 
-	ret = get_pll_limits(dev, reg, pll);
+	ret = nvbios_pll_parse(bios, reg, pll);
 	if (ret)
 		return 0;
 
-	pll->vco2.maxfreq = 0;
+	pll->vco2.max_freq = 0;
 	pll->refclk = read_pll_ref(dev, reg);
 	if (!pll->refclk)
 		return 0;
 
-	ret = nouveau_calc_pll_mnp(dev, pll, clk, &coef);
+	ret = pclk->pll_calc(pclk, pll, clk, &coef);
 	if (ret == 0)
 		return 0;
 
@@ -461,27 +476,29 @@ mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
 static u32
 mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
 {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	if (mr <= 1)
-		return nv_rd32(exec->dev, 0x1002c0 + ((mr - 0) * 4));
+		return nv_rd32(device, 0x1002c0 + ((mr - 0) * 4));
 	if (mr <= 3)
-		return nv_rd32(exec->dev, 0x1002e0 + ((mr - 2) * 4));
+		return nv_rd32(device, 0x1002e0 + ((mr - 2) * 4));
 	return 0;
 }
 
 static void
 mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
 {
-	struct drm_nouveau_private *dev_priv = exec->dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
 	struct nv50_pm_state *info = exec->priv;
 	struct hwsq_ucode *hwsq = &info->mclk_hwsq;
 
 	if (mr <= 1) {
-		if (dev_priv->vram_rank_B)
+		if (pfb->ram.ranks > 1)
 			hwsq_wr32(hwsq, 0x1002c8 + ((mr - 0) * 4), data);
 		hwsq_wr32(hwsq, 0x1002c0 + ((mr - 0) * 4), data);
 	} else
 	if (mr <= 3) {
-		if (dev_priv->vram_rank_B)
+		if (pfb->ram.ranks > 1)
 			hwsq_wr32(hwsq, 0x1002e8 + ((mr - 2) * 4), data);
 		hwsq_wr32(hwsq, 0x1002e0 + ((mr - 2) * 4), data);
 	}
@@ -490,11 +507,12 @@ mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
 static void
 mclk_clock_set(struct nouveau_mem_exec_func *exec)
 {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nv50_pm_state *info = exec->priv;
 	struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-	u32 ctrl = nv_rd32(exec->dev, 0x004008);
+	u32 ctrl = nv_rd32(device, 0x004008);
 
-	info->mmast = nv_rd32(exec->dev, 0x00c040);
+	info->mmast = nv_rd32(device, 0x00c040);
 	info->mmast &= ~0xc0000000; /* get MCLK_2 from HREF */
 	info->mmast |=  0x0000c000; /* use MCLK_2 as MPLL_BYPASS clock */
 
@@ -508,7 +526,7 @@ mclk_clock_set(struct nouveau_mem_exec_func *exec)
 static void
 mclk_timing_set(struct nouveau_mem_exec_func *exec)
 {
-	struct drm_device *dev = exec->dev;
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nv50_pm_state *info = exec->priv;
 	struct nouveau_pm_level *perflvl = info->perflvl;
 	struct hwsq_ucode *hwsq = &info->mclk_hwsq;
@@ -516,7 +534,7 @@ mclk_timing_set(struct nouveau_mem_exec_func *exec)
 
 	for (i = 0; i < 9; i++) {
 		u32 reg = 0x100220 + (i * 4);
-		u32 val = nv_rd32(dev, reg);
+		u32 val = nv_rd32(device, reg);
 		if (val != perflvl->timing.reg[i])
 			hwsq_wr32(hwsq, reg, perflvl->timing.reg[i]);
 	}
@@ -526,7 +544,8 @@ static int
 calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 	  struct nv50_pm_state *info)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
 	u32 crtc_mask = nv50_display_active_crtcs(dev);
 	struct nouveau_mem_exec_func exec = {
 		.dev = dev,
@@ -542,22 +561,22 @@ calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 		.priv = info
 	};
 	struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-	struct pll_lims pll;
+	struct nvbios_pll pll;
 	int N, M, P;
 	int ret;
 
 	/* use pcie refclock if possible, otherwise use mpll */
-	info->mctrl  = nv_rd32(dev, 0x004008);
+	info->mctrl  = nv_rd32(device, 0x004008);
 	info->mctrl &= ~0x81ff0200;
 	if (clk_same(perflvl->memory, read_clk(dev, clk_src_href))) {
-		info->mctrl |= 0x00000200 | (pll.log2p_bias << 19);
+		info->mctrl |= 0x00000200 | (pll.bias_p << 19);
 	} else {
 		ret = calc_pll(dev, 0x4008, &pll, perflvl->memory, &N, &M, &P);
 		if (ret == 0)
 			return -EINVAL;
 
 		info->mctrl |= 0x80000000 | (P << 22) | (P << 16);
-		info->mctrl |= pll.log2p_bias << 19;
+		info->mctrl |= pll.bias_p << 19;
 		info->mcoef  = (N << 8) | M;
 	}
 
@@ -567,7 +586,7 @@ calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 		hwsq_op5f(hwsq, crtc_mask, 0x00); /* wait for scanout */
 		hwsq_op5f(hwsq, crtc_mask, 0x01); /* wait for vblank */
 	}
-	if (dev_priv->chipset >= 0x92)
+	if (nv_device(drm->device)->chipset >= 0x92)
 		hwsq_wr32(hwsq, 0x611200, 0x00003300); /* disable scanout */
 	hwsq_setf(hwsq, 0x10, 0); /* disable bus access */
 	hwsq_op5f(hwsq, 0x00, 0x01); /* no idea :s */
@@ -578,7 +597,7 @@ calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 
 	hwsq_setf(hwsq, 0x10, 1); /* enable bus access */
 	hwsq_op5f(hwsq, 0x00, 0x00); /* no idea, reverse of 0x00, 0x01? */
-	if (dev_priv->chipset >= 0x92)
+	if (nv_device(drm->device)->chipset >= 0x92)
 		hwsq_wr32(hwsq, 0x611200, 0x00003330); /* enable scanout */
 	hwsq_fini(hwsq);
 	return 0;
@@ -587,16 +606,17 @@ calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
 void *
 nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nv50_pm_state *info;
 	struct hwsq_ucode *hwsq;
-	struct pll_lims pll;
+	struct nvbios_pll pll;
 	u32 out, mast, divs, ctrl;
 	int clk, ret = -EINVAL;
 	int N, M, P1, P2;
 
-	if (dev_priv->chipset == 0xaa ||
-	    dev_priv->chipset == 0xac)
+	if (nv_device(drm->device)->chipset == 0xaa ||
+	    nv_device(drm->device)->chipset == 0xac)
 		return ERR_PTR(-ENODEV);
 
 	info = kmalloc(sizeof(*info), GFP_KERNEL);
@@ -645,7 +665,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 		clk = calc_div(perflvl->core, perflvl->vdec, &P1);
 
 		/* see how close we can get using xpll/hclk as a source */
-		if (dev_priv->chipset != 0x98)
+		if (nv_device(drm->device)->chipset != 0x98)
 			out = read_pll(dev, 0x004030);
 		else
 			out = read_clk(dev, clk_src_hclkm3d2);
@@ -654,7 +674,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 		/* select whichever gets us closest */
 		if (abs((int)perflvl->vdec - clk) <=
 		    abs((int)perflvl->vdec - out)) {
-			if (dev_priv->chipset != 0x98)
+			if (nv_device(drm->device)->chipset != 0x98)
 				mast |= 0x00000c00;
 			divs |= P1 << 8;
 		} else {
@@ -682,7 +702,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	}
 
 	/* vdec/dom6: complete switch to new clocks */
-	switch (dev_priv->chipset) {
+	switch (nv_device(drm->device)->chipset) {
 	case 0x92:
 	case 0x94:
 	case 0x96:
@@ -698,7 +718,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	/* core/shader: make sure sclk/nvclk are disconnected from their
 	 * PLLs (nvclk to dom6, sclk to hclk)
 	 */
-	if (dev_priv->chipset < 0x92)
+	if (nv_device(drm->device)->chipset < 0x92)
 		mast = (mast & ~0x001000b0) | 0x00100080;
 	else
 		mast = (mast & ~0x000000b3) | 0x00000081;
@@ -710,7 +730,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	if (clk == 0)
 		goto error;
 
-	ctrl  = nv_rd32(dev, 0x004028) & ~0xc03f0100;
+	ctrl  = nv_rd32(device, 0x004028) & ~0xc03f0100;
 	mast &= ~0x00100000;
 	mast |= 3;
 
@@ -723,7 +743,7 @@ nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	 * cases will be handled by tying to nvclk, but it's possible there's
 	 * corners
 	 */
-	ctrl = nv_rd32(dev, 0x004020) & ~0xc03f0100;
+	ctrl = nv_rd32(device, 0x004020) & ~0xc03f0100;
 
 	if (P1-- && perflvl->shader == (perflvl->core << 1)) {
 		hwsq_wr32(hwsq, 0x004020, (P1 << 19) | (P1 << 16) | ctrl);
@@ -752,11 +772,12 @@ error:
 static int
 prog_hwsq(struct drm_device *dev, struct hwsq_ucode *hwsq)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u32 hwsq_data, hwsq_kick;
 	int i;
 
-	if (dev_priv->chipset < 0x94) {
+	if (nv_device(drm->device)->chipset < 0x94) {
 		hwsq_data = 0x001400;
 		hwsq_kick = 0x00000003;
 	} else {
@@ -764,22 +785,22 @@ prog_hwsq(struct drm_device *dev, struct hwsq_ucode *hwsq)
 		hwsq_kick = 0x00000001;
 	}
 	/* upload hwsq ucode */
-	nv_mask(dev, 0x001098, 0x00000008, 0x00000000);
-	nv_wr32(dev, 0x001304, 0x00000000);
-	if (dev_priv->chipset >= 0x92)
-		nv_wr32(dev, 0x001318, 0x00000000);
+	nv_mask(device, 0x001098, 0x00000008, 0x00000000);
+	nv_wr32(device, 0x001304, 0x00000000);
+	if (nv_device(drm->device)->chipset >= 0x92)
+		nv_wr32(device, 0x001318, 0x00000000);
 	for (i = 0; i < hwsq->len / 4; i++)
-		nv_wr32(dev, hwsq_data + (i * 4), hwsq->ptr.u32[i]);
-	nv_mask(dev, 0x001098, 0x00000018, 0x00000018);
+		nv_wr32(device, hwsq_data + (i * 4), hwsq->ptr.u32[i]);
+	nv_mask(device, 0x001098, 0x00000018, 0x00000018);
 
 	/* launch, and wait for completion */
-	nv_wr32(dev, 0x00130c, hwsq_kick);
-	if (!nv_wait(dev, 0x001308, 0x00000100, 0x00000000)) {
-		NV_ERROR(dev, "hwsq ucode exec timed out\n");
-		NV_ERROR(dev, "0x001308: 0x%08x\n", nv_rd32(dev, 0x001308));
+	nv_wr32(device, 0x00130c, hwsq_kick);
+	if (!nv_wait(device, 0x001308, 0x00000100, 0x00000000)) {
+		NV_ERROR(drm, "hwsq ucode exec timed out\n");
+		NV_ERROR(drm, "0x001308: 0x%08x\n", nv_rd32(device, 0x001308));
 		for (i = 0; i < hwsq->len / 4; i++) {
-			NV_ERROR(dev, "0x%06x: 0x%08x\n", 0x1400 + (i * 4),
-				 nv_rd32(dev, 0x001400 + (i * 4)));
+			NV_ERROR(drm, "0x%06x: 0x%08x\n", 0x1400 + (i * 4),
+				 nv_rd32(device, 0x001400 + (i * 4)));
 		}
 
 		return -EIO;
@@ -791,20 +812,22 @@ prog_hwsq(struct drm_device *dev, struct hwsq_ucode *hwsq)
 int
 nv50_pm_clocks_set(struct drm_device *dev, void *data)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nv50_pm_state *info = data;
 	struct bit_entry M;
 	int ret = -EBUSY;
 
 	/* halt and idle execution engines */
-	nv_mask(dev, 0x002504, 0x00000001, 0x00000001);
-	if (!nv_wait(dev, 0x002504, 0x00000010, 0x00000010))
+	nv_mask(device, 0x002504, 0x00000001, 0x00000001);
+	if (!nv_wait(device, 0x002504, 0x00000010, 0x00000010))
 		goto resume;
-	if (!nv_wait(dev, 0x00251c, 0x0000003f, 0x0000003f))
+	if (!nv_wait(device, 0x00251c, 0x0000003f, 0x0000003f))
 		goto resume;
 
 	/* program memory clock, if necessary - must come before engine clock
 	 * reprogramming due to how we construct the hwsq scripts in pre()
 	 */
+#define nouveau_bios_init_exec(a,b) nouveau_bios_run_init_table((a), (b), NULL, 0)
 	if (info->mclk_hwsq.len) {
 		/* execute some scripts that do ??? from the vbios.. */
 		if (!bit_table(dev, 'M', &M) && M.version == 1) {
@@ -826,61 +849,7 @@ nv50_pm_clocks_set(struct drm_device *dev, void *data)
 	ret = prog_hwsq(dev, &info->eclk_hwsq);
 
 resume:
-	nv_mask(dev, 0x002504, 0x00000001, 0x00000000);
+	nv_mask(device, 0x002504, 0x00000001, 0x00000000);
 	kfree(info);
 	return ret;
 }
-
-static int
-pwm_info(struct drm_device *dev, int *line, int *ctrl, int *indx)
-{
-	if (*line == 0x04) {
-		*ctrl = 0x00e100;
-		*line = 4;
-		*indx = 0;
-	} else
-	if (*line == 0x09) {
-		*ctrl = 0x00e100;
-		*line = 9;
-		*indx = 1;
-	} else
-	if (*line == 0x10) {
-		*ctrl = 0x00e28c;
-		*line = 0;
-		*indx = 0;
-	} else {
-		NV_ERROR(dev, "unknown pwm ctrl for gpio %d\n", *line);
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-int
-nv50_pm_pwm_get(struct drm_device *dev, int line, u32 *divs, u32 *duty)
-{
-	int ctrl, id, ret = pwm_info(dev, &line, &ctrl, &id);
-	if (ret)
-		return ret;
-
-	if (nv_rd32(dev, ctrl) & (1 << line)) {
-		*divs = nv_rd32(dev, 0x00e114 + (id * 8));
-		*duty = nv_rd32(dev, 0x00e118 + (id * 8));
-		return 0;
-	}
-
-	return -EINVAL;
-}
-
-int
-nv50_pm_pwm_set(struct drm_device *dev, int line, u32 divs, u32 duty)
-{
-	int ctrl, id, ret = pwm_info(dev, &line, &ctrl, &id);
-	if (ret)
-		return ret;
-
-	nv_mask(dev, ctrl, 0x00010001 << line, 0x00000001 << line);
-	nv_wr32(dev, 0x00e114 + (id * 8), divs);
-	nv_wr32(dev, 0x00e118 + (id * 8), duty | 0x80000000);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_software.c b/drivers/gpu/drm/nouveau/nv50_software.c
deleted file mode 100644
index df554d9dacb..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_software.c
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
-
-#include "nv50_display.h"
-
-struct nv50_software_priv {
-	struct nouveau_software_priv base;
-};
-
-struct nv50_software_chan {
-	struct nouveau_software_chan base;
-};
-
-static int
-mthd_dma_vblsem(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct nv50_software_chan *pch = chan->engctx[NVOBJ_ENGINE_SW];
-	struct nouveau_gpuobj *gpuobj;
-
-	gpuobj = nouveau_ramht_find(chan, data);
-	if (!gpuobj)
-		return -ENOENT;
-
-	pch->base.vblank.ctxdma = gpuobj->cinst >> 4;
-	return 0;
-}
-
-static int
-mthd_vblsem_offset(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct nv50_software_chan *pch = chan->engctx[NVOBJ_ENGINE_SW];
-	pch->base.vblank.offset = data;
-	return 0;
-}
-
-static int
-mthd_vblsem_value(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct nv50_software_chan *pch = chan->engctx[NVOBJ_ENGINE_SW];
-	pch->base.vblank.value = data;
-	return 0;
-}
-
-static int
-mthd_vblsem_release(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	struct nv50_software_priv *psw = nv_engine(chan->dev, NVOBJ_ENGINE_SW);
-	struct nv50_software_chan *pch = chan->engctx[NVOBJ_ENGINE_SW];
-	struct drm_device *dev = chan->dev;
-
-	if (data > 1)
-		return -EINVAL;
-
-	drm_vblank_get(dev, data);
-
-	pch->base.vblank.head = data;
-	list_add(&pch->base.vblank.list, &psw->base.vblank);
-	return 0;
-}
-
-static int
-mthd_flip(struct nouveau_channel *chan, u32 class, u32 mthd, u32 data)
-{
-	nouveau_finish_page_flip(chan, NULL);
-	return 0;
-}
-
-static int
-nv50_software_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv50_software_priv *psw = nv_engine(chan->dev, NVOBJ_ENGINE_SW);
-	struct nv50_display *pdisp = nv50_display(chan->dev);
-	struct nv50_software_chan *pch;
-	int ret = 0, i;
-
-	pch = kzalloc(sizeof(*pch), GFP_KERNEL);
-	if (!pch)
-		return -ENOMEM;
-
-	nouveau_software_context_new(&pch->base);
-	pch->base.vblank.channel = chan->ramin->vinst >> 12;
-	chan->engctx[engine] = pch;
-
-	/* dma objects for display sync channel semaphore blocks */
-	for (i = 0; i < chan->dev->mode_config.num_crtc; i++) {
-		struct nv50_display_crtc *dispc = &pdisp->crtc[i];
-		struct nouveau_gpuobj *obj = NULL;
-
-		ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
-					     dispc->sem.bo->bo.offset, 0x1000,
-					     NV_MEM_ACCESS_RW,
-					     NV_MEM_TARGET_VRAM, &obj);
-		if (ret)
-			break;
-
-		ret = nouveau_ramht_insert(chan, NvEvoSema0 + i, obj);
-		nouveau_gpuobj_ref(NULL, &obj);
-	}
-
-	if (ret)
-		psw->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nv50_software_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv50_software_chan *pch = chan->engctx[engine];
-	chan->engctx[engine] = NULL;
-	kfree(pch);
-}
-
-static int
-nv50_software_object_new(struct nouveau_channel *chan, int engine,
-			 u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, 0, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 0;
-	obj->class  = class;
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static int
-nv50_software_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
-static int
-nv50_software_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static void
-nv50_software_destroy(struct drm_device *dev, int engine)
-{
-	struct nv50_software_priv *psw = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, SW);
-	kfree(psw);
-}
-
-int
-nv50_software_create(struct drm_device *dev)
-{
-	struct nv50_software_priv *psw = kzalloc(sizeof(*psw), GFP_KERNEL);
-	if (!psw)
-		return -ENOMEM;
-
-	psw->base.base.destroy = nv50_software_destroy;
-	psw->base.base.init = nv50_software_init;
-	psw->base.base.fini = nv50_software_fini;
-	psw->base.base.context_new = nv50_software_context_new;
-	psw->base.base.context_del = nv50_software_context_del;
-	psw->base.base.object_new = nv50_software_object_new;
-	nouveau_software_create(&psw->base);
-
-	NVOBJ_ENGINE_ADD(dev, SW, &psw->base.base);
-	NVOBJ_CLASS(dev, 0x506e, SW);
-	NVOBJ_MTHD (dev, 0x506e, 0x018c, mthd_dma_vblsem);
-	NVOBJ_MTHD (dev, 0x506e, 0x0400, mthd_vblsem_offset);
-	NVOBJ_MTHD (dev, 0x506e, 0x0404, mthd_vblsem_value);
-	NVOBJ_MTHD (dev, 0x506e, 0x0408, mthd_vblsem_release);
-	NVOBJ_MTHD (dev, 0x506e, 0x0500, mthd_flip);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_sor.c b/drivers/gpu/drm/nouveau/nv50_sor.c
index 93240bde891..b562b59e132 100644
--- a/drivers/gpu/drm/nouveau/nv50_sor.c
+++ b/drivers/gpu/drm/nouveau/nv50_sor.c
@@ -24,40 +24,45 @@
  *
  */
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
 #define NOUVEAU_DMA_DEBUG (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
 #include "nouveau_reg.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
 #include "nouveau_crtc.h"
 #include "nv50_display.h"
 
+#include <subdev/timer.h>
+
 static u32
-nv50_sor_dp_lane_map(struct drm_device *dev, struct dcb_entry *dcb, u8 lane)
+nv50_sor_dp_lane_map(struct drm_device *dev, struct dcb_output *dcb, u8 lane)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	static const u8 nvaf[] = { 24, 16, 8, 0 }; /* thanks, apple.. */
 	static const u8 nv50[] = { 16, 8, 0, 24 };
-	if (dev_priv->chipset == 0xaf)
+	if (nv_device(drm->device)->chipset == 0xaf)
 		return nvaf[lane];
 	return nv50[lane];
 }
 
 static void
-nv50_sor_dp_train_set(struct drm_device *dev, struct dcb_entry *dcb, u8 pattern)
+nv50_sor_dp_train_set(struct drm_device *dev, struct dcb_output *dcb, u8 pattern)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
-	nv_mask(dev, NV50_SOR_DP_CTRL(or, link), 0x0f000000, pattern << 24);
+	nv_mask(device, NV50_SOR_DP_CTRL(or, link), 0x0f000000, pattern << 24);
 }
 
 static void
-nv50_sor_dp_train_adj(struct drm_device *dev, struct dcb_entry *dcb,
+nv50_sor_dp_train_adj(struct drm_device *dev, struct dcb_output *dcb,
 		      u8 lane, u8 swing, u8 preem)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
 	u32 shift = nv50_sor_dp_lane_map(dev, dcb, lane);
 	u32 mask = 0x000000ff << shift;
@@ -65,7 +70,7 @@ nv50_sor_dp_train_adj(struct drm_device *dev, struct dcb_entry *dcb,
 
 	table = nouveau_dp_bios_data(dev, dcb, &entry);
 	if (!table || (table[0] != 0x20 && table[0] != 0x21)) {
-		NV_ERROR(dev, "PDISP: unsupported DP table for chipset\n");
+		NV_ERROR(drm, "PDISP: unsupported DP table for chipset\n");
 		return;
 	}
 
@@ -76,24 +81,26 @@ nv50_sor_dp_train_adj(struct drm_device *dev, struct dcb_entry *dcb,
 			return;
 	}
 
-	nv_mask(dev, NV50_SOR_DP_UNK118(or, link), mask, config[2] << shift);
-	nv_mask(dev, NV50_SOR_DP_UNK120(or, link), mask, config[3] << shift);
-	nv_mask(dev, NV50_SOR_DP_UNK130(or, link), 0x0000ff00, config[4] << 8);
+	nv_mask(device, NV50_SOR_DP_UNK118(or, link), mask, config[2] << shift);
+	nv_mask(device, NV50_SOR_DP_UNK120(or, link), mask, config[3] << shift);
+	nv_mask(device, NV50_SOR_DP_UNK130(or, link), 0x0000ff00, config[4] << 8);
 }
 
 static void
-nv50_sor_dp_link_set(struct drm_device *dev, struct dcb_entry *dcb, int crtc,
+nv50_sor_dp_link_set(struct drm_device *dev, struct dcb_output *dcb, int crtc,
 		     int link_nr, u32 link_bw, bool enhframe)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
-	u32 dpctrl = nv_rd32(dev, NV50_SOR_DP_CTRL(or, link)) & ~0x001f4000;
-	u32 clksor = nv_rd32(dev, 0x614300 + (or * 0x800)) & ~0x000c0000;
+	u32 dpctrl = nv_rd32(device, NV50_SOR_DP_CTRL(or, link)) & ~0x001f4000;
+	u32 clksor = nv_rd32(device, 0x614300 + (or * 0x800)) & ~0x000c0000;
 	u8 *table, *entry, mask;
 	int i;
 
 	table = nouveau_dp_bios_data(dev, dcb, &entry);
 	if (!table || (table[0] != 0x20 && table[0] != 0x21)) {
-		NV_ERROR(dev, "PDISP: unsupported DP table for chipset\n");
+		NV_ERROR(drm, "PDISP: unsupported DP table for chipset\n");
 		return;
 	}
 
@@ -112,20 +119,21 @@ nv50_sor_dp_link_set(struct drm_device *dev, struct dcb_entry *dcb, int crtc,
 	if (link_bw > 162000)
 		clksor |= 0x00040000;
 
-	nv_wr32(dev, 0x614300 + (or * 0x800), clksor);
-	nv_wr32(dev, NV50_SOR_DP_CTRL(or, link), dpctrl);
+	nv_wr32(device, 0x614300 + (or * 0x800), clksor);
+	nv_wr32(device, NV50_SOR_DP_CTRL(or, link), dpctrl);
 
 	mask = 0;
 	for (i = 0; i < link_nr; i++)
 		mask |= 1 << (nv50_sor_dp_lane_map(dev, dcb, i) >> 3);
-	nv_mask(dev, NV50_SOR_DP_UNK130(or, link), 0x0000000f, mask);
+	nv_mask(device, NV50_SOR_DP_UNK130(or, link), 0x0000000f, mask);
 }
 
 static void
 nv50_sor_dp_link_get(struct drm_device *dev, u32 or, u32 link, u32 *nr, u32 *bw)
 {
-	u32 dpctrl = nv_rd32(dev, NV50_SOR_DP_CTRL(or, link)) & 0x000f0000;
-	u32 clksor = nv_rd32(dev, 0x614300 + (or * 0x800));
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 dpctrl = nv_rd32(device, NV50_SOR_DP_CTRL(or, link)) & 0x000f0000;
+	u32 clksor = nv_rd32(device, 0x614300 + (or * 0x800));
 	if (clksor & 0x000c0000)
 		*bw = 270000;
 	else
@@ -139,6 +147,8 @@ nv50_sor_dp_link_get(struct drm_device *dev, u32 or, u32 link, u32 *nr, u32 *bw)
 void
 nv50_sor_dp_calc_tu(struct drm_device *dev, int or, int link, u32 clk, u32 bpp)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	const u32 symbol = 100000;
 	int bestTU = 0, bestVTUi = 0, bestVTUf = 0, bestVTUa = 0;
 	int TU, VTUi, VTUf, VTUa;
@@ -206,7 +216,7 @@ nv50_sor_dp_calc_tu(struct drm_device *dev, int or, int link, u32 clk, u32 bpp)
 	}
 
 	if (!bestTU) {
-		NV_ERROR(dev, "DP: unable to find suitable config\n");
+		NV_ERROR(drm, "DP: unable to find suitable config\n");
 		return;
 	}
 
@@ -217,8 +227,8 @@ nv50_sor_dp_calc_tu(struct drm_device *dev, int or, int link, u32 clk, u32 bpp)
 	r = do_div(unk, symbol);
 	unk += 6;
 
-	nv_mask(dev, NV50_SOR_DP_CTRL(or, link), 0x000001fc, bestTU << 2);
-	nv_mask(dev, NV50_SOR_DP_SCFG(or, link), 0x010f7f3f, bestVTUa << 24 |
+	nv_mask(device, NV50_SOR_DP_CTRL(or, link), 0x000001fc, bestTU << 2);
+	nv_mask(device, NV50_SOR_DP_SCFG(or, link), 0x010f7f3f, bestVTUa << 24 |
 							     bestVTUf << 16 |
 							     bestVTUi << 8 |
 							     unk);
@@ -227,6 +237,7 @@ static void
 nv50_sor_disconnect(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct drm_device *dev = encoder->dev;
 	struct nouveau_channel *evo = nv50_display(dev)->master;
 	int ret;
@@ -235,11 +246,11 @@ nv50_sor_disconnect(struct drm_encoder *encoder)
 		return;
 	nv50_crtc_blank(nouveau_crtc(nv_encoder->crtc), true);
 
-	NV_DEBUG_KMS(dev, "Disconnecting SOR %d\n", nv_encoder->or);
+	NV_DEBUG(drm, "Disconnecting SOR %d\n", nv_encoder->or);
 
 	ret = RING_SPACE(evo, 4);
 	if (ret) {
-		NV_ERROR(dev, "no space while disconnecting SOR\n");
+		NV_ERROR(drm, "no space while disconnecting SOR\n");
 		return;
 	}
 	BEGIN_NV04(evo, 0, NV50_EVO_SOR(nv_encoder->or, MODE_CTRL), 1);
@@ -256,22 +267,24 @@ nv50_sor_disconnect(struct drm_encoder *encoder)
 static void
 nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 {
+	struct nouveau_device *device = nouveau_dev(encoder->dev);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct drm_device *dev = encoder->dev;
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_encoder *enc;
 	uint32_t val;
 	int or = nv_encoder->or;
 
-	NV_DEBUG_KMS(dev, "or %d type %d mode %d\n", or, nv_encoder->dcb->type, mode);
+	NV_DEBUG(drm, "or %d type %d mode %d\n", or, nv_encoder->dcb->type, mode);
 
 	nv_encoder->last_dpms = mode;
 	list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
 		struct nouveau_encoder *nvenc = nouveau_encoder(enc);
 
 		if (nvenc == nv_encoder ||
-		    (nvenc->dcb->type != OUTPUT_TMDS &&
-		     nvenc->dcb->type != OUTPUT_LVDS &&
-		     nvenc->dcb->type != OUTPUT_DP) ||
+		    (nvenc->dcb->type != DCB_OUTPUT_TMDS &&
+		     nvenc->dcb->type != DCB_OUTPUT_LVDS &&
+		     nvenc->dcb->type != DCB_OUTPUT_DP) ||
 		    nvenc->dcb->or != nv_encoder->dcb->or)
 			continue;
 
@@ -280,30 +293,30 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 	}
 
 	/* wait for it to be done */
-	if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or),
+	if (!nv_wait(device, NV50_PDISPLAY_SOR_DPMS_CTRL(or),
 		     NV50_PDISPLAY_SOR_DPMS_CTRL_PENDING, 0)) {
-		NV_ERROR(dev, "timeout: SOR_DPMS_CTRL_PENDING(%d) == 0\n", or);
-		NV_ERROR(dev, "SOR_DPMS_CTRL(%d) = 0x%08x\n", or,
-			 nv_rd32(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or)));
+		NV_ERROR(drm, "timeout: SOR_DPMS_CTRL_PENDING(%d) == 0\n", or);
+		NV_ERROR(drm, "SOR_DPMS_CTRL(%d) = 0x%08x\n", or,
+			 nv_rd32(device, NV50_PDISPLAY_SOR_DPMS_CTRL(or)));
 	}
 
-	val = nv_rd32(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or));
+	val = nv_rd32(device, NV50_PDISPLAY_SOR_DPMS_CTRL(or));
 
 	if (mode == DRM_MODE_DPMS_ON)
 		val |= NV50_PDISPLAY_SOR_DPMS_CTRL_ON;
 	else
 		val &= ~NV50_PDISPLAY_SOR_DPMS_CTRL_ON;
 
-	nv_wr32(dev, NV50_PDISPLAY_SOR_DPMS_CTRL(or), val |
+	nv_wr32(device, NV50_PDISPLAY_SOR_DPMS_CTRL(or), val |
 		NV50_PDISPLAY_SOR_DPMS_CTRL_PENDING);
-	if (!nv_wait(dev, NV50_PDISPLAY_SOR_DPMS_STATE(or),
+	if (!nv_wait(device, NV50_PDISPLAY_SOR_DPMS_STATE(or),
 		     NV50_PDISPLAY_SOR_DPMS_STATE_WAIT, 0)) {
-		NV_ERROR(dev, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", or);
-		NV_ERROR(dev, "SOR_DPMS_STATE(%d) = 0x%08x\n", or,
-			 nv_rd32(dev, NV50_PDISPLAY_SOR_DPMS_STATE(or)));
+		NV_ERROR(drm, "timeout: SOR_DPMS_STATE_WAIT(%d) == 0\n", or);
+		NV_ERROR(drm, "SOR_DPMS_STATE(%d) = 0x%08x\n", or,
+			 nv_rd32(device, NV50_PDISPLAY_SOR_DPMS_STATE(or)));
 	}
 
-	if (nv_encoder->dcb->type == OUTPUT_DP) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
 		struct dp_train_func func = {
 			.link_set = nv50_sor_dp_link_set,
 			.train_set = nv50_sor_dp_train_set,
@@ -317,13 +330,15 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode)
 static void
 nv50_sor_save(struct drm_encoder *encoder)
 {
-	NV_ERROR(encoder->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static void
 nv50_sor_restore(struct drm_encoder *encoder)
 {
-	NV_ERROR(encoder->dev, "!!\n");
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
+	NV_ERROR(drm, "!!\n");
 }
 
 static bool
@@ -331,14 +346,15 @@ nv50_sor_mode_fixup(struct drm_encoder *encoder,
 		    const struct drm_display_mode *mode,
 		    struct drm_display_mode *adjusted_mode)
 {
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_connector *connector;
 
-	NV_DEBUG_KMS(encoder->dev, "or %d\n", nv_encoder->or);
+	NV_DEBUG(drm, "or %d\n", nv_encoder->or);
 
 	connector = nouveau_encoder_connector_get(nv_encoder);
 	if (!connector) {
-		NV_ERROR(encoder->dev, "Encoder has no connector\n");
+		NV_ERROR(drm, "Encoder has no connector\n");
 		return false;
 	}
 
@@ -354,7 +370,7 @@ nv50_sor_prepare(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	nv50_sor_disconnect(encoder);
-	if (nv_encoder->dcb->type == OUTPUT_DP) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
 		/* avoid race between link training and supervisor intr */
 		nv50_display_sync(encoder->dev);
 	}
@@ -371,18 +387,18 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 {
 	struct nouveau_channel *evo = nv50_display(encoder->dev)->master;
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
-	struct drm_device *dev = encoder->dev;
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 	struct nouveau_crtc *crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_connector *nv_connector;
 	uint32_t mode_ctl = 0;
 	int ret;
 
-	NV_DEBUG_KMS(dev, "or %d type %d -> crtc %d\n",
+	NV_DEBUG(drm, "or %d type %d -> crtc %d\n",
 		     nv_encoder->or, nv_encoder->dcb->type, crtc->index);
 	nv_encoder->crtc = encoder->crtc;
 
 	switch (nv_encoder->dcb->type) {
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		if (nv_encoder->dcb->sorconf.link & 1) {
 			if (mode->clock < 165000)
 				mode_ctl = 0x0100;
@@ -393,7 +409,7 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 		nouveau_hdmi_mode_set(encoder, mode);
 		break;
-	case OUTPUT_DP:
+	case DCB_OUTPUT_DP:
 		nv_connector = nouveau_encoder_connector_get(nv_encoder);
 		if (nv_connector && nv_connector->base.display_info.bpc == 6) {
 			nv_encoder->dp.datarate = mode->clock * 18 / 8;
@@ -427,7 +443,7 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 	ret = RING_SPACE(evo, 2);
 	if (ret) {
-		NV_ERROR(dev, "no space while connecting SOR\n");
+		NV_ERROR(drm, "no space while connecting SOR\n");
 		nv_encoder->crtc = NULL;
 		return;
 	}
@@ -458,11 +474,9 @@ static void
 nv50_sor_destroy(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
+	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
 
-	if (!encoder)
-		return;
-
-	NV_DEBUG_KMS(encoder->dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	drm_encoder_cleanup(encoder);
 
@@ -474,21 +488,22 @@ static const struct drm_encoder_funcs nv50_sor_encoder_funcs = {
 };
 
 int
-nv50_sor_create(struct drm_connector *connector, struct dcb_entry *entry)
+nv50_sor_create(struct drm_connector *connector, struct dcb_output *entry)
 {
 	struct nouveau_encoder *nv_encoder = NULL;
 	struct drm_device *dev = connector->dev;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct drm_encoder *encoder;
 	int type;
 
-	NV_DEBUG_KMS(dev, "\n");
+	NV_DEBUG(drm, "\n");
 
 	switch (entry->type) {
-	case OUTPUT_TMDS:
-	case OUTPUT_DP:
+	case DCB_OUTPUT_TMDS:
+	case DCB_OUTPUT_DP:
 		type = DRM_MODE_ENCODER_TMDS;
 		break;
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		type = DRM_MODE_ENCODER_LVDS;
 		break;
 	default:
diff --git a/drivers/gpu/drm/nouveau/nv50_vram.c b/drivers/gpu/drm/nouveau/nv50_vram.c
deleted file mode 100644
index 9ed9ae397d7..00000000000
--- a/drivers/gpu/drm/nouveau/nv50_vram.c
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-
-static int types[0x80] = {
-	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 2, 2, 0, 0, 0, 0,
-	1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2, 2, 2, 2, 2, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 0, 0,
-	0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 2,
-	1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
-};
-
-bool
-nv50_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
-{
-	int type = (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) >> 8;
-
-	if (likely(type < ARRAY_SIZE(types) && types[type]))
-		return true;
-	return false;
-}
-
-void
-nv50_vram_del(struct drm_device *dev, struct nouveau_mem **pmem)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_mm *mm = &dev_priv->engine.vram.mm;
-	struct nouveau_mm_node *this;
-	struct nouveau_mem *mem;
-
-	mem = *pmem;
-	*pmem = NULL;
-	if (unlikely(mem == NULL))
-		return;
-
-	mutex_lock(&mm->mutex);
-	while (!list_empty(&mem->regions)) {
-		this = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
-
-		list_del(&this->rl_entry);
-		nouveau_mm_put(mm, this);
-	}
-
-	if (mem->tag) {
-		drm_mm_put_block(mem->tag);
-		mem->tag = NULL;
-	}
-	mutex_unlock(&mm->mutex);
-
-	kfree(mem);
-}
-
-int
-nv50_vram_new(struct drm_device *dev, u64 size, u32 align, u32 size_nc,
-	      u32 memtype, struct nouveau_mem **pmem)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_mm *mm = &dev_priv->engine.vram.mm;
-	struct nouveau_mm_node *r;
-	struct nouveau_mem *mem;
-	int comp = (memtype & 0x300) >> 8;
-	int type = (memtype & 0x07f);
-	int ret;
-
-	if (!types[type])
-		return -EINVAL;
-	size >>= 12;
-	align >>= 12;
-	size_nc >>= 12;
-
-	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
-	if (!mem)
-		return -ENOMEM;
-
-	mutex_lock(&mm->mutex);
-	if (comp) {
-		if (align == 16) {
-			struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-			int n = (size >> 4) * comp;
-
-			mem->tag = drm_mm_search_free(&pfb->tag_heap, n, 0, 0);
-			if (mem->tag)
-				mem->tag = drm_mm_get_block(mem->tag, n, 0);
-		}
-
-		if (unlikely(!mem->tag))
-			comp = 0;
-	}
-
-	INIT_LIST_HEAD(&mem->regions);
-	mem->dev = dev_priv->dev;
-	mem->memtype = (comp << 7) | type;
-	mem->size = size;
-
-	do {
-		ret = nouveau_mm_get(mm, types[type], size, size_nc, align, &r);
-		if (ret) {
-			mutex_unlock(&mm->mutex);
-			nv50_vram_del(dev, &mem);
-			return ret;
-		}
-
-		list_add_tail(&r->rl_entry, &mem->regions);
-		size -= r->length;
-	} while (size);
-	mutex_unlock(&mm->mutex);
-
-	r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
-	mem->offset = (u64)r->offset << 12;
-	*pmem = mem;
-	return 0;
-}
-
-static u32
-nv50_vram_rblock(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i, parts, colbits, rowbitsa, rowbitsb, banks;
-	u64 rowsize, predicted;
-	u32 r0, r4, rt, ru, rblock_size;
-
-	r0 = nv_rd32(dev, 0x100200);
-	r4 = nv_rd32(dev, 0x100204);
-	rt = nv_rd32(dev, 0x100250);
-	ru = nv_rd32(dev, 0x001540);
-	NV_DEBUG(dev, "memcfg 0x%08x 0x%08x 0x%08x 0x%08x\n", r0, r4, rt, ru);
-
-	for (i = 0, parts = 0; i < 8; i++) {
-		if (ru & (0x00010000 << i))
-			parts++;
-	}
-
-	colbits  =  (r4 & 0x0000f000) >> 12;
-	rowbitsa = ((r4 & 0x000f0000) >> 16) + 8;
-	rowbitsb = ((r4 & 0x00f00000) >> 20) + 8;
-	banks    = 1 << (((r4 & 0x03000000) >> 24) + 2);
-
-	rowsize = parts * banks * (1 << colbits) * 8;
-	predicted = rowsize << rowbitsa;
-	if (r0 & 0x00000004)
-		predicted += rowsize << rowbitsb;
-
-	if (predicted != dev_priv->vram_size) {
-		NV_WARN(dev, "memory controller reports %dMiB VRAM\n",
-			(u32)(dev_priv->vram_size >> 20));
-		NV_WARN(dev, "we calculated %dMiB VRAM\n",
-			(u32)(predicted >> 20));
-	}
-
-	rblock_size = rowsize;
-	if (rt & 1)
-		rblock_size *= 3;
-
-	NV_DEBUG(dev, "rblock %d bytes\n", rblock_size);
-	return rblock_size;
-}
-
-int
-nv50_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
-	const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
-	u32 pfb714 = nv_rd32(dev, 0x100714);
-	u32 rblock, length;
-
-	switch (pfb714 & 0x00000007) {
-	case 0: dev_priv->vram_type = NV_MEM_TYPE_DDR1; break;
-	case 1:
-		if (nouveau_mem_vbios_type(dev) == NV_MEM_TYPE_DDR3)
-			dev_priv->vram_type = NV_MEM_TYPE_DDR3;
-		else
-			dev_priv->vram_type = NV_MEM_TYPE_DDR2;
-		break;
-	case 2: dev_priv->vram_type = NV_MEM_TYPE_GDDR3; break;
-	case 3: dev_priv->vram_type = NV_MEM_TYPE_GDDR4; break;
-	case 4: dev_priv->vram_type = NV_MEM_TYPE_GDDR5; break;
-	default:
-		break;
-	}
-
-	dev_priv->vram_rank_B = !!(nv_rd32(dev, 0x100200) & 0x4);
-	dev_priv->vram_size  = nv_rd32(dev, 0x10020c);
-	dev_priv->vram_size |= (dev_priv->vram_size & 0xff) << 32;
-	dev_priv->vram_size &= 0xffffffff00ULL;
-
-	/* IGPs, no funky reordering happens here, they don't have VRAM */
-	if (dev_priv->chipset == 0xaa ||
-	    dev_priv->chipset == 0xac ||
-	    dev_priv->chipset == 0xaf) {
-		dev_priv->vram_sys_base = (u64)nv_rd32(dev, 0x100e10) << 12;
-		rblock = 4096 >> 12;
-	} else {
-		rblock = nv50_vram_rblock(dev) >> 12;
-	}
-
-	length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
-
-	return nouveau_mm_init(&vram->mm, rsvd_head, length, rblock);
-}
-
-void
-nv50_vram_fini(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-
-	nouveau_mm_fini(&vram->mm);
-}
diff --git a/drivers/gpu/drm/nouveau/nv84_bsp.c b/drivers/gpu/drm/nouveau/nv84_bsp.c
deleted file mode 100644
index 74875739bcc..00000000000
--- a/drivers/gpu/drm/nouveau/nv84_bsp.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-
-/*XXX: This stub is currently used on NV98+ also, as soon as this becomes
- *     more than just an enable/disable stub this needs to be split out to
- *     nv98_bsp.c...
- */
-
-struct nv84_bsp_engine {
-	struct nouveau_exec_engine base;
-};
-
-static int
-nv84_bsp_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	if (!(nv_rd32(dev, 0x000200) & 0x00008000))
-		return 0;
-
-	nv_mask(dev, 0x000200, 0x00008000, 0x00000000);
-	return 0;
-}
-
-static int
-nv84_bsp_init(struct drm_device *dev, int engine)
-{
-	nv_mask(dev, 0x000200, 0x00008000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00008000, 0x00008000);
-	return 0;
-}
-
-static void
-nv84_bsp_destroy(struct drm_device *dev, int engine)
-{
-	struct nv84_bsp_engine *pbsp = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, BSP);
-
-	kfree(pbsp);
-}
-
-int
-nv84_bsp_create(struct drm_device *dev)
-{
-	struct nv84_bsp_engine *pbsp;
-
-	pbsp = kzalloc(sizeof(*pbsp), GFP_KERNEL);
-	if (!pbsp)
-		return -ENOMEM;
-
-	pbsp->base.destroy = nv84_bsp_destroy;
-	pbsp->base.init = nv84_bsp_init;
-	pbsp->base.fini = nv84_bsp_fini;
-
-	NVOBJ_ENGINE_ADD(dev, BSP, &pbsp->base);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv84_crypt.c b/drivers/gpu/drm/nouveau/nv84_crypt.c
deleted file mode 100644
index bbfcc73b670..00000000000
--- a/drivers/gpu/drm/nouveau/nv84_crypt.c
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-
-struct nv84_crypt_engine {
-	struct nouveau_exec_engine base;
-};
-
-static int
-nv84_crypt_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramin = chan->ramin;
-	struct nouveau_gpuobj *ctx;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &ctx);
-	if (ret)
-		return ret;
-
-	nv_wo32(ramin, 0xa0, 0x00190000);
-	nv_wo32(ramin, 0xa4, ctx->vinst + ctx->size - 1);
-	nv_wo32(ramin, 0xa8, ctx->vinst);
-	nv_wo32(ramin, 0xac, 0);
-	nv_wo32(ramin, 0xb0, 0);
-	nv_wo32(ramin, 0xb4, 0);
-	dev_priv->engine.instmem.flush(dev);
-
-	atomic_inc(&chan->vm->engref[engine]);
-	chan->engctx[engine] = ctx;
-	return 0;
-}
-
-static void
-nv84_crypt_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	u32 inst;
-
-	inst  = (chan->ramin->vinst >> 12);
-	inst |= 0x80000000;
-
-	/* mark context as invalid if still on the hardware, not
-	 * doing this causes issues the next time PCRYPT is used,
-	 * unsurprisingly :)
-	 */
-	nv_wr32(dev, 0x10200c, 0x00000000);
-	if (nv_rd32(dev, 0x102188) == inst)
-		nv_mask(dev, 0x102188, 0x80000000, 0x00000000);
-	if (nv_rd32(dev, 0x10218c) == inst)
-		nv_mask(dev, 0x10218c, 0x80000000, 0x00000000);
-	nv_wr32(dev, 0x10200c, 0x00000010);
-
-	nouveau_gpuobj_ref(NULL, &ctx);
-
-	atomic_dec(&chan->vm->engref[engine]);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nv84_crypt_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *obj = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 16, 16, NVOBJ_FLAG_ZERO_FREE, &obj);
-	if (ret)
-		return ret;
-	obj->engine = 5;
-	obj->class  = class;
-
-	nv_wo32(obj, 0x00, class);
-	dev_priv->engine.instmem.flush(dev);
-
-	ret = nouveau_ramht_insert(chan, handle, obj);
-	nouveau_gpuobj_ref(NULL, &obj);
-	return ret;
-}
-
-static void
-nv84_crypt_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 0x0a);
-}
-
-static struct nouveau_bitfield nv84_crypt_intr[] = {
-	{ 0x00000001, "INVALID_STATE" },
-	{ 0x00000002, "ILLEGAL_MTHD" },
-	{ 0x00000004, "ILLEGAL_CLASS" },
-	{ 0x00000080, "QUERY" },
-	{ 0x00000100, "FAULT" },
-	{}
-};
-
-static void
-nv84_crypt_isr(struct drm_device *dev)
-{
-	u32 stat = nv_rd32(dev, 0x102130);
-	u32 mthd = nv_rd32(dev, 0x102190);
-	u32 data = nv_rd32(dev, 0x102194);
-	u64 inst = (u64)(nv_rd32(dev, 0x102188) & 0x7fffffff) << 12;
-	int show = nouveau_ratelimit();
-	int chid = nv50_graph_isr_chid(dev, inst);
-
-	if (show) {
-		NV_INFO(dev, "PCRYPT:");
-		nouveau_bitfield_print(nv84_crypt_intr, stat);
-		printk(KERN_CONT " ch %d (0x%010llx) mthd 0x%04x data 0x%08x\n",
-			chid, inst, mthd, data);
-	}
-
-	nv_wr32(dev, 0x102130, stat);
-	nv_wr32(dev, 0x10200c, 0x10);
-
-	nv50_fb_vm_trap(dev, show);
-}
-
-static int
-nv84_crypt_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_wr32(dev, 0x102140, 0x00000000);
-	return 0;
-}
-
-static int
-nv84_crypt_init(struct drm_device *dev, int engine)
-{
-	nv_mask(dev, 0x000200, 0x00004000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00004000, 0x00004000);
-
-	nv_wr32(dev, 0x102130, 0xffffffff);
-	nv_wr32(dev, 0x102140, 0xffffffbf);
-
-	nv_wr32(dev, 0x10200c, 0x00000010);
-	return 0;
-}
-
-static void
-nv84_crypt_destroy(struct drm_device *dev, int engine)
-{
-	struct nv84_crypt_engine *pcrypt = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, CRYPT);
-
-	nouveau_irq_unregister(dev, 14);
-	kfree(pcrypt);
-}
-
-int
-nv84_crypt_create(struct drm_device *dev)
-{
-	struct nv84_crypt_engine *pcrypt;
-
-	pcrypt = kzalloc(sizeof(*pcrypt), GFP_KERNEL);
-	if (!pcrypt)
-		return -ENOMEM;
-
-	pcrypt->base.destroy = nv84_crypt_destroy;
-	pcrypt->base.init = nv84_crypt_init;
-	pcrypt->base.fini = nv84_crypt_fini;
-	pcrypt->base.context_new = nv84_crypt_context_new;
-	pcrypt->base.context_del = nv84_crypt_context_del;
-	pcrypt->base.object_new = nv84_crypt_object_new;
-	pcrypt->base.tlb_flush = nv84_crypt_tlb_flush;
-
-	nouveau_irq_register(dev, 14, nv84_crypt_isr);
-
-	NVOBJ_ENGINE_ADD(dev, CRYPT, &pcrypt->base);
-	NVOBJ_CLASS (dev, 0x74c1, CRYPT);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c
index c2f889b0d34..c686650584b 100644
--- a/drivers/gpu/drm/nouveau/nv84_fence.c
+++ b/drivers/gpu/drm/nouveau/nv84_fence.c
@@ -22,13 +22,17 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <core/object.h>
+#include <core/class.h>
+
+#include <engine/fifo.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fence.h"
 
+#include "nv50_display.h"
+
 struct nv84_fence_chan {
 	struct nouveau_fence_chan base;
 };
@@ -42,13 +46,14 @@ static int
 nv84_fence_emit(struct nouveau_fence *fence)
 {
 	struct nouveau_channel *chan = fence->channel;
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
 	int ret = RING_SPACE(chan, 7);
 	if (ret == 0) {
 		BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
 		OUT_RING  (chan, NvSema);
 		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(chan->id * 16));
-		OUT_RING  (chan, lower_32_bits(chan->id * 16));
+		OUT_RING  (chan, upper_32_bits(fifo->chid * 16));
+		OUT_RING  (chan, lower_32_bits(fifo->chid * 16));
 		OUT_RING  (chan, fence->sequence);
 		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG);
 		FIRE_RING (chan);
@@ -61,13 +66,14 @@ static int
 nv84_fence_sync(struct nouveau_fence *fence,
 		struct nouveau_channel *prev, struct nouveau_channel *chan)
 {
+	struct nouveau_fifo_chan *fifo = (void *)prev->object;
 	int ret = RING_SPACE(chan, 7);
 	if (ret == 0) {
 		BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
 		OUT_RING  (chan, NvSema);
 		BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
-		OUT_RING  (chan, upper_32_bits(prev->id * 16));
-		OUT_RING  (chan, lower_32_bits(prev->id * 16));
+		OUT_RING  (chan, upper_32_bits(fifo->chid * 16));
+		OUT_RING  (chan, lower_32_bits(fifo->chid * 16));
 		OUT_RING  (chan, fence->sequence);
 		OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_GEQUAL);
 		FIRE_RING (chan);
@@ -78,100 +84,99 @@ nv84_fence_sync(struct nouveau_fence *fence,
 static u32
 nv84_fence_read(struct nouveau_channel *chan)
 {
-	struct nv84_fence_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_FENCE);
-	return nv_ro32(priv->mem, chan->id * 16);
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	struct nv84_fence_priv *priv = chan->drm->fence;
+	return nv_ro32(priv->mem, fifo->chid * 16);
 }
 
 static void
-nv84_fence_context_del(struct nouveau_channel *chan, int engine)
+nv84_fence_context_del(struct nouveau_channel *chan)
 {
-	struct nv84_fence_chan *fctx = chan->engctx[engine];
+	struct nv84_fence_chan *fctx = chan->fence;
 	nouveau_fence_context_del(&fctx->base);
-	chan->engctx[engine] = NULL;
+	chan->fence = NULL;
 	kfree(fctx);
 }
 
 static int
-nv84_fence_context_new(struct nouveau_channel *chan, int engine)
+nv84_fence_context_new(struct nouveau_channel *chan)
 {
-	struct nv84_fence_priv *priv = nv_engine(chan->dev, engine);
+	struct drm_device *dev = chan->drm->dev;
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	struct nv84_fence_priv *priv = chan->drm->fence;
 	struct nv84_fence_chan *fctx;
-	struct nouveau_gpuobj *obj;
-	int ret;
+	struct nouveau_object *object;
+	int ret, i;
 
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (!fctx)
 		return -ENOMEM;
 
 	nouveau_fence_context_new(&fctx->base);
 
-	ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
-				     priv->mem->vinst, priv->mem->size,
-				     NV_MEM_ACCESS_RW,
-				     NV_MEM_TARGET_VRAM, &obj);
-	if (ret == 0) {
-		ret = nouveau_ramht_insert(chan, NvSema, obj);
-		nouveau_gpuobj_ref(NULL, &obj);
-		nv_wo32(priv->mem, chan->id * 16, 0x00000000);
+	ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
+				 NvSema, 0x0002,
+				 &(struct nv_dma_class) {
+					.flags = NV_DMA_TARGET_VRAM |
+						 NV_DMA_ACCESS_RDWR,
+					.start = priv->mem->addr,
+					.limit = priv->mem->addr +
+						 priv->mem->size - 1,
+				 }, sizeof(struct nv_dma_class),
+				 &object);
+
+	/* dma objects for display sync channel semaphore blocks */
+	for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) {
+		struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i);
+
+		ret = nouveau_object_new(nv_object(chan->cli), chan->handle,
+					 NvEvoSema0 + i, 0x003d,
+					 &(struct nv_dma_class) {
+						.flags = NV_DMA_TARGET_VRAM |
+							 NV_DMA_ACCESS_RDWR,
+						.start = bo->bo.offset,
+						.limit = bo->bo.offset + 0xfff,
+					 }, sizeof(struct nv_dma_class),
+					 &object);
 	}
 
 	if (ret)
-		nv84_fence_context_del(chan, engine);
+		nv84_fence_context_del(chan);
+	nv_wo32(priv->mem, fifo->chid * 16, 0x00000000);
 	return ret;
 }
 
-static int
-nv84_fence_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static int
-nv84_fence_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
 static void
-nv84_fence_destroy(struct drm_device *dev, int engine)
+nv84_fence_destroy(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv84_fence_priv *priv = nv_engine(dev, engine);
-
+	struct nv84_fence_priv *priv = drm->fence;
 	nouveau_gpuobj_ref(NULL, &priv->mem);
-	dev_priv->eng[engine] = NULL;
+	drm->fence = NULL;
 	kfree(priv);
 }
 
 int
-nv84_fence_create(struct drm_device *dev)
+nv84_fence_create(struct nouveau_drm *drm)
 {
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
 	struct nv84_fence_priv *priv;
+	u32 chan = pfifo->max + 1;
 	int ret;
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	priv = drm->fence = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->base.engine.destroy = nv84_fence_destroy;
-	priv->base.engine.init = nv84_fence_init;
-	priv->base.engine.fini = nv84_fence_fini;
-	priv->base.engine.context_new = nv84_fence_context_new;
-	priv->base.engine.context_del = nv84_fence_context_del;
+	priv->base.dtor = nv84_fence_destroy;
+	priv->base.context_new = nv84_fence_context_new;
+	priv->base.context_del = nv84_fence_context_del;
 	priv->base.emit = nv84_fence_emit;
 	priv->base.sync = nv84_fence_sync;
 	priv->base.read = nv84_fence_read;
-	dev_priv->eng[NVOBJ_ENGINE_FENCE] = &priv->base.engine;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 16 * pfifo->channels,
-				 0x1000, 0, &priv->mem);
-	if (ret)
-		goto out;
 
-out:
+	ret = nouveau_gpuobj_new(drm->device, NULL, chan * 16, 0x1000, 0,
+				&priv->mem);
 	if (ret)
-		nv84_fence_destroy(dev, NVOBJ_ENGINE_FENCE);
+		nv84_fence_destroy(drm);
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nv84_fifo.c b/drivers/gpu/drm/nouveau/nv84_fifo.c
deleted file mode 100644
index c564c5e4c30..00000000000
--- a/drivers/gpu/drm/nouveau/nv84_fifo.c
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Copyright (C) 2012 Ben Skeggs.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial
- * portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
- * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
- * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
-#include "nouveau_vm.h"
-
-struct nv84_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct nouveau_gpuobj *playlist[2];
-	int cur_playlist;
-};
-
-struct nv84_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *ramfc;
-	struct nouveau_gpuobj *cache;
-};
-
-static int
-nv84_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nv84_fifo_priv *priv = nv_engine(chan->dev, engine);
-	struct nv84_fifo_chan *fctx;
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-        u64 ib_offset = chan->pushbuf_base + chan->dma.ib_base * 4;
-	u64 instance;
-	unsigned long flags;
-	int ret;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-	atomic_inc(&chan->vm->engref[engine]);
-
-	chan->user = ioremap(pci_resource_start(dev->pdev, 0) +
-			     NV50_USER(chan->id), PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	ret = nouveau_gpuobj_new(dev, chan, 256, 256, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &fctx->ramfc);
-	if (ret)
-		goto error;
-
-	instance = fctx->ramfc->vinst >> 8;
-
-	ret = nouveau_gpuobj_new(dev, chan, 4096, 1024, 0, &fctx->cache);
-	if (ret)
-		goto error;
-
-	nv_wo32(fctx->ramfc, 0x3c, 0x403f6078);
-	nv_wo32(fctx->ramfc, 0x40, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x44, 0x01003fff);
-	nv_wo32(fctx->ramfc, 0x48, chan->pushbuf->cinst >> 4);
-	nv_wo32(fctx->ramfc, 0x50, lower_32_bits(ib_offset));
-	nv_wo32(fctx->ramfc, 0x54, upper_32_bits(ib_offset) |
-				   drm_order(chan->dma.ib_max + 1) << 16);
-	nv_wo32(fctx->ramfc, 0x60, 0x7fffffff);
-	nv_wo32(fctx->ramfc, 0x78, 0x00000000);
-	nv_wo32(fctx->ramfc, 0x7c, 0x30000001);
-	nv_wo32(fctx->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) |
-				   (4 << 24) /* SEARCH_FULL */ |
-				   (chan->ramht->gpuobj->cinst >> 4));
-	nv_wo32(fctx->ramfc, 0x88, fctx->cache->vinst >> 10);
-	nv_wo32(fctx->ramfc, 0x98, chan->ramin->vinst >> 12);
-
-	nv_wo32(chan->ramin, 0x00, chan->id);
-	nv_wo32(chan->ramin, 0x04, fctx->ramfc->vinst >> 8);
-
-	dev_priv->engine.instmem.flush(dev);
-
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_wr32(dev, 0x002600 + (chan->id * 4), 0x80000000 | instance);
-	nv50_fifo_playlist_update(dev);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nv84_fifo_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv84_fifo_chan *fctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	unsigned long flags;
-	u32 save;
-
-	/* remove channel from playlist, will context switch if active */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_mask(dev, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000);
-	nv50_fifo_playlist_update(dev);
-
-	save = nv_mask(dev, 0x002520, 0x0000003f, 0x15);
-
-	/* tell any engines on this channel to unload their contexts */
-	nv_wr32(dev, 0x0032fc, chan->ramin->vinst >> 12);
-	if (!nv_wait_ne(dev, 0x0032fc, 0xffffffff, 0xffffffff))
-		NV_INFO(dev, "PFIFO: channel %d unload timeout\n", chan->id);
-
-	nv_wr32(dev, 0x002520, save);
-
-	nv_wr32(dev, 0x002600 + (chan->id * 4), 0x00000000);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
-
-	/* clean up */
-	if (chan->user) {
-		iounmap(chan->user);
-		chan->user = NULL;
-	}
-
-	nouveau_gpuobj_ref(NULL, &fctx->ramfc);
-	nouveau_gpuobj_ref(NULL, &fctx->cache);
-
-	atomic_dec(&chan->vm->engref[engine]);
-	chan->engctx[engine] = NULL;
-	kfree(fctx);
-}
-
-static int
-nv84_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv84_fifo_chan *fctx;
-	u32 instance;
-	int i;
-
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
-	nv_wr32(dev, 0x00250c, 0x6f3cfc34);
-	nv_wr32(dev, 0x002044, 0x01003fff);
-
-	nv_wr32(dev, 0x002100, 0xffffffff);
-	nv_wr32(dev, 0x002140, 0xffffffff);
-
-	for (i = 0; i < 128; i++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
-		if (chan && (fctx = chan->engctx[engine]))
-			instance = 0x80000000 | fctx->ramfc->vinst >> 8;
-		else
-			instance = 0x00000000;
-		nv_wr32(dev, 0x002600 + (i * 4), instance);
-	}
-
-	nv50_fifo_playlist_update(dev);
-
-	nv_wr32(dev, 0x003200, 1);
-	nv_wr32(dev, 0x003250, 1);
-	nv_wr32(dev, 0x002500, 1);
-	return 0;
-}
-
-static int
-nv84_fifo_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv84_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-	u32 save;
-
-	/* set playlist length to zero, fifo will unload context */
-	nv_wr32(dev, 0x0032ec, 0);
-
-	save = nv_mask(dev, 0x002520, 0x0000003f, 0x15);
-
-	/* tell all connected engines to unload their contexts */
-	for (i = 0; i < priv->base.channels; i++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
-		if (chan)
-			nv_wr32(dev, 0x0032fc, chan->ramin->vinst >> 12);
-		if (!nv_wait_ne(dev, 0x0032fc, 0xffffffff, 0xffffffff)) {
-			NV_INFO(dev, "PFIFO: channel %d unload timeout\n", i);
-			return -EBUSY;
-		}
-	}
-
-	nv_wr32(dev, 0x002520, save);
-	nv_wr32(dev, 0x002140, 0);
-	return 0;
-}
-
-int
-nv84_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv84_fifo_priv *priv;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nv50_fifo_destroy;
-	priv->base.base.init = nv84_fifo_init;
-	priv->base.base.fini = nv84_fifo_fini;
-	priv->base.base.context_new = nv84_fifo_context_new;
-	priv->base.base.context_del = nv84_fifo_context_del;
-	priv->base.base.tlb_flush = nv50_fifo_tlb_flush;
-	priv->base.channels = 127;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	ret = nouveau_gpuobj_new(dev, NULL, priv->base.channels * 4, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->playlist[0]);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, priv->base.channels * 4, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->playlist[1]);
-	if (ret)
-		goto error;
-
-	nouveau_irq_register(dev, 8, nv04_fifo_isr);
-error:
-	if (ret)
-		priv->base.base.destroy(dev, NVOBJ_ENGINE_FIFO);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nv84_vp.c b/drivers/gpu/drm/nouveau/nv84_vp.c
deleted file mode 100644
index 6570d300ab8..00000000000
--- a/drivers/gpu/drm/nouveau/nv84_vp.c
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-
-/*XXX: This stub is currently used on NV98+ also, as soon as this becomes
- *     more than just an enable/disable stub this needs to be split out to
- *     nv98_vp.c...
- */
-
-struct nv84_vp_engine {
-	struct nouveau_exec_engine base;
-};
-
-static int
-nv84_vp_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	if (!(nv_rd32(dev, 0x000200) & 0x00020000))
-		return 0;
-
-	nv_mask(dev, 0x000200, 0x00020000, 0x00000000);
-	return 0;
-}
-
-static int
-nv84_vp_init(struct drm_device *dev, int engine)
-{
-	nv_mask(dev, 0x000200, 0x00020000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00020000, 0x00020000);
-	return 0;
-}
-
-static void
-nv84_vp_destroy(struct drm_device *dev, int engine)
-{
-	struct nv84_vp_engine *pvp = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, VP);
-
-	kfree(pvp);
-}
-
-int
-nv84_vp_create(struct drm_device *dev)
-{
-	struct nv84_vp_engine *pvp;
-
-	pvp = kzalloc(sizeof(*pvp), GFP_KERNEL);
-	if (!pvp)
-		return -ENOMEM;
-
-	pvp->base.destroy = nv84_vp_destroy;
-	pvp->base.init = nv84_vp_init;
-	pvp->base.fini = nv84_vp_fini;
-
-	NVOBJ_ENGINE_ADD(dev, VP, &pvp->base);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv98_crypt.c b/drivers/gpu/drm/nouveau/nv98_crypt.c
deleted file mode 100644
index e25e13fb894..00000000000
--- a/drivers/gpu/drm/nouveau/nv98_crypt.c
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-
-#include "nv98_crypt.fuc.h"
-
-struct nv98_crypt_priv {
-	struct nouveau_exec_engine base;
-};
-
-struct nv98_crypt_chan {
-	struct nouveau_gpuobj *mem;
-};
-
-static int
-nv98_crypt_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv98_crypt_priv *priv = nv_engine(dev, engine);
-	struct nv98_crypt_chan *cctx;
-	int ret;
-
-	cctx = chan->engctx[engine] = kzalloc(sizeof(*cctx), GFP_KERNEL);
-	if (!cctx)
-		return -ENOMEM;
-
-	atomic_inc(&chan->vm->engref[engine]);
-
-	ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &cctx->mem);
-	if (ret)
-		goto error;
-
-	nv_wo32(chan->ramin, 0xa0, 0x00190000);
-	nv_wo32(chan->ramin, 0xa4, cctx->mem->vinst + cctx->mem->size - 1);
-	nv_wo32(chan->ramin, 0xa8, cctx->mem->vinst);
-	nv_wo32(chan->ramin, 0xac, 0x00000000);
-	nv_wo32(chan->ramin, 0xb0, 0x00000000);
-	nv_wo32(chan->ramin, 0xb4, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
-
-error:
-	if (ret)
-		priv->base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nv98_crypt_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nv98_crypt_chan *cctx = chan->engctx[engine];
-	int i;
-
-	for (i = 0xa0; i < 0xb4; i += 4)
-		nv_wo32(chan->ramin, i, 0x00000000);
-
-	nouveau_gpuobj_ref(NULL, &cctx->mem);
-
-	atomic_dec(&chan->vm->engref[engine]);
-	chan->engctx[engine] = NULL;
-	kfree(cctx);
-}
-
-static int
-nv98_crypt_object_new(struct nouveau_channel *chan, int engine,
-		     u32 handle, u16 class)
-{
-	struct nv98_crypt_chan *cctx = chan->engctx[engine];
-
-	/* fuc engine doesn't need an object, our ramht code does.. */
-	cctx->mem->engine = 5;
-	cctx->mem->class  = class;
-	return nouveau_ramht_insert(chan, handle, cctx->mem);
-}
-
-static void
-nv98_crypt_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 0x0a);
-}
-
-static int
-nv98_crypt_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, 0x000200, 0x00004000, 0x00000000);
-	return 0;
-}
-
-static int
-nv98_crypt_init(struct drm_device *dev, int engine)
-{
-	int i;
-
-	/* reset! */
-	nv_mask(dev, 0x000200, 0x00004000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00004000, 0x00004000);
-
-	/* wait for exit interrupt to signal */
-	nv_wait(dev, 0x087008, 0x00000010, 0x00000010);
-	nv_wr32(dev, 0x087004, 0x00000010);
-
-	/* upload microcode code and data segments */
-	nv_wr32(dev, 0x087ff8, 0x00100000);
-	for (i = 0; i < ARRAY_SIZE(nv98_pcrypt_code); i++)
-		nv_wr32(dev, 0x087ff4, nv98_pcrypt_code[i]);
-
-	nv_wr32(dev, 0x087ff8, 0x00000000);
-	for (i = 0; i < ARRAY_SIZE(nv98_pcrypt_data); i++)
-		nv_wr32(dev, 0x087ff4, nv98_pcrypt_data[i]);
-
-	/* start it running */
-	nv_wr32(dev, 0x08710c, 0x00000000);
-	nv_wr32(dev, 0x087104, 0x00000000); /* ENTRY */
-	nv_wr32(dev, 0x087100, 0x00000002); /* TRIGGER */
-	return 0;
-}
-
-static struct nouveau_enum nv98_crypt_isr_error_name[] = {
-	{ 0x0000, "ILLEGAL_MTHD" },
-	{ 0x0001, "INVALID_BITFIELD" },
-	{ 0x0002, "INVALID_ENUM" },
-	{ 0x0003, "QUERY" },
-	{}
-};
-
-static void
-nv98_crypt_isr(struct drm_device *dev)
-{
-	u32 disp = nv_rd32(dev, 0x08701c);
-	u32 stat = nv_rd32(dev, 0x087008) & disp & ~(disp >> 16);
-	u32 inst = nv_rd32(dev, 0x087050) & 0x3fffffff;
-	u32 ssta = nv_rd32(dev, 0x087040) & 0x0000ffff;
-	u32 addr = nv_rd32(dev, 0x087040) >> 16;
-	u32 mthd = (addr & 0x07ff) << 2;
-	u32 subc = (addr & 0x3800) >> 11;
-	u32 data = nv_rd32(dev, 0x087044);
-	int chid = nv50_graph_isr_chid(dev, inst);
-
-	if (stat & 0x00000040) {
-		NV_INFO(dev, "PCRYPT: DISPATCH_ERROR [");
-		nouveau_enum_print(nv98_crypt_isr_error_name, ssta);
-		printk("] ch %d [0x%08x] subc %d mthd 0x%04x data 0x%08x\n",
-			chid, inst, subc, mthd, data);
-		nv_wr32(dev, 0x087004, 0x00000040);
-		stat &= ~0x00000040;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PCRYPT: unhandled intr 0x%08x\n", stat);
-		nv_wr32(dev, 0x087004, stat);
-	}
-
-	nv50_fb_vm_trap(dev, 1);
-}
-
-static void
-nv98_crypt_destroy(struct drm_device *dev, int engine)
-{
-	struct nv98_crypt_priv *priv = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 14);
-	NVOBJ_ENGINE_DEL(dev, CRYPT);
-	kfree(priv);
-}
-
-int
-nv98_crypt_create(struct drm_device *dev)
-{
-	struct nv98_crypt_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.destroy = nv98_crypt_destroy;
-	priv->base.init = nv98_crypt_init;
-	priv->base.fini = nv98_crypt_fini;
-	priv->base.context_new = nv98_crypt_context_new;
-	priv->base.context_del = nv98_crypt_context_del;
-	priv->base.object_new = nv98_crypt_object_new;
-	priv->base.tlb_flush = nv98_crypt_tlb_flush;
-
-	nouveau_irq_register(dev, 14, nv98_crypt_isr);
-
-	NVOBJ_ENGINE_ADD(dev, CRYPT, &priv->base);
-	NVOBJ_CLASS(dev, 0x88b4, CRYPT);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nva3_copy.c b/drivers/gpu/drm/nouveau/nva3_copy.c
deleted file mode 100644
index 0387dc7f4f4..00000000000
--- a/drivers/gpu/drm/nouveau/nva3_copy.c
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-#include "nva3_copy.fuc.h"
-
-struct nva3_copy_engine {
-	struct nouveau_exec_engine base;
-};
-
-static int
-nva3_copy_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramin = chan->ramin;
-	struct nouveau_gpuobj *ctx = NULL;
-	int ret;
-
-	NV_DEBUG(dev, "ch%d\n", chan->id);
-
-	ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
-				 NVOBJ_FLAG_ZERO_FREE, &ctx);
-	if (ret)
-		return ret;
-
-	nv_wo32(ramin, 0xc0, 0x00190000);
-	nv_wo32(ramin, 0xc4, ctx->vinst + ctx->size - 1);
-	nv_wo32(ramin, 0xc8, ctx->vinst);
-	nv_wo32(ramin, 0xcc, 0x00000000);
-	nv_wo32(ramin, 0xd0, 0x00000000);
-	nv_wo32(ramin, 0xd4, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
-
-	atomic_inc(&chan->vm->engref[engine]);
-	chan->engctx[engine] = ctx;
-	return 0;
-}
-
-static int
-nva3_copy_object_new(struct nouveau_channel *chan, int engine,
-		     u32 handle, u16 class)
-{
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-
-	/* fuc engine doesn't need an object, our ramht code does.. */
-	ctx->engine = 3;
-	ctx->class  = class;
-	return nouveau_ramht_insert(chan, handle, ctx);
-}
-
-static void
-nva3_copy_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-	int i;
-
-	for (i = 0xc0; i <= 0xd4; i += 4)
-		nv_wo32(chan->ramin, i, 0x00000000);
-
-	atomic_dec(&chan->vm->engref[engine]);
-	nouveau_gpuobj_ref(NULL, &ctx);
-	chan->engctx[engine] = ctx;
-}
-
-static void
-nva3_copy_tlb_flush(struct drm_device *dev, int engine)
-{
-	nv50_vm_flush_engine(dev, 0x0d);
-}
-
-static int
-nva3_copy_init(struct drm_device *dev, int engine)
-{
-	int i;
-
-	nv_mask(dev, 0x000200, 0x00002000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00002000, 0x00002000);
-	nv_wr32(dev, 0x104014, 0xffffffff); /* disable all interrupts */
-
-	/* upload ucode */
-	nv_wr32(dev, 0x1041c0, 0x01000000);
-	for (i = 0; i < sizeof(nva3_pcopy_data) / 4; i++)
-		nv_wr32(dev, 0x1041c4, nva3_pcopy_data[i]);
-
-	nv_wr32(dev, 0x104180, 0x01000000);
-	for (i = 0; i < sizeof(nva3_pcopy_code) / 4; i++) {
-		if ((i & 0x3f) == 0)
-			nv_wr32(dev, 0x104188, i >> 6);
-		nv_wr32(dev, 0x104184, nva3_pcopy_code[i]);
-	}
-
-	/* start it running */
-	nv_wr32(dev, 0x10410c, 0x00000000);
-	nv_wr32(dev, 0x104104, 0x00000000); /* ENTRY */
-	nv_wr32(dev, 0x104100, 0x00000002); /* TRIGGER */
-	return 0;
-}
-
-static int
-nva3_copy_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	nv_mask(dev, 0x104048, 0x00000003, 0x00000000);
-	nv_wr32(dev, 0x104014, 0xffffffff);
-	return 0;
-}
-
-static struct nouveau_enum nva3_copy_isr_error_name[] = {
-	{ 0x0001, "ILLEGAL_MTHD" },
-	{ 0x0002, "INVALID_ENUM" },
-	{ 0x0003, "INVALID_BITFIELD" },
-	{}
-};
-
-static void
-nva3_copy_isr(struct drm_device *dev)
-{
-	u32 dispatch = nv_rd32(dev, 0x10401c);
-	u32 stat = nv_rd32(dev, 0x104008) & dispatch & ~(dispatch >> 16);
-	u32 inst = nv_rd32(dev, 0x104050) & 0x3fffffff;
-	u32 ssta = nv_rd32(dev, 0x104040) & 0x0000ffff;
-	u32 addr = nv_rd32(dev, 0x104040) >> 16;
-	u32 mthd = (addr & 0x07ff) << 2;
-	u32 subc = (addr & 0x3800) >> 11;
-	u32 data = nv_rd32(dev, 0x104044);
-	int chid = nv50_graph_isr_chid(dev, inst);
-
-	if (stat & 0x00000040) {
-		NV_INFO(dev, "PCOPY: DISPATCH_ERROR [");
-		nouveau_enum_print(nva3_copy_isr_error_name, ssta);
-		printk("] ch %d [0x%08x] subc %d mthd 0x%04x data 0x%08x\n",
-			chid, inst, subc, mthd, data);
-		nv_wr32(dev, 0x104004, 0x00000040);
-		stat &= ~0x00000040;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PCOPY: unhandled intr 0x%08x\n", stat);
-		nv_wr32(dev, 0x104004, stat);
-	}
-	nv50_fb_vm_trap(dev, 1);
-}
-
-static void
-nva3_copy_destroy(struct drm_device *dev, int engine)
-{
-	struct nva3_copy_engine *pcopy = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, 22);
-
-	NVOBJ_ENGINE_DEL(dev, COPY0);
-	kfree(pcopy);
-}
-
-int
-nva3_copy_create(struct drm_device *dev)
-{
-	struct nva3_copy_engine *pcopy;
-
-	pcopy = kzalloc(sizeof(*pcopy), GFP_KERNEL);
-	if (!pcopy)
-		return -ENOMEM;
-
-	pcopy->base.destroy = nva3_copy_destroy;
-	pcopy->base.init = nva3_copy_init;
-	pcopy->base.fini = nva3_copy_fini;
-	pcopy->base.context_new = nva3_copy_context_new;
-	pcopy->base.context_del = nva3_copy_context_del;
-	pcopy->base.object_new = nva3_copy_object_new;
-	pcopy->base.tlb_flush = nva3_copy_tlb_flush;
-
-	nouveau_irq_register(dev, 22, nva3_copy_isr);
-
-	NVOBJ_ENGINE_ADD(dev, COPY0, &pcopy->base);
-	NVOBJ_CLASS(dev, 0x85b5, COPY0);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nva3_pm.c b/drivers/gpu/drm/nouveau/nva3_pm.c
index 798829353fb..863f010fafe 100644
--- a/drivers/gpu/drm/nouveau/nva3_pm.c
+++ b/drivers/gpu/drm/nouveau/nva3_pm.c
@@ -22,18 +22,25 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <drm/drmP.h>
+#include "nouveau_drm.h"
 #include "nouveau_bios.h"
 #include "nouveau_pm.h"
 
+#include <subdev/bios/pll.h>
+#include <subdev/bios.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
 static u32 read_clk(struct drm_device *, int, bool);
 static u32 read_pll(struct drm_device *, int, u32);
 
 static u32
 read_vco(struct drm_device *dev, int clk)
 {
-	u32 sctl = nv_rd32(dev, 0x4120 + (clk * 4));
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 sctl = nv_rd32(device, 0x4120 + (clk * 4));
 	if ((sctl & 0x00000030) != 0x00000030)
 		return read_pll(dev, 0x41, 0x00e820);
 	return read_pll(dev, 0x42, 0x00e8a0);
@@ -42,26 +49,27 @@ read_vco(struct drm_device *dev, int clk)
 static u32
 read_clk(struct drm_device *dev, int clk, bool ignore_en)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	u32 sctl, sdiv, sclk;
 
 	/* refclk for the 0xe8xx plls is a fixed frequency */
 	if (clk >= 0x40) {
-		if (dev_priv->chipset == 0xaf) {
+		if (nv_device(drm->device)->chipset == 0xaf) {
 			/* no joke.. seriously.. sigh.. */
-			return nv_rd32(dev, 0x00471c) * 1000;
+			return nv_rd32(device, 0x00471c) * 1000;
 		}
 
-		return dev_priv->crystal;
+		return device->crystal;
 	}
 
-	sctl = nv_rd32(dev, 0x4120 + (clk * 4));
+	sctl = nv_rd32(device, 0x4120 + (clk * 4));
 	if (!ignore_en && !(sctl & 0x00000100))
 		return 0;
 
 	switch (sctl & 0x00003000) {
 	case 0x00000000:
-		return dev_priv->crystal;
+		return device->crystal;
 	case 0x00002000:
 		if (sctl & 0x00000040)
 			return 108000;
@@ -78,12 +86,13 @@ read_clk(struct drm_device *dev, int clk, bool ignore_en)
 static u32
 read_pll(struct drm_device *dev, int clk, u32 pll)
 {
-	u32 ctrl = nv_rd32(dev, pll + 0);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ctrl = nv_rd32(device, pll + 0);
 	u32 sclk = 0, P = 1, N = 1, M = 1;
 
 	if (!(ctrl & 0x00000008)) {
 		if (ctrl & 0x00000001) {
-			u32 coef = nv_rd32(dev, pll + 4);
+			u32 coef = nv_rd32(device, pll + 4);
 			M = (coef & 0x000000ff) >> 0;
 			N = (coef & 0x0000ff00) >> 8;
 			P = (coef & 0x003f0000) >> 16;
@@ -111,7 +120,10 @@ struct creg {
 static int
 calc_clk(struct drm_device *dev, int clk, u32 pll, u32 khz, struct creg *reg)
 {
-	struct pll_lims limits;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nvbios_pll limits;
 	u32 oclk, sclk, sdiv;
 	int P, N, M, diff;
 	int ret;
@@ -119,7 +131,7 @@ calc_clk(struct drm_device *dev, int clk, u32 pll, u32 khz, struct creg *reg)
 	reg->pll = 0;
 	reg->clk = 0;
 	if (!khz) {
-		NV_DEBUG(dev, "no clock for 0x%04x/0x%02x\n", pll, clk);
+		NV_DEBUG(drm, "no clock for 0x%04x/0x%02x\n", pll, clk);
 		return 0;
 	}
 
@@ -154,14 +166,14 @@ calc_clk(struct drm_device *dev, int clk, u32 pll, u32 khz, struct creg *reg)
 		}
 
 		if (!pll) {
-			NV_ERROR(dev, "bad freq %02x: %d %d\n", clk, khz, sclk);
+			NV_ERROR(drm, "bad freq %02x: %d %d\n", clk, khz, sclk);
 			return -ERANGE;
 		}
 
 		break;
 	}
 
-	ret = get_pll_limits(dev, pll, &limits);
+	ret = nvbios_pll_parse(bios, pll, &limits);
 	if (ret)
 		return ret;
 
@@ -171,54 +183,60 @@ calc_clk(struct drm_device *dev, int clk, u32 pll, u32 khz, struct creg *reg)
 
 	ret = nva3_calc_pll(dev, &limits, khz, &N, NULL, &M, &P);
 	if (ret >= 0) {
-		reg->clk = nv_rd32(dev, 0x4120 + (clk * 4));
+		reg->clk = nv_rd32(device, 0x4120 + (clk * 4));
 		reg->pll = (P << 16) | (N << 8) | M;
 	}
+
 	return ret;
 }
 
 static void
 prog_pll(struct drm_device *dev, int clk, u32 pll, struct creg *reg)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	const u32 src0 = 0x004120 + (clk * 4);
 	const u32 src1 = 0x004160 + (clk * 4);
 	const u32 ctrl = pll + 0;
 	const u32 coef = pll + 4;
 
 	if (!reg->clk && !reg->pll) {
-		NV_DEBUG(dev, "no clock for %02x\n", clk);
+		NV_DEBUG(drm, "no clock for %02x\n", clk);
 		return;
 	}
 
 	if (reg->pll) {
-		nv_mask(dev, src0, 0x00000101, 0x00000101);
-		nv_wr32(dev, coef, reg->pll);
-		nv_mask(dev, ctrl, 0x00000015, 0x00000015);
-		nv_mask(dev, ctrl, 0x00000010, 0x00000000);
-		nv_wait(dev, ctrl, 0x00020000, 0x00020000);
-		nv_mask(dev, ctrl, 0x00000010, 0x00000010);
-		nv_mask(dev, ctrl, 0x00000008, 0x00000000);
-		nv_mask(dev, src1, 0x00000100, 0x00000000);
-		nv_mask(dev, src1, 0x00000001, 0x00000000);
+		nv_mask(device, src0, 0x00000101, 0x00000101);
+		nv_wr32(device, coef, reg->pll);
+		nv_mask(device, ctrl, 0x00000015, 0x00000015);
+		nv_mask(device, ctrl, 0x00000010, 0x00000000);
+		nv_wait(device, ctrl, 0x00020000, 0x00020000);
+		nv_mask(device, ctrl, 0x00000010, 0x00000010);
+		nv_mask(device, ctrl, 0x00000008, 0x00000000);
+		nv_mask(device, src1, 0x00000100, 0x00000000);
+		nv_mask(device, src1, 0x00000001, 0x00000000);
 	} else {
-		nv_mask(dev, src1, 0x003f3141, 0x00000101 | reg->clk);
-		nv_mask(dev, ctrl, 0x00000018, 0x00000018);
+		nv_mask(device, src1, 0x003f3141, 0x00000101 | reg->clk);
+		nv_mask(device, ctrl, 0x00000018, 0x00000018);
 		udelay(20);
-		nv_mask(dev, ctrl, 0x00000001, 0x00000000);
-		nv_mask(dev, src0, 0x00000100, 0x00000000);
-		nv_mask(dev, src0, 0x00000001, 0x00000000);
+		nv_mask(device, ctrl, 0x00000001, 0x00000000);
+		nv_mask(device, src0, 0x00000100, 0x00000000);
+		nv_mask(device, src0, 0x00000001, 0x00000000);
 	}
 }
 
 static void
 prog_clk(struct drm_device *dev, int clk, struct creg *reg)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
 	if (!reg->clk) {
-		NV_DEBUG(dev, "no clock for %02x\n", clk);
+		NV_DEBUG(drm, "no clock for %02x\n", clk);
 		return;
 	}
 
-	nv_mask(dev, 0x004120 + (clk * 4), 0x003f3141, 0x00000101 | reg->clk);
+	nv_mask(device, 0x004120 + (clk * 4), 0x003f3141, 0x00000101 | reg->clk);
 }
 
 int
@@ -309,10 +327,11 @@ static bool
 nva3_pm_grcp_idle(void *data)
 {
 	struct drm_device *dev = data;
+	struct nouveau_device *device = nouveau_dev(dev);
 
-	if (!(nv_rd32(dev, 0x400304) & 0x00000001))
+	if (!(nv_rd32(device, 0x400304) & 0x00000001))
 		return true;
-	if (nv_rd32(dev, 0x400308) == 0x0050001c)
+	if (nv_rd32(device, 0x400308) == 0x0050001c)
 		return true;
 	return false;
 }
@@ -320,85 +339,91 @@ nva3_pm_grcp_idle(void *data)
 static void
 mclk_precharge(struct nouveau_mem_exec_func *exec)
 {
-	nv_wr32(exec->dev, 0x1002d4, 0x00000001);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	nv_wr32(device, 0x1002d4, 0x00000001);
 }
 
 static void
 mclk_refresh(struct nouveau_mem_exec_func *exec)
 {
-	nv_wr32(exec->dev, 0x1002d0, 0x00000001);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	nv_wr32(device, 0x1002d0, 0x00000001);
 }
 
 static void
 mclk_refresh_auto(struct nouveau_mem_exec_func *exec, bool enable)
 {
-	nv_wr32(exec->dev, 0x100210, enable ? 0x80000000 : 0x00000000);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	nv_wr32(device, 0x100210, enable ? 0x80000000 : 0x00000000);
 }
 
 static void
 mclk_refresh_self(struct nouveau_mem_exec_func *exec, bool enable)
 {
-	nv_wr32(exec->dev, 0x1002dc, enable ? 0x00000001 : 0x00000000);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	nv_wr32(device, 0x1002dc, enable ? 0x00000001 : 0x00000000);
 }
 
 static void
 mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
 {
-	volatile u32 post = nv_rd32(exec->dev, 0); (void)post;
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	volatile u32 post = nv_rd32(device, 0); (void)post;
 	udelay((nsec + 500) / 1000);
 }
 
 static u32
 mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
 {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	if (mr <= 1)
-		return nv_rd32(exec->dev, 0x1002c0 + ((mr - 0) * 4));
+		return nv_rd32(device, 0x1002c0 + ((mr - 0) * 4));
 	if (mr <= 3)
-		return nv_rd32(exec->dev, 0x1002e0 + ((mr - 2) * 4));
+		return nv_rd32(device, 0x1002e0 + ((mr - 2) * 4));
 	return 0;
 }
 
 static void
 mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
 {
-	struct drm_nouveau_private *dev_priv = exec->dev->dev_private;
-
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
 	if (mr <= 1) {
-		if (dev_priv->vram_rank_B)
-			nv_wr32(exec->dev, 0x1002c8 + ((mr - 0) * 4), data);
-		nv_wr32(exec->dev, 0x1002c0 + ((mr - 0) * 4), data);
+		if (pfb->ram.ranks > 1)
+			nv_wr32(device, 0x1002c8 + ((mr - 0) * 4), data);
+		nv_wr32(device, 0x1002c0 + ((mr - 0) * 4), data);
 	} else
 	if (mr <= 3) {
-		if (dev_priv->vram_rank_B)
-			nv_wr32(exec->dev, 0x1002e8 + ((mr - 2) * 4), data);
-		nv_wr32(exec->dev, 0x1002e0 + ((mr - 2) * 4), data);
+		if (pfb->ram.ranks > 1)
+			nv_wr32(device, 0x1002e8 + ((mr - 2) * 4), data);
+		nv_wr32(device, 0x1002e0 + ((mr - 2) * 4), data);
 	}
 }
 
 static void
 mclk_clock_set(struct nouveau_mem_exec_func *exec)
 {
-	struct drm_device *dev = exec->dev;
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nva3_pm_state *info = exec->priv;
 	u32 ctrl;
 
-	ctrl = nv_rd32(dev, 0x004000);
+	ctrl = nv_rd32(device, 0x004000);
 	if (!(ctrl & 0x00000008) && info->mclk.pll) {
-		nv_wr32(dev, 0x004000, (ctrl |=  0x00000008));
-		nv_mask(dev, 0x1110e0, 0x00088000, 0x00088000);
-		nv_wr32(dev, 0x004018, 0x00001000);
-		nv_wr32(dev, 0x004000, (ctrl &= ~0x00000001));
-		nv_wr32(dev, 0x004004, info->mclk.pll);
-		nv_wr32(dev, 0x004000, (ctrl |=  0x00000001));
+		nv_wr32(device, 0x004000, (ctrl |=  0x00000008));
+		nv_mask(device, 0x1110e0, 0x00088000, 0x00088000);
+		nv_wr32(device, 0x004018, 0x00001000);
+		nv_wr32(device, 0x004000, (ctrl &= ~0x00000001));
+		nv_wr32(device, 0x004004, info->mclk.pll);
+		nv_wr32(device, 0x004000, (ctrl |=  0x00000001));
 		udelay(64);
-		nv_wr32(dev, 0x004018, 0x00005000 | info->r004018);
+		nv_wr32(device, 0x004018, 0x00005000 | info->r004018);
 		udelay(20);
 	} else
 	if (!info->mclk.pll) {
-		nv_mask(dev, 0x004168, 0x003f3040, info->mclk.clk);
-		nv_wr32(dev, 0x004000, (ctrl |= 0x00000008));
-		nv_mask(dev, 0x1110e0, 0x00088000, 0x00088000);
-		nv_wr32(dev, 0x004018, 0x0000d000 | info->r004018);
+		nv_mask(device, 0x004168, 0x003f3040, info->mclk.clk);
+		nv_wr32(device, 0x004000, (ctrl |= 0x00000008));
+		nv_mask(device, 0x1110e0, 0x00088000, 0x00088000);
+		nv_wr32(device, 0x004018, 0x0000d000 | info->r004018);
 	}
 
 	if (info->rammap) {
@@ -410,67 +435,68 @@ mclk_clock_set(struct nouveau_mem_exec_func *exec)
 				     (info->ramcfg[3] & 0x0f) << 16 |
 				     (info->ramcfg[9] & 0x0f) |
 				     0x80000000;
-			nv_wr32(dev, 0x1005a0, unk5a0);
-			nv_wr32(dev, 0x1005a4, unk5a4);
-			nv_wr32(dev, 0x10f804, unk804);
-			nv_mask(dev, 0x10053c, 0x00001000, 0x00000000);
+			nv_wr32(device, 0x1005a0, unk5a0);
+			nv_wr32(device, 0x1005a4, unk5a4);
+			nv_wr32(device, 0x10f804, unk804);
+			nv_mask(device, 0x10053c, 0x00001000, 0x00000000);
 		} else {
-			nv_mask(dev, 0x10053c, 0x00001000, 0x00001000);
-			nv_mask(dev, 0x10f804, 0x80000000, 0x00000000);
-			nv_mask(dev, 0x100760, 0x22222222, info->r100760);
-			nv_mask(dev, 0x1007a0, 0x22222222, info->r100760);
-			nv_mask(dev, 0x1007e0, 0x22222222, info->r100760);
+			nv_mask(device, 0x10053c, 0x00001000, 0x00001000);
+			nv_mask(device, 0x10f804, 0x80000000, 0x00000000);
+			nv_mask(device, 0x100760, 0x22222222, info->r100760);
+			nv_mask(device, 0x1007a0, 0x22222222, info->r100760);
+			nv_mask(device, 0x1007e0, 0x22222222, info->r100760);
 		}
 	}
 
 	if (info->mclk.pll) {
-		nv_mask(dev, 0x1110e0, 0x00088000, 0x00011000);
-		nv_wr32(dev, 0x004000, (ctrl &= ~0x00000008));
+		nv_mask(device, 0x1110e0, 0x00088000, 0x00011000);
+		nv_wr32(device, 0x004000, (ctrl &= ~0x00000008));
 	}
 }
 
 static void
 mclk_timing_set(struct nouveau_mem_exec_func *exec)
 {
-	struct drm_device *dev = exec->dev;
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nva3_pm_state *info = exec->priv;
 	struct nouveau_pm_level *perflvl = info->perflvl;
 	int i;
 
 	for (i = 0; i < 9; i++)
-		nv_wr32(dev, 0x100220 + (i * 4), perflvl->timing.reg[i]);
+		nv_wr32(device, 0x100220 + (i * 4), perflvl->timing.reg[i]);
 
 	if (info->ramcfg) {
 		u32 data = (info->ramcfg[2] & 0x08) ? 0x00000000 : 0x00001000;
-		nv_mask(dev, 0x100200, 0x00001000, data);
+		nv_mask(device, 0x100200, 0x00001000, data);
 	}
 
 	if (info->ramcfg) {
-		u32 unk714 = nv_rd32(dev, 0x100714) & ~0xf0000010;
-		u32 unk718 = nv_rd32(dev, 0x100718) & ~0x00000100;
-		u32 unk71c = nv_rd32(dev, 0x10071c) & ~0x00000100;
+		u32 unk714 = nv_rd32(device, 0x100714) & ~0xf0000010;
+		u32 unk718 = nv_rd32(device, 0x100718) & ~0x00000100;
+		u32 unk71c = nv_rd32(device, 0x10071c) & ~0x00000100;
 		if ( (info->ramcfg[2] & 0x20))
 			unk714 |= 0xf0000000;
 		if (!(info->ramcfg[2] & 0x04))
 			unk714 |= 0x00000010;
-		nv_wr32(dev, 0x100714, unk714);
+		nv_wr32(device, 0x100714, unk714);
 
 		if (info->ramcfg[2] & 0x01)
 			unk71c |= 0x00000100;
-		nv_wr32(dev, 0x10071c, unk71c);
+		nv_wr32(device, 0x10071c, unk71c);
 
 		if (info->ramcfg[2] & 0x02)
 			unk718 |= 0x00000100;
-		nv_wr32(dev, 0x100718, unk718);
+		nv_wr32(device, 0x100718, unk718);
 
 		if (info->ramcfg[2] & 0x10)
-			nv_wr32(dev, 0x111100, 0x48000000); /*XXX*/
+			nv_wr32(device, 0x111100, 0x48000000); /*XXX*/
 	}
 }
 
 static void
 prog_mem(struct drm_device *dev, struct nva3_pm_state *info)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nouveau_mem_exec_func exec = {
 		.dev = dev,
 		.precharge = mclk_precharge,
@@ -492,17 +518,17 @@ prog_mem(struct drm_device *dev, struct nva3_pm_state *info)
 		info->r100760 = 0x22222222;
 	}
 
-	ctrl = nv_rd32(dev, 0x004000);
+	ctrl = nv_rd32(device, 0x004000);
 	if (ctrl & 0x00000008) {
 		if (info->mclk.pll) {
-			nv_mask(dev, 0x004128, 0x00000101, 0x00000101);
-			nv_wr32(dev, 0x004004, info->mclk.pll);
-			nv_wr32(dev, 0x004000, (ctrl |= 0x00000001));
-			nv_wr32(dev, 0x004000, (ctrl &= 0xffffffef));
-			nv_wait(dev, 0x004000, 0x00020000, 0x00020000);
-			nv_wr32(dev, 0x004000, (ctrl |= 0x00000010));
-			nv_wr32(dev, 0x004018, 0x00005000 | info->r004018);
-			nv_wr32(dev, 0x004000, (ctrl |= 0x00000004));
+			nv_mask(device, 0x004128, 0x00000101, 0x00000101);
+			nv_wr32(device, 0x004004, info->mclk.pll);
+			nv_wr32(device, 0x004000, (ctrl |= 0x00000001));
+			nv_wr32(device, 0x004000, (ctrl &= 0xffffffef));
+			nv_wait(device, 0x004000, 0x00020000, 0x00020000);
+			nv_wr32(device, 0x004000, (ctrl |= 0x00000010));
+			nv_wr32(device, 0x004018, 0x00005000 | info->r004018);
+			nv_wr32(device, 0x004000, (ctrl |= 0x00000004));
 		}
 	} else {
 		u32 ssel = 0x00000101;
@@ -510,68 +536,67 @@ prog_mem(struct drm_device *dev, struct nva3_pm_state *info)
 			ssel |= info->mclk.clk;
 		else
 			ssel |= 0x00080000; /* 324MHz, shouldn't matter... */
-		nv_mask(dev, 0x004168, 0x003f3141, ctrl);
+		nv_mask(device, 0x004168, 0x003f3141, ctrl);
 	}
 
 	if (info->ramcfg) {
 		if (info->ramcfg[2] & 0x10) {
-			nv_mask(dev, 0x111104, 0x00000600, 0x00000000);
+			nv_mask(device, 0x111104, 0x00000600, 0x00000000);
 		} else {
-			nv_mask(dev, 0x111100, 0x40000000, 0x40000000);
-			nv_mask(dev, 0x111104, 0x00000180, 0x00000000);
+			nv_mask(device, 0x111100, 0x40000000, 0x40000000);
+			nv_mask(device, 0x111104, 0x00000180, 0x00000000);
 		}
 	}
 	if (info->rammap && !(info->rammap[4] & 0x02))
-		nv_mask(dev, 0x100200, 0x00000800, 0x00000000);
-	nv_wr32(dev, 0x611200, 0x00003300);
+		nv_mask(device, 0x100200, 0x00000800, 0x00000000);
+	nv_wr32(device, 0x611200, 0x00003300);
 	if (!(info->ramcfg[2] & 0x10))
-		nv_wr32(dev, 0x111100, 0x4c020000); /*XXX*/
+		nv_wr32(device, 0x111100, 0x4c020000); /*XXX*/
 
 	nouveau_mem_exec(&exec, info->perflvl);
 
-	nv_wr32(dev, 0x611200, 0x00003330);
+	nv_wr32(device, 0x611200, 0x00003330);
 	if (info->rammap && (info->rammap[4] & 0x02))
-		nv_mask(dev, 0x100200, 0x00000800, 0x00000800);
+		nv_mask(device, 0x100200, 0x00000800, 0x00000800);
 	if (info->ramcfg) {
 		if (info->ramcfg[2] & 0x10) {
-			nv_mask(dev, 0x111104, 0x00000180, 0x00000180);
-			nv_mask(dev, 0x111100, 0x40000000, 0x00000000);
+			nv_mask(device, 0x111104, 0x00000180, 0x00000180);
+			nv_mask(device, 0x111100, 0x40000000, 0x00000000);
 		} else {
-			nv_mask(dev, 0x111104, 0x00000600, 0x00000600);
+			nv_mask(device, 0x111104, 0x00000600, 0x00000600);
 		}
 	}
 
 	if (info->mclk.pll) {
-		nv_mask(dev, 0x004168, 0x00000001, 0x00000000);
-		nv_mask(dev, 0x004168, 0x00000100, 0x00000000);
+		nv_mask(device, 0x004168, 0x00000001, 0x00000000);
+		nv_mask(device, 0x004168, 0x00000100, 0x00000000);
 	} else {
-		nv_mask(dev, 0x004000, 0x00000001, 0x00000000);
-		nv_mask(dev, 0x004128, 0x00000001, 0x00000000);
-		nv_mask(dev, 0x004128, 0x00000100, 0x00000000);
+		nv_mask(device, 0x004000, 0x00000001, 0x00000000);
+		nv_mask(device, 0x004128, 0x00000001, 0x00000000);
+		nv_mask(device, 0x004128, 0x00000100, 0x00000000);
 	}
 }
 
 int
 nva3_pm_clocks_set(struct drm_device *dev, void *pre_state)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nva3_pm_state *info = pre_state;
-	unsigned long flags;
 	int ret = -EAGAIN;
 
 	/* prevent any new grctx switches from starting */
-	spin_lock_irqsave(&dev_priv->context_switch_lock, flags);
-	nv_wr32(dev, 0x400324, 0x00000000);
-	nv_wr32(dev, 0x400328, 0x0050001c); /* wait flag 0x1c */
+	nv_wr32(device, 0x400324, 0x00000000);
+	nv_wr32(device, 0x400328, 0x0050001c); /* wait flag 0x1c */
 	/* wait for any pending grctx switches to complete */
-	if (!nv_wait_cb(dev, nva3_pm_grcp_idle, dev)) {
-		NV_ERROR(dev, "pm: ctxprog didn't go idle\n");
+	if (!nv_wait_cb(device, nva3_pm_grcp_idle, dev)) {
+		NV_ERROR(drm, "pm: ctxprog didn't go idle\n");
 		goto cleanup;
 	}
 	/* freeze PFIFO */
-	nv_mask(dev, 0x002504, 0x00000001, 0x00000001);
-	if (!nv_wait(dev, 0x002504, 0x00000010, 0x00000010)) {
-		NV_ERROR(dev, "pm: fifo didn't go idle\n");
+	nv_mask(device, 0x002504, 0x00000001, 0x00000001);
+	if (!nv_wait(device, 0x002504, 0x00000010, 0x00000010)) {
+		NV_ERROR(drm, "pm: fifo didn't go idle\n");
 		goto cleanup;
 	}
 
@@ -587,14 +612,13 @@ nva3_pm_clocks_set(struct drm_device *dev, void *pre_state)
 
 cleanup:
 	/* unfreeze PFIFO */
-	nv_mask(dev, 0x002504, 0x00000001, 0x00000000);
+	nv_mask(device, 0x002504, 0x00000001, 0x00000000);
 	/* restore ctxprog to normal */
-	nv_wr32(dev, 0x400324, 0x00000000);
-	nv_wr32(dev, 0x400328, 0x0070009c); /* set flag 0x1c */
+	nv_wr32(device, 0x400324, 0x00000000);
+	nv_wr32(device, 0x400328, 0x0070009c); /* set flag 0x1c */
 	/* unblock it if necessary */
-	if (nv_rd32(dev, 0x400308) == 0x0050001c)
-		nv_mask(dev, 0x400824, 0x10000000, 0x10000000);
-	spin_unlock_irqrestore(&dev_priv->context_switch_lock, flags);
+	if (nv_rd32(device, 0x400308) == 0x0050001c)
+		nv_mask(device, 0x400824, 0x10000000, 0x10000000);
 	kfree(info);
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nvc0_copy.c b/drivers/gpu/drm/nouveau/nvc0_copy.c
deleted file mode 100644
index dddf006f6d8..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_copy.c
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/firmware.h>
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_util.h"
-#include "nouveau_vm.h"
-#include "nouveau_ramht.h"
-#include "nvc0_copy.fuc.h"
-
-struct nvc0_copy_engine {
-	struct nouveau_exec_engine base;
-	u32 irq;
-	u32 pmc;
-	u32 fuc;
-	u32 ctx;
-};
-
-static int
-nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(chan->dev, engine);
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_gpuobj *ramin = chan->ramin;
-	struct nouveau_gpuobj *ctx = NULL;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 256, 256,
-				 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER |
-				 NVOBJ_FLAG_ZERO_ALLOC, &ctx);
-	if (ret)
-		return ret;
-
-	nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(ctx->linst));
-	nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(ctx->linst));
-	dev_priv->engine.instmem.flush(dev);
-
-	chan->engctx[engine] = ctx;
-	return 0;
-}
-
-static int
-nvc0_copy_object_new(struct nouveau_channel *chan, int engine,
-		     u32 handle, u16 class)
-{
-	return 0;
-}
-
-static void
-nvc0_copy_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(chan->dev, engine);
-	struct nouveau_gpuobj *ctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-	u32 inst;
-
-	inst  = (chan->ramin->vinst >> 12);
-	inst |= 0x40000000;
-
-	/* disable fifo access */
-	nv_wr32(dev, pcopy->fuc + 0x048, 0x00000000);
-	/* mark channel as unloaded if it's currently active */
-	if (nv_rd32(dev, pcopy->fuc + 0x050) == inst)
-		nv_mask(dev, pcopy->fuc + 0x050, 0x40000000, 0x00000000);
-	/* mark next channel as invalid if it's about to be loaded */
-	if (nv_rd32(dev, pcopy->fuc + 0x054) == inst)
-		nv_mask(dev, pcopy->fuc + 0x054, 0x40000000, 0x00000000);
-	/* restore fifo access */
-	nv_wr32(dev, pcopy->fuc + 0x048, 0x00000003);
-
-	nv_wo32(chan->ramin, pcopy->ctx + 0, 0x00000000);
-	nv_wo32(chan->ramin, pcopy->ctx + 4, 0x00000000);
-	nouveau_gpuobj_ref(NULL, &ctx);
-
-	chan->engctx[engine] = ctx;
-}
-
-static int
-nvc0_copy_init(struct drm_device *dev, int engine)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
-	int i;
-
-	nv_mask(dev, 0x000200, pcopy->pmc, 0x00000000);
-	nv_mask(dev, 0x000200, pcopy->pmc, pcopy->pmc);
-	nv_wr32(dev, pcopy->fuc + 0x014, 0xffffffff);
-
-	nv_wr32(dev, pcopy->fuc + 0x1c0, 0x01000000);
-	for (i = 0; i < sizeof(nvc0_pcopy_data) / 4; i++)
-		nv_wr32(dev, pcopy->fuc + 0x1c4, nvc0_pcopy_data[i]);
-
-	nv_wr32(dev, pcopy->fuc + 0x180, 0x01000000);
-	for (i = 0; i < sizeof(nvc0_pcopy_code) / 4; i++) {
-		if ((i & 0x3f) == 0)
-			nv_wr32(dev, pcopy->fuc + 0x188, i >> 6);
-		nv_wr32(dev, pcopy->fuc + 0x184, nvc0_pcopy_code[i]);
-	}
-
-	nv_wr32(dev, pcopy->fuc + 0x084, engine - NVOBJ_ENGINE_COPY0);
-	nv_wr32(dev, pcopy->fuc + 0x10c, 0x00000000);
-	nv_wr32(dev, pcopy->fuc + 0x104, 0x00000000); /* ENTRY */
-	nv_wr32(dev, pcopy->fuc + 0x100, 0x00000002); /* TRIGGER */
-	return 0;
-}
-
-static int
-nvc0_copy_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
-
-	nv_mask(dev, pcopy->fuc + 0x048, 0x00000003, 0x00000000);
-
-	/* trigger fuc context unload */
-	nv_wait(dev, pcopy->fuc + 0x008, 0x0000000c, 0x00000000);
-	nv_mask(dev, pcopy->fuc + 0x054, 0x40000000, 0x00000000);
-	nv_wr32(dev, pcopy->fuc + 0x000, 0x00000008);
-	nv_wait(dev, pcopy->fuc + 0x008, 0x00000008, 0x00000000);
-
-	nv_wr32(dev, pcopy->fuc + 0x014, 0xffffffff);
-	return 0;
-}
-
-static struct nouveau_enum nvc0_copy_isr_error_name[] = {
-	{ 0x0001, "ILLEGAL_MTHD" },
-	{ 0x0002, "INVALID_ENUM" },
-	{ 0x0003, "INVALID_BITFIELD" },
-	{}
-};
-
-static void
-nvc0_copy_isr(struct drm_device *dev, int engine)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
-	u32 disp = nv_rd32(dev, pcopy->fuc + 0x01c);
-	u32 stat = nv_rd32(dev, pcopy->fuc + 0x008) & disp & ~(disp >> 16);
-	u64 inst = (u64)(nv_rd32(dev, pcopy->fuc + 0x050) & 0x0fffffff) << 12;
-	u32 chid = nvc0_graph_isr_chid(dev, inst);
-	u32 ssta = nv_rd32(dev, pcopy->fuc + 0x040) & 0x0000ffff;
-	u32 addr = nv_rd32(dev, pcopy->fuc + 0x040) >> 16;
-	u32 mthd = (addr & 0x07ff) << 2;
-	u32 subc = (addr & 0x3800) >> 11;
-	u32 data = nv_rd32(dev, pcopy->fuc + 0x044);
-
-	if (stat & 0x00000040) {
-		NV_INFO(dev, "PCOPY: DISPATCH_ERROR [");
-		nouveau_enum_print(nvc0_copy_isr_error_name, ssta);
-		printk("] ch %d [0x%010llx] subc %d mthd 0x%04x data 0x%08x\n",
-			chid, inst, subc, mthd, data);
-		nv_wr32(dev, pcopy->fuc + 0x004, 0x00000040);
-		stat &= ~0x00000040;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PCOPY: unhandled intr 0x%08x\n", stat);
-		nv_wr32(dev, pcopy->fuc + 0x004, stat);
-	}
-}
-
-static void
-nvc0_copy_isr_0(struct drm_device *dev)
-{
-	nvc0_copy_isr(dev, NVOBJ_ENGINE_COPY0);
-}
-
-static void
-nvc0_copy_isr_1(struct drm_device *dev)
-{
-	nvc0_copy_isr(dev, NVOBJ_ENGINE_COPY1);
-}
-
-static void
-nvc0_copy_destroy(struct drm_device *dev, int engine)
-{
-	struct nvc0_copy_engine *pcopy = nv_engine(dev, engine);
-
-	nouveau_irq_unregister(dev, pcopy->irq);
-
-	if (engine == NVOBJ_ENGINE_COPY0)
-		NVOBJ_ENGINE_DEL(dev, COPY0);
-	else
-		NVOBJ_ENGINE_DEL(dev, COPY1);
-	kfree(pcopy);
-}
-
-int
-nvc0_copy_create(struct drm_device *dev, int engine)
-{
-	struct nvc0_copy_engine *pcopy;
-
-	pcopy = kzalloc(sizeof(*pcopy), GFP_KERNEL);
-	if (!pcopy)
-		return -ENOMEM;
-
-	pcopy->base.destroy = nvc0_copy_destroy;
-	pcopy->base.init = nvc0_copy_init;
-	pcopy->base.fini = nvc0_copy_fini;
-	pcopy->base.context_new = nvc0_copy_context_new;
-	pcopy->base.context_del = nvc0_copy_context_del;
-	pcopy->base.object_new = nvc0_copy_object_new;
-
-	if (engine == 0) {
-		pcopy->irq = 5;
-		pcopy->pmc = 0x00000040;
-		pcopy->fuc = 0x104000;
-		pcopy->ctx = 0x0230;
-		nouveau_irq_register(dev, pcopy->irq, nvc0_copy_isr_0);
-		NVOBJ_ENGINE_ADD(dev, COPY0, &pcopy->base);
-		NVOBJ_CLASS(dev, 0x90b5, COPY0);
-	} else {
-		pcopy->irq = 6;
-		pcopy->pmc = 0x00000080;
-		pcopy->fuc = 0x105000;
-		pcopy->ctx = 0x0240;
-		nouveau_irq_register(dev, pcopy->irq, nvc0_copy_isr_1);
-		NVOBJ_ENGINE_ADD(dev, COPY1, &pcopy->base);
-		NVOBJ_CLASS(dev, 0x90b8, COPY1);
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_fb.c b/drivers/gpu/drm/nouveau/nvc0_fb.c
deleted file mode 100644
index f376c39310d..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_fb.c
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "drm.h"
-#include "nouveau_drv.h"
-#include "nouveau_drm.h"
-
-struct nvc0_fb_priv {
-	struct page *r100c10_page;
-	dma_addr_t r100c10;
-};
-
-static inline void
-nvc0_mfb_subp_isr(struct drm_device *dev, int unit, int subp)
-{
-	u32 subp_base = 0x141000 + (unit * 0x2000) + (subp * 0x400);
-	u32 stat = nv_rd32(dev, subp_base + 0x020);
-
-	if (stat) {
-		NV_INFO(dev, "PMFB%d_SUBP%d: 0x%08x\n", unit, subp, stat);
-		nv_wr32(dev, subp_base + 0x020, stat);
-	}
-}
-
-static void
-nvc0_mfb_isr(struct drm_device *dev)
-{
-	u32 units = nv_rd32(dev, 0x00017c);
-	while (units) {
-		u32 subp, unit = ffs(units) - 1;
-		for (subp = 0; subp < 2; subp++)
-			nvc0_mfb_subp_isr(dev, unit, subp);
-		units &= ~(1 << unit);
-	}
-
-	/* we do something horribly wrong and upset PMFB a lot, so mask off
-	 * interrupts from it after the first one until it's fixed
-	 */
-	nv_mask(dev, 0x000640, 0x02000000, 0x00000000);
-}
-
-static void
-nvc0_fb_destroy(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct nvc0_fb_priv *priv = pfb->priv;
-
-	nouveau_irq_unregister(dev, 25);
-
-	if (priv->r100c10_page) {
-		pci_unmap_page(dev->pdev, priv->r100c10, PAGE_SIZE,
-			       PCI_DMA_BIDIRECTIONAL);
-		__free_page(priv->r100c10_page);
-	}
-
-	kfree(priv);
-	pfb->priv = NULL;
-}
-
-static int
-nvc0_fb_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_fb_engine *pfb = &dev_priv->engine.fb;
-	struct nvc0_fb_priv *priv;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	pfb->priv = priv;
-
-	priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (!priv->r100c10_page) {
-		nvc0_fb_destroy(dev);
-		return -ENOMEM;
-	}
-
-	priv->r100c10 = pci_map_page(dev->pdev, priv->r100c10_page, 0,
-				     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(dev->pdev, priv->r100c10)) {
-		nvc0_fb_destroy(dev);
-		return -EFAULT;
-	}
-
-	nouveau_irq_register(dev, 25, nvc0_mfb_isr);
-	return 0;
-}
-
-int
-nvc0_fb_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_fb_priv *priv;
-	int ret;
-
-	if (!dev_priv->engine.fb.priv) {
-		ret = nvc0_fb_create(dev);
-		if (ret)
-			return ret;
-	}
-	priv = dev_priv->engine.fb.priv;
-
-	nv_wr32(dev, 0x100c10, priv->r100c10 >> 8);
-	nv_mask(dev, 0x17e820, 0x00100000, 0x00000000); /* NV_PLTCG_INTR_EN */
-	return 0;
-}
-
-void
-nvc0_fb_takedown(struct drm_device *dev)
-{
-	nvc0_fb_destroy(dev);
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index 797159e7b7a..9dcd30f3e1e 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -22,20 +22,16 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fbcon.h"
-#include "nouveau_mm.h"
 
 int
 nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, rect->rop == ROP_COPY ? 7 : 11);
@@ -69,9 +65,8 @@ int
 nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	int ret;
 
 	ret = RING_SPACE(chan, 12);
@@ -98,9 +93,8 @@ int
 nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
-	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
+	struct nouveau_drm *drm = nouveau_drm(nfbdev->dev);
+	struct nouveau_channel *chan = drm->channel;
 	uint32_t width, dwords, *data = (uint32_t *)image->data;
 	uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel));
 	uint32_t *palette = info->pseudo_palette;
@@ -157,12 +151,14 @@ nvc0_fbcon_accel_init(struct fb_info *info)
 {
 	struct nouveau_fbdev *nfbdev = info->par;
 	struct drm_device *dev = nfbdev->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = dev_priv->channel;
 	struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb;
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_channel *chan = drm->channel;
+	struct nouveau_object *object;
 	int ret, format;
 
-	ret = nouveau_gpuobj_gr_new(chan, 0x902d, 0x902d);
+	ret = nouveau_object_new(nv_object(chan->cli), NVDRM_CHAN, Nv2D,
+				 0x902d, NULL, 0, &object);
 	if (ret)
 		return ret;
 
@@ -202,9 +198,6 @@ nvc0_fbcon_accel_init(struct fb_info *info)
 
 	BEGIN_NVC0(chan, NvSub2D, 0x0000, 1);
 	OUT_RING  (chan, 0x0000902d);
-	BEGIN_NVC0(chan, NvSub2D, 0x0104, 2);
-	OUT_RING  (chan, upper_32_bits(chan->notifier_vma.offset));
-	OUT_RING  (chan, lower_32_bits(chan->notifier_vma.offset));
 	BEGIN_NVC0(chan, NvSub2D, 0x0290, 1);
 	OUT_RING  (chan, 0);
 	BEGIN_NVC0(chan, NvSub2D, 0x0888, 1);
diff --git a/drivers/gpu/drm/nouveau/nvc0_fence.c b/drivers/gpu/drm/nouveau/nvc0_fence.c
index 47ab388a606..53299eac967 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fence.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fence.c
@@ -22,29 +22,44 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include <core/object.h>
+#include <core/client.h>
+#include <core/class.h>
+
+#include <engine/fifo.h>
+
+#include "nouveau_drm.h"
 #include "nouveau_dma.h"
-#include "nouveau_fifo.h"
-#include "nouveau_ramht.h"
 #include "nouveau_fence.h"
 
+#include "nv50_display.h"
+
 struct nvc0_fence_priv {
 	struct nouveau_fence_priv base;
 	struct nouveau_bo *bo;
+	u32 *suspend;
 };
 
 struct nvc0_fence_chan {
 	struct nouveau_fence_chan base;
 	struct nouveau_vma vma;
+	struct nouveau_vma dispc_vma[4];
 };
 
+u64
+nvc0_fence_crtc(struct nouveau_channel *chan, int crtc)
+{
+	struct nvc0_fence_chan *fctx = chan->fence;
+	return fctx->dispc_vma[crtc].offset;
+}
+
 static int
 nvc0_fence_emit(struct nouveau_fence *fence)
 {
 	struct nouveau_channel *chan = fence->channel;
-	struct nvc0_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
-	u64 addr = fctx->vma.offset + chan->id * 16;
+	struct nvc0_fence_chan *fctx = chan->fence;
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	u64 addr = fctx->vma.offset + fifo->chid * 16;
 	int ret;
 
 	ret = RING_SPACE(chan, 5);
@@ -64,8 +79,9 @@ static int
 nvc0_fence_sync(struct nouveau_fence *fence,
 		struct nouveau_channel *prev, struct nouveau_channel *chan)
 {
-	struct nvc0_fence_chan *fctx = chan->engctx[NVOBJ_ENGINE_FENCE];
-	u64 addr = fctx->vma.offset + prev->id * 16;
+	struct nvc0_fence_chan *fctx = chan->fence;
+	struct nouveau_fifo_chan *fifo = (void *)prev->object;
+	u64 addr = fctx->vma.offset + fifo->chid * 16;
 	int ret;
 
 	ret = RING_SPACE(chan, 5);
@@ -85,91 +101,135 @@ nvc0_fence_sync(struct nouveau_fence *fence,
 static u32
 nvc0_fence_read(struct nouveau_channel *chan)
 {
-	struct nvc0_fence_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_FENCE);
-	return nouveau_bo_rd32(priv->bo, chan->id * 16/4);
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	struct nvc0_fence_priv *priv = chan->drm->fence;
+	return nouveau_bo_rd32(priv->bo, fifo->chid * 16/4);
 }
 
 static void
-nvc0_fence_context_del(struct nouveau_channel *chan, int engine)
+nvc0_fence_context_del(struct nouveau_channel *chan)
 {
-	struct nvc0_fence_priv *priv = nv_engine(chan->dev, engine);
-	struct nvc0_fence_chan *fctx = chan->engctx[engine];
+	struct drm_device *dev = chan->drm->dev;
+	struct nvc0_fence_priv *priv = chan->drm->fence;
+	struct nvc0_fence_chan *fctx = chan->fence;
+	int i;
+
+	if (nv_device(chan->drm->device)->card_type >= NV_D0) {
+		for (i = 0; i < dev->mode_config.num_crtc; i++) {
+			struct nouveau_bo *bo = nvd0_display_crtc_sema(dev, i);
+			nouveau_bo_vma_del(bo, &fctx->dispc_vma[i]);
+		}
+	} else
+	if (nv_device(chan->drm->device)->card_type >= NV_50) {
+		for (i = 0; i < dev->mode_config.num_crtc; i++) {
+			struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i);
+			nouveau_bo_vma_del(bo, &fctx->dispc_vma[i]);
+		}
+	}
 
 	nouveau_bo_vma_del(priv->bo, &fctx->vma);
 	nouveau_fence_context_del(&fctx->base);
-	chan->engctx[engine] = NULL;
+	chan->fence = NULL;
 	kfree(fctx);
 }
 
 static int
-nvc0_fence_context_new(struct nouveau_channel *chan, int engine)
+nvc0_fence_context_new(struct nouveau_channel *chan)
 {
-	struct nvc0_fence_priv *priv = nv_engine(chan->dev, engine);
+	struct nouveau_fifo_chan *fifo = (void *)chan->object;
+	struct nouveau_client *client = nouveau_client(fifo);
+	struct nvc0_fence_priv *priv = chan->drm->fence;
 	struct nvc0_fence_chan *fctx;
-	int ret;
+	int ret, i;
 
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
+	fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL);
 	if (!fctx)
 		return -ENOMEM;
 
 	nouveau_fence_context_new(&fctx->base);
 
-	ret = nouveau_bo_vma_add(priv->bo, chan->vm, &fctx->vma);
+	ret = nouveau_bo_vma_add(priv->bo, client->vm, &fctx->vma);
 	if (ret)
-		nvc0_fence_context_del(chan, engine);
+		nvc0_fence_context_del(chan);
+
+	/* map display semaphore buffers into channel's vm */
+	for (i = 0; !ret && i < chan->drm->dev->mode_config.num_crtc; i++) {
+		struct nouveau_bo *bo;
+		if (nv_device(chan->drm->device)->card_type >= NV_D0)
+			bo = nvd0_display_crtc_sema(chan->drm->dev, i);
+		else
+			bo = nv50_display_crtc_sema(chan->drm->dev, i);
 
-	nouveau_bo_wr32(priv->bo, chan->id * 16/4, 0x00000000);
+		ret = nouveau_bo_vma_add(bo, client->vm, &fctx->dispc_vma[i]);
+	}
+
+	nouveau_bo_wr32(priv->bo, fifo->chid * 16/4, 0x00000000);
 	return ret;
 }
 
-static int
-nvc0_fence_fini(struct drm_device *dev, int engine, bool suspend)
+static bool
+nvc0_fence_suspend(struct nouveau_drm *drm)
 {
-	return 0;
+	struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
+	struct nvc0_fence_priv *priv = drm->fence;
+	int i;
+
+	priv->suspend = vmalloc((pfifo->max + 1) * sizeof(u32));
+	if (priv->suspend) {
+		for (i = 0; i <= pfifo->max; i++)
+			priv->suspend[i] = nouveau_bo_rd32(priv->bo, i);
+	}
+
+	return priv->suspend != NULL;
 }
 
-static int
-nvc0_fence_init(struct drm_device *dev, int engine)
+static void
+nvc0_fence_resume(struct nouveau_drm *drm)
 {
-	return 0;
+	struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
+	struct nvc0_fence_priv *priv = drm->fence;
+	int i;
+
+	if (priv->suspend) {
+		for (i = 0; i <= pfifo->max; i++)
+			nouveau_bo_wr32(priv->bo, i, priv->suspend[i]);
+		vfree(priv->suspend);
+		priv->suspend = NULL;
+	}
 }
 
 static void
-nvc0_fence_destroy(struct drm_device *dev, int engine)
+nvc0_fence_destroy(struct nouveau_drm *drm)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_fence_priv *priv = nv_engine(dev, engine);
-
+	struct nvc0_fence_priv *priv = drm->fence;
 	nouveau_bo_unmap(priv->bo);
 	nouveau_bo_ref(NULL, &priv->bo);
-	dev_priv->eng[engine] = NULL;
+	drm->fence = NULL;
 	kfree(priv);
 }
 
 int
-nvc0_fence_create(struct drm_device *dev)
+nvc0_fence_create(struct nouveau_drm *drm)
 {
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_fifo *pfifo = nouveau_fifo(drm->device);
 	struct nvc0_fence_priv *priv;
 	int ret;
 
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	priv = drm->fence = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	priv->base.engine.destroy = nvc0_fence_destroy;
-	priv->base.engine.init = nvc0_fence_init;
-	priv->base.engine.fini = nvc0_fence_fini;
-	priv->base.engine.context_new = nvc0_fence_context_new;
-	priv->base.engine.context_del = nvc0_fence_context_del;
+	priv->base.dtor = nvc0_fence_destroy;
+	priv->base.suspend = nvc0_fence_suspend;
+	priv->base.resume = nvc0_fence_resume;
+	priv->base.context_new = nvc0_fence_context_new;
+	priv->base.context_del = nvc0_fence_context_del;
 	priv->base.emit = nvc0_fence_emit;
 	priv->base.sync = nvc0_fence_sync;
 	priv->base.read = nvc0_fence_read;
-	dev_priv->eng[NVOBJ_ENGINE_FENCE] = &priv->base.engine;
 
-	ret = nouveau_bo_new(dev, 16 * pfifo->channels, 0, TTM_PL_FLAG_VRAM,
-			     0, 0, NULL, &priv->bo);
+	ret = nouveau_bo_new(drm->dev, 16 * (pfifo->max + 1), 0,
+			     TTM_PL_FLAG_VRAM, 0, 0, NULL, &priv->bo);
 	if (ret == 0) {
 		ret = nouveau_bo_pin(priv->bo, TTM_PL_FLAG_VRAM);
 		if (ret == 0)
@@ -179,6 +239,6 @@ nvc0_fence_create(struct drm_device *dev)
 	}
 
 	if (ret)
-		nvc0_fence_destroy(dev, NVOBJ_ENGINE_FENCE);
+		nvc0_fence_destroy(drm);
 	return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/nvc0_fifo.c b/drivers/gpu/drm/nouveau/nvc0_fifo.c
deleted file mode 100644
index cd39eb99f5b..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_fifo.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nouveau_fifo.h"
-
-static void nvc0_fifo_isr(struct drm_device *);
-
-struct nvc0_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct nouveau_gpuobj *playlist[2];
-	int cur_playlist;
-	struct nouveau_vma user_vma;
-	int spoon_nr;
-};
-
-struct nvc0_fifo_chan {
-	struct nouveau_fifo_chan base;
-	struct nouveau_gpuobj *user;
-};
-
-static void
-nvc0_fifo_playlist_update(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nvc0_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct nouveau_gpuobj *cur;
-	int i, p;
-
-	cur = priv->playlist[priv->cur_playlist];
-	priv->cur_playlist = !priv->cur_playlist;
-
-	for (i = 0, p = 0; i < 128; i++) {
-		if (!(nv_rd32(dev, 0x3004 + (i * 8)) & 1))
-			continue;
-		nv_wo32(cur, p + 0, i);
-		nv_wo32(cur, p + 4, 0x00000004);
-		p += 8;
-	}
-	pinstmem->flush(dev);
-
-	nv_wr32(dev, 0x002270, cur->vinst >> 12);
-	nv_wr32(dev, 0x002274, 0x01f00000 | (p >> 3));
-	if (!nv_wait(dev, 0x00227c, 0x00100000, 0x00000000))
-		NV_ERROR(dev, "PFIFO - playlist update failed\n");
-}
-
-static int
-nvc0_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nvc0_fifo_priv *priv = nv_engine(dev, engine);
-	struct nvc0_fifo_chan *fctx;
-	u64 ib_virt = chan->pushbuf_base + chan->dma.ib_base * 4;
-	int ret, i;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	chan->user = ioremap_wc(pci_resource_start(dev->pdev, 1) +
-				priv->user_vma.offset + (chan->id * 0x1000),
-				PAGE_SIZE);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	/* allocate vram for control regs, map into polling area */
-	ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &fctx->user);
-	if (ret)
-		goto error;
-
-	nouveau_vm_map_at(&priv->user_vma, chan->id * 0x1000,
-			  *(struct nouveau_mem **)fctx->user->node);
-
-	for (i = 0; i < 0x100; i += 4)
-		nv_wo32(chan->ramin, i, 0x00000000);
-	nv_wo32(chan->ramin, 0x08, lower_32_bits(fctx->user->vinst));
-	nv_wo32(chan->ramin, 0x0c, upper_32_bits(fctx->user->vinst));
-	nv_wo32(chan->ramin, 0x10, 0x0000face);
-	nv_wo32(chan->ramin, 0x30, 0xfffff902);
-	nv_wo32(chan->ramin, 0x48, lower_32_bits(ib_virt));
-	nv_wo32(chan->ramin, 0x4c, drm_order(chan->dma.ib_max + 1) << 16 |
-				   upper_32_bits(ib_virt));
-	nv_wo32(chan->ramin, 0x54, 0x00000002);
-	nv_wo32(chan->ramin, 0x84, 0x20400000);
-	nv_wo32(chan->ramin, 0x94, 0x30000001);
-	nv_wo32(chan->ramin, 0x9c, 0x00000100);
-	nv_wo32(chan->ramin, 0xa4, 0x1f1f1f1f);
-	nv_wo32(chan->ramin, 0xa8, 0x1f1f1f1f);
-	nv_wo32(chan->ramin, 0xac, 0x0000001f);
-	nv_wo32(chan->ramin, 0xb8, 0xf8000000);
-	nv_wo32(chan->ramin, 0xf8, 0x10003080); /* 0x002310 */
-	nv_wo32(chan->ramin, 0xfc, 0x10000010); /* 0x002350 */
-	pinstmem->flush(dev);
-
-	nv_wr32(dev, 0x003000 + (chan->id * 8), 0xc0000000 |
-						(chan->ramin->vinst >> 12));
-	nv_wr32(dev, 0x003004 + (chan->id * 8), 0x001f0001);
-	nvc0_fifo_playlist_update(dev);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nvc0_fifo_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nvc0_fifo_chan *fctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-
-	nv_mask(dev, 0x003004 + (chan->id * 8), 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x002634, chan->id);
-	if (!nv_wait(dev, 0x0002634, 0xffffffff, chan->id))
-		NV_WARN(dev, "0x2634 != chid: 0x%08x\n", nv_rd32(dev, 0x2634));
-	nvc0_fifo_playlist_update(dev);
-	nv_wr32(dev, 0x003000 + (chan->id * 8), 0x00000000);
-
-	nouveau_gpuobj_ref(NULL, &fctx->user);
-	if (chan->user) {
-		iounmap(chan->user);
-		chan->user = NULL;
-	}
-
-	chan->engctx[engine] = NULL;
-	kfree(fctx);
-}
-
-static int
-nvc0_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_fifo_priv *priv = nv_engine(dev, engine);
-	struct nouveau_channel *chan;
-	int i;
-
-	/* reset PFIFO, enable all available PSUBFIFO areas */
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
-	nv_wr32(dev, 0x000204, 0xffffffff);
-	nv_wr32(dev, 0x002204, 0xffffffff);
-
-	priv->spoon_nr = hweight32(nv_rd32(dev, 0x002204));
-	NV_DEBUG(dev, "PFIFO: %d subfifo(s)\n", priv->spoon_nr);
-
-	/* assign engines to subfifos */
-	if (priv->spoon_nr >= 3) {
-		nv_wr32(dev, 0x002208, ~(1 << 0)); /* PGRAPH */
-		nv_wr32(dev, 0x00220c, ~(1 << 1)); /* PVP */
-		nv_wr32(dev, 0x002210, ~(1 << 1)); /* PPP */
-		nv_wr32(dev, 0x002214, ~(1 << 1)); /* PBSP */
-		nv_wr32(dev, 0x002218, ~(1 << 2)); /* PCE0 */
-		nv_wr32(dev, 0x00221c, ~(1 << 1)); /* PCE1 */
-	}
-
-	/* PSUBFIFO[n] */
-	for (i = 0; i < priv->spoon_nr; i++) {
-		nv_mask(dev, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
-		nv_wr32(dev, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
-		nv_wr32(dev, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTR_EN */
-	}
-
-	nv_mask(dev, 0x002200, 0x00000001, 0x00000001);
-	nv_wr32(dev, 0x002254, 0x10000000 | priv->user_vma.offset >> 12);
-
-	nv_wr32(dev, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
-	nv_wr32(dev, 0x002100, 0xffffffff);
-	nv_wr32(dev, 0x002140, 0xbfffffff);
-
-	/* restore PFIFO context table */
-	for (i = 0; i < 128; i++) {
-		chan = dev_priv->channels.ptr[i];
-		if (!chan || !chan->engctx[engine])
-			continue;
-
-		nv_wr32(dev, 0x003000 + (i * 8), 0xc0000000 |
-						 (chan->ramin->vinst >> 12));
-		nv_wr32(dev, 0x003004 + (i * 8), 0x001f0001);
-	}
-	nvc0_fifo_playlist_update(dev);
-
-	return 0;
-}
-
-static int
-nvc0_fifo_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	int i;
-
-	for (i = 0; i < 128; i++) {
-		if (!(nv_rd32(dev, 0x003004 + (i * 8)) & 1))
-			continue;
-
-		nv_mask(dev, 0x003004 + (i * 8), 0x00000001, 0x00000000);
-		nv_wr32(dev, 0x002634, i);
-		if (!nv_wait(dev, 0x002634, 0xffffffff, i)) {
-			NV_INFO(dev, "PFIFO: kick ch %d failed: 0x%08x\n",
-				i, nv_rd32(dev, 0x002634));
-			return -EBUSY;
-		}
-	}
-
-	nv_wr32(dev, 0x002140, 0x00000000);
-	return 0;
-}
-
-
-struct nouveau_enum nvc0_fifo_fault_unit[] = {
-	{ 0x00, "PGRAPH" },
-	{ 0x03, "PEEPHOLE" },
-	{ 0x04, "BAR1" },
-	{ 0x05, "BAR3" },
-	{ 0x07, "PFIFO" },
-	{ 0x10, "PBSP" },
-	{ 0x11, "PPPP" },
-	{ 0x13, "PCOUNTER" },
-	{ 0x14, "PVP" },
-	{ 0x15, "PCOPY0" },
-	{ 0x16, "PCOPY1" },
-	{ 0x17, "PDAEMON" },
-	{}
-};
-
-struct nouveau_enum nvc0_fifo_fault_reason[] = {
-	{ 0x00, "PT_NOT_PRESENT" },
-	{ 0x01, "PT_TOO_SHORT" },
-	{ 0x02, "PAGE_NOT_PRESENT" },
-	{ 0x03, "VM_LIMIT_EXCEEDED" },
-	{ 0x04, "NO_CHANNEL" },
-	{ 0x05, "PAGE_SYSTEM_ONLY" },
-	{ 0x06, "PAGE_READ_ONLY" },
-	{ 0x0a, "COMPRESSED_SYSRAM" },
-	{ 0x0c, "INVALID_STORAGE_TYPE" },
-	{}
-};
-
-struct nouveau_enum nvc0_fifo_fault_hubclient[] = {
-	{ 0x01, "PCOPY0" },
-	{ 0x02, "PCOPY1" },
-	{ 0x04, "DISPATCH" },
-	{ 0x05, "CTXCTL" },
-	{ 0x06, "PFIFO" },
-	{ 0x07, "BAR_READ" },
-	{ 0x08, "BAR_WRITE" },
-	{ 0x0b, "PVP" },
-	{ 0x0c, "PPPP" },
-	{ 0x0d, "PBSP" },
-	{ 0x11, "PCOUNTER" },
-	{ 0x12, "PDAEMON" },
-	{ 0x14, "CCACHE" },
-	{ 0x15, "CCACHE_POST" },
-	{}
-};
-
-struct nouveau_enum nvc0_fifo_fault_gpcclient[] = {
-	{ 0x01, "TEX" },
-	{ 0x0c, "ESETUP" },
-	{ 0x0e, "CTXCTL" },
-	{ 0x0f, "PROP" },
-	{}
-};
-
-struct nouveau_bitfield nvc0_fifo_subfifo_intr[] = {
-/*	{ 0x00008000, "" }	seen with null ib push */
-	{ 0x00200000, "ILLEGAL_MTHD" },
-	{ 0x00800000, "EMPTY_SUBC" },
-	{}
-};
-
-static void
-nvc0_fifo_isr_vm_fault(struct drm_device *dev, int unit)
-{
-	u32 inst = nv_rd32(dev, 0x2800 + (unit * 0x10));
-	u32 valo = nv_rd32(dev, 0x2804 + (unit * 0x10));
-	u32 vahi = nv_rd32(dev, 0x2808 + (unit * 0x10));
-	u32 stat = nv_rd32(dev, 0x280c + (unit * 0x10));
-	u32 client = (stat & 0x00001f00) >> 8;
-
-	NV_INFO(dev, "PFIFO: %s fault at 0x%010llx [",
-		(stat & 0x00000080) ? "write" : "read", (u64)vahi << 32 | valo);
-	nouveau_enum_print(nvc0_fifo_fault_reason, stat & 0x0000000f);
-	printk("] from ");
-	nouveau_enum_print(nvc0_fifo_fault_unit, unit);
-	if (stat & 0x00000040) {
-		printk("/");
-		nouveau_enum_print(nvc0_fifo_fault_hubclient, client);
-	} else {
-		printk("/GPC%d/", (stat & 0x1f000000) >> 24);
-		nouveau_enum_print(nvc0_fifo_fault_gpcclient, client);
-	}
-	printk(" on channel 0x%010llx\n", (u64)inst << 12);
-}
-
-static int
-nvc0_fifo_page_flip(struct drm_device *dev, u32 chid)
-{
-	struct nvc0_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = NULL;
-	unsigned long flags;
-	int ret = -EINVAL;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	if (likely(chid >= 0 && chid < priv->base.channels)) {
-		chan = dev_priv->channels.ptr[chid];
-		if (likely(chan))
-			ret = nouveau_finish_page_flip(chan, NULL);
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return ret;
-}
-
-static void
-nvc0_fifo_isr_subfifo_intr(struct drm_device *dev, int unit)
-{
-	u32 stat = nv_rd32(dev, 0x040108 + (unit * 0x2000));
-	u32 addr = nv_rd32(dev, 0x0400c0 + (unit * 0x2000));
-	u32 data = nv_rd32(dev, 0x0400c4 + (unit * 0x2000));
-	u32 chid = nv_rd32(dev, 0x040120 + (unit * 0x2000)) & 0x7f;
-	u32 subc = (addr & 0x00070000);
-	u32 mthd = (addr & 0x00003ffc);
-	u32 show = stat;
-
-	if (stat & 0x00200000) {
-		if (mthd == 0x0054) {
-			if (!nvc0_fifo_page_flip(dev, chid))
-				show &= ~0x00200000;
-		}
-	}
-
-	if (show) {
-		NV_INFO(dev, "PFIFO%d:", unit);
-		nouveau_bitfield_print(nvc0_fifo_subfifo_intr, show);
-		NV_INFO(dev, "PFIFO%d: ch %d subc %d mthd 0x%04x data 0x%08x\n",
-			     unit, chid, subc, mthd, data);
-	}
-
-	nv_wr32(dev, 0x0400c0 + (unit * 0x2000), 0x80600008);
-	nv_wr32(dev, 0x040108 + (unit * 0x2000), stat);
-}
-
-static void
-nvc0_fifo_isr(struct drm_device *dev)
-{
-	u32 mask = nv_rd32(dev, 0x002140);
-	u32 stat = nv_rd32(dev, 0x002100) & mask;
-
-	if (stat & 0x00000100) {
-		NV_INFO(dev, "PFIFO: unknown status 0x00000100\n");
-		nv_wr32(dev, 0x002100, 0x00000100);
-		stat &= ~0x00000100;
-	}
-
-	if (stat & 0x10000000) {
-		u32 units = nv_rd32(dev, 0x00259c);
-		u32 u = units;
-
-		while (u) {
-			int i = ffs(u) - 1;
-			nvc0_fifo_isr_vm_fault(dev, i);
-			u &= ~(1 << i);
-		}
-
-		nv_wr32(dev, 0x00259c, units);
-		stat &= ~0x10000000;
-	}
-
-	if (stat & 0x20000000) {
-		u32 units = nv_rd32(dev, 0x0025a0);
-		u32 u = units;
-
-		while (u) {
-			int i = ffs(u) - 1;
-			nvc0_fifo_isr_subfifo_intr(dev, i);
-			u &= ~(1 << i);
-		}
-
-		nv_wr32(dev, 0x0025a0, units);
-		stat &= ~0x20000000;
-	}
-
-	if (stat & 0x40000000) {
-		NV_INFO(dev, "PFIFO: unknown status 0x40000000\n");
-		nv_mask(dev, 0x002a00, 0x00000000, 0x00000000);
-		stat &= ~0x40000000;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PFIFO: unhandled status 0x%08x\n", stat);
-		nv_wr32(dev, 0x002100, stat);
-		nv_wr32(dev, 0x002140, 0);
-	}
-}
-
-static void
-nvc0_fifo_destroy(struct drm_device *dev, int engine)
-{
-	struct nvc0_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nouveau_vm_put(&priv->user_vma);
-	nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
-	nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
-
-	dev_priv->eng[engine] = NULL;
-	kfree(priv);
-}
-
-int
-nvc0_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_fifo_priv *priv;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nvc0_fifo_destroy;
-	priv->base.base.init = nvc0_fifo_init;
-	priv->base.base.fini = nvc0_fifo_fini;
-	priv->base.base.context_new = nvc0_fifo_context_new;
-	priv->base.base.context_del = nvc0_fifo_context_del;
-	priv->base.channels = 128;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 4096, 4096, 0, &priv->playlist[0]);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 4096, 4096, 0, &priv->playlist[1]);
-	if (ret)
-		goto error;
-
-	ret = nouveau_vm_get(dev_priv->bar1_vm, priv->base.channels * 0x1000,
-			     12, NV_MEM_ACCESS_RW, &priv->user_vma);
-	if (ret)
-		goto error;
-
-	nouveau_irq_register(dev, 8, nvc0_fifo_isr);
-error:
-	if (ret)
-		priv->base.base.destroy(dev, NVOBJ_ENGINE_FIFO);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.c b/drivers/gpu/drm/nouveau/nvc0_graph.c
deleted file mode 100644
index 2a01e6e4772..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_graph.c
+++ /dev/null
@@ -1,897 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/firmware.h>
-#include <linux/module.h>
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nouveau_fifo.h"
-
-#include "nvc0_graph.h"
-#include "nvc0_grhub.fuc.h"
-#include "nvc0_grgpc.fuc.h"
-
-static void
-nvc0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
-{
-	NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
-		nv_rd32(dev, base + 0x400));
-	NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
-		nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
-		nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
-	NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
-		nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
-		nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
-}
-
-static void
-nvc0_graph_ctxctl_debug(struct drm_device *dev)
-{
-	u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
-	u32 gpc;
-
-	nvc0_graph_ctxctl_debug_unit(dev, 0x409000);
-	for (gpc = 0; gpc < gpcnr; gpc++)
-		nvc0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
-}
-
-static int
-nvc0_graph_load_context(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-
-	nv_wr32(dev, 0x409840, 0x00000030);
-	nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
-	nv_wr32(dev, 0x409504, 0x00000003);
-	if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
-		NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
-
-	return 0;
-}
-
-static int
-nvc0_graph_unload_context_to(struct drm_device *dev, u64 chan)
-{
-	nv_wr32(dev, 0x409840, 0x00000003);
-	nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
-	nv_wr32(dev, 0x409504, 0x00000009);
-	if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
-		NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-static int
-nvc0_graph_construct_context(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	int ret, i;
-	u32 *ctx;
-
-	ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	if (!nouveau_ctxfw) {
-		nv_wr32(dev, 0x409840, 0x80000000);
-		nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
-		nv_wr32(dev, 0x409504, 0x00000001);
-		if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
-			NV_ERROR(dev, "PGRAPH: HUB_SET_CHAN timeout\n");
-			nvc0_graph_ctxctl_debug(dev);
-			ret = -EBUSY;
-			goto err;
-		}
-	} else {
-		nvc0_graph_load_context(chan);
-
-		nv_wo32(grch->grctx, 0x1c, 1);
-		nv_wo32(grch->grctx, 0x20, 0);
-		nv_wo32(grch->grctx, 0x28, 0);
-		nv_wo32(grch->grctx, 0x2c, 0);
-		dev_priv->engine.instmem.flush(dev);
-	}
-
-	ret = nvc0_grctx_generate(chan);
-	if (ret)
-		goto err;
-
-	if (!nouveau_ctxfw) {
-		nv_wr32(dev, 0x409840, 0x80000000);
-		nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
-		nv_wr32(dev, 0x409504, 0x00000002);
-		if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
-			NV_ERROR(dev, "PGRAPH: HUB_CTX_SAVE timeout\n");
-			nvc0_graph_ctxctl_debug(dev);
-			ret = -EBUSY;
-			goto err;
-		}
-	} else {
-		ret = nvc0_graph_unload_context_to(dev, chan->ramin->vinst);
-		if (ret)
-			goto err;
-	}
-
-	for (i = 0; i < priv->grctx_size; i += 4)
-		ctx[i / 4] = nv_ro32(grch->grctx, i);
-
-	priv->grctx_vals = ctx;
-	return 0;
-
-err:
-	kfree(ctx);
-	return ret;
-}
-
-static int
-nvc0_graph_create_context_mmio_list(struct nouveau_channel *chan)
-{
-	struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i = 0, gpc, tp, ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x2000, 256, NVOBJ_FLAG_VM,
-				 &grch->unk408004);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x8000, 256, NVOBJ_FLAG_VM,
-				 &grch->unk40800c);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 384 * 1024, 4096,
-				 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
-				 &grch->unk418810);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x1000, 0, NVOBJ_FLAG_VM,
-				 &grch->mmio);
-	if (ret)
-		return ret;
-
-
-	nv_wo32(grch->mmio, i++ * 4, 0x00408004);
-	nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
-	nv_wo32(grch->mmio, i++ * 4, 0x00408008);
-	nv_wo32(grch->mmio, i++ * 4, 0x80000018);
-
-	nv_wo32(grch->mmio, i++ * 4, 0x0040800c);
-	nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
-	nv_wo32(grch->mmio, i++ * 4, 0x00408010);
-	nv_wo32(grch->mmio, i++ * 4, 0x80000000);
-
-	nv_wo32(grch->mmio, i++ * 4, 0x00418810);
-	nv_wo32(grch->mmio, i++ * 4, 0x80000000 | grch->unk418810->linst >> 12);
-	nv_wo32(grch->mmio, i++ * 4, 0x00419848);
-	nv_wo32(grch->mmio, i++ * 4, 0x10000000 | grch->unk418810->linst >> 12);
-
-	nv_wo32(grch->mmio, i++ * 4, 0x00419004);
-	nv_wo32(grch->mmio, i++ * 4, grch->unk40800c->linst >> 8);
-	nv_wo32(grch->mmio, i++ * 4, 0x00419008);
-	nv_wo32(grch->mmio, i++ * 4, 0x00000000);
-
-	nv_wo32(grch->mmio, i++ * 4, 0x00418808);
-	nv_wo32(grch->mmio, i++ * 4, grch->unk408004->linst >> 8);
-	nv_wo32(grch->mmio, i++ * 4, 0x0041880c);
-	nv_wo32(grch->mmio, i++ * 4, 0x80000018);
-
-	if (dev_priv->chipset != 0xc1) {
-		u32 magic = 0x02180000;
-		nv_wo32(grch->mmio, i++ * 4, 0x00405830);
-		nv_wo32(grch->mmio, i++ * 4, magic);
-		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-			for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
-				u32 reg = TP_UNIT(gpc, tp, 0x520);
-				nv_wo32(grch->mmio, i++ * 4, reg);
-				nv_wo32(grch->mmio, i++ * 4, magic);
-				magic += 0x0324;
-			}
-		}
-	} else {
-		u32 magic = 0x02180000;
-		nv_wo32(grch->mmio, i++ * 4, 0x00405830);
-		nv_wo32(grch->mmio, i++ * 4, magic | 0x0000218);
-		nv_wo32(grch->mmio, i++ * 4, 0x004064c4);
-		nv_wo32(grch->mmio, i++ * 4, 0x0086ffff);
-		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-			for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
-				u32 reg = TP_UNIT(gpc, tp, 0x520);
-				nv_wo32(grch->mmio, i++ * 4, reg);
-				nv_wo32(grch->mmio, i++ * 4, (1 << 28) | magic);
-				magic += 0x0324;
-			}
-			for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
-				u32 reg = TP_UNIT(gpc, tp, 0x544);
-				nv_wo32(grch->mmio, i++ * 4, reg);
-				nv_wo32(grch->mmio, i++ * 4, magic);
-				magic += 0x0324;
-			}
-		}
-	}
-
-	grch->mmio_nr = i / 2;
-	return 0;
-}
-
-static int
-nvc0_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nvc0_graph_priv *priv = nv_engine(dev, engine);
-	struct nvc0_graph_chan *grch;
-	struct nouveau_gpuobj *grctx;
-	int ret, i;
-
-	grch = kzalloc(sizeof(*grch), GFP_KERNEL);
-	if (!grch)
-		return -ENOMEM;
-	chan->engctx[NVOBJ_ENGINE_GR] = grch;
-
-	ret = nouveau_gpuobj_new(dev, chan, priv->grctx_size, 256,
-				 NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
-				 &grch->grctx);
-	if (ret)
-		goto error;
-	grctx = grch->grctx;
-
-	ret = nvc0_graph_create_context_mmio_list(chan);
-	if (ret)
-		goto error;
-
-	nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
-	nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
-	pinstmem->flush(dev);
-
-	if (!priv->grctx_vals) {
-		ret = nvc0_graph_construct_context(chan);
-		if (ret)
-			goto error;
-	}
-
-	for (i = 0; i < priv->grctx_size; i += 4)
-		nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
-
-	if (!nouveau_ctxfw) {
-		nv_wo32(grctx, 0x00, grch->mmio_nr);
-		nv_wo32(grctx, 0x04, grch->mmio->linst >> 8);
-	} else {
-		nv_wo32(grctx, 0xf4, 0);
-		nv_wo32(grctx, 0xf8, 0);
-		nv_wo32(grctx, 0x10, grch->mmio_nr);
-		nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
-		nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
-		nv_wo32(grctx, 0x1c, 1);
-		nv_wo32(grctx, 0x20, 0);
-		nv_wo32(grctx, 0x28, 0);
-		nv_wo32(grctx, 0x2c, 0);
-	}
-	pinstmem->flush(dev);
-	return 0;
-
-error:
-	priv->base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nvc0_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nvc0_graph_chan *grch = chan->engctx[engine];
-
-	nouveau_gpuobj_ref(NULL, &grch->mmio);
-	nouveau_gpuobj_ref(NULL, &grch->unk418810);
-	nouveau_gpuobj_ref(NULL, &grch->unk40800c);
-	nouveau_gpuobj_ref(NULL, &grch->unk408004);
-	nouveau_gpuobj_ref(NULL, &grch->grctx);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nvc0_graph_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	return 0;
-}
-
-static int
-nvc0_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static void
-nvc0_graph_init_obj418880(struct drm_device *dev)
-{
-	struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int i;
-
-	nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
-	nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
-	nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
-	nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
-}
-
-static void
-nvc0_graph_init_regs(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x400080, 0x003083c2);
-	nv_wr32(dev, 0x400088, 0x00006fe7);
-	nv_wr32(dev, 0x40008c, 0x00000000);
-	nv_wr32(dev, 0x400090, 0x00000030);
-	nv_wr32(dev, 0x40013c, 0x013901f7);
-	nv_wr32(dev, 0x400140, 0x00000100);
-	nv_wr32(dev, 0x400144, 0x00000000);
-	nv_wr32(dev, 0x400148, 0x00000110);
-	nv_wr32(dev, 0x400138, 0x00000000);
-	nv_wr32(dev, 0x400130, 0x00000000);
-	nv_wr32(dev, 0x400134, 0x00000000);
-	nv_wr32(dev, 0x400124, 0x00000002);
-}
-
-static void
-nvc0_graph_init_gpc_0(struct drm_device *dev)
-{
-	struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tp_total);
-	u32 data[TP_MAX / 8];
-	u8  tpnr[GPC_MAX];
-	int i, gpc, tpc;
-
-	nv_wr32(dev, TP_UNIT(0, 0, 0x5c), 1); /* affects TFB offset queries */
-
-	/*
-	 *      TP      ROP UNKVAL(magic_not_rop_nr)
-	 * 450: 4/0/0/0 2        3
-	 * 460: 3/4/0/0 4        1
-	 * 465: 3/4/4/0 4        7
-	 * 470: 3/3/4/4 5        5
-	 * 480: 3/4/4/4 6        6
-	 */
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
-	for (i = 0, gpc = -1; i < priv->tp_total; i++) {
-		do {
-			gpc = (gpc + 1) % priv->gpc_nr;
-		} while (!tpnr[gpc]);
-		tpc = priv->tp_nr[gpc] - tpnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nv_wr32(dev, GPC_BCAST(0x0980), data[0]);
-	nv_wr32(dev, GPC_BCAST(0x0984), data[1]);
-	nv_wr32(dev, GPC_BCAST(0x0988), data[2]);
-	nv_wr32(dev, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
-						  priv->tp_nr[gpc]);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tp_total);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
-	nv_wr32(dev, GPC_BCAST(0x08ac), nv_rd32(dev, 0x100800));
-}
-
-static void
-nvc0_graph_init_units(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x409c24, 0x000f0000);
-	nv_wr32(dev, 0x404000, 0xc0000000); /* DISPATCH */
-	nv_wr32(dev, 0x404600, 0xc0000000); /* M2MF */
-	nv_wr32(dev, 0x408030, 0xc0000000);
-	nv_wr32(dev, 0x40601c, 0xc0000000);
-	nv_wr32(dev, 0x404490, 0xc0000000); /* MACRO */
-	nv_wr32(dev, 0x406018, 0xc0000000);
-	nv_wr32(dev, 0x405840, 0xc0000000);
-	nv_wr32(dev, 0x405844, 0x00ffffff);
-	nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
-	nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
-}
-
-static void
-nvc0_graph_init_gpc_1(struct drm_device *dev)
-{
-	struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int gpc, tp;
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tp = 0; tp < priv->tp_nr[gpc]; tp++) {
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x508), 0xffffffff);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x50c), 0xffffffff);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x224), 0xc0000000);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x48c), 0xc0000000);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x084), 0xc0000000);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x644), 0x001ffffe);
-			nv_wr32(dev, TP_UNIT(gpc, tp, 0x64c), 0x0000000f);
-		}
-		nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
-}
-
-static void
-nvc0_graph_init_rop(struct drm_device *dev)
-{
-	struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int rop;
-
-	for (rop = 0; rop < priv->rop_nr; rop++) {
-		nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
-		nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
-		nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
-	}
-}
-
-static void
-nvc0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
-		    struct nvc0_graph_fuc *code, struct nvc0_graph_fuc *data)
-{
-	int i;
-
-	nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
-	for (i = 0; i < data->size / 4; i++)
-		nv_wr32(dev, fuc_base + 0x01c4, data->data[i]);
-
-	nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
-	for (i = 0; i < code->size / 4; i++) {
-		if ((i & 0x3f) == 0)
-			nv_wr32(dev, fuc_base + 0x0188, i >> 6);
-		nv_wr32(dev, fuc_base + 0x0184, code->data[i]);
-	}
-}
-
-static int
-nvc0_graph_init_ctxctl(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	u32 r000260;
-	int i;
-
-	if (!nouveau_ctxfw) {
-		/* load HUB microcode */
-		r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
-		nv_wr32(dev, 0x4091c0, 0x01000000);
-		for (i = 0; i < sizeof(nvc0_grhub_data) / 4; i++)
-			nv_wr32(dev, 0x4091c4, nvc0_grhub_data[i]);
-
-		nv_wr32(dev, 0x409180, 0x01000000);
-		for (i = 0; i < sizeof(nvc0_grhub_code) / 4; i++) {
-			if ((i & 0x3f) == 0)
-				nv_wr32(dev, 0x409188, i >> 6);
-			nv_wr32(dev, 0x409184, nvc0_grhub_code[i]);
-		}
-
-		/* load GPC microcode */
-		nv_wr32(dev, 0x41a1c0, 0x01000000);
-		for (i = 0; i < sizeof(nvc0_grgpc_data) / 4; i++)
-			nv_wr32(dev, 0x41a1c4, nvc0_grgpc_data[i]);
-
-		nv_wr32(dev, 0x41a180, 0x01000000);
-		for (i = 0; i < sizeof(nvc0_grgpc_code) / 4; i++) {
-			if ((i & 0x3f) == 0)
-				nv_wr32(dev, 0x41a188, i >> 6);
-			nv_wr32(dev, 0x41a184, nvc0_grgpc_code[i]);
-		}
-		nv_wr32(dev, 0x000260, r000260);
-
-		/* start HUB ucode running, it'll init the GPCs */
-		nv_wr32(dev, 0x409800, dev_priv->chipset);
-		nv_wr32(dev, 0x40910c, 0x00000000);
-		nv_wr32(dev, 0x409100, 0x00000002);
-		if (!nv_wait(dev, 0x409800, 0x80000000, 0x80000000)) {
-			NV_ERROR(dev, "PGRAPH: HUB_INIT timed out\n");
-			nvc0_graph_ctxctl_debug(dev);
-			return -EBUSY;
-		}
-
-		priv->grctx_size = nv_rd32(dev, 0x409804);
-		return 0;
-	}
-
-	/* load fuc microcode */
-	r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
-	nvc0_graph_init_fuc(dev, 0x409000, &priv->fuc409c, &priv->fuc409d);
-	nvc0_graph_init_fuc(dev, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
-	nv_wr32(dev, 0x000260, r000260);
-
-	/* start both of them running */
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x41a10c, 0x00000000);
-	nv_wr32(dev, 0x40910c, 0x00000000);
-	nv_wr32(dev, 0x41a100, 0x00000002);
-	nv_wr32(dev, 0x409100, 0x00000002);
-	if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
-		NV_INFO(dev, "0x409800 wait failed\n");
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x7fffffff);
-	nv_wr32(dev, 0x409504, 0x00000021);
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000010);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
-		return -EBUSY;
-	}
-	priv->grctx_size = nv_rd32(dev, 0x409800);
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000016);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000025);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-static int
-nvc0_graph_init(struct drm_device *dev, int engine)
-{
-	int ret;
-
-	nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
-
-	nvc0_graph_init_obj418880(dev);
-	nvc0_graph_init_regs(dev);
-	/*nvc0_graph_init_unitplemented_magics(dev);*/
-	nvc0_graph_init_gpc_0(dev);
-	/*nvc0_graph_init_unitplemented_c242(dev);*/
-
-	nv_wr32(dev, 0x400500, 0x00010001);
-	nv_wr32(dev, 0x400100, 0xffffffff);
-	nv_wr32(dev, 0x40013c, 0xffffffff);
-
-	nvc0_graph_init_units(dev);
-	nvc0_graph_init_gpc_1(dev);
-	nvc0_graph_init_rop(dev);
-
-	nv_wr32(dev, 0x400108, 0xffffffff);
-	nv_wr32(dev, 0x400138, 0xffffffff);
-	nv_wr32(dev, 0x400118, 0xffffffff);
-	nv_wr32(dev, 0x400130, 0xffffffff);
-	nv_wr32(dev, 0x40011c, 0xffffffff);
-	nv_wr32(dev, 0x400134, 0xffffffff);
-	nv_wr32(dev, 0x400054, 0x34ce3464);
-
-	ret = nvc0_graph_init_ctxctl(dev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-int
-nvc0_graph_isr_chid(struct drm_device *dev, u64 inst)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (i = 0; i < pfifo->channels; i++) {
-		chan = dev_priv->channels.ptr[i];
-		if (!chan || !chan->ramin)
-			continue;
-
-		if (inst == chan->ramin->vinst)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return i;
-}
-
-static void
-nvc0_graph_ctxctl_isr(struct drm_device *dev)
-{
-	u32 ustat = nv_rd32(dev, 0x409c18);
-
-	if (ustat & 0x00000001)
-		NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
-	if (ustat & 0x00080000)
-		NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
-	if (ustat & ~0x00080001)
-		NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
-
-	nvc0_graph_ctxctl_debug(dev);
-	nv_wr32(dev, 0x409c20, ustat);
-}
-
-static void
-nvc0_graph_isr(struct drm_device *dev)
-{
-	u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
-	u32 chid = nvc0_graph_isr_chid(dev, inst);
-	u32 stat = nv_rd32(dev, 0x400100);
-	u32 addr = nv_rd32(dev, 0x400704);
-	u32 mthd = (addr & 0x00003ffc);
-	u32 subc = (addr & 0x00070000) >> 16;
-	u32 data = nv_rd32(dev, 0x400708);
-	u32 code = nv_rd32(dev, 0x400110);
-	u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
-
-	if (stat & 0x00000010) {
-		if (nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data)) {
-			NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] "
-				     "subc %d class 0x%04x mthd 0x%04x "
-				     "data 0x%08x\n",
-				chid, inst, subc, class, mthd, data);
-		}
-		nv_wr32(dev, 0x400100, 0x00000010);
-		stat &= ~0x00000010;
-	}
-
-	if (stat & 0x00000020) {
-		NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
-			     "class 0x%04x mthd 0x%04x data 0x%08x\n",
-			chid, inst, subc, class, mthd, data);
-		nv_wr32(dev, 0x400100, 0x00000020);
-		stat &= ~0x00000020;
-	}
-
-	if (stat & 0x00100000) {
-		NV_INFO(dev, "PGRAPH: DATA_ERROR [");
-		nouveau_enum_print(nv50_data_error_names, code);
-		printk("] ch %d [0x%010llx] subc %d class 0x%04x "
-		       "mthd 0x%04x data 0x%08x\n",
-		       chid, inst, subc, class, mthd, data);
-		nv_wr32(dev, 0x400100, 0x00100000);
-		stat &= ~0x00100000;
-	}
-
-	if (stat & 0x00200000) {
-		u32 trap = nv_rd32(dev, 0x400108);
-		NV_INFO(dev, "PGRAPH: TRAP ch %d status 0x%08x\n", chid, trap);
-		nv_wr32(dev, 0x400108, trap);
-		nv_wr32(dev, 0x400100, 0x00200000);
-		stat &= ~0x00200000;
-	}
-
-	if (stat & 0x00080000) {
-		nvc0_graph_ctxctl_isr(dev);
-		nv_wr32(dev, 0x400100, 0x00080000);
-		stat &= ~0x00080000;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
-		nv_wr32(dev, 0x400100, stat);
-	}
-
-	nv_wr32(dev, 0x400500, 0x00010001);
-}
-
-static int
-nvc0_graph_create_fw(struct drm_device *dev, const char *fwname,
-		     struct nvc0_graph_fuc *fuc)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	const struct firmware *fw;
-	char f[32];
-	int ret;
-
-	snprintf(f, sizeof(f), "nouveau/nv%02x_%s", dev_priv->chipset, fwname);
-	ret = request_firmware(&fw, f, &dev->pdev->dev);
-	if (ret) {
-		snprintf(f, sizeof(f), "nouveau/%s", fwname);
-		ret = request_firmware(&fw, f, &dev->pdev->dev);
-		if (ret) {
-			NV_ERROR(dev, "failed to load %s\n", fwname);
-			return ret;
-		}
-	}
-
-	fuc->size = fw->size;
-	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
-	release_firmware(fw);
-	return (fuc->data != NULL) ? 0 : -ENOMEM;
-}
-
-static void
-nvc0_graph_destroy_fw(struct nvc0_graph_fuc *fuc)
-{
-	if (fuc->data) {
-		kfree(fuc->data);
-		fuc->data = NULL;
-	}
-}
-
-static void
-nvc0_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nvc0_graph_priv *priv = nv_engine(dev, engine);
-
-	if (nouveau_ctxfw) {
-		nvc0_graph_destroy_fw(&priv->fuc409c);
-		nvc0_graph_destroy_fw(&priv->fuc409d);
-		nvc0_graph_destroy_fw(&priv->fuc41ac);
-		nvc0_graph_destroy_fw(&priv->fuc41ad);
-	}
-
-	nouveau_irq_unregister(dev, 12);
-
-	nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
-	nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
-
-	if (priv->grctx_vals)
-		kfree(priv->grctx_vals);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-	kfree(priv);
-}
-
-int
-nvc0_graph_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_graph_priv *priv;
-	int ret, gpc, i;
-	u32 fermi;
-
-	fermi = nvc0_graph_class(dev);
-	if (!fermi) {
-		NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
-		return 0;
-	}
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.destroy = nvc0_graph_destroy;
-	priv->base.init = nvc0_graph_init;
-	priv->base.fini = nvc0_graph_fini;
-	priv->base.context_new = nvc0_graph_context_new;
-	priv->base.context_del = nvc0_graph_context_del;
-	priv->base.object_new = nvc0_graph_object_new;
-
-	NVOBJ_ENGINE_ADD(dev, GR, &priv->base);
-	nouveau_irq_register(dev, 12, nvc0_graph_isr);
-
-	if (nouveau_ctxfw) {
-		NV_INFO(dev, "PGRAPH: using external firmware\n");
-		if (nvc0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
-		    nvc0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
-		    nvc0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
-		    nvc0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
-			ret = 0;
-			goto error;
-		}
-	}
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
-	if (ret)
-		goto error;
-
-	for (i = 0; i < 0x1000; i += 4) {
-		nv_wo32(priv->unk4188b4, i, 0x00000010);
-		nv_wo32(priv->unk4188b8, i, 0x00000010);
-	}
-
-	priv->gpc_nr  =  nv_rd32(dev, 0x409604) & 0x0000001f;
-	priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		priv->tp_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
-		priv->tp_total += priv->tp_nr[gpc];
-	}
-
-	/*XXX: these need figuring out... */
-	switch (dev_priv->chipset) {
-	case 0xc0:
-		if (priv->tp_total == 11) { /* 465, 3/4/4/0, 4 */
-			priv->magic_not_rop_nr = 0x07;
-		} else
-		if (priv->tp_total == 14) { /* 470, 3/3/4/4, 5 */
-			priv->magic_not_rop_nr = 0x05;
-		} else
-		if (priv->tp_total == 15) { /* 480, 3/4/4/4, 6 */
-			priv->magic_not_rop_nr = 0x06;
-		}
-		break;
-	case 0xc3: /* 450, 4/0/0/0, 2 */
-		priv->magic_not_rop_nr = 0x03;
-		break;
-	case 0xc4: /* 460, 3/4/0/0, 4 */
-		priv->magic_not_rop_nr = 0x01;
-		break;
-	case 0xc1: /* 2/0/0/0, 1 */
-		priv->magic_not_rop_nr = 0x01;
-		break;
-	case 0xc8: /* 4/4/3/4, 5 */
-		priv->magic_not_rop_nr = 0x06;
-		break;
-	case 0xce: /* 4/4/0/0, 4 */
-		priv->magic_not_rop_nr = 0x03;
-		break;
-	case 0xcf: /* 4/0/0/0, 3 */
-		priv->magic_not_rop_nr = 0x03;
-		break;
-	case 0xd9: /* 1/0/0/0, 1 */
-		priv->magic_not_rop_nr = 0x01;
-		break;
-	}
-
-	if (!priv->magic_not_rop_nr) {
-		NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
-			 priv->tp_nr[0], priv->tp_nr[1], priv->tp_nr[2],
-			 priv->tp_nr[3], priv->rop_nr);
-		priv->magic_not_rop_nr = 0x00;
-	}
-
-	NVOBJ_CLASS(dev, 0x902d, GR); /* 2D */
-	NVOBJ_CLASS(dev, 0x9039, GR); /* M2MF */
-	NVOBJ_CLASS(dev, 0x9097, GR); /* 3D */
-	if (fermi >= 0x9197)
-		NVOBJ_CLASS(dev, 0x9197, GR); /* 3D (NVC1-) */
-	if (fermi >= 0x9297)
-		NVOBJ_CLASS(dev, 0x9297, GR); /* 3D (NVC8-) */
-	NVOBJ_CLASS(dev, 0x90c0, GR); /* COMPUTE */
-	return 0;
-
-error:
-	nvc0_graph_destroy(dev, NVOBJ_ENGINE_GR);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_graph.h b/drivers/gpu/drm/nouveau/nvc0_graph.h
deleted file mode 100644
index 91d44ea662d..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_graph.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#ifndef __NVC0_GRAPH_H__
-#define __NVC0_GRAPH_H__
-
-#define GPC_MAX 4
-#define TP_MAX 32
-
-#define ROP_BCAST(r)     (0x408800 + (r))
-#define ROP_UNIT(u, r)   (0x410000 + (u) * 0x400 + (r))
-#define GPC_BCAST(r)     (0x418000 + (r))
-#define GPC_UNIT(t, r)   (0x500000 + (t) * 0x8000 + (r))
-#define TP_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
-
-struct nvc0_graph_fuc {
-	u32 *data;
-	u32  size;
-};
-
-struct nvc0_graph_priv {
-	struct nouveau_exec_engine base;
-
-	struct nvc0_graph_fuc fuc409c;
-	struct nvc0_graph_fuc fuc409d;
-	struct nvc0_graph_fuc fuc41ac;
-	struct nvc0_graph_fuc fuc41ad;
-
-	u8 gpc_nr;
-	u8 rop_nr;
-	u8 tp_nr[GPC_MAX];
-	u8 tp_total;
-
-	u32  grctx_size;
-	u32 *grctx_vals;
-	struct nouveau_gpuobj *unk4188b4;
-	struct nouveau_gpuobj *unk4188b8;
-
-	u8 magic_not_rop_nr;
-};
-
-struct nvc0_graph_chan {
-	struct nouveau_gpuobj *grctx;
-	struct nouveau_gpuobj *unk408004; /* 0x418810 too */
-	struct nouveau_gpuobj *unk40800c; /* 0x419004 too */
-	struct nouveau_gpuobj *unk418810; /* 0x419848 too */
-	struct nouveau_gpuobj *mmio;
-	int mmio_nr;
-};
-
-int nvc0_grctx_generate(struct nouveau_channel *);
-
-/* nvc0_graph.c uses this also to determine supported chipsets */
-static inline u32
-nvc0_graph_class(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	switch (dev_priv->chipset) {
-	case 0xc0:
-	case 0xc3:
-	case 0xc4:
-	case 0xce: /* guess, mmio trace shows only 0x9097 state */
-	case 0xcf: /* guess, mmio trace shows only 0x9097 state */
-		return 0x9097;
-	case 0xc1:
-		return 0x9197;
-	case 0xc8:
-	case 0xd9:
-		return 0x9297;
-	default:
-		return 0;
-	}
-}
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nvc0_grctx.c b/drivers/gpu/drm/nouveau/nvc0_grctx.c
deleted file mode 100644
index de77842b31c..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_grctx.c
+++ /dev/null
@@ -1,2878 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nvc0_graph.h"
-
-static void
-nv_icmd(struct drm_device *dev, u32 icmd, u32 data)
-{
-	nv_wr32(dev, 0x400204, data);
-	nv_wr32(dev, 0x400200, icmd);
-	while (nv_rd32(dev, 0x400700) & 2) {}
-}
-
-static void
-nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
-{
-	nv_wr32(dev, 0x40448c, data);
-	nv_wr32(dev, 0x404488, 0x80000000 | (mthd << 14) | class);
-}
-
-static void
-nvc0_grctx_generate_9097(struct drm_device *dev)
-{
-	u32 fermi = nvc0_graph_class(dev);
-	u32 mthd;
-
-	nv_mthd(dev, 0x9097, 0x0800, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0840, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0880, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0900, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0940, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0980, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0804, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0844, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0884, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0904, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0944, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0984, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0808, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x0848, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x0888, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x08c8, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x0908, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x0948, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x0988, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x09c8, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x080c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x084c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x088c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x08cc, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x090c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x094c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x098c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x09cc, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x0810, 0x000000cf);
-	nv_mthd(dev, 0x9097, 0x0850, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0890, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0910, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0950, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0990, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0814, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0854, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0894, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x08d4, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0914, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0954, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0994, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x09d4, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0818, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0858, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0898, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x08d8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0918, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0958, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0998, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x09d8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x081c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x085c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x089c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x091c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x095c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x099c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0820, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0860, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x08e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0920, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0960, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x09e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2700, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2720, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2740, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2760, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2780, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2704, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2724, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2744, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2764, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2784, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2708, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2728, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2748, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2768, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2788, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x270c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x272c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x274c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x276c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x278c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x27ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2710, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x2730, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x2750, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x2770, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x2790, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x27b0, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x27d0, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x27f0, 0x00014000);
-	nv_mthd(dev, 0x9097, 0x2714, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x2734, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x2754, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x2774, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x2794, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x27b4, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x27d4, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x27f4, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x1c00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ca0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ce0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cf0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ca4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cb4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cd4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ce4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cf4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c18, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c38, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c78, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c98, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ca8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cb8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cd8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ce8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cf8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c1c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c2c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c3c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c5c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c6c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c7c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1c9c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cbc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ccc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cdc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1cfc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1da0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1db0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1de0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1df0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1da4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1db4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dd4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1de4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1df4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d18, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d38, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d78, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d98, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1da8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1db8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dd8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1de8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1df8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d1c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d2c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d3c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d5c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d6c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d7c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1d9c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dbc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dcc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ddc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1dfc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f18, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f38, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f78, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f1c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f2c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f3c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f5c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f6c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f7c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f98, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fa0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fa8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fb8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fd8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fe0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fe8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ff0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ff8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1f9c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fa4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fb4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fbc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fcc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fd4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fdc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fe4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1fec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ff4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1ffc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2200, 0x00000022);
-	nv_mthd(dev, 0x9097, 0x2210, 0x00000022);
-	nv_mthd(dev, 0x9097, 0x2220, 0x00000022);
-	nv_mthd(dev, 0x9097, 0x2230, 0x00000022);
-	nv_mthd(dev, 0x9097, 0x2240, 0x00000022);
-	nv_mthd(dev, 0x9097, 0x2000, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2040, 0x00000011);
-	nv_mthd(dev, 0x9097, 0x2080, 0x00000020);
-	nv_mthd(dev, 0x9097, 0x20c0, 0x00000030);
-	nv_mthd(dev, 0x9097, 0x2100, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x2140, 0x00000051);
-	nv_mthd(dev, 0x9097, 0x200c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x204c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x208c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x20cc, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x210c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x214c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x2010, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2050, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2090, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x20d0, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x2110, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x2150, 0x00000004);
-	nv_mthd(dev, 0x9097, 0x0380, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0384, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0388, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x038c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x03ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0700, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0710, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0720, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0730, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0704, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0714, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0724, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0734, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0708, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0718, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0728, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0738, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2800, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2804, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2808, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x280c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2810, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2814, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2818, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x281c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2820, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2824, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2828, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x282c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2830, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2834, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2838, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x283c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2840, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2844, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2848, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x284c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2850, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2854, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2858, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x285c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2860, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2864, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2868, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x286c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2870, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2874, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2878, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x287c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2880, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2884, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2888, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x288c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2890, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2894, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2898, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x289c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28b0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28b4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28b8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28d4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28d8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28f0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28f4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28f8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x28fc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2900, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2904, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2908, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x290c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2910, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2914, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2918, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x291c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2920, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2924, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2928, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x292c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2930, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2934, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2938, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x293c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2940, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2944, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2948, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x294c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2950, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2954, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2958, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x295c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2960, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2964, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2968, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x296c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2970, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2974, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2978, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x297c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2980, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2984, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2988, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x298c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2990, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2994, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2998, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x299c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29b0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29b4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29b8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29d4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29d8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29f0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29f4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29f8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x29fc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0aa0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ac0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ae0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ba0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0be0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0aa4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ac4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ae4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ba4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0be4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0aa8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ac8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ae8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ba8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0be8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a2c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a6c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0aac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0acc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0aec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b2c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b6c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bcc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ab0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ad0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0af0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bf0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0a94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ab4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ad4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0af4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0b94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bb4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bd4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0bf4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ca0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ce0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cf0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c24, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c34, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c64, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c94, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ca4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cb4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cd4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ce4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cf4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c18, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c28, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c38, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c68, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c78, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c98, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ca8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cb8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cd8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ce8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0cf8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0c0c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c1c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c2c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c3c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c4c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c5c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c6c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c7c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c8c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0c9c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0cac, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0cbc, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0ccc, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0cdc, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0cec, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0cfc, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0d00, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d08, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d10, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d18, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d20, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d28, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d30, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d38, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d04, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d0c, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d14, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d1c, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d24, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d2c, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d34, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d3c, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e00, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e20, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e30, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e60, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e70, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ea0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0eb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ec0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ed0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ee0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ef0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0e04, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e14, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e24, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e34, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e44, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e54, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e64, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e74, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e84, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e94, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ea4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0eb4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ec4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ed4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ee4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ef4, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e08, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e18, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e28, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e38, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e48, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e58, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e68, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e78, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e88, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0e98, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ea8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0eb8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ec8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ed8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ee8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0ef8, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d40, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d48, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d50, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d44, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d4c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d5c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1e00, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e20, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e40, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e60, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e80, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ea0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ec0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ee0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e04, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e24, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e44, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e64, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e84, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ea4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ec4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ee4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e08, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e28, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e48, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e68, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e88, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1ea8, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1ec8, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1ee8, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e0c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e2c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e4c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e6c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e8c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1eac, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ecc, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1eec, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e10, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e30, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e50, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e70, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e90, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1eb0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ed0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ef0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e14, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e34, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e54, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e74, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e94, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1eb4, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1ed4, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1ef4, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1e18, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e38, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e58, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e78, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1e98, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1eb8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ed8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1ef8, 0x00000001);
-	if (fermi == 0x9097) {
-		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
-			nv_mthd(dev, 0x9097, mthd, 0x00000000);
-	}
-	nv_mthd(dev, 0x9097, 0x030c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1944, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1514, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d68, 0x0000ffff);
-	nv_mthd(dev, 0x9097, 0x121c, 0x0fac6881);
-	nv_mthd(dev, 0x9097, 0x0fac, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1538, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0fe0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fe4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fe8, 0x00000014);
-	nv_mthd(dev, 0x9097, 0x0fec, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x0ff0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x179c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1228, 0x00000400);
-	nv_mthd(dev, 0x9097, 0x122c, 0x00000300);
-	nv_mthd(dev, 0x9097, 0x1230, 0x00010001);
-	nv_mthd(dev, 0x9097, 0x07f8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15b4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x15cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1534, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fb0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x153c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x16b4, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x0fbc, 0x0000ffff);
-	nv_mthd(dev, 0x9097, 0x0fc0, 0x0000ffff);
-	nv_mthd(dev, 0x9097, 0x0fc4, 0x0000ffff);
-	nv_mthd(dev, 0x9097, 0x0fc8, 0x0000ffff);
-	nv_mthd(dev, 0x9097, 0x0df8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0dfc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1948, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1970, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x161c, 0x000009f0);
-	nv_mthd(dev, 0x9097, 0x0dcc, 0x00000010);
-	nv_mthd(dev, 0x9097, 0x163c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1160, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1164, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1168, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x116c, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1170, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1174, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1178, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x117c, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1180, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1184, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1188, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x118c, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1190, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1194, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1198, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x119c, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11a0, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11a4, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11a8, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11ac, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11b0, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11b4, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11b8, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11bc, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11c0, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11c4, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11c8, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11cc, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11d0, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11d4, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11d8, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x11dc, 0x25e00040);
-	nv_mthd(dev, 0x9097, 0x1880, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1884, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1888, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x188c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1890, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1894, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1898, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x189c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18b0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18b4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18b8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18d0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18d4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18d8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18e0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18f0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18f4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18f8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x18fc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x17c8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x17cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x17d0, 0x000000ff);
-	nv_mthd(dev, 0x9097, 0x17d4, 0xffffffff);
-	nv_mthd(dev, 0x9097, 0x17d8, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x17dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15f4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15f8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1434, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1438, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d74, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0dec, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x13a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1318, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1644, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0748, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0de8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1648, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12a4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1120, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1124, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1128, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x112c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1118, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x164c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1658, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1910, 0x00000290);
-	nv_mthd(dev, 0x9097, 0x1518, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x165c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1520, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1604, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1570, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x13b0, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x13b4, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x020c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1670, 0x30201000);
-	nv_mthd(dev, 0x9097, 0x1674, 0x70605040);
-	nv_mthd(dev, 0x9097, 0x1678, 0xb8a89888);
-	nv_mthd(dev, 0x9097, 0x167c, 0xf8e8d8c8);
-	nv_mthd(dev, 0x9097, 0x166c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1680, 0x00ffff00);
-	nv_mthd(dev, 0x9097, 0x12d0, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x12d4, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1684, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1688, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0dac, 0x00001b02);
-	nv_mthd(dev, 0x9097, 0x0db0, 0x00001b02);
-	nv_mthd(dev, 0x9097, 0x0db4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x168c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x15bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x156c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x187c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1110, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0dc0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0dc4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0dc8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1234, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1690, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12ac, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x02c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0790, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0794, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0798, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x079c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x07a0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x077c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1000, 0x00000010);
-	nv_mthd(dev, 0x9097, 0x10fc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1290, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0218, 0x00000010);
-	nv_mthd(dev, 0x9097, 0x12d8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12dc, 0x00000010);
-	nv_mthd(dev, 0x9097, 0x0d94, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x155c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1560, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1564, 0x00001fff);
-	nv_mthd(dev, 0x9097, 0x1574, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1578, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x157c, 0x003fffff);
-	nv_mthd(dev, 0x9097, 0x1354, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1664, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1610, 0x00000012);
-	nv_mthd(dev, 0x9097, 0x1608, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x160c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x162c, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x0210, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0320, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0324, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0328, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x032c, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0330, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0334, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0338, 0x3f800000);
-	nv_mthd(dev, 0x9097, 0x0750, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0760, 0x39291909);
-	nv_mthd(dev, 0x9097, 0x0764, 0x79695949);
-	nv_mthd(dev, 0x9097, 0x0768, 0xb9a99989);
-	nv_mthd(dev, 0x9097, 0x076c, 0xf9e9d9c9);
-	nv_mthd(dev, 0x9097, 0x0770, 0x30201000);
-	nv_mthd(dev, 0x9097, 0x0774, 0x70605040);
-	nv_mthd(dev, 0x9097, 0x0778, 0x00009080);
-	nv_mthd(dev, 0x9097, 0x0780, 0x39291909);
-	nv_mthd(dev, 0x9097, 0x0784, 0x79695949);
-	nv_mthd(dev, 0x9097, 0x0788, 0xb9a99989);
-	nv_mthd(dev, 0x9097, 0x078c, 0xf9e9d9c9);
-	nv_mthd(dev, 0x9097, 0x07d0, 0x30201000);
-	nv_mthd(dev, 0x9097, 0x07d4, 0x70605040);
-	nv_mthd(dev, 0x9097, 0x07d8, 0x00009080);
-	nv_mthd(dev, 0x9097, 0x037c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0740, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0744, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x2600, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1918, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x191c, 0x00000900);
-	nv_mthd(dev, 0x9097, 0x1920, 0x00000405);
-	nv_mthd(dev, 0x9097, 0x1308, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1924, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x13ac, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x192c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x193c, 0x00002c1c);
-	nv_mthd(dev, 0x9097, 0x0d7c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x02c0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1510, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1940, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ff4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0ff8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x194c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1950, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1968, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1590, 0x0000003f);
-	nv_mthd(dev, 0x9097, 0x07e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x07ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x07f0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x07f4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x196c, 0x00000011);
-	nv_mthd(dev, 0x9097, 0x197c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fcc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fd0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x02d8, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x1980, 0x00000080);
-	nv_mthd(dev, 0x9097, 0x1504, 0x00000080);
-	nv_mthd(dev, 0x9097, 0x1984, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0300, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x13a8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12ec, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1310, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1314, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1380, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1384, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1388, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x138c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1390, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1394, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x139c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1398, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1594, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1598, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x159c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x15a0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x15a4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x0f54, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f58, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f5c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x19bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f9c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0fa0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12cc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x12e8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x130c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1360, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1364, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1368, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x136c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1370, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1374, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1378, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x137c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x133c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1340, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1344, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1348, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x134c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1350, 0x00000002);
-	nv_mthd(dev, 0x9097, 0x1358, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x12e4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x131c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1320, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1324, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1328, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x19c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1140, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x19c4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x19c8, 0x00001500);
-	nv_mthd(dev, 0x9097, 0x135c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x19e0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19e4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19e8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19ec, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19f0, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19f4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19f8, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19fc, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x19cc, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x15b8, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a00, 0x00001111);
-	nv_mthd(dev, 0x9097, 0x1a04, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a08, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a0c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a10, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a14, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a18, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1a1c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d6c, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x0d70, 0xffff0000);
-	nv_mthd(dev, 0x9097, 0x10f8, 0x00001010);
-	nv_mthd(dev, 0x9097, 0x0d80, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d84, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d88, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d8c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0d90, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0da0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1508, 0x80000000);
-	nv_mthd(dev, 0x9097, 0x150c, 0x40000000);
-	nv_mthd(dev, 0x9097, 0x1668, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0318, 0x00000008);
-	nv_mthd(dev, 0x9097, 0x031c, 0x00000008);
-	nv_mthd(dev, 0x9097, 0x0d9c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x07dc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x074c, 0x00000055);
-	nv_mthd(dev, 0x9097, 0x1420, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x17bc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x17c0, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x17c4, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1008, 0x00000008);
-	nv_mthd(dev, 0x9097, 0x100c, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x1010, 0x0000012c);
-	nv_mthd(dev, 0x9097, 0x0d60, 0x00000040);
-	nv_mthd(dev, 0x9097, 0x075c, 0x00000003);
-	nv_mthd(dev, 0x9097, 0x1018, 0x00000020);
-	nv_mthd(dev, 0x9097, 0x101c, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1020, 0x00000020);
-	nv_mthd(dev, 0x9097, 0x1024, 0x00000001);
-	nv_mthd(dev, 0x9097, 0x1444, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x1448, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x144c, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0360, 0x20164010);
-	nv_mthd(dev, 0x9097, 0x0364, 0x00000020);
-	nv_mthd(dev, 0x9097, 0x0368, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0de4, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0204, 0x00000006);
-	nv_mthd(dev, 0x9097, 0x0208, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x02cc, 0x003fffff);
-	nv_mthd(dev, 0x9097, 0x02d0, 0x00000c48);
-	nv_mthd(dev, 0x9097, 0x1220, 0x00000005);
-	nv_mthd(dev, 0x9097, 0x0fdc, 0x00000000);
-	nv_mthd(dev, 0x9097, 0x0f98, 0x00300008);
-	nv_mthd(dev, 0x9097, 0x1284, 0x04000080);
-	nv_mthd(dev, 0x9097, 0x1450, 0x00300008);
-	nv_mthd(dev, 0x9097, 0x1454, 0x04000080);
-	nv_mthd(dev, 0x9097, 0x0214, 0x00000000);
-	/* in trace, right after 0x90c0, not here */
-	nv_mthd(dev, 0x9097, 0x3410, 0x80002006);
-}
-
-static void
-nvc0_grctx_generate_9197(struct drm_device *dev)
-{
-	u32 fermi = nvc0_graph_class(dev);
-	u32 mthd;
-
-	if (fermi == 0x9197) {
-		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
-			nv_mthd(dev, 0x9197, mthd, 0x00000000);
-	}
-	nv_mthd(dev, 0x9197, 0x02e4, 0x0000b001);
-}
-
-static void
-nvc0_grctx_generate_9297(struct drm_device *dev)
-{
-	u32 fermi = nvc0_graph_class(dev);
-	u32 mthd;
-
-	if (fermi == 0x9297) {
-		for (mthd = 0x3400; mthd <= 0x35fc; mthd += 4)
-			nv_mthd(dev, 0x9297, mthd, 0x00000000);
-	}
-	nv_mthd(dev, 0x9297, 0x036c, 0x00000000);
-	nv_mthd(dev, 0x9297, 0x0370, 0x00000000);
-	nv_mthd(dev, 0x9297, 0x07a4, 0x00000000);
-	nv_mthd(dev, 0x9297, 0x07a8, 0x00000000);
-	nv_mthd(dev, 0x9297, 0x0374, 0x00000000);
-	nv_mthd(dev, 0x9297, 0x0378, 0x00000020);
-}
-
-static void
-nvc0_grctx_generate_902d(struct drm_device *dev)
-{
-	nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
-	nv_mthd(dev, 0x902d, 0x0204, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0208, 0x00000020);
-	nv_mthd(dev, 0x902d, 0x020c, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0210, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0214, 0x00000080);
-	nv_mthd(dev, 0x902d, 0x0218, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x021c, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x0220, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0224, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0230, 0x000000cf);
-	nv_mthd(dev, 0x902d, 0x0234, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0238, 0x00000020);
-	nv_mthd(dev, 0x902d, 0x023c, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0244, 0x00000080);
-	nv_mthd(dev, 0x902d, 0x0248, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x024c, 0x00000100);
-}
-
-static void
-nvc0_grctx_generate_9039(struct drm_device *dev)
-{
-	nv_mthd(dev, 0x9039, 0x030c, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x0310, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x0314, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x0320, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x0238, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x023c, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x0318, 0x00000000);
-	nv_mthd(dev, 0x9039, 0x031c, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_90c0(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int i;
-
-	for (i = 0; dev_priv->chipset == 0xd9 && i < 4; i++) {
-		nv_mthd(dev, 0x90c0, 0x2700 + (i * 0x40), 0x00000000);
-		nv_mthd(dev, 0x90c0, 0x2720 + (i * 0x40), 0x00000000);
-		nv_mthd(dev, 0x90c0, 0x2704 + (i * 0x40), 0x00000000);
-		nv_mthd(dev, 0x90c0, 0x2724 + (i * 0x40), 0x00000000);
-		nv_mthd(dev, 0x90c0, 0x2708 + (i * 0x40), 0x00000000);
-		nv_mthd(dev, 0x90c0, 0x2728 + (i * 0x40), 0x00000000);
-	}
-	nv_mthd(dev, 0x90c0, 0x270c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x272c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x274c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x276c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x278c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x27ac, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x27cc, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x27ec, 0x00000000);
-	for (i = 0; dev_priv->chipset == 0xd9 && i < 4; i++) {
-		nv_mthd(dev, 0x90c0, 0x2710 + (i * 0x40), 0x00014000);
-		nv_mthd(dev, 0x90c0, 0x2730 + (i * 0x40), 0x00014000);
-		nv_mthd(dev, 0x90c0, 0x2714 + (i * 0x40), 0x00000040);
-		nv_mthd(dev, 0x90c0, 0x2734 + (i * 0x40), 0x00000040);
-	}
-	nv_mthd(dev, 0x90c0, 0x030c, 0x00000001);
-	nv_mthd(dev, 0x90c0, 0x1944, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0758, 0x00000100);
-	nv_mthd(dev, 0x90c0, 0x02c4, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0790, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0794, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0798, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x079c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x07a0, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x077c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0204, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0208, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x020c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0214, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x024c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x0d94, 0x00000001);
-	nv_mthd(dev, 0x90c0, 0x1608, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x160c, 0x00000000);
-	nv_mthd(dev, 0x90c0, 0x1664, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_dispatch(struct drm_device *dev)
-{
-	int i;
-
-	nv_wr32(dev, 0x404004, 0x00000000);
-	nv_wr32(dev, 0x404008, 0x00000000);
-	nv_wr32(dev, 0x40400c, 0x00000000);
-	nv_wr32(dev, 0x404010, 0x00000000);
-	nv_wr32(dev, 0x404014, 0x00000000);
-	nv_wr32(dev, 0x404018, 0x00000000);
-	nv_wr32(dev, 0x40401c, 0x00000000);
-	nv_wr32(dev, 0x404020, 0x00000000);
-	nv_wr32(dev, 0x404024, 0x00000000);
-	nv_wr32(dev, 0x404028, 0x00000000);
-	nv_wr32(dev, 0x40402c, 0x00000000);
-	nv_wr32(dev, 0x404044, 0x00000000);
-	nv_wr32(dev, 0x404094, 0x00000000);
-	nv_wr32(dev, 0x404098, 0x00000000);
-	nv_wr32(dev, 0x40409c, 0x00000000);
-	nv_wr32(dev, 0x4040a0, 0x00000000);
-	nv_wr32(dev, 0x4040a4, 0x00000000);
-	nv_wr32(dev, 0x4040a8, 0x00000000);
-	nv_wr32(dev, 0x4040ac, 0x00000000);
-	nv_wr32(dev, 0x4040b0, 0x00000000);
-	nv_wr32(dev, 0x4040b4, 0x00000000);
-	nv_wr32(dev, 0x4040b8, 0x00000000);
-	nv_wr32(dev, 0x4040bc, 0x00000000);
-	nv_wr32(dev, 0x4040c0, 0x00000000);
-	nv_wr32(dev, 0x4040c4, 0x00000000);
-	nv_wr32(dev, 0x4040c8, 0xf0000087);
-	nv_wr32(dev, 0x4040d4, 0x00000000);
-	nv_wr32(dev, 0x4040d8, 0x00000000);
-	nv_wr32(dev, 0x4040dc, 0x00000000);
-	nv_wr32(dev, 0x4040e0, 0x00000000);
-	nv_wr32(dev, 0x4040e4, 0x00000000);
-	nv_wr32(dev, 0x4040e8, 0x00001000);
-	nv_wr32(dev, 0x4040f8, 0x00000000);
-	nv_wr32(dev, 0x404130, 0x00000000);
-	nv_wr32(dev, 0x404134, 0x00000000);
-	nv_wr32(dev, 0x404138, 0x20000040);
-	nv_wr32(dev, 0x404150, 0x0000002e);
-	nv_wr32(dev, 0x404154, 0x00000400);
-	nv_wr32(dev, 0x404158, 0x00000200);
-	nv_wr32(dev, 0x404164, 0x00000055);
-	nv_wr32(dev, 0x404168, 0x00000000);
-	nv_wr32(dev, 0x404174, 0x00000000);
-	nv_wr32(dev, 0x404178, 0x00000000);
-	nv_wr32(dev, 0x40417c, 0x00000000);
-	for (i = 0; i < 8; i++)
-		nv_wr32(dev, 0x404200 + (i * 4), 0x00000000); /* subc */
-}
-
-static void
-nvc0_grctx_generate_macro(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404404, 0x00000000);
-	nv_wr32(dev, 0x404408, 0x00000000);
-	nv_wr32(dev, 0x40440c, 0x00000000);
-	nv_wr32(dev, 0x404410, 0x00000000);
-	nv_wr32(dev, 0x404414, 0x00000000);
-	nv_wr32(dev, 0x404418, 0x00000000);
-	nv_wr32(dev, 0x40441c, 0x00000000);
-	nv_wr32(dev, 0x404420, 0x00000000);
-	nv_wr32(dev, 0x404424, 0x00000000);
-	nv_wr32(dev, 0x404428, 0x00000000);
-	nv_wr32(dev, 0x40442c, 0x00000000);
-	nv_wr32(dev, 0x404430, 0x00000000);
-	nv_wr32(dev, 0x404434, 0x00000000);
-	nv_wr32(dev, 0x404438, 0x00000000);
-	nv_wr32(dev, 0x404460, 0x00000000);
-	nv_wr32(dev, 0x404464, 0x00000000);
-	nv_wr32(dev, 0x404468, 0x00ffffff);
-	nv_wr32(dev, 0x40446c, 0x00000000);
-	nv_wr32(dev, 0x404480, 0x00000001);
-	nv_wr32(dev, 0x404498, 0x00000001);
-}
-
-static void
-nvc0_grctx_generate_m2mf(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404604, 0x00000015);
-	nv_wr32(dev, 0x404608, 0x00000000);
-	nv_wr32(dev, 0x40460c, 0x00002e00);
-	nv_wr32(dev, 0x404610, 0x00000100);
-	nv_wr32(dev, 0x404618, 0x00000000);
-	nv_wr32(dev, 0x40461c, 0x00000000);
-	nv_wr32(dev, 0x404620, 0x00000000);
-	nv_wr32(dev, 0x404624, 0x00000000);
-	nv_wr32(dev, 0x404628, 0x00000000);
-	nv_wr32(dev, 0x40462c, 0x00000000);
-	nv_wr32(dev, 0x404630, 0x00000000);
-	nv_wr32(dev, 0x404634, 0x00000000);
-	nv_wr32(dev, 0x404638, 0x00000004);
-	nv_wr32(dev, 0x40463c, 0x00000000);
-	nv_wr32(dev, 0x404640, 0x00000000);
-	nv_wr32(dev, 0x404644, 0x00000000);
-	nv_wr32(dev, 0x404648, 0x00000000);
-	nv_wr32(dev, 0x40464c, 0x00000000);
-	nv_wr32(dev, 0x404650, 0x00000000);
-	nv_wr32(dev, 0x404654, 0x00000000);
-	nv_wr32(dev, 0x404658, 0x00000000);
-	nv_wr32(dev, 0x40465c, 0x007f0100);
-	nv_wr32(dev, 0x404660, 0x00000000);
-	nv_wr32(dev, 0x404664, 0x00000000);
-	nv_wr32(dev, 0x404668, 0x00000000);
-	nv_wr32(dev, 0x40466c, 0x00000000);
-	nv_wr32(dev, 0x404670, 0x00000000);
-	nv_wr32(dev, 0x404674, 0x00000000);
-	nv_wr32(dev, 0x404678, 0x00000000);
-	nv_wr32(dev, 0x40467c, 0x00000002);
-	nv_wr32(dev, 0x404680, 0x00000000);
-	nv_wr32(dev, 0x404684, 0x00000000);
-	nv_wr32(dev, 0x404688, 0x00000000);
-	nv_wr32(dev, 0x40468c, 0x00000000);
-	nv_wr32(dev, 0x404690, 0x00000000);
-	nv_wr32(dev, 0x404694, 0x00000000);
-	nv_wr32(dev, 0x404698, 0x00000000);
-	nv_wr32(dev, 0x40469c, 0x00000000);
-	nv_wr32(dev, 0x4046a0, 0x007f0080);
-	nv_wr32(dev, 0x4046a4, 0x00000000);
-	nv_wr32(dev, 0x4046a8, 0x00000000);
-	nv_wr32(dev, 0x4046ac, 0x00000000);
-	nv_wr32(dev, 0x4046b0, 0x00000000);
-	nv_wr32(dev, 0x4046b4, 0x00000000);
-	nv_wr32(dev, 0x4046b8, 0x00000000);
-	nv_wr32(dev, 0x4046bc, 0x00000000);
-	nv_wr32(dev, 0x4046c0, 0x00000000);
-	nv_wr32(dev, 0x4046c4, 0x00000000);
-	nv_wr32(dev, 0x4046c8, 0x00000000);
-	nv_wr32(dev, 0x4046cc, 0x00000000);
-	nv_wr32(dev, 0x4046d0, 0x00000000);
-	nv_wr32(dev, 0x4046d4, 0x00000000);
-	nv_wr32(dev, 0x4046d8, 0x00000000);
-	nv_wr32(dev, 0x4046dc, 0x00000000);
-	nv_wr32(dev, 0x4046e0, 0x00000000);
-	nv_wr32(dev, 0x4046e4, 0x00000000);
-	nv_wr32(dev, 0x4046e8, 0x00000000);
-	nv_wr32(dev, 0x4046f0, 0x00000000);
-	nv_wr32(dev, 0x4046f4, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_unk47xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404700, 0x00000000);
-	nv_wr32(dev, 0x404704, 0x00000000);
-	nv_wr32(dev, 0x404708, 0x00000000);
-	nv_wr32(dev, 0x40470c, 0x00000000);
-	nv_wr32(dev, 0x404710, 0x00000000);
-	nv_wr32(dev, 0x404714, 0x00000000);
-	nv_wr32(dev, 0x404718, 0x00000000);
-	nv_wr32(dev, 0x40471c, 0x00000000);
-	nv_wr32(dev, 0x404720, 0x00000000);
-	nv_wr32(dev, 0x404724, 0x00000000);
-	nv_wr32(dev, 0x404728, 0x00000000);
-	nv_wr32(dev, 0x40472c, 0x00000000);
-	nv_wr32(dev, 0x404730, 0x00000000);
-	nv_wr32(dev, 0x404734, 0x00000100);
-	nv_wr32(dev, 0x404738, 0x00000000);
-	nv_wr32(dev, 0x40473c, 0x00000000);
-	nv_wr32(dev, 0x404740, 0x00000000);
-	nv_wr32(dev, 0x404744, 0x00000000);
-	nv_wr32(dev, 0x404748, 0x00000000);
-	nv_wr32(dev, 0x40474c, 0x00000000);
-	nv_wr32(dev, 0x404750, 0x00000000);
-	nv_wr32(dev, 0x404754, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_shaders(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->chipset == 0xd9) {
-		nv_wr32(dev, 0x405800, 0x0f8000bf);
-		nv_wr32(dev, 0x405830, 0x02180218);
-		nv_wr32(dev, 0x405834, 0x08000000);
-	} else
-	if (dev_priv->chipset == 0xc1) {
-		nv_wr32(dev, 0x405800, 0x0f8000bf);
-		nv_wr32(dev, 0x405830, 0x02180218);
-		nv_wr32(dev, 0x405834, 0x00000000);
-	} else {
-		nv_wr32(dev, 0x405800, 0x078000bf);
-		nv_wr32(dev, 0x405830, 0x02180000);
-		nv_wr32(dev, 0x405834, 0x00000000);
-	}
-	nv_wr32(dev, 0x405838, 0x00000000);
-	nv_wr32(dev, 0x405854, 0x00000000);
-	nv_wr32(dev, 0x405870, 0x00000001);
-	nv_wr32(dev, 0x405874, 0x00000001);
-	nv_wr32(dev, 0x405878, 0x00000001);
-	nv_wr32(dev, 0x40587c, 0x00000001);
-	nv_wr32(dev, 0x405a00, 0x00000000);
-	nv_wr32(dev, 0x405a04, 0x00000000);
-	nv_wr32(dev, 0x405a18, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_unk60xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x406020, 0x000103c1);
-	nv_wr32(dev, 0x406028, 0x00000001);
-	nv_wr32(dev, 0x40602c, 0x00000001);
-	nv_wr32(dev, 0x406030, 0x00000001);
-	nv_wr32(dev, 0x406034, 0x00000001);
-}
-
-static void
-nvc0_grctx_generate_unk64xx(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	nv_wr32(dev, 0x4064a8, 0x00000000);
-	nv_wr32(dev, 0x4064ac, 0x00003fff);
-	nv_wr32(dev, 0x4064b4, 0x00000000);
-	nv_wr32(dev, 0x4064b8, 0x00000000);
-	if (dev_priv->chipset == 0xd9)
-		nv_wr32(dev, 0x4064bc, 0x00000000);
-	if (dev_priv->chipset == 0xc1 ||
-	    dev_priv->chipset == 0xd9) {
-		nv_wr32(dev, 0x4064c0, 0x80140078);
-		nv_wr32(dev, 0x4064c4, 0x0086ffff);
-	}
-}
-
-static void
-nvc0_grctx_generate_tpbus(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x407804, 0x00000023);
-	nv_wr32(dev, 0x40780c, 0x0a418820);
-	nv_wr32(dev, 0x407810, 0x062080e6);
-	nv_wr32(dev, 0x407814, 0x020398a4);
-	nv_wr32(dev, 0x407818, 0x0e629062);
-	nv_wr32(dev, 0x40781c, 0x0a418820);
-	nv_wr32(dev, 0x407820, 0x000000e6);
-	nv_wr32(dev, 0x4078bc, 0x00000103);
-}
-
-static void
-nvc0_grctx_generate_ccache(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x408000, 0x00000000);
-	nv_wr32(dev, 0x408004, 0x00000000);
-	nv_wr32(dev, 0x408008, 0x00000018);
-	nv_wr32(dev, 0x40800c, 0x00000000);
-	nv_wr32(dev, 0x408010, 0x00000000);
-	nv_wr32(dev, 0x408014, 0x00000069);
-	nv_wr32(dev, 0x408018, 0xe100e100);
-	nv_wr32(dev, 0x408064, 0x00000000);
-}
-
-static void
-nvc0_grctx_generate_rop(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chipset = dev_priv->chipset;
-
-	/* ROPC_BROADCAST */
-	nv_wr32(dev, 0x408800, 0x02802a3c);
-	nv_wr32(dev, 0x408804, 0x00000040);
-	if (chipset == 0xd9) {
-		nv_wr32(dev, 0x408808, 0x1043e005);
-		nv_wr32(dev, 0x408900, 0x3080b801);
-		nv_wr32(dev, 0x408904, 0x1043e005);
-		nv_wr32(dev, 0x408908, 0x00c8102f);
-	} else
-	if (chipset == 0xc1) {
-		nv_wr32(dev, 0x408808, 0x1003e005);
-		nv_wr32(dev, 0x408900, 0x3080b801);
-		nv_wr32(dev, 0x408904, 0x62000001);
-		nv_wr32(dev, 0x408908, 0x00c80929);
-	} else {
-		nv_wr32(dev, 0x408808, 0x0003e00d);
-		nv_wr32(dev, 0x408900, 0x3080b801);
-		nv_wr32(dev, 0x408904, 0x02000001);
-		nv_wr32(dev, 0x408908, 0x00c80929);
-	}
-	nv_wr32(dev, 0x40890c, 0x00000000);
-	nv_wr32(dev, 0x408980, 0x0000011d);
-}
-
-static void
-nvc0_grctx_generate_gpc(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chipset = dev_priv->chipset;
-	int i;
-
-	/* GPC_BROADCAST */
-	nv_wr32(dev, 0x418380, 0x00000016);
-	nv_wr32(dev, 0x418400, 0x38004e00);
-	nv_wr32(dev, 0x418404, 0x71e0ffff);
-	nv_wr32(dev, 0x418408, 0x00000000);
-	nv_wr32(dev, 0x41840c, 0x00001008);
-	nv_wr32(dev, 0x418410, 0x0fff0fff);
-	nv_wr32(dev, 0x418414, chipset != 0xd9 ? 0x00200fff : 0x02200fff);
-	nv_wr32(dev, 0x418450, 0x00000000);
-	nv_wr32(dev, 0x418454, 0x00000000);
-	nv_wr32(dev, 0x418458, 0x00000000);
-	nv_wr32(dev, 0x41845c, 0x00000000);
-	nv_wr32(dev, 0x418460, 0x00000000);
-	nv_wr32(dev, 0x418464, 0x00000000);
-	nv_wr32(dev, 0x418468, 0x00000001);
-	nv_wr32(dev, 0x41846c, 0x00000000);
-	nv_wr32(dev, 0x418470, 0x00000000);
-	nv_wr32(dev, 0x418600, 0x0000001f);
-	nv_wr32(dev, 0x418684, 0x0000000f);
-	nv_wr32(dev, 0x418700, 0x00000002);
-	nv_wr32(dev, 0x418704, 0x00000080);
-	nv_wr32(dev, 0x418708, 0x00000000);
-	nv_wr32(dev, 0x41870c, chipset != 0xd9 ? 0x07c80000 : 0x00000000);
-	nv_wr32(dev, 0x418710, 0x00000000);
-	nv_wr32(dev, 0x418800, chipset != 0xd9 ? 0x0006860a : 0x7006860a);
-	nv_wr32(dev, 0x418808, 0x00000000);
-	nv_wr32(dev, 0x41880c, 0x00000000);
-	nv_wr32(dev, 0x418810, 0x00000000);
-	nv_wr32(dev, 0x418828, 0x00008442);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x418830, 0x10000001);
-	else
-		nv_wr32(dev, 0x418830, 0x00000001);
-	nv_wr32(dev, 0x4188d8, 0x00000008);
-	nv_wr32(dev, 0x4188e0, 0x01000000);
-	nv_wr32(dev, 0x4188e8, 0x00000000);
-	nv_wr32(dev, 0x4188ec, 0x00000000);
-	nv_wr32(dev, 0x4188f0, 0x00000000);
-	nv_wr32(dev, 0x4188f4, 0x00000000);
-	nv_wr32(dev, 0x4188f8, 0x00000000);
-	if (chipset == 0xd9)
-		nv_wr32(dev, 0x4188fc, 0x20100008);
-	else if (chipset == 0xc1)
-		nv_wr32(dev, 0x4188fc, 0x00100018);
-	else
-		nv_wr32(dev, 0x4188fc, 0x00100000);
-	nv_wr32(dev, 0x41891c, 0x00ff00ff);
-	nv_wr32(dev, 0x418924, 0x00000000);
-	nv_wr32(dev, 0x418928, 0x00ffff00);
-	nv_wr32(dev, 0x41892c, 0x0000ff00);
-	for (i = 0; i < 8; i++) {
-		nv_wr32(dev, 0x418a00 + (i * 0x20), 0x00000000);
-		nv_wr32(dev, 0x418a04 + (i * 0x20), 0x00000000);
-		nv_wr32(dev, 0x418a08 + (i * 0x20), 0x00000000);
-		nv_wr32(dev, 0x418a0c + (i * 0x20), 0x00010000);
-		nv_wr32(dev, 0x418a10 + (i * 0x20), 0x00000000);
-		nv_wr32(dev, 0x418a14 + (i * 0x20), 0x00000000);
-		nv_wr32(dev, 0x418a18 + (i * 0x20), 0x00000000);
-	}
-	nv_wr32(dev, 0x418b00, chipset != 0xd9 ? 0x00000000 : 0x00000006);
-	nv_wr32(dev, 0x418b08, 0x0a418820);
-	nv_wr32(dev, 0x418b0c, 0x062080e6);
-	nv_wr32(dev, 0x418b10, 0x020398a4);
-	nv_wr32(dev, 0x418b14, 0x0e629062);
-	nv_wr32(dev, 0x418b18, 0x0a418820);
-	nv_wr32(dev, 0x418b1c, 0x000000e6);
-	nv_wr32(dev, 0x418bb8, 0x00000103);
-	nv_wr32(dev, 0x418c08, 0x00000001);
-	nv_wr32(dev, 0x418c10, 0x00000000);
-	nv_wr32(dev, 0x418c14, 0x00000000);
-	nv_wr32(dev, 0x418c18, 0x00000000);
-	nv_wr32(dev, 0x418c1c, 0x00000000);
-	nv_wr32(dev, 0x418c20, 0x00000000);
-	nv_wr32(dev, 0x418c24, 0x00000000);
-	nv_wr32(dev, 0x418c28, 0x00000000);
-	nv_wr32(dev, 0x418c2c, 0x00000000);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x418c6c, 0x00000001);
-	nv_wr32(dev, 0x418c80, 0x20200004);
-	nv_wr32(dev, 0x418c8c, 0x00000001);
-	nv_wr32(dev, 0x419000, 0x00000780);
-	nv_wr32(dev, 0x419004, 0x00000000);
-	nv_wr32(dev, 0x419008, 0x00000000);
-	nv_wr32(dev, 0x419014, 0x00000004);
-}
-
-static void
-nvc0_grctx_generate_tp(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	int chipset = dev_priv->chipset;
-
-	/* GPC_BROADCAST.TP_BROADCAST */
-	nv_wr32(dev, 0x419818, 0x00000000);
-	nv_wr32(dev, 0x41983c, 0x00038bc7);
-	nv_wr32(dev, 0x419848, 0x00000000);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x419864, 0x00000129);
-	else
-		nv_wr32(dev, 0x419864, 0x0000012a);
-	nv_wr32(dev, 0x419888, 0x00000000);
-	nv_wr32(dev, 0x419a00, 0x000001f0);
-	nv_wr32(dev, 0x419a04, 0x00000001);
-	nv_wr32(dev, 0x419a08, 0x00000023);
-	nv_wr32(dev, 0x419a0c, 0x00020000);
-	nv_wr32(dev, 0x419a10, 0x00000000);
-	nv_wr32(dev, 0x419a14, 0x00000200);
-	nv_wr32(dev, 0x419a1c, 0x00000000);
-	nv_wr32(dev, 0x419a20, 0x00000800);
-	if (chipset == 0xd9)
-		nv_wr32(dev, 0x00419ac4, 0x0017f440);
-	else if (chipset != 0xc0 && chipset != 0xc8)
-		nv_wr32(dev, 0x00419ac4, 0x0007f440);
-	nv_wr32(dev, 0x419b00, 0x0a418820);
-	nv_wr32(dev, 0x419b04, 0x062080e6);
-	nv_wr32(dev, 0x419b08, 0x020398a4);
-	nv_wr32(dev, 0x419b0c, 0x0e629062);
-	nv_wr32(dev, 0x419b10, 0x0a418820);
-	nv_wr32(dev, 0x419b14, 0x000000e6);
-	nv_wr32(dev, 0x419bd0, 0x00900103);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x419be0, 0x00400001);
-	else
-		nv_wr32(dev, 0x419be0, 0x00000001);
-	nv_wr32(dev, 0x419be4, 0x00000000);
-	nv_wr32(dev, 0x419c00, chipset != 0xd9 ? 0x00000002 : 0x0000000a);
-	nv_wr32(dev, 0x419c04, 0x00000006);
-	nv_wr32(dev, 0x419c08, 0x00000002);
-	nv_wr32(dev, 0x419c20, 0x00000000);
-	if (dev_priv->chipset == 0xd9) {
-		nv_wr32(dev, 0x419c24, 0x00084210);
-		nv_wr32(dev, 0x419c28, 0x3cf3cf3c);
-		nv_wr32(dev, 0x419cb0, 0x00020048);
-	} else
-	if (chipset == 0xce || chipset == 0xcf) {
-		nv_wr32(dev, 0x419cb0, 0x00020048);
-	} else {
-		nv_wr32(dev, 0x419cb0, 0x00060048);
-	}
-	nv_wr32(dev, 0x419ce8, 0x00000000);
-	nv_wr32(dev, 0x419cf4, 0x00000183);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x419d20, 0x12180000);
-	else
-		nv_wr32(dev, 0x419d20, 0x02180000);
-	nv_wr32(dev, 0x419d24, 0x00001fff);
-	if (chipset == 0xc1 || chipset == 0xd9)
-		nv_wr32(dev, 0x419d44, 0x02180218);
-	nv_wr32(dev, 0x419e04, 0x00000000);
-	nv_wr32(dev, 0x419e08, 0x00000000);
-	nv_wr32(dev, 0x419e0c, 0x00000000);
-	nv_wr32(dev, 0x419e10, 0x00000002);
-	nv_wr32(dev, 0x419e44, 0x001beff2);
-	nv_wr32(dev, 0x419e48, 0x00000000);
-	nv_wr32(dev, 0x419e4c, 0x0000000f);
-	nv_wr32(dev, 0x419e50, 0x00000000);
-	nv_wr32(dev, 0x419e54, 0x00000000);
-	nv_wr32(dev, 0x419e58, 0x00000000);
-	nv_wr32(dev, 0x419e5c, 0x00000000);
-	nv_wr32(dev, 0x419e60, 0x00000000);
-	nv_wr32(dev, 0x419e64, 0x00000000);
-	nv_wr32(dev, 0x419e68, 0x00000000);
-	nv_wr32(dev, 0x419e6c, 0x00000000);
-	nv_wr32(dev, 0x419e70, 0x00000000);
-	nv_wr32(dev, 0x419e74, 0x00000000);
-	nv_wr32(dev, 0x419e78, 0x00000000);
-	nv_wr32(dev, 0x419e7c, 0x00000000);
-	nv_wr32(dev, 0x419e80, 0x00000000);
-	nv_wr32(dev, 0x419e84, 0x00000000);
-	nv_wr32(dev, 0x419e88, 0x00000000);
-	nv_wr32(dev, 0x419e8c, 0x00000000);
-	nv_wr32(dev, 0x419e90, 0x00000000);
-	nv_wr32(dev, 0x419e98, 0x00000000);
-	if (chipset != 0xc0 && chipset != 0xc8)
-		nv_wr32(dev, 0x419ee0, 0x00011110);
-	nv_wr32(dev, 0x419f50, 0x00000000);
-	nv_wr32(dev, 0x419f54, 0x00000000);
-	if (chipset != 0xc0 && chipset != 0xc8)
-		nv_wr32(dev, 0x419f58, 0x00000000);
-}
-
-int
-nvc0_grctx_generate(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nvc0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nvc0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	int i, gpc, tp, id;
-	u32 fermi = nvc0_graph_class(dev);
-	u32 r000260, tmp;
-
-	r000260 = nv_rd32(dev, 0x000260);
-	nv_wr32(dev, 0x000260, r000260 & ~1);
-	nv_wr32(dev, 0x400208, 0x00000000);
-
-	nvc0_grctx_generate_dispatch(dev);
-	nvc0_grctx_generate_macro(dev);
-	nvc0_grctx_generate_m2mf(dev);
-	nvc0_grctx_generate_unk47xx(dev);
-	nvc0_grctx_generate_shaders(dev);
-	nvc0_grctx_generate_unk60xx(dev);
-	nvc0_grctx_generate_unk64xx(dev);
-	nvc0_grctx_generate_tpbus(dev);
-	nvc0_grctx_generate_ccache(dev);
-	nvc0_grctx_generate_rop(dev);
-	nvc0_grctx_generate_gpc(dev);
-	nvc0_grctx_generate_tp(dev);
-
-	nv_wr32(dev, 0x404154, 0x00000000);
-
-	/* fuc "mmio list" writes */
-	for (i = 0; i < grch->mmio_nr * 8; i += 8) {
-		u32 reg = nv_ro32(grch->mmio, i + 0);
-		nv_wr32(dev, reg, nv_ro32(grch->mmio, i + 4));
-	}
-
-	for (tp = 0, id = 0; tp < 4; tp++) {
-		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-			if (tp < priv->tp_nr[gpc]) {
-				nv_wr32(dev, TP_UNIT(gpc, tp, 0x698), id);
-				nv_wr32(dev, TP_UNIT(gpc, tp, 0x4e8), id);
-				nv_wr32(dev, GPC_UNIT(gpc, 0x0c10 + tp * 4), id);
-				nv_wr32(dev, TP_UNIT(gpc, tp, 0x088), id);
-				id++;
-			}
-
-			nv_wr32(dev, GPC_UNIT(gpc, 0x0c08), priv->tp_nr[gpc]);
-			nv_wr32(dev, GPC_UNIT(gpc, 0x0c8c), priv->tp_nr[gpc]);
-		}
-	}
-
-	tmp = 0;
-	for (i = 0; i < priv->gpc_nr; i++)
-		tmp |= priv->tp_nr[i] << (i * 4);
-	nv_wr32(dev, 0x406028, tmp);
-	nv_wr32(dev, 0x405870, tmp);
-
-	nv_wr32(dev, 0x40602c, 0x00000000);
-	nv_wr32(dev, 0x405874, 0x00000000);
-	nv_wr32(dev, 0x406030, 0x00000000);
-	nv_wr32(dev, 0x405878, 0x00000000);
-	nv_wr32(dev, 0x406034, 0x00000000);
-	nv_wr32(dev, 0x40587c, 0x00000000);
-
-	if (1) {
-		u8 tpnr[GPC_MAX], data[TP_MAX];
-
-		memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
-		memset(data, 0x1f, sizeof(data));
-
-		gpc = -1;
-		for (tp = 0; tp < priv->tp_total; tp++) {
-			do {
-				gpc = (gpc + 1) % priv->gpc_nr;
-			} while (!tpnr[gpc]);
-			tpnr[gpc]--;
-			data[tp] = gpc;
-		}
-
-		for (i = 0; i < 4; i++)
-			nv_wr32(dev, 0x4060a8 + (i * 4), ((u32 *)data)[i]);
-	}
-
-	if (1) {
-		u32 data[6] = {}, data2[2] = {};
-		u8 tpnr[GPC_MAX];
-		u8 shift, ntpcv;
-
-		/* calculate first set of magics */
-		memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
-
-		gpc = -1;
-		for (tp = 0; tp < priv->tp_total; tp++) {
-			do {
-				gpc = (gpc + 1) % priv->gpc_nr;
-			} while (!tpnr[gpc]);
-			tpnr[gpc]--;
-
-			data[tp / 6] |= gpc << ((tp % 6) * 5);
-		}
-
-		for (; tp < 32; tp++)
-			data[tp / 6] |= 7 << ((tp % 6) * 5);
-
-		/* and the second... */
-		shift = 0;
-		ntpcv = priv->tp_total;
-		while (!(ntpcv & (1 << 4))) {
-			ntpcv <<= 1;
-			shift++;
-		}
-
-		data2[0]  = (ntpcv << 16);
-		data2[0] |= (shift << 21);
-		data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
-		for (i = 1; i < 7; i++)
-			data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
-
-		/* GPC_BROADCAST */
-		nv_wr32(dev, 0x418bb8, (priv->tp_total << 8) |
-					priv->magic_not_rop_nr);
-		for (i = 0; i < 6; i++)
-			nv_wr32(dev, 0x418b08 + (i * 4), data[i]);
-
-		/* GPC_BROADCAST.TP_BROADCAST */
-		nv_wr32(dev, 0x419bd0, (priv->tp_total << 8) |
-				       priv->magic_not_rop_nr |
-				       data2[0]);
-		nv_wr32(dev, 0x419be4, data2[1]);
-		for (i = 0; i < 6; i++)
-			nv_wr32(dev, 0x419b00 + (i * 4), data[i]);
-
-		/* UNK78xx */
-		nv_wr32(dev, 0x4078bc, (priv->tp_total << 8) |
-					priv->magic_not_rop_nr);
-		for (i = 0; i < 6; i++)
-			nv_wr32(dev, 0x40780c + (i * 4), data[i]);
-	}
-
-	if (1) {
-		u32 tp_mask = 0, tp_set = 0;
-		u8  tpnr[GPC_MAX], a, b;
-
-		memcpy(tpnr, priv->tp_nr, sizeof(priv->tp_nr));
-		for (gpc = 0; gpc < priv->gpc_nr; gpc++)
-			tp_mask |= ((1 << priv->tp_nr[gpc]) - 1) << (gpc * 8);
-
-		for (i = 0, gpc = -1, b = -1; i < 32; i++) {
-			a = (i * (priv->tp_total - 1)) / 32;
-			if (a != b) {
-				b = a;
-				do {
-					gpc = (gpc + 1) % priv->gpc_nr;
-				} while (!tpnr[gpc]);
-				tp = priv->tp_nr[gpc] - tpnr[gpc]--;
-
-				tp_set |= 1 << ((gpc * 8) + tp);
-			}
-
-			nv_wr32(dev, 0x406800 + (i * 0x20), tp_set);
-			nv_wr32(dev, 0x406c00 + (i * 0x20), tp_set ^ tp_mask);
-		}
-	}
-
-	nv_wr32(dev, 0x400208, 0x80000000);
-
-	nv_icmd(dev, 0x00001000, 0x00000004);
-	nv_icmd(dev, 0x000000a9, 0x0000ffff);
-	nv_icmd(dev, 0x00000038, 0x0fac6881);
-	nv_icmd(dev, 0x0000003d, 0x00000001);
-	nv_icmd(dev, 0x000000e8, 0x00000400);
-	nv_icmd(dev, 0x000000e9, 0x00000400);
-	nv_icmd(dev, 0x000000ea, 0x00000400);
-	nv_icmd(dev, 0x000000eb, 0x00000400);
-	nv_icmd(dev, 0x000000ec, 0x00000400);
-	nv_icmd(dev, 0x000000ed, 0x00000400);
-	nv_icmd(dev, 0x000000ee, 0x00000400);
-	nv_icmd(dev, 0x000000ef, 0x00000400);
-	nv_icmd(dev, 0x00000078, 0x00000300);
-	nv_icmd(dev, 0x00000079, 0x00000300);
-	nv_icmd(dev, 0x0000007a, 0x00000300);
-	nv_icmd(dev, 0x0000007b, 0x00000300);
-	nv_icmd(dev, 0x0000007c, 0x00000300);
-	nv_icmd(dev, 0x0000007d, 0x00000300);
-	nv_icmd(dev, 0x0000007e, 0x00000300);
-	nv_icmd(dev, 0x0000007f, 0x00000300);
-	nv_icmd(dev, 0x00000050, 0x00000011);
-	nv_icmd(dev, 0x00000058, 0x00000008);
-	nv_icmd(dev, 0x00000059, 0x00000008);
-	nv_icmd(dev, 0x0000005a, 0x00000008);
-	nv_icmd(dev, 0x0000005b, 0x00000008);
-	nv_icmd(dev, 0x0000005c, 0x00000008);
-	nv_icmd(dev, 0x0000005d, 0x00000008);
-	nv_icmd(dev, 0x0000005e, 0x00000008);
-	nv_icmd(dev, 0x0000005f, 0x00000008);
-	nv_icmd(dev, 0x00000208, 0x00000001);
-	nv_icmd(dev, 0x00000209, 0x00000001);
-	nv_icmd(dev, 0x0000020a, 0x00000001);
-	nv_icmd(dev, 0x0000020b, 0x00000001);
-	nv_icmd(dev, 0x0000020c, 0x00000001);
-	nv_icmd(dev, 0x0000020d, 0x00000001);
-	nv_icmd(dev, 0x0000020e, 0x00000001);
-	nv_icmd(dev, 0x0000020f, 0x00000001);
-	nv_icmd(dev, 0x00000081, 0x00000001);
-	nv_icmd(dev, 0x00000085, 0x00000004);
-	nv_icmd(dev, 0x00000088, 0x00000400);
-	nv_icmd(dev, 0x00000090, 0x00000300);
-	nv_icmd(dev, 0x00000098, 0x00001001);
-	nv_icmd(dev, 0x000000e3, 0x00000001);
-	nv_icmd(dev, 0x000000da, 0x00000001);
-	nv_icmd(dev, 0x000000f8, 0x00000003);
-	nv_icmd(dev, 0x000000fa, 0x00000001);
-	nv_icmd(dev, 0x0000009f, 0x0000ffff);
-	nv_icmd(dev, 0x000000a0, 0x0000ffff);
-	nv_icmd(dev, 0x000000a1, 0x0000ffff);
-	nv_icmd(dev, 0x000000a2, 0x0000ffff);
-	nv_icmd(dev, 0x000000b1, 0x00000001);
-	nv_icmd(dev, 0x000000b2, 0x00000000);
-	nv_icmd(dev, 0x000000b3, 0x00000000);
-	nv_icmd(dev, 0x000000b4, 0x00000000);
-	nv_icmd(dev, 0x000000b5, 0x00000000);
-	nv_icmd(dev, 0x000000b6, 0x00000000);
-	nv_icmd(dev, 0x000000b7, 0x00000000);
-	nv_icmd(dev, 0x000000b8, 0x00000000);
-	nv_icmd(dev, 0x000000b9, 0x00000000);
-	nv_icmd(dev, 0x000000ba, 0x00000000);
-	nv_icmd(dev, 0x000000bb, 0x00000000);
-	nv_icmd(dev, 0x000000bc, 0x00000000);
-	nv_icmd(dev, 0x000000bd, 0x00000000);
-	nv_icmd(dev, 0x000000be, 0x00000000);
-	nv_icmd(dev, 0x000000bf, 0x00000000);
-	nv_icmd(dev, 0x000000c0, 0x00000000);
-	nv_icmd(dev, 0x000000c1, 0x00000000);
-	nv_icmd(dev, 0x000000c2, 0x00000000);
-	nv_icmd(dev, 0x000000c3, 0x00000000);
-	nv_icmd(dev, 0x000000c4, 0x00000000);
-	nv_icmd(dev, 0x000000c5, 0x00000000);
-	nv_icmd(dev, 0x000000c6, 0x00000000);
-	nv_icmd(dev, 0x000000c7, 0x00000000);
-	nv_icmd(dev, 0x000000c8, 0x00000000);
-	nv_icmd(dev, 0x000000c9, 0x00000000);
-	nv_icmd(dev, 0x000000ca, 0x00000000);
-	nv_icmd(dev, 0x000000cb, 0x00000000);
-	nv_icmd(dev, 0x000000cc, 0x00000000);
-	nv_icmd(dev, 0x000000cd, 0x00000000);
-	nv_icmd(dev, 0x000000ce, 0x00000000);
-	nv_icmd(dev, 0x000000cf, 0x00000000);
-	nv_icmd(dev, 0x000000d0, 0x00000000);
-	nv_icmd(dev, 0x000000d1, 0x00000000);
-	nv_icmd(dev, 0x000000d2, 0x00000000);
-	nv_icmd(dev, 0x000000d3, 0x00000000);
-	nv_icmd(dev, 0x000000d4, 0x00000000);
-	nv_icmd(dev, 0x000000d5, 0x00000000);
-	nv_icmd(dev, 0x000000d6, 0x00000000);
-	nv_icmd(dev, 0x000000d7, 0x00000000);
-	nv_icmd(dev, 0x000000d8, 0x00000000);
-	nv_icmd(dev, 0x000000d9, 0x00000000);
-	nv_icmd(dev, 0x00000210, 0x00000040);
-	nv_icmd(dev, 0x00000211, 0x00000040);
-	nv_icmd(dev, 0x00000212, 0x00000040);
-	nv_icmd(dev, 0x00000213, 0x00000040);
-	nv_icmd(dev, 0x00000214, 0x00000040);
-	nv_icmd(dev, 0x00000215, 0x00000040);
-	nv_icmd(dev, 0x00000216, 0x00000040);
-	nv_icmd(dev, 0x00000217, 0x00000040);
-	if (dev_priv->chipset == 0xd9) {
-		for (i = 0x0400; i <= 0x0417; i++)
-			nv_icmd(dev, i, 0x00000040);
-	}
-	nv_icmd(dev, 0x00000218, 0x0000c080);
-	nv_icmd(dev, 0x00000219, 0x0000c080);
-	nv_icmd(dev, 0x0000021a, 0x0000c080);
-	nv_icmd(dev, 0x0000021b, 0x0000c080);
-	nv_icmd(dev, 0x0000021c, 0x0000c080);
-	nv_icmd(dev, 0x0000021d, 0x0000c080);
-	nv_icmd(dev, 0x0000021e, 0x0000c080);
-	nv_icmd(dev, 0x0000021f, 0x0000c080);
-	if (dev_priv->chipset == 0xd9) {
-		for (i = 0x0440; i <= 0x0457; i++)
-			nv_icmd(dev, i, 0x0000c080);
-	}
-	nv_icmd(dev, 0x000000ad, 0x0000013e);
-	nv_icmd(dev, 0x000000e1, 0x00000010);
-	nv_icmd(dev, 0x00000290, 0x00000000);
-	nv_icmd(dev, 0x00000291, 0x00000000);
-	nv_icmd(dev, 0x00000292, 0x00000000);
-	nv_icmd(dev, 0x00000293, 0x00000000);
-	nv_icmd(dev, 0x00000294, 0x00000000);
-	nv_icmd(dev, 0x00000295, 0x00000000);
-	nv_icmd(dev, 0x00000296, 0x00000000);
-	nv_icmd(dev, 0x00000297, 0x00000000);
-	nv_icmd(dev, 0x00000298, 0x00000000);
-	nv_icmd(dev, 0x00000299, 0x00000000);
-	nv_icmd(dev, 0x0000029a, 0x00000000);
-	nv_icmd(dev, 0x0000029b, 0x00000000);
-	nv_icmd(dev, 0x0000029c, 0x00000000);
-	nv_icmd(dev, 0x0000029d, 0x00000000);
-	nv_icmd(dev, 0x0000029e, 0x00000000);
-	nv_icmd(dev, 0x0000029f, 0x00000000);
-	nv_icmd(dev, 0x000003b0, 0x00000000);
-	nv_icmd(dev, 0x000003b1, 0x00000000);
-	nv_icmd(dev, 0x000003b2, 0x00000000);
-	nv_icmd(dev, 0x000003b3, 0x00000000);
-	nv_icmd(dev, 0x000003b4, 0x00000000);
-	nv_icmd(dev, 0x000003b5, 0x00000000);
-	nv_icmd(dev, 0x000003b6, 0x00000000);
-	nv_icmd(dev, 0x000003b7, 0x00000000);
-	nv_icmd(dev, 0x000003b8, 0x00000000);
-	nv_icmd(dev, 0x000003b9, 0x00000000);
-	nv_icmd(dev, 0x000003ba, 0x00000000);
-	nv_icmd(dev, 0x000003bb, 0x00000000);
-	nv_icmd(dev, 0x000003bc, 0x00000000);
-	nv_icmd(dev, 0x000003bd, 0x00000000);
-	nv_icmd(dev, 0x000003be, 0x00000000);
-	nv_icmd(dev, 0x000003bf, 0x00000000);
-	nv_icmd(dev, 0x000002a0, 0x00000000);
-	nv_icmd(dev, 0x000002a1, 0x00000000);
-	nv_icmd(dev, 0x000002a2, 0x00000000);
-	nv_icmd(dev, 0x000002a3, 0x00000000);
-	nv_icmd(dev, 0x000002a4, 0x00000000);
-	nv_icmd(dev, 0x000002a5, 0x00000000);
-	nv_icmd(dev, 0x000002a6, 0x00000000);
-	nv_icmd(dev, 0x000002a7, 0x00000000);
-	nv_icmd(dev, 0x000002a8, 0x00000000);
-	nv_icmd(dev, 0x000002a9, 0x00000000);
-	nv_icmd(dev, 0x000002aa, 0x00000000);
-	nv_icmd(dev, 0x000002ab, 0x00000000);
-	nv_icmd(dev, 0x000002ac, 0x00000000);
-	nv_icmd(dev, 0x000002ad, 0x00000000);
-	nv_icmd(dev, 0x000002ae, 0x00000000);
-	nv_icmd(dev, 0x000002af, 0x00000000);
-	nv_icmd(dev, 0x00000420, 0x00000000);
-	nv_icmd(dev, 0x00000421, 0x00000000);
-	nv_icmd(dev, 0x00000422, 0x00000000);
-	nv_icmd(dev, 0x00000423, 0x00000000);
-	nv_icmd(dev, 0x00000424, 0x00000000);
-	nv_icmd(dev, 0x00000425, 0x00000000);
-	nv_icmd(dev, 0x00000426, 0x00000000);
-	nv_icmd(dev, 0x00000427, 0x00000000);
-	nv_icmd(dev, 0x00000428, 0x00000000);
-	nv_icmd(dev, 0x00000429, 0x00000000);
-	nv_icmd(dev, 0x0000042a, 0x00000000);
-	nv_icmd(dev, 0x0000042b, 0x00000000);
-	nv_icmd(dev, 0x0000042c, 0x00000000);
-	nv_icmd(dev, 0x0000042d, 0x00000000);
-	nv_icmd(dev, 0x0000042e, 0x00000000);
-	nv_icmd(dev, 0x0000042f, 0x00000000);
-	nv_icmd(dev, 0x000002b0, 0x00000000);
-	nv_icmd(dev, 0x000002b1, 0x00000000);
-	nv_icmd(dev, 0x000002b2, 0x00000000);
-	nv_icmd(dev, 0x000002b3, 0x00000000);
-	nv_icmd(dev, 0x000002b4, 0x00000000);
-	nv_icmd(dev, 0x000002b5, 0x00000000);
-	nv_icmd(dev, 0x000002b6, 0x00000000);
-	nv_icmd(dev, 0x000002b7, 0x00000000);
-	nv_icmd(dev, 0x000002b8, 0x00000000);
-	nv_icmd(dev, 0x000002b9, 0x00000000);
-	nv_icmd(dev, 0x000002ba, 0x00000000);
-	nv_icmd(dev, 0x000002bb, 0x00000000);
-	nv_icmd(dev, 0x000002bc, 0x00000000);
-	nv_icmd(dev, 0x000002bd, 0x00000000);
-	nv_icmd(dev, 0x000002be, 0x00000000);
-	nv_icmd(dev, 0x000002bf, 0x00000000);
-	nv_icmd(dev, 0x00000430, 0x00000000);
-	nv_icmd(dev, 0x00000431, 0x00000000);
-	nv_icmd(dev, 0x00000432, 0x00000000);
-	nv_icmd(dev, 0x00000433, 0x00000000);
-	nv_icmd(dev, 0x00000434, 0x00000000);
-	nv_icmd(dev, 0x00000435, 0x00000000);
-	nv_icmd(dev, 0x00000436, 0x00000000);
-	nv_icmd(dev, 0x00000437, 0x00000000);
-	nv_icmd(dev, 0x00000438, 0x00000000);
-	nv_icmd(dev, 0x00000439, 0x00000000);
-	nv_icmd(dev, 0x0000043a, 0x00000000);
-	nv_icmd(dev, 0x0000043b, 0x00000000);
-	nv_icmd(dev, 0x0000043c, 0x00000000);
-	nv_icmd(dev, 0x0000043d, 0x00000000);
-	nv_icmd(dev, 0x0000043e, 0x00000000);
-	nv_icmd(dev, 0x0000043f, 0x00000000);
-	nv_icmd(dev, 0x000002c0, 0x00000000);
-	nv_icmd(dev, 0x000002c1, 0x00000000);
-	nv_icmd(dev, 0x000002c2, 0x00000000);
-	nv_icmd(dev, 0x000002c3, 0x00000000);
-	nv_icmd(dev, 0x000002c4, 0x00000000);
-	nv_icmd(dev, 0x000002c5, 0x00000000);
-	nv_icmd(dev, 0x000002c6, 0x00000000);
-	nv_icmd(dev, 0x000002c7, 0x00000000);
-	nv_icmd(dev, 0x000002c8, 0x00000000);
-	nv_icmd(dev, 0x000002c9, 0x00000000);
-	nv_icmd(dev, 0x000002ca, 0x00000000);
-	nv_icmd(dev, 0x000002cb, 0x00000000);
-	nv_icmd(dev, 0x000002cc, 0x00000000);
-	nv_icmd(dev, 0x000002cd, 0x00000000);
-	nv_icmd(dev, 0x000002ce, 0x00000000);
-	nv_icmd(dev, 0x000002cf, 0x00000000);
-	nv_icmd(dev, 0x000004d0, 0x00000000);
-	nv_icmd(dev, 0x000004d1, 0x00000000);
-	nv_icmd(dev, 0x000004d2, 0x00000000);
-	nv_icmd(dev, 0x000004d3, 0x00000000);
-	nv_icmd(dev, 0x000004d4, 0x00000000);
-	nv_icmd(dev, 0x000004d5, 0x00000000);
-	nv_icmd(dev, 0x000004d6, 0x00000000);
-	nv_icmd(dev, 0x000004d7, 0x00000000);
-	nv_icmd(dev, 0x000004d8, 0x00000000);
-	nv_icmd(dev, 0x000004d9, 0x00000000);
-	nv_icmd(dev, 0x000004da, 0x00000000);
-	nv_icmd(dev, 0x000004db, 0x00000000);
-	nv_icmd(dev, 0x000004dc, 0x00000000);
-	nv_icmd(dev, 0x000004dd, 0x00000000);
-	nv_icmd(dev, 0x000004de, 0x00000000);
-	nv_icmd(dev, 0x000004df, 0x00000000);
-	nv_icmd(dev, 0x00000720, 0x00000000);
-	nv_icmd(dev, 0x00000721, 0x00000000);
-	nv_icmd(dev, 0x00000722, 0x00000000);
-	nv_icmd(dev, 0x00000723, 0x00000000);
-	nv_icmd(dev, 0x00000724, 0x00000000);
-	nv_icmd(dev, 0x00000725, 0x00000000);
-	nv_icmd(dev, 0x00000726, 0x00000000);
-	nv_icmd(dev, 0x00000727, 0x00000000);
-	nv_icmd(dev, 0x00000728, 0x00000000);
-	nv_icmd(dev, 0x00000729, 0x00000000);
-	nv_icmd(dev, 0x0000072a, 0x00000000);
-	nv_icmd(dev, 0x0000072b, 0x00000000);
-	nv_icmd(dev, 0x0000072c, 0x00000000);
-	nv_icmd(dev, 0x0000072d, 0x00000000);
-	nv_icmd(dev, 0x0000072e, 0x00000000);
-	nv_icmd(dev, 0x0000072f, 0x00000000);
-	nv_icmd(dev, 0x000008c0, 0x00000000);
-	nv_icmd(dev, 0x000008c1, 0x00000000);
-	nv_icmd(dev, 0x000008c2, 0x00000000);
-	nv_icmd(dev, 0x000008c3, 0x00000000);
-	nv_icmd(dev, 0x000008c4, 0x00000000);
-	nv_icmd(dev, 0x000008c5, 0x00000000);
-	nv_icmd(dev, 0x000008c6, 0x00000000);
-	nv_icmd(dev, 0x000008c7, 0x00000000);
-	nv_icmd(dev, 0x000008c8, 0x00000000);
-	nv_icmd(dev, 0x000008c9, 0x00000000);
-	nv_icmd(dev, 0x000008ca, 0x00000000);
-	nv_icmd(dev, 0x000008cb, 0x00000000);
-	nv_icmd(dev, 0x000008cc, 0x00000000);
-	nv_icmd(dev, 0x000008cd, 0x00000000);
-	nv_icmd(dev, 0x000008ce, 0x00000000);
-	nv_icmd(dev, 0x000008cf, 0x00000000);
-	nv_icmd(dev, 0x00000890, 0x00000000);
-	nv_icmd(dev, 0x00000891, 0x00000000);
-	nv_icmd(dev, 0x00000892, 0x00000000);
-	nv_icmd(dev, 0x00000893, 0x00000000);
-	nv_icmd(dev, 0x00000894, 0x00000000);
-	nv_icmd(dev, 0x00000895, 0x00000000);
-	nv_icmd(dev, 0x00000896, 0x00000000);
-	nv_icmd(dev, 0x00000897, 0x00000000);
-	nv_icmd(dev, 0x00000898, 0x00000000);
-	nv_icmd(dev, 0x00000899, 0x00000000);
-	nv_icmd(dev, 0x0000089a, 0x00000000);
-	nv_icmd(dev, 0x0000089b, 0x00000000);
-	nv_icmd(dev, 0x0000089c, 0x00000000);
-	nv_icmd(dev, 0x0000089d, 0x00000000);
-	nv_icmd(dev, 0x0000089e, 0x00000000);
-	nv_icmd(dev, 0x0000089f, 0x00000000);
-	nv_icmd(dev, 0x000008e0, 0x00000000);
-	nv_icmd(dev, 0x000008e1, 0x00000000);
-	nv_icmd(dev, 0x000008e2, 0x00000000);
-	nv_icmd(dev, 0x000008e3, 0x00000000);
-	nv_icmd(dev, 0x000008e4, 0x00000000);
-	nv_icmd(dev, 0x000008e5, 0x00000000);
-	nv_icmd(dev, 0x000008e6, 0x00000000);
-	nv_icmd(dev, 0x000008e7, 0x00000000);
-	nv_icmd(dev, 0x000008e8, 0x00000000);
-	nv_icmd(dev, 0x000008e9, 0x00000000);
-	nv_icmd(dev, 0x000008ea, 0x00000000);
-	nv_icmd(dev, 0x000008eb, 0x00000000);
-	nv_icmd(dev, 0x000008ec, 0x00000000);
-	nv_icmd(dev, 0x000008ed, 0x00000000);
-	nv_icmd(dev, 0x000008ee, 0x00000000);
-	nv_icmd(dev, 0x000008ef, 0x00000000);
-	nv_icmd(dev, 0x000008a0, 0x00000000);
-	nv_icmd(dev, 0x000008a1, 0x00000000);
-	nv_icmd(dev, 0x000008a2, 0x00000000);
-	nv_icmd(dev, 0x000008a3, 0x00000000);
-	nv_icmd(dev, 0x000008a4, 0x00000000);
-	nv_icmd(dev, 0x000008a5, 0x00000000);
-	nv_icmd(dev, 0x000008a6, 0x00000000);
-	nv_icmd(dev, 0x000008a7, 0x00000000);
-	nv_icmd(dev, 0x000008a8, 0x00000000);
-	nv_icmd(dev, 0x000008a9, 0x00000000);
-	nv_icmd(dev, 0x000008aa, 0x00000000);
-	nv_icmd(dev, 0x000008ab, 0x00000000);
-	nv_icmd(dev, 0x000008ac, 0x00000000);
-	nv_icmd(dev, 0x000008ad, 0x00000000);
-	nv_icmd(dev, 0x000008ae, 0x00000000);
-	nv_icmd(dev, 0x000008af, 0x00000000);
-	nv_icmd(dev, 0x000008f0, 0x00000000);
-	nv_icmd(dev, 0x000008f1, 0x00000000);
-	nv_icmd(dev, 0x000008f2, 0x00000000);
-	nv_icmd(dev, 0x000008f3, 0x00000000);
-	nv_icmd(dev, 0x000008f4, 0x00000000);
-	nv_icmd(dev, 0x000008f5, 0x00000000);
-	nv_icmd(dev, 0x000008f6, 0x00000000);
-	nv_icmd(dev, 0x000008f7, 0x00000000);
-	nv_icmd(dev, 0x000008f8, 0x00000000);
-	nv_icmd(dev, 0x000008f9, 0x00000000);
-	nv_icmd(dev, 0x000008fa, 0x00000000);
-	nv_icmd(dev, 0x000008fb, 0x00000000);
-	nv_icmd(dev, 0x000008fc, 0x00000000);
-	nv_icmd(dev, 0x000008fd, 0x00000000);
-	nv_icmd(dev, 0x000008fe, 0x00000000);
-	nv_icmd(dev, 0x000008ff, 0x00000000);
-	nv_icmd(dev, 0x0000094c, 0x000000ff);
-	nv_icmd(dev, 0x0000094d, 0xffffffff);
-	nv_icmd(dev, 0x0000094e, 0x00000002);
-	nv_icmd(dev, 0x000002ec, 0x00000001);
-	nv_icmd(dev, 0x00000303, 0x00000001);
-	nv_icmd(dev, 0x000002e6, 0x00000001);
-	nv_icmd(dev, 0x00000466, 0x00000052);
-	nv_icmd(dev, 0x00000301, 0x3f800000);
-	nv_icmd(dev, 0x00000304, 0x30201000);
-	nv_icmd(dev, 0x00000305, 0x70605040);
-	nv_icmd(dev, 0x00000306, 0xb8a89888);
-	nv_icmd(dev, 0x00000307, 0xf8e8d8c8);
-	nv_icmd(dev, 0x0000030a, 0x00ffff00);
-	nv_icmd(dev, 0x0000030b, 0x0000001a);
-	nv_icmd(dev, 0x0000030c, 0x00000001);
-	nv_icmd(dev, 0x00000318, 0x00000001);
-	nv_icmd(dev, 0x00000340, 0x00000000);
-	nv_icmd(dev, 0x00000375, 0x00000001);
-	nv_icmd(dev, 0x00000351, 0x00000100);
-	nv_icmd(dev, 0x0000037d, 0x00000006);
-	nv_icmd(dev, 0x000003a0, 0x00000002);
-	nv_icmd(dev, 0x000003aa, 0x00000001);
-	nv_icmd(dev, 0x000003a9, 0x00000001);
-	nv_icmd(dev, 0x00000380, 0x00000001);
-	nv_icmd(dev, 0x00000360, 0x00000040);
-	nv_icmd(dev, 0x00000366, 0x00000000);
-	nv_icmd(dev, 0x00000367, 0x00000000);
-	nv_icmd(dev, 0x00000368, 0x00001fff);
-	nv_icmd(dev, 0x00000370, 0x00000000);
-	nv_icmd(dev, 0x00000371, 0x00000000);
-	nv_icmd(dev, 0x00000372, 0x003fffff);
-	nv_icmd(dev, 0x0000037a, 0x00000012);
-	nv_icmd(dev, 0x000005e0, 0x00000022);
-	nv_icmd(dev, 0x000005e1, 0x00000022);
-	nv_icmd(dev, 0x000005e2, 0x00000022);
-	nv_icmd(dev, 0x000005e3, 0x00000022);
-	nv_icmd(dev, 0x000005e4, 0x00000022);
-	nv_icmd(dev, 0x00000619, 0x00000003);
-	nv_icmd(dev, 0x00000811, 0x00000003);
-	nv_icmd(dev, 0x00000812, 0x00000004);
-	nv_icmd(dev, 0x00000813, 0x00000006);
-	nv_icmd(dev, 0x00000814, 0x00000008);
-	nv_icmd(dev, 0x00000815, 0x0000000b);
-	nv_icmd(dev, 0x00000800, 0x00000001);
-	nv_icmd(dev, 0x00000801, 0x00000001);
-	nv_icmd(dev, 0x00000802, 0x00000001);
-	nv_icmd(dev, 0x00000803, 0x00000001);
-	nv_icmd(dev, 0x00000804, 0x00000001);
-	nv_icmd(dev, 0x00000805, 0x00000001);
-	nv_icmd(dev, 0x00000632, 0x00000001);
-	nv_icmd(dev, 0x00000633, 0x00000002);
-	nv_icmd(dev, 0x00000634, 0x00000003);
-	nv_icmd(dev, 0x00000635, 0x00000004);
-	nv_icmd(dev, 0x00000654, 0x3f800000);
-	nv_icmd(dev, 0x00000657, 0x3f800000);
-	nv_icmd(dev, 0x00000655, 0x3f800000);
-	nv_icmd(dev, 0x00000656, 0x3f800000);
-	nv_icmd(dev, 0x000006cd, 0x3f800000);
-	nv_icmd(dev, 0x000007f5, 0x3f800000);
-	nv_icmd(dev, 0x000007dc, 0x39291909);
-	nv_icmd(dev, 0x000007dd, 0x79695949);
-	nv_icmd(dev, 0x000007de, 0xb9a99989);
-	nv_icmd(dev, 0x000007df, 0xf9e9d9c9);
-	nv_icmd(dev, 0x000007e8, 0x00003210);
-	nv_icmd(dev, 0x000007e9, 0x00007654);
-	nv_icmd(dev, 0x000007ea, 0x00000098);
-	nv_icmd(dev, 0x000007ec, 0x39291909);
-	nv_icmd(dev, 0x000007ed, 0x79695949);
-	nv_icmd(dev, 0x000007ee, 0xb9a99989);
-	nv_icmd(dev, 0x000007ef, 0xf9e9d9c9);
-	nv_icmd(dev, 0x000007f0, 0x00003210);
-	nv_icmd(dev, 0x000007f1, 0x00007654);
-	nv_icmd(dev, 0x000007f2, 0x00000098);
-	nv_icmd(dev, 0x000005a5, 0x00000001);
-	nv_icmd(dev, 0x00000980, 0x00000000);
-	nv_icmd(dev, 0x00000981, 0x00000000);
-	nv_icmd(dev, 0x00000982, 0x00000000);
-	nv_icmd(dev, 0x00000983, 0x00000000);
-	nv_icmd(dev, 0x00000984, 0x00000000);
-	nv_icmd(dev, 0x00000985, 0x00000000);
-	nv_icmd(dev, 0x00000986, 0x00000000);
-	nv_icmd(dev, 0x00000987, 0x00000000);
-	nv_icmd(dev, 0x00000988, 0x00000000);
-	nv_icmd(dev, 0x00000989, 0x00000000);
-	nv_icmd(dev, 0x0000098a, 0x00000000);
-	nv_icmd(dev, 0x0000098b, 0x00000000);
-	nv_icmd(dev, 0x0000098c, 0x00000000);
-	nv_icmd(dev, 0x0000098d, 0x00000000);
-	nv_icmd(dev, 0x0000098e, 0x00000000);
-	nv_icmd(dev, 0x0000098f, 0x00000000);
-	nv_icmd(dev, 0x00000990, 0x00000000);
-	nv_icmd(dev, 0x00000991, 0x00000000);
-	nv_icmd(dev, 0x00000992, 0x00000000);
-	nv_icmd(dev, 0x00000993, 0x00000000);
-	nv_icmd(dev, 0x00000994, 0x00000000);
-	nv_icmd(dev, 0x00000995, 0x00000000);
-	nv_icmd(dev, 0x00000996, 0x00000000);
-	nv_icmd(dev, 0x00000997, 0x00000000);
-	nv_icmd(dev, 0x00000998, 0x00000000);
-	nv_icmd(dev, 0x00000999, 0x00000000);
-	nv_icmd(dev, 0x0000099a, 0x00000000);
-	nv_icmd(dev, 0x0000099b, 0x00000000);
-	nv_icmd(dev, 0x0000099c, 0x00000000);
-	nv_icmd(dev, 0x0000099d, 0x00000000);
-	nv_icmd(dev, 0x0000099e, 0x00000000);
-	nv_icmd(dev, 0x0000099f, 0x00000000);
-	nv_icmd(dev, 0x000009a0, 0x00000000);
-	nv_icmd(dev, 0x000009a1, 0x00000000);
-	nv_icmd(dev, 0x000009a2, 0x00000000);
-	nv_icmd(dev, 0x000009a3, 0x00000000);
-	nv_icmd(dev, 0x000009a4, 0x00000000);
-	nv_icmd(dev, 0x000009a5, 0x00000000);
-	nv_icmd(dev, 0x000009a6, 0x00000000);
-	nv_icmd(dev, 0x000009a7, 0x00000000);
-	nv_icmd(dev, 0x000009a8, 0x00000000);
-	nv_icmd(dev, 0x000009a9, 0x00000000);
-	nv_icmd(dev, 0x000009aa, 0x00000000);
-	nv_icmd(dev, 0x000009ab, 0x00000000);
-	nv_icmd(dev, 0x000009ac, 0x00000000);
-	nv_icmd(dev, 0x000009ad, 0x00000000);
-	nv_icmd(dev, 0x000009ae, 0x00000000);
-	nv_icmd(dev, 0x000009af, 0x00000000);
-	nv_icmd(dev, 0x000009b0, 0x00000000);
-	nv_icmd(dev, 0x000009b1, 0x00000000);
-	nv_icmd(dev, 0x000009b2, 0x00000000);
-	nv_icmd(dev, 0x000009b3, 0x00000000);
-	nv_icmd(dev, 0x000009b4, 0x00000000);
-	nv_icmd(dev, 0x000009b5, 0x00000000);
-	nv_icmd(dev, 0x000009b6, 0x00000000);
-	nv_icmd(dev, 0x000009b7, 0x00000000);
-	nv_icmd(dev, 0x000009b8, 0x00000000);
-	nv_icmd(dev, 0x000009b9, 0x00000000);
-	nv_icmd(dev, 0x000009ba, 0x00000000);
-	nv_icmd(dev, 0x000009bb, 0x00000000);
-	nv_icmd(dev, 0x000009bc, 0x00000000);
-	nv_icmd(dev, 0x000009bd, 0x00000000);
-	nv_icmd(dev, 0x000009be, 0x00000000);
-	nv_icmd(dev, 0x000009bf, 0x00000000);
-	nv_icmd(dev, 0x000009c0, 0x00000000);
-	nv_icmd(dev, 0x000009c1, 0x00000000);
-	nv_icmd(dev, 0x000009c2, 0x00000000);
-	nv_icmd(dev, 0x000009c3, 0x00000000);
-	nv_icmd(dev, 0x000009c4, 0x00000000);
-	nv_icmd(dev, 0x000009c5, 0x00000000);
-	nv_icmd(dev, 0x000009c6, 0x00000000);
-	nv_icmd(dev, 0x000009c7, 0x00000000);
-	nv_icmd(dev, 0x000009c8, 0x00000000);
-	nv_icmd(dev, 0x000009c9, 0x00000000);
-	nv_icmd(dev, 0x000009ca, 0x00000000);
-	nv_icmd(dev, 0x000009cb, 0x00000000);
-	nv_icmd(dev, 0x000009cc, 0x00000000);
-	nv_icmd(dev, 0x000009cd, 0x00000000);
-	nv_icmd(dev, 0x000009ce, 0x00000000);
-	nv_icmd(dev, 0x000009cf, 0x00000000);
-	nv_icmd(dev, 0x000009d0, 0x00000000);
-	nv_icmd(dev, 0x000009d1, 0x00000000);
-	nv_icmd(dev, 0x000009d2, 0x00000000);
-	nv_icmd(dev, 0x000009d3, 0x00000000);
-	nv_icmd(dev, 0x000009d4, 0x00000000);
-	nv_icmd(dev, 0x000009d5, 0x00000000);
-	nv_icmd(dev, 0x000009d6, 0x00000000);
-	nv_icmd(dev, 0x000009d7, 0x00000000);
-	nv_icmd(dev, 0x000009d8, 0x00000000);
-	nv_icmd(dev, 0x000009d9, 0x00000000);
-	nv_icmd(dev, 0x000009da, 0x00000000);
-	nv_icmd(dev, 0x000009db, 0x00000000);
-	nv_icmd(dev, 0x000009dc, 0x00000000);
-	nv_icmd(dev, 0x000009dd, 0x00000000);
-	nv_icmd(dev, 0x000009de, 0x00000000);
-	nv_icmd(dev, 0x000009df, 0x00000000);
-	nv_icmd(dev, 0x000009e0, 0x00000000);
-	nv_icmd(dev, 0x000009e1, 0x00000000);
-	nv_icmd(dev, 0x000009e2, 0x00000000);
-	nv_icmd(dev, 0x000009e3, 0x00000000);
-	nv_icmd(dev, 0x000009e4, 0x00000000);
-	nv_icmd(dev, 0x000009e5, 0x00000000);
-	nv_icmd(dev, 0x000009e6, 0x00000000);
-	nv_icmd(dev, 0x000009e7, 0x00000000);
-	nv_icmd(dev, 0x000009e8, 0x00000000);
-	nv_icmd(dev, 0x000009e9, 0x00000000);
-	nv_icmd(dev, 0x000009ea, 0x00000000);
-	nv_icmd(dev, 0x000009eb, 0x00000000);
-	nv_icmd(dev, 0x000009ec, 0x00000000);
-	nv_icmd(dev, 0x000009ed, 0x00000000);
-	nv_icmd(dev, 0x000009ee, 0x00000000);
-	nv_icmd(dev, 0x000009ef, 0x00000000);
-	nv_icmd(dev, 0x000009f0, 0x00000000);
-	nv_icmd(dev, 0x000009f1, 0x00000000);
-	nv_icmd(dev, 0x000009f2, 0x00000000);
-	nv_icmd(dev, 0x000009f3, 0x00000000);
-	nv_icmd(dev, 0x000009f4, 0x00000000);
-	nv_icmd(dev, 0x000009f5, 0x00000000);
-	nv_icmd(dev, 0x000009f6, 0x00000000);
-	nv_icmd(dev, 0x000009f7, 0x00000000);
-	nv_icmd(dev, 0x000009f8, 0x00000000);
-	nv_icmd(dev, 0x000009f9, 0x00000000);
-	nv_icmd(dev, 0x000009fa, 0x00000000);
-	nv_icmd(dev, 0x000009fb, 0x00000000);
-	nv_icmd(dev, 0x000009fc, 0x00000000);
-	nv_icmd(dev, 0x000009fd, 0x00000000);
-	nv_icmd(dev, 0x000009fe, 0x00000000);
-	nv_icmd(dev, 0x000009ff, 0x00000000);
-	nv_icmd(dev, 0x00000468, 0x00000004);
-	nv_icmd(dev, 0x0000046c, 0x00000001);
-	nv_icmd(dev, 0x00000470, 0x00000000);
-	nv_icmd(dev, 0x00000471, 0x00000000);
-	nv_icmd(dev, 0x00000472, 0x00000000);
-	nv_icmd(dev, 0x00000473, 0x00000000);
-	nv_icmd(dev, 0x00000474, 0x00000000);
-	nv_icmd(dev, 0x00000475, 0x00000000);
-	nv_icmd(dev, 0x00000476, 0x00000000);
-	nv_icmd(dev, 0x00000477, 0x00000000);
-	nv_icmd(dev, 0x00000478, 0x00000000);
-	nv_icmd(dev, 0x00000479, 0x00000000);
-	nv_icmd(dev, 0x0000047a, 0x00000000);
-	nv_icmd(dev, 0x0000047b, 0x00000000);
-	nv_icmd(dev, 0x0000047c, 0x00000000);
-	nv_icmd(dev, 0x0000047d, 0x00000000);
-	nv_icmd(dev, 0x0000047e, 0x00000000);
-	nv_icmd(dev, 0x0000047f, 0x00000000);
-	nv_icmd(dev, 0x00000480, 0x00000000);
-	nv_icmd(dev, 0x00000481, 0x00000000);
-	nv_icmd(dev, 0x00000482, 0x00000000);
-	nv_icmd(dev, 0x00000483, 0x00000000);
-	nv_icmd(dev, 0x00000484, 0x00000000);
-	nv_icmd(dev, 0x00000485, 0x00000000);
-	nv_icmd(dev, 0x00000486, 0x00000000);
-	nv_icmd(dev, 0x00000487, 0x00000000);
-	nv_icmd(dev, 0x00000488, 0x00000000);
-	nv_icmd(dev, 0x00000489, 0x00000000);
-	nv_icmd(dev, 0x0000048a, 0x00000000);
-	nv_icmd(dev, 0x0000048b, 0x00000000);
-	nv_icmd(dev, 0x0000048c, 0x00000000);
-	nv_icmd(dev, 0x0000048d, 0x00000000);
-	nv_icmd(dev, 0x0000048e, 0x00000000);
-	nv_icmd(dev, 0x0000048f, 0x00000000);
-	nv_icmd(dev, 0x00000490, 0x00000000);
-	nv_icmd(dev, 0x00000491, 0x00000000);
-	nv_icmd(dev, 0x00000492, 0x00000000);
-	nv_icmd(dev, 0x00000493, 0x00000000);
-	nv_icmd(dev, 0x00000494, 0x00000000);
-	nv_icmd(dev, 0x00000495, 0x00000000);
-	nv_icmd(dev, 0x00000496, 0x00000000);
-	nv_icmd(dev, 0x00000497, 0x00000000);
-	nv_icmd(dev, 0x00000498, 0x00000000);
-	nv_icmd(dev, 0x00000499, 0x00000000);
-	nv_icmd(dev, 0x0000049a, 0x00000000);
-	nv_icmd(dev, 0x0000049b, 0x00000000);
-	nv_icmd(dev, 0x0000049c, 0x00000000);
-	nv_icmd(dev, 0x0000049d, 0x00000000);
-	nv_icmd(dev, 0x0000049e, 0x00000000);
-	nv_icmd(dev, 0x0000049f, 0x00000000);
-	nv_icmd(dev, 0x000004a0, 0x00000000);
-	nv_icmd(dev, 0x000004a1, 0x00000000);
-	nv_icmd(dev, 0x000004a2, 0x00000000);
-	nv_icmd(dev, 0x000004a3, 0x00000000);
-	nv_icmd(dev, 0x000004a4, 0x00000000);
-	nv_icmd(dev, 0x000004a5, 0x00000000);
-	nv_icmd(dev, 0x000004a6, 0x00000000);
-	nv_icmd(dev, 0x000004a7, 0x00000000);
-	nv_icmd(dev, 0x000004a8, 0x00000000);
-	nv_icmd(dev, 0x000004a9, 0x00000000);
-	nv_icmd(dev, 0x000004aa, 0x00000000);
-	nv_icmd(dev, 0x000004ab, 0x00000000);
-	nv_icmd(dev, 0x000004ac, 0x00000000);
-	nv_icmd(dev, 0x000004ad, 0x00000000);
-	nv_icmd(dev, 0x000004ae, 0x00000000);
-	nv_icmd(dev, 0x000004af, 0x00000000);
-	nv_icmd(dev, 0x000004b0, 0x00000000);
-	nv_icmd(dev, 0x000004b1, 0x00000000);
-	nv_icmd(dev, 0x000004b2, 0x00000000);
-	nv_icmd(dev, 0x000004b3, 0x00000000);
-	nv_icmd(dev, 0x000004b4, 0x00000000);
-	nv_icmd(dev, 0x000004b5, 0x00000000);
-	nv_icmd(dev, 0x000004b6, 0x00000000);
-	nv_icmd(dev, 0x000004b7, 0x00000000);
-	nv_icmd(dev, 0x000004b8, 0x00000000);
-	nv_icmd(dev, 0x000004b9, 0x00000000);
-	nv_icmd(dev, 0x000004ba, 0x00000000);
-	nv_icmd(dev, 0x000004bb, 0x00000000);
-	nv_icmd(dev, 0x000004bc, 0x00000000);
-	nv_icmd(dev, 0x000004bd, 0x00000000);
-	nv_icmd(dev, 0x000004be, 0x00000000);
-	nv_icmd(dev, 0x000004bf, 0x00000000);
-	nv_icmd(dev, 0x000004c0, 0x00000000);
-	nv_icmd(dev, 0x000004c1, 0x00000000);
-	nv_icmd(dev, 0x000004c2, 0x00000000);
-	nv_icmd(dev, 0x000004c3, 0x00000000);
-	nv_icmd(dev, 0x000004c4, 0x00000000);
-	nv_icmd(dev, 0x000004c5, 0x00000000);
-	nv_icmd(dev, 0x000004c6, 0x00000000);
-	nv_icmd(dev, 0x000004c7, 0x00000000);
-	nv_icmd(dev, 0x000004c8, 0x00000000);
-	nv_icmd(dev, 0x000004c9, 0x00000000);
-	nv_icmd(dev, 0x000004ca, 0x00000000);
-	nv_icmd(dev, 0x000004cb, 0x00000000);
-	nv_icmd(dev, 0x000004cc, 0x00000000);
-	nv_icmd(dev, 0x000004cd, 0x00000000);
-	nv_icmd(dev, 0x000004ce, 0x00000000);
-	nv_icmd(dev, 0x000004cf, 0x00000000);
-	nv_icmd(dev, 0x00000510, 0x3f800000);
-	nv_icmd(dev, 0x00000511, 0x3f800000);
-	nv_icmd(dev, 0x00000512, 0x3f800000);
-	nv_icmd(dev, 0x00000513, 0x3f800000);
-	nv_icmd(dev, 0x00000514, 0x3f800000);
-	nv_icmd(dev, 0x00000515, 0x3f800000);
-	nv_icmd(dev, 0x00000516, 0x3f800000);
-	nv_icmd(dev, 0x00000517, 0x3f800000);
-	nv_icmd(dev, 0x00000518, 0x3f800000);
-	nv_icmd(dev, 0x00000519, 0x3f800000);
-	nv_icmd(dev, 0x0000051a, 0x3f800000);
-	nv_icmd(dev, 0x0000051b, 0x3f800000);
-	nv_icmd(dev, 0x0000051c, 0x3f800000);
-	nv_icmd(dev, 0x0000051d, 0x3f800000);
-	nv_icmd(dev, 0x0000051e, 0x3f800000);
-	nv_icmd(dev, 0x0000051f, 0x3f800000);
-	nv_icmd(dev, 0x00000520, 0x000002b6);
-	nv_icmd(dev, 0x00000529, 0x00000001);
-	nv_icmd(dev, 0x00000530, 0xffff0000);
-	nv_icmd(dev, 0x00000531, 0xffff0000);
-	nv_icmd(dev, 0x00000532, 0xffff0000);
-	nv_icmd(dev, 0x00000533, 0xffff0000);
-	nv_icmd(dev, 0x00000534, 0xffff0000);
-	nv_icmd(dev, 0x00000535, 0xffff0000);
-	nv_icmd(dev, 0x00000536, 0xffff0000);
-	nv_icmd(dev, 0x00000537, 0xffff0000);
-	nv_icmd(dev, 0x00000538, 0xffff0000);
-	nv_icmd(dev, 0x00000539, 0xffff0000);
-	nv_icmd(dev, 0x0000053a, 0xffff0000);
-	nv_icmd(dev, 0x0000053b, 0xffff0000);
-	nv_icmd(dev, 0x0000053c, 0xffff0000);
-	nv_icmd(dev, 0x0000053d, 0xffff0000);
-	nv_icmd(dev, 0x0000053e, 0xffff0000);
-	nv_icmd(dev, 0x0000053f, 0xffff0000);
-	nv_icmd(dev, 0x00000585, 0x0000003f);
-	nv_icmd(dev, 0x00000576, 0x00000003);
-	if (dev_priv->chipset == 0xc1 ||
-	    dev_priv->chipset == 0xd9)
-		nv_icmd(dev, 0x0000057b, 0x00000059);
-	nv_icmd(dev, 0x00000586, 0x00000040);
-	nv_icmd(dev, 0x00000582, 0x00000080);
-	nv_icmd(dev, 0x00000583, 0x00000080);
-	nv_icmd(dev, 0x000005c2, 0x00000001);
-	nv_icmd(dev, 0x00000638, 0x00000001);
-	nv_icmd(dev, 0x00000639, 0x00000001);
-	nv_icmd(dev, 0x0000063a, 0x00000002);
-	nv_icmd(dev, 0x0000063b, 0x00000001);
-	nv_icmd(dev, 0x0000063c, 0x00000001);
-	nv_icmd(dev, 0x0000063d, 0x00000002);
-	nv_icmd(dev, 0x0000063e, 0x00000001);
-	nv_icmd(dev, 0x000008b8, 0x00000001);
-	nv_icmd(dev, 0x000008b9, 0x00000001);
-	nv_icmd(dev, 0x000008ba, 0x00000001);
-	nv_icmd(dev, 0x000008bb, 0x00000001);
-	nv_icmd(dev, 0x000008bc, 0x00000001);
-	nv_icmd(dev, 0x000008bd, 0x00000001);
-	nv_icmd(dev, 0x000008be, 0x00000001);
-	nv_icmd(dev, 0x000008bf, 0x00000001);
-	nv_icmd(dev, 0x00000900, 0x00000001);
-	nv_icmd(dev, 0x00000901, 0x00000001);
-	nv_icmd(dev, 0x00000902, 0x00000001);
-	nv_icmd(dev, 0x00000903, 0x00000001);
-	nv_icmd(dev, 0x00000904, 0x00000001);
-	nv_icmd(dev, 0x00000905, 0x00000001);
-	nv_icmd(dev, 0x00000906, 0x00000001);
-	nv_icmd(dev, 0x00000907, 0x00000001);
-	nv_icmd(dev, 0x00000908, 0x00000002);
-	nv_icmd(dev, 0x00000909, 0x00000002);
-	nv_icmd(dev, 0x0000090a, 0x00000002);
-	nv_icmd(dev, 0x0000090b, 0x00000002);
-	nv_icmd(dev, 0x0000090c, 0x00000002);
-	nv_icmd(dev, 0x0000090d, 0x00000002);
-	nv_icmd(dev, 0x0000090e, 0x00000002);
-	nv_icmd(dev, 0x0000090f, 0x00000002);
-	nv_icmd(dev, 0x00000910, 0x00000001);
-	nv_icmd(dev, 0x00000911, 0x00000001);
-	nv_icmd(dev, 0x00000912, 0x00000001);
-	nv_icmd(dev, 0x00000913, 0x00000001);
-	nv_icmd(dev, 0x00000914, 0x00000001);
-	nv_icmd(dev, 0x00000915, 0x00000001);
-	nv_icmd(dev, 0x00000916, 0x00000001);
-	nv_icmd(dev, 0x00000917, 0x00000001);
-	nv_icmd(dev, 0x00000918, 0x00000001);
-	nv_icmd(dev, 0x00000919, 0x00000001);
-	nv_icmd(dev, 0x0000091a, 0x00000001);
-	nv_icmd(dev, 0x0000091b, 0x00000001);
-	nv_icmd(dev, 0x0000091c, 0x00000001);
-	nv_icmd(dev, 0x0000091d, 0x00000001);
-	nv_icmd(dev, 0x0000091e, 0x00000001);
-	nv_icmd(dev, 0x0000091f, 0x00000001);
-	nv_icmd(dev, 0x00000920, 0x00000002);
-	nv_icmd(dev, 0x00000921, 0x00000002);
-	nv_icmd(dev, 0x00000922, 0x00000002);
-	nv_icmd(dev, 0x00000923, 0x00000002);
-	nv_icmd(dev, 0x00000924, 0x00000002);
-	nv_icmd(dev, 0x00000925, 0x00000002);
-	nv_icmd(dev, 0x00000926, 0x00000002);
-	nv_icmd(dev, 0x00000927, 0x00000002);
-	nv_icmd(dev, 0x00000928, 0x00000001);
-	nv_icmd(dev, 0x00000929, 0x00000001);
-	nv_icmd(dev, 0x0000092a, 0x00000001);
-	nv_icmd(dev, 0x0000092b, 0x00000001);
-	nv_icmd(dev, 0x0000092c, 0x00000001);
-	nv_icmd(dev, 0x0000092d, 0x00000001);
-	nv_icmd(dev, 0x0000092e, 0x00000001);
-	nv_icmd(dev, 0x0000092f, 0x00000001);
-	nv_icmd(dev, 0x00000648, 0x00000001);
-	nv_icmd(dev, 0x00000649, 0x00000001);
-	nv_icmd(dev, 0x0000064a, 0x00000001);
-	nv_icmd(dev, 0x0000064b, 0x00000001);
-	nv_icmd(dev, 0x0000064c, 0x00000001);
-	nv_icmd(dev, 0x0000064d, 0x00000001);
-	nv_icmd(dev, 0x0000064e, 0x00000001);
-	nv_icmd(dev, 0x0000064f, 0x00000001);
-	nv_icmd(dev, 0x00000650, 0x00000001);
-	nv_icmd(dev, 0x00000658, 0x0000000f);
-	nv_icmd(dev, 0x000007ff, 0x0000000a);
-	nv_icmd(dev, 0x0000066a, 0x40000000);
-	nv_icmd(dev, 0x0000066b, 0x10000000);
-	nv_icmd(dev, 0x0000066c, 0xffff0000);
-	nv_icmd(dev, 0x0000066d, 0xffff0000);
-	nv_icmd(dev, 0x000007af, 0x00000008);
-	nv_icmd(dev, 0x000007b0, 0x00000008);
-	nv_icmd(dev, 0x000007f6, 0x00000001);
-	nv_icmd(dev, 0x000006b2, 0x00000055);
-	nv_icmd(dev, 0x000007ad, 0x00000003);
-	nv_icmd(dev, 0x00000937, 0x00000001);
-	nv_icmd(dev, 0x00000971, 0x00000008);
-	nv_icmd(dev, 0x00000972, 0x00000040);
-	nv_icmd(dev, 0x00000973, 0x0000012c);
-	nv_icmd(dev, 0x0000097c, 0x00000040);
-	nv_icmd(dev, 0x00000979, 0x00000003);
-	nv_icmd(dev, 0x00000975, 0x00000020);
-	nv_icmd(dev, 0x00000976, 0x00000001);
-	nv_icmd(dev, 0x00000977, 0x00000020);
-	nv_icmd(dev, 0x00000978, 0x00000001);
-	nv_icmd(dev, 0x00000957, 0x00000003);
-	nv_icmd(dev, 0x0000095e, 0x20164010);
-	nv_icmd(dev, 0x0000095f, 0x00000020);
-	if (dev_priv->chipset == 0xd9)
-		nv_icmd(dev, 0x0000097d, 0x00000020);
-	nv_icmd(dev, 0x00000683, 0x00000006);
-	nv_icmd(dev, 0x00000685, 0x003fffff);
-	nv_icmd(dev, 0x00000687, 0x00000c48);
-	nv_icmd(dev, 0x000006a0, 0x00000005);
-	nv_icmd(dev, 0x00000840, 0x00300008);
-	nv_icmd(dev, 0x00000841, 0x04000080);
-	nv_icmd(dev, 0x00000842, 0x00300008);
-	nv_icmd(dev, 0x00000843, 0x04000080);
-	nv_icmd(dev, 0x00000818, 0x00000000);
-	nv_icmd(dev, 0x00000819, 0x00000000);
-	nv_icmd(dev, 0x0000081a, 0x00000000);
-	nv_icmd(dev, 0x0000081b, 0x00000000);
-	nv_icmd(dev, 0x0000081c, 0x00000000);
-	nv_icmd(dev, 0x0000081d, 0x00000000);
-	nv_icmd(dev, 0x0000081e, 0x00000000);
-	nv_icmd(dev, 0x0000081f, 0x00000000);
-	nv_icmd(dev, 0x00000848, 0x00000000);
-	nv_icmd(dev, 0x00000849, 0x00000000);
-	nv_icmd(dev, 0x0000084a, 0x00000000);
-	nv_icmd(dev, 0x0000084b, 0x00000000);
-	nv_icmd(dev, 0x0000084c, 0x00000000);
-	nv_icmd(dev, 0x0000084d, 0x00000000);
-	nv_icmd(dev, 0x0000084e, 0x00000000);
-	nv_icmd(dev, 0x0000084f, 0x00000000);
-	nv_icmd(dev, 0x00000850, 0x00000000);
-	nv_icmd(dev, 0x00000851, 0x00000000);
-	nv_icmd(dev, 0x00000852, 0x00000000);
-	nv_icmd(dev, 0x00000853, 0x00000000);
-	nv_icmd(dev, 0x00000854, 0x00000000);
-	nv_icmd(dev, 0x00000855, 0x00000000);
-	nv_icmd(dev, 0x00000856, 0x00000000);
-	nv_icmd(dev, 0x00000857, 0x00000000);
-	nv_icmd(dev, 0x00000738, 0x00000000);
-	nv_icmd(dev, 0x000006aa, 0x00000001);
-	nv_icmd(dev, 0x000006ab, 0x00000002);
-	nv_icmd(dev, 0x000006ac, 0x00000080);
-	nv_icmd(dev, 0x000006ad, 0x00000100);
-	nv_icmd(dev, 0x000006ae, 0x00000100);
-	nv_icmd(dev, 0x000006b1, 0x00000011);
-	nv_icmd(dev, 0x000006bb, 0x000000cf);
-	nv_icmd(dev, 0x000006ce, 0x2a712488);
-	nv_icmd(dev, 0x00000739, 0x4085c000);
-	nv_icmd(dev, 0x0000073a, 0x00000080);
-	nv_icmd(dev, 0x00000786, 0x80000100);
-	nv_icmd(dev, 0x0000073c, 0x00010100);
-	nv_icmd(dev, 0x0000073d, 0x02800000);
-	nv_icmd(dev, 0x00000787, 0x000000cf);
-	nv_icmd(dev, 0x0000078c, 0x00000008);
-	nv_icmd(dev, 0x00000792, 0x00000001);
-	nv_icmd(dev, 0x00000794, 0x00000001);
-	nv_icmd(dev, 0x00000795, 0x00000001);
-	nv_icmd(dev, 0x00000796, 0x00000001);
-	nv_icmd(dev, 0x00000797, 0x000000cf);
-	nv_icmd(dev, 0x00000836, 0x00000001);
-	nv_icmd(dev, 0x0000079a, 0x00000002);
-	nv_icmd(dev, 0x00000833, 0x04444480);
-	nv_icmd(dev, 0x000007a1, 0x00000001);
-	nv_icmd(dev, 0x000007a3, 0x00000001);
-	nv_icmd(dev, 0x000007a4, 0x00000001);
-	nv_icmd(dev, 0x000007a5, 0x00000001);
-	nv_icmd(dev, 0x00000831, 0x00000004);
-	nv_icmd(dev, 0x0000080c, 0x00000002);
-	nv_icmd(dev, 0x0000080d, 0x00000100);
-	nv_icmd(dev, 0x0000080e, 0x00000100);
-	nv_icmd(dev, 0x0000080f, 0x00000001);
-	nv_icmd(dev, 0x00000823, 0x00000002);
-	nv_icmd(dev, 0x00000824, 0x00000100);
-	nv_icmd(dev, 0x00000825, 0x00000100);
-	nv_icmd(dev, 0x00000826, 0x00000001);
-	nv_icmd(dev, 0x0000095d, 0x00000001);
-	nv_icmd(dev, 0x0000082b, 0x00000004);
-	nv_icmd(dev, 0x00000942, 0x00010001);
-	nv_icmd(dev, 0x00000943, 0x00000001);
-	nv_icmd(dev, 0x00000944, 0x00000022);
-	nv_icmd(dev, 0x000007c5, 0x00010001);
-	nv_icmd(dev, 0x00000834, 0x00000001);
-	nv_icmd(dev, 0x000007c7, 0x00000001);
-	nv_icmd(dev, 0x0000c1b0, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b1, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b2, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b3, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b4, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b5, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b6, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b7, 0x0000000f);
-	nv_icmd(dev, 0x0000c1b8, 0x0fac6881);
-	nv_icmd(dev, 0x0000c1b9, 0x00fac688);
-	nv_icmd(dev, 0x0001e100, 0x00000001);
-	nv_icmd(dev, 0x00001000, 0x00000002);
-	nv_icmd(dev, 0x000006aa, 0x00000001);
-	nv_icmd(dev, 0x000006ad, 0x00000100);
-	nv_icmd(dev, 0x000006ae, 0x00000100);
-	nv_icmd(dev, 0x000006b1, 0x00000011);
-	nv_icmd(dev, 0x0000078c, 0x00000008);
-	nv_icmd(dev, 0x00000792, 0x00000001);
-	nv_icmd(dev, 0x00000794, 0x00000001);
-	nv_icmd(dev, 0x00000795, 0x00000001);
-	nv_icmd(dev, 0x00000796, 0x00000001);
-	nv_icmd(dev, 0x00000797, 0x000000cf);
-	nv_icmd(dev, 0x0000079a, 0x00000002);
-	nv_icmd(dev, 0x00000833, 0x04444480);
-	nv_icmd(dev, 0x000007a1, 0x00000001);
-	nv_icmd(dev, 0x000007a3, 0x00000001);
-	nv_icmd(dev, 0x000007a4, 0x00000001);
-	nv_icmd(dev, 0x000007a5, 0x00000001);
-	nv_icmd(dev, 0x00000831, 0x00000004);
-	nv_icmd(dev, 0x0001e100, 0x00000001);
-	nv_icmd(dev, 0x00001000, 0x00000014);
-	nv_icmd(dev, 0x00000351, 0x00000100);
-	nv_icmd(dev, 0x00000957, 0x00000003);
-	nv_icmd(dev, 0x0000095d, 0x00000001);
-	nv_icmd(dev, 0x0000082b, 0x00000004);
-	nv_icmd(dev, 0x00000942, 0x00010001);
-	nv_icmd(dev, 0x00000943, 0x00000001);
-	nv_icmd(dev, 0x000007c5, 0x00010001);
-	nv_icmd(dev, 0x00000834, 0x00000001);
-	nv_icmd(dev, 0x000007c7, 0x00000001);
-	nv_icmd(dev, 0x0001e100, 0x00000001);
-	nv_icmd(dev, 0x00001000, 0x00000001);
-	nv_icmd(dev, 0x0000080c, 0x00000002);
-	nv_icmd(dev, 0x0000080d, 0x00000100);
-	nv_icmd(dev, 0x0000080e, 0x00000100);
-	nv_icmd(dev, 0x0000080f, 0x00000001);
-	nv_icmd(dev, 0x00000823, 0x00000002);
-	nv_icmd(dev, 0x00000824, 0x00000100);
-	nv_icmd(dev, 0x00000825, 0x00000100);
-	nv_icmd(dev, 0x00000826, 0x00000001);
-	nv_icmd(dev, 0x0001e100, 0x00000001);
-	nv_wr32(dev, 0x400208, 0x00000000);
-	nv_wr32(dev, 0x404154, 0x00000400);
-
-	nvc0_grctx_generate_9097(dev);
-	if (fermi >= 0x9197)
-		nvc0_grctx_generate_9197(dev);
-	if (fermi >= 0x9297)
-		nvc0_grctx_generate_9297(dev);
-	nvc0_grctx_generate_902d(dev);
-	nvc0_grctx_generate_9039(dev);
-	nvc0_grctx_generate_90c0(dev);
-
-	nv_wr32(dev, 0x000260, r000260);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_instmem.c b/drivers/gpu/drm/nouveau/nvc0_instmem.c
deleted file mode 100644
index b701c439c92..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_instmem.c
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_vm.h"
-
-struct nvc0_instmem_priv {
-	struct nouveau_gpuobj  *bar1_pgd;
-	struct nouveau_channel *bar1;
-	struct nouveau_gpuobj  *bar3_pgd;
-	struct nouveau_channel *bar3;
-};
-
-int
-nvc0_instmem_suspend(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	dev_priv->ramin_available = false;
-	return 0;
-}
-
-void
-nvc0_instmem_resume(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
-
-	nv_mask(dev, 0x100c80, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x001704, 0x80000000 | priv->bar1->ramin->vinst >> 12);
-	nv_wr32(dev, 0x001714, 0xc0000000 | priv->bar3->ramin->vinst >> 12);
-	dev_priv->ramin_available = true;
-}
-
-static void
-nvc0_channel_del(struct nouveau_channel **pchan)
-{
-	struct nouveau_channel *chan;
-
-	chan = *pchan;
-	*pchan = NULL;
-	if (!chan)
-		return;
-
-	nouveau_vm_ref(NULL, &chan->vm, NULL);
-	if (drm_mm_initialized(&chan->ramin_heap))
-		drm_mm_takedown(&chan->ramin_heap);
-	nouveau_gpuobj_ref(NULL, &chan->ramin);
-	kfree(chan);
-}
-
-static int
-nvc0_channel_new(struct drm_device *dev, u32 size, struct nouveau_vm *vm,
-		 struct nouveau_channel **pchan,
-		 struct nouveau_gpuobj *pgd, u64 vm_size)
-{
-	struct nouveau_channel *chan;
-	int ret;
-
-	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
-	if (!chan)
-		return -ENOMEM;
-	chan->dev = dev;
-
-	ret = nouveau_gpuobj_new(dev, NULL, size, 0x1000, 0, &chan->ramin);
-	if (ret) {
-		nvc0_channel_del(&chan);
-		return ret;
-	}
-
-	ret = drm_mm_init(&chan->ramin_heap, 0x1000, size - 0x1000);
-	if (ret) {
-		nvc0_channel_del(&chan);
-		return ret;
-	}
-
-	ret = nouveau_vm_ref(vm, &chan->vm, NULL);
-	if (ret) {
-		nvc0_channel_del(&chan);
-		return ret;
-	}
-
-	nv_wo32(chan->ramin, 0x0200, lower_32_bits(pgd->vinst));
-	nv_wo32(chan->ramin, 0x0204, upper_32_bits(pgd->vinst));
-	nv_wo32(chan->ramin, 0x0208, lower_32_bits(vm_size - 1));
-	nv_wo32(chan->ramin, 0x020c, upper_32_bits(vm_size - 1));
-
-	*pchan = chan;
-	return 0;
-}
-
-int
-nvc0_instmem_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct pci_dev *pdev = dev->pdev;
-	struct nvc0_instmem_priv *priv;
-	struct nouveau_vm *vm = NULL;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-	pinstmem->priv = priv;
-
-	/* BAR3 VM */
-	ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 3), 0,
-			     &dev_priv->bar3_vm);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL,
-				 (pci_resource_len(pdev, 3) >> 12) * 8, 0,
-				 NVOBJ_FLAG_DONT_MAP |
-				 NVOBJ_FLAG_ZERO_ALLOC,
-				 &dev_priv->bar3_vm->pgt[0].obj[0]);
-	if (ret)
-		goto error;
-	dev_priv->bar3_vm->pgt[0].refcount[0] = 1;
-
-	nv50_instmem_map(dev_priv->bar3_vm->pgt[0].obj[0]);
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar3_pgd);
-	if (ret)
-		goto error;
-
-	ret = nouveau_vm_ref(dev_priv->bar3_vm, &vm, priv->bar3_pgd);
-	if (ret)
-		goto error;
-	nouveau_vm_ref(NULL, &vm, NULL);
-
-	ret = nvc0_channel_new(dev, 8192, dev_priv->bar3_vm, &priv->bar3,
-			       priv->bar3_pgd, pci_resource_len(dev->pdev, 3));
-	if (ret)
-		goto error;
-
-	/* BAR1 VM */
-	ret = nouveau_vm_new(dev, 0, pci_resource_len(pdev, 1), 0, &vm);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 4096,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->bar1_pgd);
-	if (ret)
-		goto error;
-
-	ret = nouveau_vm_ref(vm, &dev_priv->bar1_vm, priv->bar1_pgd);
-	if (ret)
-		goto error;
-	nouveau_vm_ref(NULL, &vm, NULL);
-
-	ret = nvc0_channel_new(dev, 8192, dev_priv->bar1_vm, &priv->bar1,
-			       priv->bar1_pgd, pci_resource_len(dev->pdev, 1));
-	if (ret)
-		goto error;
-
-	/* channel vm */
-	ret = nouveau_vm_new(dev, 0, (1ULL << 40), 0x0008000000ULL,
-			     &dev_priv->chan_vm);
-	if (ret)
-		goto error;
-
-	nvc0_instmem_resume(dev);
-	return 0;
-error:
-	nvc0_instmem_takedown(dev);
-	return ret;
-}
-
-void
-nvc0_instmem_takedown(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_instmem_priv *priv = dev_priv->engine.instmem.priv;
-	struct nouveau_vm *vm = NULL;
-
-	nvc0_instmem_suspend(dev);
-
-	nv_wr32(dev, 0x1704, 0x00000000);
-	nv_wr32(dev, 0x1714, 0x00000000);
-
-	nouveau_vm_ref(NULL, &dev_priv->chan_vm, NULL);
-
-	nvc0_channel_del(&priv->bar1);
-	nouveau_vm_ref(NULL, &dev_priv->bar1_vm, priv->bar1_pgd);
-	nouveau_gpuobj_ref(NULL, &priv->bar1_pgd);
-
-	nvc0_channel_del(&priv->bar3);
-	nouveau_vm_ref(dev_priv->bar3_vm, &vm, NULL);
-	nouveau_vm_ref(NULL, &vm, priv->bar3_pgd);
-	nouveau_gpuobj_ref(NULL, &priv->bar3_pgd);
-	nouveau_gpuobj_ref(NULL, &dev_priv->bar3_vm->pgt[0].obj[0]);
-	nouveau_vm_ref(NULL, &dev_priv->bar3_vm, NULL);
-
-	dev_priv->engine.instmem.priv = NULL;
-	kfree(priv);
-}
-
diff --git a/drivers/gpu/drm/nouveau/nvc0_pm.c b/drivers/gpu/drm/nouveau/nvc0_pm.c
index 4e712b10ebd..0d34eb58117 100644
--- a/drivers/gpu/drm/nouveau/nvc0_pm.c
+++ b/drivers/gpu/drm/nouveau/nvc0_pm.c
@@ -22,18 +22,24 @@
  * Authors: Ben Skeggs
  */
 
-#include "drmP.h"
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
 #include "nouveau_bios.h"
 #include "nouveau_pm.h"
 
+#include <subdev/bios/pll.h>
+#include <subdev/bios.h>
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+
 static u32 read_div(struct drm_device *, int, u32, u32);
 static u32 read_pll(struct drm_device *, u32);
 
 static u32
 read_vco(struct drm_device *dev, u32 dsrc)
 {
-	u32 ssrc = nv_rd32(dev, dsrc);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ssrc = nv_rd32(device, dsrc);
 	if (!(ssrc & 0x00000100))
 		return read_pll(dev, 0x00e800);
 	return read_pll(dev, 0x00e820);
@@ -42,8 +48,9 @@ read_vco(struct drm_device *dev, u32 dsrc)
 static u32
 read_pll(struct drm_device *dev, u32 pll)
 {
-	u32 ctrl = nv_rd32(dev, pll + 0);
-	u32 coef = nv_rd32(dev, pll + 4);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ctrl = nv_rd32(device, pll + 0);
+	u32 coef = nv_rd32(device, pll + 4);
 	u32 P = (coef & 0x003f0000) >> 16;
 	u32 N = (coef & 0x0000ff00) >> 8;
 	u32 M = (coef & 0x000000ff) >> 0;
@@ -83,8 +90,9 @@ read_pll(struct drm_device *dev, u32 pll)
 static u32
 read_div(struct drm_device *dev, int doff, u32 dsrc, u32 dctl)
 {
-	u32 ssrc = nv_rd32(dev, dsrc + (doff * 4));
-	u32 sctl = nv_rd32(dev, dctl + (doff * 4));
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ssrc = nv_rd32(device, dsrc + (doff * 4));
+	u32 sctl = nv_rd32(device, dctl + (doff * 4));
 
 	switch (ssrc & 0x00000003) {
 	case 0:
@@ -109,7 +117,8 @@ read_div(struct drm_device *dev, int doff, u32 dsrc, u32 dctl)
 static u32
 read_mem(struct drm_device *dev)
 {
-	u32 ssel = nv_rd32(dev, 0x1373f0);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 ssel = nv_rd32(device, 0x1373f0);
 	if (ssel & 0x00000001)
 		return read_div(dev, 0, 0x137300, 0x137310);
 	return read_pll(dev, 0x132000);
@@ -118,8 +127,9 @@ read_mem(struct drm_device *dev)
 static u32
 read_clk(struct drm_device *dev, int clk)
 {
-	u32 sctl = nv_rd32(dev, 0x137250 + (clk * 4));
-	u32 ssel = nv_rd32(dev, 0x137100);
+	struct nouveau_device *device = nouveau_dev(dev);
+	u32 sctl = nv_rd32(device, 0x137250 + (clk * 4));
+	u32 ssel = nv_rd32(device, 0x137100);
 	u32 sclk, sdiv;
 
 	if (ssel & (1 << clk)) {
@@ -212,10 +222,12 @@ calc_src(struct drm_device *dev, int clk, u32 freq, u32 *dsrc, u32 *ddiv)
 static u32
 calc_pll(struct drm_device *dev, int clk, u32 freq, u32 *coef)
 {
-	struct pll_lims limits;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nvbios_pll limits;
 	int N, M, P, ret;
 
-	ret = get_pll_limits(dev, 0x137000 + (clk * 0x20), &limits);
+	ret = nvbios_pll_parse(bios, 0x137000 + (clk * 0x20), &limits);
 	if (ret)
 		return 0;
 
@@ -308,31 +320,33 @@ calc_clk(struct drm_device *dev, int clk, struct nvc0_pm_clock *info, u32 freq)
 static int
 calc_mem(struct drm_device *dev, struct nvc0_pm_clock *info, u32 freq)
 {
-	struct pll_lims pll;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_bios *bios = nouveau_bios(device);
+	struct nvbios_pll pll;
 	int N, M, P, ret;
 	u32 ctrl;
 
 	/* mclk pll input freq comes from another pll, make sure it's on */
-	ctrl = nv_rd32(dev, 0x132020);
+	ctrl = nv_rd32(device, 0x132020);
 	if (!(ctrl & 0x00000001)) {
 		/* if not, program it to 567MHz.  nfi where this value comes
 		 * from - it looks like it's in the pll limits table for
 		 * 132000 but the binary driver ignores all my attempts to
 		 * change this value.
 		 */
-		nv_wr32(dev, 0x137320, 0x00000103);
-		nv_wr32(dev, 0x137330, 0x81200606);
-		nv_wait(dev, 0x132020, 0x00010000, 0x00010000);
-		nv_wr32(dev, 0x132024, 0x0001150f);
-		nv_mask(dev, 0x132020, 0x00000001, 0x00000001);
-		nv_wait(dev, 0x137390, 0x00020000, 0x00020000);
-		nv_mask(dev, 0x132020, 0x00000004, 0x00000004);
+		nv_wr32(device, 0x137320, 0x00000103);
+		nv_wr32(device, 0x137330, 0x81200606);
+		nv_wait(device, 0x132020, 0x00010000, 0x00010000);
+		nv_wr32(device, 0x132024, 0x0001150f);
+		nv_mask(device, 0x132020, 0x00000001, 0x00000001);
+		nv_wait(device, 0x137390, 0x00020000, 0x00020000);
+		nv_mask(device, 0x132020, 0x00000004, 0x00000004);
 	}
 
 	/* for the moment, until the clock tree is better understood, use
 	 * pll mode for all clock frequencies
 	 */
-	ret = get_pll_limits(dev, 0x132000, &pll);
+	ret = nvbios_pll_parse(bios, 0x132000, &pll);
 	if (ret == 0) {
 		pll.refclk = read_pll(dev, 0x132020);
 		if (pll.refclk) {
@@ -350,7 +364,7 @@ calc_mem(struct drm_device *dev, struct nvc0_pm_clock *info, u32 freq)
 void *
 nvc0_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nvc0_pm_state *info;
 	int ret;
 
@@ -364,7 +378,7 @@ nvc0_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 	 * are always the same freq with the binary driver even when the
 	 * performance table says they should differ.
 	 */
-	if (dev_priv->chipset == 0xd9)
+	if (device->chipset == 0xd9)
 		perflvl->rop = 0;
 
 	if ((ret = calc_clk(dev, 0x00, &info->eng[0x00], perflvl->shader)) ||
@@ -394,38 +408,40 @@ nvc0_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
 static void
 prog_clk(struct drm_device *dev, int clk, struct nvc0_pm_clock *info)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+
 	/* program dividers at 137160/1371d0 first */
 	if (clk < 7 && !info->ssel) {
-		nv_mask(dev, 0x1371d0 + (clk * 0x04), 0x80003f3f, info->ddiv);
-		nv_wr32(dev, 0x137160 + (clk * 0x04), info->dsrc);
+		nv_mask(device, 0x1371d0 + (clk * 0x04), 0x80003f3f, info->ddiv);
+		nv_wr32(device, 0x137160 + (clk * 0x04), info->dsrc);
 	}
 
 	/* switch clock to non-pll mode */
-	nv_mask(dev, 0x137100, (1 << clk), 0x00000000);
-	nv_wait(dev, 0x137100, (1 << clk), 0x00000000);
+	nv_mask(device, 0x137100, (1 << clk), 0x00000000);
+	nv_wait(device, 0x137100, (1 << clk), 0x00000000);
 
 	/* reprogram pll */
 	if (clk < 7) {
 		/* make sure it's disabled first... */
 		u32 base = 0x137000 + (clk * 0x20);
-		u32 ctrl = nv_rd32(dev, base + 0x00);
+		u32 ctrl = nv_rd32(device, base + 0x00);
 		if (ctrl & 0x00000001) {
-			nv_mask(dev, base + 0x00, 0x00000004, 0x00000000);
-			nv_mask(dev, base + 0x00, 0x00000001, 0x00000000);
+			nv_mask(device, base + 0x00, 0x00000004, 0x00000000);
+			nv_mask(device, base + 0x00, 0x00000001, 0x00000000);
 		}
 		/* program it to new values, if necessary */
 		if (info->ssel) {
-			nv_wr32(dev, base + 0x04, info->coef);
-			nv_mask(dev, base + 0x00, 0x00000001, 0x00000001);
-			nv_wait(dev, base + 0x00, 0x00020000, 0x00020000);
-			nv_mask(dev, base + 0x00, 0x00020004, 0x00000004);
+			nv_wr32(device, base + 0x04, info->coef);
+			nv_mask(device, base + 0x00, 0x00000001, 0x00000001);
+			nv_wait(device, base + 0x00, 0x00020000, 0x00020000);
+			nv_mask(device, base + 0x00, 0x00020004, 0x00000004);
 		}
 	}
 
 	/* select pll/non-pll mode, and program final clock divider */
-	nv_mask(dev, 0x137100, (1 << clk), info->ssel);
-	nv_wait(dev, 0x137100, (1 << clk), info->ssel);
-	nv_mask(dev, 0x137250 + (clk * 0x04), 0x00003f3f, info->mdiv);
+	nv_mask(device, 0x137100, (1 << clk), info->ssel);
+	nv_wait(device, 0x137100, (1 << clk), info->ssel);
+	nv_mask(device, 0x137250 + (clk * 0x04), 0x00003f3f, info->mdiv);
 }
 
 static void
@@ -441,7 +457,8 @@ mclk_refresh(struct nouveau_mem_exec_func *exec)
 static void
 mclk_refresh_auto(struct nouveau_mem_exec_func *exec, bool enable)
 {
-	nv_wr32(exec->dev, 0x10f210, enable ? 0x80000000 : 0x00000000);
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	nv_wr32(device, 0x10f210, enable ? 0x80000000 : 0x00000000);
 }
 
 static void
@@ -458,83 +475,84 @@ mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
 static u32
 mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
 {
-	struct drm_device *dev = exec->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	if (dev_priv->vram_type != NV_MEM_TYPE_GDDR5) {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	if (pfb->ram.type != NV_MEM_TYPE_GDDR5) {
 		if (mr <= 1)
-			return nv_rd32(dev, 0x10f300 + ((mr - 0) * 4));
-		return nv_rd32(dev, 0x10f320 + ((mr - 2) * 4));
+			return nv_rd32(device, 0x10f300 + ((mr - 0) * 4));
+		return nv_rd32(device, 0x10f320 + ((mr - 2) * 4));
 	} else {
 		if (mr == 0)
-			return nv_rd32(dev, 0x10f300 + (mr * 4));
+			return nv_rd32(device, 0x10f300 + (mr * 4));
 		else
 		if (mr <= 7)
-			return nv_rd32(dev, 0x10f32c + (mr * 4));
-		return nv_rd32(dev, 0x10f34c);
+			return nv_rd32(device, 0x10f32c + (mr * 4));
+		return nv_rd32(device, 0x10f34c);
 	}
 }
 
 static void
 mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
 {
-	struct drm_device *dev = exec->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	if (dev_priv->vram_type != NV_MEM_TYPE_GDDR5) {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	if (pfb->ram.type != NV_MEM_TYPE_GDDR5) {
 		if (mr <= 1) {
-			nv_wr32(dev, 0x10f300 + ((mr - 0) * 4), data);
-			if (dev_priv->vram_rank_B)
-				nv_wr32(dev, 0x10f308 + ((mr - 0) * 4), data);
+			nv_wr32(device, 0x10f300 + ((mr - 0) * 4), data);
+			if (pfb->ram.ranks > 1)
+				nv_wr32(device, 0x10f308 + ((mr - 0) * 4), data);
 		} else
 		if (mr <= 3) {
-			nv_wr32(dev, 0x10f320 + ((mr - 2) * 4), data);
-			if (dev_priv->vram_rank_B)
-				nv_wr32(dev, 0x10f328 + ((mr - 2) * 4), data);
+			nv_wr32(device, 0x10f320 + ((mr - 2) * 4), data);
+			if (pfb->ram.ranks > 1)
+				nv_wr32(device, 0x10f328 + ((mr - 2) * 4), data);
 		}
 	} else {
-		if      (mr ==  0) nv_wr32(dev, 0x10f300 + (mr * 4), data);
-		else if (mr <=  7) nv_wr32(dev, 0x10f32c + (mr * 4), data);
-		else if (mr == 15) nv_wr32(dev, 0x10f34c, data);
+		if      (mr ==  0) nv_wr32(device, 0x10f300 + (mr * 4), data);
+		else if (mr <=  7) nv_wr32(device, 0x10f32c + (mr * 4), data);
+		else if (mr == 15) nv_wr32(device, 0x10f34c, data);
 	}
 }
 
 static void
 mclk_clock_set(struct nouveau_mem_exec_func *exec)
 {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nvc0_pm_state *info = exec->priv;
-	struct drm_device *dev = exec->dev;
-	u32 ctrl = nv_rd32(dev, 0x132000);
+	u32 ctrl = nv_rd32(device, 0x132000);
 
-	nv_wr32(dev, 0x137360, 0x00000001);
-	nv_wr32(dev, 0x137370, 0x00000000);
-	nv_wr32(dev, 0x137380, 0x00000000);
+	nv_wr32(device, 0x137360, 0x00000001);
+	nv_wr32(device, 0x137370, 0x00000000);
+	nv_wr32(device, 0x137380, 0x00000000);
 	if (ctrl & 0x00000001)
-		nv_wr32(dev, 0x132000, (ctrl &= ~0x00000001));
+		nv_wr32(device, 0x132000, (ctrl &= ~0x00000001));
 
-	nv_wr32(dev, 0x132004, info->mem.coef);
-	nv_wr32(dev, 0x132000, (ctrl |= 0x00000001));
-	nv_wait(dev, 0x137390, 0x00000002, 0x00000002);
-	nv_wr32(dev, 0x132018, 0x00005000);
+	nv_wr32(device, 0x132004, info->mem.coef);
+	nv_wr32(device, 0x132000, (ctrl |= 0x00000001));
+	nv_wait(device, 0x137390, 0x00000002, 0x00000002);
+	nv_wr32(device, 0x132018, 0x00005000);
 
-	nv_wr32(dev, 0x137370, 0x00000001);
-	nv_wr32(dev, 0x137380, 0x00000001);
-	nv_wr32(dev, 0x137360, 0x00000000);
+	nv_wr32(device, 0x137370, 0x00000001);
+	nv_wr32(device, 0x137380, 0x00000001);
+	nv_wr32(device, 0x137360, 0x00000000);
 }
 
 static void
 mclk_timing_set(struct nouveau_mem_exec_func *exec)
 {
+	struct nouveau_device *device = nouveau_dev(exec->dev);
 	struct nvc0_pm_state *info = exec->priv;
 	struct nouveau_pm_level *perflvl = info->perflvl;
 	int i;
 
 	for (i = 0; i < 5; i++)
-		nv_wr32(exec->dev, 0x10f290 + (i * 4), perflvl->timing.reg[i]);
+		nv_wr32(device, 0x10f290 + (i * 4), perflvl->timing.reg[i]);
 }
 
 static void
 prog_mem(struct drm_device *dev, struct nvc0_pm_state *info)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nouveau_mem_exec_func exec = {
 		.dev = dev,
 		.precharge = mclk_precharge,
@@ -549,17 +567,17 @@ prog_mem(struct drm_device *dev, struct nvc0_pm_state *info)
 		.priv = info
 	};
 
-	if (dev_priv->chipset < 0xd0)
-		nv_wr32(dev, 0x611200, 0x00003300);
+	if (device->chipset < 0xd0)
+		nv_wr32(device, 0x611200, 0x00003300);
 	else
-		nv_wr32(dev, 0x62c000, 0x03030000);
+		nv_wr32(device, 0x62c000, 0x03030000);
 
 	nouveau_mem_exec(&exec, info->perflvl);
 
-	if (dev_priv->chipset < 0xd0)
-		nv_wr32(dev, 0x611200, 0x00003330);
+	if (device->chipset < 0xd0)
+		nv_wr32(device, 0x611200, 0x00003330);
 	else
-		nv_wr32(dev, 0x62c000, 0x03030300);
+		nv_wr32(device, 0x62c000, 0x03030300);
 }
 int
 nvc0_pm_clocks_set(struct drm_device *dev, void *data)
diff --git a/drivers/gpu/drm/nouveau/nvc0_software.c b/drivers/gpu/drm/nouveau/nvc0_software.c
deleted file mode 100644
index 93e8c164fec..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_software.c
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_ramht.h"
-#include "nouveau_software.h"
-
-#include "nv50_display.h"
-
-struct nvc0_software_priv {
-	struct nouveau_software_priv base;
-};
-
-struct nvc0_software_chan {
-	struct nouveau_software_chan base;
-	struct nouveau_vma dispc_vma[4];
-};
-
-u64
-nvc0_software_crtc(struct nouveau_channel *chan, int crtc)
-{
-	struct nvc0_software_chan *pch = chan->engctx[NVOBJ_ENGINE_SW];
-	return pch->dispc_vma[crtc].offset;
-}
-
-static int
-nvc0_software_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_software_priv *psw = nv_engine(dev, NVOBJ_ENGINE_SW);
-	struct nvc0_software_chan *pch;
-	int ret = 0, i;
-
-	pch = kzalloc(sizeof(*pch), GFP_KERNEL);
-	if (!pch)
-		return -ENOMEM;
-
-	nouveau_software_context_new(&pch->base);
-	chan->engctx[engine] = pch;
-
-	/* map display semaphore buffers into channel's vm */
-	for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) {
-		struct nouveau_bo *bo;
-		if (dev_priv->card_type >= NV_D0)
-			bo = nvd0_display_crtc_sema(dev, i);
-		else
-			bo = nv50_display(dev)->crtc[i].sem.bo;
-
-		ret = nouveau_bo_vma_add(bo, chan->vm, &pch->dispc_vma[i]);
-	}
-
-	if (ret)
-		psw->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nvc0_software_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nvc0_software_chan *pch = chan->engctx[engine];
-	int i;
-
-	if (dev_priv->card_type >= NV_D0) {
-		for (i = 0; i < dev->mode_config.num_crtc; i++) {
-			struct nouveau_bo *bo = nvd0_display_crtc_sema(dev, i);
-			nouveau_bo_vma_del(bo, &pch->dispc_vma[i]);
-		}
-	} else
-	if (dev_priv->card_type >= NV_50) {
-		struct nv50_display *disp = nv50_display(dev);
-		for (i = 0; i < dev->mode_config.num_crtc; i++) {
-			struct nv50_display_crtc *dispc = &disp->crtc[i];
-			nouveau_bo_vma_del(dispc->sem.bo, &pch->dispc_vma[i]);
-		}
-	}
-
-	chan->engctx[engine] = NULL;
-	kfree(pch);
-}
-
-static int
-nvc0_software_object_new(struct nouveau_channel *chan, int engine,
-			 u32 handle, u16 class)
-{
-	return 0;
-}
-
-static int
-nvc0_software_init(struct drm_device *dev, int engine)
-{
-	return 0;
-}
-
-static int
-nvc0_software_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static void
-nvc0_software_destroy(struct drm_device *dev, int engine)
-{
-	struct nvc0_software_priv *psw = nv_engine(dev, engine);
-
-	NVOBJ_ENGINE_DEL(dev, SW);
-	kfree(psw);
-}
-
-int
-nvc0_software_create(struct drm_device *dev)
-{
-	struct nvc0_software_priv *psw = kzalloc(sizeof(*psw), GFP_KERNEL);
-	if (!psw)
-		return -ENOMEM;
-
-	psw->base.base.destroy = nvc0_software_destroy;
-	psw->base.base.init = nvc0_software_init;
-	psw->base.base.fini = nvc0_software_fini;
-	psw->base.base.context_new = nvc0_software_context_new;
-	psw->base.base.context_del = nvc0_software_context_del;
-	psw->base.base.object_new = nvc0_software_object_new;
-	nouveau_software_create(&psw->base);
-
-	NVOBJ_ENGINE_ADD(dev, SW, &psw->base.base);
-	NVOBJ_CLASS(dev, 0x906e, SW);
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_vram.c b/drivers/gpu/drm/nouveau/nvc0_vram.c
deleted file mode 100644
index a7eef8934c0..00000000000
--- a/drivers/gpu/drm/nouveau/nvc0_vram.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-
-/* 0 = unsupported
- * 1 = non-compressed
- * 3 = compressed
- */
-static const u8 types[256] = {
-	1, 1, 3, 3, 3, 3, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
-	0, 1, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3,
-	3, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 1, 1, 1, 1, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 3, 3, 3, 3, 1, 1, 1, 1, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
-	3, 3, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3,
-	3, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3, 0, 3, 0, 3,
-	3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 0, 3, 0, 3, 3, 0,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 1, 1, 0
-};
-
-bool
-nvc0_vram_flags_valid(struct drm_device *dev, u32 tile_flags)
-{
-	u8 memtype = (tile_flags & NOUVEAU_GEM_TILE_LAYOUT_MASK) >> 8;
-	return likely((types[memtype] == 1));
-}
-
-int
-nvc0_vram_new(struct drm_device *dev, u64 size, u32 align, u32 ncmin,
-	      u32 type, struct nouveau_mem **pmem)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_mm *mm = &dev_priv->engine.vram.mm;
-	struct nouveau_mm_node *r;
-	struct nouveau_mem *mem;
-	int ret;
-
-	size  >>= 12;
-	align >>= 12;
-	ncmin >>= 12;
-
-	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
-	if (!mem)
-		return -ENOMEM;
-
-	INIT_LIST_HEAD(&mem->regions);
-	mem->dev = dev_priv->dev;
-	mem->memtype = (type & 0xff);
-	mem->size = size;
-
-	mutex_lock(&mm->mutex);
-	do {
-		ret = nouveau_mm_get(mm, 1, size, ncmin, align, &r);
-		if (ret) {
-			mutex_unlock(&mm->mutex);
-			nv50_vram_del(dev, &mem);
-			return ret;
-		}
-
-		list_add_tail(&r->rl_entry, &mem->regions);
-		size -= r->length;
-	} while (size);
-	mutex_unlock(&mm->mutex);
-
-	r = list_first_entry(&mem->regions, struct nouveau_mm_node, rl_entry);
-	mem->offset = (u64)r->offset << 12;
-	*pmem = mem;
-	return 0;
-}
-
-int
-nvc0_vram_init(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_vram_engine *vram = &dev_priv->engine.vram;
-	const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
-	const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
-	u32 parts = nv_rd32(dev, 0x022438);
-	u32 pmask = nv_rd32(dev, 0x022554);
-	u32 bsize = nv_rd32(dev, 0x10f20c);
-	u32 offset, length;
-	bool uniform = true;
-	int ret, part;
-
-	NV_DEBUG(dev, "0x100800: 0x%08x\n", nv_rd32(dev, 0x100800));
-	NV_DEBUG(dev, "parts 0x%08x mask 0x%08x\n", parts, pmask);
-
-	dev_priv->vram_type = nouveau_mem_vbios_type(dev);
-	dev_priv->vram_rank_B = !!(nv_rd32(dev, 0x10f200) & 0x00000004);
-
-	/* read amount of vram attached to each memory controller */
-	for (part = 0; part < parts; part++) {
-		if (!(pmask & (1 << part))) {
-			u32 psize = nv_rd32(dev, 0x11020c + (part * 0x1000));
-			if (psize != bsize) {
-				if (psize < bsize)
-					bsize = psize;
-				uniform = false;
-			}
-
-			NV_DEBUG(dev, "%d: mem_amount 0x%08x\n", part, psize);
-			dev_priv->vram_size += (u64)psize << 20;
-		}
-	}
-
-	/* if all controllers have the same amount attached, there's no holes */
-	if (uniform) {
-		offset = rsvd_head;
-		length = (dev_priv->vram_size >> 12) - rsvd_head - rsvd_tail;
-		return nouveau_mm_init(&vram->mm, offset, length, 1);
-	}
-
-	/* otherwise, address lowest common amount from 0GiB */
-	ret = nouveau_mm_init(&vram->mm, rsvd_head, (bsize << 8) * parts, 1);
-	if (ret)
-		return ret;
-
-	/* and the rest starting from (8GiB + common_size) */
-	offset = (0x0200000000ULL >> 12) + (bsize << 8);
-	length = (dev_priv->vram_size >> 12) - (bsize << 8) - rsvd_tail;
-
-	ret = nouveau_mm_init(&vram->mm, offset, length, 0);
-	if (ret) {
-		nouveau_mm_fini(&vram->mm);
-		return ret;
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nvd0_display.c b/drivers/gpu/drm/nouveau/nvd0_display.c
index 8a2fc89b776..c402fca2b2b 100644
--- a/drivers/gpu/drm/nouveau/nvd0_display.c
+++ b/drivers/gpu/drm/nouveau/nvd0_display.c
@@ -24,18 +24,24 @@
 
 #include <linux/dma-mapping.h>
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 
-#include "nouveau_drv.h"
+#include "nouveau_drm.h"
+#include "nouveau_dma.h"
+#include "nouveau_gem.h"
 #include "nouveau_connector.h"
 #include "nouveau_encoder.h"
 #include "nouveau_crtc.h"
-#include "nouveau_dma.h"
-#include "nouveau_fb.h"
-#include "nouveau_software.h"
+#include "nouveau_fence.h"
 #include "nv50_display.h"
 
+#include <core/gpuobj.h>
+
+#include <subdev/timer.h>
+#include <subdev/bar.h>
+#include <subdev/fb.h>
+
 #define EVO_DMA_NR 9
 
 #define EVO_MASTER  (0x00)
@@ -72,8 +78,7 @@ struct nvd0_display {
 static struct nvd0_display *
 nvd0_display(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	return dev_priv->engine.display.priv;
+	return nouveau_display(dev)->priv;
 }
 
 static struct drm_crtc *
@@ -88,55 +93,47 @@ nvd0_display_crtc_get(struct drm_encoder *encoder)
 static inline int
 evo_icmd(struct drm_device *dev, int id, u32 mthd, u32 data)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	int ret = 0;
-	nv_mask(dev, 0x610700 + (id * 0x10), 0x00000001, 0x00000001);
-	nv_wr32(dev, 0x610704 + (id * 0x10), data);
-	nv_mask(dev, 0x610704 + (id * 0x10), 0x80000ffc, 0x80000000 | mthd);
-	if (!nv_wait(dev, 0x610704 + (id * 0x10), 0x80000000, 0x00000000))
+	nv_mask(device, 0x610700 + (id * 0x10), 0x00000001, 0x00000001);
+	nv_wr32(device, 0x610704 + (id * 0x10), data);
+	nv_mask(device, 0x610704 + (id * 0x10), 0x80000ffc, 0x80000000 | mthd);
+	if (!nv_wait(device, 0x610704 + (id * 0x10), 0x80000000, 0x00000000))
 		ret = -EBUSY;
-	nv_mask(dev, 0x610700 + (id * 0x10), 0x00000001, 0x00000000);
+	nv_mask(device, 0x610700 + (id * 0x10), 0x00000001, 0x00000000);
 	return ret;
 }
 
 static u32 *
 evo_wait(struct drm_device *dev, int id, int nr)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvd0_display *disp = nvd0_display(dev);
-	u32 put = nv_rd32(dev, 0x640000 + (id * 0x1000)) / 4;
+	u32 put = nv_rd32(device, 0x640000 + (id * 0x1000)) / 4;
 
 	if (put + nr >= (PAGE_SIZE / 4)) {
 		disp->evo[id].ptr[put] = 0x20000000;
 
-		nv_wr32(dev, 0x640000 + (id * 0x1000), 0x00000000);
-		if (!nv_wait(dev, 0x640004 + (id * 0x1000), ~0, 0x00000000)) {
-			NV_ERROR(dev, "evo %d dma stalled\n", id);
+		nv_wr32(device, 0x640000 + (id * 0x1000), 0x00000000);
+		if (!nv_wait(device, 0x640004 + (id * 0x1000), ~0, 0x00000000)) {
+			NV_ERROR(drm, "evo %d dma stalled\n", id);
 			return NULL;
 		}
 
 		put = 0;
 	}
 
-	if (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO)
-		NV_INFO(dev, "Evo%d: %p START\n", id, disp->evo[id].ptr + put);
-
 	return disp->evo[id].ptr + put;
 }
 
 static void
 evo_kick(u32 *push, struct drm_device *dev, int id)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nvd0_display *disp = nvd0_display(dev);
 
-	if (nouveau_reg_debug & NOUVEAU_REG_DEBUG_EVO) {
-		u32 curp = nv_rd32(dev, 0x640000 + (id * 0x1000)) >> 2;
-		u32 *cur = disp->evo[id].ptr + curp;
-
-		while (cur < push)
-			NV_INFO(dev, "Evo%d: 0x%08x\n", id, *cur++);
-		NV_INFO(dev, "Evo%d: %p KICK!\n", id, push);
-	}
-
-	nv_wr32(dev, 0x640000 + (id * 0x1000), (push - disp->evo[id].ptr) << 2);
+	nv_wr32(device, 0x640000 + (id * 0x1000), (push - disp->evo[id].ptr) << 2);
 }
 
 #define evo_mthd(p,m,s) *((p)++) = (((s) << 18) | (m))
@@ -145,6 +142,8 @@ evo_kick(u32 *push, struct drm_device *dev, int id)
 static int
 evo_init_dma(struct drm_device *dev, int ch)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvd0_display *disp = nvd0_display(dev);
 	u32 flags;
 
@@ -152,68 +151,76 @@ evo_init_dma(struct drm_device *dev, int ch)
 	if (ch == EVO_MASTER)
 		flags |= 0x01000000;
 
-	nv_wr32(dev, 0x610494 + (ch * 0x0010), (disp->evo[ch].handle >> 8) | 3);
-	nv_wr32(dev, 0x610498 + (ch * 0x0010), 0x00010000);
-	nv_wr32(dev, 0x61049c + (ch * 0x0010), 0x00000001);
-	nv_mask(dev, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000010);
-	nv_wr32(dev, 0x640000 + (ch * 0x1000), 0x00000000);
-	nv_wr32(dev, 0x610490 + (ch * 0x0010), 0x00000013 | flags);
-	if (!nv_wait(dev, 0x610490 + (ch * 0x0010), 0x80000000, 0x00000000)) {
-		NV_ERROR(dev, "PDISP: ch%d 0x%08x\n", ch,
-			      nv_rd32(dev, 0x610490 + (ch * 0x0010)));
+	nv_wr32(device, 0x610494 + (ch * 0x0010), (disp->evo[ch].handle >> 8) | 3);
+	nv_wr32(device, 0x610498 + (ch * 0x0010), 0x00010000);
+	nv_wr32(device, 0x61049c + (ch * 0x0010), 0x00000001);
+	nv_mask(device, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000010);
+	nv_wr32(device, 0x640000 + (ch * 0x1000), 0x00000000);
+	nv_wr32(device, 0x610490 + (ch * 0x0010), 0x00000013 | flags);
+	if (!nv_wait(device, 0x610490 + (ch * 0x0010), 0x80000000, 0x00000000)) {
+		NV_ERROR(drm, "PDISP: ch%d 0x%08x\n", ch,
+			      nv_rd32(device, 0x610490 + (ch * 0x0010)));
 		return -EBUSY;
 	}
 
-	nv_mask(dev, 0x610090, (1 << ch), (1 << ch));
-	nv_mask(dev, 0x6100a0, (1 << ch), (1 << ch));
+	nv_mask(device, 0x610090, (1 << ch), (1 << ch));
+	nv_mask(device, 0x6100a0, (1 << ch), (1 << ch));
 	return 0;
 }
 
 static void
 evo_fini_dma(struct drm_device *dev, int ch)
 {
-	if (!(nv_rd32(dev, 0x610490 + (ch * 0x0010)) & 0x00000010))
+	struct nouveau_device *device = nouveau_dev(dev);
+
+	if (!(nv_rd32(device, 0x610490 + (ch * 0x0010)) & 0x00000010))
 		return;
 
-	nv_mask(dev, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000000);
-	nv_mask(dev, 0x610490 + (ch * 0x0010), 0x00000003, 0x00000000);
-	nv_wait(dev, 0x610490 + (ch * 0x0010), 0x80000000, 0x00000000);
-	nv_mask(dev, 0x610090, (1 << ch), 0x00000000);
-	nv_mask(dev, 0x6100a0, (1 << ch), 0x00000000);
+	nv_mask(device, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000000);
+	nv_mask(device, 0x610490 + (ch * 0x0010), 0x00000003, 0x00000000);
+	nv_wait(device, 0x610490 + (ch * 0x0010), 0x80000000, 0x00000000);
+	nv_mask(device, 0x610090, (1 << ch), 0x00000000);
+	nv_mask(device, 0x6100a0, (1 << ch), 0x00000000);
 }
 
 static inline void
 evo_piow(struct drm_device *dev, int ch, u16 mthd, u32 data)
 {
-	nv_wr32(dev, 0x640000 + (ch * 0x1000) + mthd, data);
+	struct nouveau_device *device = nouveau_dev(dev);
+	nv_wr32(device, 0x640000 + (ch * 0x1000) + mthd, data);
 }
 
 static int
 evo_init_pio(struct drm_device *dev, int ch)
 {
-	nv_wr32(dev, 0x610490 + (ch * 0x0010), 0x00000001);
-	if (!nv_wait(dev, 0x610490 + (ch * 0x0010), 0x00010000, 0x00010000)) {
-		NV_ERROR(dev, "PDISP: ch%d 0x%08x\n", ch,
-			      nv_rd32(dev, 0x610490 + (ch * 0x0010)));
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+
+	nv_wr32(device, 0x610490 + (ch * 0x0010), 0x00000001);
+	if (!nv_wait(device, 0x610490 + (ch * 0x0010), 0x00010000, 0x00010000)) {
+		NV_ERROR(drm, "PDISP: ch%d 0x%08x\n", ch,
+			      nv_rd32(device, 0x610490 + (ch * 0x0010)));
 		return -EBUSY;
 	}
 
-	nv_mask(dev, 0x610090, (1 << ch), (1 << ch));
-	nv_mask(dev, 0x6100a0, (1 << ch), (1 << ch));
+	nv_mask(device, 0x610090, (1 << ch), (1 << ch));
+	nv_mask(device, 0x6100a0, (1 << ch), (1 << ch));
 	return 0;
 }
 
 static void
 evo_fini_pio(struct drm_device *dev, int ch)
 {
-	if (!(nv_rd32(dev, 0x610490 + (ch * 0x0010)) & 0x00000001))
+	struct nouveau_device *device = nouveau_dev(dev);
+
+	if (!(nv_rd32(device, 0x610490 + (ch * 0x0010)) & 0x00000001))
 		return;
 
-	nv_mask(dev, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000010);
-	nv_mask(dev, 0x610490 + (ch * 0x0010), 0x00000001, 0x00000000);
-	nv_wait(dev, 0x610490 + (ch * 0x0010), 0x00010000, 0x00000000);
-	nv_mask(dev, 0x610090, (1 << ch), 0x00000000);
-	nv_mask(dev, 0x6100a0, (1 << ch), 0x00000000);
+	nv_mask(device, 0x610490 + (ch * 0x0010), 0x00000010, 0x00000010);
+	nv_mask(device, 0x610490 + (ch * 0x0010), 0x00000001, 0x00000000);
+	nv_wait(device, 0x610490 + (ch * 0x0010), 0x00010000, 0x00000000);
+	nv_mask(device, 0x610090, (1 << ch), 0x00000000);
+	nv_mask(device, 0x6100a0, (1 << ch), 0x00000000);
 }
 
 static bool
@@ -225,6 +232,7 @@ evo_sync_wait(void *data)
 static int
 evo_sync(struct drm_device *dev, int ch)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct nvd0_display *disp = nvd0_display(dev);
 	u32 *push = evo_wait(dev, ch, 8);
 	if (push) {
@@ -235,7 +243,7 @@ evo_sync(struct drm_device *dev, int ch)
 		evo_data(push, 0x00000000);
 		evo_data(push, 0x00000000);
 		evo_kick(push, dev, ch);
-		if (nv_wait_cb(dev, evo_sync_wait, disp->sync))
+		if (nv_wait_cb(device, evo_sync_wait, disp->sync))
 			return 0;
 	}
 
@@ -300,7 +308,7 @@ nvd0_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 			return ret;
 
 
-		offset  = nvc0_software_crtc(chan, nv_crtc->index);
+		offset  = nvc0_fence_crtc(chan, nv_crtc->index);
 		offset += evo->sem.offset;
 
 		BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
@@ -363,7 +371,7 @@ nvd0_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 static int
 nvd0_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
 {
-	struct drm_nouveau_private *dev_priv = nv_crtc->base.dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(nv_crtc->base.dev);
 	struct drm_device *dev = nv_crtc->base.dev;
 	struct nouveau_connector *nv_connector;
 	struct drm_connector *connector;
@@ -386,7 +394,7 @@ nvd0_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
 		mode |= nv_connector->dithering_depth;
 	}
 
-	if (dev_priv->card_type < NV_E0)
+	if (nv_device(drm->device)->card_type < NV_E0)
 		mthd = 0x0490 + (nv_crtc->index * 0x0300);
 	else
 		mthd = 0x04a0 + (nv_crtc->index * 0x0300);
@@ -701,11 +709,12 @@ static int
 nvd0_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
 			struct drm_framebuffer *old_fb)
 {
+	struct nouveau_drm *drm = nouveau_drm(crtc->dev);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	int ret;
 
 	if (!crtc->fb) {
-		NV_DEBUG_KMS(crtc->dev, "No FB bound\n");
+		NV_DEBUG(drm, "No FB bound\n");
 		return 0;
 	}
 
@@ -923,6 +932,7 @@ nvd0_dac_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int or = nv_encoder->or;
 	u32 dpms_ctrl;
 
@@ -932,9 +942,9 @@ nvd0_dac_dpms(struct drm_encoder *encoder, int mode)
 	if (mode == DRM_MODE_DPMS_SUSPEND || mode == DRM_MODE_DPMS_OFF)
 		dpms_ctrl |= 0x00000004;
 
-	nv_wait(dev, 0x61a004 + (or * 0x0800), 0x80000000, 0x00000000);
-	nv_mask(dev, 0x61a004 + (or * 0x0800), 0xc000007f, dpms_ctrl);
-	nv_wait(dev, 0x61a004 + (or * 0x0800), 0x80000000, 0x00000000);
+	nv_wait(device, 0x61a004 + (or * 0x0800), 0x80000000, 0x00000000);
+	nv_mask(device, 0x61a004 + (or * 0x0800), 0xc000007f, dpms_ctrl);
+	nv_wait(device, 0x61a004 + (or * 0x0800), 0x80000000, 0x00000000);
 }
 
 static bool
@@ -1025,18 +1035,19 @@ nvd0_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
 	enum drm_connector_status status = connector_status_disconnected;
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int or = nv_encoder->or;
 	u32 load;
 
-	nv_wr32(dev, 0x61a00c + (or * 0x800), 0x00100000);
+	nv_wr32(device, 0x61a00c + (or * 0x800), 0x00100000);
 	udelay(9500);
-	nv_wr32(dev, 0x61a00c + (or * 0x800), 0x80000000);
+	nv_wr32(device, 0x61a00c + (or * 0x800), 0x80000000);
 
-	load = nv_rd32(dev, 0x61a00c + (or * 0x800));
+	load = nv_rd32(device, 0x61a00c + (or * 0x800));
 	if ((load & 0x38000000) == 0x38000000)
 		status = connector_status_connected;
 
-	nv_wr32(dev, 0x61a00c + (or * 0x800), 0x00000000);
+	nv_wr32(device, 0x61a00c + (or * 0x800), 0x00000000);
 	return status;
 }
 
@@ -1063,7 +1074,7 @@ static const struct drm_encoder_funcs nvd0_dac_func = {
 };
 
 static int
-nvd0_dac_create(struct drm_connector *connector, struct dcb_entry *dcbe)
+nvd0_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
 	struct drm_device *dev = connector->dev;
 	struct nouveau_encoder *nv_encoder;
@@ -1094,24 +1105,25 @@ nvd0_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_connector *nv_connector;
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int i, or = nv_encoder->or * 0x30;
 
 	nv_connector = nouveau_encoder_connector_get(nv_encoder);
 	if (!drm_detect_monitor_audio(nv_connector->edid))
 		return;
 
-	nv_mask(dev, 0x10ec10 + or, 0x80000003, 0x80000001);
+	nv_mask(device, 0x10ec10 + or, 0x80000003, 0x80000001);
 
 	drm_edid_to_eld(&nv_connector->base, nv_connector->edid);
 	if (nv_connector->base.eld[0]) {
 		u8 *eld = nv_connector->base.eld;
 
 		for (i = 0; i < eld[2] * 4; i++)
-			nv_wr32(dev, 0x10ec00 + or, (i << 8) | eld[i]);
+			nv_wr32(device, 0x10ec00 + or, (i << 8) | eld[i]);
 		for (i = eld[2] * 4; i < 0x60; i++)
-			nv_wr32(dev, 0x10ec00 + or, (i << 8) | 0x00);
+			nv_wr32(device, 0x10ec00 + or, (i << 8) | 0x00);
 
-		nv_mask(dev, 0x10ec10 + or, 0x80000002, 0x80000002);
+		nv_mask(device, 0x10ec10 + or, 0x80000002, 0x80000002);
 	}
 }
 
@@ -1120,9 +1132,10 @@ nvd0_audio_disconnect(struct drm_encoder *encoder)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int or = nv_encoder->or * 0x30;
 
-	nv_mask(dev, 0x10ec10 + or, 0x80000003, 0x80000000);
+	nv_mask(device, 0x10ec10 + or, 0x80000003, 0x80000000);
 }
 
 /******************************************************************************
@@ -1135,6 +1148,7 @@ nvd0_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_connector *nv_connector;
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int head = nv_crtc->index * 0x800;
 	u32 rekey = 56; /* binary driver, and tegra constant */
 	u32 max_ac_packet;
@@ -1149,25 +1163,25 @@ nvd0_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	max_ac_packet /= 32;
 
 	/* AVI InfoFrame */
-	nv_mask(dev, 0x616714 + head, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x61671c + head, 0x000d0282);
-	nv_wr32(dev, 0x616720 + head, 0x0000006f);
-	nv_wr32(dev, 0x616724 + head, 0x00000000);
-	nv_wr32(dev, 0x616728 + head, 0x00000000);
-	nv_wr32(dev, 0x61672c + head, 0x00000000);
-	nv_mask(dev, 0x616714 + head, 0x00000001, 0x00000001);
+	nv_mask(device, 0x616714 + head, 0x00000001, 0x00000000);
+	nv_wr32(device, 0x61671c + head, 0x000d0282);
+	nv_wr32(device, 0x616720 + head, 0x0000006f);
+	nv_wr32(device, 0x616724 + head, 0x00000000);
+	nv_wr32(device, 0x616728 + head, 0x00000000);
+	nv_wr32(device, 0x61672c + head, 0x00000000);
+	nv_mask(device, 0x616714 + head, 0x00000001, 0x00000001);
 
 	/* ??? InfoFrame? */
-	nv_mask(dev, 0x6167a4 + head, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x6167ac + head, 0x00000010);
-	nv_mask(dev, 0x6167a4 + head, 0x00000001, 0x00000001);
+	nv_mask(device, 0x6167a4 + head, 0x00000001, 0x00000000);
+	nv_wr32(device, 0x6167ac + head, 0x00000010);
+	nv_mask(device, 0x6167a4 + head, 0x00000001, 0x00000001);
 
 	/* HDMI_CTRL */
-	nv_mask(dev, 0x616798 + head, 0x401f007f, 0x40000000 | rekey |
+	nv_mask(device, 0x616798 + head, 0x401f007f, 0x40000000 | rekey |
 						  max_ac_packet << 16);
 
 	/* NFI, audio doesn't work without it though.. */
-	nv_mask(dev, 0x616548 + head, 0x00000070, 0x00000000);
+	nv_mask(device, 0x616548 + head, 0x00000070, 0x00000000);
 
 	nvd0_audio_mode_set(encoder, mode);
 }
@@ -1178,37 +1192,41 @@ nvd0_hdmi_disconnect(struct drm_encoder *encoder)
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	int head = nv_crtc->index * 0x800;
 
 	nvd0_audio_disconnect(encoder);
 
-	nv_mask(dev, 0x616798 + head, 0x40000000, 0x00000000);
-	nv_mask(dev, 0x6167a4 + head, 0x00000001, 0x00000000);
-	nv_mask(dev, 0x616714 + head, 0x00000001, 0x00000000);
+	nv_mask(device, 0x616798 + head, 0x40000000, 0x00000000);
+	nv_mask(device, 0x6167a4 + head, 0x00000001, 0x00000000);
+	nv_mask(device, 0x616714 + head, 0x00000001, 0x00000000);
 }
 
 /******************************************************************************
  * SOR
  *****************************************************************************/
 static inline u32
-nvd0_sor_dp_lane_map(struct drm_device *dev, struct dcb_entry *dcb, u8 lane)
+nvd0_sor_dp_lane_map(struct drm_device *dev, struct dcb_output *dcb, u8 lane)
 {
 	static const u8 nvd0[] = { 16, 8, 0, 24 };
 	return nvd0[lane];
 }
 
 static void
-nvd0_sor_dp_train_set(struct drm_device *dev, struct dcb_entry *dcb, u8 pattern)
+nvd0_sor_dp_train_set(struct drm_device *dev, struct dcb_output *dcb, u8 pattern)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	const u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
 	const u32 loff = (or * 0x800) + (link * 0x80);
-	nv_mask(dev, 0x61c110 + loff, 0x0f0f0f0f, 0x01010101 * pattern);
+	nv_mask(device, 0x61c110 + loff, 0x0f0f0f0f, 0x01010101 * pattern);
 }
 
 static void
-nvd0_sor_dp_train_adj(struct drm_device *dev, struct dcb_entry *dcb,
+nvd0_sor_dp_train_adj(struct drm_device *dev, struct dcb_output *dcb,
 		      u8 lane, u8 swing, u8 preem)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	const u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
 	const u32 loff = (or * 0x800) + (link * 0x80);
 	u32 shift = nvd0_sor_dp_lane_map(dev, dcb, lane);
@@ -1236,25 +1254,26 @@ nvd0_sor_dp_train_adj(struct drm_device *dev, struct dcb_entry *dcb,
 	}
 
 	if (!config) {
-		NV_ERROR(dev, "PDISP: unsupported DP table for chipset\n");
+		NV_ERROR(drm, "PDISP: unsupported DP table for chipset\n");
 		return;
 	}
 
-	nv_mask(dev, 0x61c118 + loff, mask, config[1] << shift);
-	nv_mask(dev, 0x61c120 + loff, mask, config[2] << shift);
-	nv_mask(dev, 0x61c130 + loff, 0x0000ff00, config[3] << 8);
-	nv_mask(dev, 0x61c13c + loff, 0x00000000, 0x00000000);
+	nv_mask(device, 0x61c118 + loff, mask, config[1] << shift);
+	nv_mask(device, 0x61c120 + loff, mask, config[2] << shift);
+	nv_mask(device, 0x61c130 + loff, 0x0000ff00, config[3] << 8);
+	nv_mask(device, 0x61c13c + loff, 0x00000000, 0x00000000);
 }
 
 static void
-nvd0_sor_dp_link_set(struct drm_device *dev, struct dcb_entry *dcb, int crtc,
+nvd0_sor_dp_link_set(struct drm_device *dev, struct dcb_output *dcb, int crtc,
 		     int link_nr, u32 link_bw, bool enhframe)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	const u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
 	const u32 loff = (or * 0x800) + (link * 0x80);
 	const u32 soff = (or * 0x800);
-	u32 dpctrl = nv_rd32(dev, 0x61c10c + loff) & ~0x001f4000;
-	u32 clksor = nv_rd32(dev, 0x612300 + soff) & ~0x007c0000;
+	u32 dpctrl = nv_rd32(device, 0x61c10c + loff) & ~0x001f4000;
+	u32 clksor = nv_rd32(device, 0x612300 + soff) & ~0x007c0000;
 	u32 script = 0x0000, lane_mask = 0;
 	u8 *table, *entry;
 	int i;
@@ -1284,20 +1303,21 @@ nvd0_sor_dp_link_set(struct drm_device *dev, struct dcb_entry *dcb, int crtc,
 	for (i = 0; i < link_nr; i++)
 		lane_mask |= 1 << (nvd0_sor_dp_lane_map(dev, dcb, i) >> 3);
 
-	nv_wr32(dev, 0x612300 + soff, clksor);
-	nv_wr32(dev, 0x61c10c + loff, dpctrl);
-	nv_mask(dev, 0x61c130 + loff, 0x0000000f, lane_mask);
+	nv_wr32(device, 0x612300 + soff, clksor);
+	nv_wr32(device, 0x61c10c + loff, dpctrl);
+	nv_mask(device, 0x61c130 + loff, 0x0000000f, lane_mask);
 }
 
 static void
-nvd0_sor_dp_link_get(struct drm_device *dev, struct dcb_entry *dcb,
+nvd0_sor_dp_link_get(struct drm_device *dev, struct dcb_output *dcb,
 		     u32 *link_nr, u32 *link_bw)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	const u32 or = ffs(dcb->or) - 1, link = !(dcb->sorconf.link & 1);
 	const u32 loff = (or * 0x800) + (link * 0x80);
 	const u32 soff = (or * 0x800);
-	u32 dpctrl = nv_rd32(dev, 0x61c10c + loff) & 0x000f0000;
-	u32 clksor = nv_rd32(dev, 0x612300 + soff);
+	u32 dpctrl = nv_rd32(device, 0x61c10c + loff) & 0x000f0000;
+	u32 clksor = nv_rd32(device, 0x612300 + soff);
 
 	if      (dpctrl > 0x00030000) *link_nr = 4;
 	else if (dpctrl > 0x00010000) *link_nr = 2;
@@ -1308,9 +1328,10 @@ nvd0_sor_dp_link_get(struct drm_device *dev, struct dcb_entry *dcb,
 }
 
 static void
-nvd0_sor_dp_calc_tu(struct drm_device *dev, struct dcb_entry *dcb,
+nvd0_sor_dp_calc_tu(struct drm_device *dev, struct dcb_output *dcb,
 		    u32 crtc, u32 datarate)
 {
+	struct nouveau_device *device = nouveau_dev(dev);
 	const u32 symbol = 100000;
 	const u32 TU = 64;
 	u32 link_nr, link_bw;
@@ -1330,7 +1351,7 @@ nvd0_sor_dp_calc_tu(struct drm_device *dev, struct dcb_entry *dcb,
 	value += 5;
 	value |= 0x08000000;
 
-	nv_wr32(dev, 0x616610 + (crtc * 0x800), value);
+	nv_wr32(device, 0x616610 + (crtc * 0x800), value);
 }
 
 static void
@@ -1338,6 +1359,7 @@ nvd0_sor_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct drm_device *dev = encoder->dev;
+	struct nouveau_device *device = nouveau_dev(dev);
 	struct drm_encoder *partner;
 	int or = nv_encoder->or;
 	u32 dpms_ctrl;
@@ -1361,12 +1383,12 @@ nvd0_sor_dpms(struct drm_encoder *encoder, int mode)
 	dpms_ctrl  = (mode == DRM_MODE_DPMS_ON);
 	dpms_ctrl |= 0x80000000;
 
-	nv_wait(dev, 0x61c004 + (or * 0x0800), 0x80000000, 0x00000000);
-	nv_mask(dev, 0x61c004 + (or * 0x0800), 0x80000001, dpms_ctrl);
-	nv_wait(dev, 0x61c004 + (or * 0x0800), 0x80000000, 0x00000000);
-	nv_wait(dev, 0x61c030 + (or * 0x0800), 0x10000000, 0x00000000);
+	nv_wait(device, 0x61c004 + (or * 0x0800), 0x80000000, 0x00000000);
+	nv_mask(device, 0x61c004 + (or * 0x0800), 0x80000001, dpms_ctrl);
+	nv_wait(device, 0x61c004 + (or * 0x0800), 0x80000000, 0x00000000);
+	nv_wait(device, 0x61c030 + (or * 0x0800), 0x10000000, 0x00000000);
 
-	if (nv_encoder->dcb->type == OUTPUT_DP) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
 		struct dp_train_func func = {
 			.link_set = nvd0_sor_dp_link_set,
 			.train_set = nvd0_sor_dp_train_set,
@@ -1427,7 +1449,7 @@ static void
 nvd0_sor_prepare(struct drm_encoder *encoder)
 {
 	nvd0_sor_disconnect(encoder);
-	if (nouveau_encoder(encoder)->dcb->type == OUTPUT_DP)
+	if (nouveau_encoder(encoder)->dcb->type == DCB_OUTPUT_DP)
 		evo_sync(encoder->dev, EVO_MASTER);
 }
 
@@ -1441,11 +1463,11 @@ nvd0_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 		  struct drm_display_mode *mode)
 {
 	struct drm_device *dev = encoder->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
 	struct nouveau_connector *nv_connector;
-	struct nvbios *bios = &dev_priv->vbios;
+	struct nvbios *bios = &drm->vbios;
 	u32 mode_ctrl = (1 << nv_crtc->index);
 	u32 syncs, magic, *push;
 	u32 or_config;
@@ -1462,7 +1484,7 @@ nvd0_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 	nv_connector = nouveau_encoder_connector_get(nv_encoder);
 	switch (nv_encoder->dcb->type) {
-	case OUTPUT_TMDS:
+	case DCB_OUTPUT_TMDS:
 		if (nv_encoder->dcb->sorconf.link & 1) {
 			if (mode->clock < 165000)
 				mode_ctrl |= 0x00000100;
@@ -1478,7 +1500,7 @@ nvd0_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 		nvd0_hdmi_mode_set(encoder, mode);
 		break;
-	case OUTPUT_LVDS:
+	case DCB_OUTPUT_LVDS:
 		or_config = (mode_ctrl & 0x00000f00) >> 8;
 		if (bios->fp_no_ddc) {
 			if (bios->fp.dual_link)
@@ -1507,7 +1529,7 @@ nvd0_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 		}
 		break;
-	case OUTPUT_DP:
+	case DCB_OUTPUT_DP:
 		if (nv_connector->base.display_info.bpc == 6) {
 			nv_encoder->dp.datarate = mode->clock * 18 / 8;
 			syncs |= 0x00000002 << 6;
@@ -1530,7 +1552,7 @@ nvd0_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
 
 	nvd0_sor_dpms(encoder, DRM_MODE_DPMS_ON);
 
-	if (nv_encoder->dcb->type == OUTPUT_DP) {
+	if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
 		nvd0_sor_dp_calc_tu(dev, nv_encoder->dcb, nv_crtc->index,
 					 nv_encoder->dp.datarate);
 	}
@@ -1571,7 +1593,7 @@ static const struct drm_encoder_funcs nvd0_sor_func = {
 };
 
 static int
-nvd0_sor_create(struct drm_connector *connector, struct dcb_entry *dcbe)
+nvd0_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
 {
 	struct drm_device *dev = connector->dev;
 	struct nouveau_encoder *nv_encoder;
@@ -1597,50 +1619,51 @@ nvd0_sor_create(struct drm_connector *connector, struct dcb_entry *dcbe)
 /******************************************************************************
  * IRQ
  *****************************************************************************/
-static struct dcb_entry *
+static struct dcb_output *
 lookup_dcb(struct drm_device *dev, int id, u32 mc)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int type, or, i, link = -1;
 
 	if (id < 4) {
-		type = OUTPUT_ANALOG;
+		type = DCB_OUTPUT_ANALOG;
 		or   = id;
 	} else {
 		switch (mc & 0x00000f00) {
-		case 0x00000000: link = 0; type = OUTPUT_LVDS; break;
-		case 0x00000100: link = 0; type = OUTPUT_TMDS; break;
-		case 0x00000200: link = 1; type = OUTPUT_TMDS; break;
-		case 0x00000500: link = 0; type = OUTPUT_TMDS; break;
-		case 0x00000800: link = 0; type = OUTPUT_DP; break;
-		case 0x00000900: link = 1; type = OUTPUT_DP; break;
+		case 0x00000000: link = 0; type = DCB_OUTPUT_LVDS; break;
+		case 0x00000100: link = 0; type = DCB_OUTPUT_TMDS; break;
+		case 0x00000200: link = 1; type = DCB_OUTPUT_TMDS; break;
+		case 0x00000500: link = 0; type = DCB_OUTPUT_TMDS; break;
+		case 0x00000800: link = 0; type = DCB_OUTPUT_DP; break;
+		case 0x00000900: link = 1; type = DCB_OUTPUT_DP; break;
 		default:
-			NV_ERROR(dev, "PDISP: unknown SOR mc 0x%08x\n", mc);
+			NV_ERROR(drm, "PDISP: unknown SOR mc 0x%08x\n", mc);
 			return NULL;
 		}
 
 		or = id - 4;
 	}
 
-	for (i = 0; i < dev_priv->vbios.dcb.entries; i++) {
-		struct dcb_entry *dcb = &dev_priv->vbios.dcb.entry[i];
+	for (i = 0; i < drm->vbios.dcb.entries; i++) {
+		struct dcb_output *dcb = &drm->vbios.dcb.entry[i];
 		if (dcb->type == type && (dcb->or & (1 << or)) &&
 		    (link < 0 || link == !(dcb->sorconf.link & 1)))
 			return dcb;
 	}
 
-	NV_ERROR(dev, "PDISP: DCB for %d/0x%08x not found\n", id, mc);
+	NV_ERROR(drm, "PDISP: DCB for %d/0x%08x not found\n", id, mc);
 	return NULL;
 }
 
 static void
 nvd0_display_unk1_handler(struct drm_device *dev, u32 crtc, u32 mask)
 {
-	struct dcb_entry *dcb;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct dcb_output *dcb;
 	int i;
 
 	for (i = 0; mask && i < 8; i++) {
-		u32 mcc = nv_rd32(dev, 0x640180 + (i * 0x20));
+		u32 mcc = nv_rd32(device, 0x640180 + (i * 0x20));
 		if (!(mcc & (1 << crtc)))
 			continue;
 
@@ -1651,20 +1674,22 @@ nvd0_display_unk1_handler(struct drm_device *dev, u32 crtc, u32 mask)
 		nouveau_bios_run_display_table(dev, 0x0000, -1, dcb, crtc);
 	}
 
-	nv_wr32(dev, 0x6101d4, 0x00000000);
-	nv_wr32(dev, 0x6109d4, 0x00000000);
-	nv_wr32(dev, 0x6101d0, 0x80000000);
+	nv_wr32(device, 0x6101d4, 0x00000000);
+	nv_wr32(device, 0x6109d4, 0x00000000);
+	nv_wr32(device, 0x6101d0, 0x80000000);
 }
 
 static void
 nvd0_display_unk2_handler(struct drm_device *dev, u32 crtc, u32 mask)
 {
-	struct dcb_entry *dcb;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct dcb_output *dcb;
 	u32 or, tmp, pclk;
 	int i;
 
 	for (i = 0; mask && i < 8; i++) {
-		u32 mcc = nv_rd32(dev, 0x640180 + (i * 0x20));
+		u32 mcc = nv_rd32(device, 0x640180 + (i * 0x20));
 		if (!(mcc & (1 << crtc)))
 			continue;
 
@@ -1675,16 +1700,16 @@ nvd0_display_unk2_handler(struct drm_device *dev, u32 crtc, u32 mask)
 		nouveau_bios_run_display_table(dev, 0x0000, -2, dcb, crtc);
 	}
 
-	pclk = nv_rd32(dev, 0x660450 + (crtc * 0x300)) / 1000;
-	NV_DEBUG_KMS(dev, "PDISP: crtc %d pclk %d mask 0x%08x\n",
+	pclk = nv_rd32(device, 0x660450 + (crtc * 0x300)) / 1000;
+	NV_DEBUG(drm, "PDISP: crtc %d pclk %d mask 0x%08x\n",
 			  crtc, pclk, mask);
 	if (pclk && (mask & 0x00010000)) {
 		nv50_crtc_set_clock(dev, crtc, pclk);
 	}
 
 	for (i = 0; mask && i < 8; i++) {
-		u32 mcp = nv_rd32(dev, 0x660180 + (i * 0x20));
-		u32 cfg = nv_rd32(dev, 0x660184 + (i * 0x20));
+		u32 mcp = nv_rd32(device, 0x660180 + (i * 0x20));
+		u32 cfg = nv_rd32(device, 0x660184 + (i * 0x20));
 		if (!(mcp & (1 << crtc)))
 			continue;
 
@@ -1695,20 +1720,20 @@ nvd0_display_unk2_handler(struct drm_device *dev, u32 crtc, u32 mask)
 
 		nouveau_bios_run_display_table(dev, cfg, pclk, dcb, crtc);
 
-		nv_wr32(dev, 0x612200 + (crtc * 0x800), 0x00000000);
+		nv_wr32(device, 0x612200 + (crtc * 0x800), 0x00000000);
 		switch (dcb->type) {
-		case OUTPUT_ANALOG:
-			nv_wr32(dev, 0x612280 + (or * 0x800), 0x00000000);
+		case DCB_OUTPUT_ANALOG:
+			nv_wr32(device, 0x612280 + (or * 0x800), 0x00000000);
 			break;
-		case OUTPUT_TMDS:
-		case OUTPUT_LVDS:
-		case OUTPUT_DP:
+		case DCB_OUTPUT_TMDS:
+		case DCB_OUTPUT_LVDS:
+		case DCB_OUTPUT_DP:
 			if (cfg & 0x00000100)
 				tmp = 0x00000101;
 			else
 				tmp = 0x00000000;
 
-			nv_mask(dev, 0x612300 + (or * 0x800), 0x00000707, tmp);
+			nv_mask(device, 0x612300 + (or * 0x800), 0x00000707, tmp);
 			break;
 		default:
 			break;
@@ -1717,22 +1742,23 @@ nvd0_display_unk2_handler(struct drm_device *dev, u32 crtc, u32 mask)
 		break;
 	}
 
-	nv_wr32(dev, 0x6101d4, 0x00000000);
-	nv_wr32(dev, 0x6109d4, 0x00000000);
-	nv_wr32(dev, 0x6101d0, 0x80000000);
+	nv_wr32(device, 0x6101d4, 0x00000000);
+	nv_wr32(device, 0x6109d4, 0x00000000);
+	nv_wr32(device, 0x6101d0, 0x80000000);
 }
 
 static void
 nvd0_display_unk4_handler(struct drm_device *dev, u32 crtc, u32 mask)
 {
-	struct dcb_entry *dcb;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct dcb_output *dcb;
 	int pclk, i;
 
-	pclk = nv_rd32(dev, 0x660450 + (crtc * 0x300)) / 1000;
+	pclk = nv_rd32(device, 0x660450 + (crtc * 0x300)) / 1000;
 
 	for (i = 0; mask && i < 8; i++) {
-		u32 mcp = nv_rd32(dev, 0x660180 + (i * 0x20));
-		u32 cfg = nv_rd32(dev, 0x660184 + (i * 0x20));
+		u32 mcp = nv_rd32(device, 0x660180 + (i * 0x20));
+		u32 cfg = nv_rd32(device, 0x660184 + (i * 0x20));
 		if (!(mcp & (1 << crtc)))
 			continue;
 
@@ -1743,34 +1769,36 @@ nvd0_display_unk4_handler(struct drm_device *dev, u32 crtc, u32 mask)
 		nouveau_bios_run_display_table(dev, cfg, -pclk, dcb, crtc);
 	}
 
-	nv_wr32(dev, 0x6101d4, 0x00000000);
-	nv_wr32(dev, 0x6109d4, 0x00000000);
-	nv_wr32(dev, 0x6101d0, 0x80000000);
+	nv_wr32(device, 0x6101d4, 0x00000000);
+	nv_wr32(device, 0x6109d4, 0x00000000);
+	nv_wr32(device, 0x6101d0, 0x80000000);
 }
 
 static void
 nvd0_display_bh(unsigned long data)
 {
 	struct drm_device *dev = (struct drm_device *)data;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nvd0_display *disp = nvd0_display(dev);
 	u32 mask = 0, crtc = ~0;
 	int i;
 
 	if (drm_debug & (DRM_UT_DRIVER | DRM_UT_KMS)) {
-		NV_INFO(dev, "PDISP: modeset req %d\n", disp->modeset);
-		NV_INFO(dev, " STAT: 0x%08x 0x%08x 0x%08x\n",
-			 nv_rd32(dev, 0x6101d0),
-			 nv_rd32(dev, 0x6101d4), nv_rd32(dev, 0x6109d4));
+		NV_INFO(drm, "PDISP: modeset req %d\n", disp->modeset);
+		NV_INFO(drm, " STAT: 0x%08x 0x%08x 0x%08x\n",
+			 nv_rd32(device, 0x6101d0),
+			 nv_rd32(device, 0x6101d4), nv_rd32(device, 0x6109d4));
 		for (i = 0; i < 8; i++) {
-			NV_INFO(dev, " %s%d: 0x%08x 0x%08x\n",
+			NV_INFO(drm, " %s%d: 0x%08x 0x%08x\n",
 				i < 4 ? "DAC" : "SOR", i,
-				nv_rd32(dev, 0x640180 + (i * 0x20)),
-				nv_rd32(dev, 0x660180 + (i * 0x20)));
+				nv_rd32(device, 0x640180 + (i * 0x20)),
+				nv_rd32(device, 0x660180 + (i * 0x20)));
 		}
 	}
 
 	while (!mask && ++crtc < dev->mode_config.num_crtc)
-		mask = nv_rd32(dev, 0x6101d4 + (crtc * 0x800));
+		mask = nv_rd32(device, 0x6101d4 + (crtc * 0x800));
 
 	if (disp->modeset & 0x00000001)
 		nvd0_display_unk1_handler(dev, crtc, mask);
@@ -1780,67 +1808,60 @@ nvd0_display_bh(unsigned long data)
 		nvd0_display_unk4_handler(dev, crtc, mask);
 }
 
-static void
+void
 nvd0_display_intr(struct drm_device *dev)
 {
 	struct nvd0_display *disp = nvd0_display(dev);
-	u32 intr = nv_rd32(dev, 0x610088);
-	int i;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	u32 intr = nv_rd32(device, 0x610088);
 
 	if (intr & 0x00000001) {
-		u32 stat = nv_rd32(dev, 0x61008c);
-		nv_wr32(dev, 0x61008c, stat);
+		u32 stat = nv_rd32(device, 0x61008c);
+		nv_wr32(device, 0x61008c, stat);
 		intr &= ~0x00000001;
 	}
 
 	if (intr & 0x00000002) {
-		u32 stat = nv_rd32(dev, 0x61009c);
+		u32 stat = nv_rd32(device, 0x61009c);
 		int chid = ffs(stat) - 1;
 		if (chid >= 0) {
-			u32 mthd = nv_rd32(dev, 0x6101f0 + (chid * 12));
-			u32 data = nv_rd32(dev, 0x6101f4 + (chid * 12));
-			u32 unkn = nv_rd32(dev, 0x6101f8 + (chid * 12));
+			u32 mthd = nv_rd32(device, 0x6101f0 + (chid * 12));
+			u32 data = nv_rd32(device, 0x6101f4 + (chid * 12));
+			u32 unkn = nv_rd32(device, 0x6101f8 + (chid * 12));
 
-			NV_INFO(dev, "EvoCh: chid %d mthd 0x%04x data 0x%08x "
+			NV_INFO(drm, "EvoCh: chid %d mthd 0x%04x data 0x%08x "
 				     "0x%08x 0x%08x\n",
 				chid, (mthd & 0x0000ffc), data, mthd, unkn);
-			nv_wr32(dev, 0x61009c, (1 << chid));
-			nv_wr32(dev, 0x6101f0 + (chid * 12), 0x90000000);
+			nv_wr32(device, 0x61009c, (1 << chid));
+			nv_wr32(device, 0x6101f0 + (chid * 12), 0x90000000);
 		}
 
 		intr &= ~0x00000002;
 	}
 
 	if (intr & 0x00100000) {
-		u32 stat = nv_rd32(dev, 0x6100ac);
+		u32 stat = nv_rd32(device, 0x6100ac);
 
 		if (stat & 0x00000007) {
 			disp->modeset = stat;
 			tasklet_schedule(&disp->tasklet);
 
-			nv_wr32(dev, 0x6100ac, (stat & 0x00000007));
+			nv_wr32(device, 0x6100ac, (stat & 0x00000007));
 			stat &= ~0x00000007;
 		}
 
 		if (stat) {
-			NV_INFO(dev, "PDISP: unknown intr24 0x%08x\n", stat);
-			nv_wr32(dev, 0x6100ac, stat);
+			NV_INFO(drm, "PDISP: unknown intr24 0x%08x\n", stat);
+			nv_wr32(device, 0x6100ac, stat);
 		}
 
 		intr &= ~0x00100000;
 	}
 
-	for (i = 0; i < dev->mode_config.num_crtc; i++) {
-		u32 mask = 0x01000000 << i;
-		if (intr & mask) {
-			u32 stat = nv_rd32(dev, 0x6100bc + (i * 0x800));
-			nv_wr32(dev, 0x6100bc + (i * 0x800), stat);
-			intr &= ~mask;
-		}
-	}
-
+	intr &= ~0x0f000000; /* vblank, handled in core */
 	if (intr)
-		NV_INFO(dev, "PDISP: unknown intr 0x%08x\n", intr);
+		NV_INFO(drm, "PDISP: unknown intr 0x%08x\n", intr);
 }
 
 /******************************************************************************
@@ -1867,15 +1888,17 @@ int
 nvd0_display_init(struct drm_device *dev)
 {
 	struct nvd0_display *disp = nvd0_display(dev);
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
 	int ret, i;
 	u32 *push;
 
-	if (nv_rd32(dev, 0x6100ac) & 0x00000100) {
-		nv_wr32(dev, 0x6100ac, 0x00000100);
-		nv_mask(dev, 0x6194e8, 0x00000001, 0x00000000);
-		if (!nv_wait(dev, 0x6194e8, 0x00000002, 0x00000000)) {
-			NV_ERROR(dev, "PDISP: 0x6194e8 0x%08x\n",
-				 nv_rd32(dev, 0x6194e8));
+	if (nv_rd32(device, 0x6100ac) & 0x00000100) {
+		nv_wr32(device, 0x6100ac, 0x00000100);
+		nv_mask(device, 0x6194e8, 0x00000001, 0x00000000);
+		if (!nv_wait(device, 0x6194e8, 0x00000002, 0x00000000)) {
+			NV_ERROR(drm, "PDISP: 0x6194e8 0x%08x\n",
+				 nv_rd32(device, 0x6194e8));
 			return -EBUSY;
 		}
 	}
@@ -1884,27 +1907,27 @@ nvd0_display_init(struct drm_device *dev)
 	 * work at all unless you do the SOR part below.
 	 */
 	for (i = 0; i < 3; i++) {
-		u32 dac = nv_rd32(dev, 0x61a000 + (i * 0x800));
-		nv_wr32(dev, 0x6101c0 + (i * 0x800), dac);
+		u32 dac = nv_rd32(device, 0x61a000 + (i * 0x800));
+		nv_wr32(device, 0x6101c0 + (i * 0x800), dac);
 	}
 
 	for (i = 0; i < 4; i++) {
-		u32 sor = nv_rd32(dev, 0x61c000 + (i * 0x800));
-		nv_wr32(dev, 0x6301c4 + (i * 0x800), sor);
+		u32 sor = nv_rd32(device, 0x61c000 + (i * 0x800));
+		nv_wr32(device, 0x6301c4 + (i * 0x800), sor);
 	}
 
 	for (i = 0; i < dev->mode_config.num_crtc; i++) {
-		u32 crtc0 = nv_rd32(dev, 0x616104 + (i * 0x800));
-		u32 crtc1 = nv_rd32(dev, 0x616108 + (i * 0x800));
-		u32 crtc2 = nv_rd32(dev, 0x61610c + (i * 0x800));
-		nv_wr32(dev, 0x6101b4 + (i * 0x800), crtc0);
-		nv_wr32(dev, 0x6101b8 + (i * 0x800), crtc1);
-		nv_wr32(dev, 0x6101bc + (i * 0x800), crtc2);
+		u32 crtc0 = nv_rd32(device, 0x616104 + (i * 0x800));
+		u32 crtc1 = nv_rd32(device, 0x616108 + (i * 0x800));
+		u32 crtc2 = nv_rd32(device, 0x61610c + (i * 0x800));
+		nv_wr32(device, 0x6101b4 + (i * 0x800), crtc0);
+		nv_wr32(device, 0x6101b8 + (i * 0x800), crtc1);
+		nv_wr32(device, 0x6101bc + (i * 0x800), crtc2);
 	}
 
 	/* point at our hash table / objects, enable interrupts */
-	nv_wr32(dev, 0x610010, (disp->mem->vinst >> 8) | 9);
-	nv_mask(dev, 0x6100b0, 0x00000307, 0x00000307);
+	nv_wr32(device, 0x610010, (disp->mem->addr >> 8) | 9);
+	nv_mask(device, 0x6100b0, 0x00000307, 0x00000307);
 
 	/* init master */
 	ret = evo_init_dma(dev, EVO_MASTER);
@@ -1944,7 +1967,6 @@ error:
 void
 nvd0_display_destroy(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nvd0_display *disp = nvd0_display(dev);
 	struct pci_dev *pdev = dev->pdev;
 	int i;
@@ -1957,31 +1979,36 @@ nvd0_display_destroy(struct drm_device *dev)
 	nouveau_gpuobj_ref(NULL, &disp->mem);
 	nouveau_bo_unmap(disp->sync);
 	nouveau_bo_ref(NULL, &disp->sync);
-	nouveau_irq_unregister(dev, 26);
 
-	dev_priv->engine.display.priv = NULL;
+	nouveau_display(dev)->priv = NULL;
 	kfree(disp);
 }
 
 int
 nvd0_display_create(struct drm_device *dev)
 {
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct dcb_table *dcb = &dev_priv->vbios.dcb;
+	struct nouveau_device *device = nouveau_dev(dev);
+	struct nouveau_drm *drm = nouveau_drm(dev);
+	struct nouveau_bar *bar = nouveau_bar(device);
+	struct nouveau_fb *pfb = nouveau_fb(device);
+	struct dcb_table *dcb = &drm->vbios.dcb;
 	struct drm_connector *connector, *tmp;
 	struct pci_dev *pdev = dev->pdev;
 	struct nvd0_display *disp;
-	struct dcb_entry *dcbe;
+	struct dcb_output *dcbe;
 	int crtcs, ret, i;
 
 	disp = kzalloc(sizeof(*disp), GFP_KERNEL);
 	if (!disp)
 		return -ENOMEM;
-	dev_priv->engine.display.priv = disp;
+
+	nouveau_display(dev)->priv = disp;
+	nouveau_display(dev)->dtor = nvd0_display_destroy;
+	nouveau_display(dev)->init = nvd0_display_init;
+	nouveau_display(dev)->fini = nvd0_display_fini;
 
 	/* create crtc objects to represent the hw heads */
-	crtcs = nv_rd32(dev, 0x022448);
+	crtcs = nv_rd32(device, 0x022448);
 	for (i = 0; i < crtcs; i++) {
 		ret = nvd0_crtc_create(dev, i);
 		if (ret)
@@ -1995,22 +2022,22 @@ nvd0_display_create(struct drm_device *dev)
 			continue;
 
 		if (dcbe->location != DCB_LOC_ON_CHIP) {
-			NV_WARN(dev, "skipping off-chip encoder %d/%d\n",
+			NV_WARN(drm, "skipping off-chip encoder %d/%d\n",
 				dcbe->type, ffs(dcbe->or) - 1);
 			continue;
 		}
 
 		switch (dcbe->type) {
-		case OUTPUT_TMDS:
-		case OUTPUT_LVDS:
-		case OUTPUT_DP:
+		case DCB_OUTPUT_TMDS:
+		case DCB_OUTPUT_LVDS:
+		case DCB_OUTPUT_DP:
 			nvd0_sor_create(connector, dcbe);
 			break;
-		case OUTPUT_ANALOG:
+		case DCB_OUTPUT_ANALOG:
 			nvd0_dac_create(connector, dcbe);
 			break;
 		default:
-			NV_WARN(dev, "skipping unsupported encoder %d/%d\n",
+			NV_WARN(drm, "skipping unsupported encoder %d/%d\n",
 				dcbe->type, ffs(dcbe->or) - 1);
 			continue;
 		}
@@ -2021,14 +2048,13 @@ nvd0_display_create(struct drm_device *dev)
 		if (connector->encoder_ids[0])
 			continue;
 
-		NV_WARN(dev, "%s has no encoders, removing\n",
+		NV_WARN(drm, "%s has no encoders, removing\n",
 			drm_get_connector_name(connector));
 		connector->funcs->destroy(connector);
 	}
 
 	/* setup interrupt handling */
 	tasklet_init(&disp->tasklet, nvd0_display_bh, (unsigned long)dev);
-	nouveau_irq_register(dev, 26, nvd0_display_intr);
 
 	/* small shared memory area we use for notifiers and semaphores */
 	ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
@@ -2045,7 +2071,7 @@ nvd0_display_create(struct drm_device *dev)
 		goto out;
 
 	/* hash table and dma objects for the memory areas we care about */
-	ret = nouveau_gpuobj_new(dev, NULL, 0x4000, 0x10000,
+	ret = nouveau_gpuobj_new(nv_object(device), NULL, 0x4000, 0x10000,
 				 NVOBJ_FLAG_ZERO_ALLOC, &disp->mem);
 	if (ret)
 		goto out;
@@ -2077,7 +2103,7 @@ nvd0_display_create(struct drm_device *dev)
 
 		nv_wo32(disp->mem, dmao + 0x20, 0x00000049);
 		nv_wo32(disp->mem, dmao + 0x24, 0x00000000);
-		nv_wo32(disp->mem, dmao + 0x28, (dev_priv->vram_size - 1) >> 8);
+		nv_wo32(disp->mem, dmao + 0x28, (pfb->ram.size - 1) >> 8);
 		nv_wo32(disp->mem, dmao + 0x2c, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x30, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x34, 0x00000000);
@@ -2087,7 +2113,7 @@ nvd0_display_create(struct drm_device *dev)
 
 		nv_wo32(disp->mem, dmao + 0x40, 0x00000009);
 		nv_wo32(disp->mem, dmao + 0x44, 0x00000000);
-		nv_wo32(disp->mem, dmao + 0x48, (dev_priv->vram_size - 1) >> 8);
+		nv_wo32(disp->mem, dmao + 0x48, (pfb->ram.size - 1) >> 8);
 		nv_wo32(disp->mem, dmao + 0x4c, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x50, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x54, 0x00000000);
@@ -2097,7 +2123,7 @@ nvd0_display_create(struct drm_device *dev)
 
 		nv_wo32(disp->mem, dmao + 0x60, 0x0fe00009);
 		nv_wo32(disp->mem, dmao + 0x64, 0x00000000);
-		nv_wo32(disp->mem, dmao + 0x68, (dev_priv->vram_size - 1) >> 8);
+		nv_wo32(disp->mem, dmao + 0x68, (pfb->ram.size - 1) >> 8);
 		nv_wo32(disp->mem, dmao + 0x6c, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x70, 0x00000000);
 		nv_wo32(disp->mem, dmao + 0x74, 0x00000000);
@@ -2106,7 +2132,7 @@ nvd0_display_create(struct drm_device *dev)
 						((dmao + 0x60) << 9));
 	}
 
-	pinstmem->flush(dev);
+	bar->flush(bar);
 
 out:
 	if (ret)
diff --git a/drivers/gpu/drm/nouveau/nve0_fifo.c b/drivers/gpu/drm/nouveau/nve0_fifo.c
deleted file mode 100644
index 281bece751b..00000000000
--- a/drivers/gpu/drm/nouveau/nve0_fifo.c
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nouveau_fifo.h"
-
-#define NVE0_FIFO_ENGINE_NUM 32
-
-static void nve0_fifo_isr(struct drm_device *);
-
-struct nve0_fifo_engine {
-	struct nouveau_gpuobj *playlist[2];
-	int cur_playlist;
-};
-
-struct nve0_fifo_priv {
-	struct nouveau_fifo_priv base;
-	struct nve0_fifo_engine engine[NVE0_FIFO_ENGINE_NUM];
-	struct {
-		struct nouveau_gpuobj *mem;
-		struct nouveau_vma bar;
-	} user;
-	int spoon_nr;
-};
-
-struct nve0_fifo_chan {
-	struct nouveau_fifo_chan base;
-	u32 engine;
-};
-
-static void
-nve0_fifo_playlist_update(struct drm_device *dev, u32 engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nve0_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct nve0_fifo_engine *peng = &priv->engine[engine];
-	struct nouveau_gpuobj *cur;
-	u32 match = (engine << 16) | 0x00000001;
-	int ret, i, p;
-
-	cur = peng->playlist[peng->cur_playlist];
-	if (unlikely(cur == NULL)) {
-		ret = nouveau_gpuobj_new(dev, NULL, 0x8000, 0x1000, 0, &cur);
-		if (ret) {
-			NV_ERROR(dev, "PFIFO: playlist alloc failed\n");
-			return;
-		}
-
-		peng->playlist[peng->cur_playlist] = cur;
-	}
-
-	peng->cur_playlist = !peng->cur_playlist;
-
-	for (i = 0, p = 0; i < priv->base.channels; i++) {
-		u32 ctrl = nv_rd32(dev, 0x800004 + (i * 8)) & 0x001f0001;
-		if (ctrl != match)
-			continue;
-		nv_wo32(cur, p + 0, i);
-		nv_wo32(cur, p + 4, 0x00000000);
-		p += 8;
-	}
-	pinstmem->flush(dev);
-
-	nv_wr32(dev, 0x002270, cur->vinst >> 12);
-	nv_wr32(dev, 0x002274, (engine << 20) | (p >> 3));
-	if (!nv_wait(dev, 0x002284 + (engine * 4), 0x00100000, 0x00000000))
-		NV_ERROR(dev, "PFIFO: playlist %d update timeout\n", engine);
-}
-
-static int
-nve0_fifo_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nve0_fifo_priv *priv = nv_engine(dev, engine);
-	struct nve0_fifo_chan *fctx;
-	u64 usermem = priv->user.mem->vinst + chan->id * 512;
-	u64 ib_virt = chan->pushbuf_base + chan->dma.ib_base * 4;
-	int ret = 0, i;
-
-	fctx = chan->engctx[engine] = kzalloc(sizeof(*fctx), GFP_KERNEL);
-	if (!fctx)
-		return -ENOMEM;
-
-	fctx->engine = 0; /* PGRAPH */
-
-	/* allocate vram for control regs, map into polling area */
-	chan->user = ioremap_wc(pci_resource_start(dev->pdev, 1) +
-				priv->user.bar.offset + (chan->id * 512), 512);
-	if (!chan->user) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	for (i = 0; i < 0x100; i += 4)
-		nv_wo32(chan->ramin, i, 0x00000000);
-	nv_wo32(chan->ramin, 0x08, lower_32_bits(usermem));
-	nv_wo32(chan->ramin, 0x0c, upper_32_bits(usermem));
-	nv_wo32(chan->ramin, 0x10, 0x0000face);
-	nv_wo32(chan->ramin, 0x30, 0xfffff902);
-	nv_wo32(chan->ramin, 0x48, lower_32_bits(ib_virt));
-	nv_wo32(chan->ramin, 0x4c, drm_order(chan->dma.ib_max + 1) << 16 |
-				     upper_32_bits(ib_virt));
-	nv_wo32(chan->ramin, 0x84, 0x20400000);
-	nv_wo32(chan->ramin, 0x94, 0x30000001);
-	nv_wo32(chan->ramin, 0x9c, 0x00000100);
-	nv_wo32(chan->ramin, 0xac, 0x0000001f);
-	nv_wo32(chan->ramin, 0xe4, 0x00000000);
-	nv_wo32(chan->ramin, 0xe8, chan->id);
-	nv_wo32(chan->ramin, 0xf8, 0x10003080); /* 0x002310 */
-	nv_wo32(chan->ramin, 0xfc, 0x10000010); /* 0x002350 */
-	pinstmem->flush(dev);
-
-	nv_wr32(dev, 0x800000 + (chan->id * 8), 0x80000000 |
-						(chan->ramin->vinst >> 12));
-	nv_mask(dev, 0x800004 + (chan->id * 8), 0x00000400, 0x00000400);
-	nve0_fifo_playlist_update(dev, fctx->engine);
-	nv_mask(dev, 0x800004 + (chan->id * 8), 0x00000400, 0x00000400);
-
-error:
-	if (ret)
-		priv->base.base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nve0_fifo_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nve0_fifo_chan *fctx = chan->engctx[engine];
-	struct drm_device *dev = chan->dev;
-
-	nv_mask(dev, 0x800004 + (chan->id * 8), 0x00000800, 0x00000800);
-	nv_wr32(dev, 0x002634, chan->id);
-	if (!nv_wait(dev, 0x0002634, 0xffffffff, chan->id))
-		NV_WARN(dev, "0x2634 != chid: 0x%08x\n", nv_rd32(dev, 0x2634));
-	nve0_fifo_playlist_update(dev, fctx->engine);
-	nv_wr32(dev, 0x800000 + (chan->id * 8), 0x00000000);
-
-	if (chan->user) {
-		iounmap(chan->user);
-		chan->user = NULL;
-	}
-
-	chan->engctx[NVOBJ_ENGINE_FIFO] = NULL;
-	kfree(fctx);
-}
-
-static int
-nve0_fifo_init(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nve0_fifo_priv *priv = nv_engine(dev, engine);
-	struct nve0_fifo_chan *fctx;
-	int i;
-
-	/* reset PFIFO, enable all available PSUBFIFO areas */
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000000);
-	nv_mask(dev, 0x000200, 0x00000100, 0x00000100);
-	nv_wr32(dev, 0x000204, 0xffffffff);
-
-	priv->spoon_nr = hweight32(nv_rd32(dev, 0x000204));
-	NV_DEBUG(dev, "PFIFO: %d subfifo(s)\n", priv->spoon_nr);
-
-	/* PSUBFIFO[n] */
-	for (i = 0; i < priv->spoon_nr; i++) {
-		nv_mask(dev, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
-		nv_wr32(dev, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
-		nv_wr32(dev, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTR_EN */
-	}
-
-	nv_wr32(dev, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
-
-	nv_wr32(dev, 0x002a00, 0xffffffff);
-	nv_wr32(dev, 0x002100, 0xffffffff);
-	nv_wr32(dev, 0x002140, 0xbfffffff);
-
-	/* restore PFIFO context table */
-	for (i = 0; i < priv->base.channels; i++) {
-		struct nouveau_channel *chan = dev_priv->channels.ptr[i];
-		if (!chan || !(fctx = chan->engctx[engine]))
-			continue;
-
-		nv_wr32(dev, 0x800000 + (i * 8), 0x80000000 |
-						 (chan->ramin->vinst >> 12));
-		nv_mask(dev, 0x800004 + (i * 8), 0x00000400, 0x00000400);
-		nve0_fifo_playlist_update(dev, fctx->engine);
-		nv_mask(dev, 0x800004 + (i * 8), 0x00000400, 0x00000400);
-	}
-
-	return 0;
-}
-
-static int
-nve0_fifo_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	struct nve0_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	for (i = 0; i < priv->base.channels; i++) {
-		if (!(nv_rd32(dev, 0x800004 + (i * 8)) & 1))
-			continue;
-
-		nv_mask(dev, 0x800004 + (i * 8), 0x00000800, 0x00000800);
-		nv_wr32(dev, 0x002634, i);
-		if (!nv_wait(dev, 0x002634, 0xffffffff, i)) {
-			NV_INFO(dev, "PFIFO: kick ch %d failed: 0x%08x\n",
-				i, nv_rd32(dev, 0x002634));
-			return -EBUSY;
-		}
-	}
-
-	nv_wr32(dev, 0x002140, 0x00000000);
-	return 0;
-}
-
-struct nouveau_enum nve0_fifo_fault_unit[] = {
-	{}
-};
-
-struct nouveau_enum nve0_fifo_fault_reason[] = {
-	{ 0x00, "PT_NOT_PRESENT" },
-	{ 0x01, "PT_TOO_SHORT" },
-	{ 0x02, "PAGE_NOT_PRESENT" },
-	{ 0x03, "VM_LIMIT_EXCEEDED" },
-	{ 0x04, "NO_CHANNEL" },
-	{ 0x05, "PAGE_SYSTEM_ONLY" },
-	{ 0x06, "PAGE_READ_ONLY" },
-	{ 0x0a, "COMPRESSED_SYSRAM" },
-	{ 0x0c, "INVALID_STORAGE_TYPE" },
-	{}
-};
-
-struct nouveau_enum nve0_fifo_fault_hubclient[] = {
-	{}
-};
-
-struct nouveau_enum nve0_fifo_fault_gpcclient[] = {
-	{}
-};
-
-struct nouveau_bitfield nve0_fifo_subfifo_intr[] = {
-	{ 0x00200000, "ILLEGAL_MTHD" },
-	{ 0x00800000, "EMPTY_SUBC" },
-	{}
-};
-
-static void
-nve0_fifo_isr_vm_fault(struct drm_device *dev, int unit)
-{
-	u32 inst = nv_rd32(dev, 0x2800 + (unit * 0x10));
-	u32 valo = nv_rd32(dev, 0x2804 + (unit * 0x10));
-	u32 vahi = nv_rd32(dev, 0x2808 + (unit * 0x10));
-	u32 stat = nv_rd32(dev, 0x280c + (unit * 0x10));
-	u32 client = (stat & 0x00001f00) >> 8;
-
-	NV_INFO(dev, "PFIFO: %s fault at 0x%010llx [",
-		(stat & 0x00000080) ? "write" : "read", (u64)vahi << 32 | valo);
-	nouveau_enum_print(nve0_fifo_fault_reason, stat & 0x0000000f);
-	printk("] from ");
-	nouveau_enum_print(nve0_fifo_fault_unit, unit);
-	if (stat & 0x00000040) {
-		printk("/");
-		nouveau_enum_print(nve0_fifo_fault_hubclient, client);
-	} else {
-		printk("/GPC%d/", (stat & 0x1f000000) >> 24);
-		nouveau_enum_print(nve0_fifo_fault_gpcclient, client);
-	}
-	printk(" on channel 0x%010llx\n", (u64)inst << 12);
-}
-
-static int
-nve0_fifo_page_flip(struct drm_device *dev, u32 chid)
-{
-	struct nve0_fifo_priv *priv = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan = NULL;
-	unsigned long flags;
-	int ret = -EINVAL;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	if (likely(chid >= 0 && chid < priv->base.channels)) {
-		chan = dev_priv->channels.ptr[chid];
-		if (likely(chan))
-			ret = nouveau_finish_page_flip(chan, NULL);
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return ret;
-}
-
-static void
-nve0_fifo_isr_subfifo_intr(struct drm_device *dev, int unit)
-{
-	u32 stat = nv_rd32(dev, 0x040108 + (unit * 0x2000));
-	u32 addr = nv_rd32(dev, 0x0400c0 + (unit * 0x2000));
-	u32 data = nv_rd32(dev, 0x0400c4 + (unit * 0x2000));
-	u32 chid = nv_rd32(dev, 0x040120 + (unit * 0x2000)) & 0x7f;
-	u32 subc = (addr & 0x00070000);
-	u32 mthd = (addr & 0x00003ffc);
-	u32 show = stat;
-
-	if (stat & 0x00200000) {
-		if (mthd == 0x0054) {
-			if (!nve0_fifo_page_flip(dev, chid))
-				show &= ~0x00200000;
-		}
-	}
-
-	if (show) {
-		NV_INFO(dev, "PFIFO%d:", unit);
-		nouveau_bitfield_print(nve0_fifo_subfifo_intr, show);
-		NV_INFO(dev, "PFIFO%d: ch %d subc %d mthd 0x%04x data 0x%08x\n",
-			unit, chid, subc, mthd, data);
-	}
-
-	nv_wr32(dev, 0x0400c0 + (unit * 0x2000), 0x80600008);
-	nv_wr32(dev, 0x040108 + (unit * 0x2000), stat);
-}
-
-static void
-nve0_fifo_isr(struct drm_device *dev)
-{
-	u32 mask = nv_rd32(dev, 0x002140);
-	u32 stat = nv_rd32(dev, 0x002100) & mask;
-
-	if (stat & 0x00000100) {
-		NV_INFO(dev, "PFIFO: unknown status 0x00000100\n");
-		nv_wr32(dev, 0x002100, 0x00000100);
-		stat &= ~0x00000100;
-	}
-
-	if (stat & 0x10000000) {
-		u32 units = nv_rd32(dev, 0x00259c);
-		u32 u = units;
-
-		while (u) {
-			int i = ffs(u) - 1;
-			nve0_fifo_isr_vm_fault(dev, i);
-			u &= ~(1 << i);
-		}
-
-		nv_wr32(dev, 0x00259c, units);
-		stat &= ~0x10000000;
-	}
-
-	if (stat & 0x20000000) {
-		u32 units = nv_rd32(dev, 0x0025a0);
-		u32 u = units;
-
-		while (u) {
-			int i = ffs(u) - 1;
-			nve0_fifo_isr_subfifo_intr(dev, i);
-			u &= ~(1 << i);
-		}
-
-		nv_wr32(dev, 0x0025a0, units);
-		stat &= ~0x20000000;
-	}
-
-	if (stat & 0x40000000) {
-		NV_INFO(dev, "PFIFO: unknown status 0x40000000\n");
-		nv_mask(dev, 0x002a00, 0x00000000, 0x00000000);
-		stat &= ~0x40000000;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PFIFO: unhandled status 0x%08x\n", stat);
-		nv_wr32(dev, 0x002100, stat);
-		nv_wr32(dev, 0x002140, 0);
-	}
-}
-
-static void
-nve0_fifo_destroy(struct drm_device *dev, int engine)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nve0_fifo_priv *priv = nv_engine(dev, engine);
-	int i;
-
-	nouveau_vm_put(&priv->user.bar);
-	nouveau_gpuobj_ref(NULL, &priv->user.mem);
-
-	for (i = 0; i < NVE0_FIFO_ENGINE_NUM; i++) {
-		nouveau_gpuobj_ref(NULL, &priv->engine[i].playlist[0]);
-		nouveau_gpuobj_ref(NULL, &priv->engine[i].playlist[1]);
-	}
-
-	dev_priv->eng[engine] = NULL;
-	kfree(priv);
-}
-
-int
-nve0_fifo_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nve0_fifo_priv *priv;
-	int ret;
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.base.destroy = nve0_fifo_destroy;
-	priv->base.base.init = nve0_fifo_init;
-	priv->base.base.fini = nve0_fifo_fini;
-	priv->base.base.context_new = nve0_fifo_context_new;
-	priv->base.base.context_del = nve0_fifo_context_del;
-	priv->base.channels = 4096;
-	dev_priv->eng[NVOBJ_ENGINE_FIFO] = &priv->base.base;
-
-	ret = nouveau_gpuobj_new(dev, NULL, priv->base.channels * 512, 0x1000,
-				 NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);
-	if (ret)
-		goto error;
-
-	ret = nouveau_vm_get(dev_priv->bar1_vm, priv->user.mem->size,
-			     12, NV_MEM_ACCESS_RW, &priv->user.bar);
-	if (ret)
-		goto error;
-
-	nouveau_vm_map(&priv->user.bar, *(struct nouveau_mem **)priv->user.mem->node);
-
-	nouveau_irq_register(dev, 8, nve0_fifo_isr);
-error:
-	if (ret)
-		priv->base.base.destroy(dev, NVOBJ_ENGINE_FIFO);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nve0_graph.c b/drivers/gpu/drm/nouveau/nve0_graph.c
deleted file mode 100644
index 8a8051b68f1..00000000000
--- a/drivers/gpu/drm/nouveau/nve0_graph.c
+++ /dev/null
@@ -1,831 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <linux/firmware.h>
-#include <linux/module.h>
-
-#include "drmP.h"
-
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nouveau_fifo.h"
-
-#include "nve0_graph.h"
-
-static void
-nve0_graph_ctxctl_debug_unit(struct drm_device *dev, u32 base)
-{
-	NV_INFO(dev, "PGRAPH: %06x - done 0x%08x\n", base,
-		nv_rd32(dev, base + 0x400));
-	NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
-		nv_rd32(dev, base + 0x800), nv_rd32(dev, base + 0x804),
-		nv_rd32(dev, base + 0x808), nv_rd32(dev, base + 0x80c));
-	NV_INFO(dev, "PGRAPH: %06x - stat 0x%08x 0x%08x 0x%08x 0x%08x\n", base,
-		nv_rd32(dev, base + 0x810), nv_rd32(dev, base + 0x814),
-		nv_rd32(dev, base + 0x818), nv_rd32(dev, base + 0x81c));
-}
-
-static void
-nve0_graph_ctxctl_debug(struct drm_device *dev)
-{
-	u32 gpcnr = nv_rd32(dev, 0x409604) & 0xffff;
-	u32 gpc;
-
-	nve0_graph_ctxctl_debug_unit(dev, 0x409000);
-	for (gpc = 0; gpc < gpcnr; gpc++)
-		nve0_graph_ctxctl_debug_unit(dev, 0x502000 + (gpc * 0x8000));
-}
-
-static int
-nve0_graph_load_context(struct nouveau_channel *chan)
-{
-	struct drm_device *dev = chan->dev;
-
-	nv_wr32(dev, 0x409840, 0x00000030);
-	nv_wr32(dev, 0x409500, 0x80000000 | chan->ramin->vinst >> 12);
-	nv_wr32(dev, 0x409504, 0x00000003);
-	if (!nv_wait(dev, 0x409800, 0x00000010, 0x00000010))
-		NV_ERROR(dev, "PGRAPH: load_ctx timeout\n");
-
-	return 0;
-}
-
-static int
-nve0_graph_unload_context_to(struct drm_device *dev, u64 chan)
-{
-	nv_wr32(dev, 0x409840, 0x00000003);
-	nv_wr32(dev, 0x409500, 0x80000000 | chan >> 12);
-	nv_wr32(dev, 0x409504, 0x00000009);
-	if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000000)) {
-		NV_ERROR(dev, "PGRAPH: unload_ctx timeout\n");
-		return -EBUSY;
-	}
-
-	return 0;
-}
-
-static int
-nve0_graph_construct_context(struct nouveau_channel *chan)
-{
-	struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
-	struct nve0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nve0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	int ret, i;
-	u32 *ctx;
-
-	ctx = kmalloc(priv->grctx_size, GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	nve0_graph_load_context(chan);
-
-	nv_wo32(grch->grctx, 0x1c, 1);
-	nv_wo32(grch->grctx, 0x20, 0);
-	nv_wo32(grch->grctx, 0x28, 0);
-	nv_wo32(grch->grctx, 0x2c, 0);
-	dev_priv->engine.instmem.flush(dev);
-
-	ret = nve0_grctx_generate(chan);
-	if (ret)
-		goto err;
-
-	ret = nve0_graph_unload_context_to(dev, chan->ramin->vinst);
-	if (ret)
-		goto err;
-
-	for (i = 0; i < priv->grctx_size; i += 4)
-		ctx[i / 4] = nv_ro32(grch->grctx, i);
-
-	priv->grctx_vals = ctx;
-	return 0;
-
-err:
-	kfree(ctx);
-	return ret;
-}
-
-static int
-nve0_graph_create_context_mmio_list(struct nouveau_channel *chan)
-{
-	struct nve0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nve0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	u32 magic[GPC_MAX][2];
-	u16 offset = 0x0000;
-	int gpc;
-	int ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x3000, 256, NVOBJ_FLAG_VM,
-				 &grch->unk408004);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x8000, 256, NVOBJ_FLAG_VM,
-				 &grch->unk40800c);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 384 * 1024, 4096,
-				 NVOBJ_FLAG_VM | NVOBJ_FLAG_VM_USER,
-				 &grch->unk418810);
-	if (ret)
-		return ret;
-
-	ret = nouveau_gpuobj_new(dev, chan, 0x1000, 0, NVOBJ_FLAG_VM,
-				 &grch->mmio);
-	if (ret)
-		return ret;
-
-#define mmio(r,v) do {                                                         \
-	nv_wo32(grch->mmio, (grch->mmio_nr * 8) + 0, (r));                     \
-	nv_wo32(grch->mmio, (grch->mmio_nr * 8) + 4, (v));                     \
-	grch->mmio_nr++;                                                       \
-} while (0)
-	mmio(0x40800c, grch->unk40800c->linst >> 8);
-	mmio(0x408010, 0x80000000);
-	mmio(0x419004, grch->unk40800c->linst >> 8);
-	mmio(0x419008, 0x00000000);
-	mmio(0x4064cc, 0x80000000);
-	mmio(0x408004, grch->unk408004->linst >> 8);
-	mmio(0x408008, 0x80000030);
-	mmio(0x418808, grch->unk408004->linst >> 8);
-	mmio(0x41880c, 0x80000030);
-	mmio(0x4064c8, 0x01800600);
-	mmio(0x418810, 0x80000000 | grch->unk418810->linst >> 12);
-	mmio(0x419848, 0x10000000 | grch->unk418810->linst >> 12);
-	mmio(0x405830, 0x02180648);
-	mmio(0x4064c4, 0x0192ffff);
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		u16 magic0 = 0x0218 * priv->tpc_nr[gpc];
-		u16 magic1 = 0x0648 * priv->tpc_nr[gpc];
-		magic[gpc][0]  = 0x10000000 | (magic0 << 16) | offset;
-		magic[gpc][1]  = 0x00000000 | (magic1 << 16);
-		offset += 0x0324 * priv->tpc_nr[gpc];
-	}
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		mmio(GPC_UNIT(gpc, 0x30c0), magic[gpc][0]);
-		mmio(GPC_UNIT(gpc, 0x30e4), magic[gpc][1] | offset);
-		offset += 0x07ff * priv->tpc_nr[gpc];
-	}
-
-	mmio(0x17e91c, 0x06060609);
-	mmio(0x17e920, 0x00090a05);
-#undef mmio
-	return 0;
-}
-
-static int
-nve0_graph_context_new(struct nouveau_channel *chan, int engine)
-{
-	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *pinstmem = &dev_priv->engine.instmem;
-	struct nve0_graph_priv *priv = nv_engine(dev, engine);
-	struct nve0_graph_chan *grch;
-	struct nouveau_gpuobj *grctx;
-	int ret, i;
-
-	grch = kzalloc(sizeof(*grch), GFP_KERNEL);
-	if (!grch)
-		return -ENOMEM;
-	chan->engctx[NVOBJ_ENGINE_GR] = grch;
-
-	ret = nouveau_gpuobj_new(dev, chan, priv->grctx_size, 256,
-				 NVOBJ_FLAG_VM | NVOBJ_FLAG_ZERO_ALLOC,
-				 &grch->grctx);
-	if (ret)
-		goto error;
-	grctx = grch->grctx;
-
-	ret = nve0_graph_create_context_mmio_list(chan);
-	if (ret)
-		goto error;
-
-	nv_wo32(chan->ramin, 0x0210, lower_32_bits(grctx->linst) | 4);
-	nv_wo32(chan->ramin, 0x0214, upper_32_bits(grctx->linst));
-	pinstmem->flush(dev);
-
-	if (!priv->grctx_vals) {
-		ret = nve0_graph_construct_context(chan);
-		if (ret)
-			goto error;
-	}
-
-	for (i = 0; i < priv->grctx_size; i += 4)
-		nv_wo32(grctx, i, priv->grctx_vals[i / 4]);
-	nv_wo32(grctx, 0xf4, 0);
-	nv_wo32(grctx, 0xf8, 0);
-	nv_wo32(grctx, 0x10, grch->mmio_nr);
-	nv_wo32(grctx, 0x14, lower_32_bits(grch->mmio->linst));
-	nv_wo32(grctx, 0x18, upper_32_bits(grch->mmio->linst));
-	nv_wo32(grctx, 0x1c, 1);
-	nv_wo32(grctx, 0x20, 0);
-	nv_wo32(grctx, 0x28, 0);
-	nv_wo32(grctx, 0x2c, 0);
-
-	pinstmem->flush(dev);
-	return 0;
-
-error:
-	priv->base.context_del(chan, engine);
-	return ret;
-}
-
-static void
-nve0_graph_context_del(struct nouveau_channel *chan, int engine)
-{
-	struct nve0_graph_chan *grch = chan->engctx[engine];
-
-	nouveau_gpuobj_ref(NULL, &grch->mmio);
-	nouveau_gpuobj_ref(NULL, &grch->unk418810);
-	nouveau_gpuobj_ref(NULL, &grch->unk40800c);
-	nouveau_gpuobj_ref(NULL, &grch->unk408004);
-	nouveau_gpuobj_ref(NULL, &grch->grctx);
-	chan->engctx[engine] = NULL;
-}
-
-static int
-nve0_graph_object_new(struct nouveau_channel *chan, int engine,
-		      u32 handle, u16 class)
-{
-	return 0;
-}
-
-static int
-nve0_graph_fini(struct drm_device *dev, int engine, bool suspend)
-{
-	return 0;
-}
-
-static void
-nve0_graph_init_obj418880(struct drm_device *dev)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int i;
-
-	nv_wr32(dev, GPC_BCAST(0x0880), 0x00000000);
-	nv_wr32(dev, GPC_BCAST(0x08a4), 0x00000000);
-	for (i = 0; i < 4; i++)
-		nv_wr32(dev, GPC_BCAST(0x0888) + (i * 4), 0x00000000);
-	nv_wr32(dev, GPC_BCAST(0x08b4), priv->unk4188b4->vinst >> 8);
-	nv_wr32(dev, GPC_BCAST(0x08b8), priv->unk4188b8->vinst >> 8);
-}
-
-static void
-nve0_graph_init_regs(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x400080, 0x003083c2);
-	nv_wr32(dev, 0x400088, 0x0001ffe7);
-	nv_wr32(dev, 0x40008c, 0x00000000);
-	nv_wr32(dev, 0x400090, 0x00000030);
-	nv_wr32(dev, 0x40013c, 0x003901f7);
-	nv_wr32(dev, 0x400140, 0x00000100);
-	nv_wr32(dev, 0x400144, 0x00000000);
-	nv_wr32(dev, 0x400148, 0x00000110);
-	nv_wr32(dev, 0x400138, 0x00000000);
-	nv_wr32(dev, 0x400130, 0x00000000);
-	nv_wr32(dev, 0x400134, 0x00000000);
-	nv_wr32(dev, 0x400124, 0x00000002);
-}
-
-static void
-nve0_graph_init_units(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x409ffc, 0x00000000);
-	nv_wr32(dev, 0x409c14, 0x00003e3e);
-	nv_wr32(dev, 0x409c24, 0x000f0000);
-
-	nv_wr32(dev, 0x404000, 0xc0000000);
-	nv_wr32(dev, 0x404600, 0xc0000000);
-	nv_wr32(dev, 0x408030, 0xc0000000);
-	nv_wr32(dev, 0x404490, 0xc0000000);
-	nv_wr32(dev, 0x406018, 0xc0000000);
-	nv_wr32(dev, 0x407020, 0xc0000000);
-	nv_wr32(dev, 0x405840, 0xc0000000);
-	nv_wr32(dev, 0x405844, 0x00ffffff);
-
-	nv_mask(dev, 0x419cc0, 0x00000008, 0x00000008);
-	nv_mask(dev, 0x419eb4, 0x00001000, 0x00001000);
-
-}
-
-static void
-nve0_graph_init_gpc_0(struct drm_device *dev)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
-	u32 data[TPC_MAX / 8];
-	u8  tpcnr[GPC_MAX];
-	int i, gpc, tpc;
-
-	nv_wr32(dev, GPC_UNIT(0, 0x3018), 0x00000001);
-
-	memset(data, 0x00, sizeof(data));
-	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
-	for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
-		do {
-			gpc = (gpc + 1) % priv->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
-
-		data[i / 8] |= tpc << ((i % 8) * 4);
-	}
-
-	nv_wr32(dev, GPC_BCAST(0x0980), data[0]);
-	nv_wr32(dev, GPC_BCAST(0x0984), data[1]);
-	nv_wr32(dev, GPC_BCAST(0x0988), data[2]);
-	nv_wr32(dev, GPC_BCAST(0x098c), data[3]);
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0914), priv->magic_not_rop_nr << 8 |
-						  priv->tpc_nr[gpc]);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0910), 0x00040000 | priv->tpc_total);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0918), magicgpc918);
-	}
-
-	nv_wr32(dev, GPC_BCAST(0x1bd4), magicgpc918);
-	nv_wr32(dev, GPC_BCAST(0x08ac), nv_rd32(dev, 0x100800));
-}
-
-static void
-nve0_graph_init_gpc_1(struct drm_device *dev)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int gpc, tpc;
-
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		nv_wr32(dev, GPC_UNIT(gpc, 0x3038), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0420), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0900), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x1028), 0xc0000000);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x0824), 0xc0000000);
-		for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);
-			nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);
-		}
-		nv_wr32(dev, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
-		nv_wr32(dev, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
-	}
-}
-
-static void
-nve0_graph_init_rop(struct drm_device *dev)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	int rop;
-
-	for (rop = 0; rop < priv->rop_nr; rop++) {
-		nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
-		nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
-		nv_wr32(dev, ROP_UNIT(rop, 0x204), 0xffffffff);
-		nv_wr32(dev, ROP_UNIT(rop, 0x208), 0xffffffff);
-	}
-}
-
-static void
-nve0_graph_init_fuc(struct drm_device *dev, u32 fuc_base,
-		    struct nve0_graph_fuc *code, struct nve0_graph_fuc *data)
-{
-	int i;
-
-	nv_wr32(dev, fuc_base + 0x01c0, 0x01000000);
-	for (i = 0; i < data->size / 4; i++)
-		nv_wr32(dev, fuc_base + 0x01c4, data->data[i]);
-
-	nv_wr32(dev, fuc_base + 0x0180, 0x01000000);
-	for (i = 0; i < code->size / 4; i++) {
-		if ((i & 0x3f) == 0)
-			nv_wr32(dev, fuc_base + 0x0188, i >> 6);
-		nv_wr32(dev, fuc_base + 0x0184, code->data[i]);
-	}
-}
-
-static int
-nve0_graph_init_ctxctl(struct drm_device *dev)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	u32 r000260;
-
-	/* load fuc microcode */
-	r000260 = nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
-	nve0_graph_init_fuc(dev, 0x409000, &priv->fuc409c, &priv->fuc409d);
-	nve0_graph_init_fuc(dev, 0x41a000, &priv->fuc41ac, &priv->fuc41ad);
-	nv_wr32(dev, 0x000260, r000260);
-
-	/* start both of them running */
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x41a10c, 0x00000000);
-	nv_wr32(dev, 0x40910c, 0x00000000);
-	nv_wr32(dev, 0x41a100, 0x00000002);
-	nv_wr32(dev, 0x409100, 0x00000002);
-	if (!nv_wait(dev, 0x409800, 0x00000001, 0x00000001))
-		NV_INFO(dev, "0x409800 wait failed\n");
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x7fffffff);
-	nv_wr32(dev, 0x409504, 0x00000021);
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000010);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x10 timeout\n");
-		return -EBUSY;
-	}
-	priv->grctx_size = nv_rd32(dev, 0x409800);
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000016);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x16 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409840, 0xffffffff);
-	nv_wr32(dev, 0x409500, 0x00000000);
-	nv_wr32(dev, 0x409504, 0x00000025);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x25 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409800, 0x00000000);
-	nv_wr32(dev, 0x409500, 0x00000001);
-	nv_wr32(dev, 0x409504, 0x00000030);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x30 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409810, 0xb00095c8);
-	nv_wr32(dev, 0x409800, 0x00000000);
-	nv_wr32(dev, 0x409500, 0x00000001);
-	nv_wr32(dev, 0x409504, 0x00000031);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x31 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409810, 0x00080420);
-	nv_wr32(dev, 0x409800, 0x00000000);
-	nv_wr32(dev, 0x409500, 0x00000001);
-	nv_wr32(dev, 0x409504, 0x00000032);
-	if (!nv_wait_ne(dev, 0x409800, 0xffffffff, 0x00000000)) {
-		NV_ERROR(dev, "fuc09 req 0x32 timeout\n");
-		return -EBUSY;
-	}
-
-	nv_wr32(dev, 0x409614, 0x00000070);
-	nv_wr32(dev, 0x409614, 0x00000770);
-	nv_wr32(dev, 0x40802c, 0x00000001);
-	return 0;
-}
-
-static int
-nve0_graph_init(struct drm_device *dev, int engine)
-{
-	int ret;
-
-	nv_mask(dev, 0x000200, 0x18001000, 0x00000000);
-	nv_mask(dev, 0x000200, 0x18001000, 0x18001000);
-
-	nve0_graph_init_obj418880(dev);
-	nve0_graph_init_regs(dev);
-	nve0_graph_init_gpc_0(dev);
-
-	nv_wr32(dev, 0x400500, 0x00010001);
-	nv_wr32(dev, 0x400100, 0xffffffff);
-	nv_wr32(dev, 0x40013c, 0xffffffff);
-
-	nve0_graph_init_units(dev);
-	nve0_graph_init_gpc_1(dev);
-	nve0_graph_init_rop(dev);
-
-	nv_wr32(dev, 0x400108, 0xffffffff);
-	nv_wr32(dev, 0x400138, 0xffffffff);
-	nv_wr32(dev, 0x400118, 0xffffffff);
-	nv_wr32(dev, 0x400130, 0xffffffff);
-	nv_wr32(dev, 0x40011c, 0xffffffff);
-	nv_wr32(dev, 0x400134, 0xffffffff);
-	nv_wr32(dev, 0x400054, 0x34ce3464);
-
-	ret = nve0_graph_init_ctxctl(dev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-int
-nve0_graph_isr_chid(struct drm_device *dev, u64 inst)
-{
-	struct nouveau_fifo_priv *pfifo = nv_engine(dev, NVOBJ_ENGINE_FIFO);
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_channel *chan;
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&dev_priv->channels.lock, flags);
-	for (i = 0; i < pfifo->channels; i++) {
-		chan = dev_priv->channels.ptr[i];
-		if (!chan || !chan->ramin)
-			continue;
-
-		if (inst == chan->ramin->vinst)
-			break;
-	}
-	spin_unlock_irqrestore(&dev_priv->channels.lock, flags);
-	return i;
-}
-
-static void
-nve0_graph_ctxctl_isr(struct drm_device *dev)
-{
-	u32 ustat = nv_rd32(dev, 0x409c18);
-
-	if (ustat & 0x00000001)
-		NV_INFO(dev, "PGRAPH: CTXCTRL ucode error\n");
-	if (ustat & 0x00080000)
-		NV_INFO(dev, "PGRAPH: CTXCTRL watchdog timeout\n");
-	if (ustat & ~0x00080001)
-		NV_INFO(dev, "PGRAPH: CTXCTRL 0x%08x\n", ustat);
-
-	nve0_graph_ctxctl_debug(dev);
-	nv_wr32(dev, 0x409c20, ustat);
-}
-
-static void
-nve0_graph_trap_isr(struct drm_device *dev, int chid)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, NVOBJ_ENGINE_GR);
-	u32 trap = nv_rd32(dev, 0x400108);
-	int rop;
-
-	if (trap & 0x00000001) {
-		u32 stat = nv_rd32(dev, 0x404000);
-		NV_INFO(dev, "PGRAPH: DISPATCH ch %d 0x%08x\n", chid, stat);
-		nv_wr32(dev, 0x404000, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x00000001);
-		trap &= ~0x00000001;
-	}
-
-	if (trap & 0x00000010) {
-		u32 stat = nv_rd32(dev, 0x405840);
-		NV_INFO(dev, "PGRAPH: SHADER ch %d 0x%08x\n", chid, stat);
-		nv_wr32(dev, 0x405840, 0xc0000000);
-		nv_wr32(dev, 0x400108, 0x00000010);
-		trap &= ~0x00000010;
-	}
-
-	if (trap & 0x02000000) {
-		for (rop = 0; rop < priv->rop_nr; rop++) {
-			u32 statz = nv_rd32(dev, ROP_UNIT(rop, 0x070));
-			u32 statc = nv_rd32(dev, ROP_UNIT(rop, 0x144));
-			NV_INFO(dev, "PGRAPH: ROP%d ch %d 0x%08x 0x%08x\n",
-				     rop, chid, statz, statc);
-			nv_wr32(dev, ROP_UNIT(rop, 0x070), 0xc0000000);
-			nv_wr32(dev, ROP_UNIT(rop, 0x144), 0xc0000000);
-		}
-		nv_wr32(dev, 0x400108, 0x02000000);
-		trap &= ~0x02000000;
-	}
-
-	if (trap) {
-		NV_INFO(dev, "PGRAPH: TRAP ch %d 0x%08x\n", chid, trap);
-		nv_wr32(dev, 0x400108, trap);
-	}
-}
-
-static void
-nve0_graph_isr(struct drm_device *dev)
-{
-	u64 inst = (u64)(nv_rd32(dev, 0x409b00) & 0x0fffffff) << 12;
-	u32 chid = nve0_graph_isr_chid(dev, inst);
-	u32 stat = nv_rd32(dev, 0x400100);
-	u32 addr = nv_rd32(dev, 0x400704);
-	u32 mthd = (addr & 0x00003ffc);
-	u32 subc = (addr & 0x00070000) >> 16;
-	u32 data = nv_rd32(dev, 0x400708);
-	u32 code = nv_rd32(dev, 0x400110);
-	u32 class = nv_rd32(dev, 0x404200 + (subc * 4));
-
-	if (stat & 0x00000010) {
-		if (nouveau_gpuobj_mthd_call2(dev, chid, class, mthd, data)) {
-			NV_INFO(dev, "PGRAPH: ILLEGAL_MTHD ch %d [0x%010llx] "
-				     "subc %d class 0x%04x mthd 0x%04x "
-				     "data 0x%08x\n",
-				chid, inst, subc, class, mthd, data);
-		}
-		nv_wr32(dev, 0x400100, 0x00000010);
-		stat &= ~0x00000010;
-	}
-
-	if (stat & 0x00000020) {
-		NV_INFO(dev, "PGRAPH: ILLEGAL_CLASS ch %d [0x%010llx] subc %d "
-			     "class 0x%04x mthd 0x%04x data 0x%08x\n",
-			chid, inst, subc, class, mthd, data);
-		nv_wr32(dev, 0x400100, 0x00000020);
-		stat &= ~0x00000020;
-	}
-
-	if (stat & 0x00100000) {
-		NV_INFO(dev, "PGRAPH: DATA_ERROR [");
-		nouveau_enum_print(nv50_data_error_names, code);
-		printk("] ch %d [0x%010llx] subc %d class 0x%04x "
-		       "mthd 0x%04x data 0x%08x\n",
-		       chid, inst, subc, class, mthd, data);
-		nv_wr32(dev, 0x400100, 0x00100000);
-		stat &= ~0x00100000;
-	}
-
-	if (stat & 0x00200000) {
-		nve0_graph_trap_isr(dev, chid);
-		nv_wr32(dev, 0x400100, 0x00200000);
-		stat &= ~0x00200000;
-	}
-
-	if (stat & 0x00080000) {
-		nve0_graph_ctxctl_isr(dev);
-		nv_wr32(dev, 0x400100, 0x00080000);
-		stat &= ~0x00080000;
-	}
-
-	if (stat) {
-		NV_INFO(dev, "PGRAPH: unknown stat 0x%08x\n", stat);
-		nv_wr32(dev, 0x400100, stat);
-	}
-
-	nv_wr32(dev, 0x400500, 0x00010001);
-}
-
-static int
-nve0_graph_create_fw(struct drm_device *dev, const char *fwname,
-		     struct nve0_graph_fuc *fuc)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	const struct firmware *fw;
-	char f[32];
-	int ret;
-
-	snprintf(f, sizeof(f), "nouveau/nv%02x_%s", dev_priv->chipset, fwname);
-	ret = request_firmware(&fw, f, &dev->pdev->dev);
-	if (ret)
-		return ret;
-
-	fuc->size = fw->size;
-	fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL);
-	release_firmware(fw);
-	return (fuc->data != NULL) ? 0 : -ENOMEM;
-}
-
-static void
-nve0_graph_destroy_fw(struct nve0_graph_fuc *fuc)
-{
-	if (fuc->data) {
-		kfree(fuc->data);
-		fuc->data = NULL;
-	}
-}
-
-static void
-nve0_graph_destroy(struct drm_device *dev, int engine)
-{
-	struct nve0_graph_priv *priv = nv_engine(dev, engine);
-
-	nve0_graph_destroy_fw(&priv->fuc409c);
-	nve0_graph_destroy_fw(&priv->fuc409d);
-	nve0_graph_destroy_fw(&priv->fuc41ac);
-	nve0_graph_destroy_fw(&priv->fuc41ad);
-
-	nouveau_irq_unregister(dev, 12);
-
-	nouveau_gpuobj_ref(NULL, &priv->unk4188b8);
-	nouveau_gpuobj_ref(NULL, &priv->unk4188b4);
-
-	if (priv->grctx_vals)
-		kfree(priv->grctx_vals);
-
-	NVOBJ_ENGINE_DEL(dev, GR);
-	kfree(priv);
-}
-
-int
-nve0_graph_create(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nve0_graph_priv *priv;
-	int ret, gpc, i;
-	u32 kepler;
-
-	kepler = nve0_graph_class(dev);
-	if (!kepler) {
-		NV_ERROR(dev, "PGRAPH: unsupported chipset, please report!\n");
-		return 0;
-	}
-
-	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->base.destroy = nve0_graph_destroy;
-	priv->base.init = nve0_graph_init;
-	priv->base.fini = nve0_graph_fini;
-	priv->base.context_new = nve0_graph_context_new;
-	priv->base.context_del = nve0_graph_context_del;
-	priv->base.object_new = nve0_graph_object_new;
-
-	NVOBJ_ENGINE_ADD(dev, GR, &priv->base);
-	nouveau_irq_register(dev, 12, nve0_graph_isr);
-
-	NV_INFO(dev, "PGRAPH: using external firmware\n");
-	if (nve0_graph_create_fw(dev, "fuc409c", &priv->fuc409c) ||
-	    nve0_graph_create_fw(dev, "fuc409d", &priv->fuc409d) ||
-	    nve0_graph_create_fw(dev, "fuc41ac", &priv->fuc41ac) ||
-	    nve0_graph_create_fw(dev, "fuc41ad", &priv->fuc41ad)) {
-		ret = 0;
-		goto error;
-	}
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b4);
-	if (ret)
-		goto error;
-
-	ret = nouveau_gpuobj_new(dev, NULL, 0x1000, 256, 0, &priv->unk4188b8);
-	if (ret)
-		goto error;
-
-	for (i = 0; i < 0x1000; i += 4) {
-		nv_wo32(priv->unk4188b4, i, 0x00000010);
-		nv_wo32(priv->unk4188b8, i, 0x00000010);
-	}
-
-	priv->gpc_nr  =  nv_rd32(dev, 0x409604) & 0x0000001f;
-	priv->rop_nr = (nv_rd32(dev, 0x409604) & 0x001f0000) >> 16;
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-		priv->tpc_nr[gpc] = nv_rd32(dev, GPC_UNIT(gpc, 0x2608));
-		priv->tpc_total += priv->tpc_nr[gpc];
-	}
-
-	switch (dev_priv->chipset) {
-	case 0xe4:
-		if (priv->tpc_total == 8)
-			priv->magic_not_rop_nr = 3;
-		else
-		if (priv->tpc_total == 7)
-			priv->magic_not_rop_nr = 1;
-		break;
-	case 0xe7:
-		priv->magic_not_rop_nr = 1;
-		break;
-	default:
-		break;
-	}
-
-	if (!priv->magic_not_rop_nr) {
-		NV_ERROR(dev, "PGRAPH: unknown config: %d/%d/%d/%d, %d\n",
-			 priv->tpc_nr[0], priv->tpc_nr[1], priv->tpc_nr[2],
-			 priv->tpc_nr[3], priv->rop_nr);
-		priv->magic_not_rop_nr = 0x00;
-	}
-
-	NVOBJ_CLASS(dev, 0xa097, GR); /* subc 0: 3D */
-	NVOBJ_CLASS(dev, 0xa0c0, GR); /* subc 1: COMPUTE */
-	NVOBJ_CLASS(dev, 0xa040, GR); /* subc 2: P2MF */
-	NVOBJ_CLASS(dev, 0x902d, GR); /* subc 3: 2D */
-	NVOBJ_CLASS(dev, 0xa0b5, GR); /* subc 4: COPY */
-	return 0;
-
-error:
-	nve0_graph_destroy(dev, NVOBJ_ENGINE_GR);
-	return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nve0_graph.h b/drivers/gpu/drm/nouveau/nve0_graph.h
deleted file mode 100644
index 2ba70449ba0..00000000000
--- a/drivers/gpu/drm/nouveau/nve0_graph.h
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#ifndef __NVE0_GRAPH_H__
-#define __NVE0_GRAPH_H__
-
-#define GPC_MAX 4
-#define TPC_MAX 32
-
-#define ROP_BCAST(r)     (0x408800 + (r))
-#define ROP_UNIT(u, r)   (0x410000 + (u) * 0x400 + (r))
-#define GPC_BCAST(r)     (0x418000 + (r))
-#define GPC_UNIT(t, r)   (0x500000 + (t) * 0x8000 + (r))
-#define TPC_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
-
-struct nve0_graph_fuc {
-	u32 *data;
-	u32  size;
-};
-
-struct nve0_graph_priv {
-	struct nouveau_exec_engine base;
-
-	struct nve0_graph_fuc fuc409c;
-	struct nve0_graph_fuc fuc409d;
-	struct nve0_graph_fuc fuc41ac;
-	struct nve0_graph_fuc fuc41ad;
-
-	u8 gpc_nr;
-	u8 rop_nr;
-	u8 tpc_nr[GPC_MAX];
-	u8 tpc_total;
-
-	u32  grctx_size;
-	u32 *grctx_vals;
-	struct nouveau_gpuobj *unk4188b4;
-	struct nouveau_gpuobj *unk4188b8;
-
-	u8 magic_not_rop_nr;
-};
-
-struct nve0_graph_chan {
-	struct nouveau_gpuobj *grctx;
-	struct nouveau_gpuobj *unk408004; /* 0x418810 too */
-	struct nouveau_gpuobj *unk40800c; /* 0x419004 too */
-	struct nouveau_gpuobj *unk418810; /* 0x419848 too */
-	struct nouveau_gpuobj *mmio;
-	int mmio_nr;
-};
-
-int nve0_grctx_generate(struct nouveau_channel *);
-
-/* nve0_graph.c uses this also to determine supported chipsets */
-static inline u32
-nve0_graph_class(struct drm_device *dev)
-{
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-
-	switch (dev_priv->chipset) {
-	case 0xe4:
-	case 0xe7:
-		return 0xa097;
-	default:
-		return 0;
-	}
-}
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nve0_grctx.c b/drivers/gpu/drm/nouveau/nve0_grctx.c
deleted file mode 100644
index d8cb360e92c..00000000000
--- a/drivers/gpu/drm/nouveau/nve0_grctx.c
+++ /dev/null
@@ -1,2777 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "drmP.h"
-#include "nouveau_drv.h"
-#include "nouveau_mm.h"
-#include "nve0_graph.h"
-
-static void
-nv_icmd(struct drm_device *dev, u32 icmd, u32 data)
-{
-	nv_wr32(dev, 0x400204, data);
-	nv_wr32(dev, 0x400200, icmd);
-	while (nv_rd32(dev, 0x400700) & 0x00000002) {}
-}
-
-static void
-nve0_grctx_generate_icmd(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x400208, 0x80000000);
-	nv_icmd(dev, 0x001000, 0x00000004);
-	nv_icmd(dev, 0x000039, 0x00000000);
-	nv_icmd(dev, 0x00003a, 0x00000000);
-	nv_icmd(dev, 0x00003b, 0x00000000);
-	nv_icmd(dev, 0x0000a9, 0x0000ffff);
-	nv_icmd(dev, 0x000038, 0x0fac6881);
-	nv_icmd(dev, 0x00003d, 0x00000001);
-	nv_icmd(dev, 0x0000e8, 0x00000400);
-	nv_icmd(dev, 0x0000e9, 0x00000400);
-	nv_icmd(dev, 0x0000ea, 0x00000400);
-	nv_icmd(dev, 0x0000eb, 0x00000400);
-	nv_icmd(dev, 0x0000ec, 0x00000400);
-	nv_icmd(dev, 0x0000ed, 0x00000400);
-	nv_icmd(dev, 0x0000ee, 0x00000400);
-	nv_icmd(dev, 0x0000ef, 0x00000400);
-	nv_icmd(dev, 0x000078, 0x00000300);
-	nv_icmd(dev, 0x000079, 0x00000300);
-	nv_icmd(dev, 0x00007a, 0x00000300);
-	nv_icmd(dev, 0x00007b, 0x00000300);
-	nv_icmd(dev, 0x00007c, 0x00000300);
-	nv_icmd(dev, 0x00007d, 0x00000300);
-	nv_icmd(dev, 0x00007e, 0x00000300);
-	nv_icmd(dev, 0x00007f, 0x00000300);
-	nv_icmd(dev, 0x000050, 0x00000011);
-	nv_icmd(dev, 0x000058, 0x00000008);
-	nv_icmd(dev, 0x000059, 0x00000008);
-	nv_icmd(dev, 0x00005a, 0x00000008);
-	nv_icmd(dev, 0x00005b, 0x00000008);
-	nv_icmd(dev, 0x00005c, 0x00000008);
-	nv_icmd(dev, 0x00005d, 0x00000008);
-	nv_icmd(dev, 0x00005e, 0x00000008);
-	nv_icmd(dev, 0x00005f, 0x00000008);
-	nv_icmd(dev, 0x000208, 0x00000001);
-	nv_icmd(dev, 0x000209, 0x00000001);
-	nv_icmd(dev, 0x00020a, 0x00000001);
-	nv_icmd(dev, 0x00020b, 0x00000001);
-	nv_icmd(dev, 0x00020c, 0x00000001);
-	nv_icmd(dev, 0x00020d, 0x00000001);
-	nv_icmd(dev, 0x00020e, 0x00000001);
-	nv_icmd(dev, 0x00020f, 0x00000001);
-	nv_icmd(dev, 0x000081, 0x00000001);
-	nv_icmd(dev, 0x000085, 0x00000004);
-	nv_icmd(dev, 0x000088, 0x00000400);
-	nv_icmd(dev, 0x000090, 0x00000300);
-	nv_icmd(dev, 0x000098, 0x00001001);
-	nv_icmd(dev, 0x0000e3, 0x00000001);
-	nv_icmd(dev, 0x0000da, 0x00000001);
-	nv_icmd(dev, 0x0000f8, 0x00000003);
-	nv_icmd(dev, 0x0000fa, 0x00000001);
-	nv_icmd(dev, 0x00009f, 0x0000ffff);
-	nv_icmd(dev, 0x0000a0, 0x0000ffff);
-	nv_icmd(dev, 0x0000a1, 0x0000ffff);
-	nv_icmd(dev, 0x0000a2, 0x0000ffff);
-	nv_icmd(dev, 0x0000b1, 0x00000001);
-	nv_icmd(dev, 0x0000ad, 0x0000013e);
-	nv_icmd(dev, 0x0000e1, 0x00000010);
-	nv_icmd(dev, 0x000290, 0x00000000);
-	nv_icmd(dev, 0x000291, 0x00000000);
-	nv_icmd(dev, 0x000292, 0x00000000);
-	nv_icmd(dev, 0x000293, 0x00000000);
-	nv_icmd(dev, 0x000294, 0x00000000);
-	nv_icmd(dev, 0x000295, 0x00000000);
-	nv_icmd(dev, 0x000296, 0x00000000);
-	nv_icmd(dev, 0x000297, 0x00000000);
-	nv_icmd(dev, 0x000298, 0x00000000);
-	nv_icmd(dev, 0x000299, 0x00000000);
-	nv_icmd(dev, 0x00029a, 0x00000000);
-	nv_icmd(dev, 0x00029b, 0x00000000);
-	nv_icmd(dev, 0x00029c, 0x00000000);
-	nv_icmd(dev, 0x00029d, 0x00000000);
-	nv_icmd(dev, 0x00029e, 0x00000000);
-	nv_icmd(dev, 0x00029f, 0x00000000);
-	nv_icmd(dev, 0x0003b0, 0x00000000);
-	nv_icmd(dev, 0x0003b1, 0x00000000);
-	nv_icmd(dev, 0x0003b2, 0x00000000);
-	nv_icmd(dev, 0x0003b3, 0x00000000);
-	nv_icmd(dev, 0x0003b4, 0x00000000);
-	nv_icmd(dev, 0x0003b5, 0x00000000);
-	nv_icmd(dev, 0x0003b6, 0x00000000);
-	nv_icmd(dev, 0x0003b7, 0x00000000);
-	nv_icmd(dev, 0x0003b8, 0x00000000);
-	nv_icmd(dev, 0x0003b9, 0x00000000);
-	nv_icmd(dev, 0x0003ba, 0x00000000);
-	nv_icmd(dev, 0x0003bb, 0x00000000);
-	nv_icmd(dev, 0x0003bc, 0x00000000);
-	nv_icmd(dev, 0x0003bd, 0x00000000);
-	nv_icmd(dev, 0x0003be, 0x00000000);
-	nv_icmd(dev, 0x0003bf, 0x00000000);
-	nv_icmd(dev, 0x0002a0, 0x00000000);
-	nv_icmd(dev, 0x0002a1, 0x00000000);
-	nv_icmd(dev, 0x0002a2, 0x00000000);
-	nv_icmd(dev, 0x0002a3, 0x00000000);
-	nv_icmd(dev, 0x0002a4, 0x00000000);
-	nv_icmd(dev, 0x0002a5, 0x00000000);
-	nv_icmd(dev, 0x0002a6, 0x00000000);
-	nv_icmd(dev, 0x0002a7, 0x00000000);
-	nv_icmd(dev, 0x0002a8, 0x00000000);
-	nv_icmd(dev, 0x0002a9, 0x00000000);
-	nv_icmd(dev, 0x0002aa, 0x00000000);
-	nv_icmd(dev, 0x0002ab, 0x00000000);
-	nv_icmd(dev, 0x0002ac, 0x00000000);
-	nv_icmd(dev, 0x0002ad, 0x00000000);
-	nv_icmd(dev, 0x0002ae, 0x00000000);
-	nv_icmd(dev, 0x0002af, 0x00000000);
-	nv_icmd(dev, 0x000420, 0x00000000);
-	nv_icmd(dev, 0x000421, 0x00000000);
-	nv_icmd(dev, 0x000422, 0x00000000);
-	nv_icmd(dev, 0x000423, 0x00000000);
-	nv_icmd(dev, 0x000424, 0x00000000);
-	nv_icmd(dev, 0x000425, 0x00000000);
-	nv_icmd(dev, 0x000426, 0x00000000);
-	nv_icmd(dev, 0x000427, 0x00000000);
-	nv_icmd(dev, 0x000428, 0x00000000);
-	nv_icmd(dev, 0x000429, 0x00000000);
-	nv_icmd(dev, 0x00042a, 0x00000000);
-	nv_icmd(dev, 0x00042b, 0x00000000);
-	nv_icmd(dev, 0x00042c, 0x00000000);
-	nv_icmd(dev, 0x00042d, 0x00000000);
-	nv_icmd(dev, 0x00042e, 0x00000000);
-	nv_icmd(dev, 0x00042f, 0x00000000);
-	nv_icmd(dev, 0x0002b0, 0x00000000);
-	nv_icmd(dev, 0x0002b1, 0x00000000);
-	nv_icmd(dev, 0x0002b2, 0x00000000);
-	nv_icmd(dev, 0x0002b3, 0x00000000);
-	nv_icmd(dev, 0x0002b4, 0x00000000);
-	nv_icmd(dev, 0x0002b5, 0x00000000);
-	nv_icmd(dev, 0x0002b6, 0x00000000);
-	nv_icmd(dev, 0x0002b7, 0x00000000);
-	nv_icmd(dev, 0x0002b8, 0x00000000);
-	nv_icmd(dev, 0x0002b9, 0x00000000);
-	nv_icmd(dev, 0x0002ba, 0x00000000);
-	nv_icmd(dev, 0x0002bb, 0x00000000);
-	nv_icmd(dev, 0x0002bc, 0x00000000);
-	nv_icmd(dev, 0x0002bd, 0x00000000);
-	nv_icmd(dev, 0x0002be, 0x00000000);
-	nv_icmd(dev, 0x0002bf, 0x00000000);
-	nv_icmd(dev, 0x000430, 0x00000000);
-	nv_icmd(dev, 0x000431, 0x00000000);
-	nv_icmd(dev, 0x000432, 0x00000000);
-	nv_icmd(dev, 0x000433, 0x00000000);
-	nv_icmd(dev, 0x000434, 0x00000000);
-	nv_icmd(dev, 0x000435, 0x00000000);
-	nv_icmd(dev, 0x000436, 0x00000000);
-	nv_icmd(dev, 0x000437, 0x00000000);
-	nv_icmd(dev, 0x000438, 0x00000000);
-	nv_icmd(dev, 0x000439, 0x00000000);
-	nv_icmd(dev, 0x00043a, 0x00000000);
-	nv_icmd(dev, 0x00043b, 0x00000000);
-	nv_icmd(dev, 0x00043c, 0x00000000);
-	nv_icmd(dev, 0x00043d, 0x00000000);
-	nv_icmd(dev, 0x00043e, 0x00000000);
-	nv_icmd(dev, 0x00043f, 0x00000000);
-	nv_icmd(dev, 0x0002c0, 0x00000000);
-	nv_icmd(dev, 0x0002c1, 0x00000000);
-	nv_icmd(dev, 0x0002c2, 0x00000000);
-	nv_icmd(dev, 0x0002c3, 0x00000000);
-	nv_icmd(dev, 0x0002c4, 0x00000000);
-	nv_icmd(dev, 0x0002c5, 0x00000000);
-	nv_icmd(dev, 0x0002c6, 0x00000000);
-	nv_icmd(dev, 0x0002c7, 0x00000000);
-	nv_icmd(dev, 0x0002c8, 0x00000000);
-	nv_icmd(dev, 0x0002c9, 0x00000000);
-	nv_icmd(dev, 0x0002ca, 0x00000000);
-	nv_icmd(dev, 0x0002cb, 0x00000000);
-	nv_icmd(dev, 0x0002cc, 0x00000000);
-	nv_icmd(dev, 0x0002cd, 0x00000000);
-	nv_icmd(dev, 0x0002ce, 0x00000000);
-	nv_icmd(dev, 0x0002cf, 0x00000000);
-	nv_icmd(dev, 0x0004d0, 0x00000000);
-	nv_icmd(dev, 0x0004d1, 0x00000000);
-	nv_icmd(dev, 0x0004d2, 0x00000000);
-	nv_icmd(dev, 0x0004d3, 0x00000000);
-	nv_icmd(dev, 0x0004d4, 0x00000000);
-	nv_icmd(dev, 0x0004d5, 0x00000000);
-	nv_icmd(dev, 0x0004d6, 0x00000000);
-	nv_icmd(dev, 0x0004d7, 0x00000000);
-	nv_icmd(dev, 0x0004d8, 0x00000000);
-	nv_icmd(dev, 0x0004d9, 0x00000000);
-	nv_icmd(dev, 0x0004da, 0x00000000);
-	nv_icmd(dev, 0x0004db, 0x00000000);
-	nv_icmd(dev, 0x0004dc, 0x00000000);
-	nv_icmd(dev, 0x0004dd, 0x00000000);
-	nv_icmd(dev, 0x0004de, 0x00000000);
-	nv_icmd(dev, 0x0004df, 0x00000000);
-	nv_icmd(dev, 0x000720, 0x00000000);
-	nv_icmd(dev, 0x000721, 0x00000000);
-	nv_icmd(dev, 0x000722, 0x00000000);
-	nv_icmd(dev, 0x000723, 0x00000000);
-	nv_icmd(dev, 0x000724, 0x00000000);
-	nv_icmd(dev, 0x000725, 0x00000000);
-	nv_icmd(dev, 0x000726, 0x00000000);
-	nv_icmd(dev, 0x000727, 0x00000000);
-	nv_icmd(dev, 0x000728, 0x00000000);
-	nv_icmd(dev, 0x000729, 0x00000000);
-	nv_icmd(dev, 0x00072a, 0x00000000);
-	nv_icmd(dev, 0x00072b, 0x00000000);
-	nv_icmd(dev, 0x00072c, 0x00000000);
-	nv_icmd(dev, 0x00072d, 0x00000000);
-	nv_icmd(dev, 0x00072e, 0x00000000);
-	nv_icmd(dev, 0x00072f, 0x00000000);
-	nv_icmd(dev, 0x0008c0, 0x00000000);
-	nv_icmd(dev, 0x0008c1, 0x00000000);
-	nv_icmd(dev, 0x0008c2, 0x00000000);
-	nv_icmd(dev, 0x0008c3, 0x00000000);
-	nv_icmd(dev, 0x0008c4, 0x00000000);
-	nv_icmd(dev, 0x0008c5, 0x00000000);
-	nv_icmd(dev, 0x0008c6, 0x00000000);
-	nv_icmd(dev, 0x0008c7, 0x00000000);
-	nv_icmd(dev, 0x0008c8, 0x00000000);
-	nv_icmd(dev, 0x0008c9, 0x00000000);
-	nv_icmd(dev, 0x0008ca, 0x00000000);
-	nv_icmd(dev, 0x0008cb, 0x00000000);
-	nv_icmd(dev, 0x0008cc, 0x00000000);
-	nv_icmd(dev, 0x0008cd, 0x00000000);
-	nv_icmd(dev, 0x0008ce, 0x00000000);
-	nv_icmd(dev, 0x0008cf, 0x00000000);
-	nv_icmd(dev, 0x000890, 0x00000000);
-	nv_icmd(dev, 0x000891, 0x00000000);
-	nv_icmd(dev, 0x000892, 0x00000000);
-	nv_icmd(dev, 0x000893, 0x00000000);
-	nv_icmd(dev, 0x000894, 0x00000000);
-	nv_icmd(dev, 0x000895, 0x00000000);
-	nv_icmd(dev, 0x000896, 0x00000000);
-	nv_icmd(dev, 0x000897, 0x00000000);
-	nv_icmd(dev, 0x000898, 0x00000000);
-	nv_icmd(dev, 0x000899, 0x00000000);
-	nv_icmd(dev, 0x00089a, 0x00000000);
-	nv_icmd(dev, 0x00089b, 0x00000000);
-	nv_icmd(dev, 0x00089c, 0x00000000);
-	nv_icmd(dev, 0x00089d, 0x00000000);
-	nv_icmd(dev, 0x00089e, 0x00000000);
-	nv_icmd(dev, 0x00089f, 0x00000000);
-	nv_icmd(dev, 0x0008e0, 0x00000000);
-	nv_icmd(dev, 0x0008e1, 0x00000000);
-	nv_icmd(dev, 0x0008e2, 0x00000000);
-	nv_icmd(dev, 0x0008e3, 0x00000000);
-	nv_icmd(dev, 0x0008e4, 0x00000000);
-	nv_icmd(dev, 0x0008e5, 0x00000000);
-	nv_icmd(dev, 0x0008e6, 0x00000000);
-	nv_icmd(dev, 0x0008e7, 0x00000000);
-	nv_icmd(dev, 0x0008e8, 0x00000000);
-	nv_icmd(dev, 0x0008e9, 0x00000000);
-	nv_icmd(dev, 0x0008ea, 0x00000000);
-	nv_icmd(dev, 0x0008eb, 0x00000000);
-	nv_icmd(dev, 0x0008ec, 0x00000000);
-	nv_icmd(dev, 0x0008ed, 0x00000000);
-	nv_icmd(dev, 0x0008ee, 0x00000000);
-	nv_icmd(dev, 0x0008ef, 0x00000000);
-	nv_icmd(dev, 0x0008a0, 0x00000000);
-	nv_icmd(dev, 0x0008a1, 0x00000000);
-	nv_icmd(dev, 0x0008a2, 0x00000000);
-	nv_icmd(dev, 0x0008a3, 0x00000000);
-	nv_icmd(dev, 0x0008a4, 0x00000000);
-	nv_icmd(dev, 0x0008a5, 0x00000000);
-	nv_icmd(dev, 0x0008a6, 0x00000000);
-	nv_icmd(dev, 0x0008a7, 0x00000000);
-	nv_icmd(dev, 0x0008a8, 0x00000000);
-	nv_icmd(dev, 0x0008a9, 0x00000000);
-	nv_icmd(dev, 0x0008aa, 0x00000000);
-	nv_icmd(dev, 0x0008ab, 0x00000000);
-	nv_icmd(dev, 0x0008ac, 0x00000000);
-	nv_icmd(dev, 0x0008ad, 0x00000000);
-	nv_icmd(dev, 0x0008ae, 0x00000000);
-	nv_icmd(dev, 0x0008af, 0x00000000);
-	nv_icmd(dev, 0x0008f0, 0x00000000);
-	nv_icmd(dev, 0x0008f1, 0x00000000);
-	nv_icmd(dev, 0x0008f2, 0x00000000);
-	nv_icmd(dev, 0x0008f3, 0x00000000);
-	nv_icmd(dev, 0x0008f4, 0x00000000);
-	nv_icmd(dev, 0x0008f5, 0x00000000);
-	nv_icmd(dev, 0x0008f6, 0x00000000);
-	nv_icmd(dev, 0x0008f7, 0x00000000);
-	nv_icmd(dev, 0x0008f8, 0x00000000);
-	nv_icmd(dev, 0x0008f9, 0x00000000);
-	nv_icmd(dev, 0x0008fa, 0x00000000);
-	nv_icmd(dev, 0x0008fb, 0x00000000);
-	nv_icmd(dev, 0x0008fc, 0x00000000);
-	nv_icmd(dev, 0x0008fd, 0x00000000);
-	nv_icmd(dev, 0x0008fe, 0x00000000);
-	nv_icmd(dev, 0x0008ff, 0x00000000);
-	nv_icmd(dev, 0x00094c, 0x000000ff);
-	nv_icmd(dev, 0x00094d, 0xffffffff);
-	nv_icmd(dev, 0x00094e, 0x00000002);
-	nv_icmd(dev, 0x0002ec, 0x00000001);
-	nv_icmd(dev, 0x000303, 0x00000001);
-	nv_icmd(dev, 0x0002e6, 0x00000001);
-	nv_icmd(dev, 0x000466, 0x00000052);
-	nv_icmd(dev, 0x000301, 0x3f800000);
-	nv_icmd(dev, 0x000304, 0x30201000);
-	nv_icmd(dev, 0x000305, 0x70605040);
-	nv_icmd(dev, 0x000306, 0xb8a89888);
-	nv_icmd(dev, 0x000307, 0xf8e8d8c8);
-	nv_icmd(dev, 0x00030a, 0x00ffff00);
-	nv_icmd(dev, 0x00030b, 0x0000001a);
-	nv_icmd(dev, 0x00030c, 0x00000001);
-	nv_icmd(dev, 0x000318, 0x00000001);
-	nv_icmd(dev, 0x000340, 0x00000000);
-	nv_icmd(dev, 0x000375, 0x00000001);
-	nv_icmd(dev, 0x00037d, 0x00000006);
-	nv_icmd(dev, 0x0003a0, 0x00000002);
-	nv_icmd(dev, 0x0003aa, 0x00000001);
-	nv_icmd(dev, 0x0003a9, 0x00000001);
-	nv_icmd(dev, 0x000380, 0x00000001);
-	nv_icmd(dev, 0x000383, 0x00000011);
-	nv_icmd(dev, 0x000360, 0x00000040);
-	nv_icmd(dev, 0x000366, 0x00000000);
-	nv_icmd(dev, 0x000367, 0x00000000);
-	nv_icmd(dev, 0x000368, 0x00000fff);
-	nv_icmd(dev, 0x000370, 0x00000000);
-	nv_icmd(dev, 0x000371, 0x00000000);
-	nv_icmd(dev, 0x000372, 0x000fffff);
-	nv_icmd(dev, 0x00037a, 0x00000012);
-	nv_icmd(dev, 0x000619, 0x00000003);
-	nv_icmd(dev, 0x000811, 0x00000003);
-	nv_icmd(dev, 0x000812, 0x00000004);
-	nv_icmd(dev, 0x000813, 0x00000006);
-	nv_icmd(dev, 0x000814, 0x00000008);
-	nv_icmd(dev, 0x000815, 0x0000000b);
-	nv_icmd(dev, 0x000800, 0x00000001);
-	nv_icmd(dev, 0x000801, 0x00000001);
-	nv_icmd(dev, 0x000802, 0x00000001);
-	nv_icmd(dev, 0x000803, 0x00000001);
-	nv_icmd(dev, 0x000804, 0x00000001);
-	nv_icmd(dev, 0x000805, 0x00000001);
-	nv_icmd(dev, 0x000632, 0x00000001);
-	nv_icmd(dev, 0x000633, 0x00000002);
-	nv_icmd(dev, 0x000634, 0x00000003);
-	nv_icmd(dev, 0x000635, 0x00000004);
-	nv_icmd(dev, 0x000654, 0x3f800000);
-	nv_icmd(dev, 0x000657, 0x3f800000);
-	nv_icmd(dev, 0x000655, 0x3f800000);
-	nv_icmd(dev, 0x000656, 0x3f800000);
-	nv_icmd(dev, 0x0006cd, 0x3f800000);
-	nv_icmd(dev, 0x0007f5, 0x3f800000);
-	nv_icmd(dev, 0x0007dc, 0x39291909);
-	nv_icmd(dev, 0x0007dd, 0x79695949);
-	nv_icmd(dev, 0x0007de, 0xb9a99989);
-	nv_icmd(dev, 0x0007df, 0xf9e9d9c9);
-	nv_icmd(dev, 0x0007e8, 0x00003210);
-	nv_icmd(dev, 0x0007e9, 0x00007654);
-	nv_icmd(dev, 0x0007ea, 0x00000098);
-	nv_icmd(dev, 0x0007ec, 0x39291909);
-	nv_icmd(dev, 0x0007ed, 0x79695949);
-	nv_icmd(dev, 0x0007ee, 0xb9a99989);
-	nv_icmd(dev, 0x0007ef, 0xf9e9d9c9);
-	nv_icmd(dev, 0x0007f0, 0x00003210);
-	nv_icmd(dev, 0x0007f1, 0x00007654);
-	nv_icmd(dev, 0x0007f2, 0x00000098);
-	nv_icmd(dev, 0x0005a5, 0x00000001);
-	nv_icmd(dev, 0x000980, 0x00000000);
-	nv_icmd(dev, 0x000981, 0x00000000);
-	nv_icmd(dev, 0x000982, 0x00000000);
-	nv_icmd(dev, 0x000983, 0x00000000);
-	nv_icmd(dev, 0x000984, 0x00000000);
-	nv_icmd(dev, 0x000985, 0x00000000);
-	nv_icmd(dev, 0x000986, 0x00000000);
-	nv_icmd(dev, 0x000987, 0x00000000);
-	nv_icmd(dev, 0x000988, 0x00000000);
-	nv_icmd(dev, 0x000989, 0x00000000);
-	nv_icmd(dev, 0x00098a, 0x00000000);
-	nv_icmd(dev, 0x00098b, 0x00000000);
-	nv_icmd(dev, 0x00098c, 0x00000000);
-	nv_icmd(dev, 0x00098d, 0x00000000);
-	nv_icmd(dev, 0x00098e, 0x00000000);
-	nv_icmd(dev, 0x00098f, 0x00000000);
-	nv_icmd(dev, 0x000990, 0x00000000);
-	nv_icmd(dev, 0x000991, 0x00000000);
-	nv_icmd(dev, 0x000992, 0x00000000);
-	nv_icmd(dev, 0x000993, 0x00000000);
-	nv_icmd(dev, 0x000994, 0x00000000);
-	nv_icmd(dev, 0x000995, 0x00000000);
-	nv_icmd(dev, 0x000996, 0x00000000);
-	nv_icmd(dev, 0x000997, 0x00000000);
-	nv_icmd(dev, 0x000998, 0x00000000);
-	nv_icmd(dev, 0x000999, 0x00000000);
-	nv_icmd(dev, 0x00099a, 0x00000000);
-	nv_icmd(dev, 0x00099b, 0x00000000);
-	nv_icmd(dev, 0x00099c, 0x00000000);
-	nv_icmd(dev, 0x00099d, 0x00000000);
-	nv_icmd(dev, 0x00099e, 0x00000000);
-	nv_icmd(dev, 0x00099f, 0x00000000);
-	nv_icmd(dev, 0x0009a0, 0x00000000);
-	nv_icmd(dev, 0x0009a1, 0x00000000);
-	nv_icmd(dev, 0x0009a2, 0x00000000);
-	nv_icmd(dev, 0x0009a3, 0x00000000);
-	nv_icmd(dev, 0x0009a4, 0x00000000);
-	nv_icmd(dev, 0x0009a5, 0x00000000);
-	nv_icmd(dev, 0x0009a6, 0x00000000);
-	nv_icmd(dev, 0x0009a7, 0x00000000);
-	nv_icmd(dev, 0x0009a8, 0x00000000);
-	nv_icmd(dev, 0x0009a9, 0x00000000);
-	nv_icmd(dev, 0x0009aa, 0x00000000);
-	nv_icmd(dev, 0x0009ab, 0x00000000);
-	nv_icmd(dev, 0x0009ac, 0x00000000);
-	nv_icmd(dev, 0x0009ad, 0x00000000);
-	nv_icmd(dev, 0x0009ae, 0x00000000);
-	nv_icmd(dev, 0x0009af, 0x00000000);
-	nv_icmd(dev, 0x0009b0, 0x00000000);
-	nv_icmd(dev, 0x0009b1, 0x00000000);
-	nv_icmd(dev, 0x0009b2, 0x00000000);
-	nv_icmd(dev, 0x0009b3, 0x00000000);
-	nv_icmd(dev, 0x0009b4, 0x00000000);
-	nv_icmd(dev, 0x0009b5, 0x00000000);
-	nv_icmd(dev, 0x0009b6, 0x00000000);
-	nv_icmd(dev, 0x0009b7, 0x00000000);
-	nv_icmd(dev, 0x0009b8, 0x00000000);
-	nv_icmd(dev, 0x0009b9, 0x00000000);
-	nv_icmd(dev, 0x0009ba, 0x00000000);
-	nv_icmd(dev, 0x0009bb, 0x00000000);
-	nv_icmd(dev, 0x0009bc, 0x00000000);
-	nv_icmd(dev, 0x0009bd, 0x00000000);
-	nv_icmd(dev, 0x0009be, 0x00000000);
-	nv_icmd(dev, 0x0009bf, 0x00000000);
-	nv_icmd(dev, 0x0009c0, 0x00000000);
-	nv_icmd(dev, 0x0009c1, 0x00000000);
-	nv_icmd(dev, 0x0009c2, 0x00000000);
-	nv_icmd(dev, 0x0009c3, 0x00000000);
-	nv_icmd(dev, 0x0009c4, 0x00000000);
-	nv_icmd(dev, 0x0009c5, 0x00000000);
-	nv_icmd(dev, 0x0009c6, 0x00000000);
-	nv_icmd(dev, 0x0009c7, 0x00000000);
-	nv_icmd(dev, 0x0009c8, 0x00000000);
-	nv_icmd(dev, 0x0009c9, 0x00000000);
-	nv_icmd(dev, 0x0009ca, 0x00000000);
-	nv_icmd(dev, 0x0009cb, 0x00000000);
-	nv_icmd(dev, 0x0009cc, 0x00000000);
-	nv_icmd(dev, 0x0009cd, 0x00000000);
-	nv_icmd(dev, 0x0009ce, 0x00000000);
-	nv_icmd(dev, 0x0009cf, 0x00000000);
-	nv_icmd(dev, 0x0009d0, 0x00000000);
-	nv_icmd(dev, 0x0009d1, 0x00000000);
-	nv_icmd(dev, 0x0009d2, 0x00000000);
-	nv_icmd(dev, 0x0009d3, 0x00000000);
-	nv_icmd(dev, 0x0009d4, 0x00000000);
-	nv_icmd(dev, 0x0009d5, 0x00000000);
-	nv_icmd(dev, 0x0009d6, 0x00000000);
-	nv_icmd(dev, 0x0009d7, 0x00000000);
-	nv_icmd(dev, 0x0009d8, 0x00000000);
-	nv_icmd(dev, 0x0009d9, 0x00000000);
-	nv_icmd(dev, 0x0009da, 0x00000000);
-	nv_icmd(dev, 0x0009db, 0x00000000);
-	nv_icmd(dev, 0x0009dc, 0x00000000);
-	nv_icmd(dev, 0x0009dd, 0x00000000);
-	nv_icmd(dev, 0x0009de, 0x00000000);
-	nv_icmd(dev, 0x0009df, 0x00000000);
-	nv_icmd(dev, 0x0009e0, 0x00000000);
-	nv_icmd(dev, 0x0009e1, 0x00000000);
-	nv_icmd(dev, 0x0009e2, 0x00000000);
-	nv_icmd(dev, 0x0009e3, 0x00000000);
-	nv_icmd(dev, 0x0009e4, 0x00000000);
-	nv_icmd(dev, 0x0009e5, 0x00000000);
-	nv_icmd(dev, 0x0009e6, 0x00000000);
-	nv_icmd(dev, 0x0009e7, 0x00000000);
-	nv_icmd(dev, 0x0009e8, 0x00000000);
-	nv_icmd(dev, 0x0009e9, 0x00000000);
-	nv_icmd(dev, 0x0009ea, 0x00000000);
-	nv_icmd(dev, 0x0009eb, 0x00000000);
-	nv_icmd(dev, 0x0009ec, 0x00000000);
-	nv_icmd(dev, 0x0009ed, 0x00000000);
-	nv_icmd(dev, 0x0009ee, 0x00000000);
-	nv_icmd(dev, 0x0009ef, 0x00000000);
-	nv_icmd(dev, 0x0009f0, 0x00000000);
-	nv_icmd(dev, 0x0009f1, 0x00000000);
-	nv_icmd(dev, 0x0009f2, 0x00000000);
-	nv_icmd(dev, 0x0009f3, 0x00000000);
-	nv_icmd(dev, 0x0009f4, 0x00000000);
-	nv_icmd(dev, 0x0009f5, 0x00000000);
-	nv_icmd(dev, 0x0009f6, 0x00000000);
-	nv_icmd(dev, 0x0009f7, 0x00000000);
-	nv_icmd(dev, 0x0009f8, 0x00000000);
-	nv_icmd(dev, 0x0009f9, 0x00000000);
-	nv_icmd(dev, 0x0009fa, 0x00000000);
-	nv_icmd(dev, 0x0009fb, 0x00000000);
-	nv_icmd(dev, 0x0009fc, 0x00000000);
-	nv_icmd(dev, 0x0009fd, 0x00000000);
-	nv_icmd(dev, 0x0009fe, 0x00000000);
-	nv_icmd(dev, 0x0009ff, 0x00000000);
-	nv_icmd(dev, 0x000468, 0x00000004);
-	nv_icmd(dev, 0x00046c, 0x00000001);
-	nv_icmd(dev, 0x000470, 0x00000000);
-	nv_icmd(dev, 0x000471, 0x00000000);
-	nv_icmd(dev, 0x000472, 0x00000000);
-	nv_icmd(dev, 0x000473, 0x00000000);
-	nv_icmd(dev, 0x000474, 0x00000000);
-	nv_icmd(dev, 0x000475, 0x00000000);
-	nv_icmd(dev, 0x000476, 0x00000000);
-	nv_icmd(dev, 0x000477, 0x00000000);
-	nv_icmd(dev, 0x000478, 0x00000000);
-	nv_icmd(dev, 0x000479, 0x00000000);
-	nv_icmd(dev, 0x00047a, 0x00000000);
-	nv_icmd(dev, 0x00047b, 0x00000000);
-	nv_icmd(dev, 0x00047c, 0x00000000);
-	nv_icmd(dev, 0x00047d, 0x00000000);
-	nv_icmd(dev, 0x00047e, 0x00000000);
-	nv_icmd(dev, 0x00047f, 0x00000000);
-	nv_icmd(dev, 0x000480, 0x00000000);
-	nv_icmd(dev, 0x000481, 0x00000000);
-	nv_icmd(dev, 0x000482, 0x00000000);
-	nv_icmd(dev, 0x000483, 0x00000000);
-	nv_icmd(dev, 0x000484, 0x00000000);
-	nv_icmd(dev, 0x000485, 0x00000000);
-	nv_icmd(dev, 0x000486, 0x00000000);
-	nv_icmd(dev, 0x000487, 0x00000000);
-	nv_icmd(dev, 0x000488, 0x00000000);
-	nv_icmd(dev, 0x000489, 0x00000000);
-	nv_icmd(dev, 0x00048a, 0x00000000);
-	nv_icmd(dev, 0x00048b, 0x00000000);
-	nv_icmd(dev, 0x00048c, 0x00000000);
-	nv_icmd(dev, 0x00048d, 0x00000000);
-	nv_icmd(dev, 0x00048e, 0x00000000);
-	nv_icmd(dev, 0x00048f, 0x00000000);
-	nv_icmd(dev, 0x000490, 0x00000000);
-	nv_icmd(dev, 0x000491, 0x00000000);
-	nv_icmd(dev, 0x000492, 0x00000000);
-	nv_icmd(dev, 0x000493, 0x00000000);
-	nv_icmd(dev, 0x000494, 0x00000000);
-	nv_icmd(dev, 0x000495, 0x00000000);
-	nv_icmd(dev, 0x000496, 0x00000000);
-	nv_icmd(dev, 0x000497, 0x00000000);
-	nv_icmd(dev, 0x000498, 0x00000000);
-	nv_icmd(dev, 0x000499, 0x00000000);
-	nv_icmd(dev, 0x00049a, 0x00000000);
-	nv_icmd(dev, 0x00049b, 0x00000000);
-	nv_icmd(dev, 0x00049c, 0x00000000);
-	nv_icmd(dev, 0x00049d, 0x00000000);
-	nv_icmd(dev, 0x00049e, 0x00000000);
-	nv_icmd(dev, 0x00049f, 0x00000000);
-	nv_icmd(dev, 0x0004a0, 0x00000000);
-	nv_icmd(dev, 0x0004a1, 0x00000000);
-	nv_icmd(dev, 0x0004a2, 0x00000000);
-	nv_icmd(dev, 0x0004a3, 0x00000000);
-	nv_icmd(dev, 0x0004a4, 0x00000000);
-	nv_icmd(dev, 0x0004a5, 0x00000000);
-	nv_icmd(dev, 0x0004a6, 0x00000000);
-	nv_icmd(dev, 0x0004a7, 0x00000000);
-	nv_icmd(dev, 0x0004a8, 0x00000000);
-	nv_icmd(dev, 0x0004a9, 0x00000000);
-	nv_icmd(dev, 0x0004aa, 0x00000000);
-	nv_icmd(dev, 0x0004ab, 0x00000000);
-	nv_icmd(dev, 0x0004ac, 0x00000000);
-	nv_icmd(dev, 0x0004ad, 0x00000000);
-	nv_icmd(dev, 0x0004ae, 0x00000000);
-	nv_icmd(dev, 0x0004af, 0x00000000);
-	nv_icmd(dev, 0x0004b0, 0x00000000);
-	nv_icmd(dev, 0x0004b1, 0x00000000);
-	nv_icmd(dev, 0x0004b2, 0x00000000);
-	nv_icmd(dev, 0x0004b3, 0x00000000);
-	nv_icmd(dev, 0x0004b4, 0x00000000);
-	nv_icmd(dev, 0x0004b5, 0x00000000);
-	nv_icmd(dev, 0x0004b6, 0x00000000);
-	nv_icmd(dev, 0x0004b7, 0x00000000);
-	nv_icmd(dev, 0x0004b8, 0x00000000);
-	nv_icmd(dev, 0x0004b9, 0x00000000);
-	nv_icmd(dev, 0x0004ba, 0x00000000);
-	nv_icmd(dev, 0x0004bb, 0x00000000);
-	nv_icmd(dev, 0x0004bc, 0x00000000);
-	nv_icmd(dev, 0x0004bd, 0x00000000);
-	nv_icmd(dev, 0x0004be, 0x00000000);
-	nv_icmd(dev, 0x0004bf, 0x00000000);
-	nv_icmd(dev, 0x0004c0, 0x00000000);
-	nv_icmd(dev, 0x0004c1, 0x00000000);
-	nv_icmd(dev, 0x0004c2, 0x00000000);
-	nv_icmd(dev, 0x0004c3, 0x00000000);
-	nv_icmd(dev, 0x0004c4, 0x00000000);
-	nv_icmd(dev, 0x0004c5, 0x00000000);
-	nv_icmd(dev, 0x0004c6, 0x00000000);
-	nv_icmd(dev, 0x0004c7, 0x00000000);
-	nv_icmd(dev, 0x0004c8, 0x00000000);
-	nv_icmd(dev, 0x0004c9, 0x00000000);
-	nv_icmd(dev, 0x0004ca, 0x00000000);
-	nv_icmd(dev, 0x0004cb, 0x00000000);
-	nv_icmd(dev, 0x0004cc, 0x00000000);
-	nv_icmd(dev, 0x0004cd, 0x00000000);
-	nv_icmd(dev, 0x0004ce, 0x00000000);
-	nv_icmd(dev, 0x0004cf, 0x00000000);
-	nv_icmd(dev, 0x000510, 0x3f800000);
-	nv_icmd(dev, 0x000511, 0x3f800000);
-	nv_icmd(dev, 0x000512, 0x3f800000);
-	nv_icmd(dev, 0x000513, 0x3f800000);
-	nv_icmd(dev, 0x000514, 0x3f800000);
-	nv_icmd(dev, 0x000515, 0x3f800000);
-	nv_icmd(dev, 0x000516, 0x3f800000);
-	nv_icmd(dev, 0x000517, 0x3f800000);
-	nv_icmd(dev, 0x000518, 0x3f800000);
-	nv_icmd(dev, 0x000519, 0x3f800000);
-	nv_icmd(dev, 0x00051a, 0x3f800000);
-	nv_icmd(dev, 0x00051b, 0x3f800000);
-	nv_icmd(dev, 0x00051c, 0x3f800000);
-	nv_icmd(dev, 0x00051d, 0x3f800000);
-	nv_icmd(dev, 0x00051e, 0x3f800000);
-	nv_icmd(dev, 0x00051f, 0x3f800000);
-	nv_icmd(dev, 0x000520, 0x000002b6);
-	nv_icmd(dev, 0x000529, 0x00000001);
-	nv_icmd(dev, 0x000530, 0xffff0000);
-	nv_icmd(dev, 0x000531, 0xffff0000);
-	nv_icmd(dev, 0x000532, 0xffff0000);
-	nv_icmd(dev, 0x000533, 0xffff0000);
-	nv_icmd(dev, 0x000534, 0xffff0000);
-	nv_icmd(dev, 0x000535, 0xffff0000);
-	nv_icmd(dev, 0x000536, 0xffff0000);
-	nv_icmd(dev, 0x000537, 0xffff0000);
-	nv_icmd(dev, 0x000538, 0xffff0000);
-	nv_icmd(dev, 0x000539, 0xffff0000);
-	nv_icmd(dev, 0x00053a, 0xffff0000);
-	nv_icmd(dev, 0x00053b, 0xffff0000);
-	nv_icmd(dev, 0x00053c, 0xffff0000);
-	nv_icmd(dev, 0x00053d, 0xffff0000);
-	nv_icmd(dev, 0x00053e, 0xffff0000);
-	nv_icmd(dev, 0x00053f, 0xffff0000);
-	nv_icmd(dev, 0x000585, 0x0000003f);
-	nv_icmd(dev, 0x000576, 0x00000003);
-	nv_icmd(dev, 0x00057b, 0x00000059);
-	nv_icmd(dev, 0x000586, 0x00000040);
-	nv_icmd(dev, 0x000582, 0x00000080);
-	nv_icmd(dev, 0x000583, 0x00000080);
-	nv_icmd(dev, 0x0005c2, 0x00000001);
-	nv_icmd(dev, 0x000638, 0x00000001);
-	nv_icmd(dev, 0x000639, 0x00000001);
-	nv_icmd(dev, 0x00063a, 0x00000002);
-	nv_icmd(dev, 0x00063b, 0x00000001);
-	nv_icmd(dev, 0x00063c, 0x00000001);
-	nv_icmd(dev, 0x00063d, 0x00000002);
-	nv_icmd(dev, 0x00063e, 0x00000001);
-	nv_icmd(dev, 0x0008b8, 0x00000001);
-	nv_icmd(dev, 0x0008b9, 0x00000001);
-	nv_icmd(dev, 0x0008ba, 0x00000001);
-	nv_icmd(dev, 0x0008bb, 0x00000001);
-	nv_icmd(dev, 0x0008bc, 0x00000001);
-	nv_icmd(dev, 0x0008bd, 0x00000001);
-	nv_icmd(dev, 0x0008be, 0x00000001);
-	nv_icmd(dev, 0x0008bf, 0x00000001);
-	nv_icmd(dev, 0x000900, 0x00000001);
-	nv_icmd(dev, 0x000901, 0x00000001);
-	nv_icmd(dev, 0x000902, 0x00000001);
-	nv_icmd(dev, 0x000903, 0x00000001);
-	nv_icmd(dev, 0x000904, 0x00000001);
-	nv_icmd(dev, 0x000905, 0x00000001);
-	nv_icmd(dev, 0x000906, 0x00000001);
-	nv_icmd(dev, 0x000907, 0x00000001);
-	nv_icmd(dev, 0x000908, 0x00000002);
-	nv_icmd(dev, 0x000909, 0x00000002);
-	nv_icmd(dev, 0x00090a, 0x00000002);
-	nv_icmd(dev, 0x00090b, 0x00000002);
-	nv_icmd(dev, 0x00090c, 0x00000002);
-	nv_icmd(dev, 0x00090d, 0x00000002);
-	nv_icmd(dev, 0x00090e, 0x00000002);
-	nv_icmd(dev, 0x00090f, 0x00000002);
-	nv_icmd(dev, 0x000910, 0x00000001);
-	nv_icmd(dev, 0x000911, 0x00000001);
-	nv_icmd(dev, 0x000912, 0x00000001);
-	nv_icmd(dev, 0x000913, 0x00000001);
-	nv_icmd(dev, 0x000914, 0x00000001);
-	nv_icmd(dev, 0x000915, 0x00000001);
-	nv_icmd(dev, 0x000916, 0x00000001);
-	nv_icmd(dev, 0x000917, 0x00000001);
-	nv_icmd(dev, 0x000918, 0x00000001);
-	nv_icmd(dev, 0x000919, 0x00000001);
-	nv_icmd(dev, 0x00091a, 0x00000001);
-	nv_icmd(dev, 0x00091b, 0x00000001);
-	nv_icmd(dev, 0x00091c, 0x00000001);
-	nv_icmd(dev, 0x00091d, 0x00000001);
-	nv_icmd(dev, 0x00091e, 0x00000001);
-	nv_icmd(dev, 0x00091f, 0x00000001);
-	nv_icmd(dev, 0x000920, 0x00000002);
-	nv_icmd(dev, 0x000921, 0x00000002);
-	nv_icmd(dev, 0x000922, 0x00000002);
-	nv_icmd(dev, 0x000923, 0x00000002);
-	nv_icmd(dev, 0x000924, 0x00000002);
-	nv_icmd(dev, 0x000925, 0x00000002);
-	nv_icmd(dev, 0x000926, 0x00000002);
-	nv_icmd(dev, 0x000927, 0x00000002);
-	nv_icmd(dev, 0x000928, 0x00000001);
-	nv_icmd(dev, 0x000929, 0x00000001);
-	nv_icmd(dev, 0x00092a, 0x00000001);
-	nv_icmd(dev, 0x00092b, 0x00000001);
-	nv_icmd(dev, 0x00092c, 0x00000001);
-	nv_icmd(dev, 0x00092d, 0x00000001);
-	nv_icmd(dev, 0x00092e, 0x00000001);
-	nv_icmd(dev, 0x00092f, 0x00000001);
-	nv_icmd(dev, 0x000648, 0x00000001);
-	nv_icmd(dev, 0x000649, 0x00000001);
-	nv_icmd(dev, 0x00064a, 0x00000001);
-	nv_icmd(dev, 0x00064b, 0x00000001);
-	nv_icmd(dev, 0x00064c, 0x00000001);
-	nv_icmd(dev, 0x00064d, 0x00000001);
-	nv_icmd(dev, 0x00064e, 0x00000001);
-	nv_icmd(dev, 0x00064f, 0x00000001);
-	nv_icmd(dev, 0x000650, 0x00000001);
-	nv_icmd(dev, 0x000658, 0x0000000f);
-	nv_icmd(dev, 0x0007ff, 0x0000000a);
-	nv_icmd(dev, 0x00066a, 0x40000000);
-	nv_icmd(dev, 0x00066b, 0x10000000);
-	nv_icmd(dev, 0x00066c, 0xffff0000);
-	nv_icmd(dev, 0x00066d, 0xffff0000);
-	nv_icmd(dev, 0x0007af, 0x00000008);
-	nv_icmd(dev, 0x0007b0, 0x00000008);
-	nv_icmd(dev, 0x0007f6, 0x00000001);
-	nv_icmd(dev, 0x0006b2, 0x00000055);
-	nv_icmd(dev, 0x0007ad, 0x00000003);
-	nv_icmd(dev, 0x000937, 0x00000001);
-	nv_icmd(dev, 0x000971, 0x00000008);
-	nv_icmd(dev, 0x000972, 0x00000040);
-	nv_icmd(dev, 0x000973, 0x0000012c);
-	nv_icmd(dev, 0x00097c, 0x00000040);
-	nv_icmd(dev, 0x000979, 0x00000003);
-	nv_icmd(dev, 0x000975, 0x00000020);
-	nv_icmd(dev, 0x000976, 0x00000001);
-	nv_icmd(dev, 0x000977, 0x00000020);
-	nv_icmd(dev, 0x000978, 0x00000001);
-	nv_icmd(dev, 0x000957, 0x00000003);
-	nv_icmd(dev, 0x00095e, 0x20164010);
-	nv_icmd(dev, 0x00095f, 0x00000020);
-	nv_icmd(dev, 0x00097d, 0x00000020);
-	nv_icmd(dev, 0x000683, 0x00000006);
-	nv_icmd(dev, 0x000685, 0x003fffff);
-	nv_icmd(dev, 0x000687, 0x003fffff);
-	nv_icmd(dev, 0x0006a0, 0x00000005);
-	nv_icmd(dev, 0x000840, 0x00400008);
-	nv_icmd(dev, 0x000841, 0x08000080);
-	nv_icmd(dev, 0x000842, 0x00400008);
-	nv_icmd(dev, 0x000843, 0x08000080);
-	nv_icmd(dev, 0x000818, 0x00000000);
-	nv_icmd(dev, 0x000819, 0x00000000);
-	nv_icmd(dev, 0x00081a, 0x00000000);
-	nv_icmd(dev, 0x00081b, 0x00000000);
-	nv_icmd(dev, 0x00081c, 0x00000000);
-	nv_icmd(dev, 0x00081d, 0x00000000);
-	nv_icmd(dev, 0x00081e, 0x00000000);
-	nv_icmd(dev, 0x00081f, 0x00000000);
-	nv_icmd(dev, 0x000848, 0x00000000);
-	nv_icmd(dev, 0x000849, 0x00000000);
-	nv_icmd(dev, 0x00084a, 0x00000000);
-	nv_icmd(dev, 0x00084b, 0x00000000);
-	nv_icmd(dev, 0x00084c, 0x00000000);
-	nv_icmd(dev, 0x00084d, 0x00000000);
-	nv_icmd(dev, 0x00084e, 0x00000000);
-	nv_icmd(dev, 0x00084f, 0x00000000);
-	nv_icmd(dev, 0x000850, 0x00000000);
-	nv_icmd(dev, 0x000851, 0x00000000);
-	nv_icmd(dev, 0x000852, 0x00000000);
-	nv_icmd(dev, 0x000853, 0x00000000);
-	nv_icmd(dev, 0x000854, 0x00000000);
-	nv_icmd(dev, 0x000855, 0x00000000);
-	nv_icmd(dev, 0x000856, 0x00000000);
-	nv_icmd(dev, 0x000857, 0x00000000);
-	nv_icmd(dev, 0x000738, 0x00000000);
-	nv_icmd(dev, 0x0006aa, 0x00000001);
-	nv_icmd(dev, 0x0006ab, 0x00000002);
-	nv_icmd(dev, 0x0006ac, 0x00000080);
-	nv_icmd(dev, 0x0006ad, 0x00000100);
-	nv_icmd(dev, 0x0006ae, 0x00000100);
-	nv_icmd(dev, 0x0006b1, 0x00000011);
-	nv_icmd(dev, 0x0006bb, 0x000000cf);
-	nv_icmd(dev, 0x0006ce, 0x2a712488);
-	nv_icmd(dev, 0x000739, 0x4085c000);
-	nv_icmd(dev, 0x00073a, 0x00000080);
-	nv_icmd(dev, 0x000786, 0x80000100);
-	nv_icmd(dev, 0x00073c, 0x00010100);
-	nv_icmd(dev, 0x00073d, 0x02800000);
-	nv_icmd(dev, 0x000787, 0x000000cf);
-	nv_icmd(dev, 0x00078c, 0x00000008);
-	nv_icmd(dev, 0x000792, 0x00000001);
-	nv_icmd(dev, 0x000794, 0x00000001);
-	nv_icmd(dev, 0x000795, 0x00000001);
-	nv_icmd(dev, 0x000796, 0x00000001);
-	nv_icmd(dev, 0x000797, 0x000000cf);
-	nv_icmd(dev, 0x000836, 0x00000001);
-	nv_icmd(dev, 0x00079a, 0x00000002);
-	nv_icmd(dev, 0x000833, 0x04444480);
-	nv_icmd(dev, 0x0007a1, 0x00000001);
-	nv_icmd(dev, 0x0007a3, 0x00000001);
-	nv_icmd(dev, 0x0007a4, 0x00000001);
-	nv_icmd(dev, 0x0007a5, 0x00000001);
-	nv_icmd(dev, 0x000831, 0x00000004);
-	nv_icmd(dev, 0x000b07, 0x00000002);
-	nv_icmd(dev, 0x000b08, 0x00000100);
-	nv_icmd(dev, 0x000b09, 0x00000100);
-	nv_icmd(dev, 0x000b0a, 0x00000001);
-	nv_icmd(dev, 0x000a04, 0x000000ff);
-	nv_icmd(dev, 0x000a0b, 0x00000040);
-	nv_icmd(dev, 0x00097f, 0x00000100);
-	nv_icmd(dev, 0x000a02, 0x00000001);
-	nv_icmd(dev, 0x000809, 0x00000007);
-	nv_icmd(dev, 0x00c221, 0x00000040);
-	nv_icmd(dev, 0x00c1b0, 0x0000000f);
-	nv_icmd(dev, 0x00c1b1, 0x0000000f);
-	nv_icmd(dev, 0x00c1b2, 0x0000000f);
-	nv_icmd(dev, 0x00c1b3, 0x0000000f);
-	nv_icmd(dev, 0x00c1b4, 0x0000000f);
-	nv_icmd(dev, 0x00c1b5, 0x0000000f);
-	nv_icmd(dev, 0x00c1b6, 0x0000000f);
-	nv_icmd(dev, 0x00c1b7, 0x0000000f);
-	nv_icmd(dev, 0x00c1b8, 0x0fac6881);
-	nv_icmd(dev, 0x00c1b9, 0x00fac688);
-	nv_icmd(dev, 0x00c401, 0x00000001);
-	nv_icmd(dev, 0x00c402, 0x00010001);
-	nv_icmd(dev, 0x00c403, 0x00000001);
-	nv_icmd(dev, 0x00c404, 0x00000001);
-	nv_icmd(dev, 0x00c40e, 0x00000020);
-	nv_icmd(dev, 0x00c500, 0x00000003);
-	nv_icmd(dev, 0x01e100, 0x00000001);
-	nv_icmd(dev, 0x001000, 0x00000002);
-	nv_icmd(dev, 0x0006aa, 0x00000001);
-	nv_icmd(dev, 0x0006ad, 0x00000100);
-	nv_icmd(dev, 0x0006ae, 0x00000100);
-	nv_icmd(dev, 0x0006b1, 0x00000011);
-	nv_icmd(dev, 0x00078c, 0x00000008);
-	nv_icmd(dev, 0x000792, 0x00000001);
-	nv_icmd(dev, 0x000794, 0x00000001);
-	nv_icmd(dev, 0x000795, 0x00000001);
-	nv_icmd(dev, 0x000796, 0x00000001);
-	nv_icmd(dev, 0x000797, 0x000000cf);
-	nv_icmd(dev, 0x00079a, 0x00000002);
-	nv_icmd(dev, 0x000833, 0x04444480);
-	nv_icmd(dev, 0x0007a1, 0x00000001);
-	nv_icmd(dev, 0x0007a3, 0x00000001);
-	nv_icmd(dev, 0x0007a4, 0x00000001);
-	nv_icmd(dev, 0x0007a5, 0x00000001);
-	nv_icmd(dev, 0x000831, 0x00000004);
-	nv_icmd(dev, 0x01e100, 0x00000001);
-	nv_icmd(dev, 0x001000, 0x00000008);
-	nv_icmd(dev, 0x000039, 0x00000000);
-	nv_icmd(dev, 0x00003a, 0x00000000);
-	nv_icmd(dev, 0x00003b, 0x00000000);
-	nv_icmd(dev, 0x000380, 0x00000001);
-	nv_icmd(dev, 0x000366, 0x00000000);
-	nv_icmd(dev, 0x000367, 0x00000000);
-	nv_icmd(dev, 0x000368, 0x00000fff);
-	nv_icmd(dev, 0x000370, 0x00000000);
-	nv_icmd(dev, 0x000371, 0x00000000);
-	nv_icmd(dev, 0x000372, 0x000fffff);
-	nv_icmd(dev, 0x000813, 0x00000006);
-	nv_icmd(dev, 0x000814, 0x00000008);
-	nv_icmd(dev, 0x000957, 0x00000003);
-	nv_icmd(dev, 0x000818, 0x00000000);
-	nv_icmd(dev, 0x000819, 0x00000000);
-	nv_icmd(dev, 0x00081a, 0x00000000);
-	nv_icmd(dev, 0x00081b, 0x00000000);
-	nv_icmd(dev, 0x00081c, 0x00000000);
-	nv_icmd(dev, 0x00081d, 0x00000000);
-	nv_icmd(dev, 0x00081e, 0x00000000);
-	nv_icmd(dev, 0x00081f, 0x00000000);
-	nv_icmd(dev, 0x000848, 0x00000000);
-	nv_icmd(dev, 0x000849, 0x00000000);
-	nv_icmd(dev, 0x00084a, 0x00000000);
-	nv_icmd(dev, 0x00084b, 0x00000000);
-	nv_icmd(dev, 0x00084c, 0x00000000);
-	nv_icmd(dev, 0x00084d, 0x00000000);
-	nv_icmd(dev, 0x00084e, 0x00000000);
-	nv_icmd(dev, 0x00084f, 0x00000000);
-	nv_icmd(dev, 0x000850, 0x00000000);
-	nv_icmd(dev, 0x000851, 0x00000000);
-	nv_icmd(dev, 0x000852, 0x00000000);
-	nv_icmd(dev, 0x000853, 0x00000000);
-	nv_icmd(dev, 0x000854, 0x00000000);
-	nv_icmd(dev, 0x000855, 0x00000000);
-	nv_icmd(dev, 0x000856, 0x00000000);
-	nv_icmd(dev, 0x000857, 0x00000000);
-	nv_icmd(dev, 0x000738, 0x00000000);
-	nv_icmd(dev, 0x000b07, 0x00000002);
-	nv_icmd(dev, 0x000b08, 0x00000100);
-	nv_icmd(dev, 0x000b09, 0x00000100);
-	nv_icmd(dev, 0x000b0a, 0x00000001);
-	nv_icmd(dev, 0x000a04, 0x000000ff);
-	nv_icmd(dev, 0x00097f, 0x00000100);
-	nv_icmd(dev, 0x000a02, 0x00000001);
-	nv_icmd(dev, 0x000809, 0x00000007);
-	nv_icmd(dev, 0x00c221, 0x00000040);
-	nv_icmd(dev, 0x00c401, 0x00000001);
-	nv_icmd(dev, 0x00c402, 0x00010001);
-	nv_icmd(dev, 0x00c403, 0x00000001);
-	nv_icmd(dev, 0x00c404, 0x00000001);
-	nv_icmd(dev, 0x00c40e, 0x00000020);
-	nv_icmd(dev, 0x00c500, 0x00000003);
-	nv_icmd(dev, 0x01e100, 0x00000001);
-	nv_icmd(dev, 0x001000, 0x00000001);
-	nv_icmd(dev, 0x000b07, 0x00000002);
-	nv_icmd(dev, 0x000b08, 0x00000100);
-	nv_icmd(dev, 0x000b09, 0x00000100);
-	nv_icmd(dev, 0x000b0a, 0x00000001);
-	nv_icmd(dev, 0x01e100, 0x00000001);
-	nv_wr32(dev, 0x400208, 0x00000000);
-}
-
-static void
-nv_mthd(struct drm_device *dev, u32 class, u32 mthd, u32 data)
-{
-	nv_wr32(dev, 0x40448c, data);
-	nv_wr32(dev, 0x404488, 0x80000000 | (mthd << 14) | class);
-}
-
-static void
-nve0_grctx_generate_a097(struct drm_device *dev)
-{
-	nv_mthd(dev, 0xa097, 0x0800, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0840, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0880, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0900, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0940, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0980, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0804, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0844, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0884, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0904, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0944, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0984, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0808, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x0848, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x0888, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x08c8, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x0908, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x0948, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x0988, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x09c8, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x080c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x084c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x088c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x08cc, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x090c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x094c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x098c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x09cc, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x0810, 0x000000cf);
-	nv_mthd(dev, 0xa097, 0x0850, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0890, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0910, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0950, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0990, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0814, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0854, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0894, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x08d4, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0914, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0954, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0994, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x09d4, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0818, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0858, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0898, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x08d8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0918, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0958, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0998, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x09d8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x081c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x085c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x089c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x091c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x095c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x099c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0820, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0860, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x08e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0920, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0960, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x09e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ca0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ce0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cf0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ca4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cb4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cd4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ce4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cf4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c18, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c38, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c78, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c98, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ca8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cb8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cd8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ce8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cf8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c1c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c2c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c3c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c5c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c6c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c7c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1c9c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cbc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ccc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cdc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1cfc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1da0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1db0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1de0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1df0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1da4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1db4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dd4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1de4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1df4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d18, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d38, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d78, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d98, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1da8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1db8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dd8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1de8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1df8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d1c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d2c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d3c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d5c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d6c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d7c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1d9c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dbc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dcc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ddc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1dfc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f18, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f38, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f78, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f1c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f2c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f3c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f5c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f6c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f7c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f98, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fa0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fa8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fb8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fd8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fe0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fe8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ff0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ff8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1f9c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fa4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fb4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fbc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fcc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fd4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fdc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fe4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1fec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ff4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1ffc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2000, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2040, 0x00000011);
-	nv_mthd(dev, 0xa097, 0x2080, 0x00000020);
-	nv_mthd(dev, 0xa097, 0x20c0, 0x00000030);
-	nv_mthd(dev, 0xa097, 0x2100, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x2140, 0x00000051);
-	nv_mthd(dev, 0xa097, 0x200c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x204c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x208c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x20cc, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x210c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x214c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x2010, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2050, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2090, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x20d0, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x2110, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x2150, 0x00000004);
-	nv_mthd(dev, 0xa097, 0x0380, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0384, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0388, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x038c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x03ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0700, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0710, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0720, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0730, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0704, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0714, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0724, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0734, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0708, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0718, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0728, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0738, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2800, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2804, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2808, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x280c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2810, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2814, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2818, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x281c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2820, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2824, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2828, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x282c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2830, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2834, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2838, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x283c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2840, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2844, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2848, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x284c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2850, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2854, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2858, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x285c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2860, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2864, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2868, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x286c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2870, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2874, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2878, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x287c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2880, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2884, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2888, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x288c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2890, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2894, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2898, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x289c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28b0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28b4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28d4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x28fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2900, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2904, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2908, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x290c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2910, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2914, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2918, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x291c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2920, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2924, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2928, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x292c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2930, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2934, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2938, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x293c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2940, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2944, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2948, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x294c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2950, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2954, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2958, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x295c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2960, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2964, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2968, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x296c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2970, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2974, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2978, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x297c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2980, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2984, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2988, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x298c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2990, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2994, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2998, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x299c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29b0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29b4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29d4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x29fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0aa0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ac0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ae0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ba0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0be0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0aa4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ac4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ae4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ba4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0be4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0aa8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ac8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ae8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ba8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0be8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a2c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a6c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0aac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0acc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0aec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b2c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b6c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bcc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ab0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ad0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0af0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bf0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0a94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ab4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ad4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0af4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0b94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bb4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bd4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0bf4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ca0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ce0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cf0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c24, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c34, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c64, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c94, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ca4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cb4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cd4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ce4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cf4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c18, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c28, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c38, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c68, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c78, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c98, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ca8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cb8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cd8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ce8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0cf8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0c0c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c1c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c2c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c3c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c4c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c5c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c6c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c7c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c8c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0c9c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0cac, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0cbc, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0ccc, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0cdc, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0cec, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0cfc, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0d00, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d08, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d10, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d18, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d20, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d28, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d30, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d38, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d04, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d0c, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d14, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d1c, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d24, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d2c, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d34, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d3c, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e00, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e20, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e30, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e60, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e70, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ea0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0eb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ec0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ed0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ee0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ef0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0e04, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e14, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e24, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e34, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e44, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e54, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e64, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e74, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e84, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e94, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ea4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0eb4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ec4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ed4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ee4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ef4, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e08, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e18, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e28, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e38, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e48, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e58, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e68, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e78, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e88, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0e98, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ea8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0eb8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ec8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ed8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ee8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0ef8, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d40, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d48, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d50, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d44, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d4c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d5c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1e00, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e20, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e40, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e60, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e80, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ea0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ec0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ee0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e04, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e24, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e44, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e64, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e84, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ea4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ec4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ee4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e08, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e28, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e48, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e68, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e88, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1ea8, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1ec8, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1ee8, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e0c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e2c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e4c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e6c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e8c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1eac, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ecc, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1eec, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e10, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e30, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e50, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e70, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e90, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1eb0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ed0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ef0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e14, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e34, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e54, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e74, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e94, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1eb4, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1ed4, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1ef4, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1e18, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e38, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e58, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e78, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1e98, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1eb8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ed8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1ef8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x3400, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3404, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3408, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x340c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3410, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3414, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3418, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x341c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3420, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3424, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3428, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x342c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3430, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3434, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3438, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x343c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3440, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3444, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3448, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x344c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3450, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3454, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3458, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x345c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3460, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3464, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3468, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x346c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3470, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3474, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3478, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x347c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3480, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3484, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3488, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x348c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3490, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3494, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3498, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x349c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34b0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34b4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34d4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x34fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3500, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3504, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3508, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x350c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3510, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3514, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3518, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x351c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3520, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3524, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3528, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x352c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3530, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3534, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3538, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x353c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3540, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3544, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3548, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x354c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3550, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3554, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3558, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x355c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3560, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3564, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3568, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x356c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3570, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3574, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3578, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x357c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3580, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3584, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3588, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x358c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3590, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3594, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x3598, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x359c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35b0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35b4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35d4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x35fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x030c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1944, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1514, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d68, 0x0000ffff);
-	nv_mthd(dev, 0xa097, 0x121c, 0x0fac6881);
-	nv_mthd(dev, 0xa097, 0x0fac, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1538, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0fe0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fe4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fe8, 0x00000014);
-	nv_mthd(dev, 0xa097, 0x0fec, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x0ff0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x179c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1228, 0x00000400);
-	nv_mthd(dev, 0xa097, 0x122c, 0x00000300);
-	nv_mthd(dev, 0xa097, 0x1230, 0x00010001);
-	nv_mthd(dev, 0xa097, 0x07f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15b4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x15cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1534, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fb0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x153c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x16b4, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x0fbc, 0x0000ffff);
-	nv_mthd(dev, 0xa097, 0x0fc0, 0x0000ffff);
-	nv_mthd(dev, 0xa097, 0x0fc4, 0x0000ffff);
-	nv_mthd(dev, 0xa097, 0x0fc8, 0x0000ffff);
-	nv_mthd(dev, 0xa097, 0x0df8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0dfc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1948, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1970, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x161c, 0x000009f0);
-	nv_mthd(dev, 0xa097, 0x0dcc, 0x00000010);
-	nv_mthd(dev, 0xa097, 0x163c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1160, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1164, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1168, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x116c, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1170, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1174, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1178, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x117c, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1180, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1184, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1188, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x118c, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1190, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1194, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1198, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x119c, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11a0, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11a4, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11a8, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11ac, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11b0, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11b4, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11b8, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11bc, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11c0, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11c4, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11c8, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11cc, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11d0, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11d4, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11d8, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x11dc, 0x25e00040);
-	nv_mthd(dev, 0xa097, 0x1880, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1884, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1888, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x188c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1890, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1894, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1898, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x189c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18b0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18b4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18d0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18d4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18e0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x18fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x17c8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x17cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x17d0, 0x000000ff);
-	nv_mthd(dev, 0xa097, 0x17d4, 0xffffffff);
-	nv_mthd(dev, 0xa097, 0x17d8, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x17dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15f8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1434, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1438, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d74, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0dec, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x13a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1318, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1644, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0748, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0de8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1648, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1120, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1124, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1128, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x112c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1118, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x164c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1658, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1910, 0x00000290);
-	nv_mthd(dev, 0xa097, 0x1518, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x165c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1520, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1604, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1570, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x13b0, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x13b4, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x020c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1670, 0x30201000);
-	nv_mthd(dev, 0xa097, 0x1674, 0x70605040);
-	nv_mthd(dev, 0xa097, 0x1678, 0xb8a89888);
-	nv_mthd(dev, 0xa097, 0x167c, 0xf8e8d8c8);
-	nv_mthd(dev, 0xa097, 0x166c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1680, 0x00ffff00);
-	nv_mthd(dev, 0xa097, 0x12d0, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x12d4, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1684, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1688, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0dac, 0x00001b02);
-	nv_mthd(dev, 0xa097, 0x0db0, 0x00001b02);
-	nv_mthd(dev, 0xa097, 0x0db4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x168c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x15bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x156c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x187c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1110, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0dc0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0dc4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0dc8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1234, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1690, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12ac, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0790, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0794, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0798, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x079c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07a0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x077c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1000, 0x00000010);
-	nv_mthd(dev, 0xa097, 0x10fc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1290, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0218, 0x00000010);
-	nv_mthd(dev, 0xa097, 0x12d8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12dc, 0x00000010);
-	nv_mthd(dev, 0xa097, 0x0d94, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x155c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1560, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1564, 0x00000fff);
-	nv_mthd(dev, 0xa097, 0x1574, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1578, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x157c, 0x000fffff);
-	nv_mthd(dev, 0xa097, 0x1354, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1610, 0x00000012);
-	nv_mthd(dev, 0xa097, 0x1608, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x160c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x260c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x162c, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x0210, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0320, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0324, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0328, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x032c, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0330, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0334, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0338, 0x3f800000);
-	nv_mthd(dev, 0xa097, 0x0750, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0760, 0x39291909);
-	nv_mthd(dev, 0xa097, 0x0764, 0x79695949);
-	nv_mthd(dev, 0xa097, 0x0768, 0xb9a99989);
-	nv_mthd(dev, 0xa097, 0x076c, 0xf9e9d9c9);
-	nv_mthd(dev, 0xa097, 0x0770, 0x30201000);
-	nv_mthd(dev, 0xa097, 0x0774, 0x70605040);
-	nv_mthd(dev, 0xa097, 0x0778, 0x00009080);
-	nv_mthd(dev, 0xa097, 0x0780, 0x39291909);
-	nv_mthd(dev, 0xa097, 0x0784, 0x79695949);
-	nv_mthd(dev, 0xa097, 0x0788, 0xb9a99989);
-	nv_mthd(dev, 0xa097, 0x078c, 0xf9e9d9c9);
-	nv_mthd(dev, 0xa097, 0x07d0, 0x30201000);
-	nv_mthd(dev, 0xa097, 0x07d4, 0x70605040);
-	nv_mthd(dev, 0xa097, 0x07d8, 0x00009080);
-	nv_mthd(dev, 0xa097, 0x037c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0740, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0744, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x2600, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1918, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x191c, 0x00000900);
-	nv_mthd(dev, 0xa097, 0x1920, 0x00000405);
-	nv_mthd(dev, 0xa097, 0x1308, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1924, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x13ac, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x192c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x193c, 0x00002c1c);
-	nv_mthd(dev, 0xa097, 0x0d7c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x02c0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1510, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1940, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ff4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0ff8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x194c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1950, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1968, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1590, 0x0000003f);
-	nv_mthd(dev, 0xa097, 0x07e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07f0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07f4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x196c, 0x00000011);
-	nv_mthd(dev, 0xa097, 0x02e4, 0x0000b001);
-	nv_mthd(dev, 0xa097, 0x036c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0370, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x197c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fcc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fd0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x02d8, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x1980, 0x00000080);
-	nv_mthd(dev, 0xa097, 0x1504, 0x00000080);
-	nv_mthd(dev, 0xa097, 0x1984, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0300, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x13a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12ec, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1310, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1314, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1380, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1384, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1388, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x138c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1390, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1394, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x139c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1398, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1594, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1598, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x159c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x15a0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x15a4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0f54, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f58, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f5c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x19bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f9c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0fa0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12cc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x12e8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x130c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1360, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1364, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1368, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x136c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1370, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1374, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1378, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x137c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x133c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1340, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1344, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1348, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x134c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1350, 0x00000002);
-	nv_mthd(dev, 0xa097, 0x1358, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x12e4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x131c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1320, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1324, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1328, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x19c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1140, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x19c4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x19c8, 0x00001500);
-	nv_mthd(dev, 0xa097, 0x135c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x19e0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19e4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19e8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19ec, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19f0, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19f4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19f8, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19fc, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x19cc, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x15b8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a00, 0x00001111);
-	nv_mthd(dev, 0xa097, 0x1a04, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a08, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a0c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a10, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a14, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a18, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1a1c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d6c, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x0d70, 0xffff0000);
-	nv_mthd(dev, 0xa097, 0x10f8, 0x00001010);
-	nv_mthd(dev, 0xa097, 0x0d80, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d84, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d88, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d8c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0d90, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0da0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07a4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x07a8, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1508, 0x80000000);
-	nv_mthd(dev, 0xa097, 0x150c, 0x40000000);
-	nv_mthd(dev, 0xa097, 0x1668, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0318, 0x00000008);
-	nv_mthd(dev, 0xa097, 0x031c, 0x00000008);
-	nv_mthd(dev, 0xa097, 0x0d9c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x0374, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0378, 0x00000020);
-	nv_mthd(dev, 0xa097, 0x07dc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x074c, 0x00000055);
-	nv_mthd(dev, 0xa097, 0x1420, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x17bc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x17c0, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x17c4, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1008, 0x00000008);
-	nv_mthd(dev, 0xa097, 0x100c, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x1010, 0x0000012c);
-	nv_mthd(dev, 0xa097, 0x0d60, 0x00000040);
-	nv_mthd(dev, 0xa097, 0x075c, 0x00000003);
-	nv_mthd(dev, 0xa097, 0x1018, 0x00000020);
-	nv_mthd(dev, 0xa097, 0x101c, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1020, 0x00000020);
-	nv_mthd(dev, 0xa097, 0x1024, 0x00000001);
-	nv_mthd(dev, 0xa097, 0x1444, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x1448, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x144c, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0360, 0x20164010);
-	nv_mthd(dev, 0xa097, 0x0364, 0x00000020);
-	nv_mthd(dev, 0xa097, 0x0368, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0de4, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0204, 0x00000006);
-	nv_mthd(dev, 0xa097, 0x0208, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x02cc, 0x003fffff);
-	nv_mthd(dev, 0xa097, 0x02d0, 0x003fffff);
-	nv_mthd(dev, 0xa097, 0x1220, 0x00000005);
-	nv_mthd(dev, 0xa097, 0x0fdc, 0x00000000);
-	nv_mthd(dev, 0xa097, 0x0f98, 0x00400008);
-	nv_mthd(dev, 0xa097, 0x1284, 0x08000080);
-	nv_mthd(dev, 0xa097, 0x1450, 0x00400008);
-	nv_mthd(dev, 0xa097, 0x1454, 0x08000080);
-	nv_mthd(dev, 0xa097, 0x0214, 0x00000000);
-}
-
-static void
-nve0_grctx_generate_902d(struct drm_device *dev)
-{
-	nv_mthd(dev, 0x902d, 0x0200, 0x000000cf);
-	nv_mthd(dev, 0x902d, 0x0204, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0208, 0x00000020);
-	nv_mthd(dev, 0x902d, 0x020c, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0210, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0214, 0x00000080);
-	nv_mthd(dev, 0x902d, 0x0218, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x021c, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x0220, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0224, 0x00000000);
-	nv_mthd(dev, 0x902d, 0x0230, 0x000000cf);
-	nv_mthd(dev, 0x902d, 0x0234, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0238, 0x00000020);
-	nv_mthd(dev, 0x902d, 0x023c, 0x00000001);
-	nv_mthd(dev, 0x902d, 0x0244, 0x00000080);
-	nv_mthd(dev, 0x902d, 0x0248, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x024c, 0x00000100);
-	nv_mthd(dev, 0x902d, 0x3410, 0x00000000);
-}
-
-static void
-nve0_graph_generate_unk40xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404010, 0x0);
-	nv_wr32(dev, 0x404014, 0x0);
-	nv_wr32(dev, 0x404018, 0x0);
-	nv_wr32(dev, 0x40401c, 0x0);
-	nv_wr32(dev, 0x404020, 0x0);
-	nv_wr32(dev, 0x404024, 0xe000);
-	nv_wr32(dev, 0x404028, 0x0);
-	nv_wr32(dev, 0x4040a8, 0x0);
-	nv_wr32(dev, 0x4040ac, 0x0);
-	nv_wr32(dev, 0x4040b0, 0x0);
-	nv_wr32(dev, 0x4040b4, 0x0);
-	nv_wr32(dev, 0x4040b8, 0x0);
-	nv_wr32(dev, 0x4040bc, 0x0);
-	nv_wr32(dev, 0x4040c0, 0x0);
-	nv_wr32(dev, 0x4040c4, 0x0);
-	nv_wr32(dev, 0x4040c8, 0xf800008f);
-	nv_wr32(dev, 0x4040d0, 0x0);
-	nv_wr32(dev, 0x4040d4, 0x0);
-	nv_wr32(dev, 0x4040d8, 0x0);
-	nv_wr32(dev, 0x4040dc, 0x0);
-	nv_wr32(dev, 0x4040e0, 0x0);
-	nv_wr32(dev, 0x4040e4, 0x0);
-	nv_wr32(dev, 0x4040e8, 0x1000);
-	nv_wr32(dev, 0x4040f8, 0x0);
-	nv_wr32(dev, 0x404130, 0x0);
-	nv_wr32(dev, 0x404134, 0x0);
-	nv_wr32(dev, 0x404138, 0x20000040);
-	nv_wr32(dev, 0x404150, 0x2e);
-	nv_wr32(dev, 0x404154, 0x400);
-	nv_wr32(dev, 0x404158, 0x200);
-	nv_wr32(dev, 0x404164, 0x55);
-	nv_wr32(dev, 0x4041a0, 0x0);
-	nv_wr32(dev, 0x4041a4, 0x0);
-	nv_wr32(dev, 0x4041a8, 0x0);
-	nv_wr32(dev, 0x4041ac, 0x0);
-	nv_wr32(dev, 0x404200, 0x0);
-	nv_wr32(dev, 0x404204, 0x0);
-	nv_wr32(dev, 0x404208, 0x0);
-	nv_wr32(dev, 0x40420c, 0x0);
-}
-
-static void
-nve0_graph_generate_unk44xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404404, 0x0);
-	nv_wr32(dev, 0x404408, 0x0);
-	nv_wr32(dev, 0x40440c, 0x0);
-	nv_wr32(dev, 0x404410, 0x0);
-	nv_wr32(dev, 0x404414, 0x0);
-	nv_wr32(dev, 0x404418, 0x0);
-	nv_wr32(dev, 0x40441c, 0x0);
-	nv_wr32(dev, 0x404420, 0x0);
-	nv_wr32(dev, 0x404424, 0x0);
-	nv_wr32(dev, 0x404428, 0x0);
-	nv_wr32(dev, 0x40442c, 0x0);
-	nv_wr32(dev, 0x404430, 0x0);
-	nv_wr32(dev, 0x404434, 0x0);
-	nv_wr32(dev, 0x404438, 0x0);
-	nv_wr32(dev, 0x404460, 0x0);
-	nv_wr32(dev, 0x404464, 0x0);
-	nv_wr32(dev, 0x404468, 0xffffff);
-	nv_wr32(dev, 0x40446c, 0x0);
-	nv_wr32(dev, 0x404480, 0x1);
-	nv_wr32(dev, 0x404498, 0x1);
-}
-
-static void
-nve0_graph_generate_unk46xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404604, 0x14);
-	nv_wr32(dev, 0x404608, 0x0);
-	nv_wr32(dev, 0x40460c, 0x3fff);
-	nv_wr32(dev, 0x404610, 0x100);
-	nv_wr32(dev, 0x404618, 0x0);
-	nv_wr32(dev, 0x40461c, 0x0);
-	nv_wr32(dev, 0x404620, 0x0);
-	nv_wr32(dev, 0x404624, 0x0);
-	nv_wr32(dev, 0x40462c, 0x0);
-	nv_wr32(dev, 0x404630, 0x0);
-	nv_wr32(dev, 0x404640, 0x0);
-	nv_wr32(dev, 0x404654, 0x0);
-	nv_wr32(dev, 0x404660, 0x0);
-	nv_wr32(dev, 0x404678, 0x0);
-	nv_wr32(dev, 0x40467c, 0x2);
-	nv_wr32(dev, 0x404680, 0x0);
-	nv_wr32(dev, 0x404684, 0x0);
-	nv_wr32(dev, 0x404688, 0x0);
-	nv_wr32(dev, 0x40468c, 0x0);
-	nv_wr32(dev, 0x404690, 0x0);
-	nv_wr32(dev, 0x404694, 0x0);
-	nv_wr32(dev, 0x404698, 0x0);
-	nv_wr32(dev, 0x40469c, 0x0);
-	nv_wr32(dev, 0x4046a0, 0x7f0080);
-	nv_wr32(dev, 0x4046a4, 0x0);
-	nv_wr32(dev, 0x4046a8, 0x0);
-	nv_wr32(dev, 0x4046ac, 0x0);
-	nv_wr32(dev, 0x4046b0, 0x0);
-	nv_wr32(dev, 0x4046b4, 0x0);
-	nv_wr32(dev, 0x4046b8, 0x0);
-	nv_wr32(dev, 0x4046bc, 0x0);
-	nv_wr32(dev, 0x4046c0, 0x0);
-	nv_wr32(dev, 0x4046c8, 0x0);
-	nv_wr32(dev, 0x4046cc, 0x0);
-	nv_wr32(dev, 0x4046d0, 0x0);
-}
-
-static void
-nve0_graph_generate_unk47xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x404700, 0x0);
-	nv_wr32(dev, 0x404704, 0x0);
-	nv_wr32(dev, 0x404708, 0x0);
-	nv_wr32(dev, 0x404718, 0x0);
-	nv_wr32(dev, 0x40471c, 0x0);
-	nv_wr32(dev, 0x404720, 0x0);
-	nv_wr32(dev, 0x404724, 0x0);
-	nv_wr32(dev, 0x404728, 0x0);
-	nv_wr32(dev, 0x40472c, 0x0);
-	nv_wr32(dev, 0x404730, 0x0);
-	nv_wr32(dev, 0x404734, 0x100);
-	nv_wr32(dev, 0x404738, 0x0);
-	nv_wr32(dev, 0x40473c, 0x0);
-	nv_wr32(dev, 0x404744, 0x0);
-	nv_wr32(dev, 0x404748, 0x0);
-	nv_wr32(dev, 0x404754, 0x0);
-}
-
-static void
-nve0_graph_generate_unk58xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x405800, 0xf8000bf);
-	nv_wr32(dev, 0x405830, 0x2180648);
-	nv_wr32(dev, 0x405834, 0x8000000);
-	nv_wr32(dev, 0x405838, 0x0);
-	nv_wr32(dev, 0x405854, 0x0);
-	nv_wr32(dev, 0x405870, 0x1);
-	nv_wr32(dev, 0x405874, 0x1);
-	nv_wr32(dev, 0x405878, 0x1);
-	nv_wr32(dev, 0x40587c, 0x1);
-	nv_wr32(dev, 0x405a00, 0x0);
-	nv_wr32(dev, 0x405a04, 0x0);
-	nv_wr32(dev, 0x405a18, 0x0);
-	nv_wr32(dev, 0x405b00, 0x0);
-	nv_wr32(dev, 0x405b10, 0x1000);
-}
-
-static void
-nve0_graph_generate_unk60xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x406020, 0x4103c1);
-	nv_wr32(dev, 0x406028, 0x1);
-	nv_wr32(dev, 0x40602c, 0x1);
-	nv_wr32(dev, 0x406030, 0x1);
-	nv_wr32(dev, 0x406034, 0x1);
-}
-
-static void
-nve0_graph_generate_unk64xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x4064a8, 0x0);
-	nv_wr32(dev, 0x4064ac, 0x3fff);
-	nv_wr32(dev, 0x4064b4, 0x0);
-	nv_wr32(dev, 0x4064b8, 0x0);
-	nv_wr32(dev, 0x4064c0, 0x801a00f0);
-	nv_wr32(dev, 0x4064c4, 0x192ffff);
-	nv_wr32(dev, 0x4064c8, 0x1800600);
-	nv_wr32(dev, 0x4064cc, 0x0);
-	nv_wr32(dev, 0x4064d0, 0x0);
-	nv_wr32(dev, 0x4064d4, 0x0);
-	nv_wr32(dev, 0x4064d8, 0x0);
-	nv_wr32(dev, 0x4064dc, 0x0);
-	nv_wr32(dev, 0x4064e0, 0x0);
-	nv_wr32(dev, 0x4064e4, 0x0);
-	nv_wr32(dev, 0x4064e8, 0x0);
-	nv_wr32(dev, 0x4064ec, 0x0);
-	nv_wr32(dev, 0x4064fc, 0x22a);
-}
-
-static void
-nve0_graph_generate_unk70xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x407040, 0x0);
-}
-
-static void
-nve0_graph_generate_unk78xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x407804, 0x23);
-	nv_wr32(dev, 0x40780c, 0xa418820);
-	nv_wr32(dev, 0x407810, 0x62080e6);
-	nv_wr32(dev, 0x407814, 0x20398a4);
-	nv_wr32(dev, 0x407818, 0xe629062);
-	nv_wr32(dev, 0x40781c, 0xa418820);
-	nv_wr32(dev, 0x407820, 0xe6);
-	nv_wr32(dev, 0x4078bc, 0x103);
-}
-
-static void
-nve0_graph_generate_unk80xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x408000, 0x0);
-	nv_wr32(dev, 0x408004, 0x0);
-	nv_wr32(dev, 0x408008, 0x30);
-	nv_wr32(dev, 0x40800c, 0x0);
-	nv_wr32(dev, 0x408010, 0x0);
-	nv_wr32(dev, 0x408014, 0x69);
-	nv_wr32(dev, 0x408018, 0xe100e100);
-	nv_wr32(dev, 0x408064, 0x0);
-}
-
-static void
-nve0_graph_generate_unk88xx(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x408800, 0x2802a3c);
-	nv_wr32(dev, 0x408804, 0x40);
-	nv_wr32(dev, 0x408808, 0x1043e005);
-	nv_wr32(dev, 0x408840, 0xb);
-	nv_wr32(dev, 0x408900, 0x3080b801);
-	nv_wr32(dev, 0x408904, 0x62000001);
-	nv_wr32(dev, 0x408908, 0xc8102f);
-	nv_wr32(dev, 0x408980, 0x11d);
-}
-
-static void
-nve0_graph_generate_gpc(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x418380, 0x16);
-	nv_wr32(dev, 0x418400, 0x38004e00);
-	nv_wr32(dev, 0x418404, 0x71e0ffff);
-	nv_wr32(dev, 0x41840c, 0x1008);
-	nv_wr32(dev, 0x418410, 0xfff0fff);
-	nv_wr32(dev, 0x418414, 0x2200fff);
-	nv_wr32(dev, 0x418450, 0x0);
-	nv_wr32(dev, 0x418454, 0x0);
-	nv_wr32(dev, 0x418458, 0x0);
-	nv_wr32(dev, 0x41845c, 0x0);
-	nv_wr32(dev, 0x418460, 0x0);
-	nv_wr32(dev, 0x418464, 0x0);
-	nv_wr32(dev, 0x418468, 0x1);
-	nv_wr32(dev, 0x41846c, 0x0);
-	nv_wr32(dev, 0x418470, 0x0);
-	nv_wr32(dev, 0x418600, 0x1f);
-	nv_wr32(dev, 0x418684, 0xf);
-	nv_wr32(dev, 0x418700, 0x2);
-	nv_wr32(dev, 0x418704, 0x80);
-	nv_wr32(dev, 0x418708, 0x0);
-	nv_wr32(dev, 0x41870c, 0x0);
-	nv_wr32(dev, 0x418710, 0x0);
-	nv_wr32(dev, 0x418800, 0x7006860a);
-	nv_wr32(dev, 0x418808, 0x0);
-	nv_wr32(dev, 0x41880c, 0x0);
-	nv_wr32(dev, 0x418810, 0x0);
-	nv_wr32(dev, 0x418828, 0x44);
-	nv_wr32(dev, 0x418830, 0x10000001);
-	nv_wr32(dev, 0x4188d8, 0x8);
-	nv_wr32(dev, 0x4188e0, 0x1000000);
-	nv_wr32(dev, 0x4188e8, 0x0);
-	nv_wr32(dev, 0x4188ec, 0x0);
-	nv_wr32(dev, 0x4188f0, 0x0);
-	nv_wr32(dev, 0x4188f4, 0x0);
-	nv_wr32(dev, 0x4188f8, 0x0);
-	nv_wr32(dev, 0x4188fc, 0x20100018);
-	nv_wr32(dev, 0x41891c, 0xff00ff);
-	nv_wr32(dev, 0x418924, 0x0);
-	nv_wr32(dev, 0x418928, 0xffff00);
-	nv_wr32(dev, 0x41892c, 0xff00);
-	nv_wr32(dev, 0x418a00, 0x0);
-	nv_wr32(dev, 0x418a04, 0x0);
-	nv_wr32(dev, 0x418a08, 0x0);
-	nv_wr32(dev, 0x418a0c, 0x10000);
-	nv_wr32(dev, 0x418a10, 0x0);
-	nv_wr32(dev, 0x418a14, 0x0);
-	nv_wr32(dev, 0x418a18, 0x0);
-	nv_wr32(dev, 0x418a20, 0x0);
-	nv_wr32(dev, 0x418a24, 0x0);
-	nv_wr32(dev, 0x418a28, 0x0);
-	nv_wr32(dev, 0x418a2c, 0x10000);
-	nv_wr32(dev, 0x418a30, 0x0);
-	nv_wr32(dev, 0x418a34, 0x0);
-	nv_wr32(dev, 0x418a38, 0x0);
-	nv_wr32(dev, 0x418a40, 0x0);
-	nv_wr32(dev, 0x418a44, 0x0);
-	nv_wr32(dev, 0x418a48, 0x0);
-	nv_wr32(dev, 0x418a4c, 0x10000);
-	nv_wr32(dev, 0x418a50, 0x0);
-	nv_wr32(dev, 0x418a54, 0x0);
-	nv_wr32(dev, 0x418a58, 0x0);
-	nv_wr32(dev, 0x418a60, 0x0);
-	nv_wr32(dev, 0x418a64, 0x0);
-	nv_wr32(dev, 0x418a68, 0x0);
-	nv_wr32(dev, 0x418a6c, 0x10000);
-	nv_wr32(dev, 0x418a70, 0x0);
-	nv_wr32(dev, 0x418a74, 0x0);
-	nv_wr32(dev, 0x418a78, 0x0);
-	nv_wr32(dev, 0x418a80, 0x0);
-	nv_wr32(dev, 0x418a84, 0x0);
-	nv_wr32(dev, 0x418a88, 0x0);
-	nv_wr32(dev, 0x418a8c, 0x10000);
-	nv_wr32(dev, 0x418a90, 0x0);
-	nv_wr32(dev, 0x418a94, 0x0);
-	nv_wr32(dev, 0x418a98, 0x0);
-	nv_wr32(dev, 0x418aa0, 0x0);
-	nv_wr32(dev, 0x418aa4, 0x0);
-	nv_wr32(dev, 0x418aa8, 0x0);
-	nv_wr32(dev, 0x418aac, 0x10000);
-	nv_wr32(dev, 0x418ab0, 0x0);
-	nv_wr32(dev, 0x418ab4, 0x0);
-	nv_wr32(dev, 0x418ab8, 0x0);
-	nv_wr32(dev, 0x418ac0, 0x0);
-	nv_wr32(dev, 0x418ac4, 0x0);
-	nv_wr32(dev, 0x418ac8, 0x0);
-	nv_wr32(dev, 0x418acc, 0x10000);
-	nv_wr32(dev, 0x418ad0, 0x0);
-	nv_wr32(dev, 0x418ad4, 0x0);
-	nv_wr32(dev, 0x418ad8, 0x0);
-	nv_wr32(dev, 0x418ae0, 0x0);
-	nv_wr32(dev, 0x418ae4, 0x0);
-	nv_wr32(dev, 0x418ae8, 0x0);
-	nv_wr32(dev, 0x418aec, 0x10000);
-	nv_wr32(dev, 0x418af0, 0x0);
-	nv_wr32(dev, 0x418af4, 0x0);
-	nv_wr32(dev, 0x418af8, 0x0);
-	nv_wr32(dev, 0x418b00, 0x6);
-	nv_wr32(dev, 0x418b08, 0xa418820);
-	nv_wr32(dev, 0x418b0c, 0x62080e6);
-	nv_wr32(dev, 0x418b10, 0x20398a4);
-	nv_wr32(dev, 0x418b14, 0xe629062);
-	nv_wr32(dev, 0x418b18, 0xa418820);
-	nv_wr32(dev, 0x418b1c, 0xe6);
-	nv_wr32(dev, 0x418bb8, 0x103);
-	nv_wr32(dev, 0x418c08, 0x1);
-	nv_wr32(dev, 0x418c10, 0x0);
-	nv_wr32(dev, 0x418c14, 0x0);
-	nv_wr32(dev, 0x418c18, 0x0);
-	nv_wr32(dev, 0x418c1c, 0x0);
-	nv_wr32(dev, 0x418c20, 0x0);
-	nv_wr32(dev, 0x418c24, 0x0);
-	nv_wr32(dev, 0x418c28, 0x0);
-	nv_wr32(dev, 0x418c2c, 0x0);
-	nv_wr32(dev, 0x418c40, 0xffffffff);
-	nv_wr32(dev, 0x418c6c, 0x1);
-	nv_wr32(dev, 0x418c80, 0x20200004);
-	nv_wr32(dev, 0x418c8c, 0x1);
-	nv_wr32(dev, 0x419000, 0x780);
-	nv_wr32(dev, 0x419004, 0x0);
-	nv_wr32(dev, 0x419008, 0x0);
-	nv_wr32(dev, 0x419014, 0x4);
-}
-
-static void
-nve0_graph_generate_tpc(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x419848, 0x0);
-	nv_wr32(dev, 0x419864, 0x129);
-	nv_wr32(dev, 0x419888, 0x0);
-	nv_wr32(dev, 0x419a00, 0xf0);
-	nv_wr32(dev, 0x419a04, 0x1);
-	nv_wr32(dev, 0x419a08, 0x21);
-	nv_wr32(dev, 0x419a0c, 0x20000);
-	nv_wr32(dev, 0x419a10, 0x0);
-	nv_wr32(dev, 0x419a14, 0x200);
-	nv_wr32(dev, 0x419a1c, 0xc000);
-	nv_wr32(dev, 0x419a20, 0x800);
-	nv_wr32(dev, 0x419a30, 0x1);
-	nv_wr32(dev, 0x419ac4, 0x37f440);
-	nv_wr32(dev, 0x419c00, 0xa);
-	nv_wr32(dev, 0x419c04, 0x80000006);
-	nv_wr32(dev, 0x419c08, 0x2);
-	nv_wr32(dev, 0x419c20, 0x0);
-	nv_wr32(dev, 0x419c24, 0x84210);
-	nv_wr32(dev, 0x419c28, 0x3efbefbe);
-	nv_wr32(dev, 0x419ce8, 0x0);
-	nv_wr32(dev, 0x419cf4, 0x3203);
-	nv_wr32(dev, 0x419e04, 0x0);
-	nv_wr32(dev, 0x419e08, 0x0);
-	nv_wr32(dev, 0x419e0c, 0x0);
-	nv_wr32(dev, 0x419e10, 0x402);
-	nv_wr32(dev, 0x419e44, 0x13eff2);
-	nv_wr32(dev, 0x419e48, 0x0);
-	nv_wr32(dev, 0x419e4c, 0x7f);
-	nv_wr32(dev, 0x419e50, 0x0);
-	nv_wr32(dev, 0x419e54, 0x0);
-	nv_wr32(dev, 0x419e58, 0x0);
-	nv_wr32(dev, 0x419e5c, 0x0);
-	nv_wr32(dev, 0x419e60, 0x0);
-	nv_wr32(dev, 0x419e64, 0x0);
-	nv_wr32(dev, 0x419e68, 0x0);
-	nv_wr32(dev, 0x419e6c, 0x0);
-	nv_wr32(dev, 0x419e70, 0x0);
-	nv_wr32(dev, 0x419e74, 0x0);
-	nv_wr32(dev, 0x419e78, 0x0);
-	nv_wr32(dev, 0x419e7c, 0x0);
-	nv_wr32(dev, 0x419e80, 0x0);
-	nv_wr32(dev, 0x419e84, 0x0);
-	nv_wr32(dev, 0x419e88, 0x0);
-	nv_wr32(dev, 0x419e8c, 0x0);
-	nv_wr32(dev, 0x419e90, 0x0);
-	nv_wr32(dev, 0x419e94, 0x0);
-	nv_wr32(dev, 0x419e98, 0x0);
-	nv_wr32(dev, 0x419eac, 0x1fcf);
-	nv_wr32(dev, 0x419eb0, 0xd3f);
-	nv_wr32(dev, 0x419ec8, 0x1304f);
-	nv_wr32(dev, 0x419f30, 0x0);
-	nv_wr32(dev, 0x419f34, 0x0);
-	nv_wr32(dev, 0x419f38, 0x0);
-	nv_wr32(dev, 0x419f3c, 0x0);
-	nv_wr32(dev, 0x419f40, 0x0);
-	nv_wr32(dev, 0x419f44, 0x0);
-	nv_wr32(dev, 0x419f48, 0x0);
-	nv_wr32(dev, 0x419f4c, 0x0);
-	nv_wr32(dev, 0x419f58, 0x0);
-	nv_wr32(dev, 0x419f78, 0xb);
-}
-
-static void
-nve0_graph_generate_tpcunk(struct drm_device *dev)
-{
-	nv_wr32(dev, 0x41be24, 0x6);
-	nv_wr32(dev, 0x41bec0, 0x12180000);
-	nv_wr32(dev, 0x41bec4, 0x37f7f);
-	nv_wr32(dev, 0x41bee4, 0x6480430);
-	nv_wr32(dev, 0x41bf00, 0xa418820);
-	nv_wr32(dev, 0x41bf04, 0x62080e6);
-	nv_wr32(dev, 0x41bf08, 0x20398a4);
-	nv_wr32(dev, 0x41bf0c, 0xe629062);
-	nv_wr32(dev, 0x41bf10, 0xa418820);
-	nv_wr32(dev, 0x41bf14, 0xe6);
-	nv_wr32(dev, 0x41bfd0, 0x900103);
-	nv_wr32(dev, 0x41bfe0, 0x400001);
-	nv_wr32(dev, 0x41bfe4, 0x0);
-}
-
-int
-nve0_grctx_generate(struct nouveau_channel *chan)
-{
-	struct nve0_graph_priv *priv = nv_engine(chan->dev, NVOBJ_ENGINE_GR);
-	struct nve0_graph_chan *grch = chan->engctx[NVOBJ_ENGINE_GR];
-	struct drm_device *dev = chan->dev;
-	u32 data[6] = {}, data2[2] = {}, tmp;
-	u32 tpc_set = 0, tpc_mask = 0;
-	u8 tpcnr[GPC_MAX], a, b;
-	u8 shift, ntpcv;
-	int i, gpc, tpc, id;
-
-	nv_mask(dev, 0x000260, 0x00000001, 0x00000000);
-	nv_wr32(dev, 0x400204, 0x00000000);
-	nv_wr32(dev, 0x400208, 0x00000000);
-
-	nve0_graph_generate_unk40xx(dev);
-	nve0_graph_generate_unk44xx(dev);
-	nve0_graph_generate_unk46xx(dev);
-	nve0_graph_generate_unk47xx(dev);
-	nve0_graph_generate_unk58xx(dev);
-	nve0_graph_generate_unk60xx(dev);
-	nve0_graph_generate_unk64xx(dev);
-	nve0_graph_generate_unk70xx(dev);
-	nve0_graph_generate_unk78xx(dev);
-	nve0_graph_generate_unk80xx(dev);
-	nve0_graph_generate_unk88xx(dev);
-	nve0_graph_generate_gpc(dev);
-	nve0_graph_generate_tpc(dev);
-	nve0_graph_generate_tpcunk(dev);
-
-	nv_wr32(dev, 0x404154, 0x0);
-
-	for (i = 0; i < grch->mmio_nr * 8; i += 8) {
-		u32 reg = nv_ro32(grch->mmio, i + 0);
-		u32 val = nv_ro32(grch->mmio, i + 4);
-		nv_wr32(dev, reg, val);
-	}
-
-	nv_wr32(dev, 0x418c6c, 0x1);
-	nv_wr32(dev, 0x41980c, 0x10);
-	nv_wr32(dev, 0x41be08, 0x4);
-	nv_wr32(dev, 0x4064c0, 0x801a00f0);
-	nv_wr32(dev, 0x405800, 0xf8000bf);
-	nv_wr32(dev, 0x419c00, 0xa);
-
-	for (tpc = 0, id = 0; tpc < 4; tpc++) {
-		for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
-			if (tpc < priv->tpc_nr[gpc]) {
-				nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x0698), id);
-				nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x04e8), id);
-				nv_wr32(dev, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
-				nv_wr32(dev, TPC_UNIT(gpc, tpc, 0x0088), id++);
-			}
-
-			nv_wr32(dev, GPC_UNIT(gpc, 0x0c08), priv->tpc_nr[gpc]);
-			nv_wr32(dev, GPC_UNIT(gpc, 0x0c8c), priv->tpc_nr[gpc]);
-		}
-	}
-
-	tmp = 0;
-	for (i = 0; i < priv->gpc_nr; i++)
-		tmp |= priv->tpc_nr[i] << (i * 4);
-	nv_wr32(dev, 0x406028, tmp);
-	nv_wr32(dev, 0x405870, tmp);
-
-	nv_wr32(dev, 0x40602c, 0x0);
-	nv_wr32(dev, 0x405874, 0x0);
-	nv_wr32(dev, 0x406030, 0x0);
-	nv_wr32(dev, 0x405878, 0x0);
-	nv_wr32(dev, 0x406034, 0x0);
-	nv_wr32(dev, 0x40587c, 0x0);
-
-	/* calculate first set of magics */
-	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
-
-	gpc = -1;
-	for (tpc = 0; tpc < priv->tpc_total; tpc++) {
-		do {
-			gpc = (gpc + 1) % priv->gpc_nr;
-		} while (!tpcnr[gpc]);
-		tpcnr[gpc]--;
-
-		data[tpc / 6] |= gpc << ((tpc % 6) * 5);
-	}
-
-	for (; tpc < 32; tpc++)
-		data[tpc / 6] |= 7 << ((tpc % 6) * 5);
-
-	/* and the second... */
-	shift = 0;
-	ntpcv = priv->tpc_total;
-	while (!(ntpcv & (1 << 4))) {
-		ntpcv <<= 1;
-		shift++;
-	}
-
-	data2[0]  = ntpcv << 16;
-	data2[0] |= shift << 21;
-	data2[0] |= (((1 << (0 + 5)) % ntpcv) << 24);
-	data2[0] |= priv->tpc_total << 8;
-	data2[0] |= priv->magic_not_rop_nr;
-	for (i = 1; i < 7; i++)
-		data2[1] |= ((1 << (i + 5)) % ntpcv) << ((i - 1) * 5);
-
-	/* and write it all the various parts of PGRAPH */
-	nv_wr32(dev, 0x418bb8, (priv->tpc_total << 8) | priv->magic_not_rop_nr);
-	for (i = 0; i < 6; i++)
-		nv_wr32(dev, 0x418b08 + (i * 4), data[i]);
-
-	nv_wr32(dev, 0x41bfd0, data2[0]);
-	nv_wr32(dev, 0x41bfe4, data2[1]);
-	for (i = 0; i < 6; i++)
-		nv_wr32(dev, 0x41bf00 + (i * 4), data[i]);
-
-	nv_wr32(dev, 0x4078bc, (priv->tpc_total << 8) | priv->magic_not_rop_nr);
-	for (i = 0; i < 6; i++)
-		nv_wr32(dev, 0x40780c + (i * 4), data[i]);
-
-
-	memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
-	for (gpc = 0; gpc < priv->gpc_nr; gpc++)
-		tpc_mask |= ((1 << priv->tpc_nr[gpc]) - 1) << (gpc * 8);
-
-	for (i = 0, gpc = -1, b = -1; i < 32; i++) {
-		a = (i * (priv->tpc_total - 1)) / 32;
-		if (a != b) {
-			b = a;
-			do {
-				gpc = (gpc + 1) % priv->gpc_nr;
-			} while (!tpcnr[gpc]);
-			tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
-
-			tpc_set |= 1 << ((gpc * 8) + tpc);
-		}
-
-		nv_wr32(dev, 0x406800 + (i * 0x20), tpc_set);
-		nv_wr32(dev, 0x406c00 + (i * 0x20), tpc_set ^ tpc_mask);
-	}
-
-	for (i = 0; i < 8; i++)
-		nv_wr32(dev, 0x4064d0 + (i * 0x04), 0x00000000);
-
-	nv_wr32(dev, 0x405b00, 0x201);
-	nv_wr32(dev, 0x408850, 0x2);
-	nv_wr32(dev, 0x408958, 0x2);
-	nv_wr32(dev, 0x419f78, 0xa);
-
-	nve0_grctx_generate_icmd(dev);
-	nve0_grctx_generate_a097(dev);
-	nve0_grctx_generate_902d(dev);
-
-	nv_mask(dev, 0x000260, 0x00000001, 0x00000001);
-	nv_wr32(dev, 0x418800, 0x7026860a); //XXX
-	nv_wr32(dev, 0x41be10, 0x00bb8bc7); //XXX
-	return 0;
-}
diff --git a/drivers/gpu/drm/r128/r128_cce.c b/drivers/gpu/drm/r128/r128_cce.c
index bcac90b543a..d4660cf942a 100644
--- a/drivers/gpu/drm/r128/r128_cce.c
+++ b/drivers/gpu/drm/r128/r128_cce.c
@@ -34,9 +34,8 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "r128_drm.h"
+#include <drm/drmP.h>
+#include <drm/r128_drm.h>
 #include "r128_drv.h"
 
 #define R128_FIFO_DEBUG		0
diff --git a/drivers/gpu/drm/r128/r128_drv.c b/drivers/gpu/drm/r128/r128_drv.c
index 2666a5308ab..472c38fe123 100644
--- a/drivers/gpu/drm/r128/r128_drv.c
+++ b/drivers/gpu/drm/r128/r128_drv.c
@@ -31,12 +31,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "r128_drm.h"
+#include <drm/drmP.h>
+#include <drm/r128_drm.h>
 #include "r128_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static struct pci_device_id pciidlist[] = {
 	r128_PCI_IDS
diff --git a/drivers/gpu/drm/r128/r128_ioc32.c b/drivers/gpu/drm/r128/r128_ioc32.c
index 51c99fc4dd3..a954c548201 100644
--- a/drivers/gpu/drm/r128/r128_ioc32.c
+++ b/drivers/gpu/drm/r128/r128_ioc32.c
@@ -31,9 +31,8 @@
  */
 #include <linux/compat.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "r128_drm.h"
+#include <drm/drmP.h>
+#include <drm/r128_drm.h>
 
 typedef struct drm_r128_init32 {
 	int func;
diff --git a/drivers/gpu/drm/r128/r128_irq.c b/drivers/gpu/drm/r128/r128_irq.c
index 429d5a02695..2ea4f09d269 100644
--- a/drivers/gpu/drm/r128/r128_irq.c
+++ b/drivers/gpu/drm/r128/r128_irq.c
@@ -30,9 +30,8 @@
  *    Eric Anholt <anholt@FreeBSD.org>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "r128_drm.h"
+#include <drm/drmP.h>
+#include <drm/r128_drm.h>
 #include "r128_drv.h"
 
 u32 r128_get_vblank_counter(struct drm_device *dev, int crtc)
diff --git a/drivers/gpu/drm/r128/r128_state.c b/drivers/gpu/drm/r128/r128_state.c
index a9e33ce6591..19bb7e6f3d9 100644
--- a/drivers/gpu/drm/r128/r128_state.c
+++ b/drivers/gpu/drm/r128/r128_state.c
@@ -28,9 +28,8 @@
  *    Gareth Hughes <gareth@valinux.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "r128_drm.h"
+#include <drm/drmP.h>
+#include <drm/r128_drm.h>
 #include "r128_drv.h"
 
 /* ================================================================
diff --git a/drivers/gpu/drm/radeon/atom.h b/drivers/gpu/drm/radeon/atom.h
index 25fea631dad..feba6b8d36b 100644
--- a/drivers/gpu/drm/radeon/atom.h
+++ b/drivers/gpu/drm/radeon/atom.h
@@ -26,7 +26,7 @@
 #define ATOM_H
 
 #include <linux/types.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 #define ATOM_BIOS_MAGIC		0xAA55
 #define ATOM_ATI_MAGIC_PTR	0x30
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 2817101fb16..96184d02c8d 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -83,25 +83,19 @@ static void atombios_scaler_setup(struct drm_crtc *crtc)
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	ENABLE_SCALER_PS_ALLOCATION args;
 	int index = GetIndexIntoMasterTable(COMMAND, EnableScaler);
-
+	struct radeon_encoder *radeon_encoder =
+		to_radeon_encoder(radeon_crtc->encoder);
 	/* fixme - fill in enc_priv for atom dac */
 	enum radeon_tv_std tv_std = TV_STD_NTSC;
 	bool is_tv = false, is_cv = false;
-	struct drm_encoder *encoder;
 
 	if (!ASIC_IS_AVIVO(rdev) && radeon_crtc->crtc_id)
 		return;
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		/* find tv std */
-		if (encoder->crtc == crtc) {
-			struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-			if (radeon_encoder->active_device & ATOM_DEVICE_TV_SUPPORT) {
-				struct radeon_encoder_atom_dac *tv_dac = radeon_encoder->enc_priv;
-				tv_std = tv_dac->tv_std;
-				is_tv = true;
-			}
-		}
+	if (radeon_encoder->active_device & ATOM_DEVICE_TV_SUPPORT) {
+		struct radeon_encoder_atom_dac *tv_dac = radeon_encoder->enc_priv;
+		tv_std = tv_dac->tv_std;
+		is_tv = true;
 	}
 
 	memset(&args, 0, sizeof(args));
@@ -533,99 +527,87 @@ union adjust_pixel_clock {
 };
 
 static u32 atombios_adjust_pll(struct drm_crtc *crtc,
-			       struct drm_display_mode *mode,
-			       struct radeon_pll *pll,
-			       bool ss_enabled,
-			       struct radeon_atom_ss *ss)
+			       struct drm_display_mode *mode)
 {
+	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct drm_encoder *encoder = NULL;
-	struct radeon_encoder *radeon_encoder = NULL;
-	struct drm_connector *connector = NULL;
+	struct drm_encoder *encoder = radeon_crtc->encoder;
+	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
 	u32 adjusted_clock = mode->clock;
-	int encoder_mode = 0;
+	int encoder_mode = atombios_get_encoder_mode(encoder);
 	u32 dp_clock = mode->clock;
-	int bpc = 8;
-	bool is_duallink = false;
+	int bpc = radeon_get_monitor_bpc(connector);
+	bool is_duallink = radeon_dig_monitor_is_duallink(encoder, mode->clock);
 
 	/* reset the pll flags */
-	pll->flags = 0;
+	radeon_crtc->pll_flags = 0;
 
 	if (ASIC_IS_AVIVO(rdev)) {
 		if ((rdev->family == CHIP_RS600) ||
 		    (rdev->family == CHIP_RS690) ||
 		    (rdev->family == CHIP_RS740))
-			pll->flags |= (/*RADEON_PLL_USE_FRAC_FB_DIV |*/
-				       RADEON_PLL_PREFER_CLOSEST_LOWER);
+			radeon_crtc->pll_flags |= (/*RADEON_PLL_USE_FRAC_FB_DIV |*/
+				RADEON_PLL_PREFER_CLOSEST_LOWER);
 
 		if (ASIC_IS_DCE32(rdev) && mode->clock > 200000)	/* range limits??? */
-			pll->flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
 		else
-			pll->flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
 
 		if (rdev->family < CHIP_RV770)
-			pll->flags |= RADEON_PLL_PREFER_MINM_OVER_MAXP;
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_MINM_OVER_MAXP;
 		/* use frac fb div on APUs */
 		if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev))
-			pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
+			radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
 	} else {
-		pll->flags |= RADEON_PLL_LEGACY;
+		radeon_crtc->pll_flags |= RADEON_PLL_LEGACY;
 
 		if (mode->clock > 200000)	/* range limits??? */
-			pll->flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_HIGH_FB_DIV;
 		else
-			pll->flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_LOW_REF_DIV;
 	}
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		if (encoder->crtc == crtc) {
-			radeon_encoder = to_radeon_encoder(encoder);
-			connector = radeon_get_connector_for_encoder(encoder);
-			bpc = radeon_get_monitor_bpc(connector);
-			encoder_mode = atombios_get_encoder_mode(encoder);
-			is_duallink = radeon_dig_monitor_is_duallink(encoder, mode->clock);
-			if ((radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
-			    (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) {
-				if (connector) {
-					struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-					struct radeon_connector_atom_dig *dig_connector =
-						radeon_connector->con_priv;
-
-					dp_clock = dig_connector->dp_clock;
-				}
-			}
+	if ((radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
+	    (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) {
+		if (connector) {
+			struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+			struct radeon_connector_atom_dig *dig_connector =
+				radeon_connector->con_priv;
 
-			/* use recommended ref_div for ss */
-			if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
-				if (ss_enabled) {
-					if (ss->refdiv) {
-						pll->flags |= RADEON_PLL_USE_REF_DIV;
-						pll->reference_div = ss->refdiv;
-						if (ASIC_IS_AVIVO(rdev))
-							pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
-					}
-				}
-			}
+			dp_clock = dig_connector->dp_clock;
+		}
+	}
 
-			if (ASIC_IS_AVIVO(rdev)) {
-				/* DVO wants 2x pixel clock if the DVO chip is in 12 bit mode */
-				if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1)
-					adjusted_clock = mode->clock * 2;
-				if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
-					pll->flags |= RADEON_PLL_PREFER_CLOSEST_LOWER;
-				if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
-					pll->flags |= RADEON_PLL_IS_LCD;
-			} else {
-				if (encoder->encoder_type != DRM_MODE_ENCODER_DAC)
-					pll->flags |= RADEON_PLL_NO_ODD_POST_DIV;
-				if (encoder->encoder_type == DRM_MODE_ENCODER_LVDS)
-					pll->flags |= RADEON_PLL_USE_REF_DIV;
+	/* use recommended ref_div for ss */
+	if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+		if (radeon_crtc->ss_enabled) {
+			if (radeon_crtc->ss.refdiv) {
+				radeon_crtc->pll_flags |= RADEON_PLL_USE_REF_DIV;
+				radeon_crtc->pll_reference_div = radeon_crtc->ss.refdiv;
+				if (ASIC_IS_AVIVO(rdev))
+					radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
 			}
-			break;
 		}
 	}
 
+	if (ASIC_IS_AVIVO(rdev)) {
+		/* DVO wants 2x pixel clock if the DVO chip is in 12 bit mode */
+		if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1)
+			adjusted_clock = mode->clock * 2;
+		if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
+			radeon_crtc->pll_flags |= RADEON_PLL_PREFER_CLOSEST_LOWER;
+		if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+			radeon_crtc->pll_flags |= RADEON_PLL_IS_LCD;
+	} else {
+		if (encoder->encoder_type != DRM_MODE_ENCODER_DAC)
+			radeon_crtc->pll_flags |= RADEON_PLL_NO_ODD_POST_DIV;
+		if (encoder->encoder_type == DRM_MODE_ENCODER_LVDS)
+			radeon_crtc->pll_flags |= RADEON_PLL_USE_REF_DIV;
+	}
+
 	/* DCE3+ has an AdjustDisplayPll that will adjust the pixel clock
 	 * accordingly based on the encoder/transmitter to work around
 	 * special hw requirements.
@@ -650,7 +632,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
 				args.v1.usPixelClock = cpu_to_le16(mode->clock / 10);
 				args.v1.ucTransmitterID = radeon_encoder->encoder_id;
 				args.v1.ucEncodeMode = encoder_mode;
-				if (ss_enabled && ss->percentage)
+				if (radeon_crtc->ss_enabled && radeon_crtc->ss.percentage)
 					args.v1.ucConfig |=
 						ADJUST_DISPLAY_CONFIG_SS_ENABLE;
 
@@ -663,7 +645,7 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
 				args.v3.sInput.ucTransmitterID = radeon_encoder->encoder_id;
 				args.v3.sInput.ucEncodeMode = encoder_mode;
 				args.v3.sInput.ucDispPllConfig = 0;
-				if (ss_enabled && ss->percentage)
+				if (radeon_crtc->ss_enabled && radeon_crtc->ss.percentage)
 					args.v3.sInput.ucDispPllConfig |=
 						DISPPLL_CONFIG_SS_ENABLE;
 				if (ENCODER_MODE_IS_DP(encoder_mode)) {
@@ -695,14 +677,14 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
 						   index, (uint32_t *)&args);
 				adjusted_clock = le32_to_cpu(args.v3.sOutput.ulDispPllFreq) * 10;
 				if (args.v3.sOutput.ucRefDiv) {
-					pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
-					pll->flags |= RADEON_PLL_USE_REF_DIV;
-					pll->reference_div = args.v3.sOutput.ucRefDiv;
+					radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
+					radeon_crtc->pll_flags |= RADEON_PLL_USE_REF_DIV;
+					radeon_crtc->pll_reference_div = args.v3.sOutput.ucRefDiv;
 				}
 				if (args.v3.sOutput.ucPostDiv) {
-					pll->flags |= RADEON_PLL_USE_FRAC_FB_DIV;
-					pll->flags |= RADEON_PLL_USE_POST_DIV;
-					pll->post_div = args.v3.sOutput.ucPostDiv;
+					radeon_crtc->pll_flags |= RADEON_PLL_USE_FRAC_FB_DIV;
+					radeon_crtc->pll_flags |= RADEON_PLL_USE_POST_DIV;
+					radeon_crtc->pll_post_div = args.v3.sOutput.ucPostDiv;
 				}
 				break;
 			default:
@@ -837,7 +819,10 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
 			args.v3.ucFracFbDiv = frac_fb_div;
 			args.v3.ucPostDiv = post_div;
 			args.v3.ucPpll = pll_id;
-			args.v3.ucMiscInfo = (pll_id << 2);
+			if (crtc_id == ATOM_CRTC2)
+				args.v3.ucMiscInfo = PIXEL_CLOCK_MISC_CRTC_SEL_CRTC2;
+			else
+				args.v3.ucMiscInfo = PIXEL_CLOCK_MISC_CRTC_SEL_CRTC1;
 			if (ss_enabled && (ss->type & ATOM_EXTERNAL_SS_MASK))
 				args.v3.ucMiscInfo |= PIXEL_CLOCK_MISC_REF_DIV_SRC;
 			args.v3.ucTransmitterId = encoder_id;
@@ -907,58 +892,29 @@ static void atombios_crtc_program_pll(struct drm_crtc *crtc,
 	atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
 }
 
-static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
+static bool atombios_crtc_prepare_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
 {
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct drm_encoder *encoder = NULL;
-	struct radeon_encoder *radeon_encoder = NULL;
-	u32 pll_clock = mode->clock;
-	u32 ref_div = 0, fb_div = 0, frac_fb_div = 0, post_div = 0;
-	struct radeon_pll *pll;
-	u32 adjusted_clock;
-	int encoder_mode = 0;
-	struct radeon_atom_ss ss;
-	bool ss_enabled = false;
-	int bpc = 8;
+	struct radeon_encoder *radeon_encoder =
+		to_radeon_encoder(radeon_crtc->encoder);
+	int encoder_mode = atombios_get_encoder_mode(radeon_crtc->encoder);
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		if (encoder->crtc == crtc) {
-			radeon_encoder = to_radeon_encoder(encoder);
-			encoder_mode = atombios_get_encoder_mode(encoder);
-			break;
-		}
-	}
-
-	if (!radeon_encoder)
-		return;
-
-	switch (radeon_crtc->pll_id) {
-	case ATOM_PPLL1:
-		pll = &rdev->clock.p1pll;
-		break;
-	case ATOM_PPLL2:
-		pll = &rdev->clock.p2pll;
-		break;
-	case ATOM_DCPLL:
-	case ATOM_PPLL_INVALID:
-	default:
-		pll = &rdev->clock.dcpll;
-		break;
-	}
+	radeon_crtc->bpc = 8;
+	radeon_crtc->ss_enabled = false;
 
 	if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
-	    (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)) {
+	    (radeon_encoder_get_dp_bridge_encoder_id(radeon_crtc->encoder) != ENCODER_OBJECT_ID_NONE)) {
 		struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
 		struct drm_connector *connector =
-			radeon_get_connector_for_encoder(encoder);
+			radeon_get_connector_for_encoder(radeon_crtc->encoder);
 		struct radeon_connector *radeon_connector =
 			to_radeon_connector(connector);
 		struct radeon_connector_atom_dig *dig_connector =
 			radeon_connector->con_priv;
 		int dp_clock;
-		bpc = radeon_get_monitor_bpc(connector);
+		radeon_crtc->bpc = radeon_get_monitor_bpc(connector);
 
 		switch (encoder_mode) {
 		case ATOM_ENCODER_MODE_DP_MST:
@@ -966,45 +922,54 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
 			/* DP/eDP */
 			dp_clock = dig_connector->dp_clock / 10;
 			if (ASIC_IS_DCE4(rdev))
-				ss_enabled =
-					radeon_atombios_get_asic_ss_info(rdev, &ss,
+				radeon_crtc->ss_enabled =
+					radeon_atombios_get_asic_ss_info(rdev, &radeon_crtc->ss,
 									 ASIC_INTERNAL_SS_ON_DP,
 									 dp_clock);
 			else {
 				if (dp_clock == 16200) {
-					ss_enabled =
-						radeon_atombios_get_ppll_ss_info(rdev, &ss,
+					radeon_crtc->ss_enabled =
+						radeon_atombios_get_ppll_ss_info(rdev,
+										 &radeon_crtc->ss,
 										 ATOM_DP_SS_ID2);
-					if (!ss_enabled)
-						ss_enabled =
-							radeon_atombios_get_ppll_ss_info(rdev, &ss,
+					if (!radeon_crtc->ss_enabled)
+						radeon_crtc->ss_enabled =
+							radeon_atombios_get_ppll_ss_info(rdev,
+											 &radeon_crtc->ss,
 											 ATOM_DP_SS_ID1);
 				} else
-					ss_enabled =
-						radeon_atombios_get_ppll_ss_info(rdev, &ss,
+					radeon_crtc->ss_enabled =
+						radeon_atombios_get_ppll_ss_info(rdev,
+										 &radeon_crtc->ss,
 										 ATOM_DP_SS_ID1);
 			}
 			break;
 		case ATOM_ENCODER_MODE_LVDS:
 			if (ASIC_IS_DCE4(rdev))
-				ss_enabled = radeon_atombios_get_asic_ss_info(rdev, &ss,
-									      dig->lcd_ss_id,
-									      mode->clock / 10);
+				radeon_crtc->ss_enabled =
+					radeon_atombios_get_asic_ss_info(rdev,
+									 &radeon_crtc->ss,
+									 dig->lcd_ss_id,
+									 mode->clock / 10);
 			else
-				ss_enabled = radeon_atombios_get_ppll_ss_info(rdev, &ss,
-									      dig->lcd_ss_id);
+				radeon_crtc->ss_enabled =
+					radeon_atombios_get_ppll_ss_info(rdev,
+									 &radeon_crtc->ss,
+									 dig->lcd_ss_id);
 			break;
 		case ATOM_ENCODER_MODE_DVI:
 			if (ASIC_IS_DCE4(rdev))
-				ss_enabled =
-					radeon_atombios_get_asic_ss_info(rdev, &ss,
+				radeon_crtc->ss_enabled =
+					radeon_atombios_get_asic_ss_info(rdev,
+									 &radeon_crtc->ss,
 									 ASIC_INTERNAL_SS_ON_TMDS,
 									 mode->clock / 10);
 			break;
 		case ATOM_ENCODER_MODE_HDMI:
 			if (ASIC_IS_DCE4(rdev))
-				ss_enabled =
-					radeon_atombios_get_asic_ss_info(rdev, &ss,
+				radeon_crtc->ss_enabled =
+					radeon_atombios_get_asic_ss_info(rdev,
+									 &radeon_crtc->ss,
 									 ASIC_INTERNAL_SS_ON_HDMI,
 									 mode->clock / 10);
 			break;
@@ -1014,43 +979,80 @@ static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode
 	}
 
 	/* adjust pixel clock as needed */
-	adjusted_clock = atombios_adjust_pll(crtc, mode, pll, ss_enabled, &ss);
+	radeon_crtc->adjusted_clock = atombios_adjust_pll(crtc, mode);
+
+	return true;
+}
+
+static void atombios_crtc_set_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
+{
+	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct radeon_device *rdev = dev->dev_private;
+	struct radeon_encoder *radeon_encoder =
+		to_radeon_encoder(radeon_crtc->encoder);
+	u32 pll_clock = mode->clock;
+	u32 ref_div = 0, fb_div = 0, frac_fb_div = 0, post_div = 0;
+	struct radeon_pll *pll;
+	int encoder_mode = atombios_get_encoder_mode(radeon_crtc->encoder);
+
+	switch (radeon_crtc->pll_id) {
+	case ATOM_PPLL1:
+		pll = &rdev->clock.p1pll;
+		break;
+	case ATOM_PPLL2:
+		pll = &rdev->clock.p2pll;
+		break;
+	case ATOM_DCPLL:
+	case ATOM_PPLL_INVALID:
+	default:
+		pll = &rdev->clock.dcpll;
+		break;
+	}
+
+	/* update pll params */
+	pll->flags = radeon_crtc->pll_flags;
+	pll->reference_div = radeon_crtc->pll_reference_div;
+	pll->post_div = radeon_crtc->pll_post_div;
 
 	if (radeon_encoder->active_device & (ATOM_DEVICE_TV_SUPPORT))
 		/* TV seems to prefer the legacy algo on some boards */
-		radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
-					  &ref_div, &post_div);
+		radeon_compute_pll_legacy(pll, radeon_crtc->adjusted_clock, &pll_clock,
+					  &fb_div, &frac_fb_div, &ref_div, &post_div);
 	else if (ASIC_IS_AVIVO(rdev))
-		radeon_compute_pll_avivo(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
-					 &ref_div, &post_div);
+		radeon_compute_pll_avivo(pll, radeon_crtc->adjusted_clock, &pll_clock,
+					 &fb_div, &frac_fb_div, &ref_div, &post_div);
 	else
-		radeon_compute_pll_legacy(pll, adjusted_clock, &pll_clock, &fb_div, &frac_fb_div,
-					  &ref_div, &post_div);
+		radeon_compute_pll_legacy(pll, radeon_crtc->adjusted_clock, &pll_clock,
+					  &fb_div, &frac_fb_div, &ref_div, &post_div);
 
-	atombios_crtc_program_ss(rdev, ATOM_DISABLE, radeon_crtc->pll_id, radeon_crtc->crtc_id, &ss);
+	atombios_crtc_program_ss(rdev, ATOM_DISABLE, radeon_crtc->pll_id,
+				 radeon_crtc->crtc_id, &radeon_crtc->ss);
 
 	atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id,
 				  encoder_mode, radeon_encoder->encoder_id, mode->clock,
-				  ref_div, fb_div, frac_fb_div, post_div, bpc, ss_enabled, &ss);
+				  ref_div, fb_div, frac_fb_div, post_div,
+				  radeon_crtc->bpc, radeon_crtc->ss_enabled, &radeon_crtc->ss);
 
-	if (ss_enabled) {
+	if (radeon_crtc->ss_enabled) {
 		/* calculate ss amount and step size */
 		if (ASIC_IS_DCE4(rdev)) {
 			u32 step_size;
-			u32 amount = (((fb_div * 10) + frac_fb_div) * ss.percentage) / 10000;
-			ss.amount = (amount / 10) & ATOM_PPLL_SS_AMOUNT_V2_FBDIV_MASK;
-			ss.amount |= ((amount - (amount / 10)) << ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT) &
+			u32 amount = (((fb_div * 10) + frac_fb_div) * radeon_crtc->ss.percentage) / 10000;
+			radeon_crtc->ss.amount = (amount / 10) & ATOM_PPLL_SS_AMOUNT_V2_FBDIV_MASK;
+			radeon_crtc->ss.amount |= ((amount - (amount / 10)) << ATOM_PPLL_SS_AMOUNT_V2_NFRAC_SHIFT) &
 				ATOM_PPLL_SS_AMOUNT_V2_NFRAC_MASK;
-			if (ss.type & ATOM_PPLL_SS_TYPE_V2_CENTRE_SPREAD)
-				step_size = (4 * amount * ref_div * (ss.rate * 2048)) /
+			if (radeon_crtc->ss.type & ATOM_PPLL_SS_TYPE_V2_CENTRE_SPREAD)
+				step_size = (4 * amount * ref_div * (radeon_crtc->ss.rate * 2048)) /
 					(125 * 25 * pll->reference_freq / 100);
 			else
-				step_size = (2 * amount * ref_div * (ss.rate * 2048)) /
+				step_size = (2 * amount * ref_div * (radeon_crtc->ss.rate * 2048)) /
 					(125 * 25 * pll->reference_freq / 100);
-			ss.step = step_size;
+			radeon_crtc->ss.step = step_size;
 		}
 
-		atombios_crtc_program_ss(rdev, ATOM_ENABLE, radeon_crtc->pll_id, radeon_crtc->crtc_id, &ss);
+		atombios_crtc_program_ss(rdev, ATOM_ENABLE, radeon_crtc->pll_id,
+					 radeon_crtc->crtc_id, &radeon_crtc->ss);
 	}
 }
 
@@ -1479,85 +1481,251 @@ static void radeon_legacy_atom_fixup(struct drm_crtc *crtc)
 	}
 }
 
+/**
+ * radeon_get_pll_use_mask - look up a mask of which pplls are in use
+ *
+ * @crtc: drm crtc
+ *
+ * Returns the mask of which PPLLs (Pixel PLLs) are in use.
+ */
+static u32 radeon_get_pll_use_mask(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_crtc *test_crtc;
+	struct radeon_crtc *test_radeon_crtc;
+	u32 pll_in_use = 0;
+
+	list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc == test_crtc)
+			continue;
+
+		test_radeon_crtc = to_radeon_crtc(test_crtc);
+		if (test_radeon_crtc->pll_id != ATOM_PPLL_INVALID)
+			pll_in_use |= (1 << test_radeon_crtc->pll_id);
+	}
+	return pll_in_use;
+}
+
+/**
+ * radeon_get_shared_dp_ppll - return the PPLL used by another crtc for DP
+ *
+ * @crtc: drm crtc
+ *
+ * Returns the PPLL (Pixel PLL) used by another crtc/encoder which is
+ * also in DP mode.  For DP, a single PPLL can be used for all DP
+ * crtcs/encoders.
+ */
+static int radeon_get_shared_dp_ppll(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_crtc *test_crtc;
+	struct radeon_crtc *test_radeon_crtc;
+
+	list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc == test_crtc)
+			continue;
+		test_radeon_crtc = to_radeon_crtc(test_crtc);
+		if (test_radeon_crtc->encoder &&
+		    ENCODER_MODE_IS_DP(atombios_get_encoder_mode(test_radeon_crtc->encoder))) {
+			/* for DP use the same PLL for all */
+			if (test_radeon_crtc->pll_id != ATOM_PPLL_INVALID)
+				return test_radeon_crtc->pll_id;
+		}
+	}
+	return ATOM_PPLL_INVALID;
+}
+
+/**
+ * radeon_get_shared_nondp_ppll - return the PPLL used by another non-DP crtc
+ *
+ * @crtc: drm crtc
+ * @encoder: drm encoder
+ *
+ * Returns the PPLL (Pixel PLL) used by another non-DP crtc/encoder which can
+ * be shared (i.e., same clock).
+ */
+static int radeon_get_shared_nondp_ppll(struct drm_crtc *crtc)
+{
+	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct drm_crtc *test_crtc;
+	struct radeon_crtc *test_radeon_crtc;
+	u32 adjusted_clock, test_adjusted_clock;
+
+	adjusted_clock = radeon_crtc->adjusted_clock;
+
+	if (adjusted_clock == 0)
+		return ATOM_PPLL_INVALID;
+
+	list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc == test_crtc)
+			continue;
+		test_radeon_crtc = to_radeon_crtc(test_crtc);
+		if (test_radeon_crtc->encoder &&
+		    !ENCODER_MODE_IS_DP(atombios_get_encoder_mode(test_radeon_crtc->encoder))) {
+			/* check if we are already driving this connector with another crtc */
+			if (test_radeon_crtc->connector == radeon_crtc->connector) {
+				/* if we are, return that pll */
+				if (test_radeon_crtc->pll_id != ATOM_PPLL_INVALID)
+					return test_radeon_crtc->pll_id;
+			}
+			/* for non-DP check the clock */
+			test_adjusted_clock = test_radeon_crtc->adjusted_clock;
+			if ((crtc->mode.clock == test_crtc->mode.clock) &&
+			    (adjusted_clock == test_adjusted_clock) &&
+			    (radeon_crtc->ss_enabled == test_radeon_crtc->ss_enabled) &&
+			    (test_radeon_crtc->pll_id != ATOM_PPLL_INVALID))
+				return test_radeon_crtc->pll_id;
+		}
+	}
+	return ATOM_PPLL_INVALID;
+}
+
+/**
+ * radeon_atom_pick_pll - Allocate a PPLL for use by the crtc.
+ *
+ * @crtc: drm crtc
+ *
+ * Returns the PPLL (Pixel PLL) to be used by the crtc.  For DP monitors
+ * a single PPLL can be used for all DP crtcs/encoders.  For non-DP
+ * monitors a dedicated PPLL must be used.  If a particular board has
+ * an external DP PLL, return ATOM_PPLL_INVALID to skip PLL programming
+ * as there is no need to program the PLL itself.  If we are not able to
+ * allocate a PLL, return ATOM_PPLL_INVALID to skip PLL programming to
+ * avoid messing up an existing monitor.
+ *
+ * Asic specific PLL information
+ *
+ * DCE 6.1
+ * - PPLL2 is only available to UNIPHYA (both DP and non-DP)
+ * - PPLL0, PPLL1 are available for UNIPHYB/C/D/E/F (both DP and non-DP)
+ *
+ * DCE 6.0
+ * - PPLL0 is available to all UNIPHY (DP only)
+ * - PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) and DAC
+ *
+ * DCE 5.0
+ * - DCPLL is available to all UNIPHY (DP only)
+ * - PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) and DAC
+ *
+ * DCE 3.0/4.0/4.1
+ * - PPLL1, PPLL2 are available for all UNIPHY (both DP and non-DP) and DAC
+ *
+ */
 static int radeon_atom_pick_pll(struct drm_crtc *crtc)
 {
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct drm_encoder *test_encoder;
-	struct drm_crtc *test_crtc;
-	uint32_t pll_in_use = 0;
+	struct radeon_encoder *radeon_encoder =
+		to_radeon_encoder(radeon_crtc->encoder);
+	u32 pll_in_use;
+	int pll;
 
 	if (ASIC_IS_DCE61(rdev)) {
-		list_for_each_entry(test_encoder, &dev->mode_config.encoder_list, head) {
-			if (test_encoder->crtc && (test_encoder->crtc == crtc)) {
-				struct radeon_encoder *test_radeon_encoder =
-					to_radeon_encoder(test_encoder);
-				struct radeon_encoder_atom_dig *dig =
-					test_radeon_encoder->enc_priv;
-
-				if ((test_radeon_encoder->encoder_id ==
-				     ENCODER_OBJECT_ID_INTERNAL_UNIPHY) &&
-				    (dig->linkb == false)) /* UNIPHY A uses PPLL2 */
-					return ATOM_PPLL2;
+		struct radeon_encoder_atom_dig *dig =
+			radeon_encoder->enc_priv;
+
+		if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_UNIPHY) &&
+		    (dig->linkb == false))
+			/* UNIPHY A uses PPLL2 */
+			return ATOM_PPLL2;
+		else if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder))) {
+			/* UNIPHY B/C/D/E/F */
+			if (rdev->clock.dp_extclk)
+				/* skip PPLL programming if using ext clock */
+				return ATOM_PPLL_INVALID;
+			else {
+				/* use the same PPLL for all DP monitors */
+				pll = radeon_get_shared_dp_ppll(crtc);
+				if (pll != ATOM_PPLL_INVALID)
+					return pll;
 			}
+		} else {
+			/* use the same PPLL for all monitors with the same clock */
+			pll = radeon_get_shared_nondp_ppll(crtc);
+			if (pll != ATOM_PPLL_INVALID)
+				return pll;
 		}
 		/* UNIPHY B/C/D/E/F */
-		list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) {
-			struct radeon_crtc *radeon_test_crtc;
-
-			if (crtc == test_crtc)
-				continue;
-
-			radeon_test_crtc = to_radeon_crtc(test_crtc);
-			if ((radeon_test_crtc->pll_id == ATOM_PPLL0) ||
-			    (radeon_test_crtc->pll_id == ATOM_PPLL1))
-				pll_in_use |= (1 << radeon_test_crtc->pll_id);
-		}
-		if (!(pll_in_use & 4))
+		pll_in_use = radeon_get_pll_use_mask(crtc);
+		if (!(pll_in_use & (1 << ATOM_PPLL0)))
 			return ATOM_PPLL0;
-		return ATOM_PPLL1;
+		if (!(pll_in_use & (1 << ATOM_PPLL1)))
+			return ATOM_PPLL1;
+		DRM_ERROR("unable to allocate a PPLL\n");
+		return ATOM_PPLL_INVALID;
 	} else if (ASIC_IS_DCE4(rdev)) {
-		list_for_each_entry(test_encoder, &dev->mode_config.encoder_list, head) {
-			if (test_encoder->crtc && (test_encoder->crtc == crtc)) {
-				/* in DP mode, the DP ref clock can come from PPLL, DCPLL, or ext clock,
-				 * depending on the asic:
-				 * DCE4: PPLL or ext clock
-				 * DCE5: DCPLL or ext clock
-				 *
-				 * Setting ATOM_PPLL_INVALID will cause SetPixelClock to skip
-				 * PPLL/DCPLL programming and only program the DP DTO for the
-				 * crtc virtual pixel clock.
-				 */
-				if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(test_encoder))) {
-					if (rdev->clock.dp_extclk)
-						return ATOM_PPLL_INVALID;
-					else if (ASIC_IS_DCE6(rdev))
-						return ATOM_PPLL0;
-					else if (ASIC_IS_DCE5(rdev))
-						return ATOM_DCPLL;
-				}
+		/* in DP mode, the DP ref clock can come from PPLL, DCPLL, or ext clock,
+		 * depending on the asic:
+		 * DCE4: PPLL or ext clock
+		 * DCE5: PPLL, DCPLL, or ext clock
+		 * DCE6: PPLL, PPLL0, or ext clock
+		 *
+		 * Setting ATOM_PPLL_INVALID will cause SetPixelClock to skip
+		 * PPLL/DCPLL programming and only program the DP DTO for the
+		 * crtc virtual pixel clock.
+		 */
+		if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder))) {
+			if (rdev->clock.dp_extclk)
+				/* skip PPLL programming if using ext clock */
+				return ATOM_PPLL_INVALID;
+			else if (ASIC_IS_DCE6(rdev))
+				/* use PPLL0 for all DP */
+				return ATOM_PPLL0;
+			else if (ASIC_IS_DCE5(rdev))
+				/* use DCPLL for all DP */
+				return ATOM_DCPLL;
+			else {
+				/* use the same PPLL for all DP monitors */
+				pll = radeon_get_shared_dp_ppll(crtc);
+				if (pll != ATOM_PPLL_INVALID)
+					return pll;
 			}
+		} else {
+			/* use the same PPLL for all monitors with the same clock */
+			pll = radeon_get_shared_nondp_ppll(crtc);
+			if (pll != ATOM_PPLL_INVALID)
+				return pll;
 		}
-
-		/* otherwise, pick one of the plls */
-		list_for_each_entry(test_crtc, &dev->mode_config.crtc_list, head) {
-			struct radeon_crtc *radeon_test_crtc;
-
-			if (crtc == test_crtc)
-				continue;
-
-			radeon_test_crtc = to_radeon_crtc(test_crtc);
-			if ((radeon_test_crtc->pll_id >= ATOM_PPLL1) &&
-			    (radeon_test_crtc->pll_id <= ATOM_PPLL2))
-				pll_in_use |= (1 << radeon_test_crtc->pll_id);
-		}
-		if (!(pll_in_use & 1))
+		/* all other cases */
+		pll_in_use = radeon_get_pll_use_mask(crtc);
+		if (!(pll_in_use & (1 << ATOM_PPLL2)))
+			return ATOM_PPLL2;
+		if (!(pll_in_use & (1 << ATOM_PPLL1)))
 			return ATOM_PPLL1;
-		return ATOM_PPLL2;
-	} else
-		return radeon_crtc->crtc_id;
-
+		DRM_ERROR("unable to allocate a PPLL\n");
+		return ATOM_PPLL_INVALID;
+	} else {
+		if (ASIC_IS_AVIVO(rdev)) {
+			/* in DP mode, the DP ref clock can come from either PPLL
+			 * depending on the asic:
+			 * DCE3: PPLL1 or PPLL2
+			 */
+			if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder))) {
+				/* use the same PPLL for all DP monitors */
+				pll = radeon_get_shared_dp_ppll(crtc);
+				if (pll != ATOM_PPLL_INVALID)
+					return pll;
+			} else {
+				/* use the same PPLL for all monitors with the same clock */
+				pll = radeon_get_shared_nondp_ppll(crtc);
+				if (pll != ATOM_PPLL_INVALID)
+					return pll;
+			}
+			/* all other cases */
+			pll_in_use = radeon_get_pll_use_mask(crtc);
+			if (!(pll_in_use & (1 << ATOM_PPLL2)))
+				return ATOM_PPLL2;
+			if (!(pll_in_use & (1 << ATOM_PPLL1)))
+				return ATOM_PPLL1;
+			DRM_ERROR("unable to allocate a PPLL\n");
+			return ATOM_PPLL_INVALID;
+		} else {
+			/* on pre-R5xx asics, the crtc to pll mapping is hardcoded */
+			return radeon_crtc->crtc_id;
+		}
+	}
 }
 
 void radeon_atom_disp_eng_pll_init(struct radeon_device *rdev)
@@ -1588,18 +1756,13 @@ int atombios_crtc_mode_set(struct drm_crtc *crtc,
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct radeon_device *rdev = dev->dev_private;
-	struct drm_encoder *encoder;
+	struct radeon_encoder *radeon_encoder =
+		to_radeon_encoder(radeon_crtc->encoder);
 	bool is_tvcv = false;
 
-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-		/* find tv std */
-		if (encoder->crtc == crtc) {
-			struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-			if (radeon_encoder->active_device &
-			    (ATOM_DEVICE_TV_SUPPORT | ATOM_DEVICE_CV_SUPPORT))
-				is_tvcv = true;
-		}
-	}
+	if (radeon_encoder->active_device &
+	    (ATOM_DEVICE_TV_SUPPORT | ATOM_DEVICE_CV_SUPPORT))
+		is_tvcv = true;
 
 	atombios_crtc_set_pll(crtc, adjusted_mode);
 
@@ -1626,8 +1789,34 @@ static bool atombios_crtc_mode_fixup(struct drm_crtc *crtc,
 				     const struct drm_display_mode *mode,
 				     struct drm_display_mode *adjusted_mode)
 {
+	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	struct drm_encoder *encoder;
+
+	/* assign the encoder to the radeon crtc to avoid repeated lookups later */
+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
+		if (encoder->crtc == crtc) {
+			radeon_crtc->encoder = encoder;
+			radeon_crtc->connector = radeon_get_connector_for_encoder(encoder);
+			break;
+		}
+	}
+	if ((radeon_crtc->encoder == NULL) || (radeon_crtc->connector == NULL)) {
+		radeon_crtc->encoder = NULL;
+		radeon_crtc->connector = NULL;
+		return false;
+	}
 	if (!radeon_crtc_scaling_mode_fixup(crtc, mode, adjusted_mode))
 		return false;
+	if (!atombios_crtc_prepare_pll(crtc, adjusted_mode))
+		return false;
+	/* pick pll */
+	radeon_crtc->pll_id = radeon_atom_pick_pll(crtc);
+	/* if we can't get a PPLL for a non-DP encoder, fail */
+	if ((radeon_crtc->pll_id == ATOM_PPLL_INVALID) &&
+	    !ENCODER_MODE_IS_DP(atombios_get_encoder_mode(radeon_crtc->encoder)))
+		return false;
+
 	return true;
 }
 
@@ -1638,8 +1827,6 @@ static void atombios_crtc_prepare(struct drm_crtc *crtc)
 	struct radeon_device *rdev = dev->dev_private;
 
 	radeon_crtc->in_mode_set = true;
-	/* pick pll */
-	radeon_crtc->pll_id = radeon_atom_pick_pll(crtc);
 
 	/* disable crtc pair power gating before programming */
 	if (ASIC_IS_DCE6(rdev))
@@ -1697,7 +1884,10 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
 		break;
 	}
 done:
-	radeon_crtc->pll_id = -1;
+	radeon_crtc->pll_id = ATOM_PPLL_INVALID;
+	radeon_crtc->adjusted_clock = 0;
+	radeon_crtc->encoder = NULL;
+	radeon_crtc->connector = NULL;
 }
 
 static const struct drm_crtc_helper_funcs atombios_helper_funcs = {
@@ -1746,6 +1936,9 @@ void radeon_atombios_init_crtc(struct drm_device *dev,
 		else
 			radeon_crtc->crtc_offset = 0;
 	}
-	radeon_crtc->pll_id = -1;
+	radeon_crtc->pll_id = ATOM_PPLL_INVALID;
+	radeon_crtc->adjusted_clock = 0;
+	radeon_crtc->encoder = NULL;
+	radeon_crtc->connector = NULL;
 	drm_crtc_helper_add(&radeon_crtc->base, &atombios_helper_funcs);
 }
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index 3623b98ed3f..d5699fe4f1e 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -24,13 +24,13 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #include "atom.h"
 #include "atom-bits.h"
-#include "drm_dp_helper.h"
+#include <drm/drm_dp_helper.h>
 
 /* move these to drm_dp_helper.c/h */
 #define DP_LINK_CONFIGURATION_SIZE 9
@@ -653,9 +653,7 @@ static bool radeon_dp_get_link_status(struct radeon_connector *radeon_connector,
 		return false;
 	}
 
-	DRM_DEBUG_KMS("link status %02x %02x %02x %02x %02x %02x\n",
-		  link_status[0], link_status[1], link_status[2],
-		  link_status[3], link_status[4], link_status[5]);
+	DRM_DEBUG_KMS("link status %*ph\n", 6, link_status);
 	return true;
 }
 
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 6e8803a1170..49cbb3795a1 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -23,14 +23,256 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
+#include <linux/backlight.h>
 
 extern int atom_debug;
 
+static u8
+radeon_atom_get_backlight_level_from_reg(struct radeon_device *rdev)
+{
+	u8 backlight_level;
+	u32 bios_2_scratch;
+
+	if (rdev->family >= CHIP_R600)
+		bios_2_scratch = RREG32(R600_BIOS_2_SCRATCH);
+	else
+		bios_2_scratch = RREG32(RADEON_BIOS_2_SCRATCH);
+
+	backlight_level = ((bios_2_scratch & ATOM_S2_CURRENT_BL_LEVEL_MASK) >>
+			   ATOM_S2_CURRENT_BL_LEVEL_SHIFT);
+
+	return backlight_level;
+}
+
+static void
+radeon_atom_set_backlight_level_to_reg(struct radeon_device *rdev,
+				       u8 backlight_level)
+{
+	u32 bios_2_scratch;
+
+	if (rdev->family >= CHIP_R600)
+		bios_2_scratch = RREG32(R600_BIOS_2_SCRATCH);
+	else
+		bios_2_scratch = RREG32(RADEON_BIOS_2_SCRATCH);
+
+	bios_2_scratch &= ~ATOM_S2_CURRENT_BL_LEVEL_MASK;
+	bios_2_scratch |= ((backlight_level << ATOM_S2_CURRENT_BL_LEVEL_SHIFT) &
+			   ATOM_S2_CURRENT_BL_LEVEL_MASK);
+
+	if (rdev->family >= CHIP_R600)
+		WREG32(R600_BIOS_2_SCRATCH, bios_2_scratch);
+	else
+		WREG32(RADEON_BIOS_2_SCRATCH, bios_2_scratch);
+}
+
+u8
+atombios_get_backlight_level(struct radeon_encoder *radeon_encoder)
+{
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+
+	if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
+		return 0;
+
+	return radeon_atom_get_backlight_level_from_reg(rdev);
+}
+
+void
+atombios_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level)
+{
+	struct drm_encoder *encoder = &radeon_encoder->base;
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+	struct radeon_encoder_atom_dig *dig;
+	DISPLAY_DEVICE_OUTPUT_CONTROL_PS_ALLOCATION args;
+	int index;
+
+	if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
+		return;
+
+	if ((radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) &&
+	    radeon_encoder->enc_priv) {
+		dig = radeon_encoder->enc_priv;
+		dig->backlight_level = level;
+		radeon_atom_set_backlight_level_to_reg(rdev, dig->backlight_level);
+
+		switch (radeon_encoder->encoder_id) {
+		case ENCODER_OBJECT_ID_INTERNAL_LVDS:
+		case ENCODER_OBJECT_ID_INTERNAL_LVTM1:
+			index = GetIndexIntoMasterTable(COMMAND, LCD1OutputControl);
+			if (dig->backlight_level == 0) {
+				args.ucAction = ATOM_LCD_BLOFF;
+				atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+			} else {
+				args.ucAction = ATOM_LCD_BL_BRIGHTNESS_CONTROL;
+				atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+				args.ucAction = ATOM_LCD_BLON;
+				atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
+			}
+			break;
+		case ENCODER_OBJECT_ID_INTERNAL_UNIPHY:
+		case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
+		case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
+		case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+			if (dig->backlight_level == 0)
+				atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
+			else {
+				atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_BL_BRIGHTNESS_CONTROL, 0, 0);
+				atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
+			}
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
+
+static u8 radeon_atom_bl_level(struct backlight_device *bd)
+{
+	u8 level;
+
+	/* Convert brightness to hardware level */
+	if (bd->props.brightness < 0)
+		level = 0;
+	else if (bd->props.brightness > RADEON_MAX_BL_LEVEL)
+		level = RADEON_MAX_BL_LEVEL;
+	else
+		level = bd->props.brightness;
+
+	return level;
+}
+
+static int radeon_atom_backlight_update_status(struct backlight_device *bd)
+{
+	struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+	struct radeon_encoder *radeon_encoder = pdata->encoder;
+
+	atombios_set_backlight_level(radeon_encoder, radeon_atom_bl_level(bd));
+
+	return 0;
+}
+
+static int radeon_atom_backlight_get_brightness(struct backlight_device *bd)
+{
+	struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+	struct radeon_encoder *radeon_encoder = pdata->encoder;
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+
+	return radeon_atom_get_backlight_level_from_reg(rdev);
+}
+
+static const struct backlight_ops radeon_atom_backlight_ops = {
+	.get_brightness = radeon_atom_backlight_get_brightness,
+	.update_status	= radeon_atom_backlight_update_status,
+};
+
+void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
+				struct drm_connector *drm_connector)
+{
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+	struct backlight_device *bd;
+	struct backlight_properties props;
+	struct radeon_backlight_privdata *pdata;
+	struct radeon_encoder_atom_dig *dig;
+	u8 backlight_level;
+
+	if (!radeon_encoder->enc_priv)
+		return;
+
+	if (!rdev->is_atom_bios)
+		return;
+
+	if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
+		return;
+
+	pdata = kmalloc(sizeof(struct radeon_backlight_privdata), GFP_KERNEL);
+	if (!pdata) {
+		DRM_ERROR("Memory allocation failed\n");
+		goto error;
+	}
+
+	memset(&props, 0, sizeof(props));
+	props.max_brightness = RADEON_MAX_BL_LEVEL;
+	props.type = BACKLIGHT_RAW;
+	bd = backlight_device_register("radeon_bl", &drm_connector->kdev,
+				       pdata, &radeon_atom_backlight_ops, &props);
+	if (IS_ERR(bd)) {
+		DRM_ERROR("Backlight registration failed\n");
+		goto error;
+	}
+
+	pdata->encoder = radeon_encoder;
+
+	backlight_level = radeon_atom_get_backlight_level_from_reg(rdev);
+
+	dig = radeon_encoder->enc_priv;
+	dig->bl_dev = bd;
+
+	bd->props.brightness = radeon_atom_backlight_get_brightness(bd);
+	bd->props.power = FB_BLANK_UNBLANK;
+	backlight_update_status(bd);
+
+	DRM_INFO("radeon atom DIG backlight initialized\n");
+
+	return;
+
+error:
+	kfree(pdata);
+	return;
+}
+
+static void radeon_atom_backlight_exit(struct radeon_encoder *radeon_encoder)
+{
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+	struct backlight_device *bd = NULL;
+	struct radeon_encoder_atom_dig *dig;
+
+	if (!radeon_encoder->enc_priv)
+		return;
+
+	if (!rdev->is_atom_bios)
+		return;
+
+	if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
+		return;
+
+	dig = radeon_encoder->enc_priv;
+	bd = dig->bl_dev;
+	dig->bl_dev = NULL;
+
+	if (bd) {
+		struct radeon_legacy_backlight_privdata *pdata;
+
+		pdata = bl_get_data(bd);
+		backlight_device_unregister(bd);
+		kfree(pdata);
+
+		DRM_INFO("radeon atom LVDS backlight unloaded\n");
+	}
+}
+
+#else /* !CONFIG_BACKLIGHT_CLASS_DEVICE */
+
+void radeon_atom_backlight_init(struct radeon_encoder *encoder)
+{
+}
+
+static void radeon_atom_backlight_exit(struct radeon_encoder *encoder)
+{
+}
+
+#endif
+
 /* evil but including atombios.h is much worse */
 bool radeon_atom_get_tv_timings(struct radeon_device *rdev, int index,
 				struct drm_display_mode *mode);
@@ -209,6 +451,32 @@ atombios_tv_setup(struct drm_encoder *encoder, int action)
 
 }
 
+static u8 radeon_atom_get_bpc(struct drm_encoder *encoder)
+{
+	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+	int bpc = 8;
+
+	if (connector)
+		bpc = radeon_get_monitor_bpc(connector);
+
+	switch (bpc) {
+	case 0:
+		return PANEL_BPC_UNDEFINE;
+	case 6:
+		return PANEL_6BIT_PER_COLOR;
+	case 8:
+	default:
+		return PANEL_8BIT_PER_COLOR;
+	case 10:
+		return PANEL_10BIT_PER_COLOR;
+	case 12:
+		return PANEL_12BIT_PER_COLOR;
+	case 16:
+		return PANEL_16BIT_PER_COLOR;
+	}
+}
+
+
 union dvo_encoder_control {
 	ENABLE_EXTERNAL_TMDS_ENCODER_PS_ALLOCATION ext_tmds;
 	DVO_ENCODER_CONTROL_PS_ALLOCATION dvo;
@@ -406,7 +674,8 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
 		return ATOM_ENCODER_MODE_DP;
 
 	/* DVO is always DVO */
-	if (radeon_encoder->encoder_id == ATOM_ENCODER_MODE_DVO)
+	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_DVO1) ||
+	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1))
 		return ATOM_ENCODER_MODE_DVO;
 
 	connector = radeon_get_connector_for_encoder(encoder);
@@ -535,7 +804,6 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo
 	int dp_clock = 0;
 	int dp_lane_count = 0;
 	int hpd_id = RADEON_HPD_NONE;
-	int bpc = 8;
 
 	if (connector) {
 		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -545,7 +813,6 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo
 		dp_clock = dig_connector->dp_clock;
 		dp_lane_count = dig_connector->dp_lane_count;
 		hpd_id = radeon_connector->hpd.hpd;
-		bpc = radeon_get_monitor_bpc(connector);
 	}
 
 	/* no dig encoder assigned */
@@ -612,37 +879,17 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo
 			else
 				args.v3.ucEncoderMode = atombios_get_encoder_mode(encoder);
 
-			if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode))
+			if (ENCODER_MODE_IS_DP(args.v3.ucEncoderMode))
 				args.v3.ucLaneNum = dp_lane_count;
 			else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 				args.v3.ucLaneNum = 8;
 			else
 				args.v3.ucLaneNum = 4;
 
-			if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode) && (dp_clock == 270000))
+			if (ENCODER_MODE_IS_DP(args.v3.ucEncoderMode) && (dp_clock == 270000))
 				args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V3_DPLINKRATE_2_70GHZ;
 			args.v3.acConfig.ucDigSel = dig->dig_encoder;
-			switch (bpc) {
-			case 0:
-				args.v3.ucBitPerColor = PANEL_BPC_UNDEFINE;
-				break;
-			case 6:
-				args.v3.ucBitPerColor = PANEL_6BIT_PER_COLOR;
-				break;
-			case 8:
-			default:
-				args.v3.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-				break;
-			case 10:
-				args.v3.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-				break;
-			case 12:
-				args.v3.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-				break;
-			case 16:
-				args.v3.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-				break;
-			}
+			args.v3.ucBitPerColor = radeon_atom_get_bpc(encoder);
 			break;
 		case 4:
 			args.v4.ucAction = action;
@@ -652,41 +899,21 @@ atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mo
 			else
 				args.v4.ucEncoderMode = atombios_get_encoder_mode(encoder);
 
-			if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode))
+			if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode))
 				args.v4.ucLaneNum = dp_lane_count;
 			else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 				args.v4.ucLaneNum = 8;
 			else
 				args.v4.ucLaneNum = 4;
 
-			if (ENCODER_MODE_IS_DP(args.v1.ucEncoderMode)) {
+			if (ENCODER_MODE_IS_DP(args.v4.ucEncoderMode)) {
 				if (dp_clock == 270000)
 					args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_2_70GHZ;
 				else if (dp_clock == 540000)
 					args.v1.ucConfig |= ATOM_ENCODER_CONFIG_V4_DPLINKRATE_5_40GHZ;
 			}
 			args.v4.acConfig.ucDigSel = dig->dig_encoder;
-			switch (bpc) {
-			case 0:
-				args.v4.ucBitPerColor = PANEL_BPC_UNDEFINE;
-				break;
-			case 6:
-				args.v4.ucBitPerColor = PANEL_6BIT_PER_COLOR;
-				break;
-			case 8:
-			default:
-				args.v4.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-				break;
-			case 10:
-				args.v4.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-				break;
-			case 12:
-				args.v4.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-				break;
-			case 16:
-				args.v4.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-				break;
-			}
+			args.v4.ucBitPerColor = radeon_atom_get_bpc(encoder);
 			if (hpd_id == RADEON_HPD_NONE)
 				args.v4.ucHPD_ID = 0;
 			else
@@ -799,8 +1026,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
 				args.v1.asMode.ucLaneSet = lane_set;
 			} else {
 				if (is_dp)
-					args.v1.usPixelClock =
-						cpu_to_le16(dp_clock / 10);
+					args.v1.usPixelClock = cpu_to_le16(dp_clock / 10);
 				else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 					args.v1.usPixelClock = cpu_to_le16((radeon_encoder->pixel_clock / 2) / 10);
 				else
@@ -857,8 +1083,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
 				args.v2.asMode.ucLaneSet = lane_set;
 			} else {
 				if (is_dp)
-					args.v2.usPixelClock =
-						cpu_to_le16(dp_clock / 10);
+					args.v2.usPixelClock = cpu_to_le16(dp_clock / 10);
 				else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 					args.v2.usPixelClock = cpu_to_le16((radeon_encoder->pixel_clock / 2) / 10);
 				else
@@ -900,8 +1125,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
 				args.v3.asMode.ucLaneSet = lane_set;
 			} else {
 				if (is_dp)
-					args.v3.usPixelClock =
-						cpu_to_le16(dp_clock / 10);
+					args.v3.usPixelClock = cpu_to_le16(dp_clock / 10);
 				else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 					args.v3.usPixelClock = cpu_to_le16((radeon_encoder->pixel_clock / 2) / 10);
 				else
@@ -960,8 +1184,7 @@ atombios_dig_transmitter_setup(struct drm_encoder *encoder, int action, uint8_t
 				args.v4.asMode.ucLaneSet = lane_set;
 			} else {
 				if (is_dp)
-					args.v4.usPixelClock =
-						cpu_to_le16(dp_clock / 10);
+					args.v4.usPixelClock = cpu_to_le16(dp_clock / 10);
 				else if (radeon_dig_monitor_is_duallink(encoder, radeon_encoder->pixel_clock))
 					args.v4.usPixelClock = cpu_to_le16((radeon_encoder->pixel_clock / 2) / 10);
 				else
@@ -1147,7 +1370,6 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
 	int dp_lane_count = 0;
 	int connector_object_id = 0;
 	u32 ext_enum = (ext_radeon_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT;
-	int bpc = 8;
 
 	if (action == EXTERNAL_ENCODER_ACTION_V3_ENCODER_INIT)
 		connector = radeon_get_connector_for_encoder_init(encoder);
@@ -1163,7 +1385,6 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
 		dp_lane_count = dig_connector->dp_lane_count;
 		connector_object_id =
 			(radeon_connector->connector_object_id & OBJECT_ID_MASK) >> OBJECT_ID_SHIFT;
-		bpc = radeon_get_monitor_bpc(connector);
 	}
 
 	memset(&args, 0, sizeof(args));
@@ -1221,27 +1442,7 @@ atombios_external_encoder_setup(struct drm_encoder *encoder,
 				args.v3.sExtEncoder.ucConfig |= EXTERNAL_ENCODER_CONFIG_V3_ENCODER3;
 				break;
 			}
-			switch (bpc) {
-			case 0:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_BPC_UNDEFINE;
-				break;
-			case 6:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_6BIT_PER_COLOR;
-				break;
-			case 8:
-			default:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_8BIT_PER_COLOR;
-				break;
-			case 10:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_10BIT_PER_COLOR;
-				break;
-			case 12:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_12BIT_PER_COLOR;
-				break;
-			case 16:
-				args.v3.sExtEncoder.ucBitPerColor = PANEL_16BIT_PER_COLOR;
-				break;
-			}
+			args.v3.sExtEncoder.ucBitPerColor = radeon_atom_get_bpc(encoder);
 			break;
 		default:
 			DRM_ERROR("Unknown table version: %d, %d\n", frev, crev);
@@ -2286,6 +2487,8 @@ static const struct drm_encoder_helper_funcs radeon_atom_dac_helper_funcs = {
 void radeon_enc_destroy(struct drm_encoder *encoder)
 {
 	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+	if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+		radeon_atom_backlight_exit(radeon_encoder);
 	kfree(radeon_encoder->enc_priv);
 	drm_encoder_cleanup(encoder);
 	kfree(radeon_encoder);
@@ -2295,7 +2498,7 @@ static const struct drm_encoder_funcs radeon_atom_enc_funcs = {
 	.destroy = radeon_enc_destroy,
 };
 
-struct radeon_encoder_atom_dac *
+static struct radeon_encoder_atom_dac *
 radeon_atombios_set_dac_info(struct radeon_encoder *radeon_encoder)
 {
 	struct drm_device *dev = radeon_encoder->base.dev;
@@ -2309,7 +2512,7 @@ radeon_atombios_set_dac_info(struct radeon_encoder *radeon_encoder)
 	return dac;
 }
 
-struct radeon_encoder_atom_dig *
+static struct radeon_encoder_atom_dig *
 radeon_atombios_set_dig_info(struct radeon_encoder *radeon_encoder)
 {
 	int encoder_enum = (radeon_encoder->encoder_enum & ENUM_ID_MASK) >> ENUM_ID_SHIFT;
diff --git a/drivers/gpu/drm/radeon/atombios_i2c.c b/drivers/gpu/drm/radeon/atombios_i2c.c
index 44d87b6b422..082338df708 100644
--- a/drivers/gpu/drm/radeon/atombios_i2c.c
+++ b/drivers/gpu/drm/radeon/atombios_i2c.c
@@ -22,8 +22,8 @@
  * Authors: Alex Deucher
  *
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index ed3340adeb6..a1f49c5fd74 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -24,10 +24,10 @@
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "evergreend.h"
 #include "atom.h"
 #include "avivod.h"
@@ -37,6 +37,16 @@
 #define EVERGREEN_PFP_UCODE_SIZE 1120
 #define EVERGREEN_PM4_UCODE_SIZE 1376
 
+static const u32 crtc_offsets[6] =
+{
+	EVERGREEN_CRTC0_REGISTER_OFFSET,
+	EVERGREEN_CRTC1_REGISTER_OFFSET,
+	EVERGREEN_CRTC2_REGISTER_OFFSET,
+	EVERGREEN_CRTC3_REGISTER_OFFSET,
+	EVERGREEN_CRTC4_REGISTER_OFFSET,
+	EVERGREEN_CRTC5_REGISTER_OFFSET
+};
+
 static void evergreen_gpu_init(struct radeon_device *rdev);
 void evergreen_fini(struct radeon_device *rdev);
 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
@@ -105,17 +115,19 @@ void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
  */
 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
 {
-	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
 	int i;
 
-	if (RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset) & EVERGREEN_CRTC_MASTER_EN) {
+	if (crtc >= rdev->num_crtc)
+		return;
+
+	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
 		for (i = 0; i < rdev->usec_timeout; i++) {
-			if (!(RREG32(EVERGREEN_CRTC_STATUS + radeon_crtc->crtc_offset) & EVERGREEN_CRTC_V_BLANK))
+			if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
 				break;
 			udelay(1);
 		}
 		for (i = 0; i < rdev->usec_timeout; i++) {
-			if (RREG32(EVERGREEN_CRTC_STATUS + radeon_crtc->crtc_offset) & EVERGREEN_CRTC_V_BLANK)
+			if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
 				break;
 			udelay(1);
 		}
@@ -310,6 +322,64 @@ void sumo_pm_init_profile(struct radeon_device *rdev)
 }
 
 /**
+ * btc_pm_init_profile - Initialize power profiles callback.
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Initialize the power states used in profile mode
+ * (BTC, cayman).
+ * Used for profile mode only.
+ */
+void btc_pm_init_profile(struct radeon_device *rdev)
+{
+	int idx;
+
+	/* default */
+	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
+	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
+	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
+	/* starting with BTC, there is one state that is used for both
+	 * MH and SH.  Difference is that we always use the high clock index for
+	 * mclk.
+	 */
+	if (rdev->flags & RADEON_IS_MOBILITY)
+		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
+	else
+		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
+	/* low sh */
+	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
+	/* mid sh */
+	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
+	/* high sh */
+	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
+	/* low mh */
+	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
+	/* mid mh */
+	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
+	/* high mh */
+	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
+	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
+	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
+}
+
+/**
  * evergreen_pm_misc - set additional pm hw parameters callback.
  *
  * @rdev: radeon_device pointer
@@ -1105,7 +1175,7 @@ void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
 	}
 }
 
-int evergreen_pcie_gart_enable(struct radeon_device *rdev)
+static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int r;
@@ -1164,7 +1234,7 @@ int evergreen_pcie_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void evergreen_pcie_gart_disable(struct radeon_device *rdev)
+static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
 {
 	u32 tmp;
 
@@ -1189,7 +1259,7 @@ void evergreen_pcie_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void evergreen_pcie_gart_fini(struct radeon_device *rdev)
+static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
 {
 	evergreen_pcie_gart_disable(rdev);
 	radeon_gart_table_vram_free(rdev);
@@ -1197,7 +1267,7 @@ void evergreen_pcie_gart_fini(struct radeon_device *rdev)
 }
 
 
-void evergreen_agp_enable(struct radeon_device *rdev)
+static void evergreen_agp_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 
@@ -1225,116 +1295,103 @@ void evergreen_agp_enable(struct radeon_device *rdev)
 
 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
 {
+	u32 crtc_enabled, tmp, frame_count, blackout;
+	int i, j;
+
 	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
 	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
 
-	/* Stop all video */
+	/* disable VGA render */
 	WREG32(VGA_RENDER_CONTROL, 0);
-	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
-	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
-	if (rdev->num_crtc >= 4) {
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
-	}
-	if (rdev->num_crtc >= 6) {
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
-	}
-	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
-	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-	if (rdev->num_crtc >= 4) {
-		WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-		WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-	}
-	if (rdev->num_crtc >= 6) {
-		WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-		WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
-	}
-	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
-	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
-	if (rdev->num_crtc >= 4) {
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
-	}
-	if (rdev->num_crtc >= 6) {
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
-		WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
+	/* blank the display controllers */
+	for (i = 0; i < rdev->num_crtc; i++) {
+		crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
+		if (crtc_enabled) {
+			save->crtc_enabled[i] = true;
+			if (ASIC_IS_DCE6(rdev)) {
+				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
+				if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
+					radeon_wait_for_vblank(rdev, i);
+					tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
+					WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
+				}
+			} else {
+				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
+				if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
+					radeon_wait_for_vblank(rdev, i);
+					tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
+					WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
+				}
+			}
+			/* wait for the next frame */
+			frame_count = radeon_get_vblank_counter(rdev, i);
+			for (j = 0; j < rdev->usec_timeout; j++) {
+				if (radeon_get_vblank_counter(rdev, i) != frame_count)
+					break;
+				udelay(1);
+			}
+		}
 	}
 
-	WREG32(D1VGA_CONTROL, 0);
-	WREG32(D2VGA_CONTROL, 0);
-	if (rdev->num_crtc >= 4) {
-		WREG32(EVERGREEN_D3VGA_CONTROL, 0);
-		WREG32(EVERGREEN_D4VGA_CONTROL, 0);
-	}
-	if (rdev->num_crtc >= 6) {
-		WREG32(EVERGREEN_D5VGA_CONTROL, 0);
-		WREG32(EVERGREEN_D6VGA_CONTROL, 0);
+	radeon_mc_wait_for_idle(rdev);
+
+	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
+	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
+		/* Block CPU access */
+		WREG32(BIF_FB_EN, 0);
+		/* blackout the MC */
+		blackout &= ~BLACKOUT_MODE_MASK;
+		WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
 	}
 }
 
 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
 {
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
-	       upper_32_bits(rdev->mc.vram_start));
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
-	       upper_32_bits(rdev->mc.vram_start));
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
-	       (u32)rdev->mc.vram_start);
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
-	       (u32)rdev->mc.vram_start);
-
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
-	       upper_32_bits(rdev->mc.vram_start));
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
-	       upper_32_bits(rdev->mc.vram_start));
-	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
-	       (u32)rdev->mc.vram_start);
-	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
-	       (u32)rdev->mc.vram_start);
-
-	if (rdev->num_crtc >= 4) {
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
-
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
-	}
-	if (rdev->num_crtc >= 6) {
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
-		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
-		       (u32)rdev->mc.vram_start);
+	u32 tmp, frame_count;
+	int i, j;
 
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
+	/* update crtc base addresses */
+	for (i = 0; i < rdev->num_crtc; i++) {
+		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
 		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
+		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
 		       upper_32_bits(rdev->mc.vram_start));
-		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
+		WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
 		       (u32)rdev->mc.vram_start);
-		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
+		WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
 		       (u32)rdev->mc.vram_start);
 	}
-
 	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
 	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
-	/* Unlock host access */
+
+	/* unblackout the MC */
+	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
+	tmp &= ~BLACKOUT_MODE_MASK;
+	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
+	/* allow CPU access */
+	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
+
+	for (i = 0; i < rdev->num_crtc; i++) {
+		if (save->crtc_enabled) {
+			if (ASIC_IS_DCE6(rdev)) {
+				tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
+				tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
+				WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
+			} else {
+				tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
+				tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
+				WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
+			}
+			/* wait for the next frame */
+			frame_count = radeon_get_vblank_counter(rdev, i);
+			for (j = 0; j < rdev->usec_timeout; j++) {
+				if (radeon_get_vblank_counter(rdev, i) != frame_count)
+					break;
+				udelay(1);
+			}
+		}
+	}
+	/* Unlock vga access */
 	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
 	mdelay(1);
 	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
@@ -1553,7 +1610,7 @@ static int evergreen_cp_start(struct radeon_device *rdev)
 	return 0;
 }
 
-int evergreen_cp_resume(struct radeon_device *rdev)
+static int evergreen_cp_resume(struct radeon_device *rdev)
 {
 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
 	u32 tmp;
@@ -2329,22 +2386,10 @@ int evergreen_asic_reset(struct radeon_device *rdev)
 
 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
 {
-	switch (crtc) {
-	case 0:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
-	case 1:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
-	case 2:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
-	case 3:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
-	case 4:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
-	case 5:
-		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
-	default:
+	if (crtc >= rdev->num_crtc)
 		return 0;
-	}
+	else
+		return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
 }
 
 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
@@ -2537,10 +2582,6 @@ int evergreen_irq_set(struct radeon_device *rdev)
 		DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
 		afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
 	}
-	if (rdev->irq.gui_idle) {
-		DRM_DEBUG("gui idle\n");
-		grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
-	}
 
 	if (rdev->family >= CHIP_CAYMAN) {
 		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
@@ -2722,7 +2763,7 @@ static void evergreen_irq_ack(struct radeon_device *rdev)
 	}
 }
 
-void evergreen_irq_disable(struct radeon_device *rdev)
+static void evergreen_irq_disable(struct radeon_device *rdev)
 {
 	r600_disable_interrupts(rdev);
 	/* Wait and acknowledge irq */
@@ -3075,7 +3116,6 @@ restart_ih:
 			break;
 		case 233: /* GUI IDLE */
 			DRM_DEBUG("IH: GUI idle\n");
-			wake_up(&rdev->irq.idle_queue);
 			break;
 		default:
 			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
diff --git a/drivers/gpu/drm/radeon/evergreen_blit_kms.c b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
index 89cb9feb565..057c87b6515 100644
--- a/drivers/gpu/drm/radeon/evergreen_blit_kms.c
+++ b/drivers/gpu/drm/radeon/evergreen_blit_kms.c
@@ -24,9 +24,8 @@
  *     Alex Deucher <alexander.deucher@amd.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #include "evergreend.h"
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index e44a62a07fe..573ed1bc6cf 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -25,7 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "evergreend.h"
 #include "evergreen_reg_safe.h"
@@ -846,6 +846,16 @@ static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p,
 		return -EINVAL;
 	}
 
+	if (!mipmap) {
+		if (llevel) {
+			dev_warn(p->dev, "%s:%i got NULL MIP_ADDRESS relocation\n",
+				 __func__, __LINE__);
+			return -EINVAL;
+		} else {
+			return 0; /* everything's ok */
+		}
+	}
+
 	/* check mipmap size */
 	for (i = 1; i <= llevel; i++) {
 		unsigned w, h, d;
@@ -995,7 +1005,7 @@ static int evergreen_cs_track_check(struct radeon_cs_parser *p)
  * Assume that chunk_ib_index is properly set. Will return -EINVAL
  * if packet is bigger than remaining ib size. or if packets is unknown.
  **/
-int evergreen_cs_packet_parse(struct radeon_cs_parser *p,
+static int evergreen_cs_packet_parse(struct radeon_cs_parser *p,
 			      struct radeon_cs_packet *pkt,
 			      unsigned idx)
 {
@@ -1081,6 +1091,27 @@ static int evergreen_cs_packet_next_reloc(struct radeon_cs_parser *p,
 }
 
 /**
+ * evergreen_cs_packet_next_is_pkt3_nop() - test if the next packet is NOP
+ * @p:		structure holding the parser context.
+ *
+ * Check if the next packet is a relocation packet3.
+ **/
+static bool evergreen_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p)
+{
+	struct radeon_cs_packet p3reloc;
+	int r;
+
+	r = evergreen_cs_packet_parse(p, &p3reloc, p->idx);
+	if (r) {
+		return false;
+	}
+	if (p3reloc.type != PACKET_TYPE3 || p3reloc.opcode != PACKET3_NOP) {
+		return false;
+	}
+	return true;
+}
+
+/**
  * evergreen_cs_packet_next_vline() - parse userspace VLINE packet
  * @parser:		parser structure holding parsing context.
  *
@@ -2330,7 +2361,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
 		for (i = 0; i < (pkt->count / 8); i++) {
 			struct radeon_bo *texture, *mipmap;
 			u32 toffset, moffset;
-			u32 size, offset;
+			u32 size, offset, mip_address, tex_dim;
 
 			switch (G__SQ_CONSTANT_TYPE(radeon_get_ib_value(p, idx+1+(i*8)+7))) {
 			case SQ_TEX_VTX_VALID_TEXTURE:
@@ -2359,14 +2390,28 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
 				}
 				texture = reloc->robj;
 				toffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+
 				/* tex mip base */
-				r = evergreen_cs_packet_next_reloc(p, &reloc);
-				if (r) {
-					DRM_ERROR("bad SET_RESOURCE (tex)\n");
-					return -EINVAL;
+				tex_dim = ib[idx+1+(i*8)+0] & 0x7;
+				mip_address = ib[idx+1+(i*8)+3];
+
+				if ((tex_dim == SQ_TEX_DIM_2D_MSAA || tex_dim == SQ_TEX_DIM_2D_ARRAY_MSAA) &&
+				    !mip_address &&
+				    !evergreen_cs_packet_next_is_pkt3_nop(p)) {
+					/* MIP_ADDRESS should point to FMASK for an MSAA texture.
+					 * It should be 0 if FMASK is disabled. */
+					moffset = 0;
+					mipmap = NULL;
+				} else {
+					r = evergreen_cs_packet_next_reloc(p, &reloc);
+					if (r) {
+						DRM_ERROR("bad SET_RESOURCE (tex)\n");
+						return -EINVAL;
+					}
+					moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+					mipmap = reloc->robj;
 				}
-				moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
-				mipmap = reloc->robj;
+
 				r = evergreen_cs_track_validate_texture(p, texture, mipmap, idx+1+(i*8));
 				if (r)
 					return r;
diff --git a/drivers/gpu/drm/radeon/evergreen_hdmi.c b/drivers/gpu/drm/radeon/evergreen_hdmi.c
index 65c54160028..327c08b5418 100644
--- a/drivers/gpu/drm/radeon/evergreen_hdmi.c
+++ b/drivers/gpu/drm/radeon/evergreen_hdmi.c
@@ -24,8 +24,8 @@
  * Authors: Christian König
  *          Rafał Miłecki
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "evergreend.h"
diff --git a/drivers/gpu/drm/radeon/evergreen_reg.h b/drivers/gpu/drm/radeon/evergreen_reg.h
index 8beac106502..034f4c22e5d 100644
--- a/drivers/gpu/drm/radeon/evergreen_reg.h
+++ b/drivers/gpu/drm/radeon/evergreen_reg.h
@@ -218,6 +218,8 @@
 #define EVERGREEN_CRTC_CONTROL                          0x6e70
 #       define EVERGREEN_CRTC_MASTER_EN                 (1 << 0)
 #       define EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE (1 << 24)
+#define EVERGREEN_CRTC_BLANK_CONTROL                    0x6e74
+#       define EVERGREEN_CRTC_BLANK_DATA_EN             (1 << 8)
 #define EVERGREEN_CRTC_STATUS                           0x6e8c
 #       define EVERGREEN_CRTC_V_BLANK                   (1 << 0)
 #define EVERGREEN_CRTC_STATUS_POSITION                  0x6e90
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 79347855d9b..df542f1a5df 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -87,6 +87,10 @@
 
 #define	CONFIG_MEMSIZE					0x5428
 
+#define	BIF_FB_EN						0x5490
+#define		FB_READ_EN					(1 << 0)
+#define		FB_WRITE_EN					(1 << 1)
+
 #define	CP_COHER_BASE					0x85F8
 #define	CP_STALLED_STAT1			0x8674
 #define	CP_STALLED_STAT2			0x8678
@@ -430,6 +434,9 @@
 #define		NOOFCHAN_MASK					0x00003000
 #define MC_SHARED_CHREMAP					0x2008
 
+#define MC_SHARED_BLACKOUT_CNTL           		0x20ac
+#define		BLACKOUT_MODE_MASK			0x00000007
+
 #define	MC_ARB_RAMCFG					0x2760
 #define		NOOFBANK_SHIFT					0
 #define		NOOFBANK_MASK					0x00000003
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 853800e8582..8bcb554ea0c 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -25,10 +25,10 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "nid.h"
 #include "atom.h"
 #include "ni_reg.h"
@@ -726,7 +726,7 @@ void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
 	WREG32(VM_INVALIDATE_REQUEST, 1);
 }
 
-int cayman_pcie_gart_enable(struct radeon_device *rdev)
+static int cayman_pcie_gart_enable(struct radeon_device *rdev)
 {
 	int i, r;
 
@@ -782,7 +782,7 @@ int cayman_pcie_gart_enable(struct radeon_device *rdev)
 	       (u32)(rdev->dummy_page.addr >> 12));
 	WREG32(VM_CONTEXT1_CNTL2, 0);
 	WREG32(VM_CONTEXT1_CNTL, 0);
-	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 
 	cayman_pcie_gart_tlb_flush(rdev);
@@ -793,7 +793,7 @@ int cayman_pcie_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void cayman_pcie_gart_disable(struct radeon_device *rdev)
+static void cayman_pcie_gart_disable(struct radeon_device *rdev)
 {
 	/* Disable all tables */
 	WREG32(VM_CONTEXT0_CNTL, 0);
@@ -813,7 +813,7 @@ void cayman_pcie_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void cayman_pcie_gart_fini(struct radeon_device *rdev)
+static void cayman_pcie_gart_fini(struct radeon_device *rdev)
 {
 	cayman_pcie_gart_disable(rdev);
 	radeon_gart_table_vram_free(rdev);
@@ -879,12 +879,13 @@ void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
 #endif
 			  (ib->gpu_addr & 0xFFFFFFFC));
 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
-	radeon_ring_write(ring, ib->length_dw | (ib->vm_id << 24));
+	radeon_ring_write(ring, ib->length_dw | 
+			  (ib->vm ? (ib->vm->id << 24) : 0));
 
 	/* flush read cache over gart for this vmid */
 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
 	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
-	radeon_ring_write(ring, ib->vm_id);
+	radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
 	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA);
 	radeon_ring_write(ring, 0xFFFFFFFF);
@@ -1004,7 +1005,7 @@ static void cayman_cp_fini(struct radeon_device *rdev)
 	radeon_scratch_free(rdev, ring->rptr_save_reg);
 }
 
-int cayman_cp_resume(struct radeon_device *rdev)
+static int cayman_cp_resume(struct radeon_device *rdev)
 {
 	static const int ridx[] = {
 		RADEON_RING_TYPE_GFX_INDEX,
@@ -1496,53 +1497,16 @@ void cayman_vm_fini(struct radeon_device *rdev)
 {
 }
 
-int cayman_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
-{
-	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (id << 2), 0);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (id << 2), vm->last_pfn);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
-	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-	/* bits 0-7 are the VM contexts0-7 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << id);
-	return 0;
-}
-
-void cayman_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (vm->id << 2), 0);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (vm->id << 2), 0);
-	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0);
-	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-	/* bits 0-7 are the VM contexts0-7 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
-}
-
-void cayman_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-	if (vm->id == -1)
-		return;
-
-	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-	/* bits 0-7 are the VM contexts0-7 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
-}
-
-#define R600_PTE_VALID     (1 << 0)
+#define R600_ENTRY_VALID   (1 << 0)
 #define R600_PTE_SYSTEM    (1 << 1)
 #define R600_PTE_SNOOPED   (1 << 2)
 #define R600_PTE_READABLE  (1 << 5)
 #define R600_PTE_WRITEABLE (1 << 6)
 
-uint32_t cayman_vm_page_flags(struct radeon_device *rdev,
-			      struct radeon_vm *vm,
-			      uint32_t flags)
+uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
 {
 	uint32_t r600_flags = 0;
-
-	r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
+	r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
 	r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
 	r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
 	if (flags & RADEON_VM_PAGE_SYSTEM) {
@@ -1552,12 +1516,76 @@ uint32_t cayman_vm_page_flags(struct radeon_device *rdev,
 	return r600_flags;
 }
 
-void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
-			unsigned pfn, uint64_t addr, uint32_t flags)
+/**
+ * cayman_vm_set_page - update the page tables using the CP
+ *
+ * @rdev: radeon_device pointer
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using the CP (cayman-si).
+ */
+void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe,
+			uint64_t addr, unsigned count,
+			uint32_t incr, uint32_t flags)
+{
+	struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
+	uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
+	int i;
+
+	radeon_ring_write(ring, PACKET3(PACKET3_ME_WRITE, 1 + count * 2));
+	radeon_ring_write(ring, pe);
+	radeon_ring_write(ring, upper_32_bits(pe) & 0xff);
+	for (i = 0; i < count; ++i) {
+		uint64_t value = 0;
+		if (flags & RADEON_VM_PAGE_SYSTEM) {
+			value = radeon_vm_map_gart(rdev, addr);
+			value &= 0xFFFFFFFFFFFFF000ULL;
+			addr += incr;
+
+		} else if (flags & RADEON_VM_PAGE_VALID) {
+			value = addr;
+			addr += incr;
+		}
+
+		value |= r600_flags;
+		radeon_ring_write(ring, value);
+		radeon_ring_write(ring, upper_32_bits(value));
+	}
+}
+
+/**
+ * cayman_vm_flush - vm flush using the CP
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Update the page table base and flush the VM TLB
+ * using the CP (cayman-si).
+ */
+void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 {
-	void __iomem *ptr = (void *)vm->pt;
+	struct radeon_ring *ring = &rdev->ring[ridx];
+
+	if (vm == NULL)
+		return;
+
+	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (vm->id << 2), 0));
+	radeon_ring_write(ring, 0);
+
+	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (vm->id << 2), 0));
+	radeon_ring_write(ring, vm->last_pfn);
 
-	addr = addr & 0xFFFFFFFFFFFFF000ULL;
-	addr |= flags;
-	writeq(addr, ptr + (pfn * 8));
+	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
+	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
+
+	/* flush hdp cache */
+	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
+	radeon_ring_write(ring, 0x1);
+
+	/* bits 0-7 are the VM contexts0-7 */
+	radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
+	radeon_ring_write(ring, 1 << vm->id);
 }
diff --git a/drivers/gpu/drm/radeon/nid.h b/drivers/gpu/drm/radeon/nid.h
index 870db340d37..2423d1b5d38 100644
--- a/drivers/gpu/drm/radeon/nid.h
+++ b/drivers/gpu/drm/radeon/nid.h
@@ -585,6 +585,7 @@
 #define	PACKET3_SET_CONTEXT_REG_INDIRECT		0x73
 #define	PACKET3_SET_RESOURCE_INDIRECT			0x74
 #define	PACKET3_SET_APPEND_CNT			        0x75
+#define	PACKET3_ME_WRITE				0x7A
 
 #endif
 
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 8d7e33a0b24..376884f1bcd 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -27,9 +27,8 @@
  */
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_asic.h"
@@ -80,10 +79,12 @@ MODULE_FIRMWARE(FIRMWARE_R520);
  */
 void r100_wait_for_vblank(struct radeon_device *rdev, int crtc)
 {
-	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
 	int i;
 
-	if (radeon_crtc->crtc_id == 0) {
+	if (crtc >= rdev->num_crtc)
+		return;
+
+	if (crtc == 0) {
 		if (RREG32(RADEON_CRTC_GEN_CNTL) & RADEON_CRTC_EN) {
 			for (i = 0; i < rdev->usec_timeout; i++) {
 				if (!(RREG32(RADEON_CRTC_STATUS) & RADEON_CRTC_VBLANK_CUR))
@@ -698,9 +699,6 @@ int r100_irq_set(struct radeon_device *rdev)
 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
 		tmp |= RADEON_SW_INT_ENABLE;
 	}
-	if (rdev->irq.gui_idle) {
-		tmp |= RADEON_GUI_IDLE_MASK;
-	}
 	if (rdev->irq.crtc_vblank_int[0] ||
 	    atomic_read(&rdev->irq.pflip[0])) {
 		tmp |= RADEON_CRTC_VBLANK_MASK;
@@ -737,12 +735,6 @@ static uint32_t r100_irq_ack(struct radeon_device *rdev)
 		RADEON_CRTC_VBLANK_STAT | RADEON_CRTC2_VBLANK_STAT |
 		RADEON_FP_DETECT_STAT | RADEON_FP2_DETECT_STAT;
 
-	/* the interrupt works, but the status bit is permanently asserted */
-	if (rdev->irq.gui_idle && radeon_gui_idle(rdev)) {
-		if (!rdev->irq.gui_idle_acked)
-			irq_mask |= RADEON_GUI_IDLE_STAT;
-	}
-
 	if (irqs) {
 		WREG32(RADEON_GEN_INT_STATUS, irqs);
 	}
@@ -754,9 +746,6 @@ int r100_irq_process(struct radeon_device *rdev)
 	uint32_t status, msi_rearm;
 	bool queue_hotplug = false;
 
-	/* reset gui idle ack.  the status bit is broken */
-	rdev->irq.gui_idle_acked = false;
-
 	status = r100_irq_ack(rdev);
 	if (!status) {
 		return IRQ_NONE;
@@ -769,11 +758,6 @@ int r100_irq_process(struct radeon_device *rdev)
 		if (status & RADEON_SW_INT_TEST) {
 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
 		}
-		/* gui idle interrupt */
-		if (status & RADEON_GUI_IDLE_STAT) {
-			rdev->irq.gui_idle_acked = true;
-			wake_up(&rdev->irq.idle_queue);
-		}
 		/* Vertical blank interrupts */
 		if (status & RADEON_CRTC_VBLANK_STAT) {
 			if (rdev->irq.crtc_vblank_int[0]) {
@@ -803,8 +787,6 @@ int r100_irq_process(struct radeon_device *rdev)
 		}
 		status = r100_irq_ack(rdev);
 	}
-	/* reset gui idle ack.  the status bit is broken */
-	rdev->irq.gui_idle_acked = false;
 	if (queue_hotplug)
 		schedule_work(&rdev->hotplug_work);
 	if (rdev->msi_enabled) {
@@ -2530,7 +2512,7 @@ void r100_cs_track_clear(struct radeon_device *rdev, struct r100_cs_track *track
 /*
  * Global GPU functions
  */
-void r100_errata(struct radeon_device *rdev)
+static void r100_errata(struct radeon_device *rdev)
 {
 	rdev->pll_errata = 0;
 
@@ -2545,51 +2527,7 @@ void r100_errata(struct radeon_device *rdev)
 	}
 }
 
-/* Wait for vertical sync on primary CRTC */
-void r100_gpu_wait_for_vsync(struct radeon_device *rdev)
-{
-	uint32_t crtc_gen_cntl, tmp;
-	int i;
-
-	crtc_gen_cntl = RREG32(RADEON_CRTC_GEN_CNTL);
-	if ((crtc_gen_cntl & RADEON_CRTC_DISP_REQ_EN_B) ||
-	    !(crtc_gen_cntl & RADEON_CRTC_EN)) {
-		return;
-	}
-	/* Clear the CRTC_VBLANK_SAVE bit */
-	WREG32(RADEON_CRTC_STATUS, RADEON_CRTC_VBLANK_SAVE_CLEAR);
-	for (i = 0; i < rdev->usec_timeout; i++) {
-		tmp = RREG32(RADEON_CRTC_STATUS);
-		if (tmp & RADEON_CRTC_VBLANK_SAVE) {
-			return;
-		}
-		DRM_UDELAY(1);
-	}
-}
-
-/* Wait for vertical sync on secondary CRTC */
-void r100_gpu_wait_for_vsync2(struct radeon_device *rdev)
-{
-	uint32_t crtc2_gen_cntl, tmp;
-	int i;
-
-	crtc2_gen_cntl = RREG32(RADEON_CRTC2_GEN_CNTL);
-	if ((crtc2_gen_cntl & RADEON_CRTC2_DISP_REQ_EN_B) ||
-	    !(crtc2_gen_cntl & RADEON_CRTC2_EN))
-		return;
-
-	/* Clear the CRTC_VBLANK_SAVE bit */
-	WREG32(RADEON_CRTC2_STATUS, RADEON_CRTC2_VBLANK_SAVE_CLEAR);
-	for (i = 0; i < rdev->usec_timeout; i++) {
-		tmp = RREG32(RADEON_CRTC2_STATUS);
-		if (tmp & RADEON_CRTC2_VBLANK_SAVE) {
-			return;
-		}
-		DRM_UDELAY(1);
-	}
-}
-
-int r100_rbbm_fifo_wait_for_entry(struct radeon_device *rdev, unsigned n)
+static int r100_rbbm_fifo_wait_for_entry(struct radeon_device *rdev, unsigned n)
 {
 	unsigned i;
 	uint32_t tmp;
@@ -2950,7 +2888,7 @@ void r100_vga_set_state(struct radeon_device *rdev, bool state)
 	WREG32(RADEON_CONFIG_CNTL, temp);
 }
 
-void r100_mc_init(struct radeon_device *rdev)
+static void r100_mc_init(struct radeon_device *rdev)
 {
 	u64 base;
 
@@ -3022,7 +2960,7 @@ void r100_pll_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
 	r100_pll_errata_after_data(rdev);
 }
 
-void r100_set_safe_registers(struct radeon_device *rdev)
+static void r100_set_safe_registers(struct radeon_device *rdev)
 {
 	if (ASIC_IS_RN50(rdev)) {
 		rdev->config.r100.reg_safe_bm = rn50_reg_safe_bm;
@@ -3817,9 +3755,10 @@ int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 		return r;
 	}
 	WREG32(scratch, 0xCAFEDEAD);
-	r = radeon_ib_get(rdev, RADEON_RING_TYPE_GFX_INDEX, &ib, 256);
+	r = radeon_ib_get(rdev, RADEON_RING_TYPE_GFX_INDEX, &ib, NULL, 256);
 	if (r) {
-		return r;
+		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
+		goto free_scratch;
 	}
 	ib.ptr[0] = PACKET0(scratch, 0);
 	ib.ptr[1] = 0xDEADBEEF;
@@ -3832,13 +3771,13 @@ int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 	ib.length_dw = 8;
 	r = radeon_ib_schedule(rdev, &ib, NULL);
 	if (r) {
-		radeon_scratch_free(rdev, scratch);
-		radeon_ib_free(rdev, &ib);
-		return r;
+		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
+		goto free_ib;
 	}
 	r = radeon_fence_wait(ib.fence, false);
 	if (r) {
-		return r;
+		DRM_ERROR("radeon: fence wait failed (%d).\n", r);
+		goto free_ib;
 	}
 	for (i = 0; i < rdev->usec_timeout; i++) {
 		tmp = RREG32(scratch);
@@ -3854,8 +3793,10 @@ int r100_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 			  scratch, tmp);
 		r = -EINVAL;
 	}
-	radeon_scratch_free(rdev, scratch);
+free_ib:
 	radeon_ib_free(rdev, &ib);
+free_scratch:
+	radeon_scratch_free(rdev, scratch);
 	return r;
 }
 
@@ -3964,7 +3905,7 @@ static void r100_mc_program(struct radeon_device *rdev)
 	r100_mc_resume(rdev, &save);
 }
 
-void r100_clock_startup(struct radeon_device *rdev)
+static void r100_clock_startup(struct radeon_device *rdev)
 {
 	u32 tmp;
 
diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c
index f0889259eb0..98143a5c5b7 100644
--- a/drivers/gpu/drm/radeon/r200.c
+++ b/drivers/gpu/drm/radeon/r200.c
@@ -25,9 +25,8 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_asic.h"
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index 646a1927dda..d0ba6023a1f 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -33,7 +33,7 @@
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_asic.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "r100_track.h"
 #include "r300d.h"
 #include "rv350d.h"
@@ -296,7 +296,7 @@ void r300_ring_start(struct radeon_device *rdev, struct radeon_ring *ring)
 	radeon_ring_unlock_commit(rdev, ring);
 }
 
-void r300_errata(struct radeon_device *rdev)
+static void r300_errata(struct radeon_device *rdev)
 {
 	rdev->pll_errata = 0;
 
@@ -322,7 +322,7 @@ int r300_mc_wait_for_idle(struct radeon_device *rdev)
 	return -1;
 }
 
-void r300_gpu_init(struct radeon_device *rdev)
+static void r300_gpu_init(struct radeon_device *rdev)
 {
 	uint32_t gb_tile_config, tmp;
 
diff --git a/drivers/gpu/drm/radeon/r300_cmdbuf.c b/drivers/gpu/drm/radeon/r300_cmdbuf.c
index 1fe98b421c9..002ab038d2a 100644
--- a/drivers/gpu/drm/radeon/r300_cmdbuf.c
+++ b/drivers/gpu/drm/radeon/r300_cmdbuf.c
@@ -31,10 +31,9 @@
  *    Nicolai Haehnle <prefect_@gmx.net>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_buffer.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_buffer.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 #include "r300_reg.h"
 
diff --git a/drivers/gpu/drm/radeon/r420.c b/drivers/gpu/drm/radeon/r420.c
index f2f5bf6d339..6fce2eb4dd1 100644
--- a/drivers/gpu/drm/radeon/r420.c
+++ b/drivers/gpu/drm/radeon/r420.c
@@ -27,7 +27,7 @@
  */
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_asic.h"
diff --git a/drivers/gpu/drm/radeon/r520.c b/drivers/gpu/drm/radeon/r520.c
index 079d3c52c08..f795a4e092c 100644
--- a/drivers/gpu/drm/radeon/r520.c
+++ b/drivers/gpu/drm/radeon/r520.c
@@ -25,7 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "atom.h"
@@ -119,7 +119,7 @@ static void r520_vram_get_type(struct radeon_device *rdev)
 		rdev->mc.vram_width *= 2;
 }
 
-void r520_mc_init(struct radeon_device *rdev)
+static void r520_mc_init(struct radeon_device *rdev)
 {
 
 	r520_vram_get_type(rdev);
@@ -131,7 +131,7 @@ void r520_mc_init(struct radeon_device *rdev)
 	radeon_update_bandwidth_info(rdev);
 }
 
-void r520_mc_program(struct radeon_device *rdev)
+static void r520_mc_program(struct radeon_device *rdev)
 {
 	struct rv515_mc_save save;
 
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index d79c639ae73..70c800ff619 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -30,8 +30,8 @@
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
 #include <linux/module.h>
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "radeon_mode.h"
@@ -98,7 +98,7 @@ int r600_debugfs_mc_info_init(struct radeon_device *rdev);
 
 /* r600,rv610,rv630,rv620,rv635,rv670 */
 int r600_mc_wait_for_idle(struct radeon_device *rdev);
-void r600_gpu_init(struct radeon_device *rdev);
+static void r600_gpu_init(struct radeon_device *rdev);
 void r600_fini(struct radeon_device *rdev);
 void r600_irq_disable(struct radeon_device *rdev);
 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
@@ -881,7 +881,7 @@ int r600_pcie_gart_init(struct radeon_device *rdev)
 	return radeon_gart_table_vram_alloc(rdev);
 }
 
-int r600_pcie_gart_enable(struct radeon_device *rdev)
+static int r600_pcie_gart_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int r, i;
@@ -938,7 +938,7 @@ int r600_pcie_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void r600_pcie_gart_disable(struct radeon_device *rdev)
+static void r600_pcie_gart_disable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int i;
@@ -971,14 +971,14 @@ void r600_pcie_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void r600_pcie_gart_fini(struct radeon_device *rdev)
+static void r600_pcie_gart_fini(struct radeon_device *rdev)
 {
 	radeon_gart_fini(rdev);
 	r600_pcie_gart_disable(rdev);
 	radeon_gart_table_vram_free(rdev);
 }
 
-void r600_agp_enable(struct radeon_device *rdev)
+static void r600_agp_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int i;
@@ -1158,7 +1158,7 @@ static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc
 	}
 }
 
-int r600_mc_init(struct radeon_device *rdev)
+static int r600_mc_init(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int chansize, numchan;
@@ -1258,7 +1258,7 @@ void r600_vram_scratch_fini(struct radeon_device *rdev)
  * reset, it's up to the caller to determine if the GPU needs one. We
  * might add an helper function to check that.
  */
-int r600_gpu_soft_reset(struct radeon_device *rdev)
+static int r600_gpu_soft_reset(struct radeon_device *rdev)
 {
 	struct rv515_mc_save save;
 	u32 grbm_busy_mask = S_008010_VC_BUSY(1) | S_008010_VGT_BUSY_NO_DMA(1) |
@@ -1433,7 +1433,7 @@ int r600_count_pipe_bits(uint32_t val)
 	return ret;
 }
 
-void r600_gpu_init(struct radeon_device *rdev)
+static void r600_gpu_init(struct radeon_device *rdev)
 {
 	u32 tiling_config;
 	u32 ramcfg;
@@ -2347,7 +2347,7 @@ void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
 	/* FIXME: implement */
 }
 
-int r600_startup(struct radeon_device *rdev)
+static int r600_startup(struct radeon_device *rdev)
 {
 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
 	int r;
@@ -2635,10 +2635,10 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 		return r;
 	}
 	WREG32(scratch, 0xCAFEDEAD);
-	r = radeon_ib_get(rdev, ring->idx, &ib, 256);
+	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
 	if (r) {
 		DRM_ERROR("radeon: failed to get ib (%d).\n", r);
-		return r;
+		goto free_scratch;
 	}
 	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
 	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
@@ -2646,15 +2646,13 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 	ib.length_dw = 3;
 	r = radeon_ib_schedule(rdev, &ib, NULL);
 	if (r) {
-		radeon_scratch_free(rdev, scratch);
-		radeon_ib_free(rdev, &ib);
 		DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
-		return r;
+		goto free_ib;
 	}
 	r = radeon_fence_wait(ib.fence, false);
 	if (r) {
 		DRM_ERROR("radeon: fence wait failed (%d).\n", r);
-		return r;
+		goto free_ib;
 	}
 	for (i = 0; i < rdev->usec_timeout; i++) {
 		tmp = RREG32(scratch);
@@ -2669,8 +2667,10 @@ int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
 			  scratch, tmp);
 		r = -EINVAL;
 	}
-	radeon_scratch_free(rdev, scratch);
+free_ib:
 	radeon_ib_free(rdev, &ib);
+free_scratch:
+	radeon_scratch_free(rdev, scratch);
 	return r;
 }
 
@@ -3088,10 +3088,6 @@ int r600_irq_set(struct radeon_device *rdev)
 		DRM_DEBUG("r600_irq_set: hdmi 0\n");
 		hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
 	}
-	if (rdev->irq.gui_idle) {
-		DRM_DEBUG("gui idle\n");
-		grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
-	}
 
 	WREG32(CP_INT_CNTL, cp_int_cntl);
 	WREG32(DxMODE_INT_MASK, mode_int);
@@ -3475,7 +3471,6 @@ restart_ih:
 			break;
 		case 233: /* GUI IDLE */
 			DRM_DEBUG("IH: GUI idle\n");
-			wake_up(&rdev->irq.idle_queue);
 			break;
 		default:
 			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
diff --git a/drivers/gpu/drm/radeon/r600_audio.c b/drivers/gpu/drm/radeon/r600_audio.c
index 79b55916cf9..cb03fe22b0a 100644
--- a/drivers/gpu/drm/radeon/r600_audio.c
+++ b/drivers/gpu/drm/radeon/r600_audio.c
@@ -23,7 +23,7 @@
  *
  * Authors: Christian König
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_reg.h"
 #include "radeon_asic.h"
diff --git a/drivers/gpu/drm/radeon/r600_blit.c b/drivers/gpu/drm/radeon/r600_blit.c
index 3c031a48205..77da1f9c0b8 100644
--- a/drivers/gpu/drm/radeon/r600_blit.c
+++ b/drivers/gpu/drm/radeon/r600_blit.c
@@ -23,9 +23,8 @@
  * Authors:
  *     Alex Deucher <alexander.deucher@amd.com>
  */
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 #include "r600_blit_shaders.h"
@@ -489,31 +488,36 @@ set_default_state(drm_radeon_private_t *dev_priv)
 	ADVANCE_RING();
 }
 
-static uint32_t i2f(uint32_t input)
+/* 23 bits of float fractional data */
+#define I2F_FRAC_BITS  23
+#define I2F_MASK ((1 << I2F_FRAC_BITS) - 1)
+
+/*
+ * Converts unsigned integer into 32-bit IEEE floating point representation.
+ * Will be exact from 0 to 2^24.  Above that, we round towards zero
+ * as the fractional bits will not fit in a float.  (It would be better to
+ * round towards even as the fpu does, but that is slower.)
+ */
+__pure uint32_t int2float(uint32_t x)
 {
-	u32 result, i, exponent, fraction;
-
-	if ((input & 0x3fff) == 0)
-		result = 0; /* 0 is a special case */
-	else {
-		exponent = 140; /* exponent biased by 127; */
-		fraction = (input & 0x3fff) << 10; /* cheat and only
-						      handle numbers below 2^^15 */
-		for (i = 0; i < 14; i++) {
-			if (fraction & 0x800000)
-				break;
-			else {
-				fraction = fraction << 1; /* keep
-							     shifting left until top bit = 1 */
-				exponent = exponent - 1;
-			}
-		}
-		result = exponent << 23 | (fraction & 0x7fffff); /* mask
-								    off top bit; assumed 1 */
-	}
-	return result;
-}
+	uint32_t msb, exponent, fraction;
+
+	/* Zero is special */
+	if (!x) return 0;
+
+	/* Get location of the most significant bit */
+	msb = __fls(x);
 
+	/*
+	 * Use a rotate instead of a shift because that works both leftwards
+	 * and rightwards due to the mod(32) behaviour.  This means we don't
+	 * need to check to see if we are above 2^24 or not.
+	 */
+	fraction = ror32(x, (msb - I2F_FRAC_BITS) & 0x1f) & I2F_MASK;
+	exponent = (127 + msb) << I2F_FRAC_BITS;
+
+	return fraction + exponent;
+}
 
 static int r600_nomm_get_vb(struct drm_device *dev)
 {
@@ -632,20 +636,20 @@ r600_blit_copy(struct drm_device *dev,
 				vb = r600_nomm_get_vb_ptr(dev);
 			}
 
-			vb[0] = i2f(dst_x);
+			vb[0] = int2float(dst_x);
 			vb[1] = 0;
-			vb[2] = i2f(src_x);
+			vb[2] = int2float(src_x);
 			vb[3] = 0;
 
-			vb[4] = i2f(dst_x);
-			vb[5] = i2f(h);
-			vb[6] = i2f(src_x);
-			vb[7] = i2f(h);
+			vb[4] = int2float(dst_x);
+			vb[5] = int2float(h);
+			vb[6] = int2float(src_x);
+			vb[7] = int2float(h);
 
-			vb[8] = i2f(dst_x + cur_size);
-			vb[9] = i2f(h);
-			vb[10] = i2f(src_x + cur_size);
-			vb[11] = i2f(h);
+			vb[8] = int2float(dst_x + cur_size);
+			vb[9] = int2float(h);
+			vb[10] = int2float(src_x + cur_size);
+			vb[11] = int2float(h);
 
 			/* src */
 			set_tex_resource(dev_priv, FMT_8,
@@ -721,20 +725,20 @@ r600_blit_copy(struct drm_device *dev,
 				vb = r600_nomm_get_vb_ptr(dev);
 			}
 
-			vb[0] = i2f(dst_x / 4);
+			vb[0] = int2float(dst_x / 4);
 			vb[1] = 0;
-			vb[2] = i2f(src_x / 4);
+			vb[2] = int2float(src_x / 4);
 			vb[3] = 0;
 
-			vb[4] = i2f(dst_x / 4);
-			vb[5] = i2f(h);
-			vb[6] = i2f(src_x / 4);
-			vb[7] = i2f(h);
+			vb[4] = int2float(dst_x / 4);
+			vb[5] = int2float(h);
+			vb[6] = int2float(src_x / 4);
+			vb[7] = int2float(h);
 
-			vb[8] = i2f((dst_x + cur_size) / 4);
-			vb[9] = i2f(h);
-			vb[10] = i2f((src_x + cur_size) / 4);
-			vb[11] = i2f(h);
+			vb[8] = int2float((dst_x + cur_size) / 4);
+			vb[9] = int2float(h);
+			vb[10] = int2float((src_x + cur_size) / 4);
+			vb[11] = int2float(h);
 
 			/* src */
 			set_tex_resource(dev_priv, FMT_8_8_8_8,
@@ -804,20 +808,20 @@ r600_blit_swap(struct drm_device *dev,
 	dx2 = dx + w;
 	dy2 = dy + h;
 
-	vb[0] = i2f(dx);
-	vb[1] = i2f(dy);
-	vb[2] = i2f(sx);
-	vb[3] = i2f(sy);
+	vb[0] = int2float(dx);
+	vb[1] = int2float(dy);
+	vb[2] = int2float(sx);
+	vb[3] = int2float(sy);
 
-	vb[4] = i2f(dx);
-	vb[5] = i2f(dy2);
-	vb[6] = i2f(sx);
-	vb[7] = i2f(sy2);
+	vb[4] = int2float(dx);
+	vb[5] = int2float(dy2);
+	vb[6] = int2float(sx);
+	vb[7] = int2float(sy2);
 
-	vb[8] = i2f(dx2);
-	vb[9] = i2f(dy2);
-	vb[10] = i2f(sx2);
-	vb[11] = i2f(sy2);
+	vb[8] = int2float(dx2);
+	vb[9] = int2float(dy2);
+	vb[10] = int2float(sx2);
+	vb[11] = int2float(sy2);
 
 	switch(cpp) {
 	case 4:
diff --git a/drivers/gpu/drm/radeon/r600_blit_kms.c b/drivers/gpu/drm/radeon/r600_blit_kms.c
index 2bef8549ddf..e082dca6fee 100644
--- a/drivers/gpu/drm/radeon/r600_blit_kms.c
+++ b/drivers/gpu/drm/radeon/r600_blit_kms.c
@@ -23,9 +23,8 @@
  *
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #include "r600d.h"
@@ -455,46 +454,6 @@ set_default_state(struct radeon_device *rdev)
 	radeon_ring_write(ring, sq_stack_resource_mgmt_2);
 }
 
-#define I2F_MAX_BITS 15
-#define I2F_MAX_INPUT  ((1 << I2F_MAX_BITS) - 1)
-#define I2F_SHIFT (24 - I2F_MAX_BITS)
-
-/*
- * Converts unsigned integer into 32-bit IEEE floating point representation.
- * Conversion is not universal and only works for the range from 0
- * to 2^I2F_MAX_BITS-1. Currently we only use it with inputs between
- * 0 and 16384 (inclusive), so I2F_MAX_BITS=15 is enough. If necessary,
- * I2F_MAX_BITS can be increased, but that will add to the loop iterations
- * and slow us down. Conversion is done by shifting the input and counting
- * down until the first 1 reaches bit position 23. The resulting counter
- * and the shifted input are, respectively, the exponent and the fraction.
- * The sign is always zero.
- */
-static uint32_t i2f(uint32_t input)
-{
-	u32 result, i, exponent, fraction;
-
-	WARN_ON_ONCE(input > I2F_MAX_INPUT);
-
-	if ((input & I2F_MAX_INPUT) == 0)
-		result = 0;
-	else {
-		exponent = 126 + I2F_MAX_BITS;
-		fraction = (input & I2F_MAX_INPUT) << I2F_SHIFT;
-
-		for (i = 0; i < I2F_MAX_BITS; i++) {
-			if (fraction & 0x800000)
-				break;
-			else {
-				fraction = fraction << 1;
-				exponent = exponent - 1;
-			}
-		}
-		result = exponent << 23 | (fraction & 0x7fffff);
-	}
-	return result;
-}
-
 int r600_blit_init(struct radeon_device *rdev)
 {
 	u32 obj_size;
@@ -766,14 +725,14 @@ void r600_kms_blit_copy(struct radeon_device *rdev,
 		vb_cpu_addr[3] = 0;
 
 		vb_cpu_addr[4] = 0;
-		vb_cpu_addr[5] = i2f(h);
+		vb_cpu_addr[5] = int2float(h);
 		vb_cpu_addr[6] = 0;
-		vb_cpu_addr[7] = i2f(h);
+		vb_cpu_addr[7] = int2float(h);
 
-		vb_cpu_addr[8] = i2f(w);
-		vb_cpu_addr[9] = i2f(h);
-		vb_cpu_addr[10] = i2f(w);
-		vb_cpu_addr[11] = i2f(h);
+		vb_cpu_addr[8] = int2float(w);
+		vb_cpu_addr[9] = int2float(h);
+		vb_cpu_addr[10] = int2float(w);
+		vb_cpu_addr[11] = int2float(h);
 
 		rdev->r600_blit.primitives.set_tex_resource(rdev, FMT_8_8_8_8,
 							    w, h, w, src_gpu_addr, size_in_bytes);
diff --git a/drivers/gpu/drm/radeon/r600_blit_shaders.h b/drivers/gpu/drm/radeon/r600_blit_shaders.h
index f437d36dd98..2f3ce7a7597 100644
--- a/drivers/gpu/drm/radeon/r600_blit_shaders.h
+++ b/drivers/gpu/drm/radeon/r600_blit_shaders.h
@@ -35,4 +35,5 @@ extern const u32 r6xx_default_state[];
 extern const u32 r6xx_ps_size, r6xx_vs_size;
 extern const u32 r6xx_default_size, r7xx_default_size;
 
+__pure uint32_t int2float(uint32_t x);
 #endif
diff --git a/drivers/gpu/drm/radeon/r600_cp.c b/drivers/gpu/drm/radeon/r600_cp.c
index 75ed17c9611..2514123d2d0 100644
--- a/drivers/gpu/drm/radeon/r600_cp.c
+++ b/drivers/gpu/drm/radeon/r600_cp.c
@@ -28,9 +28,8 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 #define PFP_UCODE_SIZE 576
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index f37676d7f21..211c40252fe 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -26,7 +26,7 @@
  *          Jerome Glisse
  */
 #include <linux/kernel.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "r600d.h"
 #include "r600_reg_safe.h"
@@ -847,7 +847,7 @@ static int r600_cs_track_check(struct radeon_cs_parser *p)
  * Assume that chunk_ib_index is properly set. Will return -EINVAL
  * if packet is bigger than remaining ib size. or if packets is unknown.
  **/
-int r600_cs_packet_parse(struct radeon_cs_parser *p,
+static int r600_cs_packet_parse(struct radeon_cs_parser *p,
 			struct radeon_cs_packet *pkt,
 			unsigned idx)
 {
@@ -2180,7 +2180,8 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
 		}
 		break;
 	case PACKET3_STRMOUT_BASE_UPDATE:
-		if (p->family < CHIP_RV770) {
+		/* RS780 and RS880 also need this */
+		if (p->family < CHIP_RS780) {
 			DRM_ERROR("STRMOUT_BASE_UPDATE only supported on 7xx\n");
 			return -EINVAL;
 		}
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index e3558c3ef24..ff80efe9cb7 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -23,8 +23,8 @@
  *
  * Authors: Christian König
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "r600d.h"
@@ -53,7 +53,7 @@ enum r600_hdmi_iec_status_bits {
 	AUDIO_STATUS_LEVEL        = 0x80
 };
 
-struct radeon_hdmi_acr r600_hdmi_predefined_acr[] = {
+static const struct radeon_hdmi_acr r600_hdmi_predefined_acr[] = {
     /*	     32kHz	  44.1kHz	48kHz    */
     /* Clock      N     CTS      N     CTS      N     CTS */
     {  25174,  4576,  28125,  7007,  31250,  6864,  28125 }, /*  25,20/1.001 MHz */
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 59a15315ae9..b04c06444d8 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -123,6 +123,7 @@ extern int radeon_lockup_timeout;
 #define CAYMAN_RING_TYPE_CP2_INDEX		2
 
 /* hardcode those limit for now */
+#define RADEON_VA_IB_OFFSET			(1 << 20)
 #define RADEON_VA_RESERVED_SIZE			(8 << 20)
 #define RADEON_IB_VM_MAX_SIZE			(64 << 10)
 
@@ -253,6 +254,22 @@ static inline struct radeon_fence *radeon_fence_later(struct radeon_fence *a,
 	}
 }
 
+static inline bool radeon_fence_is_earlier(struct radeon_fence *a,
+					   struct radeon_fence *b)
+{
+	if (!a) {
+		return false;
+	}
+
+	if (!b) {
+		return true;
+	}
+
+	BUG_ON(a->ring != b->ring);
+
+	return a->seq < b->seq;
+}
+
 /*
  * Tiling registers
  */
@@ -275,18 +292,20 @@ struct radeon_mman {
 
 /* bo virtual address in a specific vm */
 struct radeon_bo_va {
-	/* bo list is protected by bo being reserved */
+	/* protected by bo being reserved */
 	struct list_head		bo_list;
-	/* vm list is protected by vm mutex */
-	struct list_head		vm_list;
-	/* constant after initialization */
-	struct radeon_vm		*vm;
-	struct radeon_bo		*bo;
 	uint64_t			soffset;
 	uint64_t			eoffset;
 	uint32_t			flags;
-	struct radeon_fence		*fence;
 	bool				valid;
+	unsigned			ref_count;
+
+	/* protected by vm mutex */
+	struct list_head		vm_list;
+
+	/* constant after initialization */
+	struct radeon_vm		*vm;
+	struct radeon_bo		*bo;
 };
 
 struct radeon_bo {
@@ -566,9 +585,6 @@ struct radeon_irq {
 	atomic_t			pflip[RADEON_MAX_CRTCS];
 	wait_queue_head_t		vblank_queue;
 	bool				hpd[RADEON_MAX_HPD_PINS];
-	bool				gui_idle;
-	bool				gui_idle_acked;
-	wait_queue_head_t		idle_queue;
 	bool				afmt[RADEON_MAX_AFMT_BLOCKS];
 	union radeon_irq_stat_regs	stat_regs;
 };
@@ -583,7 +599,6 @@ void radeon_irq_kms_enable_afmt(struct radeon_device *rdev, int block);
 void radeon_irq_kms_disable_afmt(struct radeon_device *rdev, int block);
 void radeon_irq_kms_enable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
 void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask);
-int radeon_irq_kms_wait_gui_idle(struct radeon_device *rdev);
 
 /*
  * CP & rings.
@@ -596,7 +611,7 @@ struct radeon_ib {
 	uint32_t			*ptr;
 	int				ring;
 	struct radeon_fence		*fence;
-	unsigned			vm_id;
+	struct radeon_vm		*vm;
 	bool				is_const_ib;
 	struct radeon_fence		*sync_to[RADEON_NUM_RINGS];
 	struct radeon_semaphore		*semaphore;
@@ -632,41 +647,38 @@ struct radeon_ring {
 /*
  * VM
  */
+
+/* maximum number of VMIDs */
+#define RADEON_NUM_VM	16
+
+/* defines number of bits in page table versus page directory,
+ * a page is 4KB so we have 12 bits offset, 9 bits in the page
+ * table and the remaining 19 bits are in the page directory */
+#define RADEON_VM_BLOCK_SIZE   9
+
+/* number of entries in page table */
+#define RADEON_VM_PTE_COUNT (1 << RADEON_VM_BLOCK_SIZE)
+
 struct radeon_vm {
 	struct list_head		list;
 	struct list_head		va;
-	int				id;
+	unsigned			id;
 	unsigned			last_pfn;
-	u64				pt_gpu_addr;
-	u64				*pt;
+	u64				pd_gpu_addr;
 	struct radeon_sa_bo		*sa_bo;
 	struct mutex			mutex;
 	/* last fence for cs using this vm */
 	struct radeon_fence		*fence;
-};
-
-struct radeon_vm_funcs {
-	int (*init)(struct radeon_device *rdev);
-	void (*fini)(struct radeon_device *rdev);
-	/* cs mutex must be lock for schedule_ib */
-	int (*bind)(struct radeon_device *rdev, struct radeon_vm *vm, int id);
-	void (*unbind)(struct radeon_device *rdev, struct radeon_vm *vm);
-	void (*tlb_flush)(struct radeon_device *rdev, struct radeon_vm *vm);
-	uint32_t (*page_flags)(struct radeon_device *rdev,
-			       struct radeon_vm *vm,
-			       uint32_t flags);
-	void (*set_page)(struct radeon_device *rdev, struct radeon_vm *vm,
-			unsigned pfn, uint64_t addr, uint32_t flags);
+	/* last flush or NULL if we still need to flush */
+	struct radeon_fence		*last_flush;
 };
 
 struct radeon_vm_manager {
 	struct mutex			lock;
 	struct list_head		lru_vm;
-	uint32_t			use_bitmap;
+	struct radeon_fence		*active[RADEON_NUM_VM];
 	struct radeon_sa_manager	sa_manager;
 	uint32_t			max_pfn;
-	/* fields constant after init */
-	const struct radeon_vm_funcs	*funcs;
 	/* number of VMIDs */
 	unsigned			nvm;
 	/* vram base address for page table entry  */
@@ -738,7 +750,8 @@ struct si_rlc {
 };
 
 int radeon_ib_get(struct radeon_device *rdev, int ring,
-		  struct radeon_ib *ib, unsigned size);
+		  struct radeon_ib *ib, struct radeon_vm *vm,
+		  unsigned size);
 void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib);
 int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
 		       struct radeon_ib *const_ib);
@@ -1131,6 +1144,15 @@ struct radeon_asic {
 		void (*tlb_flush)(struct radeon_device *rdev);
 		int (*set_page)(struct radeon_device *rdev, int i, uint64_t addr);
 	} gart;
+	struct {
+		int (*init)(struct radeon_device *rdev);
+		void (*fini)(struct radeon_device *rdev);
+
+		u32 pt_ring_index;
+		void (*set_page)(struct radeon_device *rdev, uint64_t pe,
+				 uint64_t addr, unsigned count,
+				 uint32_t incr, uint32_t flags);
+	} vm;
 	/* ring specific callbacks */
 	struct {
 		void (*ib_execute)(struct radeon_device *rdev, struct radeon_ib *ib);
@@ -1143,6 +1165,7 @@ struct radeon_asic {
 		int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
 		int (*ib_test)(struct radeon_device *rdev, struct radeon_ring *cp);
 		bool (*is_lockup)(struct radeon_device *rdev, struct radeon_ring *cp);
+		void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 	} ring[RADEON_NUM_RINGS];
 	/* irqs */
 	struct {
@@ -1157,6 +1180,10 @@ struct radeon_asic {
 		u32 (*get_vblank_counter)(struct radeon_device *rdev, int crtc);
 		/* wait for vblank */
 		void (*wait_for_vblank)(struct radeon_device *rdev, int crtc);
+		/* set backlight level */
+		void (*set_backlight_level)(struct radeon_encoder *radeon_encoder, u8 level);
+		/* get backlight level */
+		u8 (*get_backlight_level)(struct radeon_encoder *radeon_encoder);
 	} display;
 	/* copy functions for bo handling */
 	struct {
@@ -1428,6 +1455,56 @@ struct r600_vram_scratch {
 	u64				gpu_addr;
 };
 
+/*
+ * ACPI
+ */
+struct radeon_atif_notification_cfg {
+	bool enabled;
+	int command_code;
+};
+
+struct radeon_atif_notifications {
+	bool display_switch;
+	bool expansion_mode_change;
+	bool thermal_state;
+	bool forced_power_state;
+	bool system_power_state;
+	bool display_conf_change;
+	bool px_gfx_switch;
+	bool brightness_change;
+	bool dgpu_display_event;
+};
+
+struct radeon_atif_functions {
+	bool system_params;
+	bool sbios_requests;
+	bool select_active_disp;
+	bool lid_state;
+	bool get_tv_standard;
+	bool set_tv_standard;
+	bool get_panel_expansion_mode;
+	bool set_panel_expansion_mode;
+	bool temperature_change;
+	bool graphics_device_types;
+};
+
+struct radeon_atif {
+	struct radeon_atif_notifications notifications;
+	struct radeon_atif_functions functions;
+	struct radeon_atif_notification_cfg notification_cfg;
+	struct radeon_encoder *encoder_for_bl;
+};
+
+struct radeon_atcs_functions {
+	bool get_ext_state;
+	bool pcie_perf_req;
+	bool pcie_dev_rdy;
+	bool pcie_bus_width;
+};
+
+struct radeon_atcs {
+	struct radeon_atcs_functions functions;
+};
 
 /*
  * Core structure, functions and helpers.
@@ -1520,6 +1597,9 @@ struct radeon_device {
 	/* virtual memory */
 	struct radeon_vm_manager	vm_manager;
 	struct mutex			gpu_clock_mutex;
+	/* ACPI interface */
+	struct radeon_atif		atif;
+	struct radeon_atcs		atcs;
 };
 
 int radeon_device_init(struct radeon_device *rdev,
@@ -1683,15 +1763,21 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_asic_reset(rdev) (rdev)->asic->asic_reset((rdev))
 #define radeon_gart_tlb_flush(rdev) (rdev)->asic->gart.tlb_flush((rdev))
 #define radeon_gart_set_page(rdev, i, p) (rdev)->asic->gart.set_page((rdev), (i), (p))
+#define radeon_asic_vm_init(rdev) (rdev)->asic->vm.init((rdev))
+#define radeon_asic_vm_fini(rdev) (rdev)->asic->vm.fini((rdev))
+#define radeon_asic_vm_set_page(rdev, pe, addr, count, incr, flags) ((rdev)->asic->vm.set_page((rdev), (pe), (addr), (count), (incr), (flags)))
 #define radeon_ring_start(rdev, r, cp) (rdev)->asic->ring[(r)].ring_start((rdev), (cp))
 #define radeon_ring_test(rdev, r, cp) (rdev)->asic->ring[(r)].ring_test((rdev), (cp))
 #define radeon_ib_test(rdev, r, cp) (rdev)->asic->ring[(r)].ib_test((rdev), (cp))
 #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)].ib_execute((rdev), (ib))
 #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)].ib_parse((rdev), (ib))
 #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)].is_lockup((rdev), (cp))
+#define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)].vm_flush((rdev), (r), (vm))
 #define radeon_irq_set(rdev) (rdev)->asic->irq.set((rdev))
 #define radeon_irq_process(rdev) (rdev)->asic->irq.process((rdev))
 #define radeon_get_vblank_counter(rdev, crtc) (rdev)->asic->display.get_vblank_counter((rdev), (crtc))
+#define radeon_set_backlight_level(rdev, e, l) (rdev)->asic->display.set_backlight_level((e), (l))
+#define radeon_get_backlight_level(rdev, e) (rdev)->asic->display.get_backlight_level((e))
 #define radeon_fence_ring_emit(rdev, r, fence) (rdev)->asic->ring[(r)].emit_fence((rdev), (fence))
 #define radeon_semaphore_ring_emit(rdev, r, cp, semaphore, emit_wait) (rdev)->asic->ring[(r)].emit_semaphore((rdev), (cp), (semaphore), (emit_wait))
 #define radeon_copy_blit(rdev, s, d, np, f) (rdev)->asic->copy.blit((rdev), (s), (d), (np), (f))
@@ -1759,22 +1845,30 @@ int radeon_vm_manager_init(struct radeon_device *rdev);
 void radeon_vm_manager_fini(struct radeon_device *rdev);
 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm);
-int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm);
-void radeon_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm);
+int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm);
+struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
+				       struct radeon_vm *vm, int ring);
+void radeon_vm_fence(struct radeon_device *rdev,
+		     struct radeon_vm *vm,
+		     struct radeon_fence *fence);
+uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr);
 int radeon_vm_bo_update_pte(struct radeon_device *rdev,
 			    struct radeon_vm *vm,
 			    struct radeon_bo *bo,
 			    struct ttm_mem_reg *mem);
 void radeon_vm_bo_invalidate(struct radeon_device *rdev,
 			     struct radeon_bo *bo);
-int radeon_vm_bo_add(struct radeon_device *rdev,
-		     struct radeon_vm *vm,
-		     struct radeon_bo *bo,
-		     uint64_t offset,
-		     uint32_t flags);
+struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
+				       struct radeon_bo *bo);
+struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
+				      struct radeon_vm *vm,
+				      struct radeon_bo *bo);
+int radeon_vm_bo_set_addr(struct radeon_device *rdev,
+			  struct radeon_bo_va *bo_va,
+			  uint64_t offset,
+			  uint32_t flags);
 int radeon_vm_bo_rmv(struct radeon_device *rdev,
-		     struct radeon_vm *vm,
-		     struct radeon_bo *bo);
+		     struct radeon_bo_va *bo_va);
 
 /* audio */
 void r600_audio_update_hdmi(struct work_struct *work);
@@ -1832,12 +1926,14 @@ extern void evergreen_hdmi_setmode(struct drm_encoder *encoder, struct drm_displ
 extern int ni_init_microcode(struct radeon_device *rdev);
 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 
-/* radeon_acpi.c */ 
-#if defined(CONFIG_ACPI) 
-extern int radeon_acpi_init(struct radeon_device *rdev); 
-#else 
-static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; } 
-#endif 
+/* radeon_acpi.c */
+#if defined(CONFIG_ACPI)
+extern int radeon_acpi_init(struct radeon_device *rdev);
+extern void radeon_acpi_fini(struct radeon_device *rdev);
+#else
+static inline int radeon_acpi_init(struct radeon_device *rdev) { return 0; }
+static inline void radeon_acpi_fini(struct radeon_device *rdev) { }
+#endif
 
 #include "radeon_object.h"
 
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 3516a6081dc..b0a5688c67f 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -1,35 +1,118 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
 #include <linux/pci.h>
 #include <linux/acpi.h>
 #include <linux/slab.h>
+#include <linux/power_supply.h>
 #include <acpi/acpi_drivers.h>
 #include <acpi/acpi_bus.h>
+#include <acpi/video.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 #include "radeon.h"
+#include "radeon_acpi.h"
+#include "atom.h"
 
 #include <linux/vga_switcheroo.h>
 
+#define ACPI_AC_CLASS           "ac_adapter"
+
+extern void radeon_pm_acpi_event_handler(struct radeon_device *rdev);
+
+struct atif_verify_interface {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 version;		/* version */
+	u32 notification_mask;	/* supported notifications mask */
+	u32 function_bits;	/* supported functions bit vector */
+} __packed;
+
+struct atif_system_params {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u32 valid_mask;		/* valid flags mask */
+	u32 flags;		/* flags */
+	u8 command_code;	/* notify command code */
+} __packed;
+
+struct atif_sbios_requests {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u32 pending;		/* pending sbios requests */
+	u8 panel_exp_mode;	/* panel expansion mode */
+	u8 thermal_gfx;		/* thermal state: target gfx controller */
+	u8 thermal_state;	/* thermal state: state id (0: exit state, non-0: state) */
+	u8 forced_power_gfx;	/* forced power state: target gfx controller */
+	u8 forced_power_state;	/* forced power state: state id */
+	u8 system_power_src;	/* system power source */
+	u8 backlight_level;	/* panel backlight level (0-255) */
+} __packed;
+
+#define ATIF_NOTIFY_MASK	0x3
+#define ATIF_NOTIFY_NONE	0
+#define ATIF_NOTIFY_81		1
+#define ATIF_NOTIFY_N		2
+
+struct atcs_verify_interface {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 version;		/* version */
+	u32 function_bits;	/* supported functions bit vector */
+} __packed;
+
 /* Call the ATIF method
+ */
+/**
+ * radeon_atif_call - call an ATIF method
  *
- * Note: currently we discard the output
+ * @handle: acpi handle
+ * @function: the ATIF function to execute
+ * @params: ATIF function params
+ *
+ * Executes the requested ATIF function (all asics).
+ * Returns a pointer to the acpi output buffer.
  */
-static int radeon_atif_call(acpi_handle handle)
+static union acpi_object *radeon_atif_call(acpi_handle handle, int function,
+		struct acpi_buffer *params)
 {
 	acpi_status status;
 	union acpi_object atif_arg_elements[2];
 	struct acpi_object_list atif_arg;
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL};
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 
 	atif_arg.count = 2;
 	atif_arg.pointer = &atif_arg_elements[0];
 
 	atif_arg_elements[0].type = ACPI_TYPE_INTEGER;
-	atif_arg_elements[0].integer.value = 0;
-	atif_arg_elements[1].type = ACPI_TYPE_INTEGER;
-	atif_arg_elements[1].integer.value = 0;
+	atif_arg_elements[0].integer.value = function;
+
+	if (params) {
+		atif_arg_elements[1].type = ACPI_TYPE_BUFFER;
+		atif_arg_elements[1].buffer.length = params->length;
+		atif_arg_elements[1].buffer.pointer = params->pointer;
+	} else {
+		/* We need a second fake parameter */
+		atif_arg_elements[1].type = ACPI_TYPE_INTEGER;
+		atif_arg_elements[1].integer.value = 0;
+	}
 
 	status = acpi_evaluate_object(handle, "ATIF", &atif_arg, &buffer);
 
@@ -38,17 +121,434 @@ static int radeon_atif_call(acpi_handle handle)
 		DRM_DEBUG_DRIVER("failed to evaluate ATIF got %s\n",
 				 acpi_format_exception(status));
 		kfree(buffer.pointer);
-		return 1;
+		return NULL;
 	}
 
-	kfree(buffer.pointer);
-	return 0;
+	return buffer.pointer;
+}
+
+/**
+ * radeon_atif_parse_notification - parse supported notifications
+ *
+ * @n: supported notifications struct
+ * @mask: supported notifications mask from ATIF
+ *
+ * Use the supported notifications mask from ATIF function
+ * ATIF_FUNCTION_VERIFY_INTERFACE to determine what notifications
+ * are supported (all asics).
+ */
+static void radeon_atif_parse_notification(struct radeon_atif_notifications *n, u32 mask)
+{
+	n->display_switch = mask & ATIF_DISPLAY_SWITCH_REQUEST_SUPPORTED;
+	n->expansion_mode_change = mask & ATIF_EXPANSION_MODE_CHANGE_REQUEST_SUPPORTED;
+	n->thermal_state = mask & ATIF_THERMAL_STATE_CHANGE_REQUEST_SUPPORTED;
+	n->forced_power_state = mask & ATIF_FORCED_POWER_STATE_CHANGE_REQUEST_SUPPORTED;
+	n->system_power_state = mask & ATIF_SYSTEM_POWER_SOURCE_CHANGE_REQUEST_SUPPORTED;
+	n->display_conf_change = mask & ATIF_DISPLAY_CONF_CHANGE_REQUEST_SUPPORTED;
+	n->px_gfx_switch = mask & ATIF_PX_GFX_SWITCH_REQUEST_SUPPORTED;
+	n->brightness_change = mask & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST_SUPPORTED;
+	n->dgpu_display_event = mask & ATIF_DGPU_DISPLAY_EVENT_SUPPORTED;
+}
+
+/**
+ * radeon_atif_parse_functions - parse supported functions
+ *
+ * @f: supported functions struct
+ * @mask: supported functions mask from ATIF
+ *
+ * Use the supported functions mask from ATIF function
+ * ATIF_FUNCTION_VERIFY_INTERFACE to determine what functions
+ * are supported (all asics).
+ */
+static void radeon_atif_parse_functions(struct radeon_atif_functions *f, u32 mask)
+{
+	f->system_params = mask & ATIF_GET_SYSTEM_PARAMETERS_SUPPORTED;
+	f->sbios_requests = mask & ATIF_GET_SYSTEM_BIOS_REQUESTS_SUPPORTED;
+	f->select_active_disp = mask & ATIF_SELECT_ACTIVE_DISPLAYS_SUPPORTED;
+	f->lid_state = mask & ATIF_GET_LID_STATE_SUPPORTED;
+	f->get_tv_standard = mask & ATIF_GET_TV_STANDARD_FROM_CMOS_SUPPORTED;
+	f->set_tv_standard = mask & ATIF_SET_TV_STANDARD_IN_CMOS_SUPPORTED;
+	f->get_panel_expansion_mode = mask & ATIF_GET_PANEL_EXPANSION_MODE_FROM_CMOS_SUPPORTED;
+	f->set_panel_expansion_mode = mask & ATIF_SET_PANEL_EXPANSION_MODE_IN_CMOS_SUPPORTED;
+	f->temperature_change = mask & ATIF_TEMPERATURE_CHANGE_NOTIFICATION_SUPPORTED;
+	f->graphics_device_types = mask & ATIF_GET_GRAPHICS_DEVICE_TYPES_SUPPORTED;
+}
+
+/**
+ * radeon_atif_verify_interface - verify ATIF
+ *
+ * @handle: acpi handle
+ * @atif: radeon atif struct
+ *
+ * Execute the ATIF_FUNCTION_VERIFY_INTERFACE ATIF function
+ * to initialize ATIF and determine what features are supported
+ * (all asics).
+ * returns 0 on success, error on failure.
+ */
+static int radeon_atif_verify_interface(acpi_handle handle,
+		struct radeon_atif *atif)
+{
+	union acpi_object *info;
+	struct atif_verify_interface output;
+	size_t size;
+	int err = 0;
+
+	info = radeon_atif_call(handle, ATIF_FUNCTION_VERIFY_INTERFACE, NULL);
+	if (!info)
+		return -EIO;
+
+	memset(&output, 0, sizeof(output));
+
+	size = *(u16 *) info->buffer.pointer;
+	if (size < 12) {
+		DRM_INFO("ATIF buffer is too small: %lu\n", size);
+		err = -EINVAL;
+		goto out;
+	}
+	size = min(sizeof(output), size);
+
+	memcpy(&output, info->buffer.pointer, size);
+
+	/* TODO: check version? */
+	DRM_DEBUG_DRIVER("ATIF version %u\n", output.version);
+
+	radeon_atif_parse_notification(&atif->notifications, output.notification_mask);
+	radeon_atif_parse_functions(&atif->functions, output.function_bits);
+
+out:
+	kfree(info);
+	return err;
+}
+
+/**
+ * radeon_atif_get_notification_params - determine notify configuration
+ *
+ * @handle: acpi handle
+ * @n: atif notification configuration struct
+ *
+ * Execute the ATIF_FUNCTION_GET_SYSTEM_PARAMETERS ATIF function
+ * to determine if a notifier is used and if so which one
+ * (all asics).  This is either Notify(VGA, 0x81) or Notify(VGA, n)
+ * where n is specified in the result if a notifier is used.
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atif_get_notification_params(acpi_handle handle,
+		struct radeon_atif_notification_cfg *n)
+{
+	union acpi_object *info;
+	struct atif_system_params params;
+	size_t size;
+	int err = 0;
+
+	info = radeon_atif_call(handle, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS, NULL);
+	if (!info) {
+		err = -EIO;
+		goto out;
+	}
+
+	size = *(u16 *) info->buffer.pointer;
+	if (size < 10) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	memset(&params, 0, sizeof(params));
+	size = min(sizeof(params), size);
+	memcpy(&params, info->buffer.pointer, size);
+
+	DRM_DEBUG_DRIVER("SYSTEM_PARAMS: mask = %#x, flags = %#x\n",
+			params.flags, params.valid_mask);
+	params.flags = params.flags & params.valid_mask;
+
+	if ((params.flags & ATIF_NOTIFY_MASK) == ATIF_NOTIFY_NONE) {
+		n->enabled = false;
+		n->command_code = 0;
+	} else if ((params.flags & ATIF_NOTIFY_MASK) == ATIF_NOTIFY_81) {
+		n->enabled = true;
+		n->command_code = 0x81;
+	} else {
+		if (size < 11) {
+			err = -EINVAL;
+			goto out;
+		}
+		n->enabled = true;
+		n->command_code = params.command_code;
+	}
+
+out:
+	DRM_DEBUG_DRIVER("Notification %s, command code = %#x\n",
+			(n->enabled ? "enabled" : "disabled"),
+			n->command_code);
+	kfree(info);
+	return err;
+}
+
+/**
+ * radeon_atif_get_sbios_requests - get requested sbios event
+ *
+ * @handle: acpi handle
+ * @req: atif sbios request struct
+ *
+ * Execute the ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS ATIF function
+ * to determine what requests the sbios is making to the driver
+ * (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atif_get_sbios_requests(acpi_handle handle,
+		struct atif_sbios_requests *req)
+{
+	union acpi_object *info;
+	size_t size;
+	int count = 0;
+
+	info = radeon_atif_call(handle, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS, NULL);
+	if (!info)
+		return -EIO;
+
+	size = *(u16 *)info->buffer.pointer;
+	if (size < 0xd) {
+		count = -EINVAL;
+		goto out;
+	}
+	memset(req, 0, sizeof(*req));
+
+	size = min(sizeof(*req), size);
+	memcpy(req, info->buffer.pointer, size);
+	DRM_DEBUG_DRIVER("SBIOS pending requests: %#x\n", req->pending);
+
+	count = hweight32(req->pending);
+
+out:
+	kfree(info);
+	return count;
+}
+
+/**
+ * radeon_atif_handler - handle ATIF notify requests
+ *
+ * @rdev: radeon_device pointer
+ * @event: atif sbios request struct
+ *
+ * Checks the acpi event and if it matches an atif event,
+ * handles it.
+ * Returns NOTIFY code
+ */
+int radeon_atif_handler(struct radeon_device *rdev,
+		struct acpi_bus_event *event)
+{
+	struct radeon_atif *atif = &rdev->atif;
+	struct atif_sbios_requests req;
+	acpi_handle handle;
+	int count;
+
+	DRM_DEBUG_DRIVER("event, device_class = %s, type = %#x\n",
+			event->device_class, event->type);
+
+	if (strcmp(event->device_class, ACPI_VIDEO_CLASS) != 0)
+		return NOTIFY_DONE;
+
+	if (!atif->notification_cfg.enabled ||
+			event->type != atif->notification_cfg.command_code)
+		/* Not our event */
+		return NOTIFY_DONE;
+
+	/* Check pending SBIOS requests */
+	handle = DEVICE_ACPI_HANDLE(&rdev->pdev->dev);
+	count = radeon_atif_get_sbios_requests(handle, &req);
+
+	if (count <= 0)
+		return NOTIFY_DONE;
+
+	DRM_DEBUG_DRIVER("ATIF: %d pending SBIOS requests\n", count);
+
+	if (req.pending & ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST) {
+		struct radeon_encoder *enc = atif->encoder_for_bl;
+
+		if (enc) {
+			DRM_DEBUG_DRIVER("Changing brightness to %d\n",
+					req.backlight_level);
+
+			radeon_set_backlight_level(rdev, enc, req.backlight_level);
+
+			if (rdev->is_atom_bios) {
+				struct radeon_encoder_atom_dig *dig = enc->enc_priv;
+				backlight_force_update(dig->bl_dev,
+						       BACKLIGHT_UPDATE_HOTKEY);
+			} else {
+				struct radeon_encoder_lvds *dig = enc->enc_priv;
+				backlight_force_update(dig->bl_dev,
+						       BACKLIGHT_UPDATE_HOTKEY);
+			}
+		}
+	}
+	/* TODO: check other events */
+
+	/* We've handled the event, stop the notifier chain. The ACPI interface
+	 * overloads ACPI_VIDEO_NOTIFY_PROBE, we don't want to send that to
+	 * userspace if the event was generated only to signal a SBIOS
+	 * request.
+	 */
+	return NOTIFY_BAD;
+}
+
+/* Call the ATCS method
+ */
+/**
+ * radeon_atcs_call - call an ATCS method
+ *
+ * @handle: acpi handle
+ * @function: the ATCS function to execute
+ * @params: ATCS function params
+ *
+ * Executes the requested ATCS function (all asics).
+ * Returns a pointer to the acpi output buffer.
+ */
+static union acpi_object *radeon_atcs_call(acpi_handle handle, int function,
+					   struct acpi_buffer *params)
+{
+	acpi_status status;
+	union acpi_object atcs_arg_elements[2];
+	struct acpi_object_list atcs_arg;
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+
+	atcs_arg.count = 2;
+	atcs_arg.pointer = &atcs_arg_elements[0];
+
+	atcs_arg_elements[0].type = ACPI_TYPE_INTEGER;
+	atcs_arg_elements[0].integer.value = function;
+
+	if (params) {
+		atcs_arg_elements[1].type = ACPI_TYPE_BUFFER;
+		atcs_arg_elements[1].buffer.length = params->length;
+		atcs_arg_elements[1].buffer.pointer = params->pointer;
+	} else {
+		/* We need a second fake parameter */
+		atcs_arg_elements[1].type = ACPI_TYPE_INTEGER;
+		atcs_arg_elements[1].integer.value = 0;
+	}
+
+	status = acpi_evaluate_object(handle, "ATCS", &atcs_arg, &buffer);
+
+	/* Fail only if calling the method fails and ATIF is supported */
+	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
+		DRM_DEBUG_DRIVER("failed to evaluate ATCS got %s\n",
+				 acpi_format_exception(status));
+		kfree(buffer.pointer);
+		return NULL;
+	}
+
+	return buffer.pointer;
+}
+
+/**
+ * radeon_atcs_parse_functions - parse supported functions
+ *
+ * @f: supported functions struct
+ * @mask: supported functions mask from ATCS
+ *
+ * Use the supported functions mask from ATCS function
+ * ATCS_FUNCTION_VERIFY_INTERFACE to determine what functions
+ * are supported (all asics).
+ */
+static void radeon_atcs_parse_functions(struct radeon_atcs_functions *f, u32 mask)
+{
+	f->get_ext_state = mask & ATCS_GET_EXTERNAL_STATE_SUPPORTED;
+	f->pcie_perf_req = mask & ATCS_PCIE_PERFORMANCE_REQUEST_SUPPORTED;
+	f->pcie_dev_rdy = mask & ATCS_PCIE_DEVICE_READY_NOTIFICATION_SUPPORTED;
+	f->pcie_bus_width = mask & ATCS_SET_PCIE_BUS_WIDTH_SUPPORTED;
+}
+
+/**
+ * radeon_atcs_verify_interface - verify ATCS
+ *
+ * @handle: acpi handle
+ * @atcs: radeon atcs struct
+ *
+ * Execute the ATCS_FUNCTION_VERIFY_INTERFACE ATCS function
+ * to initialize ATCS and determine what features are supported
+ * (all asics).
+ * returns 0 on success, error on failure.
+ */
+static int radeon_atcs_verify_interface(acpi_handle handle,
+					struct radeon_atcs *atcs)
+{
+	union acpi_object *info;
+	struct atcs_verify_interface output;
+	size_t size;
+	int err = 0;
+
+	info = radeon_atcs_call(handle, ATCS_FUNCTION_VERIFY_INTERFACE, NULL);
+	if (!info)
+		return -EIO;
+
+	memset(&output, 0, sizeof(output));
+
+	size = *(u16 *) info->buffer.pointer;
+	if (size < 8) {
+		DRM_INFO("ATCS buffer is too small: %lu\n", size);
+		err = -EINVAL;
+		goto out;
+	}
+	size = min(sizeof(output), size);
+
+	memcpy(&output, info->buffer.pointer, size);
+
+	/* TODO: check version? */
+	DRM_DEBUG_DRIVER("ATCS version %u\n", output.version);
+
+	radeon_atcs_parse_functions(&atcs->functions, output.function_bits);
+
+out:
+	kfree(info);
+	return err;
+}
+
+/**
+ * radeon_acpi_event - handle notify events
+ *
+ * @nb: notifier block
+ * @val: val
+ * @data: acpi event
+ *
+ * Calls relevant radeon functions in response to various
+ * acpi events.
+ * Returns NOTIFY code
+ */
+static int radeon_acpi_event(struct notifier_block *nb,
+			     unsigned long val,
+			     void *data)
+{
+	struct radeon_device *rdev = container_of(nb, struct radeon_device, acpi_nb);
+	struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
+
+	if (strcmp(entry->device_class, ACPI_AC_CLASS) == 0) {
+		if (power_supply_is_system_supplied() > 0)
+			DRM_DEBUG_DRIVER("pm: AC\n");
+		else
+			DRM_DEBUG_DRIVER("pm: DC\n");
+
+		radeon_pm_acpi_event_handler(rdev);
+	}
+
+	/* Check for pending SBIOS requests */
+	return radeon_atif_handler(rdev, entry);
 }
 
 /* Call all ACPI methods here */
+/**
+ * radeon_acpi_init - init driver acpi support
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Verifies the AMD ACPI interfaces and registers with the acpi
+ * notifier chain (all asics).
+ * Returns 0 on success, error on failure.
+ */
 int radeon_acpi_init(struct radeon_device *rdev)
 {
 	acpi_handle handle;
+	struct radeon_atif *atif = &rdev->atif;
+	struct radeon_atcs *atcs = &rdev->atcs;
 	int ret;
 
 	/* Get the device handle */
@@ -58,11 +558,90 @@ int radeon_acpi_init(struct radeon_device *rdev)
 	if (!ASIC_IS_AVIVO(rdev) || !rdev->bios || !handle)
 		return 0;
 
+	/* Call the ATCS method */
+	ret = radeon_atcs_verify_interface(handle, atcs);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Call to ATCS verify_interface failed: %d\n", ret);
+	}
+
 	/* Call the ATIF method */
-	ret = radeon_atif_call(handle);
-	if (ret)
-		return ret;
+	ret = radeon_atif_verify_interface(handle, atif);
+	if (ret) {
+		DRM_DEBUG_DRIVER("Call to ATIF verify_interface failed: %d\n", ret);
+		goto out;
+	}
+
+	if (atif->notifications.brightness_change) {
+		struct drm_encoder *tmp;
+		struct radeon_encoder *target = NULL;
+
+		/* Find the encoder controlling the brightness */
+		list_for_each_entry(tmp, &rdev->ddev->mode_config.encoder_list,
+				head) {
+			struct radeon_encoder *enc = to_radeon_encoder(tmp);
+
+			if ((enc->devices & (ATOM_DEVICE_LCD_SUPPORT)) &&
+			    enc->enc_priv) {
+				if (rdev->is_atom_bios) {
+					struct radeon_encoder_atom_dig *dig = enc->enc_priv;
+					if (dig->bl_dev) {
+						target = enc;
+						break;
+					}
+				} else {
+					struct radeon_encoder_lvds *dig = enc->enc_priv;
+					if (dig->bl_dev) {
+						target = enc;
+						break;
+					}
+				}
+			}
+		}
+
+		atif->encoder_for_bl = target;
+		if (!target) {
+			/* Brightness change notification is enabled, but we
+			 * didn't find a backlight controller, this should
+			 * never happen.
+			 */
+			DRM_ERROR("Cannot find a backlight controller\n");
+		}
+	}
 
-	return 0;
+	if (atif->functions.sbios_requests && !atif->functions.system_params) {
+		/* XXX check this workraround, if sbios request function is
+		 * present we have to see how it's configured in the system
+		 * params
+		 */
+		atif->functions.system_params = true;
+	}
+
+	if (atif->functions.system_params) {
+		ret = radeon_atif_get_notification_params(handle,
+				&atif->notification_cfg);
+		if (ret) {
+			DRM_DEBUG_DRIVER("Call to GET_SYSTEM_PARAMS failed: %d\n",
+					ret);
+			/* Disable notification */
+			atif->notification_cfg.enabled = false;
+		}
+	}
+
+out:
+	rdev->acpi_nb.notifier_call = radeon_acpi_event;
+	register_acpi_notifier(&rdev->acpi_nb);
+
+	return ret;
 }
 
+/**
+ * radeon_acpi_fini - tear down driver acpi support
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Unregisters with the acpi notifier chain (all asics).
+ */
+void radeon_acpi_fini(struct radeon_device *rdev)
+{
+	unregister_acpi_notifier(&rdev->acpi_nb);
+}
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.h b/drivers/gpu/drm/radeon/radeon_acpi.h
new file mode 100644
index 00000000000..be4af76f213
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_acpi.h
@@ -0,0 +1,445 @@
+/*
+ * Copyright 2012 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef RADEON_ACPI_H
+#define RADEON_ACPI_H
+
+struct radeon_device;
+struct acpi_bus_event;
+
+int radeon_atif_handler(struct radeon_device *rdev,
+		struct acpi_bus_event *event);
+
+/* AMD hw uses four ACPI control methods:
+ * 1. ATIF
+ * ARG0: (ACPI_INTEGER) function code
+ * ARG1: (ACPI_BUFFER) parameter buffer, 256 bytes
+ * OUTPUT: (ACPI_BUFFER) output buffer, 256 bytes
+ * ATIF provides an entry point for the gfx driver to interact with the sbios.
+ * The AMD ACPI notification mechanism uses Notify (VGA, 0x81) or a custom
+ * notification. Which notification is used as indicated by the ATIF Control
+ * Method GET_SYSTEM_PARAMETERS. When the driver receives Notify (VGA, 0x81) or
+ * a custom notification it invokes ATIF Control Method GET_SYSTEM_BIOS_REQUESTS
+ * to identify pending System BIOS requests and associated parameters. For
+ * example, if one of the pending requests is DISPLAY_SWITCH_REQUEST, the driver
+ * will perform display device detection and invoke ATIF Control Method
+ * SELECT_ACTIVE_DISPLAYS.
+ *
+ * 2. ATPX
+ * ARG0: (ACPI_INTEGER) function code
+ * ARG1: (ACPI_BUFFER) parameter buffer, 256 bytes
+ * OUTPUT: (ACPI_BUFFER) output buffer, 256 bytes
+ * ATPX methods are used on PowerXpress systems to handle mux switching and
+ * discrete GPU power control.
+ *
+ * 3. ATRM
+ * ARG0: (ACPI_INTEGER) offset of vbios rom data
+ * ARG1: (ACPI_BUFFER) size of the buffer to fill (up to 4K).
+ * OUTPUT: (ACPI_BUFFER) output buffer
+ * ATRM provides an interfacess to access the discrete GPU vbios image on
+ * PowerXpress systems with multiple GPUs.
+ *
+ * 4. ATCS
+ * ARG0: (ACPI_INTEGER) function code
+ * ARG1: (ACPI_BUFFER) parameter buffer, 256 bytes
+ * OUTPUT: (ACPI_BUFFER) output buffer, 256 bytes
+ * ATCS provides an interface to AMD chipset specific functionality.
+ *
+ */
+/* ATIF */
+#define ATIF_FUNCTION_VERIFY_INTERFACE                             0x0
+/* ARG0: ATIF_FUNCTION_VERIFY_INTERFACE
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - version
+ * DWORD - supported notifications mask
+ * DWORD - supported functions bit vector
+ */
+/* Notifications mask */
+#       define ATIF_DISPLAY_SWITCH_REQUEST_SUPPORTED               (1 << 0)
+#       define ATIF_EXPANSION_MODE_CHANGE_REQUEST_SUPPORTED        (1 << 1)
+#       define ATIF_THERMAL_STATE_CHANGE_REQUEST_SUPPORTED         (1 << 2)
+#       define ATIF_FORCED_POWER_STATE_CHANGE_REQUEST_SUPPORTED    (1 << 3)
+#       define ATIF_SYSTEM_POWER_SOURCE_CHANGE_REQUEST_SUPPORTED   (1 << 4)
+#       define ATIF_DISPLAY_CONF_CHANGE_REQUEST_SUPPORTED          (1 << 5)
+#       define ATIF_PX_GFX_SWITCH_REQUEST_SUPPORTED                (1 << 6)
+#       define ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST_SUPPORTED      (1 << 7)
+#       define ATIF_DGPU_DISPLAY_EVENT_SUPPORTED                   (1 << 8)
+/* supported functions vector */
+#       define ATIF_GET_SYSTEM_PARAMETERS_SUPPORTED               (1 << 0)
+#       define ATIF_GET_SYSTEM_BIOS_REQUESTS_SUPPORTED            (1 << 1)
+#       define ATIF_SELECT_ACTIVE_DISPLAYS_SUPPORTED              (1 << 2)
+#       define ATIF_GET_LID_STATE_SUPPORTED                       (1 << 3)
+#       define ATIF_GET_TV_STANDARD_FROM_CMOS_SUPPORTED           (1 << 4)
+#       define ATIF_SET_TV_STANDARD_IN_CMOS_SUPPORTED             (1 << 5)
+#       define ATIF_GET_PANEL_EXPANSION_MODE_FROM_CMOS_SUPPORTED  (1 << 6)
+#       define ATIF_SET_PANEL_EXPANSION_MODE_IN_CMOS_SUPPORTED    (1 << 7)
+#       define ATIF_TEMPERATURE_CHANGE_NOTIFICATION_SUPPORTED     (1 << 12)
+#       define ATIF_GET_GRAPHICS_DEVICE_TYPES_SUPPORTED           (1 << 14)
+#define ATIF_FUNCTION_GET_SYSTEM_PARAMETERS                        0x1
+/* ARG0: ATIF_FUNCTION_GET_SYSTEM_PARAMETERS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * DWORD - valid flags mask
+ * DWORD - flags
+ *
+ * OR
+ *
+ * WORD  - structure size in bytes (includes size field)
+ * DWORD - valid flags mask
+ * DWORD - flags
+ * BYTE  - notify command code
+ *
+ * flags
+ * bits 1:0:
+ * 0 - Notify(VGA, 0x81) is not used for notification
+ * 1 - Notify(VGA, 0x81) is used for notification
+ * 2 - Notify(VGA, n) is used for notification where
+ * n (0xd0-0xd9) is specified in notify command code.
+ * bit 2:
+ * 1 - lid changes not reported though int10
+ */
+#define ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS                     0x2
+/* ARG0: ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * DWORD - pending sbios requests
+ * BYTE  - panel expansion mode
+ * BYTE  - thermal state: target gfx controller
+ * BYTE  - thermal state: state id (0: exit state, non-0: state)
+ * BYTE  - forced power state: target gfx controller
+ * BYTE  - forced power state: state id
+ * BYTE  - system power source
+ * BYTE  - panel backlight level (0-255)
+ */
+/* pending sbios requests */
+#       define ATIF_DISPLAY_SWITCH_REQUEST                         (1 << 0)
+#       define ATIF_EXPANSION_MODE_CHANGE_REQUEST                  (1 << 1)
+#       define ATIF_THERMAL_STATE_CHANGE_REQUEST                   (1 << 2)
+#       define ATIF_FORCED_POWER_STATE_CHANGE_REQUEST              (1 << 3)
+#       define ATIF_SYSTEM_POWER_SOURCE_CHANGE_REQUEST             (1 << 4)
+#       define ATIF_DISPLAY_CONF_CHANGE_REQUEST                    (1 << 5)
+#       define ATIF_PX_GFX_SWITCH_REQUEST                          (1 << 6)
+#       define ATIF_PANEL_BRIGHTNESS_CHANGE_REQUEST                (1 << 7)
+#       define ATIF_DGPU_DISPLAY_EVENT                             (1 << 8)
+/* panel expansion mode */
+#       define ATIF_PANEL_EXPANSION_DISABLE                        0
+#       define ATIF_PANEL_EXPANSION_FULL                           1
+#       define ATIF_PANEL_EXPANSION_ASPECT                         2
+/* target gfx controller */
+#       define ATIF_TARGET_GFX_SINGLE                              0
+#       define ATIF_TARGET_GFX_PX_IGPU                             1
+#       define ATIF_TARGET_GFX_PX_DGPU                             2
+/* system power source */
+#       define ATIF_POWER_SOURCE_AC                                1
+#       define ATIF_POWER_SOURCE_DC                                2
+#       define ATIF_POWER_SOURCE_RESTRICTED_AC_1                   3
+#       define ATIF_POWER_SOURCE_RESTRICTED_AC_2                   4
+#define ATIF_FUNCTION_SELECT_ACTIVE_DISPLAYS                       0x3
+/* ARG0: ATIF_FUNCTION_SELECT_ACTIVE_DISPLAYS
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - selected displays
+ * WORD  - connected displays
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - selected displays
+ */
+#       define ATIF_LCD1                                           (1 << 0)
+#       define ATIF_CRT1                                           (1 << 1)
+#       define ATIF_TV                                             (1 << 2)
+#       define ATIF_DFP1                                           (1 << 3)
+#       define ATIF_CRT2                                           (1 << 4)
+#       define ATIF_LCD2                                           (1 << 5)
+#       define ATIF_DFP2                                           (1 << 7)
+#       define ATIF_CV                                             (1 << 8)
+#       define ATIF_DFP3                                           (1 << 9)
+#       define ATIF_DFP4                                           (1 << 10)
+#       define ATIF_DFP5                                           (1 << 11)
+#       define ATIF_DFP6                                           (1 << 12)
+#define ATIF_FUNCTION_GET_LID_STATE                                0x4
+/* ARG0: ATIF_FUNCTION_GET_LID_STATE
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - lid state (0: open, 1: closed)
+ *
+ * GET_LID_STATE only works at boot and resume, for general lid
+ * status, use the kernel provided status
+ */
+#define ATIF_FUNCTION_GET_TV_STANDARD_FROM_CMOS                    0x5
+/* ARG0: ATIF_FUNCTION_GET_TV_STANDARD_FROM_CMOS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - 0
+ * BYTE  - TV standard
+ */
+#       define ATIF_TV_STD_NTSC                                    0
+#       define ATIF_TV_STD_PAL                                     1
+#       define ATIF_TV_STD_PALM                                    2
+#       define ATIF_TV_STD_PAL60                                   3
+#       define ATIF_TV_STD_NTSCJ                                   4
+#       define ATIF_TV_STD_PALCN                                   5
+#       define ATIF_TV_STD_PALN                                    6
+#       define ATIF_TV_STD_SCART_RGB                               9
+#define ATIF_FUNCTION_SET_TV_STANDARD_IN_CMOS                      0x6
+/* ARG0: ATIF_FUNCTION_SET_TV_STANDARD_IN_CMOS
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - 0
+ * BYTE  - TV standard
+ * OUTPUT: none
+ */
+#define ATIF_FUNCTION_GET_PANEL_EXPANSION_MODE_FROM_CMOS           0x7
+/* ARG0: ATIF_FUNCTION_GET_PANEL_EXPANSION_MODE_FROM_CMOS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - panel expansion mode
+ */
+#define ATIF_FUNCTION_SET_PANEL_EXPANSION_MODE_IN_CMOS             0x8
+/* ARG0: ATIF_FUNCTION_SET_PANEL_EXPANSION_MODE_IN_CMOS
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - panel expansion mode
+ * OUTPUT: none
+ */
+#define ATIF_FUNCTION_TEMPERATURE_CHANGE_NOTIFICATION              0xD
+/* ARG0: ATIF_FUNCTION_TEMPERATURE_CHANGE_NOTIFICATION
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - gfx controller id
+ * BYTE  - current temperature (degress Celsius)
+ * OUTPUT: none
+ */
+#define ATIF_FUNCTION_GET_GRAPHICS_DEVICE_TYPES                    0xF
+/* ARG0: ATIF_FUNCTION_GET_GRAPHICS_DEVICE_TYPES
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - number of gfx devices
+ * WORD  - device structure size in bytes (excludes device size field)
+ * DWORD - flags         \
+ * WORD  - bus number     } repeated structure
+ * WORD  - device number /
+ */
+/* flags */
+#       define ATIF_PX_REMOVABLE_GRAPHICS_DEVICE                   (1 << 0)
+#       define ATIF_XGP_PORT                                       (1 << 1)
+#       define ATIF_VGA_ENABLED_GRAPHICS_DEVICE                    (1 << 2)
+#       define ATIF_XGP_PORT_IN_DOCK                               (1 << 3)
+
+/* ATPX */
+#define ATPX_FUNCTION_VERIFY_INTERFACE                             0x0
+/* ARG0: ATPX_FUNCTION_VERIFY_INTERFACE
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - version
+ * DWORD - supported functions bit vector
+ */
+/* supported functions vector */
+#       define ATPX_GET_PX_PARAMETERS_SUPPORTED                    (1 << 0)
+#       define ATPX_POWER_CONTROL_SUPPORTED                        (1 << 1)
+#       define ATPX_DISPLAY_MUX_CONTROL_SUPPORTED                  (1 << 2)
+#       define ATPX_I2C_MUX_CONTROL_SUPPORTED                      (1 << 3)
+#       define ATPX_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION_SUPPORTED (1 << 4)
+#       define ATPX_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION_SUPPORTED   (1 << 5)
+#       define ATPX_GET_DISPLAY_CONNECTORS_MAPPING_SUPPORTED       (1 << 7)
+#       define ATPX_GET_DISPLAY_DETECTION_PORTS_SUPPORTED          (1 << 8)
+#define ATPX_FUNCTION_GET_PX_PARAMETERS                            0x1
+/* ARG0: ATPX_FUNCTION_GET_PX_PARAMETERS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * DWORD - valid flags mask
+ * DWORD - flags
+ */
+/* flags */
+#       define ATPX_LVDS_I2C_AVAILABLE_TO_BOTH_GPUS                (1 << 0)
+#       define ATPX_CRT1_I2C_AVAILABLE_TO_BOTH_GPUS                (1 << 1)
+#       define ATPX_DVI1_I2C_AVAILABLE_TO_BOTH_GPUS                (1 << 2)
+#       define ATPX_CRT1_RGB_SIGNAL_MUXED                          (1 << 3)
+#       define ATPX_TV_SIGNAL_MUXED                                (1 << 4)
+#       define ATPX_DFP_SIGNAL_MUXED                               (1 << 5)
+#       define ATPX_SEPARATE_MUX_FOR_I2C                           (1 << 6)
+#       define ATPX_DYNAMIC_PX_SUPPORTED                           (1 << 7)
+#       define ATPX_ACF_NOT_SUPPORTED                              (1 << 8)
+#       define ATPX_FIXED_NOT_SUPPORTED                            (1 << 9)
+#       define ATPX_DYNAMIC_DGPU_POWER_OFF_SUPPORTED               (1 << 10)
+#       define ATPX_DGPU_REQ_POWER_FOR_DISPLAYS                    (1 << 11)
+#define ATPX_FUNCTION_POWER_CONTROL                                0x2
+/* ARG0: ATPX_FUNCTION_POWER_CONTROL
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - dGPU power state (0: power off, 1: power on)
+ * OUTPUT: none
+ */
+#define ATPX_FUNCTION_DISPLAY_MUX_CONTROL                          0x3
+/* ARG0: ATPX_FUNCTION_DISPLAY_MUX_CONTROL
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - display mux control (0: iGPU, 1: dGPU)
+ * OUTPUT: none
+ */
+#       define ATPX_INTEGRATED_GPU                                 0
+#       define ATPX_DISCRETE_GPU                                   1
+#define ATPX_FUNCTION_I2C_MUX_CONTROL                              0x4
+/* ARG0: ATPX_FUNCTION_I2C_MUX_CONTROL
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - i2c/aux/hpd mux control (0: iGPU, 1: dGPU)
+ * OUTPUT: none
+ */
+#define ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION    0x5
+/* ARG0: ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - target gpu (0: iGPU, 1: dGPU)
+ * OUTPUT: none
+ */
+#define ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION      0x6
+/* ARG0: ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - target gpu (0: iGPU, 1: dGPU)
+ * OUTPUT: none
+ */
+#define ATPX_FUNCTION_GET_DISPLAY_CONNECTORS_MAPPING               0x8
+/* ARG0: ATPX_FUNCTION_GET_DISPLAY_CONNECTORS_MAPPING
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - number of display connectors
+ * WORD  - connector structure size in bytes (excludes connector size field)
+ * BYTE  - flags                                                     \
+ * BYTE  - ATIF display vector bit position                           } repeated
+ * BYTE  - adapter id (0: iGPU, 1-n: dGPU ordered by pcie bus number) } structure
+ * WORD  - connector ACPI id                                         /
+ */
+/* flags */
+#       define ATPX_DISPLAY_OUTPUT_SUPPORTED_BY_ADAPTER_ID_DEVICE  (1 << 0)
+#       define ATPX_DISPLAY_HPD_SUPPORTED_BY_ADAPTER_ID_DEVICE     (1 << 1)
+#       define ATPX_DISPLAY_I2C_SUPPORTED_BY_ADAPTER_ID_DEVICE     (1 << 2)
+#define ATPX_FUNCTION_GET_DISPLAY_DETECTION_PORTS                  0x9
+/* ARG0: ATPX_FUNCTION_GET_DISPLAY_DETECTION_PORTS
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - number of HPD/DDC ports
+ * WORD  - port structure size in bytes (excludes port size field)
+ * BYTE  - ATIF display vector bit position \
+ * BYTE  - hpd id                            } reapeated structure
+ * BYTE  - ddc id                           /
+ *
+ * available on A+A systems only
+ */
+/* hpd id */
+#       define ATPX_HPD_NONE                                       0
+#       define ATPX_HPD1                                           1
+#       define ATPX_HPD2                                           2
+#       define ATPX_HPD3                                           3
+#       define ATPX_HPD4                                           4
+#       define ATPX_HPD5                                           5
+#       define ATPX_HPD6                                           6
+/* ddc id */
+#       define ATPX_DDC_NONE                                       0
+#       define ATPX_DDC1                                           1
+#       define ATPX_DDC2                                           2
+#       define ATPX_DDC3                                           3
+#       define ATPX_DDC4                                           4
+#       define ATPX_DDC5                                           5
+#       define ATPX_DDC6                                           6
+#       define ATPX_DDC7                                           7
+#       define ATPX_DDC8                                           8
+
+/* ATCS */
+#define ATCS_FUNCTION_VERIFY_INTERFACE                             0x0
+/* ARG0: ATCS_FUNCTION_VERIFY_INTERFACE
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - version
+ * DWORD - supported functions bit vector
+ */
+/* supported functions vector */
+#       define ATCS_GET_EXTERNAL_STATE_SUPPORTED                   (1 << 0)
+#       define ATCS_PCIE_PERFORMANCE_REQUEST_SUPPORTED             (1 << 1)
+#       define ATCS_PCIE_DEVICE_READY_NOTIFICATION_SUPPORTED       (1 << 2)
+#       define ATCS_SET_PCIE_BUS_WIDTH_SUPPORTED                   (1 << 3)
+#define ATCS_FUNCTION_GET_EXTERNAL_STATE                           0x1
+/* ARG0: ATCS_FUNCTION_GET_EXTERNAL_STATE
+ * ARG1: none
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * DWORD - valid flags mask
+ * DWORD - flags (0: undocked, 1: docked)
+ */
+/* flags */
+#       define ATCS_DOCKED                                         (1 << 0)
+#define ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST                     0x2
+/* ARG0: ATCS_FUNCTION_PCIE_PERFORMANCE_REQUEST
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num)
+ * WORD  - valid flags mask
+ * WORD  - flags
+ * BYTE  - request type
+ * BYTE  - performance request
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - return value
+ */
+/* flags */
+#       define ATCS_ADVERTISE_CAPS                                 (1 << 0)
+#       define ATCS_WAIT_FOR_COMPLETION                            (1 << 1)
+/* request type */
+#       define ATCS_PCIE_LINK_SPEED                                1
+/* performance request */
+#       define ATCS_REMOVE                                         0
+#       define ATCS_FORCE_LOW_POWER                                1
+#       define ATCS_PERF_LEVEL_1                                   2 /* PCIE Gen 1 */
+#       define ATCS_PERF_LEVEL_2                                   3 /* PCIE Gen 2 */
+#       define ATCS_PERF_LEVEL_3                                   4 /* PCIE Gen 3 */
+/* return value */
+#       define ATCS_REQUEST_REFUSED                                1
+#       define ATCS_REQUEST_COMPLETE                               2
+#       define ATCS_REQUEST_IN_PROGRESS                            3
+#define ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION               0x3
+/* ARG0: ATCS_FUNCTION_PCIE_DEVICE_READY_NOTIFICATION
+ * ARG1: none
+ * OUTPUT: none
+ */
+#define ATCS_FUNCTION_SET_PCIE_BUS_WIDTH                           0x4
+/* ARG0: ATCS_FUNCTION_SET_PCIE_BUS_WIDTH
+ * ARG1:
+ * WORD  - structure size in bytes (includes size field)
+ * WORD  - client id (bit 2-0: func num, 7-3: dev num, 15-8: bus num)
+ * BYTE  - number of active lanes
+ * OUTPUT:
+ * WORD  - structure size in bytes (includes size field)
+ * BYTE  - number of active lanes
+ */
+
+#endif
diff --git a/drivers/gpu/drm/radeon/radeon_agp.c b/drivers/gpu/drm/radeon/radeon_agp.c
index bd2f33e5c91..10ea17a6b2a 100644
--- a/drivers/gpu/drm/radeon/radeon_agp.c
+++ b/drivers/gpu/drm/radeon/radeon_agp.c
@@ -24,10 +24,9 @@
  *    Dave Airlie
  *    Jerome Glisse <glisse@freedesktop.org>
  */
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "radeon.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 
 #if __OS_HAS_AGP
 
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 973417c4b01..654520b95ab 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -198,6 +198,8 @@ static struct radeon_asic r100_asic = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &radeon_legacy_set_backlight_level,
+		.get_backlight_level = &radeon_legacy_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -272,6 +274,8 @@ static struct radeon_asic r200_asic = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &radeon_legacy_set_backlight_level,
+		.get_backlight_level = &radeon_legacy_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -346,6 +350,8 @@ static struct radeon_asic r300_asic = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &radeon_legacy_set_backlight_level,
+		.get_backlight_level = &radeon_legacy_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -420,6 +426,8 @@ static struct radeon_asic r300_asic_pcie = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &radeon_legacy_set_backlight_level,
+		.get_backlight_level = &radeon_legacy_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -494,6 +502,8 @@ static struct radeon_asic r420_asic = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -568,6 +578,8 @@ static struct radeon_asic rs400_asic = {
 		.bandwidth_update = &r100_bandwidth_update,
 		.get_vblank_counter = &r100_get_vblank_counter,
 		.wait_for_vblank = &r100_wait_for_vblank,
+		.set_backlight_level = &radeon_legacy_set_backlight_level,
+		.get_backlight_level = &radeon_legacy_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -642,6 +654,8 @@ static struct radeon_asic rs600_asic = {
 		.bandwidth_update = &rs600_bandwidth_update,
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -716,6 +730,8 @@ static struct radeon_asic rs690_asic = {
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.bandwidth_update = &rs690_bandwidth_update,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -790,6 +806,8 @@ static struct radeon_asic rv515_asic = {
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.bandwidth_update = &rv515_bandwidth_update,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -864,6 +882,8 @@ static struct radeon_asic r520_asic = {
 		.bandwidth_update = &rv515_bandwidth_update,
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r100_copy_blit,
@@ -937,6 +957,8 @@ static struct radeon_asic r600_asic = {
 		.bandwidth_update = &rv515_bandwidth_update,
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1010,6 +1032,8 @@ static struct radeon_asic rs780_asic = {
 		.bandwidth_update = &rs690_bandwidth_update,
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1083,6 +1107,8 @@ static struct radeon_asic rv770_asic = {
 		.bandwidth_update = &rv515_bandwidth_update,
 		.get_vblank_counter = &rs600_get_vblank_counter,
 		.wait_for_vblank = &avivo_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1156,6 +1182,8 @@ static struct radeon_asic evergreen_asic = {
 		.bandwidth_update = &evergreen_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1229,6 +1257,8 @@ static struct radeon_asic sumo_asic = {
 		.bandwidth_update = &evergreen_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1302,6 +1332,8 @@ static struct radeon_asic btc_asic = {
 		.bandwidth_update = &evergreen_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1325,7 +1357,7 @@ static struct radeon_asic btc_asic = {
 		.misc = &evergreen_pm_misc,
 		.prepare = &evergreen_pm_prepare,
 		.finish = &evergreen_pm_finish,
-		.init_profile = &r600_pm_init_profile,
+		.init_profile = &btc_pm_init_profile,
 		.get_dynpm_state = &r600_pm_get_dynpm_state,
 		.get_engine_clock = &radeon_atom_get_engine_clock,
 		.set_engine_clock = &radeon_atom_set_engine_clock,
@@ -1342,16 +1374,6 @@ static struct radeon_asic btc_asic = {
 	},
 };
 
-static const struct radeon_vm_funcs cayman_vm_funcs = {
-	.init = &cayman_vm_init,
-	.fini = &cayman_vm_fini,
-	.bind = &cayman_vm_bind,
-	.unbind = &cayman_vm_unbind,
-	.tlb_flush = &cayman_vm_tlb_flush,
-	.page_flags = &cayman_vm_page_flags,
-	.set_page = &cayman_vm_set_page,
-};
-
 static struct radeon_asic cayman_asic = {
 	.init = &cayman_init,
 	.fini = &cayman_fini,
@@ -1366,6 +1388,12 @@ static struct radeon_asic cayman_asic = {
 		.tlb_flush = &cayman_pcie_gart_tlb_flush,
 		.set_page = &rs600_gart_set_page,
 	},
+	.vm = {
+		.init = &cayman_vm_init,
+		.fini = &cayman_vm_fini,
+		.pt_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+		.set_page = &cayman_vm_set_page,
+	},
 	.ring = {
 		[RADEON_RING_TYPE_GFX_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1376,6 +1404,7 @@ static struct radeon_asic cayman_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP1_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1386,6 +1415,7 @@ static struct radeon_asic cayman_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP2_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1396,6 +1426,7 @@ static struct radeon_asic cayman_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		}
 	},
 	.irq = {
@@ -1406,6 +1437,8 @@ static struct radeon_asic cayman_asic = {
 		.bandwidth_update = &evergreen_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1429,7 +1462,7 @@ static struct radeon_asic cayman_asic = {
 		.misc = &evergreen_pm_misc,
 		.prepare = &evergreen_pm_prepare,
 		.finish = &evergreen_pm_finish,
-		.init_profile = &r600_pm_init_profile,
+		.init_profile = &btc_pm_init_profile,
 		.get_dynpm_state = &r600_pm_get_dynpm_state,
 		.get_engine_clock = &radeon_atom_get_engine_clock,
 		.set_engine_clock = &radeon_atom_set_engine_clock,
@@ -1460,6 +1493,12 @@ static struct radeon_asic trinity_asic = {
 		.tlb_flush = &cayman_pcie_gart_tlb_flush,
 		.set_page = &rs600_gart_set_page,
 	},
+	.vm = {
+		.init = &cayman_vm_init,
+		.fini = &cayman_vm_fini,
+		.pt_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+		.set_page = &cayman_vm_set_page,
+	},
 	.ring = {
 		[RADEON_RING_TYPE_GFX_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1470,6 +1509,7 @@ static struct radeon_asic trinity_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP1_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1480,6 +1520,7 @@ static struct radeon_asic trinity_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP2_INDEX] = {
 			.ib_execute = &cayman_ring_ib_execute,
@@ -1490,6 +1531,7 @@ static struct radeon_asic trinity_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &evergreen_gpu_is_lockup,
+			.vm_flush = &cayman_vm_flush,
 		}
 	},
 	.irq = {
@@ -1500,6 +1542,8 @@ static struct radeon_asic trinity_asic = {
 		.bandwidth_update = &dce6_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = &r600_copy_blit,
@@ -1540,16 +1584,6 @@ static struct radeon_asic trinity_asic = {
 	},
 };
 
-static const struct radeon_vm_funcs si_vm_funcs = {
-	.init = &si_vm_init,
-	.fini = &si_vm_fini,
-	.bind = &si_vm_bind,
-	.unbind = &si_vm_unbind,
-	.tlb_flush = &si_vm_tlb_flush,
-	.page_flags = &cayman_vm_page_flags,
-	.set_page = &cayman_vm_set_page,
-};
-
 static struct radeon_asic si_asic = {
 	.init = &si_init,
 	.fini = &si_fini,
@@ -1564,6 +1598,12 @@ static struct radeon_asic si_asic = {
 		.tlb_flush = &si_pcie_gart_tlb_flush,
 		.set_page = &rs600_gart_set_page,
 	},
+	.vm = {
+		.init = &si_vm_init,
+		.fini = &si_vm_fini,
+		.pt_ring_index = RADEON_RING_TYPE_GFX_INDEX,
+		.set_page = &si_vm_set_page,
+	},
 	.ring = {
 		[RADEON_RING_TYPE_GFX_INDEX] = {
 			.ib_execute = &si_ring_ib_execute,
@@ -1574,6 +1614,7 @@ static struct radeon_asic si_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &si_gpu_is_lockup,
+			.vm_flush = &si_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP1_INDEX] = {
 			.ib_execute = &si_ring_ib_execute,
@@ -1584,6 +1625,7 @@ static struct radeon_asic si_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &si_gpu_is_lockup,
+			.vm_flush = &si_vm_flush,
 		},
 		[CAYMAN_RING_TYPE_CP2_INDEX] = {
 			.ib_execute = &si_ring_ib_execute,
@@ -1594,6 +1636,7 @@ static struct radeon_asic si_asic = {
 			.ring_test = &r600_ring_test,
 			.ib_test = &r600_ib_test,
 			.is_lockup = &si_gpu_is_lockup,
+			.vm_flush = &si_vm_flush,
 		}
 	},
 	.irq = {
@@ -1604,6 +1647,8 @@ static struct radeon_asic si_asic = {
 		.bandwidth_update = &dce6_bandwidth_update,
 		.get_vblank_counter = &evergreen_get_vblank_counter,
 		.wait_for_vblank = &dce4_wait_for_vblank,
+		.set_backlight_level = &atombios_set_backlight_level,
+		.get_backlight_level = &atombios_get_backlight_level,
 	},
 	.copy = {
 		.blit = NULL,
@@ -1697,6 +1742,7 @@ int radeon_asic_init(struct radeon_device *rdev)
 			rdev->asic->pm.set_engine_clock = &radeon_legacy_set_engine_clock;
 			rdev->asic->pm.get_memory_clock = &radeon_legacy_get_memory_clock;
 			rdev->asic->pm.set_memory_clock = NULL;
+			rdev->asic->display.set_backlight_level = &radeon_legacy_set_backlight_level;
 		}
 		break;
 	case CHIP_RS400:
@@ -1769,13 +1815,11 @@ int radeon_asic_init(struct radeon_device *rdev)
 		rdev->asic = &cayman_asic;
 		/* set num crtcs */
 		rdev->num_crtc = 6;
-		rdev->vm_manager.funcs = &cayman_vm_funcs;
 		break;
 	case CHIP_ARUBA:
 		rdev->asic = &trinity_asic;
 		/* set num crtcs */
 		rdev->num_crtc = 4;
-		rdev->vm_manager.funcs = &cayman_vm_funcs;
 		break;
 	case CHIP_TAHITI:
 	case CHIP_PITCAIRN:
@@ -1783,7 +1827,6 @@ int radeon_asic_init(struct radeon_device *rdev)
 		rdev->asic = &si_asic;
 		/* set num crtcs */
 		rdev->num_crtc = 6;
-		rdev->vm_manager.funcs = &si_vm_funcs;
 		break;
 	default:
 		/* FIXME: not supported yet */
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 18c38d14c8c..5e3a0e5c6be 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -42,6 +42,12 @@ uint32_t radeon_atom_get_memory_clock(struct radeon_device *rdev);
 void radeon_atom_set_memory_clock(struct radeon_device *rdev, uint32_t mem_clock);
 void radeon_atom_set_clock_gating(struct radeon_device *rdev, int enable);
 
+void atombios_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level);
+u8 atombios_get_backlight_level(struct radeon_encoder *radeon_encoder);
+void radeon_legacy_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level);
+u8 radeon_legacy_get_backlight_level(struct radeon_encoder *radeon_encoder);
+
+
 /*
  * r100,rv100,rs100,rv200,rs200
  */
@@ -389,6 +395,7 @@ void r700_cp_fini(struct radeon_device *rdev);
 struct evergreen_mc_save {
 	u32 vga_render_control;
 	u32 vga_hdp_control;
+	bool crtc_enabled[RADEON_MAX_CRTCS];
 };
 
 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev);
@@ -413,6 +420,7 @@ extern void evergreen_pm_misc(struct radeon_device *rdev);
 extern void evergreen_pm_prepare(struct radeon_device *rdev);
 extern void evergreen_pm_finish(struct radeon_device *rdev);
 extern void sumo_pm_init_profile(struct radeon_device *rdev);
+extern void btc_pm_init_profile(struct radeon_device *rdev);
 extern void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc);
 extern u32 evergreen_page_flip(struct radeon_device *rdev, int crtc, u64 crtc_base);
 extern void evergreen_post_page_flip(struct radeon_device *rdev, int crtc);
@@ -435,14 +443,11 @@ int cayman_asic_reset(struct radeon_device *rdev);
 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
 int cayman_vm_init(struct radeon_device *rdev);
 void cayman_vm_fini(struct radeon_device *rdev);
-int cayman_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id);
-void cayman_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm);
-void cayman_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm);
-uint32_t cayman_vm_page_flags(struct radeon_device *rdev,
-			      struct radeon_vm *vm,
-			      uint32_t flags);
-void cayman_vm_set_page(struct radeon_device *rdev, struct radeon_vm *vm,
-			unsigned pfn, uint64_t addr, uint32_t flags);
+void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
+uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags);
+void cayman_vm_set_page(struct radeon_device *rdev, uint64_t pe,
+			uint64_t addr, unsigned count,
+			uint32_t incr, uint32_t flags);
 int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 
 /* DCE6 - SI */
@@ -465,9 +470,10 @@ int si_irq_set(struct radeon_device *rdev);
 int si_irq_process(struct radeon_device *rdev);
 int si_vm_init(struct radeon_device *rdev);
 void si_vm_fini(struct radeon_device *rdev);
-int si_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id);
-void si_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm);
-void si_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm);
+void si_vm_set_page(struct radeon_device *rdev, uint64_t pe,
+		    uint64_t addr, unsigned count,
+		    uint32_t incr, uint32_t flags);
+void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 uint64_t si_get_gpu_clock(struct radeon_device *rdev);
 
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index d67d4f3eb6f..f22eb571352 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -23,8 +23,8 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #include "atom.h"
@@ -1254,6 +1254,10 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
 		if (rdev->clock.max_pixel_clock == 0)
 			rdev->clock.max_pixel_clock = 40000;
 
+		/* not technically a clock, but... */
+		rdev->mode_info.firmware_flags =
+			le16_to_cpu(firmware_info->info.usFirmwareCapability.susAccess);
+
 		return true;
 	}
 
@@ -2005,7 +2009,8 @@ static int radeon_atombios_parse_power_table_1_3(struct radeon_device *rdev)
 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
 
 	/* add the i2c bus for thermal/fan chip */
-	if (power_info->info.ucOverdriveThermalController > 0) {
+	if ((power_info->info.ucOverdriveThermalController > 0) &&
+	    (power_info->info.ucOverdriveThermalController < ARRAY_SIZE(thermal_controller_names))) {
 		DRM_INFO("Possible %s thermal controller at 0x%02x\n",
 			 thermal_controller_names[power_info->info.ucOverdriveThermalController],
 			 power_info->info.ucOverdriveControllerAddress >> 1);
@@ -2209,7 +2214,7 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r
 			   (controller->ucType ==
 			    ATOM_PP_THERMALCONTROLLER_EMC2103_WITH_INTERNAL)) {
 			DRM_INFO("Special thermal controller config\n");
-		} else {
+		} else if (controller->ucType < ARRAY_SIZE(pp_lib_thermal_controller_names)) {
 			DRM_INFO("Possible %s thermal controller at 0x%02x %s fan control\n",
 				 pp_lib_thermal_controller_names[controller->ucType],
 				 controller->ucI2cAddress >> 1,
@@ -2224,6 +2229,12 @@ static void radeon_atombios_add_pplib_thermal_controller(struct radeon_device *r
 				strlcpy(info.type, name, sizeof(info.type));
 				i2c_new_device(&rdev->pm.i2c_bus->adapter, &info);
 			}
+		} else {
+			DRM_INFO("Unknown thermal controller type %d at 0x%02x %s fan control\n",
+				 controller->ucType,
+				 controller->ucI2cAddress >> 1,
+				 (controller->ucFanParameters &
+				  ATOM_PP_FANPARAMETERS_NOFAN) ? "without" : "with");
 		}
 	}
 }
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index 2a2cf0b88a2..582e99449c1 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -12,30 +12,62 @@
 #include <acpi/acpi_bus.h>
 #include <linux/pci.h>
 
-#define ATPX_VERSION 0
-#define ATPX_GPU_PWR 2
-#define ATPX_MUX_SELECT 3
-#define ATPX_I2C_MUX_SELECT 4
-#define ATPX_SWITCH_START 5
-#define ATPX_SWITCH_END 6
-
-#define ATPX_INTEGRATED 0
-#define ATPX_DISCRETE 1
+#include "radeon_acpi.h"
+
+struct radeon_atpx_functions {
+	bool px_params;
+	bool power_cntl;
+	bool disp_mux_cntl;
+	bool i2c_mux_cntl;
+	bool switch_start;
+	bool switch_end;
+	bool disp_connectors_mapping;
+	bool disp_detetion_ports;
+};
 
-#define ATPX_MUX_IGD 0
-#define ATPX_MUX_DISCRETE 1
+struct radeon_atpx {
+	acpi_handle handle;
+	struct radeon_atpx_functions functions;
+};
 
 static struct radeon_atpx_priv {
 	bool atpx_detected;
 	/* handle for device - and atpx */
 	acpi_handle dhandle;
-	acpi_handle atpx_handle;
+	struct radeon_atpx atpx;
 } radeon_atpx_priv;
 
-static int radeon_atpx_get_version(acpi_handle handle)
+struct atpx_verify_interface {
+	u16 size;		/* structure size in bytes (includes size field) */
+	u16 version;		/* version */
+	u32 function_bits;	/* supported functions bit vector */
+} __packed;
+
+struct atpx_power_control {
+	u16 size;
+	u8 dgpu_state;
+} __packed;
+
+struct atpx_mux {
+	u16 size;
+	u16 mux;
+} __packed;
+
+/**
+ * radeon_atpx_call - call an ATPX method
+ *
+ * @handle: acpi handle
+ * @function: the ATPX function to execute
+ * @params: ATPX function params
+ *
+ * Executes the requested ATPX function (all asics).
+ * Returns a pointer to the acpi output buffer.
+ */
+static union acpi_object *radeon_atpx_call(acpi_handle handle, int function,
+					   struct acpi_buffer *params)
 {
 	acpi_status status;
-	union acpi_object atpx_arg_elements[2], *obj;
+	union acpi_object atpx_arg_elements[2];
 	struct acpi_object_list atpx_arg;
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 
@@ -43,99 +75,292 @@ static int radeon_atpx_get_version(acpi_handle handle)
 	atpx_arg.pointer = &atpx_arg_elements[0];
 
 	atpx_arg_elements[0].type = ACPI_TYPE_INTEGER;
-	atpx_arg_elements[0].integer.value = ATPX_VERSION;
+	atpx_arg_elements[0].integer.value = function;
+
+	if (params) {
+		atpx_arg_elements[1].type = ACPI_TYPE_BUFFER;
+		atpx_arg_elements[1].buffer.length = params->length;
+		atpx_arg_elements[1].buffer.pointer = params->pointer;
+	} else {
+		/* We need a second fake parameter */
+		atpx_arg_elements[1].type = ACPI_TYPE_INTEGER;
+		atpx_arg_elements[1].integer.value = 0;
+	}
 
-	atpx_arg_elements[1].type = ACPI_TYPE_INTEGER;
-	atpx_arg_elements[1].integer.value = ATPX_VERSION;
+	status = acpi_evaluate_object(handle, "ATPX", &atpx_arg, &buffer);
 
-	status = acpi_evaluate_object(handle, NULL, &atpx_arg, &buffer);
-	if (ACPI_FAILURE(status)) {
-		printk("%s: failed to call ATPX: %s\n", __func__, acpi_format_exception(status));
-		return -ENOSYS;
+	/* Fail only if calling the method fails and ATPX is supported */
+	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
+		printk("failed to evaluate ATPX got %s\n",
+		       acpi_format_exception(status));
+		kfree(buffer.pointer);
+		return NULL;
 	}
-	obj = (union acpi_object *)buffer.pointer;
-	if (obj && (obj->type == ACPI_TYPE_BUFFER))
-		printk(KERN_INFO "radeon atpx: version is %d\n", *((u8 *)(obj->buffer.pointer) + 2));
-	kfree(buffer.pointer);
-	return 0;
+
+	return buffer.pointer;
 }
 
-static int radeon_atpx_execute(acpi_handle handle, int cmd_id, u16 value)
+/**
+ * radeon_atpx_parse_functions - parse supported functions
+ *
+ * @f: supported functions struct
+ * @mask: supported functions mask from ATPX
+ *
+ * Use the supported functions mask from ATPX function
+ * ATPX_FUNCTION_VERIFY_INTERFACE to determine what functions
+ * are supported (all asics).
+ */
+static void radeon_atpx_parse_functions(struct radeon_atpx_functions *f, u32 mask)
 {
-	acpi_status status;
-	union acpi_object atpx_arg_elements[2];
-	struct acpi_object_list atpx_arg;
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	uint8_t buf[4] = {0};
-
-	if (!handle)
-		return -EINVAL;
-
-	atpx_arg.count = 2;
-	atpx_arg.pointer = &atpx_arg_elements[0];
+	f->px_params = mask & ATPX_GET_PX_PARAMETERS_SUPPORTED;
+	f->power_cntl = mask & ATPX_POWER_CONTROL_SUPPORTED;
+	f->disp_mux_cntl = mask & ATPX_DISPLAY_MUX_CONTROL_SUPPORTED;
+	f->i2c_mux_cntl = mask & ATPX_I2C_MUX_CONTROL_SUPPORTED;
+	f->switch_start = mask & ATPX_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION_SUPPORTED;
+	f->switch_end = mask & ATPX_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION_SUPPORTED;
+	f->disp_connectors_mapping = mask & ATPX_GET_DISPLAY_CONNECTORS_MAPPING_SUPPORTED;
+	f->disp_detetion_ports = mask & ATPX_GET_DISPLAY_DETECTION_PORTS_SUPPORTED;
+}
 
-	atpx_arg_elements[0].type = ACPI_TYPE_INTEGER;
-	atpx_arg_elements[0].integer.value = cmd_id;
+/**
+ * radeon_atpx_verify_interface - verify ATPX
+ *
+ * @handle: acpi handle
+ * @atpx: radeon atpx struct
+ *
+ * Execute the ATPX_FUNCTION_VERIFY_INTERFACE ATPX function
+ * to initialize ATPX and determine what features are supported
+ * (all asics).
+ * returns 0 on success, error on failure.
+ */
+static int radeon_atpx_verify_interface(struct radeon_atpx *atpx)
+{
+	union acpi_object *info;
+	struct atpx_verify_interface output;
+	size_t size;
+	int err = 0;
+
+	info = radeon_atpx_call(atpx->handle, ATPX_FUNCTION_VERIFY_INTERFACE, NULL);
+	if (!info)
+		return -EIO;
+
+	memset(&output, 0, sizeof(output));
+
+	size = *(u16 *) info->buffer.pointer;
+	if (size < 8) {
+		printk("ATPX buffer is too small: %lu\n", size);
+		err = -EINVAL;
+		goto out;
+	}
+	size = min(sizeof(output), size);
 
-	buf[2] = value & 0xff;
-	buf[3] = (value >> 8) & 0xff;
+	memcpy(&output, info->buffer.pointer, size);
 
-	atpx_arg_elements[1].type = ACPI_TYPE_BUFFER;
-	atpx_arg_elements[1].buffer.length = 4;
-	atpx_arg_elements[1].buffer.pointer = buf;
+	/* TODO: check version? */
+	printk("ATPX version %u\n", output.version);
 
-	status = acpi_evaluate_object(handle, NULL, &atpx_arg, &buffer);
-	if (ACPI_FAILURE(status)) {
-		printk("%s: failed to call ATPX: %s\n", __func__, acpi_format_exception(status));
-		return -ENOSYS;
-	}
-	kfree(buffer.pointer);
+	radeon_atpx_parse_functions(&atpx->functions, output.function_bits);
 
-	return 0;
+out:
+	kfree(info);
+	return err;
 }
 
-static int radeon_atpx_set_discrete_state(acpi_handle handle, int state)
+/**
+ * radeon_atpx_set_discrete_state - power up/down discrete GPU
+ *
+ * @atpx: atpx info struct
+ * @state: discrete GPU state (0 = power down, 1 = power up)
+ *
+ * Execute the ATPX_FUNCTION_POWER_CONTROL ATPX function to
+ * power down/up the discrete GPU (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atpx_set_discrete_state(struct radeon_atpx *atpx, u8 state)
 {
-	return radeon_atpx_execute(handle, ATPX_GPU_PWR, state);
+	struct acpi_buffer params;
+	union acpi_object *info;
+	struct atpx_power_control input;
+
+	if (atpx->functions.power_cntl) {
+		input.size = 3;
+		input.dgpu_state = state;
+		params.length = input.size;
+		params.pointer = &input;
+		info = radeon_atpx_call(atpx->handle,
+					ATPX_FUNCTION_POWER_CONTROL,
+					&params);
+		if (!info)
+			return -EIO;
+		kfree(info);
+	}
+	return 0;
 }
 
-static int radeon_atpx_switch_mux(acpi_handle handle, int mux_id)
+/**
+ * radeon_atpx_switch_disp_mux - switch display mux
+ *
+ * @atpx: atpx info struct
+ * @mux_id: mux state (0 = integrated GPU, 1 = discrete GPU)
+ *
+ * Execute the ATPX_FUNCTION_DISPLAY_MUX_CONTROL ATPX function to
+ * switch the display mux between the discrete GPU and integrated GPU
+ * (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atpx_switch_disp_mux(struct radeon_atpx *atpx, u16 mux_id)
 {
-	return radeon_atpx_execute(handle, ATPX_MUX_SELECT, mux_id);
+	struct acpi_buffer params;
+	union acpi_object *info;
+	struct atpx_mux input;
+
+	if (atpx->functions.disp_mux_cntl) {
+		input.size = 4;
+		input.mux = mux_id;
+		params.length = input.size;
+		params.pointer = &input;
+		info = radeon_atpx_call(atpx->handle,
+					ATPX_FUNCTION_DISPLAY_MUX_CONTROL,
+					&params);
+		if (!info)
+			return -EIO;
+		kfree(info);
+	}
+	return 0;
 }
 
-static int radeon_atpx_switch_i2c_mux(acpi_handle handle, int mux_id)
+/**
+ * radeon_atpx_switch_i2c_mux - switch i2c/hpd mux
+ *
+ * @atpx: atpx info struct
+ * @mux_id: mux state (0 = integrated GPU, 1 = discrete GPU)
+ *
+ * Execute the ATPX_FUNCTION_I2C_MUX_CONTROL ATPX function to
+ * switch the i2c/hpd mux between the discrete GPU and integrated GPU
+ * (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atpx_switch_i2c_mux(struct radeon_atpx *atpx, u16 mux_id)
 {
-	return radeon_atpx_execute(handle, ATPX_I2C_MUX_SELECT, mux_id);
+	struct acpi_buffer params;
+	union acpi_object *info;
+	struct atpx_mux input;
+
+	if (atpx->functions.i2c_mux_cntl) {
+		input.size = 4;
+		input.mux = mux_id;
+		params.length = input.size;
+		params.pointer = &input;
+		info = radeon_atpx_call(atpx->handle,
+					ATPX_FUNCTION_I2C_MUX_CONTROL,
+					&params);
+		if (!info)
+			return -EIO;
+		kfree(info);
+	}
+	return 0;
 }
 
-static int radeon_atpx_switch_start(acpi_handle handle, int gpu_id)
+/**
+ * radeon_atpx_switch_start - notify the sbios of a GPU switch
+ *
+ * @atpx: atpx info struct
+ * @mux_id: mux state (0 = integrated GPU, 1 = discrete GPU)
+ *
+ * Execute the ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION ATPX
+ * function to notify the sbios that a switch between the discrete GPU and
+ * integrated GPU has begun (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atpx_switch_start(struct radeon_atpx *atpx, u16 mux_id)
 {
-	return radeon_atpx_execute(handle, ATPX_SWITCH_START, gpu_id);
+	struct acpi_buffer params;
+	union acpi_object *info;
+	struct atpx_mux input;
+
+	if (atpx->functions.switch_start) {
+		input.size = 4;
+		input.mux = mux_id;
+		params.length = input.size;
+		params.pointer = &input;
+		info = radeon_atpx_call(atpx->handle,
+					ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_START_NOTIFICATION,
+					&params);
+		if (!info)
+			return -EIO;
+		kfree(info);
+	}
+	return 0;
 }
 
-static int radeon_atpx_switch_end(acpi_handle handle, int gpu_id)
+/**
+ * radeon_atpx_switch_end - notify the sbios of a GPU switch
+ *
+ * @atpx: atpx info struct
+ * @mux_id: mux state (0 = integrated GPU, 1 = discrete GPU)
+ *
+ * Execute the ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION ATPX
+ * function to notify the sbios that a switch between the discrete GPU and
+ * integrated GPU has ended (all asics).
+ * Returns 0 on success, error on failure.
+ */
+static int radeon_atpx_switch_end(struct radeon_atpx *atpx, u16 mux_id)
 {
-	return radeon_atpx_execute(handle, ATPX_SWITCH_END, gpu_id);
+	struct acpi_buffer params;
+	union acpi_object *info;
+	struct atpx_mux input;
+
+	if (atpx->functions.switch_end) {
+		input.size = 4;
+		input.mux = mux_id;
+		params.length = input.size;
+		params.pointer = &input;
+		info = radeon_atpx_call(atpx->handle,
+					ATPX_FUNCTION_GRAPHICS_DEVICE_SWITCH_END_NOTIFICATION,
+					&params);
+		if (!info)
+			return -EIO;
+		kfree(info);
+	}
+	return 0;
 }
 
+/**
+ * radeon_atpx_switchto - switch to the requested GPU
+ *
+ * @id: GPU to switch to
+ *
+ * Execute the necessary ATPX functions to switch between the discrete GPU and
+ * integrated GPU (all asics).
+ * Returns 0 on success, error on failure.
+ */
 static int radeon_atpx_switchto(enum vga_switcheroo_client_id id)
 {
-	int gpu_id;
+	u16 gpu_id;
 
 	if (id == VGA_SWITCHEROO_IGD)
-		gpu_id = ATPX_INTEGRATED;
+		gpu_id = ATPX_INTEGRATED_GPU;
 	else
-		gpu_id = ATPX_DISCRETE;
+		gpu_id = ATPX_DISCRETE_GPU;
 
-	radeon_atpx_switch_start(radeon_atpx_priv.atpx_handle, gpu_id);
-	radeon_atpx_switch_mux(radeon_atpx_priv.atpx_handle, gpu_id);
-	radeon_atpx_switch_i2c_mux(radeon_atpx_priv.atpx_handle, gpu_id);
-	radeon_atpx_switch_end(radeon_atpx_priv.atpx_handle, gpu_id);
+	radeon_atpx_switch_start(&radeon_atpx_priv.atpx, gpu_id);
+	radeon_atpx_switch_disp_mux(&radeon_atpx_priv.atpx, gpu_id);
+	radeon_atpx_switch_i2c_mux(&radeon_atpx_priv.atpx, gpu_id);
+	radeon_atpx_switch_end(&radeon_atpx_priv.atpx, gpu_id);
 
 	return 0;
 }
 
+/**
+ * radeon_atpx_switchto - switch to the requested GPU
+ *
+ * @id: GPU to switch to
+ * @state: requested power state (0 = off, 1 = on)
+ *
+ * Execute the necessary ATPX function to power down/up the discrete GPU
+ * (all asics).
+ * Returns 0 on success, error on failure.
+ */
 static int radeon_atpx_power_state(enum vga_switcheroo_client_id id,
 				   enum vga_switcheroo_state state)
 {
@@ -143,10 +368,18 @@ static int radeon_atpx_power_state(enum vga_switcheroo_client_id id,
 	if (id == VGA_SWITCHEROO_IGD)
 		return 0;
 
-	radeon_atpx_set_discrete_state(radeon_atpx_priv.atpx_handle, state);
+	radeon_atpx_set_discrete_state(&radeon_atpx_priv.atpx, state);
 	return 0;
 }
 
+/**
+ * radeon_atpx_pci_probe_handle - look up the ATRM and ATPX handles
+ *
+ * @pdev: pci device
+ *
+ * Look up the ATPX and ATRM handles (all asics).
+ * Returns true if the handles are found, false if not.
+ */
 static bool radeon_atpx_pci_probe_handle(struct pci_dev *pdev)
 {
 	acpi_handle dhandle, atpx_handle;
@@ -161,18 +394,30 @@ static bool radeon_atpx_pci_probe_handle(struct pci_dev *pdev)
 		return false;
 
 	radeon_atpx_priv.dhandle = dhandle;
-	radeon_atpx_priv.atpx_handle = atpx_handle;
+	radeon_atpx_priv.atpx.handle = atpx_handle;
 	return true;
 }
 
+/**
+ * radeon_atpx_init - verify the ATPX interface
+ *
+ * Verify the ATPX interface (all asics).
+ * Returns 0 on success, error on failure.
+ */
 static int radeon_atpx_init(void)
 {
 	/* set up the ATPX handle */
-
-	radeon_atpx_get_version(radeon_atpx_priv.atpx_handle);
-	return 0;
+	return radeon_atpx_verify_interface(&radeon_atpx_priv.atpx);
 }
 
+/**
+ * radeon_atpx_get_client_id - get the client id
+ *
+ * @pdev: pci device
+ *
+ * look up whether we are the integrated or discrete GPU (all asics).
+ * Returns the client id.
+ */
 static int radeon_atpx_get_client_id(struct pci_dev *pdev)
 {
 	if (radeon_atpx_priv.dhandle == DEVICE_ACPI_HANDLE(&pdev->dev))
@@ -188,6 +433,12 @@ static struct vga_switcheroo_handler radeon_atpx_handler = {
 	.get_client_id = radeon_atpx_get_client_id,
 };
 
+/**
+ * radeon_atpx_detect - detect whether we have PX
+ *
+ * Check if we have a PX system (all asics).
+ * Returns true if we have a PX system, false if not.
+ */
 static bool radeon_atpx_detect(void)
 {
 	char acpi_method_name[255] = { 0 };
@@ -203,7 +454,7 @@ static bool radeon_atpx_detect(void)
 	}
 
 	if (has_atpx && vga_count == 2) {
-		acpi_get_name(radeon_atpx_priv.atpx_handle, ACPI_FULL_PATHNAME, &buffer);
+		acpi_get_name(radeon_atpx_priv.atpx.handle, ACPI_FULL_PATHNAME, &buffer);
 		printk(KERN_INFO "VGA switcheroo: detected switching method %s handle\n",
 		       acpi_method_name);
 		radeon_atpx_priv.atpx_detected = true;
@@ -212,6 +463,11 @@ static bool radeon_atpx_detect(void)
 	return false;
 }
 
+/**
+ * radeon_register_atpx_handler - register with vga_switcheroo
+ *
+ * Register the PX callbacks with vga_switcheroo (all asics).
+ */
 void radeon_register_atpx_handler(void)
 {
 	bool r;
@@ -224,6 +480,11 @@ void radeon_register_atpx_handler(void)
 	vga_switcheroo_register_handler(&radeon_atpx_handler);
 }
 
+/**
+ * radeon_unregister_atpx_handler - unregister with vga_switcheroo
+ *
+ * Unregister the PX callbacks with vga_switcheroo (all asics).
+ */
 void radeon_unregister_atpx_handler(void)
 {
 	vga_switcheroo_unregister_handler();
diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c
index d306cc8fdea..b8015913d38 100644
--- a/drivers/gpu/drm/radeon/radeon_bios.c
+++ b/drivers/gpu/drm/radeon/radeon_bios.c
@@ -25,7 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "atom.h"
diff --git a/drivers/gpu/drm/radeon/radeon_clocks.c b/drivers/gpu/drm/radeon/radeon_clocks.c
index 9c6b29a4192..38e396dae0a 100644
--- a/drivers/gpu/drm/radeon/radeon_clocks.c
+++ b/drivers/gpu/drm/radeon/radeon_clocks.c
@@ -25,8 +25,8 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "atom.h"
diff --git a/drivers/gpu/drm/radeon/radeon_combios.c b/drivers/gpu/drm/radeon/radeon_combios.c
index f75247d42ff..45b660b27cf 100644
--- a/drivers/gpu/drm/radeon/radeon_combios.c
+++ b/drivers/gpu/drm/radeon/radeon_combios.c
@@ -24,8 +24,8 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 
@@ -3319,15 +3319,6 @@ static void combios_write_ram_size(struct drm_device *dev)
 	WREG32(RADEON_CONFIG_MEMSIZE, mem_size);
 }
 
-void radeon_combios_dyn_clk_setup(struct drm_device *dev, int enable)
-{
-	uint16_t dyn_clk_info =
-	    combios_get_table_offset(dev, COMBIOS_DYN_CLK_1_TABLE);
-
-	if (dyn_clk_info)
-		combios_parse_pll_table(dev, dyn_clk_info);
-}
-
 void radeon_combios_asic_init(struct drm_device *dev)
 {
 	struct radeon_device *rdev = dev->dev_private;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 895e628b60f..67cfc1795ec 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -23,11 +23,11 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "drm_edid.h"
-#include "drm_crtc_helper.h"
-#include "drm_fb_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 
@@ -40,10 +40,6 @@ radeon_atombios_connected_scratch_regs(struct drm_connector *connector,
 				       struct drm_encoder *encoder,
 				       bool connected);
 
-extern void
-radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
-			     struct drm_connector *drm_connector);
-
 void radeon_connector_hotplug(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
@@ -198,7 +194,7 @@ radeon_connector_update_scratch_regs(struct drm_connector *connector, enum drm_c
 	}
 }
 
-struct drm_encoder *radeon_find_encoder(struct drm_connector *connector, int encoder_type)
+static struct drm_encoder *radeon_find_encoder(struct drm_connector *connector, int encoder_type)
 {
 	struct drm_mode_object *obj;
 	struct drm_encoder *encoder;
@@ -219,7 +215,7 @@ struct drm_encoder *radeon_find_encoder(struct drm_connector *connector, int enc
 	return NULL;
 }
 
-struct drm_encoder *radeon_best_single_encoder(struct drm_connector *connector)
+static struct drm_encoder *radeon_best_single_encoder(struct drm_connector *connector)
 {
 	int enc_id = connector->encoder_ids[0];
 	struct drm_mode_object *obj;
@@ -370,7 +366,7 @@ static void radeon_add_common_modes(struct drm_encoder *encoder, struct drm_conn
 	}
 }
 
-int radeon_connector_set_property(struct drm_connector *connector, struct drm_property *property,
+static int radeon_connector_set_property(struct drm_connector *connector, struct drm_property *property,
 				  uint64_t val)
 {
 	struct drm_device *dev = connector->dev;
@@ -691,13 +687,13 @@ static int radeon_lvds_set_property(struct drm_connector *connector,
 }
 
 
-struct drm_connector_helper_funcs radeon_lvds_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs radeon_lvds_connector_helper_funcs = {
 	.get_modes = radeon_lvds_get_modes,
 	.mode_valid = radeon_lvds_mode_valid,
 	.best_encoder = radeon_best_single_encoder,
 };
 
-struct drm_connector_funcs radeon_lvds_connector_funcs = {
+static const struct drm_connector_funcs radeon_lvds_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = radeon_lvds_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -809,13 +805,13 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
 	return ret;
 }
 
-struct drm_connector_helper_funcs radeon_vga_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs radeon_vga_connector_helper_funcs = {
 	.get_modes = radeon_vga_get_modes,
 	.mode_valid = radeon_vga_mode_valid,
 	.best_encoder = radeon_best_single_encoder,
 };
 
-struct drm_connector_funcs radeon_vga_connector_funcs = {
+static const struct drm_connector_funcs radeon_vga_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = radeon_vga_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -879,13 +875,13 @@ radeon_tv_detect(struct drm_connector *connector, bool force)
 	return ret;
 }
 
-struct drm_connector_helper_funcs radeon_tv_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs radeon_tv_connector_helper_funcs = {
 	.get_modes = radeon_tv_get_modes,
 	.mode_valid = radeon_tv_mode_valid,
 	.best_encoder = radeon_best_single_encoder,
 };
 
-struct drm_connector_funcs radeon_tv_connector_funcs = {
+static const struct drm_connector_funcs radeon_tv_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = radeon_tv_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -1089,7 +1085,7 @@ out:
 }
 
 /* okay need to be smart in here about which encoder to pick */
-struct drm_encoder *radeon_dvi_encoder(struct drm_connector *connector)
+static struct drm_encoder *radeon_dvi_encoder(struct drm_connector *connector)
 {
 	int enc_id = connector->encoder_ids[0];
 	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
@@ -1179,13 +1175,13 @@ static int radeon_dvi_mode_valid(struct drm_connector *connector,
 	return MODE_OK;
 }
 
-struct drm_connector_helper_funcs radeon_dvi_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs radeon_dvi_connector_helper_funcs = {
 	.get_modes = radeon_dvi_get_modes,
 	.mode_valid = radeon_dvi_mode_valid,
 	.best_encoder = radeon_dvi_encoder,
 };
 
-struct drm_connector_funcs radeon_dvi_connector_funcs = {
+static const struct drm_connector_funcs radeon_dvi_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = radeon_dvi_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -1462,13 +1458,13 @@ static int radeon_dp_mode_valid(struct drm_connector *connector,
 	}
 }
 
-struct drm_connector_helper_funcs radeon_dp_connector_helper_funcs = {
+static const struct drm_connector_helper_funcs radeon_dp_connector_helper_funcs = {
 	.get_modes = radeon_dp_get_modes,
 	.mode_valid = radeon_dp_mode_valid,
 	.best_encoder = radeon_dvi_encoder,
 };
 
-struct drm_connector_funcs radeon_dp_connector_funcs = {
+static const struct drm_connector_funcs radeon_dp_connector_funcs = {
 	.dpms = drm_helper_connector_dpms,
 	.detect = radeon_dp_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
@@ -2008,15 +2004,4 @@ radeon_add_legacy_connector(struct drm_device *dev,
 		connector->polled = DRM_CONNECTOR_POLL_HPD;
 	connector->display_info.subpixel_order = subpixel_order;
 	drm_sysfs_connector_add(connector);
-	if (connector_type == DRM_MODE_CONNECTOR_LVDS) {
-		struct drm_encoder *drm_encoder;
-
-		list_for_each_entry(drm_encoder, &dev->mode_config.encoder_list, head) {
-			struct radeon_encoder *radeon_encoder;
-
-			radeon_encoder = to_radeon_encoder(drm_encoder);
-			if (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_LVDS)
-				radeon_legacy_backlight_init(radeon_encoder, connector);
-		}
-	}
 }
diff --git a/drivers/gpu/drm/radeon/radeon_cp.c b/drivers/gpu/drm/radeon/radeon_cp.c
index ef67e181377..8b2797dc7b6 100644
--- a/drivers/gpu/drm/radeon/radeon_cp.c
+++ b/drivers/gpu/drm/radeon/radeon_cp.c
@@ -31,10 +31,8 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_sarea.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 #include "r300_reg.h"
 
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index b4a0db24f4d..cb7b7c062fe 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -24,15 +24,15 @@
  * Authors:
  *    Jerome Glisse <glisse@freedesktop.org>
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 
 void r100_cs_dump_packet(struct radeon_cs_parser *p,
 			 struct radeon_cs_packet *pkt);
 
-int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
+static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 {
 	struct drm_device *ddev = p->rdev->ddev;
 	struct radeon_cs_chunk *chunk;
@@ -115,19 +115,27 @@ static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority
 	return 0;
 }
 
+static void radeon_cs_sync_to(struct radeon_cs_parser *p,
+			      struct radeon_fence *fence)
+{
+	struct radeon_fence *other;
+
+	if (!fence)
+		return;
+
+	other = p->ib.sync_to[fence->ring];
+	p->ib.sync_to[fence->ring] = radeon_fence_later(fence, other);
+}
+
 static void radeon_cs_sync_rings(struct radeon_cs_parser *p)
 {
 	int i;
 
 	for (i = 0; i < p->nrelocs; i++) {
-		struct radeon_fence *a, *b;
-
-		if (!p->relocs[i].robj || !p->relocs[i].robj->tbo.sync_obj)
+		if (!p->relocs[i].robj)
 			continue;
 
-		a = p->relocs[i].robj->tbo.sync_obj;
-		b = p->ib.sync_to[a->ring];
-		p->ib.sync_to[a->ring] = radeon_fence_later(a, b);
+		radeon_cs_sync_to(p, p->relocs[i].robj->tbo.sync_obj);
 	}
 }
 
@@ -278,30 +286,6 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
 	return 0;
 }
 
-static void radeon_bo_vm_fence_va(struct radeon_cs_parser *parser,
-				  struct radeon_fence *fence)
-{
-	struct radeon_fpriv *fpriv = parser->filp->driver_priv;
-	struct radeon_vm *vm = &fpriv->vm;
-	struct radeon_bo_list *lobj;
-
-	if (parser->chunk_ib_idx == -1) {
-		return;
-	}
-	if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) {
-		return;
-	}
-
-	list_for_each_entry(lobj, &parser->validated, tv.head) {
-		struct radeon_bo_va *bo_va;
-		struct radeon_bo *rbo = lobj->bo;
-
-		bo_va = radeon_bo_va(rbo, vm);
-		radeon_fence_unref(&bo_va->fence);
-		bo_va->fence = radeon_fence_ref(fence);
-	}
-}
-
 /**
  * cs_parser_fini() - clean parser states
  * @parser:	parser structure holding parsing context.
@@ -315,8 +299,6 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error)
 	unsigned i;
 
 	if (!error) {
-		/* fence all bo va before ttm_eu_fence_buffer_objects so bo are still reserved */
-		radeon_bo_vm_fence_va(parser, parser->ib.fence);
 		ttm_eu_fence_buffer_objects(&parser->validated,
 					    parser->ib.fence);
 	} else {
@@ -363,7 +345,7 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
 	 * uncached).
 	 */
 	r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
-			   ib_chunk->length_dw * 4);
+			   NULL, ib_chunk->length_dw * 4);
 	if (r) {
 		DRM_ERROR("Failed to get ib !\n");
 		return r;
@@ -380,7 +362,6 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
 		return r;
 	}
 	radeon_cs_sync_rings(parser);
-	parser->ib.vm_id = 0;
 	r = radeon_ib_schedule(rdev, &parser->ib, NULL);
 	if (r) {
 		DRM_ERROR("Failed to schedule IB !\n");
@@ -391,10 +372,15 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
 static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
 				   struct radeon_vm *vm)
 {
+	struct radeon_device *rdev = parser->rdev;
 	struct radeon_bo_list *lobj;
 	struct radeon_bo *bo;
 	int r;
 
+	r = radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem);
+	if (r) {
+		return r;
+	}
 	list_for_each_entry(lobj, &parser->validated, tv.head) {
 		bo = lobj->bo;
 		r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem);
@@ -426,7 +412,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
 			return -EINVAL;
 		}
 		r =  radeon_ib_get(rdev, parser->ring, &parser->const_ib,
-				   ib_chunk->length_dw * 4);
+				   vm, ib_chunk->length_dw * 4);
 		if (r) {
 			DRM_ERROR("Failed to get const ib !\n");
 			return r;
@@ -450,7 +436,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
 		return -EINVAL;
 	}
 	r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
-			   ib_chunk->length_dw * 4);
+			   vm, ib_chunk->length_dw * 4);
 	if (r) {
 		DRM_ERROR("Failed to get ib !\n");
 		return r;
@@ -468,7 +454,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
 
 	mutex_lock(&rdev->vm_manager.lock);
 	mutex_lock(&vm->mutex);
-	r = radeon_vm_bind(rdev, vm);
+	r = radeon_vm_alloc_pt(rdev, vm);
 	if (r) {
 		goto out;
 	}
@@ -477,32 +463,21 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
 		goto out;
 	}
 	radeon_cs_sync_rings(parser);
-
-	parser->ib.vm_id = vm->id;
-	/* ib pool is bind at 0 in virtual address space,
-	 * so gpu_addr is the offset inside the pool bo
-	 */
-	parser->ib.gpu_addr = parser->ib.sa_bo->soffset;
+	radeon_cs_sync_to(parser, vm->fence);
+	radeon_cs_sync_to(parser, radeon_vm_grab_id(rdev, vm, parser->ring));
 
 	if ((rdev->family >= CHIP_TAHITI) &&
 	    (parser->chunk_const_ib_idx != -1)) {
-		parser->const_ib.vm_id = vm->id;
-		/* ib pool is bind at 0 in virtual address space,
-		 * so gpu_addr is the offset inside the pool bo
-		 */
-		parser->const_ib.gpu_addr = parser->const_ib.sa_bo->soffset;
 		r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib);
 	} else {
 		r = radeon_ib_schedule(rdev, &parser->ib, NULL);
 	}
 
-out:
 	if (!r) {
-		if (vm->fence) {
-			radeon_fence_unref(&vm->fence);
-		}
-		vm->fence = radeon_fence_ref(parser->ib.fence);
+		radeon_vm_fence(rdev, vm, parser->ib.fence);
 	}
+
+out:
 	mutex_unlock(&vm->mutex);
 	mutex_unlock(&rdev->vm_manager.lock);
 	return r;
diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c
index 8794744cdf1..0fe56c9f64b 100644
--- a/drivers/gpu/drm/radeon/radeon_cursor.c
+++ b/drivers/gpu/drm/radeon/radeon_cursor.c
@@ -23,8 +23,8 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #define CURSOR_WIDTH 64
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 7a3daebd732..64a42647f08 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -842,7 +842,7 @@ static unsigned int radeon_vga_set_decode(void *cookie, bool state)
  * Validates certain module parameters and updates
  * the associated values used by the driver (all asics).
  */
-void radeon_check_arguments(struct radeon_device *rdev)
+static void radeon_check_arguments(struct radeon_device *rdev)
 {
 	/* vramlimit must be a power of two */
 	switch (radeon_vram_limit) {
@@ -1013,13 +1013,11 @@ int radeon_device_init(struct radeon_device *rdev,
 	init_rwsem(&rdev->pm.mclk_lock);
 	init_rwsem(&rdev->exclusive_lock);
 	init_waitqueue_head(&rdev->irq.vblank_queue);
-	init_waitqueue_head(&rdev->irq.idle_queue);
 	r = radeon_gem_init(rdev);
 	if (r)
 		return r;
 	/* initialize vm here */
 	mutex_init(&rdev->vm_manager.lock);
-	rdev->vm_manager.use_bitmap = 1;
 	rdev->vm_manager.max_pfn = 1 << 20;
 	INIT_LIST_HEAD(&rdev->vm_manager.lru_vm);
 
@@ -1284,6 +1282,13 @@ int radeon_resume_kms(struct drm_device *dev)
 	if (rdev->is_atom_bios) {
 		radeon_atom_encoder_init(rdev);
 		radeon_atom_disp_eng_pll_init(rdev);
+		/* turn on the BL */
+		if (rdev->mode_info.bl_encoder) {
+			u8 bl_level = radeon_get_backlight_level(rdev,
+								 rdev->mode_info.bl_encoder);
+			radeon_set_backlight_level(rdev, rdev->mode_info.bl_encoder,
+						   bl_level);
+		}
 	}
 	/* reset hpd state */
 	radeon_hpd_init(rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 7ddef8f30d0..bfa2a601572 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -23,15 +23,15 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #include "atom.h"
 #include <asm/div64.h>
 
-#include "drm_crtc_helper.h"
-#include "drm_edid.h"
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
 
 static void avivo_crtc_load_lut(struct drm_crtc *crtc)
 {
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 8c593ea82c4..07eb84e8a8a 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -29,12 +29,11 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 #include <linux/console.h>
 #include <linux/module.h>
 
@@ -64,9 +63,11 @@
  *   2.20.0 - r600-si: RADEON_INFO_TIMESTAMP query
  *   2.21.0 - r600-r700: FMASK and CMASK
  *   2.22.0 - r600 only: RESOLVE_BOX allowed
+ *   2.23.0 - allow STRMOUT_BASE_UPDATE on RS780 and RS880
+ *   2.24.0 - eg only: allow MIP_ADDRESS=0 for MSAA textures
  */
 #define KMS_DRIVER_MAJOR	2
-#define KMS_DRIVER_MINOR	22
+#define KMS_DRIVER_MINOR	24
 #define KMS_DRIVER_PATCHLEVEL	0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index 74670696277..bd4959ca23a 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -23,12 +23,20 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 
+extern void
+radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
+			     struct drm_connector *drm_connector);
+extern void
+radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
+			   struct drm_connector *drm_connector);
+
+
 static uint32_t radeon_encoder_clones(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
@@ -153,6 +161,7 @@ radeon_get_encoder_enum(struct drm_device *dev, uint32_t supported_device, uint8
 void
 radeon_link_encoder_connector(struct drm_device *dev)
 {
+	struct radeon_device *rdev = dev->dev_private;
 	struct drm_connector *connector;
 	struct radeon_connector *radeon_connector;
 	struct drm_encoder *encoder;
@@ -163,8 +172,16 @@ radeon_link_encoder_connector(struct drm_device *dev)
 		radeon_connector = to_radeon_connector(connector);
 		list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 			radeon_encoder = to_radeon_encoder(encoder);
-			if (radeon_encoder->devices & radeon_connector->devices)
+			if (radeon_encoder->devices & radeon_connector->devices) {
 				drm_mode_connector_attach_encoder(connector, encoder);
+				if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
+					if (rdev->is_atom_bios)
+						radeon_atom_backlight_init(radeon_encoder, connector);
+					else
+						radeon_legacy_backlight_init(radeon_encoder, connector);
+					rdev->mode_info.bl_encoder = radeon_encoder;
+				}
+			}
 		}
 	}
 }
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 5906914a78b..cc8489d8c6d 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -27,14 +27,13 @@
 #include <linux/slab.h>
 #include <linux/fb.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
-#include "drm_fb_helper.h"
+#include <drm/drm_fb_helper.h>
 
 #include <linux/vga_switcheroo.h>
 
@@ -316,22 +315,6 @@ static int radeon_fb_find_or_create_single(struct drm_fb_helper *helper,
 	return new_fb;
 }
 
-static char *mode_option;
-int radeon_parse_options(char *options)
-{
-	char *this_opt;
-
-	if (!options || !*options)
-		return 0;
-
-	while ((this_opt = strsep(&options, ",")) != NULL) {
-		if (!*this_opt)
-			continue;
-		mode_option = this_opt;
-	}
-	return 0;
-}
-
 void radeon_fb_output_poll_changed(struct radeon_device *rdev)
 {
 	drm_fb_helper_hotplug_event(&rdev->mode_info.rfbdev->helper);
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index 2a59375dbe5..22bd6c2c274 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -34,8 +34,7 @@
 #include <linux/list.h>
 #include <linux/kref.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_trace.h"
@@ -399,7 +398,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
 	return 0;
 }
 
-bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
+static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
 {
 	unsigned i;
 
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index bb3b7fe05cc..f0c06d196b7 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -25,8 +25,8 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_reg.h"
 
@@ -423,6 +423,18 @@ void radeon_gart_fini(struct radeon_device *rdev)
  */
 
 /**
+ * radeon_vm_directory_size - returns the size of the page directory in bytes
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Calculate the size of the page directory in bytes (cayman+).
+ */
+static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
+{
+	return (rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE) * 8;
+}
+
+/**
  * radeon_vm_manager_init - init the vm manager
  *
  * @rdev: radeon_device pointer
@@ -435,12 +447,15 @@ int radeon_vm_manager_init(struct radeon_device *rdev)
 	struct radeon_vm *vm;
 	struct radeon_bo_va *bo_va;
 	int r;
+	unsigned size;
 
 	if (!rdev->vm_manager.enabled) {
-		/* mark first vm as always in use, it's the system one */
 		/* allocate enough for 2 full VM pts */
+		size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev));
+		size += RADEON_GPU_PAGE_ALIGN(rdev->vm_manager.max_pfn * 8);
+		size *= 2;
 		r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager,
-					      rdev->vm_manager.max_pfn * 8 * 2,
+					      size,
 					      RADEON_GEM_DOMAIN_VRAM);
 		if (r) {
 			dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n",
@@ -448,10 +463,10 @@ int radeon_vm_manager_init(struct radeon_device *rdev)
 			return r;
 		}
 
-		r = rdev->vm_manager.funcs->init(rdev);
+		r = radeon_asic_vm_init(rdev);
 		if (r)
 			return r;
-	
+
 		rdev->vm_manager.enabled = true;
 
 		r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager);
@@ -461,73 +476,36 @@ int radeon_vm_manager_init(struct radeon_device *rdev)
 
 	/* restore page table */
 	list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) {
-		if (vm->id == -1)
+		if (vm->sa_bo == NULL)
 			continue;
 
 		list_for_each_entry(bo_va, &vm->va, vm_list) {
-			struct ttm_mem_reg *mem = NULL;
-			if (bo_va->valid)
-				mem = &bo_va->bo->tbo.mem;
-
 			bo_va->valid = false;
-			r = radeon_vm_bo_update_pte(rdev, vm, bo_va->bo, mem);
-			if (r) {
-				DRM_ERROR("Failed to update pte for vm %d!\n", vm->id);
-			}
-		}
-
-		r = rdev->vm_manager.funcs->bind(rdev, vm, vm->id);
-		if (r) {
-			DRM_ERROR("Failed to bind vm %d!\n", vm->id);
 		}
 	}
 	return 0;
 }
 
-/* global mutex must be lock */
 /**
- * radeon_vm_unbind_locked - unbind a specific vm
+ * radeon_vm_free_pt - free the page table for a specific vm
  *
  * @rdev: radeon_device pointer
  * @vm: vm to unbind
  *
- * Unbind the requested vm (cayman+).
- * Wait for use of the VM to finish, then unbind the page table,
- * and free the page table memory.
+ * Free the page table of a specific vm (cayman+).
+ *
+ * Global and local mutex must be lock!
  */
-static void radeon_vm_unbind_locked(struct radeon_device *rdev,
+static void radeon_vm_free_pt(struct radeon_device *rdev,
 				    struct radeon_vm *vm)
 {
 	struct radeon_bo_va *bo_va;
 
-	if (vm->id == -1) {
+	if (!vm->sa_bo)
 		return;
-	}
 
-	/* wait for vm use to end */
-	while (vm->fence) {
-		int r;
-		r = radeon_fence_wait(vm->fence, false);
-		if (r)
-			DRM_ERROR("error while waiting for fence: %d\n", r);
-		if (r == -EDEADLK) {
-			mutex_unlock(&rdev->vm_manager.lock);
-			r = radeon_gpu_reset(rdev);
-			mutex_lock(&rdev->vm_manager.lock);
-			if (!r)
-				continue;
-		}
-		break;
-	}
-	radeon_fence_unref(&vm->fence);
-
-	/* hw unbind */
-	rdev->vm_manager.funcs->unbind(rdev, vm);
-	rdev->vm_manager.use_bitmap &= ~(1 << vm->id);
 	list_del_init(&vm->list);
-	vm->id = -1;
-	radeon_sa_bo_free(rdev, &vm->sa_bo, NULL);
-	vm->pt = NULL;
+	radeon_sa_bo_free(rdev, &vm->sa_bo, vm->fence);
 
 	list_for_each_entry(bo_va, &vm->va, vm_list) {
 		bo_va->valid = false;
@@ -544,16 +522,22 @@ static void radeon_vm_unbind_locked(struct radeon_device *rdev,
 void radeon_vm_manager_fini(struct radeon_device *rdev)
 {
 	struct radeon_vm *vm, *tmp;
+	int i;
 
 	if (!rdev->vm_manager.enabled)
 		return;
 
 	mutex_lock(&rdev->vm_manager.lock);
-	/* unbind all active vm */
+	/* free all allocated page tables */
 	list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) {
-		radeon_vm_unbind_locked(rdev, vm);
+		mutex_lock(&vm->mutex);
+		radeon_vm_free_pt(rdev, vm);
+		mutex_unlock(&vm->mutex);
 	}
-	rdev->vm_manager.funcs->fini(rdev);
+	for (i = 0; i < RADEON_NUM_VM; ++i) {
+		radeon_fence_unref(&rdev->vm_manager.active[i]);
+	}
+	radeon_asic_vm_fini(rdev);
 	mutex_unlock(&rdev->vm_manager.lock);
 
 	radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager);
@@ -561,46 +545,34 @@ void radeon_vm_manager_fini(struct radeon_device *rdev)
 	rdev->vm_manager.enabled = false;
 }
 
-/* global mutex must be locked */
 /**
- * radeon_vm_unbind - locked version of unbind
- *
- * @rdev: radeon_device pointer
- * @vm: vm to unbind
- *
- * Locked version that wraps radeon_vm_unbind_locked (cayman+).
- */
-void radeon_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-	mutex_lock(&vm->mutex);
-	radeon_vm_unbind_locked(rdev, vm);
-	mutex_unlock(&vm->mutex);
-}
-
-/* global and local mutex must be locked */
-/**
- * radeon_vm_bind - bind a page table to a VMID
+ * radeon_vm_alloc_pt - allocates a page table for a VM
  *
  * @rdev: radeon_device pointer
  * @vm: vm to bind
  *
- * Bind the requested vm (cayman+).
- * Suballocate memory for the page table, allocate a VMID
- * and bind the page table to it, and finally start to populate
- * the page table.
+ * Allocate a page table for the requested vm (cayman+).
+ * Also starts to populate the page table.
  * Returns 0 for success, error for failure.
+ *
+ * Global and local mutex must be locked!
  */
-int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm)
+int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
 {
 	struct radeon_vm *vm_evict;
-	unsigned i;
-	int id = -1, r;
+	int r;
+	u64 *pd_addr;
+	int tables_size;
 
 	if (vm == NULL) {
 		return -EINVAL;
 	}
 
-	if (vm->id != -1) {
+	/* allocate enough to cover the current VM size */
+	tables_size = RADEON_GPU_PAGE_ALIGN(radeon_vm_directory_size(rdev));
+	tables_size += RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8);
+
+	if (vm->sa_bo != NULL) {
 		/* update lru */
 		list_del_init(&vm->list);
 		list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
@@ -609,98 +581,215 @@ int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm)
 
 retry:
 	r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo,
-			     RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8),
-			     RADEON_GPU_PAGE_SIZE, false);
-	if (r) {
+			     tables_size, RADEON_GPU_PAGE_SIZE, false);
+	if (r == -ENOMEM) {
 		if (list_empty(&rdev->vm_manager.lru_vm)) {
 			return r;
 		}
 		vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, struct radeon_vm, list);
-		radeon_vm_unbind(rdev, vm_evict);
+		mutex_lock(&vm_evict->mutex);
+		radeon_vm_free_pt(rdev, vm_evict);
+		mutex_unlock(&vm_evict->mutex);
 		goto retry;
+
+	} else if (r) {
+		return r;
 	}
-	vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo);
-	vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
-	memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8));
 
-retry_id:
-	/* search for free vm */
-	for (i = 0; i < rdev->vm_manager.nvm; i++) {
-		if (!(rdev->vm_manager.use_bitmap & (1 << i))) {
-			id = i;
-			break;
+	pd_addr = radeon_sa_bo_cpu_addr(vm->sa_bo);
+	vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo);
+	memset(pd_addr, 0, tables_size);
+
+	list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
+	return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo,
+				       &rdev->ring_tmp_bo.bo->tbo.mem);
+}
+
+/**
+ * radeon_vm_grab_id - allocate the next free VMID
+ *
+ * @rdev: radeon_device pointer
+ * @vm: vm to allocate id for
+ * @ring: ring we want to submit job to
+ *
+ * Allocate an id for the vm (cayman+).
+ * Returns the fence we need to sync to (if any).
+ *
+ * Global and local mutex must be locked!
+ */
+struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
+				       struct radeon_vm *vm, int ring)
+{
+	struct radeon_fence *best[RADEON_NUM_RINGS] = {};
+	unsigned choices[2] = {};
+	unsigned i;
+
+	/* check if the id is still valid */
+	if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id])
+		return NULL;
+
+	/* we definately need to flush */
+	radeon_fence_unref(&vm->last_flush);
+
+	/* skip over VMID 0, since it is the system VM */
+	for (i = 1; i < rdev->vm_manager.nvm; ++i) {
+		struct radeon_fence *fence = rdev->vm_manager.active[i];
+
+		if (fence == NULL) {
+			/* found a free one */
+			vm->id = i;
+			return NULL;
+		}
+
+		if (radeon_fence_is_earlier(fence, best[fence->ring])) {
+			best[fence->ring] = fence;
+			choices[fence->ring == ring ? 0 : 1] = i;
 		}
 	}
-	/* evict vm if necessary */
-	if (id == -1) {
-		vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, struct radeon_vm, list);
-		radeon_vm_unbind(rdev, vm_evict);
-		goto retry_id;
+
+	for (i = 0; i < 2; ++i) {
+		if (choices[i]) {
+			vm->id = choices[i];
+			return rdev->vm_manager.active[choices[i]];
+		}
 	}
 
-	/* do hw bind */
-	r = rdev->vm_manager.funcs->bind(rdev, vm, id);
-	if (r) {
-		radeon_sa_bo_free(rdev, &vm->sa_bo, NULL);
-		return r;
+	/* should never happen */
+	BUG();
+	return NULL;
+}
+
+/**
+ * radeon_vm_fence - remember fence for vm
+ *
+ * @rdev: radeon_device pointer
+ * @vm: vm we want to fence
+ * @fence: fence to remember
+ *
+ * Fence the vm (cayman+).
+ * Set the fence used to protect page table and id.
+ *
+ * Global and local mutex must be locked!
+ */
+void radeon_vm_fence(struct radeon_device *rdev,
+		     struct radeon_vm *vm,
+		     struct radeon_fence *fence)
+{
+	radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
+	rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
+
+	radeon_fence_unref(&vm->fence);
+	vm->fence = radeon_fence_ref(fence);
+}
+
+/**
+ * radeon_vm_bo_find - find the bo_va for a specific vm & bo
+ *
+ * @vm: requested vm
+ * @bo: requested buffer object
+ *
+ * Find @bo inside the requested vm (cayman+).
+ * Search inside the @bos vm list for the requested vm
+ * Returns the found bo_va or NULL if none is found
+ *
+ * Object has to be reserved!
+ */
+struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
+				       struct radeon_bo *bo)
+{
+	struct radeon_bo_va *bo_va;
+
+	list_for_each_entry(bo_va, &bo->va, bo_list) {
+		if (bo_va->vm == vm) {
+			return bo_va;
+		}
 	}
-	rdev->vm_manager.use_bitmap |= 1 << id;
-	vm->id = id;
-	list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
-	return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo,
-				       &rdev->ring_tmp_bo.bo->tbo.mem);
+	return NULL;
 }
 
-/* object have to be reserved */
 /**
  * radeon_vm_bo_add - add a bo to a specific vm
  *
  * @rdev: radeon_device pointer
  * @vm: requested vm
  * @bo: radeon buffer object
- * @offset: requested offset of the buffer in the VM address space
- * @flags: attributes of pages (read/write/valid/etc.)
  *
  * Add @bo into the requested vm (cayman+).
- * Add @bo to the list of bos associated with the vm and validate
- * the offset requested within the vm address space.
- * Returns 0 for success, error for failure.
+ * Add @bo to the list of bos associated with the vm
+ * Returns newly added bo_va or NULL for failure
+ *
+ * Object has to be reserved!
  */
-int radeon_vm_bo_add(struct radeon_device *rdev,
-		     struct radeon_vm *vm,
-		     struct radeon_bo *bo,
-		     uint64_t offset,
-		     uint32_t flags)
+struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
+				      struct radeon_vm *vm,
+				      struct radeon_bo *bo)
 {
-	struct radeon_bo_va *bo_va, *tmp;
-	struct list_head *head;
-	uint64_t size = radeon_bo_size(bo), last_offset = 0;
-	unsigned last_pfn;
+	struct radeon_bo_va *bo_va;
 
 	bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL);
 	if (bo_va == NULL) {
-		return -ENOMEM;
+		return NULL;
 	}
 	bo_va->vm = vm;
 	bo_va->bo = bo;
-	bo_va->soffset = offset;
-	bo_va->eoffset = offset + size;
-	bo_va->flags = flags;
+	bo_va->soffset = 0;
+	bo_va->eoffset = 0;
+	bo_va->flags = 0;
 	bo_va->valid = false;
+	bo_va->ref_count = 1;
 	INIT_LIST_HEAD(&bo_va->bo_list);
 	INIT_LIST_HEAD(&bo_va->vm_list);
-	/* make sure object fit at this offset */
-	if (bo_va->soffset >= bo_va->eoffset) {
-		kfree(bo_va);
-		return -EINVAL;
-	}
 
-	last_pfn = bo_va->eoffset / RADEON_GPU_PAGE_SIZE;
-	if (last_pfn > rdev->vm_manager.max_pfn) {
-		kfree(bo_va);
-		dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
-			last_pfn, rdev->vm_manager.max_pfn);
-		return -EINVAL;
+	mutex_lock(&vm->mutex);
+	list_add(&bo_va->vm_list, &vm->va);
+	list_add_tail(&bo_va->bo_list, &bo->va);
+	mutex_unlock(&vm->mutex);
+
+	return bo_va;
+}
+
+/**
+ * radeon_vm_bo_set_addr - set bos virtual address inside a vm
+ *
+ * @rdev: radeon_device pointer
+ * @bo_va: bo_va to store the address
+ * @soffset: requested offset of the buffer in the VM address space
+ * @flags: attributes of pages (read/write/valid/etc.)
+ *
+ * Set offset of @bo_va (cayman+).
+ * Validate and set the offset requested within the vm address space.
+ * Returns 0 for success, error for failure.
+ *
+ * Object has to be reserved!
+ */
+int radeon_vm_bo_set_addr(struct radeon_device *rdev,
+			  struct radeon_bo_va *bo_va,
+			  uint64_t soffset,
+			  uint32_t flags)
+{
+	uint64_t size = radeon_bo_size(bo_va->bo);
+	uint64_t eoffset, last_offset = 0;
+	struct radeon_vm *vm = bo_va->vm;
+	struct radeon_bo_va *tmp;
+	struct list_head *head;
+	unsigned last_pfn;
+
+	if (soffset) {
+		/* make sure object fit at this offset */
+		eoffset = soffset + size;
+		if (soffset >= eoffset) {
+			return -EINVAL;
+		}
+
+		last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
+		if (last_pfn > rdev->vm_manager.max_pfn) {
+			dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
+				last_pfn, rdev->vm_manager.max_pfn);
+			return -EINVAL;
+		}
+
+	} else {
+		eoffset = last_pfn = 0;
 	}
 
 	mutex_lock(&vm->mutex);
@@ -713,7 +802,7 @@ int radeon_vm_bo_add(struct radeon_device *rdev,
 		if (last_pfn > vm->last_pfn) {
 			/* grow va space 32M by 32M */
 			unsigned align = ((32 << 20) >> 12) - 1;
-			radeon_vm_unbind_locked(rdev, vm);
+			radeon_vm_free_pt(rdev, vm);
 			vm->last_pfn = (last_pfn + align) & ~align;
 		}
 		mutex_unlock(&rdev->vm_manager.lock);
@@ -721,68 +810,60 @@ int radeon_vm_bo_add(struct radeon_device *rdev,
 	head = &vm->va;
 	last_offset = 0;
 	list_for_each_entry(tmp, &vm->va, vm_list) {
-		if (bo_va->soffset >= last_offset && bo_va->eoffset < tmp->soffset) {
+		if (bo_va == tmp) {
+			/* skip over currently modified bo */
+			continue;
+		}
+
+		if (soffset >= last_offset && eoffset <= tmp->soffset) {
 			/* bo can be added before this one */
 			break;
 		}
-		if (bo_va->soffset >= tmp->soffset && bo_va->soffset < tmp->eoffset) {
+		if (eoffset > tmp->soffset && soffset < tmp->eoffset) {
 			/* bo and tmp overlap, invalid offset */
 			dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n",
-				bo, (unsigned)bo_va->soffset, tmp->bo,
+				bo_va->bo, (unsigned)bo_va->soffset, tmp->bo,
 				(unsigned)tmp->soffset, (unsigned)tmp->eoffset);
-			kfree(bo_va);
 			mutex_unlock(&vm->mutex);
 			return -EINVAL;
 		}
 		last_offset = tmp->eoffset;
 		head = &tmp->vm_list;
 	}
-	list_add(&bo_va->vm_list, head);
-	list_add_tail(&bo_va->bo_list, &bo->va);
+
+	bo_va->soffset = soffset;
+	bo_va->eoffset = eoffset;
+	bo_va->flags = flags;
+	bo_va->valid = false;
+	list_move(&bo_va->vm_list, head);
+
 	mutex_unlock(&vm->mutex);
 	return 0;
 }
 
 /**
- * radeon_vm_get_addr - get the physical address of the page
+ * radeon_vm_map_gart - get the physical address of a gart page
  *
  * @rdev: radeon_device pointer
- * @mem: ttm mem
- * @pfn: pfn
+ * @addr: the unmapped addr
  *
  * Look up the physical address of the page that the pte resolves
  * to (cayman+).
  * Returns the physical address of the page.
  */
-static u64 radeon_vm_get_addr(struct radeon_device *rdev,
-			      struct ttm_mem_reg *mem,
-			      unsigned pfn)
+uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
 {
-	u64 addr = 0;
-
-	switch (mem->mem_type) {
-	case TTM_PL_VRAM:
-		addr = (mem->start << PAGE_SHIFT);
-		addr += pfn * RADEON_GPU_PAGE_SIZE;
-		addr += rdev->vm_manager.vram_base_offset;
-		break;
-	case TTM_PL_TT:
-		/* offset inside page table */
-		addr = mem->start << PAGE_SHIFT;
-		addr += pfn * RADEON_GPU_PAGE_SIZE;
-		addr = addr >> PAGE_SHIFT;
-		/* page table offset */
-		addr = rdev->gart.pages_addr[addr];
-		/* in case cpu page size != gpu page size*/
-		addr += (pfn * RADEON_GPU_PAGE_SIZE) & (~PAGE_MASK);
-		break;
-	default:
-		break;
-	}
-	return addr;
+	uint64_t result;
+
+	/* page table offset */
+	result = rdev->gart.pages_addr[addr >> PAGE_SHIFT];
+
+	/* in case cpu page size != gpu page size*/
+	result |= addr & (~PAGE_MASK);
+
+	return result;
 }
 
-/* object have to be reserved & global and local mutex must be locked */
 /**
  * radeon_vm_bo_update_pte - map a bo into the vm page table
  *
@@ -793,103 +874,160 @@ static u64 radeon_vm_get_addr(struct radeon_device *rdev,
  *
  * Fill in the page table entries for @bo (cayman+).
  * Returns 0 for success, -EINVAL for failure.
+ *
+ * Object have to be reserved & global and local mutex must be locked!
  */
 int radeon_vm_bo_update_pte(struct radeon_device *rdev,
 			    struct radeon_vm *vm,
 			    struct radeon_bo *bo,
 			    struct ttm_mem_reg *mem)
 {
+	unsigned ridx = rdev->asic->vm.pt_ring_index;
+	struct radeon_ring *ring = &rdev->ring[ridx];
+	struct radeon_semaphore *sem = NULL;
 	struct radeon_bo_va *bo_va;
-	unsigned ngpu_pages, i;
-	uint64_t addr = 0, pfn;
-	uint32_t flags;
+	unsigned nptes, npdes, ndw;
+	uint64_t pe, addr;
+	uint64_t pfn;
+	int r;
 
 	/* nothing to do if vm isn't bound */
-	if (vm->id == -1)
+	if (vm->sa_bo == NULL)
 		return 0;
 
-	bo_va = radeon_bo_va(bo, vm);
+	bo_va = radeon_vm_bo_find(vm, bo);
 	if (bo_va == NULL) {
 		dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
 		return -EINVAL;
 	}
 
-	if (bo_va->valid && mem)
+	if (!bo_va->soffset) {
+		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
+			bo, vm);
+		return -EINVAL;
+	}
+
+	if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
 		return 0;
 
-	ngpu_pages = radeon_bo_ngpu_pages(bo);
 	bo_va->flags &= ~RADEON_VM_PAGE_VALID;
 	bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
 	if (mem) {
+		addr = mem->start << PAGE_SHIFT;
 		if (mem->mem_type != TTM_PL_SYSTEM) {
 			bo_va->flags |= RADEON_VM_PAGE_VALID;
 			bo_va->valid = true;
 		}
 		if (mem->mem_type == TTM_PL_TT) {
 			bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
+		} else {
+			addr += rdev->vm_manager.vram_base_offset;
 		}
+	} else {
+		addr = 0;
+		bo_va->valid = false;
 	}
-	pfn = bo_va->soffset / RADEON_GPU_PAGE_SIZE;
-	flags = rdev->vm_manager.funcs->page_flags(rdev, bo_va->vm, bo_va->flags);
-	for (i = 0, addr = 0; i < ngpu_pages; i++) {
-		if (mem && bo_va->valid) {
-			addr = radeon_vm_get_addr(rdev, mem, i);
+
+	if (vm->fence && radeon_fence_signaled(vm->fence)) {
+		radeon_fence_unref(&vm->fence);
+	}
+
+	if (vm->fence && vm->fence->ring != ridx) {
+		r = radeon_semaphore_create(rdev, &sem);
+		if (r) {
+			return r;
 		}
-		rdev->vm_manager.funcs->set_page(rdev, bo_va->vm, i + pfn, addr, flags);
 	}
-	rdev->vm_manager.funcs->tlb_flush(rdev, bo_va->vm);
+
+	/* estimate number of dw needed */
+	/* reserve space for 32-bit padding */
+	ndw = 32;
+
+	nptes = radeon_bo_ngpu_pages(bo);
+
+	pfn = (bo_va->soffset / RADEON_GPU_PAGE_SIZE);
+
+	/* handle cases where a bo spans several pdes  */
+	npdes = (ALIGN(pfn + nptes, RADEON_VM_PTE_COUNT) -
+		 (pfn & ~(RADEON_VM_PTE_COUNT - 1))) >> RADEON_VM_BLOCK_SIZE;
+
+	/* reserve space for one header for every 2k dwords */
+	ndw += (nptes >> 11) * 3;
+	/* reserve space for pte addresses */
+	ndw += nptes * 2;
+
+	/* reserve space for one header for every 2k dwords */
+	ndw += (npdes >> 11) * 3;
+	/* reserve space for pde addresses */
+	ndw += npdes * 2;
+
+	r = radeon_ring_lock(rdev, ring, ndw);
+	if (r) {
+		return r;
+	}
+
+	if (sem && radeon_fence_need_sync(vm->fence, ridx)) {
+		radeon_semaphore_sync_rings(rdev, sem, vm->fence->ring, ridx);
+		radeon_fence_note_sync(vm->fence, ridx);
+	}
+
+	/* update page table entries */
+	pe = vm->pd_gpu_addr;
+	pe += radeon_vm_directory_size(rdev);
+	pe += (bo_va->soffset / RADEON_GPU_PAGE_SIZE) * 8;
+
+	radeon_asic_vm_set_page(rdev, pe, addr, nptes,
+				RADEON_GPU_PAGE_SIZE, bo_va->flags);
+
+	/* update page directory entries */
+	addr = pe;
+
+	pe = vm->pd_gpu_addr;
+	pe += ((bo_va->soffset / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE) * 8;
+
+	radeon_asic_vm_set_page(rdev, pe, addr, npdes,
+				RADEON_VM_PTE_COUNT * 8, RADEON_VM_PAGE_VALID);
+
+	radeon_fence_unref(&vm->fence);
+	r = radeon_fence_emit(rdev, &vm->fence, ridx);
+	if (r) {
+		radeon_ring_unlock_undo(rdev, ring);
+		return r;
+	}
+	radeon_ring_unlock_commit(rdev, ring);
+	radeon_semaphore_free(rdev, &sem, vm->fence);
+	radeon_fence_unref(&vm->last_flush);
 	return 0;
 }
 
-/* object have to be reserved */
 /**
  * radeon_vm_bo_rmv - remove a bo to a specific vm
  *
  * @rdev: radeon_device pointer
- * @vm: requested vm
- * @bo: radeon buffer object
+ * @bo_va: requested bo_va
  *
- * Remove @bo from the requested vm (cayman+).
- * Remove @bo from the list of bos associated with the vm and
- * remove the ptes for @bo in the page table.
+ * Remove @bo_va->bo from the requested vm (cayman+).
+ * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and
+ * remove the ptes for @bo_va in the page table.
  * Returns 0 for success.
+ *
+ * Object have to be reserved!
  */
 int radeon_vm_bo_rmv(struct radeon_device *rdev,
-		     struct radeon_vm *vm,
-		     struct radeon_bo *bo)
+		     struct radeon_bo_va *bo_va)
 {
-	struct radeon_bo_va *bo_va;
 	int r;
 
-	bo_va = radeon_bo_va(bo, vm);
-	if (bo_va == NULL)
-		return 0;
-
-	/* wait for va use to end */
-	while (bo_va->fence) {
-		r = radeon_fence_wait(bo_va->fence, false);
-		if (r) {
-			DRM_ERROR("error while waiting for fence: %d\n", r);
-		}
-		if (r == -EDEADLK) {
-			r = radeon_gpu_reset(rdev);
-			if (!r)
-				continue;
-		}
-		break;
-	}
-	radeon_fence_unref(&bo_va->fence);
-
 	mutex_lock(&rdev->vm_manager.lock);
-	mutex_lock(&vm->mutex);
-	radeon_vm_bo_update_pte(rdev, vm, bo, NULL);
+	mutex_lock(&bo_va->vm->mutex);
+	r = radeon_vm_bo_update_pte(rdev, bo_va->vm, bo_va->bo, NULL);
 	mutex_unlock(&rdev->vm_manager.lock);
 	list_del(&bo_va->vm_list);
-	mutex_unlock(&vm->mutex);
+	mutex_unlock(&bo_va->vm->mutex);
 	list_del(&bo_va->bo_list);
 
 	kfree(bo_va);
-	return 0;
+	return r;
 }
 
 /**
@@ -925,27 +1063,23 @@ void radeon_vm_bo_invalidate(struct radeon_device *rdev,
  */
 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
 {
+	struct radeon_bo_va *bo_va;
 	int r;
 
-	vm->id = -1;
+	vm->id = 0;
 	vm->fence = NULL;
+	vm->last_pfn = 0;
 	mutex_init(&vm->mutex);
 	INIT_LIST_HEAD(&vm->list);
 	INIT_LIST_HEAD(&vm->va);
-	/* SI requires equal sized PTs for all VMs, so always set
-	 * last_pfn to max_pfn.  cayman allows variable sized
-	 * pts so we can grow then as needed.  Once we switch
-	 * to two level pts we can unify this again.
-	 */
-	if (rdev->family >= CHIP_TAHITI)
-		vm->last_pfn = rdev->vm_manager.max_pfn;
-	else
-		vm->last_pfn = 0;
+
 	/* map the ib pool buffer at 0 in virtual address space, set
 	 * read only
 	 */
-	r = radeon_vm_bo_add(rdev, vm, rdev->ring_tmp_bo.bo, 0,
-			     RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED);
+	bo_va = radeon_vm_bo_add(rdev, vm, rdev->ring_tmp_bo.bo);
+	r = radeon_vm_bo_set_addr(rdev, bo_va, RADEON_VA_IB_OFFSET,
+				  RADEON_VM_PAGE_READABLE |
+				  RADEON_VM_PAGE_SNOOPED);
 	return r;
 }
 
@@ -965,7 +1099,7 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
 
 	mutex_lock(&rdev->vm_manager.lock);
 	mutex_lock(&vm->mutex);
-	radeon_vm_unbind_locked(rdev, vm);
+	radeon_vm_free_pt(rdev, vm);
 	mutex_unlock(&rdev->vm_manager.lock);
 
 	/* remove all bo at this point non are busy any more because unbind
@@ -973,10 +1107,9 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
 	 */
 	r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
 	if (!r) {
-		bo_va = radeon_bo_va(rdev->ring_tmp_bo.bo, vm);
+		bo_va = radeon_vm_bo_find(vm, rdev->ring_tmp_bo.bo);
 		list_del_init(&bo_va->bo_list);
 		list_del_init(&bo_va->vm_list);
-		radeon_fence_unref(&bo_va->fence);
 		radeon_bo_unreserve(rdev->ring_tmp_bo.bo);
 		kfree(bo_va);
 	}
@@ -988,10 +1121,11 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
 		r = radeon_bo_reserve(bo_va->bo, false);
 		if (!r) {
 			list_del_init(&bo_va->bo_list);
-			radeon_fence_unref(&bo_va->fence);
 			radeon_bo_unreserve(bo_va->bo);
 			kfree(bo_va);
 		}
 	}
+	radeon_fence_unref(&vm->fence);
+	radeon_fence_unref(&vm->last_flush);
 	mutex_unlock(&vm->mutex);
 }
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 1b57b0058ad..f38fbcc4693 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -25,9 +25,8 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 int radeon_gem_object_init(struct drm_gem_object *obj)
@@ -124,6 +123,30 @@ void radeon_gem_fini(struct radeon_device *rdev)
  */
 int radeon_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_priv)
 {
+	struct radeon_bo *rbo = gem_to_radeon_bo(obj);
+	struct radeon_device *rdev = rbo->rdev;
+	struct radeon_fpriv *fpriv = file_priv->driver_priv;
+	struct radeon_vm *vm = &fpriv->vm;
+	struct radeon_bo_va *bo_va;
+	int r;
+
+	if (rdev->family < CHIP_CAYMAN) {
+		return 0;
+	}
+
+	r = radeon_bo_reserve(rbo, false);
+	if (r) {
+		return r;
+	}
+
+	bo_va = radeon_vm_bo_find(vm, rbo);
+	if (!bo_va) {
+		bo_va = radeon_vm_bo_add(rdev, vm, rbo);
+	} else {
+		++bo_va->ref_count;
+	}
+	radeon_bo_unreserve(rbo);
+
 	return 0;
 }
 
@@ -134,16 +157,25 @@ void radeon_gem_object_close(struct drm_gem_object *obj,
 	struct radeon_device *rdev = rbo->rdev;
 	struct radeon_fpriv *fpriv = file_priv->driver_priv;
 	struct radeon_vm *vm = &fpriv->vm;
+	struct radeon_bo_va *bo_va;
+	int r;
 
 	if (rdev->family < CHIP_CAYMAN) {
 		return;
 	}
 
-	if (radeon_bo_reserve(rbo, false)) {
-		dev_err(rdev->dev, "leaking bo va because we fail to reserve bo\n");
+	r = radeon_bo_reserve(rbo, true);
+	if (r) {
+		dev_err(rdev->dev, "leaking bo va because "
+			"we fail to reserve bo (%d)\n", r);
 		return;
 	}
-	radeon_vm_bo_rmv(rdev, vm, rbo);
+	bo_va = radeon_vm_bo_find(vm, rbo);
+	if (bo_va) {
+		if (--bo_va->ref_count == 0) {
+			radeon_vm_bo_rmv(rdev, bo_va);
+		}
+	}
 	radeon_bo_unreserve(rbo);
 }
 
@@ -459,19 +491,24 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
 		drm_gem_object_unreference_unlocked(gobj);
 		return r;
 	}
+	bo_va = radeon_vm_bo_find(&fpriv->vm, rbo);
+	if (!bo_va) {
+		args->operation = RADEON_VA_RESULT_ERROR;
+		drm_gem_object_unreference_unlocked(gobj);
+		return -ENOENT;
+	}
+
 	switch (args->operation) {
 	case RADEON_VA_MAP:
-		bo_va = radeon_bo_va(rbo, &fpriv->vm);
-		if (bo_va) {
+		if (bo_va->soffset) {
 			args->operation = RADEON_VA_RESULT_VA_EXIST;
 			args->offset = bo_va->soffset;
 			goto out;
 		}
-		r = radeon_vm_bo_add(rdev, &fpriv->vm, rbo,
-				     args->offset, args->flags);
+		r = radeon_vm_bo_set_addr(rdev, bo_va, args->offset, args->flags);
 		break;
 	case RADEON_VA_UNMAP:
-		r = radeon_vm_bo_rmv(rdev, &fpriv->vm, rbo);
+		r = radeon_vm_bo_set_addr(rdev, bo_va, 0, 0);
 		break;
 	default:
 		break;
diff --git a/drivers/gpu/drm/radeon/radeon_i2c.c b/drivers/gpu/drm/radeon/radeon_i2c.c
index 3edec1c198e..c5bddd630eb 100644
--- a/drivers/gpu/drm/radeon/radeon_i2c.c
+++ b/drivers/gpu/drm/radeon/radeon_i2c.c
@@ -25,9 +25,9 @@
  */
 #include <linux/export.h>
 
-#include "drmP.h"
-#include "drm_edid.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_edid.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 
diff --git a/drivers/gpu/drm/radeon/radeon_ioc32.c b/drivers/gpu/drm/radeon/radeon_ioc32.c
index 48b7cea31e0..c180df8e84d 100644
--- a/drivers/gpu/drm/radeon/radeon_ioc32.c
+++ b/drivers/gpu/drm/radeon/radeon_ioc32.c
@@ -29,9 +29,8 @@
  */
 #include <linux/compat.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 typedef struct drm_radeon_init32 {
@@ -369,7 +368,7 @@ static int compat_radeon_cp_setparam(struct file *file, unsigned int cmd,
 #define compat_radeon_cp_setparam NULL
 #endif /* X86_64 || IA64 */
 
-drm_ioctl_compat_t *radeon_compat_ioctls[] = {
+static drm_ioctl_compat_t *radeon_compat_ioctls[] = {
 	[DRM_RADEON_CP_INIT] = compat_radeon_cp_init,
 	[DRM_RADEON_CLEAR] = compat_radeon_cp_clear,
 	[DRM_RADEON_STIPPLE] = compat_radeon_cp_stipple,
diff --git a/drivers/gpu/drm/radeon/radeon_irq.c b/drivers/gpu/drm/radeon/radeon_irq.c
index 00da38424df..e7710339a6a 100644
--- a/drivers/gpu/drm/radeon/radeon_irq.c
+++ b/drivers/gpu/drm/radeon/radeon_irq.c
@@ -30,9 +30,8 @@
  *    Michel D�zer <michel@daenzer.net>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 void radeon_irq_set_state(struct drm_device *dev, u32 mask, int state)
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 50b596ec7b7..90374dd7796 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -25,9 +25,9 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "atom.h"
@@ -99,7 +99,6 @@ void radeon_driver_irq_preinstall_kms(struct drm_device *dev)
 	/* Disable *all* interrupts */
 	for (i = 0; i < RADEON_NUM_RINGS; i++)
 		atomic_set(&rdev->irq.ring_int[i], 0);
-	rdev->irq.gui_idle = false;
 	for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
 		rdev->irq.hpd[i] = false;
 	for (i = 0; i < RADEON_MAX_CRTCS; i++) {
@@ -147,7 +146,6 @@ void radeon_driver_irq_uninstall_kms(struct drm_device *dev)
 	/* Disable *all* interrupts */
 	for (i = 0; i < RADEON_NUM_RINGS; i++)
 		atomic_set(&rdev->irq.ring_int[i], 0);
-	rdev->irq.gui_idle = false;
 	for (i = 0; i < RADEON_MAX_HPD_PINS; i++)
 		rdev->irq.hpd[i] = false;
 	for (i = 0; i < RADEON_MAX_CRTCS; i++) {
@@ -204,6 +202,16 @@ static bool radeon_msi_ok(struct radeon_device *rdev)
 	    (rdev->pdev->subsystem_device == 0x01fd))
 		return true;
 
+	/* Gateway RS690 only seems to work with MSIs. */
+	if ((rdev->pdev->device == 0x791f) &&
+	    (rdev->pdev->subsystem_vendor == 0x107b) &&
+	    (rdev->pdev->subsystem_device == 0x0185))
+		return true;
+
+	/* try and enable MSIs by default on all RS690s */
+	if (rdev->family == CHIP_RS690)
+		return true;
+
 	/* RV515 seems to have MSI issues where it loses
 	 * MSI rearms occasionally. This leads to lockups and freezes.
 	 * disable it by default.
@@ -457,34 +465,3 @@ void radeon_irq_kms_disable_hpd(struct radeon_device *rdev, unsigned hpd_mask)
 	spin_unlock_irqrestore(&rdev->irq.lock, irqflags);
 }
 
-/**
- * radeon_irq_kms_wait_gui_idle - waits for drawing engine to be idle
- *
- * @rdev: radeon device pointer
- *
- * Enabled the GUI idle interrupt and waits for it to fire (r6xx+).
- * This is currently used to make sure the 3D engine is idle for power
- * management, but should be replaces with proper fence waits.
- * GUI idle interrupts don't work very well on pre-r6xx hw and it also
- * does not take into account other aspects of the chip that may be busy.
- * DO NOT USE GOING FORWARD.
- */
-int radeon_irq_kms_wait_gui_idle(struct radeon_device *rdev)
-{
-	unsigned long irqflags;
-	int r;
-
-	spin_lock_irqsave(&rdev->irq.lock, irqflags);
-	rdev->irq.gui_idle = true;
-	radeon_irq_set(rdev);
-	spin_unlock_irqrestore(&rdev->irq.lock, irqflags);
-
-	r = wait_event_timeout(rdev->irq.idle_queue, radeon_gui_idle(rdev),
-			       msecs_to_jiffies(RADEON_WAIT_IDLE_TIMEOUT));
-
-	spin_lock_irqsave(&rdev->irq.lock, irqflags);
-	rdev->irq.gui_idle = false;
-	radeon_irq_set(rdev);
-	spin_unlock_irqrestore(&rdev->irq.lock, irqflags);
-	return r;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 414b4acf694..83b8d8aa71c 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -25,10 +25,9 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
-#include "drm_sarea.h"
+#include <drm/drmP.h>
 #include "radeon.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "radeon_asic.h"
 
 #include <linux/vga_switcheroo.h>
@@ -51,6 +50,7 @@ int radeon_driver_unload_kms(struct drm_device *dev)
 
 	if (rdev == NULL)
 		return 0;
+	radeon_acpi_fini(rdev);
 	radeon_modeset_fini(rdev);
 	radeon_device_fini(rdev);
 	kfree(rdev);
@@ -103,11 +103,6 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
 		goto out;
 	}
 
-	/* Call ACPI methods */
-	acpi_status = radeon_acpi_init(rdev);
-	if (acpi_status)
-		dev_dbg(&dev->pdev->dev, "Error during ACPI methods call\n");
-
 	/* Again modeset_init should fail only on fatal error
 	 * otherwise it should provide enough functionalities
 	 * for shadowfb to run
@@ -115,6 +110,17 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
 	r = radeon_modeset_init(rdev);
 	if (r)
 		dev_err(&dev->pdev->dev, "Fatal error during modeset init\n");
+
+	/* Call ACPI methods: require modeset init
+	 * but failure is not fatal
+	 */
+	if (!r) {
+		acpi_status = radeon_acpi_init(rdev);
+		if (acpi_status)
+		dev_dbg(&dev->pdev->dev,
+				"Error during ACPI methods call\n");
+	}
+
 out:
 	if (r)
 		radeon_driver_unload_kms(dev);
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 94b4a1c1289..5677a424b58 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -206,11 +206,6 @@ static void radeon_legacy_rmx_mode_set(struct drm_crtc *crtc,
 	WREG32(RADEON_FP_CRTC_V_TOTAL_DISP, fp_crtc_v_total_disp);
 }
 
-void radeon_restore_common_regs(struct drm_device *dev)
-{
-	/* don't need this yet */
-}
-
 static void radeon_pll_wait_for_read_update_complete(struct drm_device *dev)
 {
 	struct radeon_device *rdev = dev->dev_private;
@@ -295,7 +290,7 @@ static uint8_t radeon_compute_pll_gain(uint16_t ref_freq, uint16_t ref_div,
 		return 1;
 }
 
-void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
+static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 670e9910f86..92487e61477 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -23,9 +23,9 @@
  * Authors: Dave Airlie
  *          Alex Deucher
  */
-#include "drmP.h"
-#include "drm_crtc_helper.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "atom.h"
 #include <linux/backlight.h>
@@ -271,13 +271,6 @@ static const struct drm_encoder_helper_funcs radeon_legacy_lvds_helper_funcs = {
 
 #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
 
-#define MAX_RADEON_LEVEL 0xFF
-
-struct radeon_backlight_privdata {
-	struct radeon_encoder *encoder;
-	uint8_t negative;
-};
-
 static uint8_t radeon_legacy_lvds_level(struct backlight_device *bd)
 {
 	struct radeon_backlight_privdata *pdata = bl_get_data(bd);
@@ -286,21 +279,33 @@ static uint8_t radeon_legacy_lvds_level(struct backlight_device *bd)
 	/* Convert brightness to hardware level */
 	if (bd->props.brightness < 0)
 		level = 0;
-	else if (bd->props.brightness > MAX_RADEON_LEVEL)
-		level = MAX_RADEON_LEVEL;
+	else if (bd->props.brightness > RADEON_MAX_BL_LEVEL)
+		level = RADEON_MAX_BL_LEVEL;
 	else
 		level = bd->props.brightness;
 
 	if (pdata->negative)
-		level = MAX_RADEON_LEVEL - level;
+		level = RADEON_MAX_BL_LEVEL - level;
 
 	return level;
 }
 
-static int radeon_legacy_backlight_update_status(struct backlight_device *bd)
+u8
+radeon_legacy_get_backlight_level(struct radeon_encoder *radeon_encoder)
+{
+	struct drm_device *dev = radeon_encoder->base.dev;
+	struct radeon_device *rdev = dev->dev_private;
+	u8 backlight_level;
+
+	backlight_level = (RREG32(RADEON_LVDS_GEN_CNTL) >>
+			   RADEON_LVDS_BL_MOD_LEVEL_SHIFT) & 0xff;
+
+	return backlight_level;
+}
+
+void
+radeon_legacy_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level)
 {
-	struct radeon_backlight_privdata *pdata = bl_get_data(bd);
-	struct radeon_encoder *radeon_encoder = pdata->encoder;
 	struct drm_device *dev = radeon_encoder->base.dev;
 	struct radeon_device *rdev = dev->dev_private;
 	int dpms_mode = DRM_MODE_DPMS_ON;
@@ -308,19 +313,31 @@ static int radeon_legacy_backlight_update_status(struct backlight_device *bd)
 	if (radeon_encoder->enc_priv) {
 		if (rdev->is_atom_bios) {
 			struct radeon_encoder_atom_dig *lvds = radeon_encoder->enc_priv;
-			dpms_mode = lvds->dpms_mode;
-			lvds->backlight_level = radeon_legacy_lvds_level(bd);
+			if (lvds->backlight_level > 0)
+				dpms_mode = lvds->dpms_mode;
+			else
+				dpms_mode = DRM_MODE_DPMS_OFF;
+			lvds->backlight_level = level;
 		} else {
 			struct radeon_encoder_lvds *lvds = radeon_encoder->enc_priv;
-			dpms_mode = lvds->dpms_mode;
-			lvds->backlight_level = radeon_legacy_lvds_level(bd);
+			if (lvds->backlight_level > 0)
+				dpms_mode = lvds->dpms_mode;
+			else
+				dpms_mode = DRM_MODE_DPMS_OFF;
+			lvds->backlight_level = level;
 		}
 	}
 
-	if (bd->props.brightness > 0)
-		radeon_legacy_lvds_update(&radeon_encoder->base, dpms_mode);
-	else
-		radeon_legacy_lvds_update(&radeon_encoder->base, DRM_MODE_DPMS_OFF);
+	radeon_legacy_lvds_update(&radeon_encoder->base, dpms_mode);
+}
+
+static int radeon_legacy_backlight_update_status(struct backlight_device *bd)
+{
+	struct radeon_backlight_privdata *pdata = bl_get_data(bd);
+	struct radeon_encoder *radeon_encoder = pdata->encoder;
+
+	radeon_legacy_set_backlight_level(radeon_encoder,
+					  radeon_legacy_lvds_level(bd));
 
 	return 0;
 }
@@ -336,7 +353,7 @@ static int radeon_legacy_backlight_get_brightness(struct backlight_device *bd)
 	backlight_level = (RREG32(RADEON_LVDS_GEN_CNTL) >>
 			   RADEON_LVDS_BL_MOD_LEVEL_SHIFT) & 0xff;
 
-	return pdata->negative ? MAX_RADEON_LEVEL - backlight_level : backlight_level;
+	return pdata->negative ? RADEON_MAX_BL_LEVEL - backlight_level : backlight_level;
 }
 
 static const struct backlight_ops radeon_backlight_ops = {
@@ -370,7 +387,7 @@ void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
 	}
 
 	memset(&props, 0, sizeof(props));
-	props.max_brightness = MAX_RADEON_LEVEL;
+	props.max_brightness = RADEON_MAX_BL_LEVEL;
 	props.type = BACKLIGHT_RAW;
 	bd = backlight_device_register("radeon_bl", &drm_connector->kdev,
 				       pdata, &radeon_backlight_ops, &props);
@@ -449,7 +466,7 @@ static void radeon_legacy_backlight_exit(struct radeon_encoder *radeon_encoder)
 	}
 
 	if (bd) {
-		struct radeon_legacy_backlight_privdata *pdata;
+		struct radeon_backlight_privdata *pdata;
 
 		pdata = bl_get_data(bd);
 		backlight_device_unregister(bd);
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_tv.c b/drivers/gpu/drm/radeon/radeon_legacy_tv.c
index b37ec0f1413..49750d07ab7 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_tv.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_tv.c
@@ -1,5 +1,5 @@
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 #include "radeon.h"
 
 /*
diff --git a/drivers/gpu/drm/radeon/radeon_mem.c b/drivers/gpu/drm/radeon/radeon_mem.c
index 988548efea9..b9f06724163 100644
--- a/drivers/gpu/drm/radeon/radeon_mem.c
+++ b/drivers/gpu/drm/radeon/radeon_mem.c
@@ -29,9 +29,8 @@
  *    Keith Whitwell <keith@tungstengraphics.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 /* Very simple allocator for GART memory, working on a static range
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index d56978949f3..92c5f473cf0 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -30,12 +30,11 @@
 #ifndef RADEON_MODE_H
 #define RADEON_MODE_H
 
-#include <drm_crtc.h>
-#include <drm_mode.h>
-#include <drm_edid.h>
-#include <drm_dp_helper.h>
-#include <drm_fixed.h>
-#include <drm_crtc_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_fixed.h>
+#include <drm/drm_crtc_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 
@@ -252,8 +251,23 @@ struct radeon_mode_info {
 
 	/* pointer to fbdev info structure */
 	struct radeon_fbdev *rfbdev;
+	/* firmware flags */
+	u16 firmware_flags;
+	/* pointer to backlight encoder */
+	struct radeon_encoder *bl_encoder;
 };
 
+#define RADEON_MAX_BL_LEVEL 0xFF
+
+#if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE)
+
+struct radeon_backlight_privdata {
+	struct radeon_encoder *encoder;
+	uint8_t negative;
+};
+
+#endif
+
 #define MAX_H_CODE_TIMING_LEN 32
 #define MAX_V_CODE_TIMING_LEN 32
 
@@ -269,6 +283,18 @@ struct radeon_tv_regs {
 	uint16_t v_code_timing[MAX_V_CODE_TIMING_LEN];
 };
 
+struct radeon_atom_ss {
+	uint16_t percentage;
+	uint8_t type;
+	uint16_t step;
+	uint8_t delay;
+	uint8_t range;
+	uint8_t refdiv;
+	/* asic_ss */
+	uint16_t rate;
+	uint16_t amount;
+};
+
 struct radeon_crtc {
 	struct drm_crtc base;
 	int crtc_id;
@@ -293,6 +319,16 @@ struct radeon_crtc {
 	/* page flipping */
 	struct radeon_unpin_work *unpin_work;
 	int deferred_flip_completion;
+	/* pll sharing */
+	struct radeon_atom_ss ss;
+	bool ss_enabled;
+	u32 adjusted_clock;
+	int bpc;
+	u32 pll_reference_div;
+	u32 pll_post_div;
+	u32 pll_flags;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
 };
 
 struct radeon_encoder_primary_dac {
@@ -346,18 +382,6 @@ struct radeon_encoder_ext_tmds {
 };
 
 /* spread spectrum */
-struct radeon_atom_ss {
-	uint16_t percentage;
-	uint8_t type;
-	uint16_t step;
-	uint8_t delay;
-	uint8_t range;
-	uint8_t refdiv;
-	/* asic_ss */
-	uint16_t rate;
-	uint16_t amount;
-};
-
 struct radeon_encoder_atom_dig {
 	bool linkb;
 	/* atom dig */
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 9024e722283..8b27dd6e314 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -32,7 +32,7 @@
 #include <linux/list.h>
 #include <linux/slab.h>
 #include <drm/drmP.h>
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 #include "radeon_trace.h"
 
@@ -52,7 +52,7 @@ void radeon_bo_clear_va(struct radeon_bo *bo)
 
 	list_for_each_entry_safe(bo_va, tmp, &bo->va, bo_list) {
 		/* remove from all vm address space */
-		radeon_vm_bo_rmv(bo->rdev, bo_va->vm, bo);
+		radeon_vm_bo_rmv(bo->rdev, bo_va);
 	}
 }
 
@@ -627,18 +627,17 @@ int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)
 /**
  * radeon_bo_reserve - reserve bo
  * @bo:		bo structure
- * @no_wait:		don't sleep while trying to reserve (return -EBUSY)
+ * @no_intr:	don't return -ERESTARTSYS on pending signal
  *
  * Returns:
- * -EBUSY: buffer is busy and @no_wait is true
  * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
  * a signal. Release all buffer reservations and return to user-space.
  */
-int radeon_bo_reserve(struct radeon_bo *bo, bool no_wait)
+int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr)
 {
 	int r;
 
-	r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, 0);
+	r = ttm_bo_reserve(&bo->tbo, !no_intr, false, false, 0);
 	if (unlikely(r != 0)) {
 		if (r != -ERESTARTSYS)
 			dev_err(bo->rdev->dev, "%p reserve failed\n", bo);
@@ -646,16 +645,3 @@ int radeon_bo_reserve(struct radeon_bo *bo, bool no_wait)
 	}
 	return 0;
 }
-
-/* object have to be reserved */
-struct radeon_bo_va *radeon_bo_va(struct radeon_bo *rbo, struct radeon_vm *vm)
-{
-	struct radeon_bo_va *bo_va;
-
-	list_for_each_entry(bo_va, &rbo->va, bo_list) {
-		if (bo_va->vm == vm) {
-			return bo_va;
-		}
-	}
-	return NULL;
-}
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 17fb99f177c..93cd491fff2 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -52,7 +52,7 @@ static inline unsigned radeon_mem_type_to_domain(u32 mem_type)
 	return 0;
 }
 
-int radeon_bo_reserve(struct radeon_bo *bo, bool no_wait);
+int radeon_bo_reserve(struct radeon_bo *bo, bool no_intr);
 
 static inline void radeon_bo_unreserve(struct radeon_bo *bo)
 {
@@ -141,8 +141,6 @@ extern void radeon_bo_move_notify(struct ttm_buffer_object *bo,
 					struct ttm_mem_reg *mem);
 extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 extern int radeon_bo_get_surface_reg(struct radeon_bo *bo);
-extern struct radeon_bo_va *radeon_bo_va(struct radeon_bo *rbo,
-					 struct radeon_vm *vm);
 
 /*
  * sub allocation
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 7ae60660010..aa14dbb7e4f 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -20,13 +20,10 @@
  * Authors: Rafał Miłecki <zajec5@gmail.com>
  *          Alex Deucher <alexdeucher@gmail.com>
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "avivod.h"
 #include "atom.h"
-#ifdef CONFIG_ACPI
-#include <linux/acpi.h>
-#endif
 #include <linux/power_supply.h>
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
@@ -36,7 +33,7 @@
 #define RADEON_WAIT_VBLANK_TIMEOUT 200
 
 static const char *radeon_pm_state_type_name[5] = {
-	"Default",
+	"",
 	"Powersave",
 	"Battery",
 	"Balanced",
@@ -50,8 +47,6 @@ static bool radeon_pm_debug_check_in_vbl(struct radeon_device *rdev, bool finish
 static void radeon_pm_update_profile(struct radeon_device *rdev);
 static void radeon_pm_set_clocks(struct radeon_device *rdev);
 
-#define ACPI_AC_CLASS           "ac_adapter"
-
 int radeon_pm_get_type_index(struct radeon_device *rdev,
 			     enum radeon_pm_state_type ps_type,
 			     int instance)
@@ -70,33 +65,17 @@ int radeon_pm_get_type_index(struct radeon_device *rdev,
 	return rdev->pm.default_power_state_index;
 }
 
-#ifdef CONFIG_ACPI
-static int radeon_acpi_event(struct notifier_block *nb,
-			     unsigned long val,
-			     void *data)
+void radeon_pm_acpi_event_handler(struct radeon_device *rdev)
 {
-	struct radeon_device *rdev = container_of(nb, struct radeon_device, acpi_nb);
-	struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
-
-	if (strcmp(entry->device_class, ACPI_AC_CLASS) == 0) {
-		if (power_supply_is_system_supplied() > 0)
-			DRM_DEBUG_DRIVER("pm: AC\n");
-		else
-			DRM_DEBUG_DRIVER("pm: DC\n");
-
-		if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
-			if (rdev->pm.profile == PM_PROFILE_AUTO) {
-				mutex_lock(&rdev->pm.mutex);
-				radeon_pm_update_profile(rdev);
-				radeon_pm_set_clocks(rdev);
-				mutex_unlock(&rdev->pm.mutex);
-			}
+	if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
+		if (rdev->pm.profile == PM_PROFILE_AUTO) {
+			mutex_lock(&rdev->pm.mutex);
+			radeon_pm_update_profile(rdev);
+			radeon_pm_set_clocks(rdev);
+			mutex_unlock(&rdev->pm.mutex);
 		}
 	}
-
-	return NOTIFY_OK;
 }
-#endif
 
 static void radeon_pm_update_profile(struct radeon_device *rdev)
 {
@@ -188,8 +167,21 @@ static void radeon_set_power_state(struct radeon_device *rdev)
 		if (sclk > rdev->pm.default_sclk)
 			sclk = rdev->pm.default_sclk;
 
-		mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
-			clock_info[rdev->pm.requested_clock_mode_index].mclk;
+		/* starting with BTC, there is one state that is used for both
+		 * MH and SH.  Difference is that we always use the high clock index for
+		 * mclk.
+		 */
+		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
+		    (rdev->family >= CHIP_BARTS) &&
+		    rdev->pm.active_crtc_count &&
+		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
+		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
+			mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
+				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].mclk;
+		else
+			mclk = rdev->pm.power_state[rdev->pm.requested_power_state_index].
+				clock_info[rdev->pm.requested_clock_mode_index].mclk;
+
 		if (mclk > rdev->pm.default_mclk)
 			mclk = rdev->pm.default_mclk;
 
@@ -253,18 +245,13 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
 	down_write(&rdev->pm.mclk_lock);
 	mutex_lock(&rdev->ring_lock);
 
-	/* gui idle int has issues on older chips it seems */
-	if (rdev->family >= CHIP_R600) {
-		if (rdev->irq.installed) {
-			/* wait for GPU to become idle */
-			radeon_irq_kms_wait_gui_idle(rdev);
-		}
-	} else {
-		struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
-		if (ring->ready) {
-			radeon_fence_wait_empty_locked(rdev, RADEON_RING_TYPE_GFX_INDEX);
-		}
+	/* wait for the rings to drain */
+	for (i = 0; i < RADEON_NUM_RINGS; i++) {
+		struct radeon_ring *ring = &rdev->ring[i];
+		if (ring->ready)
+			radeon_fence_wait_empty_locked(rdev, i);
 	}
+
 	radeon_unmap_vram_bos(rdev);
 
 	if (rdev->irq.installed) {
@@ -320,17 +307,15 @@ static void radeon_pm_print_states(struct radeon_device *rdev)
 		for (j = 0; j < power_state->num_clock_modes; j++) {
 			clock_info = &(power_state->clock_info[j]);
 			if (rdev->flags & RADEON_IS_IGP)
-				DRM_DEBUG_DRIVER("\t\t%d e: %d%s\n",
-					j,
-					clock_info->sclk * 10,
-					clock_info->flags & RADEON_PM_MODE_NO_DISPLAY ? "\tNo display only" : "");
+				DRM_DEBUG_DRIVER("\t\t%d e: %d\n",
+						 j,
+						 clock_info->sclk * 10);
 			else
-				DRM_DEBUG_DRIVER("\t\t%d e: %d\tm: %d\tv: %d%s\n",
-					j,
-					clock_info->sclk * 10,
-					clock_info->mclk * 10,
-					clock_info->voltage.voltage,
-					clock_info->flags & RADEON_PM_MODE_NO_DISPLAY ? "\tNo display only" : "");
+				DRM_DEBUG_DRIVER("\t\t%d e: %d\tm: %d\tv: %d\n",
+						 j,
+						 clock_info->sclk * 10,
+						 clock_info->mclk * 10,
+						 clock_info->voltage.voltage);
 		}
 	}
 }
@@ -547,7 +532,9 @@ void radeon_pm_suspend(struct radeon_device *rdev)
 void radeon_pm_resume(struct radeon_device *rdev)
 {
 	/* set up the default clocks if the MC ucode is loaded */
-	if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+	if ((rdev->family >= CHIP_BARTS) &&
+	    (rdev->family <= CHIP_CAYMAN) &&
+	    rdev->mc_fw) {
 		if (rdev->pm.default_vddc)
 			radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
 						SET_VOLTAGE_TYPE_ASIC_VDDC);
@@ -602,7 +589,9 @@ int radeon_pm_init(struct radeon_device *rdev)
 		radeon_pm_print_states(rdev);
 		radeon_pm_init_profile(rdev);
 		/* set up the default clocks if the MC ucode is loaded */
-		if (ASIC_IS_DCE5(rdev) && rdev->mc_fw) {
+		if ((rdev->family >= CHIP_BARTS) &&
+		    (rdev->family <= CHIP_CAYMAN) &&
+		    rdev->mc_fw) {
 			if (rdev->pm.default_vddc)
 				radeon_atom_set_voltage(rdev, rdev->pm.default_vddc,
 							SET_VOLTAGE_TYPE_ASIC_VDDC);
@@ -632,10 +621,6 @@ int radeon_pm_init(struct radeon_device *rdev)
 		if (ret)
 			DRM_ERROR("failed to create device file for power method\n");
 
-#ifdef CONFIG_ACPI
-		rdev->acpi_nb.notifier_call = radeon_acpi_event;
-		register_acpi_notifier(&rdev->acpi_nb);
-#endif
 		if (radeon_debugfs_pm_init(rdev)) {
 			DRM_ERROR("Failed to register debugfs file for PM!\n");
 		}
@@ -666,9 +651,6 @@ void radeon_pm_fini(struct radeon_device *rdev)
 
 		device_remove_file(rdev->dev, &dev_attr_power_profile);
 		device_remove_file(rdev->dev, &dev_attr_power_method);
-#ifdef CONFIG_ACPI
-		unregister_acpi_notifier(&rdev->acpi_nb);
-#endif
 	}
 
 	if (rdev->pm.power_state)
diff --git a/drivers/gpu/drm/radeon/radeon_prime.c b/drivers/gpu/drm/radeon/radeon_prime.c
index 6bef46ace83..e09521858f6 100644
--- a/drivers/gpu/drm/radeon/radeon_prime.c
+++ b/drivers/gpu/drm/radeon/radeon_prime.c
@@ -23,11 +23,10 @@
  *
  * Authors: Alex Deucher
  */
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 
 #include "radeon.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 
 #include <linux/dma-buf.h>
 
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 43c431a2686..bba66902c83 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -28,8 +28,8 @@
  */
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include "drmP.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "atom.h"
@@ -43,7 +43,7 @@
  * produce command buffers which are send to the kernel and
  * put in IBs for execution by the requested ring.
  */
-int radeon_debugfs_sa_init(struct radeon_device *rdev);
+static int radeon_debugfs_sa_init(struct radeon_device *rdev);
 
 /**
  * radeon_ib_get - request an IB (Indirect Buffer)
@@ -58,7 +58,8 @@ int radeon_debugfs_sa_init(struct radeon_device *rdev);
  * Returns 0 on success, error on failure.
  */
 int radeon_ib_get(struct radeon_device *rdev, int ring,
-		  struct radeon_ib *ib, unsigned size)
+		  struct radeon_ib *ib, struct radeon_vm *vm,
+		  unsigned size)
 {
 	int i, r;
 
@@ -76,8 +77,15 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 	ib->ring = ring;
 	ib->fence = NULL;
 	ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
-	ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
-	ib->vm_id = 0;
+	ib->vm = vm;
+	if (vm) {
+		/* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address
+		 * space and soffset is the offset inside the pool bo
+		 */
+		ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
+	} else {
+		ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
+	}
 	ib->is_const_ib = false;
 	for (i = 0; i < RADEON_NUM_RINGS; ++i)
 		ib->sync_to[i] = NULL;
@@ -152,6 +160,10 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
 	if (!need_sync) {
 		radeon_semaphore_free(rdev, &ib->semaphore, NULL);
 	}
+	/* if we can't remember our last VM flush then flush now! */
+	if (ib->vm && !ib->vm->last_flush) {
+		radeon_ring_vm_flush(rdev, ib->ring, ib->vm);
+	}
 	if (const_ib) {
 		radeon_ring_ib_execute(rdev, const_ib->ring, const_ib);
 		radeon_semaphore_free(rdev, &const_ib->semaphore, NULL);
@@ -166,6 +178,10 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
 	if (const_ib) {
 		const_ib->fence = radeon_fence_ref(ib->fence);
 	}
+	/* we just flushed the VM, remember that */
+	if (ib->vm && !ib->vm->last_flush) {
+		ib->vm->last_flush = radeon_fence_ref(ib->fence);
+	}
 	radeon_ring_unlock_commit(rdev, ring);
 	return 0;
 }
@@ -275,7 +291,7 @@ int radeon_ib_ring_tests(struct radeon_device *rdev)
  * wptr.  The GPU then starts fetching commands and executes
  * them until the pointers are equal again.
  */
-int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
+static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
 
 /**
  * radeon_ring_write - write a value to the ring
@@ -803,7 +819,7 @@ static struct drm_info_list radeon_debugfs_sa_list[] = {
 
 #endif
 
-int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
+static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
 {
 #if defined(CONFIG_DEBUG_FS)
 	unsigned i;
@@ -823,7 +839,7 @@ int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *rin
 	return 0;
 }
 
-int radeon_debugfs_sa_init(struct radeon_device *rdev)
+static int radeon_debugfs_sa_init(struct radeon_device *rdev)
 {
 #if defined(CONFIG_DEBUG_FS)
 	return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1);
diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c
index 4e771240fdd..cb800995d4f 100644
--- a/drivers/gpu/drm/radeon/radeon_sa.c
+++ b/drivers/gpu/drm/radeon/radeon_sa.c
@@ -41,8 +41,7 @@
  * If we are asked to block we wait on all the oldest fence of all
  * rings. We just wait for any of those fence to complete.
  */
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 
 static void radeon_sa_bo_remove_locked(struct radeon_sa_bo *sa_bo);
@@ -316,7 +315,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev,
 {
 	struct radeon_fence *fences[RADEON_NUM_RINGS];
 	unsigned tries[RADEON_NUM_RINGS];
-	int i, r = -ENOMEM;
+	int i, r;
 
 	BUG_ON(align > RADEON_GPU_PAGE_SIZE);
 	BUG_ON(size > sa_manager->size);
@@ -331,7 +330,7 @@ int radeon_sa_bo_new(struct radeon_device *rdev,
 	INIT_LIST_HEAD(&(*sa_bo)->flist);
 
 	spin_lock(&sa_manager->wq.lock);
-	while(1) {
+	do {
 		for (i = 0; i < RADEON_NUM_RINGS; ++i) {
 			fences[i] = NULL;
 			tries[i] = 0;
@@ -349,26 +348,22 @@ int radeon_sa_bo_new(struct radeon_device *rdev,
 			/* see if we can skip over some allocations */
 		} while (radeon_sa_bo_next_hole(sa_manager, fences, tries));
 
-		if (!block) {
-			break;
-		}
-
 		spin_unlock(&sa_manager->wq.lock);
 		r = radeon_fence_wait_any(rdev, fences, false);
 		spin_lock(&sa_manager->wq.lock);
 		/* if we have nothing to wait for block */
-		if (r == -ENOENT) {
+		if (r == -ENOENT && block) {
 			r = wait_event_interruptible_locked(
 				sa_manager->wq, 
 				radeon_sa_event(sa_manager, size, align)
 			);
+
+		} else if (r == -ENOENT) {
+			r = -ENOMEM;
 		}
-		if (r) {
-			goto out_err;
-		}
-	};
 
-out_err:
+	} while (!r);
+
 	spin_unlock(&sa_manager->wq.lock);
 	kfree(*sa_bo);
 	*sa_bo = NULL;
diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c
index 7cc78de6ddc..97f3ece81cd 100644
--- a/drivers/gpu/drm/radeon/radeon_semaphore.c
+++ b/drivers/gpu/drm/radeon/radeon_semaphore.c
@@ -27,8 +27,7 @@
  * Authors:
  *    Christian König <deathsimple@vodafone.de>
  */
-#include "drmP.h"
-#include "drm.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 
 
diff --git a/drivers/gpu/drm/radeon/radeon_state.c b/drivers/gpu/drm/radeon/radeon_state.c
index e8422ae7fe7..8e9057b6a36 100644
--- a/drivers/gpu/drm/radeon/radeon_state.c
+++ b/drivers/gpu/drm/radeon/radeon_state.c
@@ -27,11 +27,9 @@
  *    Kevin E. Martin <martin@valinux.com>
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_buffer.h"
-#include "drm_sarea.h"
-#include "radeon_drm.h"
+#include <drm/drmP.h>
+#include <drm/drm_buffer.h>
+#include <drm/radeon_drm.h>
 #include "radeon_drv.h"
 
 /* ================================================================
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 7c16540c10f..587c09a00ba 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -313,7 +313,7 @@ out_cleanup:
 		printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
 }
 
-void radeon_test_ring_sync2(struct radeon_device *rdev,
+static void radeon_test_ring_sync2(struct radeon_device *rdev,
 			    struct radeon_ring *ringA,
 			    struct radeon_ring *ringB,
 			    struct radeon_ring *ringC)
diff --git a/drivers/gpu/drm/radeon/radeon_trace_points.c b/drivers/gpu/drm/radeon/radeon_trace_points.c
index 8175993df84..e51d3575976 100644
--- a/drivers/gpu/drm/radeon/radeon_trace_points.c
+++ b/drivers/gpu/drm/radeon/radeon_trace_points.c
@@ -2,7 +2,7 @@
  * Author : Dave Airlie <airlied@redhat.com>
  */
 #include <drm/drmP.h>
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "radeon.h"
 
 #define CREATE_TRACE_POINTS
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 5b71c716d83..5ebe1b3e5db 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -549,7 +549,7 @@ static struct ttm_backend_func radeon_backend_func = {
 	.destroy = &radeon_ttm_backend_destroy,
 };
 
-struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev,
+static struct ttm_tt *radeon_ttm_tt_create(struct ttm_bo_device *bdev,
 				    unsigned long size, uint32_t page_flags,
 				    struct page *dummy_read_page)
 {
diff --git a/drivers/gpu/drm/radeon/rs400.c b/drivers/gpu/drm/radeon/rs400.c
index 2752f7f7823..73051ce3121 100644
--- a/drivers/gpu/drm/radeon/rs400.c
+++ b/drivers/gpu/drm/radeon/rs400.c
@@ -242,7 +242,7 @@ int rs400_mc_wait_for_idle(struct radeon_device *rdev)
 	return -1;
 }
 
-void rs400_gpu_init(struct radeon_device *rdev)
+static void rs400_gpu_init(struct radeon_device *rdev)
 {
 	/* FIXME: is this correct ? */
 	r420_pipes_init(rdev);
@@ -252,7 +252,7 @@ void rs400_gpu_init(struct radeon_device *rdev)
 	}
 }
 
-void rs400_mc_init(struct radeon_device *rdev)
+static void rs400_mc_init(struct radeon_device *rdev)
 {
 	u64 base;
 
@@ -370,7 +370,7 @@ static int rs400_debugfs_pcie_gart_info_init(struct radeon_device *rdev)
 #endif
 }
 
-void rs400_mc_program(struct radeon_device *rdev)
+static void rs400_mc_program(struct radeon_device *rdev)
 {
 	struct r100_mc_save save;
 
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 5301b3df846..5a0fc74c2ba 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -35,7 +35,7 @@
  * close to the one of the R600 family (R600 likely being an evolution
  * of the RS600 GART block).
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "atom.h"
@@ -43,22 +43,30 @@
 
 #include "rs600_reg_safe.h"
 
-void rs600_gpu_init(struct radeon_device *rdev);
+static void rs600_gpu_init(struct radeon_device *rdev);
 int rs600_mc_wait_for_idle(struct radeon_device *rdev);
 
+static const u32 crtc_offsets[2] =
+{
+	0,
+	AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
+};
+
 void avivo_wait_for_vblank(struct radeon_device *rdev, int crtc)
 {
-	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc];
 	int i;
 
-	if (RREG32(AVIVO_D1CRTC_CONTROL + radeon_crtc->crtc_offset) & AVIVO_CRTC_EN) {
+	if (crtc >= rdev->num_crtc)
+		return;
+
+	if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[crtc]) & AVIVO_CRTC_EN) {
 		for (i = 0; i < rdev->usec_timeout; i++) {
-			if (!(RREG32(AVIVO_D1CRTC_STATUS + radeon_crtc->crtc_offset) & AVIVO_D1CRTC_V_BLANK))
+			if (!(RREG32(AVIVO_D1CRTC_STATUS + crtc_offsets[crtc]) & AVIVO_D1CRTC_V_BLANK))
 				break;
 			udelay(1);
 		}
 		for (i = 0; i < rdev->usec_timeout; i++) {
-			if (RREG32(AVIVO_D1CRTC_STATUS + radeon_crtc->crtc_offset) & AVIVO_D1CRTC_V_BLANK)
+			if (RREG32(AVIVO_D1CRTC_STATUS + crtc_offsets[crtc]) & AVIVO_D1CRTC_V_BLANK)
 				break;
 			udelay(1);
 		}
@@ -424,7 +432,7 @@ void rs600_gart_tlb_flush(struct radeon_device *rdev)
 	tmp = RREG32_MC(R_000100_MC_PT0_CNTL);
 }
 
-int rs600_gart_init(struct radeon_device *rdev)
+static int rs600_gart_init(struct radeon_device *rdev)
 {
 	int r;
 
@@ -506,7 +514,7 @@ static int rs600_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void rs600_gart_disable(struct radeon_device *rdev)
+static void rs600_gart_disable(struct radeon_device *rdev)
 {
 	u32 tmp;
 
@@ -517,7 +525,7 @@ void rs600_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void rs600_gart_fini(struct radeon_device *rdev)
+static void rs600_gart_fini(struct radeon_device *rdev)
 {
 	radeon_gart_fini(rdev);
 	rs600_gart_disable(rdev);
@@ -567,9 +575,6 @@ int rs600_irq_set(struct radeon_device *rdev)
 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
 		tmp |= S_000040_SW_INT_EN(1);
 	}
-	if (rdev->irq.gui_idle) {
-		tmp |= S_000040_GUI_IDLE(1);
-	}
 	if (rdev->irq.crtc_vblank_int[0] ||
 	    atomic_read(&rdev->irq.pflip[0])) {
 		mode_int |= S_006540_D1MODE_VBLANK_INT_MASK(1);
@@ -602,12 +607,6 @@ static inline u32 rs600_irq_ack(struct radeon_device *rdev)
 	uint32_t irq_mask = S_000044_SW_INT(1);
 	u32 tmp;
 
-	/* the interrupt works, but the status bit is permanently asserted */
-	if (rdev->irq.gui_idle && radeon_gui_idle(rdev)) {
-		if (!rdev->irq.gui_idle_acked)
-			irq_mask |= S_000044_GUI_IDLE_STAT(1);
-	}
-
 	if (G_000044_DISPLAY_INT_STAT(irqs)) {
 		rdev->irq.stat_regs.r500.disp_int = RREG32(R_007EDC_DISP_INTERRUPT_STATUS);
 		if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
@@ -667,9 +666,6 @@ int rs600_irq_process(struct radeon_device *rdev)
 	bool queue_hotplug = false;
 	bool queue_hdmi = false;
 
-	/* reset gui idle ack.  the status bit is broken */
-	rdev->irq.gui_idle_acked = false;
-
 	status = rs600_irq_ack(rdev);
 	if (!status &&
 	    !rdev->irq.stat_regs.r500.disp_int &&
@@ -683,11 +679,6 @@ int rs600_irq_process(struct radeon_device *rdev)
 		if (G_000044_SW_INT(status)) {
 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
 		}
-		/* GUI idle */
-		if (G_000040_GUI_IDLE(status)) {
-			rdev->irq.gui_idle_acked = true;
-			wake_up(&rdev->irq.idle_queue);
-		}
 		/* Vertical blank interrupts */
 		if (G_007EDC_LB_D1_VBLANK_INTERRUPT(rdev->irq.stat_regs.r500.disp_int)) {
 			if (rdev->irq.crtc_vblank_int[0]) {
@@ -721,8 +712,6 @@ int rs600_irq_process(struct radeon_device *rdev)
 		}
 		status = rs600_irq_ack(rdev);
 	}
-	/* reset gui idle ack.  the status bit is broken */
-	rdev->irq.gui_idle_acked = false;
 	if (queue_hotplug)
 		schedule_work(&rdev->hotplug_work);
 	if (queue_hdmi)
@@ -764,7 +753,7 @@ int rs600_mc_wait_for_idle(struct radeon_device *rdev)
 	return -1;
 }
 
-void rs600_gpu_init(struct radeon_device *rdev)
+static void rs600_gpu_init(struct radeon_device *rdev)
 {
 	r420_pipes_init(rdev);
 	/* Wait for mc idle */
@@ -772,7 +761,7 @@ void rs600_gpu_init(struct radeon_device *rdev)
 		dev_warn(rdev->dev, "Wait MC idle timeout before updating MC.\n");
 }
 
-void rs600_mc_init(struct radeon_device *rdev)
+static void rs600_mc_init(struct radeon_device *rdev)
 {
 	u64 base;
 
@@ -834,7 +823,7 @@ void rs600_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
 	WREG32(R_000074_MC_IND_DATA, v);
 }
 
-void rs600_debugfs(struct radeon_device *rdev)
+static void rs600_debugfs(struct radeon_device *rdev)
 {
 	if (r100_debugfs_rbbm_init(rdev))
 		DRM_ERROR("Failed to register debugfs file for RBBM !\n");
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 3b663fcfe06..5706d2ac75a 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -25,7 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "atom.h"
@@ -145,7 +145,7 @@ void rs690_pm_info(struct radeon_device *rdev)
 	rdev->pm.sideport_bandwidth.full = dfixed_div(rdev->pm.sideport_bandwidth, tmp);
 }
 
-void rs690_mc_init(struct radeon_device *rdev)
+static void rs690_mc_init(struct radeon_device *rdev)
 {
 	u64 base;
 
@@ -224,7 +224,7 @@ struct rs690_watermark {
 	fixed20_12 sclk;
 };
 
-void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
+static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
 				  struct radeon_crtc *crtc,
 				  struct rs690_watermark *wm)
 {
@@ -581,7 +581,7 @@ void rs690_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
 	WREG32(R_000078_MC_INDEX, 0x7F);
 }
 
-void rs690_mc_program(struct radeon_device *rdev)
+static void rs690_mc_program(struct radeon_device *rdev)
 {
 	struct rv515_mc_save save;
 
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index aa8ef491ef3..785d09590b2 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -27,7 +27,7 @@
  */
 #include <linux/seq_file.h>
 #include <linux/slab.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "rv515d.h"
 #include "radeon.h"
 #include "radeon_asic.h"
@@ -35,9 +35,9 @@
 #include "rv515_reg_safe.h"
 
 /* This files gather functions specifics to: rv515 */
-int rv515_debugfs_pipes_info_init(struct radeon_device *rdev);
-int rv515_debugfs_ga_info_init(struct radeon_device *rdev);
-void rv515_gpu_init(struct radeon_device *rdev);
+static int rv515_debugfs_pipes_info_init(struct radeon_device *rdev);
+static int rv515_debugfs_ga_info_init(struct radeon_device *rdev);
+static void rv515_gpu_init(struct radeon_device *rdev);
 int rv515_mc_wait_for_idle(struct radeon_device *rdev);
 
 void rv515_debugfs(struct radeon_device *rdev)
@@ -143,7 +143,7 @@ void rv515_vga_render_disable(struct radeon_device *rdev)
 		RREG32(R_000300_VGA_RENDER_CONTROL) & C_000300_VGA_VSTATUS_CNTL);
 }
 
-void rv515_gpu_init(struct radeon_device *rdev)
+static void rv515_gpu_init(struct radeon_device *rdev)
 {
 	unsigned pipe_select_current, gb_pipe_select, tmp;
 
@@ -189,7 +189,7 @@ static void rv515_vram_get_type(struct radeon_device *rdev)
 	}
 }
 
-void rv515_mc_init(struct radeon_device *rdev)
+static void rv515_mc_init(struct radeon_device *rdev)
 {
 
 	rv515_vram_get_type(rdev);
@@ -261,7 +261,7 @@ static struct drm_info_list rv515_ga_info_list[] = {
 };
 #endif
 
-int rv515_debugfs_pipes_info_init(struct radeon_device *rdev)
+static int rv515_debugfs_pipes_info_init(struct radeon_device *rdev)
 {
 #if defined(CONFIG_DEBUG_FS)
 	return radeon_debugfs_add_files(rdev, rv515_pipes_info_list, 1);
@@ -270,7 +270,7 @@ int rv515_debugfs_pipes_info_init(struct radeon_device *rdev)
 #endif
 }
 
-int rv515_debugfs_ga_info_init(struct radeon_device *rdev)
+static int rv515_debugfs_ga_info_init(struct radeon_device *rdev)
 {
 #if defined(CONFIG_DEBUG_FS)
 	return radeon_debugfs_add_files(rdev, rv515_ga_info_list, 1);
@@ -310,7 +310,7 @@ void rv515_mc_resume(struct radeon_device *rdev, struct rv515_mc_save *save)
 	WREG32(R_000300_VGA_RENDER_CONTROL, save->vga_render_control);
 }
 
-void rv515_mc_program(struct radeon_device *rdev)
+static void rv515_mc_program(struct radeon_device *rdev)
 {
 	struct rv515_mc_save save;
 
@@ -787,7 +787,7 @@ struct rv515_watermark {
 	fixed20_12 sclk;
 };
 
-void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
+static void rv515_crtc_bandwidth_compute(struct radeon_device *rdev,
 				  struct radeon_crtc *crtc,
 				  struct rv515_watermark *wm)
 {
diff --git a/drivers/gpu/drm/radeon/rv770.c b/drivers/gpu/drm/radeon/rv770.c
index ca8ffec10ff..79814a08c8e 100644
--- a/drivers/gpu/drm/radeon/rv770.c
+++ b/drivers/gpu/drm/radeon/rv770.c
@@ -28,10 +28,10 @@
 #include <linux/firmware.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "rv770d.h"
 #include "atom.h"
 #include "avivod.h"
@@ -124,7 +124,7 @@ void rv770_pm_misc(struct radeon_device *rdev)
 /*
  * GART
  */
-int rv770_pcie_gart_enable(struct radeon_device *rdev)
+static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int r, i;
@@ -175,7 +175,7 @@ int rv770_pcie_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void rv770_pcie_gart_disable(struct radeon_device *rdev)
+static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int i;
@@ -201,7 +201,7 @@ void rv770_pcie_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void rv770_pcie_gart_fini(struct radeon_device *rdev)
+static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 {
 	radeon_gart_fini(rdev);
 	rv770_pcie_gart_disable(rdev);
@@ -209,7 +209,7 @@ void rv770_pcie_gart_fini(struct radeon_device *rdev)
 }
 
 
-void rv770_agp_enable(struct radeon_device *rdev)
+static void rv770_agp_enable(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int i;
@@ -839,7 +839,7 @@ void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
 	}
 }
 
-int rv770_mc_init(struct radeon_device *rdev)
+static int rv770_mc_init(struct radeon_device *rdev)
 {
 	u32 tmp;
 	int chansize, numchan;
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 0139e227e3c..f79633a036c 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -25,10 +25,10 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
-#include "radeon_drm.h"
+#include <drm/radeon_drm.h>
 #include "sid.h"
 #include "atom.h"
 #include "si_blit_shaders.h"
@@ -1806,13 +1806,14 @@ void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
 #endif
 			  (ib->gpu_addr & 0xFFFFFFFC));
 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
-	radeon_ring_write(ring, ib->length_dw | (ib->vm_id << 24));
+	radeon_ring_write(ring, ib->length_dw |
+			  (ib->vm ? (ib->vm->id << 24) : 0));
 
 	if (!ib->is_const_ib) {
 		/* flush read cache over gart for this vmid */
 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
 		radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
-		radeon_ring_write(ring, ib->vm_id);
+		radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
 		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
 		radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
 				  PACKET3_TC_ACTION_ENA |
@@ -2363,7 +2364,7 @@ void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
 	WREG32(VM_INVALIDATE_REQUEST, 1);
 }
 
-int si_pcie_gart_enable(struct radeon_device *rdev)
+static int si_pcie_gart_enable(struct radeon_device *rdev)
 {
 	int r, i;
 
@@ -2425,7 +2426,7 @@ int si_pcie_gart_enable(struct radeon_device *rdev)
 	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
 	       (u32)(rdev->dummy_page.addr >> 12));
 	WREG32(VM_CONTEXT1_CNTL2, 0);
-	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
+	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 
 	si_pcie_gart_tlb_flush(rdev);
@@ -2436,7 +2437,7 @@ int si_pcie_gart_enable(struct radeon_device *rdev)
 	return 0;
 }
 
-void si_pcie_gart_disable(struct radeon_device *rdev)
+static void si_pcie_gart_disable(struct radeon_device *rdev)
 {
 	/* Disable all tables */
 	WREG32(VM_CONTEXT0_CNTL, 0);
@@ -2455,7 +2456,7 @@ void si_pcie_gart_disable(struct radeon_device *rdev)
 	radeon_gart_table_vram_unpin(rdev);
 }
 
-void si_pcie_gart_fini(struct radeon_device *rdev)
+static void si_pcie_gart_fini(struct radeon_device *rdev)
 {
 	si_pcie_gart_disable(rdev);
 	radeon_gart_table_vram_free(rdev);
@@ -2788,41 +2789,84 @@ void si_vm_fini(struct radeon_device *rdev)
 {
 }
 
-int si_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm, int id)
+/**
+ * si_vm_set_page - update the page tables using the CP
+ *
+ * @rdev: radeon_device pointer
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: access flags
+ *
+ * Update the page tables using the CP (cayman-si).
+ */
+void si_vm_set_page(struct radeon_device *rdev, uint64_t pe,
+		    uint64_t addr, unsigned count,
+		    uint32_t incr, uint32_t flags)
 {
-	if (id < 8)
-		WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (id << 2), vm->pt_gpu_addr >> 12);
-	else
-		WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((id - 8) << 2),
-		       vm->pt_gpu_addr >> 12);
-	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-	/* bits 0-15 are the VM contexts0-15 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << id);
-	return 0;
+	struct radeon_ring *ring = &rdev->ring[rdev->asic->vm.pt_ring_index];
+	uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
+	int i;
+	uint64_t value;
+
+	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 2 + count * 2));
+	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+				 WRITE_DATA_DST_SEL(1)));
+	radeon_ring_write(ring, pe);
+	radeon_ring_write(ring, upper_32_bits(pe));
+	for (i = 0; i < count; ++i) {
+		if (flags & RADEON_VM_PAGE_SYSTEM) {
+			value = radeon_vm_map_gart(rdev, addr);
+			value &= 0xFFFFFFFFFFFFF000ULL;
+		} else if (flags & RADEON_VM_PAGE_VALID)
+			value = addr;
+		else
+			value = 0;
+		addr += incr;
+		value |= r600_flags;
+		radeon_ring_write(ring, value);
+		radeon_ring_write(ring, upper_32_bits(value));
+	}
 }
 
-void si_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm)
+void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 {
-	if (vm->id < 8)
-		WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0);
-	else
-		WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2), 0);
-	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
-	/* bits 0-15 are the VM contexts0-15 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
-}
+	struct radeon_ring *ring = &rdev->ring[ridx];
 
-void si_vm_tlb_flush(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-	if (vm->id == -1)
+	if (vm == NULL)
 		return;
 
+	/* write new base address */
+	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+				 WRITE_DATA_DST_SEL(0)));
+
+	if (vm->id < 8) {
+		radeon_ring_write(ring,
+				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
+	} else {
+		radeon_ring_write(ring,
+				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
+	}
+	radeon_ring_write(ring, 0);
+	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
+
 	/* flush hdp cache */
-	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
+	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+				 WRITE_DATA_DST_SEL(0)));
+	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+	radeon_ring_write(ring, 0);
+	radeon_ring_write(ring, 0x1);
+
 	/* bits 0-15 are the VM contexts0-15 */
-	WREG32(VM_INVALIDATE_REQUEST, 1 << vm->id);
+	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
+	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
+				 WRITE_DATA_DST_SEL(0)));
+	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
+	radeon_ring_write(ring, 0);
+	radeon_ring_write(ring, 1 << vm->id);
 }
 
 /*
@@ -3199,10 +3243,6 @@ int si_irq_set(struct radeon_device *rdev)
 		DRM_DEBUG("si_irq_set: hpd 6\n");
 		hpd6 |= DC_HPDx_INT_EN;
 	}
-	if (rdev->irq.gui_idle) {
-		DRM_DEBUG("gui idle\n");
-		grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
-	}
 
 	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
 	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
@@ -3658,7 +3698,6 @@ restart_ih:
 			break;
 		case 233: /* GUI IDLE */
 			DRM_DEBUG("IH: GUI idle\n");
-			wake_up(&rdev->irq.idle_queue);
 			break;
 		default:
 			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index ef4815c27b1..7d2a20e5657 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -812,6 +812,21 @@
 #define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
 #define	PACKET3_DRAW_INDEX_MULTI_ELEMENT		0x36
 #define	PACKET3_WRITE_DATA				0x37
+#define		WRITE_DATA_DST_SEL(x)                   ((x) << 8)
+                /* 0 - register
+		 * 1 - memory (sync - via GRBM)
+		 * 2 - tc/l2
+		 * 3 - gds
+		 * 4 - reserved
+		 * 5 - memory (async - direct)
+		 */
+#define		WR_ONE_ADDR                             (1 << 16)
+#define		WR_CONFIRM                              (1 << 20)
+#define		WRITE_DATA_ENGINE_SEL(x)                ((x) << 30)
+                /* 0 - me
+		 * 1 - pfp
+		 * 2 - ce
+		 */
 #define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
 #define	PACKET3_MEM_SEMAPHORE				0x39
 #define	PACKET3_MPEG_INDEX				0x3A
diff --git a/drivers/gpu/drm/savage/savage_bci.c b/drivers/gpu/drm/savage/savage_bci.c
index 1efbb907583..b55c1d66114 100644
--- a/drivers/gpu/drm/savage/savage_bci.c
+++ b/drivers/gpu/drm/savage/savage_bci.c
@@ -22,8 +22,8 @@
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#include "drmP.h"
-#include "savage_drm.h"
+#include <drm/drmP.h>
+#include <drm/savage_drm.h>
 #include "savage_drv.h"
 
 /* Need a long timeout for shadow status updates can take a while
@@ -547,6 +547,8 @@ int savage_driver_load(struct drm_device *dev, unsigned long chipset)
 
 	dev_priv->chipset = (enum savage_family)chipset;
 
+	pci_set_master(dev->pdev);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/savage/savage_drv.c b/drivers/gpu/drm/savage/savage_drv.c
index c5a164337bd..71b2081e783 100644
--- a/drivers/gpu/drm/savage/savage_drv.c
+++ b/drivers/gpu/drm/savage/savage_drv.c
@@ -25,11 +25,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "savage_drm.h"
+#include <drm/drmP.h>
+#include <drm/savage_drm.h>
 #include "savage_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static struct pci_device_id pciidlist[] = {
 	savage_PCI_IDS
diff --git a/drivers/gpu/drm/savage/savage_state.c b/drivers/gpu/drm/savage/savage_state.c
index b6d8608375c..b35e75ed890 100644
--- a/drivers/gpu/drm/savage/savage_state.c
+++ b/drivers/gpu/drm/savage/savage_state.c
@@ -22,8 +22,8 @@
  * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
-#include "drmP.h"
-#include "savage_drm.h"
+#include <drm/drmP.h>
+#include <drm/savage_drm.h>
 #include "savage_drv.h"
 
 void savage_emit_clip_rect_s3d(drm_savage_private_t * dev_priv,
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
new file mode 100644
index 00000000000..7e7d52b2a2f
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -0,0 +1,10 @@
+config DRM_SHMOBILE
+	tristate "DRM Support for SH Mobile"
+	depends on DRM && (SUPERH || ARCH_SHMOBILE)
+	select DRM_KMS_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DRM_GEM_CMA_HELPER
+	help
+	  Choose this option if you have an SH Mobile chipset.
+	  If M is selected the module will be called shmob-drm.
+
diff --git a/drivers/gpu/drm/shmobile/Makefile b/drivers/gpu/drm/shmobile/Makefile
new file mode 100644
index 00000000000..4c3eeb35563
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/Makefile
@@ -0,0 +1,7 @@
+shmob-drm-y := shmob_drm_backlight.o \
+	       shmob_drm_crtc.o \
+	       shmob_drm_drv.o \
+	       shmob_drm_kms.o \
+	       shmob_drm_plane.o
+
+obj-$(CONFIG_DRM_SHMOBILE)	+= shmob-drm.o
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_backlight.c b/drivers/gpu/drm/shmobile/shmob_drm_backlight.c
new file mode 100644
index 00000000000..463aee18f77
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_backlight.c
@@ -0,0 +1,90 @@
+/*
+ * shmob_drm_backlight.c  --  SH Mobile DRM Backlight
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/backlight.h>
+
+#include "shmob_drm_backlight.h"
+#include "shmob_drm_crtc.h"
+#include "shmob_drm_drv.h"
+
+static int shmob_drm_backlight_update(struct backlight_device *bdev)
+{
+	struct shmob_drm_connector *scon = bl_get_data(bdev);
+	struct shmob_drm_device *sdev = scon->connector.dev->dev_private;
+	const struct shmob_drm_backlight_data *bdata = &sdev->pdata->backlight;
+	int brightness = bdev->props.brightness;
+
+	if (bdev->props.power != FB_BLANK_UNBLANK ||
+	    bdev->props.state & BL_CORE_SUSPENDED)
+		brightness = 0;
+
+	return bdata->set_brightness(brightness);
+}
+
+static int shmob_drm_backlight_get_brightness(struct backlight_device *bdev)
+{
+	struct shmob_drm_connector *scon = bl_get_data(bdev);
+	struct shmob_drm_device *sdev = scon->connector.dev->dev_private;
+	const struct shmob_drm_backlight_data *bdata = &sdev->pdata->backlight;
+
+	return bdata->get_brightness();
+}
+
+static const struct backlight_ops shmob_drm_backlight_ops = {
+	.options	= BL_CORE_SUSPENDRESUME,
+	.update_status	= shmob_drm_backlight_update,
+	.get_brightness	= shmob_drm_backlight_get_brightness,
+};
+
+void shmob_drm_backlight_dpms(struct shmob_drm_connector *scon, int mode)
+{
+	if (scon->backlight == NULL)
+		return;
+
+	scon->backlight->props.power = mode == DRM_MODE_DPMS_ON
+				     ? FB_BLANK_UNBLANK : FB_BLANK_POWERDOWN;
+	backlight_update_status(scon->backlight);
+}
+
+int shmob_drm_backlight_init(struct shmob_drm_connector *scon)
+{
+	struct shmob_drm_device *sdev = scon->connector.dev->dev_private;
+	const struct shmob_drm_backlight_data *bdata = &sdev->pdata->backlight;
+	struct drm_connector *connector = &scon->connector;
+	struct drm_device *dev = connector->dev;
+	struct backlight_device *backlight;
+
+	if (!bdata->max_brightness)
+		return 0;
+
+	backlight = backlight_device_register(bdata->name, dev->dev, scon,
+					      &shmob_drm_backlight_ops, NULL);
+	if (IS_ERR(backlight)) {
+		dev_err(dev->dev, "unable to register backlight device: %ld\n",
+			PTR_ERR(backlight));
+		return PTR_ERR(backlight);
+	}
+
+	backlight->props.max_brightness = bdata->max_brightness;
+	backlight->props.brightness = bdata->max_brightness;
+	backlight->props.power = FB_BLANK_POWERDOWN;
+	backlight_update_status(backlight);
+
+	scon->backlight = backlight;
+	return 0;
+}
+
+void shmob_drm_backlight_exit(struct shmob_drm_connector *scon)
+{
+	backlight_device_unregister(scon->backlight);
+}
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_backlight.h b/drivers/gpu/drm/shmobile/shmob_drm_backlight.h
new file mode 100644
index 00000000000..9477595d2ff
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_backlight.h
@@ -0,0 +1,23 @@
+/*
+ * shmob_drm_backlight.h  --  SH Mobile DRM Backlight
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_BACKLIGHT_H__
+#define __SHMOB_DRM_BACKLIGHT_H__
+
+struct shmob_drm_connector;
+
+void shmob_drm_backlight_dpms(struct shmob_drm_connector *scon, int mode);
+int shmob_drm_backlight_init(struct shmob_drm_connector *scon);
+void shmob_drm_backlight_exit(struct shmob_drm_connector *scon);
+
+#endif /* __SHMOB_DRM_BACKLIGHT_H__ */
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
new file mode 100644
index 00000000000..0e7a9306bd0
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -0,0 +1,763 @@
+/*
+ * shmob_drm_crtc.c  --  SH Mobile DRM CRTCs
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/backlight.h>
+#include <linux/clk.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include <video/sh_mobile_meram.h>
+
+#include "shmob_drm_backlight.h"
+#include "shmob_drm_crtc.h"
+#include "shmob_drm_drv.h"
+#include "shmob_drm_kms.h"
+#include "shmob_drm_plane.h"
+#include "shmob_drm_regs.h"
+
+/*
+ * TODO: panel support
+ */
+
+/* -----------------------------------------------------------------------------
+ * Clock management
+ */
+
+static void shmob_drm_clk_on(struct shmob_drm_device *sdev)
+{
+	if (sdev->clock)
+		clk_enable(sdev->clock);
+#if 0
+	if (sdev->meram_dev && sdev->meram_dev->pdev)
+		pm_runtime_get_sync(&sdev->meram_dev->pdev->dev);
+#endif
+}
+
+static void shmob_drm_clk_off(struct shmob_drm_device *sdev)
+{
+#if 0
+	if (sdev->meram_dev && sdev->meram_dev->pdev)
+		pm_runtime_put_sync(&sdev->meram_dev->pdev->dev);
+#endif
+	if (sdev->clock)
+		clk_disable(sdev->clock);
+}
+
+/* -----------------------------------------------------------------------------
+ * CRTC
+ */
+
+static void shmob_drm_crtc_setup_geometry(struct shmob_drm_crtc *scrtc)
+{
+	struct drm_crtc *crtc = &scrtc->crtc;
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+	const struct shmob_drm_interface_data *idata = &sdev->pdata->iface;
+	const struct drm_display_mode *mode = &crtc->mode;
+	u32 value;
+
+	value = sdev->ldmt1r
+	      | ((mode->flags & DRM_MODE_FLAG_PVSYNC) ? 0 : LDMT1R_VPOL)
+	      | ((mode->flags & DRM_MODE_FLAG_PHSYNC) ? 0 : LDMT1R_HPOL)
+	      | ((idata->flags & SHMOB_DRM_IFACE_FL_DWPOL) ? LDMT1R_DWPOL : 0)
+	      | ((idata->flags & SHMOB_DRM_IFACE_FL_DIPOL) ? LDMT1R_DIPOL : 0)
+	      | ((idata->flags & SHMOB_DRM_IFACE_FL_DAPOL) ? LDMT1R_DAPOL : 0)
+	      | ((idata->flags & SHMOB_DRM_IFACE_FL_HSCNT) ? LDMT1R_HSCNT : 0)
+	      | ((idata->flags & SHMOB_DRM_IFACE_FL_DWCNT) ? LDMT1R_DWCNT : 0);
+	lcdc_write(sdev, LDMT1R, value);
+
+	if (idata->interface >= SHMOB_DRM_IFACE_SYS8A &&
+	    idata->interface <= SHMOB_DRM_IFACE_SYS24) {
+		/* Setup SYS bus. */
+		value = (idata->sys.cs_setup << LDMT2R_CSUP_SHIFT)
+		      | (idata->sys.vsync_active_high ? LDMT2R_RSV : 0)
+		      | (idata->sys.vsync_dir_input ? LDMT2R_VSEL : 0)
+		      | (idata->sys.write_setup << LDMT2R_WCSC_SHIFT)
+		      | (idata->sys.write_cycle << LDMT2R_WCEC_SHIFT)
+		      | (idata->sys.write_strobe << LDMT2R_WCLW_SHIFT);
+		lcdc_write(sdev, LDMT2R, value);
+
+		value = (idata->sys.read_latch << LDMT3R_RDLC_SHIFT)
+		      | (idata->sys.read_setup << LDMT3R_RCSC_SHIFT)
+		      | (idata->sys.read_cycle << LDMT3R_RCEC_SHIFT)
+		      | (idata->sys.read_strobe << LDMT3R_RCLW_SHIFT);
+		lcdc_write(sdev, LDMT3R, value);
+	}
+
+	value = ((mode->hdisplay / 8) << 16)			/* HDCN */
+	      | (mode->htotal / 8);				/* HTCN */
+	lcdc_write(sdev, LDHCNR, value);
+
+	value = (((mode->hsync_end - mode->hsync_start) / 8) << 16) /* HSYNW */
+	      | (mode->hsync_start / 8);			/* HSYNP */
+	lcdc_write(sdev, LDHSYNR, value);
+
+	value = ((mode->hdisplay & 7) << 24) | ((mode->htotal & 7) << 16)
+	      | (((mode->hsync_end - mode->hsync_start) & 7) << 8)
+	      | (mode->hsync_start & 7);
+	lcdc_write(sdev, LDHAJR, value);
+
+	value = ((mode->vdisplay) << 16)			/* VDLN */
+	      | mode->vtotal;					/* VTLN */
+	lcdc_write(sdev, LDVLNR, value);
+
+	value = ((mode->vsync_end - mode->vsync_start) << 16)	/* VSYNW */
+	      | mode->vsync_start;				/* VSYNP */
+	lcdc_write(sdev, LDVSYNR, value);
+}
+
+static void shmob_drm_crtc_start_stop(struct shmob_drm_crtc *scrtc, bool start)
+{
+	struct shmob_drm_device *sdev = scrtc->crtc.dev->dev_private;
+	u32 value;
+
+	value = lcdc_read(sdev, LDCNT2R);
+	if (start)
+		lcdc_write(sdev, LDCNT2R, value | LDCNT2R_DO);
+	else
+		lcdc_write(sdev, LDCNT2R, value & ~LDCNT2R_DO);
+
+	/* Wait until power is applied/stopped. */
+	while (1) {
+		value = lcdc_read(sdev, LDPMR) & LDPMR_LPS;
+		if ((start && value) || (!start && !value))
+			break;
+
+		cpu_relax();
+	}
+
+	if (!start) {
+		/* Stop the dot clock. */
+		lcdc_write(sdev, LDDCKSTPR, LDDCKSTPR_DCKSTP);
+	}
+}
+
+/*
+ * shmob_drm_crtc_start - Configure and start the LCDC
+ * @scrtc: the SH Mobile CRTC
+ *
+ * Configure and start the LCDC device. External devices (clocks, MERAM, panels,
+ * ...) are not touched by this function.
+ */
+static void shmob_drm_crtc_start(struct shmob_drm_crtc *scrtc)
+{
+	struct drm_crtc *crtc = &scrtc->crtc;
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+	const struct shmob_drm_interface_data *idata = &sdev->pdata->iface;
+	const struct shmob_drm_format_info *format;
+	struct drm_device *dev = sdev->ddev;
+	struct drm_plane *plane;
+	u32 value;
+
+	if (scrtc->started)
+		return;
+
+	format = shmob_drm_format_info(crtc->fb->pixel_format);
+	if (WARN_ON(format == NULL))
+		return;
+
+	/* Enable clocks before accessing the hardware. */
+	shmob_drm_clk_on(sdev);
+
+	/* Reset and enable the LCDC. */
+	lcdc_write(sdev, LDCNT2R, lcdc_read(sdev, LDCNT2R) | LDCNT2R_BR);
+	lcdc_wait_bit(sdev, LDCNT2R, LDCNT2R_BR, 0);
+	lcdc_write(sdev, LDCNT2R, LDCNT2R_ME);
+
+	/* Stop the LCDC first and disable all interrupts. */
+	shmob_drm_crtc_start_stop(scrtc, false);
+	lcdc_write(sdev, LDINTR, 0);
+
+	/* Configure power supply, dot clocks and start them. */
+	lcdc_write(sdev, LDPMR, 0);
+
+	value = sdev->lddckr;
+	if (idata->clk_div) {
+		/* FIXME: sh7724 can only use 42, 48, 54 and 60 for the divider
+		 * denominator.
+		 */
+		lcdc_write(sdev, LDDCKPAT1R, 0);
+		lcdc_write(sdev, LDDCKPAT2R, (1 << (idata->clk_div / 2)) - 1);
+
+		if (idata->clk_div == 1)
+			value |= LDDCKR_MOSEL;
+		else
+			value |= idata->clk_div;
+	}
+
+	lcdc_write(sdev, LDDCKR, value);
+	lcdc_write(sdev, LDDCKSTPR, 0);
+	lcdc_wait_bit(sdev, LDDCKSTPR, ~0, 0);
+
+	/* TODO: Setup SYS panel */
+
+	/* Setup geometry, format, frame buffer memory and operation mode. */
+	shmob_drm_crtc_setup_geometry(scrtc);
+
+	/* TODO: Handle YUV colorspaces. Hardcode REC709 for now. */
+	lcdc_write(sdev, LDDFR, format->lddfr | LDDFR_CF1);
+	lcdc_write(sdev, LDMLSR, scrtc->line_size);
+	lcdc_write(sdev, LDSA1R, scrtc->dma[0]);
+	if (format->yuv)
+		lcdc_write(sdev, LDSA2R, scrtc->dma[1]);
+	lcdc_write(sdev, LDSM1R, 0);
+
+	/* Word and long word swap. */
+	switch (format->fourcc) {
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV42:
+		value = LDDDSR_LS | LDDDSR_WS;
+		break;
+	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV24:
+		value = LDDDSR_LS | LDDDSR_WS | LDDDSR_BS;
+		break;
+	case DRM_FORMAT_ARGB8888:
+	default:
+		value = LDDDSR_LS;
+		break;
+	}
+	lcdc_write(sdev, LDDDSR, value);
+
+	/* Setup planes. */
+	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+		if (plane->crtc == crtc)
+			shmob_drm_plane_setup(plane);
+	}
+
+	/* Enable the display output. */
+	lcdc_write(sdev, LDCNT1R, LDCNT1R_DE);
+
+	shmob_drm_crtc_start_stop(scrtc, true);
+
+	scrtc->started = true;
+}
+
+static void shmob_drm_crtc_stop(struct shmob_drm_crtc *scrtc)
+{
+	struct drm_crtc *crtc = &scrtc->crtc;
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+
+	if (!scrtc->started)
+		return;
+
+	/* Disable the MERAM cache. */
+	if (scrtc->cache) {
+		sh_mobile_meram_cache_free(sdev->meram, scrtc->cache);
+		scrtc->cache = NULL;
+	}
+
+	/* Stop the LCDC. */
+	shmob_drm_crtc_start_stop(scrtc, false);
+
+	/* Disable the display output. */
+	lcdc_write(sdev, LDCNT1R, 0);
+
+	/* Stop clocks. */
+	shmob_drm_clk_off(sdev);
+
+	scrtc->started = false;
+}
+
+void shmob_drm_crtc_suspend(struct shmob_drm_crtc *scrtc)
+{
+	shmob_drm_crtc_stop(scrtc);
+}
+
+void shmob_drm_crtc_resume(struct shmob_drm_crtc *scrtc)
+{
+	if (scrtc->dpms != DRM_MODE_DPMS_ON)
+		return;
+
+	shmob_drm_crtc_start(scrtc);
+}
+
+static void shmob_drm_crtc_compute_base(struct shmob_drm_crtc *scrtc,
+					int x, int y)
+{
+	struct drm_crtc *crtc = &scrtc->crtc;
+	struct drm_framebuffer *fb = crtc->fb;
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+	struct drm_gem_cma_object *gem;
+	unsigned int bpp;
+
+	bpp = scrtc->format->yuv ? 8 : scrtc->format->bpp;
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	scrtc->dma[0] = gem->paddr + fb->offsets[0]
+		      + y * fb->pitches[0] + x * bpp / 8;
+
+	if (scrtc->format->yuv) {
+		bpp = scrtc->format->bpp - 8;
+		gem = drm_fb_cma_get_gem_obj(fb, 1);
+		scrtc->dma[1] = gem->paddr + fb->offsets[1]
+			      + y / (bpp == 4 ? 2 : 1) * fb->pitches[1]
+			      + x * (bpp == 16 ? 2 : 1);
+	}
+
+	if (scrtc->cache)
+		sh_mobile_meram_cache_update(sdev->meram, scrtc->cache,
+					     scrtc->dma[0], scrtc->dma[1],
+					     &scrtc->dma[0], &scrtc->dma[1]);
+}
+
+static void shmob_drm_crtc_update_base(struct shmob_drm_crtc *scrtc)
+{
+	struct drm_crtc *crtc = &scrtc->crtc;
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+
+	shmob_drm_crtc_compute_base(scrtc, crtc->x, crtc->y);
+
+	lcdc_write_mirror(sdev, LDSA1R, scrtc->dma[0]);
+	if (scrtc->format->yuv)
+		lcdc_write_mirror(sdev, LDSA2R, scrtc->dma[1]);
+
+	lcdc_write(sdev, LDRCNTR, lcdc_read(sdev, LDRCNTR) ^ LDRCNTR_MRS);
+}
+
+#define to_shmob_crtc(c)	container_of(c, struct shmob_drm_crtc, crtc)
+
+static void shmob_drm_crtc_dpms(struct drm_crtc *crtc, int mode)
+{
+	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
+
+	if (scrtc->dpms == mode)
+		return;
+
+	if (mode == DRM_MODE_DPMS_ON)
+		shmob_drm_crtc_start(scrtc);
+	else
+		shmob_drm_crtc_stop(scrtc);
+
+	scrtc->dpms = mode;
+}
+
+static bool shmob_drm_crtc_mode_fixup(struct drm_crtc *crtc,
+				      const struct drm_display_mode *mode,
+				      struct drm_display_mode *adjusted_mode)
+{
+	return true;
+}
+
+static void shmob_drm_crtc_mode_prepare(struct drm_crtc *crtc)
+{
+	shmob_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+}
+
+static int shmob_drm_crtc_mode_set(struct drm_crtc *crtc,
+				   struct drm_display_mode *mode,
+				   struct drm_display_mode *adjusted_mode,
+				   int x, int y,
+				   struct drm_framebuffer *old_fb)
+{
+	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
+	struct shmob_drm_device *sdev = crtc->dev->dev_private;
+	const struct sh_mobile_meram_cfg *mdata = sdev->pdata->meram;
+	const struct shmob_drm_format_info *format;
+	void *cache;
+
+	format = shmob_drm_format_info(crtc->fb->pixel_format);
+	if (format == NULL) {
+		dev_dbg(sdev->dev, "mode_set: unsupported format %08x\n",
+			crtc->fb->pixel_format);
+		return -EINVAL;
+	}
+
+	scrtc->format = format;
+	scrtc->line_size = crtc->fb->pitches[0];
+
+	if (sdev->meram) {
+		/* Enable MERAM cache if configured. We need to de-init
+		 * configured ICBs before we can re-initialize them.
+		 */
+		if (scrtc->cache) {
+			sh_mobile_meram_cache_free(sdev->meram, scrtc->cache);
+			scrtc->cache = NULL;
+		}
+
+		cache = sh_mobile_meram_cache_alloc(sdev->meram, mdata,
+						    crtc->fb->pitches[0],
+						    adjusted_mode->vdisplay,
+						    format->meram,
+						    &scrtc->line_size);
+		if (!IS_ERR(cache))
+			scrtc->cache = cache;
+	}
+
+	shmob_drm_crtc_compute_base(scrtc, x, y);
+
+	return 0;
+}
+
+static void shmob_drm_crtc_mode_commit(struct drm_crtc *crtc)
+{
+	shmob_drm_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
+}
+
+static int shmob_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+					struct drm_framebuffer *old_fb)
+{
+	shmob_drm_crtc_update_base(to_shmob_crtc(crtc));
+
+	return 0;
+}
+
+static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
+	.dpms = shmob_drm_crtc_dpms,
+	.mode_fixup = shmob_drm_crtc_mode_fixup,
+	.prepare = shmob_drm_crtc_mode_prepare,
+	.commit = shmob_drm_crtc_mode_commit,
+	.mode_set = shmob_drm_crtc_mode_set,
+	.mode_set_base = shmob_drm_crtc_mode_set_base,
+};
+
+void shmob_drm_crtc_cancel_page_flip(struct shmob_drm_crtc *scrtc,
+				     struct drm_file *file)
+{
+	struct drm_pending_vblank_event *event;
+	struct drm_device *dev = scrtc->crtc.dev;
+	unsigned long flags;
+
+	/* Destroy the pending vertical blanking event associated with the
+	 * pending page flip, if any, and disable vertical blanking interrupts.
+	 */
+	spin_lock_irqsave(&dev->event_lock, flags);
+	event = scrtc->event;
+	if (event && event->base.file_priv == file) {
+		scrtc->event = NULL;
+		event->base.destroy(&event->base);
+		drm_vblank_put(dev, 0);
+	}
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+void shmob_drm_crtc_finish_page_flip(struct shmob_drm_crtc *scrtc)
+{
+	struct drm_pending_vblank_event *event;
+	struct drm_device *dev = scrtc->crtc.dev;
+	struct timeval vblanktime;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	event = scrtc->event;
+	scrtc->event = NULL;
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+
+	if (event == NULL)
+		return;
+
+	event->event.sequence = drm_vblank_count_and_time(dev, 0, &vblanktime);
+	event->event.tv_sec = vblanktime.tv_sec;
+	event->event.tv_usec = vblanktime.tv_usec;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	list_add_tail(&event->base.link, &event->base.file_priv->event_list);
+	wake_up_interruptible(&event->base.file_priv->event_wait);
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+
+	drm_vblank_put(dev, 0);
+}
+
+static int shmob_drm_crtc_page_flip(struct drm_crtc *crtc,
+				    struct drm_framebuffer *fb,
+				    struct drm_pending_vblank_event *event)
+{
+	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
+	struct drm_device *dev = scrtc->crtc.dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	if (scrtc->event != NULL) {
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+		return -EBUSY;
+	}
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+
+	crtc->fb = fb;
+	shmob_drm_crtc_update_base(scrtc);
+
+	if (event) {
+		event->pipe = 0;
+		spin_lock_irqsave(&dev->event_lock, flags);
+		scrtc->event = event;
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+		drm_vblank_get(dev, 0);
+	}
+
+	return 0;
+}
+
+static const struct drm_crtc_funcs crtc_funcs = {
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_crtc_helper_set_config,
+	.page_flip = shmob_drm_crtc_page_flip,
+};
+
+int shmob_drm_crtc_create(struct shmob_drm_device *sdev)
+{
+	struct drm_crtc *crtc = &sdev->crtc.crtc;
+	int ret;
+
+	sdev->crtc.dpms = DRM_MODE_DPMS_OFF;
+
+	ret = drm_crtc_init(sdev->ddev, crtc, &crtc_funcs);
+	if (ret < 0)
+		return ret;
+
+	drm_crtc_helper_add(crtc, &crtc_helper_funcs);
+
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * Encoder
+ */
+
+#define to_shmob_encoder(e) \
+	container_of(e, struct shmob_drm_encoder, encoder)
+
+static void shmob_drm_encoder_dpms(struct drm_encoder *encoder, int mode)
+{
+	struct shmob_drm_encoder *senc = to_shmob_encoder(encoder);
+	struct shmob_drm_device *sdev = encoder->dev->dev_private;
+	struct shmob_drm_connector *scon = &sdev->connector;
+
+	if (senc->dpms == mode)
+		return;
+
+	shmob_drm_backlight_dpms(scon, mode);
+
+	senc->dpms = mode;
+}
+
+static bool shmob_drm_encoder_mode_fixup(struct drm_encoder *encoder,
+					 const struct drm_display_mode *mode,
+					 struct drm_display_mode *adjusted_mode)
+{
+	struct drm_device *dev = encoder->dev;
+	struct shmob_drm_device *sdev = dev->dev_private;
+	struct drm_connector *connector = &sdev->connector.connector;
+	const struct drm_display_mode *panel_mode;
+
+	if (list_empty(&connector->modes)) {
+		dev_dbg(dev->dev, "mode_fixup: empty modes list\n");
+		return false;
+	}
+
+	/* The flat panel mode is fixed, just copy it to the adjusted mode. */
+	panel_mode = list_first_entry(&connector->modes,
+				      struct drm_display_mode, head);
+	drm_mode_copy(adjusted_mode, panel_mode);
+
+	return true;
+}
+
+static void shmob_drm_encoder_mode_prepare(struct drm_encoder *encoder)
+{
+	/* No-op, everything is handled in the CRTC code. */
+}
+
+static void shmob_drm_encoder_mode_set(struct drm_encoder *encoder,
+				       struct drm_display_mode *mode,
+				       struct drm_display_mode *adjusted_mode)
+{
+	/* No-op, everything is handled in the CRTC code. */
+}
+
+static void shmob_drm_encoder_mode_commit(struct drm_encoder *encoder)
+{
+	/* No-op, everything is handled in the CRTC code. */
+}
+
+static const struct drm_encoder_helper_funcs encoder_helper_funcs = {
+	.dpms = shmob_drm_encoder_dpms,
+	.mode_fixup = shmob_drm_encoder_mode_fixup,
+	.prepare = shmob_drm_encoder_mode_prepare,
+	.commit = shmob_drm_encoder_mode_commit,
+	.mode_set = shmob_drm_encoder_mode_set,
+};
+
+static void shmob_drm_encoder_destroy(struct drm_encoder *encoder)
+{
+	drm_encoder_cleanup(encoder);
+}
+
+static const struct drm_encoder_funcs encoder_funcs = {
+	.destroy = shmob_drm_encoder_destroy,
+};
+
+int shmob_drm_encoder_create(struct shmob_drm_device *sdev)
+{
+	struct drm_encoder *encoder = &sdev->encoder.encoder;
+	int ret;
+
+	sdev->encoder.dpms = DRM_MODE_DPMS_OFF;
+
+	encoder->possible_crtcs = 1;
+
+	ret = drm_encoder_init(sdev->ddev, encoder, &encoder_funcs,
+			       DRM_MODE_ENCODER_LVDS);
+	if (ret < 0)
+		return ret;
+
+	drm_encoder_helper_add(encoder, &encoder_helper_funcs);
+
+	return 0;
+}
+
+void shmob_drm_crtc_enable_vblank(struct shmob_drm_device *sdev, bool enable)
+{
+	unsigned long flags;
+	u32 ldintr;
+
+	/* Be careful not to acknowledge any pending interrupt. */
+	spin_lock_irqsave(&sdev->irq_lock, flags);
+	ldintr = lcdc_read(sdev, LDINTR) | LDINTR_STATUS_MASK;
+	if (enable)
+		ldintr |= LDINTR_VEE;
+	else
+		ldintr &= ~LDINTR_VEE;
+	lcdc_write(sdev, LDINTR, ldintr);
+	spin_unlock_irqrestore(&sdev->irq_lock, flags);
+}
+
+/* -----------------------------------------------------------------------------
+ * Connector
+ */
+
+#define to_shmob_connector(c) \
+	container_of(c, struct shmob_drm_connector, connector)
+
+static int shmob_drm_connector_get_modes(struct drm_connector *connector)
+{
+	struct shmob_drm_device *sdev = connector->dev->dev_private;
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_create(connector->dev);
+	if (mode == NULL)
+		return 0;
+
+	mode->type = DRM_MODE_TYPE_PREFERRED | DRM_MODE_TYPE_DRIVER;
+	mode->clock = sdev->pdata->panel.mode.clock;
+	mode->hdisplay = sdev->pdata->panel.mode.hdisplay;
+	mode->hsync_start = sdev->pdata->panel.mode.hsync_start;
+	mode->hsync_end = sdev->pdata->panel.mode.hsync_end;
+	mode->htotal = sdev->pdata->panel.mode.htotal;
+	mode->vdisplay = sdev->pdata->panel.mode.vdisplay;
+	mode->vsync_start = sdev->pdata->panel.mode.vsync_start;
+	mode->vsync_end = sdev->pdata->panel.mode.vsync_end;
+	mode->vtotal = sdev->pdata->panel.mode.vtotal;
+	mode->flags = sdev->pdata->panel.mode.flags;
+
+	drm_mode_set_name(mode);
+	drm_mode_probed_add(connector, mode);
+
+	connector->display_info.width_mm = sdev->pdata->panel.width_mm;
+	connector->display_info.height_mm = sdev->pdata->panel.height_mm;
+
+	return 1;
+}
+
+static int shmob_drm_connector_mode_valid(struct drm_connector *connector,
+					  struct drm_display_mode *mode)
+{
+	return MODE_OK;
+}
+
+static struct drm_encoder *
+shmob_drm_connector_best_encoder(struct drm_connector *connector)
+{
+	struct shmob_drm_connector *scon = to_shmob_connector(connector);
+
+	return scon->encoder;
+}
+
+static const struct drm_connector_helper_funcs connector_helper_funcs = {
+	.get_modes = shmob_drm_connector_get_modes,
+	.mode_valid = shmob_drm_connector_mode_valid,
+	.best_encoder = shmob_drm_connector_best_encoder,
+};
+
+static void shmob_drm_connector_destroy(struct drm_connector *connector)
+{
+	struct shmob_drm_connector *scon = to_shmob_connector(connector);
+
+	shmob_drm_backlight_exit(scon);
+	drm_sysfs_connector_remove(connector);
+	drm_connector_cleanup(connector);
+}
+
+static enum drm_connector_status
+shmob_drm_connector_detect(struct drm_connector *connector, bool force)
+{
+	return connector_status_connected;
+}
+
+static const struct drm_connector_funcs connector_funcs = {
+	.dpms = drm_helper_connector_dpms,
+	.detect = shmob_drm_connector_detect,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = shmob_drm_connector_destroy,
+};
+
+int shmob_drm_connector_create(struct shmob_drm_device *sdev,
+			       struct drm_encoder *encoder)
+{
+	struct drm_connector *connector = &sdev->connector.connector;
+	int ret;
+
+	sdev->connector.encoder = encoder;
+
+	connector->display_info.width_mm = sdev->pdata->panel.width_mm;
+	connector->display_info.height_mm = sdev->pdata->panel.height_mm;
+
+	ret = drm_connector_init(sdev->ddev, connector, &connector_funcs,
+				 DRM_MODE_CONNECTOR_LVDS);
+	if (ret < 0)
+		return ret;
+
+	drm_connector_helper_add(connector, &connector_helper_funcs);
+	ret = drm_sysfs_connector_add(connector);
+	if (ret < 0)
+		goto err_cleanup;
+
+	ret = shmob_drm_backlight_init(&sdev->connector);
+	if (ret < 0)
+		goto err_sysfs;
+
+	ret = drm_mode_connector_attach_encoder(connector, encoder);
+	if (ret < 0)
+		goto err_backlight;
+
+	connector->encoder = encoder;
+
+	drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+	drm_connector_property_set_value(connector,
+		sdev->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
+
+	return 0;
+
+err_backlight:
+	shmob_drm_backlight_exit(&sdev->connector);
+err_sysfs:
+	drm_sysfs_connector_remove(connector);
+err_cleanup:
+	drm_connector_cleanup(connector);
+	return ret;
+}
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.h b/drivers/gpu/drm/shmobile/shmob_drm_crtc.h
new file mode 100644
index 00000000000..e5bd109c4c3
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.h
@@ -0,0 +1,60 @@
+/*
+ * shmob_drm_crtc.h  --  SH Mobile DRM CRTCs
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_CRTC_H__
+#define __SHMOB_DRM_CRTC_H__
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+
+struct backlight_device;
+struct shmob_drm_device;
+
+struct shmob_drm_crtc {
+	struct drm_crtc crtc;
+
+	struct drm_pending_vblank_event *event;
+	int dpms;
+
+	const struct shmob_drm_format_info *format;
+	void *cache;
+	unsigned long dma[2];
+	unsigned int line_size;
+	bool started;
+};
+
+struct shmob_drm_encoder {
+	struct drm_encoder encoder;
+	int dpms;
+};
+
+struct shmob_drm_connector {
+	struct drm_connector connector;
+	struct drm_encoder *encoder;
+
+	struct backlight_device *backlight;
+};
+
+int shmob_drm_crtc_create(struct shmob_drm_device *sdev);
+void shmob_drm_crtc_enable_vblank(struct shmob_drm_device *sdev, bool enable);
+void shmob_drm_crtc_cancel_page_flip(struct shmob_drm_crtc *scrtc,
+				     struct drm_file *file);
+void shmob_drm_crtc_finish_page_flip(struct shmob_drm_crtc *scrtc);
+void shmob_drm_crtc_suspend(struct shmob_drm_crtc *scrtc);
+void shmob_drm_crtc_resume(struct shmob_drm_crtc *scrtc);
+
+int shmob_drm_encoder_create(struct shmob_drm_device *sdev);
+int shmob_drm_connector_create(struct shmob_drm_device *sdev,
+			       struct drm_encoder *encoder);
+
+#endif /* __SHMOB_DRM_CRTC_H__ */
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
new file mode 100644
index 00000000000..c71d493fd0c
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -0,0 +1,361 @@
+/*
+ * shmob_drm_drv.c  --  SH Mobile DRM driver
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/slab.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include "shmob_drm_crtc.h"
+#include "shmob_drm_drv.h"
+#include "shmob_drm_kms.h"
+#include "shmob_drm_plane.h"
+#include "shmob_drm_regs.h"
+
+/* -----------------------------------------------------------------------------
+ * Hardware initialization
+ */
+
+static int __devinit shmob_drm_init_interface(struct shmob_drm_device *sdev)
+{
+	static const u32 ldmt1r[] = {
+		[SHMOB_DRM_IFACE_RGB8] = LDMT1R_MIFTYP_RGB8,
+		[SHMOB_DRM_IFACE_RGB9] = LDMT1R_MIFTYP_RGB9,
+		[SHMOB_DRM_IFACE_RGB12A] = LDMT1R_MIFTYP_RGB12A,
+		[SHMOB_DRM_IFACE_RGB12B] = LDMT1R_MIFTYP_RGB12B,
+		[SHMOB_DRM_IFACE_RGB16] = LDMT1R_MIFTYP_RGB16,
+		[SHMOB_DRM_IFACE_RGB18] = LDMT1R_MIFTYP_RGB18,
+		[SHMOB_DRM_IFACE_RGB24] = LDMT1R_MIFTYP_RGB24,
+		[SHMOB_DRM_IFACE_YUV422] = LDMT1R_MIFTYP_YCBCR,
+		[SHMOB_DRM_IFACE_SYS8A] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS8A,
+		[SHMOB_DRM_IFACE_SYS8B] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS8B,
+		[SHMOB_DRM_IFACE_SYS8C] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS8C,
+		[SHMOB_DRM_IFACE_SYS8D] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS8D,
+		[SHMOB_DRM_IFACE_SYS9] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS9,
+		[SHMOB_DRM_IFACE_SYS12] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS12,
+		[SHMOB_DRM_IFACE_SYS16A] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS16A,
+		[SHMOB_DRM_IFACE_SYS16B] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS16B,
+		[SHMOB_DRM_IFACE_SYS16C] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS16C,
+		[SHMOB_DRM_IFACE_SYS18] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS18,
+		[SHMOB_DRM_IFACE_SYS24] = LDMT1R_IFM | LDMT1R_MIFTYP_SYS24,
+	};
+
+	if (sdev->pdata->iface.interface >= ARRAY_SIZE(ldmt1r)) {
+		dev_err(sdev->dev, "invalid interface type %u\n",
+			sdev->pdata->iface.interface);
+		return -EINVAL;
+	}
+
+	sdev->ldmt1r = ldmt1r[sdev->pdata->iface.interface];
+	return 0;
+}
+
+static int __devinit shmob_drm_setup_clocks(struct shmob_drm_device *sdev,
+					    enum shmob_drm_clk_source clksrc)
+{
+	struct clk *clk;
+	char *clkname;
+
+	switch (clksrc) {
+	case SHMOB_DRM_CLK_BUS:
+		clkname = "bus_clk";
+		sdev->lddckr = LDDCKR_ICKSEL_BUS;
+		break;
+	case SHMOB_DRM_CLK_PERIPHERAL:
+		clkname = "peripheral_clk";
+		sdev->lddckr = LDDCKR_ICKSEL_MIPI;
+		break;
+	case SHMOB_DRM_CLK_EXTERNAL:
+		clkname = NULL;
+		sdev->lddckr = LDDCKR_ICKSEL_HDMI;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	clk = clk_get(sdev->dev, clkname);
+	if (IS_ERR(clk)) {
+		dev_err(sdev->dev, "cannot get dot clock %s\n", clkname);
+		return PTR_ERR(clk);
+	}
+
+	sdev->clock = clk;
+	return 0;
+}
+
+/* -----------------------------------------------------------------------------
+ * DRM operations
+ */
+
+static int shmob_drm_unload(struct drm_device *dev)
+{
+	struct shmob_drm_device *sdev = dev->dev_private;
+
+	drm_kms_helper_poll_fini(dev);
+	drm_mode_config_cleanup(dev);
+	drm_vblank_cleanup(dev);
+	drm_irq_uninstall(dev);
+
+	if (sdev->clock)
+		clk_put(sdev->clock);
+
+	if (sdev->mmio)
+		iounmap(sdev->mmio);
+
+	dev->dev_private = NULL;
+	kfree(sdev);
+
+	return 0;
+}
+
+static int shmob_drm_load(struct drm_device *dev, unsigned long flags)
+{
+	struct shmob_drm_platform_data *pdata = dev->dev->platform_data;
+	struct platform_device *pdev = dev->platformdev;
+	struct shmob_drm_device *sdev;
+	struct resource *res;
+	unsigned int i;
+	int ret;
+
+	if (pdata == NULL) {
+		dev_err(dev->dev, "no platform data\n");
+		return -EINVAL;
+	}
+
+	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
+	if (sdev == NULL) {
+		dev_err(dev->dev, "failed to allocate private data\n");
+		return -ENOMEM;
+	}
+
+	sdev->dev = &pdev->dev;
+	sdev->pdata = pdata;
+	spin_lock_init(&sdev->irq_lock);
+
+	sdev->ddev = dev;
+	dev->dev_private = sdev;
+
+	/* I/O resources and clocks */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (res == NULL) {
+		dev_err(&pdev->dev, "failed to get memory resource\n");
+		ret = -EINVAL;
+		goto done;
+	}
+
+	sdev->mmio = ioremap_nocache(res->start, resource_size(res));
+	if (sdev->mmio == NULL) {
+		dev_err(&pdev->dev, "failed to remap memory resource\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	ret = shmob_drm_setup_clocks(sdev, pdata->clk_source);
+	if (ret < 0)
+		goto done;
+
+	ret = shmob_drm_init_interface(sdev);
+	if (ret < 0)
+		goto done;
+
+	ret = shmob_drm_modeset_init(sdev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to initialize mode setting\n");
+		goto done;
+	}
+
+	for (i = 0; i < 4; ++i) {
+		ret = shmob_drm_plane_create(sdev, i);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "failed to create plane %u\n", i);
+			goto done;
+		}
+	}
+
+	ret = drm_vblank_init(dev, 1);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to initialize vblank\n");
+		goto done;
+	}
+
+	ret = drm_irq_install(dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to install IRQ handler\n");
+		goto done;
+	}
+
+done:
+	if (ret)
+		shmob_drm_unload(dev);
+
+	return ret;
+}
+
+static void shmob_drm_preclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct shmob_drm_device *sdev = dev->dev_private;
+
+	shmob_drm_crtc_cancel_page_flip(&sdev->crtc, file);
+}
+
+static irqreturn_t shmob_drm_irq(int irq, void *arg)
+{
+	struct drm_device *dev = arg;
+	struct shmob_drm_device *sdev = dev->dev_private;
+	unsigned long flags;
+	u32 status;
+
+	/* Acknowledge interrupts. Putting interrupt enable and interrupt flag
+	 * bits in the same register is really brain-dead design and requires
+	 * taking a spinlock.
+	 */
+	spin_lock_irqsave(&sdev->irq_lock, flags);
+	status = lcdc_read(sdev, LDINTR);
+	lcdc_write(sdev, LDINTR, status ^ LDINTR_STATUS_MASK);
+	spin_unlock_irqrestore(&sdev->irq_lock, flags);
+
+	if (status & LDINTR_VES) {
+		drm_handle_vblank(dev, 0);
+		shmob_drm_crtc_finish_page_flip(&sdev->crtc);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int shmob_drm_enable_vblank(struct drm_device *dev, int crtc)
+{
+	struct shmob_drm_device *sdev = dev->dev_private;
+
+	shmob_drm_crtc_enable_vblank(sdev, true);
+
+	return 0;
+}
+
+static void shmob_drm_disable_vblank(struct drm_device *dev, int crtc)
+{
+	struct shmob_drm_device *sdev = dev->dev_private;
+
+	shmob_drm_crtc_enable_vblank(sdev, false);
+}
+
+static const struct file_operations shmob_drm_fops = {
+	.owner		= THIS_MODULE,
+	.open		= drm_open,
+	.release	= drm_release,
+	.unlocked_ioctl	= drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= drm_compat_ioctl,
+#endif
+	.poll		= drm_poll,
+	.read		= drm_read,
+	.fasync		= drm_fasync,
+	.llseek		= no_llseek,
+	.mmap		= drm_gem_cma_mmap,
+};
+
+static struct drm_driver shmob_drm_driver = {
+	.driver_features	= DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET,
+	.load			= shmob_drm_load,
+	.unload			= shmob_drm_unload,
+	.preclose		= shmob_drm_preclose,
+	.irq_handler		= shmob_drm_irq,
+	.get_vblank_counter	= drm_vblank_count,
+	.enable_vblank		= shmob_drm_enable_vblank,
+	.disable_vblank		= shmob_drm_disable_vblank,
+	.gem_free_object	= drm_gem_cma_free_object,
+	.gem_vm_ops		= &drm_gem_cma_vm_ops,
+	.dumb_create		= drm_gem_cma_dumb_create,
+	.dumb_map_offset	= drm_gem_cma_dumb_map_offset,
+	.dumb_destroy		= drm_gem_cma_dumb_destroy,
+	.fops			= &shmob_drm_fops,
+	.name			= "shmob-drm",
+	.desc			= "Renesas SH Mobile DRM",
+	.date			= "20120424",
+	.major			= 1,
+	.minor			= 0,
+};
+
+/* -----------------------------------------------------------------------------
+ * Power management
+ */
+
+#if CONFIG_PM_SLEEP
+static int shmob_drm_pm_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+	struct shmob_drm_device *sdev = ddev->dev_private;
+
+	drm_kms_helper_poll_disable(ddev);
+	shmob_drm_crtc_suspend(&sdev->crtc);
+
+	return 0;
+}
+
+static int shmob_drm_pm_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *ddev = platform_get_drvdata(pdev);
+	struct shmob_drm_device *sdev = ddev->dev_private;
+
+	mutex_lock(&sdev->ddev->mode_config.mutex);
+	shmob_drm_crtc_resume(&sdev->crtc);
+	mutex_unlock(&sdev->ddev->mode_config.mutex);
+
+	drm_kms_helper_poll_enable(sdev->ddev);
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops shmob_drm_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(shmob_drm_pm_suspend, shmob_drm_pm_resume)
+};
+
+/* -----------------------------------------------------------------------------
+ * Platform driver
+ */
+
+static int __devinit shmob_drm_probe(struct platform_device *pdev)
+{
+	return drm_platform_init(&shmob_drm_driver, pdev);
+}
+
+static int __devexit shmob_drm_remove(struct platform_device *pdev)
+{
+	drm_platform_exit(&shmob_drm_driver, pdev);
+
+	return 0;
+}
+
+static struct platform_driver shmob_drm_platform_driver = {
+	.probe		= shmob_drm_probe,
+	.remove		= __devexit_p(shmob_drm_remove),
+	.driver		= {
+		.owner	= THIS_MODULE,
+		.name	= "shmob-drm",
+		.pm	= &shmob_drm_pm_ops,
+	},
+};
+
+module_platform_driver(shmob_drm_platform_driver);
+
+MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
+MODULE_DESCRIPTION("Renesas SH Mobile DRM Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.h b/drivers/gpu/drm/shmobile/shmob_drm_drv.h
new file mode 100644
index 00000000000..4d46b811b5a
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.h
@@ -0,0 +1,47 @@
+/*
+ * shmob_drm.h  --  SH Mobile DRM driver
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_DRV_H__
+#define __SHMOB_DRM_DRV_H__
+
+#include <linux/kernel.h>
+#include <linux/platform_data/shmob_drm.h>
+#include <linux/spinlock.h>
+
+#include "shmob_drm_crtc.h"
+
+struct clk;
+struct device;
+struct drm_device;
+struct sh_mobile_meram_info;
+
+struct shmob_drm_device {
+	struct device *dev;
+	const struct shmob_drm_platform_data *pdata;
+
+	void __iomem *mmio;
+	struct clk *clock;
+	struct sh_mobile_meram_info *meram;
+	u32 lddckr;
+	u32 ldmt1r;
+
+	spinlock_t irq_lock;		/* Protects hardware LDINTR register */
+
+	struct drm_device *ddev;
+
+	struct shmob_drm_crtc crtc;
+	struct shmob_drm_encoder encoder;
+	struct shmob_drm_connector connector;
+};
+
+#endif /* __SHMOB_DRM_DRV_H__ */
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.c b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
new file mode 100644
index 00000000000..c291ee385b4
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
@@ -0,0 +1,160 @@
+/*
+ * shmob_drm_kms.c  --  SH Mobile DRM Mode Setting
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include <video/sh_mobile_meram.h>
+
+#include "shmob_drm_crtc.h"
+#include "shmob_drm_drv.h"
+#include "shmob_drm_kms.h"
+#include "shmob_drm_regs.h"
+
+/* -----------------------------------------------------------------------------
+ * Format helpers
+ */
+
+static const struct shmob_drm_format_info shmob_drm_format_infos[] = {
+	{
+		.fourcc = DRM_FORMAT_RGB565,
+		.bpp = 16,
+		.yuv = false,
+		.lddfr = LDDFR_PKF_RGB16,
+		.meram = SH_MOBILE_MERAM_PF_RGB,
+	}, {
+		.fourcc = DRM_FORMAT_RGB888,
+		.bpp = 24,
+		.yuv = false,
+		.lddfr = LDDFR_PKF_RGB24,
+		.meram = SH_MOBILE_MERAM_PF_RGB,
+	}, {
+		.fourcc = DRM_FORMAT_ARGB8888,
+		.bpp = 32,
+		.yuv = false,
+		.lddfr = LDDFR_PKF_ARGB32,
+		.meram = SH_MOBILE_MERAM_PF_RGB,
+	}, {
+		.fourcc = DRM_FORMAT_NV12,
+		.bpp = 12,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_420,
+		.meram = SH_MOBILE_MERAM_PF_NV,
+	}, {
+		.fourcc = DRM_FORMAT_NV21,
+		.bpp = 12,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_420,
+		.meram = SH_MOBILE_MERAM_PF_NV,
+	}, {
+		.fourcc = DRM_FORMAT_NV16,
+		.bpp = 16,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_422,
+		.meram = SH_MOBILE_MERAM_PF_NV,
+	}, {
+		.fourcc = DRM_FORMAT_NV61,
+		.bpp = 16,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_422,
+		.meram = SH_MOBILE_MERAM_PF_NV,
+	}, {
+		.fourcc = DRM_FORMAT_NV24,
+		.bpp = 24,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_444,
+		.meram = SH_MOBILE_MERAM_PF_NV24,
+	}, {
+		.fourcc = DRM_FORMAT_NV42,
+		.bpp = 24,
+		.yuv = true,
+		.lddfr = LDDFR_CC | LDDFR_YF_444,
+		.meram = SH_MOBILE_MERAM_PF_NV24,
+	},
+};
+
+const struct shmob_drm_format_info *shmob_drm_format_info(u32 fourcc)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(shmob_drm_format_infos); ++i) {
+		if (shmob_drm_format_infos[i].fourcc == fourcc)
+			return &shmob_drm_format_infos[i];
+	}
+
+	return NULL;
+}
+
+/* -----------------------------------------------------------------------------
+ * Frame buffer
+ */
+
+static struct drm_framebuffer *
+shmob_drm_fb_create(struct drm_device *dev, struct drm_file *file_priv,
+		    struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	const struct shmob_drm_format_info *format;
+
+	format = shmob_drm_format_info(mode_cmd->pixel_format);
+	if (format == NULL) {
+		dev_dbg(dev->dev, "unsupported pixel format %08x\n",
+			mode_cmd->pixel_format);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (mode_cmd->pitches[0] & 7 || mode_cmd->pitches[0] >= 65536) {
+		dev_dbg(dev->dev, "valid pitch value %u\n",
+			mode_cmd->pitches[0]);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (format->yuv) {
+		unsigned int chroma_cpp = format->bpp == 24 ? 2 : 1;
+
+		if (mode_cmd->pitches[1] != mode_cmd->pitches[0] * chroma_cpp) {
+			dev_dbg(dev->dev,
+				"luma and chroma pitches do not match\n");
+			return ERR_PTR(-EINVAL);
+		}
+	}
+
+	return drm_fb_cma_create(dev, file_priv, mode_cmd);
+}
+
+static const struct drm_mode_config_funcs shmob_drm_mode_config_funcs = {
+	.fb_create = shmob_drm_fb_create,
+};
+
+int shmob_drm_modeset_init(struct shmob_drm_device *sdev)
+{
+	drm_mode_config_init(sdev->ddev);
+
+	shmob_drm_crtc_create(sdev);
+	shmob_drm_encoder_create(sdev);
+	shmob_drm_connector_create(sdev, &sdev->encoder.encoder);
+
+	drm_kms_helper_poll_init(sdev->ddev);
+
+	sdev->ddev->mode_config.min_width = 0;
+	sdev->ddev->mode_config.min_height = 0;
+	sdev->ddev->mode_config.max_width = 4095;
+	sdev->ddev->mode_config.max_height = 4095;
+	sdev->ddev->mode_config.funcs = &shmob_drm_mode_config_funcs;
+
+	drm_helper_disable_unused_functions(sdev->ddev);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.h b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
new file mode 100644
index 00000000000..9495c911130
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
@@ -0,0 +1,34 @@
+/*
+ * shmob_drm_kms.h  --  SH Mobile DRM Mode Setting
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_KMS_H__
+#define __SHMOB_DRM_KMS_H__
+
+#include <linux/types.h>
+
+struct drm_gem_cma_object;
+struct shmob_drm_device;
+
+struct shmob_drm_format_info {
+	u32 fourcc;
+	unsigned int bpp;
+	bool yuv;
+	u32 lddfr;
+	unsigned int meram;
+};
+
+const struct shmob_drm_format_info *shmob_drm_format_info(u32 fourcc);
+
+int shmob_drm_modeset_init(struct shmob_drm_device *sdev);
+
+#endif /* __SHMOB_DRM_KMS_H__ */
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
new file mode 100644
index 00000000000..e1eb899b028
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
@@ -0,0 +1,268 @@
+/*
+ * shmob_drm_plane.c  --  SH Mobile DRM Planes
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+
+#include <video/sh_mobile_meram.h>
+
+#include "shmob_drm_drv.h"
+#include "shmob_drm_kms.h"
+#include "shmob_drm_plane.h"
+#include "shmob_drm_regs.h"
+
+struct shmob_drm_plane {
+	struct drm_plane plane;
+	unsigned int index;
+	unsigned int alpha;
+
+	const struct shmob_drm_format_info *format;
+	unsigned long dma[2];
+
+	unsigned int src_x;
+	unsigned int src_y;
+	unsigned int crtc_x;
+	unsigned int crtc_y;
+	unsigned int crtc_w;
+	unsigned int crtc_h;
+};
+
+#define to_shmob_plane(p)	container_of(p, struct shmob_drm_plane, plane)
+
+static void shmob_drm_plane_compute_base(struct shmob_drm_plane *splane,
+					 struct drm_framebuffer *fb,
+					 int x, int y)
+{
+	struct drm_gem_cma_object *gem;
+	unsigned int bpp;
+
+	bpp = splane->format->yuv ? 8 : splane->format->bpp;
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	splane->dma[0] = gem->paddr + fb->offsets[0]
+		       + y * fb->pitches[0] + x * bpp / 8;
+
+	if (splane->format->yuv) {
+		bpp = splane->format->bpp - 8;
+		gem = drm_fb_cma_get_gem_obj(fb, 1);
+		splane->dma[1] = gem->paddr + fb->offsets[1]
+			       + y / (bpp == 4 ? 2 : 1) * fb->pitches[1]
+			       + x * (bpp == 16 ? 2 : 1);
+	}
+}
+
+static void __shmob_drm_plane_setup(struct shmob_drm_plane *splane,
+				    struct drm_framebuffer *fb)
+{
+	struct shmob_drm_device *sdev = splane->plane.dev->dev_private;
+	u32 format;
+
+	/* TODO: Support ROP3 mode */
+	format = LDBBSIFR_EN | (splane->alpha << LDBBSIFR_LAY_SHIFT);
+
+	switch (splane->format->fourcc) {
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_NV42:
+		format |= LDBBSIFR_SWPL | LDBBSIFR_SWPW;
+		break;
+	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV24:
+		format |= LDBBSIFR_SWPL | LDBBSIFR_SWPW | LDBBSIFR_SWPB;
+		break;
+	case DRM_FORMAT_ARGB8888:
+	default:
+		format |= LDBBSIFR_SWPL;
+		break;
+	}
+
+	switch (splane->format->fourcc) {
+	case DRM_FORMAT_RGB565:
+		format |= LDBBSIFR_AL_1 | LDBBSIFR_RY | LDBBSIFR_RPKF_RGB16;
+		break;
+	case DRM_FORMAT_RGB888:
+		format |= LDBBSIFR_AL_1 | LDBBSIFR_RY | LDBBSIFR_RPKF_RGB24;
+		break;
+	case DRM_FORMAT_ARGB8888:
+		format |= LDBBSIFR_AL_PK | LDBBSIFR_RY | LDDFR_PKF_ARGB32;
+		break;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+		format |= LDBBSIFR_AL_1 | LDBBSIFR_CHRR_420;
+		break;
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+		format |= LDBBSIFR_AL_1 | LDBBSIFR_CHRR_422;
+		break;
+	case DRM_FORMAT_NV24:
+	case DRM_FORMAT_NV42:
+		format |= LDBBSIFR_AL_1 | LDBBSIFR_CHRR_444;
+		break;
+	}
+
+#define plane_reg_dump(sdev, splane, reg) \
+	dev_dbg(sdev->ddev->dev, "%s(%u): %s 0x%08x 0x%08x\n", __func__, \
+		splane->index, #reg, \
+		lcdc_read(sdev, reg(splane->index)), \
+		lcdc_read(sdev, reg(splane->index) + LCDC_SIDE_B_OFFSET))
+
+	plane_reg_dump(sdev, splane, LDBnBSIFR);
+	plane_reg_dump(sdev, splane, LDBnBSSZR);
+	plane_reg_dump(sdev, splane, LDBnBLOCR);
+	plane_reg_dump(sdev, splane, LDBnBSMWR);
+	plane_reg_dump(sdev, splane, LDBnBSAYR);
+	plane_reg_dump(sdev, splane, LDBnBSACR);
+
+	lcdc_write(sdev, LDBCR, LDBCR_UPC(splane->index));
+	dev_dbg(sdev->ddev->dev, "%s(%u): %s 0x%08x\n", __func__, splane->index,
+		"LDBCR", lcdc_read(sdev, LDBCR));
+
+	lcdc_write(sdev, LDBnBSIFR(splane->index), format);
+
+	lcdc_write(sdev, LDBnBSSZR(splane->index),
+		   (splane->crtc_h << LDBBSSZR_BVSS_SHIFT) |
+		   (splane->crtc_w << LDBBSSZR_BHSS_SHIFT));
+	lcdc_write(sdev, LDBnBLOCR(splane->index),
+		   (splane->crtc_y << LDBBLOCR_CVLC_SHIFT) |
+		   (splane->crtc_x << LDBBLOCR_CHLC_SHIFT));
+	lcdc_write(sdev, LDBnBSMWR(splane->index),
+		   fb->pitches[0] << LDBBSMWR_BSMW_SHIFT);
+
+	shmob_drm_plane_compute_base(splane, fb, splane->src_x, splane->src_y);
+
+	lcdc_write(sdev, LDBnBSAYR(splane->index), splane->dma[0]);
+	if (splane->format->yuv)
+		lcdc_write(sdev, LDBnBSACR(splane->index), splane->dma[1]);
+
+	lcdc_write(sdev, LDBCR,
+		   LDBCR_UPF(splane->index) | LDBCR_UPD(splane->index));
+	dev_dbg(sdev->ddev->dev, "%s(%u): %s 0x%08x\n", __func__, splane->index,
+		"LDBCR", lcdc_read(sdev, LDBCR));
+
+	plane_reg_dump(sdev, splane, LDBnBSIFR);
+	plane_reg_dump(sdev, splane, LDBnBSSZR);
+	plane_reg_dump(sdev, splane, LDBnBLOCR);
+	plane_reg_dump(sdev, splane, LDBnBSMWR);
+	plane_reg_dump(sdev, splane, LDBnBSAYR);
+	plane_reg_dump(sdev, splane, LDBnBSACR);
+}
+
+void shmob_drm_plane_setup(struct drm_plane *plane)
+{
+	struct shmob_drm_plane *splane = to_shmob_plane(plane);
+
+	if (plane->fb == NULL || !plane->enabled)
+		return;
+
+	__shmob_drm_plane_setup(splane, plane->fb);
+}
+
+static int
+shmob_drm_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+		       struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+		       unsigned int crtc_w, unsigned int crtc_h,
+		       uint32_t src_x, uint32_t src_y,
+		       uint32_t src_w, uint32_t src_h)
+{
+	struct shmob_drm_plane *splane = to_shmob_plane(plane);
+	struct shmob_drm_device *sdev = plane->dev->dev_private;
+	const struct shmob_drm_format_info *format;
+
+	format = shmob_drm_format_info(fb->pixel_format);
+	if (format == NULL) {
+		dev_dbg(sdev->dev, "update_plane: unsupported format %08x\n",
+			fb->pixel_format);
+		return -EINVAL;
+	}
+
+	if (src_w >> 16 != crtc_w || src_h >> 16 != crtc_h) {
+		dev_dbg(sdev->dev, "%s: scaling not supported\n", __func__);
+		return -EINVAL;
+	}
+
+	splane->format = format;
+
+	splane->src_x = src_x >> 16;
+	splane->src_y = src_y >> 16;
+	splane->crtc_x = crtc_x;
+	splane->crtc_y = crtc_y;
+	splane->crtc_w = crtc_w;
+	splane->crtc_h = crtc_h;
+
+	__shmob_drm_plane_setup(splane, fb);
+	return 0;
+}
+
+static int shmob_drm_plane_disable(struct drm_plane *plane)
+{
+	struct shmob_drm_plane *splane = to_shmob_plane(plane);
+	struct shmob_drm_device *sdev = plane->dev->dev_private;
+
+	splane->format = NULL;
+
+	lcdc_write(sdev, LDBnBSIFR(splane->index), 0);
+	return 0;
+}
+
+static void shmob_drm_plane_destroy(struct drm_plane *plane)
+{
+	struct shmob_drm_plane *splane = to_shmob_plane(plane);
+
+	shmob_drm_plane_disable(plane);
+	drm_plane_cleanup(plane);
+	kfree(splane);
+}
+
+static const struct drm_plane_funcs shmob_drm_plane_funcs = {
+	.update_plane = shmob_drm_plane_update,
+	.disable_plane = shmob_drm_plane_disable,
+	.destroy = shmob_drm_plane_destroy,
+};
+
+static const uint32_t formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
+	DRM_FORMAT_NV16,
+	DRM_FORMAT_NV61,
+	DRM_FORMAT_NV24,
+	DRM_FORMAT_NV42,
+};
+
+int shmob_drm_plane_create(struct shmob_drm_device *sdev, unsigned int index)
+{
+	struct shmob_drm_plane *splane;
+	int ret;
+
+	splane = kzalloc(sizeof(*splane), GFP_KERNEL);
+	if (splane == NULL)
+		return -ENOMEM;
+
+	splane->index = index;
+	splane->alpha = 255;
+
+	ret = drm_plane_init(sdev->ddev, &splane->plane, 1,
+			     &shmob_drm_plane_funcs, formats,
+			     ARRAY_SIZE(formats), false);
+	if (ret < 0)
+		kfree(splane);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_plane.h b/drivers/gpu/drm/shmobile/shmob_drm_plane.h
new file mode 100644
index 00000000000..99623d05e3b
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_plane.h
@@ -0,0 +1,22 @@
+/*
+ * shmob_drm_plane.h  --  SH Mobile DRM Planes
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_PLANE_H__
+#define __SHMOB_DRM_PLANE_H__
+
+struct shmob_drm_device;
+
+int shmob_drm_plane_create(struct shmob_drm_device *sdev, unsigned int index);
+void shmob_drm_plane_setup(struct drm_plane *plane);
+
+#endif /* __SHMOB_DRM_PLANE_H__ */
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_regs.h b/drivers/gpu/drm/shmobile/shmob_drm_regs.h
new file mode 100644
index 00000000000..7923cdd6368
--- /dev/null
+++ b/drivers/gpu/drm/shmobile/shmob_drm_regs.h
@@ -0,0 +1,311 @@
+/*
+ * shmob_drm_regs.h  --  SH Mobile DRM registers
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_REGS_H__
+#define __SHMOB_DRM_REGS_H__
+
+#include <linux/io.h>
+
+/* Register definitions */
+#define LDDCKPAT1R		0x400
+#define LDDCKPAT2R		0x404
+#define LDDCKR			0x410
+#define LDDCKR_ICKSEL_BUS	(0 << 16)
+#define LDDCKR_ICKSEL_MIPI	(1 << 16)
+#define LDDCKR_ICKSEL_HDMI	(2 << 16)
+#define LDDCKR_ICKSEL_EXT	(3 << 16)
+#define LDDCKR_ICKSEL_MASK	(7 << 16)
+#define LDDCKR_MOSEL		(1 << 6)
+#define LDDCKSTPR		0x414
+#define LDDCKSTPR_DCKSTS	(1 << 16)
+#define LDDCKSTPR_DCKSTP	(1 << 0)
+#define LDMT1R			0x418
+#define LDMT1R_VPOL		(1 << 28)
+#define LDMT1R_HPOL		(1 << 27)
+#define LDMT1R_DWPOL		(1 << 26)
+#define LDMT1R_DIPOL		(1 << 25)
+#define LDMT1R_DAPOL		(1 << 24)
+#define LDMT1R_HSCNT		(1 << 17)
+#define LDMT1R_DWCNT		(1 << 16)
+#define LDMT1R_IFM		(1 << 12)
+#define LDMT1R_MIFTYP_RGB8	(0x0 << 0)
+#define LDMT1R_MIFTYP_RGB9	(0x4 << 0)
+#define LDMT1R_MIFTYP_RGB12A	(0x5 << 0)
+#define LDMT1R_MIFTYP_RGB12B	(0x6 << 0)
+#define LDMT1R_MIFTYP_RGB16	(0x7 << 0)
+#define LDMT1R_MIFTYP_RGB18	(0xa << 0)
+#define LDMT1R_MIFTYP_RGB24	(0xb << 0)
+#define LDMT1R_MIFTYP_YCBCR	(0xf << 0)
+#define LDMT1R_MIFTYP_SYS8A	(0x0 << 0)
+#define LDMT1R_MIFTYP_SYS8B	(0x1 << 0)
+#define LDMT1R_MIFTYP_SYS8C	(0x2 << 0)
+#define LDMT1R_MIFTYP_SYS8D	(0x3 << 0)
+#define LDMT1R_MIFTYP_SYS9	(0x4 << 0)
+#define LDMT1R_MIFTYP_SYS12	(0x5 << 0)
+#define LDMT1R_MIFTYP_SYS16A	(0x7 << 0)
+#define LDMT1R_MIFTYP_SYS16B	(0x8 << 0)
+#define LDMT1R_MIFTYP_SYS16C	(0x9 << 0)
+#define LDMT1R_MIFTYP_SYS18	(0xa << 0)
+#define LDMT1R_MIFTYP_SYS24	(0xb << 0)
+#define LDMT1R_MIFTYP_MASK	(0xf << 0)
+#define LDMT2R			0x41c
+#define LDMT2R_CSUP_MASK	(7 << 26)
+#define LDMT2R_CSUP_SHIFT	26
+#define LDMT2R_RSV		(1 << 25)
+#define LDMT2R_VSEL		(1 << 24)
+#define LDMT2R_WCSC_MASK	(0xff << 16)
+#define LDMT2R_WCSC_SHIFT	16
+#define LDMT2R_WCEC_MASK	(0xff << 8)
+#define LDMT2R_WCEC_SHIFT	8
+#define LDMT2R_WCLW_MASK	(0xff << 0)
+#define LDMT2R_WCLW_SHIFT	0
+#define LDMT3R			0x420
+#define LDMT3R_RDLC_MASK	(0x3f << 24)
+#define LDMT3R_RDLC_SHIFT	24
+#define LDMT3R_RCSC_MASK	(0xff << 16)
+#define LDMT3R_RCSC_SHIFT	16
+#define LDMT3R_RCEC_MASK	(0xff << 8)
+#define LDMT3R_RCEC_SHIFT	8
+#define LDMT3R_RCLW_MASK	(0xff << 0)
+#define LDMT3R_RCLW_SHIFT	0
+#define LDDFR			0x424
+#define LDDFR_CF1		(1 << 18)
+#define LDDFR_CF0		(1 << 17)
+#define LDDFR_CC		(1 << 16)
+#define LDDFR_YF_420		(0 << 8)
+#define LDDFR_YF_422		(1 << 8)
+#define LDDFR_YF_444		(2 << 8)
+#define LDDFR_YF_MASK		(3 << 8)
+#define LDDFR_PKF_ARGB32	(0x00 << 0)
+#define LDDFR_PKF_RGB16		(0x03 << 0)
+#define LDDFR_PKF_RGB24		(0x0b << 0)
+#define LDDFR_PKF_MASK		(0x1f << 0)
+#define LDSM1R			0x428
+#define LDSM1R_OS		(1 << 0)
+#define LDSM2R			0x42c
+#define LDSM2R_OSTRG		(1 << 0)
+#define LDSA1R			0x430
+#define LDSA2R			0x434
+#define LDMLSR			0x438
+#define LDWBFR			0x43c
+#define LDWBCNTR		0x440
+#define LDWBAR			0x444
+#define LDHCNR			0x448
+#define LDHSYNR			0x44c
+#define LDVLNR			0x450
+#define LDVSYNR			0x454
+#define LDHPDR			0x458
+#define LDVPDR			0x45c
+#define LDPMR			0x460
+#define LDPMR_LPS		(3 << 0)
+#define LDINTR			0x468
+#define LDINTR_FE		(1 << 10)
+#define LDINTR_VSE		(1 << 9)
+#define LDINTR_VEE		(1 << 8)
+#define LDINTR_FS		(1 << 2)
+#define LDINTR_VSS		(1 << 1)
+#define LDINTR_VES		(1 << 0)
+#define LDINTR_STATUS_MASK	(0xff << 0)
+#define LDSR			0x46c
+#define LDSR_MSS		(1 << 10)
+#define LDSR_MRS		(1 << 8)
+#define LDSR_AS			(1 << 1)
+#define LDCNT1R			0x470
+#define LDCNT1R_DE		(1 << 0)
+#define LDCNT2R			0x474
+#define LDCNT2R_BR		(1 << 8)
+#define LDCNT2R_MD		(1 << 3)
+#define LDCNT2R_SE		(1 << 2)
+#define LDCNT2R_ME		(1 << 1)
+#define LDCNT2R_DO		(1 << 0)
+#define LDRCNTR			0x478
+#define LDRCNTR_SRS		(1 << 17)
+#define LDRCNTR_SRC		(1 << 16)
+#define LDRCNTR_MRS		(1 << 1)
+#define LDRCNTR_MRC		(1 << 0)
+#define LDDDSR			0x47c
+#define LDDDSR_LS		(1 << 2)
+#define LDDDSR_WS		(1 << 1)
+#define LDDDSR_BS		(1 << 0)
+#define LDHAJR			0x4a0
+
+#define LDDWD0R			0x800
+#define LDDWDxR_WDACT		(1 << 28)
+#define LDDWDxR_RSW		(1 << 24)
+#define LDDRDR			0x840
+#define LDDRDR_RSR		(1 << 24)
+#define LDDRDR_DRD_MASK		(0x3ffff << 0)
+#define LDDWAR			0x900
+#define LDDWAR_WA		(1 << 0)
+#define LDDRAR			0x904
+#define LDDRAR_RA		(1 << 0)
+
+#define LDBCR			0xb00
+#define LDBCR_UPC(n)		(1 << ((n) + 16))
+#define LDBCR_UPF(n)		(1 << ((n) + 8))
+#define LDBCR_UPD(n)		(1 << ((n) + 0))
+#define LDBnBSIFR(n)		(0xb20 + (n) * 0x20 + 0x00)
+#define LDBBSIFR_EN		(1 << 31)
+#define LDBBSIFR_VS		(1 << 29)
+#define LDBBSIFR_BRSEL		(1 << 28)
+#define LDBBSIFR_MX		(1 << 27)
+#define LDBBSIFR_MY		(1 << 26)
+#define LDBBSIFR_CV3		(3 << 24)
+#define LDBBSIFR_CV2		(2 << 24)
+#define LDBBSIFR_CV1		(1 << 24)
+#define LDBBSIFR_CV0		(0 << 24)
+#define LDBBSIFR_CV_MASK	(3 << 24)
+#define LDBBSIFR_LAY_MASK	(0xff << 16)
+#define LDBBSIFR_LAY_SHIFT	16
+#define LDBBSIFR_ROP3_MASK	(0xff << 16)
+#define LDBBSIFR_ROP3_SHIFT	16
+#define LDBBSIFR_AL_PL8		(3 << 14)
+#define LDBBSIFR_AL_PL1		(2 << 14)
+#define LDBBSIFR_AL_PK		(1 << 14)
+#define LDBBSIFR_AL_1		(0 << 14)
+#define LDBBSIFR_AL_MASK	(3 << 14)
+#define LDBBSIFR_SWPL		(1 << 10)
+#define LDBBSIFR_SWPW		(1 << 9)
+#define LDBBSIFR_SWPB		(1 << 8)
+#define LDBBSIFR_RY		(1 << 7)
+#define LDBBSIFR_CHRR_420	(2 << 0)
+#define LDBBSIFR_CHRR_422	(1 << 0)
+#define LDBBSIFR_CHRR_444	(0 << 0)
+#define LDBBSIFR_RPKF_ARGB32	(0x00 << 0)
+#define LDBBSIFR_RPKF_RGB16	(0x03 << 0)
+#define LDBBSIFR_RPKF_RGB24	(0x0b << 0)
+#define LDBBSIFR_RPKF_MASK	(0x1f << 0)
+#define LDBnBSSZR(n)		(0xb20 + (n) * 0x20 + 0x04)
+#define LDBBSSZR_BVSS_MASK	(0xfff << 16)
+#define LDBBSSZR_BVSS_SHIFT	16
+#define LDBBSSZR_BHSS_MASK	(0xfff << 0)
+#define LDBBSSZR_BHSS_SHIFT	0
+#define LDBnBLOCR(n)		(0xb20 + (n) * 0x20 + 0x08)
+#define LDBBLOCR_CVLC_MASK	(0xfff << 16)
+#define LDBBLOCR_CVLC_SHIFT	16
+#define LDBBLOCR_CHLC_MASK	(0xfff << 0)
+#define LDBBLOCR_CHLC_SHIFT	0
+#define LDBnBSMWR(n)		(0xb20 + (n) * 0x20 + 0x0c)
+#define LDBBSMWR_BSMWA_MASK	(0xffff << 16)
+#define LDBBSMWR_BSMWA_SHIFT	16
+#define LDBBSMWR_BSMW_MASK	(0xffff << 0)
+#define LDBBSMWR_BSMW_SHIFT	0
+#define LDBnBSAYR(n)		(0xb20 + (n) * 0x20 + 0x10)
+#define LDBBSAYR_FG1A_MASK	(0xff << 24)
+#define LDBBSAYR_FG1A_SHIFT	24
+#define LDBBSAYR_FG1R_MASK	(0xff << 16)
+#define LDBBSAYR_FG1R_SHIFT	16
+#define LDBBSAYR_FG1G_MASK	(0xff << 8)
+#define LDBBSAYR_FG1G_SHIFT	8
+#define LDBBSAYR_FG1B_MASK	(0xff << 0)
+#define LDBBSAYR_FG1B_SHIFT	0
+#define LDBnBSACR(n)		(0xb20 + (n) * 0x20 + 0x14)
+#define LDBBSACR_FG2A_MASK	(0xff << 24)
+#define LDBBSACR_FG2A_SHIFT	24
+#define LDBBSACR_FG2R_MASK	(0xff << 16)
+#define LDBBSACR_FG2R_SHIFT	16
+#define LDBBSACR_FG2G_MASK	(0xff << 8)
+#define LDBBSACR_FG2G_SHIFT	8
+#define LDBBSACR_FG2B_MASK	(0xff << 0)
+#define LDBBSACR_FG2B_SHIFT	0
+#define LDBnBSAAR(n)		(0xb20 + (n) * 0x20 + 0x18)
+#define LDBBSAAR_AP_MASK	(0xff << 24)
+#define LDBBSAAR_AP_SHIFT	24
+#define LDBBSAAR_R_MASK		(0xff << 16)
+#define LDBBSAAR_R_SHIFT	16
+#define LDBBSAAR_GY_MASK	(0xff << 8)
+#define LDBBSAAR_GY_SHIFT	8
+#define LDBBSAAR_B_MASK		(0xff << 0)
+#define LDBBSAAR_B_SHIFT	0
+#define LDBnBPPCR(n)		(0xb20 + (n) * 0x20 + 0x1c)
+#define LDBBPPCR_AP_MASK	(0xff << 24)
+#define LDBBPPCR_AP_SHIFT	24
+#define LDBBPPCR_R_MASK		(0xff << 16)
+#define LDBBPPCR_R_SHIFT	16
+#define LDBBPPCR_GY_MASK	(0xff << 8)
+#define LDBBPPCR_GY_SHIFT	8
+#define LDBBPPCR_B_MASK		(0xff << 0)
+#define LDBBPPCR_B_SHIFT	0
+#define LDBnBBGCL(n)		(0xb10 + (n) * 0x04)
+#define LDBBBGCL_BGA_MASK	(0xff << 24)
+#define LDBBBGCL_BGA_SHIFT	24
+#define LDBBBGCL_BGR_MASK	(0xff << 16)
+#define LDBBBGCL_BGR_SHIFT	16
+#define LDBBBGCL_BGG_MASK	(0xff << 8)
+#define LDBBBGCL_BGG_SHIFT	8
+#define LDBBBGCL_BGB_MASK	(0xff << 0)
+#define LDBBBGCL_BGB_SHIFT	0
+
+#define LCDC_SIDE_B_OFFSET	0x1000
+#define LCDC_MIRROR_OFFSET	0x2000
+
+static inline bool lcdc_is_banked(u32 reg)
+{
+	switch (reg) {
+	case LDMT1R:
+	case LDMT2R:
+	case LDMT3R:
+	case LDDFR:
+	case LDSM1R:
+	case LDSA1R:
+	case LDSA2R:
+	case LDMLSR:
+	case LDWBFR:
+	case LDWBCNTR:
+	case LDWBAR:
+	case LDHCNR:
+	case LDHSYNR:
+	case LDVLNR:
+	case LDVSYNR:
+	case LDHPDR:
+	case LDVPDR:
+	case LDHAJR:
+		return true;
+	default:
+		return reg >= LDBnBBGCL(0) && reg <= LDBnBPPCR(3);
+	}
+}
+
+static inline void lcdc_write_mirror(struct shmob_drm_device *sdev, u32 reg,
+				     u32 data)
+{
+	iowrite32(data, sdev->mmio + reg + LCDC_MIRROR_OFFSET);
+}
+
+static inline void lcdc_write(struct shmob_drm_device *sdev, u32 reg, u32 data)
+{
+	iowrite32(data, sdev->mmio + reg);
+	if (lcdc_is_banked(reg))
+		iowrite32(data, sdev->mmio + reg + LCDC_SIDE_B_OFFSET);
+}
+
+static inline u32 lcdc_read(struct shmob_drm_device *sdev, u32 reg)
+{
+	return ioread32(sdev->mmio + reg);
+}
+
+static inline int lcdc_wait_bit(struct shmob_drm_device *sdev, u32 reg,
+				u32 mask, u32 until)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(5);
+
+	while ((lcdc_read(sdev, reg) & mask) != until) {
+		if (time_after(jiffies, timeout))
+			return -ETIMEDOUT;
+		cpu_relax();
+	}
+
+	return 0;
+}
+
+#endif /* __SHMOB_DRM_REGS_H__ */
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index 867dc03000e..841065b998a 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -27,11 +27,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "sis_drm.h"
+#include <drm/drmP.h>
+#include <drm/sis_drm.h>
 #include "sis_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static struct pci_device_id pciidlist[] = {
 	sisdrv_PCI_IDS
diff --git a/drivers/gpu/drm/sis/sis_drv.h b/drivers/gpu/drm/sis/sis_drv.h
index 573758b2d2d..13b527bb83b 100644
--- a/drivers/gpu/drm/sis/sis_drv.h
+++ b/drivers/gpu/drm/sis/sis_drv.h
@@ -44,7 +44,7 @@ enum sis_family {
 	SIS_CHIP_315 = 1,
 };
 
-#include "drm_mm.h"
+#include <drm/drm_mm.h>
 
 
 #define SIS_BASE (dev_priv->mmio)
diff --git a/drivers/gpu/drm/sis/sis_mm.c b/drivers/gpu/drm/sis/sis_mm.c
index 2c231070d25..2b2f78c428a 100644
--- a/drivers/gpu/drm/sis/sis_mm.c
+++ b/drivers/gpu/drm/sis/sis_mm.c
@@ -31,8 +31,8 @@
  *    Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  */
 
-#include "drmP.h"
-#include "sis_drm.h"
+#include <drm/drmP.h>
+#include <drm/sis_drm.h>
 #include "sis_drv.h"
 
 #include <video/sisfb.h>
diff --git a/drivers/gpu/drm/tdfx/tdfx_drv.c b/drivers/gpu/drm/tdfx/tdfx_drv.c
index a7f4d6bd133..ddfa743459d 100644
--- a/drivers/gpu/drm/tdfx/tdfx_drv.c
+++ b/drivers/gpu/drm/tdfx/tdfx_drv.c
@@ -32,10 +32,10 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "tdfx_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static struct pci_device_id pciidlist[] = {
 	tdfx_PCI_IDS
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index 4a872829136..3302f99e749 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -31,11 +31,11 @@
 
 #define pr_fmt(fmt) "[TTM] " fmt
 
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_page_alloc.h>
 #ifdef TTM_HAS_AGP
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_placement.h>
 #include <linux/agp_backend.h>
 #include <linux/module.h>
 #include <linux/slab.h>
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 36f4b28c1b9..402ab69f9f9 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -30,9 +30,9 @@
 
 #define pr_fmt(fmt) "[TTM] " fmt
 
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
 #include <linux/jiffies.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
index 038e947d00f..9212494e907 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -28,10 +28,10 @@
  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  */
 
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "drm_mm.h"
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/drm_mm.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/module.h>
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index f8187ead7b3..2026060f03e 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -28,8 +28,8 @@
  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  */
 
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
 #include <linux/io.h>
 #include <linux/highmem.h>
 #include <linux/wait.h>
@@ -472,7 +472,7 @@ pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 	else
 		tmp = pgprot_noncached(tmp);
 #endif
-#if defined(__sparc__)
+#if defined(__sparc__) || defined(__mips__)
 	if (!(caching_flags & TTM_PL_FLAG_CACHED))
 		tmp = pgprot_noncached(tmp);
 #endif
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 3832fe10b4d..1937069432c 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -25,9 +25,9 @@
  *
  **************************************************************************/
 
-#include "ttm/ttm_execbuf_util.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_execbuf_util.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
 #include <linux/wait.h>
 #include <linux/sched.h>
 #include <linux/module.h>
diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c
index 075daf44bce..3daa9a3930b 100644
--- a/drivers/gpu/drm/ttm/ttm_lock.c
+++ b/drivers/gpu/drm/ttm/ttm_lock.c
@@ -28,8 +28,8 @@
  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  */
 
-#include "ttm/ttm_lock.h"
-#include "ttm/ttm_module.h"
+#include <drm/ttm/ttm_lock.h>
+#include <drm/ttm/ttm_module.h>
 #include <linux/atomic.h>
 #include <linux/errno.h>
 #include <linux/wait.h>
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index 23d2ecbaed5..479c6b0467c 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -27,9 +27,9 @@
 
 #define pr_fmt(fmt) "[TTM] " fmt
 
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/ttm/ttm_memory.h>
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_page_alloc.h>
 #include <linux/spinlock.h>
 #include <linux/sched.h>
 #include <linux/wait.h>
diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c
index 902d7cf9fb4..d7f92fe9d90 100644
--- a/drivers/gpu/drm/ttm/ttm_module.c
+++ b/drivers/gpu/drm/ttm/ttm_module.c
@@ -31,8 +31,8 @@
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/sched.h>
-#include "ttm/ttm_module.h"
-#include "drm_sysfs.h"
+#include <drm/ttm/ttm_module.h>
+#include <drm/drm_sysfs.h>
 
 static DECLARE_WAIT_QUEUE_HEAD(exit_q);
 atomic_t device_released;
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 68daca412cb..c7857874956 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -51,8 +51,8 @@
 
 #define pr_fmt(fmt) "[TTM] " fmt
 
-#include "ttm/ttm_object.h"
-#include "ttm/ttm_module.h"
+#include <drm/ttm/ttm_object.h>
+#include <drm/ttm/ttm_module.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
 #include <linux/slab.h>
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index ebc6fac96e3..860dc4813e9 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -45,8 +45,8 @@
 
 #include <linux/atomic.h>
 
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_page_alloc.h>
 
 #ifdef TTM_HAS_AGP
 #include <asm/agp.h>
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index 4f9e548b2ee..b8b394319b4 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -47,8 +47,8 @@
 #include <linux/atomic.h>
 #include <linux/device.h>
 #include <linux/kthread.h>
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_page_alloc.h>
 #ifdef TTM_HAS_AGP
 #include <asm/agp.h>
 #endif
@@ -1060,7 +1060,7 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 
 	_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
 	if (!_manager)
-		goto err_manager;
+		goto err;
 
 	mutex_init(&_manager->lock);
 	INIT_LIST_HEAD(&_manager->pools);
@@ -1078,9 +1078,6 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 	}
 	ttm_dma_pool_mm_shrink_init(_manager);
 	return 0;
-err_manager:
-	kfree(_manager);
-	_manager = NULL;
 err:
 	return ret;
 }
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index fa09daf9a50..bf8260133ea 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -38,12 +38,12 @@
 #include <linux/swap.h>
 #include <linux/slab.h>
 #include <linux/export.h>
-#include "drm_cache.h"
-#include "drm_mem_util.h"
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/drm_cache.h>
+#include <drm/drm_mem_util.h>
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_page_alloc.h>
 
 /**
  * Allocates storage for pointers to the pages that back the ttm.
@@ -290,8 +290,6 @@ int ttm_tt_swapin(struct ttm_tt *ttm)
 	struct file *swap_storage;
 	struct page *from_page;
 	struct page *to_page;
-	void *from_virtual;
-	void *to_virtual;
 	int i;
 	int ret = -ENOMEM;
 
@@ -311,11 +309,7 @@ int ttm_tt_swapin(struct ttm_tt *ttm)
 			goto out_err;
 
 		preempt_disable();
-		from_virtual = kmap_atomic(from_page);
-		to_virtual = kmap_atomic(to_page);
-		memcpy(to_virtual, from_virtual, PAGE_SIZE);
-		kunmap_atomic(to_virtual);
-		kunmap_atomic(from_virtual);
+		copy_highpage(to_page, from_page);
 		preempt_enable();
 		page_cache_release(from_page);
 	}
@@ -336,8 +330,6 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
 	struct file *swap_storage;
 	struct page *from_page;
 	struct page *to_page;
-	void *from_virtual;
-	void *to_virtual;
 	int i;
 	int ret = -ENOMEM;
 
@@ -367,11 +359,7 @@ int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
 			goto out_err;
 		}
 		preempt_disable();
-		from_virtual = kmap_atomic(from_page);
-		to_virtual = kmap_atomic(to_page);
-		memcpy(to_virtual, from_virtual, PAGE_SIZE);
-		kunmap_atomic(to_virtual);
-		kunmap_atomic(from_virtual);
+		copy_highpage(to_page, from_page);
 		preempt_enable();
 		set_page_dirty(to_page);
 		mark_page_accessed(to_page);
diff --git a/drivers/gpu/drm/udl/udl_connector.c b/drivers/gpu/drm/udl/udl_connector.c
index 8d9dc44f1f9..b3b2cedf674 100644
--- a/drivers/gpu/drm/udl/udl_connector.c
+++ b/drivers/gpu/drm/udl/udl_connector.c
@@ -10,10 +10,10 @@
  * more details.
  */
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_edid.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
 /* dummy connector to just get EDID,
@@ -57,11 +57,8 @@ static int udl_get_modes(struct drm_connector *connector)
 
 	edid = (struct edid *)udl_get_edid(udl);
 
-	connector->display_info.raw_edid = (char *)edid;
-
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
-	connector->display_info.raw_edid = NULL;
 	kfree(edid);
 	return ret;
 }
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index 9f84128505b..c0770dbba74 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -7,8 +7,8 @@
  */
 
 #include <linux/module.h>
-#include "drm_usb.h"
-#include "drm_crtc_helper.h"
+#include <drm/drm_usb.h>
+#include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
 static struct drm_driver driver;
diff --git a/drivers/gpu/drm/udl/udl_encoder.c b/drivers/gpu/drm/udl/udl_encoder.c
index 0731ab2e6c0..4052c465649 100644
--- a/drivers/gpu/drm/udl/udl_encoder.c
+++ b/drivers/gpu/drm/udl/udl_encoder.c
@@ -10,13 +10,13 @@
  * more details.
  */
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
 /* dummy encoder */
-void udl_enc_destroy(struct drm_encoder *encoder)
+static void udl_enc_destroy(struct drm_encoder *encoder)
 {
 	drm_encoder_cleanup(encoder);
 	kfree(encoder);
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index ce9a6117992..67df842fbb3 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -13,14 +13,14 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/fb.h>
+#include <linux/dma-buf.h>
 
-#include "drmP.h"
-#include "drm.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
-#include "drm_fb_helper.h"
+#include <drm/drm_fb_helper.h>
 
 #define DL_DEFIO_WRITE_DELAY    5 /* fb_deferred_io.delay in jiffies */
 
@@ -356,12 +356,12 @@ static struct fb_ops udlfb_ops = {
 	.fb_release = udl_fb_release,
 };
 
-void udl_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
+static void udl_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
 			   u16 blue, int regno)
 {
 }
 
-void udl_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
+static void udl_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
 			     u16 *blue, int regno)
 {
 	*red = 0;
@@ -377,16 +377,33 @@ static int udl_user_framebuffer_dirty(struct drm_framebuffer *fb,
 {
 	struct udl_framebuffer *ufb = to_udl_fb(fb);
 	int i;
+	int ret = 0;
 
 	if (!ufb->active_16)
 		return 0;
 
+	if (ufb->obj->base.import_attach) {
+		ret = dma_buf_begin_cpu_access(ufb->obj->base.import_attach->dmabuf,
+					       0, ufb->obj->base.size,
+					       DMA_FROM_DEVICE);
+		if (ret)
+			return ret;
+	}
+
 	for (i = 0; i < num_clips; i++) {
-		udl_handle_damage(ufb, clips[i].x1, clips[i].y1,
+		ret = udl_handle_damage(ufb, clips[i].x1, clips[i].y1,
 				  clips[i].x2 - clips[i].x1,
 				  clips[i].y2 - clips[i].y1);
+		if (ret)
+			break;
 	}
-	return 0;
+
+	if (ufb->obj->base.import_attach) {
+		dma_buf_end_cpu_access(ufb->obj->base.import_attach->dmabuf,
+				       0, ufb->obj->base.size,
+				       DMA_FROM_DEVICE);
+	}
+	return ret;
 }
 
 static void udl_user_framebuffer_destroy(struct drm_framebuffer *fb)
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index 291ecc14558..afd212c9921 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -6,7 +6,7 @@
  * more details.
  */
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "udl_drv.h"
 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
@@ -181,11 +181,6 @@ int udl_gem_vmap(struct udl_gem_object *obj)
 	int ret;
 
 	if (obj->base.import_attach) {
-		ret = dma_buf_begin_cpu_access(obj->base.import_attach->dmabuf,
-					       0, obj->base.size, DMA_BIDIRECTIONAL);
-		if (ret)
-			return -EINVAL;
-
 		obj->vmapping = dma_buf_vmap(obj->base.import_attach->dmabuf);
 		if (!obj->vmapping)
 			return -ENOMEM;
@@ -206,8 +201,6 @@ void udl_gem_vunmap(struct udl_gem_object *obj)
 {
 	if (obj->base.import_attach) {
 		dma_buf_vunmap(obj->base.import_attach->dmabuf, obj->vmapping);
-		dma_buf_end_cpu_access(obj->base.import_attach->dmabuf, 0,
-				       obj->base.size, DMA_BIDIRECTIONAL);
 		return;
 	}
 
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index 4c2d836a089..0ce2d719525 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -10,7 +10,7 @@
  * License v2. See the file COPYING in the main directory of this archive for
  * more details.
  */
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "udl_drv.h"
 
 /* -BULK_SIZE as per usb-skeleton. Can we get full page and avoid overhead? */
@@ -41,11 +41,8 @@ static int udl_parse_vendor_descriptor(struct drm_device *dev,
 	total_len = usb_get_descriptor(usbdev, 0x5f, /* vendor specific */
 				    0, desc, MAX_VENDOR_DESCRIPTOR_SIZE);
 	if (total_len > 5) {
-		DRM_INFO("vendor descriptor length:%x data:%02x %02x %02x %02x" \
-			"%02x %02x %02x %02x %02x %02x %02x\n",
-			total_len, desc[0],
-			desc[1], desc[2], desc[3], desc[4], desc[5], desc[6],
-			desc[7], desc[8], desc[9], desc[10]);
+		DRM_INFO("vendor descriptor length:%x data:%*ph\n",
+			total_len, 11, desc);
 
 		if ((desc[0] != total_len) || /* descriptor length */
 		    (desc[1] != 0x5f) ||   /* vendor descriptor type */
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 9159d48d1df..e96d2349bd5 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -11,9 +11,9 @@
  * more details.
  */
 
-#include "drmP.h"
-#include "drm_crtc.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
 #include "udl_drv.h"
 
 /*
@@ -391,7 +391,7 @@ static const struct drm_crtc_funcs udl_crtc_funcs = {
 	.destroy = udl_crtc_destroy,
 };
 
-int udl_crtc_init(struct drm_device *dev)
+static int udl_crtc_init(struct drm_device *dev)
 {
 	struct drm_crtc *crtc;
 
diff --git a/drivers/gpu/drm/udl/udl_transfer.c b/drivers/gpu/drm/udl/udl_transfer.c
index b9320e2608d..dc095526ffb 100644
--- a/drivers/gpu/drm/udl/udl_transfer.c
+++ b/drivers/gpu/drm/udl/udl_transfer.c
@@ -15,7 +15,7 @@
 #include <linux/fb.h>
 #include <linux/prefetch.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "udl_drv.h"
 
 #define MAX_CMD_PIXELS		255
@@ -126,10 +126,10 @@ static void udl_compress_hline16(
 
 	while ((pixel_end > pixel) &&
 	       (cmd_buffer_end - MIN_RLX_CMD_BYTES > cmd)) {
-		uint8_t *raw_pixels_count_byte = 0;
-		uint8_t *cmd_pixels_count_byte = 0;
-		const u8 *raw_pixel_start = 0;
-		const u8 *cmd_pixel_start, *cmd_pixel_end = 0;
+		uint8_t *raw_pixels_count_byte = NULL;
+		uint8_t *cmd_pixels_count_byte = NULL;
+		const u8 *raw_pixel_start = NULL;
+		const u8 *cmd_pixel_start, *cmd_pixel_end = NULL;
 
 		prefetchw((void *) cmd); /* pull in one cache line at least */
 
diff --git a/drivers/gpu/drm/via/via_dma.c b/drivers/gpu/drm/via/via_dma.c
index cc0ffa9abd0..13558f5a242 100644
--- a/drivers/gpu/drm/via/via_dma.c
+++ b/drivers/gpu/drm/via/via_dma.c
@@ -34,9 +34,8 @@
  *    Thomas Hellstrom.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 #include "via_3d_reg.h"
 
diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c
index 3e038a394c5..8b0f25904e6 100644
--- a/drivers/gpu/drm/via/via_dmablit.c
+++ b/drivers/gpu/drm/via/via_dmablit.c
@@ -34,8 +34,8 @@
  * the same DMA mappings?
  */
 
-#include "drmP.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 #include "via_dmablit.h"
 
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
index af1b914b17e..f4ae2032794 100644
--- a/drivers/gpu/drm/via/via_drv.c
+++ b/drivers/gpu/drm/via/via_drv.c
@@ -24,11 +24,11 @@
 
 #include <linux/module.h>
 
-#include "drmP.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 
-#include "drm_pciids.h"
+#include <drm/drm_pciids.h>
 
 static int via_driver_open(struct drm_device *dev, struct drm_file *file)
 {
diff --git a/drivers/gpu/drm/via/via_drv.h b/drivers/gpu/drm/via/via_drv.h
index 88edacc9300..893a65090c3 100644
--- a/drivers/gpu/drm/via/via_drv.h
+++ b/drivers/gpu/drm/via/via_drv.h
@@ -24,7 +24,7 @@
 #ifndef _VIA_DRV_H_
 #define _VIA_DRV_H_
 
-#include "drm_mm.h"
+#include <drm/drm_mm.h>
 #define DRIVER_AUTHOR	"Various"
 
 #define DRIVER_NAME		"via"
diff --git a/drivers/gpu/drm/via/via_irq.c b/drivers/gpu/drm/via/via_irq.c
index d391f48ef87..ac98964297c 100644
--- a/drivers/gpu/drm/via/via_irq.c
+++ b/drivers/gpu/drm/via/via_irq.c
@@ -35,9 +35,8 @@
  * The refresh rate is also calculated for video playback sync purposes.
  */
 
-#include "drmP.h"
-#include "drm.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 
 #define VIA_REG_INTERRUPT       0x200
diff --git a/drivers/gpu/drm/via/via_map.c b/drivers/gpu/drm/via/via_map.c
index c126182ac07..c0f1cc7f5ca 100644
--- a/drivers/gpu/drm/via/via_map.c
+++ b/drivers/gpu/drm/via/via_map.c
@@ -21,8 +21,8 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  */
-#include "drmP.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 
 static int via_do_init_map(struct drm_device *dev, drm_via_init_t *init)
diff --git a/drivers/gpu/drm/via/via_mm.c b/drivers/gpu/drm/via/via_mm.c
index acfcb358e7b..0d55432e02a 100644
--- a/drivers/gpu/drm/via/via_mm.c
+++ b/drivers/gpu/drm/via/via_mm.c
@@ -25,8 +25,8 @@
  * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  */
 
-#include "drmP.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 
 #define VIA_MM_ALIGN_SHIFT 4
diff --git a/drivers/gpu/drm/via/via_verifier.c b/drivers/gpu/drm/via/via_verifier.c
index 48957b856d4..9dbc92bd151 100644
--- a/drivers/gpu/drm/via/via_verifier.c
+++ b/drivers/gpu/drm/via/via_verifier.c
@@ -29,9 +29,8 @@
  */
 
 #include "via_3d_reg.h"
-#include "drmP.h"
-#include "drm.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_verifier.h"
 #include "via_drv.h"
 
diff --git a/drivers/gpu/drm/via/via_video.c b/drivers/gpu/drm/via/via_video.c
index 675d311f038..6569efa2ff6 100644
--- a/drivers/gpu/drm/via/via_video.c
+++ b/drivers/gpu/drm/via/via_video.c
@@ -25,8 +25,8 @@
  * Video and XvMC related functions.
  */
 
-#include "drmP.h"
-#include "via_drm.h"
+#include <drm/drmP.h>
+#include <drm/via_drm.h>
 #include "via_drv.h"
 
 void via_init_futex(drm_via_private_t *dev_priv)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 1e2c0fb7f78..9826fbc8815 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -26,9 +26,9 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_page_alloc.h"
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_page_alloc.h>
 
 static uint32_t vram_placement_flags = TTM_PL_FLAG_VRAM |
 	TTM_PL_FLAG_CACHED;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
index 3fa884db08a..3ce68a2e312 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
@@ -25,9 +25,9 @@
  *
  **************************************************************************/
 
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_placement.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index ba2c35dbf10..ed3c1e7ddde 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -26,12 +26,12 @@
  **************************************************************************/
 #include <linux/module.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
-#include "ttm/ttm_placement.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_object.h"
-#include "ttm/ttm_module.h"
+#include <drm/ttm/ttm_placement.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_object.h>
+#include <drm/ttm/ttm_module.h>
 
 #define VMWGFX_DRIVER_NAME "vmwgfx"
 #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices"
@@ -438,7 +438,6 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 		DRM_ERROR("Failed allocating a device private struct.\n");
 		return -ENOMEM;
 	}
-	memset(dev_priv, 0, sizeof(*dev_priv));
 
 	pci_set_master(dev->pdev);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 29c984ff7f2..88a179e26de 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -29,15 +29,15 @@
 #define _VMWGFX_DRV_H_
 
 #include "vmwgfx_reg.h"
-#include "drmP.h"
-#include "vmwgfx_drm.h"
-#include "drm_hashtab.h"
-#include "linux/suspend.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_object.h"
-#include "ttm/ttm_lock.h"
-#include "ttm/ttm_execbuf_util.h"
-#include "ttm/ttm_module.h"
+#include <drm/drmP.h>
+#include <drm/vmwgfx_drm.h>
+#include <drm/drm_hashtab.h>
+#include <linux/suspend.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_object.h>
+#include <drm/ttm/ttm_lock.h>
+#include <drm/ttm/ttm_execbuf_util.h>
+#include <drm/ttm/ttm_module.h>
 #include "vmwgfx_fence.h"
 
 #define VMWGFX_DRIVER_DATE "20120209"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 4acced44a62..30654b4cc97 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -27,8 +27,8 @@
 
 #include "vmwgfx_drv.h"
 #include "vmwgfx_reg.h"
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_bo_api.h>
+#include <drm/ttm/ttm_placement.h>
 
 static int vmw_cmd_invalid(struct vmw_private *dev_priv,
 			   struct vmw_sw_context *sw_context,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index a32f2e96dd0..ed5ce2a41bb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -28,10 +28,10 @@
 
 #include <linux/export.h>
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_placement.h>
 
 #define VMW_DIRTY_DELAY (HZ / 30)
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index 7e0743358df..bc187fafd58 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -25,7 +25,7 @@
  *
  **************************************************************************/
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
 #define VMW_FENCE_WRAP (1 << 31)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
index a0c2f12b1e1..3eb148667d6 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
@@ -26,8 +26,8 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
-#include "drmP.h"
-#include "ttm/ttm_placement.h"
+#include <drm/drmP.h>
+#include <drm/ttm/ttm_placement.h>
 
 bool vmw_fifo_have_3d(struct vmw_private *dev_priv)
 {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
index 21ee7822656..3751730764a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
@@ -26,8 +26,8 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
-#include "drmP.h"
-#include "ttm/ttm_bo_driver.h"
+#include <drm/drmP.h>
+#include <drm/ttm/ttm_bo_driver.h>
 
 #define VMW_PPN_SIZE sizeof(unsigned long)
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
index 5f717152cff..c5c054ae905 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmrid_manager.c
@@ -29,9 +29,9 @@
  */
 
 #include "vmwgfx_drv.h"
-#include "ttm/ttm_module.h"
-#include "ttm/ttm_bo_driver.h"
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_module.h>
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_placement.h>
 #include <linux/idr.h>
 #include <linux/spinlock.h>
 #include <linux/kernel.h>
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 66917c6c381..b07ca2e4d04 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -26,7 +26,7 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
-#include "vmwgfx_drm.h"
+#include <drm/vmwgfx_drm.h>
 #include "vmwgfx_kms.h"
 
 int vmw_getparam_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
index cabc95f7517..4640adbcaf9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
@@ -25,7 +25,7 @@
  *
  **************************************************************************/
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
 #define VMW_FENCE_WRAP (1 << 24)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index c50724bd30f..54743943d8b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -483,7 +483,6 @@ static int do_surface_dirty_sou(struct vmw_private *dev_priv,
 	}
 
 	/* only need to do this once */
-	memset(cmd, 0, fifo_size);
 	cmd->header.id = cpu_to_le32(SVGA_3D_CMD_BLIT_SURFACE_TO_SCREEN);
 	cmd->header.size = cpu_to_le32(fifo_size - sizeof(cmd->header));
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 8184bc5b173..6fa89c9d621 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -28,8 +28,8 @@
 #ifndef VMWGFX_KMS_H_
 #define VMWGFX_KMS_H_
 
-#include "drmP.h"
-#include "drm_crtc_helper.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
 #include "vmwgfx_drv.h"
 
 #define VMWGFX_NUM_DISPLAY_UNITS 8
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index 14399eec9c3..cb55b7b6637 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -26,10 +26,10 @@
  **************************************************************************/
 
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
-#include "ttm/ttm_placement.h"
+#include <drm/ttm/ttm_placement.h>
 
 #include "svga_overlay.h"
 #include "svga_escape.h"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 2c6ffe0e2c0..da3c6b5b98a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -26,10 +26,10 @@
  **************************************************************************/
 
 #include "vmwgfx_drv.h"
-#include "vmwgfx_drm.h"
-#include "ttm/ttm_object.h"
-#include "ttm/ttm_placement.h"
-#include "drmP.h"
+#include <drm/vmwgfx_drm.h>
+#include <drm/ttm/ttm_object.h>
+#include <drm/ttm/ttm_placement.h>
+#include <drm/drmP.h>
 
 struct vmw_user_context {
 	struct ttm_base_object base;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
index d3c11f5184f..98d6bfb3a99 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
@@ -25,7 +25,7 @@
  *
  **************************************************************************/
 
-#include "drmP.h"
+#include <drm/drmP.h>
 #include "vmwgfx_drv.h"
 
 int vmw_mmap(struct file *filp, struct vm_area_struct *vma)
diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c
index 2091ae8f539..2421d95130d 100644
--- a/drivers/i2c/i2c-core.c
+++ b/drivers/i2c/i2c-core.c
@@ -982,7 +982,7 @@ int i2c_add_numbered_adapter(struct i2c_adapter *adap)
 
 	if (adap->nr == -1) /* -1 means dynamically assign bus id */
 		return i2c_add_adapter(adap);
-	if (adap->nr & ~MAX_ID_MASK)
+	if (adap->nr & ~MAX_IDR_MASK)
 		return -EINVAL;
 
 retry:
diff --git a/drivers/ide/aec62xx.c b/drivers/ide/aec62xx.c
index 57d00caefc8..01451940393 100644
--- a/drivers/ide/aec62xx.c
+++ b/drivers/ide/aec62xx.c
@@ -181,7 +181,7 @@ static const struct ide_port_ops atp86x_port_ops = {
 	.cable_detect		= atp86x_cable_detect,
 };
 
-static const struct ide_port_info aec62xx_chipsets[] __devinitdata = {
+static const struct ide_port_info aec62xx_chipsets[] __devinitconst = {
 	{	/* 0: AEC6210 */
 		.name		= DRV_NAME,
 		.init_chipset	= init_chipset_aec62xx,
diff --git a/drivers/ide/ali14xx.c b/drivers/ide/ali14xx.c
index d3be99fb415..8f3570ee64c 100644
--- a/drivers/ide/ali14xx.c
+++ b/drivers/ide/ali14xx.c
@@ -52,13 +52,13 @@
 
 /* port addresses for auto-detection */
 #define ALI_NUM_PORTS 4
-static const int ports[ALI_NUM_PORTS] __initdata =
+static const int ports[ALI_NUM_PORTS] __initconst =
 	{ 0x074, 0x0f4, 0x034, 0x0e4 };
 
 /* register initialization data */
 typedef struct { u8 reg, data; } RegInitializer;
 
-static const RegInitializer initData[] __initdata = {
+static const RegInitializer initData[] __initconst = {
 	{0x01, 0x0f}, {0x02, 0x00}, {0x03, 0x00}, {0x04, 0x00},
 	{0x05, 0x00}, {0x06, 0x00}, {0x07, 0x2b}, {0x0a, 0x0f},
 	{0x25, 0x00}, {0x26, 0x00}, {0x27, 0x00}, {0x28, 0x00},
diff --git a/drivers/ide/alim15x3.c b/drivers/ide/alim15x3.c
index 2c8016ad0e2..911a27ca356 100644
--- a/drivers/ide/alim15x3.c
+++ b/drivers/ide/alim15x3.c
@@ -512,7 +512,7 @@ static const struct ide_dma_ops ali_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info ali15x3_chipset __devinitdata = {
+static const struct ide_port_info ali15x3_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_ali15x3,
 	.init_hwif	= init_hwif_ali15x3,
diff --git a/drivers/ide/amd74xx.c b/drivers/ide/amd74xx.c
index 3747b2561f0..56fc99557ba 100644
--- a/drivers/ide/amd74xx.c
+++ b/drivers/ide/amd74xx.c
@@ -223,7 +223,7 @@ static const struct ide_port_ops amd_port_ops = {
 		.udma_mask	= udma,					\
 	}
 
-static const struct ide_port_info amd74xx_chipsets[] __devinitdata = {
+static const struct ide_port_info amd74xx_chipsets[] __devinitconst = {
 	/* 0: AMD7401 */	DECLARE_AMD_DEV(0x00, ATA_UDMA2),
 	/* 1: AMD7409 */	DECLARE_AMD_DEV(ATA_SWDMA2, ATA_UDMA4),
 	/* 2: AMD7411/7441 */	DECLARE_AMD_DEV(ATA_SWDMA2, ATA_UDMA5),
diff --git a/drivers/ide/atiixp.c b/drivers/ide/atiixp.c
index 15f0ead89f5..cb43480b1bd 100644
--- a/drivers/ide/atiixp.c
+++ b/drivers/ide/atiixp.c
@@ -139,7 +139,7 @@ static const struct ide_port_ops atiixp_port_ops = {
 	.cable_detect		= atiixp_cable_detect,
 };
 
-static const struct ide_port_info atiixp_pci_info[] __devinitdata = {
+static const struct ide_port_info atiixp_pci_info[] __devinitconst = {
 	{	/* 0: IXP200/300/400/700 */
 		.name		= DRV_NAME,
 		.enablebits	= {{0x48,0x01,0x00}, {0x48,0x08,0x00}},
diff --git a/drivers/ide/cmd640.c b/drivers/ide/cmd640.c
index 14717304b38..70f0a2754c1 100644
--- a/drivers/ide/cmd640.c
+++ b/drivers/ide/cmd640.c
@@ -685,7 +685,7 @@ static int pci_conf2(void)
 	return 0;
 }
 
-static const struct ide_port_info cmd640_port_info __initdata = {
+static const struct ide_port_info cmd640_port_info __initconst = {
 	.chipset		= ide_cmd640,
 	.host_flags		= IDE_HFLAG_SERIALIZE |
 				  IDE_HFLAG_NO_DMA |
diff --git a/drivers/ide/cmd64x.c b/drivers/ide/cmd64x.c
index 5f80312e636..d1fc43802f5 100644
--- a/drivers/ide/cmd64x.c
+++ b/drivers/ide/cmd64x.c
@@ -327,7 +327,7 @@ static const struct ide_dma_ops cmd646_rev1_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info cmd64x_chipsets[] __devinitdata = {
+static const struct ide_port_info cmd64x_chipsets[] __devinitconst = {
 	{	/* 0: CMD643 */
 		.name		= DRV_NAME,
 		.init_chipset	= init_chipset_cmd64x,
diff --git a/drivers/ide/cs5520.c b/drivers/ide/cs5520.c
index 2c1e5f7cd26..14447621e60 100644
--- a/drivers/ide/cs5520.c
+++ b/drivers/ide/cs5520.c
@@ -94,7 +94,7 @@ static const struct ide_port_ops cs5520_port_ops = {
 	.set_dma_mode		= cs5520_set_dma_mode,
 };
 
-static const struct ide_port_info cyrix_chipset __devinitdata = {
+static const struct ide_port_info cyrix_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= { { 0x60, 0x01, 0x01 }, { 0x60, 0x02, 0x02 } },
 	.port_ops	= &cs5520_port_ops,
diff --git a/drivers/ide/cs5530.c b/drivers/ide/cs5530.c
index 4dc4eb92b07..49b40ad59d1 100644
--- a/drivers/ide/cs5530.c
+++ b/drivers/ide/cs5530.c
@@ -245,7 +245,7 @@ static const struct ide_port_ops cs5530_port_ops = {
 	.udma_filter		= cs5530_udma_filter,
 };
 
-static const struct ide_port_info cs5530_chipset __devinitdata = {
+static const struct ide_port_info cs5530_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_cs5530,
 	.init_hwif	= init_hwif_cs5530,
diff --git a/drivers/ide/cs5535.c b/drivers/ide/cs5535.c
index 5059fafadf2..18d4c852602 100644
--- a/drivers/ide/cs5535.c
+++ b/drivers/ide/cs5535.c
@@ -170,7 +170,7 @@ static const struct ide_port_ops cs5535_port_ops = {
 	.cable_detect		= cs5535_cable_detect,
 };
 
-static const struct ide_port_info cs5535_chipset __devinitdata = {
+static const struct ide_port_info cs5535_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.port_ops	= &cs5535_port_ops,
 	.host_flags	= IDE_HFLAG_SINGLE | IDE_HFLAG_POST_SET_MODE,
diff --git a/drivers/ide/cy82c693.c b/drivers/ide/cy82c693.c
index 847553fd8b9..3ffb49dab57 100644
--- a/drivers/ide/cy82c693.c
+++ b/drivers/ide/cy82c693.c
@@ -163,7 +163,7 @@ static const struct ide_port_ops cy82c693_port_ops = {
 	.set_dma_mode		= cy82c693_set_dma_mode,
 };
 
-static const struct ide_port_info cy82c693_chipset __devinitdata = {
+static const struct ide_port_info cy82c693_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_iops	= init_iops_cy82c693,
 	.port_ops	= &cy82c693_port_ops,
diff --git a/drivers/ide/dtc2278.c b/drivers/ide/dtc2278.c
index 46af4743b3e..8722df329cb 100644
--- a/drivers/ide/dtc2278.c
+++ b/drivers/ide/dtc2278.c
@@ -91,7 +91,7 @@ static const struct ide_port_ops dtc2278_port_ops = {
 	.set_pio_mode		= dtc2278_set_pio_mode,
 };
 
-static const struct ide_port_info dtc2278_port_info __initdata = {
+static const struct ide_port_info dtc2278_port_info __initconst = {
 	.name			= DRV_NAME,
 	.chipset		= ide_dtc2278,
 	.port_ops		= &dtc2278_port_ops,
diff --git a/drivers/ide/hpt366.c b/drivers/ide/hpt366.c
index 58c51cddc10..4aec3b87ff9 100644
--- a/drivers/ide/hpt366.c
+++ b/drivers/ide/hpt366.c
@@ -443,7 +443,7 @@ static struct hpt_timings hpt37x_timings = {
 	}
 };
 
-static const struct hpt_info hpt36x __devinitdata = {
+static const struct hpt_info hpt36x __devinitconst = {
 	.chip_name	= "HPT36x",
 	.chip_type	= HPT36x,
 	.udma_mask	= HPT366_ALLOW_ATA66_3 ? (HPT366_ALLOW_ATA66_4 ? ATA_UDMA4 : ATA_UDMA3) : ATA_UDMA2,
@@ -451,7 +451,7 @@ static const struct hpt_info hpt36x __devinitdata = {
 	.timings	= &hpt36x_timings
 };
 
-static const struct hpt_info hpt370 __devinitdata = {
+static const struct hpt_info hpt370 __devinitconst = {
 	.chip_name	= "HPT370",
 	.chip_type	= HPT370,
 	.udma_mask	= HPT370_ALLOW_ATA100_5 ? ATA_UDMA5 : ATA_UDMA4,
@@ -459,7 +459,7 @@ static const struct hpt_info hpt370 __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt370a __devinitdata = {
+static const struct hpt_info hpt370a __devinitconst = {
 	.chip_name	= "HPT370A",
 	.chip_type	= HPT370A,
 	.udma_mask	= HPT370_ALLOW_ATA100_5 ? ATA_UDMA5 : ATA_UDMA4,
@@ -467,7 +467,7 @@ static const struct hpt_info hpt370a __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt374 __devinitdata = {
+static const struct hpt_info hpt374 __devinitconst = {
 	.chip_name	= "HPT374",
 	.chip_type	= HPT374,
 	.udma_mask	= ATA_UDMA5,
@@ -475,7 +475,7 @@ static const struct hpt_info hpt374 __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt372 __devinitdata = {
+static const struct hpt_info hpt372 __devinitconst = {
 	.chip_name	= "HPT372",
 	.chip_type	= HPT372,
 	.udma_mask	= HPT372_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -483,7 +483,7 @@ static const struct hpt_info hpt372 __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt372a __devinitdata = {
+static const struct hpt_info hpt372a __devinitconst = {
 	.chip_name	= "HPT372A",
 	.chip_type	= HPT372A,
 	.udma_mask	= HPT372_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -491,7 +491,7 @@ static const struct hpt_info hpt372a __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt302 __devinitdata = {
+static const struct hpt_info hpt302 __devinitconst = {
 	.chip_name	= "HPT302",
 	.chip_type	= HPT302,
 	.udma_mask	= HPT302_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -499,7 +499,7 @@ static const struct hpt_info hpt302 __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt371 __devinitdata = {
+static const struct hpt_info hpt371 __devinitconst = {
 	.chip_name	= "HPT371",
 	.chip_type	= HPT371,
 	.udma_mask	= HPT371_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -507,7 +507,7 @@ static const struct hpt_info hpt371 __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt372n __devinitdata = {
+static const struct hpt_info hpt372n __devinitconst = {
 	.chip_name	= "HPT372N",
 	.chip_type	= HPT372N,
 	.udma_mask	= HPT372_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -515,7 +515,7 @@ static const struct hpt_info hpt372n __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt302n __devinitdata = {
+static const struct hpt_info hpt302n __devinitconst = {
 	.chip_name	= "HPT302N",
 	.chip_type	= HPT302N,
 	.udma_mask	= HPT302_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -523,7 +523,7 @@ static const struct hpt_info hpt302n __devinitdata = {
 	.timings	= &hpt37x_timings
 };
 
-static const struct hpt_info hpt371n __devinitdata = {
+static const struct hpt_info hpt371n __devinitconst = {
 	.chip_name	= "HPT371N",
 	.chip_type	= HPT371N,
 	.udma_mask	= HPT371_ALLOW_ATA133_6 ? ATA_UDMA6 : ATA_UDMA5,
@@ -1361,7 +1361,7 @@ static const struct ide_dma_ops hpt36x_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info hpt366_chipsets[] __devinitdata = {
+static const struct ide_port_info hpt366_chipsets[] __devinitconst = {
 	{	/* 0: HPT36x */
 		.name		= DRV_NAME,
 		.init_chipset	= init_chipset_hpt366,
diff --git a/drivers/ide/ht6560b.c b/drivers/ide/ht6560b.c
index 986f2513eab..1e0fd3aa962 100644
--- a/drivers/ide/ht6560b.c
+++ b/drivers/ide/ht6560b.c
@@ -341,7 +341,7 @@ static const struct ide_port_ops ht6560b_port_ops = {
 	.set_pio_mode		= ht6560b_set_pio_mode,
 };
 
-static const struct ide_port_info ht6560b_port_info __initdata = {
+static const struct ide_port_info ht6560b_port_info __initconst = {
 	.name			= DRV_NAME,
 	.chipset		= ide_ht6560b,
 	.tp_ops 		= &ht6560b_tp_ops,
diff --git a/drivers/ide/icside.c b/drivers/ide/icside.c
index bcb507b0cfd..e640d0ac3af 100644
--- a/drivers/ide/icside.c
+++ b/drivers/ide/icside.c
@@ -451,7 +451,7 @@ err_free:
 	return ret;
 }
 
-static const struct ide_port_info icside_v6_port_info __initdata = {
+static const struct ide_port_info icside_v6_port_info __initconst = {
 	.init_dma		= icside_dma_off_init,
 	.port_ops		= &icside_v6_no_dma_port_ops,
 	.host_flags		= IDE_HFLAG_SERIALIZE | IDE_HFLAG_MMIO,
diff --git a/drivers/ide/ide-pci-generic.c b/drivers/ide/ide-pci-generic.c
index 7f56b738d76..dab5b670bfb 100644
--- a/drivers/ide/ide-pci-generic.c
+++ b/drivers/ide/ide-pci-generic.c
@@ -53,7 +53,7 @@ static const struct ide_port_ops netcell_port_ops = {
 		.udma_mask	= ATA_UDMA6, \
 	}
 
-static const struct ide_port_info generic_chipsets[] __devinitdata = {
+static const struct ide_port_info generic_chipsets[] __devinitconst = {
 	/*  0: Unknown */
 	DECLARE_GENERIC_PCI_DEV(0),
 
diff --git a/drivers/ide/it8172.c b/drivers/ide/it8172.c
index 560e66d0765..d5dd180c4b8 100644
--- a/drivers/ide/it8172.c
+++ b/drivers/ide/it8172.c
@@ -115,7 +115,7 @@ static const struct ide_port_ops it8172_port_ops = {
 	.set_dma_mode	= it8172_set_dma_mode,
 };
 
-static const struct ide_port_info it8172_port_info __devinitdata = {
+static const struct ide_port_info it8172_port_info __devinitconst = {
 	.name		= DRV_NAME,
 	.port_ops	= &it8172_port_ops,
 	.enablebits	= { {0x41, 0x80, 0x80}, {0x00, 0x00, 0x00} },
diff --git a/drivers/ide/it8213.c b/drivers/ide/it8213.c
index 46816ba2641..1847aeb5450 100644
--- a/drivers/ide/it8213.c
+++ b/drivers/ide/it8213.c
@@ -156,7 +156,7 @@ static const struct ide_port_ops it8213_port_ops = {
 	.cable_detect		= it8213_cable_detect,
 };
 
-static const struct ide_port_info it8213_chipset __devinitdata = {
+static const struct ide_port_info it8213_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= { {0x41, 0x80, 0x80} },
 	.port_ops	= &it8213_port_ops,
diff --git a/drivers/ide/it821x.c b/drivers/ide/it821x.c
index 2e3169f2acd..c5611dbca34 100644
--- a/drivers/ide/it821x.c
+++ b/drivers/ide/it821x.c
@@ -630,7 +630,7 @@ static const struct ide_port_ops it821x_port_ops = {
 	.cable_detect		= it821x_cable_detect,
 };
 
-static const struct ide_port_info it821x_chipset __devinitdata = {
+static const struct ide_port_info it821x_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_it821x,
 	.init_hwif	= init_hwif_it821x,
diff --git a/drivers/ide/jmicron.c b/drivers/ide/jmicron.c
index 74c2c4a6d90..efddd7d9f92 100644
--- a/drivers/ide/jmicron.c
+++ b/drivers/ide/jmicron.c
@@ -102,7 +102,7 @@ static const struct ide_port_ops jmicron_port_ops = {
 	.cable_detect		= jmicron_cable_detect,
 };
 
-static const struct ide_port_info jmicron_chipset __devinitdata = {
+static const struct ide_port_info jmicron_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= { { 0x40, 0x01, 0x01 }, { 0x40, 0x10, 0x10 } },
 	.port_ops	= &jmicron_port_ops,
diff --git a/drivers/ide/ns87415.c b/drivers/ide/ns87415.c
index 95327a2c242..73f78d872d5 100644
--- a/drivers/ide/ns87415.c
+++ b/drivers/ide/ns87415.c
@@ -293,7 +293,7 @@ static const struct ide_dma_ops ns87415_dma_ops = {
 	.dma_sff_read_status	= superio_dma_sff_read_status,
 };
 
-static const struct ide_port_info ns87415_chipset __devinitdata = {
+static const struct ide_port_info ns87415_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_hwif	= init_hwif_ns87415,
 	.tp_ops 	= &ns87415_tp_ops,
diff --git a/drivers/ide/opti621.c b/drivers/ide/opti621.c
index 1a53a4c375e..39edc66cb96 100644
--- a/drivers/ide/opti621.c
+++ b/drivers/ide/opti621.c
@@ -131,7 +131,7 @@ static const struct ide_port_ops opti621_port_ops = {
 	.set_pio_mode		= opti621_set_pio_mode,
 };
 
-static const struct ide_port_info opti621_chipset __devinitdata = {
+static const struct ide_port_info opti621_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= { {0x45, 0x80, 0x00}, {0x40, 0x08, 0x00} },
 	.port_ops	= &opti621_port_ops,
diff --git a/drivers/ide/pdc202xx_new.c b/drivers/ide/pdc202xx_new.c
index 9546fe2a93f..2e5ceb62fb3 100644
--- a/drivers/ide/pdc202xx_new.c
+++ b/drivers/ide/pdc202xx_new.c
@@ -465,7 +465,7 @@ static const struct ide_port_ops pdcnew_port_ops = {
 		.udma_mask	= udma, \
 	}
 
-static const struct ide_port_info pdcnew_chipsets[] __devinitdata = {
+static const struct ide_port_info pdcnew_chipsets[] __devinitconst = {
 	/* 0: PDC202{68,70} */		DECLARE_PDCNEW_DEV(ATA_UDMA5),
 	/* 1: PDC202{69,71,75,76,77} */	DECLARE_PDCNEW_DEV(ATA_UDMA6),
 };
diff --git a/drivers/ide/pdc202xx_old.c b/drivers/ide/pdc202xx_old.c
index 3a35ec6193d..56345109681 100644
--- a/drivers/ide/pdc202xx_old.c
+++ b/drivers/ide/pdc202xx_old.c
@@ -270,7 +270,7 @@ static const struct ide_dma_ops pdc2026x_dma_ops = {
 		.max_sectors	= sectors, \
 	}
 
-static const struct ide_port_info pdc202xx_chipsets[] __devinitdata = {
+static const struct ide_port_info pdc202xx_chipsets[] __devinitconst = {
 	{	/* 0: PDC20246 */
 		.name		= DRV_NAME,
 		.init_chipset	= init_chipset_pdc202xx,
diff --git a/drivers/ide/piix.c b/drivers/ide/piix.c
index 1892e81fb00..fe0fd60cfc0 100644
--- a/drivers/ide/piix.c
+++ b/drivers/ide/piix.c
@@ -344,7 +344,7 @@ static const struct ide_port_ops ich_port_ops = {
 		.udma_mask	= udma, \
 	}
 
-static const struct ide_port_info piix_pci_info[] __devinitdata = {
+static const struct ide_port_info piix_pci_info[] __devinitconst = {
 	/* 0: MPIIX */
 	{	/*
 		 * MPIIX actually has only a single IDE channel mapped to
diff --git a/drivers/ide/qd65xx.c b/drivers/ide/qd65xx.c
index e03f4f19c1d..a6fb6a894c7 100644
--- a/drivers/ide/qd65xx.c
+++ b/drivers/ide/qd65xx.c
@@ -335,7 +335,7 @@ static const struct ide_port_ops qd6580_port_ops = {
 	.set_pio_mode		= qd6580_set_pio_mode,
 };
 
-static const struct ide_port_info qd65xx_port_info __initdata = {
+static const struct ide_port_info qd65xx_port_info __initconst = {
 	.name			= DRV_NAME,
 	.tp_ops 		= &qd65xx_tp_ops,
 	.chipset		= ide_qd65xx,
diff --git a/drivers/ide/rz1000.c b/drivers/ide/rz1000.c
index a6414a884eb..c04173e9fc3 100644
--- a/drivers/ide/rz1000.c
+++ b/drivers/ide/rz1000.c
@@ -38,7 +38,7 @@ static int __devinit rz1000_disable_readahead(struct pci_dev *dev)
 	}
 }
 
-static const struct ide_port_info rz1000_chipset __devinitdata = {
+static const struct ide_port_info rz1000_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.host_flags	= IDE_HFLAG_NO_DMA,
 };
diff --git a/drivers/ide/sc1200.c b/drivers/ide/sc1200.c
index 356b9b504ff..d4758ebe77d 100644
--- a/drivers/ide/sc1200.c
+++ b/drivers/ide/sc1200.c
@@ -291,7 +291,7 @@ static const struct ide_dma_ops sc1200_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info sc1200_chipset __devinitdata = {
+static const struct ide_port_info sc1200_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.port_ops	= &sc1200_port_ops,
 	.dma_ops	= &sc1200_dma_ops,
diff --git a/drivers/ide/scc_pata.c b/drivers/ide/scc_pata.c
index b7f5b0c4310..97010381002 100644
--- a/drivers/ide/scc_pata.c
+++ b/drivers/ide/scc_pata.c
@@ -811,7 +811,7 @@ static const struct ide_dma_ops scc_dma_ops = {
 	.dma_sff_read_status	= scc_dma_sff_read_status,
 };
 
-static const struct ide_port_info scc_chipset __devinitdata = {
+static const struct ide_port_info scc_chipset __devinitconst = {
 	.name		= "sccIDE",
 	.init_iops	= init_iops_scc,
 	.init_dma	= scc_init_dma,
diff --git a/drivers/ide/serverworks.c b/drivers/ide/serverworks.c
index 35fb8dabb55..24d72ef23df 100644
--- a/drivers/ide/serverworks.c
+++ b/drivers/ide/serverworks.c
@@ -337,7 +337,7 @@ static const struct ide_port_ops svwks_port_ops = {
 	.cable_detect		= svwks_cable_detect,
 };
 
-static const struct ide_port_info serverworks_chipsets[] __devinitdata = {
+static const struct ide_port_info serverworks_chipsets[] __devinitconst = {
 	{	/* 0: OSB4 */
 		.name		= DRV_NAME,
 		.init_chipset	= init_chipset_svwks,
diff --git a/drivers/ide/siimage.c b/drivers/ide/siimage.c
index ddeda444a27..46f7e30d379 100644
--- a/drivers/ide/siimage.c
+++ b/drivers/ide/siimage.c
@@ -719,7 +719,7 @@ static const struct ide_dma_ops sil_dma_ops = {
 		.udma_mask	= ATA_UDMA6,		\
 	}
 
-static const struct ide_port_info siimage_chipsets[] __devinitdata = {
+static const struct ide_port_info siimage_chipsets[] __devinitconst = {
 	/* 0: SiI680 */  DECLARE_SII_DEV(&sil_pata_port_ops),
 	/* 1: SiI3112 */ DECLARE_SII_DEV(&sil_sata_port_ops)
 };
diff --git a/drivers/ide/sis5513.c b/drivers/ide/sis5513.c
index 4a002256775..09e61b4c5e9 100644
--- a/drivers/ide/sis5513.c
+++ b/drivers/ide/sis5513.c
@@ -563,7 +563,7 @@ static const struct ide_port_ops sis_ata133_port_ops = {
 	.cable_detect		= sis_cable_detect,
 };
 
-static const struct ide_port_info sis5513_chipset __devinitdata = {
+static const struct ide_port_info sis5513_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_sis5513,
 	.enablebits	= { {0x4a, 0x02, 0x02}, {0x4a, 0x04, 0x04} },
diff --git a/drivers/ide/sl82c105.c b/drivers/ide/sl82c105.c
index f21dc2ad768..d051cd224bd 100644
--- a/drivers/ide/sl82c105.c
+++ b/drivers/ide/sl82c105.c
@@ -299,7 +299,7 @@ static const struct ide_dma_ops sl82c105_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info sl82c105_chipset __devinitdata = {
+static const struct ide_port_info sl82c105_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_sl82c105,
 	.enablebits	= {{0x40,0x01,0x01}, {0x40,0x10,0x10}},
diff --git a/drivers/ide/slc90e66.c b/drivers/ide/slc90e66.c
index 864ffe0e26d..863a5e9283c 100644
--- a/drivers/ide/slc90e66.c
+++ b/drivers/ide/slc90e66.c
@@ -132,7 +132,7 @@ static const struct ide_port_ops slc90e66_port_ops = {
 	.cable_detect		= slc90e66_cable_detect,
 };
 
-static const struct ide_port_info slc90e66_chipset __devinitdata = {
+static const struct ide_port_info slc90e66_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= { {0x41, 0x80, 0x80}, {0x43, 0x80, 0x80} },
 	.port_ops	= &slc90e66_port_ops,
diff --git a/drivers/ide/tc86c001.c b/drivers/ide/tc86c001.c
index 4799d5c384e..17946785ebf 100644
--- a/drivers/ide/tc86c001.c
+++ b/drivers/ide/tc86c001.c
@@ -192,7 +192,7 @@ static const struct ide_dma_ops tc86c001_dma_ops = {
 	.dma_sff_read_status	= ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info tc86c001_chipset __devinitdata = {
+static const struct ide_port_info tc86c001_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_hwif	= init_hwif_tc86c001,
 	.port_ops	= &tc86c001_port_ops,
diff --git a/drivers/ide/triflex.c b/drivers/ide/triflex.c
index 281c9142634..55ce1b80efc 100644
--- a/drivers/ide/triflex.c
+++ b/drivers/ide/triflex.c
@@ -92,7 +92,7 @@ static const struct ide_port_ops triflex_port_ops = {
 	.set_dma_mode		= triflex_set_mode,
 };
 
-static const struct ide_port_info triflex_device __devinitdata = {
+static const struct ide_port_info triflex_device __devinitconst = {
 	.name		= DRV_NAME,
 	.enablebits	= {{0x80, 0x01, 0x01}, {0x80, 0x02, 0x02}},
 	.port_ops	= &triflex_port_ops,
diff --git a/drivers/ide/trm290.c b/drivers/ide/trm290.c
index 4b42ca09153..e494a98a43a 100644
--- a/drivers/ide/trm290.c
+++ b/drivers/ide/trm290.c
@@ -324,7 +324,7 @@ static struct ide_dma_ops trm290_dma_ops = {
 	.dma_check		= trm290_dma_check,
 };
 
-static const struct ide_port_info trm290_chipset __devinitdata = {
+static const struct ide_port_info trm290_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_hwif	= init_hwif_trm290,
 	.tp_ops 	= &trm290_tp_ops,
diff --git a/drivers/ide/tx4938ide.c b/drivers/ide/tx4938ide.c
index 7002765b593..91d49dd957e 100644
--- a/drivers/ide/tx4938ide.c
+++ b/drivers/ide/tx4938ide.c
@@ -117,7 +117,7 @@ static const struct ide_port_ops tx4938ide_port_ops = {
 	.set_pio_mode		= tx4938ide_set_pio_mode,
 };
 
-static const struct ide_port_info tx4938ide_port_info __initdata = {
+static const struct ide_port_info tx4938ide_port_info __initconst = {
 	.port_ops		= &tx4938ide_port_ops,
 #ifdef __BIG_ENDIAN
 	.tp_ops			= &tx4938ide_tp_ops,
diff --git a/drivers/ide/tx4939ide.c b/drivers/ide/tx4939ide.c
index 71c23195497..c0ab800b7bb 100644
--- a/drivers/ide/tx4939ide.c
+++ b/drivers/ide/tx4939ide.c
@@ -522,7 +522,7 @@ static const struct ide_dma_ops tx4939ide_dma_ops = {
 	.dma_sff_read_status	= tx4939ide_dma_sff_read_status,
 };
 
-static const struct ide_port_info tx4939ide_port_info __initdata = {
+static const struct ide_port_info tx4939ide_port_info __initconst = {
 	.init_hwif		= tx4939ide_init_hwif,
 	.init_dma		= tx4939ide_init_dma,
 	.port_ops		= &tx4939ide_port_ops,
diff --git a/drivers/ide/umc8672.c b/drivers/ide/umc8672.c
index 5cfb7812066..3aa0fea0f3d 100644
--- a/drivers/ide/umc8672.c
+++ b/drivers/ide/umc8672.c
@@ -128,7 +128,7 @@ static const struct ide_port_ops umc8672_port_ops = {
 	.set_pio_mode		= umc_set_pio_mode,
 };
 
-static const struct ide_port_info umc8672_port_info __initdata = {
+static const struct ide_port_info umc8672_port_info __initconst = {
 	.name			= DRV_NAME,
 	.chipset		= ide_umc8672,
 	.port_ops		= &umc8672_port_ops,
diff --git a/drivers/ide/via82cxxx.c b/drivers/ide/via82cxxx.c
index f46f49cfcc2..eb7767864d1 100644
--- a/drivers/ide/via82cxxx.c
+++ b/drivers/ide/via82cxxx.c
@@ -403,7 +403,7 @@ static const struct ide_port_ops via_port_ops = {
 	.cable_detect		= via82cxxx_cable_detect,
 };
 
-static const struct ide_port_info via82cxxx_chipset __devinitdata = {
+static const struct ide_port_info via82cxxx_chipset __devinitconst = {
 	.name		= DRV_NAME,
 	.init_chipset	= init_chipset_via82cxxx,
 	.enablebits	= { { 0x40, 0x02, 0x02 }, { 0x40, 0x01, 0x01 } },
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index d67999f6e34..394fea2ba1b 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -390,7 +390,7 @@ static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 		ret = idr_get_new_above(&cm.local_id_table, cm_id_priv,
 					next_id, &id);
 		if (!ret)
-			next_id = ((unsigned) id + 1) & MAX_ID_MASK;
+			next_id = ((unsigned) id + 1) & MAX_IDR_MASK;
 		spin_unlock_irqrestore(&cm.lock, flags);
 	} while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) );
 
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index 7972bae2e9b..2709ff58139 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -1183,7 +1183,7 @@ static ssize_t ucma_migrate_id(struct ucma_file *new_file,
 	struct rdma_ucm_migrate_id cmd;
 	struct rdma_ucm_migrate_resp resp;
 	struct ucma_context *ctx;
-	struct file *filp;
+	struct fd f;
 	struct ucma_file *cur_file;
 	int ret = 0;
 
@@ -1191,12 +1191,12 @@ static ssize_t ucma_migrate_id(struct ucma_file *new_file,
 		return -EFAULT;
 
 	/* Get current fd to protect against it being closed */
-	filp = fget(cmd.fd);
-	if (!filp)
+	f = fdget(cmd.fd);
+	if (!f.file)
 		return -ENOENT;
 
 	/* Validate current fd and prevent destruction of id. */
-	ctx = ucma_get_ctx(filp->private_data, cmd.id);
+	ctx = ucma_get_ctx(f.file->private_data, cmd.id);
 	if (IS_ERR(ctx)) {
 		ret = PTR_ERR(ctx);
 		goto file_put;
@@ -1230,7 +1230,7 @@ response:
 
 	ucma_put_ctx(ctx);
 file_put:
-	fput(filp);
+	fdput(f);
 	return ret;
 }
 
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index f9d0d7c413a..0cb0007724a 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -705,7 +705,7 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 	struct ib_udata			udata;
 	struct ib_uxrcd_object         *obj;
 	struct ib_xrcd                 *xrcd = NULL;
-	struct file                    *f = NULL;
+	struct fd			f = {NULL, 0};
 	struct inode                   *inode = NULL;
 	int				ret = 0;
 	int				new_xrcd = 0;
@@ -724,18 +724,13 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 
 	if (cmd.fd != -1) {
 		/* search for file descriptor */
-		f = fget(cmd.fd);
-		if (!f) {
-			ret = -EBADF;
-			goto err_tree_mutex_unlock;
-		}
-
-		inode = f->f_dentry->d_inode;
-		if (!inode) {
+		f = fdget(cmd.fd);
+		if (!f.file) {
 			ret = -EBADF;
 			goto err_tree_mutex_unlock;
 		}
 
+		inode = f.file->f_path.dentry->d_inode;
 		xrcd = find_xrcd(file->device, inode);
 		if (!xrcd && !(cmd.oflags & O_CREAT)) {
 			/* no file descriptor. Need CREATE flag */
@@ -800,8 +795,8 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 		goto err_copy;
 	}
 
-	if (f)
-		fput(f);
+	if (f.file)
+		fdput(f);
 
 	mutex_lock(&file->mutex);
 	list_add_tail(&obj->uobject.list, &file->ucontext->xrcd_list);
@@ -830,8 +825,8 @@ err:
 	put_uobj_write(&obj->uobject);
 
 err_tree_mutex_unlock:
-	if (f)
-		fput(f);
+	if (f.file)
+		fdput(f);
 
 	mutex_unlock(&file->device->xrcd_tree_mutex);
 
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 604556d73d2..6f2ce6fa98f 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -541,16 +541,15 @@ struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
 struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
 {
 	struct ib_uverbs_event_file *ev_file = NULL;
-	struct file *filp;
+	struct fd f = fdget(fd);
 
-	filp = fget(fd);
-	if (!filp)
+	if (!f.file)
 		return NULL;
 
-	if (filp->f_op != &uverbs_event_fops)
+	if (f.file->f_op != &uverbs_event_fops)
 		goto out;
 
-	ev_file = filp->private_data;
+	ev_file = f.file->private_data;
 	if (ev_file->is_async) {
 		ev_file = NULL;
 		goto out;
@@ -559,7 +558,7 @@ struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd)
 	kref_get(&ev_file->ref);
 
 out:
-	fput(filp);
+	fdput(f);
 	return ev_file;
 }
 
diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c
index e25e4dafb8a..80079e5a2e3 100644
--- a/drivers/infiniband/hw/mlx4/cm.c
+++ b/drivers/infiniband/hw/mlx4/cm.c
@@ -225,7 +225,7 @@ id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id)
 		ret = idr_get_new_above(&sriov->pv_id_table, ent,
 					next_id, &id);
 		if (!ret) {
-			next_id = ((unsigned) id + 1) & MAX_ID_MASK;
+			next_id = ((unsigned) id + 1) & MAX_IDR_MASK;
 			ent->pv_cm_id = (u32)id;
 			sl_id_map_add(ibdev, ent);
 		}
diff --git a/drivers/input/misc/twl4030-vibra.c b/drivers/input/misc/twl4030-vibra.c
index fc0ed9b4342..2194a3c7236 100644
--- a/drivers/input/misc/twl4030-vibra.c
+++ b/drivers/input/misc/twl4030-vibra.c
@@ -26,6 +26,7 @@
 #include <linux/module.h>
 #include <linux/jiffies.h>
 #include <linux/platform_device.h>
+#include <linux/of.h>
 #include <linux/workqueue.h>
 #include <linux/i2c/twl.h>
 #include <linux/mfd/twl4030-audio.h>
@@ -194,13 +195,26 @@ static int twl4030_vibra_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(twl4030_vibra_pm_ops,
 			 twl4030_vibra_suspend, twl4030_vibra_resume);
 
+static bool twl4030_vibra_check_coexist(struct twl4030_vibra_data *pdata,
+			      struct device_node *node)
+{
+	if (pdata && pdata->coexist)
+		return true;
+
+	if (of_find_node_by_name(node, "codec"))
+		return true;
+
+	return false;
+}
+
 static int __devinit twl4030_vibra_probe(struct platform_device *pdev)
 {
 	struct twl4030_vibra_data *pdata = pdev->dev.platform_data;
+	struct device_node *twl4030_core_node = pdev->dev.parent->of_node;
 	struct vibra_info *info;
 	int ret;
 
-	if (!pdata) {
+	if (!pdata && !twl4030_core_node) {
 		dev_dbg(&pdev->dev, "platform_data not available\n");
 		return -EINVAL;
 	}
@@ -210,7 +224,7 @@ static int __devinit twl4030_vibra_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	info->dev = &pdev->dev;
-	info->coexist = pdata->coexist;
+	info->coexist = twl4030_vibra_check_coexist(pdata, twl4030_core_node);
 	INIT_WORK(&info->play_work, vibra_play_work);
 
 	info->input_dev = input_allocate_device();
diff --git a/drivers/input/touchscreen/88pm860x-ts.c b/drivers/input/touchscreen/88pm860x-ts.c
index 05f30b73c3c..326218dbd6e 100644
--- a/drivers/input/touchscreen/88pm860x-ts.c
+++ b/drivers/input/touchscreen/88pm860x-ts.c
@@ -10,6 +10,7 @@
  */
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
 #include <linux/input.h>
@@ -113,14 +114,69 @@ static void pm860x_touch_close(struct input_dev *dev)
 	pm860x_set_bits(touch->i2c, MEAS_EN3, data, 0);
 }
 
+#ifdef CONFIG_OF
+static int __devinit pm860x_touch_dt_init(struct platform_device *pdev,
+					  struct pm860x_chip *chip,
+					  int *res_x)
+{
+	struct device_node *np = pdev->dev.parent->of_node;
+	struct i2c_client *i2c = (chip->id == CHIP_PM8607) ? chip->client \
+				 : chip->companion;
+	int data, n, ret;
+	if (!np)
+		return -ENODEV;
+	np = of_find_node_by_name(np, "touch");
+	if (!np) {
+		dev_err(&pdev->dev, "Can't find touch node\n");
+		return -EINVAL;
+	}
+	/* set GPADC MISC1 register */
+	data = 0;
+	if (!of_property_read_u32(np, "marvell,88pm860x-gpadc-prebias", &n))
+		data |= (n << 1) & PM8607_GPADC_PREBIAS_MASK;
+	if (!of_property_read_u32(np, "marvell,88pm860x-gpadc-slot-cycle", &n))
+		data |= (n << 3) & PM8607_GPADC_SLOT_CYCLE_MASK;
+	if (!of_property_read_u32(np, "marvell,88pm860x-gpadc-off-scale", &n))
+		data |= (n << 5) & PM8607_GPADC_OFF_SCALE_MASK;
+	if (!of_property_read_u32(np, "marvell,88pm860x-gpadc-sw-cal", &n))
+		data |= (n << 7) & PM8607_GPADC_SW_CAL_MASK;
+	if (data) {
+		ret = pm860x_reg_write(i2c, PM8607_GPADC_MISC1, data);
+		if (ret < 0)
+			return -EINVAL;
+	}
+	/* set tsi prebias time */
+	if (!of_property_read_u32(np, "marvell,88pm860x-tsi-prebias", &data)) {
+		ret = pm860x_reg_write(i2c, PM8607_TSI_PREBIAS, data);
+		if (ret < 0)
+			return -EINVAL;
+	}
+	/* set prebias & prechg time of pen detect */
+	data = 0;
+	if (!of_property_read_u32(np, "marvell,88pm860x-pen-prebias", &n))
+		data |= n & PM8607_PD_PREBIAS_MASK;
+	if (!of_property_read_u32(np, "marvell,88pm860x-pen-prechg", &n))
+		data |= n & PM8607_PD_PRECHG_MASK;
+	if (data) {
+		ret = pm860x_reg_write(i2c, PM8607_PD_PREBIAS, data);
+		if (ret < 0)
+			return -EINVAL;
+	}
+	of_property_read_u32(np, "marvell,88pm860x-resistor-X", res_x);
+	return 0;
+}
+#else
+#define pm860x_touch_dt_init(x, y, z)	(-1)
+#endif
+
 static int __devinit pm860x_touch_probe(struct platform_device *pdev)
 {
 	struct pm860x_chip *chip = dev_get_drvdata(pdev->dev.parent);
-	struct pm860x_platform_data *pm860x_pdata =		\
-				pdev->dev.parent->platform_data;
-	struct pm860x_touch_pdata *pdata = NULL;
+	struct pm860x_touch_pdata *pdata = pdev->dev.platform_data;
 	struct pm860x_touch *touch;
-	int irq, ret;
+	struct i2c_client *i2c = (chip->id == CHIP_PM8607) ? chip->client \
+				 : chip->companion;
+	int irq, ret, res_x = 0, data = 0;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
@@ -128,16 +184,55 @@ static int __devinit pm860x_touch_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	if (!pm860x_pdata) {
-		dev_err(&pdev->dev, "platform data is missing\n");
-		return -EINVAL;
-	}
-
-	pdata = pm860x_pdata->touch;
-	if (!pdata) {
-		dev_err(&pdev->dev, "touchscreen data is missing\n");
-		return -EINVAL;
+	if (pm860x_touch_dt_init(pdev, chip, &res_x)) {
+		if (pdata) {
+			/* set GPADC MISC1 register */
+			data = 0;
+			data |= (pdata->gpadc_prebias << 1)
+				& PM8607_GPADC_PREBIAS_MASK;
+			data |= (pdata->slot_cycle << 3)
+				& PM8607_GPADC_SLOT_CYCLE_MASK;
+			data |= (pdata->off_scale << 5)
+				& PM8607_GPADC_OFF_SCALE_MASK;
+			data |= (pdata->sw_cal << 7)
+				& PM8607_GPADC_SW_CAL_MASK;
+			if (data) {
+				ret = pm860x_reg_write(i2c,
+					PM8607_GPADC_MISC1, data);
+				if (ret < 0)
+					return -EINVAL;
+			}
+			/* set tsi prebias time */
+			if (pdata->tsi_prebias) {
+				data = pdata->tsi_prebias;
+				ret = pm860x_reg_write(i2c,
+					PM8607_TSI_PREBIAS, data);
+				if (ret < 0)
+					return -EINVAL;
+			}
+			/* set prebias & prechg time of pen detect */
+			data = 0;
+			data |= pdata->pen_prebias
+				& PM8607_PD_PREBIAS_MASK;
+			data |= (pdata->pen_prechg << 5)
+				& PM8607_PD_PRECHG_MASK;
+			if (data) {
+				ret = pm860x_reg_write(i2c,
+					PM8607_PD_PREBIAS, data);
+				if (ret < 0)
+					return -EINVAL;
+			}
+			res_x = pdata->res_x;
+		} else {
+			dev_err(&pdev->dev, "failed to get platform data\n");
+			return -EINVAL;
+		}
 	}
+	/* enable GPADC */
+	ret = pm860x_set_bits(i2c, PM8607_GPADC_MISC1, PM8607_GPADC_EN,
+			      PM8607_GPADC_EN);
+	if (ret)
+		return ret;
 
 	touch = kzalloc(sizeof(struct pm860x_touch), GFP_KERNEL);
 	if (touch == NULL)
@@ -158,9 +253,9 @@ static int __devinit pm860x_touch_probe(struct platform_device *pdev)
 	touch->idev->open = pm860x_touch_open;
 	touch->idev->close = pm860x_touch_close;
 	touch->chip = chip;
-	touch->i2c = (chip->id == CHIP_PM8607) ? chip->client : chip->companion;
-	touch->irq = irq + chip->irq_base;
-	touch->res_x = pdata->res_x;
+	touch->i2c = i2c;
+	touch->irq = irq;
+	touch->res_x = res_x;
 	input_set_drvdata(touch->idev, touch);
 
 	ret = request_threaded_irq(touch->irq, NULL, pm860x_touch_handler,
diff --git a/drivers/leds/leds-88pm860x.c b/drivers/leds/leds-88pm860x.c
index 61897cfeeda..b7e8cc0957f 100644
--- a/drivers/leds/leds-88pm860x.c
+++ b/drivers/leds/leds-88pm860x.c
@@ -12,6 +12,7 @@
 
 #include <linux/kernel.h>
 #include <linux/init.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
 #include <linux/leds.h>
@@ -20,18 +21,12 @@
 #include <linux/mfd/88pm860x.h>
 #include <linux/module.h>
 
-#define LED_PWM_SHIFT		(3)
 #define LED_PWM_MASK		(0x1F)
 #define LED_CURRENT_MASK	(0x07 << 5)
 
-#define LED_BLINK_ON_MASK	(0x07)
 #define LED_BLINK_MASK		(0x7F)
 
-#define LED_BLINK_ON(x)		((x & 0x7) * 66 + 66)
-#define LED_BLINK_ON_MIN	LED_BLINK_ON(0)
-#define LED_BLINK_ON_MAX	LED_BLINK_ON(0x7)
 #define LED_ON_CONTINUOUS	(0x0F << 3)
-#define LED_TO_ON(x)		((x - 66) / 66)
 
 #define LED1_BLINK_EN		(1 << 1)
 #define LED2_BLINK_EN		(1 << 2)
@@ -49,85 +44,25 @@ struct pm860x_led {
 	unsigned char brightness;
 	unsigned char current_brightness;
 
-	int blink_data;
-	int blink_time;
-	int blink_on;
-	int blink_off;
+	int reg_control;
+	int reg_blink;
+	int blink_mask;
 };
 
-/* return offset of color register */
-static inline int __led_off(int port)
-{
-	int ret = -EINVAL;
-
-	switch (port) {
-	case PM8606_LED1_RED:
-	case PM8606_LED1_GREEN:
-	case PM8606_LED1_BLUE:
-		ret = port - PM8606_LED1_RED + PM8606_RGB1B;
-		break;
-	case PM8606_LED2_RED:
-	case PM8606_LED2_GREEN:
-	case PM8606_LED2_BLUE:
-		ret = port - PM8606_LED2_RED + PM8606_RGB2B;
-		break;
-	}
-	return ret;
-}
-
-/* return offset of blink register */
-static inline int __blink_off(int port)
-{
-	int ret = -EINVAL;
-
-	switch (port) {
-	case PM8606_LED1_RED:
-	case PM8606_LED1_GREEN:
-	case PM8606_LED1_BLUE:
-		ret = PM8606_RGB1A;
-		break;
-	case PM8606_LED2_RED:
-	case PM8606_LED2_GREEN:
-	case PM8606_LED2_BLUE:
-		ret = PM8606_RGB2A;
-		break;
-	}
-	return ret;
-}
-
-static inline int __blink_ctl_mask(int port)
-{
-	int ret = -EINVAL;
-
-	switch (port) {
-	case PM8606_LED1_RED:
-	case PM8606_LED1_GREEN:
-	case PM8606_LED1_BLUE:
-		ret = LED1_BLINK_EN;
-		break;
-	case PM8606_LED2_RED:
-	case PM8606_LED2_GREEN:
-	case PM8606_LED2_BLUE:
-		ret = LED2_BLINK_EN;
-		break;
-	}
-	return ret;
-}
-
 static int led_power_set(struct pm860x_chip *chip, int port, int on)
 {
 	int ret = -EINVAL;
 
 	switch (port) {
-	case PM8606_LED1_RED:
-	case PM8606_LED1_GREEN:
-	case PM8606_LED1_BLUE:
+	case 0:
+	case 1:
+	case 2:
 		ret = on ? pm8606_osc_enable(chip, RGB1_ENABLE) :
 			pm8606_osc_disable(chip, RGB1_ENABLE);
 		break;
-	case PM8606_LED2_RED:
-	case PM8606_LED2_GREEN:
-	case PM8606_LED2_BLUE:
+	case 3:
+	case 4:
+	case 5:
 		ret = on ? pm8606_osc_enable(chip, RGB2_ENABLE) :
 			pm8606_osc_disable(chip, RGB2_ENABLE);
 		break;
@@ -141,7 +76,7 @@ static void pm860x_led_work(struct work_struct *work)
 	struct pm860x_led *led;
 	struct pm860x_chip *chip;
 	unsigned char buf[3];
-	int mask, ret;
+	int ret;
 
 	led = container_of(work, struct pm860x_led, work);
 	chip = led->chip;
@@ -149,34 +84,34 @@ static void pm860x_led_work(struct work_struct *work)
 	if ((led->current_brightness == 0) && led->brightness) {
 		led_power_set(chip, led->port, 1);
 		if (led->iset) {
-			pm860x_set_bits(led->i2c, __led_off(led->port),
+			pm860x_set_bits(led->i2c, led->reg_control,
 					LED_CURRENT_MASK, led->iset);
 		}
-		pm860x_set_bits(led->i2c, __blink_off(led->port),
+		pm860x_set_bits(led->i2c, led->reg_blink,
 				LED_BLINK_MASK, LED_ON_CONTINUOUS);
-		mask = __blink_ctl_mask(led->port);
-		pm860x_set_bits(led->i2c, PM8606_WLED3B, mask, mask);
+		pm860x_set_bits(led->i2c, PM8606_WLED3B, led->blink_mask,
+				led->blink_mask);
 	}
-	pm860x_set_bits(led->i2c, __led_off(led->port), LED_PWM_MASK,
+	pm860x_set_bits(led->i2c, led->reg_control, LED_PWM_MASK,
 			led->brightness);
 
 	if (led->brightness == 0) {
-		pm860x_bulk_read(led->i2c, __led_off(led->port), 3, buf);
+		pm860x_bulk_read(led->i2c, led->reg_control, 3, buf);
 		ret = buf[0] & LED_PWM_MASK;
 		ret |= buf[1] & LED_PWM_MASK;
 		ret |= buf[2] & LED_PWM_MASK;
 		if (ret == 0) {
 			/* unset current since no led is lighting */
-			pm860x_set_bits(led->i2c, __led_off(led->port),
+			pm860x_set_bits(led->i2c, led->reg_control,
 					LED_CURRENT_MASK, 0);
-			mask = __blink_ctl_mask(led->port);
-			pm860x_set_bits(led->i2c, PM8606_WLED3B, mask, 0);
+			pm860x_set_bits(led->i2c, PM8606_WLED3B,
+					led->blink_mask, 0);
 			led_power_set(chip, led->port, 0);
 		}
 	}
 	led->current_brightness = led->brightness;
 	dev_dbg(chip->dev, "Update LED. (reg:%d, brightness:%d)\n",
-		__led_off(led->port), led->brightness);
+		led->reg_control, led->brightness);
 	mutex_unlock(&led->lock);
 }
 
@@ -189,39 +124,92 @@ static void pm860x_led_set(struct led_classdev *cdev,
 	schedule_work(&data->work);
 }
 
+#ifdef CONFIG_OF
+static int pm860x_led_dt_init(struct platform_device *pdev,
+			      struct pm860x_led *data)
+{
+	struct device_node *nproot = pdev->dev.parent->of_node, *np;
+	int iset = 0;
+	if (!nproot)
+		return -ENODEV;
+	nproot = of_find_node_by_name(nproot, "leds");
+	if (!nproot) {
+		dev_err(&pdev->dev, "failed to find leds node\n");
+		return -ENODEV;
+	}
+	for_each_child_of_node(nproot, np) {
+		if (!of_node_cmp(np->name, data->name)) {
+			of_property_read_u32(np, "marvell,88pm860x-iset",
+					     &iset);
+			data->iset = PM8606_LED_CURRENT(iset);
+			break;
+		}
+	}
+	return 0;
+}
+#else
+#define pm860x_led_dt_init(x, y)	(-1)
+#endif
+
 static int pm860x_led_probe(struct platform_device *pdev)
 {
 	struct pm860x_chip *chip = dev_get_drvdata(pdev->dev.parent);
-	struct pm860x_led_pdata *pdata;
+	struct pm860x_led_pdata *pdata = pdev->dev.platform_data;
 	struct pm860x_led *data;
 	struct resource *res;
-	int ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource!\n");
-		return -EINVAL;
-	}
-
-	pdata = pdev->dev.platform_data;
-	if (pdata == NULL) {
-		dev_err(&pdev->dev, "No platform data!\n");
-		return -EINVAL;
-	}
+	int ret = 0;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(struct pm860x_led), GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
-	strncpy(data->name, res->name, MFD_NAME_SIZE - 1);
+	res = platform_get_resource_byname(pdev, IORESOURCE_REG, "control");
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for control\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_control = res->start;
+	res = platform_get_resource_byname(pdev, IORESOURCE_REG, "blink");
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for blink\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_blink = res->start;
+	memset(data->name, 0, MFD_NAME_SIZE);
+	switch (pdev->id) {
+	case 0:
+		data->blink_mask = LED1_BLINK_EN;
+		sprintf(data->name, "led0-red");
+		break;
+	case 1:
+		data->blink_mask = LED1_BLINK_EN;
+		sprintf(data->name, "led0-green");
+		break;
+	case 2:
+		data->blink_mask = LED1_BLINK_EN;
+		sprintf(data->name, "led0-blue");
+		break;
+	case 3:
+		data->blink_mask = LED2_BLINK_EN;
+		sprintf(data->name, "led1-red");
+		break;
+	case 4:
+		data->blink_mask = LED2_BLINK_EN;
+		sprintf(data->name, "led1-green");
+		break;
+	case 5:
+		data->blink_mask = LED2_BLINK_EN;
+		sprintf(data->name, "led1-blue");
+		break;
+	}
 	dev_set_drvdata(&pdev->dev, data);
 	data->chip = chip;
 	data->i2c = (chip->id == CHIP_PM8606) ? chip->client : chip->companion;
-	data->iset = pdata->iset;
-	data->port = pdata->flags;
-	if (data->port < 0) {
-		dev_err(&pdev->dev, "check device failed\n");
-		return -EINVAL;
-	}
+	data->port = pdev->id;
+	if (pm860x_led_dt_init(pdev, data))
+		if (pdata)
+			data->iset = pdata->iset;
 
 	data->current_brightness = 0;
 	data->cdev.name = data->name;
@@ -236,6 +224,9 @@ static int pm860x_led_probe(struct platform_device *pdev)
 	}
 	pm860x_led_set(&data->cdev, 0);
 	return 0;
+out:
+	devm_kfree(&pdev->dev, data);
+	return ret;
 }
 
 static int pm860x_led_remove(struct platform_device *pdev)
diff --git a/drivers/macintosh/macio_asic.c b/drivers/macintosh/macio_asic.c
index 20e5c2cda43..ef87310b766 100644
--- a/drivers/macintosh/macio_asic.c
+++ b/drivers/macintosh/macio_asic.c
@@ -748,7 +748,7 @@ static void __devexit macio_pci_remove(struct pci_dev* pdev)
  * MacIO is matched against any Apple ID, it's probe() function
  * will then decide wether it applies or not
  */
-static const struct pci_device_id __devinitdata pci_ids [] = { {
+static const struct pci_device_id __devinitconst pci_ids[] = { {
 	.vendor		= PCI_VENDOR_ID_APPLE,
 	.device		= PCI_ANY_ID,
 	.subvendor	= PCI_ANY_ID,
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index d99db5623ac..fb69baa06ca 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -1666,7 +1666,7 @@ mpt_mapresources(MPT_ADAPTER *ioc)
 	if (pci_request_selected_regions(pdev, ioc->bars, "mpt")) {
 		printk(MYIOC_s_ERR_FMT "pci_request_selected_regions() with "
 		    "MEM failed\n", ioc->name);
-		return r;
+		goto out_pci_disable_device;
 	}
 
 	if (sizeof(dma_addr_t) > 4) {
@@ -1690,8 +1690,7 @@ mpt_mapresources(MPT_ADAPTER *ioc)
 		} else {
 			printk(MYIOC_s_WARN_FMT "no suitable DMA mask for %s\n",
 			    ioc->name, pci_name(pdev));
-			pci_release_selected_regions(pdev, ioc->bars);
-			return r;
+			goto out_pci_release_region;
 		}
 	} else {
 		if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))
@@ -1704,8 +1703,7 @@ mpt_mapresources(MPT_ADAPTER *ioc)
 		} else {
 			printk(MYIOC_s_WARN_FMT "no suitable DMA mask for %s\n",
 			    ioc->name, pci_name(pdev));
-			pci_release_selected_regions(pdev, ioc->bars);
-			return r;
+			goto out_pci_release_region;
 		}
 	}
 
@@ -1735,8 +1733,8 @@ mpt_mapresources(MPT_ADAPTER *ioc)
 	if (mem == NULL) {
 		printk(MYIOC_s_ERR_FMT ": ERROR - Unable to map adapter"
 			" memory!\n", ioc->name);
-		pci_release_selected_regions(pdev, ioc->bars);
-		return -EINVAL;
+		r = -EINVAL;
+		goto out_pci_release_region;
 	}
 	ioc->memmap = mem;
 	dinitprintk(ioc, printk(MYIOC_s_INFO_FMT "mem = %p, mem_phys = %llx\n",
@@ -1750,6 +1748,12 @@ mpt_mapresources(MPT_ADAPTER *ioc)
 	ioc->pio_chip = (SYSIF_REGS __iomem *)port;
 
 	return 0;
+
+out_pci_release_region:
+	pci_release_selected_regions(pdev, ioc->bars);
+out_pci_disable_device:
+	pci_disable_device(pdev);
+	return r;
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
diff --git a/drivers/mfd/88pm860x-core.c b/drivers/mfd/88pm860x-core.c
index b73f033b2c6..59d117e9fa3 100644
--- a/drivers/mfd/88pm860x-core.c
+++ b/drivers/mfd/88pm860x-core.c
@@ -11,50 +11,116 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/88pm860x.h>
 #include <linux/regulator/machine.h>
 
 #define INT_STATUS_NUM			3
 
-static struct resource bk_resources[] __devinitdata = {
-	{PM8606_BACKLIGHT1, PM8606_BACKLIGHT1, "backlight-0", IORESOURCE_IO,},
-	{PM8606_BACKLIGHT2, PM8606_BACKLIGHT2, "backlight-1", IORESOURCE_IO,},
-	{PM8606_BACKLIGHT3, PM8606_BACKLIGHT3, "backlight-2", IORESOURCE_IO,},
-};
-
-static struct resource led_resources[] __devinitdata = {
-	{PM8606_LED1_RED,   PM8606_LED1_RED,   "led0-red",   IORESOURCE_IO,},
-	{PM8606_LED1_GREEN, PM8606_LED1_GREEN, "led0-green", IORESOURCE_IO,},
-	{PM8606_LED1_BLUE,  PM8606_LED1_BLUE,  "led0-blue",  IORESOURCE_IO,},
-	{PM8606_LED2_RED,   PM8606_LED2_RED,   "led1-red",   IORESOURCE_IO,},
-	{PM8606_LED2_GREEN, PM8606_LED2_GREEN, "led1-green", IORESOURCE_IO,},
-	{PM8606_LED2_BLUE,  PM8606_LED2_BLUE,  "led1-blue",  IORESOURCE_IO,},
-};
-
-static struct resource regulator_resources[] __devinitdata = {
-	{PM8607_ID_BUCK1, PM8607_ID_BUCK1, "buck-1", IORESOURCE_IO,},
-	{PM8607_ID_BUCK2, PM8607_ID_BUCK2, "buck-2", IORESOURCE_IO,},
-	{PM8607_ID_BUCK3, PM8607_ID_BUCK3, "buck-3", IORESOURCE_IO,},
-	{PM8607_ID_LDO1,  PM8607_ID_LDO1,  "ldo-01", IORESOURCE_IO,},
-	{PM8607_ID_LDO2,  PM8607_ID_LDO2,  "ldo-02", IORESOURCE_IO,},
-	{PM8607_ID_LDO3,  PM8607_ID_LDO3,  "ldo-03", IORESOURCE_IO,},
-	{PM8607_ID_LDO4,  PM8607_ID_LDO4,  "ldo-04", IORESOURCE_IO,},
-	{PM8607_ID_LDO5,  PM8607_ID_LDO5,  "ldo-05", IORESOURCE_IO,},
-	{PM8607_ID_LDO6,  PM8607_ID_LDO6,  "ldo-06", IORESOURCE_IO,},
-	{PM8607_ID_LDO7,  PM8607_ID_LDO7,  "ldo-07", IORESOURCE_IO,},
-	{PM8607_ID_LDO8,  PM8607_ID_LDO8,  "ldo-08", IORESOURCE_IO,},
-	{PM8607_ID_LDO9,  PM8607_ID_LDO9,  "ldo-09", IORESOURCE_IO,},
-	{PM8607_ID_LDO10, PM8607_ID_LDO10, "ldo-10", IORESOURCE_IO,},
-	{PM8607_ID_LDO11, PM8607_ID_LDO11, "ldo-11", IORESOURCE_IO,},
-	{PM8607_ID_LDO12, PM8607_ID_LDO12, "ldo-12", IORESOURCE_IO,},
-	{PM8607_ID_LDO13, PM8607_ID_LDO13, "ldo-13", IORESOURCE_IO,},
-	{PM8607_ID_LDO14, PM8607_ID_LDO14, "ldo-14", IORESOURCE_IO,},
-	{PM8607_ID_LDO15, PM8607_ID_LDO15, "ldo-15", IORESOURCE_IO,},
+static struct resource bk0_resources[] __devinitdata = {
+	{2, 2, "duty cycle", IORESOURCE_REG, },
+	{3, 3, "always on",  IORESOURCE_REG, },
+	{3, 3, "current",    IORESOURCE_REG, },
+};
+static struct resource bk1_resources[] __devinitdata = {
+	{4, 4, "duty cycle", IORESOURCE_REG, },
+	{5, 5, "always on",  IORESOURCE_REG, },
+	{5, 5, "current",    IORESOURCE_REG, },
+};
+static struct resource bk2_resources[] __devinitdata = {
+	{6, 6, "duty cycle", IORESOURCE_REG, },
+	{7, 7, "always on",  IORESOURCE_REG, },
+	{5, 5, "current",    IORESOURCE_REG, },
+};
+
+static struct resource led0_resources[] __devinitdata = {
+	/* RGB1 Red LED */
+	{0xd, 0xd, "control", IORESOURCE_REG, },
+	{0xc, 0xc, "blink",   IORESOURCE_REG, },
+};
+static struct resource led1_resources[] __devinitdata = {
+	/* RGB1 Green LED */
+	{0xe, 0xe, "control", IORESOURCE_REG, },
+	{0xc, 0xc, "blink",   IORESOURCE_REG, },
+};
+static struct resource led2_resources[] __devinitdata = {
+	/* RGB1 Blue LED */
+	{0xf, 0xf, "control", IORESOURCE_REG, },
+	{0xc, 0xc, "blink",   IORESOURCE_REG, },
+};
+static struct resource led3_resources[] __devinitdata = {
+	/* RGB2 Red LED */
+	{0x9, 0x9, "control", IORESOURCE_REG, },
+	{0x8, 0x8, "blink",   IORESOURCE_REG, },
+};
+static struct resource led4_resources[] __devinitdata = {
+	/* RGB2 Green LED */
+	{0xa, 0xa, "control", IORESOURCE_REG, },
+	{0x8, 0x8, "blink",   IORESOURCE_REG, },
+};
+static struct resource led5_resources[] __devinitdata = {
+	/* RGB2 Blue LED */
+	{0xb, 0xb, "control", IORESOURCE_REG, },
+	{0x8, 0x8, "blink",   IORESOURCE_REG, },
+};
+
+static struct resource buck1_resources[] __devinitdata = {
+	{0x24, 0x24, "buck set", IORESOURCE_REG, },
+};
+static struct resource buck2_resources[] __devinitdata = {
+	{0x25, 0x25, "buck set", IORESOURCE_REG, },
+};
+static struct resource buck3_resources[] __devinitdata = {
+	{0x26, 0x26, "buck set", IORESOURCE_REG, },
+};
+static struct resource ldo1_resources[] __devinitdata = {
+	{0x10, 0x10, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo2_resources[] __devinitdata = {
+	{0x11, 0x11, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo3_resources[] __devinitdata = {
+	{0x12, 0x12, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo4_resources[] __devinitdata = {
+	{0x13, 0x13, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo5_resources[] __devinitdata = {
+	{0x14, 0x14, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo6_resources[] __devinitdata = {
+	{0x15, 0x15, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo7_resources[] __devinitdata = {
+	{0x16, 0x16, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo8_resources[] __devinitdata = {
+	{0x17, 0x17, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo9_resources[] __devinitdata = {
+	{0x18, 0x18, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo10_resources[] __devinitdata = {
+	{0x19, 0x19, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo12_resources[] __devinitdata = {
+	{0x1a, 0x1a, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo_vibrator_resources[] __devinitdata = {
+	{0x28, 0x28, "ldo set", IORESOURCE_REG, },
+};
+static struct resource ldo14_resources[] __devinitdata = {
+	{0x1b, 0x1b, "ldo set", IORESOURCE_REG, },
 };
 
 static struct resource touch_resources[] __devinitdata = {
@@ -90,48 +156,145 @@ static struct resource charger_resources[] __devinitdata = {
 	{PM8607_IRQ_VCHG, PM8607_IRQ_VCHG, "vchg voltage",    IORESOURCE_IRQ,},
 };
 
-static struct resource preg_resources[] __devinitdata = {
-	{PM8606_ID_PREG,  PM8606_ID_PREG,  "preg",   IORESOURCE_IO,},
-};
-
 static struct resource rtc_resources[] __devinitdata = {
 	{PM8607_IRQ_RTC, PM8607_IRQ_RTC, "rtc", IORESOURCE_IRQ,},
 };
 
-static struct mfd_cell bk_devs[] = {
-	{"88pm860x-backlight", 0,},
-	{"88pm860x-backlight", 1,},
-	{"88pm860x-backlight", 2,},
-};
-
-static struct mfd_cell led_devs[] = {
-	{"88pm860x-led", 0,},
-	{"88pm860x-led", 1,},
-	{"88pm860x-led", 2,},
-	{"88pm860x-led", 3,},
-	{"88pm860x-led", 4,},
-	{"88pm860x-led", 5,},
-};
-
-static struct mfd_cell regulator_devs[] = {
-	{"88pm860x-regulator", 0,},
-	{"88pm860x-regulator", 1,},
-	{"88pm860x-regulator", 2,},
-	{"88pm860x-regulator", 3,},
-	{"88pm860x-regulator", 4,},
-	{"88pm860x-regulator", 5,},
-	{"88pm860x-regulator", 6,},
-	{"88pm860x-regulator", 7,},
-	{"88pm860x-regulator", 8,},
-	{"88pm860x-regulator", 9,},
-	{"88pm860x-regulator", 10,},
-	{"88pm860x-regulator", 11,},
-	{"88pm860x-regulator", 12,},
-	{"88pm860x-regulator", 13,},
-	{"88pm860x-regulator", 14,},
-	{"88pm860x-regulator", 15,},
-	{"88pm860x-regulator", 16,},
-	{"88pm860x-regulator", 17,},
+static struct mfd_cell bk_devs[] __devinitdata = {
+	{
+		.name = "88pm860x-backlight",
+		.id = 0,
+		.num_resources = ARRAY_SIZE(bk0_resources),
+		.resources = bk0_resources,
+	}, {
+		.name = "88pm860x-backlight",
+		.id = 1,
+		.num_resources = ARRAY_SIZE(bk1_resources),
+		.resources = bk1_resources,
+	}, {
+		.name = "88pm860x-backlight",
+		.id = 2,
+		.num_resources = ARRAY_SIZE(bk2_resources),
+		.resources = bk2_resources,
+	},
+};
+
+static struct mfd_cell led_devs[] __devinitdata = {
+	{
+		.name = "88pm860x-led",
+		.id = 0,
+		.num_resources = ARRAY_SIZE(led0_resources),
+		.resources = led0_resources,
+	}, {
+		.name = "88pm860x-led",
+		.id = 1,
+		.num_resources = ARRAY_SIZE(led1_resources),
+		.resources = led1_resources,
+	}, {
+		.name = "88pm860x-led",
+		.id = 2,
+		.num_resources = ARRAY_SIZE(led2_resources),
+		.resources = led2_resources,
+	}, {
+		.name = "88pm860x-led",
+		.id = 3,
+		.num_resources = ARRAY_SIZE(led3_resources),
+		.resources = led3_resources,
+	}, {
+		.name = "88pm860x-led",
+		.id = 4,
+		.num_resources = ARRAY_SIZE(led4_resources),
+		.resources = led4_resources,
+	}, {
+		.name = "88pm860x-led",
+		.id = 5,
+		.num_resources = ARRAY_SIZE(led5_resources),
+		.resources = led5_resources,
+	},
+};
+
+static struct mfd_cell reg_devs[] __devinitdata = {
+	{
+		.name = "88pm860x-regulator",
+		.id = 0,
+		.num_resources = ARRAY_SIZE(buck1_resources),
+		.resources = buck1_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 1,
+		.num_resources = ARRAY_SIZE(buck2_resources),
+		.resources = buck2_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 2,
+		.num_resources = ARRAY_SIZE(buck3_resources),
+		.resources = buck3_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 3,
+		.num_resources = ARRAY_SIZE(ldo1_resources),
+		.resources = ldo1_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 4,
+		.num_resources = ARRAY_SIZE(ldo2_resources),
+		.resources = ldo2_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 5,
+		.num_resources = ARRAY_SIZE(ldo3_resources),
+		.resources = ldo3_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 6,
+		.num_resources = ARRAY_SIZE(ldo4_resources),
+		.resources = ldo4_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 7,
+		.num_resources = ARRAY_SIZE(ldo5_resources),
+		.resources = ldo5_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 8,
+		.num_resources = ARRAY_SIZE(ldo6_resources),
+		.resources = ldo6_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 9,
+		.num_resources = ARRAY_SIZE(ldo7_resources),
+		.resources = ldo7_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 10,
+		.num_resources = ARRAY_SIZE(ldo8_resources),
+		.resources = ldo8_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 11,
+		.num_resources = ARRAY_SIZE(ldo9_resources),
+		.resources = ldo9_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 12,
+		.num_resources = ARRAY_SIZE(ldo10_resources),
+		.resources = ldo10_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 13,
+		.num_resources = ARRAY_SIZE(ldo12_resources),
+		.resources = ldo12_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 14,
+		.num_resources = ARRAY_SIZE(ldo_vibrator_resources),
+		.resources = ldo_vibrator_resources,
+	}, {
+		.name = "88pm860x-regulator",
+		.id = 15,
+		.num_resources = ARRAY_SIZE(ldo14_resources),
+		.resources = ldo14_resources,
+	},
 };
 
 static struct mfd_cell touch_devs[] = {
@@ -360,15 +523,12 @@ static void pm860x_irq_sync_unlock(struct irq_data *data)
 
 static void pm860x_irq_enable(struct irq_data *data)
 {
-	struct pm860x_chip *chip = irq_data_get_irq_chip_data(data);
-	pm860x_irqs[data->irq - chip->irq_base].enable
-		= pm860x_irqs[data->irq - chip->irq_base].offs;
+	pm860x_irqs[data->hwirq].enable = pm860x_irqs[data->hwirq].offs;
 }
 
 static void pm860x_irq_disable(struct irq_data *data)
 {
-	struct pm860x_chip *chip = irq_data_get_irq_chip_data(data);
-	pm860x_irqs[data->irq - chip->irq_base].enable = 0;
+	pm860x_irqs[data->hwirq].enable = 0;
 }
 
 static struct irq_chip pm860x_irq_chip = {
@@ -379,53 +539,25 @@ static struct irq_chip pm860x_irq_chip = {
 	.irq_disable	= pm860x_irq_disable,
 };
 
-static int __devinit device_gpadc_init(struct pm860x_chip *chip,
-				       struct pm860x_platform_data *pdata)
+static int pm860x_irq_domain_map(struct irq_domain *d, unsigned int virq,
+				 irq_hw_number_t hw)
 {
-	struct i2c_client *i2c = (chip->id == CHIP_PM8607) ? chip->client \
-				: chip->companion;
-	int data;
-	int ret;
-
-	/* initialize GPADC without activating it */
-
-	if (!pdata || !pdata->touch)
-		return -EINVAL;
-
-	/* set GPADC MISC1 register */
-	data = 0;
-	data |= (pdata->touch->gpadc_prebias << 1) & PM8607_GPADC_PREBIAS_MASK;
-	data |= (pdata->touch->slot_cycle << 3) & PM8607_GPADC_SLOT_CYCLE_MASK;
-	data |= (pdata->touch->off_scale << 5) & PM8607_GPADC_OFF_SCALE_MASK;
-	data |= (pdata->touch->sw_cal << 7) & PM8607_GPADC_SW_CAL_MASK;
-	if (data) {
-		ret = pm860x_reg_write(i2c, PM8607_GPADC_MISC1, data);
-		if (ret < 0)
-			goto out;
-	}
-	/* set tsi prebias time */
-	if (pdata->touch->tsi_prebias) {
-		data = pdata->touch->tsi_prebias;
-		ret = pm860x_reg_write(i2c, PM8607_TSI_PREBIAS, data);
-		if (ret < 0)
-			goto out;
-	}
-	/* set prebias & prechg time of pen detect */
-	data = 0;
-	data |= pdata->touch->pen_prebias & PM8607_PD_PREBIAS_MASK;
-	data |= (pdata->touch->pen_prechg << 5) & PM8607_PD_PRECHG_MASK;
-	if (data) {
-		ret = pm860x_reg_write(i2c, PM8607_PD_PREBIAS, data);
-		if (ret < 0)
-			goto out;
-	}
-
-	ret = pm860x_set_bits(i2c, PM8607_GPADC_MISC1,
-			      PM8607_GPADC_EN, PM8607_GPADC_EN);
-out:
-	return ret;
+	irq_set_chip_data(virq, d->host_data);
+	irq_set_chip_and_handler(virq, &pm860x_irq_chip, handle_edge_irq);
+	irq_set_nested_thread(virq, 1);
+#ifdef CONFIG_ARM
+	set_irq_flags(virq, IRQF_VALID);
+#else
+	irq_set_noprobe(virq);
+#endif
+	return 0;
 }
 
+static struct irq_domain_ops pm860x_irq_domain_ops = {
+	.map	= pm860x_irq_domain_map,
+	.xlate	= irq_domain_xlate_onetwocell,
+};
+
 static int __devinit device_irq_init(struct pm860x_chip *chip,
 				     struct pm860x_platform_data *pdata)
 {
@@ -433,13 +565,9 @@ static int __devinit device_irq_init(struct pm860x_chip *chip,
 				: chip->companion;
 	unsigned char status_buf[INT_STATUS_NUM];
 	unsigned long flags = IRQF_TRIGGER_FALLING | IRQF_ONESHOT;
-	int i, data, mask, ret = -EINVAL;
-	int __irq;
-
-	if (!pdata || !pdata->irq_base) {
-		dev_warn(chip->dev, "No interrupt support on IRQ base\n");
-		return -EINVAL;
-	}
+	int data, mask, ret = -EINVAL;
+	int nr_irqs, irq_base = -1;
+	struct device_node *node = i2c->dev.of_node;
 
 	mask = PM8607_B0_MISC1_INV_INT | PM8607_B0_MISC1_INT_CLEAR
 		| PM8607_B0_MISC1_INT_MASK;
@@ -479,26 +607,24 @@ static int __devinit device_irq_init(struct pm860x_chip *chip,
 		goto out;
 
 	mutex_init(&chip->irq_lock);
-	chip->irq_base = pdata->irq_base;
+
+	if (pdata && pdata->irq_base)
+		irq_base = pdata->irq_base;
+	nr_irqs = ARRAY_SIZE(pm860x_irqs);
+	chip->irq_base = irq_alloc_descs(irq_base, 0, nr_irqs, 0);
+	if (chip->irq_base < 0) {
+		dev_err(&i2c->dev, "Failed to allocate interrupts, ret:%d\n",
+			chip->irq_base);
+		ret = -EBUSY;
+		goto out;
+	}
+	irq_domain_add_legacy(node, nr_irqs, chip->irq_base, 0,
+			      &pm860x_irq_domain_ops, chip);
 	chip->core_irq = i2c->irq;
 	if (!chip->core_irq)
 		goto out;
 
-	/* register IRQ by genirq */
-	for (i = 0; i < ARRAY_SIZE(pm860x_irqs); i++) {
-		__irq = i + chip->irq_base;
-		irq_set_chip_data(__irq, chip);
-		irq_set_chip_and_handler(__irq, &pm860x_irq_chip,
-					 handle_edge_irq);
-		irq_set_nested_thread(__irq, 1);
-#ifdef CONFIG_ARM
-		set_irq_flags(__irq, IRQF_VALID);
-#else
-		irq_set_noprobe(__irq);
-#endif
-	}
-
-	ret = request_threaded_irq(chip->core_irq, NULL, pm860x_irq, flags,
+	ret = request_threaded_irq(chip->core_irq, NULL, pm860x_irq, flags | IRQF_ONESHOT,
 				   "88pm860x", chip);
 	if (ret) {
 		dev_err(chip->dev, "Failed to request IRQ: %d\n", ret);
@@ -615,108 +741,122 @@ static void __devinit device_osc_init(struct i2c_client *i2c)
 static void __devinit device_bk_init(struct pm860x_chip *chip,
 				     struct pm860x_platform_data *pdata)
 {
-	int ret;
-	int i, j, id;
-
-	if ((pdata == NULL) || (pdata->backlight == NULL))
-		return;
-
-	if (pdata->num_backlights > ARRAY_SIZE(bk_devs))
-		pdata->num_backlights = ARRAY_SIZE(bk_devs);
-
-	for (i = 0; i < pdata->num_backlights; i++) {
-		bk_devs[i].platform_data = &pdata->backlight[i];
-		bk_devs[i].pdata_size = sizeof(struct pm860x_backlight_pdata);
-
-		for (j = 0; j < ARRAY_SIZE(bk_devs); j++) {
-			id = bk_resources[j].start;
-			if (pdata->backlight[i].flags != id)
-				continue;
-
-			bk_devs[i].num_resources = 1;
-			bk_devs[i].resources = &bk_resources[j];
-			ret = mfd_add_devices(chip->dev, 0,
-					      &bk_devs[i], 1,
-					      &bk_resources[j], 0, NULL);
-			if (ret < 0) {
-				dev_err(chip->dev, "Failed to add "
-					"backlight subdev\n");
-				return;
-			}
+	int ret, i;
+
+	if (pdata && pdata->backlight) {
+		if (pdata->num_backlights > ARRAY_SIZE(bk_devs))
+			pdata->num_backlights = ARRAY_SIZE(bk_devs);
+		for (i = 0; i < pdata->num_backlights; i++) {
+			bk_devs[i].platform_data = &pdata->backlight[i];
+			bk_devs[i].pdata_size =
+				sizeof(struct pm860x_backlight_pdata);
 		}
 	}
+	ret = mfd_add_devices(chip->dev, 0, bk_devs,
+			      ARRAY_SIZE(bk_devs), NULL, 0, NULL);
+	if (ret < 0)
+		dev_err(chip->dev, "Failed to add backlight subdev\n");
 }
 
 static void __devinit device_led_init(struct pm860x_chip *chip,
 				      struct pm860x_platform_data *pdata)
 {
-	int ret;
-	int i, j, id;
-
-	if ((pdata == NULL) || (pdata->led == NULL))
-		return;
-
-	if (pdata->num_leds > ARRAY_SIZE(led_devs))
-		pdata->num_leds = ARRAY_SIZE(led_devs);
-
-	for (i = 0; i < pdata->num_leds; i++) {
-		led_devs[i].platform_data = &pdata->led[i];
-		led_devs[i].pdata_size = sizeof(struct pm860x_led_pdata);
-
-		for (j = 0; j < ARRAY_SIZE(led_devs); j++) {
-			id = led_resources[j].start;
-			if (pdata->led[i].flags != id)
-				continue;
-
-			led_devs[i].num_resources = 1;
-			led_devs[i].resources = &led_resources[j],
-			ret = mfd_add_devices(chip->dev, 0,
-					      &led_devs[i], 1,
-					      &led_resources[j], 0, NULL);
-			if (ret < 0) {
-				dev_err(chip->dev, "Failed to add "
-					"led subdev\n");
-				return;
-			}
+	int ret, i;
+
+	if (pdata && pdata->led) {
+		if (pdata->num_leds > ARRAY_SIZE(led_devs))
+			pdata->num_leds = ARRAY_SIZE(led_devs);
+		for (i = 0; i < pdata->num_leds; i++) {
+			led_devs[i].platform_data = &pdata->led[i];
+			led_devs[i].pdata_size =
+				sizeof(struct pm860x_led_pdata);
 		}
 	}
+	ret = mfd_add_devices(chip->dev, 0, led_devs,
+			      ARRAY_SIZE(led_devs), NULL, 0, NULL);
+	if (ret < 0) {
+		dev_err(chip->dev, "Failed to add led subdev\n");
+		return;
+	}
 }
 
 static void __devinit device_regulator_init(struct pm860x_chip *chip,
 					    struct pm860x_platform_data *pdata)
 {
-	struct regulator_init_data *initdata;
 	int ret;
-	int i, seq;
 
-	if ((pdata == NULL) || (pdata->regulator == NULL))
+	if (pdata == NULL)
+		return;
+	if (pdata->buck1) {
+		reg_devs[0].platform_data = pdata->buck1;
+		reg_devs[0].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->buck2) {
+		reg_devs[1].platform_data = pdata->buck2;
+		reg_devs[1].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->buck3) {
+		reg_devs[2].platform_data = pdata->buck3;
+		reg_devs[2].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo1) {
+		reg_devs[3].platform_data = pdata->ldo1;
+		reg_devs[3].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo2) {
+		reg_devs[4].platform_data = pdata->ldo2;
+		reg_devs[4].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo3) {
+		reg_devs[5].platform_data = pdata->ldo3;
+		reg_devs[5].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo4) {
+		reg_devs[6].platform_data = pdata->ldo4;
+		reg_devs[6].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo5) {
+		reg_devs[7].platform_data = pdata->ldo5;
+		reg_devs[7].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo6) {
+		reg_devs[8].platform_data = pdata->ldo6;
+		reg_devs[8].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo7) {
+		reg_devs[9].platform_data = pdata->ldo7;
+		reg_devs[9].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo8) {
+		reg_devs[10].platform_data = pdata->ldo8;
+		reg_devs[10].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo9) {
+		reg_devs[11].platform_data = pdata->ldo9;
+		reg_devs[11].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo10) {
+		reg_devs[12].platform_data = pdata->ldo10;
+		reg_devs[12].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo12) {
+		reg_devs[13].platform_data = pdata->ldo12;
+		reg_devs[13].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo_vibrator) {
+		reg_devs[14].platform_data = pdata->ldo_vibrator;
+		reg_devs[14].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo14) {
+		reg_devs[15].platform_data = pdata->ldo14;
+		reg_devs[15].pdata_size = sizeof(struct regulator_init_data);
+	}
+	ret = mfd_add_devices(chip->dev, 0, reg_devs,
+			      ARRAY_SIZE(reg_devs), NULL, 0, NULL);
+	if (ret < 0) {
+		dev_err(chip->dev, "Failed to add regulator subdev\n");
 		return;
-
-	if (pdata->num_regulators > ARRAY_SIZE(regulator_devs))
-		pdata->num_regulators = ARRAY_SIZE(regulator_devs);
-
-	for (i = 0, seq = -1; i < pdata->num_regulators; i++) {
-		initdata = &pdata->regulator[i];
-		seq = *(unsigned int *)initdata->driver_data;
-		if ((seq < 0) || (seq > PM8607_ID_RG_MAX)) {
-			dev_err(chip->dev, "Wrong ID(%d) on regulator(%s)\n",
-				seq, initdata->constraints.name);
-			goto out;
-		}
-		regulator_devs[i].platform_data = &pdata->regulator[i];
-		regulator_devs[i].pdata_size = sizeof(struct regulator_init_data);
-		regulator_devs[i].num_resources = 1;
-		regulator_devs[i].resources = &regulator_resources[seq];
-
-		ret = mfd_add_devices(chip->dev, 0, &regulator_devs[i], 1,
-				      &regulator_resources[seq], 0, NULL);
-		if (ret < 0) {
-			dev_err(chip->dev, "Failed to add regulator subdev\n");
-			goto out;
-		}
 	}
-out:
-	return;
 }
 
 static void __devinit device_rtc_init(struct pm860x_chip *chip,
@@ -785,10 +925,8 @@ static void __devinit device_power_init(struct pm860x_chip *chip,
 
 	power_devs[2].platform_data = &preg_init_data;
 	power_devs[2].pdata_size = sizeof(struct regulator_init_data);
-	power_devs[2].num_resources = ARRAY_SIZE(preg_resources);
-	power_devs[2].resources = &preg_resources[0],
 	ret = mfd_add_devices(chip->dev, 0, &power_devs[2], 1,
-			      &preg_resources[0], chip->irq_base, NULL);
+			      NULL, chip->irq_base, NULL);
 	if (ret < 0)
 		dev_err(chip->dev, "Failed to add preg subdev\n");
 }
@@ -868,10 +1006,6 @@ static void __devinit device_8607_init(struct pm860x_chip *chip,
 		goto out;
 	}
 
-	ret = device_gpadc_init(chip, pdata);
-	if (ret < 0)
-		goto out;
-
 	ret = device_irq_init(chip, pdata);
 	if (ret < 0)
 		goto out;
@@ -895,8 +1029,8 @@ static void __devinit device_8606_init(struct pm860x_chip *chip,
 	device_led_init(chip, pdata);
 }
 
-int __devinit pm860x_device_init(struct pm860x_chip *chip,
-		       struct pm860x_platform_data *pdata)
+static int __devinit pm860x_device_init(struct pm860x_chip *chip,
+					struct pm860x_platform_data *pdata)
 {
 	chip->core_irq = 0;
 
@@ -923,12 +1057,207 @@ int __devinit pm860x_device_init(struct pm860x_chip *chip,
 	return 0;
 }
 
-void __devexit pm860x_device_exit(struct pm860x_chip *chip)
+static void __devexit pm860x_device_exit(struct pm860x_chip *chip)
 {
 	device_irq_exit(chip);
 	mfd_remove_devices(chip->dev);
 }
 
+static int verify_addr(struct i2c_client *i2c)
+{
+	unsigned short addr_8607[] = {0x30, 0x34};
+	unsigned short addr_8606[] = {0x10, 0x11};
+	int size, i;
+
+	if (i2c == NULL)
+		return 0;
+	size = ARRAY_SIZE(addr_8606);
+	for (i = 0; i < size; i++) {
+		if (i2c->addr == *(addr_8606 + i))
+			return CHIP_PM8606;
+	}
+	size = ARRAY_SIZE(addr_8607);
+	for (i = 0; i < size; i++) {
+		if (i2c->addr == *(addr_8607 + i))
+			return CHIP_PM8607;
+	}
+	return 0;
+}
+
+static struct regmap_config pm860x_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static int __devinit pm860x_dt_init(struct device_node *np,
+				    struct device *dev,
+				    struct pm860x_platform_data *pdata)
+{
+	int ret;
+
+	if (of_get_property(np, "marvell,88pm860x-irq-read-clr", NULL))
+		pdata->irq_mode = 1;
+	ret = of_property_read_u32(np, "marvell,88pm860x-slave-addr",
+				   &pdata->companion_addr);
+	if (ret) {
+		dev_err(dev, "Not found \"marvell,88pm860x-slave-addr\" "
+			"property\n");
+		pdata->companion_addr = 0;
+	}
+	return 0;
+}
+
+static int __devinit pm860x_probe(struct i2c_client *client,
+				  const struct i2c_device_id *id)
+{
+	struct pm860x_platform_data *pdata = client->dev.platform_data;
+	struct device_node *node = client->dev.of_node;
+	struct pm860x_chip *chip;
+	int ret;
+
+	if (node && !pdata) {
+		/* parse DT to get platform data */
+		pdata = devm_kzalloc(&client->dev,
+				     sizeof(struct pm860x_platform_data),
+				     GFP_KERNEL);
+		if (!pdata)
+			return -ENOMEM;
+		ret = pm860x_dt_init(node, &client->dev, pdata);
+		if (ret)
+			goto err;
+	} else if (!pdata) {
+		pr_info("No platform data in %s!\n", __func__);
+		return -EINVAL;
+	}
+
+	chip = kzalloc(sizeof(struct pm860x_chip), GFP_KERNEL);
+	if (chip == NULL) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	chip->id = verify_addr(client);
+	chip->regmap = regmap_init_i2c(client, &pm860x_regmap_config);
+	if (IS_ERR(chip->regmap)) {
+		ret = PTR_ERR(chip->regmap);
+		dev_err(&client->dev, "Failed to allocate register map: %d\n",
+				ret);
+		kfree(chip);
+		return ret;
+	}
+	chip->client = client;
+	i2c_set_clientdata(client, chip);
+	chip->dev = &client->dev;
+	dev_set_drvdata(chip->dev, chip);
+
+	/*
+	 * Both client and companion client shares same platform driver.
+	 * Driver distinguishes them by pdata->companion_addr.
+	 * pdata->companion_addr is only assigned if companion chip exists.
+	 * At the same time, the companion_addr shouldn't equal to client
+	 * address.
+	 */
+	if (pdata->companion_addr && (pdata->companion_addr != client->addr)) {
+		chip->companion_addr = pdata->companion_addr;
+		chip->companion = i2c_new_dummy(chip->client->adapter,
+						chip->companion_addr);
+		chip->regmap_companion = regmap_init_i2c(chip->companion,
+							&pm860x_regmap_config);
+		if (IS_ERR(chip->regmap_companion)) {
+			ret = PTR_ERR(chip->regmap_companion);
+			dev_err(&chip->companion->dev,
+				"Failed to allocate register map: %d\n", ret);
+			return ret;
+		}
+		i2c_set_clientdata(chip->companion, chip);
+	}
+
+	pm860x_device_init(chip, pdata);
+	return 0;
+err:
+	if (node)
+		devm_kfree(&client->dev, pdata);
+	return ret;
+}
+
+static int __devexit pm860x_remove(struct i2c_client *client)
+{
+	struct pm860x_chip *chip = i2c_get_clientdata(client);
+
+	pm860x_device_exit(chip);
+	if (chip->companion) {
+		regmap_exit(chip->regmap_companion);
+		i2c_unregister_device(chip->companion);
+	}
+	regmap_exit(chip->regmap);
+	kfree(chip);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int pm860x_suspend(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct pm860x_chip *chip = i2c_get_clientdata(client);
+
+	if (device_may_wakeup(dev) && chip->wakeup_flag)
+		enable_irq_wake(chip->core_irq);
+	return 0;
+}
+
+static int pm860x_resume(struct device *dev)
+{
+	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
+	struct pm860x_chip *chip = i2c_get_clientdata(client);
+
+	if (device_may_wakeup(dev) && chip->wakeup_flag)
+		disable_irq_wake(chip->core_irq);
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(pm860x_pm_ops, pm860x_suspend, pm860x_resume);
+
+static const struct i2c_device_id pm860x_id_table[] = {
+	{ "88PM860x", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, pm860x_id_table);
+
+static const struct of_device_id pm860x_dt_ids[] = {
+	{ .compatible = "marvell,88pm860x", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, pm860x_dt_ids);
+
+static struct i2c_driver pm860x_driver = {
+	.driver	= {
+		.name	= "88PM860x",
+		.owner	= THIS_MODULE,
+		.pm     = &pm860x_pm_ops,
+		.of_match_table	= of_match_ptr(pm860x_dt_ids),
+	},
+	.probe		= pm860x_probe,
+	.remove		= __devexit_p(pm860x_remove),
+	.id_table	= pm860x_id_table,
+};
+
+static int __init pm860x_i2c_init(void)
+{
+	int ret;
+	ret = i2c_add_driver(&pm860x_driver);
+	if (ret != 0)
+		pr_err("Failed to register 88PM860x I2C driver: %d\n", ret);
+	return ret;
+}
+subsys_initcall(pm860x_i2c_init);
+
+static void __exit pm860x_i2c_exit(void)
+{
+	i2c_del_driver(&pm860x_driver);
+}
+module_exit(pm860x_i2c_exit);
+
 MODULE_DESCRIPTION("PMIC Driver for Marvell 88PM860x");
 MODULE_AUTHOR("Haojian Zhuang <haojian.zhuang@marvell.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/88pm860x-i2c.c b/drivers/mfd/88pm860x-i2c.c
index b2cfdc45856..ff8f803ce83 100644
--- a/drivers/mfd/88pm860x-i2c.c
+++ b/drivers/mfd/88pm860x-i2c.c
@@ -10,12 +10,9 @@
  */
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
 #include <linux/i2c.h>
-#include <linux/err.h>
 #include <linux/regmap.h>
 #include <linux/mfd/88pm860x.h>
-#include <linux/slab.h>
 
 int pm860x_reg_read(struct i2c_client *i2c, int reg)
 {
@@ -91,8 +88,18 @@ static int read_device(struct i2c_client *i2c, int reg,
 	unsigned char msgbuf0[I2C_SMBUS_BLOCK_MAX + 3];
 	unsigned char msgbuf1[I2C_SMBUS_BLOCK_MAX + 2];
 	struct i2c_adapter *adap = i2c->adapter;
-	struct i2c_msg msg[2] = {{i2c->addr, 0, 1, msgbuf0},
-				 {i2c->addr, I2C_M_RD, 0, msgbuf1},
+	struct i2c_msg msg[2] = {
+					{
+						.addr = i2c->addr,
+						.flags = 0,
+						.len = 1,
+						.buf = msgbuf0
+					},
+					{	.addr = i2c->addr,
+						.flags = I2C_M_RD,
+						.len = 0,
+						.buf = msgbuf1
+					},
 				};
 	int num = 1, ret = 0;
 
@@ -231,160 +238,3 @@ out:
 	return ret;
 }
 EXPORT_SYMBOL(pm860x_page_set_bits);
-
-static const struct i2c_device_id pm860x_id_table[] = {
-	{ "88PM860x", 0 },
-	{}
-};
-MODULE_DEVICE_TABLE(i2c, pm860x_id_table);
-
-static int verify_addr(struct i2c_client *i2c)
-{
-	unsigned short addr_8607[] = {0x30, 0x34};
-	unsigned short addr_8606[] = {0x10, 0x11};
-	int size, i;
-
-	if (i2c == NULL)
-		return 0;
-	size = ARRAY_SIZE(addr_8606);
-	for (i = 0; i < size; i++) {
-		if (i2c->addr == *(addr_8606 + i))
-			return CHIP_PM8606;
-	}
-	size = ARRAY_SIZE(addr_8607);
-	for (i = 0; i < size; i++) {
-		if (i2c->addr == *(addr_8607 + i))
-			return CHIP_PM8607;
-	}
-	return 0;
-}
-
-static struct regmap_config pm860x_regmap_config = {
-	.reg_bits = 8,
-	.val_bits = 8,
-};
-
-static int __devinit pm860x_probe(struct i2c_client *client,
-				  const struct i2c_device_id *id)
-{
-	struct pm860x_platform_data *pdata = client->dev.platform_data;
-	struct pm860x_chip *chip;
-	int ret;
-
-	if (!pdata) {
-		pr_info("No platform data in %s!\n", __func__);
-		return -EINVAL;
-	}
-
-	chip = kzalloc(sizeof(struct pm860x_chip), GFP_KERNEL);
-	if (chip == NULL)
-		return -ENOMEM;
-
-	chip->id = verify_addr(client);
-	chip->regmap = regmap_init_i2c(client, &pm860x_regmap_config);
-	if (IS_ERR(chip->regmap)) {
-		ret = PTR_ERR(chip->regmap);
-		dev_err(&client->dev, "Failed to allocate register map: %d\n",
-				ret);
-		kfree(chip);
-		return ret;
-	}
-	chip->client = client;
-	i2c_set_clientdata(client, chip);
-	chip->dev = &client->dev;
-	dev_set_drvdata(chip->dev, chip);
-
-	/*
-	 * Both client and companion client shares same platform driver.
-	 * Driver distinguishes them by pdata->companion_addr.
-	 * pdata->companion_addr is only assigned if companion chip exists.
-	 * At the same time, the companion_addr shouldn't equal to client
-	 * address.
-	 */
-	if (pdata->companion_addr && (pdata->companion_addr != client->addr)) {
-		chip->companion_addr = pdata->companion_addr;
-		chip->companion = i2c_new_dummy(chip->client->adapter,
-						chip->companion_addr);
-		chip->regmap_companion = regmap_init_i2c(chip->companion,
-							&pm860x_regmap_config);
-		if (IS_ERR(chip->regmap_companion)) {
-			ret = PTR_ERR(chip->regmap_companion);
-			dev_err(&chip->companion->dev,
-				"Failed to allocate register map: %d\n", ret);
-			return ret;
-		}
-		i2c_set_clientdata(chip->companion, chip);
-	}
-
-	pm860x_device_init(chip, pdata);
-	return 0;
-}
-
-static int __devexit pm860x_remove(struct i2c_client *client)
-{
-	struct pm860x_chip *chip = i2c_get_clientdata(client);
-
-	pm860x_device_exit(chip);
-	if (chip->companion) {
-		regmap_exit(chip->regmap_companion);
-		i2c_unregister_device(chip->companion);
-	}
-	regmap_exit(chip->regmap);
-	kfree(chip);
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int pm860x_suspend(struct device *dev)
-{
-	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
-	struct pm860x_chip *chip = i2c_get_clientdata(client);
-
-	if (device_may_wakeup(dev) && chip->wakeup_flag)
-		enable_irq_wake(chip->core_irq);
-	return 0;
-}
-
-static int pm860x_resume(struct device *dev)
-{
-	struct i2c_client *client = container_of(dev, struct i2c_client, dev);
-	struct pm860x_chip *chip = i2c_get_clientdata(client);
-
-	if (device_may_wakeup(dev) && chip->wakeup_flag)
-		disable_irq_wake(chip->core_irq);
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(pm860x_pm_ops, pm860x_suspend, pm860x_resume);
-
-static struct i2c_driver pm860x_driver = {
-	.driver	= {
-		.name	= "88PM860x",
-		.owner	= THIS_MODULE,
-		.pm     = &pm860x_pm_ops,
-	},
-	.probe		= pm860x_probe,
-	.remove		= __devexit_p(pm860x_remove),
-	.id_table	= pm860x_id_table,
-};
-
-static int __init pm860x_i2c_init(void)
-{
-	int ret;
-	ret = i2c_add_driver(&pm860x_driver);
-	if (ret != 0)
-		pr_err("Failed to register 88PM860x I2C driver: %d\n", ret);
-	return ret;
-}
-subsys_initcall(pm860x_i2c_init);
-
-static void __exit pm860x_i2c_exit(void)
-{
-	i2c_del_driver(&pm860x_driver);
-}
-module_exit(pm860x_i2c_exit);
-
-MODULE_DESCRIPTION("I2C Driver for Marvell 88PM860x");
-MODULE_AUTHOR("Haojian Zhuang <haojian.zhuang@marvell.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index b1a146205c0..acab3ef8a31 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -298,16 +298,6 @@ config MFD_TWL4030_AUDIO
 	select MFD_CORE
 	default n
 
-config TWL6030_PWM
-	tristate "TWL6030 PWM (Pulse Width Modulator) Support"
-	depends on TWL4030_CORE
-	select HAVE_PWM
-	depends on !PWM
-	default n
-	help
-	  Say yes here if you want support for TWL6030 PWM.
-	  This is used to control charging LED brightness.
-
 config TWL6040_CORE
 	bool "Support for TWL6040 audio codec"
 	depends on I2C=y && GENERIC_HARDIRQS
@@ -385,6 +375,18 @@ config MFD_T7L66XB
 	help
 	  Support for Toshiba Mobile IO Controller T7L66XB
 
+config MFD_SMSC
+       bool "Support for the SMSC ECE1099 series chips"
+       depends on I2C=y
+       select MFD_CORE
+       select REGMAP_I2C
+       help
+        If you say yes here you get support for the
+        ece1099 chips from SMSC.
+
+        To compile this driver as a module, choose M here: the
+        module will be called smsc.
+
 config MFD_TC6387XB
 	bool "Support Toshiba TC6387XB"
 	depends on ARM && HAVE_CLK
@@ -441,6 +443,23 @@ config MFD_DA9052_I2C
 	  for accessing the device, additional drivers must be enabled in
 	  order to use the functionality of the device.
 
+config MFD_DA9055
+	bool "Dialog Semiconductor DA9055 PMIC Support"
+	select REGMAP_I2C
+	select REGMAP_IRQ
+	select PMIC_DA9055
+	select MFD_CORE
+	depends on I2C=y
+	help
+	  Say yes here for support of Dialog Semiconductor DA9055. This is
+	  a Power Management IC. This driver provides common support for
+	  accessing the device as well as the I2C interface to the chip itself.
+	  Additional drivers must be enabled in order to use the functionality
+	  of the device.
+
+	  This driver can be built as a module. If built as a module it will be
+	  called "da9055"
+
 config PMIC_ADP5520
 	bool "Analog Devices ADP5520/01 MFD PMIC Core Support"
 	depends on I2C=y
@@ -451,6 +470,16 @@ config PMIC_ADP5520
 	  individual components like LCD backlight, LEDs, GPIOs and Kepad
 	  under the corresponding menus.
 
+config MFD_LP8788
+	bool "Texas Instruments LP8788 Power Management Unit Driver"
+	depends on I2C=y
+	select MFD_CORE
+	select REGMAP_I2C
+	select IRQ_DOMAIN
+	help
+	  TI LP8788 PMU supports regulators, battery charger, RTC,
+	  ADC, backlight driver and current sinks.
+
 config MFD_MAX77686
 	bool "Maxim Semiconductor MAX77686 PMIC Support"
 	depends on I2C=y && GENERIC_HARDIRQS
@@ -477,6 +506,18 @@ config MFD_MAX77693
 	  additional drivers must be enabled in order to use the functionality
 	  of the device.
 
+config MFD_MAX8907
+	tristate "Maxim Semiconductor MAX8907 PMIC Support"
+	select MFD_CORE
+	depends on I2C=y && GENERIC_HARDIRQS
+	select REGMAP_I2C
+	select REGMAP_IRQ
+	help
+	  Say yes here to support for Maxim Semiconductor MAX8907. This is
+	  a Power Management IC. This driver provides common support for
+	  accessing the device; additional drivers must be enabled in order
+	  to use the functionality of the device.
+
 config MFD_MAX8925
 	bool "Maxim Semiconductor MAX8925 PMIC Support"
 	depends on I2C=y && GENERIC_HARDIRQS
@@ -896,7 +937,7 @@ config MFD_WL1273_CORE
 	  audio codec.
 
 config MFD_OMAP_USB_HOST
-	bool "Support OMAP USBHS core driver"
+	bool "Support OMAP USBHS core and TLL driver"
 	depends on USB_EHCI_HCD_OMAP || USB_OHCI_HCD_OMAP3
 	default y
 	help
@@ -985,13 +1026,13 @@ config MFD_STA2X11
 	depends on STA2X11
 	select MFD_CORE
 
-config MFD_ANATOP
-	bool "Support for Freescale i.MX on-chip ANATOP controller"
-	depends on SOC_IMX6Q
+config MFD_SYSCON
+	bool "System Controller Register R/W Based on Regmap"
+	depends on OF
+	select REGMAP_MMIO
 	help
-	  Select this option to enable Freescale i.MX on-chip ANATOP
-	  MFD controller. This controller embeds regulator and
-	  thermal devices for Freescale i.MX platforms.
+	  Select this option to enable accessing system control registers
+	  via regmap.
 
 config MFD_PALMAS
 	bool "Support for the TI Palmas series chips"
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index 79dd22d1dc3..d8ccb630ddb 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -63,7 +63,6 @@ obj-$(CONFIG_TWL4030_CORE)	+= twl-core.o twl4030-irq.o twl6030-irq.o
 obj-$(CONFIG_TWL4030_MADC)      += twl4030-madc.o
 obj-$(CONFIG_TWL4030_POWER)    += twl4030-power.o
 obj-$(CONFIG_MFD_TWL4030_AUDIO)	+= twl4030-audio.o
-obj-$(CONFIG_TWL6030_PWM)	+= twl6030-pwm.o
 obj-$(CONFIG_TWL6040_CORE)	+= twl6040-core.o twl6040-irq.o
 
 obj-$(CONFIG_MFD_MC13XXX)	+= mc13xxx-core.o
@@ -77,6 +76,7 @@ obj-$(CONFIG_EZX_PCAP)		+= ezx-pcap.o
 obj-$(CONFIG_MCP)		+= mcp-core.o
 obj-$(CONFIG_MCP_SA11X0)	+= mcp-sa11x0.o
 obj-$(CONFIG_MCP_UCB1200)	+= ucb1x00-core.o
+obj-$(CONFIG_MFD_SMSC)        += smsc-ece1099.o
 obj-$(CONFIG_MCP_UCB1200_TS)	+= ucb1x00-ts.o
 
 ifeq ($(CONFIG_SA1100_ASSABET),y)
@@ -90,8 +90,14 @@ obj-$(CONFIG_PMIC_DA9052)	+= da9052-core.o
 obj-$(CONFIG_MFD_DA9052_SPI)	+= da9052-spi.o
 obj-$(CONFIG_MFD_DA9052_I2C)	+= da9052-i2c.o
 
+obj-$(CONFIG_MFD_LP8788)	+= lp8788.o lp8788-irq.o
+
+da9055-objs			:= da9055-core.o da9055-i2c.o
+obj-$(CONFIG_MFD_DA9055)	+= da9055.o
+
 obj-$(CONFIG_MFD_MAX77686)	+= max77686.o max77686-irq.o
 obj-$(CONFIG_MFD_MAX77693)	+= max77693.o max77693-irq.o
+obj-$(CONFIG_MFD_MAX8907)	+= max8907.o
 max8925-objs			:= max8925-core.o max8925-i2c.o
 obj-$(CONFIG_MFD_MAX8925)	+= max8925.o
 obj-$(CONFIG_MFD_MAX8997)	+= max8997.o max8997-irq.o
@@ -120,7 +126,7 @@ obj-$(CONFIG_MFD_TPS6586X)	+= tps6586x.o
 obj-$(CONFIG_MFD_VX855)		+= vx855.o
 obj-$(CONFIG_MFD_WL1273_CORE)	+= wl1273-core.o
 obj-$(CONFIG_MFD_CS5535)	+= cs5535-mfd.o
-obj-$(CONFIG_MFD_OMAP_USB_HOST)	+= omap-usb-host.o
+obj-$(CONFIG_MFD_OMAP_USB_HOST)	+= omap-usb-host.o omap-usb-tll.o
 obj-$(CONFIG_MFD_PM8921_CORE) 	+= pm8921-core.o
 obj-$(CONFIG_MFD_PM8XXX_IRQ) 	+= pm8xxx-irq.o
 obj-$(CONFIG_TPS65911_COMPARATOR)	+= tps65911-comparator.o
@@ -130,5 +136,5 @@ obj-$(CONFIG_MFD_INTEL_MSIC)	+= intel_msic.o
 obj-$(CONFIG_MFD_PALMAS)	+= palmas.o
 obj-$(CONFIG_MFD_RC5T583)	+= rc5t583.o rc5t583-irq.o
 obj-$(CONFIG_MFD_SEC_CORE)	+= sec-core.o sec-irq.o
-obj-$(CONFIG_MFD_ANATOP)	+= anatop-mfd.o
+obj-$(CONFIG_MFD_SYSCON)	+= syscon.o
 obj-$(CONFIG_MFD_LM3533)	+= lm3533-core.o lm3533-ctrlbank.o
diff --git a/drivers/mfd/ab3100-core.c b/drivers/mfd/ab3100-core.c
index 01781ae5d0d..2b3dde571a5 100644
--- a/drivers/mfd/ab3100-core.c
+++ b/drivers/mfd/ab3100-core.c
@@ -21,6 +21,7 @@
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
 #include <linux/mfd/core.h>
+#include <linux/mfd/ab3100.h>
 #include <linux/mfd/abx500.h>
 
 /* These are the only registers inside AB3100 used in this main file */
diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c
index 47adf800024..1667c77b5cd 100644
--- a/drivers/mfd/ab8500-core.c
+++ b/drivers/mfd/ab8500-core.c
@@ -472,6 +472,22 @@ static irqreturn_t ab8500_hierarchical_irq(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
+/**
+ * ab8500_irq_get_virq(): Map an interrupt on a chip to a virtual IRQ
+ *
+ * @ab8500: ab8500_irq controller to operate on.
+ * @irq: index of the interrupt requested in the chip IRQs
+ *
+ * Useful for drivers to request their own IRQs.
+ */
+static int ab8500_irq_get_virq(struct ab8500 *ab8500, int irq)
+{
+	if (!ab8500)
+		return -EINVAL;
+
+	return irq_create_mapping(ab8500->domain, irq);
+}
+
 static irqreturn_t ab8500_irq(int irq, void *dev)
 {
 	struct ab8500 *ab8500 = dev;
@@ -501,8 +517,9 @@ static irqreturn_t ab8500_irq(int irq, void *dev)
 		do {
 			int bit = __ffs(value);
 			int line = i * 8 + bit;
+			int virq = ab8500_irq_get_virq(ab8500, line);
 
-			handle_nested_irq(ab8500->irq_base + line);
+			handle_nested_irq(virq);
 			value &= ~(1 << bit);
 
 		} while (value);
@@ -511,23 +528,6 @@ static irqreturn_t ab8500_irq(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
-/**
- * ab8500_irq_get_virq(): Map an interrupt on a chip to a virtual IRQ
- *
- * @ab8500: ab8500_irq controller to operate on.
- * @irq: index of the interrupt requested in the chip IRQs
- *
- * Useful for drivers to request their own IRQs.
- */
-int ab8500_irq_get_virq(struct ab8500 *ab8500, int irq)
-{
-	if (!ab8500)
-		return -EINVAL;
-
-	return irq_create_mapping(ab8500->domain, irq);
-}
-EXPORT_SYMBOL_GPL(ab8500_irq_get_virq);
-
 static int ab8500_irq_map(struct irq_domain *d, unsigned int virq,
 				irq_hw_number_t hwirq)
 {
@@ -1076,6 +1076,7 @@ static struct mfd_cell __devinitdata ab8500_devs[] = {
 	},
 	{
 		.name = "ab8500-codec",
+		.of_compatible = "stericsson,ab8500-codec",
 	},
 };
 
diff --git a/drivers/mfd/anatop-mfd.c b/drivers/mfd/anatop-mfd.c
deleted file mode 100644
index 5576e07576d..00000000000
--- a/drivers/mfd/anatop-mfd.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Anatop MFD driver
- *
- * Copyright (C) 2012 Ying-Chun Liu (PaulLiu) <paul.liu@linaro.org>
- * Copyright (C) 2012 Linaro
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- */
-
-#include <linux/io.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/of.h>
-#include <linux/of_platform.h>
-#include <linux/of_address.h>
-#include <linux/mfd/anatop.h>
-
-u32 anatop_read_reg(struct anatop *adata, u32 addr)
-{
-	return readl(adata->ioreg + addr);
-}
-EXPORT_SYMBOL_GPL(anatop_read_reg);
-
-void anatop_write_reg(struct anatop *adata, u32 addr, u32 data, u32 mask)
-{
-	u32 val;
-
-	data &= mask;
-
-	spin_lock(&adata->reglock);
-	val = readl(adata->ioreg + addr);
-	val &= ~mask;
-	val |= data;
-	writel(val, adata->ioreg + addr);
-	spin_unlock(&adata->reglock);
-}
-EXPORT_SYMBOL_GPL(anatop_write_reg);
-
-static const struct of_device_id of_anatop_match[] = {
-	{ .compatible = "fsl,imx6q-anatop", },
-	{ },
-};
-
-static int __devinit of_anatop_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	void *ioreg;
-	struct anatop *drvdata;
-
-	ioreg = of_iomap(np, 0);
-	if (!ioreg)
-		return -EADDRNOTAVAIL;
-	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
-	if (!drvdata)
-		return -ENOMEM;
-	drvdata->ioreg = ioreg;
-	spin_lock_init(&drvdata->reglock);
-	platform_set_drvdata(pdev, drvdata);
-	of_platform_populate(np, NULL, NULL, dev);
-
-	return 0;
-}
-
-static int __devexit of_anatop_remove(struct platform_device *pdev)
-{
-	struct anatop *drvdata;
-	drvdata = platform_get_drvdata(pdev);
-	iounmap(drvdata->ioreg);
-
-	return 0;
-}
-
-static struct platform_driver anatop_of_driver = {
-	.driver = {
-		.name = "anatop-mfd",
-		.owner = THIS_MODULE,
-		.of_match_table = of_anatop_match,
-	},
-	.probe		= of_anatop_probe,
-	.remove		= of_anatop_remove,
-};
-
-static int __init anatop_init(void)
-{
-	return platform_driver_register(&anatop_of_driver);
-}
-postcore_initcall(anatop_init);
-
-static void __exit anatop_exit(void)
-{
-	platform_driver_unregister(&anatop_of_driver);
-}
-module_exit(anatop_exit);
-
-MODULE_AUTHOR("Ying-Chun Liu (PaulLiu) <paul.liu@linaro.org>");
-MODULE_DESCRIPTION("ANATOP MFD driver");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/arizona-irq.c b/drivers/mfd/arizona-irq.c
index 98ac345f468..ef0f2d001df 100644
--- a/drivers/mfd/arizona-irq.c
+++ b/drivers/mfd/arizona-irq.c
@@ -94,7 +94,8 @@ static irqreturn_t arizona_ctrlif_err(int irq, void *data)
 static irqreturn_t arizona_irq_thread(int irq, void *data)
 {
 	struct arizona *arizona = data;
-	int i, ret;
+	unsigned int val;
+	int ret;
 
 	ret = pm_runtime_get_sync(arizona->dev);
 	if (ret < 0) {
@@ -102,9 +103,20 @@ static irqreturn_t arizona_irq_thread(int irq, void *data)
 		return IRQ_NONE;
 	}
 
-	/* Check both domains */
-	for (i = 0; i < 2; i++)
-		handle_nested_irq(irq_find_mapping(arizona->virq, i));
+	/* Always handle the AoD domain */
+	handle_nested_irq(irq_find_mapping(arizona->virq, 0));
+
+	/*
+	 * Check if one of the main interrupts is asserted and only
+	 * check that domain if it is.
+	 */
+	ret = regmap_read(arizona->regmap, ARIZONA_IRQ_PIN_STATUS, &val);
+	if (ret == 0 && val & ARIZONA_IRQ1_STS) {
+		handle_nested_irq(irq_find_mapping(arizona->virq, 1));
+	} else if (ret != 0) {
+		dev_err(arizona->dev, "Failed to read main IRQ status: %d\n",
+			ret);
+	}
 
 	pm_runtime_mark_last_busy(arizona->dev);
 	pm_runtime_put_autosuspend(arizona->dev);
@@ -156,18 +168,36 @@ int arizona_irq_init(struct arizona *arizona)
 	int flags = IRQF_ONESHOT;
 	int ret, i;
 	const struct regmap_irq_chip *aod, *irq;
+	bool ctrlif_error = true;
 
 	switch (arizona->type) {
 #ifdef CONFIG_MFD_WM5102
 	case WM5102:
 		aod = &wm5102_aod;
 		irq = &wm5102_irq;
+
+		switch (arizona->rev) {
+		case 0:
+			ctrlif_error = false;
+			break;
+		default:
+			break;
+		}
 		break;
 #endif
 #ifdef CONFIG_MFD_WM5110
 	case WM5110:
 		aod = &wm5110_aod;
 		irq = &wm5110_irq;
+
+		switch (arizona->rev) {
+		case 0:
+		case 1:
+			ctrlif_error = false;
+			break;
+		default:
+			break;
+		}
 		break;
 #endif
 	default:
@@ -226,13 +256,17 @@ int arizona_irq_init(struct arizona *arizona)
 	}
 
 	/* Handle control interface errors in the core */
-	i = arizona_map_irq(arizona, ARIZONA_IRQ_CTRLIF_ERR);
-	ret = request_threaded_irq(i, NULL, arizona_ctrlif_err, IRQF_ONESHOT,
-				   "Control interface error", arizona);
-	if (ret != 0) {
-		dev_err(arizona->dev, "Failed to request boot done %d: %d\n",
-			arizona->irq, ret);
-		goto err_ctrlif;
+	if (ctrlif_error) {
+		i = arizona_map_irq(arizona, ARIZONA_IRQ_CTRLIF_ERR);
+		ret = request_threaded_irq(i, NULL, arizona_ctrlif_err,
+					   IRQF_ONESHOT,
+					   "Control interface error", arizona);
+		if (ret != 0) {
+			dev_err(arizona->dev,
+				"Failed to request CTRLIF_ERR %d: %d\n",
+				arizona->irq, ret);
+			goto err_ctrlif;
+		}
 	}
 
 	ret = request_threaded_irq(arizona->irq, NULL, arizona_irq_thread,
diff --git a/drivers/mfd/da9055-core.c b/drivers/mfd/da9055-core.c
new file mode 100644
index 00000000000..ff6c77f392b
--- /dev/null
+++ b/drivers/mfd/da9055-core.c
@@ -0,0 +1,423 @@
+/*
+ * Device access for Dialog DA9055 PMICs.
+ *
+ * Copyright(c) 2012 Dialog Semiconductor Ltd.
+ *
+ * Author: David Dajun Chen <dchen@diasemi.com>
+ *
+ *  This program is free software; you can redistribute  it and/or modify it
+ *  under  the terms of  the GNU General  Public License as published by the
+ *  Free Software Foundation;  either version 2 of the  License, or (at your
+ *  option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/input.h>
+#include <linux/irq.h>
+#include <linux/mutex.h>
+
+#include <linux/mfd/core.h>
+#include <linux/mfd/da9055/core.h>
+#include <linux/mfd/da9055/pdata.h>
+#include <linux/mfd/da9055/reg.h>
+
+#define DA9055_IRQ_NONKEY_MASK		0x01
+#define DA9055_IRQ_ALM_MASK		0x02
+#define DA9055_IRQ_TICK_MASK		0x04
+#define DA9055_IRQ_ADC_MASK		0x08
+#define DA9055_IRQ_BUCK_ILIM_MASK	0x08
+
+static bool da9055_register_readable(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case DA9055_REG_STATUS_A:
+	case DA9055_REG_STATUS_B:
+	case DA9055_REG_EVENT_A:
+	case DA9055_REG_EVENT_B:
+	case DA9055_REG_EVENT_C:
+	case DA9055_REG_IRQ_MASK_A:
+	case DA9055_REG_IRQ_MASK_B:
+	case DA9055_REG_IRQ_MASK_C:
+
+	case DA9055_REG_CONTROL_A:
+	case DA9055_REG_CONTROL_B:
+	case DA9055_REG_CONTROL_C:
+	case DA9055_REG_CONTROL_D:
+	case DA9055_REG_CONTROL_E:
+
+	case DA9055_REG_ADC_MAN:
+	case DA9055_REG_ADC_CONT:
+	case DA9055_REG_VSYS_MON:
+	case DA9055_REG_ADC_RES_L:
+	case DA9055_REG_ADC_RES_H:
+	case DA9055_REG_VSYS_RES:
+	case DA9055_REG_ADCIN1_RES:
+	case DA9055_REG_ADCIN2_RES:
+	case DA9055_REG_ADCIN3_RES:
+
+	case DA9055_REG_COUNT_S:
+	case DA9055_REG_COUNT_MI:
+	case DA9055_REG_COUNT_H:
+	case DA9055_REG_COUNT_D:
+	case DA9055_REG_COUNT_MO:
+	case DA9055_REG_COUNT_Y:
+	case DA9055_REG_ALARM_H:
+	case DA9055_REG_ALARM_D:
+	case DA9055_REG_ALARM_MI:
+	case DA9055_REG_ALARM_MO:
+	case DA9055_REG_ALARM_Y:
+
+	case DA9055_REG_GPIO0_1:
+	case DA9055_REG_GPIO2:
+	case DA9055_REG_GPIO_MODE0_2:
+
+	case DA9055_REG_BCORE_CONT:
+	case DA9055_REG_BMEM_CONT:
+	case DA9055_REG_LDO1_CONT:
+	case DA9055_REG_LDO2_CONT:
+	case DA9055_REG_LDO3_CONT:
+	case DA9055_REG_LDO4_CONT:
+	case DA9055_REG_LDO5_CONT:
+	case DA9055_REG_LDO6_CONT:
+	case DA9055_REG_BUCK_LIM:
+	case DA9055_REG_BCORE_MODE:
+	case DA9055_REG_VBCORE_A:
+	case DA9055_REG_VBMEM_A:
+	case DA9055_REG_VLDO1_A:
+	case DA9055_REG_VLDO2_A:
+	case DA9055_REG_VLDO3_A:
+	case DA9055_REG_VLDO4_A:
+	case DA9055_REG_VLDO5_A:
+	case DA9055_REG_VLDO6_A:
+	case DA9055_REG_VBCORE_B:
+	case DA9055_REG_VBMEM_B:
+	case DA9055_REG_VLDO1_B:
+	case DA9055_REG_VLDO2_B:
+	case DA9055_REG_VLDO3_B:
+	case DA9055_REG_VLDO4_B:
+	case DA9055_REG_VLDO5_B:
+	case DA9055_REG_VLDO6_B:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool da9055_register_writeable(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case DA9055_REG_STATUS_A:
+	case DA9055_REG_STATUS_B:
+	case DA9055_REG_EVENT_A:
+	case DA9055_REG_EVENT_B:
+	case DA9055_REG_EVENT_C:
+	case DA9055_REG_IRQ_MASK_A:
+	case DA9055_REG_IRQ_MASK_B:
+	case DA9055_REG_IRQ_MASK_C:
+
+	case DA9055_REG_CONTROL_A:
+	case DA9055_REG_CONTROL_B:
+	case DA9055_REG_CONTROL_C:
+	case DA9055_REG_CONTROL_D:
+	case DA9055_REG_CONTROL_E:
+
+	case DA9055_REG_ADC_MAN:
+	case DA9055_REG_ADC_CONT:
+	case DA9055_REG_VSYS_MON:
+	case DA9055_REG_ADC_RES_L:
+	case DA9055_REG_ADC_RES_H:
+	case DA9055_REG_VSYS_RES:
+	case DA9055_REG_ADCIN1_RES:
+	case DA9055_REG_ADCIN2_RES:
+	case DA9055_REG_ADCIN3_RES:
+
+	case DA9055_REG_COUNT_S:
+	case DA9055_REG_COUNT_MI:
+	case DA9055_REG_COUNT_H:
+	case DA9055_REG_COUNT_D:
+	case DA9055_REG_COUNT_MO:
+	case DA9055_REG_COUNT_Y:
+	case DA9055_REG_ALARM_H:
+	case DA9055_REG_ALARM_D:
+	case DA9055_REG_ALARM_MI:
+	case DA9055_REG_ALARM_MO:
+	case DA9055_REG_ALARM_Y:
+
+	case DA9055_REG_GPIO0_1:
+	case DA9055_REG_GPIO2:
+	case DA9055_REG_GPIO_MODE0_2:
+
+	case DA9055_REG_BCORE_CONT:
+	case DA9055_REG_BMEM_CONT:
+	case DA9055_REG_LDO1_CONT:
+	case DA9055_REG_LDO2_CONT:
+	case DA9055_REG_LDO3_CONT:
+	case DA9055_REG_LDO4_CONT:
+	case DA9055_REG_LDO5_CONT:
+	case DA9055_REG_LDO6_CONT:
+	case DA9055_REG_BUCK_LIM:
+	case DA9055_REG_BCORE_MODE:
+	case DA9055_REG_VBCORE_A:
+	case DA9055_REG_VBMEM_A:
+	case DA9055_REG_VLDO1_A:
+	case DA9055_REG_VLDO2_A:
+	case DA9055_REG_VLDO3_A:
+	case DA9055_REG_VLDO4_A:
+	case DA9055_REG_VLDO5_A:
+	case DA9055_REG_VLDO6_A:
+	case DA9055_REG_VBCORE_B:
+	case DA9055_REG_VBMEM_B:
+	case DA9055_REG_VLDO1_B:
+	case DA9055_REG_VLDO2_B:
+	case DA9055_REG_VLDO3_B:
+	case DA9055_REG_VLDO4_B:
+	case DA9055_REG_VLDO5_B:
+	case DA9055_REG_VLDO6_B:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool da9055_register_volatile(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case DA9055_REG_STATUS_A:
+	case DA9055_REG_STATUS_B:
+	case DA9055_REG_EVENT_A:
+	case DA9055_REG_EVENT_B:
+	case DA9055_REG_EVENT_C:
+
+	case DA9055_REG_CONTROL_A:
+	case DA9055_REG_CONTROL_E:
+
+	case DA9055_REG_ADC_MAN:
+	case DA9055_REG_ADC_RES_L:
+	case DA9055_REG_ADC_RES_H:
+	case DA9055_REG_VSYS_RES:
+	case DA9055_REG_ADCIN1_RES:
+	case DA9055_REG_ADCIN2_RES:
+	case DA9055_REG_ADCIN3_RES:
+
+	case DA9055_REG_COUNT_S:
+	case DA9055_REG_COUNT_MI:
+	case DA9055_REG_COUNT_H:
+	case DA9055_REG_COUNT_D:
+	case DA9055_REG_COUNT_MO:
+	case DA9055_REG_COUNT_Y:
+	case DA9055_REG_ALARM_MI:
+
+	case DA9055_REG_BCORE_CONT:
+	case DA9055_REG_BMEM_CONT:
+	case DA9055_REG_LDO1_CONT:
+	case DA9055_REG_LDO2_CONT:
+	case DA9055_REG_LDO3_CONT:
+	case DA9055_REG_LDO4_CONT:
+	case DA9055_REG_LDO5_CONT:
+	case DA9055_REG_LDO6_CONT:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static struct regmap_irq da9055_irqs[] = {
+	[DA9055_IRQ_NONKEY] = {
+		.reg_offset = 0,
+		.mask = DA9055_IRQ_NONKEY_MASK,
+	},
+	[DA9055_IRQ_ALARM] = {
+		.reg_offset = 0,
+		.mask = DA9055_IRQ_ALM_MASK,
+	},
+	[DA9055_IRQ_TICK] = {
+		.reg_offset = 0,
+		.mask = DA9055_IRQ_TICK_MASK,
+	},
+	[DA9055_IRQ_HWMON] = {
+		.reg_offset = 0,
+		.mask = DA9055_IRQ_ADC_MASK,
+	},
+	[DA9055_IRQ_REGULATOR] = {
+		.reg_offset = 1,
+		.mask = DA9055_IRQ_BUCK_ILIM_MASK,
+	},
+};
+
+struct regmap_config da9055_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.cache_type = REGCACHE_RBTREE,
+
+	.max_register = DA9055_MAX_REGISTER_CNT,
+	.readable_reg = da9055_register_readable,
+	.writeable_reg = da9055_register_writeable,
+	.volatile_reg = da9055_register_volatile,
+};
+EXPORT_SYMBOL_GPL(da9055_regmap_config);
+
+static struct resource da9055_onkey_resource = {
+	.name = "ONKEY",
+	.start = DA9055_IRQ_NONKEY,
+	.end   = DA9055_IRQ_NONKEY,
+	.flags = IORESOURCE_IRQ,
+};
+
+static struct resource da9055_rtc_resource[] = {
+	{
+		.name = "ALM",
+		.start = DA9055_IRQ_ALARM,
+		.end   = DA9055_IRQ_ALARM,
+		.flags = IORESOURCE_IRQ,
+	},
+	{
+		.name = "TICK",
+		.start = DA9055_IRQ_TICK,
+		.end   = DA9055_IRQ_TICK,
+		.flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct resource da9055_hwmon_resource = {
+	.name = "HWMON",
+	.start = DA9055_IRQ_HWMON,
+	.end   = DA9055_IRQ_HWMON,
+	.flags = IORESOURCE_IRQ,
+};
+
+static struct resource da9055_ld05_6_resource = {
+	.name = "REGULATOR",
+	.start = DA9055_IRQ_REGULATOR,
+	.end   = DA9055_IRQ_REGULATOR,
+	.flags = IORESOURCE_IRQ,
+};
+
+static struct mfd_cell da9055_devs[] = {
+	{
+		.of_compatible = "dialog,da9055-gpio",
+		.name = "da9055-gpio",
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 1,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 2,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 3,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 4,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 5,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 6,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.id = 7,
+		.resources = &da9055_ld05_6_resource,
+		.num_resources = 1,
+	},
+	{
+		.of_compatible = "dialog,da9055-regulator",
+		.name = "da9055-regulator",
+		.resources = &da9055_ld05_6_resource,
+		.num_resources = 1,
+		.id = 8,
+	},
+	{
+		.of_compatible = "dialog,da9055-onkey",
+		.name = "da9055-onkey",
+		.resources = &da9055_onkey_resource,
+		.num_resources = 1,
+	},
+	{
+		.of_compatible = "dialog,da9055-rtc",
+		.name = "da9055-rtc",
+		.resources = da9055_rtc_resource,
+		.num_resources = ARRAY_SIZE(da9055_rtc_resource),
+	},
+	{
+		.of_compatible = "dialog,da9055-hwmon",
+		.name = "da9055-hwmon",
+		.resources = &da9055_hwmon_resource,
+		.num_resources = 1,
+	},
+	{
+		.of_compatible = "dialog,da9055-watchdog",
+		.name = "da9055-watchdog",
+	},
+};
+
+static struct regmap_irq_chip da9055_regmap_irq_chip = {
+	.name = "da9055_irq",
+	.status_base = DA9055_REG_EVENT_A,
+	.mask_base = DA9055_REG_IRQ_MASK_A,
+	.ack_base = DA9055_REG_EVENT_A,
+	.num_regs = 3,
+	.irqs = da9055_irqs,
+	.num_irqs = ARRAY_SIZE(da9055_irqs),
+};
+
+int __devinit da9055_device_init(struct da9055 *da9055)
+{
+	struct da9055_pdata *pdata = da9055->dev->platform_data;
+	int ret;
+
+	if (pdata && pdata->init != NULL)
+		pdata->init(da9055);
+
+	if (!pdata || !pdata->irq_base)
+		da9055->irq_base = -1;
+	else
+		da9055->irq_base = pdata->irq_base;
+
+	ret = regmap_add_irq_chip(da9055->regmap, da9055->chip_irq,
+				  IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+				  da9055->irq_base, &da9055_regmap_irq_chip,
+				  &da9055->irq_data);
+	if (ret < 0)
+		return ret;
+
+	da9055->irq_base = regmap_irq_chip_get_base(da9055->irq_data);
+
+	ret = mfd_add_devices(da9055->dev, -1,
+			      da9055_devs, ARRAY_SIZE(da9055_devs),
+			      NULL, da9055->irq_base, NULL);
+	if (ret)
+		goto err;
+
+	return 0;
+
+err:
+	mfd_remove_devices(da9055->dev);
+	return ret;
+}
+
+void __devexit da9055_device_exit(struct da9055 *da9055)
+{
+	regmap_del_irq_chip(da9055->chip_irq, da9055->irq_data);
+	mfd_remove_devices(da9055->dev);
+}
+
+MODULE_DESCRIPTION("Core support for the DA9055 PMIC");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("David Dajun Chen <dchen@diasemi.com>");
diff --git a/drivers/mfd/da9055-i2c.c b/drivers/mfd/da9055-i2c.c
new file mode 100644
index 00000000000..88f6dca53ba
--- /dev/null
+++ b/drivers/mfd/da9055-i2c.c
@@ -0,0 +1,93 @@
+ /* I2C access for DA9055 PMICs.
+ *
+ * Copyright(c) 2012 Dialog Semiconductor Ltd.
+ *
+ * Author: David Dajun Chen <dchen@diasemi.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/i2c.h>
+#include <linux/err.h>
+
+#include <linux/mfd/da9055/core.h>
+
+static int __devinit da9055_i2c_probe(struct i2c_client *i2c,
+				      const struct i2c_device_id *id)
+{
+	struct da9055 *da9055;
+	int ret;
+
+	da9055 = devm_kzalloc(&i2c->dev, sizeof(struct da9055), GFP_KERNEL);
+	if (!da9055)
+		return -ENOMEM;
+
+	da9055->regmap = devm_regmap_init_i2c(i2c, &da9055_regmap_config);
+	if (IS_ERR(da9055->regmap)) {
+		ret = PTR_ERR(da9055->regmap);
+		dev_err(&i2c->dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	da9055->dev = &i2c->dev;
+	da9055->chip_irq = i2c->irq;
+
+	i2c_set_clientdata(i2c, da9055);
+
+	return da9055_device_init(da9055);
+}
+
+static int __devexit da9055_i2c_remove(struct i2c_client *i2c)
+{
+	struct da9055 *da9055 = i2c_get_clientdata(i2c);
+
+	da9055_device_exit(da9055);
+
+	return 0;
+}
+
+static struct i2c_device_id da9055_i2c_id[] = {
+	{"da9055-pmic", 0},
+	{ }
+};
+
+static struct i2c_driver da9055_i2c_driver = {
+	.probe = da9055_i2c_probe,
+	.remove = __devexit_p(da9055_i2c_remove),
+	.id_table = da9055_i2c_id,
+	.driver = {
+		.name = "da9055",
+		.owner = THIS_MODULE,
+	},
+};
+
+static int __init da9055_i2c_init(void)
+{
+	int ret;
+
+	ret = i2c_add_driver(&da9055_i2c_driver);
+	if (ret != 0) {
+		pr_err("DA9055 I2C registration failed %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+subsys_initcall(da9055_i2c_init);
+
+static void __exit da9055_i2c_exit(void)
+{
+	i2c_del_driver(&da9055_i2c_driver);
+}
+module_exit(da9055_i2c_exit);
+
+MODULE_AUTHOR("David Dajun Chen <dchen@diasemi.com>");
+MODULE_DESCRIPTION("I2C driver for Dialog DA9055 PMIC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c
index 6b67edbdbd0..00b8b0f3dfb 100644
--- a/drivers/mfd/db8500-prcmu.c
+++ b/drivers/mfd/db8500-prcmu.c
@@ -270,6 +270,8 @@ static struct {
 	struct prcmu_fw_version version;
 } fw_info;
 
+static struct irq_domain *db8500_irq_domain;
+
 /*
  * This vector maps irq numbers to the bits in the bit field used in
  * communication with the PRCMU firmware.
@@ -2624,7 +2626,7 @@ static void prcmu_irq_mask(struct irq_data *d)
 
 	spin_lock_irqsave(&mb0_transfer.dbb_irqs_lock, flags);
 
-	mb0_transfer.req.dbb_irqs &= ~prcmu_irq_bit[d->irq - IRQ_PRCMU_BASE];
+	mb0_transfer.req.dbb_irqs &= ~prcmu_irq_bit[d->hwirq];
 
 	spin_unlock_irqrestore(&mb0_transfer.dbb_irqs_lock, flags);
 
@@ -2638,7 +2640,7 @@ static void prcmu_irq_unmask(struct irq_data *d)
 
 	spin_lock_irqsave(&mb0_transfer.dbb_irqs_lock, flags);
 
-	mb0_transfer.req.dbb_irqs |= prcmu_irq_bit[d->irq - IRQ_PRCMU_BASE];
+	mb0_transfer.req.dbb_irqs |= prcmu_irq_bit[d->hwirq];
 
 	spin_unlock_irqrestore(&mb0_transfer.dbb_irqs_lock, flags);
 
@@ -2678,9 +2680,37 @@ static char *fw_project_name(u8 project)
 	}
 }
 
+static int db8500_irq_map(struct irq_domain *d, unsigned int virq,
+				irq_hw_number_t hwirq)
+{
+	irq_set_chip_and_handler(virq, &prcmu_irq_chip,
+				handle_simple_irq);
+	set_irq_flags(virq, IRQF_VALID);
+
+	return 0;
+}
+
+static struct irq_domain_ops db8500_irq_ops = {
+        .map    = db8500_irq_map,
+        .xlate  = irq_domain_xlate_twocell,
+};
+
+static int db8500_irq_init(struct device_node *np)
+{
+	db8500_irq_domain = irq_domain_add_legacy(
+		np, NUM_PRCMU_WAKEUPS, IRQ_PRCMU_BASE,
+		0, &db8500_irq_ops, NULL);
+
+	if (!db8500_irq_domain) {
+		pr_err("Failed to create irqdomain\n");
+		return -ENOSYS;
+	}
+
+	return 0;
+}
+
 void __init db8500_prcmu_early_init(void)
 {
-	unsigned int i;
 	if (cpu_is_u8500v2()) {
 		void *tcpm_base = ioremap_nocache(U8500_PRCMU_TCPM_BASE, SZ_4K);
 
@@ -2725,15 +2755,6 @@ void __init db8500_prcmu_early_init(void)
 
 	INIT_WORK(&mb0_transfer.mask_work, prcmu_mask_work);
 
-	/* Initalize irqs. */
-	for (i = 0; i < NUM_PRCMU_WAKEUPS; i++) {
-		unsigned int irq;
-
-		irq = IRQ_PRCMU_BASE + i;
-		irq_set_chip_and_handler(irq, &prcmu_irq_chip,
-					 handle_simple_irq);
-		set_irq_flags(irq, IRQF_VALID);
-	}
 	compute_armss_rate();
 }
 
@@ -3041,6 +3062,8 @@ static int __devinit db8500_prcmu_probe(struct platform_device *pdev)
 		goto no_irq_return;
 	}
 
+	db8500_irq_init(np);
+
 	for (i = 0; i < ARRAY_SIZE(db8500_prcmu_devs); i++) {
 		if (!strcmp(db8500_prcmu_devs[i].name, "ab8500-core")) {
 			db8500_prcmu_devs[i].platform_data = ab8500_platdata;
diff --git a/drivers/mfd/lp8788-irq.c b/drivers/mfd/lp8788-irq.c
new file mode 100644
index 00000000000..c84ded5f8ec
--- /dev/null
+++ b/drivers/mfd/lp8788-irq.c
@@ -0,0 +1,198 @@
+/*
+ * TI LP8788 MFD - interrupt handler
+ *
+ * Copyright 2012 Texas Instruments
+ *
+ * Author: Milo(Woogyom) Kim <milo.kim@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/device.h>
+#include <linux/mfd/lp8788.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+/* register address */
+#define LP8788_INT_1			0x00
+#define LP8788_INTEN_1			0x03
+
+#define BASE_INTEN_ADDR			LP8788_INTEN_1
+#define SIZE_REG			8
+#define NUM_REGS			3
+
+/*
+ * struct lp8788_irq_data
+ * @lp               : used for accessing to lp8788 registers
+ * @irq_lock         : mutex for enabling/disabling the interrupt
+ * @domain           : IRQ domain for handling nested interrupt
+ * @enabled          : status of enabled interrupt
+ */
+struct lp8788_irq_data {
+	struct lp8788 *lp;
+	struct mutex irq_lock;
+	struct irq_domain *domain;
+	int enabled[LP8788_INT_MAX];
+};
+
+static inline u8 _irq_to_addr(enum lp8788_int_id id)
+{
+	return id / SIZE_REG;
+}
+
+static inline u8 _irq_to_enable_addr(enum lp8788_int_id id)
+{
+	return _irq_to_addr(id) + BASE_INTEN_ADDR;
+}
+
+static inline u8 _irq_to_mask(enum lp8788_int_id id)
+{
+	return 1 << (id % SIZE_REG);
+}
+
+static inline u8 _irq_to_val(enum lp8788_int_id id, int enable)
+{
+	return enable << (id % SIZE_REG);
+}
+
+static void lp8788_irq_enable(struct irq_data *data)
+{
+	struct lp8788_irq_data *irqd = irq_data_get_irq_chip_data(data);
+	irqd->enabled[data->hwirq] = 1;
+}
+
+static void lp8788_irq_disable(struct irq_data *data)
+{
+	struct lp8788_irq_data *irqd = irq_data_get_irq_chip_data(data);
+	irqd->enabled[data->hwirq] = 0;
+}
+
+static void lp8788_irq_bus_lock(struct irq_data *data)
+{
+	struct lp8788_irq_data *irqd = irq_data_get_irq_chip_data(data);
+
+	mutex_lock(&irqd->irq_lock);
+}
+
+static void lp8788_irq_bus_sync_unlock(struct irq_data *data)
+{
+	struct lp8788_irq_data *irqd = irq_data_get_irq_chip_data(data);
+	enum lp8788_int_id irq = data->hwirq;
+	u8 addr, mask, val;
+
+	addr = _irq_to_enable_addr(irq);
+	mask = _irq_to_mask(irq);
+	val = _irq_to_val(irq, irqd->enabled[irq]);
+
+	lp8788_update_bits(irqd->lp, addr, mask, val);
+
+	mutex_unlock(&irqd->irq_lock);
+}
+
+static struct irq_chip lp8788_irq_chip = {
+	.name			= "lp8788",
+	.irq_enable		= lp8788_irq_enable,
+	.irq_disable		= lp8788_irq_disable,
+	.irq_bus_lock		= lp8788_irq_bus_lock,
+	.irq_bus_sync_unlock	= lp8788_irq_bus_sync_unlock,
+};
+
+static irqreturn_t lp8788_irq_handler(int irq, void *ptr)
+{
+	struct lp8788_irq_data *irqd = ptr;
+	struct lp8788 *lp = irqd->lp;
+	u8 status[NUM_REGS], addr, mask;
+	bool handled;
+	int i;
+
+	if (lp8788_read_multi_bytes(lp, LP8788_INT_1, status, NUM_REGS))
+		return IRQ_NONE;
+
+	for (i = 0 ; i < LP8788_INT_MAX ; i++) {
+		addr = _irq_to_addr(i);
+		mask = _irq_to_mask(i);
+
+		/* reporting only if the irq is enabled */
+		if (status[addr] & mask) {
+			handle_nested_irq(irq_find_mapping(irqd->domain, i));
+			handled = true;
+		}
+	}
+
+	return handled ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static int lp8788_irq_map(struct irq_domain *d, unsigned int virq,
+			irq_hw_number_t hwirq)
+{
+	struct lp8788_irq_data *irqd = d->host_data;
+	struct irq_chip *chip = &lp8788_irq_chip;
+
+	irq_set_chip_data(virq, irqd);
+	irq_set_chip_and_handler(virq, chip, handle_edge_irq);
+	irq_set_nested_thread(virq, 1);
+
+#ifdef CONFIG_ARM
+	set_irq_flags(virq, IRQF_VALID);
+#else
+	irq_set_noprobe(virq);
+#endif
+
+	return 0;
+}
+
+static struct irq_domain_ops lp8788_domain_ops = {
+	.map = lp8788_irq_map,
+};
+
+int lp8788_irq_init(struct lp8788 *lp, int irq)
+{
+	struct lp8788_irq_data *irqd;
+	int ret;
+
+	if (irq <= 0) {
+		dev_warn(lp->dev, "invalid irq number: %d\n", irq);
+		return 0;
+	}
+
+	irqd = devm_kzalloc(lp->dev, sizeof(*irqd), GFP_KERNEL);
+	if (!irqd)
+		return -ENOMEM;
+
+	irqd->lp = lp;
+	irqd->domain = irq_domain_add_linear(lp->dev->of_node, LP8788_INT_MAX,
+					&lp8788_domain_ops, irqd);
+	if (!irqd->domain) {
+		dev_err(lp->dev, "failed to add irq domain err\n");
+		return -EINVAL;
+	}
+
+	lp->irqdm = irqd->domain;
+	mutex_init(&irqd->irq_lock);
+
+	ret = request_threaded_irq(irq, NULL, lp8788_irq_handler,
+				IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+				"lp8788-irq", irqd);
+	if (ret) {
+		dev_err(lp->dev, "failed to create a thread for IRQ_N\n");
+		return ret;
+	}
+
+	lp->irq = irq;
+
+	return 0;
+}
+
+void lp8788_irq_exit(struct lp8788 *lp)
+{
+	if (lp->irq)
+		free_irq(lp->irq, lp->irqdm);
+}
diff --git a/drivers/mfd/lp8788.c b/drivers/mfd/lp8788.c
new file mode 100644
index 00000000000..3e94a699833
--- /dev/null
+++ b/drivers/mfd/lp8788.c
@@ -0,0 +1,245 @@
+/*
+ * TI LP8788 MFD - core interface
+ *
+ * Copyright 2012 Texas Instruments
+ *
+ * Author: Milo(Woogyom) Kim <milo.kim@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/lp8788.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#define MAX_LP8788_REGISTERS		0xA2
+
+#define MFD_DEV_SIMPLE(_name)					\
+{								\
+	.name = LP8788_DEV_##_name,				\
+}
+
+#define MFD_DEV_WITH_ID(_name, _id)				\
+{								\
+	.name = LP8788_DEV_##_name,				\
+	.id = _id,						\
+}
+
+#define MFD_DEV_WITH_RESOURCE(_name, _resource, num_resource)	\
+{								\
+	.name = LP8788_DEV_##_name,				\
+	.resources = _resource,					\
+	.num_resources = num_resource,				\
+}
+
+static struct resource chg_irqs[] = {
+	/* Charger Interrupts */
+	{
+		.start = LP8788_INT_CHG_INPUT_STATE,
+		.end   = LP8788_INT_PRECHG_TIMEOUT,
+		.name  = LP8788_CHG_IRQ,
+		.flags = IORESOURCE_IRQ,
+	},
+	/* Power Routing Switch Interrupts */
+	{
+		.start = LP8788_INT_ENTER_SYS_SUPPORT,
+		.end   = LP8788_INT_EXIT_SYS_SUPPORT,
+		.name  = LP8788_PRSW_IRQ,
+		.flags = IORESOURCE_IRQ,
+	},
+	/* Battery Interrupts */
+	{
+		.start = LP8788_INT_BATT_LOW,
+		.end   = LP8788_INT_NO_BATT,
+		.name  = LP8788_BATT_IRQ,
+		.flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct resource rtc_irqs[] = {
+	{
+		.start = LP8788_INT_RTC_ALARM1,
+		.end   = LP8788_INT_RTC_ALARM2,
+		.name  = LP8788_ALM_IRQ,
+		.flags = IORESOURCE_IRQ,
+	},
+};
+
+static struct mfd_cell lp8788_devs[] = {
+	/* 4 bucks */
+	MFD_DEV_WITH_ID(BUCK, 1),
+	MFD_DEV_WITH_ID(BUCK, 2),
+	MFD_DEV_WITH_ID(BUCK, 3),
+	MFD_DEV_WITH_ID(BUCK, 4),
+
+	/* 12 digital ldos */
+	MFD_DEV_WITH_ID(DLDO, 1),
+	MFD_DEV_WITH_ID(DLDO, 2),
+	MFD_DEV_WITH_ID(DLDO, 3),
+	MFD_DEV_WITH_ID(DLDO, 4),
+	MFD_DEV_WITH_ID(DLDO, 5),
+	MFD_DEV_WITH_ID(DLDO, 6),
+	MFD_DEV_WITH_ID(DLDO, 7),
+	MFD_DEV_WITH_ID(DLDO, 8),
+	MFD_DEV_WITH_ID(DLDO, 9),
+	MFD_DEV_WITH_ID(DLDO, 10),
+	MFD_DEV_WITH_ID(DLDO, 11),
+	MFD_DEV_WITH_ID(DLDO, 12),
+
+	/* 10 analog ldos */
+	MFD_DEV_WITH_ID(ALDO, 1),
+	MFD_DEV_WITH_ID(ALDO, 2),
+	MFD_DEV_WITH_ID(ALDO, 3),
+	MFD_DEV_WITH_ID(ALDO, 4),
+	MFD_DEV_WITH_ID(ALDO, 5),
+	MFD_DEV_WITH_ID(ALDO, 6),
+	MFD_DEV_WITH_ID(ALDO, 7),
+	MFD_DEV_WITH_ID(ALDO, 8),
+	MFD_DEV_WITH_ID(ALDO, 9),
+	MFD_DEV_WITH_ID(ALDO, 10),
+
+	/* ADC */
+	MFD_DEV_SIMPLE(ADC),
+
+	/* battery charger */
+	MFD_DEV_WITH_RESOURCE(CHARGER, chg_irqs, ARRAY_SIZE(chg_irqs)),
+
+	/* rtc */
+	MFD_DEV_WITH_RESOURCE(RTC, rtc_irqs, ARRAY_SIZE(rtc_irqs)),
+
+	/* backlight */
+	MFD_DEV_SIMPLE(BACKLIGHT),
+
+	/* current sink for vibrator */
+	MFD_DEV_SIMPLE(VIBRATOR),
+
+	/* current sink for keypad LED */
+	MFD_DEV_SIMPLE(KEYLED),
+};
+
+int lp8788_read_byte(struct lp8788 *lp, u8 reg, u8 *data)
+{
+	int ret;
+	unsigned int val;
+
+	ret = regmap_read(lp->regmap, reg, &val);
+	if (ret < 0) {
+		dev_err(lp->dev, "failed to read 0x%.2x\n", reg);
+		return ret;
+	}
+
+	*data = (u8)val;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(lp8788_read_byte);
+
+int lp8788_read_multi_bytes(struct lp8788 *lp, u8 reg, u8 *data, size_t count)
+{
+	return regmap_bulk_read(lp->regmap, reg, data, count);
+}
+EXPORT_SYMBOL_GPL(lp8788_read_multi_bytes);
+
+int lp8788_write_byte(struct lp8788 *lp, u8 reg, u8 data)
+{
+	return regmap_write(lp->regmap, reg, data);
+}
+EXPORT_SYMBOL_GPL(lp8788_write_byte);
+
+int lp8788_update_bits(struct lp8788 *lp, u8 reg, u8 mask, u8 data)
+{
+	return regmap_update_bits(lp->regmap, reg, mask, data);
+}
+EXPORT_SYMBOL_GPL(lp8788_update_bits);
+
+static int lp8788_platform_init(struct lp8788 *lp)
+{
+	struct lp8788_platform_data *pdata = lp->pdata;
+
+	return (pdata && pdata->init_func) ? pdata->init_func(lp) : 0;
+}
+
+static const struct regmap_config lp8788_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MAX_LP8788_REGISTERS,
+};
+
+static int lp8788_probe(struct i2c_client *cl, const struct i2c_device_id *id)
+{
+	struct lp8788 *lp;
+	struct lp8788_platform_data *pdata = cl->dev.platform_data;
+	int ret;
+
+	lp = devm_kzalloc(&cl->dev, sizeof(struct lp8788), GFP_KERNEL);
+	if (!lp)
+		return -ENOMEM;
+
+	lp->regmap = devm_regmap_init_i2c(cl, &lp8788_regmap_config);
+	if (IS_ERR(lp->regmap)) {
+		ret = PTR_ERR(lp->regmap);
+		dev_err(&cl->dev, "regmap init i2c err: %d\n", ret);
+		return ret;
+	}
+
+	lp->pdata = pdata;
+	lp->dev = &cl->dev;
+	i2c_set_clientdata(cl, lp);
+
+	ret = lp8788_platform_init(lp);
+	if (ret)
+		return ret;
+
+	ret = lp8788_irq_init(lp, cl->irq);
+	if (ret)
+		return ret;
+
+	return mfd_add_devices(lp->dev, -1, lp8788_devs,
+			       ARRAY_SIZE(lp8788_devs), NULL, 0, NULL);
+}
+
+static int __devexit lp8788_remove(struct i2c_client *cl)
+{
+	struct lp8788 *lp = i2c_get_clientdata(cl);
+
+	mfd_remove_devices(lp->dev);
+	lp8788_irq_exit(lp);
+	return 0;
+}
+
+static const struct i2c_device_id lp8788_ids[] = {
+	{"lp8788", 0},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, lp8788_ids);
+
+static struct i2c_driver lp8788_driver = {
+	.driver = {
+		.name = "lp8788",
+		.owner = THIS_MODULE,
+	},
+	.probe = lp8788_probe,
+	.remove = __devexit_p(lp8788_remove),
+	.id_table = lp8788_ids,
+};
+
+static int __init lp8788_init(void)
+{
+	return i2c_add_driver(&lp8788_driver);
+}
+subsys_initcall(lp8788_init);
+
+static void __exit lp8788_exit(void)
+{
+	i2c_del_driver(&lp8788_driver);
+}
+module_exit(lp8788_exit);
+
+MODULE_DESCRIPTION("TI LP8788 MFD Driver");
+MODULE_AUTHOR("Milo Kim");
+MODULE_LICENSE("GPL");
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index 092ad4b44b6..a22544fe531 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -49,6 +49,7 @@
  *	document number TBD : DH89xxCC
  *	document number TBD : Panther Point
  *	document number TBD : Lynx Point
+ *	document number TBD : Lynx Point-LP
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -192,6 +193,7 @@ enum lpc_chipsets {
 	LPC_DH89XXCC,	/* DH89xxCC */
 	LPC_PPT,	/* Panther Point */
 	LPC_LPT,	/* Lynx Point */
+	LPC_LPT_LP,	/* Lynx Point-LP */
 };
 
 struct lpc_ich_info lpc_chipset_info[] __devinitdata = {
@@ -468,6 +470,10 @@ struct lpc_ich_info lpc_chipset_info[] __devinitdata = {
 		.name = "Lynx Point",
 		.iTCO_version = 2,
 	},
+	[LPC_LPT_LP] = {
+		.name = "Lynx Point_LP",
+		.iTCO_version = 2,
+	},
 };
 
 /*
@@ -641,6 +647,14 @@ static DEFINE_PCI_DEVICE_TABLE(lpc_ich_ids) = {
 	{ PCI_VDEVICE(INTEL, 0x8c5d), LPC_LPT},
 	{ PCI_VDEVICE(INTEL, 0x8c5e), LPC_LPT},
 	{ PCI_VDEVICE(INTEL, 0x8c5f), LPC_LPT},
+	{ PCI_VDEVICE(INTEL, 0x9c40), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c41), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c42), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c43), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c44), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c45), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c46), LPC_LPT_LP},
+	{ PCI_VDEVICE(INTEL, 0x9c47), LPC_LPT_LP},
 	{ 0, },			/* End of list */
 };
 MODULE_DEVICE_TABLE(pci, lpc_ich_ids);
@@ -683,6 +697,30 @@ static void __devinit lpc_ich_finalize_cell(struct mfd_cell *cell,
 	cell->pdata_size = sizeof(struct lpc_ich_info);
 }
 
+/*
+ * We don't check for resource conflict globally. There are 2 or 3 independent
+ * GPIO groups and it's enough to have access to one of these to instantiate
+ * the device.
+ */
+static int __devinit lpc_ich_check_conflict_gpio(struct resource *res)
+{
+	int ret;
+	u8 use_gpio = 0;
+
+	if (resource_size(res) >= 0x50 &&
+	    !acpi_check_region(res->start + 0x40, 0x10, "LPC ICH GPIO3"))
+		use_gpio |= 1 << 2;
+
+	if (!acpi_check_region(res->start + 0x30, 0x10, "LPC ICH GPIO2"))
+		use_gpio |= 1 << 1;
+
+	ret = acpi_check_region(res->start + 0x00, 0x30, "LPC ICH GPIO1");
+	if (!ret)
+		use_gpio |= 1 << 0;
+
+	return use_gpio ? use_gpio : ret;
+}
+
 static int __devinit lpc_ich_init_gpio(struct pci_dev *dev,
 				const struct pci_device_id *id)
 {
@@ -740,12 +778,13 @@ gpe0_done:
 		break;
 	}
 
-	ret = acpi_check_resource_conflict(res);
-	if (ret) {
+	ret = lpc_ich_check_conflict_gpio(res);
+	if (ret < 0) {
 		/* this isn't necessarily fatal for the GPIO */
 		acpi_conflict = true;
 		goto gpio_done;
 	}
+	lpc_chipset_info[id->driver_data].use_gpio = ret;
 	lpc_ich_enable_gpio_space(dev);
 
 	lpc_ich_finalize_cell(&lpc_ich_cells[LPC_GPIO], id);
diff --git a/drivers/mfd/max8907.c b/drivers/mfd/max8907.c
new file mode 100644
index 00000000000..17f2593d82b
--- /dev/null
+++ b/drivers/mfd/max8907.c
@@ -0,0 +1,351 @@
+/*
+ * max8907.c - mfd driver for MAX8907
+ *
+ * Copyright (C) 2010 Gyungoh Yoo <jack.yoo@maxim-ic.com>
+ * Copyright (C) 2010-2012, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/max8907.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+static struct mfd_cell max8907_cells[] = {
+	{ .name = "max8907-regulator", },
+	{ .name = "max8907-rtc", },
+};
+
+static bool max8907_gen_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MAX8907_REG_ON_OFF_IRQ1:
+	case MAX8907_REG_ON_OFF_STAT:
+	case MAX8907_REG_ON_OFF_IRQ2:
+	case MAX8907_REG_CHG_IRQ1:
+	case MAX8907_REG_CHG_IRQ2:
+	case MAX8907_REG_CHG_STAT:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool max8907_gen_is_precious_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MAX8907_REG_ON_OFF_IRQ1:
+	case MAX8907_REG_ON_OFF_IRQ2:
+	case MAX8907_REG_CHG_IRQ1:
+	case MAX8907_REG_CHG_IRQ2:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool max8907_gen_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return !max8907_gen_is_volatile_reg(dev, reg);
+}
+
+static const struct regmap_config max8907_regmap_gen_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.volatile_reg = max8907_gen_is_volatile_reg,
+	.precious_reg = max8907_gen_is_precious_reg,
+	.writeable_reg = max8907_gen_is_writeable_reg,
+	.max_register = MAX8907_REG_LDO20VOUT,
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static bool max8907_rtc_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	if (reg <= MAX8907_REG_RTC_YEAR2)
+		return true;
+
+	switch (reg) {
+	case MAX8907_REG_RTC_STATUS:
+	case MAX8907_REG_RTC_IRQ:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool max8907_rtc_is_precious_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MAX8907_REG_RTC_IRQ:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool max8907_rtc_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MAX8907_REG_RTC_STATUS:
+	case MAX8907_REG_RTC_IRQ:
+		return false;
+	default:
+		return true;
+	}
+}
+
+static const struct regmap_config max8907_regmap_rtc_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.volatile_reg = max8907_rtc_is_volatile_reg,
+	.precious_reg = max8907_rtc_is_precious_reg,
+	.writeable_reg = max8907_rtc_is_writeable_reg,
+	.max_register = MAX8907_REG_MPL_CNTL,
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static const struct regmap_irq max8907_chg_irqs[] = {
+	{ .reg_offset = 0, .mask = 1 << 0, },
+	{ .reg_offset = 0, .mask = 1 << 1, },
+	{ .reg_offset = 0, .mask = 1 << 2, },
+	{ .reg_offset = 1, .mask = 1 << 0, },
+	{ .reg_offset = 1, .mask = 1 << 1, },
+	{ .reg_offset = 1, .mask = 1 << 2, },
+	{ .reg_offset = 1, .mask = 1 << 3, },
+	{ .reg_offset = 1, .mask = 1 << 4, },
+	{ .reg_offset = 1, .mask = 1 << 5, },
+	{ .reg_offset = 1, .mask = 1 << 6, },
+	{ .reg_offset = 1, .mask = 1 << 7, },
+};
+
+static const struct regmap_irq_chip max8907_chg_irq_chip = {
+	.name = "max8907 chg",
+	.status_base = MAX8907_REG_CHG_IRQ1,
+	.mask_base = MAX8907_REG_CHG_IRQ1_MASK,
+	.wake_base = MAX8907_REG_CHG_IRQ1_MASK,
+	.irq_reg_stride = MAX8907_REG_CHG_IRQ2 - MAX8907_REG_CHG_IRQ1,
+	.num_regs = 2,
+	.irqs = max8907_chg_irqs,
+	.num_irqs = ARRAY_SIZE(max8907_chg_irqs),
+};
+
+static const struct regmap_irq max8907_on_off_irqs[] = {
+	{ .reg_offset = 0, .mask = 1 << 0, },
+	{ .reg_offset = 0, .mask = 1 << 1, },
+	{ .reg_offset = 0, .mask = 1 << 2, },
+	{ .reg_offset = 0, .mask = 1 << 3, },
+	{ .reg_offset = 0, .mask = 1 << 4, },
+	{ .reg_offset = 0, .mask = 1 << 5, },
+	{ .reg_offset = 0, .mask = 1 << 6, },
+	{ .reg_offset = 0, .mask = 1 << 7, },
+	{ .reg_offset = 1, .mask = 1 << 0, },
+	{ .reg_offset = 1, .mask = 1 << 1, },
+};
+
+static const struct regmap_irq_chip max8907_on_off_irq_chip = {
+	.name = "max8907 on_off",
+	.status_base = MAX8907_REG_ON_OFF_IRQ1,
+	.mask_base = MAX8907_REG_ON_OFF_IRQ1_MASK,
+	.irq_reg_stride = MAX8907_REG_ON_OFF_IRQ2 - MAX8907_REG_ON_OFF_IRQ1,
+	.num_regs = 2,
+	.irqs = max8907_on_off_irqs,
+	.num_irqs = ARRAY_SIZE(max8907_on_off_irqs),
+};
+
+static const struct regmap_irq max8907_rtc_irqs[] = {
+	{ .reg_offset = 0, .mask = 1 << 2, },
+	{ .reg_offset = 0, .mask = 1 << 3, },
+};
+
+static const struct regmap_irq_chip max8907_rtc_irq_chip = {
+	.name = "max8907 rtc",
+	.status_base = MAX8907_REG_RTC_IRQ,
+	.mask_base = MAX8907_REG_RTC_IRQ_MASK,
+	.num_regs = 1,
+	.irqs = max8907_rtc_irqs,
+	.num_irqs = ARRAY_SIZE(max8907_rtc_irqs),
+};
+
+static struct max8907 *max8907_pm_off;
+static void max8907_power_off(void)
+{
+	regmap_update_bits(max8907_pm_off->regmap_gen, MAX8907_REG_RESET_CNFG,
+			MAX8907_MASK_POWER_OFF, MAX8907_MASK_POWER_OFF);
+}
+
+static __devinit int max8907_i2c_probe(struct i2c_client *i2c,
+				       const struct i2c_device_id *id)
+{
+	struct max8907 *max8907;
+	int ret;
+	struct max8907_platform_data *pdata = dev_get_platdata(&i2c->dev);
+	bool pm_off = false;
+
+	if (pdata)
+		pm_off = pdata->pm_off;
+	else if (i2c->dev.of_node)
+		pm_off = of_property_read_bool(i2c->dev.of_node,
+					"maxim,system-power-controller");
+
+	max8907 = devm_kzalloc(&i2c->dev, sizeof(struct max8907), GFP_KERNEL);
+	if (!max8907) {
+		ret = -ENOMEM;
+		goto err_alloc_drvdata;
+	}
+
+	max8907->dev = &i2c->dev;
+	dev_set_drvdata(max8907->dev, max8907);
+
+	max8907->i2c_gen = i2c;
+	i2c_set_clientdata(i2c, max8907);
+	max8907->regmap_gen = devm_regmap_init_i2c(i2c,
+						&max8907_regmap_gen_config);
+	if (IS_ERR(max8907->regmap_gen)) {
+		ret = PTR_ERR(max8907->regmap_gen);
+		dev_err(&i2c->dev, "gen regmap init failed: %d\n", ret);
+		goto err_regmap_gen;
+	}
+
+	max8907->i2c_rtc = i2c_new_dummy(i2c->adapter, MAX8907_RTC_I2C_ADDR);
+	if (!max8907->i2c_rtc) {
+		ret = -ENOMEM;
+		goto err_dummy_rtc;
+	}
+	i2c_set_clientdata(max8907->i2c_rtc, max8907);
+	max8907->regmap_rtc = devm_regmap_init_i2c(max8907->i2c_rtc,
+						&max8907_regmap_rtc_config);
+	if (IS_ERR(max8907->regmap_rtc)) {
+		ret = PTR_ERR(max8907->regmap_rtc);
+		dev_err(&i2c->dev, "rtc regmap init failed: %d\n", ret);
+		goto err_regmap_rtc;
+	}
+
+	irq_set_status_flags(max8907->i2c_gen->irq, IRQ_NOAUTOEN);
+
+	ret = regmap_add_irq_chip(max8907->regmap_gen, max8907->i2c_gen->irq,
+				  IRQF_ONESHOT | IRQF_SHARED, -1,
+				  &max8907_chg_irq_chip,
+				  &max8907->irqc_chg);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "failed to add chg irq chip: %d\n", ret);
+		goto err_irqc_chg;
+	}
+	ret = regmap_add_irq_chip(max8907->regmap_gen, max8907->i2c_gen->irq,
+				  IRQF_ONESHOT | IRQF_SHARED, -1,
+				  &max8907_on_off_irq_chip,
+				  &max8907->irqc_on_off);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "failed to add on off irq chip: %d\n", ret);
+		goto err_irqc_on_off;
+	}
+	ret = regmap_add_irq_chip(max8907->regmap_rtc, max8907->i2c_gen->irq,
+				  IRQF_ONESHOT | IRQF_SHARED, -1,
+				  &max8907_rtc_irq_chip,
+				  &max8907->irqc_rtc);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "failed to add rtc irq chip: %d\n", ret);
+		goto err_irqc_rtc;
+	}
+
+	enable_irq(max8907->i2c_gen->irq);
+
+	ret = mfd_add_devices(max8907->dev, -1, max8907_cells,
+			      ARRAY_SIZE(max8907_cells), NULL, 0, NULL);
+	if (ret != 0) {
+		dev_err(&i2c->dev, "failed to add MFD devices %d\n", ret);
+		goto err_add_devices;
+	}
+
+	if (pm_off && !pm_power_off) {
+		max8907_pm_off = max8907;
+		pm_power_off = max8907_power_off;
+	}
+
+	return 0;
+
+err_add_devices:
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_rtc);
+err_irqc_rtc:
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_on_off);
+err_irqc_on_off:
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_chg);
+err_irqc_chg:
+err_regmap_rtc:
+	i2c_unregister_device(max8907->i2c_rtc);
+err_dummy_rtc:
+err_regmap_gen:
+err_alloc_drvdata:
+	return ret;
+}
+
+static __devexit int max8907_i2c_remove(struct i2c_client *i2c)
+{
+	struct max8907 *max8907 = i2c_get_clientdata(i2c);
+
+	mfd_remove_devices(max8907->dev);
+
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_rtc);
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_on_off);
+	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_chg);
+
+	i2c_unregister_device(max8907->i2c_rtc);
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static struct of_device_id max8907_of_match[] = {
+	{ .compatible = "maxim,max8907" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, max8907_of_match);
+#endif
+
+static const struct i2c_device_id max8907_i2c_id[] = {
+	{"max8907", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, max8907_i2c_id);
+
+static struct i2c_driver max8907_i2c_driver = {
+	.driver = {
+		.name = "max8907",
+		.owner = THIS_MODULE,
+		.of_match_table = of_match_ptr(max8907_of_match),
+	},
+	.probe = max8907_i2c_probe,
+	.remove = max8907_i2c_remove,
+	.id_table = max8907_i2c_id,
+};
+
+static int __init max8907_i2c_init(void)
+{
+	int ret = -ENODEV;
+
+	ret = i2c_add_driver(&max8907_i2c_driver);
+	if (ret != 0)
+		pr_err("Failed to register I2C driver: %d\n", ret);
+
+	return ret;
+}
+subsys_initcall(max8907_i2c_init);
+
+static void __exit max8907_i2c_exit(void)
+{
+	i2c_del_driver(&max8907_i2c_driver);
+}
+module_exit(max8907_i2c_exit);
+
+MODULE_DESCRIPTION("MAX8907 multi-function core driver");
+MODULE_AUTHOR("Gyungoh Yoo <jack.yoo@maxim-ic.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/max8925-core.c b/drivers/mfd/max8925-core.c
index ee53757beca..9f54c04912f 100644
--- a/drivers/mfd/max8925-core.c
+++ b/drivers/mfd/max8925-core.c
@@ -15,23 +15,20 @@
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/regulator/machine.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/max8925.h>
 
-static struct resource backlight_resources[] = {
-	{
-		.name	= "max8925-backlight",
-		.start	= MAX8925_WLED_MODE_CNTL,
-		.end	= MAX8925_WLED_CNTL,
-		.flags	= IORESOURCE_IO,
-	},
+static struct resource bk_resources[] __devinitdata = {
+	{ 0x84, 0x84, "mode control", IORESOURCE_REG, },
+	{ 0x85, 0x85, "control",      IORESOURCE_REG, },
 };
 
-static struct mfd_cell backlight_devs[] = {
+static struct mfd_cell bk_devs[] __devinitdata = {
 	{
 		.name		= "max8925-backlight",
-		.num_resources	= 1,
-		.resources	= &backlight_resources[0],
+		.num_resources	= ARRAY_SIZE(bk_resources),
+		.resources	= &bk_resources[0],
 		.id		= -1,
 	},
 };
@@ -41,7 +38,7 @@ static struct resource touch_resources[] = {
 		.name	= "max8925-tsc",
 		.start	= MAX8925_TSC_IRQ,
 		.end	= MAX8925_ADC_RES_END,
-		.flags	= IORESOURCE_IO,
+		.flags	= IORESOURCE_REG,
 	},
 };
 
@@ -59,7 +56,7 @@ static struct resource power_supply_resources[] = {
 		.name	= "max8925-power",
 		.start	= MAX8925_CHG_IRQ1,
 		.end	= MAX8925_CHG_IRQ1_MASK,
-		.flags	= IORESOURCE_IO,
+		.flags	= IORESOURCE_REG,
 	},
 };
 
@@ -113,71 +110,215 @@ static struct mfd_cell onkey_devs[] = {
 	},
 };
 
-#define MAX8925_REG_RESOURCE(_start, _end)	\
-{						\
-	.start	= MAX8925_##_start,		\
-	.end	= MAX8925_##_end,		\
-	.flags	= IORESOURCE_IO,		\
-}
+static struct resource sd1_resources[] __devinitdata = {
+	{0x06, 0x06, "sdv", IORESOURCE_REG, },
+};
 
-static struct resource regulator_resources[] = {
-	MAX8925_REG_RESOURCE(SDCTL1, SDCTL1),
-	MAX8925_REG_RESOURCE(SDCTL2, SDCTL2),
-	MAX8925_REG_RESOURCE(SDCTL3, SDCTL3),
-	MAX8925_REG_RESOURCE(LDOCTL1, LDOCTL1),
-	MAX8925_REG_RESOURCE(LDOCTL2, LDOCTL2),
-	MAX8925_REG_RESOURCE(LDOCTL3, LDOCTL3),
-	MAX8925_REG_RESOURCE(LDOCTL4, LDOCTL4),
-	MAX8925_REG_RESOURCE(LDOCTL5, LDOCTL5),
-	MAX8925_REG_RESOURCE(LDOCTL6, LDOCTL6),
-	MAX8925_REG_RESOURCE(LDOCTL7, LDOCTL7),
-	MAX8925_REG_RESOURCE(LDOCTL8, LDOCTL8),
-	MAX8925_REG_RESOURCE(LDOCTL9, LDOCTL9),
-	MAX8925_REG_RESOURCE(LDOCTL10, LDOCTL10),
-	MAX8925_REG_RESOURCE(LDOCTL11, LDOCTL11),
-	MAX8925_REG_RESOURCE(LDOCTL12, LDOCTL12),
-	MAX8925_REG_RESOURCE(LDOCTL13, LDOCTL13),
-	MAX8925_REG_RESOURCE(LDOCTL14, LDOCTL14),
-	MAX8925_REG_RESOURCE(LDOCTL15, LDOCTL15),
-	MAX8925_REG_RESOURCE(LDOCTL16, LDOCTL16),
-	MAX8925_REG_RESOURCE(LDOCTL17, LDOCTL17),
-	MAX8925_REG_RESOURCE(LDOCTL18, LDOCTL18),
-	MAX8925_REG_RESOURCE(LDOCTL19, LDOCTL19),
-	MAX8925_REG_RESOURCE(LDOCTL20, LDOCTL20),
-};
-
-#define MAX8925_REG_DEVS(_id)						\
-{									\
-	.name		= "max8925-regulator",				\
-	.num_resources	= 1,						\
-	.resources	= &regulator_resources[MAX8925_ID_##_id],	\
-	.id		= MAX8925_ID_##_id,				\
-}
+static struct resource sd2_resources[] __devinitdata = {
+	{0x09, 0x09, "sdv", IORESOURCE_REG, },
+};
+
+static struct resource sd3_resources[] __devinitdata = {
+	{0x0c, 0x0c, "sdv", IORESOURCE_REG, },
+};
+
+static struct resource ldo1_resources[] __devinitdata = {
+	{0x1a, 0x1a, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo2_resources[] __devinitdata = {
+	{0x1e, 0x1e, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo3_resources[] __devinitdata = {
+	{0x22, 0x22, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo4_resources[] __devinitdata = {
+	{0x26, 0x26, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo5_resources[] __devinitdata = {
+	{0x2a, 0x2a, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo6_resources[] __devinitdata = {
+	{0x2e, 0x2e, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo7_resources[] __devinitdata = {
+	{0x32, 0x32, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo8_resources[] __devinitdata = {
+	{0x36, 0x36, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo9_resources[] __devinitdata = {
+	{0x3a, 0x3a, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo10_resources[] __devinitdata = {
+	{0x3e, 0x3e, "ldov", IORESOURCE_REG, },
+};
 
-static struct mfd_cell regulator_devs[] = {
-	MAX8925_REG_DEVS(SD1),
-	MAX8925_REG_DEVS(SD2),
-	MAX8925_REG_DEVS(SD3),
-	MAX8925_REG_DEVS(LDO1),
-	MAX8925_REG_DEVS(LDO2),
-	MAX8925_REG_DEVS(LDO3),
-	MAX8925_REG_DEVS(LDO4),
-	MAX8925_REG_DEVS(LDO5),
-	MAX8925_REG_DEVS(LDO6),
-	MAX8925_REG_DEVS(LDO7),
-	MAX8925_REG_DEVS(LDO8),
-	MAX8925_REG_DEVS(LDO9),
-	MAX8925_REG_DEVS(LDO10),
-	MAX8925_REG_DEVS(LDO11),
-	MAX8925_REG_DEVS(LDO12),
-	MAX8925_REG_DEVS(LDO13),
-	MAX8925_REG_DEVS(LDO14),
-	MAX8925_REG_DEVS(LDO15),
-	MAX8925_REG_DEVS(LDO16),
-	MAX8925_REG_DEVS(LDO17),
-	MAX8925_REG_DEVS(LDO18),
-	MAX8925_REG_DEVS(LDO19),
-	MAX8925_REG_DEVS(LDO20),
+static struct resource ldo11_resources[] __devinitdata = {
+	{0x42, 0x42, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo12_resources[] __devinitdata = {
+	{0x46, 0x46, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo13_resources[] __devinitdata = {
+	{0x4a, 0x4a, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo14_resources[] __devinitdata = {
+	{0x4e, 0x4e, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo15_resources[] __devinitdata = {
+	{0x52, 0x52, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo16_resources[] __devinitdata = {
+	{0x12, 0x12, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo17_resources[] __devinitdata = {
+	{0x16, 0x16, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo18_resources[] __devinitdata = {
+	{0x74, 0x74, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo19_resources[] __devinitdata = {
+	{0x5e, 0x5e, "ldov", IORESOURCE_REG, },
+};
+
+static struct resource ldo20_resources[] __devinitdata = {
+	{0x9e, 0x9e, "ldov", IORESOURCE_REG, },
+};
+
+static struct mfd_cell reg_devs[] __devinitdata = {
+	{
+		.name = "max8925-regulator",
+		.id = 0,
+		.num_resources = ARRAY_SIZE(sd1_resources),
+		.resources = sd1_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 1,
+		.num_resources = ARRAY_SIZE(sd2_resources),
+		.resources = sd2_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 2,
+		.num_resources = ARRAY_SIZE(sd3_resources),
+		.resources = sd3_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 3,
+		.num_resources = ARRAY_SIZE(ldo1_resources),
+		.resources = ldo1_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 4,
+		.num_resources = ARRAY_SIZE(ldo2_resources),
+		.resources = ldo2_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 5,
+		.num_resources = ARRAY_SIZE(ldo3_resources),
+		.resources = ldo3_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 6,
+		.num_resources = ARRAY_SIZE(ldo4_resources),
+		.resources = ldo4_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 7,
+		.num_resources = ARRAY_SIZE(ldo5_resources),
+		.resources = ldo5_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 8,
+		.num_resources = ARRAY_SIZE(ldo6_resources),
+		.resources = ldo6_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 9,
+		.num_resources = ARRAY_SIZE(ldo7_resources),
+		.resources = ldo7_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 10,
+		.num_resources = ARRAY_SIZE(ldo8_resources),
+		.resources = ldo8_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 11,
+		.num_resources = ARRAY_SIZE(ldo9_resources),
+		.resources = ldo9_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 12,
+		.num_resources = ARRAY_SIZE(ldo10_resources),
+		.resources = ldo10_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 13,
+		.num_resources = ARRAY_SIZE(ldo11_resources),
+		.resources = ldo11_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 14,
+		.num_resources = ARRAY_SIZE(ldo12_resources),
+		.resources = ldo12_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 15,
+		.num_resources = ARRAY_SIZE(ldo13_resources),
+		.resources = ldo13_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 16,
+		.num_resources = ARRAY_SIZE(ldo14_resources),
+		.resources = ldo14_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 17,
+		.num_resources = ARRAY_SIZE(ldo15_resources),
+		.resources = ldo15_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 18,
+		.num_resources = ARRAY_SIZE(ldo16_resources),
+		.resources = ldo16_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 19,
+		.num_resources = ARRAY_SIZE(ldo17_resources),
+		.resources = ldo17_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 20,
+		.num_resources = ARRAY_SIZE(ldo18_resources),
+		.resources = ldo18_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 21,
+		.num_resources = ARRAY_SIZE(ldo19_resources),
+		.resources = ldo19_resources,
+	}, {
+		.name = "max8925-regulator",
+		.id = 22,
+		.num_resources = ARRAY_SIZE(ldo20_resources),
+		.resources = ldo20_resources,
+	},
 };
 
 enum {
@@ -547,7 +688,7 @@ static int max8925_irq_init(struct max8925_chip *chip, int irq,
 		goto tsc_irq;
 	}
 
-	ret = request_threaded_irq(irq, NULL, max8925_irq, flags,
+	ret = request_threaded_irq(irq, NULL, max8925_irq, flags | IRQF_ONESHOT,
 				   "max8925", chip);
 	if (ret) {
 		dev_err(chip->dev, "Failed to request core IRQ: %d\n", ret);
@@ -565,7 +706,7 @@ tsc_irq:
 	chip->tsc_irq = pdata->tsc_irq;
 
 	ret = request_threaded_irq(chip->tsc_irq, NULL, max8925_tsc_irq,
-				   flags, "max8925-tsc", chip);
+				   flags | IRQF_ONESHOT, "max8925-tsc", chip);
 	if (ret) {
 		dev_err(chip->dev, "Failed to request TSC IRQ: %d\n", ret);
 		chip->tsc_irq = 0;
@@ -573,6 +714,113 @@ tsc_irq:
 	return 0;
 }
 
+static void __devinit init_regulator(struct max8925_chip *chip,
+				     struct max8925_platform_data *pdata)
+{
+	int ret;
+
+	if (!pdata)
+		return;
+	if (pdata->sd1) {
+		reg_devs[0].platform_data = pdata->sd1;
+		reg_devs[0].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->sd2) {
+		reg_devs[1].platform_data = pdata->sd2;
+		reg_devs[1].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->sd3) {
+		reg_devs[2].platform_data = pdata->sd3;
+		reg_devs[2].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo1) {
+		reg_devs[3].platform_data = pdata->ldo1;
+		reg_devs[3].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo2) {
+		reg_devs[4].platform_data = pdata->ldo2;
+		reg_devs[4].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo3) {
+		reg_devs[5].platform_data = pdata->ldo3;
+		reg_devs[5].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo4) {
+		reg_devs[6].platform_data = pdata->ldo4;
+		reg_devs[6].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo5) {
+		reg_devs[7].platform_data = pdata->ldo5;
+		reg_devs[7].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo6) {
+		reg_devs[8].platform_data = pdata->ldo6;
+		reg_devs[8].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo7) {
+		reg_devs[9].platform_data = pdata->ldo7;
+		reg_devs[9].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo8) {
+		reg_devs[10].platform_data = pdata->ldo8;
+		reg_devs[10].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo9) {
+		reg_devs[11].platform_data = pdata->ldo9;
+		reg_devs[11].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo10) {
+		reg_devs[12].platform_data = pdata->ldo10;
+		reg_devs[12].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo11) {
+		reg_devs[13].platform_data = pdata->ldo11;
+		reg_devs[13].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo12) {
+		reg_devs[14].platform_data = pdata->ldo12;
+		reg_devs[14].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo13) {
+		reg_devs[15].platform_data = pdata->ldo13;
+		reg_devs[15].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo14) {
+		reg_devs[16].platform_data = pdata->ldo14;
+		reg_devs[16].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo15) {
+		reg_devs[17].platform_data = pdata->ldo15;
+		reg_devs[17].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo16) {
+		reg_devs[18].platform_data = pdata->ldo16;
+		reg_devs[18].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo17) {
+		reg_devs[19].platform_data = pdata->ldo17;
+		reg_devs[19].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo18) {
+		reg_devs[20].platform_data = pdata->ldo18;
+		reg_devs[20].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo19) {
+		reg_devs[21].platform_data = pdata->ldo19;
+		reg_devs[21].pdata_size = sizeof(struct regulator_init_data);
+	}
+	if (pdata->ldo20) {
+		reg_devs[22].platform_data = pdata->ldo20;
+		reg_devs[22].pdata_size = sizeof(struct regulator_init_data);
+	}
+	ret = mfd_add_devices(chip->dev, 0, reg_devs, ARRAY_SIZE(reg_devs),
+			      NULL, 0, NULL);
+	if (ret < 0) {
+		dev_err(chip->dev, "Failed to add regulator subdev\n");
+		return;
+	}
+}
+
 int __devinit max8925_device_init(struct max8925_chip *chip,
 				  struct max8925_platform_data *pdata)
 {
@@ -612,24 +860,17 @@ int __devinit max8925_device_init(struct max8925_chip *chip,
 		goto out_dev;
 	}
 
-	if (pdata) {
-		ret = mfd_add_devices(chip->dev, 0, &regulator_devs[0],
-				      ARRAY_SIZE(regulator_devs),
-				      &regulator_resources[0], 0, NULL);
-		if (ret < 0) {
-			dev_err(chip->dev, "Failed to add regulator subdev\n");
-			goto out_dev;
-		}
-	}
+	init_regulator(chip, pdata);
 
 	if (pdata && pdata->backlight) {
-		ret = mfd_add_devices(chip->dev, 0, &backlight_devs[0],
-				      ARRAY_SIZE(backlight_devs),
-				      &backlight_resources[0], 0, NULL);
-		if (ret < 0) {
-			dev_err(chip->dev, "Failed to add backlight subdev\n");
-			goto out_dev;
-		}
+		bk_devs[0].platform_data = &pdata->backlight;
+		bk_devs[0].pdata_size = sizeof(struct max8925_backlight_pdata);
+	}
+	ret = mfd_add_devices(chip->dev, 0, bk_devs, ARRAY_SIZE(bk_devs),
+			      NULL, 0, NULL);
+	if (ret < 0) {
+		dev_err(chip->dev, "Failed to add backlight subdev\n");
+		goto out_dev;
 	}
 
 	if (pdata && pdata->power) {
diff --git a/drivers/mfd/mc13xxx-core.c b/drivers/mfd/mc13xxx-core.c
index 1ec79b54bd2..1aba0238f42 100644
--- a/drivers/mfd/mc13xxx-core.c
+++ b/drivers/mfd/mc13xxx-core.c
@@ -676,7 +676,6 @@ int mc13xxx_common_init(struct mc13xxx *mc13xxx,
 err_mask:
 err_revision:
 		mc13xxx_unlock(mc13xxx);
-		kfree(mc13xxx);
 		return ret;
 	}
 
diff --git a/drivers/mfd/omap-usb-host.c b/drivers/mfd/omap-usb-host.c
index 41088ecbb2a..23cec57c02b 100644
--- a/drivers/mfd/omap-usb-host.c
+++ b/drivers/mfd/omap-usb-host.c
@@ -21,7 +21,6 @@
 #include <linux/types.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/spinlock.h>
@@ -36,63 +35,6 @@
 
 /* OMAP USBHOST Register addresses  */
 
-/* TLL Register Set */
-#define	OMAP_USBTLL_REVISION				(0x00)
-#define	OMAP_USBTLL_SYSCONFIG				(0x10)
-#define	OMAP_USBTLL_SYSCONFIG_CACTIVITY			(1 << 8)
-#define	OMAP_USBTLL_SYSCONFIG_SIDLEMODE			(1 << 3)
-#define	OMAP_USBTLL_SYSCONFIG_ENAWAKEUP			(1 << 2)
-#define	OMAP_USBTLL_SYSCONFIG_SOFTRESET			(1 << 1)
-#define	OMAP_USBTLL_SYSCONFIG_AUTOIDLE			(1 << 0)
-
-#define	OMAP_USBTLL_SYSSTATUS				(0x14)
-#define	OMAP_USBTLL_SYSSTATUS_RESETDONE			(1 << 0)
-
-#define	OMAP_USBTLL_IRQSTATUS				(0x18)
-#define	OMAP_USBTLL_IRQENABLE				(0x1C)
-
-#define	OMAP_TLL_SHARED_CONF				(0x30)
-#define	OMAP_TLL_SHARED_CONF_USB_90D_DDR_EN		(1 << 6)
-#define	OMAP_TLL_SHARED_CONF_USB_180D_SDR_EN		(1 << 5)
-#define	OMAP_TLL_SHARED_CONF_USB_DIVRATION		(1 << 2)
-#define	OMAP_TLL_SHARED_CONF_FCLK_REQ			(1 << 1)
-#define	OMAP_TLL_SHARED_CONF_FCLK_IS_ON			(1 << 0)
-
-#define	OMAP_TLL_CHANNEL_CONF(num)			(0x040 + 0x004 * num)
-#define OMAP_TLL_CHANNEL_CONF_FSLSMODE_SHIFT		24
-#define	OMAP_TLL_CHANNEL_CONF_ULPINOBITSTUFF		(1 << 11)
-#define	OMAP_TLL_CHANNEL_CONF_ULPI_ULPIAUTOIDLE		(1 << 10)
-#define	OMAP_TLL_CHANNEL_CONF_UTMIAUTOIDLE		(1 << 9)
-#define	OMAP_TLL_CHANNEL_CONF_ULPIDDRMODE		(1 << 8)
-#define OMAP_TLL_CHANNEL_CONF_CHANMODE_FSLS		(1 << 1)
-#define	OMAP_TLL_CHANNEL_CONF_CHANEN			(1 << 0)
-
-#define OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0		0x0
-#define OMAP_TLL_FSLSMODE_6PIN_PHY_DP_DM		0x1
-#define OMAP_TLL_FSLSMODE_3PIN_PHY			0x2
-#define OMAP_TLL_FSLSMODE_4PIN_PHY			0x3
-#define OMAP_TLL_FSLSMODE_6PIN_TLL_DAT_SE0		0x4
-#define OMAP_TLL_FSLSMODE_6PIN_TLL_DP_DM		0x5
-#define OMAP_TLL_FSLSMODE_3PIN_TLL			0x6
-#define OMAP_TLL_FSLSMODE_4PIN_TLL			0x7
-#define OMAP_TLL_FSLSMODE_2PIN_TLL_DAT_SE0		0xA
-#define OMAP_TLL_FSLSMODE_2PIN_DAT_DP_DM		0xB
-
-#define	OMAP_TLL_ULPI_FUNCTION_CTRL(num)		(0x804 + 0x100 * num)
-#define	OMAP_TLL_ULPI_INTERFACE_CTRL(num)		(0x807 + 0x100 * num)
-#define	OMAP_TLL_ULPI_OTG_CTRL(num)			(0x80A + 0x100 * num)
-#define	OMAP_TLL_ULPI_INT_EN_RISE(num)			(0x80D + 0x100 * num)
-#define	OMAP_TLL_ULPI_INT_EN_FALL(num)			(0x810 + 0x100 * num)
-#define	OMAP_TLL_ULPI_INT_STATUS(num)			(0x813 + 0x100 * num)
-#define	OMAP_TLL_ULPI_INT_LATCH(num)			(0x814 + 0x100 * num)
-#define	OMAP_TLL_ULPI_DEBUG(num)			(0x815 + 0x100 * num)
-#define	OMAP_TLL_ULPI_SCRATCH_REGISTER(num)		(0x816 + 0x100 * num)
-
-#define OMAP_TLL_CHANNEL_COUNT				3
-#define OMAP_TLL_CHANNEL_1_EN_MASK			(1 << 0)
-#define OMAP_TLL_CHANNEL_2_EN_MASK			(1 << 1)
-#define OMAP_TLL_CHANNEL_3_EN_MASK			(1 << 2)
-
 /* UHH Register Set */
 #define	OMAP_UHH_REVISION				(0x00)
 #define	OMAP_UHH_SYSCONFIG				(0x10)
@@ -132,8 +74,6 @@
 #define OMAP4_P2_MODE_TLL				(1 << 18)
 #define OMAP4_P2_MODE_HSIC				(3 << 18)
 
-#define OMAP_REV2_TLL_CHANNEL_COUNT			2
-
 #define	OMAP_UHH_DEBUG_CSR				(0x44)
 
 /* Values of UHH_REVISION - Note: these are not given in the TRM */
@@ -153,15 +93,12 @@ struct usbhs_hcd_omap {
 	struct clk			*xclk60mhsp2_ck;
 	struct clk			*utmi_p1_fck;
 	struct clk			*usbhost_p1_fck;
-	struct clk			*usbtll_p1_fck;
 	struct clk			*utmi_p2_fck;
 	struct clk			*usbhost_p2_fck;
-	struct clk			*usbtll_p2_fck;
 	struct clk			*init_60m_fclk;
 	struct clk			*ehci_logic_fck;
 
 	void __iomem			*uhh_base;
-	void __iomem			*tll_base;
 
 	struct usbhs_omap_platform_data	platdata;
 
@@ -336,93 +273,6 @@ static bool is_ohci_port(enum usbhs_omap_port_mode pmode)
 	}
 }
 
-/*
- * convert the port-mode enum to a value we can use in the FSLSMODE
- * field of USBTLL_CHANNEL_CONF
- */
-static unsigned ohci_omap3_fslsmode(enum usbhs_omap_port_mode mode)
-{
-	switch (mode) {
-	case OMAP_USBHS_PORT_MODE_UNUSED:
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
-		return OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0;
-
-	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
-		return OMAP_TLL_FSLSMODE_6PIN_PHY_DP_DM;
-
-	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
-		return OMAP_TLL_FSLSMODE_3PIN_PHY;
-
-	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
-		return OMAP_TLL_FSLSMODE_4PIN_PHY;
-
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
-		return OMAP_TLL_FSLSMODE_6PIN_TLL_DAT_SE0;
-
-	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
-		return OMAP_TLL_FSLSMODE_6PIN_TLL_DP_DM;
-
-	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
-		return OMAP_TLL_FSLSMODE_3PIN_TLL;
-
-	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
-		return OMAP_TLL_FSLSMODE_4PIN_TLL;
-
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
-		return OMAP_TLL_FSLSMODE_2PIN_TLL_DAT_SE0;
-
-	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
-		return OMAP_TLL_FSLSMODE_2PIN_DAT_DP_DM;
-	default:
-		pr_warning("Invalid port mode, using default\n");
-		return OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0;
-	}
-}
-
-static void usbhs_omap_tll_init(struct device *dev, u8 tll_channel_count)
-{
-	struct usbhs_hcd_omap		*omap = dev_get_drvdata(dev);
-	struct usbhs_omap_platform_data	*pdata = dev->platform_data;
-	unsigned			reg;
-	int				i;
-
-	/* Program Common TLL register */
-	reg = usbhs_read(omap->tll_base, OMAP_TLL_SHARED_CONF);
-	reg |= (OMAP_TLL_SHARED_CONF_FCLK_IS_ON
-		| OMAP_TLL_SHARED_CONF_USB_DIVRATION);
-	reg &= ~OMAP_TLL_SHARED_CONF_USB_90D_DDR_EN;
-	reg &= ~OMAP_TLL_SHARED_CONF_USB_180D_SDR_EN;
-
-	usbhs_write(omap->tll_base, OMAP_TLL_SHARED_CONF, reg);
-
-	/* Enable channels now */
-	for (i = 0; i < tll_channel_count; i++) {
-		reg = usbhs_read(omap->tll_base,
-				OMAP_TLL_CHANNEL_CONF(i));
-
-		if (is_ohci_port(pdata->port_mode[i])) {
-			reg |= ohci_omap3_fslsmode(pdata->port_mode[i])
-				<< OMAP_TLL_CHANNEL_CONF_FSLSMODE_SHIFT;
-			reg |= OMAP_TLL_CHANNEL_CONF_CHANMODE_FSLS;
-		} else if (pdata->port_mode[i] == OMAP_EHCI_PORT_MODE_TLL) {
-
-			/* Disable AutoIdle, BitStuffing and use SDR Mode */
-			reg &= ~(OMAP_TLL_CHANNEL_CONF_UTMIAUTOIDLE
-				| OMAP_TLL_CHANNEL_CONF_ULPINOBITSTUFF
-				| OMAP_TLL_CHANNEL_CONF_ULPIDDRMODE);
-
-		} else
-			continue;
-
-		reg |= OMAP_TLL_CHANNEL_CONF_CHANEN;
-		usbhs_write(omap->tll_base,
-				OMAP_TLL_CHANNEL_CONF(i), reg);
-
-		usbhs_writeb(omap->tll_base,
-				OMAP_TLL_ULPI_SCRATCH_REGISTER(i), 0xbe);
-	}
-}
-
 static int usbhs_runtime_resume(struct device *dev)
 {
 	struct usbhs_hcd_omap		*omap = dev_get_drvdata(dev);
@@ -436,19 +286,17 @@ static int usbhs_runtime_resume(struct device *dev)
 		return  -ENODEV;
 	}
 
+	omap_tll_enable();
 	spin_lock_irqsave(&omap->lock, flags);
 
 	if (omap->ehci_logic_fck && !IS_ERR(omap->ehci_logic_fck))
 		clk_enable(omap->ehci_logic_fck);
 
-	if (is_ehci_tll_mode(pdata->port_mode[0])) {
+	if (is_ehci_tll_mode(pdata->port_mode[0]))
 		clk_enable(omap->usbhost_p1_fck);
-		clk_enable(omap->usbtll_p1_fck);
-	}
-	if (is_ehci_tll_mode(pdata->port_mode[1])) {
+	if (is_ehci_tll_mode(pdata->port_mode[1]))
 		clk_enable(omap->usbhost_p2_fck);
-		clk_enable(omap->usbtll_p2_fck);
-	}
+
 	clk_enable(omap->utmi_p1_fck);
 	clk_enable(omap->utmi_p2_fck);
 
@@ -472,14 +320,11 @@ static int usbhs_runtime_suspend(struct device *dev)
 
 	spin_lock_irqsave(&omap->lock, flags);
 
-	if (is_ehci_tll_mode(pdata->port_mode[0])) {
+	if (is_ehci_tll_mode(pdata->port_mode[0]))
 		clk_disable(omap->usbhost_p1_fck);
-		clk_disable(omap->usbtll_p1_fck);
-	}
-	if (is_ehci_tll_mode(pdata->port_mode[1])) {
+	if (is_ehci_tll_mode(pdata->port_mode[1]))
 		clk_disable(omap->usbhost_p2_fck);
-		clk_disable(omap->usbtll_p2_fck);
-	}
+
 	clk_disable(omap->utmi_p2_fck);
 	clk_disable(omap->utmi_p1_fck);
 
@@ -487,6 +332,7 @@ static int usbhs_runtime_suspend(struct device *dev)
 		clk_disable(omap->ehci_logic_fck);
 
 	spin_unlock_irqrestore(&omap->lock, flags);
+	omap_tll_disable();
 
 	return 0;
 }
@@ -500,8 +346,6 @@ static void omap_usbhs_init(struct device *dev)
 
 	dev_dbg(dev, "starting TI HSUSB Controller\n");
 
-	pm_runtime_get_sync(dev);
-
 	if (pdata->ehci_data->phy_reset) {
 		if (gpio_is_valid(pdata->ehci_data->reset_gpio_port[0]))
 			gpio_request_one(pdata->ehci_data->reset_gpio_port[0],
@@ -515,6 +359,7 @@ static void omap_usbhs_init(struct device *dev)
 		udelay(10);
 	}
 
+	pm_runtime_get_sync(dev);
 	spin_lock_irqsave(&omap->lock, flags);
 	omap->usbhs_rev = usbhs_read(omap->uhh_base, OMAP_UHH_REVISION);
 	dev_dbg(dev, "OMAP UHH_REVISION 0x%x\n", omap->usbhs_rev);
@@ -580,22 +425,9 @@ static void omap_usbhs_init(struct device *dev)
 	usbhs_write(omap->uhh_base, OMAP_UHH_HOSTCONFIG, reg);
 	dev_dbg(dev, "UHH setup done, uhh_hostconfig=%x\n", reg);
 
-	if (is_ehci_tll_mode(pdata->port_mode[0]) ||
-		is_ehci_tll_mode(pdata->port_mode[1]) ||
-		is_ehci_tll_mode(pdata->port_mode[2]) ||
-		(is_ohci_port(pdata->port_mode[0])) ||
-		(is_ohci_port(pdata->port_mode[1])) ||
-		(is_ohci_port(pdata->port_mode[2]))) {
-
-		/* Enable UTMI mode for required TLL channels */
-		if (is_omap_usbhs_rev2(omap))
-			usbhs_omap_tll_init(dev, OMAP_REV2_TLL_CHANNEL_COUNT);
-		else
-			usbhs_omap_tll_init(dev, OMAP_TLL_CHANNEL_COUNT);
-	}
-
 	spin_unlock_irqrestore(&omap->lock, flags);
 
+	pm_runtime_put_sync(dev);
 	if (pdata->ehci_data->phy_reset) {
 		/* Hold the PHY in RESET for enough time till
 		 * PHY is settled and ready
@@ -610,8 +442,6 @@ static void omap_usbhs_init(struct device *dev)
 			gpio_set_value_cansleep
 				(pdata->ehci_data->reset_gpio_port[1], 1);
 	}
-
-	pm_runtime_put_sync(dev);
 }
 
 static void omap_usbhs_deinit(struct device *dev)
@@ -714,32 +544,18 @@ static int __devinit usbhs_omap_probe(struct platform_device *pdev)
 		goto err_xclk60mhsp2_ck;
 	}
 
-	omap->usbtll_p1_fck = clk_get(dev, "usb_tll_hs_usb_ch0_clk");
-	if (IS_ERR(omap->usbtll_p1_fck)) {
-		ret = PTR_ERR(omap->usbtll_p1_fck);
-		dev_err(dev, "usbtll_p1_fck failed error:%d\n", ret);
-		goto err_usbhost_p1_fck;
-	}
-
 	omap->usbhost_p2_fck = clk_get(dev, "usb_host_hs_utmi_p2_clk");
 	if (IS_ERR(omap->usbhost_p2_fck)) {
 		ret = PTR_ERR(omap->usbhost_p2_fck);
 		dev_err(dev, "usbhost_p2_fck failed error:%d\n", ret);
-		goto err_usbtll_p1_fck;
-	}
-
-	omap->usbtll_p2_fck = clk_get(dev, "usb_tll_hs_usb_ch1_clk");
-	if (IS_ERR(omap->usbtll_p2_fck)) {
-		ret = PTR_ERR(omap->usbtll_p2_fck);
-		dev_err(dev, "usbtll_p2_fck failed error:%d\n", ret);
-		goto err_usbhost_p2_fck;
+		goto err_usbhost_p1_fck;
 	}
 
 	omap->init_60m_fclk = clk_get(dev, "init_60m_fclk");
 	if (IS_ERR(omap->init_60m_fclk)) {
 		ret = PTR_ERR(omap->init_60m_fclk);
 		dev_err(dev, "init_60m_fclk failed error:%d\n", ret);
-		goto err_usbtll_p2_fck;
+		goto err_usbhost_p2_fck;
 	}
 
 	if (is_ehci_phy_mode(pdata->port_mode[0])) {
@@ -785,20 +601,6 @@ static int __devinit usbhs_omap_probe(struct platform_device *pdev)
 		goto err_init_60m_fclk;
 	}
 
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tll");
-	if (!res) {
-		dev_err(dev, "UHH EHCI get resource failed\n");
-		ret = -ENODEV;
-		goto err_tll;
-	}
-
-	omap->tll_base = ioremap(res->start, resource_size(res));
-	if (!omap->tll_base) {
-		dev_err(dev, "TLL ioremap failed\n");
-		ret = -ENOMEM;
-		goto err_tll;
-	}
-
 	platform_set_drvdata(pdev, omap);
 
 	omap_usbhs_init(dev);
@@ -812,23 +614,14 @@ static int __devinit usbhs_omap_probe(struct platform_device *pdev)
 
 err_alloc:
 	omap_usbhs_deinit(&pdev->dev);
-	iounmap(omap->tll_base);
-
-err_tll:
 	iounmap(omap->uhh_base);
 
 err_init_60m_fclk:
 	clk_put(omap->init_60m_fclk);
 
-err_usbtll_p2_fck:
-	clk_put(omap->usbtll_p2_fck);
-
 err_usbhost_p2_fck:
 	clk_put(omap->usbhost_p2_fck);
 
-err_usbtll_p1_fck:
-	clk_put(omap->usbtll_p1_fck);
-
 err_usbhost_p1_fck:
 	clk_put(omap->usbhost_p1_fck);
 
@@ -864,12 +657,9 @@ static int __devexit usbhs_omap_remove(struct platform_device *pdev)
 	struct usbhs_hcd_omap *omap = platform_get_drvdata(pdev);
 
 	omap_usbhs_deinit(&pdev->dev);
-	iounmap(omap->tll_base);
 	iounmap(omap->uhh_base);
 	clk_put(omap->init_60m_fclk);
-	clk_put(omap->usbtll_p2_fck);
 	clk_put(omap->usbhost_p2_fck);
-	clk_put(omap->usbtll_p1_fck);
 	clk_put(omap->usbhost_p1_fck);
 	clk_put(omap->xclk60mhsp2_ck);
 	clk_put(omap->utmi_p2_fck);
@@ -910,8 +700,10 @@ static int __init omap_usbhs_drvinit(void)
  * init before ehci and ohci drivers;
  * The usbhs core driver should be initialized much before
  * the omap ehci and ohci probe functions are called.
+ * This usbhs core driver should be initialized after
+ * usb tll driver
  */
-fs_initcall(omap_usbhs_drvinit);
+fs_initcall_sync(omap_usbhs_drvinit);
 
 static void __exit omap_usbhs_drvexit(void)
 {
diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c
new file mode 100644
index 00000000000..4b7757b8430
--- /dev/null
+++ b/drivers/mfd/omap-usb-tll.c
@@ -0,0 +1,471 @@
+/**
+ * omap-usb-tll.c - The USB TLL driver for OMAP EHCI & OHCI
+ *
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com
+ * Author: Keshava Munegowda <keshava_mgowda@ti.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2  of
+ * the License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/err.h>
+#include <plat/usb.h>
+#include <linux/pm_runtime.h>
+
+#define USBTLL_DRIVER_NAME	"usbhs_tll"
+
+/* TLL Register Set */
+#define	OMAP_USBTLL_REVISION				(0x00)
+#define	OMAP_USBTLL_SYSCONFIG				(0x10)
+#define	OMAP_USBTLL_SYSCONFIG_CACTIVITY			(1 << 8)
+#define	OMAP_USBTLL_SYSCONFIG_SIDLEMODE			(1 << 3)
+#define	OMAP_USBTLL_SYSCONFIG_ENAWAKEUP			(1 << 2)
+#define	OMAP_USBTLL_SYSCONFIG_SOFTRESET			(1 << 1)
+#define	OMAP_USBTLL_SYSCONFIG_AUTOIDLE			(1 << 0)
+
+#define	OMAP_USBTLL_SYSSTATUS				(0x14)
+#define	OMAP_USBTLL_SYSSTATUS_RESETDONE			(1 << 0)
+
+#define	OMAP_USBTLL_IRQSTATUS				(0x18)
+#define	OMAP_USBTLL_IRQENABLE				(0x1C)
+
+#define	OMAP_TLL_SHARED_CONF				(0x30)
+#define	OMAP_TLL_SHARED_CONF_USB_90D_DDR_EN		(1 << 6)
+#define	OMAP_TLL_SHARED_CONF_USB_180D_SDR_EN		(1 << 5)
+#define	OMAP_TLL_SHARED_CONF_USB_DIVRATION		(1 << 2)
+#define	OMAP_TLL_SHARED_CONF_FCLK_REQ			(1 << 1)
+#define	OMAP_TLL_SHARED_CONF_FCLK_IS_ON			(1 << 0)
+
+#define	OMAP_TLL_CHANNEL_CONF(num)			(0x040 + 0x004 * num)
+#define OMAP_TLL_CHANNEL_CONF_FSLSMODE_SHIFT		24
+#define	OMAP_TLL_CHANNEL_CONF_ULPINOBITSTUFF		(1 << 11)
+#define	OMAP_TLL_CHANNEL_CONF_ULPI_ULPIAUTOIDLE		(1 << 10)
+#define	OMAP_TLL_CHANNEL_CONF_UTMIAUTOIDLE		(1 << 9)
+#define	OMAP_TLL_CHANNEL_CONF_ULPIDDRMODE		(1 << 8)
+#define OMAP_TLL_CHANNEL_CONF_CHANMODE_FSLS		(1 << 1)
+#define	OMAP_TLL_CHANNEL_CONF_CHANEN			(1 << 0)
+
+#define OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0		0x0
+#define OMAP_TLL_FSLSMODE_6PIN_PHY_DP_DM		0x1
+#define OMAP_TLL_FSLSMODE_3PIN_PHY			0x2
+#define OMAP_TLL_FSLSMODE_4PIN_PHY			0x3
+#define OMAP_TLL_FSLSMODE_6PIN_TLL_DAT_SE0		0x4
+#define OMAP_TLL_FSLSMODE_6PIN_TLL_DP_DM		0x5
+#define OMAP_TLL_FSLSMODE_3PIN_TLL			0x6
+#define OMAP_TLL_FSLSMODE_4PIN_TLL			0x7
+#define OMAP_TLL_FSLSMODE_2PIN_TLL_DAT_SE0		0xA
+#define OMAP_TLL_FSLSMODE_2PIN_DAT_DP_DM		0xB
+
+#define	OMAP_TLL_ULPI_FUNCTION_CTRL(num)		(0x804 + 0x100 * num)
+#define	OMAP_TLL_ULPI_INTERFACE_CTRL(num)		(0x807 + 0x100 * num)
+#define	OMAP_TLL_ULPI_OTG_CTRL(num)			(0x80A + 0x100 * num)
+#define	OMAP_TLL_ULPI_INT_EN_RISE(num)			(0x80D + 0x100 * num)
+#define	OMAP_TLL_ULPI_INT_EN_FALL(num)			(0x810 + 0x100 * num)
+#define	OMAP_TLL_ULPI_INT_STATUS(num)			(0x813 + 0x100 * num)
+#define	OMAP_TLL_ULPI_INT_LATCH(num)			(0x814 + 0x100 * num)
+#define	OMAP_TLL_ULPI_DEBUG(num)			(0x815 + 0x100 * num)
+#define	OMAP_TLL_ULPI_SCRATCH_REGISTER(num)		(0x816 + 0x100 * num)
+
+#define OMAP_REV2_TLL_CHANNEL_COUNT			2
+#define OMAP_TLL_CHANNEL_COUNT				3
+#define OMAP_TLL_CHANNEL_1_EN_MASK			(1 << 0)
+#define OMAP_TLL_CHANNEL_2_EN_MASK			(1 << 1)
+#define OMAP_TLL_CHANNEL_3_EN_MASK			(1 << 2)
+
+/* Values of USBTLL_REVISION - Note: these are not given in the TRM */
+#define OMAP_USBTLL_REV1		0x00000015	/* OMAP3 */
+#define OMAP_USBTLL_REV2		0x00000018	/* OMAP 3630 */
+#define OMAP_USBTLL_REV3		0x00000004	/* OMAP4 */
+
+#define is_ehci_tll_mode(x)	(x == OMAP_EHCI_PORT_MODE_TLL)
+
+struct usbtll_omap {
+	struct clk				*usbtll_p1_fck;
+	struct clk				*usbtll_p2_fck;
+	struct usbtll_omap_platform_data	platdata;
+	/* secure the register updates */
+	spinlock_t				lock;
+};
+
+/*-------------------------------------------------------------------------*/
+
+const char usbtll_driver_name[] = USBTLL_DRIVER_NAME;
+struct platform_device	*tll_pdev;
+
+/*-------------------------------------------------------------------------*/
+
+static inline void usbtll_write(void __iomem *base, u32 reg, u32 val)
+{
+	__raw_writel(val, base + reg);
+}
+
+static inline u32 usbtll_read(void __iomem *base, u32 reg)
+{
+	return __raw_readl(base + reg);
+}
+
+static inline void usbtll_writeb(void __iomem *base, u8 reg, u8 val)
+{
+	__raw_writeb(val, base + reg);
+}
+
+static inline u8 usbtll_readb(void __iomem *base, u8 reg)
+{
+	return __raw_readb(base + reg);
+}
+
+/*-------------------------------------------------------------------------*/
+
+static bool is_ohci_port(enum usbhs_omap_port_mode pmode)
+{
+	switch (pmode) {
+	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
+	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
+	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
+	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
+	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
+	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
+	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
+	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
+	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
+	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
+		return true;
+
+	default:
+		return false;
+	}
+}
+
+/*
+ * convert the port-mode enum to a value we can use in the FSLSMODE
+ * field of USBTLL_CHANNEL_CONF
+ */
+static unsigned ohci_omap3_fslsmode(enum usbhs_omap_port_mode mode)
+{
+	switch (mode) {
+	case OMAP_USBHS_PORT_MODE_UNUSED:
+	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DATSE0:
+		return OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0;
+
+	case OMAP_OHCI_PORT_MODE_PHY_6PIN_DPDM:
+		return OMAP_TLL_FSLSMODE_6PIN_PHY_DP_DM;
+
+	case OMAP_OHCI_PORT_MODE_PHY_3PIN_DATSE0:
+		return OMAP_TLL_FSLSMODE_3PIN_PHY;
+
+	case OMAP_OHCI_PORT_MODE_PHY_4PIN_DPDM:
+		return OMAP_TLL_FSLSMODE_4PIN_PHY;
+
+	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DATSE0:
+		return OMAP_TLL_FSLSMODE_6PIN_TLL_DAT_SE0;
+
+	case OMAP_OHCI_PORT_MODE_TLL_6PIN_DPDM:
+		return OMAP_TLL_FSLSMODE_6PIN_TLL_DP_DM;
+
+	case OMAP_OHCI_PORT_MODE_TLL_3PIN_DATSE0:
+		return OMAP_TLL_FSLSMODE_3PIN_TLL;
+
+	case OMAP_OHCI_PORT_MODE_TLL_4PIN_DPDM:
+		return OMAP_TLL_FSLSMODE_4PIN_TLL;
+
+	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DATSE0:
+		return OMAP_TLL_FSLSMODE_2PIN_TLL_DAT_SE0;
+
+	case OMAP_OHCI_PORT_MODE_TLL_2PIN_DPDM:
+		return OMAP_TLL_FSLSMODE_2PIN_DAT_DP_DM;
+	default:
+		pr_warn("Invalid port mode, using default\n");
+		return OMAP_TLL_FSLSMODE_6PIN_PHY_DAT_SE0;
+	}
+}
+
+/**
+ * usbtll_omap_probe - initialize TI-based HCDs
+ *
+ * Allocates basic resources for this USB host controller.
+ */
+static int __devinit usbtll_omap_probe(struct platform_device *pdev)
+{
+	struct device				*dev =  &pdev->dev;
+	struct usbtll_omap_platform_data	*pdata = dev->platform_data;
+	void __iomem				*base;
+	struct resource				*res;
+	struct usbtll_omap			*tll;
+	unsigned				reg;
+	unsigned long				flags;
+	int					ret = 0;
+	int					i, ver, count;
+
+	dev_dbg(dev, "starting TI HSUSB TLL Controller\n");
+
+	tll = kzalloc(sizeof(struct usbtll_omap), GFP_KERNEL);
+	if (!tll) {
+		dev_err(dev, "Memory allocation failed\n");
+		ret = -ENOMEM;
+		goto end;
+	}
+
+	spin_lock_init(&tll->lock);
+
+	for (i = 0; i < OMAP3_HS_USB_PORTS; i++)
+		tll->platdata.port_mode[i] = pdata->port_mode[i];
+
+	tll->usbtll_p1_fck = clk_get(dev, "usb_tll_hs_usb_ch0_clk");
+	if (IS_ERR(tll->usbtll_p1_fck)) {
+		ret = PTR_ERR(tll->usbtll_p1_fck);
+		dev_err(dev, "usbtll_p1_fck failed error:%d\n", ret);
+		goto err_tll;
+	}
+
+	tll->usbtll_p2_fck = clk_get(dev, "usb_tll_hs_usb_ch1_clk");
+	if (IS_ERR(tll->usbtll_p2_fck)) {
+		ret = PTR_ERR(tll->usbtll_p2_fck);
+		dev_err(dev, "usbtll_p2_fck failed error:%d\n", ret);
+		goto err_usbtll_p1_fck;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "usb tll get resource failed\n");
+		ret = -ENODEV;
+		goto err_usbtll_p2_fck;
+	}
+
+	base = ioremap(res->start, resource_size(res));
+	if (!base) {
+		dev_err(dev, "TLL ioremap failed\n");
+		ret = -ENOMEM;
+		goto err_usbtll_p2_fck;
+	}
+
+	platform_set_drvdata(pdev, tll);
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+
+	spin_lock_irqsave(&tll->lock, flags);
+
+	ver =  usbtll_read(base, OMAP_USBTLL_REVISION);
+	switch (ver) {
+	case OMAP_USBTLL_REV1:
+	case OMAP_USBTLL_REV2:
+		count = OMAP_TLL_CHANNEL_COUNT;
+		break;
+	case OMAP_USBTLL_REV3:
+		count = OMAP_REV2_TLL_CHANNEL_COUNT;
+		break;
+	default:
+		dev_err(dev, "TLL version failed\n");
+		ret = -ENODEV;
+		goto err_ioremap;
+	}
+
+	if (is_ehci_tll_mode(pdata->port_mode[0]) ||
+	    is_ehci_tll_mode(pdata->port_mode[1]) ||
+	    is_ehci_tll_mode(pdata->port_mode[2]) ||
+	    is_ohci_port(pdata->port_mode[0]) ||
+	    is_ohci_port(pdata->port_mode[1]) ||
+	    is_ohci_port(pdata->port_mode[2])) {
+
+		/* Program Common TLL register */
+		reg = usbtll_read(base, OMAP_TLL_SHARED_CONF);
+		reg |= (OMAP_TLL_SHARED_CONF_FCLK_IS_ON
+			| OMAP_TLL_SHARED_CONF_USB_DIVRATION);
+		reg &= ~OMAP_TLL_SHARED_CONF_USB_90D_DDR_EN;
+		reg &= ~OMAP_TLL_SHARED_CONF_USB_180D_SDR_EN;
+
+		usbtll_write(base, OMAP_TLL_SHARED_CONF, reg);
+
+		/* Enable channels now */
+		for (i = 0; i < count; i++) {
+			reg = usbtll_read(base,	OMAP_TLL_CHANNEL_CONF(i));
+
+			if (is_ohci_port(pdata->port_mode[i])) {
+				reg |= ohci_omap3_fslsmode(pdata->port_mode[i])
+				<< OMAP_TLL_CHANNEL_CONF_FSLSMODE_SHIFT;
+				reg |= OMAP_TLL_CHANNEL_CONF_CHANMODE_FSLS;
+			} else if (pdata->port_mode[i] ==
+					OMAP_EHCI_PORT_MODE_TLL) {
+				/*
+				 * Disable AutoIdle, BitStuffing
+				 * and use SDR Mode
+				 */
+				reg &= ~(OMAP_TLL_CHANNEL_CONF_UTMIAUTOIDLE
+					| OMAP_TLL_CHANNEL_CONF_ULPINOBITSTUFF
+					| OMAP_TLL_CHANNEL_CONF_ULPIDDRMODE);
+			} else {
+				continue;
+			}
+			reg |= OMAP_TLL_CHANNEL_CONF_CHANEN;
+			usbtll_write(base, OMAP_TLL_CHANNEL_CONF(i), reg);
+
+			usbtll_writeb(base,
+				      OMAP_TLL_ULPI_SCRATCH_REGISTER(i),
+				      0xbe);
+		}
+	}
+
+err_ioremap:
+	spin_unlock_irqrestore(&tll->lock, flags);
+	iounmap(base);
+	pm_runtime_put_sync(dev);
+	tll_pdev = pdev;
+	if (!ret)
+		goto end;
+	pm_runtime_disable(dev);
+
+err_usbtll_p2_fck:
+	clk_put(tll->usbtll_p2_fck);
+
+err_usbtll_p1_fck:
+	clk_put(tll->usbtll_p1_fck);
+
+err_tll:
+	kfree(tll);
+
+end:
+	return ret;
+}
+
+/**
+ * usbtll_omap_remove - shutdown processing for UHH & TLL HCDs
+ * @pdev: USB Host Controller being removed
+ *
+ * Reverses the effect of usbtll_omap_probe().
+ */
+static int __devexit usbtll_omap_remove(struct platform_device *pdev)
+{
+	struct usbtll_omap *tll = platform_get_drvdata(pdev);
+
+	clk_put(tll->usbtll_p2_fck);
+	clk_put(tll->usbtll_p1_fck);
+	pm_runtime_disable(&pdev->dev);
+	kfree(tll);
+	return 0;
+}
+
+static int usbtll_runtime_resume(struct device *dev)
+{
+	struct usbtll_omap			*tll = dev_get_drvdata(dev);
+	struct usbtll_omap_platform_data	*pdata = &tll->platdata;
+	unsigned long				flags;
+
+	dev_dbg(dev, "usbtll_runtime_resume\n");
+
+	if (!pdata) {
+		dev_dbg(dev, "missing platform_data\n");
+		return  -ENODEV;
+	}
+
+	spin_lock_irqsave(&tll->lock, flags);
+
+	if (is_ehci_tll_mode(pdata->port_mode[0]))
+		clk_enable(tll->usbtll_p1_fck);
+
+	if (is_ehci_tll_mode(pdata->port_mode[1]))
+		clk_enable(tll->usbtll_p2_fck);
+
+	spin_unlock_irqrestore(&tll->lock, flags);
+
+	return 0;
+}
+
+static int usbtll_runtime_suspend(struct device *dev)
+{
+	struct usbtll_omap			*tll = dev_get_drvdata(dev);
+	struct usbtll_omap_platform_data	*pdata = &tll->platdata;
+	unsigned long				flags;
+
+	dev_dbg(dev, "usbtll_runtime_suspend\n");
+
+	if (!pdata) {
+		dev_dbg(dev, "missing platform_data\n");
+		return  -ENODEV;
+	}
+
+	spin_lock_irqsave(&tll->lock, flags);
+
+	if (is_ehci_tll_mode(pdata->port_mode[0]))
+		clk_disable(tll->usbtll_p1_fck);
+
+	if (is_ehci_tll_mode(pdata->port_mode[1]))
+		clk_disable(tll->usbtll_p2_fck);
+
+	spin_unlock_irqrestore(&tll->lock, flags);
+
+	return 0;
+}
+
+static const struct dev_pm_ops usbtllomap_dev_pm_ops = {
+	SET_RUNTIME_PM_OPS(usbtll_runtime_suspend,
+			   usbtll_runtime_resume,
+			   NULL)
+};
+
+static struct platform_driver usbtll_omap_driver = {
+	.driver = {
+		.name		= (char *)usbtll_driver_name,
+		.owner		= THIS_MODULE,
+		.pm		= &usbtllomap_dev_pm_ops,
+	},
+	.probe		= usbtll_omap_probe,
+	.remove		= __devexit_p(usbtll_omap_remove),
+};
+
+int omap_tll_enable(void)
+{
+	if (!tll_pdev) {
+		pr_err("missing omap usbhs tll platform_data\n");
+		return  -ENODEV;
+	}
+	return pm_runtime_get_sync(&tll_pdev->dev);
+}
+EXPORT_SYMBOL_GPL(omap_tll_enable);
+
+int omap_tll_disable(void)
+{
+	if (!tll_pdev) {
+		pr_err("missing omap usbhs tll platform_data\n");
+		return  -ENODEV;
+	}
+	return pm_runtime_put_sync(&tll_pdev->dev);
+}
+EXPORT_SYMBOL_GPL(omap_tll_disable);
+
+MODULE_AUTHOR("Keshava Munegowda <keshava_mgowda@ti.com>");
+MODULE_ALIAS("platform:" USBHS_DRIVER_NAME);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("usb tll driver for TI OMAP EHCI and OHCI controllers");
+
+static int __init omap_usbtll_drvinit(void)
+{
+	return platform_driver_register(&usbtll_omap_driver);
+}
+
+/*
+ * init before usbhs core driver;
+ * The usbtll driver should be initialized before
+ * the usbhs core driver probe function is called.
+ */
+fs_initcall(omap_usbtll_drvinit);
+
+static void __exit omap_usbtll_drvexit(void)
+{
+	platform_driver_unregister(&usbtll_omap_driver);
+}
+module_exit(omap_usbtll_drvexit);
diff --git a/drivers/mfd/palmas.c b/drivers/mfd/palmas.c
index a345f9bb7b4..4f8d6e6b19a 100644
--- a/drivers/mfd/palmas.c
+++ b/drivers/mfd/palmas.c
@@ -23,60 +23,7 @@
 #include <linux/err.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/palmas.h>
-
-static const struct resource gpadc_resource[] = {
-	{
-		.name = "EOC_SW",
-		.start = PALMAS_GPADC_EOC_SW_IRQ,
-		.end = PALMAS_GPADC_EOC_SW_IRQ,
-		.flags = IORESOURCE_IRQ,
-	}
-};
-
-static const struct resource usb_resource[] = {
-	{
-		.name = "ID",
-		.start = PALMAS_ID_OTG_IRQ,
-		.end = PALMAS_ID_OTG_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.name = "ID_WAKEUP",
-		.start = PALMAS_ID_IRQ,
-		.end = PALMAS_ID_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.name = "VBUS",
-		.start = PALMAS_VBUS_OTG_IRQ,
-		.end = PALMAS_VBUS_OTG_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-	{
-		.name = "VBUS_WAKEUP",
-		.start = PALMAS_VBUS_IRQ,
-		.end = PALMAS_VBUS_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-};
-
-static const struct resource rtc_resource[] = {
-	{
-		.name = "RTC_ALARM",
-		.start = PALMAS_RTC_ALARM_IRQ,
-		.end = PALMAS_RTC_ALARM_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-};
-
-static const struct resource pwron_resource[] = {
-	{
-		.name = "PWRON_BUTTON",
-		.start = PALMAS_PWRON_IRQ,
-		.end = PALMAS_PWRON_IRQ,
-		.flags = IORESOURCE_IRQ,
-	},
-};
+#include <linux/of_platform.h>
 
 enum palmas_ids {
 	PALMAS_PMIC_ID,
@@ -111,20 +58,14 @@ static const struct mfd_cell palmas_children[] = {
 	},
 	{
 		.name = "palmas-rtc",
-		.num_resources = ARRAY_SIZE(rtc_resource),
-		.resources = rtc_resource,
 		.id = PALMAS_RTC_ID,
 	},
 	{
 		.name = "palmas-pwrbutton",
-		.num_resources = ARRAY_SIZE(pwron_resource),
-		.resources = pwron_resource,
 		.id = PALMAS_PWRBUTTON_ID,
 	},
 	{
 		.name = "palmas-gpadc",
-		.num_resources = ARRAY_SIZE(gpadc_resource),
-		.resources = gpadc_resource,
 		.id = PALMAS_GPADC_ID,
 	},
 	{
@@ -141,8 +82,6 @@ static const struct mfd_cell palmas_children[] = {
 	},
 	{
 		.name = "palmas-usb",
-		.num_resources = ARRAY_SIZE(usb_resource),
-		.resources = usb_resource,
 		.id = PALMAS_USB_ID,
 	}
 };
@@ -308,17 +247,56 @@ static struct regmap_irq_chip palmas_irq_chip = {
 			PALMAS_INT1_MASK),
 };
 
+static void __devinit palmas_dt_to_pdata(struct device_node *node,
+		struct palmas_platform_data *pdata)
+{
+	int ret;
+	u32 prop;
+
+	ret = of_property_read_u32(node, "ti,mux_pad1", &prop);
+	if (!ret) {
+		pdata->mux_from_pdata = 1;
+		pdata->pad1 = prop;
+	}
+
+	ret = of_property_read_u32(node, "ti,mux_pad2", &prop);
+	if (!ret) {
+		pdata->mux_from_pdata = 1;
+		pdata->pad2 = prop;
+	}
+
+	/* The default for this register is all masked */
+	ret = of_property_read_u32(node, "ti,power_ctrl", &prop);
+	if (!ret)
+		pdata->power_ctrl = prop;
+	else
+		pdata->power_ctrl = PALMAS_POWER_CTRL_NSLEEP_MASK |
+					PALMAS_POWER_CTRL_ENABLE1_MASK |
+					PALMAS_POWER_CTRL_ENABLE2_MASK;
+}
+
 static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 			    const struct i2c_device_id *id)
 {
 	struct palmas *palmas;
 	struct palmas_platform_data *pdata;
+	struct device_node *node = i2c->dev.of_node;
 	int ret = 0, i;
 	unsigned int reg, addr;
 	int slave;
 	struct mfd_cell *children;
 
 	pdata = dev_get_platdata(&i2c->dev);
+
+	if (node && !pdata) {
+		pdata = devm_kzalloc(&i2c->dev, sizeof(*pdata), GFP_KERNEL);
+
+		if (!pdata)
+			return -ENOMEM;
+
+		palmas_dt_to_pdata(node, pdata);
+	}
+
 	if (!pdata)
 		return -EINVAL;
 
@@ -364,7 +342,7 @@ static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 	regmap_write(palmas->regmap[slave], addr, reg);
 
 	ret = regmap_add_irq_chip(palmas->regmap[slave], palmas->irq,
-			IRQF_ONESHOT | IRQF_TRIGGER_LOW, -1, &palmas_irq_chip,
+			IRQF_ONESHOT | IRQF_TRIGGER_LOW, 0, &palmas_irq_chip,
 			&palmas->irq_data);
 	if (ret < 0)
 		goto err;
@@ -377,11 +355,11 @@ static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 		reg = pdata->pad1;
 		ret = regmap_write(palmas->regmap[slave], addr, reg);
 		if (ret)
-			goto err;
+			goto err_irq;
 	} else {
 		ret = regmap_read(palmas->regmap[slave], addr, &reg);
 		if (ret)
-			goto err;
+			goto err_irq;
 	}
 
 	if (!(reg & PALMAS_PRIMARY_SECONDARY_PAD1_GPIO_0))
@@ -412,11 +390,11 @@ static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 		reg = pdata->pad2;
 		ret = regmap_write(palmas->regmap[slave], addr, reg);
 		if (ret)
-			goto err;
+			goto err_irq;
 	} else {
 		ret = regmap_read(palmas->regmap[slave], addr, &reg);
 		if (ret)
-			goto err;
+			goto err_irq;
 	}
 
 	if (!(reg & PALMAS_PRIMARY_SECONDARY_PAD2_GPIO_4))
@@ -439,18 +417,43 @@ static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 
 	ret = regmap_write(palmas->regmap[slave], addr, reg);
 	if (ret)
-		goto err;
+		goto err_irq;
+
+	/*
+	 * If we are probing with DT do this the DT way and return here
+	 * otherwise continue and add devices using mfd helpers.
+	 */
+	if (node) {
+		ret = of_platform_populate(node, NULL, NULL, &i2c->dev);
+		if (ret < 0)
+			goto err_irq;
+		else
+			return ret;
+	}
 
 	children = kmemdup(palmas_children, sizeof(palmas_children),
 			   GFP_KERNEL);
 	if (!children) {
 		ret = -ENOMEM;
-		goto err;
+		goto err_irq;
 	}
 
 	children[PALMAS_PMIC_ID].platform_data = pdata->pmic_pdata;
 	children[PALMAS_PMIC_ID].pdata_size = sizeof(*pdata->pmic_pdata);
 
+	children[PALMAS_GPADC_ID].platform_data = pdata->gpadc_pdata;
+	children[PALMAS_GPADC_ID].pdata_size = sizeof(*pdata->gpadc_pdata);
+
+	children[PALMAS_RESOURCE_ID].platform_data = pdata->resource_pdata;
+	children[PALMAS_RESOURCE_ID].pdata_size =
+			sizeof(*pdata->resource_pdata);
+
+	children[PALMAS_USB_ID].platform_data = pdata->usb_pdata;
+	children[PALMAS_USB_ID].pdata_size = sizeof(*pdata->usb_pdata);
+
+	children[PALMAS_CLK_ID].platform_data = pdata->clk_pdata;
+	children[PALMAS_CLK_ID].pdata_size = sizeof(*pdata->clk_pdata);
+
 	ret = mfd_add_devices(palmas->dev, -1,
 			      children, ARRAY_SIZE(palmas_children),
 			      NULL, regmap_irq_chip_get_base(palmas->irq_data),
@@ -458,13 +461,15 @@ static int __devinit palmas_i2c_probe(struct i2c_client *i2c,
 	kfree(children);
 
 	if (ret < 0)
-		goto err;
+		goto err_devices;
 
 	return ret;
 
-err:
+err_devices:
 	mfd_remove_devices(palmas->dev);
-	kfree(palmas);
+err_irq:
+	regmap_del_irq_chip(palmas->irq, palmas->irq_data);
+err:
 	return ret;
 }
 
diff --git a/drivers/mfd/rc5t583-irq.c b/drivers/mfd/rc5t583-irq.c
index fa6f80fad5f..fe00cdd6f83 100644
--- a/drivers/mfd/rc5t583-irq.c
+++ b/drivers/mfd/rc5t583-irq.c
@@ -255,7 +255,7 @@ static irqreturn_t rc5t583_irq(int irq, void *data)
 {
 	struct rc5t583 *rc5t583 = data;
 	uint8_t int_sts[RC5T583_MAX_INTERRUPT_MASK_REGS];
-	uint8_t master_int;
+	uint8_t master_int = 0;
 	int i;
 	int ret;
 	unsigned int rtc_int_sts = 0;
diff --git a/drivers/mfd/rc5t583.c b/drivers/mfd/rc5t583.c
index ff61efc76ce..f1a024ecdb1 100644
--- a/drivers/mfd/rc5t583.c
+++ b/drivers/mfd/rc5t583.c
@@ -85,7 +85,7 @@ static int __rc5t583_set_ext_pwrreq1_control(struct device *dev,
 	int id, int ext_pwr, int slots)
 {
 	int ret;
-	uint8_t sleepseq_val;
+	uint8_t sleepseq_val = 0;
 	unsigned int en_bit;
 	unsigned int slot_bit;
 
diff --git a/drivers/mfd/smsc-ece1099.c b/drivers/mfd/smsc-ece1099.c
new file mode 100644
index 00000000000..24ae3d8421c
--- /dev/null
+++ b/drivers/mfd/smsc-ece1099.c
@@ -0,0 +1,113 @@
+/*
+ * TI SMSC MFD Driver
+ *
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com
+ *
+ * Author: Sourav Poddar <sourav.poddar@ti.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under  the terms of the GNU General  Public License as published by the
+ *  Free Software Foundation;  GPL v2.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/gpio.h>
+#include <linux/workqueue.h>
+#include <linux/irq.h>
+#include <linux/regmap.h>
+#include <linux/err.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/smsc.h>
+#include <linux/of_platform.h>
+
+static struct regmap_config smsc_regmap_config = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = SMSC_VEN_ID_H,
+		.cache_type = REGCACHE_RBTREE,
+};
+
+static int smsc_i2c_probe(struct i2c_client *i2c,
+			const struct i2c_device_id *id)
+{
+	struct smsc *smsc;
+	int devid, rev, venid_l, venid_h;
+	int ret = 0;
+
+	smsc = devm_kzalloc(&i2c->dev, sizeof(struct smsc),
+				GFP_KERNEL);
+	if (!smsc) {
+		dev_err(&i2c->dev, "smsc mfd driver memory allocation failed\n");
+		return -ENOMEM;
+	}
+
+	smsc->regmap = devm_regmap_init_i2c(i2c, &smsc_regmap_config);
+	if (IS_ERR(smsc->regmap)) {
+		ret = PTR_ERR(smsc->regmap);
+		goto err;
+	}
+
+	i2c_set_clientdata(i2c, smsc);
+	smsc->dev = &i2c->dev;
+
+#ifdef CONFIG_OF
+	of_property_read_u32(i2c->dev.of_node, "clock", &smsc->clk);
+#endif
+
+	regmap_read(smsc->regmap, SMSC_DEV_ID, &devid);
+	regmap_read(smsc->regmap, SMSC_DEV_REV, &rev);
+	regmap_read(smsc->regmap, SMSC_VEN_ID_L, &venid_l);
+	regmap_read(smsc->regmap, SMSC_VEN_ID_H, &venid_h);
+
+	dev_info(&i2c->dev, "SMSCxxx devid: %02x rev: %02x venid: %02x\n",
+		devid, rev, (venid_h << 8) | venid_l);
+
+	ret = regmap_write(smsc->regmap, SMSC_CLK_CTRL, smsc->clk);
+	if (ret)
+		goto err;
+
+#ifdef CONFIG_OF
+	if (i2c->dev.of_node)
+		ret = of_platform_populate(i2c->dev.of_node,
+					   NULL, NULL, &i2c->dev);
+#endif
+
+err:
+	return ret;
+}
+
+static int smsc_i2c_remove(struct i2c_client *i2c)
+{
+	struct smsc *smsc = i2c_get_clientdata(i2c);
+
+	mfd_remove_devices(smsc->dev);
+
+	return 0;
+}
+
+static const struct i2c_device_id smsc_i2c_id[] = {
+	{ "smscece1099", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, smsc_i2c_id);
+
+static struct i2c_driver smsc_i2c_driver = {
+	.driver = {
+		   .name = "smsc",
+		   .owner = THIS_MODULE,
+	},
+	.probe = smsc_i2c_probe,
+	.remove = smsc_i2c_remove,
+	.id_table = smsc_i2c_id,
+};
+
+module_i2c_driver(smsc_i2c_driver);
+
+MODULE_AUTHOR("Sourav Poddar <sourav.poddar@ti.com>");
+MODULE_DESCRIPTION("SMSC chip multi-function driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c
new file mode 100644
index 00000000000..65fe609026c
--- /dev/null
+++ b/drivers/mfd/syscon.c
@@ -0,0 +1,176 @@
+/*
+ * System Control Driver
+ *
+ * Copyright (C) 2012 Freescale Semiconductor, Inc.
+ * Copyright (C) 2012 Linaro Ltd.
+ *
+ * Author: Dong Aisheng <dong.aisheng@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+static struct platform_driver syscon_driver;
+
+struct syscon {
+	struct device *dev;
+	void __iomem *base;
+	struct regmap *regmap;
+};
+
+static int syscon_match(struct device *dev, void *data)
+{
+	struct syscon *syscon = dev_get_drvdata(dev);
+	struct device_node *dn = data;
+
+	return (syscon->dev->of_node == dn) ? 1 : 0;
+}
+
+struct regmap *syscon_node_to_regmap(struct device_node *np)
+{
+	struct syscon *syscon;
+	struct device *dev;
+
+	dev = driver_find_device(&syscon_driver.driver, NULL, np,
+				 syscon_match);
+	if (!dev)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	syscon = dev_get_drvdata(dev);
+
+	return syscon->regmap;
+}
+EXPORT_SYMBOL_GPL(syscon_node_to_regmap);
+
+struct regmap *syscon_regmap_lookup_by_compatible(const char *s)
+{
+	struct device_node *syscon_np;
+	struct regmap *regmap;
+
+	syscon_np = of_find_compatible_node(NULL, NULL, s);
+	if (!syscon_np)
+		return ERR_PTR(-ENODEV);
+
+	regmap = syscon_node_to_regmap(syscon_np);
+	of_node_put(syscon_np);
+
+	return regmap;
+}
+EXPORT_SYMBOL_GPL(syscon_regmap_lookup_by_compatible);
+
+struct regmap *syscon_regmap_lookup_by_phandle(struct device_node *np,
+					const char *property)
+{
+	struct device_node *syscon_np;
+	struct regmap *regmap;
+
+	syscon_np = of_parse_phandle(np, property, 0);
+	if (!syscon_np)
+		return ERR_PTR(-ENODEV);
+
+	regmap = syscon_node_to_regmap(syscon_np);
+	of_node_put(syscon_np);
+
+	return regmap;
+}
+EXPORT_SYMBOL_GPL(syscon_regmap_lookup_by_phandle);
+
+static const struct of_device_id of_syscon_match[] = {
+	{ .compatible = "syscon", },
+	{ },
+};
+
+static struct regmap_config syscon_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static int __devinit syscon_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct syscon *syscon;
+	struct resource res;
+	int ret;
+
+	if (!np)
+		return -ENOENT;
+
+	syscon = devm_kzalloc(dev, sizeof(struct syscon),
+			    GFP_KERNEL);
+	if (!syscon)
+		return -ENOMEM;
+
+	syscon->base = of_iomap(np, 0);
+	if (!syscon->base)
+		return -EADDRNOTAVAIL;
+
+	ret = of_address_to_resource(np, 0, &res);
+	if (ret)
+		return ret;
+
+	syscon_regmap_config.max_register = res.end - res.start - 3;
+	syscon->regmap = devm_regmap_init_mmio(dev, syscon->base,
+					&syscon_regmap_config);
+	if (IS_ERR(syscon->regmap)) {
+		dev_err(dev, "regmap init failed\n");
+		return PTR_ERR(syscon->regmap);
+	}
+
+	syscon->dev = dev;
+	platform_set_drvdata(pdev, syscon);
+
+	dev_info(dev, "syscon regmap start 0x%x end 0x%x registered\n",
+		res.start, res.end);
+
+	return 0;
+}
+
+static int __devexit syscon_remove(struct platform_device *pdev)
+{
+	struct syscon *syscon;
+
+	syscon = platform_get_drvdata(pdev);
+	iounmap(syscon->base);
+	platform_set_drvdata(pdev, NULL);
+
+	return 0;
+}
+
+static struct platform_driver syscon_driver = {
+	.driver = {
+		.name = "syscon",
+		.owner = THIS_MODULE,
+		.of_match_table = of_syscon_match,
+	},
+	.probe		= syscon_probe,
+	.remove		= __devexit_p(syscon_remove),
+};
+
+static int __init syscon_init(void)
+{
+	return platform_driver_register(&syscon_driver);
+}
+postcore_initcall(syscon_init);
+
+static void __exit syscon_exit(void)
+{
+	platform_driver_unregister(&syscon_driver);
+}
+module_exit(syscon_exit);
+
+MODULE_AUTHOR("Dong Aisheng <dong.aisheng@linaro.org>");
+MODULE_DESCRIPTION("System Control driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mfd/tc3589x.c b/drivers/mfd/tc3589x.c
index b56ba6b4329..8f4c853ca11 100644
--- a/drivers/mfd/tc3589x.c
+++ b/drivers/mfd/tc3589x.c
@@ -9,8 +9,10 @@
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqdomain.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
+#include <linux/of.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/tc3589x.h>
 
@@ -145,6 +147,7 @@ static struct mfd_cell tc3589x_dev_gpio[] = {
 		.name		= "tc3589x-gpio",
 		.num_resources	= ARRAY_SIZE(gpio_resources),
 		.resources	= &gpio_resources[0],
+		.of_compatible	= "tc3589x-gpio",
 	},
 };
 
@@ -153,6 +156,7 @@ static struct mfd_cell tc3589x_dev_keypad[] = {
 		.name           = "tc3589x-keypad",
 		.num_resources  = ARRAY_SIZE(keypad_resources),
 		.resources      = &keypad_resources[0],
+		.of_compatible	= "tc3589x-keypad",
 	},
 };
 
@@ -168,8 +172,9 @@ again:
 
 	while (status) {
 		int bit = __ffs(status);
+		int virq = irq_create_mapping(tc3589x->domain, bit);
 
-		handle_nested_irq(tc3589x->irq_base + bit);
+		handle_nested_irq(virq);
 		status &= ~(1 << bit);
 	}
 
@@ -186,38 +191,60 @@ again:
 	return IRQ_HANDLED;
 }
 
-static int tc3589x_irq_init(struct tc3589x *tc3589x)
+static int tc3589x_irq_map(struct irq_domain *d, unsigned int virq,
+				irq_hw_number_t hwirq)
 {
-	int base = tc3589x->irq_base;
-	int irq;
+	struct tc3589x *tc3589x = d->host_data;
 
-	for (irq = base; irq < base + TC3589x_NR_INTERNAL_IRQS; irq++) {
-		irq_set_chip_data(irq, tc3589x);
-		irq_set_chip_and_handler(irq, &dummy_irq_chip,
-					 handle_edge_irq);
-		irq_set_nested_thread(irq, 1);
+	irq_set_chip_data(virq, tc3589x);
+	irq_set_chip_and_handler(virq, &dummy_irq_chip,
+				handle_edge_irq);
+	irq_set_nested_thread(virq, 1);
 #ifdef CONFIG_ARM
-		set_irq_flags(irq, IRQF_VALID);
+	set_irq_flags(virq, IRQF_VALID);
 #else
-		irq_set_noprobe(irq);
+	irq_set_noprobe(virq);
 #endif
-	}
 
 	return 0;
 }
 
-static void tc3589x_irq_remove(struct tc3589x *tc3589x)
+static void tc3589x_irq_unmap(struct irq_domain *d, unsigned int virq)
 {
-	int base = tc3589x->irq_base;
-	int irq;
-
-	for (irq = base; irq < base + TC3589x_NR_INTERNAL_IRQS; irq++) {
 #ifdef CONFIG_ARM
-		set_irq_flags(irq, 0);
+	set_irq_flags(virq, 0);
 #endif
-		irq_set_chip_and_handler(irq, NULL, NULL);
-		irq_set_chip_data(irq, NULL);
+	irq_set_chip_and_handler(virq, NULL, NULL);
+	irq_set_chip_data(virq, NULL);
+}
+
+static struct irq_domain_ops tc3589x_irq_ops = {
+        .map    = tc3589x_irq_map,
+	.unmap  = tc3589x_irq_unmap,
+        .xlate  = irq_domain_xlate_twocell,
+};
+
+static int tc3589x_irq_init(struct tc3589x *tc3589x, struct device_node *np)
+{
+	int base = tc3589x->irq_base;
+
+	if (base) {
+		tc3589x->domain = irq_domain_add_legacy(
+			NULL, TC3589x_NR_INTERNAL_IRQS, base,
+			0, &tc3589x_irq_ops, tc3589x);
+	}
+	else {
+		tc3589x->domain = irq_domain_add_linear(
+			np, TC3589x_NR_INTERNAL_IRQS,
+			&tc3589x_irq_ops, tc3589x);
 	}
+
+	if (!tc3589x->domain) {
+		dev_err(tc3589x->dev, "Failed to create irqdomain\n");
+		return -ENOSYS;
+	}
+
+	return 0;
 }
 
 static int tc3589x_chip_init(struct tc3589x *tc3589x)
@@ -263,7 +290,7 @@ static int __devinit tc3589x_device_init(struct tc3589x *tc3589x)
 	if (blocks & TC3589x_BLOCK_GPIO) {
 		ret = mfd_add_devices(tc3589x->dev, -1, tc3589x_dev_gpio,
 				      ARRAY_SIZE(tc3589x_dev_gpio), NULL,
-				      tc3589x->irq_base, NULL);
+				      tc3589x->irq_base, tc3589x->domain);
 		if (ret) {
 			dev_err(tc3589x->dev, "failed to add gpio child\n");
 			return ret;
@@ -274,7 +301,7 @@ static int __devinit tc3589x_device_init(struct tc3589x *tc3589x)
 	if (blocks & TC3589x_BLOCK_KEYPAD) {
 		ret = mfd_add_devices(tc3589x->dev, -1, tc3589x_dev_keypad,
 				      ARRAY_SIZE(tc3589x_dev_keypad), NULL,
-				      tc3589x->irq_base, NULL);
+				      tc3589x->irq_base, tc3589x->domain);
 		if (ret) {
 			dev_err(tc3589x->dev, "failed to keypad child\n");
 			return ret;
@@ -285,13 +312,47 @@ static int __devinit tc3589x_device_init(struct tc3589x *tc3589x)
 	return ret;
 }
 
+static int tc3589x_of_probe(struct device_node *np,
+			struct tc3589x_platform_data *pdata)
+{
+	struct device_node *child;
+
+	for_each_child_of_node(np, child) {
+		if (!strcmp(child->name, "tc3589x_gpio")) {
+			pdata->block |= TC3589x_BLOCK_GPIO;
+		}
+		if (!strcmp(child->name, "tc3589x_keypad")) {
+			pdata->block |= TC3589x_BLOCK_KEYPAD;
+		}
+	}
+
+	return 0;
+}
+
 static int __devinit tc3589x_probe(struct i2c_client *i2c,
 				   const struct i2c_device_id *id)
 {
 	struct tc3589x_platform_data *pdata = i2c->dev.platform_data;
+	struct device_node *np = i2c->dev.of_node;
 	struct tc3589x *tc3589x;
 	int ret;
 
+	if (!pdata) {
+		if (np) {
+			pdata = devm_kzalloc(&i2c->dev, sizeof(*pdata), GFP_KERNEL);
+			if (!pdata)
+				return -ENOMEM;
+
+			ret = tc3589x_of_probe(np, pdata);
+			if (ret)
+				return ret;
+		}
+		else {
+			dev_err(&i2c->dev, "No platform data or DT found\n");
+			return -EINVAL;
+		}
+	}
+
 	if (!i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_BYTE_DATA
 				     | I2C_FUNC_SMBUS_I2C_BLOCK))
 		return -EIO;
@@ -314,7 +375,7 @@ static int __devinit tc3589x_probe(struct i2c_client *i2c,
 	if (ret)
 		goto out_free;
 
-	ret = tc3589x_irq_init(tc3589x);
+	ret = tc3589x_irq_init(tc3589x, np);
 	if (ret)
 		goto out_free;
 
@@ -323,7 +384,7 @@ static int __devinit tc3589x_probe(struct i2c_client *i2c,
 				   "tc3589x", tc3589x);
 	if (ret) {
 		dev_err(tc3589x->dev, "failed to request IRQ: %d\n", ret);
-		goto out_removeirq;
+		goto out_free;
 	}
 
 	ret = tc3589x_device_init(tc3589x);
@@ -336,8 +397,6 @@ static int __devinit tc3589x_probe(struct i2c_client *i2c,
 
 out_freeirq:
 	free_irq(tc3589x->i2c->irq, tc3589x);
-out_removeirq:
-	tc3589x_irq_remove(tc3589x);
 out_free:
 	kfree(tc3589x);
 	return ret;
@@ -350,7 +409,6 @@ static int __devexit tc3589x_remove(struct i2c_client *client)
 	mfd_remove_devices(tc3589x->dev);
 
 	free_irq(tc3589x->i2c->irq, tc3589x);
-	tc3589x_irq_remove(tc3589x);
 
 	kfree(tc3589x);
 
diff --git a/drivers/mfd/tps65090.c b/drivers/mfd/tps65090.c
index 50fd87c87a1..074ae32b0d2 100644
--- a/drivers/mfd/tps65090.c
+++ b/drivers/mfd/tps65090.c
@@ -236,7 +236,7 @@ static int __devinit tps65090_irq_init(struct tps65090 *tps65090, int irq,
 
 static bool is_volatile_reg(struct device *dev, unsigned int reg)
 {
-	if ((reg == TPS65090_INT_STS) || (reg == TPS65090_INT_STS))
+	if (reg == TPS65090_INT_STS)
 		return true;
 	else
 		return false;
diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c
index a95e9421b73..3fb32e65525 100644
--- a/drivers/mfd/tps65217.c
+++ b/drivers/mfd/tps65217.c
@@ -34,6 +34,9 @@ static struct mfd_cell tps65217s[] = {
 	{
 		.name = "tps65217-pmic",
 	},
+	{
+		.name = "tps65217-bl",
+	},
 };
 
 /**
diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c
index 345960ca2fd..46746436877 100644
--- a/drivers/mfd/tps6586x.c
+++ b/drivers/mfd/tps6586x.c
@@ -30,6 +30,10 @@
 #include <linux/mfd/core.h>
 #include <linux/mfd/tps6586x.h>
 
+#define TPS6586X_SUPPLYENE	0x14
+#define EXITSLREQ_BIT		BIT(1)
+#define SLEEP_MODE_BIT		BIT(3)
+
 /* interrupt control registers */
 #define TPS6586X_INT_ACK1	0xb5
 #define TPS6586X_INT_ACK2	0xb6
@@ -422,6 +426,7 @@ static struct tps6586x_platform_data *tps6586x_parse_dt(struct i2c_client *clien
 	pdata->subdevs = devs;
 	pdata->gpio_base = -1;
 	pdata->irq_base = -1;
+	pdata->pm_off = of_property_read_bool(np, "ti,system-power-controller");
 
 	return pdata;
 }
@@ -454,6 +459,15 @@ static const struct regmap_config tps6586x_regmap_config = {
 	.cache_type = REGCACHE_RBTREE,
 };
 
+static struct device *tps6586x_dev;
+static void tps6586x_power_off(void)
+{
+	if (tps6586x_clr_bits(tps6586x_dev, TPS6586X_SUPPLYENE, EXITSLREQ_BIT))
+		return;
+
+	tps6586x_set_bits(tps6586x_dev, TPS6586X_SUPPLYENE, SLEEP_MODE_BIT);
+}
+
 static int __devinit tps6586x_i2c_probe(struct i2c_client *client,
 					const struct i2c_device_id *id)
 {
@@ -519,6 +533,11 @@ static int __devinit tps6586x_i2c_probe(struct i2c_client *client,
 		goto err_add_devs;
 	}
 
+	if (pdata->pm_off && !pm_power_off) {
+		tps6586x_dev = &client->dev;
+		pm_power_off = tps6586x_power_off;
+	}
+
 	return 0;
 
 err_add_devs:
diff --git a/drivers/mfd/tps65910.c b/drivers/mfd/tps65910.c
index d3ce4d569de..0d79ce2b501 100644
--- a/drivers/mfd/tps65910.c
+++ b/drivers/mfd/tps65910.c
@@ -24,6 +24,14 @@
 #include <linux/mfd/tps65910.h>
 #include <linux/of_device.h>
 
+static struct resource rtc_resources[] = {
+	{
+		.start  = TPS65910_IRQ_RTC_ALARM,
+		.end    = TPS65910_IRQ_RTC_ALARM,
+		.flags  = IORESOURCE_IRQ,
+	}
+};
+
 static struct mfd_cell tps65910s[] = {
 	{
 		.name = "tps65910-gpio",
@@ -33,6 +41,8 @@ static struct mfd_cell tps65910s[] = {
 	},
 	{
 		.name = "tps65910-rtc",
+		.num_resources = ARRAY_SIZE(rtc_resources),
+		.resources = &rtc_resources[0],
 	},
 	{
 		.name = "tps65910-power",
@@ -198,6 +208,8 @@ static struct tps65910_board *tps65910_parse_dt(struct i2c_client *client,
 
 	board_info->irq = client->irq;
 	board_info->irq_base = -1;
+	board_info->pm_off = of_property_read_bool(np,
+			"ti,system-power-controller");
 
 	return board_info;
 }
@@ -210,6 +222,21 @@ struct tps65910_board *tps65910_parse_dt(struct i2c_client *client,
 }
 #endif
 
+static struct i2c_client *tps65910_i2c_client;
+static void tps65910_power_off(void)
+{
+	struct tps65910 *tps65910;
+
+	tps65910 = dev_get_drvdata(&tps65910_i2c_client->dev);
+
+	if (tps65910_reg_set_bits(tps65910, TPS65910_DEVCTRL,
+			DEVCTRL_PWR_OFF_MASK) < 0)
+		return;
+
+	tps65910_reg_clear_bits(tps65910, TPS65910_DEVCTRL,
+			DEVCTRL_DEV_ON_MASK);
+}
+
 static __devinit int tps65910_i2c_probe(struct i2c_client *i2c,
 					const struct i2c_device_id *id)
 {
@@ -267,6 +294,11 @@ static __devinit int tps65910_i2c_probe(struct i2c_client *i2c,
 	tps65910_ck32k_init(tps65910, pmic_plat_data);
 	tps65910_sleepinit(tps65910, pmic_plat_data);
 
+	if (pmic_plat_data->pm_off && !pm_power_off) {
+		tps65910_i2c_client = i2c;
+		pm_power_off = tps65910_power_off;
+	}
+
 	return ret;
 }
 
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index 9d3a0bc1a65..4ae64232020 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -63,70 +63,6 @@
 
 #define DRIVER_NAME			"twl"
 
-#if defined(CONFIG_KEYBOARD_TWL4030) || defined(CONFIG_KEYBOARD_TWL4030_MODULE)
-#define twl_has_keypad()	true
-#else
-#define twl_has_keypad()	false
-#endif
-
-#if defined(CONFIG_GPIO_TWL4030) || defined(CONFIG_GPIO_TWL4030_MODULE)
-#define twl_has_gpio()	true
-#else
-#define twl_has_gpio()	false
-#endif
-
-#if defined(CONFIG_REGULATOR_TWL4030) \
-	|| defined(CONFIG_REGULATOR_TWL4030_MODULE)
-#define twl_has_regulator()	true
-#else
-#define twl_has_regulator()	false
-#endif
-
-#if defined(CONFIG_TWL4030_MADC) || defined(CONFIG_TWL4030_MADC_MODULE)
-#define twl_has_madc()	true
-#else
-#define twl_has_madc()	false
-#endif
-
-#ifdef CONFIG_TWL4030_POWER
-#define twl_has_power()        true
-#else
-#define twl_has_power()        false
-#endif
-
-#if defined(CONFIG_RTC_DRV_TWL4030) || defined(CONFIG_RTC_DRV_TWL4030_MODULE)
-#define twl_has_rtc()	true
-#else
-#define twl_has_rtc()	false
-#endif
-
-#if defined(CONFIG_TWL4030_USB) || defined(CONFIG_TWL4030_USB_MODULE) ||\
-	defined(CONFIG_TWL6030_USB) || defined(CONFIG_TWL6030_USB_MODULE)
-#define twl_has_usb()	true
-#else
-#define twl_has_usb()	false
-#endif
-
-#if defined(CONFIG_TWL4030_WATCHDOG) || \
-	defined(CONFIG_TWL4030_WATCHDOG_MODULE)
-#define twl_has_watchdog()        true
-#else
-#define twl_has_watchdog()        false
-#endif
-
-#if defined(CONFIG_MFD_TWL4030_AUDIO) || \
-	defined(CONFIG_MFD_TWL4030_AUDIO_MODULE)
-#define twl_has_codec()	true
-#else
-#define twl_has_codec()	false
-#endif
-
-#if defined(CONFIG_CHARGER_TWL4030) || defined(CONFIG_CHARGER_TWL4030_MODULE)
-#define twl_has_bci()	true
-#else
-#define twl_has_bci()	false
-#endif
-
 /* Triton Core internal information (BEGIN) */
 
 /* Last - for index max*/
@@ -134,13 +70,6 @@
 
 #define TWL_NUM_SLAVES		4
 
-#if defined(CONFIG_INPUT_TWL4030_PWRBUTTON) \
-	|| defined(CONFIG_INPUT_TWL4030_PWRBUTTON_MODULE)
-#define twl_has_pwrbutton()	true
-#else
-#define twl_has_pwrbutton()	false
-#endif
-
 #define SUB_CHIP_ID0 0
 #define SUB_CHIP_ID1 1
 #define SUB_CHIP_ID2 2
@@ -552,6 +481,38 @@ int twl_get_version(void)
 }
 EXPORT_SYMBOL_GPL(twl_get_version);
 
+/**
+ * twl_get_hfclk_rate - API to get TWL external HFCLK clock rate.
+ *
+ * Api to get the TWL HFCLK rate based on BOOT_CFG register.
+ */
+int twl_get_hfclk_rate(void)
+{
+	u8 ctrl;
+	int rate;
+
+	twl_i2c_read_u8(TWL_MODULE_PM_MASTER, &ctrl, R_CFG_BOOT);
+
+	switch (ctrl & 0x3) {
+	case HFCLK_FREQ_19p2_MHZ:
+		rate = 19200000;
+		break;
+	case HFCLK_FREQ_26_MHZ:
+		rate = 26000000;
+		break;
+	case HFCLK_FREQ_38p4_MHZ:
+		rate = 38400000;
+		break;
+	default:
+		pr_err("TWL4030: HFCLK is not configured\n");
+		rate = -EINVAL;
+		break;
+	}
+
+	return rate;
+}
+EXPORT_SYMBOL_GPL(twl_get_hfclk_rate);
+
 static struct device *
 add_numbered_child(unsigned chip, const char *name, int num,
 		void *pdata, unsigned pdata_len,
@@ -669,7 +630,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	struct device	*child;
 	unsigned sub_chip_id;
 
-	if (twl_has_gpio() && pdata->gpio) {
+	if (IS_ENABLED(CONFIG_GPIO_TWL4030) && pdata->gpio) {
 		child = add_child(SUB_CHIP_ID1, "twl4030_gpio",
 				pdata->gpio, sizeof(*pdata->gpio),
 				false, irq_base + GPIO_INTR_OFFSET, 0);
@@ -677,7 +638,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_keypad() && pdata->keypad) {
+	if (IS_ENABLED(CONFIG_KEYBOARD_TWL4030) && pdata->keypad) {
 		child = add_child(SUB_CHIP_ID2, "twl4030_keypad",
 				pdata->keypad, sizeof(*pdata->keypad),
 				true, irq_base + KEYPAD_INTR_OFFSET, 0);
@@ -685,7 +646,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_madc() && pdata->madc) {
+	if (IS_ENABLED(CONFIG_TWL4030_MADC) && pdata->madc) {
 		child = add_child(2, "twl4030_madc",
 				pdata->madc, sizeof(*pdata->madc),
 				true, irq_base + MADC_INTR_OFFSET, 0);
@@ -693,7 +654,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_rtc()) {
+	if (IS_ENABLED(CONFIG_RTC_DRV_TWL4030)) {
 		/*
 		 * REVISIT platform_data here currently might expose the
 		 * "msecure" line ... but for now we just expect board
@@ -709,7 +670,15 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_usb() && pdata->usb && twl_class_is_4030()) {
+	if (IS_ENABLED(CONFIG_PWM_TWL6030) && twl_class_is_6030()) {
+		child = add_child(TWL6030_MODULE_ID1, "twl6030-pwm", NULL, 0,
+				  false, 0, 0);
+		if (IS_ERR(child))
+			return PTR_ERR(child);
+	}
+
+	if (IS_ENABLED(CONFIG_TWL4030_USB) && pdata->usb &&
+	    twl_class_is_4030()) {
 
 		static struct regulator_consumer_supply usb1v5 = {
 			.supply =	"usb1v5",
@@ -723,7 +692,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 		};
 
 	/* First add the regulators so that they can be used by transceiver */
-		if (twl_has_regulator()) {
+		if (IS_ENABLED(CONFIG_REGULATOR_TWL4030)) {
 			/* this is a template that gets copied */
 			struct regulator_init_data usb_fixed = {
 				.constraints.valid_modes_mask =
@@ -765,18 +734,19 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 			return PTR_ERR(child);
 
 		/* we need to connect regulators to this transceiver */
-		if (twl_has_regulator() && child) {
+		if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && child) {
 			usb1v5.dev_name = dev_name(child);
 			usb1v8.dev_name = dev_name(child);
 			usb3v1[0].dev_name = dev_name(child);
 		}
 	}
-	if (twl_has_usb() && pdata->usb && twl_class_is_6030()) {
+	if (IS_ENABLED(CONFIG_TWL6030_USB) && pdata->usb &&
+	    twl_class_is_6030()) {
 
 		static struct regulator_consumer_supply usb3v3;
 		int regulator;
 
-		if (twl_has_regulator()) {
+		if (IS_ENABLED(CONFIG_REGULATOR_TWL4030)) {
 			/* this is a template that gets copied */
 			struct regulator_init_data usb_fixed = {
 				.constraints.valid_modes_mask =
@@ -813,9 +783,10 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 		if (IS_ERR(child))
 			return PTR_ERR(child);
 		/* we need to connect regulators to this transceiver */
-		if (twl_has_regulator() && child)
+		if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && child)
 			usb3v3.dev_name = dev_name(child);
-	} else if (twl_has_regulator() && twl_class_is_6030()) {
+	} else if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) &&
+		   twl_class_is_6030()) {
 		if (features & TWL6025_SUBCLASS)
 			child = add_regulator(TWL6025_REG_LDOUSB,
 						pdata->ldousb, features);
@@ -827,20 +798,21 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 					return PTR_ERR(child);
 	}
 
-	if (twl_has_watchdog() && twl_class_is_4030()) {
+	if (IS_ENABLED(CONFIG_TWL4030_WATCHDOG) && twl_class_is_4030()) {
 		child = add_child(0, "twl4030_wdt", NULL, 0, false, 0, 0);
 		if (IS_ERR(child))
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_pwrbutton() && twl_class_is_4030()) {
+	if (IS_ENABLED(CONFIG_INPUT_TWL4030_PWRBUTTON) && twl_class_is_4030()) {
 		child = add_child(1, "twl4030_pwrbutton",
 				NULL, 0, true, irq_base + 8 + 0, 0);
 		if (IS_ERR(child))
 			return PTR_ERR(child);
 	}
 
-	if (twl_has_codec() && pdata->audio && twl_class_is_4030()) {
+	if (IS_ENABLED(CONFIG_MFD_TWL4030_AUDIO) && pdata->audio &&
+	    twl_class_is_4030()) {
 		sub_chip_id = twl_map[TWL_MODULE_AUDIO_VOICE].sid;
 		child = add_child(sub_chip_id, "twl4030-audio",
 				pdata->audio, sizeof(*pdata->audio),
@@ -850,7 +822,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	}
 
 	/* twl4030 regulators */
-	if (twl_has_regulator() && twl_class_is_4030()) {
+	if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && twl_class_is_4030()) {
 		child = add_regulator(TWL4030_REG_VPLL1, pdata->vpll1,
 					features);
 		if (IS_ERR(child))
@@ -905,7 +877,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	}
 
 	/* maybe add LDOs that are omitted on cost-reduced parts */
-	if (twl_has_regulator() && !(features & TPS_SUBSET)
+	if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && !(features & TPS_SUBSET)
 	  && twl_class_is_4030()) {
 		child = add_regulator(TWL4030_REG_VPLL2, pdata->vpll2,
 					features);
@@ -939,7 +911,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	}
 
 	/* twl6030 regulators */
-	if (twl_has_regulator() && twl_class_is_6030() &&
+	if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && twl_class_is_6030() &&
 			!(features & TWL6025_SUBCLASS)) {
 		child = add_regulator(TWL6030_REG_VDD1, pdata->vdd1,
 					features);
@@ -1013,7 +985,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	}
 
 	/* 6030 and 6025 share this regulator */
-	if (twl_has_regulator() && twl_class_is_6030()) {
+	if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && twl_class_is_6030()) {
 		child = add_regulator(TWL6030_REG_VANA, pdata->vana,
 					features);
 		if (IS_ERR(child))
@@ -1021,7 +993,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 	}
 
 	/* twl6025 regulators */
-	if (twl_has_regulator() && twl_class_is_6030() &&
+	if (IS_ENABLED(CONFIG_REGULATOR_TWL4030) && twl_class_is_6030() &&
 			(features & TWL6025_SUBCLASS)) {
 		child = add_regulator(TWL6025_REG_LDO5, pdata->ldo5,
 					features);
@@ -1080,7 +1052,7 @@ add_children(struct twl4030_platform_data *pdata, unsigned irq_base,
 
 	}
 
-	if (twl_has_bci() && pdata->bci &&
+	if (IS_ENABLED(CONFIG_CHARGER_TWL4030) && pdata->bci &&
 			!(features & (TPS_SUBSET | TWL5031))) {
 		child = add_child(3, "twl4030_bci",
 				pdata->bci, sizeof(*pdata->bci), false,
@@ -1295,7 +1267,7 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	}
 
 	/* load power event scripts */
-	if (twl_has_power() && pdata->power)
+	if (IS_ENABLED(CONFIG_TWL4030_POWER) && pdata->power)
 		twl4030_power_init(pdata->power);
 
 	/* Maybe init the T2 Interrupt subsystem */
diff --git a/drivers/mfd/twl4030-audio.c b/drivers/mfd/twl4030-audio.c
index 77c9acb1458..5c11acf9e0f 100644
--- a/drivers/mfd/twl4030-audio.c
+++ b/drivers/mfd/twl4030-audio.c
@@ -28,6 +28,8 @@
 #include <linux/kernel.h>
 #include <linux/fs.h>
 #include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/i2c/twl.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/twl4030-audio.h>
@@ -156,47 +158,70 @@ unsigned int twl4030_audio_get_mclk(void)
 }
 EXPORT_SYMBOL_GPL(twl4030_audio_get_mclk);
 
+static bool twl4030_audio_has_codec(struct twl4030_audio_data *pdata,
+			      struct device_node *node)
+{
+	if (pdata && pdata->codec)
+		return true;
+
+	if (of_find_node_by_name(node, "codec"))
+		return true;
+
+	return false;
+}
+
+static bool twl4030_audio_has_vibra(struct twl4030_audio_data *pdata,
+			      struct device_node *node)
+{
+	int vibra;
+
+	if (pdata && pdata->vibra)
+		return true;
+
+	if (!of_property_read_u32(node, "ti,enable-vibra", &vibra) && vibra)
+		return true;
+
+	return false;
+}
+
 static int __devinit twl4030_audio_probe(struct platform_device *pdev)
 {
 	struct twl4030_audio *audio;
 	struct twl4030_audio_data *pdata = pdev->dev.platform_data;
+	struct device_node *node = pdev->dev.of_node;
 	struct mfd_cell *cell = NULL;
 	int ret, childs = 0;
 	u8 val;
 
-	if (!pdata) {
+	if (!pdata && !node) {
 		dev_err(&pdev->dev, "Platform data is missing\n");
 		return -EINVAL;
 	}
 
+	audio = devm_kzalloc(&pdev->dev, sizeof(struct twl4030_audio),
+			     GFP_KERNEL);
+	if (!audio)
+		return -ENOMEM;
+
+	mutex_init(&audio->mutex);
+	audio->audio_mclk = twl_get_hfclk_rate();
+
 	/* Configure APLL_INFREQ and disable APLL if enabled */
-	val = 0;
-	switch (pdata->audio_mclk) {
+	switch (audio->audio_mclk) {
 	case 19200000:
-		val |= TWL4030_APLL_INFREQ_19200KHZ;
+		val = TWL4030_APLL_INFREQ_19200KHZ;
 		break;
 	case 26000000:
-		val |= TWL4030_APLL_INFREQ_26000KHZ;
+		val = TWL4030_APLL_INFREQ_26000KHZ;
 		break;
 	case 38400000:
-		val |= TWL4030_APLL_INFREQ_38400KHZ;
+		val = TWL4030_APLL_INFREQ_38400KHZ;
 		break;
 	default:
 		dev_err(&pdev->dev, "Invalid audio_mclk\n");
 		return -EINVAL;
 	}
-	twl_i2c_write_u8(TWL4030_MODULE_AUDIO_VOICE,
-					val, TWL4030_REG_APLL_CTL);
-
-	audio = kzalloc(sizeof(struct twl4030_audio), GFP_KERNEL);
-	if (!audio)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, audio);
-
-	twl4030_audio_dev = pdev;
-	mutex_init(&audio->mutex);
-	audio->audio_mclk = pdata->audio_mclk;
+	twl_i2c_write_u8(TWL4030_MODULE_AUDIO_VOICE, val, TWL4030_REG_APLL_CTL);
 
 	/* Codec power */
 	audio->resource[TWL4030_AUDIO_RES_POWER].reg = TWL4030_REG_CODEC_MODE;
@@ -206,21 +231,28 @@ static int __devinit twl4030_audio_probe(struct platform_device *pdev)
 	audio->resource[TWL4030_AUDIO_RES_APLL].reg = TWL4030_REG_APLL_CTL;
 	audio->resource[TWL4030_AUDIO_RES_APLL].mask = TWL4030_APLL_EN;
 
-	if (pdata->codec) {
+	if (twl4030_audio_has_codec(pdata, node)) {
 		cell = &audio->cells[childs];
 		cell->name = "twl4030-codec";
-		cell->platform_data = pdata->codec;
-		cell->pdata_size = sizeof(*pdata->codec);
+		if (pdata) {
+			cell->platform_data = pdata->codec;
+			cell->pdata_size = sizeof(*pdata->codec);
+		}
 		childs++;
 	}
-	if (pdata->vibra) {
+	if (twl4030_audio_has_vibra(pdata, node)) {
 		cell = &audio->cells[childs];
 		cell->name = "twl4030-vibra";
-		cell->platform_data = pdata->vibra;
-		cell->pdata_size = sizeof(*pdata->vibra);
+		if (pdata) {
+			cell->platform_data = pdata->vibra;
+			cell->pdata_size = sizeof(*pdata->vibra);
+		}
 		childs++;
 	}
 
+	platform_set_drvdata(pdev, audio);
+	twl4030_audio_dev = pdev;
+
 	if (childs)
 		ret = mfd_add_devices(&pdev->dev, pdev->id, audio->cells,
 				      childs, NULL, 0, NULL);
@@ -229,39 +261,42 @@ static int __devinit twl4030_audio_probe(struct platform_device *pdev)
 		ret = -ENODEV;
 	}
 
-	if (!ret)
-		return 0;
+	if (ret) {
+		platform_set_drvdata(pdev, NULL);
+		twl4030_audio_dev = NULL;
+	}
 
-	platform_set_drvdata(pdev, NULL);
-	kfree(audio);
-	twl4030_audio_dev = NULL;
 	return ret;
 }
 
 static int __devexit twl4030_audio_remove(struct platform_device *pdev)
 {
-	struct twl4030_audio *audio = platform_get_drvdata(pdev);
-
 	mfd_remove_devices(&pdev->dev);
 	platform_set_drvdata(pdev, NULL);
-	kfree(audio);
 	twl4030_audio_dev = NULL;
 
 	return 0;
 }
 
-MODULE_ALIAS("platform:twl4030-audio");
+static const struct of_device_id twl4030_audio_of_match[] = {
+	{.compatible = "ti,twl4030-audio", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, twl4030_audio_of_match);
 
 static struct platform_driver twl4030_audio_driver = {
-	.probe		= twl4030_audio_probe,
-	.remove		= __devexit_p(twl4030_audio_remove),
 	.driver		= {
 		.owner	= THIS_MODULE,
 		.name	= "twl4030-audio",
+		.of_match_table = twl4030_audio_of_match,
 	},
+	.probe		= twl4030_audio_probe,
+	.remove		= __devexit_p(twl4030_audio_remove),
 };
 
 module_platform_driver(twl4030_audio_driver);
 
 MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>");
+MODULE_DESCRIPTION("TWL4030 audio block MFD driver");
 MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:twl4030-audio");
diff --git a/drivers/mfd/twl6040-core.c b/drivers/mfd/twl6040-core.c
index 3dca5c195a2..3f2a1cf02fc 100644
--- a/drivers/mfd/twl6040-core.c
+++ b/drivers/mfd/twl6040-core.c
@@ -584,7 +584,7 @@ static int __devinit twl6040_probe(struct i2c_client *client,
 		goto irq_init_err;
 
 	ret = request_threaded_irq(twl6040->irq_base + TWL6040_IRQ_READY,
-				   NULL, twl6040_naudint_handler, 0,
+				   NULL, twl6040_naudint_handler, IRQF_ONESHOT,
 				   "twl6040_irq_ready", twl6040);
 	if (ret) {
 		dev_err(twl6040->dev, "READY IRQ request failed: %d\n",
@@ -631,6 +631,21 @@ static int __devinit twl6040_probe(struct i2c_client *client,
 		children++;
 	}
 
+	/*
+	 * Enable the GPO driver in the following cases:
+	 * DT booted kernel or legacy boot with valid gpo platform_data
+	 */
+	if (!pdata || (pdata && pdata->gpo)) {
+		cell = &twl6040->cells[children];
+		cell->name = "twl6040-gpo";
+
+		if (pdata) {
+			cell->platform_data = pdata->gpo;
+			cell->pdata_size = sizeof(*pdata->gpo);
+		}
+		children++;
+	}
+
 	ret = mfd_add_devices(&client->dev, -1, twl6040->cells, children,
 			      NULL, 0, NULL);
 	if (ret)
diff --git a/drivers/mfd/wm5110-tables.c b/drivers/mfd/wm5110-tables.c
index bd8782c8896..adda6b10b90 100644
--- a/drivers/mfd/wm5110-tables.c
+++ b/drivers/mfd/wm5110-tables.c
@@ -133,15 +133,109 @@ static const struct reg_default wm5110_reva_patch[] = {
 	{ 0x209, 0x002A },
 };
 
+static const struct reg_default wm5110_revb_patch[] = {
+	{ 0x80, 0x3 },
+	{ 0x36e, 0x0210 },
+	{ 0x370, 0x0210 },
+	{ 0x372, 0x0210 },
+	{ 0x374, 0x0210 },
+	{ 0x376, 0x0210 },
+	{ 0x378, 0x0210 },
+	{ 0x36d, 0x0028 },
+	{ 0x36f, 0x0028 },
+	{ 0x371, 0x0028 },
+	{ 0x373, 0x0028 },
+	{ 0x375, 0x0028 },
+	{ 0x377, 0x0028 },
+	{ 0x280, 0x2002 },
+	{ 0x44, 0x20 },
+	{ 0x45, 0x40 },
+	{ 0x46, 0x60 },
+	{ 0x47, 0x80 },
+	{ 0x48, 0xa0 },
+	{ 0x51, 0x13 },
+	{ 0x52, 0x33 },
+	{ 0x53, 0x53 },
+	{ 0x54, 0x73 },
+	{ 0x55, 0x93 },
+	{ 0x56, 0xb3 },
+	{ 0xc30, 0x3e3e },
+	{ 0xc31, 0x3e },
+	{ 0xc32, 0x3e3e },
+	{ 0xc33, 0x3e3e },
+	{ 0xc34, 0x3e3e },
+	{ 0xc35, 0x3e3e },
+	{ 0xc36, 0x3e3e },
+	{ 0xc37, 0x3e3e },
+	{ 0xc38, 0x3e3e },
+	{ 0xc39, 0x3e3e },
+	{ 0xc3a, 0x3e3e },
+	{ 0xc3b, 0x3e3e },
+	{ 0xc3c, 0x3e },
+	{ 0x201, 0x18a5 },
+	{ 0x202, 0x4100 },
+	{ 0x460, 0x0c40 },
+	{ 0x461, 0x8000 },
+	{ 0x462, 0x0c41 },
+	{ 0x463, 0x4820 },
+	{ 0x464, 0x0c41 },
+	{ 0x465, 0x4040 },
+	{ 0x466, 0x0841 },
+	{ 0x467, 0x3940 },
+	{ 0x468, 0x0841 },
+	{ 0x469, 0x2030 },
+	{ 0x46a, 0x0842 },
+	{ 0x46b, 0x1990 },
+	{ 0x46c, 0x08c2 },
+	{ 0x46d, 0x1450 },
+	{ 0x46e, 0x08c6 },
+	{ 0x46f, 0x1020 },
+	{ 0x470, 0x08c6 },
+	{ 0x471, 0x0cd0 },
+	{ 0x472, 0x08c6 },
+	{ 0x473, 0x0a30 },
+	{ 0x474, 0x0442 },
+	{ 0x475, 0x0660 },
+	{ 0x476, 0x0446 },
+	{ 0x477, 0x0510 },
+	{ 0x478, 0x04c6 },
+	{ 0x479, 0x0400 },
+	{ 0x47a, 0x04ce },
+	{ 0x47b, 0x0330 },
+	{ 0x47c, 0x05df },
+	{ 0x47d, 0x0001 },
+	{ 0x47e, 0x07ff },
+	{ 0x2db, 0x0a00 },
+	{ 0x2dd, 0x0023 },
+	{ 0x2df, 0x0102 },
+	{ 0x2ef, 0x924 },
+	{ 0x2f0, 0x924 },
+	{ 0x2f1, 0x924 },
+	{ 0x2f2, 0x924 },
+	{ 0x2f3, 0x924 },
+	{ 0x2f4, 0x924 },
+	{ 0x2eb, 0x60 },
+	{ 0x2ec, 0x60 },
+	{ 0x2ed, 0x60 },
+	{ 0x4f2, 0x33e },
+	{ 0x458, 0x0000 },
+	{ 0x15a, 0x0003 },
+	{ 0x80, 0x0 },
+};
+
 /* We use a function so we can use ARRAY_SIZE() */
 int wm5110_patch(struct arizona *arizona)
 {
 	switch (arizona->rev) {
 	case 0:
-	case 1:
 		return regmap_register_patch(arizona->regmap,
 					     wm5110_reva_patch,
 					     ARRAY_SIZE(wm5110_reva_patch));
+	case 1:
+		return regmap_register_patch(arizona->regmap,
+					     wm5110_revb_patch,
+					     ARRAY_SIZE(wm5110_revb_patch));
+
 	default:
 		return 0;
 	}
diff --git a/drivers/mfd/wm831x-core.c b/drivers/mfd/wm831x-core.c
index 30173103594..521340a708d 100644
--- a/drivers/mfd/wm831x-core.c
+++ b/drivers/mfd/wm831x-core.c
@@ -614,18 +614,11 @@ int wm831x_set_bits(struct wm831x *wm831x, unsigned short reg,
 }
 EXPORT_SYMBOL_GPL(wm831x_set_bits);
 
-static struct resource wm831x_io_parent = {
-	.start = 0,
-	.end   = 0xffffffff,
-	.flags = IORESOURCE_IO,
-};
-
 static struct resource wm831x_dcdc1_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_DC1_CONTROL_1,
 		.end   = WM831X_DC1_DVS_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -644,10 +637,9 @@ static struct resource wm831x_dcdc1_resources[] = {
 
 static struct resource wm831x_dcdc2_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_DC2_CONTROL_1,
 		.end   = WM831X_DC2_DVS_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -665,10 +657,9 @@ static struct resource wm831x_dcdc2_resources[] = {
 
 static struct resource wm831x_dcdc3_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_DC3_CONTROL_1,
 		.end   = WM831X_DC3_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -680,10 +671,9 @@ static struct resource wm831x_dcdc3_resources[] = {
 
 static struct resource wm831x_dcdc4_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_DC4_CONTROL,
 		.end   = WM831X_DC4_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -695,10 +685,9 @@ static struct resource wm831x_dcdc4_resources[] = {
 
 static struct resource wm8320_dcdc4_buck_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_DC4_CONTROL,
 		.end   = WM832X_DC4_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -718,10 +707,9 @@ static struct resource wm831x_gpio_resources[] = {
 
 static struct resource wm831x_isink1_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_CURRENT_SINK_1,
 		.end   = WM831X_CURRENT_SINK_1,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.start = WM831X_IRQ_CS1,
@@ -732,10 +720,9 @@ static struct resource wm831x_isink1_resources[] = {
 
 static struct resource wm831x_isink2_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_CURRENT_SINK_2,
 		.end   = WM831X_CURRENT_SINK_2,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.start = WM831X_IRQ_CS2,
@@ -746,10 +733,9 @@ static struct resource wm831x_isink2_resources[] = {
 
 static struct resource wm831x_ldo1_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO1_CONTROL,
 		.end   = WM831X_LDO1_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -761,10 +747,9 @@ static struct resource wm831x_ldo1_resources[] = {
 
 static struct resource wm831x_ldo2_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO2_CONTROL,
 		.end   = WM831X_LDO2_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -776,10 +761,9 @@ static struct resource wm831x_ldo2_resources[] = {
 
 static struct resource wm831x_ldo3_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO3_CONTROL,
 		.end   = WM831X_LDO3_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -791,10 +775,9 @@ static struct resource wm831x_ldo3_resources[] = {
 
 static struct resource wm831x_ldo4_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO4_CONTROL,
 		.end   = WM831X_LDO4_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -806,10 +789,9 @@ static struct resource wm831x_ldo4_resources[] = {
 
 static struct resource wm831x_ldo5_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO5_CONTROL,
 		.end   = WM831X_LDO5_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -821,10 +803,9 @@ static struct resource wm831x_ldo5_resources[] = {
 
 static struct resource wm831x_ldo6_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO6_CONTROL,
 		.end   = WM831X_LDO6_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -836,10 +817,9 @@ static struct resource wm831x_ldo6_resources[] = {
 
 static struct resource wm831x_ldo7_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO7_CONTROL,
 		.end   = WM831X_LDO7_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -851,10 +831,9 @@ static struct resource wm831x_ldo7_resources[] = {
 
 static struct resource wm831x_ldo8_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO8_CONTROL,
 		.end   = WM831X_LDO8_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -866,10 +845,9 @@ static struct resource wm831x_ldo8_resources[] = {
 
 static struct resource wm831x_ldo9_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO9_CONTROL,
 		.end   = WM831X_LDO9_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -881,10 +859,9 @@ static struct resource wm831x_ldo9_resources[] = {
 
 static struct resource wm831x_ldo10_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO10_CONTROL,
 		.end   = WM831X_LDO10_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 	{
 		.name  = "UV",
@@ -896,10 +873,9 @@ static struct resource wm831x_ldo10_resources[] = {
 
 static struct resource wm831x_ldo11_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_LDO11_ON_CONTROL,
 		.end   = WM831X_LDO11_SLEEP_CONTROL,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 };
 
@@ -998,19 +974,17 @@ static struct resource wm831x_rtc_resources[] = {
 
 static struct resource wm831x_status1_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_STATUS_LED_1,
 		.end   = WM831X_STATUS_LED_1,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 };
 
 static struct resource wm831x_status2_resources[] = {
 	{
-		.parent = &wm831x_io_parent,
 		.start = WM831X_STATUS_LED_2,
 		.end   = WM831X_STATUS_LED_2,
-		.flags = IORESOURCE_IO,
+		.flags = IORESOURCE_REG,
 	},
 };
 
diff --git a/drivers/mfd/wm8994-core.c b/drivers/mfd/wm8994-core.c
index 2febf88cfce..8fefc961ec0 100644
--- a/drivers/mfd/wm8994-core.c
+++ b/drivers/mfd/wm8994-core.c
@@ -374,23 +374,23 @@ static int wm8994_ldo_in_use(struct wm8994_pdata *pdata, int ldo)
 }
 #endif
 
-static const __devinitdata struct reg_default wm8994_revc_patch[] = {
+static const __devinitconst struct reg_default wm8994_revc_patch[] = {
 	{ 0x102, 0x3 },
 	{ 0x56, 0x3 },
 	{ 0x817, 0x0 },
 	{ 0x102, 0x0 },
 };
 
-static const __devinitdata struct reg_default wm8958_reva_patch[] = {
+static const __devinitconst struct reg_default wm8958_reva_patch[] = {
 	{ 0x102, 0x3 },
 	{ 0xcb, 0x81 },
 	{ 0x817, 0x0 },
 	{ 0x102, 0x0 },
 };
 
-static const __devinitdata struct reg_default wm1811_reva_patch[] = {
+static const __devinitconst struct reg_default wm1811_reva_patch[] = {
 	{ 0x102, 0x3 },
-	{ 0x56, 0x7 },
+	{ 0x56, 0xc07 },
 	{ 0x5d, 0x7e },
 	{ 0x5e, 0x0 },
 	{ 0x102, 0x0 },
diff --git a/drivers/mfd/wm8994-regmap.c b/drivers/mfd/wm8994-regmap.c
index 52e9e294494..2fbce9c5950 100644
--- a/drivers/mfd/wm8994-regmap.c
+++ b/drivers/mfd/wm8994-regmap.c
@@ -1136,7 +1136,7 @@ static bool wm1811_volatile_register(struct device *dev, unsigned int reg)
 
 	switch (reg) {
 	case WM8994_GPIO_6:
-		if (wm8994->revision > 1)
+		if (wm8994->cust_id > 1 || wm8994->revision > 1)
 			return true;
 		else
 			return false;
diff --git a/drivers/mmc/host/sdhci-pci.c b/drivers/mmc/host/sdhci-pci.c
index 504da715a41..9722d43d614 100644
--- a/drivers/mmc/host/sdhci-pci.c
+++ b/drivers/mmc/host/sdhci-pci.c
@@ -653,7 +653,7 @@ static const struct sdhci_pci_fixes sdhci_via = {
 	.probe		= via_probe,
 };
 
-static const struct pci_device_id pci_ids[] __devinitdata = {
+static const struct pci_device_id pci_ids[] __devinitconst = {
 	{
 		.vendor		= PCI_VENDOR_ID_RICOH,
 		.device		= PCI_DEVICE_ID_RICOH_R5C822,
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index d518e4db8a0..3a49e6de5e6 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -744,7 +744,7 @@ int parse_mtd_partitions(struct mtd_info *master, const char **types,
 	return ret;
 }
 
-int mtd_is_partition(struct mtd_info *mtd)
+int mtd_is_partition(const struct mtd_info *mtd)
 {
 	struct mtd_part *part;
 	int ispart = 0;
@@ -760,3 +760,13 @@ int mtd_is_partition(struct mtd_info *mtd)
 	return ispart;
 }
 EXPORT_SYMBOL_GPL(mtd_is_partition);
+
+/* Returns the size of the entire flash chip */
+uint64_t mtd_get_device_size(const struct mtd_info *mtd)
+{
+	if (!mtd_is_partition(mtd))
+		return mtd->size;
+
+	return PART(mtd)->master->size;
+}
+EXPORT_SYMBOL_GPL(mtd_get_device_size);
diff --git a/drivers/mtd/ubi/Kconfig b/drivers/mtd/ubi/Kconfig
index ea4b95b5451..271a842f8c3 100644
--- a/drivers/mtd/ubi/Kconfig
+++ b/drivers/mtd/ubi/Kconfig
@@ -27,20 +27,34 @@ config MTD_UBI_WL_THRESHOLD
 	  life-cycle less than 10000, the threshold should be lessened (e.g.,
 	  to 128 or 256, although it does not have to be power of 2).
 
-config MTD_UBI_BEB_RESERVE
-	int "Percentage of reserved eraseblocks for bad eraseblocks handling"
-	default 2
-	range 0 25
+config MTD_UBI_BEB_LIMIT
+	int "Maximum expected bad eraseblock count per 1024 eraseblocks"
+	default 20
+	range 0 768
 	help
-	  If the MTD device admits of bad eraseblocks (e.g. NAND flash), UBI
-	  reserves some amount of physical eraseblocks to handle new bad
-	  eraseblocks. For example, if a flash physical eraseblock becomes bad,
-	  UBI uses these reserved physical eraseblocks to relocate the bad one.
-	  This option specifies how many physical eraseblocks will be reserved
-	  for bad eraseblock handling (percents of total number of good flash
-	  eraseblocks). If the underlying flash does not admit of bad
-	  eraseblocks (e.g. NOR flash), this value is ignored and nothing is
-	  reserved. Leave the default value if unsure.
+	  This option specifies the maximum bad physical eraseblocks UBI
+	  expects on the MTD device (per 1024 eraseblocks). If the underlying
+	  flash does not admit of bad eraseblocks (e.g. NOR flash), this value
+	  is ignored.
+
+	  NAND datasheets often specify the minimum and maximum NVM (Number of
+	  Valid Blocks) for the flashes' endurance lifetime. The maximum
+	  expected bad eraseblocks per 1024 eraseblocks then can be calculated
+	  as "1024 * (1 - MinNVB / MaxNVB)", which gives 20 for most NANDs
+	  (MaxNVB is basically the total count of eraseblocks on the chip).
+
+	  To put it differently, if this value is 20, UBI will try to reserve
+	  about 1.9% of physical eraseblocks for bad blocks handling. And that
+	  will be 1.9% of eraseblocks on the entire NAND chip, not just the MTD
+	  partition UBI attaches. This means that if you have, say, a NAND
+	  flash chip admits maximum 40 bad eraseblocks, and it is split on two
+	  MTD partitions of the same size, UBI will reserve 40 eraseblocks when
+	  attaching a partition.
+
+	  This option can be overridden by the "mtd=" UBI module parameter or
+	  by the "attach" ioctl.
+
+	  Leave the default value if unsure.
 
 config MTD_UBI_GLUEBI
 	tristate "MTD devices emulation driver (gluebi)"
diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c
index bd27cbbb406..f7adf53e4f4 100644
--- a/drivers/mtd/ubi/attach.c
+++ b/drivers/mtd/ubi/attach.c
@@ -79,7 +79,7 @@
  *     NAND), it is probably a PEB which was being erased when power cut
  *     happened, so this is corruption type 1. However, this is just a guess,
  *     which might be wrong.
- *   o Otherwise this it corruption type 2.
+ *   o Otherwise this is corruption type 2.
  */
 
 #include <linux/err.h>
@@ -378,8 +378,8 @@ static int compare_lebs(struct ubi_device *ubi, const struct ubi_ainf_peb *aeb,
 			if (err == UBI_IO_BITFLIPS)
 				bitflips = 1;
 			else {
-				ubi_err("VID of PEB %d header is bad, but it "
-					"was OK earlier, err %d", pnum, err);
+				ubi_err("VID of PEB %d header is bad, but it was OK earlier, err %d",
+					pnum, err);
 				if (err > 0)
 					err = -EIO;
 
@@ -790,12 +790,12 @@ static int check_corruption(struct ubi_device *ubi, struct ubi_vid_hdr *vid_hdr,
 	if (ubi_check_pattern(ubi->peb_buf, 0xFF, ubi->leb_size))
 		goto out_unlock;
 
-	ubi_err("PEB %d contains corrupted VID header, and the data does not "
-		"contain all 0xFF, this may be a non-UBI PEB or a severe VID "
-		"header corruption which requires manual inspection", pnum);
+	ubi_err("PEB %d contains corrupted VID header, and the data does not contain all 0xFF",
+		pnum);
+	ubi_err("this may be a non-UBI PEB or a severe VID header corruption which requires manual inspection");
 	ubi_dump_vid_hdr(vid_hdr);
-	dbg_msg("hexdump of PEB %d offset %d, length %d",
-		pnum, ubi->leb_start, ubi->leb_size);
+	pr_err("hexdump of PEB %d offset %d, length %d",
+	       pnum, ubi->leb_start, ubi->leb_size);
 	ubi_dbg_print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 32, 1,
 			       ubi->peb_buf, ubi->leb_size, 1);
 	err = 1;
@@ -907,8 +907,8 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			ubi->image_seq = image_seq;
 		if (ubi->image_seq && image_seq &&
 		    ubi->image_seq != image_seq) {
-			ubi_err("bad image sequence number %d in PEB %d, "
-				"expected %d", image_seq, pnum, ubi->image_seq);
+			ubi_err("bad image sequence number %d in PEB %d, expected %d",
+				image_seq, pnum, ubi->image_seq);
 			ubi_dump_ec_hdr(ech);
 			return -EINVAL;
 		}
@@ -975,7 +975,7 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			return err;
 		goto adjust_mean_ec;
 	case UBI_IO_FF:
-		if (ec_err)
+		if (ec_err || bitflips)
 			err = add_to_list(ai, pnum, UBI_UNKNOWN,
 					  UBI_UNKNOWN, ec, 1, &ai->erase);
 		else
@@ -997,8 +997,8 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		/* Unsupported internal volume */
 		switch (vidh->compat) {
 		case UBI_COMPAT_DELETE:
-			ubi_msg("\"delete\" compatible internal volume %d:%d"
-				" found, will remove it", vol_id, lnum);
+			ubi_msg("\"delete\" compatible internal volume %d:%d found, will remove it",
+				vol_id, lnum);
 			err = add_to_list(ai, pnum, vol_id, lnum,
 					  ec, 1, &ai->erase);
 			if (err)
@@ -1006,15 +1006,14 @@ static int scan_peb(struct ubi_device *ubi, struct ubi_attach_info *ai,
 			return 0;
 
 		case UBI_COMPAT_RO:
-			ubi_msg("read-only compatible internal volume %d:%d"
-				" found, switch to read-only mode",
+			ubi_msg("read-only compatible internal volume %d:%d found, switch to read-only mode",
 				vol_id, lnum);
 			ubi->ro_mode = 1;
 			break;
 
 		case UBI_COMPAT_PRESERVE:
-			ubi_msg("\"preserve\" compatible internal volume %d:%d"
-				" found", vol_id, lnum);
+			ubi_msg("\"preserve\" compatible internal volume %d:%d found",
+				vol_id, lnum);
 			err = add_to_list(ai, pnum, vol_id, lnum,
 					  ec, 0, &ai->alien);
 			if (err)
@@ -1075,10 +1074,10 @@ static int late_analysis(struct ubi_device *ubi, struct ubi_attach_info *ai)
 	if (ai->corr_peb_count) {
 		ubi_err("%d PEBs are corrupted and preserved",
 			ai->corr_peb_count);
-		printk(KERN_ERR "Corrupted PEBs are:");
+		pr_err("Corrupted PEBs are:");
 		list_for_each_entry(aeb, &ai->corr, u.list)
-			printk(KERN_CONT " %d", aeb->pnum);
-		printk(KERN_CONT "\n");
+			pr_cont(" %d", aeb->pnum);
+		pr_cont("\n");
 
 		/*
 		 * If too many PEBs are corrupted, we refuse attaching,
@@ -1112,8 +1111,7 @@ static int late_analysis(struct ubi_device *ubi, struct ubi_attach_info *ai)
 			get_random_bytes(&ubi->image_seq,
 					 sizeof(ubi->image_seq));
 		} else {
-			ubi_err("MTD device is not UBI-formatted and possibly "
-				"contains non-UBI data - refusing it");
+			ubi_err("MTD device is not UBI-formatted and possibly contains non-UBI data - refusing it");
 			return -EINVAL;
 		}
 
@@ -1172,7 +1170,7 @@ static struct ubi_attach_info *scan_all(struct ubi_device *ubi)
 			goto out_vidh;
 	}
 
-	dbg_msg("scanning is finished");
+	ubi_msg("scanning is finished");
 
 	/* Calculate mean erase counter */
 	if (ai->ec_count)
@@ -1244,7 +1242,7 @@ int ubi_attach(struct ubi_device *ubi)
 	ubi->corr_peb_count = ai->corr_peb_count;
 	ubi->max_ec = ai->max_ec;
 	ubi->mean_ec = ai->mean_ec;
-	ubi_msg("max. sequence number:       %llu", ai->max_sqnum);
+	dbg_gen("max. sequence number:       %llu", ai->max_sqnum);
 
 	err = ubi_read_volume_table(ubi, ai);
 	if (err)
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index 2c5ed5ca9c3..34977039850 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -36,6 +36,7 @@
 #include <linux/namei.h>
 #include <linux/stat.h>
 #include <linux/miscdevice.h>
+#include <linux/mtd/partitions.h>
 #include <linux/log2.h>
 #include <linux/kthread.h>
 #include <linux/kernel.h>
@@ -45,6 +46,12 @@
 /* Maximum length of the 'mtd=' parameter */
 #define MTD_PARAM_LEN_MAX 64
 
+/* Maximum number of comma-separated items in the 'mtd=' parameter */
+#define MTD_PARAM_MAX_COUNT 3
+
+/* Maximum value for the number of bad PEBs per 1024 PEBs */
+#define MAX_MTD_UBI_BEB_LIMIT 768
+
 #ifdef CONFIG_MTD_UBI_MODULE
 #define ubi_is_module() 1
 #else
@@ -56,10 +63,12 @@
  * @name: MTD character device node path, MTD device name, or MTD device number
  *        string
  * @vid_hdr_offs: VID header offset
+ * @max_beb_per1024: maximum expected number of bad PEBs per 1024 PEBs
  */
 struct mtd_dev_param {
 	char name[MTD_PARAM_LEN_MAX];
 	int vid_hdr_offs;
+	int max_beb_per1024;
 };
 
 /* Numbers of elements set in the @mtd_dev_param array */
@@ -564,9 +573,38 @@ void ubi_free_internal_volumes(struct ubi_device *ubi)
 	}
 }
 
+static int get_bad_peb_limit(const struct ubi_device *ubi, int max_beb_per1024)
+{
+	int limit, device_pebs;
+	uint64_t device_size;
+
+	if (!max_beb_per1024)
+		return 0;
+
+	/*
+	 * Here we are using size of the entire flash chip and
+	 * not just the MTD partition size because the maximum
+	 * number of bad eraseblocks is a percentage of the
+	 * whole device and bad eraseblocks are not fairly
+	 * distributed over the flash chip. So the worst case
+	 * is that all the bad eraseblocks of the chip are in
+	 * the MTD partition we are attaching (ubi->mtd).
+	 */
+	device_size = mtd_get_device_size(ubi->mtd);
+	device_pebs = mtd_div_by_eb(device_size, ubi->mtd);
+	limit = mult_frac(device_pebs, max_beb_per1024, 1024);
+
+	/* Round it up */
+	if (mult_frac(limit, 1024, max_beb_per1024) < device_pebs)
+		limit += 1;
+
+	return limit;
+}
+
 /**
  * io_init - initialize I/O sub-system for a given UBI device.
  * @ubi: UBI device description object
+ * @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs
  *
  * If @ubi->vid_hdr_offset or @ubi->leb_start is zero, default offsets are
  * assumed:
@@ -579,8 +617,11 @@ void ubi_free_internal_volumes(struct ubi_device *ubi)
  * This function returns zero in case of success and a negative error code in
  * case of failure.
  */
-static int io_init(struct ubi_device *ubi)
+static int io_init(struct ubi_device *ubi, int max_beb_per1024)
 {
+	dbg_gen("sizeof(struct ubi_ainf_peb) %zu", sizeof(struct ubi_ainf_peb));
+	dbg_gen("sizeof(struct ubi_wl_entry) %zu", sizeof(struct ubi_wl_entry));
+
 	if (ubi->mtd->numeraseregions != 0) {
 		/*
 		 * Some flashes have several erase regions. Different regions
@@ -607,8 +648,10 @@ static int io_init(struct ubi_device *ubi)
 	ubi->peb_count  = mtd_div_by_eb(ubi->mtd->size, ubi->mtd);
 	ubi->flash_size = ubi->mtd->size;
 
-	if (mtd_can_have_bb(ubi->mtd))
+	if (mtd_can_have_bb(ubi->mtd)) {
 		ubi->bad_allowed = 1;
+		ubi->bad_peb_limit = get_bad_peb_limit(ubi, max_beb_per1024);
+	}
 
 	if (ubi->mtd->type == MTD_NORFLASH) {
 		ubi_assert(ubi->mtd->writesize == 1);
@@ -650,11 +693,11 @@ static int io_init(struct ubi_device *ubi)
 	ubi->ec_hdr_alsize = ALIGN(UBI_EC_HDR_SIZE, ubi->hdrs_min_io_size);
 	ubi->vid_hdr_alsize = ALIGN(UBI_VID_HDR_SIZE, ubi->hdrs_min_io_size);
 
-	dbg_msg("min_io_size      %d", ubi->min_io_size);
-	dbg_msg("max_write_size   %d", ubi->max_write_size);
-	dbg_msg("hdrs_min_io_size %d", ubi->hdrs_min_io_size);
-	dbg_msg("ec_hdr_alsize    %d", ubi->ec_hdr_alsize);
-	dbg_msg("vid_hdr_alsize   %d", ubi->vid_hdr_alsize);
+	dbg_gen("min_io_size      %d", ubi->min_io_size);
+	dbg_gen("max_write_size   %d", ubi->max_write_size);
+	dbg_gen("hdrs_min_io_size %d", ubi->hdrs_min_io_size);
+	dbg_gen("ec_hdr_alsize    %d", ubi->ec_hdr_alsize);
+	dbg_gen("vid_hdr_alsize   %d", ubi->vid_hdr_alsize);
 
 	if (ubi->vid_hdr_offset == 0)
 		/* Default offset */
@@ -671,10 +714,10 @@ static int io_init(struct ubi_device *ubi)
 	ubi->leb_start = ubi->vid_hdr_offset + UBI_VID_HDR_SIZE;
 	ubi->leb_start = ALIGN(ubi->leb_start, ubi->min_io_size);
 
-	dbg_msg("vid_hdr_offset   %d", ubi->vid_hdr_offset);
-	dbg_msg("vid_hdr_aloffset %d", ubi->vid_hdr_aloffset);
-	dbg_msg("vid_hdr_shift    %d", ubi->vid_hdr_shift);
-	dbg_msg("leb_start        %d", ubi->leb_start);
+	dbg_gen("vid_hdr_offset   %d", ubi->vid_hdr_offset);
+	dbg_gen("vid_hdr_aloffset %d", ubi->vid_hdr_aloffset);
+	dbg_gen("vid_hdr_shift    %d", ubi->vid_hdr_shift);
+	dbg_gen("leb_start        %d", ubi->leb_start);
 
 	/* The shift must be aligned to 32-bit boundary */
 	if (ubi->vid_hdr_shift % 4) {
@@ -700,7 +743,7 @@ static int io_init(struct ubi_device *ubi)
 	ubi->max_erroneous = ubi->peb_count / 10;
 	if (ubi->max_erroneous < 16)
 		ubi->max_erroneous = 16;
-	dbg_msg("max_erroneous    %d", ubi->max_erroneous);
+	dbg_gen("max_erroneous    %d", ubi->max_erroneous);
 
 	/*
 	 * It may happen that EC and VID headers are situated in one minimal
@@ -708,30 +751,18 @@ static int io_init(struct ubi_device *ubi)
 	 * read-only mode.
 	 */
 	if (ubi->vid_hdr_offset + UBI_VID_HDR_SIZE <= ubi->hdrs_min_io_size) {
-		ubi_warn("EC and VID headers are in the same minimal I/O unit, "
-			 "switch to read-only mode");
+		ubi_warn("EC and VID headers are in the same minimal I/O unit, switch to read-only mode");
 		ubi->ro_mode = 1;
 	}
 
 	ubi->leb_size = ubi->peb_size - ubi->leb_start;
 
 	if (!(ubi->mtd->flags & MTD_WRITEABLE)) {
-		ubi_msg("MTD device %d is write-protected, attach in "
-			"read-only mode", ubi->mtd->index);
+		ubi_msg("MTD device %d is write-protected, attach in read-only mode",
+			ubi->mtd->index);
 		ubi->ro_mode = 1;
 	}
 
-	ubi_msg("physical eraseblock size:   %d bytes (%d KiB)",
-		ubi->peb_size, ubi->peb_size >> 10);
-	ubi_msg("logical eraseblock size:    %d bytes", ubi->leb_size);
-	ubi_msg("smallest flash I/O unit:    %d", ubi->min_io_size);
-	if (ubi->hdrs_min_io_size != ubi->min_io_size)
-		ubi_msg("sub-page size:              %d",
-			ubi->hdrs_min_io_size);
-	ubi_msg("VID header offset:          %d (aligned %d)",
-		ubi->vid_hdr_offset, ubi->vid_hdr_aloffset);
-	ubi_msg("data offset:                %d", ubi->leb_start);
-
 	/*
 	 * Note, ideally, we have to initialize @ubi->bad_peb_count here. But
 	 * unfortunately, MTD does not provide this information. We should loop
@@ -759,6 +790,11 @@ static int autoresize(struct ubi_device *ubi, int vol_id)
 	struct ubi_volume *vol = ubi->volumes[vol_id];
 	int err, old_reserved_pebs = vol->reserved_pebs;
 
+	if (ubi->ro_mode) {
+		ubi_warn("skip auto-resize because of R/O mode");
+		return 0;
+	}
+
 	/*
 	 * Clear the auto-resize flag in the volume in-memory copy of the
 	 * volume table, and 'ubi_resize_volume()' will propagate this change
@@ -800,6 +836,7 @@ static int autoresize(struct ubi_device *ubi, int vol_id)
  * @mtd: MTD device description object
  * @ubi_num: number to assign to the new UBI device
  * @vid_hdr_offset: VID header offset
+ * @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs
  *
  * This function attaches MTD device @mtd_dev to UBI and assign @ubi_num number
  * to the newly created UBI device, unless @ubi_num is %UBI_DEV_NUM_AUTO, in
@@ -810,11 +847,18 @@ static int autoresize(struct ubi_device *ubi, int vol_id)
  * Note, the invocations of this function has to be serialized by the
  * @ubi_devices_mutex.
  */
-int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, int vid_hdr_offset)
+int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
+		       int vid_hdr_offset, int max_beb_per1024)
 {
 	struct ubi_device *ubi;
 	int i, err, ref = 0;
 
+	if (max_beb_per1024 < 0 || max_beb_per1024 > MAX_MTD_UBI_BEB_LIMIT)
+		return -EINVAL;
+
+	if (!max_beb_per1024)
+		max_beb_per1024 = CONFIG_MTD_UBI_BEB_LIMIT;
+
 	/*
 	 * Check if we already have the same MTD device attached.
 	 *
@@ -839,8 +883,8 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, int vid_hdr_offset)
 	 * no sense to attach emulated MTD devices, so we prohibit this.
 	 */
 	if (mtd->type == MTD_UBIVOLUME) {
-		ubi_err("refuse attaching mtd%d - it is already emulated on "
-			"top of UBI", mtd->index);
+		ubi_err("refuse attaching mtd%d - it is already emulated on top of UBI",
+			mtd->index);
 		return -EINVAL;
 	}
 
@@ -880,10 +924,8 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, int vid_hdr_offset)
 	spin_lock_init(&ubi->volumes_lock);
 
 	ubi_msg("attaching mtd%d to ubi%d", mtd->index, ubi_num);
-	dbg_msg("sizeof(struct ubi_ainf_peb) %zu", sizeof(struct ubi_ainf_peb));
-	dbg_msg("sizeof(struct ubi_wl_entry) %zu", sizeof(struct ubi_wl_entry));
 
-	err = io_init(ubi);
+	err = io_init(ubi, max_beb_per1024);
 	if (err)
 		goto out_free;
 
@@ -924,23 +966,24 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, int vid_hdr_offset)
 		goto out_debugfs;
 	}
 
-	ubi_msg("attached mtd%d to ubi%d", mtd->index, ubi_num);
-	ubi_msg("MTD device name:            \"%s\"", mtd->name);
-	ubi_msg("MTD device size:            %llu MiB", ubi->flash_size >> 20);
-	ubi_msg("number of good PEBs:        %d", ubi->good_peb_count);
-	ubi_msg("number of bad PEBs:         %d", ubi->bad_peb_count);
-	ubi_msg("number of corrupted PEBs:   %d", ubi->corr_peb_count);
-	ubi_msg("max. allowed volumes:       %d", ubi->vtbl_slots);
-	ubi_msg("wear-leveling threshold:    %d", CONFIG_MTD_UBI_WL_THRESHOLD);
-	ubi_msg("number of internal volumes: %d", UBI_INT_VOL_COUNT);
-	ubi_msg("number of user volumes:     %d",
-		ubi->vol_count - UBI_INT_VOL_COUNT);
-	ubi_msg("available PEBs:             %d", ubi->avail_pebs);
-	ubi_msg("total number of reserved PEBs: %d", ubi->rsvd_pebs);
-	ubi_msg("number of PEBs reserved for bad PEB handling: %d",
-		ubi->beb_rsvd_pebs);
-	ubi_msg("max/mean erase counter: %d/%d", ubi->max_ec, ubi->mean_ec);
-	ubi_msg("image sequence number:  %d", ubi->image_seq);
+	ubi_msg("attached mtd%d (name \"%s\", size %llu MiB) to ubi%d",
+		mtd->index, mtd->name, ubi->flash_size >> 20, ubi_num);
+	ubi_msg("PEB size: %d bytes (%d KiB), LEB size: %d bytes",
+		ubi->peb_size, ubi->peb_size >> 10, ubi->leb_size);
+	ubi_msg("min./max. I/O unit sizes: %d/%d, sub-page size %d",
+		ubi->min_io_size, ubi->max_write_size, ubi->hdrs_min_io_size);
+	ubi_msg("VID header offset: %d (aligned %d), data offset: %d",
+		ubi->vid_hdr_offset, ubi->vid_hdr_aloffset, ubi->leb_start);
+	ubi_msg("good PEBs: %d, bad PEBs: %d, corrupted PEBs: %d",
+		ubi->good_peb_count, ubi->bad_peb_count, ubi->corr_peb_count);
+	ubi_msg("user volume: %d, internal volumes: %d, max. volumes count: %d",
+		ubi->vol_count - UBI_INT_VOL_COUNT, UBI_INT_VOL_COUNT,
+		ubi->vtbl_slots);
+	ubi_msg("max/mean erase counter: %d/%d, WL threshold: %d, image sequence number: %u",
+		ubi->max_ec, ubi->mean_ec, CONFIG_MTD_UBI_WL_THRESHOLD,
+		ubi->image_seq);
+	ubi_msg("available PEBs: %d, total reserved PEBs: %d, PEBs reserved for bad PEB handling: %d",
+		ubi->avail_pebs, ubi->rsvd_pebs, ubi->beb_rsvd_pebs);
 
 	/*
 	 * The below lock makes sure we do not race with 'ubi_thread()' which
@@ -1017,7 +1060,7 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway)
 
 	ubi_assert(ubi_num == ubi->ubi_num);
 	ubi_notify_all(ubi, UBI_VOLUME_REMOVED, NULL);
-	dbg_msg("detaching mtd%d from ubi%d", ubi->mtd->index, ubi_num);
+	ubi_msg("detaching mtd%d from ubi%d", ubi->mtd->index, ubi_num);
 
 	/*
 	 * Before freeing anything, we have to stop the background thread to
@@ -1172,7 +1215,7 @@ static int __init ubi_init(void)
 
 		mutex_lock(&ubi_devices_mutex);
 		err = ubi_attach_mtd_dev(mtd, UBI_DEV_NUM_AUTO,
-					 p->vid_hdr_offs);
+					 p->vid_hdr_offs, p->max_beb_per1024);
 		mutex_unlock(&ubi_devices_mutex);
 		if (err < 0) {
 			ubi_err("cannot attach mtd%d", mtd->index);
@@ -1218,7 +1261,7 @@ out:
 	ubi_err("UBI error: cannot initialize UBI, error %d", err);
 	return err;
 }
-module_init(ubi_init);
+late_initcall(ubi_init);
 
 static void __exit ubi_exit(void)
 {
@@ -1252,8 +1295,7 @@ static int __init bytes_str_to_int(const char *str)
 
 	result = simple_strtoul(str, &endp, 0);
 	if (str == endp || result >= INT_MAX) {
-		printk(KERN_ERR "UBI error: incorrect bytes count: \"%s\"\n",
-		       str);
+		ubi_err("UBI error: incorrect bytes count: \"%s\"\n", str);
 		return -EINVAL;
 	}
 
@@ -1269,8 +1311,7 @@ static int __init bytes_str_to_int(const char *str)
 	case '\0':
 		break;
 	default:
-		printk(KERN_ERR "UBI error: incorrect bytes count: \"%s\"\n",
-		       str);
+		ubi_err("UBI error: incorrect bytes count: \"%s\"\n", str);
 		return -EINVAL;
 	}
 
@@ -1291,27 +1332,26 @@ static int __init ubi_mtd_param_parse(const char *val, struct kernel_param *kp)
 	struct mtd_dev_param *p;
 	char buf[MTD_PARAM_LEN_MAX];
 	char *pbuf = &buf[0];
-	char *tokens[2] = {NULL, NULL};
+	char *tokens[MTD_PARAM_MAX_COUNT];
 
 	if (!val)
 		return -EINVAL;
 
 	if (mtd_devs == UBI_MAX_DEVICES) {
-		printk(KERN_ERR "UBI error: too many parameters, max. is %d\n",
-		       UBI_MAX_DEVICES);
+		ubi_err("UBI error: too many parameters, max. is %d\n",
+			UBI_MAX_DEVICES);
 		return -EINVAL;
 	}
 
 	len = strnlen(val, MTD_PARAM_LEN_MAX);
 	if (len == MTD_PARAM_LEN_MAX) {
-		printk(KERN_ERR "UBI error: parameter \"%s\" is too long, "
-		       "max. is %d\n", val, MTD_PARAM_LEN_MAX);
+		ubi_err("UBI error: parameter \"%s\" is too long, max. is %d\n",
+			val, MTD_PARAM_LEN_MAX);
 		return -EINVAL;
 	}
 
 	if (len == 0) {
-		printk(KERN_WARNING "UBI warning: empty 'mtd=' parameter - "
-		       "ignored\n");
+		pr_warn("UBI warning: empty 'mtd=' parameter - ignored\n");
 		return 0;
 	}
 
@@ -1321,12 +1361,11 @@ static int __init ubi_mtd_param_parse(const char *val, struct kernel_param *kp)
 	if (buf[len - 1] == '\n')
 		buf[len - 1] = '\0';
 
-	for (i = 0; i < 2; i++)
+	for (i = 0; i < MTD_PARAM_MAX_COUNT; i++)
 		tokens[i] = strsep(&pbuf, ",");
 
 	if (pbuf) {
-		printk(KERN_ERR "UBI error: too many arguments at \"%s\"\n",
-		       val);
+		ubi_err("UBI error: too many arguments at \"%s\"\n", val);
 		return -EINVAL;
 	}
 
@@ -1339,23 +1378,32 @@ static int __init ubi_mtd_param_parse(const char *val, struct kernel_param *kp)
 	if (p->vid_hdr_offs < 0)
 		return p->vid_hdr_offs;
 
+	if (tokens[2]) {
+		int err = kstrtoint(tokens[2], 10, &p->max_beb_per1024);
+
+		if (err) {
+			ubi_err("UBI error: bad value for max_beb_per1024 parameter: %s",
+				tokens[2]);
+			return -EINVAL;
+		}
+	}
+
 	mtd_devs += 1;
 	return 0;
 }
 
 module_param_call(mtd, ubi_mtd_param_parse, NULL, NULL, 000);
-MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: "
-		      "mtd=<name|num|path>[,<vid_hdr_offs>].\n"
+MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=<name|num|path>[,<vid_hdr_offs>[,max_beb_per1024]].\n"
 		      "Multiple \"mtd\" parameters may be specified.\n"
-		      "MTD devices may be specified by their number, name, or "
-		      "path to the MTD character device node.\n"
-		      "Optional \"vid_hdr_offs\" parameter specifies UBI VID "
-		      "header position to be used by UBI.\n"
-		      "Example 1: mtd=/dev/mtd0 - attach MTD device "
-		      "/dev/mtd0.\n"
-		      "Example 2: mtd=content,1984 mtd=4 - attach MTD device "
-		      "with name \"content\" using VID header offset 1984, and "
-		      "MTD device number 4 with default VID header offset.");
+		      "MTD devices may be specified by their number, name, or path to the MTD character device node.\n"
+		      "Optional \"vid_hdr_offs\" parameter specifies UBI VID header position to be used by UBI. (default value if 0)\n"
+		      "Optional \"max_beb_per1024\" parameter specifies the maximum expected bad eraseblock per 1024 eraseblocks. (default value ("
+		      __stringify(CONFIG_MTD_UBI_BEB_LIMIT) ") if 0)\n"
+		      "\n"
+		      "Example 1: mtd=/dev/mtd0 - attach MTD device /dev/mtd0.\n"
+		      "Example 2: mtd=content,1984 mtd=4 - attach MTD device with name \"content\" using VID header offset 1984, and MTD device number 4 with default VID header offset.\n"
+		      "Example 3: mtd=/dev/mtd1,0,25 - attach MTD device /dev/mtd1 using default VID header offset and reserve 25*nand_size_in_blocks/1024 erase blocks for bad block handling.\n"
+		      "\t(e.g. if the NAND *chipset* has 4096 PEB, 100 will be reserved for this UBI device).");
 
 MODULE_VERSION(__stringify(UBI_VERSION));
 MODULE_DESCRIPTION("UBI - Unsorted Block Images");
diff --git a/drivers/mtd/ubi/cdev.c b/drivers/mtd/ubi/cdev.c
index fb556787818..dfcc65b33e9 100644
--- a/drivers/mtd/ubi/cdev.c
+++ b/drivers/mtd/ubi/cdev.c
@@ -140,9 +140,9 @@ static int vol_cdev_release(struct inode *inode, struct file *file)
 		vol->updating = 0;
 		vfree(vol->upd_buf);
 	} else if (vol->changing_leb) {
-		dbg_gen("only %lld of %lld bytes received for atomic LEB change"
-			" for volume %d:%d, cancel", vol->upd_received,
-			vol->upd_bytes, vol->ubi->ubi_num, vol->vol_id);
+		dbg_gen("only %lld of %lld bytes received for atomic LEB change for volume %d:%d, cancel",
+			vol->upd_received, vol->upd_bytes, vol->ubi->ubi_num,
+			vol->vol_id);
 		vol->changing_leb = 0;
 		vfree(vol->upd_buf);
 	}
@@ -189,7 +189,8 @@ static loff_t vol_cdev_llseek(struct file *file, loff_t offset, int origin)
 	return new_offset;
 }
 
-static int vol_cdev_fsync(struct file *file, loff_t start, loff_t end, int datasync)
+static int vol_cdev_fsync(struct file *file, loff_t start, loff_t end,
+			  int datasync)
 {
 	struct ubi_volume_desc *desc = file->private_data;
 	struct ubi_device *ubi = desc->vol->ubi;
@@ -753,7 +754,7 @@ static int rename_volumes(struct ubi_device *ubi,
 		re->new_name_len = name_len;
 		memcpy(re->new_name, name, name_len);
 		list_add_tail(&re->list, &rename_list);
-		dbg_msg("will rename volume %d from \"%s\" to \"%s\"",
+		dbg_gen("will rename volume %d from \"%s\" to \"%s\"",
 			vol_id, re->desc->vol->name, name);
 	}
 
@@ -811,7 +812,7 @@ static int rename_volumes(struct ubi_device *ubi,
 		re1->remove = 1;
 		re1->desc = desc;
 		list_add(&re1->list, &rename_list);
-		dbg_msg("will remove volume %d, name \"%s\"",
+		dbg_gen("will remove volume %d, name \"%s\"",
 			re1->desc->vol->vol_id, re1->desc->vol->name);
 	}
 
@@ -942,7 +943,7 @@ static long ubi_cdev_ioctl(struct file *file, unsigned int cmd,
 	{
 		struct ubi_rnvol_req *req;
 
-		dbg_msg("re-name volumes");
+		dbg_gen("re-name volumes");
 		req = kmalloc(sizeof(struct ubi_rnvol_req), GFP_KERNEL);
 		if (!req) {
 			err = -ENOMEM;
@@ -1010,7 +1011,8 @@ static long ctrl_cdev_ioctl(struct file *file, unsigned int cmd,
 		 * 'ubi_attach_mtd_dev()'.
 		 */
 		mutex_lock(&ubi_devices_mutex);
-		err = ubi_attach_mtd_dev(mtd, req.ubi_num, req.vid_hdr_offset);
+		err = ubi_attach_mtd_dev(mtd, req.ubi_num, req.vid_hdr_offset,
+					 req.max_beb_per1024);
 		mutex_unlock(&ubi_devices_mutex);
 		if (err < 0)
 			put_mtd_device(mtd);
diff --git a/drivers/mtd/ubi/debug.c b/drivers/mtd/ubi/debug.c
index 7c138030521..26908a59506 100644
--- a/drivers/mtd/ubi/debug.c
+++ b/drivers/mtd/ubi/debug.c
@@ -43,8 +43,8 @@ void ubi_dump_flash(struct ubi_device *ubi, int pnum, int offset, int len)
 		return;
 	err = mtd_read(ubi->mtd, addr, len, &read, buf);
 	if (err && err != -EUCLEAN) {
-		ubi_err("error %d while reading %d bytes from PEB %d:%d, "
-			"read %zd bytes", err, len, pnum, offset, read);
+		ubi_err("error %d while reading %d bytes from PEB %d:%d, read %zd bytes",
+			err, len, pnum, offset, read);
 		goto out;
 	}
 
@@ -62,21 +62,15 @@ out:
  */
 void ubi_dump_ec_hdr(const struct ubi_ec_hdr *ec_hdr)
 {
-	printk(KERN_DEBUG "Erase counter header dump:\n");
-	printk(KERN_DEBUG "\tmagic          %#08x\n",
-	       be32_to_cpu(ec_hdr->magic));
-	printk(KERN_DEBUG "\tversion        %d\n", (int)ec_hdr->version);
-	printk(KERN_DEBUG "\tec             %llu\n",
-	       (long long)be64_to_cpu(ec_hdr->ec));
-	printk(KERN_DEBUG "\tvid_hdr_offset %d\n",
-	       be32_to_cpu(ec_hdr->vid_hdr_offset));
-	printk(KERN_DEBUG "\tdata_offset    %d\n",
-	       be32_to_cpu(ec_hdr->data_offset));
-	printk(KERN_DEBUG "\timage_seq      %d\n",
-	       be32_to_cpu(ec_hdr->image_seq));
-	printk(KERN_DEBUG "\thdr_crc        %#08x\n",
-	       be32_to_cpu(ec_hdr->hdr_crc));
-	printk(KERN_DEBUG "erase counter header hexdump:\n");
+	pr_err("Erase counter header dump:\n");
+	pr_err("\tmagic          %#08x\n", be32_to_cpu(ec_hdr->magic));
+	pr_err("\tversion        %d\n", (int)ec_hdr->version);
+	pr_err("\tec             %llu\n", (long long)be64_to_cpu(ec_hdr->ec));
+	pr_err("\tvid_hdr_offset %d\n", be32_to_cpu(ec_hdr->vid_hdr_offset));
+	pr_err("\tdata_offset    %d\n", be32_to_cpu(ec_hdr->data_offset));
+	pr_err("\timage_seq      %d\n", be32_to_cpu(ec_hdr->image_seq));
+	pr_err("\thdr_crc        %#08x\n", be32_to_cpu(ec_hdr->hdr_crc));
+	pr_err("erase counter header hexdump:\n");
 	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 32, 1,
 		       ec_hdr, UBI_EC_HDR_SIZE, 1);
 }
@@ -87,21 +81,21 @@ void ubi_dump_ec_hdr(const struct ubi_ec_hdr *ec_hdr)
  */
 void ubi_dump_vid_hdr(const struct ubi_vid_hdr *vid_hdr)
 {
-	printk(KERN_DEBUG "Volume identifier header dump:\n");
-	printk(KERN_DEBUG "\tmagic     %08x\n", be32_to_cpu(vid_hdr->magic));
-	printk(KERN_DEBUG "\tversion   %d\n",  (int)vid_hdr->version);
-	printk(KERN_DEBUG "\tvol_type  %d\n",  (int)vid_hdr->vol_type);
-	printk(KERN_DEBUG "\tcopy_flag %d\n",  (int)vid_hdr->copy_flag);
-	printk(KERN_DEBUG "\tcompat    %d\n",  (int)vid_hdr->compat);
-	printk(KERN_DEBUG "\tvol_id    %d\n",  be32_to_cpu(vid_hdr->vol_id));
-	printk(KERN_DEBUG "\tlnum      %d\n",  be32_to_cpu(vid_hdr->lnum));
-	printk(KERN_DEBUG "\tdata_size %d\n",  be32_to_cpu(vid_hdr->data_size));
-	printk(KERN_DEBUG "\tused_ebs  %d\n",  be32_to_cpu(vid_hdr->used_ebs));
-	printk(KERN_DEBUG "\tdata_pad  %d\n",  be32_to_cpu(vid_hdr->data_pad));
-	printk(KERN_DEBUG "\tsqnum     %llu\n",
+	pr_err("Volume identifier header dump:\n");
+	pr_err("\tmagic     %08x\n", be32_to_cpu(vid_hdr->magic));
+	pr_err("\tversion   %d\n",  (int)vid_hdr->version);
+	pr_err("\tvol_type  %d\n",  (int)vid_hdr->vol_type);
+	pr_err("\tcopy_flag %d\n",  (int)vid_hdr->copy_flag);
+	pr_err("\tcompat    %d\n",  (int)vid_hdr->compat);
+	pr_err("\tvol_id    %d\n",  be32_to_cpu(vid_hdr->vol_id));
+	pr_err("\tlnum      %d\n",  be32_to_cpu(vid_hdr->lnum));
+	pr_err("\tdata_size %d\n",  be32_to_cpu(vid_hdr->data_size));
+	pr_err("\tused_ebs  %d\n",  be32_to_cpu(vid_hdr->used_ebs));
+	pr_err("\tdata_pad  %d\n",  be32_to_cpu(vid_hdr->data_pad));
+	pr_err("\tsqnum     %llu\n",
 		(unsigned long long)be64_to_cpu(vid_hdr->sqnum));
-	printk(KERN_DEBUG "\thdr_crc   %08x\n", be32_to_cpu(vid_hdr->hdr_crc));
-	printk(KERN_DEBUG "Volume identifier header hexdump:\n");
+	pr_err("\thdr_crc   %08x\n", be32_to_cpu(vid_hdr->hdr_crc));
+	pr_err("Volume identifier header hexdump:\n");
 	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 32, 1,
 		       vid_hdr, UBI_VID_HDR_SIZE, 1);
 }
@@ -112,25 +106,25 @@ void ubi_dump_vid_hdr(const struct ubi_vid_hdr *vid_hdr)
  */
 void ubi_dump_vol_info(const struct ubi_volume *vol)
 {
-	printk(KERN_DEBUG "Volume information dump:\n");
-	printk(KERN_DEBUG "\tvol_id          %d\n", vol->vol_id);
-	printk(KERN_DEBUG "\treserved_pebs   %d\n", vol->reserved_pebs);
-	printk(KERN_DEBUG "\talignment       %d\n", vol->alignment);
-	printk(KERN_DEBUG "\tdata_pad        %d\n", vol->data_pad);
-	printk(KERN_DEBUG "\tvol_type        %d\n", vol->vol_type);
-	printk(KERN_DEBUG "\tname_len        %d\n", vol->name_len);
-	printk(KERN_DEBUG "\tusable_leb_size %d\n", vol->usable_leb_size);
-	printk(KERN_DEBUG "\tused_ebs        %d\n", vol->used_ebs);
-	printk(KERN_DEBUG "\tused_bytes      %lld\n", vol->used_bytes);
-	printk(KERN_DEBUG "\tlast_eb_bytes   %d\n", vol->last_eb_bytes);
-	printk(KERN_DEBUG "\tcorrupted       %d\n", vol->corrupted);
-	printk(KERN_DEBUG "\tupd_marker      %d\n", vol->upd_marker);
+	pr_err("Volume information dump:\n");
+	pr_err("\tvol_id          %d\n", vol->vol_id);
+	pr_err("\treserved_pebs   %d\n", vol->reserved_pebs);
+	pr_err("\talignment       %d\n", vol->alignment);
+	pr_err("\tdata_pad        %d\n", vol->data_pad);
+	pr_err("\tvol_type        %d\n", vol->vol_type);
+	pr_err("\tname_len        %d\n", vol->name_len);
+	pr_err("\tusable_leb_size %d\n", vol->usable_leb_size);
+	pr_err("\tused_ebs        %d\n", vol->used_ebs);
+	pr_err("\tused_bytes      %lld\n", vol->used_bytes);
+	pr_err("\tlast_eb_bytes   %d\n", vol->last_eb_bytes);
+	pr_err("\tcorrupted       %d\n", vol->corrupted);
+	pr_err("\tupd_marker      %d\n", vol->upd_marker);
 
 	if (vol->name_len <= UBI_VOL_NAME_MAX &&
 	    strnlen(vol->name, vol->name_len + 1) == vol->name_len) {
-		printk(KERN_DEBUG "\tname            %s\n", vol->name);
+		pr_err("\tname            %s\n", vol->name);
 	} else {
-		printk(KERN_DEBUG "\t1st 5 characters of name: %c%c%c%c%c\n",
+		pr_err("\t1st 5 characters of name: %c%c%c%c%c\n",
 		       vol->name[0], vol->name[1], vol->name[2],
 		       vol->name[3], vol->name[4]);
 	}
@@ -145,29 +139,28 @@ void ubi_dump_vtbl_record(const struct ubi_vtbl_record *r, int idx)
 {
 	int name_len = be16_to_cpu(r->name_len);
 
-	printk(KERN_DEBUG "Volume table record %d dump:\n", idx);
-	printk(KERN_DEBUG "\treserved_pebs   %d\n",
-	       be32_to_cpu(r->reserved_pebs));
-	printk(KERN_DEBUG "\talignment       %d\n", be32_to_cpu(r->alignment));
-	printk(KERN_DEBUG "\tdata_pad        %d\n", be32_to_cpu(r->data_pad));
-	printk(KERN_DEBUG "\tvol_type        %d\n", (int)r->vol_type);
-	printk(KERN_DEBUG "\tupd_marker      %d\n", (int)r->upd_marker);
-	printk(KERN_DEBUG "\tname_len        %d\n", name_len);
+	pr_err("Volume table record %d dump:\n", idx);
+	pr_err("\treserved_pebs   %d\n", be32_to_cpu(r->reserved_pebs));
+	pr_err("\talignment       %d\n", be32_to_cpu(r->alignment));
+	pr_err("\tdata_pad        %d\n", be32_to_cpu(r->data_pad));
+	pr_err("\tvol_type        %d\n", (int)r->vol_type);
+	pr_err("\tupd_marker      %d\n", (int)r->upd_marker);
+	pr_err("\tname_len        %d\n", name_len);
 
 	if (r->name[0] == '\0') {
-		printk(KERN_DEBUG "\tname            NULL\n");
+		pr_err("\tname            NULL\n");
 		return;
 	}
 
 	if (name_len <= UBI_VOL_NAME_MAX &&
 	    strnlen(&r->name[0], name_len + 1) == name_len) {
-		printk(KERN_DEBUG "\tname            %s\n", &r->name[0]);
+		pr_err("\tname            %s\n", &r->name[0]);
 	} else {
-		printk(KERN_DEBUG "\t1st 5 characters of name: %c%c%c%c%c\n",
+		pr_err("\t1st 5 characters of name: %c%c%c%c%c\n",
 			r->name[0], r->name[1], r->name[2], r->name[3],
 			r->name[4]);
 	}
-	printk(KERN_DEBUG "\tcrc             %#08x\n", be32_to_cpu(r->crc));
+	pr_err("\tcrc             %#08x\n", be32_to_cpu(r->crc));
 }
 
 /**
@@ -176,15 +169,15 @@ void ubi_dump_vtbl_record(const struct ubi_vtbl_record *r, int idx)
  */
 void ubi_dump_av(const struct ubi_ainf_volume *av)
 {
-	printk(KERN_DEBUG "Volume attaching information dump:\n");
-	printk(KERN_DEBUG "\tvol_id         %d\n", av->vol_id);
-	printk(KERN_DEBUG "\thighest_lnum   %d\n", av->highest_lnum);
-	printk(KERN_DEBUG "\tleb_count      %d\n", av->leb_count);
-	printk(KERN_DEBUG "\tcompat         %d\n", av->compat);
-	printk(KERN_DEBUG "\tvol_type       %d\n", av->vol_type);
-	printk(KERN_DEBUG "\tused_ebs       %d\n", av->used_ebs);
-	printk(KERN_DEBUG "\tlast_data_size %d\n", av->last_data_size);
-	printk(KERN_DEBUG "\tdata_pad       %d\n", av->data_pad);
+	pr_err("Volume attaching information dump:\n");
+	pr_err("\tvol_id         %d\n", av->vol_id);
+	pr_err("\thighest_lnum   %d\n", av->highest_lnum);
+	pr_err("\tleb_count      %d\n", av->leb_count);
+	pr_err("\tcompat         %d\n", av->compat);
+	pr_err("\tvol_type       %d\n", av->vol_type);
+	pr_err("\tused_ebs       %d\n", av->used_ebs);
+	pr_err("\tlast_data_size %d\n", av->last_data_size);
+	pr_err("\tdata_pad       %d\n", av->data_pad);
 }
 
 /**
@@ -194,13 +187,13 @@ void ubi_dump_av(const struct ubi_ainf_volume *av)
  */
 void ubi_dump_aeb(const struct ubi_ainf_peb *aeb, int type)
 {
-	printk(KERN_DEBUG "eraseblock attaching information dump:\n");
-	printk(KERN_DEBUG "\tec       %d\n", aeb->ec);
-	printk(KERN_DEBUG "\tpnum     %d\n", aeb->pnum);
+	pr_err("eraseblock attaching information dump:\n");
+	pr_err("\tec       %d\n", aeb->ec);
+	pr_err("\tpnum     %d\n", aeb->pnum);
 	if (type == 0) {
-		printk(KERN_DEBUG "\tlnum     %d\n", aeb->lnum);
-		printk(KERN_DEBUG "\tscrub    %d\n", aeb->scrub);
-		printk(KERN_DEBUG "\tsqnum    %llu\n", aeb->sqnum);
+		pr_err("\tlnum     %d\n", aeb->lnum);
+		pr_err("\tscrub    %d\n", aeb->scrub);
+		pr_err("\tsqnum    %llu\n", aeb->sqnum);
 	}
 }
 
@@ -212,16 +205,16 @@ void ubi_dump_mkvol_req(const struct ubi_mkvol_req *req)
 {
 	char nm[17];
 
-	printk(KERN_DEBUG "Volume creation request dump:\n");
-	printk(KERN_DEBUG "\tvol_id    %d\n",   req->vol_id);
-	printk(KERN_DEBUG "\talignment %d\n",   req->alignment);
-	printk(KERN_DEBUG "\tbytes     %lld\n", (long long)req->bytes);
-	printk(KERN_DEBUG "\tvol_type  %d\n",   req->vol_type);
-	printk(KERN_DEBUG "\tname_len  %d\n",   req->name_len);
+	pr_err("Volume creation request dump:\n");
+	pr_err("\tvol_id    %d\n",   req->vol_id);
+	pr_err("\talignment %d\n",   req->alignment);
+	pr_err("\tbytes     %lld\n", (long long)req->bytes);
+	pr_err("\tvol_type  %d\n",   req->vol_type);
+	pr_err("\tname_len  %d\n",   req->name_len);
 
 	memcpy(nm, req->name, 16);
 	nm[16] = 0;
-	printk(KERN_DEBUG "\t1st 16 characters of name: %s\n", nm);
+	pr_err("\t1st 16 characters of name: %s\n", nm);
 }
 
 /**
diff --git a/drivers/mtd/ubi/debug.h b/drivers/mtd/ubi/debug.h
index d5d2645b51a..3dbc877d966 100644
--- a/drivers/mtd/ubi/debug.h
+++ b/drivers/mtd/ubi/debug.h
@@ -29,22 +29,18 @@ void ubi_dump_vid_hdr(const struct ubi_vid_hdr *vid_hdr);
 
 #define ubi_assert(expr)  do {                                               \
 	if (unlikely(!(expr))) {                                             \
-		printk(KERN_CRIT "UBI assert failed in %s at %u (pid %d)\n", \
+		pr_crit("UBI assert failed in %s at %u (pid %d)\n",          \
 		       __func__, __LINE__, current->pid);                    \
 		dump_stack();                                                \
 	}                                                                    \
 } while (0)
 
-#define ubi_dbg_print_hex_dump(l, ps, pt, r, g, b, len, a)  \
+#define ubi_dbg_print_hex_dump(l, ps, pt, r, g, b, len, a)                   \
 		print_hex_dump(l, ps, pt, r, g, b, len, a)
 
 #define ubi_dbg_msg(type, fmt, ...) \
-	pr_debug("UBI DBG " type ": " fmt "\n", ##__VA_ARGS__)
-
-/* Just a debugging messages not related to any specific UBI subsystem */
-#define dbg_msg(fmt, ...)                                    \
-	printk(KERN_DEBUG "UBI DBG (pid %d): %s: " fmt "\n", \
-	       current->pid, __func__, ##__VA_ARGS__)
+	pr_debug("UBI DBG " type " (pid %d): " fmt "\n", current->pid,       \
+		 ##__VA_ARGS__)
 
 /* General debugging messages */
 #define dbg_gen(fmt, ...) ubi_dbg_msg("gen", fmt, ##__VA_ARGS__)
diff --git a/drivers/mtd/ubi/eba.c b/drivers/mtd/ubi/eba.c
index b703ac7729c..a26d7d25317 100644
--- a/drivers/mtd/ubi/eba.c
+++ b/drivers/mtd/ubi/eba.c
@@ -420,9 +420,8 @@ retry:
 				 */
 				if (err == UBI_IO_BAD_HDR_EBADMSG ||
 				    err == UBI_IO_BAD_HDR) {
-					ubi_warn("corrupted VID header at PEB "
-						 "%d, LEB %d:%d", pnum, vol_id,
-						 lnum);
+					ubi_warn("corrupted VID header at PEB %d, LEB %d:%d",
+						 pnum, vol_id, lnum);
 					err = -EBADMSG;
 				} else
 					ubi_ro_mode(ubi);
@@ -660,9 +659,8 @@ retry:
 	if (len) {
 		err = ubi_io_write_data(ubi, buf, pnum, offset, len);
 		if (err) {
-			ubi_warn("failed to write %d bytes at offset %d of "
-				 "LEB %d:%d, PEB %d", len, offset, vol_id,
-				 lnum, pnum);
+			ubi_warn("failed to write %d bytes at offset %d of LEB %d:%d, PEB %d",
+				 len, offset, vol_id, lnum, pnum);
 			goto write_error;
 		}
 	}
@@ -1040,9 +1038,8 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 	 * cancel it.
 	 */
 	if (vol->eba_tbl[lnum] != from) {
-		dbg_wl("LEB %d:%d is no longer mapped to PEB %d, mapped to "
-		       "PEB %d, cancel", vol_id, lnum, from,
-		       vol->eba_tbl[lnum]);
+		dbg_wl("LEB %d:%d is no longer mapped to PEB %d, mapped to PEB %d, cancel",
+		       vol_id, lnum, from, vol->eba_tbl[lnum]);
 		err = MOVE_CANCEL_RACE;
 		goto out_unlock_leb;
 	}
@@ -1107,8 +1104,8 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 	err = ubi_io_read_vid_hdr(ubi, to, vid_hdr, 1);
 	if (err) {
 		if (err != UBI_IO_BITFLIPS) {
-			ubi_warn("error %d while reading VID header back from "
-				  "PEB %d", err, to);
+			ubi_warn("error %d while reading VID header back from PEB %d",
+				 err, to);
 			if (is_error_sane(err))
 				err = MOVE_TARGET_RD_ERR;
 		} else
@@ -1134,8 +1131,8 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 		err = ubi_io_read_data(ubi, ubi->peb_buf, to, 0, aldata_size);
 		if (err) {
 			if (err != UBI_IO_BITFLIPS) {
-				ubi_warn("error %d while reading data back "
-					 "from PEB %d", err, to);
+				ubi_warn("error %d while reading data back from PEB %d",
+					 err, to);
 				if (is_error_sane(err))
 					err = MOVE_TARGET_RD_ERR;
 			} else
@@ -1146,8 +1143,8 @@ int ubi_eba_copy_leb(struct ubi_device *ubi, int from, int to,
 		cond_resched();
 
 		if (crc != crc32(UBI_CRC32_INIT, ubi->peb_buf, data_size)) {
-			ubi_warn("read data back from PEB %d and it is "
-				 "different", to);
+			ubi_warn("read data back from PEB %d and it is different",
+				 to);
 			err = -EINVAL;
 			goto out_unlock_buf;
 		}
@@ -1197,11 +1194,11 @@ static void print_rsvd_warning(struct ubi_device *ubi,
 			return;
 	}
 
-	ubi_warn("cannot reserve enough PEBs for bad PEB handling, reserved %d,"
-		 " need %d", ubi->beb_rsvd_pebs, ubi->beb_rsvd_level);
+	ubi_warn("cannot reserve enough PEBs for bad PEB handling, reserved %d, need %d",
+		 ubi->beb_rsvd_pebs, ubi->beb_rsvd_level);
 	if (ubi->corr_peb_count)
 		ubi_warn("%d PEBs are corrupted and not used",
-			ubi->corr_peb_count);
+			 ubi->corr_peb_count);
 }
 
 /**
diff --git a/drivers/mtd/ubi/gluebi.c b/drivers/mtd/ubi/gluebi.c
index 4e44bee4c56..4bd4db8c84c 100644
--- a/drivers/mtd/ubi/gluebi.c
+++ b/drivers/mtd/ubi/gluebi.c
@@ -41,7 +41,7 @@
 #include "ubi-media.h"
 
 #define err_msg(fmt, ...)                                   \
-	printk(KERN_DEBUG "gluebi (pid %d): %s: " fmt "\n", \
+	pr_err("gluebi (pid %d): %s: " fmt "\n",            \
 	       current->pid, __func__, ##__VA_ARGS__)
 
 /**
@@ -341,9 +341,8 @@ static int gluebi_create(struct ubi_device_info *di,
 	mutex_lock(&devices_mutex);
 	g = find_gluebi_nolock(vi->ubi_num, vi->vol_id);
 	if (g)
-		err_msg("gluebi MTD device %d form UBI device %d volume %d "
-			"already exists", g->mtd.index, vi->ubi_num,
-			vi->vol_id);
+		err_msg("gluebi MTD device %d form UBI device %d volume %d already exists",
+			g->mtd.index, vi->ubi_num, vi->vol_id);
 	mutex_unlock(&devices_mutex);
 
 	if (mtd_device_register(mtd, NULL, 0)) {
@@ -376,8 +375,8 @@ static int gluebi_remove(struct ubi_volume_info *vi)
 	mutex_lock(&devices_mutex);
 	gluebi = find_gluebi_nolock(vi->ubi_num, vi->vol_id);
 	if (!gluebi) {
-		err_msg("got remove notification for unknown UBI device %d "
-			"volume %d", vi->ubi_num, vi->vol_id);
+		err_msg("got remove notification for unknown UBI device %d volume %d",
+			vi->ubi_num, vi->vol_id);
 		err = -ENOENT;
 	} else if (gluebi->refcnt)
 		err = -EBUSY;
@@ -390,9 +389,8 @@ static int gluebi_remove(struct ubi_volume_info *vi)
 	mtd = &gluebi->mtd;
 	err = mtd_device_unregister(mtd);
 	if (err) {
-		err_msg("cannot remove fake MTD device %d, UBI device %d, "
-			"volume %d, error %d", mtd->index, gluebi->ubi_num,
-			gluebi->vol_id, err);
+		err_msg("cannot remove fake MTD device %d, UBI device %d, volume %d, error %d",
+			mtd->index, gluebi->ubi_num, gluebi->vol_id, err);
 		mutex_lock(&devices_mutex);
 		list_add_tail(&gluebi->list, &gluebi_devices);
 		mutex_unlock(&devices_mutex);
@@ -422,8 +420,8 @@ static int gluebi_updated(struct ubi_volume_info *vi)
 	gluebi = find_gluebi_nolock(vi->ubi_num, vi->vol_id);
 	if (!gluebi) {
 		mutex_unlock(&devices_mutex);
-		err_msg("got update notification for unknown UBI device %d "
-			"volume %d", vi->ubi_num, vi->vol_id);
+		err_msg("got update notification for unknown UBI device %d volume %d",
+			vi->ubi_num, vi->vol_id);
 		return -ENOENT;
 	}
 
@@ -449,8 +447,8 @@ static int gluebi_resized(struct ubi_volume_info *vi)
 	gluebi = find_gluebi_nolock(vi->ubi_num, vi->vol_id);
 	if (!gluebi) {
 		mutex_unlock(&devices_mutex);
-		err_msg("got update notification for unknown UBI device %d "
-			"volume %d", vi->ubi_num, vi->vol_id);
+		err_msg("got update notification for unknown UBI device %d volume %d",
+			vi->ubi_num, vi->vol_id);
 		return -ENOENT;
 	}
 	gluebi->mtd.size = vi->used_bytes;
@@ -507,9 +505,9 @@ static void __exit ubi_gluebi_exit(void)
 
 		err = mtd_device_unregister(mtd);
 		if (err)
-			err_msg("error %d while removing gluebi MTD device %d, "
-				"UBI device %d, volume %d - ignoring", err,
-				mtd->index, gluebi->ubi_num, gluebi->vol_id);
+			err_msg("error %d while removing gluebi MTD device %d, UBI device %d, volume %d - ignoring",
+				err, mtd->index, gluebi->ubi_num,
+				gluebi->vol_id);
 		kfree(mtd->name);
 		kfree(gluebi);
 	}
diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c
index a8d523794b5..78a1dcbf210 100644
--- a/drivers/mtd/ubi/io.c
+++ b/drivers/mtd/ubi/io.c
@@ -177,21 +177,20 @@ retry:
 			 * enabled. A corresponding message will be printed
 			 * later, when it is has been scrubbed.
 			 */
-			dbg_msg("fixable bit-flip detected at PEB %d", pnum);
+			ubi_msg("fixable bit-flip detected at PEB %d", pnum);
 			ubi_assert(len == read);
 			return UBI_IO_BITFLIPS;
 		}
 
 		if (retries++ < UBI_IO_RETRIES) {
-			ubi_warn("error %d%s while reading %d bytes from PEB "
-				 "%d:%d, read only %zd bytes, retry",
+			ubi_warn("error %d%s while reading %d bytes from PEB %d:%d, read only %zd bytes, retry",
 				 err, errstr, len, pnum, offset, read);
 			yield();
 			goto retry;
 		}
 
-		ubi_err("error %d%s while reading %d bytes from PEB %d:%d, "
-			"read %zd bytes", err, errstr, len, pnum, offset, read);
+		ubi_err("error %d%s while reading %d bytes from PEB %d:%d, read %zd bytes",
+			err, errstr, len, pnum, offset, read);
 		dump_stack();
 
 		/*
@@ -274,8 +273,8 @@ int ubi_io_write(struct ubi_device *ubi, const void *buf, int pnum, int offset,
 	}
 
 	if (ubi_dbg_is_write_failure(ubi)) {
-		ubi_err("cannot write %d bytes to PEB %d:%d "
-			"(emulated)", len, pnum, offset);
+		ubi_err("cannot write %d bytes to PEB %d:%d (emulated)",
+			len, pnum, offset);
 		dump_stack();
 		return -EIO;
 	}
@@ -283,8 +282,8 @@ int ubi_io_write(struct ubi_device *ubi, const void *buf, int pnum, int offset,
 	addr = (loff_t)pnum * ubi->peb_size + offset;
 	err = mtd_write(ubi->mtd, addr, len, &written, buf);
 	if (err) {
-		ubi_err("error %d while writing %d bytes to PEB %d:%d, written "
-			"%zd bytes", err, len, pnum, offset, written);
+		ubi_err("error %d while writing %d bytes to PEB %d:%d, written %zd bytes",
+			err, len, pnum, offset, written);
 		dump_stack();
 		ubi_dump_flash(ubi, pnum, offset, len);
 	} else
@@ -685,8 +684,7 @@ static int validate_ec_hdr(const struct ubi_device *ubi,
 	leb_start = be32_to_cpu(ec_hdr->data_offset);
 
 	if (ec_hdr->version != UBI_VERSION) {
-		ubi_err("node with incompatible UBI version found: "
-			"this UBI version is %d, image version is %d",
+		ubi_err("node with incompatible UBI version found: this UBI version is %d, image version is %d",
 			UBI_VERSION, (int)ec_hdr->version);
 		goto bad;
 	}
@@ -777,10 +775,10 @@ int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum,
 		if (ubi_check_pattern(ec_hdr, 0xFF, UBI_EC_HDR_SIZE)) {
 			/* The physical eraseblock is supposedly empty */
 			if (verbose)
-				ubi_warn("no EC header found at PEB %d, "
-					 "only 0xFF bytes", pnum);
-			dbg_bld("no EC header found at PEB %d, "
-				"only 0xFF bytes", pnum);
+				ubi_warn("no EC header found at PEB %d, only 0xFF bytes",
+					 pnum);
+			dbg_bld("no EC header found at PEB %d, only 0xFF bytes",
+				pnum);
 			if (!read_err)
 				return UBI_IO_FF;
 			else
@@ -792,12 +790,12 @@ int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum,
 		 * 0xFF bytes. Report that the header is corrupted.
 		 */
 		if (verbose) {
-			ubi_warn("bad magic number at PEB %d: %08x instead of "
-				 "%08x", pnum, magic, UBI_EC_HDR_MAGIC);
+			ubi_warn("bad magic number at PEB %d: %08x instead of %08x",
+				 pnum, magic, UBI_EC_HDR_MAGIC);
 			ubi_dump_ec_hdr(ec_hdr);
 		}
-		dbg_bld("bad magic number at PEB %d: %08x instead of "
-			"%08x", pnum, magic, UBI_EC_HDR_MAGIC);
+		dbg_bld("bad magic number at PEB %d: %08x instead of %08x",
+			pnum, magic, UBI_EC_HDR_MAGIC);
 		return UBI_IO_BAD_HDR;
 	}
 
@@ -806,12 +804,12 @@ int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum,
 
 	if (hdr_crc != crc) {
 		if (verbose) {
-			ubi_warn("bad EC header CRC at PEB %d, calculated "
-				 "%#08x, read %#08x", pnum, crc, hdr_crc);
+			ubi_warn("bad EC header CRC at PEB %d, calculated %#08x, read %#08x",
+				 pnum, crc, hdr_crc);
 			ubi_dump_ec_hdr(ec_hdr);
 		}
-		dbg_bld("bad EC header CRC at PEB %d, calculated "
-			"%#08x, read %#08x", pnum, crc, hdr_crc);
+		dbg_bld("bad EC header CRC at PEB %d, calculated %#08x, read %#08x",
+			pnum, crc, hdr_crc);
 
 		if (!read_err)
 			return UBI_IO_BAD_HDR;
@@ -1032,10 +1030,10 @@ int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
 
 		if (ubi_check_pattern(vid_hdr, 0xFF, UBI_VID_HDR_SIZE)) {
 			if (verbose)
-				ubi_warn("no VID header found at PEB %d, "
-					 "only 0xFF bytes", pnum);
-			dbg_bld("no VID header found at PEB %d, "
-				"only 0xFF bytes", pnum);
+				ubi_warn("no VID header found at PEB %d, only 0xFF bytes",
+					 pnum);
+			dbg_bld("no VID header found at PEB %d, only 0xFF bytes",
+				pnum);
 			if (!read_err)
 				return UBI_IO_FF;
 			else
@@ -1043,12 +1041,12 @@ int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
 		}
 
 		if (verbose) {
-			ubi_warn("bad magic number at PEB %d: %08x instead of "
-				 "%08x", pnum, magic, UBI_VID_HDR_MAGIC);
+			ubi_warn("bad magic number at PEB %d: %08x instead of %08x",
+				 pnum, magic, UBI_VID_HDR_MAGIC);
 			ubi_dump_vid_hdr(vid_hdr);
 		}
-		dbg_bld("bad magic number at PEB %d: %08x instead of "
-			"%08x", pnum, magic, UBI_VID_HDR_MAGIC);
+		dbg_bld("bad magic number at PEB %d: %08x instead of %08x",
+			pnum, magic, UBI_VID_HDR_MAGIC);
 		return UBI_IO_BAD_HDR;
 	}
 
@@ -1057,12 +1055,12 @@ int ubi_io_read_vid_hdr(struct ubi_device *ubi, int pnum,
 
 	if (hdr_crc != crc) {
 		if (verbose) {
-			ubi_warn("bad CRC at PEB %d, calculated %#08x, "
-				 "read %#08x", pnum, crc, hdr_crc);
+			ubi_warn("bad CRC at PEB %d, calculated %#08x, read %#08x",
+				 pnum, crc, hdr_crc);
 			ubi_dump_vid_hdr(vid_hdr);
 		}
-		dbg_bld("bad CRC at PEB %d, calculated %#08x, "
-			"read %#08x", pnum, crc, hdr_crc);
+		dbg_bld("bad CRC at PEB %d, calculated %#08x, read %#08x",
+			pnum, crc, hdr_crc);
 		if (!read_err)
 			return UBI_IO_BAD_HDR;
 		else
@@ -1300,8 +1298,8 @@ static int self_check_peb_vid_hdr(const struct ubi_device *ubi, int pnum)
 	crc = crc32(UBI_CRC32_INIT, vid_hdr, UBI_EC_HDR_SIZE_CRC);
 	hdr_crc = be32_to_cpu(vid_hdr->hdr_crc);
 	if (hdr_crc != crc) {
-		ubi_err("bad VID header CRC at PEB %d, calculated %#08x, "
-			"read %#08x", pnum, crc, hdr_crc);
+		ubi_err("bad VID header CRC at PEB %d, calculated %#08x, read %#08x",
+			pnum, crc, hdr_crc);
 		ubi_err("self-check failed for PEB %d", pnum);
 		ubi_dump_vid_hdr(vid_hdr);
 		dump_stack();
@@ -1411,15 +1409,15 @@ int ubi_self_check_all_ff(struct ubi_device *ubi, int pnum, int offset, int len)
 
 	err = mtd_read(ubi->mtd, addr, len, &read, buf);
 	if (err && !mtd_is_bitflip(err)) {
-		ubi_err("error %d while reading %d bytes from PEB %d:%d, "
-			"read %zd bytes", err, len, pnum, offset, read);
+		ubi_err("error %d while reading %d bytes from PEB %d:%d, read %zd bytes",
+			err, len, pnum, offset, read);
 		goto error;
 	}
 
 	err = ubi_check_pattern(buf, 0xFF, len);
 	if (err == 0) {
-		ubi_err("flash region at PEB %d:%d, length %d does not "
-			"contain all 0xFF bytes", pnum, offset, len);
+		ubi_err("flash region at PEB %d:%d, length %d does not contain all 0xFF bytes",
+			pnum, offset, len);
 		goto fail;
 	}
 
diff --git a/drivers/mtd/ubi/misc.c b/drivers/mtd/ubi/misc.c
index 8bbfb444b89..f913d701a5b 100644
--- a/drivers/mtd/ubi/misc.c
+++ b/drivers/mtd/ubi/misc.c
@@ -121,10 +121,16 @@ void ubi_update_reserved(struct ubi_device *ubi)
  */
 void ubi_calculate_reserved(struct ubi_device *ubi)
 {
-	ubi->beb_rsvd_level = ubi->good_peb_count/100;
-	ubi->beb_rsvd_level *= CONFIG_MTD_UBI_BEB_RESERVE;
-	if (ubi->beb_rsvd_level < MIN_RESEVED_PEBS)
-		ubi->beb_rsvd_level = MIN_RESEVED_PEBS;
+	/*
+	 * Calculate the actual number of PEBs currently needed to be reserved
+	 * for future bad eraseblock handling.
+	 */
+	ubi->beb_rsvd_level = ubi->bad_peb_limit - ubi->bad_peb_count;
+	if (ubi->beb_rsvd_level < 0) {
+		ubi->beb_rsvd_level = 0;
+		ubi_warn("number of bad PEBs (%d) is above the expected limit (%d), not reserving any PEBs for bad PEB handling, will use available PEBs (if any)",
+			 ubi->bad_peb_count, ubi->bad_peb_limit);
+	}
 }
 
 /**
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index 84f66e3fa05..383ee43d242 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -51,17 +51,14 @@
 #define UBI_NAME_STR "ubi"
 
 /* Normal UBI messages */
-#define ubi_msg(fmt, ...) printk(KERN_NOTICE "UBI: " fmt "\n", ##__VA_ARGS__)
+#define ubi_msg(fmt, ...) pr_notice("UBI: " fmt "\n", ##__VA_ARGS__)
 /* UBI warning messages */
-#define ubi_warn(fmt, ...) printk(KERN_WARNING "UBI warning: %s: " fmt "\n", \
-				  __func__, ##__VA_ARGS__)
+#define ubi_warn(fmt, ...) pr_warn("UBI warning: %s: " fmt "\n",  \
+				   __func__, ##__VA_ARGS__)
 /* UBI error messages */
-#define ubi_err(fmt, ...) printk(KERN_ERR "UBI error: %s: " fmt "\n", \
+#define ubi_err(fmt, ...) pr_err("UBI error: %s: " fmt "\n",      \
 				 __func__, ##__VA_ARGS__)
 
-/* Lowest number PEBs reserved for bad PEB handling */
-#define MIN_RESEVED_PEBS 2
-
 /* Background thread name pattern */
 #define UBI_BGT_NAME_PATTERN "ubi_bgt%dd"
 
@@ -363,6 +360,7 @@ struct ubi_wl_entry;
  * @flash_size: underlying MTD device size (in bytes)
  * @peb_count: count of physical eraseblocks on the MTD device
  * @peb_size: physical eraseblock size
+ * @bad_peb_limit: top limit of expected bad physical eraseblocks
  * @bad_peb_count: count of bad physical eraseblocks
  * @good_peb_count: count of good physical eraseblocks
  * @corr_peb_count: count of corrupted physical eraseblocks (preserved and not
@@ -410,6 +408,7 @@ struct ubi_device {
 	int avail_pebs;
 	int beb_rsvd_pebs;
 	int beb_rsvd_level;
+	int bad_peb_limit;
 
 	int autoresize_vol_id;
 	int vtbl_slots;
@@ -694,7 +693,8 @@ int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum,
 			 struct ubi_vid_hdr *vid_hdr);
 
 /* build.c */
-int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, int vid_hdr_offset);
+int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num,
+		       int vid_hdr_offset, int max_beb_per1024);
 int ubi_detach_mtd_dev(int ubi_num, int anyway);
 struct ubi_device *ubi_get_device(int ubi_num);
 void ubi_put_device(struct ubi_device *ubi);
diff --git a/drivers/mtd/ubi/vtbl.c b/drivers/mtd/ubi/vtbl.c
index 568307cc7ca..926e3df14fb 100644
--- a/drivers/mtd/ubi/vtbl.c
+++ b/drivers/mtd/ubi/vtbl.c
@@ -270,8 +270,8 @@ static int vtbl_check(const struct ubi_device *ubi,
 
 			if (len1 > 0 && len1 == len2 &&
 			    !strncmp(vtbl[i].name, vtbl[n].name, len1)) {
-				ubi_err("volumes %d and %d have the same name"
-					" \"%s\"", i, n, vtbl[i].name);
+				ubi_err("volumes %d and %d have the same name \"%s\"",
+					i, n, vtbl[i].name);
 				ubi_dump_vtbl_record(&vtbl[i], i);
 				ubi_dump_vtbl_record(&vtbl[n], n);
 				return -EINVAL;
@@ -304,7 +304,7 @@ static int create_vtbl(struct ubi_device *ubi, struct ubi_attach_info *ai,
 	struct ubi_vid_hdr *vid_hdr;
 	struct ubi_ainf_peb *new_aeb;
 
-	ubi_msg("create volume table (copy #%d)", copy + 1);
+	dbg_gen("create volume table (copy #%d)", copy + 1);
 
 	vid_hdr = ubi_zalloc_vid_hdr(ubi, GFP_KERNEL);
 	if (!vid_hdr)
@@ -562,8 +562,8 @@ static int init_volumes(struct ubi_device *ubi,
 		if (vtbl[i].flags & UBI_VTBL_AUTORESIZE_FLG) {
 			/* Auto re-size flag may be set only for one volume */
 			if (ubi->autoresize_vol_id != -1) {
-				ubi_err("more than one auto-resize volume (%d "
-					"and %d)", ubi->autoresize_vol_id, i);
+				ubi_err("more than one auto-resize volume (%d and %d)",
+					ubi->autoresize_vol_id, i);
 				kfree(vol);
 				return -EINVAL;
 			}
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c
index b6be644e7b8..032fc57f109 100644
--- a/drivers/mtd/ubi/wl.c
+++ b/drivers/mtd/ubi/wl.c
@@ -978,9 +978,10 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
 			int cancel)
 {
 	struct ubi_wl_entry *e = wl_wrk->e;
-	int pnum = e->pnum, err, need;
+	int pnum = e->pnum;
 	int vol_id = wl_wrk->vol_id;
 	int lnum = wl_wrk->lnum;
+	int err, available_consumed = 0;
 
 	if (cancel) {
 		dbg_wl("cancel erasure of PEB %d EC %d", pnum, e->ec);
@@ -1045,20 +1046,14 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
 	}
 
 	spin_lock(&ubi->volumes_lock);
-	need = ubi->beb_rsvd_level - ubi->beb_rsvd_pebs + 1;
-	if (need > 0) {
-		need = ubi->avail_pebs >= need ? need : ubi->avail_pebs;
-		ubi->avail_pebs -= need;
-		ubi->rsvd_pebs += need;
-		ubi->beb_rsvd_pebs += need;
-		if (need > 0)
-			ubi_msg("reserve more %d PEBs", need);
-	}
-
 	if (ubi->beb_rsvd_pebs == 0) {
-		spin_unlock(&ubi->volumes_lock);
-		ubi_err("no reserved physical eraseblocks");
-		goto out_ro;
+		if (ubi->avail_pebs == 0) {
+			spin_unlock(&ubi->volumes_lock);
+			ubi_err("no reserved/available physical eraseblocks");
+			goto out_ro;
+		}
+		ubi->avail_pebs -= 1;
+		available_consumed = 1;
 	}
 	spin_unlock(&ubi->volumes_lock);
 
@@ -1068,19 +1063,36 @@ static int erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk,
 		goto out_ro;
 
 	spin_lock(&ubi->volumes_lock);
-	ubi->beb_rsvd_pebs -= 1;
+	if (ubi->beb_rsvd_pebs > 0) {
+		if (available_consumed) {
+			/*
+			 * The amount of reserved PEBs increased since we last
+			 * checked.
+			 */
+			ubi->avail_pebs += 1;
+			available_consumed = 0;
+		}
+		ubi->beb_rsvd_pebs -= 1;
+	}
 	ubi->bad_peb_count += 1;
 	ubi->good_peb_count -= 1;
 	ubi_calculate_reserved(ubi);
-	if (ubi->beb_rsvd_pebs)
+	if (available_consumed)
+		ubi_warn("no PEBs in the reserved pool, used an available PEB");
+	else if (ubi->beb_rsvd_pebs)
 		ubi_msg("%d PEBs left in the reserve", ubi->beb_rsvd_pebs);
 	else
-		ubi_warn("last PEB from the reserved pool was used");
+		ubi_warn("last PEB from the reserve was used");
 	spin_unlock(&ubi->volumes_lock);
 
 	return err;
 
 out_ro:
+	if (available_consumed) {
+		spin_lock(&ubi->volumes_lock);
+		ubi->avail_pebs += 1;
+		spin_unlock(&ubi->volumes_lock);
+	}
 	ubi_ro_mode(ubi);
 	return err;
 }
@@ -1189,7 +1201,7 @@ int ubi_wl_scrub_peb(struct ubi_device *ubi, int pnum)
 {
 	struct ubi_wl_entry *e;
 
-	dbg_msg("schedule PEB %d for scrubbing", pnum);
+	ubi_msg("schedule PEB %d for scrubbing", pnum);
 
 retry:
 	spin_lock(&ubi->wl_lock);
diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c
index 034c16b60e9..adc3708d882 100644
--- a/drivers/net/can/slcan.c
+++ b/drivers/net/can/slcan.c
@@ -56,7 +56,7 @@
 #include <linux/kernel.h>
 #include <linux/can.h>
 
-static __initdata const char banner[] =
+static __initconst const char banner[] =
 	KERN_INFO "slcan: serial line CAN interface driver\n";
 
 MODULE_ALIAS_LDISC(N_SLCAN);
diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c
index 4f93c0be005..0a2a5ee79a1 100644
--- a/drivers/net/can/vcan.c
+++ b/drivers/net/can/vcan.c
@@ -49,7 +49,7 @@
 #include <linux/slab.h>
 #include <net/rtnetlink.h>
 
-static __initdata const char banner[] =
+static __initconst const char banner[] =
 	KERN_INFO "vcan: Virtual CAN interface driver\n";
 
 MODULE_DESCRIPTION("virtual CAN interface");
diff --git a/drivers/net/ethernet/8390/ne3210.c b/drivers/net/ethernet/8390/ne3210.c
index a2f8b2b8e27..e3f57427d5c 100644
--- a/drivers/net/ethernet/8390/ne3210.c
+++ b/drivers/net/ethernet/8390/ne3210.c
@@ -81,7 +81,7 @@ static void ne3210_block_output(struct net_device *dev, int count, const unsigne
 
 static unsigned char irq_map[] __initdata = {15, 12, 11, 10, 9, 7, 5, 3};
 static unsigned int shmem_map[] __initdata = {0xff0, 0xfe0, 0xfff0, 0xd8, 0xffe0, 0xffc0, 0xd0, 0x0};
-static const char *ifmap[] __initdata = {"UTP", "?", "BNC", "AUI"};
+static const char * const ifmap[] __initconst = {"UTP", "?", "BNC", "AUI"};
 static int ifmap_val[] __initdata = {
 		IF_PORT_10BASET,
 		IF_PORT_UNKNOWN,
diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c
index d920a529ba2..5b65992c2a0 100644
--- a/drivers/net/ethernet/adaptec/starfire.c
+++ b/drivers/net/ethernet/adaptec/starfire.c
@@ -295,7 +295,7 @@ MODULE_DEVICE_TABLE(pci, starfire_pci_tbl);
 static const struct chip_info {
 	const char *name;
 	int drv_flags;
-} netdrv_tbl[] __devinitdata = {
+} netdrv_tbl[] __devinitconst = {
 	{ "Adaptec Starfire 6915", CanHaveMII },
 };
 
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
index 55a2e379505..d19f82f7597 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
@@ -702,7 +702,7 @@ struct atl1c_platform_patch {
 	u32 patch_flag;
 #define ATL1C_LINK_PATCH	0x1
 };
-static const struct atl1c_platform_patch plats[] __devinitdata = {
+static const struct atl1c_platform_patch plats[] __devinitconst = {
 {0x2060, 0xC1, 0x1019, 0x8152, 0x1},
 {0x2060, 0xC1, 0x1019, 0x2060, 0x1},
 {0x2060, 0xC1, 0x1019, 0xE000, 0x1},
diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c
index 57d64b80fd7..623dd8635c4 100644
--- a/drivers/net/ethernet/atheros/atlx/atl2.c
+++ b/drivers/net/ethernet/atheros/atlx/atl2.c
@@ -2845,7 +2845,7 @@ static void atl2_force_ps(struct atl2_hw *hw)
  */
 
 #define ATL2_PARAM(X, desc) \
-    static const int __devinitdata X[ATL2_MAX_NIC + 1] = ATL2_PARAM_INIT; \
+    static const int __devinitconst X[ATL2_MAX_NIC + 1] = ATL2_PARAM_INIT; \
     MODULE_PARM(X, "1-" __MODULE_STRING(ATL2_MAX_NIC) "i"); \
     MODULE_PARM_DESC(X, desc);
 #else
diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c
index 61cc0934286..77335853ac3 100644
--- a/drivers/net/ethernet/dec/tulip/de2104x.c
+++ b/drivers/net/ethernet/dec/tulip/de2104x.c
@@ -661,9 +661,6 @@ static netdev_tx_t de_start_xmit (struct sk_buff *skb,
    new frame, not around filling de->setup_frame.  This is non-deterministic
    when re-entered but still correct. */
 
-#undef set_bit_le
-#define set_bit_le(i,p) do { ((char *)(p))[(i)/8] |= (1<<((i)%8)); } while(0)
-
 static void build_setup_frame_hash(u16 *setup_frm, struct net_device *dev)
 {
 	struct de_private *de = netdev_priv(dev);
@@ -673,12 +670,12 @@ static void build_setup_frame_hash(u16 *setup_frm, struct net_device *dev)
 	u16 *eaddrs;
 
 	memset(hash_table, 0, sizeof(hash_table));
-	set_bit_le(255, hash_table); 			/* Broadcast entry */
+	__set_bit_le(255, hash_table);			/* Broadcast entry */
 	/* This should work on big-endian machines as well. */
 	netdev_for_each_mc_addr(ha, dev) {
 		int index = ether_crc_le(ETH_ALEN, ha->addr) & 0x1ff;
 
-		set_bit_le(index, hash_table);
+		__set_bit_le(index, hash_table);
 	}
 
 	for (i = 0; i < 32; i++) {
diff --git a/drivers/net/ethernet/dec/tulip/eeprom.c b/drivers/net/ethernet/dec/tulip/eeprom.c
index ed7d1dcd956..44f7e8e82d8 100644
--- a/drivers/net/ethernet/dec/tulip/eeprom.c
+++ b/drivers/net/ethernet/dec/tulip/eeprom.c
@@ -79,7 +79,7 @@ static struct eeprom_fixup eeprom_fixups[] __devinitdata = {
   {NULL}};
 
 
-static const char *block_name[] __devinitdata = {
+static const char *const block_name[] __devinitconst = {
 	"21140 non-MII",
 	"21140 MII PHY",
 	"21142 Serial PHY",
diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c
index c4f37aca226..885700a1997 100644
--- a/drivers/net/ethernet/dec/tulip/tulip_core.c
+++ b/drivers/net/ethernet/dec/tulip/tulip_core.c
@@ -1010,9 +1010,6 @@ static int private_ioctl (struct net_device *dev, struct ifreq *rq, int cmd)
    new frame, not around filling tp->setup_frame.  This is non-deterministic
    when re-entered but still correct. */
 
-#undef set_bit_le
-#define set_bit_le(i,p) do { ((char *)(p))[(i)/8] |= (1<<((i)%8)); } while(0)
-
 static void build_setup_frame_hash(u16 *setup_frm, struct net_device *dev)
 {
 	struct tulip_private *tp = netdev_priv(dev);
@@ -1022,12 +1019,12 @@ static void build_setup_frame_hash(u16 *setup_frm, struct net_device *dev)
 	u16 *eaddrs;
 
 	memset(hash_table, 0, sizeof(hash_table));
-	set_bit_le(255, hash_table); 			/* Broadcast entry */
+	__set_bit_le(255, hash_table);			/* Broadcast entry */
 	/* This should work on big-endian machines as well. */
 	netdev_for_each_mc_addr(ha, dev) {
 		int index = ether_crc_le(ETH_ALEN, ha->addr) & 0x1ff;
 
-		set_bit_le(index, hash_table);
+		__set_bit_le(index, hash_table);
 	}
 	for (i = 0; i < 32; i++) {
 		*setup_frm++ = hash_table[i];
diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c
index 4d1ffca83c8..7c1ec4d7920 100644
--- a/drivers/net/ethernet/dec/tulip/winbond-840.c
+++ b/drivers/net/ethernet/dec/tulip/winbond-840.c
@@ -236,7 +236,7 @@ struct pci_id_info {
         int drv_flags;		/* Driver use, intended as capability flags. */
 };
 
-static const struct pci_id_info pci_id_tbl[] __devinitdata = {
+static const struct pci_id_info pci_id_tbl[] __devinitconst = {
 	{ 				/* Sometime a Level-One switch card. */
 	  "Winbond W89c840",	CanHaveMII | HasBrokenTx | FDXOnNoMII},
 	{ "Winbond W89c840",	CanHaveMII | HasBrokenTx},
diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c
index d7bb52a7bda..3b83588e51f 100644
--- a/drivers/net/ethernet/dlink/sundance.c
+++ b/drivers/net/ethernet/dlink/sundance.c
@@ -218,7 +218,7 @@ enum {
 struct pci_id_info {
         const char *name;
 };
-static const struct pci_id_info pci_id_tbl[] __devinitdata = {
+static const struct pci_id_info pci_id_tbl[] __devinitconst = {
 	{"D-Link DFE-550TX FAST Ethernet Adapter"},
 	{"D-Link DFE-550FX 100Mbps Fiber-optics Adapter"},
 	{"D-Link DFE-580TX 4 port Server Adapter"},
diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c
index 9d71c9cc300..0e4a0ac86aa 100644
--- a/drivers/net/ethernet/fealnx.c
+++ b/drivers/net/ethernet/fealnx.c
@@ -150,7 +150,7 @@ struct chip_info {
 	int flags;
 };
 
-static const struct chip_info skel_netdrv_tbl[] __devinitdata = {
+static const struct chip_info skel_netdrv_tbl[] __devinitconst = {
  	{ "100/10M Ethernet PCI Adapter",	HAS_MII_XCVR },
 	{ "100/10M Ethernet PCI Adapter",	HAS_CHIP_XCVR },
 	{ "1000/100/10M Ethernet PCI Adapter",	HAS_MII_XCVR },
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c
index b528e52a8ee..2a0c9dc48eb 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_hw.c
@@ -38,7 +38,7 @@ static inline void writeq(u64 val, void __iomem *addr)
 }
 #endif
 
-static const struct crb_128M_2M_block_map
+static struct crb_128M_2M_block_map
 crb_128M_2M_map[64] __cacheline_aligned_in_smp = {
     {{{0, 0,         0,         0} } },		/* 0: PCI */
     {{{1, 0x0100000, 0x0102000, 0x120000},	/* 1: PCIE */
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c
index 1d83565cc6a..3ed7add23c1 100644
--- a/drivers/net/ethernet/realtek/8139too.c
+++ b/drivers/net/ethernet/realtek/8139too.c
@@ -228,7 +228,7 @@ typedef enum {
 static const struct {
 	const char *name;
 	u32 hw_flags;
-} board_info[] __devinitdata = {
+} board_info[] __devinitconst = {
 	{ "RealTek RTL8139", RTL8139_CAPS },
 	{ "RealTek RTL8129", RTL8129_CAPS },
 };
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 96bd980e828..4f86d0cd516 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -2019,14 +2019,14 @@ static void efx_set_rx_mode(struct net_device *net_dev)
 		netdev_for_each_mc_addr(ha, net_dev) {
 			crc = ether_crc_le(ETH_ALEN, ha->addr);
 			bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
-			set_bit_le(bit, mc_hash->byte);
+			__set_bit_le(bit, mc_hash);
 		}
 
 		/* Broadcast packets go through the multicast hash filter.
 		 * ether_crc_le() of the broadcast address is 0xbe2612ff
 		 * so we always add bit 0xff to the mask.
 		 */
-		set_bit_le(0xff, mc_hash->byte);
+		__set_bit_le(0xff, mc_hash);
 	}
 
 	if (efx->port_enabled)
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index c1a010cda89..576a3109116 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -1101,18 +1101,6 @@ static inline struct efx_rx_buffer *efx_rx_buffer(struct efx_rx_queue *rx_queue,
 	return &rx_queue->buffer[index];
 }
 
-/* Set bit in a little-endian bitfield */
-static inline void set_bit_le(unsigned nr, unsigned char *addr)
-{
-	addr[nr / 8] |= (1 << (nr % 8));
-}
-
-/* Clear bit in a little-endian bitfield */
-static inline void clear_bit_le(unsigned nr, unsigned char *addr)
-{
-	addr[nr / 8] &= ~(1 << (nr % 8));
-}
-
 
 /**
  * EFX_MAX_FRAME_LEN - calculate maximum frame length
diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c
index cdff40b6572..aab7cacb2e3 100644
--- a/drivers/net/ethernet/sfc/nic.c
+++ b/drivers/net/ethernet/sfc/nic.c
@@ -472,9 +472,9 @@ void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 
 		efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 		if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
-			clear_bit_le(tx_queue->queue, (void *)&reg);
+			__clear_bit_le(tx_queue->queue, &reg);
 		else
-			set_bit_le(tx_queue->queue, (void *)&reg);
+			__set_bit_le(tx_queue->queue, &reg);
 		efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 	}
 
diff --git a/drivers/net/ethernet/sis/sis190.c b/drivers/net/ethernet/sis/sis190.c
index 4613591b43e..d8166012b7d 100644
--- a/drivers/net/ethernet/sis/sis190.c
+++ b/drivers/net/ethernet/sis/sis190.c
@@ -1618,7 +1618,7 @@ static int __devinit sis190_get_mac_addr_from_eeprom(struct pci_dev *pdev,
 static int __devinit sis190_get_mac_addr_from_apc(struct pci_dev *pdev,
 						  struct net_device *dev)
 {
-	static const u16 __devinitdata ids[] = { 0x0965, 0x0966, 0x0968 };
+	static const u16 __devinitconst ids[] = { 0x0965, 0x0966, 0x0968 };
 	struct sis190_private *tp = netdev_priv(dev);
 	struct pci_dev *isa_bridge;
 	u8 reg, tmp8;
diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c
index 64783a0d545..1450e33fc25 100644
--- a/drivers/net/hamradio/6pack.c
+++ b/drivers/net/hamradio/6pack.c
@@ -811,9 +811,9 @@ static struct tty_ldisc_ops sp_ldisc = {
 
 /* Initialize 6pack control device -- register 6pack line discipline */
 
-static const char msg_banner[]  __initdata = KERN_INFO \
+static const char msg_banner[]  __initconst = KERN_INFO \
 	"AX.25: 6pack driver, " SIXPACK_VERSION "\n";
-static const char msg_regfail[] __initdata = KERN_ERR  \
+static const char msg_regfail[] __initconst = KERN_ERR  \
 	"6pack: can't register line discipline (err = %d)\n";
 
 static int __init sixpack_init_driver(void)
@@ -829,7 +829,7 @@ static int __init sixpack_init_driver(void)
 	return status;
 }
 
-static const char msg_unregfail[] __exitdata = KERN_ERR \
+static const char msg_unregfail[] = KERN_ERR \
 	"6pack: can't unregister line discipline (err = %d)\n";
 
 static void __exit sixpack_exit_driver(void)
diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c
index 76d54774ba8..c2e5497397d 100644
--- a/drivers/net/hamradio/bpqether.c
+++ b/drivers/net/hamradio/bpqether.c
@@ -87,7 +87,7 @@
 
 #include <linux/bpqether.h>
 
-static const char banner[] __initdata = KERN_INFO \
+static const char banner[] __initconst = KERN_INFO \
 	"AX.25: bpqether driver version 004\n";
 
 static char bcast_addr[6]={0xFF,0xFF,0xFF,0xFF,0xFF,0xFF};
diff --git a/drivers/net/hamradio/mkiss.c b/drivers/net/hamradio/mkiss.c
index 2c0894a92ab..8e01c457015 100644
--- a/drivers/net/hamradio/mkiss.c
+++ b/drivers/net/hamradio/mkiss.c
@@ -997,9 +997,9 @@ static struct tty_ldisc_ops ax_ldisc = {
 	.write_wakeup	= mkiss_write_wakeup
 };
 
-static const char banner[] __initdata = KERN_INFO \
+static const char banner[] __initconst = KERN_INFO \
 	"mkiss: AX.25 Multikiss, Hans Albas PE1AYX\n";
-static const char msg_regfail[] __initdata = KERN_ERR \
+static const char msg_regfail[] __initconst = KERN_ERR \
 	"mkiss: can't register line discipline (err = %d)\n";
 
 static int __init mkiss_init_driver(void)
@@ -1015,7 +1015,7 @@ static int __init mkiss_init_driver(void)
 	return status;
 }
 
-static const char msg_unregfail[] __exitdata = KERN_ERR \
+static const char msg_unregfail[] = KERN_ERR \
 	"mkiss: can't unregister line discipline (err = %d)\n";
 
 static void __exit mkiss_exit_driver(void)
diff --git a/drivers/net/hamradio/scc.c b/drivers/net/hamradio/scc.c
index efc6c97163a..1b4a47bd32b 100644
--- a/drivers/net/hamradio/scc.c
+++ b/drivers/net/hamradio/scc.c
@@ -182,7 +182,7 @@
 
 #include "z8530.h"
 
-static const char banner[] __initdata = KERN_INFO \
+static const char banner[] __initconst = KERN_INFO \
 	"AX.25: Z8530 SCC driver version "VERSION".dl1bke\n";
 
 static void t_dwait(unsigned long);
diff --git a/drivers/net/hamradio/yam.c b/drivers/net/hamradio/yam.c
index 5a6412ecce7..c6645f1017a 100644
--- a/drivers/net/hamradio/yam.c
+++ b/drivers/net/hamradio/yam.c
@@ -76,7 +76,7 @@
 /* --------------------------------------------------------------------- */
 
 static const char yam_drvname[] = "yam";
-static const char yam_drvinfo[] __initdata = KERN_INFO \
+static const char yam_drvinfo[] __initconst = KERN_INFO \
 	"YAM driver version 0.8 by F1OAT/F6FBB\n";
 
 /* --------------------------------------------------------------------- */
diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c
index 91d25888a1b..d8b9b1e8ee0 100644
--- a/drivers/net/rionet.c
+++ b/drivers/net/rionet.c
@@ -26,7 +26,7 @@
 #include <linux/ethtool.h>
 
 #define DRV_NAME        "rionet"
-#define DRV_VERSION     "0.2"
+#define DRV_VERSION     "0.3"
 #define DRV_AUTHOR      "Matt Porter <mporter@kernel.crashing.org>"
 #define DRV_DESC        "Ethernet over RapidIO"
 
@@ -47,8 +47,7 @@ MODULE_LICENSE("GPL");
 
 #define RIONET_TX_RING_SIZE	CONFIG_RIONET_TX_SIZE
 #define RIONET_RX_RING_SIZE	CONFIG_RIONET_RX_SIZE
-
-static LIST_HEAD(rionet_peers);
+#define RIONET_MAX_NETS		8
 
 struct rionet_private {
 	struct rio_mport *mport;
@@ -69,16 +68,14 @@ struct rionet_peer {
 	struct resource *res;
 };
 
-static int rionet_check = 0;
-static int rionet_capable = 1;
+struct rionet_net {
+	struct net_device *ndev;
+	struct list_head peers;
+	struct rio_dev **active;
+	int nact;	/* number of active peers */
+};
 
-/*
- * This is a fast lookup table for translating TX
- * Ethernet packets into a destination RIO device. It
- * could be made into a hash table to save memory depending
- * on system trade-offs.
- */
-static struct rio_dev **rionet_active;
+static struct rionet_net nets[RIONET_MAX_NETS];
 
 #define is_rionet_capable(src_ops, dst_ops)			\
 			((src_ops & RIO_SRC_OPS_DATA_MSG) &&	\
@@ -175,6 +172,7 @@ static int rionet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	struct ethhdr *eth = (struct ethhdr *)skb->data;
 	u16 destid;
 	unsigned long flags;
+	int add_num = 1;
 
 	local_irq_save(flags);
 	if (!spin_trylock(&rnet->tx_lock)) {
@@ -182,7 +180,10 @@ static int rionet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 		return NETDEV_TX_LOCKED;
 	}
 
-	if ((rnet->tx_cnt + 1) > RIONET_TX_RING_SIZE) {
+	if (is_multicast_ether_addr(eth->h_dest))
+		add_num = nets[rnet->mport->id].nact;
+
+	if ((rnet->tx_cnt + add_num) > RIONET_TX_RING_SIZE) {
 		netif_stop_queue(ndev);
 		spin_unlock_irqrestore(&rnet->tx_lock, flags);
 		printk(KERN_ERR "%s: BUG! Tx Ring full when queue awake!\n",
@@ -191,15 +192,22 @@ static int rionet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	}
 
 	if (is_multicast_ether_addr(eth->h_dest)) {
+		int count = 0;
+
 		for (i = 0; i < RIO_MAX_ROUTE_ENTRIES(rnet->mport->sys_size);
 				i++)
-			if (rionet_active[i])
+			if (nets[rnet->mport->id].active[i]) {
 				rionet_queue_tx_msg(skb, ndev,
-						    rionet_active[i]);
+					nets[rnet->mport->id].active[i]);
+				if (count)
+					atomic_inc(&skb->users);
+				count++;
+			}
 	} else if (RIONET_MAC_MATCH(eth->h_dest)) {
 		destid = RIONET_GET_DESTID(eth->h_dest);
-		if (rionet_active[destid])
-			rionet_queue_tx_msg(skb, ndev, rionet_active[destid]);
+		if (nets[rnet->mport->id].active[destid])
+			rionet_queue_tx_msg(skb, ndev,
+					nets[rnet->mport->id].active[destid]);
 	}
 
 	spin_unlock_irqrestore(&rnet->tx_lock, flags);
@@ -218,16 +226,21 @@ static void rionet_dbell_event(struct rio_mport *mport, void *dev_id, u16 sid, u
 		printk(KERN_INFO "%s: doorbell sid %4.4x tid %4.4x info %4.4x",
 		       DRV_NAME, sid, tid, info);
 	if (info == RIONET_DOORBELL_JOIN) {
-		if (!rionet_active[sid]) {
-			list_for_each_entry(peer, &rionet_peers, node) {
-				if (peer->rdev->destid == sid)
-					rionet_active[sid] = peer->rdev;
+		if (!nets[rnet->mport->id].active[sid]) {
+			list_for_each_entry(peer,
+					   &nets[rnet->mport->id].peers, node) {
+				if (peer->rdev->destid == sid) {
+					nets[rnet->mport->id].active[sid] =
+								peer->rdev;
+					nets[rnet->mport->id].nact++;
+				}
 			}
 			rio_mport_send_doorbell(mport, sid,
 						RIONET_DOORBELL_JOIN);
 		}
 	} else if (info == RIONET_DOORBELL_LEAVE) {
-		rionet_active[sid] = NULL;
+		nets[rnet->mport->id].active[sid] = NULL;
+		nets[rnet->mport->id].nact--;
 	} else {
 		if (netif_msg_intr(rnet))
 			printk(KERN_WARNING "%s: unhandled doorbell\n",
@@ -321,7 +334,8 @@ static int rionet_open(struct net_device *ndev)
 	netif_carrier_on(ndev);
 	netif_start_queue(ndev);
 
-	list_for_each_entry_safe(peer, tmp, &rionet_peers, node) {
+	list_for_each_entry_safe(peer, tmp,
+				 &nets[rnet->mport->id].peers, node) {
 		if (!(peer->res = rio_request_outb_dbell(peer->rdev,
 							 RIONET_DOORBELL_JOIN,
 							 RIONET_DOORBELL_LEAVE)))
@@ -346,7 +360,7 @@ static int rionet_close(struct net_device *ndev)
 	int i;
 
 	if (netif_msg_ifup(rnet))
-		printk(KERN_INFO "%s: close\n", DRV_NAME);
+		printk(KERN_INFO "%s: close %s\n", DRV_NAME, ndev->name);
 
 	netif_stop_queue(ndev);
 	netif_carrier_off(ndev);
@@ -354,10 +368,11 @@ static int rionet_close(struct net_device *ndev)
 	for (i = 0; i < RIONET_RX_RING_SIZE; i++)
 		kfree_skb(rnet->rx_skb[i]);
 
-	list_for_each_entry_safe(peer, tmp, &rionet_peers, node) {
-		if (rionet_active[peer->rdev->destid]) {
+	list_for_each_entry_safe(peer, tmp,
+				 &nets[rnet->mport->id].peers, node) {
+		if (nets[rnet->mport->id].active[peer->rdev->destid]) {
 			rio_send_doorbell(peer->rdev, RIONET_DOORBELL_LEAVE);
-			rionet_active[peer->rdev->destid] = NULL;
+			nets[rnet->mport->id].active[peer->rdev->destid] = NULL;
 		}
 		rio_release_outb_dbell(peer->rdev, peer->res);
 	}
@@ -373,17 +388,21 @@ static int rionet_close(struct net_device *ndev)
 static void rionet_remove(struct rio_dev *rdev)
 {
 	struct net_device *ndev = rio_get_drvdata(rdev);
+	unsigned char netid = rdev->net->hport->id;
 	struct rionet_peer *peer, *tmp;
 
-	free_pages((unsigned long)rionet_active, get_order(sizeof(void *) *
-			RIO_MAX_ROUTE_ENTRIES(rdev->net->hport->sys_size)));
 	unregister_netdev(ndev);
-	free_netdev(ndev);
 
-	list_for_each_entry_safe(peer, tmp, &rionet_peers, node) {
+	free_pages((unsigned long)nets[netid].active, get_order(sizeof(void *) *
+			RIO_MAX_ROUTE_ENTRIES(rdev->net->hport->sys_size)));
+	nets[netid].active = NULL;
+
+	list_for_each_entry_safe(peer, tmp, &nets[netid].peers, node) {
 		list_del(&peer->node);
 		kfree(peer);
 	}
+
+	free_netdev(ndev);
 }
 
 static void rionet_get_drvinfo(struct net_device *ndev,
@@ -435,13 +454,13 @@ static int rionet_setup_netdev(struct rio_mport *mport, struct net_device *ndev)
 	const size_t rionet_active_bytes = sizeof(void *) *
 				RIO_MAX_ROUTE_ENTRIES(mport->sys_size);
 
-	rionet_active = (struct rio_dev **)__get_free_pages(GFP_KERNEL,
-			get_order(rionet_active_bytes));
-	if (!rionet_active) {
+	nets[mport->id].active = (struct rio_dev **)__get_free_pages(GFP_KERNEL,
+						get_order(rionet_active_bytes));
+	if (!nets[mport->id].active) {
 		rc = -ENOMEM;
 		goto out;
 	}
-	memset((void *)rionet_active, 0, rionet_active_bytes);
+	memset((void *)nets[mport->id].active, 0, rionet_active_bytes);
 
 	/* Set up private area */
 	rnet = netdev_priv(ndev);
@@ -470,60 +489,62 @@ static int rionet_setup_netdev(struct rio_mport *mport, struct net_device *ndev)
 	if (rc != 0)
 		goto out;
 
-	printk("%s: %s %s Version %s, MAC %pM\n",
+	printk(KERN_INFO "%s: %s %s Version %s, MAC %pM, %s\n",
 	       ndev->name,
 	       DRV_NAME,
 	       DRV_DESC,
 	       DRV_VERSION,
-	       ndev->dev_addr);
+	       ndev->dev_addr,
+	       mport->name);
 
       out:
 	return rc;
 }
 
-/*
- * XXX Make multi-net safe
- */
+static unsigned long net_table[RIONET_MAX_NETS/sizeof(unsigned long) + 1];
+
 static int rionet_probe(struct rio_dev *rdev, const struct rio_device_id *id)
 {
 	int rc = -ENODEV;
 	u32 lsrc_ops, ldst_ops;
 	struct rionet_peer *peer;
 	struct net_device *ndev = NULL;
+	unsigned char netid = rdev->net->hport->id;
+	int oldnet;
 
-	/* If local device is not rionet capable, give up quickly */
-	if (!rionet_capable)
-		goto out;
+	if (netid >= RIONET_MAX_NETS)
+		return rc;
 
-	/* Allocate our net_device structure */
-	ndev = alloc_etherdev(sizeof(struct rionet_private));
-	if (ndev == NULL) {
-		rc = -ENOMEM;
-		goto out;
-	}
+	oldnet = test_and_set_bit(netid, net_table);
 
 	/*
 	 * First time through, make sure local device is rionet
-	 * capable, setup netdev,  and set flags so this is skipped
-	 * on later probes
+	 * capable, setup netdev (will be skipped on later probes)
 	 */
-	if (!rionet_check) {
+	if (!oldnet) {
 		rio_local_read_config_32(rdev->net->hport, RIO_SRC_OPS_CAR,
 					 &lsrc_ops);
 		rio_local_read_config_32(rdev->net->hport, RIO_DST_OPS_CAR,
 					 &ldst_ops);
 		if (!is_rionet_capable(lsrc_ops, ldst_ops)) {
 			printk(KERN_ERR
-			       "%s: local device is not network capable\n",
-			       DRV_NAME);
-			rionet_check = 1;
-			rionet_capable = 0;
+			       "%s: local device %s is not network capable\n",
+			       DRV_NAME, rdev->net->hport->name);
 			goto out;
 		}
 
+		/* Allocate our net_device structure */
+		ndev = alloc_etherdev(sizeof(struct rionet_private));
+		if (ndev == NULL) {
+			rc = -ENOMEM;
+			goto out;
+		}
+		nets[netid].ndev = ndev;
 		rc = rionet_setup_netdev(rdev->net->hport, ndev);
-		rionet_check = 1;
-	}
+		INIT_LIST_HEAD(&nets[netid].peers);
+		nets[netid].nact = 0;
+	} else if (nets[netid].ndev == NULL)
+		goto out;
 
 	/*
 	 * If the remote device has mailbox/doorbell capabilities,
@@ -535,10 +556,10 @@ static int rionet_probe(struct rio_dev *rdev, const struct rio_device_id *id)
 			goto out;
 		}
 		peer->rdev = rdev;
-		list_add_tail(&peer->node, &rionet_peers);
+		list_add_tail(&peer->node, &nets[netid].peers);
 	}
 
-	rio_set_drvdata(rdev, ndev);
+	rio_set_drvdata(rdev, nets[netid].ndev);
 
       out:
 	return rc;
diff --git a/drivers/net/wan/z85230.c b/drivers/net/wan/z85230.c
index 0e576906170..feacc3b994b 100644
--- a/drivers/net/wan/z85230.c
+++ b/drivers/net/wan/z85230.c
@@ -1775,7 +1775,7 @@ EXPORT_SYMBOL(z8530_queue_xmit);
 /*
  *	Module support
  */
-static const char banner[] __initdata =
+static const char banner[] __initconst =
 	KERN_INFO "Generic Z85C30/Z85230 interface driver v0.02\n";
 
 static int __init z85230_init_driver(void)
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 682633bfe00..05593d88202 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -635,9 +635,7 @@ static void xen_netbk_rx_action(struct xen_netbk *netbk)
 		return;
 
 	BUG_ON(npo.copy_prod > ARRAY_SIZE(netbk->grant_copy_op));
-	ret = HYPERVISOR_grant_table_op(GNTTABOP_copy, &netbk->grant_copy_op,
-					npo.copy_prod);
-	BUG_ON(ret != 0);
+	gnttab_batch_copy(netbk->grant_copy_op, npo.copy_prod);
 
 	while ((skb = __skb_dequeue(&rxq)) != NULL) {
 		sco = (struct skb_cb_overlay *)skb->cb;
@@ -1460,18 +1458,15 @@ static void xen_netbk_tx_submit(struct xen_netbk *netbk)
 static void xen_netbk_tx_action(struct xen_netbk *netbk)
 {
 	unsigned nr_gops;
-	int ret;
 
 	nr_gops = xen_netbk_tx_build_gops(netbk);
 
 	if (nr_gops == 0)
 		return;
-	ret = HYPERVISOR_grant_table_op(GNTTABOP_copy,
-					netbk->tx_copy_ops, nr_gops);
-	BUG_ON(ret);
 
-	xen_netbk_tx_submit(netbk);
+	gnttab_batch_copy(netbk->tx_copy_ops, nr_gops);
 
+	xen_netbk_tx_submit(netbk);
 }
 
 static void xen_netbk_idx_release(struct xen_netbk *netbk, u16 pending_idx)
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 7e262a6124c..72e496f1e9b 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -9,8 +9,8 @@
 
 /* Max address size we deal with */
 #define OF_MAX_ADDR_CELLS	4
-#define OF_CHECK_COUNTS(na, ns)	((na) > 0 && (na) <= OF_MAX_ADDR_CELLS && \
-			(ns) > 0)
+#define OF_CHECK_ADDR_COUNT(na)	((na) > 0 && (na) <= OF_MAX_ADDR_CELLS)
+#define OF_CHECK_COUNTS(na, ns)	(OF_CHECK_ADDR_COUNT(na) && (ns) > 0)
 
 static struct of_bus *of_match_bus(struct device_node *np);
 static int __of_address_to_resource(struct device_node *dev,
@@ -69,6 +69,14 @@ static u64 of_bus_default_map(u32 *addr, const __be32 *range,
 		 (unsigned long long)cp, (unsigned long long)s,
 		 (unsigned long long)da);
 
+	/*
+	 * If the number of address cells is larger than 2 we assume the
+	 * mapping doesn't specify a physical address. Rather, the address
+	 * specifies an identifier that must match exactly.
+	 */
+	if (na > 2 && memcmp(range, addr, na * 4) != 0)
+		return OF_BAD_ADDR;
+
 	if (da < cp || da >= (cp + s))
 		return OF_BAD_ADDR;
 	return da - cp;
@@ -182,7 +190,7 @@ const __be32 *of_get_pci_address(struct device_node *dev, int bar_no, u64 *size,
 	}
 	bus->count_cells(dev, &na, &ns);
 	of_node_put(parent);
-	if (!OF_CHECK_COUNTS(na, ns))
+	if (!OF_CHECK_ADDR_COUNT(na))
 		return NULL;
 
 	/* Get "reg" or "assigned-addresses" property */
@@ -490,6 +498,25 @@ u64 of_translate_dma_address(struct device_node *dev, const __be32 *in_addr)
 }
 EXPORT_SYMBOL(of_translate_dma_address);
 
+bool of_can_translate_address(struct device_node *dev)
+{
+	struct device_node *parent;
+	struct of_bus *bus;
+	int na, ns;
+
+	parent = of_get_parent(dev);
+	if (parent == NULL)
+		return false;
+
+	bus = of_match_bus(parent);
+	bus->count_cells(dev, &na, &ns);
+
+	of_node_put(parent);
+
+	return OF_CHECK_COUNTS(na, ns);
+}
+EXPORT_SYMBOL(of_can_translate_address);
+
 const __be32 *of_get_address(struct device_node *dev, int index, u64 *size,
 		    unsigned int *flags)
 {
@@ -506,7 +533,7 @@ const __be32 *of_get_address(struct device_node *dev, int index, u64 *size,
 	bus = of_match_bus(parent);
 	bus->count_cells(dev, &na, &ns);
 	of_node_put(parent);
-	if (!OF_CHECK_COUNTS(na, ns))
+	if (!OF_CHECK_ADDR_COUNT(na))
 		return NULL;
 
 	/* Get "reg" or "assigned-addresses" property */
diff --git a/drivers/of/base.c b/drivers/of/base.c
index d4a1c9a043e..af3b22ac762 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -391,6 +391,29 @@ struct device_node *of_get_next_available_child(const struct device_node *node,
 EXPORT_SYMBOL(of_get_next_available_child);
 
 /**
+ *	of_get_child_by_name - Find the child node by name for a given parent
+ *	@node:	parent node
+ *	@name:	child name to look for.
+ *
+ *      This function looks for child node for given matching name
+ *
+ *	Returns a node pointer if found, with refcount incremented, use
+ *	of_node_put() on it when done.
+ *	Returns NULL if node is not found.
+ */
+struct device_node *of_get_child_by_name(const struct device_node *node,
+				const char *name)
+{
+	struct device_node *child;
+
+	for_each_child_of_node(node, child)
+		if (child->name && (of_node_cmp(child->name, name) == 0))
+			break;
+	return child;
+}
+EXPORT_SYMBOL(of_get_child_by_name);
+
+/**
  *	of_find_node_by_path - Find a node matching a full OF path
  *	@path:	The full path to match
  *
diff --git a/drivers/of/irq.c b/drivers/of/irq.c
index ff8ab7b2737..a23ec777999 100644
--- a/drivers/of/irq.c
+++ b/drivers/of/irq.c
@@ -392,6 +392,7 @@ int of_irq_to_resource_table(struct device_node *dev, struct resource *res,
 
 	return i;
 }
+EXPORT_SYMBOL_GPL(of_irq_to_resource_table);
 
 struct intc_desc {
 	struct list_head	list;
diff --git a/drivers/of/of_i2c.c b/drivers/of/of_i2c.c
index 1e173f35767..3550f3bf4f9 100644
--- a/drivers/of/of_i2c.c
+++ b/drivers/of/of_i2c.c
@@ -61,6 +61,9 @@ void of_i2c_register_devices(struct i2c_adapter *adap)
 		info.of_node = of_node_get(node);
 		info.archdata = &dev_ad;
 
+		if (of_get_property(node, "wakeup-source", NULL))
+			info.flags |= I2C_CLIENT_WAKE;
+
 		request_module("%s%s", I2C_MODULE_PREFIX, info.type);
 
 		result = i2c_new_device(adap, &info);
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index e44f8c2d239..9bdeaf30b17 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -78,6 +78,7 @@ void of_device_make_bus_id(struct device *dev)
 	struct device_node *node = dev->of_node;
 	const u32 *reg;
 	u64 addr;
+	const __be32 *addrp;
 	int magic;
 
 #ifdef CONFIG_PPC_DCR
@@ -105,7 +106,15 @@ void of_device_make_bus_id(struct device *dev)
 	 */
 	reg = of_get_property(node, "reg", NULL);
 	if (reg) {
-		addr = of_translate_address(node, reg);
+		if (of_can_translate_address(node)) {
+			addr = of_translate_address(node, reg);
+		} else {
+			addrp = of_get_address(node, 0, NULL, NULL);
+			if (addrp)
+				addr = of_read_number(addrp, 1);
+			else
+				addr = OF_BAD_ADDR;
+		}
 		if (addr != OF_BAD_ADDR) {
 			dev_set_name(dev, "%llx.%s",
 				     (unsigned long long)addr, node->name);
@@ -140,8 +149,9 @@ struct platform_device *of_device_alloc(struct device_node *np,
 		return NULL;
 
 	/* count the io and irq resources */
-	while (of_address_to_resource(np, num_reg, &temp_res) == 0)
-		num_reg++;
+	if (of_can_translate_address(np))
+		while (of_address_to_resource(np, num_reg, &temp_res) == 0)
+			num_reg++;
 	num_irq = of_irq_count(np);
 
 	/* Populate the resource table */
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index 9e1d2959e22..94c6e2aa03d 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -607,21 +607,6 @@ static int pci_pm_prepare(struct device *dev)
 	int error = 0;
 
 	/*
-	 * If a PCI device configured to wake up the system from sleep states
-	 * has been suspended at run time and there's a resume request pending
-	 * for it, this is equivalent to the device signaling wakeup, so the
-	 * system suspend operation should be aborted.
-	 */
-	pm_runtime_get_noresume(dev);
-	if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
-		pm_wakeup_event(dev, 0);
-
-	if (pm_wakeup_pending()) {
-		pm_runtime_put_sync(dev);
-		return -EBUSY;
-	}
-
-	/*
 	 * PCI devices suspended at run time need to be resumed at this
 	 * point, because in general it is necessary to reconfigure them for
 	 * system suspend.  Namely, if the device is supposed to wake up the
@@ -644,8 +629,6 @@ static void pci_pm_complete(struct device *dev)
 
 	if (drv && drv->pm && drv->pm->complete)
 		drv->pm->complete(dev);
-
-	pm_runtime_put_sync(dev);
 }
 
 #else /* !CONFIG_PM_SLEEP */
diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c
index def8d0b5620..0aab85a5155 100644
--- a/drivers/pci/xen-pcifront.c
+++ b/drivers/pci/xen-pcifront.c
@@ -21,6 +21,7 @@
 #include <linux/bitops.h>
 #include <linux/time.h>
 
+#include <asm/xen/swiotlb-xen.h>
 #define INVALID_GRANT_REF (0)
 #define INVALID_EVTCHN    (-1)
 
@@ -236,7 +237,7 @@ static int pcifront_bus_write(struct pci_bus *bus, unsigned int devfn,
 	return errno_to_pcibios_err(do_pci_op(pdev, &op));
 }
 
-struct pci_ops pcifront_bus_ops = {
+static struct pci_ops pcifront_bus_ops = {
 	.read = pcifront_bus_read,
 	.write = pcifront_bus_write,
 };
@@ -668,7 +669,7 @@ static irqreturn_t pcifront_handler_aer(int irq, void *dev)
 	schedule_pcifront_aer_op(pdev);
 	return IRQ_HANDLED;
 }
-static int pcifront_connect(struct pcifront_device *pdev)
+static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
 {
 	int err = 0;
 
@@ -681,9 +682,13 @@ static int pcifront_connect(struct pcifront_device *pdev)
 		dev_err(&pdev->xdev->dev, "PCI frontend already installed!\n");
 		err = -EEXIST;
 	}
-
 	spin_unlock(&pcifront_dev_lock);
 
+	if (!err && !swiotlb_nr_tbl()) {
+		err = pci_xen_swiotlb_init_late();
+		if (err)
+			dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
+	}
 	return err;
 }
 
@@ -842,10 +847,10 @@ static int __devinit pcifront_try_connect(struct pcifront_device *pdev)
 	    XenbusStateInitialised)
 		goto out;
 
-	err = pcifront_connect(pdev);
+	err = pcifront_connect_and_init_dma(pdev);
 	if (err) {
 		xenbus_dev_fatal(pdev->xdev, err,
-				 "Error connecting PCI Frontend");
+				 "Error setting up PCI Frontend");
 		goto out;
 	}
 
diff --git a/drivers/platform/x86/amilo-rfkill.c b/drivers/platform/x86/amilo-rfkill.c
index a514bf66fdd..1deca7f6c4e 100644
--- a/drivers/platform/x86/amilo-rfkill.c
+++ b/drivers/platform/x86/amilo-rfkill.c
@@ -74,7 +74,7 @@ static const struct rfkill_ops amilo_m7440_rfkill_ops = {
 	.set_block = amilo_m7440_rfkill_set_block
 };
 
-static const struct dmi_system_id __devinitdata amilo_rfkill_id_table[] = {
+static const struct dmi_system_id __devinitconst amilo_rfkill_id_table[] = {
 	{
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU SIEMENS"),
diff --git a/drivers/platform/x86/fujitsu-tablet.c b/drivers/platform/x86/fujitsu-tablet.c
index 7acae3f85f3..f77484528b1 100644
--- a/drivers/platform/x86/fujitsu-tablet.c
+++ b/drivers/platform/x86/fujitsu-tablet.c
@@ -52,7 +52,7 @@ struct fujitsu_config {
 	unsigned int quirks;
 };
 
-static unsigned short keymap_Lifebook_Tseries[KEYMAP_LEN] __initconst = {
+static unsigned short keymap_Lifebook_Tseries[KEYMAP_LEN] __initdata = {
 	KEY_RESERVED,
 	KEY_RESERVED,
 	KEY_RESERVED,
@@ -71,7 +71,7 @@ static unsigned short keymap_Lifebook_Tseries[KEYMAP_LEN] __initconst = {
 	KEY_LEFTALT
 };
 
-static unsigned short keymap_Lifebook_U810[KEYMAP_LEN] __initconst = {
+static unsigned short keymap_Lifebook_U810[KEYMAP_LEN] __initdata = {
 	KEY_RESERVED,
 	KEY_RESERVED,
 	KEY_RESERVED,
@@ -90,7 +90,7 @@ static unsigned short keymap_Lifebook_U810[KEYMAP_LEN] __initconst = {
 	KEY_LEFTALT
 };
 
-static unsigned short keymap_Stylistic_Tseries[KEYMAP_LEN] __initconst = {
+static unsigned short keymap_Stylistic_Tseries[KEYMAP_LEN] __initdata = {
 	KEY_RESERVED,
 	KEY_RESERVED,
 	KEY_RESERVED,
@@ -109,7 +109,7 @@ static unsigned short keymap_Stylistic_Tseries[KEYMAP_LEN] __initconst = {
 	KEY_LEFTALT
 };
 
-static unsigned short keymap_Stylistic_ST5xxx[KEYMAP_LEN] __initconst = {
+static unsigned short keymap_Stylistic_ST5xxx[KEYMAP_LEN] __initdata = {
 	KEY_RESERVED,
 	KEY_RESERVED,
 	KEY_RESERVED,
@@ -299,7 +299,7 @@ static int __devinit fujitsu_dmi_stylistic(const struct dmi_system_id *dmi)
 	return 1;
 }
 
-static struct dmi_system_id dmi_ids[] __initconst = {
+static const struct dmi_system_id dmi_ids[] __initconst = {
 	{
 		.callback = fujitsu_dmi_lifebook,
 		.ident = "Fujitsu Siemens P/T Series",
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index 9da5fe715e6..75dd651664a 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -522,7 +522,7 @@ static acpi_handle ec_handle;
 
 #define TPACPI_HANDLE(object, parent, paths...)			\
 	static acpi_handle  object##_handle;			\
-	static const acpi_handle *object##_parent __initdata =	\
+	static const acpi_handle * const object##_parent __initconst =	\
 						&parent##_handle; \
 	static char *object##_paths[] __initdata = { paths }
 
diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c
index e771487132f..2420d5af058 100644
--- a/drivers/pps/pps.c
+++ b/drivers/pps/pps.c
@@ -306,7 +306,7 @@ int pps_register_cdev(struct pps_device *pps)
 	if (err < 0)
 		return err;
 
-	pps->id &= MAX_ID_MASK;
+	pps->id &= MAX_IDR_MASK;
 	if (pps->id >= PPS_MAX_SOURCES) {
 		pr_err("%s: too many PPS sources in the system\n",
 					pps->info.name);
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index 90c5c7357a5..d7c6b83097c 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -115,6 +115,15 @@ config  PWM_TIEHRPWM
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-tiehrpwm.
 
+config PWM_TWL6030
+	tristate "TWL6030 PWM support"
+	depends on TWL4030_CORE
+	help
+	  Generic PWM framework driver for TWL6030.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-twl6030.
+
 config PWM_VT8500
 	tristate "vt8500 pwm support"
 	depends on ARCH_VT8500
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index e4b2c898964..78f123dca30 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -8,4 +8,5 @@ obj-$(CONFIG_PWM_SAMSUNG)	+= pwm-samsung.o
 obj-$(CONFIG_PWM_TEGRA)		+= pwm-tegra.o
 obj-$(CONFIG_PWM_TIECAP)	+= pwm-tiecap.o
 obj-$(CONFIG_PWM_TIEHRPWM)	+= pwm-tiehrpwm.o
+obj-$(CONFIG_PWM_TWL6030)	+= pwm-twl6030.o
 obj-$(CONFIG_PWM_VT8500)	+= pwm-vt8500.o
diff --git a/drivers/mfd/twl6030-pwm.c b/drivers/pwm/pwm-twl6030.c
index e8fee147678..8e6387864ca 100644
--- a/drivers/mfd/twl6030-pwm.c
+++ b/drivers/pwm/pwm-twl6030.c
@@ -20,6 +20,7 @@
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pwm.h>
 #include <linux/i2c/twl.h>
 #include <linux/slab.h>
 
@@ -45,40 +46,54 @@
 
 #define PWM_CTRL2_MODE_MASK	0x3
 
-struct pwm_device {
-	const char *label;
-	unsigned int pwm_id;
+struct twl6030_pwm_chip {
+	struct pwm_chip chip;
 };
 
-int pwm_config(struct pwm_device *pwm, int duty_ns, int period_ns)
+static int twl6030_pwm_request(struct pwm_chip *chip, struct pwm_device *pwm)
 {
-	u8 duty_cycle;
 	int ret;
+	u8 val;
 
-	if (pwm == NULL || period_ns == 0 || duty_ns > period_ns)
-		return -EINVAL;
+	/* Configure PWM */
+	val = PWM_CTRL2_DIS_PD | PWM_CTRL2_CURR_02 | PWM_CTRL2_SRC_VAC |
+	      PWM_CTRL2_MODE_HW;
 
-	duty_cycle = (duty_ns * PWM_CTRL1_MAX) / period_ns;
+	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, LED_PWM_CTRL2);
+	if (ret < 0) {
+		dev_err(chip->dev, "%s: Failed to configure PWM, Error %d\n",
+			pwm->label, ret);
+		return ret;
+	}
 
-	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, duty_cycle, LED_PWM_CTRL1);
+	return 0;
+}
 
+static int twl6030_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm,
+			      int duty_ns, int period_ns)
+{
+	u8 duty_cycle = (duty_ns * PWM_CTRL1_MAX) / period_ns;
+	int ret;
+
+	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, duty_cycle, LED_PWM_CTRL1);
 	if (ret < 0) {
 		pr_err("%s: Failed to configure PWM, Error %d\n",
 			pwm->label, ret);
 		return ret;
 	}
+
 	return 0;
 }
-EXPORT_SYMBOL(pwm_config);
 
-int pwm_enable(struct pwm_device *pwm)
+static int twl6030_pwm_enable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
-	u8 val;
 	int ret;
+	u8 val;
 
 	ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, LED_PWM_CTRL2);
 	if (ret < 0) {
-		pr_err("%s: Failed to enable PWM, Error %d\n", pwm->label, ret);
+		dev_err(chip->dev, "%s: Failed to enable PWM, Error %d\n",
+			pwm->label, ret);
 		return ret;
 	}
 
@@ -88,23 +103,23 @@ int pwm_enable(struct pwm_device *pwm)
 
 	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, LED_PWM_CTRL2);
 	if (ret < 0) {
-		pr_err("%s: Failed to enable PWM, Error %d\n", pwm->label, ret);
+		dev_err(chip->dev, "%s: Failed to enable PWM, Error %d\n",
+			pwm->label, ret);
 		return ret;
 	}
 
 	twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, LED_PWM_CTRL2);
 	return 0;
 }
-EXPORT_SYMBOL(pwm_enable);
 
-void pwm_disable(struct pwm_device *pwm)
+static void twl6030_pwm_disable(struct pwm_chip *chip, struct pwm_device *pwm)
 {
-	u8 val;
 	int ret;
+	u8 val;
 
 	ret = twl_i2c_read_u8(TWL6030_MODULE_ID1, &val, LED_PWM_CTRL2);
 	if (ret < 0) {
-		pr_err("%s: Failed to disable PWM, Error %d\n",
+		dev_err(chip->dev, "%s: Failed to disable PWM, Error %d\n",
 			pwm->label, ret);
 		return;
 	}
@@ -114,52 +129,56 @@ void pwm_disable(struct pwm_device *pwm)
 
 	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, LED_PWM_CTRL2);
 	if (ret < 0) {
-		pr_err("%s: Failed to disable PWM, Error %d\n",
+		dev_err(chip->dev, "%s: Failed to disable PWM, Error %d\n",
 			pwm->label, ret);
-		return;
 	}
-	return;
 }
-EXPORT_SYMBOL(pwm_disable);
 
-struct pwm_device *pwm_request(int pwm_id, const char *label)
+static const struct pwm_ops twl6030_pwm_ops = {
+	.request = twl6030_pwm_request,
+	.config = twl6030_pwm_config,
+	.enable = twl6030_pwm_enable,
+	.disable = twl6030_pwm_disable,
+};
+
+static int twl6030_pwm_probe(struct platform_device *pdev)
 {
-	u8 val;
+	struct twl6030_pwm_chip *twl6030;
 	int ret;
-	struct pwm_device *pwm;
-
-	pwm = kzalloc(sizeof(struct pwm_device), GFP_KERNEL);
-	if (pwm == NULL) {
-		pr_err("%s: failed to allocate memory\n", label);
-		return NULL;
-	}
 
-	pwm->label = label;
-	pwm->pwm_id = pwm_id;
-
-	/* Configure PWM */
-	val = PWM_CTRL2_DIS_PD | PWM_CTRL2_CURR_02 | PWM_CTRL2_SRC_VAC |
-		PWM_CTRL2_MODE_HW;
+	twl6030 = devm_kzalloc(&pdev->dev, sizeof(*twl6030), GFP_KERNEL);
+	if (!twl6030)
+		return -ENOMEM;
 
-	ret = twl_i2c_write_u8(TWL6030_MODULE_ID1, val, LED_PWM_CTRL2);
+	twl6030->chip.dev = &pdev->dev;
+	twl6030->chip.ops = &twl6030_pwm_ops;
+	twl6030->chip.base = -1;
+	twl6030->chip.npwm = 1;
 
-	if (ret < 0) {
-		pr_err("%s: Failed to configure PWM, Error %d\n",
-			 pwm->label, ret);
+	ret = pwmchip_add(&twl6030->chip);
+	if (ret < 0)
+		return ret;
 
-		kfree(pwm);
-		return NULL;
-	}
+	platform_set_drvdata(pdev, twl6030);
 
-	return pwm;
+	return 0;
 }
-EXPORT_SYMBOL(pwm_request);
 
-void pwm_free(struct pwm_device *pwm)
+static int twl6030_pwm_remove(struct platform_device *pdev)
 {
-	pwm_disable(pwm);
-	kfree(pwm);
+	struct twl6030_pwm_chip *twl6030 = platform_get_drvdata(pdev);
+
+	return pwmchip_remove(&twl6030->chip);
 }
-EXPORT_SYMBOL(pwm_free);
 
+static struct platform_driver twl6030_pwm_driver = {
+	.driver = {
+		.name = "twl6030-pwm",
+	},
+	.probe = twl6030_pwm_probe,
+	.remove = __devexit_p(twl6030_pwm_remove),
+};
+module_platform_driver(twl6030_pwm_driver);
+
+MODULE_ALIAS("platform:twl6030-pwm");
 MODULE_LICENSE("GPL");
diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c
index d5e1625bbac..38ecd8f4d60 100644
--- a/drivers/rapidio/devices/tsi721.c
+++ b/drivers/rapidio/devices/tsi721.c
@@ -862,6 +862,90 @@ static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv)
 }
 
 /**
+ * tsi721_rio_map_inb_mem -- Mapping inbound memory region.
+ * @mport: RapidIO master port
+ * @lstart: Local memory space start address.
+ * @rstart: RapidIO space start address.
+ * @size: The mapping region size.
+ * @flags: Flags for mapping. 0 for using default flags.
+ *
+ * Return: 0 -- Success.
+ *
+ * This function will create the inbound mapping
+ * from rstart to lstart.
+ */
+static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart,
+		u64 rstart, u32 size, u32 flags)
+{
+	struct tsi721_device *priv = mport->priv;
+	int i;
+	u32 regval;
+
+	if (!is_power_of_2(size) || size < 0x1000 ||
+	    ((u64)lstart & (size - 1)) || (rstart & (size - 1)))
+		return -EINVAL;
+
+	/* Search for free inbound translation window */
+	for (i = 0; i < TSI721_IBWIN_NUM; i++) {
+		regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
+		if (!(regval & TSI721_IBWIN_LB_WEN))
+			break;
+	}
+
+	if (i >= TSI721_IBWIN_NUM) {
+		dev_err(&priv->pdev->dev,
+			"Unable to find free inbound window\n");
+		return -EBUSY;
+	}
+
+	iowrite32(TSI721_IBWIN_SIZE(size) << 8,
+			priv->regs + TSI721_IBWIN_SZ(i));
+
+	iowrite32(((u64)lstart >> 32), priv->regs + TSI721_IBWIN_TUA(i));
+	iowrite32(((u64)lstart & TSI721_IBWIN_TLA_ADD),
+		  priv->regs + TSI721_IBWIN_TLA(i));
+
+	iowrite32(rstart >> 32, priv->regs + TSI721_IBWIN_UB(i));
+	iowrite32((rstart & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN,
+		priv->regs + TSI721_IBWIN_LB(i));
+	dev_dbg(&priv->pdev->dev,
+		"Configured IBWIN%d mapping (RIO_0x%llx -> PCIe_0x%llx)\n",
+		i, rstart, (unsigned long long)lstart);
+
+	return 0;
+}
+
+/**
+ * fsl_rio_unmap_inb_mem -- Unmapping inbound memory region.
+ * @mport: RapidIO master port
+ * @lstart: Local memory space start address.
+ */
+static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport,
+				dma_addr_t lstart)
+{
+	struct tsi721_device *priv = mport->priv;
+	int i;
+	u64 addr;
+	u32 regval;
+
+	/* Search for matching active inbound translation window */
+	for (i = 0; i < TSI721_IBWIN_NUM; i++) {
+		regval = ioread32(priv->regs + TSI721_IBWIN_LB(i));
+		if (regval & TSI721_IBWIN_LB_WEN) {
+			regval = ioread32(priv->regs + TSI721_IBWIN_TUA(i));
+			addr = (u64)regval << 32;
+			regval = ioread32(priv->regs + TSI721_IBWIN_TLA(i));
+			addr |= regval & TSI721_IBWIN_TLA_ADD;
+
+			if (addr == (u64)lstart) {
+				iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
+				break;
+			}
+		}
+	}
+}
+
+/**
  * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe)
  * translation regions.
  * @priv: pointer to tsi721 private data
@@ -874,7 +958,7 @@ static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv)
 
 	/* Disable all SR2PC inbound windows */
 	for (i = 0; i < TSI721_IBWIN_NUM; i++)
-		iowrite32(0, priv->regs + TSI721_IBWINLB(i));
+		iowrite32(0, priv->regs + TSI721_IBWIN_LB(i));
 }
 
 /**
@@ -2144,6 +2228,8 @@ static int __devinit tsi721_setup_mport(struct tsi721_device *priv)
 	ops->add_outb_message = tsi721_add_outb_message;
 	ops->add_inb_buffer = tsi721_add_inb_buffer;
 	ops->get_inb_message = tsi721_get_inb_message;
+	ops->map_inb = tsi721_rio_map_inb_mem;
+	ops->unmap_inb = tsi721_rio_unmap_inb_mem;
 
 	mport = kzalloc(sizeof(struct rio_mport), GFP_KERNEL);
 	if (!mport) {
@@ -2165,7 +2251,8 @@ static int __devinit tsi721_setup_mport(struct tsi721_device *priv)
 	rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff);
 	rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3);
 	rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3);
-	strcpy(mport->name, "Tsi721 mport");
+	snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)",
+		 dev_driver_string(&pdev->dev), dev_name(&pdev->dev));
 
 	/* Hook up interrupt handler */
 
@@ -2315,7 +2402,8 @@ static int __devinit tsi721_probe(struct pci_dev *pdev,
 
 	/* Configure DMA attributes. */
 	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
-		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
+		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (err) {
 			dev_info(&pdev->dev, "Unable to set DMA mask\n");
 			goto err_unmap_bars;
 		}
diff --git a/drivers/rapidio/devices/tsi721.h b/drivers/rapidio/devices/tsi721.h
index 59de9d7be34..7d5b13ba8d4 100644
--- a/drivers/rapidio/devices/tsi721.h
+++ b/drivers/rapidio/devices/tsi721.h
@@ -156,9 +156,18 @@
 
 #define TSI721_IBWIN_NUM	8
 
-#define TSI721_IBWINLB(x)	(0x29000 + (x) * 0x20)
-#define TSI721_IBWINLB_BA	0xfffff000
-#define TSI721_IBWINLB_WEN	0x00000001
+#define TSI721_IBWIN_LB(x)	(0x29000 + (x) * 0x20)
+#define TSI721_IBWIN_LB_BA	0xfffff000
+#define TSI721_IBWIN_LB_WEN	0x00000001
+
+#define TSI721_IBWIN_UB(x)	(0x29004 + (x) * 0x20)
+#define TSI721_IBWIN_SZ(x)	(0x29008 + (x) * 0x20)
+#define TSI721_IBWIN_SZ_SIZE	0x00001f00
+#define TSI721_IBWIN_SIZE(size)	(__fls(size) - 12)
+
+#define TSI721_IBWIN_TLA(x)	(0x2900c + (x) * 0x20)
+#define TSI721_IBWIN_TLA_ADD	0xfffff000
+#define TSI721_IBWIN_TUA(x)	(0x29010 + (x) * 0x20)
 
 #define TSI721_SR2PC_GEN_INTE	0x29800
 #define TSI721_SR2PC_PWE	0x29804
diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c
index 2bebd791a09..48e9041dd1e 100644
--- a/drivers/rapidio/rio-scan.c
+++ b/drivers/rapidio/rio-scan.c
@@ -31,27 +31,21 @@
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/timer.h>
+#include <linux/sched.h>
 #include <linux/jiffies.h>
 #include <linux/slab.h>
 
 #include "rio.h"
 
 LIST_HEAD(rio_devices);
-static LIST_HEAD(rio_switches);
-
-static void rio_enum_timeout(unsigned long);
 
 static void rio_init_em(struct rio_dev *rdev);
 
 DEFINE_SPINLOCK(rio_global_list_lock);
 
 static int next_destid = 0;
-static int next_net = 0;
 static int next_comptag = 1;
 
-static struct timer_list rio_enum_timer =
-TIMER_INITIALIZER(rio_enum_timeout, 0, 0);
-
 static int rio_mport_phys_table[] = {
 	RIO_EFB_PAR_EP_ID,
 	RIO_EFB_PAR_EP_REC_ID,
@@ -60,6 +54,114 @@ static int rio_mport_phys_table[] = {
 	-1,
 };
 
+
+/*
+ * rio_destid_alloc - Allocate next available destID for given network
+ * net: RIO network
+ *
+ * Returns next available device destination ID for the specified RIO network.
+ * Marks allocated ID as one in use.
+ * Returns RIO_INVALID_DESTID if new destID is not available.
+ */
+static u16 rio_destid_alloc(struct rio_net *net)
+{
+	int destid;
+	struct rio_id_table *idtab = &net->destid_table;
+
+	spin_lock(&idtab->lock);
+	destid = find_next_zero_bit(idtab->table, idtab->max, idtab->next);
+	if (destid >= idtab->max)
+		destid = find_first_zero_bit(idtab->table, idtab->max);
+
+	if (destid < idtab->max) {
+		idtab->next = destid + 1;
+		if (idtab->next >= idtab->max)
+			idtab->next = 0;
+		set_bit(destid, idtab->table);
+		destid += idtab->start;
+	} else
+		destid = RIO_INVALID_DESTID;
+
+	spin_unlock(&idtab->lock);
+	return (u16)destid;
+}
+
+/*
+ * rio_destid_reserve - Reserve the specivied destID
+ * net: RIO network
+ * destid: destID to reserve
+ *
+ * Tries to reserve the specified destID.
+ * Returns 0 if successfull.
+ */
+static int rio_destid_reserve(struct rio_net *net, u16 destid)
+{
+	int oldbit;
+	struct rio_id_table *idtab = &net->destid_table;
+
+	destid -= idtab->start;
+	spin_lock(&idtab->lock);
+	oldbit = test_and_set_bit(destid, idtab->table);
+	spin_unlock(&idtab->lock);
+	return oldbit;
+}
+
+/*
+ * rio_destid_free - free a previously allocated destID
+ * net: RIO network
+ * destid: destID to free
+ *
+ * Makes the specified destID available for use.
+ */
+static void rio_destid_free(struct rio_net *net, u16 destid)
+{
+	struct rio_id_table *idtab = &net->destid_table;
+
+	destid -= idtab->start;
+	spin_lock(&idtab->lock);
+	clear_bit(destid, idtab->table);
+	spin_unlock(&idtab->lock);
+}
+
+/*
+ * rio_destid_first - return first destID in use
+ * net: RIO network
+ */
+static u16 rio_destid_first(struct rio_net *net)
+{
+	int destid;
+	struct rio_id_table *idtab = &net->destid_table;
+
+	spin_lock(&idtab->lock);
+	destid = find_first_bit(idtab->table, idtab->max);
+	if (destid >= idtab->max)
+		destid = RIO_INVALID_DESTID;
+	else
+		destid += idtab->start;
+	spin_unlock(&idtab->lock);
+	return (u16)destid;
+}
+
+/*
+ * rio_destid_next - return next destID in use
+ * net: RIO network
+ * from: destination ID from which search shall continue
+ */
+static u16 rio_destid_next(struct rio_net *net, u16 from)
+{
+	int destid;
+	struct rio_id_table *idtab = &net->destid_table;
+
+	spin_lock(&idtab->lock);
+	destid = find_next_bit(idtab->table, idtab->max, from);
+	if (destid >= idtab->max)
+		destid = RIO_INVALID_DESTID;
+	else
+		destid += idtab->start;
+	spin_unlock(&idtab->lock);
+	return (u16)destid;
+}
+
 /**
  * rio_get_device_id - Get the base/extended device id for a device
  * @port: RIO master port
@@ -108,14 +210,15 @@ static void rio_local_set_device_id(struct rio_mport *port, u16 did)
 
 /**
  * rio_clear_locks- Release all host locks and signal enumeration complete
- * @port: Master port to issue transaction
+ * @net: RIO network to run on
  *
  * Marks the component tag CSR on each device with the enumeration
  * complete flag. When complete, it then release the host locks on
  * each device. Returns 0 on success or %-EINVAL on failure.
  */
-static int rio_clear_locks(struct rio_mport *port)
+static int rio_clear_locks(struct rio_net *net)
 {
+	struct rio_mport *port = net->hport;
 	struct rio_dev *rdev;
 	u32 result;
 	int ret = 0;
@@ -130,7 +233,7 @@ static int rio_clear_locks(struct rio_mport *port)
 		       result);
 		ret = -EINVAL;
 	}
-	list_for_each_entry(rdev, &rio_devices, global_list) {
+	list_for_each_entry(rdev, &net->devices, net_list) {
 		rio_write_config_32(rdev, RIO_HOST_DID_LOCK_CSR,
 				    port->host_deviceid);
 		rio_read_config_32(rdev, RIO_HOST_DID_LOCK_CSR, &result);
@@ -176,10 +279,6 @@ static int rio_enum_host(struct rio_mport *port)
 
 	/* Set master port destid and init destid ctr */
 	rio_local_set_device_id(port, port->host_deviceid);
-
-	if (next_destid == port->host_deviceid)
-		next_destid++;
-
 	return 0;
 }
 
@@ -446,9 +545,8 @@ static struct rio_dev __devinit *rio_setup_device(struct rio_net *net,
 	if (rio_device_has_destid(port, rdev->src_ops, rdev->dst_ops)) {
 		if (do_enum) {
 			rio_set_device_id(port, destid, hopcount, next_destid);
-			rdev->destid = next_destid++;
-			if (next_destid == port->host_deviceid)
-				next_destid++;
+			rdev->destid = next_destid;
+			next_destid = rio_destid_alloc(net);
 		} else
 			rdev->destid = rio_get_device_id(port, destid, hopcount);
 
@@ -483,7 +581,7 @@ static struct rio_dev __devinit *rio_setup_device(struct rio_net *net,
 			rswitch->clr_table(port, destid, hopcount,
 					   RIO_GLOBAL_TABLE);
 
-		list_add_tail(&rswitch->node, &rio_switches);
+		list_add_tail(&rswitch->node, &net->switches);
 
 	} else {
 		if (do_enum)
@@ -747,12 +845,7 @@ static u16 rio_get_host_deviceid_lock(struct rio_mport *port, u8 hopcount)
 static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 			 u8 hopcount, struct rio_dev *prev, int prev_port)
 {
-	int port_num;
-	int cur_destid;
-	int sw_destid;
-	int sw_inport;
 	struct rio_dev *rdev;
-	u16 destid;
 	u32 regval;
 	int tmp;
 
@@ -818,19 +911,26 @@ static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 		return -1;
 
 	if (rio_is_switch(rdev)) {
+		int sw_destid;
+		int cur_destid;
+		int sw_inport;
+		u16 destid;
+		int port_num;
+
 		sw_inport = RIO_GET_PORT_NUM(rdev->swpinfo);
 		rio_route_add_entry(rdev, RIO_GLOBAL_TABLE,
 				    port->host_deviceid, sw_inport, 0);
 		rdev->rswitch->route_table[port->host_deviceid] = sw_inport;
 
-		for (destid = 0; destid < next_destid; destid++) {
-			if (destid == port->host_deviceid)
-				continue;
-			rio_route_add_entry(rdev, RIO_GLOBAL_TABLE,
-					    destid, sw_inport, 0);
-			rdev->rswitch->route_table[destid] = sw_inport;
+		destid = rio_destid_first(net);
+		while (destid != RIO_INVALID_DESTID && destid < next_destid) {
+			if (destid != port->host_deviceid) {
+				rio_route_add_entry(rdev, RIO_GLOBAL_TABLE,
+						    destid, sw_inport, 0);
+				rdev->rswitch->route_table[destid] = sw_inport;
+			}
+			destid = rio_destid_next(net, destid + 1);
 		}
-
 		pr_debug(
 		    "RIO: found %s (vid %4.4x did %4.4x) with %d ports\n",
 		    rio_name(rdev), rdev->vid, rdev->did,
@@ -839,12 +939,10 @@ static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 		for (port_num = 0;
 		     port_num < RIO_GET_TOTAL_PORTS(rdev->swpinfo);
 		     port_num++) {
-			/*Enable Input Output Port (transmitter reviever)*/
-			rio_enable_rx_tx_port(port, 0,
+			if (sw_inport == port_num) {
+				rio_enable_rx_tx_port(port, 0,
 					      RIO_ANY_DESTID(port->sys_size),
 					      hopcount, port_num);
-
-			if (sw_inport == port_num) {
 				rdev->rswitch->port_ok |= (1 << port_num);
 				continue;
 			}
@@ -857,6 +955,9 @@ static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 				pr_debug(
 				    "RIO: scanning device on port %d\n",
 				    port_num);
+				rio_enable_rx_tx_port(port, 0,
+					      RIO_ANY_DESTID(port->sys_size),
+					      hopcount, port_num);
 				rdev->rswitch->port_ok |= (1 << port_num);
 				rio_route_add_entry(rdev, RIO_GLOBAL_TABLE,
 						RIO_ANY_DESTID(port->sys_size),
@@ -867,19 +968,22 @@ static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 					return -1;
 
 				/* Update routing tables */
-				if (next_destid > cur_destid) {
+				destid = rio_destid_next(net, cur_destid + 1);
+				if (destid != RIO_INVALID_DESTID) {
 					for (destid = cur_destid;
-					     destid < next_destid; destid++) {
-						if (destid == port->host_deviceid)
-							continue;
-						rio_route_add_entry(rdev,
+					     destid < next_destid;) {
+						if (destid != port->host_deviceid) {
+							rio_route_add_entry(rdev,
 								    RIO_GLOBAL_TABLE,
 								    destid,
 								    port_num,
 								    0);
-						rdev->rswitch->
-						    route_table[destid] =
-						    port_num;
+							rdev->rswitch->
+								route_table[destid] =
+								port_num;
+						}
+						destid = rio_destid_next(net,
+								destid + 1);
 					}
 				}
 			} else {
@@ -905,11 +1009,8 @@ static int __devinit rio_enum_peer(struct rio_net *net, struct rio_mport *port,
 		rio_init_em(rdev);
 
 		/* Check for empty switch */
-		if (next_destid == sw_destid) {
-			next_destid++;
-			if (next_destid == port->host_deviceid)
-				next_destid++;
-		}
+		if (next_destid == sw_destid)
+			next_destid = rio_destid_alloc(net);
 
 		rdev->destid = sw_destid;
 	} else
@@ -1047,48 +1148,71 @@ static int rio_mport_is_active(struct rio_mport *port)
 /**
  * rio_alloc_net- Allocate and configure a new RIO network
  * @port: Master port associated with the RIO network
+ * @do_enum: Enumeration/Discovery mode flag
+ * @start: logical minimal start id for new net
  *
  * Allocates a RIO network structure, initializes per-network
  * list heads, and adds the associated master port to the
  * network list of associated master ports. Returns a
  * RIO network pointer on success or %NULL on failure.
  */
-static struct rio_net __devinit *rio_alloc_net(struct rio_mport *port)
+static struct rio_net __devinit *rio_alloc_net(struct rio_mport *port,
+					       int do_enum, u16 start)
 {
 	struct rio_net *net;
 
 	net = kzalloc(sizeof(struct rio_net), GFP_KERNEL);
+	if (net && do_enum) {
+		net->destid_table.table = kzalloc(
+			BITS_TO_LONGS(RIO_MAX_ROUTE_ENTRIES(port->sys_size)) *
+			sizeof(long),
+			GFP_KERNEL);
+
+		if (net->destid_table.table == NULL) {
+			pr_err("RIO: failed to allocate destID table\n");
+			kfree(net);
+			net = NULL;
+		} else {
+			net->destid_table.start = start;
+			net->destid_table.next = 0;
+			net->destid_table.max =
+					RIO_MAX_ROUTE_ENTRIES(port->sys_size);
+			spin_lock_init(&net->destid_table.lock);
+		}
+	}
+
 	if (net) {
 		INIT_LIST_HEAD(&net->node);
 		INIT_LIST_HEAD(&net->devices);
+		INIT_LIST_HEAD(&net->switches);
 		INIT_LIST_HEAD(&net->mports);
 		list_add_tail(&port->nnode, &net->mports);
 		net->hport = port;
-		net->id = next_net++;
+		net->id = port->id;
 	}
 	return net;
 }
 
 /**
  * rio_update_route_tables- Updates route tables in switches
- * @port: Master port associated with the RIO network
+ * @net: RIO network to run update on
  *
  * For each enumerated device, ensure that each switch in a system
  * has correct routing entries. Add routes for devices that where
  * unknown dirung the first enumeration pass through the switch.
  */
-static void rio_update_route_tables(struct rio_mport *port)
+static void rio_update_route_tables(struct rio_net *net)
 {
 	struct rio_dev *rdev, *swrdev;
 	struct rio_switch *rswitch;
 	u8 sport;
 	u16 destid;
 
-	list_for_each_entry(rdev, &rio_devices, global_list) {
+	list_for_each_entry(rdev, &net->devices, net_list) {
 
 		destid = rdev->destid;
 
-		list_for_each_entry(rswitch, &rio_switches, node) {
+		list_for_each_entry(rswitch, &net->switches, node) {
 
 			if (rio_is_switch(rdev)	&& (rdev->rswitch == rswitch))
 				continue;
@@ -1166,12 +1290,16 @@ int __devinit rio_enum_mport(struct rio_mport *mport)
 
 	/* If master port has an active link, allocate net and enum peers */
 	if (rio_mport_is_active(mport)) {
-		if (!(net = rio_alloc_net(mport))) {
+		net = rio_alloc_net(mport, 1, 0);
+		if (!net) {
 			printk(KERN_ERR "RIO: failed to allocate new net\n");
 			rc = -ENOMEM;
 			goto out;
 		}
 
+		/* reserve mport destID in new net */
+		rio_destid_reserve(net, mport->host_deviceid);
+
 		/* Enable Input Output Port (transmitter reviever) */
 		rio_enable_rx_tx_port(mport, 1, 0, 0, 0);
 
@@ -1179,17 +1307,21 @@ int __devinit rio_enum_mport(struct rio_mport *mport)
 		rio_local_write_config_32(mport, RIO_COMPONENT_TAG_CSR,
 					  next_comptag++);
 
+		next_destid = rio_destid_alloc(net);
+
 		if (rio_enum_peer(net, mport, 0, NULL, 0) < 0) {
 			/* A higher priority host won enumeration, bail. */
 			printk(KERN_INFO
 			       "RIO: master port %d device has lost enumeration to a remote host\n",
 			       mport->id);
-			rio_clear_locks(mport);
+			rio_clear_locks(net);
 			rc = -EBUSY;
 			goto out;
 		}
-		rio_update_route_tables(mport);
-		rio_clear_locks(mport);
+		/* free the last allocated destID (unused) */
+		rio_destid_free(net, next_destid);
+		rio_update_route_tables(net);
+		rio_clear_locks(net);
 		rio_pw_enable(mport, 1);
 	} else {
 		printk(KERN_INFO "RIO: master port %d link inactive\n",
@@ -1203,47 +1335,34 @@ int __devinit rio_enum_mport(struct rio_mport *mport)
 
 /**
  * rio_build_route_tables- Generate route tables from switch route entries
+ * @net: RIO network to run route tables scan on
  *
  * For each switch device, generate a route table by copying existing
  * route entries from the switch.
  */
-static void rio_build_route_tables(void)
+static void rio_build_route_tables(struct rio_net *net)
 {
+	struct rio_switch *rswitch;
 	struct rio_dev *rdev;
 	int i;
 	u8 sport;
 
-	list_for_each_entry(rdev, &rio_devices, global_list)
-		if (rio_is_switch(rdev)) {
-			rio_lock_device(rdev->net->hport, rdev->destid,
-					rdev->hopcount, 1000);
-			for (i = 0;
-			     i < RIO_MAX_ROUTE_ENTRIES(rdev->net->hport->sys_size);
-			     i++) {
-				if (rio_route_get_entry(rdev,
-					RIO_GLOBAL_TABLE, i, &sport, 0) < 0)
-					continue;
-				rdev->rswitch->route_table[i] = sport;
-			}
+	list_for_each_entry(rswitch, &net->switches, node) {
+		rdev = sw_to_rio_dev(rswitch);
 
-			rio_unlock_device(rdev->net->hport,
-					  rdev->destid,
-					  rdev->hopcount);
+		rio_lock_device(net->hport, rdev->destid,
+				rdev->hopcount, 1000);
+		for (i = 0;
+		     i < RIO_MAX_ROUTE_ENTRIES(net->hport->sys_size);
+		     i++) {
+			if (rio_route_get_entry(rdev, RIO_GLOBAL_TABLE,
+						i, &sport, 0) < 0)
+				continue;
+			rswitch->route_table[i] = sport;
 		}
-}
 
-/**
- * rio_enum_timeout- Signal that enumeration timed out
- * @data: Address of timeout flag.
- *
- * When the enumeration complete timer expires, set a flag that
- * signals to the discovery process that enumeration did not
- * complete in a sane amount of time.
- */
-static void rio_enum_timeout(unsigned long data)
-{
-	/* Enumeration timed out, set flag */
-	*(int *)data = 1;
+		rio_unlock_device(net->hport, rdev->destid, rdev->hopcount);
+	}
 }
 
 /**
@@ -1259,34 +1378,33 @@ static void rio_enum_timeout(unsigned long data)
 int __devinit rio_disc_mport(struct rio_mport *mport)
 {
 	struct rio_net *net = NULL;
-	int enum_timeout_flag = 0;
+	unsigned long to_end;
 
 	printk(KERN_INFO "RIO: discover master port %d, %s\n", mport->id,
 	       mport->name);
 
 	/* If master port has an active link, allocate net and discover peers */
 	if (rio_mport_is_active(mport)) {
-		if (!(net = rio_alloc_net(mport))) {
-			printk(KERN_ERR "RIO: Failed to allocate new net\n");
-			goto bail;
-		}
+		pr_debug("RIO: wait for enumeration to complete...\n");
 
-		pr_debug("RIO: wait for enumeration complete...");
-
-		rio_enum_timer.expires =
-		    jiffies + CONFIG_RAPIDIO_DISC_TIMEOUT * HZ;
-		rio_enum_timer.data = (unsigned long)&enum_timeout_flag;
-		add_timer(&rio_enum_timer);
-		while (!rio_enum_complete(mport)) {
-			mdelay(1);
-			if (enum_timeout_flag) {
-				del_timer_sync(&rio_enum_timer);
-				goto timeout;
-			}
+		to_end = jiffies + CONFIG_RAPIDIO_DISC_TIMEOUT * HZ;
+		while (time_before(jiffies, to_end)) {
+			if (rio_enum_complete(mport))
+				goto enum_done;
+			schedule_timeout_uninterruptible(msecs_to_jiffies(10));
 		}
-		del_timer_sync(&rio_enum_timer);
 
-		pr_debug("done\n");
+		pr_debug("RIO: discovery timeout on mport %d %s\n",
+			 mport->id, mport->name);
+		goto bail;
+enum_done:
+		pr_debug("RIO: ... enumeration done\n");
+
+		net = rio_alloc_net(mport, 0, 0);
+		if (!net) {
+			printk(KERN_ERR "RIO: Failed to allocate new net\n");
+			goto bail;
+		}
 
 		/* Read DestID assigned by enumerator */
 		rio_local_read_config_32(mport, RIO_DID_CSR,
@@ -1302,13 +1420,10 @@ int __devinit rio_disc_mport(struct rio_mport *mport)
 			goto bail;
 		}
 
-		rio_build_route_tables();
+		rio_build_route_tables(net);
 	}
 
 	return 0;
-
-      timeout:
-	pr_debug("timeout\n");
-      bail:
+bail:
 	return -EBUSY;
 }
diff --git a/drivers/rapidio/rio.c b/drivers/rapidio/rio.c
index c40665a4fa3..d4bd69013c5 100644
--- a/drivers/rapidio/rio.c
+++ b/drivers/rapidio/rio.c
@@ -33,6 +33,7 @@
 
 static LIST_HEAD(rio_mports);
 static unsigned char next_portid;
+static DEFINE_SPINLOCK(rio_mmap_lock);
 
 /**
  * rio_local_get_device_id - Get the base/extended device id for a port
@@ -398,6 +399,49 @@ int rio_release_inb_pwrite(struct rio_dev *rdev)
 EXPORT_SYMBOL_GPL(rio_release_inb_pwrite);
 
 /**
+ * rio_map_inb_region -- Map inbound memory region.
+ * @mport: Master port.
+ * @lstart: physical address of memory region to be mapped
+ * @rbase: RIO base address assigned to this window
+ * @size: Size of the memory region
+ * @rflags: Flags for mapping.
+ *
+ * Return: 0 -- Success.
+ *
+ * This function will create the mapping from RIO space to local memory.
+ */
+int rio_map_inb_region(struct rio_mport *mport, dma_addr_t local,
+			u64 rbase, u32 size, u32 rflags)
+{
+	int rc = 0;
+	unsigned long flags;
+
+	if (!mport->ops->map_inb)
+		return -1;
+	spin_lock_irqsave(&rio_mmap_lock, flags);
+	rc = mport->ops->map_inb(mport, local, rbase, size, rflags);
+	spin_unlock_irqrestore(&rio_mmap_lock, flags);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(rio_map_inb_region);
+
+/**
+ * rio_unmap_inb_region -- Unmap the inbound memory region
+ * @mport: Master port
+ * @lstart: physical address of memory region to be unmapped
+ */
+void rio_unmap_inb_region(struct rio_mport *mport, dma_addr_t lstart)
+{
+	unsigned long flags;
+	if (!mport->ops->unmap_inb)
+		return;
+	spin_lock_irqsave(&rio_mmap_lock, flags);
+	mport->ops->unmap_inb(mport, lstart);
+	spin_unlock_irqrestore(&rio_mmap_lock, flags);
+}
+EXPORT_SYMBOL_GPL(rio_unmap_inb_region);
+
+/**
  * rio_mport_get_physefb - Helper function that returns register offset
  *                      for Physical Layer Extended Features Block.
  * @port: Master port to issue transaction
@@ -1216,15 +1260,62 @@ static int __devinit rio_init(void)
 	return 0;
 }
 
+static struct workqueue_struct *rio_wq;
+
+struct rio_disc_work {
+	struct work_struct	work;
+	struct rio_mport	*mport;
+};
+
+static void __devinit disc_work_handler(struct work_struct *_work)
+{
+	struct rio_disc_work *work;
+
+	work = container_of(_work, struct rio_disc_work, work);
+	pr_debug("RIO: discovery work for mport %d %s\n",
+		 work->mport->id, work->mport->name);
+	rio_disc_mport(work->mport);
+
+	kfree(work);
+}
+
 int __devinit rio_init_mports(void)
 {
 	struct rio_mport *port;
+	struct rio_disc_work *work;
+	int no_disc = 0;
 
 	list_for_each_entry(port, &rio_mports, node) {
 		if (port->host_deviceid >= 0)
 			rio_enum_mport(port);
-		else
-			rio_disc_mport(port);
+		else if (!no_disc) {
+			if (!rio_wq) {
+				rio_wq = alloc_workqueue("riodisc", 0, 0);
+				if (!rio_wq) {
+					pr_err("RIO: unable allocate rio_wq\n");
+					no_disc = 1;
+					continue;
+				}
+			}
+
+			work = kzalloc(sizeof *work, GFP_KERNEL);
+			if (!work) {
+				pr_err("RIO: no memory for work struct\n");
+				no_disc = 1;
+				continue;
+			}
+
+			work->mport = port;
+			INIT_WORK(&work->work, disc_work_handler);
+			queue_work(rio_wq, &work->work);
+		}
+	}
+
+	if (rio_wq) {
+		pr_debug("RIO: flush discovery workqueue\n");
+		flush_workqueue(rio_wq);
+		pr_debug("RIO: flush discovery workqueue finished\n");
+		destroy_workqueue(rio_wq);
 	}
 
 	rio_init();
diff --git a/drivers/regulator/88pm8607.c b/drivers/regulator/88pm8607.c
index c3482b954cb..1c5ab0172ea 100644
--- a/drivers/regulator/88pm8607.c
+++ b/drivers/regulator/88pm8607.c
@@ -12,6 +12,8 @@
 #include <linux/init.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/regulator/of_regulator.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
@@ -23,6 +25,7 @@ struct pm8607_regulator_info {
 	struct pm860x_chip	*chip;
 	struct regulator_dev	*regulator;
 	struct i2c_client	*i2c;
+	struct i2c_client	*i2c_8606;
 
 	unsigned int	*vol_table;
 	unsigned int	*vol_suspend;
@@ -242,6 +245,35 @@ static int pm8607_set_voltage_sel(struct regulator_dev *rdev, unsigned selector)
 	return ret;
 }
 
+static int pm8606_preg_enable(struct regulator_dev *rdev)
+{
+	struct pm8607_regulator_info *info = rdev_get_drvdata(rdev);
+
+	return pm860x_set_bits(info->i2c, rdev->desc->enable_reg,
+			       1 << rdev->desc->enable_mask, 0);
+}
+
+static int pm8606_preg_disable(struct regulator_dev *rdev)
+{
+	struct pm8607_regulator_info *info = rdev_get_drvdata(rdev);
+
+	return pm860x_set_bits(info->i2c, rdev->desc->enable_reg,
+			       1 << rdev->desc->enable_mask,
+			       1 << rdev->desc->enable_mask);
+}
+
+static int pm8606_preg_is_enabled(struct regulator_dev *rdev)
+{
+	struct pm8607_regulator_info *info = rdev_get_drvdata(rdev);
+	int ret;
+
+	ret = pm860x_reg_read(info->i2c, rdev->desc->enable_reg);
+	if (ret < 0)
+		return ret;
+
+	return !((unsigned char)ret & (1 << rdev->desc->enable_mask));
+}
+
 static struct regulator_ops pm8607_regulator_ops = {
 	.list_voltage	= pm8607_list_voltage,
 	.set_voltage_sel = pm8607_set_voltage_sel,
@@ -251,6 +283,25 @@ static struct regulator_ops pm8607_regulator_ops = {
 	.is_enabled = regulator_is_enabled_regmap,
 };
 
+static struct regulator_ops pm8606_preg_ops = {
+	.enable		= pm8606_preg_enable,
+	.disable	= pm8606_preg_disable,
+	.is_enabled	= pm8606_preg_is_enabled,
+};
+
+#define PM8606_PREG(ereg, ebit)						\
+{									\
+	.desc	= {							\
+		.name	= "PREG",					\
+		.ops	= &pm8606_preg_ops,				\
+		.type	= REGULATOR_CURRENT,				\
+		.id	= PM8606_ID_PREG,				\
+		.owner	= THIS_MODULE,					\
+		.enable_reg = PM8606_##ereg,				\
+		.enable_mask = (ebit),					\
+	},								\
+}
+
 #define PM8607_DVC(vreg, ureg, ubit, ereg, ebit)			\
 {									\
 	.desc	= {							\
@@ -311,6 +362,38 @@ static struct pm8607_regulator_info pm8607_regulator_info[] = {
 	PM8607_LDO(14,        LDO14, 0, SUPPLIES_EN12, 6),
 };
 
+static struct pm8607_regulator_info pm8606_regulator_info[] = {
+	PM8606_PREG(PREREGULATORB, 5),
+};
+
+#ifdef CONFIG_OF
+static int pm8607_regulator_dt_init(struct platform_device *pdev,
+				    struct pm8607_regulator_info *info,
+				    struct regulator_config *config)
+{
+	struct device_node *nproot, *np;
+	nproot = pdev->dev.parent->of_node;
+	if (!nproot)
+		return -ENODEV;
+	nproot = of_find_node_by_name(nproot, "regulators");
+	if (!nproot) {
+		dev_err(&pdev->dev, "failed to find regulators node\n");
+		return -ENODEV;
+	}
+	for_each_child_of_node(nproot, np) {
+		if (!of_node_cmp(np->name, info->desc.name)) {
+			config->init_data =
+				of_get_regulator_init_data(&pdev->dev, np);
+			config->of_node = np;
+			break;
+		}
+	}
+	return 0;
+}
+#else
+#define pm8607_regulator_dt_init(x, y, z)	(-1)
+#endif
+
 static int __devinit pm8607_regulator_probe(struct platform_device *pdev)
 {
 	struct pm860x_chip *chip = dev_get_drvdata(pdev->dev.parent);
@@ -320,22 +403,28 @@ static int __devinit pm8607_regulator_probe(struct platform_device *pdev)
 	struct resource *res;
 	int i;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource!\n");
-		return -EINVAL;
-	}
-	for (i = 0; i < ARRAY_SIZE(pm8607_regulator_info); i++) {
-		info = &pm8607_regulator_info[i];
-		if (info->desc.id == res->start)
-			break;
-	}
-	if (i == ARRAY_SIZE(pm8607_regulator_info)) {
-		dev_err(&pdev->dev, "Failed to find regulator %llu\n",
-			(unsigned long long)res->start);
-		return -EINVAL;
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
+	if (res) {
+		/* There're resources in 88PM8607 regulator driver */
+		for (i = 0; i < ARRAY_SIZE(pm8607_regulator_info); i++) {
+			info = &pm8607_regulator_info[i];
+			if (info->desc.vsel_reg == res->start)
+				break;
+		}
+		if (i == ARRAY_SIZE(pm8607_regulator_info)) {
+			dev_err(&pdev->dev, "Failed to find regulator %llu\n",
+				(unsigned long long)res->start);
+			return -EINVAL;
+		}
+	} else {
+		/* There's no resource in 88PM8606 PREG regulator driver */
+		info = &pm8606_regulator_info[0];
+		/* i is used to check regulator ID */
+		i = -1;
 	}
 	info->i2c = (chip->id == CHIP_PM8607) ? chip->client : chip->companion;
+	info->i2c_8606 = (chip->id == CHIP_PM8607) ? chip->companion :
+			chip->client;
 	info->chip = chip;
 
 	/* check DVC ramp slope double */
@@ -343,15 +432,17 @@ static int __devinit pm8607_regulator_probe(struct platform_device *pdev)
 		info->slope_double = 1;
 
 	config.dev = &pdev->dev;
-	config.init_data = pdata;
 	config.driver_data = info;
 
+	if (pm8607_regulator_dt_init(pdev, info, &config))
+		if (pdata)
+			config.init_data = pdata;
+
 	if (chip->id == CHIP_PM8607)
 		config.regmap = chip->regmap;
 	else
 		config.regmap = chip->regmap_companion;
 
-	/* replace driver_data with info */
 	info->regulator = regulator_register(&info->desc, &config);
 	if (IS_ERR(info->regulator)) {
 		dev_err(&pdev->dev, "failed to register regulator %s\n",
@@ -372,6 +463,18 @@ static int __devexit pm8607_regulator_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static struct platform_device_id pm8607_regulator_driver_ids[] = {
+	{
+		.name	= "88pm860x-regulator",
+		.driver_data	= 0,
+	}, {
+		.name	= "88pm860x-preg",
+		.driver_data	= 0,
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(platform, pm8607_regulator_driver_ids);
+
 static struct platform_driver pm8607_regulator_driver = {
 	.driver		= {
 		.name	= "88pm860x-regulator",
@@ -379,6 +482,7 @@ static struct platform_driver pm8607_regulator_driver = {
 	},
 	.probe		= pm8607_regulator_probe,
 	.remove		= __devexit_p(pm8607_regulator_remove),
+	.id_table	= pm8607_regulator_driver_ids,
 };
 
 static int __init pm8607_regulator_init(void)
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index e98a5e7827d..67d47b59a66 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -122,7 +122,7 @@ config REGULATOR_FAN53555
 
 config REGULATOR_ANATOP
 	tristate "Freescale i.MX on-chip ANATOP LDO regulators"
-	depends on MFD_ANATOP
+	depends on MFD_SYSCON
 	help
 	  Say y here to support Freescale i.MX on-chip ANATOP LDOs
 	  regulators. It is recommended that this option be
diff --git a/drivers/regulator/ab3100.c b/drivers/regulator/ab3100.c
index 65ad2b36ce3..df4ad8927f0 100644
--- a/drivers/regulator/ab3100.c
+++ b/drivers/regulator/ab3100.c
@@ -15,6 +15,7 @@
 #include <linux/err.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/driver.h>
+#include <linux/mfd/ab3100.h>
 #include <linux/mfd/abx500.h>
 
 /* LDO registers and some handy masking definitions for AB3100 */
diff --git a/drivers/regulator/anatop-regulator.c b/drivers/regulator/anatop-regulator.c
index ce0fe72a428..1af97686f44 100644
--- a/drivers/regulator/anatop-regulator.c
+++ b/drivers/regulator/anatop-regulator.c
@@ -21,19 +21,20 @@
 #include <linux/slab.h>
 #include <linux/device.h>
 #include <linux/module.h>
+#include <linux/mfd/syscon.h>
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
-#include <linux/mfd/anatop.h>
+#include <linux/regmap.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/of_regulator.h>
 
 struct anatop_regulator {
 	const char *name;
 	u32 control_reg;
-	struct anatop *mfd;
+	struct regmap *anatop;
 	int vol_bit_shift;
 	int vol_bit_width;
 	int min_bit_val;
@@ -43,7 +44,8 @@ struct anatop_regulator {
 	struct regulator_init_data *initdata;
 };
 
-static int anatop_set_voltage_sel(struct regulator_dev *reg, unsigned selector)
+static int anatop_regmap_set_voltage_sel(struct regulator_dev *reg,
+					unsigned selector)
 {
 	struct anatop_regulator *anatop_reg = rdev_get_drvdata(reg);
 	u32 val, mask;
@@ -56,12 +58,13 @@ static int anatop_set_voltage_sel(struct regulator_dev *reg, unsigned selector)
 	mask = ((1 << anatop_reg->vol_bit_width) - 1) <<
 		anatop_reg->vol_bit_shift;
 	val <<= anatop_reg->vol_bit_shift;
-	anatop_write_reg(anatop_reg->mfd, anatop_reg->control_reg, val, mask);
+	regmap_update_bits(anatop_reg->anatop, anatop_reg->control_reg,
+				mask, val);
 
 	return 0;
 }
 
-static int anatop_get_voltage_sel(struct regulator_dev *reg)
+static int anatop_regmap_get_voltage_sel(struct regulator_dev *reg)
 {
 	struct anatop_regulator *anatop_reg = rdev_get_drvdata(reg);
 	u32 val, mask;
@@ -69,7 +72,7 @@ static int anatop_get_voltage_sel(struct regulator_dev *reg)
 	if (!anatop_reg->control_reg)
 		return -ENOTSUPP;
 
-	val = anatop_read_reg(anatop_reg->mfd, anatop_reg->control_reg);
+	regmap_read(anatop_reg->anatop, anatop_reg->control_reg, &val);
 	mask = ((1 << anatop_reg->vol_bit_width) - 1) <<
 		anatop_reg->vol_bit_shift;
 	val = (val & mask) >> anatop_reg->vol_bit_shift;
@@ -78,8 +81,8 @@ static int anatop_get_voltage_sel(struct regulator_dev *reg)
 }
 
 static struct regulator_ops anatop_rops = {
-	.set_voltage_sel = anatop_set_voltage_sel,
-	.get_voltage_sel = anatop_get_voltage_sel,
+	.set_voltage_sel = anatop_regmap_set_voltage_sel,
+	.get_voltage_sel = anatop_regmap_get_voltage_sel,
 	.list_voltage = regulator_list_voltage_linear,
 	.map_voltage = regulator_map_voltage_linear,
 };
@@ -88,11 +91,11 @@ static int __devinit anatop_regulator_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
+	struct device_node *anatop_np;
 	struct regulator_desc *rdesc;
 	struct regulator_dev *rdev;
 	struct anatop_regulator *sreg;
 	struct regulator_init_data *initdata;
-	struct anatop *anatopmfd = dev_get_drvdata(pdev->dev.parent);
 	struct regulator_config config = { };
 	int ret = 0;
 
@@ -109,7 +112,15 @@ static int __devinit anatop_regulator_probe(struct platform_device *pdev)
 	rdesc->ops = &anatop_rops;
 	rdesc->type = REGULATOR_VOLTAGE;
 	rdesc->owner = THIS_MODULE;
-	sreg->mfd = anatopmfd;
+
+	anatop_np = of_get_parent(np);
+	if (!anatop_np)
+		return -ENODEV;
+	sreg->anatop = syscon_node_to_regmap(anatop_np);
+	of_node_put(anatop_np);
+	if (IS_ERR(sreg->anatop))
+		return PTR_ERR(sreg->anatop);
+
 	ret = of_property_read_u32(np, "anatop-reg-offset",
 				   &sreg->control_reg);
 	if (ret) {
diff --git a/drivers/regulator/max8925-regulator.c b/drivers/regulator/max8925-regulator.c
index 43dc97ec393..9bb0be37495 100644
--- a/drivers/regulator/max8925-regulator.c
+++ b/drivers/regulator/max8925-regulator.c
@@ -214,37 +214,36 @@ static struct max8925_regulator_info max8925_regulator_info[] = {
 	MAX8925_LDO(20, 750, 3900, 50),
 };
 
-static struct max8925_regulator_info * __devinit find_regulator_info(int id)
-{
-	struct max8925_regulator_info *ri;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(max8925_regulator_info); i++) {
-		ri = &max8925_regulator_info[i];
-		if (ri->desc.id == id)
-			return ri;
-	}
-	return NULL;
-}
-
 static int __devinit max8925_regulator_probe(struct platform_device *pdev)
 {
 	struct max8925_chip *chip = dev_get_drvdata(pdev->dev.parent);
-	struct max8925_platform_data *pdata = chip->dev->platform_data;
+	struct regulator_init_data *pdata = pdev->dev.platform_data;
 	struct regulator_config config = { };
 	struct max8925_regulator_info *ri;
+	struct resource *res;
 	struct regulator_dev *rdev;
+	int i;
 
-	ri = find_regulator_info(pdev->id);
-	if (ri == NULL) {
-		dev_err(&pdev->dev, "invalid regulator ID specified\n");
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource!\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < ARRAY_SIZE(max8925_regulator_info); i++) {
+		ri = &max8925_regulator_info[i];
+		if (ri->vol_reg == res->start)
+			break;
+	}
+	if (i == ARRAY_SIZE(max8925_regulator_info)) {
+		dev_err(&pdev->dev, "Failed to find regulator %llu\n",
+			(unsigned long long)res->start);
 		return -EINVAL;
 	}
 	ri->i2c = chip->i2c;
 	ri->chip = chip;
 
 	config.dev = &pdev->dev;
-	config.init_data = pdata->regulator[pdev->id];
+	config.init_data = pdata;
 	config.driver_data = ri;
 
 	rdev = regulator_register(&ri->desc, &config);
diff --git a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
index 2ba7502fa3b..07aee694ba9 100644
--- a/drivers/regulator/palmas-regulator.c
+++ b/drivers/regulator/palmas-regulator.c
@@ -22,6 +22,9 @@
 #include <linux/slab.h>
 #include <linux/regmap.h>
 #include <linux/mfd/palmas.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/regulator/of_regulator.h>
 
 struct regs_info {
 	char	*name;
@@ -568,10 +571,103 @@ static int palmas_ldo_init(struct palmas *palmas, int id,
 	return 0;
 }
 
+static struct of_regulator_match palmas_matches[] = {
+	{ .name = "smps12", },
+	{ .name = "smps123", },
+	{ .name = "smps3", },
+	{ .name = "smps45", },
+	{ .name = "smps457", },
+	{ .name = "smps6", },
+	{ .name = "smps7", },
+	{ .name = "smps8", },
+	{ .name = "smps9", },
+	{ .name = "smps10", },
+	{ .name = "ldo1", },
+	{ .name = "ldo2", },
+	{ .name = "ldo3", },
+	{ .name = "ldo4", },
+	{ .name = "ldo5", },
+	{ .name = "ldo6", },
+	{ .name = "ldo7", },
+	{ .name = "ldo8", },
+	{ .name = "ldo9", },
+	{ .name = "ldoln", },
+	{ .name = "ldousb", },
+};
+
+static void __devinit palmas_dt_to_pdata(struct device *dev,
+		struct device_node *node,
+		struct palmas_pmic_platform_data *pdata)
+{
+	struct device_node *regulators;
+	u32 prop;
+	int idx, ret;
+
+	regulators = of_find_node_by_name(node, "regulators");
+	if (!regulators) {
+		dev_info(dev, "regulator node not found\n");
+		return;
+	}
+
+	ret = of_regulator_match(dev, regulators, palmas_matches,
+			PALMAS_NUM_REGS);
+	if (ret < 0) {
+		dev_err(dev, "Error parsing regulator init data: %d\n", ret);
+		return;
+	}
+
+	for (idx = 0; idx < PALMAS_NUM_REGS; idx++) {
+		if (!palmas_matches[idx].init_data ||
+				!palmas_matches[idx].of_node)
+			continue;
+
+		pdata->reg_data[idx] = palmas_matches[idx].init_data;
+
+		pdata->reg_init[idx] = devm_kzalloc(dev,
+				sizeof(struct palmas_reg_init), GFP_KERNEL);
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,warm_reset", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->warm_reset = prop;
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,roof_floor", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->roof_floor = prop;
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,mode_sleep", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->mode_sleep = prop;
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,warm_reset", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->warm_reset = prop;
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,tstep", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->tstep = prop;
+
+		ret = of_property_read_u32(palmas_matches[idx].of_node,
+				"ti,vsel", &prop);
+		if (!ret)
+			pdata->reg_init[idx]->vsel = prop;
+	}
+
+	ret = of_property_read_u32(node, "ti,ldo6_vibrator", &prop);
+	if (!ret)
+		pdata->ldo6_vibrator = prop;
+}
+
+
 static __devinit int palmas_probe(struct platform_device *pdev)
 {
 	struct palmas *palmas = dev_get_drvdata(pdev->dev.parent);
 	struct palmas_pmic_platform_data *pdata = pdev->dev.platform_data;
+	struct device_node *node = pdev->dev.of_node;
 	struct regulator_dev *rdev;
 	struct regulator_config config = { };
 	struct palmas_pmic *pmic;
@@ -579,10 +675,14 @@ static __devinit int palmas_probe(struct platform_device *pdev)
 	int id = 0, ret;
 	unsigned int addr, reg;
 
-	if (!pdata)
-		return -EINVAL;
-	if (!pdata->reg_data)
-		return -EINVAL;
+	if (node && !pdata) {
+		pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+
+		if (!pdata)
+			return -ENOMEM;
+
+		palmas_dt_to_pdata(&pdev->dev, node, pdata);
+	}
 
 	pmic = devm_kzalloc(&pdev->dev, sizeof(*pmic), GFP_KERNEL);
 	if (!pmic)
@@ -661,7 +761,7 @@ static __devinit int palmas_probe(struct platform_device *pdev)
 		pmic->desc[id].owner = THIS_MODULE;
 
 		/* Initialise sleep/init values from platform data */
-		if (pdata && pdata->reg_init) {
+		if (pdata) {
 			reg_init = pdata->reg_init[id];
 			if (reg_init) {
 				ret = palmas_smps_init(palmas, id, reg_init);
@@ -685,11 +785,13 @@ static __devinit int palmas_probe(struct platform_device *pdev)
 				pmic->range[id] = 1;
 		}
 
-		if (pdata && pdata->reg_data)
+		if (pdata)
 			config.init_data = pdata->reg_data[id];
 		else
 			config.init_data = NULL;
 
+		config.of_node = palmas_matches[id].of_node;
+
 		rdev = regulator_register(&pmic->desc[id], &config);
 		if (IS_ERR(rdev)) {
 			dev_err(&pdev->dev,
@@ -726,11 +828,13 @@ static __devinit int palmas_probe(struct platform_device *pdev)
 						palmas_regs_info[id].ctrl_addr);
 		pmic->desc[id].enable_mask = PALMAS_LDO1_CTRL_MODE_ACTIVE;
 
-		if (pdata && pdata->reg_data)
+		if (pdata)
 			config.init_data = pdata->reg_data[id];
 		else
 			config.init_data = NULL;
 
+		config.of_node = palmas_matches[id].of_node;
+
 		rdev = regulator_register(&pmic->desc[id], &config);
 		if (IS_ERR(rdev)) {
 			dev_err(&pdev->dev,
@@ -744,7 +848,7 @@ static __devinit int palmas_probe(struct platform_device *pdev)
 		pmic->rdev[id] = rdev;
 
 		/* Initialise sleep/init values from platform data */
-		if (pdata->reg_init) {
+		if (pdata) {
 			reg_init = pdata->reg_init[id];
 			if (reg_init) {
 				ret = palmas_ldo_init(palmas, id, reg_init);
@@ -774,9 +878,15 @@ static int __devexit palmas_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static struct of_device_id __devinitdata of_palmas_match_tbl[] = {
+	{ .compatible = "ti,palmas-pmic", },
+	{ /* end */ }
+};
+
 static struct platform_driver palmas_driver = {
 	.driver = {
 		.name = "palmas-pmic",
+		.of_match_table = of_palmas_match_tbl,
 		.owner = THIS_MODULE,
 	},
 	.probe = palmas_probe,
@@ -799,3 +909,4 @@ MODULE_AUTHOR("Graeme Gregory <gg@slimlogic.co.uk>");
 MODULE_DESCRIPTION("Palmas voltage regulator driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:palmas-pmic");
+MODULE_DEVICE_TABLE(of, of_palmas_match_tbl);
diff --git a/drivers/regulator/wm831x-dcdc.c b/drivers/regulator/wm831x-dcdc.c
index 90cbcc68370..782c228a19b 100644
--- a/drivers/regulator/wm831x-dcdc.c
+++ b/drivers/regulator/wm831x-dcdc.c
@@ -475,9 +475,9 @@ static __devinit int wm831x_buckv_probe(struct platform_device *pdev)
 
 	dcdc->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
@@ -650,9 +650,9 @@ static __devinit int wm831x_buckp_probe(struct platform_device *pdev)
 
 	dcdc->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
@@ -794,9 +794,9 @@ static __devinit int wm831x_boostp_probe(struct platform_device *pdev)
 
 	dcdc->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
diff --git a/drivers/regulator/wm831x-isink.c b/drivers/regulator/wm831x-isink.c
index 0d207c29771..2646a1902b3 100644
--- a/drivers/regulator/wm831x-isink.c
+++ b/drivers/regulator/wm831x-isink.c
@@ -172,9 +172,9 @@ static __devinit int wm831x_isink_probe(struct platform_device *pdev)
 
 	isink->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
diff --git a/drivers/regulator/wm831x-ldo.c b/drivers/regulator/wm831x-ldo.c
index 9af512672be..c2dc03993dc 100644
--- a/drivers/regulator/wm831x-ldo.c
+++ b/drivers/regulator/wm831x-ldo.c
@@ -273,9 +273,9 @@ static __devinit int wm831x_gp_ldo_probe(struct platform_device *pdev)
 
 	ldo->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
@@ -530,9 +530,9 @@ static __devinit int wm831x_aldo_probe(struct platform_device *pdev)
 
 	ldo->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
@@ -687,9 +687,9 @@ static __devinit int wm831x_alive_ldo_probe(struct platform_device *pdev)
 
 	ldo->wm831x = wm831x;
 
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
 	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource\n");
+		dev_err(&pdev->dev, "No REG resource\n");
 		ret = -EINVAL;
 		goto err;
 	}
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index f8d818abf98..96ce101b906 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -4,11 +4,14 @@ menu "Remoteproc drivers (EXPERIMENTAL)"
 config REMOTEPROC
 	tristate
 	depends on EXPERIMENTAL
+	depends on HAS_DMA
 	select FW_CONFIG
+	select VIRTIO
 
 config OMAP_REMOTEPROC
 	tristate "OMAP remoteproc support"
 	depends on EXPERIMENTAL
+	depends on HAS_DMA
 	depends on ARCH_OMAP4
 	depends on OMAP_IOMMU
 	select REMOTEPROC
@@ -27,4 +30,15 @@ config OMAP_REMOTEPROC
 	  It's safe to say n here if you're not interested in multimedia
 	  offloading or just want a bare minimum kernel.
 
+config STE_MODEM_RPROC
+	tristate "STE-Modem remoteproc support"
+	depends on EXPERIMENTAL
+	depends on HAS_DMA
+	select REMOTEPROC
+	default n
+	help
+	  Say y or m here to support STE-Modem shared memory driver.
+	  This can be either built-in or a loadable module.
+	  If unsure say N.
+
 endmenu
diff --git a/drivers/remoteproc/Makefile b/drivers/remoteproc/Makefile
index 934ce6e2c66..391b65181c0 100644
--- a/drivers/remoteproc/Makefile
+++ b/drivers/remoteproc/Makefile
@@ -8,3 +8,4 @@ remoteproc-y				+= remoteproc_debugfs.o
 remoteproc-y				+= remoteproc_virtio.o
 remoteproc-y				+= remoteproc_elf_loader.o
 obj-$(CONFIG_OMAP_REMOTEPROC)		+= omap_remoteproc.o
+obj-$(CONFIG_STE_MODEM_RPROC)	 	+= ste_modem_rproc.o
diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c
index b54504ee61f..32c289c2ba1 100644
--- a/drivers/remoteproc/omap_remoteproc.c
+++ b/drivers/remoteproc/omap_remoteproc.c
@@ -116,6 +116,9 @@ static int omap_rproc_start(struct rproc *rproc)
 	struct omap_rproc_pdata *pdata = pdev->dev.platform_data;
 	int ret;
 
+	if (pdata->set_bootaddr)
+		pdata->set_bootaddr(rproc->bootaddr);
+
 	oproc->nb.notifier_call = omap_rproc_mbox_callback;
 
 	/* every omap rproc is assigned a mailbox instance for messaging */
diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c
index d5c2dbfc744..dd3bfaf1ad4 100644
--- a/drivers/remoteproc/remoteproc_core.c
+++ b/drivers/remoteproc/remoteproc_core.c
@@ -50,6 +50,18 @@ typedef int (*rproc_handle_resource_t)(struct rproc *rproc, void *, int avail);
 /* Unique indices for remoteproc devices */
 static DEFINE_IDA(rproc_dev_index);
 
+static const char * const rproc_crash_names[] = {
+	[RPROC_MMUFAULT]	= "mmufault",
+};
+
+/* translate rproc_crash_type to string */
+static const char *rproc_crash_to_string(enum rproc_crash_type type)
+{
+	if (type < ARRAY_SIZE(rproc_crash_names))
+		return rproc_crash_names[type];
+	return "unkown";
+}
+
 /*
  * This is the IOMMU fault handler we register with the IOMMU API
  * (when relevant; not all remote processors access memory through
@@ -57,18 +69,19 @@ static DEFINE_IDA(rproc_dev_index);
  *
  * IOMMU core will invoke this handler whenever the remote processor
  * will try to access an unmapped device address.
- *
- * Currently this is mostly a stub, but it will be later used to trigger
- * the recovery of the remote processor.
  */
 static int rproc_iommu_fault(struct iommu_domain *domain, struct device *dev,
 		unsigned long iova, int flags, void *token)
 {
+	struct rproc *rproc = token;
+
 	dev_err(dev, "iommu fault: da 0x%lx flags 0x%x\n", iova, flags);
 
+	rproc_report_crash(rproc, RPROC_MMUFAULT);
+
 	/*
 	 * Let the iommu core know we're not really handling this fault;
-	 * we just plan to use this as a recovery trigger.
+	 * we just used it as a recovery trigger.
 	 */
 	return -ENOSYS;
 }
@@ -215,8 +228,11 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i)
 		return ret;
 	}
 
-	dev_dbg(dev, "vring%d: va %p dma %x size %x idr %d\n", i, va,
-					dma, size, notifyid);
+	/* Store largest notifyid */
+	rproc->max_notifyid = max(rproc->max_notifyid, notifyid);
+
+	dev_dbg(dev, "vring%d: va %p dma %llx size %x idr %d\n", i, va,
+				(unsigned long long)dma, size, notifyid);
 
 	rvring->va = va;
 	rvring->dma = dma;
@@ -256,13 +272,25 @@ rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i)
 	return 0;
 }
 
+static int rproc_max_notifyid(int id, void *p, void *data)
+{
+	int *maxid = data;
+	*maxid = max(*maxid, id);
+	return 0;
+}
+
 void rproc_free_vring(struct rproc_vring *rvring)
 {
 	int size = PAGE_ALIGN(vring_size(rvring->len, rvring->align));
 	struct rproc *rproc = rvring->rvdev->rproc;
+	int maxid = 0;
 
 	dma_free_coherent(rproc->dev.parent, size, rvring->va, rvring->dma);
 	idr_remove(&rproc->notifyids, rvring->notifyid);
+
+	/* Find the largest remaining notifyid */
+	idr_for_each(&rproc->notifyids, rproc_max_notifyid, &maxid);
+	rproc->max_notifyid = maxid;
 }
 
 /**
@@ -545,17 +573,10 @@ static int rproc_handle_carveout(struct rproc *rproc,
 	dev_dbg(dev, "carveout rsc: da %x, pa %x, len %x, flags %x\n",
 			rsc->da, rsc->pa, rsc->len, rsc->flags);
 
-	mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
-	if (!mapping) {
-		dev_err(dev, "kzalloc mapping failed\n");
-		return -ENOMEM;
-	}
-
 	carveout = kzalloc(sizeof(*carveout), GFP_KERNEL);
 	if (!carveout) {
 		dev_err(dev, "kzalloc carveout failed\n");
-		ret = -ENOMEM;
-		goto free_mapping;
+		return -ENOMEM;
 	}
 
 	va = dma_alloc_coherent(dev->parent, rsc->len, &dma, GFP_KERNEL);
@@ -565,7 +586,8 @@ static int rproc_handle_carveout(struct rproc *rproc,
 		goto free_carv;
 	}
 
-	dev_dbg(dev, "carveout va %p, dma %x, len 0x%x\n", va, dma, rsc->len);
+	dev_dbg(dev, "carveout va %p, dma %llx, len 0x%x\n", va,
+					(unsigned long long)dma, rsc->len);
 
 	/*
 	 * Ok, this is non-standard.
@@ -585,11 +607,18 @@ static int rproc_handle_carveout(struct rproc *rproc,
 	 * physical address in this case.
 	 */
 	if (rproc->domain) {
+		mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
+		if (!mapping) {
+			dev_err(dev, "kzalloc mapping failed\n");
+			ret = -ENOMEM;
+			goto dma_free;
+		}
+
 		ret = iommu_map(rproc->domain, rsc->da, dma, rsc->len,
 								rsc->flags);
 		if (ret) {
 			dev_err(dev, "iommu_map failed: %d\n", ret);
-			goto dma_free;
+			goto free_mapping;
 		}
 
 		/*
@@ -603,7 +632,8 @@ static int rproc_handle_carveout(struct rproc *rproc,
 		mapping->len = rsc->len;
 		list_add_tail(&mapping->node, &rproc->mappings);
 
-		dev_dbg(dev, "carveout mapped 0x%x to 0x%x\n", rsc->da, dma);
+		dev_dbg(dev, "carveout mapped 0x%x to 0x%llx\n",
+					rsc->da, (unsigned long long)dma);
 	}
 
 	/*
@@ -634,12 +664,12 @@ static int rproc_handle_carveout(struct rproc *rproc,
 
 	return 0;
 
+free_mapping:
+	kfree(mapping);
 dma_free:
 	dma_free_coherent(dev->parent, rsc->len, va, dma);
 free_carv:
 	kfree(carveout);
-free_mapping:
-	kfree(mapping);
 	return ret;
 }
 
@@ -871,6 +901,91 @@ out:
 	complete_all(&rproc->firmware_loading_complete);
 }
 
+static int rproc_add_virtio_devices(struct rproc *rproc)
+{
+	int ret;
+
+	/* rproc_del() calls must wait until async loader completes */
+	init_completion(&rproc->firmware_loading_complete);
+
+	/*
+	 * We must retrieve early virtio configuration info from
+	 * the firmware (e.g. whether to register a virtio device,
+	 * what virtio features does it support, ...).
+	 *
+	 * We're initiating an asynchronous firmware loading, so we can
+	 * be built-in kernel code, without hanging the boot process.
+	 */
+	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG,
+				      rproc->firmware, &rproc->dev, GFP_KERNEL,
+				      rproc, rproc_fw_config_virtio);
+	if (ret < 0) {
+		dev_err(&rproc->dev, "request_firmware_nowait err: %d\n", ret);
+		complete_all(&rproc->firmware_loading_complete);
+	}
+
+	return ret;
+}
+
+/**
+ * rproc_trigger_recovery() - recover a remoteproc
+ * @rproc: the remote processor
+ *
+ * The recovery is done by reseting all the virtio devices, that way all the
+ * rpmsg drivers will be reseted along with the remote processor making the
+ * remoteproc functional again.
+ *
+ * This function can sleep, so it cannot be called from atomic context.
+ */
+int rproc_trigger_recovery(struct rproc *rproc)
+{
+	struct rproc_vdev *rvdev, *rvtmp;
+
+	dev_err(&rproc->dev, "recovering %s\n", rproc->name);
+
+	init_completion(&rproc->crash_comp);
+
+	/* clean up remote vdev entries */
+	list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node)
+		rproc_remove_virtio_dev(rvdev);
+
+	/* wait until there is no more rproc users */
+	wait_for_completion(&rproc->crash_comp);
+
+	return rproc_add_virtio_devices(rproc);
+}
+
+/**
+ * rproc_crash_handler_work() - handle a crash
+ *
+ * This function needs to handle everything related to a crash, like cpu
+ * registers and stack dump, information to help to debug the fatal error, etc.
+ */
+static void rproc_crash_handler_work(struct work_struct *work)
+{
+	struct rproc *rproc = container_of(work, struct rproc, crash_handler);
+	struct device *dev = &rproc->dev;
+
+	dev_dbg(dev, "enter %s\n", __func__);
+
+	mutex_lock(&rproc->lock);
+
+	if (rproc->state == RPROC_CRASHED || rproc->state == RPROC_OFFLINE) {
+		/* handle only the first crash detected */
+		mutex_unlock(&rproc->lock);
+		return;
+	}
+
+	rproc->state = RPROC_CRASHED;
+	dev_err(dev, "handling crash #%u in %s\n", ++rproc->crash_cnt,
+		rproc->name);
+
+	mutex_unlock(&rproc->lock);
+
+	if (!rproc->recovery_disabled)
+		rproc_trigger_recovery(rproc);
+}
+
 /**
  * rproc_boot() - boot a remote processor
  * @rproc: handle of a remote processor
@@ -992,6 +1107,10 @@ void rproc_shutdown(struct rproc *rproc)
 
 	rproc_disable_iommu(rproc);
 
+	/* if in crash state, unlock crash handler */
+	if (rproc->state == RPROC_CRASHED)
+		complete_all(&rproc->crash_comp);
+
 	rproc->state = RPROC_OFFLINE;
 
 	dev_info(dev, "stopped remote processor %s\n", rproc->name);
@@ -1026,7 +1145,7 @@ EXPORT_SYMBOL(rproc_shutdown);
 int rproc_add(struct rproc *rproc)
 {
 	struct device *dev = &rproc->dev;
-	int ret = 0;
+	int ret;
 
 	ret = device_add(dev);
 	if (ret < 0)
@@ -1040,26 +1159,7 @@ int rproc_add(struct rproc *rproc)
 	/* create debugfs entries */
 	rproc_create_debug_dir(rproc);
 
-	/* rproc_del() calls must wait until async loader completes */
-	init_completion(&rproc->firmware_loading_complete);
-
-	/*
-	 * We must retrieve early virtio configuration info from
-	 * the firmware (e.g. whether to register a virtio device,
-	 * what virtio features does it support, ...).
-	 *
-	 * We're initiating an asynchronous firmware loading, so we can
-	 * be built-in kernel code, without hanging the boot process.
-	 */
-	ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_HOTPLUG,
-					rproc->firmware, dev, GFP_KERNEL,
-					rproc, rproc_fw_config_virtio);
-	if (ret < 0) {
-		dev_err(dev, "request_firmware_nowait failed: %d\n", ret);
-		complete_all(&rproc->firmware_loading_complete);
-	}
-
-	return ret;
+	return rproc_add_virtio_devices(rproc);
 }
 EXPORT_SYMBOL(rproc_add);
 
@@ -1165,6 +1265,9 @@ struct rproc *rproc_alloc(struct device *dev, const char *name,
 	INIT_LIST_HEAD(&rproc->traces);
 	INIT_LIST_HEAD(&rproc->rvdevs);
 
+	INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work);
+	init_completion(&rproc->crash_comp);
+
 	rproc->state = RPROC_OFFLINE;
 
 	return rproc;
@@ -1221,6 +1324,32 @@ int rproc_del(struct rproc *rproc)
 }
 EXPORT_SYMBOL(rproc_del);
 
+/**
+ * rproc_report_crash() - rproc crash reporter function
+ * @rproc: remote processor
+ * @type: crash type
+ *
+ * This function must be called every time a crash is detected by the low-level
+ * drivers implementing a specific remoteproc. This should not be called from a
+ * non-remoteproc driver.
+ *
+ * This function can be called from atomic/interrupt context.
+ */
+void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type)
+{
+	if (!rproc) {
+		pr_err("NULL rproc pointer\n");
+		return;
+	}
+
+	dev_err(&rproc->dev, "crash detected in %s: type %s\n",
+		rproc->name, rproc_crash_to_string(type));
+
+	/* create a new task to handle the error */
+	schedule_work(&rproc->crash_handler);
+}
+EXPORT_SYMBOL(rproc_report_crash);
+
 static int __init remoteproc_init(void)
 {
 	rproc_init_debugfs();
diff --git a/drivers/remoteproc/remoteproc_debugfs.c b/drivers/remoteproc/remoteproc_debugfs.c
index 03833850f21..157a5730960 100644
--- a/drivers/remoteproc/remoteproc_debugfs.c
+++ b/drivers/remoteproc/remoteproc_debugfs.c
@@ -28,6 +28,9 @@
 #include <linux/debugfs.h>
 #include <linux/remoteproc.h>
 #include <linux/device.h>
+#include <linux/uaccess.h>
+
+#include "remoteproc_internal.h"
 
 /* remoteproc debugfs parent dir */
 static struct dentry *rproc_dbg;
@@ -79,7 +82,7 @@ static ssize_t rproc_state_read(struct file *filp, char __user *userbuf,
 
 	state = rproc->state > RPROC_LAST ? RPROC_LAST : rproc->state;
 
-	i = snprintf(buf, 30, "%.28s (%d)\n", rproc_state_string[state],
+	i = scnprintf(buf, 30, "%.28s (%d)\n", rproc_state_string[state],
 							rproc->state);
 
 	return simple_read_from_buffer(userbuf, count, ppos, buf, i);
@@ -100,7 +103,7 @@ static ssize_t rproc_name_read(struct file *filp, char __user *userbuf,
 	char buf[100];
 	int i;
 
-	i = snprintf(buf, sizeof(buf), "%.98s\n", rproc->name);
+	i = scnprintf(buf, sizeof(buf), "%.98s\n", rproc->name);
 
 	return simple_read_from_buffer(userbuf, count, ppos, buf, i);
 }
@@ -111,6 +114,82 @@ static const struct file_operations rproc_name_ops = {
 	.llseek	= generic_file_llseek,
 };
 
+/* expose recovery flag via debugfs */
+static ssize_t rproc_recovery_read(struct file *filp, char __user *userbuf,
+				   size_t count, loff_t *ppos)
+{
+	struct rproc *rproc = filp->private_data;
+	char *buf = rproc->recovery_disabled ? "disabled\n" : "enabled\n";
+
+	return simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf));
+}
+
+/*
+ * By writing to the 'recovery' debugfs entry, we control the behavior of the
+ * recovery mechanism dynamically. The default value of this entry is "enabled".
+ *
+ * The 'recovery' debugfs entry supports these commands:
+ *
+ * enabled:	When enabled, the remote processor will be automatically
+ *		recovered whenever it crashes. Moreover, if the remote
+ *		processor crashes while recovery is disabled, it will
+ *		be automatically recovered too as soon as recovery is enabled.
+ *
+ * disabled:	When disabled, a remote processor will remain in a crashed
+ *		state if it crashes. This is useful for debugging purposes;
+ *		without it, debugging a crash is substantially harder.
+ *
+ * recover:	This function will trigger an immediate recovery if the
+ *		remote processor is in a crashed state, without changing
+ *		or checking the recovery state (enabled/disabled).
+ *		This is useful during debugging sessions, when one expects
+ *		additional crashes to happen after enabling recovery. In this
+ *		case, enabling recovery will make it hard to debug subsequent
+ *		crashes, so it's recommended to keep recovery disabled, and
+ *		instead use the "recover" command as needed.
+ */
+static ssize_t
+rproc_recovery_write(struct file *filp, const char __user *user_buf,
+		     size_t count, loff_t *ppos)
+{
+	struct rproc *rproc = filp->private_data;
+	char buf[10];
+	int ret;
+
+	if (count > sizeof(buf))
+		return count;
+
+	ret = copy_from_user(buf, user_buf, count);
+	if (ret)
+		return -EFAULT;
+
+	/* remove end of line */
+	if (buf[count - 1] == '\n')
+		buf[count - 1] = '\0';
+
+	if (!strncmp(buf, "enabled", count)) {
+		rproc->recovery_disabled = false;
+		/* if rproc has crashed, trigger recovery */
+		if (rproc->state == RPROC_CRASHED)
+			rproc_trigger_recovery(rproc);
+	} else if (!strncmp(buf, "disabled", count)) {
+		rproc->recovery_disabled = true;
+	} else if (!strncmp(buf, "recover", count)) {
+		/* if rproc has crashed, trigger recovery */
+		if (rproc->state == RPROC_CRASHED)
+			rproc_trigger_recovery(rproc);
+	}
+
+	return count;
+}
+
+static const struct file_operations rproc_recovery_ops = {
+	.read = rproc_recovery_read,
+	.write = rproc_recovery_write,
+	.open = simple_open,
+	.llseek = generic_file_llseek,
+};
+
 void rproc_remove_trace_file(struct dentry *tfile)
 {
 	debugfs_remove(tfile);
@@ -154,6 +233,8 @@ void rproc_create_debug_dir(struct rproc *rproc)
 					rproc, &rproc_name_ops);
 	debugfs_create_file("state", 0400, rproc->dbg_dir,
 					rproc, &rproc_state_ops);
+	debugfs_create_file("recovery", 0400, rproc->dbg_dir,
+					rproc, &rproc_recovery_ops);
 }
 
 void __init rproc_init_debugfs(void)
diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h
index a690ebe7aa5..7bb66482d06 100644
--- a/drivers/remoteproc/remoteproc_internal.h
+++ b/drivers/remoteproc/remoteproc_internal.h
@@ -63,6 +63,7 @@ void rproc_free_vring(struct rproc_vring *rvring);
 int rproc_alloc_vring(struct rproc_vdev *rvdev, int i);
 
 void *rproc_da_to_va(struct rproc *rproc, u64 da, int len);
+int rproc_trigger_recovery(struct rproc *rproc);
 
 static inline
 int rproc_fw_sanity_check(struct rproc *rproc, const struct firmware *fw)
diff --git a/drivers/remoteproc/ste_modem_rproc.c b/drivers/remoteproc/ste_modem_rproc.c
new file mode 100644
index 00000000000..a7743c06933
--- /dev/null
+++ b/drivers/remoteproc/ste_modem_rproc.c
@@ -0,0 +1,322 @@
+/*
+ * Copyright (C) ST-Ericsson AB 2012
+ * Author: Sjur Brændeland <sjur.brandeland@stericsson.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+#include <linux/remoteproc.h>
+#include <linux/ste_modem_shm.h>
+#include "remoteproc_internal.h"
+
+#define SPROC_FW_SIZE (50 * 4096)
+#define SPROC_MAX_TOC_ENTRIES 32
+#define SPROC_MAX_NOTIFY_ID 14
+#define SPROC_RESOURCE_NAME "rsc-table"
+#define SPROC_MODEM_NAME "ste-modem"
+#define SPROC_MODEM_FIRMWARE SPROC_MODEM_NAME "-fw.bin"
+
+#define sproc_dbg(sproc, fmt, ...) \
+	dev_dbg(&sproc->mdev->pdev.dev, fmt, ##__VA_ARGS__)
+#define sproc_err(sproc, fmt, ...) \
+	dev_err(&sproc->mdev->pdev.dev, fmt, ##__VA_ARGS__)
+
+/* STE-modem control structure */
+struct sproc {
+	struct rproc *rproc;
+	struct ste_modem_device *mdev;
+	int error;
+	void *fw_addr;
+	size_t fw_size;
+	dma_addr_t fw_dma_addr;
+};
+
+/* STE-Modem firmware entry */
+struct ste_toc_entry {
+	__le32 start;
+	__le32 size;
+	__le32 flags;
+	__le32 entry_point;
+	__le32 load_addr;
+	char name[12];
+};
+
+/*
+ * The Table Of Content is located at the start of the firmware image and
+ * at offset zero in the shared memory region. The resource table typically
+ * contains the initial boot image (boot strap) and other information elements
+ * such as remoteproc resource table. Each entry is identified by a unique
+ * name.
+ */
+struct ste_toc {
+	struct ste_toc_entry table[SPROC_MAX_TOC_ENTRIES];
+};
+
+/* Loads the firmware to shared memory. */
+static int sproc_load_segments(struct rproc *rproc, const struct firmware *fw)
+{
+	struct sproc *sproc = rproc->priv;
+
+	memcpy(sproc->fw_addr, fw->data, fw->size);
+
+	return 0;
+}
+
+/* Find the entry for resource table in the Table of Content */
+static struct ste_toc_entry *sproc_find_rsc_entry(const struct firmware *fw)
+{
+	int i;
+	struct ste_toc *toc;
+
+	if (!fw)
+		return NULL;
+
+	toc = (void *)fw->data;
+
+	/* Search the table for the resource table */
+	for (i = 0; i < SPROC_MAX_TOC_ENTRIES &&
+		    toc->table[i].start != 0xffffffff; i++) {
+
+		if (!strncmp(toc->table[i].name, SPROC_RESOURCE_NAME,
+			     sizeof(toc->table[i].name))) {
+			if (toc->table[i].start > fw->size)
+				return NULL;
+			return &toc->table[i];
+		}
+	}
+
+	return NULL;
+}
+
+/* Find the resource table inside the remote processor's firmware. */
+static struct resource_table *
+sproc_find_rsc_table(struct rproc *rproc, const struct firmware *fw,
+		     int *tablesz)
+{
+	struct sproc *sproc = rproc->priv;
+	struct resource_table *table;
+	struct ste_toc_entry *entry;
+
+	entry = sproc_find_rsc_entry(fw);
+	if (!entry) {
+		sproc_err(sproc, "resource table not found in fw\n");
+		return NULL;
+	}
+
+	table = (void *)(fw->data + entry->start);
+
+	/* sanity check size and offset of resource table */
+	if (entry->start > SPROC_FW_SIZE ||
+	    entry->size > SPROC_FW_SIZE ||
+	    fw->size > SPROC_FW_SIZE ||
+	    entry->start + entry->size > fw->size ||
+	    sizeof(struct resource_table) > entry->size) {
+		sproc_err(sproc, "bad size of fw or resource table\n");
+		return NULL;
+	}
+
+	/* we don't support any version beyond the first */
+	if (table->ver != 1) {
+		sproc_err(sproc, "unsupported fw ver: %d\n", table->ver);
+		return NULL;
+	}
+
+	/* make sure reserved bytes are zeroes */
+	if (table->reserved[0] || table->reserved[1]) {
+		sproc_err(sproc, "non zero reserved bytes\n");
+		return NULL;
+	}
+
+	/* make sure the offsets array isn't truncated */
+	if (table->num > SPROC_MAX_TOC_ENTRIES ||
+	    table->num * sizeof(table->offset[0]) +
+	    sizeof(struct resource_table) > entry->size) {
+		sproc_err(sproc, "resource table incomplete\n");
+		return NULL;
+	}
+
+	/* If the fw size has grown, release the previous fw allocation */
+	if (SPROC_FW_SIZE < fw->size) {
+		sproc_err(sproc, "Insufficient space for fw (%d < %zd)\n",
+			  SPROC_FW_SIZE, fw->size);
+		return NULL;
+	}
+
+	sproc->fw_size = fw->size;
+	*tablesz = entry->size;
+
+	return table;
+}
+
+/* STE modem firmware handler operations */
+const struct rproc_fw_ops sproc_fw_ops = {
+	.load = sproc_load_segments,
+	.find_rsc_table = sproc_find_rsc_table,
+};
+
+/* Kick the modem with specified notification id */
+static void sproc_kick(struct rproc *rproc, int vqid)
+{
+	struct sproc *sproc = rproc->priv;
+
+	sproc_dbg(sproc, "kick vqid:%d\n", vqid);
+
+	/*
+	 * We need different notification IDs for RX and TX so add
+	 * an offset on TX notification IDs.
+	 */
+	sproc->mdev->ops.kick(sproc->mdev, vqid + SPROC_MAX_NOTIFY_ID);
+}
+
+/* Received a kick from a modem, kick the virtqueue */
+static void sproc_kick_callback(struct ste_modem_device *mdev, int vqid)
+{
+	struct sproc *sproc = mdev->drv_data;
+
+	if (rproc_vq_interrupt(sproc->rproc, vqid) == IRQ_NONE)
+		sproc_dbg(sproc, "no message was found in vqid %d\n", vqid);
+}
+
+struct ste_modem_dev_cb sproc_dev_cb = {
+	.kick = sproc_kick_callback,
+};
+
+/* Start the STE modem */
+static int sproc_start(struct rproc *rproc)
+{
+	struct sproc *sproc = rproc->priv;
+	int i, err;
+
+	sproc_dbg(sproc, "start ste-modem\n");
+
+	/* Sanity test the max_notifyid */
+	if (rproc->max_notifyid > SPROC_MAX_NOTIFY_ID) {
+		sproc_err(sproc, "Notification IDs too high:%d\n",
+			  rproc->max_notifyid);
+		return -EINVAL;
+	}
+
+	/* Subscribe to notifications */
+	for (i = 0; i < rproc->max_notifyid; i++) {
+		err = sproc->mdev->ops.kick_subscribe(sproc->mdev, i);
+		if (err) {
+			sproc_err(sproc,
+				  "subscription of kicks failed:%d\n", err);
+			return err;
+		}
+	}
+
+	/* Request modem start-up*/
+	return sproc->mdev->ops.power(sproc->mdev, true);
+}
+
+/* Stop the STE modem */
+static int sproc_stop(struct rproc *rproc)
+{
+	struct sproc *sproc = rproc->priv;
+	sproc_dbg(sproc, "stop ste-modem\n");
+
+	return sproc->mdev->ops.power(sproc->mdev, false);
+}
+
+static struct rproc_ops sproc_ops = {
+	.start		= sproc_start,
+	.stop		= sproc_stop,
+	.kick		= sproc_kick,
+};
+
+/* STE modem device is unregistered */
+static int sproc_drv_remove(struct platform_device *pdev)
+{
+	struct ste_modem_device *mdev =
+		container_of(pdev, struct ste_modem_device, pdev);
+	struct sproc *sproc = mdev->drv_data;
+
+	sproc_dbg(sproc, "remove ste-modem\n");
+
+	/* Reset device callback functions */
+	sproc->mdev->ops.setup(sproc->mdev, NULL);
+
+	/* Unregister as remoteproc device */
+	rproc_del(sproc->rproc);
+	rproc_put(sproc->rproc);
+
+	mdev->drv_data = NULL;
+
+	return 0;
+}
+
+/* Handle probe of a modem device */
+static int sproc_probe(struct platform_device *pdev)
+{
+	struct ste_modem_device *mdev =
+		container_of(pdev, struct ste_modem_device, pdev);
+	struct sproc *sproc;
+	struct rproc *rproc;
+	int err;
+
+	dev_dbg(&mdev->pdev.dev, "probe ste-modem\n");
+
+	if (!mdev->ops.setup || !mdev->ops.kick || !mdev->ops.kick_subscribe ||
+	    !mdev->ops.power) {
+		dev_err(&mdev->pdev.dev, "invalid mdev ops\n");
+		return -EINVAL;
+	}
+
+	rproc = rproc_alloc(&mdev->pdev.dev, mdev->pdev.name, &sproc_ops,
+			    SPROC_MODEM_FIRMWARE, sizeof(*sproc));
+	if (!rproc)
+		return -ENOMEM;
+
+	sproc = rproc->priv;
+	sproc->mdev = mdev;
+	sproc->rproc = rproc;
+	mdev->drv_data = sproc;
+
+	/* Provide callback functions to modem device */
+	sproc->mdev->ops.setup(sproc->mdev, &sproc_dev_cb);
+
+	/* Set the STE-modem specific firmware handler */
+	rproc->fw_ops = &sproc_fw_ops;
+
+	/*
+	 * STE-modem requires the firmware to be located
+	 * at the start of the shared memory region. So we need to
+	 * reserve space for firmware at the start.
+	 */
+	sproc->fw_addr = dma_alloc_coherent(rproc->dev.parent, SPROC_FW_SIZE,
+					    &sproc->fw_dma_addr,
+					    GFP_KERNEL);
+	if (!sproc->fw_addr) {
+		sproc_err(sproc, "Cannot allocate memory for fw\n");
+		err = -ENOMEM;
+		goto free_rproc;
+	}
+
+	/* Register as a remoteproc device */
+	err = rproc_add(rproc);
+	if (err)
+		goto free_rproc;
+
+	return 0;
+
+free_rproc:
+	/* Reset device data upon error */
+	mdev->drv_data = NULL;
+	rproc_put(rproc);
+	return err;
+}
+
+static struct platform_driver sproc_driver = {
+	.driver	= {
+		.name	= SPROC_MODEM_NAME,
+		.owner	= THIS_MODULE,
+	},
+	.probe	= sproc_probe,
+	.remove	= sproc_drv_remove,
+};
+
+module_platform_driver(sproc_driver);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("STE Modem driver using the Remote Processor Framework");
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index fabc99a75c6..e069f176a82 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -19,7 +19,6 @@ if RTC_CLASS
 
 config RTC_HCTOSYS
 	bool "Set system time from RTC on startup and resume"
-	depends on RTC_CLASS = y
 	default y
 	help
 	  If you say yes here, the system time (wall clock) will be set using
@@ -51,7 +50,6 @@ config RTC_HCTOSYS_DEVICE
 
 config RTC_DEBUG
 	bool "RTC debug support"
-	depends on RTC_CLASS = y
 	help
 	  Say yes here to enable debugging support in the RTC framework
 	  and individual RTC drivers.
@@ -61,7 +59,6 @@ comment "RTC interfaces"
 config RTC_INTF_SYSFS
 	boolean "/sys/class/rtc/rtcN (sysfs)"
 	depends on SYSFS
-	default RTC_CLASS
 	help
 	  Say yes here if you want to use your RTCs using sysfs interfaces,
 	  /sys/class/rtc/rtc0 through /sys/.../rtcN.
@@ -69,19 +66,19 @@ config RTC_INTF_SYSFS
 	  If unsure, say Y.
 
 config RTC_INTF_PROC
-	boolean "/proc/driver/rtc (procfs for rtc0)"
+	boolean "/proc/driver/rtc (procfs for rtcN)"
 	depends on PROC_FS
-	default RTC_CLASS
 	help
-	  Say yes here if you want to use your first RTC through the proc
-	  interface, /proc/driver/rtc. Other RTCs will not be available
-	  through that API.
+	  Say yes here if you want to use your system clock RTC through
+	  the proc interface, /proc/driver/rtc.
+	  Other RTCs will not be available through that API.
+	  If there is no RTC for the system clock, then the first RTC(rtc0)
+	  is used by default.
 
 	  If unsure, say Y.
 
 config RTC_INTF_DEV
 	boolean "/dev/rtcN (character devices)"
-	default RTC_CLASS
 	help
 	  Say yes here if you want to use your RTCs using the /dev
 	  interfaces, which "udev" sets up as /dev/rtc0 through
@@ -127,7 +124,7 @@ if I2C
 
 config RTC_DRV_88PM860X
 	tristate "Marvell 88PM860x"
-	depends on RTC_CLASS && I2C && MFD_88PM860X
+	depends on I2C && MFD_88PM860X
 	help
 	  If you say yes here you get support for RTC function in Marvell
 	  88PM860x chips.
@@ -137,7 +134,7 @@ config RTC_DRV_88PM860X
 
 config RTC_DRV_88PM80X
 	tristate "Marvell 88PM80x"
-	depends on RTC_CLASS && I2C && MFD_88PM800
+	depends on I2C && MFD_88PM800
 	help
 	  If you say yes here you get support for RTC function in Marvell
 	  88PM80x chips.
@@ -165,7 +162,7 @@ config RTC_DRV_DS1307
 
 config RTC_DRV_DS1374
 	tristate "Dallas/Maxim DS1374"
-	depends on RTC_CLASS && I2C
+	depends on I2C
 	help
 	  If you say yes here you get support for Dallas Semiconductor
 	  DS1374 real-time clock chips. If an interrupt is associated
@@ -185,7 +182,7 @@ config RTC_DRV_DS1672
 
 config RTC_DRV_DS3232
 	tristate "Dallas/Maxim DS3232"
-	depends on RTC_CLASS && I2C
+	depends on I2C
 	help
 	  If you say yes here you get support for Dallas Semiconductor
 	  DS3232 real-time clock chips. If an interrupt is associated
@@ -203,6 +200,16 @@ config RTC_DRV_MAX6900
 	  This driver can also be built as a module. If so, the module
 	  will be called rtc-max6900.
 
+config RTC_DRV_MAX8907
+	tristate "Maxim MAX8907"
+	depends on MFD_MAX8907
+	help
+	  If you say yes here you will get support for the
+	  RTC of Maxim MAX8907 PMIC.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-max8907.
+
 config RTC_DRV_MAX8925
 	tristate "Maxim MAX8925"
 	depends on MFD_MAX8925
@@ -325,7 +332,7 @@ config RTC_DRV_TWL92330
 
 config RTC_DRV_TWL4030
 	tristate "TI TWL4030/TWL5030/TWL6030/TPS659x0"
-	depends on RTC_CLASS && TWL4030_CORE
+	depends on TWL4030_CORE
 	help
 	  If you say yes here you get support for the RTC on the
 	  TWL4030/TWL5030/TWL6030 family chips, used mostly with OMAP3 platforms.
@@ -333,6 +340,26 @@ config RTC_DRV_TWL4030
 	  This driver can also be built as a module. If so, the module
 	  will be called rtc-twl.
 
+config RTC_DRV_TPS65910
+	tristate "TI TPS65910 RTC driver"
+	depends on RTC_CLASS && MFD_TPS65910
+	help
+	  If you say yes here you get support for the RTC on the
+	  TPS65910 chips.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-tps65910.
+
+config RTC_DRV_RC5T583
+	tristate "RICOH 5T583 RTC driver"
+	depends on MFD_RC5T583
+	help
+	  If you say yes here you get support for the RTC on the
+	  RICOH 5T583 chips.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-rc5t583.
+
 config RTC_DRV_S35390A
 	tristate "Seiko Instruments S-35390A"
 	select BITREVERSE
@@ -538,7 +565,6 @@ config RTC_DRV_DS1302
 
 config RTC_DRV_DS1511
 	tristate "Dallas DS1511"
-	depends on RTC_CLASS
 	help
 	  If you say yes here you get support for the
 	  Dallas DS1511 timekeeping/watchdog chip.
@@ -583,7 +609,6 @@ config RTC_DRV_EFI
 
 config RTC_DRV_STK17TA8
 	tristate "Simtek STK17TA8"
-	depends on RTC_CLASS
 	help
 	  If you say yes here you get support for the
 	  Simtek STK17TA8 timekeeping chip.
@@ -658,6 +683,15 @@ config RTC_DRV_V3020
 	  This driver can also be built as a module. If so, the module
 	  will be called rtc-v3020.
 
+config RTC_DRV_DS2404
+	tristate "Dallas DS2404"
+	help
+	  If you say yes here you get support for the
+	  Dallas DS2404 RTC chip.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called rtc-ds2404.
+
 config RTC_DRV_WM831X
 	tristate "Wolfson Microelectronics WM831x RTC"
 	depends on MFD_WM831X
@@ -704,6 +738,7 @@ config RTC_DRV_AB3100
 config RTC_DRV_AB8500
 	tristate "ST-Ericsson AB8500 RTC"
 	depends on AB8500_CORE
+	select RTC_INTF_DEV
 	select RTC_INTF_DEV_UIE_EMUL
 	help
 	  Select this to enable the ST-Ericsson AB8500 power management IC RTC
@@ -711,7 +746,7 @@ config RTC_DRV_AB8500
 
 config RTC_DRV_NUC900
 	tristate "NUC910/NUC920 RTC driver"
-	depends on RTC_CLASS && ARCH_W90X900
+	depends on ARCH_W90X900
 	help
 	  If you say yes here you get support for the RTC subsystem of the
 	  NUC910/NUC920 used in embedded systems.
@@ -731,7 +766,6 @@ config RTC_DRV_DAVINCI
 config RTC_DRV_IMXDI
 	tristate "Freescale IMX DryIce Real Time Clock"
 	depends on SOC_IMX25
-	depends on RTC_CLASS
 	help
 	   Support for Freescale IMX DryIce RTC
 
@@ -791,7 +825,7 @@ config RTC_DRV_SA1100
 
 config RTC_DRV_SH
 	tristate "SuperH On-Chip RTC"
-	depends on RTC_CLASS && SUPERH && HAVE_CLK
+	depends on SUPERH && HAVE_CLK
 	help
 	  Say Y here to enable support for the on-chip RTC found in
 	  most SuperH processors.
@@ -1023,7 +1057,6 @@ config RTC_DRV_MPC5121
 
 config RTC_DRV_JZ4740
 	tristate "Ingenic JZ4740 SoC"
-	depends on RTC_CLASS
 	depends on MACH_JZ4740
 	help
 	  If you say yes here you get support for the Ingenic JZ4740 SoC RTC
@@ -1053,7 +1086,7 @@ config RTC_DRV_PM8XXX
 
 config RTC_DRV_TEGRA
 	tristate "NVIDIA Tegra Internal RTC driver"
-	depends on RTC_CLASS && ARCH_TEGRA
+	depends on ARCH_TEGRA
 	help
 	  If you say yes here you get support for the
 	  Tegra 200 series internal RTC module.
@@ -1090,7 +1123,6 @@ config RTC_DRV_LOONGSON1
 config RTC_DRV_MXC
 	tristate "Freescale MXC Real Time Clock"
 	depends on ARCH_MXC
-	depends on RTC_CLASS
 	help
 	   If you say yes here you get support for the Freescale MXC
 	   RTC module.
@@ -1098,4 +1130,15 @@ config RTC_DRV_MXC
 	   This driver can also be built as a module, if so, the module
 	   will be called "rtc-mxc".
 
+config RTC_DRV_SNVS
+	tristate "Freescale SNVS RTC support"
+	depends on HAS_IOMEM
+	depends on OF
+	help
+	   If you say yes here you get support for the Freescale SNVS
+	   Low Power (LP) RTC module.
+
+	   This driver can also be built as a module, if so, the module
+	   will be called "rtc-snvs".
+
 endif # RTC_CLASS
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index 0d5b2b66f90..56297f0fd38 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -43,6 +43,7 @@ obj-$(CONFIG_RTC_DRV_DS1511)	+= rtc-ds1511.o
 obj-$(CONFIG_RTC_DRV_DS1553)	+= rtc-ds1553.o
 obj-$(CONFIG_RTC_DRV_DS1672)	+= rtc-ds1672.o
 obj-$(CONFIG_RTC_DRV_DS1742)	+= rtc-ds1742.o
+obj-$(CONFIG_RTC_DRV_DS2404)    += rtc-ds2404.o
 obj-$(CONFIG_RTC_DRV_DS3232)	+= rtc-ds3232.o
 obj-$(CONFIG_RTC_DRV_DS3234)	+= rtc-ds3234.o
 obj-$(CONFIG_RTC_DRV_EFI)	+= rtc-efi.o
@@ -64,6 +65,7 @@ obj-$(CONFIG_RTC_DRV_M48T59)	+= rtc-m48t59.o
 obj-$(CONFIG_RTC_DRV_M48T86)	+= rtc-m48t86.o
 obj-$(CONFIG_RTC_DRV_MXC)	+= rtc-mxc.o
 obj-$(CONFIG_RTC_DRV_MAX6900)	+= rtc-max6900.o
+obj-$(CONFIG_RTC_DRV_MAX8907)	+= rtc-max8907.o
 obj-$(CONFIG_RTC_DRV_MAX8925)	+= rtc-max8925.o
 obj-$(CONFIG_RTC_DRV_MAX8998)	+= rtc-max8998.o
 obj-$(CONFIG_RTC_DRV_MAX6902)	+= rtc-max6902.o
@@ -85,6 +87,7 @@ obj-$(CONFIG_RTC_DRV_PS3)	+= rtc-ps3.o
 obj-$(CONFIG_RTC_DRV_PUV3)	+= rtc-puv3.o
 obj-$(CONFIG_RTC_DRV_PXA)	+= rtc-pxa.o
 obj-$(CONFIG_RTC_DRV_R9701)	+= rtc-r9701.o
+obj-$(CONFIG_RTC_DRV_RC5T583)	+= rtc-rc5t583.o
 obj-$(CONFIG_RTC_DRV_RP5C01)	+= rtc-rp5c01.o
 obj-$(CONFIG_RTC_DRV_RS5C313)	+= rtc-rs5c313.o
 obj-$(CONFIG_RTC_DRV_RS5C348)	+= rtc-rs5c348.o
@@ -96,6 +99,7 @@ obj-$(CONFIG_RTC_DRV_S35390A)	+= rtc-s35390a.o
 obj-$(CONFIG_RTC_DRV_S3C)	+= rtc-s3c.o
 obj-$(CONFIG_RTC_DRV_SA1100)	+= rtc-sa1100.o
 obj-$(CONFIG_RTC_DRV_SH)	+= rtc-sh.o
+obj-$(CONFIG_RTC_DRV_SNVS)	+= rtc-snvs.o
 obj-$(CONFIG_RTC_DRV_SPEAR)	+= rtc-spear.o
 obj-$(CONFIG_RTC_DRV_STARFIRE)	+= rtc-starfire.o
 obj-$(CONFIG_RTC_DRV_STK17TA8)	+= rtc-stk17ta8.o
@@ -105,6 +109,7 @@ obj-$(CONFIG_RTC_DRV_TEGRA)	+= rtc-tegra.o
 obj-$(CONFIG_RTC_DRV_TEST)	+= rtc-test.o
 obj-$(CONFIG_RTC_DRV_TILE)	+= rtc-tile.o
 obj-$(CONFIG_RTC_DRV_TWL4030)	+= rtc-twl.o
+obj-$(CONFIG_RTC_DRV_TPS65910)	+= rtc-tps65910.o
 obj-$(CONFIG_RTC_DRV_TX4939)	+= rtc-tx4939.o
 obj-$(CONFIG_RTC_DRV_V3020)	+= rtc-v3020.o
 obj-$(CONFIG_RTC_DRV_VR41XX)	+= rtc-vr41xx.o
diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c
index dc4c2748bbc..f8a0aab218c 100644
--- a/drivers/rtc/class.c
+++ b/drivers/rtc/class.c
@@ -31,8 +31,12 @@ static void rtc_device_release(struct device *dev)
 	kfree(rtc);
 }
 
-#if defined(CONFIG_PM) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
+#ifdef CONFIG_RTC_HCTOSYS_DEVICE
+/* Result of the last RTC to system clock attempt. */
+int rtc_hctosys_ret = -ENODEV;
+#endif
 
+#if defined(CONFIG_PM) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
 /*
  * On suspend(), measure the delta between one RTC and the
  * system's wall clock; restore it on resume().
@@ -84,6 +88,7 @@ static int rtc_resume(struct device *dev)
 	struct timespec		new_system, new_rtc;
 	struct timespec		sleep_time;
 
+	rtc_hctosys_ret = -ENODEV;
 	if (strcmp(dev_name(&rtc->dev), CONFIG_RTC_HCTOSYS_DEVICE) != 0)
 		return 0;
 
@@ -117,6 +122,7 @@ static int rtc_resume(struct device *dev)
 
 	if (sleep_time.tv_sec >= 0)
 		timekeeping_inject_sleeptime(&sleep_time);
+	rtc_hctosys_ret = 0;
 	return 0;
 }
 
@@ -238,6 +244,7 @@ void rtc_device_unregister(struct rtc_device *rtc)
 		rtc_proc_del_device(rtc);
 		device_unregister(&rtc->dev);
 		rtc->ops = NULL;
+		ida_simple_remove(&rtc_ida, rtc->id);
 		mutex_unlock(&rtc->ops_lock);
 		put_device(&rtc->dev);
 	}
diff --git a/drivers/rtc/hctosys.c b/drivers/rtc/hctosys.c
index bc90b091f19..4aa60d74004 100644
--- a/drivers/rtc/hctosys.c
+++ b/drivers/rtc/hctosys.c
@@ -22,8 +22,6 @@
  * the best guess is to add 0.5s.
  */
 
-int rtc_hctosys_ret = -ENODEV;
-
 static int __init rtc_hctosys(void)
 {
 	int err = -ENODEV;
@@ -56,7 +54,7 @@ static int __init rtc_hctosys(void)
 
 	rtc_tm_to_time(&tm, &tv.tv_sec);
 
-	do_settimeofday(&tv);
+	err = do_settimeofday(&tv);
 
 	dev_info(rtc->dev.parent,
 		"setting system clock to "
diff --git a/drivers/rtc/rtc-88pm860x.c b/drivers/rtc/rtc-88pm860x.c
index feddefc4210..de9e854b326 100644
--- a/drivers/rtc/rtc-88pm860x.c
+++ b/drivers/rtc/rtc-88pm860x.c
@@ -11,6 +11,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
@@ -284,6 +285,28 @@ out:
 }
 #endif
 
+#ifdef CONFIG_OF
+static int __devinit pm860x_rtc_dt_init(struct platform_device *pdev,
+					struct pm860x_rtc_info *info)
+{
+	struct device_node *np = pdev->dev.parent->of_node;
+	int ret;
+	if (!np)
+		return -ENODEV;
+	np = of_find_node_by_name(np, "rtc");
+	if (!np) {
+		dev_err(&pdev->dev, "failed to find rtc node\n");
+		return -ENODEV;
+	}
+	ret = of_property_read_u32(np, "marvell,88pm860x-vrtc", &info->vrtc);
+	if (ret)
+		info->vrtc = 0;
+	return 0;
+}
+#else
+#define pm860x_rtc_dt_init(x, y)	(-1)
+#endif
+
 static int __devinit pm860x_rtc_probe(struct platform_device *pdev)
 {
 	struct pm860x_chip *chip = dev_get_drvdata(pdev->dev.parent);
@@ -294,8 +317,6 @@ static int __devinit pm860x_rtc_probe(struct platform_device *pdev)
 	int ret;
 
 	pdata = pdev->dev.platform_data;
-	if (pdata == NULL)
-		dev_warn(&pdev->dev, "No platform data!\n");
 
 	info = kzalloc(sizeof(struct pm860x_rtc_info), GFP_KERNEL);
 	if (!info)
@@ -345,9 +366,11 @@ static int __devinit pm860x_rtc_probe(struct platform_device *pdev)
 		}
 	}
 	rtc_tm_to_time(&tm, &ticks);
-	if (pdata && pdata->sync) {
-		pdata->sync(ticks);
-		info->sync = pdata->sync;
+	if (pm860x_rtc_dt_init(pdev, info)) {
+		if (pdata && pdata->sync) {
+			pdata->sync(ticks);
+			info->sync = pdata->sync;
+		}
 	}
 
 	info->rtc_dev = rtc_device_register("88pm860x-rtc", &pdev->dev,
@@ -366,10 +389,12 @@ static int __devinit pm860x_rtc_probe(struct platform_device *pdev)
 
 #ifdef VRTC_CALIBRATION
 	/* <00> -- 2.7V, <01> -- 2.9V, <10> -- 3.1V, <11> -- 3.3V */
-	if (pdata && pdata->vrtc)
-		info->vrtc = pdata->vrtc & 0x3;
-	else
-		info->vrtc = 1;
+	if (pm860x_rtc_dt_init(pdev, info)) {
+		if (pdata && pdata->vrtc)
+			info->vrtc = pdata->vrtc & 0x3;
+		else
+			info->vrtc = 1;
+	}
 	pm860x_set_bits(info->i2c, PM8607_MEAS_EN2, MEAS2_VRTC, MEAS2_VRTC);
 
 	/* calibrate VRTC */
diff --git a/drivers/rtc/rtc-at91sam9.c b/drivers/rtc/rtc-at91sam9.c
index 1dd61f402b0..2dfe7a2fb99 100644
--- a/drivers/rtc/rtc-at91sam9.c
+++ b/drivers/rtc/rtc-at91sam9.c
@@ -473,18 +473,7 @@ static struct platform_driver at91_rtc_driver = {
 	},
 };
 
-static int __init at91_rtc_init(void)
-{
-	return platform_driver_register(&at91_rtc_driver);
-}
-module_init(at91_rtc_init);
-
-static void __exit at91_rtc_exit(void)
-{
-	platform_driver_unregister(&at91_rtc_driver);
-}
-module_exit(at91_rtc_exit);
-
+module_platform_driver(at91_rtc_driver);
 
 MODULE_AUTHOR("Michel Benoit");
 MODULE_DESCRIPTION("RTC driver for Atmel AT91SAM9x");
diff --git a/drivers/rtc/rtc-coh901331.c b/drivers/rtc/rtc-coh901331.c
index 76b2156d3c6..c8115b83e5a 100644
--- a/drivers/rtc/rtc-coh901331.c
+++ b/drivers/rtc/rtc-coh901331.c
@@ -276,8 +276,7 @@ static void coh901331_shutdown(struct platform_device *pdev)
 
 	clk_enable(rtap->clk);
 	writel(0, rtap->virtbase + COH901331_IRQ_MASK);
-	clk_disable(rtap->clk);
-	clk_unprepare(rtap->clk);
+	clk_disable_unprepare(rtap->clk);
 }
 
 static struct platform_driver coh901331_driver = {
diff --git a/drivers/rtc/rtc-ds1672.c b/drivers/rtc/rtc-ds1672.c
index 7fa67d0df17..45d65c0b3a8 100644
--- a/drivers/rtc/rtc-ds1672.c
+++ b/drivers/rtc/rtc-ds1672.c
@@ -37,8 +37,17 @@ static int ds1672_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	unsigned char buf[4];
 
 	struct i2c_msg msgs[] = {
-		{client->addr, 0, 1, &addr},	/* setup read ptr */
-		{client->addr, I2C_M_RD, 4, buf},	/* read date */
+		{/* setup read ptr */
+			.addr = client->addr,
+			.len = 1,
+			.buf = &addr
+		},
+		{/* read date */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 4,
+			.buf = buf
+		},
 	};
 
 	/* read date registers */
@@ -99,8 +108,17 @@ static int ds1672_get_control(struct i2c_client *client, u8 *status)
 	unsigned char addr = DS1672_REG_CONTROL;
 
 	struct i2c_msg msgs[] = {
-		{client->addr, 0, 1, &addr},	/* setup read ptr */
-		{client->addr, I2C_M_RD, 1, status},	/* read control */
+		{/* setup read ptr */
+			.addr = client->addr,
+			.len = 1,
+			.buf = &addr
+		},
+		{/* read control */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = status
+		},
 	};
 
 	/* read control register */
diff --git a/drivers/rtc/rtc-ds2404.c b/drivers/rtc/rtc-ds2404.c
new file mode 100644
index 00000000000..5ea9df7c8c3
--- /dev/null
+++ b/drivers/rtc/rtc-ds2404.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (C) 2012 Sven Schnelle <svens@stackframe.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/rtc.h>
+#include <linux/types.h>
+#include <linux/bcd.h>
+#include <linux/rtc-ds2404.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/slab.h>
+
+#include <linux/io.h>
+
+#define DS2404_STATUS_REG 0x200
+#define DS2404_CONTROL_REG 0x201
+#define DS2404_RTC_REG 0x202
+
+#define DS2404_WRITE_SCRATCHPAD_CMD 0x0f
+#define DS2404_READ_SCRATCHPAD_CMD 0xaa
+#define DS2404_COPY_SCRATCHPAD_CMD 0x55
+#define DS2404_READ_MEMORY_CMD 0xf0
+
+struct ds2404;
+
+struct ds2404_chip_ops {
+	int (*map_io)(struct ds2404 *chip, struct platform_device *pdev,
+		      struct ds2404_platform_data *pdata);
+	void (*unmap_io)(struct ds2404 *chip);
+};
+
+#define DS2404_RST	0
+#define DS2404_CLK	1
+#define DS2404_DQ	2
+
+struct ds2404_gpio {
+	const char *name;
+	unsigned int gpio;
+};
+
+struct ds2404 {
+	struct ds2404_gpio *gpio;
+	struct ds2404_chip_ops *ops;
+	struct rtc_device *rtc;
+};
+
+static struct ds2404_gpio ds2404_gpio[] = {
+	{ "RTC RST", 0 },
+	{ "RTC CLK", 0 },
+	{ "RTC DQ", 0 },
+};
+
+static int ds2404_gpio_map(struct ds2404 *chip, struct platform_device *pdev,
+			  struct ds2404_platform_data *pdata)
+{
+	int i, err;
+
+	ds2404_gpio[DS2404_RST].gpio = pdata->gpio_rst;
+	ds2404_gpio[DS2404_CLK].gpio = pdata->gpio_clk;
+	ds2404_gpio[DS2404_DQ].gpio = pdata->gpio_dq;
+
+	for (i = 0; i < ARRAY_SIZE(ds2404_gpio); i++) {
+		err = gpio_request(ds2404_gpio[i].gpio, ds2404_gpio[i].name);
+		if (err) {
+			printk(KERN_ERR "error mapping gpio %s: %d\n",
+				ds2404_gpio[i].name, err);
+			goto err_request;
+		}
+		if (i != DS2404_DQ)
+			gpio_direction_output(ds2404_gpio[i].gpio, 1);
+	}
+
+	chip->gpio = ds2404_gpio;
+	return 0;
+
+err_request:
+	while (--i >= 0)
+		gpio_free(ds2404_gpio[i].gpio);
+	return err;
+}
+
+static void ds2404_gpio_unmap(struct ds2404 *chip)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ds2404_gpio); i++)
+		gpio_free(ds2404_gpio[i].gpio);
+}
+
+static struct ds2404_chip_ops ds2404_gpio_ops = {
+	.map_io		= ds2404_gpio_map,
+	.unmap_io	= ds2404_gpio_unmap,
+};
+
+static void ds2404_reset(struct device *dev)
+{
+	gpio_set_value(ds2404_gpio[DS2404_RST].gpio, 0);
+	udelay(1000);
+	gpio_set_value(ds2404_gpio[DS2404_RST].gpio, 1);
+	gpio_set_value(ds2404_gpio[DS2404_CLK].gpio, 0);
+	gpio_direction_output(ds2404_gpio[DS2404_DQ].gpio, 0);
+	udelay(10);
+}
+
+static void ds2404_write_byte(struct device *dev, u8 byte)
+{
+	int i;
+
+	gpio_direction_output(ds2404_gpio[DS2404_DQ].gpio, 1);
+	for (i = 0; i < 8; i++) {
+		gpio_set_value(ds2404_gpio[DS2404_DQ].gpio, byte & (1 << i));
+		udelay(10);
+		gpio_set_value(ds2404_gpio[DS2404_CLK].gpio, 1);
+		udelay(10);
+		gpio_set_value(ds2404_gpio[DS2404_CLK].gpio, 0);
+		udelay(10);
+	}
+}
+
+static u8 ds2404_read_byte(struct device *dev)
+{
+	int i;
+	u8 ret = 0;
+
+	gpio_direction_input(ds2404_gpio[DS2404_DQ].gpio);
+
+	for (i = 0; i < 8; i++) {
+		gpio_set_value(ds2404_gpio[DS2404_CLK].gpio, 0);
+		udelay(10);
+		if (gpio_get_value(ds2404_gpio[DS2404_DQ].gpio))
+			ret |= 1 << i;
+		gpio_set_value(ds2404_gpio[DS2404_CLK].gpio, 1);
+		udelay(10);
+	}
+	return ret;
+}
+
+static void ds2404_read_memory(struct device *dev, u16 offset,
+			       int length, u8 *out)
+{
+	ds2404_reset(dev);
+	ds2404_write_byte(dev, DS2404_READ_MEMORY_CMD);
+	ds2404_write_byte(dev, offset & 0xff);
+	ds2404_write_byte(dev, (offset >> 8) & 0xff);
+	while (length--)
+		*out++ = ds2404_read_byte(dev);
+}
+
+static void ds2404_write_memory(struct device *dev, u16 offset,
+				int length, u8 *out)
+{
+	int i;
+	u8 ta01, ta02, es;
+
+	ds2404_reset(dev);
+	ds2404_write_byte(dev, DS2404_WRITE_SCRATCHPAD_CMD);
+	ds2404_write_byte(dev, offset & 0xff);
+	ds2404_write_byte(dev, (offset >> 8) & 0xff);
+
+	for (i = 0; i < length; i++)
+		ds2404_write_byte(dev, out[i]);
+
+	ds2404_reset(dev);
+	ds2404_write_byte(dev, DS2404_READ_SCRATCHPAD_CMD);
+
+	ta01 = ds2404_read_byte(dev);
+	ta02 = ds2404_read_byte(dev);
+	es = ds2404_read_byte(dev);
+
+	for (i = 0; i < length; i++) {
+		if (out[i] != ds2404_read_byte(dev)) {
+			printk(KERN_ERR "read invalid data\n");
+			return;
+		}
+	}
+
+	ds2404_reset(dev);
+	ds2404_write_byte(dev, DS2404_COPY_SCRATCHPAD_CMD);
+	ds2404_write_byte(dev, ta01);
+	ds2404_write_byte(dev, ta02);
+	ds2404_write_byte(dev, es);
+
+	gpio_direction_input(ds2404_gpio[DS2404_DQ].gpio);
+	while (gpio_get_value(ds2404_gpio[DS2404_DQ].gpio))
+		;
+}
+
+static void ds2404_enable_osc(struct device *dev)
+{
+	u8 in[1] = { 0x10 }; /* enable oscillator */
+	ds2404_write_memory(dev, 0x201, 1, in);
+}
+
+static int ds2404_read_time(struct device *dev, struct rtc_time *dt)
+{
+	unsigned long time = 0;
+
+	ds2404_read_memory(dev, 0x203, 4, (u8 *)&time);
+	time = le32_to_cpu(time);
+
+	rtc_time_to_tm(time, dt);
+	return rtc_valid_tm(dt);
+}
+
+static int ds2404_set_mmss(struct device *dev, unsigned long secs)
+{
+	u32 time = cpu_to_le32(secs);
+	ds2404_write_memory(dev, 0x203, 4, (u8 *)&time);
+	return 0;
+}
+
+static const struct rtc_class_ops ds2404_rtc_ops = {
+	.read_time	= ds2404_read_time,
+	.set_mmss	= ds2404_set_mmss,
+};
+
+static int rtc_probe(struct platform_device *pdev)
+{
+	struct ds2404_platform_data *pdata = pdev->dev.platform_data;
+	struct ds2404 *chip;
+	int retval = -EBUSY;
+
+	chip = kzalloc(sizeof(struct ds2404), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	chip->ops = &ds2404_gpio_ops;
+
+	retval = chip->ops->map_io(chip, pdev, pdata);
+	if (retval)
+		goto err_chip;
+
+	dev_info(&pdev->dev, "using GPIOs RST:%d, CLK:%d, DQ:%d\n",
+		 chip->gpio[DS2404_RST].gpio, chip->gpio[DS2404_CLK].gpio,
+		 chip->gpio[DS2404_DQ].gpio);
+
+	platform_set_drvdata(pdev, chip);
+
+	chip->rtc = rtc_device_register("ds2404",
+				&pdev->dev, &ds2404_rtc_ops, THIS_MODULE);
+	if (IS_ERR(chip->rtc)) {
+		retval = PTR_ERR(chip->rtc);
+		goto err_io;
+	}
+
+	ds2404_enable_osc(&pdev->dev);
+	return 0;
+
+err_io:
+	chip->ops->unmap_io(chip);
+err_chip:
+	kfree(chip);
+	return retval;
+}
+
+static int rtc_remove(struct platform_device *dev)
+{
+	struct ds2404 *chip = platform_get_drvdata(dev);
+	struct rtc_device *rtc = chip->rtc;
+
+	if (rtc)
+		rtc_device_unregister(rtc);
+
+	chip->ops->unmap_io(chip);
+	kfree(chip);
+
+	return 0;
+}
+
+static struct platform_driver rtc_device_driver = {
+	.probe	= rtc_probe,
+	.remove = rtc_remove,
+	.driver = {
+		.name	= "ds2404",
+		.owner	= THIS_MODULE,
+	},
+};
+
+static __init int ds2404_init(void)
+{
+	return platform_driver_register(&rtc_device_driver);
+}
+
+static __exit void ds2404_exit(void)
+{
+	platform_driver_unregister(&rtc_device_driver);
+}
+
+module_init(ds2404_init);
+module_exit(ds2404_exit);
+
+MODULE_DESCRIPTION("DS2404 RTC");
+MODULE_AUTHOR("Sven Schnelle");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:ds2404");
diff --git a/drivers/rtc/rtc-em3027.c b/drivers/rtc/rtc-em3027.c
index 0104ea7ebe5..f6c24ce35d3 100644
--- a/drivers/rtc/rtc-em3027.c
+++ b/drivers/rtc/rtc-em3027.c
@@ -49,8 +49,17 @@ static int em3027_get_time(struct device *dev, struct rtc_time *tm)
 	unsigned char buf[7];
 
 	struct i2c_msg msgs[] = {
-		{client->addr, 0, 1, &addr},		/* setup read addr */
-		{client->addr, I2C_M_RD, 7, buf},	/* read time/date */
+		{/* setup read addr */
+			.addr = client->addr,
+			.len = 1,
+			.buf = &addr
+		},
+		{/* read time/date */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 7,
+			.buf = buf
+		},
 	};
 
 	/* read time/date registers */
@@ -76,7 +85,9 @@ static int em3027_set_time(struct device *dev, struct rtc_time *tm)
 	unsigned char buf[8];
 
 	struct i2c_msg msg = {
-		client->addr, 0, 8, buf,	/* write time/date */
+		.addr = client->addr,
+		.len = 8,
+		.buf = buf,	/* write time/date */
 	};
 
 	buf[0] = EM3027_REG_WATCH_SEC;
diff --git a/drivers/rtc/rtc-isl1208.c b/drivers/rtc/rtc-isl1208.c
index dd2aeee6c66..26c81f23360 100644
--- a/drivers/rtc/rtc-isl1208.c
+++ b/drivers/rtc/rtc-isl1208.c
@@ -68,9 +68,17 @@ isl1208_i2c_read_regs(struct i2c_client *client, u8 reg, u8 buf[],
 {
 	u8 reg_addr[1] = { reg };
 	struct i2c_msg msgs[2] = {
-		{client->addr, 0, sizeof(reg_addr), reg_addr}
-		,
-		{client->addr, I2C_M_RD, len, buf}
+		{
+			.addr = client->addr,
+			.len = sizeof(reg_addr),
+			.buf = reg_addr
+		},
+		{
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = len,
+			.buf = buf
+		}
 	};
 	int ret;
 
@@ -90,7 +98,11 @@ isl1208_i2c_set_regs(struct i2c_client *client, u8 reg, u8 const buf[],
 {
 	u8 i2c_buf[ISL1208_REG_USR2 + 2];
 	struct i2c_msg msgs[1] = {
-		{client->addr, 0, len + 1, i2c_buf}
+		{
+			.addr = client->addr,
+			.len = len + 1,
+			.buf = i2c_buf
+		}
 	};
 	int ret;
 
@@ -697,6 +709,7 @@ isl1208_remove(struct i2c_client *client)
 
 static const struct i2c_device_id isl1208_id[] = {
 	{ "isl1208", 0 },
+	{ "isl1218", 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, isl1208_id);
diff --git a/drivers/rtc/rtc-jz4740.c b/drivers/rtc/rtc-jz4740.c
index 05ab227eeff..1224182d3ea 100644
--- a/drivers/rtc/rtc-jz4740.c
+++ b/drivers/rtc/rtc-jz4740.c
@@ -42,7 +42,7 @@ struct jz4740_rtc {
 
 	struct rtc_device *rtc;
 
-	unsigned int irq;
+	int irq;
 
 	spinlock_t lock;
 };
diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c
index 4e0f84af99a..b885bcd0890 100644
--- a/drivers/rtc/rtc-m41t80.c
+++ b/drivers/rtc/rtc-m41t80.c
@@ -213,163 +213,14 @@ static int m41t80_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	return m41t80_set_datetime(to_i2c_client(dev), tm);
 }
 
-static int m41t80_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	int rc;
-
-	rc = i2c_smbus_read_byte_data(client, M41T80_REG_ALARM_MON);
-	if (rc < 0)
-		goto err;
-
-	if (enabled)
-		rc |= M41T80_ALMON_AFE;
-	else
-		rc &= ~M41T80_ALMON_AFE;
-
-	if (i2c_smbus_write_byte_data(client, M41T80_REG_ALARM_MON, rc) < 0)
-		goto err;
-
-	return 0;
-err:
-	return -EIO;
-}
-
-static int m41t80_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *t)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	u8 wbuf[1 + M41T80_ALARM_REG_SIZE];
-	u8 *buf = &wbuf[1];
-	u8 *reg = buf - M41T80_REG_ALARM_MON;
-	u8 dt_addr[1] = { M41T80_REG_ALARM_MON };
-	struct i2c_msg msgs_in[] = {
-		{
-			.addr	= client->addr,
-			.flags	= 0,
-			.len	= 1,
-			.buf	= dt_addr,
-		},
-		{
-			.addr	= client->addr,
-			.flags	= I2C_M_RD,
-			.len	= M41T80_ALARM_REG_SIZE,
-			.buf	= buf,
-		},
-	};
-	struct i2c_msg msgs[] = {
-		{
-			.addr	= client->addr,
-			.flags	= 0,
-			.len	= 1 + M41T80_ALARM_REG_SIZE,
-			.buf	= wbuf,
-		 },
-	};
-
-	if (i2c_transfer(client->adapter, msgs_in, 2) < 0) {
-		dev_err(&client->dev, "read error\n");
-		return -EIO;
-	}
-	reg[M41T80_REG_ALARM_MON] &= ~(0x1f | M41T80_ALMON_AFE);
-	reg[M41T80_REG_ALARM_DAY] = 0;
-	reg[M41T80_REG_ALARM_HOUR] &= ~(0x3f | 0x80);
-	reg[M41T80_REG_ALARM_MIN] = 0;
-	reg[M41T80_REG_ALARM_SEC] = 0;
-
-	wbuf[0] = M41T80_REG_ALARM_MON; /* offset into rtc's regs */
-	reg[M41T80_REG_ALARM_SEC] |= t->time.tm_sec >= 0 ?
-		bin2bcd(t->time.tm_sec) : 0x80;
-	reg[M41T80_REG_ALARM_MIN] |= t->time.tm_min >= 0 ?
-		bin2bcd(t->time.tm_min) : 0x80;
-	reg[M41T80_REG_ALARM_HOUR] |= t->time.tm_hour >= 0 ?
-		bin2bcd(t->time.tm_hour) : 0x80;
-	reg[M41T80_REG_ALARM_DAY] |= t->time.tm_mday >= 0 ?
-		bin2bcd(t->time.tm_mday) : 0x80;
-	if (t->time.tm_mon >= 0)
-		reg[M41T80_REG_ALARM_MON] |= bin2bcd(t->time.tm_mon + 1);
-	else
-		reg[M41T80_REG_ALARM_DAY] |= 0x40;
-
-	if (i2c_transfer(client->adapter, msgs, 1) != 1) {
-		dev_err(&client->dev, "write error\n");
-		return -EIO;
-	}
-
-	if (t->enabled) {
-		reg[M41T80_REG_ALARM_MON] |= M41T80_ALMON_AFE;
-		if (i2c_smbus_write_byte_data(client, M41T80_REG_ALARM_MON,
-					      reg[M41T80_REG_ALARM_MON]) < 0) {
-			dev_err(&client->dev, "write error\n");
-			return -EIO;
-		}
-	}
-	return 0;
-}
-
-static int m41t80_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *t)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	u8 buf[M41T80_ALARM_REG_SIZE + 1]; /* all alarm regs and flags */
-	u8 dt_addr[1] = { M41T80_REG_ALARM_MON };
-	u8 *reg = buf - M41T80_REG_ALARM_MON;
-	struct i2c_msg msgs[] = {
-		{
-			.addr	= client->addr,
-			.flags	= 0,
-			.len	= 1,
-			.buf	= dt_addr,
-		},
-		{
-			.addr	= client->addr,
-			.flags	= I2C_M_RD,
-			.len	= M41T80_ALARM_REG_SIZE + 1,
-			.buf	= buf,
-		},
-	};
-
-	if (i2c_transfer(client->adapter, msgs, 2) < 0) {
-		dev_err(&client->dev, "read error\n");
-		return -EIO;
-	}
-	t->time.tm_sec = -1;
-	t->time.tm_min = -1;
-	t->time.tm_hour = -1;
-	t->time.tm_mday = -1;
-	t->time.tm_mon = -1;
-	if (!(reg[M41T80_REG_ALARM_SEC] & 0x80))
-		t->time.tm_sec = bcd2bin(reg[M41T80_REG_ALARM_SEC] & 0x7f);
-	if (!(reg[M41T80_REG_ALARM_MIN] & 0x80))
-		t->time.tm_min = bcd2bin(reg[M41T80_REG_ALARM_MIN] & 0x7f);
-	if (!(reg[M41T80_REG_ALARM_HOUR] & 0x80))
-		t->time.tm_hour = bcd2bin(reg[M41T80_REG_ALARM_HOUR] & 0x3f);
-	if (!(reg[M41T80_REG_ALARM_DAY] & 0x80))
-		t->time.tm_mday = bcd2bin(reg[M41T80_REG_ALARM_DAY] & 0x3f);
-	if (!(reg[M41T80_REG_ALARM_DAY] & 0x40))
-		t->time.tm_mon = bcd2bin(reg[M41T80_REG_ALARM_MON] & 0x1f) - 1;
-	t->time.tm_year = -1;
-	t->time.tm_wday = -1;
-	t->time.tm_yday = -1;
-	t->time.tm_isdst = -1;
-	t->enabled = !!(reg[M41T80_REG_ALARM_MON] & M41T80_ALMON_AFE);
-	t->pending = !!(reg[M41T80_REG_FLAGS] & M41T80_FLAGS_AF);
-	return 0;
-}
-
+/*
+ * XXX - m41t80 alarm functionality is reported broken.
+ * until it is fixed, don't register alarm functions.
+ */
 static struct rtc_class_ops m41t80_rtc_ops = {
 	.read_time = m41t80_rtc_read_time,
 	.set_time = m41t80_rtc_set_time,
-	/*
-	 * XXX - m41t80 alarm functionality is reported broken.
-	 * until it is fixed, don't register alarm functions.
-	 *
-	.read_alarm = m41t80_rtc_read_alarm,
-	.set_alarm = m41t80_rtc_set_alarm,
-	*/
 	.proc = m41t80_rtc_proc,
-	/*
-	 * See above comment on broken alarm
-	 *
-	.alarm_irq_enable = m41t80_rtc_alarm_irq_enable,
-	*/
 };
 
 #if defined(CONFIG_RTC_INTF_SYSFS) || defined(CONFIG_RTC_INTF_SYSFS_MODULE)
diff --git a/drivers/rtc/rtc-max8907.c b/drivers/rtc/rtc-max8907.c
new file mode 100644
index 00000000000..e094ffa434f
--- /dev/null
+++ b/drivers/rtc/rtc-max8907.c
@@ -0,0 +1,244 @@
+/*
+ * RTC driver for Maxim MAX8907
+ *
+ * Copyright (c) 2011-2012, NVIDIA Corporation.
+ *
+ * Based on drivers/rtc/rtc-max8925.c,
+ * Copyright (C) 2009-2010 Marvell International Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/bcd.h>
+#include <linux/i2c.h>
+#include <linux/mfd/max8907.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/rtc.h>
+#include <linux/slab.h>
+
+enum {
+	RTC_SEC = 0,
+	RTC_MIN,
+	RTC_HOUR,
+	RTC_WEEKDAY,
+	RTC_DATE,
+	RTC_MONTH,
+	RTC_YEAR1,
+	RTC_YEAR2,
+};
+
+#define TIME_NUM			8
+#define ALARM_1SEC			(1 << 7)
+#define HOUR_12				(1 << 7)
+#define HOUR_AM_PM			(1 << 5)
+#define ALARM0_IRQ			(1 << 3)
+#define ALARM1_IRQ			(1 << 2)
+#define ALARM0_STATUS			(1 << 2)
+#define ALARM1_STATUS			(1 << 1)
+
+struct max8907_rtc {
+	struct max8907		*max8907;
+	struct regmap		*regmap;
+	struct rtc_device	*rtc_dev;
+	int			irq;
+};
+
+static irqreturn_t max8907_irq_handler(int irq, void *data)
+{
+	struct max8907_rtc *rtc = data;
+
+	regmap_update_bits(rtc->regmap, MAX8907_REG_ALARM0_CNTL, 0x7f, 0);
+
+	rtc_update_irq(rtc->rtc_dev, 1, RTC_IRQF | RTC_AF);
+
+	return IRQ_HANDLED;
+}
+
+static void regs_to_tm(u8 *regs, struct rtc_time *tm)
+{
+	tm->tm_year = bcd2bin(regs[RTC_YEAR2]) * 100 +
+		bcd2bin(regs[RTC_YEAR1]) - 1900;
+	tm->tm_mon = bcd2bin(regs[RTC_MONTH] & 0x1f) - 1;
+	tm->tm_mday = bcd2bin(regs[RTC_DATE] & 0x3f);
+	tm->tm_wday = (regs[RTC_WEEKDAY] & 0x07) - 1;
+	if (regs[RTC_HOUR] & HOUR_12) {
+		tm->tm_hour = bcd2bin(regs[RTC_HOUR] & 0x01f);
+		if (tm->tm_hour == 12)
+			tm->tm_hour = 0;
+		if (regs[RTC_HOUR] & HOUR_AM_PM)
+			tm->tm_hour += 12;
+	} else {
+		tm->tm_hour = bcd2bin(regs[RTC_HOUR] & 0x03f);
+	}
+	tm->tm_min = bcd2bin(regs[RTC_MIN] & 0x7f);
+	tm->tm_sec = bcd2bin(regs[RTC_SEC] & 0x7f);
+}
+
+static void tm_to_regs(struct rtc_time *tm, u8 *regs)
+{
+	u8 high, low;
+
+	high = (tm->tm_year + 1900) / 100;
+	low = tm->tm_year % 100;
+	regs[RTC_YEAR2] = bin2bcd(high);
+	regs[RTC_YEAR1] = bin2bcd(low);
+	regs[RTC_MONTH] = bin2bcd(tm->tm_mon + 1);
+	regs[RTC_DATE] = bin2bcd(tm->tm_mday);
+	regs[RTC_WEEKDAY] = tm->tm_wday + 1;
+	regs[RTC_HOUR] = bin2bcd(tm->tm_hour);
+	regs[RTC_MIN] = bin2bcd(tm->tm_min);
+	regs[RTC_SEC] = bin2bcd(tm->tm_sec);
+}
+
+static int max8907_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct max8907_rtc *rtc = dev_get_drvdata(dev);
+	u8 regs[TIME_NUM];
+	int ret;
+
+	ret = regmap_bulk_read(rtc->regmap, MAX8907_REG_RTC_SEC, regs,
+			       TIME_NUM);
+	if (ret < 0)
+		return ret;
+
+	regs_to_tm(regs, tm);
+
+	return 0;
+}
+
+static int max8907_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct max8907_rtc *rtc = dev_get_drvdata(dev);
+	u8 regs[TIME_NUM];
+
+	tm_to_regs(tm, regs);
+
+	return regmap_bulk_write(rtc->regmap, MAX8907_REG_RTC_SEC, regs,
+				 TIME_NUM);
+}
+
+static int max8907_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct max8907_rtc *rtc = dev_get_drvdata(dev);
+	u8 regs[TIME_NUM];
+	unsigned int val;
+	int ret;
+
+	ret = regmap_bulk_read(rtc->regmap, MAX8907_REG_ALARM0_SEC, regs,
+			       TIME_NUM);
+	if (ret < 0)
+		return ret;
+
+	regs_to_tm(regs, &alrm->time);
+
+	ret = regmap_read(rtc->regmap, MAX8907_REG_ALARM0_CNTL, &val);
+	if (ret < 0)
+		return ret;
+
+	alrm->enabled = !!(val & 0x7f);
+
+	return 0;
+}
+
+static int max8907_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct max8907_rtc *rtc = dev_get_drvdata(dev);
+	u8 regs[TIME_NUM];
+	int ret;
+
+	tm_to_regs(&alrm->time, regs);
+
+	/* Disable alarm while we update the target time */
+	ret = regmap_update_bits(rtc->regmap, MAX8907_REG_ALARM0_CNTL, 0x7f, 0);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_bulk_write(rtc->regmap, MAX8907_REG_ALARM0_SEC, regs,
+				TIME_NUM);
+	if (ret < 0)
+		return ret;
+
+	if (alrm->enabled)
+		ret = regmap_update_bits(rtc->regmap, MAX8907_REG_ALARM0_CNTL,
+					 0x7f, 0x7f);
+
+	return ret;
+}
+
+static const struct rtc_class_ops max8907_rtc_ops = {
+	.read_time	= max8907_rtc_read_time,
+	.set_time	= max8907_rtc_set_time,
+	.read_alarm	= max8907_rtc_read_alarm,
+	.set_alarm	= max8907_rtc_set_alarm,
+};
+
+static int __devinit max8907_rtc_probe(struct platform_device *pdev)
+{
+	struct max8907 *max8907 = dev_get_drvdata(pdev->dev.parent);
+	struct max8907_rtc *rtc;
+	int ret;
+
+	rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL);
+	if (!rtc)
+		return -ENOMEM;
+	platform_set_drvdata(pdev, rtc);
+
+	rtc->max8907 = max8907;
+	rtc->regmap = max8907->regmap_rtc;
+
+	rtc->rtc_dev = rtc_device_register("max8907-rtc", &pdev->dev,
+					&max8907_rtc_ops, THIS_MODULE);
+	if (IS_ERR(rtc->rtc_dev)) {
+		ret = PTR_ERR(rtc->rtc_dev);
+		dev_err(&pdev->dev, "Failed to register RTC device: %d\n", ret);
+		return ret;
+	}
+
+	rtc->irq = regmap_irq_get_virq(max8907->irqc_rtc,
+				       MAX8907_IRQ_RTC_ALARM0);
+	if (rtc->irq < 0) {
+		ret = rtc->irq;
+		goto err_unregister;
+	}
+
+	ret = request_threaded_irq(rtc->irq, NULL, max8907_irq_handler,
+				   IRQF_ONESHOT, "max8907-alarm0", rtc);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to request IRQ%d: %d\n",
+			rtc->irq, ret);
+		goto err_unregister;
+	}
+
+	return 0;
+
+err_unregister:
+	rtc_device_unregister(rtc->rtc_dev);
+	return ret;
+}
+
+static int __devexit max8907_rtc_remove(struct platform_device *pdev)
+{
+	struct max8907_rtc *rtc = platform_get_drvdata(pdev);
+
+	free_irq(rtc->irq, rtc);
+	rtc_device_unregister(rtc->rtc_dev);
+
+	return 0;
+}
+
+static struct platform_driver max8907_rtc_driver = {
+	.driver = {
+		.name = "max8907-rtc",
+		.owner = THIS_MODULE,
+	},
+	.probe = max8907_rtc_probe,
+	.remove = __devexit_p(max8907_rtc_remove),
+};
+module_platform_driver(max8907_rtc_driver);
+
+MODULE_DESCRIPTION("Maxim MAX8907 RTC driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rtc/rtc-mxc.c b/drivers/rtc/rtc-mxc.c
index e3e50d69baf..cd0106293a4 100644
--- a/drivers/rtc/rtc-mxc.c
+++ b/drivers/rtc/rtc-mxc.c
@@ -343,7 +343,7 @@ static struct rtc_class_ops mxc_rtc_ops = {
 	.alarm_irq_enable	= mxc_rtc_alarm_irq_enable,
 };
 
-static int __init mxc_rtc_probe(struct platform_device *pdev)
+static int __devinit mxc_rtc_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct rtc_device *rtc;
@@ -367,14 +367,14 @@ static int __init mxc_rtc_probe(struct platform_device *pdev)
 	pdata->ioaddr = devm_ioremap(&pdev->dev, res->start,
 				     resource_size(res));
 
-	pdata->clk = clk_get(&pdev->dev, "rtc");
+	pdata->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(pdata->clk)) {
 		dev_err(&pdev->dev, "unable to get clock!\n");
 		ret = PTR_ERR(pdata->clk);
 		goto exit_free_pdata;
 	}
 
-	clk_enable(pdata->clk);
+	clk_prepare_enable(pdata->clk);
 	rate = clk_get_rate(pdata->clk);
 
 	if (rate == 32768)
@@ -426,22 +426,20 @@ static int __init mxc_rtc_probe(struct platform_device *pdev)
 exit_clr_drvdata:
 	platform_set_drvdata(pdev, NULL);
 exit_put_clk:
-	clk_disable(pdata->clk);
-	clk_put(pdata->clk);
+	clk_disable_unprepare(pdata->clk);
 
 exit_free_pdata:
 
 	return ret;
 }
 
-static int __exit mxc_rtc_remove(struct platform_device *pdev)
+static int __devexit mxc_rtc_remove(struct platform_device *pdev)
 {
 	struct rtc_plat_data *pdata = platform_get_drvdata(pdev);
 
 	rtc_device_unregister(pdata->rtc);
 
-	clk_disable(pdata->clk);
-	clk_put(pdata->clk);
+	clk_disable_unprepare(pdata->clk);
 	platform_set_drvdata(pdev, NULL);
 
 	return 0;
@@ -482,21 +480,11 @@ static struct platform_driver mxc_rtc_driver = {
 #endif
 		   .owner	= THIS_MODULE,
 	},
-	.remove		= __exit_p(mxc_rtc_remove),
+	.probe = mxc_rtc_probe,
+	.remove = __devexit_p(mxc_rtc_remove),
 };
 
-static int __init mxc_rtc_init(void)
-{
-	return platform_driver_probe(&mxc_rtc_driver, mxc_rtc_probe);
-}
-
-static void __exit mxc_rtc_exit(void)
-{
-	platform_driver_unregister(&mxc_rtc_driver);
-}
-
-module_init(mxc_rtc_init);
-module_exit(mxc_rtc_exit);
+module_platform_driver(mxc_rtc_driver)
 
 MODULE_AUTHOR("Daniel Mack <daniel@caiaq.de>");
 MODULE_DESCRIPTION("RTC driver for Freescale MXC");
diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c
index c2fe426a6ef..98e3a2b681e 100644
--- a/drivers/rtc/rtc-pcf8563.c
+++ b/drivers/rtc/rtc-pcf8563.c
@@ -78,8 +78,17 @@ static int pcf8563_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	unsigned char buf[13] = { PCF8563_REG_ST1 };
 
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 1, buf },	/* setup read ptr */
-		{ client->addr, I2C_M_RD, 13, buf },	/* read status + date */
+		{/* setup read ptr */
+			.addr = client->addr,
+			.len = 1,
+			.buf = buf
+		},
+		{/* read status + date */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 13,
+			.buf = buf
+		},
 	};
 
 	/* read registers */
diff --git a/drivers/rtc/rtc-proc.c b/drivers/rtc/rtc-proc.c
index 0a59fda5c09..e96236ac2e7 100644
--- a/drivers/rtc/rtc-proc.c
+++ b/drivers/rtc/rtc-proc.c
@@ -18,6 +18,26 @@
 
 #include "rtc-core.h"
 
+#define NAME_SIZE	10
+
+#if defined(CONFIG_RTC_HCTOSYS_DEVICE)
+static bool is_rtc_hctosys(struct rtc_device *rtc)
+{
+	int size;
+	char name[NAME_SIZE];
+
+	size = scnprintf(name, NAME_SIZE, "rtc%d", rtc->id);
+	if (size > NAME_SIZE)
+		return false;
+
+	return !strncmp(name, CONFIG_RTC_HCTOSYS_DEVICE, NAME_SIZE);
+}
+#else
+static bool is_rtc_hctosys(struct rtc_device *rtc)
+{
+	return (rtc->id == 0);
+}
+#endif
 
 static int rtc_proc_show(struct seq_file *seq, void *offset)
 {
@@ -117,12 +137,12 @@ static const struct file_operations rtc_proc_fops = {
 
 void rtc_proc_add_device(struct rtc_device *rtc)
 {
-	if (rtc->id == 0)
+	if (is_rtc_hctosys(rtc))
 		proc_create_data("driver/rtc", 0, NULL, &rtc_proc_fops, rtc);
 }
 
 void rtc_proc_del_device(struct rtc_device *rtc)
 {
-	if (rtc->id == 0)
+	if (is_rtc_hctosys(rtc))
 		remove_proc_entry("driver/rtc", NULL);
 }
diff --git a/drivers/rtc/rtc-rc5t583.c b/drivers/rtc/rtc-rc5t583.c
new file mode 100644
index 00000000000..cdb140c29c5
--- /dev/null
+++ b/drivers/rtc/rtc-rc5t583.c
@@ -0,0 +1,331 @@
+/*
+ * rtc-rc5t583.c -- RICOH RC5T583 Real Time Clock
+ *
+ * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
+ * Author: Venu Byravarasu <vbyravarasu@nvidia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/rtc.h>
+#include <linux/bcd.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/rc5t583.h>
+
+struct rc5t583_rtc {
+	struct rtc_device	*rtc;
+	/* To store the list of enabled interrupts, during system suspend */
+	u32 irqen;
+};
+
+/* Total number of RTC registers needed to set time*/
+#define NUM_TIME_REGS	(RC5T583_RTC_YEAR - RC5T583_RTC_SEC + 1)
+
+/* Total number of RTC registers needed to set Y-Alarm*/
+#define NUM_YAL_REGS	(RC5T583_RTC_AY_YEAR - RC5T583_RTC_AY_MIN + 1)
+
+/* Set Y-Alarm interrupt */
+#define SET_YAL BIT(5)
+
+/* Get Y-Alarm interrupt status*/
+#define GET_YAL_STATUS BIT(3)
+
+static int rc5t583_rtc_alarm_irq_enable(struct device *dev, unsigned enabled)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	u8 val;
+
+	/* Set Y-Alarm, based on 'enabled' */
+	val = enabled ? SET_YAL : 0;
+
+	return regmap_update_bits(rc5t583->regmap, RC5T583_RTC_CTL1, SET_YAL,
+		val);
+}
+
+/*
+ * Gets current rc5t583 RTC time and date parameters.
+ *
+ * The RTC's time/alarm representation is not what gmtime(3) requires
+ * Linux to use:
+ *
+ *  - Months are 1..12 vs Linux 0-11
+ *  - Years are 0..99 vs Linux 1900..N (we assume 21st century)
+ */
+static int rc5t583_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	u8 rtc_data[NUM_TIME_REGS];
+	int ret;
+
+	ret = regmap_bulk_read(rc5t583->regmap, RC5T583_RTC_SEC, rtc_data,
+		NUM_TIME_REGS);
+	if (ret < 0) {
+		dev_err(dev, "RTC read time failed with err:%d\n", ret);
+		return ret;
+	}
+
+	tm->tm_sec = bcd2bin(rtc_data[0]);
+	tm->tm_min = bcd2bin(rtc_data[1]);
+	tm->tm_hour = bcd2bin(rtc_data[2]);
+	tm->tm_wday = bcd2bin(rtc_data[3]);
+	tm->tm_mday = bcd2bin(rtc_data[4]);
+	tm->tm_mon = bcd2bin(rtc_data[5]) - 1;
+	tm->tm_year = bcd2bin(rtc_data[6]) + 100;
+
+	return ret;
+}
+
+static int rc5t583_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	unsigned char rtc_data[NUM_TIME_REGS];
+	int ret;
+
+	rtc_data[0] = bin2bcd(tm->tm_sec);
+	rtc_data[1] = bin2bcd(tm->tm_min);
+	rtc_data[2] = bin2bcd(tm->tm_hour);
+	rtc_data[3] = bin2bcd(tm->tm_wday);
+	rtc_data[4] = bin2bcd(tm->tm_mday);
+	rtc_data[5] = bin2bcd(tm->tm_mon + 1);
+	rtc_data[6] = bin2bcd(tm->tm_year - 100);
+
+	ret = regmap_bulk_write(rc5t583->regmap, RC5T583_RTC_SEC, rtc_data,
+		NUM_TIME_REGS);
+	if (ret < 0) {
+		dev_err(dev, "RTC set time failed with error %d\n", ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int rc5t583_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	unsigned char alarm_data[NUM_YAL_REGS];
+	u32 interrupt_enable;
+	int ret;
+
+	ret = regmap_bulk_read(rc5t583->regmap, RC5T583_RTC_AY_MIN, alarm_data,
+		NUM_YAL_REGS);
+	if (ret < 0) {
+		dev_err(dev, "rtc_read_alarm error %d\n", ret);
+		return ret;
+	}
+
+	alm->time.tm_min = bcd2bin(alarm_data[0]);
+	alm->time.tm_hour = bcd2bin(alarm_data[1]);
+	alm->time.tm_mday = bcd2bin(alarm_data[2]);
+	alm->time.tm_mon = bcd2bin(alarm_data[3]) - 1;
+	alm->time.tm_year = bcd2bin(alarm_data[4]) + 100;
+
+	ret = regmap_read(rc5t583->regmap, RC5T583_RTC_CTL1, &interrupt_enable);
+	if (ret < 0)
+		return ret;
+
+	/* check if YALE is set */
+	if (interrupt_enable & SET_YAL)
+		alm->enabled = 1;
+
+	return ret;
+}
+
+static int rc5t583_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	unsigned char alarm_data[NUM_YAL_REGS];
+	int ret;
+
+	ret = rc5t583_rtc_alarm_irq_enable(dev, 0);
+	if (ret)
+		return ret;
+
+	alarm_data[0] = bin2bcd(alm->time.tm_min);
+	alarm_data[1] = bin2bcd(alm->time.tm_hour);
+	alarm_data[2] = bin2bcd(alm->time.tm_mday);
+	alarm_data[3] = bin2bcd(alm->time.tm_mon + 1);
+	alarm_data[4] = bin2bcd(alm->time.tm_year - 100);
+
+	ret = regmap_bulk_write(rc5t583->regmap, RC5T583_RTC_AY_MIN, alarm_data,
+		NUM_YAL_REGS);
+	if (ret) {
+		dev_err(dev, "rtc_set_alarm error %d\n", ret);
+		return ret;
+	}
+
+	if (alm->enabled)
+		ret = rc5t583_rtc_alarm_irq_enable(dev, 1);
+
+	return ret;
+}
+
+static irqreturn_t rc5t583_rtc_interrupt(int irq, void *rtc)
+{
+	struct device *dev = rtc;
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	struct rc5t583_rtc *rc5t583_rtc = dev_get_drvdata(dev);
+	unsigned long events = 0;
+	int ret;
+	u32 rtc_reg;
+
+	ret = regmap_read(rc5t583->regmap, RC5T583_RTC_CTL2, &rtc_reg);
+	if (ret < 0)
+		return IRQ_NONE;
+
+	if (rtc_reg & GET_YAL_STATUS) {
+		events = RTC_IRQF | RTC_AF;
+		/* clear pending Y-alarm interrupt bit */
+		rtc_reg &= ~GET_YAL_STATUS;
+	}
+
+	ret = regmap_write(rc5t583->regmap, RC5T583_RTC_CTL2, rtc_reg);
+	if (ret)
+		return IRQ_NONE;
+
+	/* Notify RTC core on event */
+	rtc_update_irq(rc5t583_rtc->rtc, 1, events);
+
+	return IRQ_HANDLED;
+}
+
+static const struct rtc_class_ops rc5t583_rtc_ops = {
+	.read_time	= rc5t583_rtc_read_time,
+	.set_time	= rc5t583_rtc_set_time,
+	.read_alarm	= rc5t583_rtc_read_alarm,
+	.set_alarm	= rc5t583_rtc_set_alarm,
+	.alarm_irq_enable = rc5t583_rtc_alarm_irq_enable,
+};
+
+static int __devinit rc5t583_rtc_probe(struct platform_device *pdev)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(pdev->dev.parent);
+	struct rc5t583_rtc *ricoh_rtc;
+	struct rc5t583_platform_data *pmic_plat_data;
+	int ret;
+	int irq;
+
+	ricoh_rtc = devm_kzalloc(&pdev->dev, sizeof(struct rc5t583_rtc),
+			GFP_KERNEL);
+	if (!ricoh_rtc)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ricoh_rtc);
+
+	/* Clear pending interrupts */
+	ret = regmap_write(rc5t583->regmap, RC5T583_RTC_CTL2, 0);
+	if (ret < 0)
+		return ret;
+
+	/* clear RTC Adjust register */
+	ret = regmap_write(rc5t583->regmap, RC5T583_RTC_ADJ, 0);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "unable to program rtc_adjust reg\n");
+		return -EBUSY;
+	}
+
+	pmic_plat_data = dev_get_platdata(rc5t583->dev);
+	irq = pmic_plat_data->irq_base;
+	if (irq <= 0) {
+		dev_warn(&pdev->dev, "Wake up is not possible as irq = %d\n",
+			irq);
+		return ret;
+	}
+
+	irq += RC5T583_IRQ_YALE;
+	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+		rc5t583_rtc_interrupt, IRQF_TRIGGER_LOW,
+		"rtc-rc5t583", &pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "IRQ is not free.\n");
+		return ret;
+	}
+	device_init_wakeup(&pdev->dev, 1);
+
+	ricoh_rtc->rtc = rtc_device_register(pdev->name, &pdev->dev,
+		&rc5t583_rtc_ops, THIS_MODULE);
+	if (IS_ERR(ricoh_rtc->rtc)) {
+		ret = PTR_ERR(ricoh_rtc->rtc);
+		dev_err(&pdev->dev, "RTC device register: err %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Disable rc5t583 RTC interrupts.
+ * Sets status flag to free.
+ */
+static int __devexit rc5t583_rtc_remove(struct platform_device *pdev)
+{
+	struct rc5t583_rtc *rc5t583_rtc = dev_get_drvdata(&pdev->dev);
+
+	rc5t583_rtc_alarm_irq_enable(&rc5t583_rtc->rtc->dev, 0);
+
+	rtc_device_unregister(rc5t583_rtc->rtc);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+
+static int rc5t583_rtc_suspend(struct device *dev)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	struct rc5t583_rtc *rc5t583_rtc = dev_get_drvdata(dev);
+	int ret;
+
+	/* Store current list of enabled interrupts*/
+	ret = regmap_read(rc5t583->regmap, RC5T583_RTC_CTL1,
+		&rc5t583_rtc->irqen);
+	return ret;
+}
+
+static int rc5t583_rtc_resume(struct device *dev)
+{
+	struct rc5t583 *rc5t583 = dev_get_drvdata(dev->parent);
+	struct rc5t583_rtc *rc5t583_rtc = dev_get_drvdata(dev);
+
+	/* Restore list of enabled interrupts before suspend */
+	return regmap_write(rc5t583->regmap, RC5T583_RTC_CTL1,
+		rc5t583_rtc->irqen);
+}
+
+static const struct dev_pm_ops rc5t583_rtc_pm_ops = {
+	.suspend	= rc5t583_rtc_suspend,
+	.resume		= rc5t583_rtc_resume,
+};
+
+#define DEV_PM_OPS     (&rc5t583_rtc_pm_ops)
+#else
+#define DEV_PM_OPS     NULL
+#endif
+
+static struct platform_driver rc5t583_rtc_driver = {
+	.probe		= rc5t583_rtc_probe,
+	.remove		= __devexit_p(rc5t583_rtc_remove),
+	.driver		= {
+		.owner	= THIS_MODULE,
+		.name	= "rtc-rc5t583",
+		.pm	= DEV_PM_OPS,
+	},
+};
+
+module_platform_driver(rc5t583_rtc_driver);
+MODULE_ALIAS("platform:rtc-rc5t583");
+MODULE_AUTHOR("Venu Byravarasu <vbyravarasu@nvidia.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/rtc/rtc-rs5c372.c b/drivers/rtc/rtc-rs5c372.c
index fb4842c3544..76f565ae384 100644
--- a/drivers/rtc/rtc-rs5c372.c
+++ b/drivers/rtc/rtc-rs5c372.c
@@ -104,7 +104,12 @@ static int rs5c_get_regs(struct rs5c372 *rs5c)
 {
 	struct i2c_client	*client = rs5c->client;
 	struct i2c_msg		msgs[] = {
-		{ client->addr, I2C_M_RD, sizeof rs5c->buf, rs5c->buf },
+		{
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = sizeof(rs5c->buf),
+			.buf = rs5c->buf
+		},
 	};
 
 	/* This implements the third reading method from the datasheet, using
diff --git a/drivers/rtc/rtc-s35390a.c b/drivers/rtc/rtc-s35390a.c
index c9562ceedef..8a092325188 100644
--- a/drivers/rtc/rtc-s35390a.c
+++ b/drivers/rtc/rtc-s35390a.c
@@ -19,6 +19,8 @@
 #define S35390A_CMD_STATUS1	0
 #define S35390A_CMD_STATUS2	1
 #define S35390A_CMD_TIME1	2
+#define S35390A_CMD_TIME2	3
+#define S35390A_CMD_INT2_REG1	5
 
 #define S35390A_BYTE_YEAR	0
 #define S35390A_BYTE_MONTH	1
@@ -28,12 +30,23 @@
 #define S35390A_BYTE_MINS	5
 #define S35390A_BYTE_SECS	6
 
+#define S35390A_ALRM_BYTE_WDAY	0
+#define S35390A_ALRM_BYTE_HOURS	1
+#define S35390A_ALRM_BYTE_MINS	2
+
 #define S35390A_FLAG_POC	0x01
 #define S35390A_FLAG_BLD	0x02
 #define S35390A_FLAG_24H	0x40
 #define S35390A_FLAG_RESET	0x80
 #define S35390A_FLAG_TEST	0x01
 
+#define S35390A_INT2_MODE_MASK		0xF0
+
+#define S35390A_INT2_MODE_NOINTR	0x00
+#define S35390A_INT2_MODE_FREQ		0x10
+#define S35390A_INT2_MODE_ALARM		0x40
+#define S35390A_INT2_MODE_PMIN_EDG	0x20
+
 static const struct i2c_device_id s35390a_id[] = {
 	{ "s35390a", 0 },
 	{ }
@@ -50,7 +63,11 @@ static int s35390a_set_reg(struct s35390a *s35390a, int reg, char *buf, int len)
 {
 	struct i2c_client *client = s35390a->client[reg];
 	struct i2c_msg msg[] = {
-		{ client->addr, 0, len, buf },
+		{
+			.addr = client->addr,
+			.len = len,
+			.buf = buf
+		},
 	};
 
 	if ((i2c_transfer(client->adapter, msg, 1)) != 1)
@@ -63,7 +80,12 @@ static int s35390a_get_reg(struct s35390a *s35390a, int reg, char *buf, int len)
 {
 	struct i2c_client *client = s35390a->client[reg];
 	struct i2c_msg msg[] = {
-		{ client->addr, I2C_M_RD, len, buf },
+		{
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = len,
+			.buf = buf
+		},
 	};
 
 	if ((i2c_transfer(client->adapter, msg, 1)) != 1)
@@ -184,6 +206,104 @@ static int s35390a_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	return rtc_valid_tm(tm);
 }
 
+static int s35390a_set_alarm(struct i2c_client *client, struct rtc_wkalrm *alm)
+{
+	struct s35390a *s35390a = i2c_get_clientdata(client);
+	char buf[3], sts = 0;
+	int err, i;
+
+	dev_dbg(&client->dev, "%s: alm is secs=%d, mins=%d, hours=%d mday=%d, "\
+		"mon=%d, year=%d, wday=%d\n", __func__, alm->time.tm_sec,
+		alm->time.tm_min, alm->time.tm_hour, alm->time.tm_mday,
+		alm->time.tm_mon, alm->time.tm_year, alm->time.tm_wday);
+
+	/* disable interrupt */
+	err = s35390a_set_reg(s35390a, S35390A_CMD_STATUS2, &sts, sizeof(sts));
+	if (err < 0)
+		return err;
+
+	/* clear pending interrupt, if any */
+	err = s35390a_get_reg(s35390a, S35390A_CMD_STATUS1, &sts, sizeof(sts));
+	if (err < 0)
+		return err;
+
+	if (alm->enabled)
+		sts = S35390A_INT2_MODE_ALARM;
+	else
+		sts = S35390A_INT2_MODE_NOINTR;
+
+	/* This chip expects the bits of each byte to be in reverse order */
+	sts = bitrev8(sts);
+
+	/* set interupt mode*/
+	err = s35390a_set_reg(s35390a, S35390A_CMD_STATUS2, &sts, sizeof(sts));
+	if (err < 0)
+		return err;
+
+	if (alm->time.tm_wday != -1)
+		buf[S35390A_ALRM_BYTE_WDAY] = bin2bcd(alm->time.tm_wday) | 0x80;
+
+	buf[S35390A_ALRM_BYTE_HOURS] = s35390a_hr2reg(s35390a,
+			alm->time.tm_hour) | 0x80;
+	buf[S35390A_ALRM_BYTE_MINS] = bin2bcd(alm->time.tm_min) | 0x80;
+
+	if (alm->time.tm_hour >= 12)
+		buf[S35390A_ALRM_BYTE_HOURS] |= 0x40;
+
+	for (i = 0; i < 3; ++i)
+		buf[i] = bitrev8(buf[i]);
+
+	err = s35390a_set_reg(s35390a, S35390A_CMD_INT2_REG1, buf,
+								sizeof(buf));
+
+	return err;
+}
+
+static int s35390a_read_alarm(struct i2c_client *client, struct rtc_wkalrm *alm)
+{
+	struct s35390a *s35390a = i2c_get_clientdata(client);
+	char buf[3], sts;
+	int i, err;
+
+	err = s35390a_get_reg(s35390a, S35390A_CMD_STATUS2, &sts, sizeof(sts));
+	if (err < 0)
+		return err;
+
+	if (bitrev8(sts) != S35390A_INT2_MODE_ALARM)
+		return -EINVAL;
+
+	err = s35390a_get_reg(s35390a, S35390A_CMD_INT2_REG1, buf, sizeof(buf));
+	if (err < 0)
+		return err;
+
+	/* This chip returns the bits of each byte in reverse order */
+	for (i = 0; i < 3; ++i) {
+		buf[i] = bitrev8(buf[i]);
+		buf[i] &= ~0x80;
+	}
+
+	alm->time.tm_wday = bcd2bin(buf[S35390A_ALRM_BYTE_WDAY]);
+	alm->time.tm_hour = s35390a_reg2hr(s35390a,
+						buf[S35390A_ALRM_BYTE_HOURS]);
+	alm->time.tm_min = bcd2bin(buf[S35390A_ALRM_BYTE_MINS]);
+
+	dev_dbg(&client->dev, "%s: alm is mins=%d, hours=%d, wday=%d\n",
+			__func__, alm->time.tm_min, alm->time.tm_hour,
+			alm->time.tm_wday);
+
+	return 0;
+}
+
+static int s35390a_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	return s35390a_read_alarm(to_i2c_client(dev), alm);
+}
+
+static int s35390a_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	return s35390a_set_alarm(to_i2c_client(dev), alm);
+}
+
 static int s35390a_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
 	return s35390a_get_datetime(to_i2c_client(dev), tm);
@@ -197,6 +317,9 @@ static int s35390a_rtc_set_time(struct device *dev, struct rtc_time *tm)
 static const struct rtc_class_ops s35390a_rtc_ops = {
 	.read_time	= s35390a_rtc_read_time,
 	.set_time	= s35390a_rtc_set_time,
+	.set_alarm	= s35390a_rtc_set_alarm,
+	.read_alarm	= s35390a_rtc_read_alarm,
+
 };
 
 static struct i2c_driver s35390a_driver;
@@ -261,6 +384,8 @@ static int s35390a_probe(struct i2c_client *client,
 	if (s35390a_get_datetime(client, &tm) < 0)
 		dev_warn(&client->dev, "clock needs to be set\n");
 
+	device_set_wakeup_capable(&client->dev, 1);
+
 	s35390a->rtc = rtc_device_register(s35390a_driver.driver.name,
 				&client->dev, &s35390a_rtc_ops, THIS_MODULE);
 
diff --git a/drivers/rtc/rtc-s3c.c b/drivers/rtc/rtc-s3c.c
index bfbd92c8d1c..77823d21d31 100644
--- a/drivers/rtc/rtc-s3c.c
+++ b/drivers/rtc/rtc-s3c.c
@@ -476,13 +476,13 @@ static int __devinit s3c_rtc_probe(struct platform_device *pdev)
 	s3c_rtc_tickno = platform_get_irq(pdev, 1);
 	if (s3c_rtc_tickno < 0) {
 		dev_err(&pdev->dev, "no irq for rtc tick\n");
-		return -ENOENT;
+		return s3c_rtc_tickno;
 	}
 
 	s3c_rtc_alarmno = platform_get_irq(pdev, 0);
 	if (s3c_rtc_alarmno < 0) {
 		dev_err(&pdev->dev, "no irq for alarm\n");
-		return -ENOENT;
+		return s3c_rtc_alarmno;
 	}
 
 	pr_debug("s3c2410_rtc: tick irq %d, alarm irq %d\n",
diff --git a/drivers/rtc/rtc-snvs.c b/drivers/rtc/rtc-snvs.c
new file mode 100644
index 00000000000..3c0da333f46
--- /dev/null
+++ b/drivers/rtc/rtc-snvs.c
@@ -0,0 +1,350 @@
+/*
+ * Copyright (C) 2011-2012 Freescale Semiconductor, Inc.
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/rtc.h>
+
+/* These register offsets are relative to LP (Low Power) range */
+#define SNVS_LPCR		0x04
+#define SNVS_LPSR		0x18
+#define SNVS_LPSRTCMR		0x1c
+#define SNVS_LPSRTCLR		0x20
+#define SNVS_LPTAR		0x24
+#define SNVS_LPPGDR		0x30
+
+#define SNVS_LPCR_SRTC_ENV	(1 << 0)
+#define SNVS_LPCR_LPTA_EN	(1 << 1)
+#define SNVS_LPCR_LPWUI_EN	(1 << 3)
+#define SNVS_LPSR_LPTA		(1 << 0)
+
+#define SNVS_LPPGDR_INIT	0x41736166
+#define CNTR_TO_SECS_SH		15
+
+struct snvs_rtc_data {
+	struct rtc_device *rtc;
+	void __iomem *ioaddr;
+	int irq;
+	spinlock_t lock;
+};
+
+static u32 rtc_read_lp_counter(void __iomem *ioaddr)
+{
+	u64 read1, read2;
+
+	do {
+		read1 = readl(ioaddr + SNVS_LPSRTCMR);
+		read1 <<= 32;
+		read1 |= readl(ioaddr + SNVS_LPSRTCLR);
+
+		read2 = readl(ioaddr + SNVS_LPSRTCMR);
+		read2 <<= 32;
+		read2 |= readl(ioaddr + SNVS_LPSRTCLR);
+	} while (read1 != read2);
+
+	/* Convert 47-bit counter to 32-bit raw second count */
+	return (u32) (read1 >> CNTR_TO_SECS_SH);
+}
+
+static void rtc_write_sync_lp(void __iomem *ioaddr)
+{
+	u32 count1, count2, count3;
+	int i;
+
+	/* Wait for 3 CKIL cycles */
+	for (i = 0; i < 3; i++) {
+		do {
+			count1 = readl(ioaddr + SNVS_LPSRTCLR);
+			count2 = readl(ioaddr + SNVS_LPSRTCLR);
+		} while (count1 != count2);
+
+		/* Now wait until counter value changes */
+		do {
+			do {
+				count2 = readl(ioaddr + SNVS_LPSRTCLR);
+				count3 = readl(ioaddr + SNVS_LPSRTCLR);
+			} while (count2 != count3);
+		} while (count3 == count1);
+	}
+}
+
+static int snvs_rtc_enable(struct snvs_rtc_data *data, bool enable)
+{
+	unsigned long flags;
+	int timeout = 1000;
+	u32 lpcr;
+
+	spin_lock_irqsave(&data->lock, flags);
+
+	lpcr = readl(data->ioaddr + SNVS_LPCR);
+	if (enable)
+		lpcr |= SNVS_LPCR_SRTC_ENV;
+	else
+		lpcr &= ~SNVS_LPCR_SRTC_ENV;
+	writel(lpcr, data->ioaddr + SNVS_LPCR);
+
+	spin_unlock_irqrestore(&data->lock, flags);
+
+	while (--timeout) {
+		lpcr = readl(data->ioaddr + SNVS_LPCR);
+
+		if (enable) {
+			if (lpcr & SNVS_LPCR_SRTC_ENV)
+				break;
+		} else {
+			if (!(lpcr & SNVS_LPCR_SRTC_ENV))
+				break;
+		}
+	}
+
+	if (!timeout)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int snvs_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	unsigned long time = rtc_read_lp_counter(data->ioaddr);
+
+	rtc_time_to_tm(time, tm);
+
+	return 0;
+}
+
+static int snvs_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	unsigned long time;
+
+	rtc_tm_to_time(tm, &time);
+
+	/* Disable RTC first */
+	snvs_rtc_enable(data, false);
+
+	/* Write 32-bit time to 47-bit timer, leaving 15 LSBs blank */
+	writel(time << CNTR_TO_SECS_SH, data->ioaddr + SNVS_LPSRTCLR);
+	writel(time >> (32 - CNTR_TO_SECS_SH), data->ioaddr + SNVS_LPSRTCMR);
+
+	/* Enable RTC again */
+	snvs_rtc_enable(data, true);
+
+	return 0;
+}
+
+static int snvs_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	u32 lptar, lpsr;
+
+	lptar = readl(data->ioaddr + SNVS_LPTAR);
+	rtc_time_to_tm(lptar, &alrm->time);
+
+	lpsr = readl(data->ioaddr + SNVS_LPSR);
+	alrm->pending = (lpsr & SNVS_LPSR_LPTA) ? 1 : 0;
+
+	return 0;
+}
+
+static int snvs_rtc_alarm_irq_enable(struct device *dev, unsigned int enable)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	u32 lpcr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&data->lock, flags);
+
+	lpcr = readl(data->ioaddr + SNVS_LPCR);
+	if (enable)
+		lpcr |= (SNVS_LPCR_LPTA_EN | SNVS_LPCR_LPWUI_EN);
+	else
+		lpcr &= ~(SNVS_LPCR_LPTA_EN | SNVS_LPCR_LPWUI_EN);
+	writel(lpcr, data->ioaddr + SNVS_LPCR);
+
+	spin_unlock_irqrestore(&data->lock, flags);
+
+	rtc_write_sync_lp(data->ioaddr);
+
+	return 0;
+}
+
+static int snvs_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	struct rtc_time *alrm_tm = &alrm->time;
+	unsigned long time;
+	unsigned long flags;
+	u32 lpcr;
+
+	rtc_tm_to_time(alrm_tm, &time);
+
+	spin_lock_irqsave(&data->lock, flags);
+
+	/* Have to clear LPTA_EN before programming new alarm time in LPTAR */
+	lpcr = readl(data->ioaddr + SNVS_LPCR);
+	lpcr &= ~SNVS_LPCR_LPTA_EN;
+	writel(lpcr, data->ioaddr + SNVS_LPCR);
+
+	spin_unlock_irqrestore(&data->lock, flags);
+
+	writel(time, data->ioaddr + SNVS_LPTAR);
+
+	/* Clear alarm interrupt status bit */
+	writel(SNVS_LPSR_LPTA, data->ioaddr + SNVS_LPSR);
+
+	return snvs_rtc_alarm_irq_enable(dev, alrm->enabled);
+}
+
+static const struct rtc_class_ops snvs_rtc_ops = {
+	.read_time = snvs_rtc_read_time,
+	.set_time = snvs_rtc_set_time,
+	.read_alarm = snvs_rtc_read_alarm,
+	.set_alarm = snvs_rtc_set_alarm,
+	.alarm_irq_enable = snvs_rtc_alarm_irq_enable,
+};
+
+static irqreturn_t snvs_rtc_irq_handler(int irq, void *dev_id)
+{
+	struct device *dev = dev_id;
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+	u32 lpsr;
+	u32 events = 0;
+
+	lpsr = readl(data->ioaddr + SNVS_LPSR);
+
+	if (lpsr & SNVS_LPSR_LPTA) {
+		events |= (RTC_AF | RTC_IRQF);
+
+		/* RTC alarm should be one-shot */
+		snvs_rtc_alarm_irq_enable(dev, 0);
+
+		rtc_update_irq(data->rtc, 1, events);
+	}
+
+	/* clear interrupt status */
+	writel(lpsr, data->ioaddr + SNVS_LPSR);
+
+	return events ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static int __devinit snvs_rtc_probe(struct platform_device *pdev)
+{
+	struct snvs_rtc_data *data;
+	struct resource *res;
+	int ret;
+
+	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	data->ioaddr = devm_request_and_ioremap(&pdev->dev, res);
+	if (!data->ioaddr)
+		return -EADDRNOTAVAIL;
+
+	data->irq = platform_get_irq(pdev, 0);
+	if (data->irq < 0)
+		return data->irq;
+
+	platform_set_drvdata(pdev, data);
+
+	spin_lock_init(&data->lock);
+
+	/* Initialize glitch detect */
+	writel(SNVS_LPPGDR_INIT, data->ioaddr + SNVS_LPPGDR);
+
+	/* Clear interrupt status */
+	writel(0xffffffff, data->ioaddr + SNVS_LPSR);
+
+	/* Enable RTC */
+	snvs_rtc_enable(data, true);
+
+	device_init_wakeup(&pdev->dev, true);
+
+	ret = devm_request_irq(&pdev->dev, data->irq, snvs_rtc_irq_handler,
+			       IRQF_SHARED, "rtc alarm", &pdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to request irq %d: %d\n",
+			data->irq, ret);
+		return ret;
+	}
+
+	data->rtc = rtc_device_register(pdev->name, &pdev->dev,
+					&snvs_rtc_ops, THIS_MODULE);
+	if (IS_ERR(data->rtc)) {
+		ret = PTR_ERR(data->rtc);
+		dev_err(&pdev->dev, "failed to register rtc: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int __devexit snvs_rtc_remove(struct platform_device *pdev)
+{
+	struct snvs_rtc_data *data = platform_get_drvdata(pdev);
+
+	rtc_device_unregister(data->rtc);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int snvs_rtc_suspend(struct device *dev)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+
+	if (device_may_wakeup(dev))
+		enable_irq_wake(data->irq);
+
+	return 0;
+}
+
+static int snvs_rtc_resume(struct device *dev)
+{
+	struct snvs_rtc_data *data = dev_get_drvdata(dev);
+
+	if (device_may_wakeup(dev))
+		disable_irq_wake(data->irq);
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(snvs_rtc_pm_ops, snvs_rtc_suspend, snvs_rtc_resume);
+
+static const struct of_device_id __devinitconst snvs_dt_ids[] = {
+	{ .compatible = "fsl,sec-v4.0-mon-rtc-lp", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, snvs_dt_ids);
+
+static struct platform_driver snvs_rtc_driver = {
+	.driver = {
+		.name	= "snvs_rtc",
+		.owner	= THIS_MODULE,
+		.pm	= &snvs_rtc_pm_ops,
+		.of_match_table = snvs_dt_ids,
+	},
+	.probe		= snvs_rtc_probe,
+	.remove		= __devexit_p(snvs_rtc_remove),
+};
+module_platform_driver(snvs_rtc_driver);
+
+MODULE_AUTHOR("Freescale Semiconductor, Inc.");
+MODULE_DESCRIPTION("Freescale SNVS RTC Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rtc/rtc-spear.c b/drivers/rtc/rtc-spear.c
index e2785479113..bb507d23f6c 100644
--- a/drivers/rtc/rtc-spear.c
+++ b/drivers/rtc/rtc-spear.c
@@ -235,7 +235,7 @@ static int spear_rtc_read_time(struct device *dev, struct rtc_time *tm)
 static int spear_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct spear_rtc_config *config = dev_get_drvdata(dev);
-	unsigned int time, date, err = 0;
+	unsigned int time, date;
 
 	if (tm2bcd(tm) < 0)
 		return -EINVAL;
@@ -247,11 +247,8 @@ static int spear_rtc_set_time(struct device *dev, struct rtc_time *tm)
 		(tm->tm_year << YEAR_SHIFT);
 	writel(time, config->ioaddr + TIME_REG);
 	writel(date, config->ioaddr + DATE_REG);
-	err = is_write_complete(config);
-	if (err < 0)
-		return err;
 
-	return 0;
+	return is_write_complete(config);
 }
 
 /*
@@ -295,7 +292,8 @@ static int spear_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm)
 static int spear_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
 {
 	struct spear_rtc_config *config = dev_get_drvdata(dev);
-	unsigned int time, date, err = 0;
+	unsigned int time, date;
+	int err;
 
 	if (tm2bcd(&alm->time) < 0)
 		return -EINVAL;
@@ -357,7 +355,7 @@ static int __devinit spear_rtc_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct spear_rtc_config *config;
-	unsigned int status = 0;
+	int status = 0;
 	int irq;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
diff --git a/drivers/rtc/rtc-sysfs.c b/drivers/rtc/rtc-sysfs.c
index 380083ca572..b70e2bb6364 100644
--- a/drivers/rtc/rtc-sysfs.c
+++ b/drivers/rtc/rtc-sysfs.c
@@ -102,6 +102,12 @@ rtc_sysfs_set_max_user_freq(struct device *dev, struct device_attribute *attr,
 	return n;
 }
 
+/**
+ * rtc_sysfs_show_hctosys - indicate if the given RTC set the system time
+ *
+ * Returns 1 if the system clock was set by this RTC at the last
+ * boot or resume event.
+ */
 static ssize_t
 rtc_sysfs_show_hctosys(struct device *dev, struct device_attribute *attr,
 		char *buf)
diff --git a/drivers/rtc/rtc-tps65910.c b/drivers/rtc/rtc-tps65910.c
new file mode 100644
index 00000000000..7a82337e4de
--- /dev/null
+++ b/drivers/rtc/rtc-tps65910.c
@@ -0,0 +1,349 @@
+/*
+ * rtc-tps65910.c -- TPS65910 Real Time Clock interface
+ *
+ * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
+ * Author: Venu Byravarasu <vbyravarasu@nvidia.com>
+ *
+ * Based on original TI driver rtc-twl.c
+ *   Copyright (C) 2007 MontaVista Software, Inc
+ *   Author: Alexandre Rusev <source@mvista.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/rtc.h>
+#include <linux/bcd.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/mfd/tps65910.h>
+
+struct tps65910_rtc {
+	struct rtc_device	*rtc;
+	/* To store the list of enabled interrupts */
+	u32 irqstat;
+};
+
+/* Total number of RTC registers needed to set time*/
+#define NUM_TIME_REGS	(TPS65910_YEARS - TPS65910_SECONDS + 1)
+
+static int tps65910_rtc_alarm_irq_enable(struct device *dev, unsigned enabled)
+{
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	u8 val = 0;
+
+	if (enabled)
+		val = TPS65910_RTC_INTERRUPTS_IT_ALARM;
+
+	return regmap_write(tps->regmap, TPS65910_RTC_INTERRUPTS, val);
+}
+
+/*
+ * Gets current tps65910 RTC time and date parameters.
+ *
+ * The RTC's time/alarm representation is not what gmtime(3) requires
+ * Linux to use:
+ *
+ *  - Months are 1..12 vs Linux 0-11
+ *  - Years are 0..99 vs Linux 1900..N (we assume 21st century)
+ */
+static int tps65910_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	unsigned char rtc_data[NUM_TIME_REGS];
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	int ret;
+
+	/* Copy RTC counting registers to static registers or latches */
+	ret = regmap_update_bits(tps->regmap, TPS65910_RTC_CTRL,
+		TPS65910_RTC_CTRL_GET_TIME, TPS65910_RTC_CTRL_GET_TIME);
+	if (ret < 0) {
+		dev_err(dev, "RTC CTRL reg update failed with err:%d\n", ret);
+		return ret;
+	}
+
+	ret = regmap_bulk_read(tps->regmap, TPS65910_SECONDS, rtc_data,
+		NUM_TIME_REGS);
+	if (ret < 0) {
+		dev_err(dev, "reading from RTC failed with err:%d\n", ret);
+		return ret;
+	}
+
+	tm->tm_sec = bcd2bin(rtc_data[0]);
+	tm->tm_min = bcd2bin(rtc_data[1]);
+	tm->tm_hour = bcd2bin(rtc_data[2]);
+	tm->tm_mday = bcd2bin(rtc_data[3]);
+	tm->tm_mon = bcd2bin(rtc_data[4]) - 1;
+	tm->tm_year = bcd2bin(rtc_data[5]) + 100;
+
+	return ret;
+}
+
+static int tps65910_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	unsigned char rtc_data[NUM_TIME_REGS];
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	int ret;
+
+	rtc_data[0] = bin2bcd(tm->tm_sec);
+	rtc_data[1] = bin2bcd(tm->tm_min);
+	rtc_data[2] = bin2bcd(tm->tm_hour);
+	rtc_data[3] = bin2bcd(tm->tm_mday);
+	rtc_data[4] = bin2bcd(tm->tm_mon + 1);
+	rtc_data[5] = bin2bcd(tm->tm_year - 100);
+
+	/* Stop RTC while updating the RTC time registers */
+	ret = regmap_update_bits(tps->regmap, TPS65910_RTC_CTRL,
+		TPS65910_RTC_CTRL_STOP_RTC, 0);
+	if (ret < 0) {
+		dev_err(dev, "RTC stop failed with err:%d\n", ret);
+		return ret;
+	}
+
+	/* update all the time registers in one shot */
+	ret = regmap_bulk_write(tps->regmap, TPS65910_SECONDS, rtc_data,
+		NUM_TIME_REGS);
+	if (ret < 0) {
+		dev_err(dev, "rtc_set_time error %d\n", ret);
+		return ret;
+	}
+
+	/* Start back RTC */
+	ret = regmap_update_bits(tps->regmap, TPS65910_RTC_CTRL,
+		TPS65910_RTC_CTRL_STOP_RTC, 1);
+	if (ret < 0)
+		dev_err(dev, "RTC start failed with err:%d\n", ret);
+
+	return ret;
+}
+
+/*
+ * Gets current tps65910 RTC alarm time.
+ */
+static int tps65910_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	unsigned char alarm_data[NUM_TIME_REGS];
+	u32 int_val;
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	int ret;
+
+	ret = regmap_bulk_read(tps->regmap, TPS65910_SECONDS, alarm_data,
+		NUM_TIME_REGS);
+	if (ret < 0) {
+		dev_err(dev, "rtc_read_alarm error %d\n", ret);
+		return ret;
+	}
+
+	alm->time.tm_sec = bcd2bin(alarm_data[0]);
+	alm->time.tm_min = bcd2bin(alarm_data[1]);
+	alm->time.tm_hour = bcd2bin(alarm_data[2]);
+	alm->time.tm_mday = bcd2bin(alarm_data[3]);
+	alm->time.tm_mon = bcd2bin(alarm_data[4]) - 1;
+	alm->time.tm_year = bcd2bin(alarm_data[5]) + 100;
+
+	ret = regmap_read(tps->regmap, TPS65910_RTC_INTERRUPTS, &int_val);
+	if (ret < 0)
+		return ret;
+
+	if (int_val & TPS65910_RTC_INTERRUPTS_IT_ALARM)
+		alm->enabled = 1;
+
+	return ret;
+}
+
+static int tps65910_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
+{
+	unsigned char alarm_data[NUM_TIME_REGS];
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	int ret;
+
+	ret = tps65910_rtc_alarm_irq_enable(dev, 0);
+	if (ret)
+		return ret;
+
+	alarm_data[0] = bin2bcd(alm->time.tm_sec);
+	alarm_data[1] = bin2bcd(alm->time.tm_min);
+	alarm_data[2] = bin2bcd(alm->time.tm_hour);
+	alarm_data[3] = bin2bcd(alm->time.tm_mday);
+	alarm_data[4] = bin2bcd(alm->time.tm_mon + 1);
+	alarm_data[5] = bin2bcd(alm->time.tm_year - 100);
+
+	/* update all the alarm registers in one shot */
+	ret = regmap_bulk_write(tps->regmap, TPS65910_ALARM_SECONDS,
+		alarm_data, NUM_TIME_REGS);
+	if (ret) {
+		dev_err(dev, "rtc_set_alarm error %d\n", ret);
+		return ret;
+	}
+
+	if (alm->enabled)
+		ret = tps65910_rtc_alarm_irq_enable(dev, 1);
+
+	return ret;
+}
+
+static irqreturn_t tps65910_rtc_interrupt(int irq, void *rtc)
+{
+	struct device *dev = rtc;
+	unsigned long events = 0;
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	struct tps65910_rtc *tps_rtc = dev_get_drvdata(dev);
+	int ret;
+	u32 rtc_reg;
+
+	ret = regmap_read(tps->regmap, TPS65910_RTC_STATUS, &rtc_reg);
+	if (ret)
+		return IRQ_NONE;
+
+	if (rtc_reg & TPS65910_RTC_STATUS_ALARM)
+		events = RTC_IRQF | RTC_AF;
+
+	ret = regmap_write(tps->regmap, TPS65910_RTC_STATUS, rtc_reg);
+	if (ret)
+		return IRQ_NONE;
+
+	/* Notify RTC core on event */
+	rtc_update_irq(tps_rtc->rtc, 1, events);
+
+	return IRQ_HANDLED;
+}
+
+static const struct rtc_class_ops tps65910_rtc_ops = {
+	.read_time	= tps65910_rtc_read_time,
+	.set_time	= tps65910_rtc_set_time,
+	.read_alarm	= tps65910_rtc_read_alarm,
+	.set_alarm	= tps65910_rtc_set_alarm,
+	.alarm_irq_enable = tps65910_rtc_alarm_irq_enable,
+};
+
+static int __devinit tps65910_rtc_probe(struct platform_device *pdev)
+{
+	struct tps65910 *tps65910 = NULL;
+	struct tps65910_rtc *tps_rtc = NULL;
+	int ret;
+	int irq;
+	u32 rtc_reg;
+
+	tps65910 = dev_get_drvdata(pdev->dev.parent);
+
+	tps_rtc = devm_kzalloc(&pdev->dev, sizeof(struct tps65910_rtc),
+			GFP_KERNEL);
+	if (!tps_rtc)
+		return -ENOMEM;
+
+	/* Clear pending interrupts */
+	ret = regmap_read(tps65910->regmap, TPS65910_RTC_STATUS, &rtc_reg);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_write(tps65910->regmap, TPS65910_RTC_STATUS, rtc_reg);
+	if (ret < 0)
+		return ret;
+
+	dev_dbg(&pdev->dev, "Enabling rtc-tps65910.\n");
+	rtc_reg = TPS65910_RTC_CTRL_STOP_RTC;
+	ret = regmap_write(tps65910->regmap, TPS65910_RTC_CTRL, rtc_reg);
+	if (ret < 0)
+		return ret;
+
+	irq  = platform_get_irq(pdev, 0);
+	if (irq <= 0) {
+		dev_warn(&pdev->dev, "Wake up is not possible as irq = %d\n",
+			irq);
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
+		tps65910_rtc_interrupt, IRQF_TRIGGER_LOW,
+		"rtc-tps65910", &pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "IRQ is not free.\n");
+		return ret;
+	}
+	device_init_wakeup(&pdev->dev, 1);
+
+	tps_rtc->rtc = rtc_device_register(pdev->name, &pdev->dev,
+		&tps65910_rtc_ops, THIS_MODULE);
+	if (IS_ERR(tps_rtc->rtc)) {
+		ret = PTR_ERR(tps_rtc->rtc);
+		dev_err(&pdev->dev, "RTC device register: err %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, tps_rtc);
+
+	return 0;
+}
+
+/*
+ * Disable tps65910 RTC interrupts.
+ * Sets status flag to free.
+ */
+static int __devexit tps65910_rtc_remove(struct platform_device *pdev)
+{
+	/* leave rtc running, but disable irqs */
+	struct rtc_device *rtc = platform_get_drvdata(pdev);
+
+	tps65910_rtc_alarm_irq_enable(&rtc->dev, 0);
+
+	rtc_device_unregister(rtc);
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+
+static int tps65910_rtc_suspend(struct device *dev)
+{
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+	u8 alarm = TPS65910_RTC_INTERRUPTS_IT_ALARM;
+	int ret;
+
+	/* Store current list of enabled interrupts*/
+	ret = regmap_read(tps->regmap, TPS65910_RTC_INTERRUPTS,
+		&tps->rtc->irqstat);
+	if (ret < 0)
+		return ret;
+
+	/* Enable RTC ALARM interrupt only */
+	return regmap_write(tps->regmap, TPS65910_RTC_INTERRUPTS, alarm);
+}
+
+static int tps65910_rtc_resume(struct device *dev)
+{
+	struct tps65910 *tps = dev_get_drvdata(dev->parent);
+
+	/* Restore list of enabled interrupts before suspend */
+	return regmap_write(tps->regmap, TPS65910_RTC_INTERRUPTS,
+		tps->rtc->irqstat);
+}
+
+static const struct dev_pm_ops tps65910_rtc_pm_ops = {
+	.suspend	= tps65910_rtc_suspend,
+	.resume		= tps65910_rtc_resume,
+};
+
+#define DEV_PM_OPS     (&tps65910_rtc_pm_ops)
+#else
+#define DEV_PM_OPS     NULL
+#endif
+
+static struct platform_driver tps65910_rtc_driver = {
+	.probe		= tps65910_rtc_probe,
+	.remove		= __devexit_p(tps65910_rtc_remove),
+	.driver		= {
+		.owner	= THIS_MODULE,
+		.name	= "tps65910-rtc",
+		.pm	= DEV_PM_OPS,
+	},
+};
+
+module_platform_driver(tps65910_rtc_driver);
+MODULE_ALIAS("platform:rtc-tps65910");
+MODULE_AUTHOR("Venu Byravarasu <vbyravarasu@nvidia.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rtc/rtc-x1205.c b/drivers/rtc/rtc-x1205.c
index 403b3d41d10..f36e59c6bc0 100644
--- a/drivers/rtc/rtc-x1205.c
+++ b/drivers/rtc/rtc-x1205.c
@@ -97,8 +97,17 @@ static int x1205_get_datetime(struct i2c_client *client, struct rtc_time *tm,
 	int i;
 
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 2, dt_addr },	/* setup read ptr */
-		{ client->addr, I2C_M_RD, 8, buf },	/* read date */
+		{/* setup read ptr */
+			.addr = client->addr,
+			.len = 2,
+			.buf = dt_addr
+		},
+		{/* read date */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 8,
+			.buf = buf
+		},
 	};
 
 	/* read date registers */
@@ -142,8 +151,17 @@ static int x1205_get_status(struct i2c_client *client, unsigned char *sr)
 	static unsigned char sr_addr[2] = { 0, X1205_REG_SR };
 
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 2, sr_addr },	/* setup read ptr */
-		{ client->addr, I2C_M_RD, 1, sr },	/* read status */
+		{     /* setup read ptr */
+			.addr = client->addr,
+			.len = 2,
+			.buf = sr_addr
+		},
+		{    /* read status */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = sr
+		},
 	};
 
 	/* read status register */
@@ -279,8 +297,17 @@ static int x1205_get_dtrim(struct i2c_client *client, int *trim)
 	static unsigned char dtr_addr[2] = { 0, X1205_REG_DTR };
 
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 2, dtr_addr },	/* setup read ptr */
-		{ client->addr, I2C_M_RD, 1, &dtr }, 	/* read dtr */
+		{	/* setup read ptr */
+			.addr = client->addr,
+			.len = 2,
+			.buf = dtr_addr
+		},
+		{      /* read dtr */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = &dtr
+		},
 	};
 
 	/* read dtr register */
@@ -311,8 +338,17 @@ static int x1205_get_atrim(struct i2c_client *client, int *trim)
 	static unsigned char atr_addr[2] = { 0, X1205_REG_ATR };
 
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 2, atr_addr },	/* setup read ptr */
-		{ client->addr, I2C_M_RD, 1, &atr }, 	/* read atr */
+		{/* setup read ptr */
+			.addr = client->addr,
+			.len = 2,
+			.buf = atr_addr
+		},
+		{/* read atr */
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = &atr
+		},
 	};
 
 	/* read atr register */
@@ -381,8 +417,17 @@ static int x1205_validate_client(struct i2c_client *client)
 		unsigned char addr[2] = { 0, probe_zero_pattern[i] };
 
 		struct i2c_msg msgs[2] = {
-			{ client->addr, 0, 2, addr },
-			{ client->addr, I2C_M_RD, 1, &buf },
+			{
+				.addr = client->addr,
+				.len = 2,
+				.buf = addr
+			},
+			{
+				.addr = client->addr,
+				.flags = I2C_M_RD,
+				.len = 1,
+				.buf = &buf
+			},
 		};
 
 		if ((xfer = i2c_transfer(client->adapter, msgs, 2)) != 2) {
@@ -409,8 +454,17 @@ static int x1205_validate_client(struct i2c_client *client)
 		unsigned char addr[2] = { 0, probe_limits_pattern[i].reg };
 
 		struct i2c_msg msgs[2] = {
-			{ client->addr, 0, 2, addr },
-			{ client->addr, I2C_M_RD, 1, &reg },
+			{
+				.addr = client->addr,
+				.len = 2,
+				.buf = addr
+			},
+			{
+				.addr = client->addr,
+				.flags = I2C_M_RD,
+				.len = 1,
+				.buf = &reg
+			},
 		};
 
 		if ((xfer = i2c_transfer(client->adapter, msgs, 2)) != 2) {
@@ -444,8 +498,18 @@ static int x1205_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	static unsigned char int_addr[2] = { 0, X1205_REG_INT };
 	struct i2c_client *client = to_i2c_client(dev);
 	struct i2c_msg msgs[] = {
-		{ client->addr, 0, 2, int_addr },        /* setup read ptr */
-		{ client->addr, I2C_M_RD, 1, &intreg },  /* read INT register */
+		{ /* setup read ptr */
+			.addr = client->addr,
+			.len = 2,
+			.buf = int_addr
+		},
+		{/* read INT register */
+
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = 1,
+			.buf = &intreg
+		},
 	};
 
 	/* read interrupt register and status register */
diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c
index aff8621de80..f6adde44f22 100644
--- a/drivers/s390/scsi/zfcp_aux.c
+++ b/drivers/s390/scsi/zfcp_aux.c
@@ -519,6 +519,7 @@ struct zfcp_port *zfcp_port_enqueue(struct zfcp_adapter *adapter, u64 wwpn,
 
 	rwlock_init(&port->unit_list_lock);
 	INIT_LIST_HEAD(&port->unit_list);
+	atomic_set(&port->units, 0);
 
 	INIT_WORK(&port->gid_pn_work, zfcp_fc_port_did_lookup);
 	INIT_WORK(&port->test_link_work, zfcp_fc_link_test_work);
diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c
index e37f0455194..f2dd3a0a39e 100644
--- a/drivers/s390/scsi/zfcp_ccw.c
+++ b/drivers/s390/scsi/zfcp_ccw.c
@@ -39,19 +39,25 @@ void zfcp_ccw_adapter_put(struct zfcp_adapter *adapter)
 	spin_unlock_irqrestore(&zfcp_ccw_adapter_ref_lock, flags);
 }
 
-static int zfcp_ccw_activate(struct ccw_device *cdev)
-
+/**
+ * zfcp_ccw_activate - activate adapter and wait for it to finish
+ * @cdev: pointer to belonging ccw device
+ * @clear: Status flags to clear.
+ * @tag: s390dbf trace record tag
+ */
+static int zfcp_ccw_activate(struct ccw_device *cdev, int clear, char *tag)
 {
 	struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev);
 
 	if (!adapter)
 		return 0;
 
+	zfcp_erp_clear_adapter_status(adapter, clear);
 	zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING);
 	zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED,
-				"ccresu2");
+				tag);
 	zfcp_erp_wait(adapter);
-	flush_work(&adapter->scan_work);
+	flush_work(&adapter->scan_work); /* ok to call even if nothing queued */
 
 	zfcp_ccw_adapter_put(adapter);
 
@@ -164,26 +170,34 @@ static int zfcp_ccw_set_online(struct ccw_device *cdev)
 	BUG_ON(!zfcp_reqlist_isempty(adapter->req_list));
 	adapter->req_no = 0;
 
-	zfcp_ccw_activate(cdev);
+	zfcp_ccw_activate(cdev, 0, "ccsonl1");
+	/* scan for remote ports
+	   either at the end of any successful adapter recovery
+	   or only after the adapter recovery for setting a device online */
+	zfcp_fc_inverse_conditional_port_scan(adapter);
+	flush_work(&adapter->scan_work); /* ok to call even if nothing queued */
 	zfcp_ccw_adapter_put(adapter);
 	return 0;
 }
 
 /**
- * zfcp_ccw_set_offline - set_offline function of zfcp driver
+ * zfcp_ccw_offline_sync - shut down adapter and wait for it to finish
  * @cdev: pointer to belonging ccw device
+ * @set: Status flags to set.
+ * @tag: s390dbf trace record tag
  *
  * This function gets called by the common i/o layer and sets an adapter
  * into state offline.
  */
-static int zfcp_ccw_set_offline(struct ccw_device *cdev)
+static int zfcp_ccw_offline_sync(struct ccw_device *cdev, int set, char *tag)
 {
 	struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev);
 
 	if (!adapter)
 		return 0;
 
-	zfcp_erp_adapter_shutdown(adapter, 0, "ccsoff1");
+	zfcp_erp_set_adapter_status(adapter, set);
+	zfcp_erp_adapter_shutdown(adapter, 0, tag);
 	zfcp_erp_wait(adapter);
 
 	zfcp_ccw_adapter_put(adapter);
@@ -191,6 +205,18 @@ static int zfcp_ccw_set_offline(struct ccw_device *cdev)
 }
 
 /**
+ * zfcp_ccw_set_offline - set_offline function of zfcp driver
+ * @cdev: pointer to belonging ccw device
+ *
+ * This function gets called by the common i/o layer and sets an adapter
+ * into state offline.
+ */
+static int zfcp_ccw_set_offline(struct ccw_device *cdev)
+{
+	return zfcp_ccw_offline_sync(cdev, 0, "ccsoff1");
+}
+
+/**
  * zfcp_ccw_notify - ccw notify function
  * @cdev: pointer to belonging ccw device
  * @event: indicates if adapter was detached or attached
@@ -207,6 +233,11 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event)
 
 	switch (event) {
 	case CIO_GONE:
+		if (atomic_read(&adapter->status) &
+		    ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */
+			zfcp_dbf_hba_basic("ccnigo1", adapter);
+			break;
+		}
 		dev_warn(&cdev->dev, "The FCP device has been detached\n");
 		zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti1");
 		break;
@@ -216,6 +247,11 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event)
 		zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti2");
 		break;
 	case CIO_OPER:
+		if (atomic_read(&adapter->status) &
+		    ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */
+			zfcp_dbf_hba_basic("ccniop1", adapter);
+			break;
+		}
 		dev_info(&cdev->dev, "The FCP device is operational again\n");
 		zfcp_erp_set_adapter_status(adapter,
 					    ZFCP_STATUS_COMMON_RUNNING);
@@ -251,6 +287,28 @@ static void zfcp_ccw_shutdown(struct ccw_device *cdev)
 	zfcp_ccw_adapter_put(adapter);
 }
 
+static int zfcp_ccw_suspend(struct ccw_device *cdev)
+{
+	zfcp_ccw_offline_sync(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccsusp1");
+	return 0;
+}
+
+static int zfcp_ccw_thaw(struct ccw_device *cdev)
+{
+	/* trace records for thaw and final shutdown during suspend
+	   can only be found in system dump until the end of suspend
+	   but not after resume because it's based on the memory image
+	   right after the very first suspend (freeze) callback */
+	zfcp_ccw_activate(cdev, 0, "ccthaw1");
+	return 0;
+}
+
+static int zfcp_ccw_resume(struct ccw_device *cdev)
+{
+	zfcp_ccw_activate(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccresu1");
+	return 0;
+}
+
 struct ccw_driver zfcp_ccw_driver = {
 	.driver = {
 		.owner	= THIS_MODULE,
@@ -263,7 +321,7 @@ struct ccw_driver zfcp_ccw_driver = {
 	.set_offline = zfcp_ccw_set_offline,
 	.notify      = zfcp_ccw_notify,
 	.shutdown    = zfcp_ccw_shutdown,
-	.freeze      = zfcp_ccw_set_offline,
-	.thaw	     = zfcp_ccw_activate,
-	.restore     = zfcp_ccw_activate,
+	.freeze      = zfcp_ccw_suspend,
+	.thaw	     = zfcp_ccw_thaw,
+	.restore     = zfcp_ccw_resume,
 };
diff --git a/drivers/s390/scsi/zfcp_cfdc.c b/drivers/s390/scsi/zfcp_cfdc.c
index fbd8b4db602..49b82e46629 100644
--- a/drivers/s390/scsi/zfcp_cfdc.c
+++ b/drivers/s390/scsi/zfcp_cfdc.c
@@ -293,7 +293,7 @@ void zfcp_cfdc_adapter_access_changed(struct zfcp_adapter *adapter)
 	}
 	read_unlock_irqrestore(&adapter->port_list_lock, flags);
 
-	shost_for_each_device(sdev, port->adapter->scsi_host) {
+	shost_for_each_device(sdev, adapter->scsi_host) {
 		zfcp_sdev = sdev_to_zfcp(sdev);
 		status = atomic_read(&zfcp_sdev->status);
 		if ((status & ZFCP_STATUS_COMMON_ACCESS_DENIED) ||
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index 3c1d22097ad..e1a8cc2526e 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -191,7 +191,7 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount,
 	length = min((u16)sizeof(struct qdio_buffer),
 		     (u16)ZFCP_DBF_PAY_MAX_REC);
 
-	while ((char *)pl[payload->counter] && payload->counter < scount) {
+	while (payload->counter < scount && (char *)pl[payload->counter]) {
 		memcpy(payload->data, (char *)pl[payload->counter], length);
 		debug_event(dbf->pay, 1, payload, zfcp_dbf_plen(length));
 		payload->counter++;
@@ -200,6 +200,26 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount,
 	spin_unlock_irqrestore(&dbf->pay_lock, flags);
 }
 
+/**
+ * zfcp_dbf_hba_basic - trace event for basic adapter events
+ * @adapter: pointer to struct zfcp_adapter
+ */
+void zfcp_dbf_hba_basic(char *tag, struct zfcp_adapter *adapter)
+{
+	struct zfcp_dbf *dbf = adapter->dbf;
+	struct zfcp_dbf_hba *rec = &dbf->hba_buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dbf->hba_lock, flags);
+	memset(rec, 0, sizeof(*rec));
+
+	memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN);
+	rec->id = ZFCP_DBF_HBA_BASIC;
+
+	debug_event(dbf->hba, 1, rec, sizeof(*rec));
+	spin_unlock_irqrestore(&dbf->hba_lock, flags);
+}
+
 static void zfcp_dbf_set_common(struct zfcp_dbf_rec *rec,
 				struct zfcp_adapter *adapter,
 				struct zfcp_port *port,
diff --git a/drivers/s390/scsi/zfcp_dbf.h b/drivers/s390/scsi/zfcp_dbf.h
index 714f087eb7a..3ac7a4b30dd 100644
--- a/drivers/s390/scsi/zfcp_dbf.h
+++ b/drivers/s390/scsi/zfcp_dbf.h
@@ -154,6 +154,7 @@ enum zfcp_dbf_hba_id {
 	ZFCP_DBF_HBA_RES	= 1,
 	ZFCP_DBF_HBA_USS	= 2,
 	ZFCP_DBF_HBA_BIT	= 3,
+	ZFCP_DBF_HBA_BASIC	= 4,
 };
 
 /**
diff --git a/drivers/s390/scsi/zfcp_def.h b/drivers/s390/scsi/zfcp_def.h
index 2955e1a3dea..1305955cbf5 100644
--- a/drivers/s390/scsi/zfcp_def.h
+++ b/drivers/s390/scsi/zfcp_def.h
@@ -77,6 +77,7 @@ struct zfcp_reqlist;
 #define ZFCP_STATUS_ADAPTER_SIOSL_ISSUED	0x00000004
 #define ZFCP_STATUS_ADAPTER_XCONFIG_OK		0x00000008
 #define ZFCP_STATUS_ADAPTER_HOST_CON_INIT	0x00000010
+#define ZFCP_STATUS_ADAPTER_SUSPENDED		0x00000040
 #define ZFCP_STATUS_ADAPTER_ERP_PENDING		0x00000100
 #define ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED	0x00000200
 #define ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED	0x00000400
@@ -204,6 +205,7 @@ struct zfcp_port {
 	struct zfcp_adapter    *adapter;       /* adapter used to access port */
 	struct list_head	unit_list;	/* head of logical unit list */
 	rwlock_t		unit_list_lock; /* unit list lock */
+	atomic_t		units;	       /* zfcp_unit count */
 	atomic_t	       status;	       /* status of this remote port */
 	u64		       wwnn;	       /* WWNN if known */
 	u64		       wwpn;	       /* WWPN */
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 92d3df6ac8b..4133ab6e20f 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -1230,7 +1230,7 @@ static void zfcp_erp_action_cleanup(struct zfcp_erp_action *act, int result)
 	case ZFCP_ERP_ACTION_REOPEN_ADAPTER:
 		if (result == ZFCP_ERP_SUCCEEDED) {
 			register_service_level(&adapter->service_level);
-			queue_work(adapter->work_queue, &adapter->scan_work);
+			zfcp_fc_conditional_port_scan(adapter);
 			queue_work(adapter->work_queue, &adapter->ns_up_work);
 		} else
 			unregister_service_level(&adapter->service_level);
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 36f422770ff..1d3dd3f7d69 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -54,6 +54,7 @@ extern void zfcp_dbf_hba_fsf_res(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_berr(struct zfcp_dbf *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **);
+extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *);
 extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32);
 extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *);
 extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *);
@@ -98,6 +99,8 @@ extern void zfcp_fc_gs_destroy(struct zfcp_adapter *);
 extern int zfcp_fc_exec_bsg_job(struct fc_bsg_job *);
 extern int zfcp_fc_timeout_bsg_job(struct fc_bsg_job *);
 extern void zfcp_fc_sym_name_update(struct work_struct *);
+extern void zfcp_fc_conditional_port_scan(struct zfcp_adapter *);
+extern void zfcp_fc_inverse_conditional_port_scan(struct zfcp_adapter *);
 
 /* zfcp_fsf.c */
 extern struct kmem_cache *zfcp_fsf_qtcb_cache;
@@ -158,6 +161,7 @@ extern void zfcp_scsi_dif_sense_error(struct scsi_cmnd *, int);
 extern struct attribute_group zfcp_sysfs_unit_attrs;
 extern struct attribute_group zfcp_sysfs_adapter_attrs;
 extern struct attribute_group zfcp_sysfs_port_attrs;
+extern struct mutex zfcp_sysfs_port_units_mutex;
 extern struct device_attribute *zfcp_sysfs_sdev_attrs[];
 extern struct device_attribute *zfcp_sysfs_shost_attrs[];
 
diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c
index 88688a80b2c..ff598cd68b2 100644
--- a/drivers/s390/scsi/zfcp_fc.c
+++ b/drivers/s390/scsi/zfcp_fc.c
@@ -26,6 +26,27 @@ static u32 zfcp_fc_rscn_range_mask[] = {
 	[ELS_ADDR_FMT_FAB]		= 0x000000,
 };
 
+static bool no_auto_port_rescan;
+module_param_named(no_auto_port_rescan, no_auto_port_rescan, bool, 0600);
+MODULE_PARM_DESC(no_auto_port_rescan,
+		 "no automatic port_rescan (default off)");
+
+void zfcp_fc_conditional_port_scan(struct zfcp_adapter *adapter)
+{
+	if (no_auto_port_rescan)
+		return;
+
+	queue_work(adapter->work_queue, &adapter->scan_work);
+}
+
+void zfcp_fc_inverse_conditional_port_scan(struct zfcp_adapter *adapter)
+{
+	if (!no_auto_port_rescan)
+		return;
+
+	queue_work(adapter->work_queue, &adapter->scan_work);
+}
+
 /**
  * zfcp_fc_post_event - post event to userspace via fc_transport
  * @work: work struct with enqueued events
@@ -206,7 +227,7 @@ static void zfcp_fc_incoming_rscn(struct zfcp_fsf_req *fsf_req)
 		zfcp_fc_enqueue_event(fsf_req->adapter, FCH_EVT_RSCN,
 				      *(u32 *)page);
 	}
-	queue_work(fsf_req->adapter->work_queue, &fsf_req->adapter->scan_work);
+	zfcp_fc_conditional_port_scan(fsf_req->adapter);
 }
 
 static void zfcp_fc_incoming_wwpn(struct zfcp_fsf_req *req, u64 wwpn)
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index e1c1efc2c5a..c96320d79fb 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -219,7 +219,7 @@ static void zfcp_fsf_status_read_handler(struct zfcp_fsf_req *req)
 		return;
 	}
 
-	zfcp_dbf_hba_fsf_uss("fssrh_2", req);
+	zfcp_dbf_hba_fsf_uss("fssrh_4", req);
 
 	switch (sr_buf->status_type) {
 	case FSF_STATUS_READ_PORT_CLOSED:
@@ -257,7 +257,7 @@ static void zfcp_fsf_status_read_handler(struct zfcp_fsf_req *req)
 		if (sr_buf->status_subtype & FSF_STATUS_READ_SUB_ACT_UPDATED)
 			zfcp_cfdc_adapter_access_changed(adapter);
 		if (sr_buf->status_subtype & FSF_STATUS_READ_SUB_INCOMING_ELS)
-			queue_work(adapter->work_queue, &adapter->scan_work);
+			zfcp_fc_conditional_port_scan(adapter);
 		break;
 	case FSF_STATUS_READ_CFDC_UPDATED:
 		zfcp_cfdc_adapter_access_changed(adapter);
@@ -437,6 +437,34 @@ void zfcp_fsf_req_dismiss_all(struct zfcp_adapter *adapter)
 	}
 }
 
+#define ZFCP_FSF_PORTSPEED_1GBIT	(1 <<  0)
+#define ZFCP_FSF_PORTSPEED_2GBIT	(1 <<  1)
+#define ZFCP_FSF_PORTSPEED_4GBIT	(1 <<  2)
+#define ZFCP_FSF_PORTSPEED_10GBIT	(1 <<  3)
+#define ZFCP_FSF_PORTSPEED_8GBIT	(1 <<  4)
+#define ZFCP_FSF_PORTSPEED_16GBIT	(1 <<  5)
+#define ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED (1 << 15)
+
+static u32 zfcp_fsf_convert_portspeed(u32 fsf_speed)
+{
+	u32 fdmi_speed = 0;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_1GBIT)
+		fdmi_speed |= FC_PORTSPEED_1GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_2GBIT)
+		fdmi_speed |= FC_PORTSPEED_2GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_4GBIT)
+		fdmi_speed |= FC_PORTSPEED_4GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_10GBIT)
+		fdmi_speed |= FC_PORTSPEED_10GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_8GBIT)
+		fdmi_speed |= FC_PORTSPEED_8GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_16GBIT)
+		fdmi_speed |= FC_PORTSPEED_16GBIT;
+	if (fsf_speed & ZFCP_FSF_PORTSPEED_NOT_NEGOTIATED)
+		fdmi_speed |= FC_PORTSPEED_NOT_NEGOTIATED;
+	return fdmi_speed;
+}
+
 static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 {
 	struct fsf_qtcb_bottom_config *bottom = &req->qtcb->bottom.config;
@@ -456,7 +484,8 @@ static int zfcp_fsf_exchange_config_evaluate(struct zfcp_fsf_req *req)
 	fc_host_port_name(shost) = nsp->fl_wwpn;
 	fc_host_node_name(shost) = nsp->fl_wwnn;
 	fc_host_port_id(shost) = ntoh24(bottom->s_id);
-	fc_host_speed(shost) = bottom->fc_link_speed;
+	fc_host_speed(shost) =
+		zfcp_fsf_convert_portspeed(bottom->fc_link_speed);
 	fc_host_supported_classes(shost) = FC_COS_CLASS2 | FC_COS_CLASS3;
 
 	adapter->hydra_version = bottom->adapter_type;
@@ -580,7 +609,8 @@ static void zfcp_fsf_exchange_port_evaluate(struct zfcp_fsf_req *req)
 	} else
 		fc_host_permanent_port_name(shost) = fc_host_port_name(shost);
 	fc_host_maxframe_size(shost) = bottom->maximum_frame_size;
-	fc_host_supported_speeds(shost) = bottom->supported_speed;
+	fc_host_supported_speeds(shost) =
+		zfcp_fsf_convert_portspeed(bottom->supported_speed);
 	memcpy(fc_host_supported_fc4s(shost), bottom->supported_fc4_types,
 	       FC_FC4_LIST_SIZE);
 	memcpy(fc_host_active_fc4s(shost), bottom->active_fc4_types,
@@ -771,12 +801,14 @@ out:
 static void zfcp_fsf_abort_fcp_command_handler(struct zfcp_fsf_req *req)
 {
 	struct scsi_device *sdev = req->data;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
+	struct zfcp_scsi_dev *zfcp_sdev;
 	union fsf_status_qual *fsq = &req->qtcb->header.fsf_status_qual;
 
 	if (req->status & ZFCP_STATUS_FSFREQ_ERROR)
 		return;
 
+	zfcp_sdev = sdev_to_zfcp(sdev);
+
 	switch (req->qtcb->header.fsf_status) {
 	case FSF_PORT_HANDLE_NOT_VALID:
 		if (fsq->word[0] == fsq->word[1]) {
@@ -885,7 +917,7 @@ static void zfcp_fsf_send_ct_handler(struct zfcp_fsf_req *req)
 
 	switch (header->fsf_status) {
         case FSF_GOOD:
-		zfcp_dbf_san_res("fsscth1", req);
+		zfcp_dbf_san_res("fsscth2", req);
 		ct->status = 0;
 		break;
         case FSF_SERVICE_CLASS_NOT_SUPPORTED:
@@ -1739,13 +1771,15 @@ static void zfcp_fsf_open_lun_handler(struct zfcp_fsf_req *req)
 {
 	struct zfcp_adapter *adapter = req->adapter;
 	struct scsi_device *sdev = req->data;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
+	struct zfcp_scsi_dev *zfcp_sdev;
 	struct fsf_qtcb_header *header = &req->qtcb->header;
 	struct fsf_qtcb_bottom_support *bottom = &req->qtcb->bottom.support;
 
 	if (req->status & ZFCP_STATUS_FSFREQ_ERROR)
 		return;
 
+	zfcp_sdev = sdev_to_zfcp(sdev);
+
 	atomic_clear_mask(ZFCP_STATUS_COMMON_ACCESS_DENIED |
 			  ZFCP_STATUS_COMMON_ACCESS_BOXED |
 			  ZFCP_STATUS_LUN_SHARED |
@@ -1856,11 +1890,13 @@ out:
 static void zfcp_fsf_close_lun_handler(struct zfcp_fsf_req *req)
 {
 	struct scsi_device *sdev = req->data;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
+	struct zfcp_scsi_dev *zfcp_sdev;
 
 	if (req->status & ZFCP_STATUS_FSFREQ_ERROR)
 		return;
 
+	zfcp_sdev = sdev_to_zfcp(sdev);
+
 	switch (req->qtcb->header.fsf_status) {
 	case FSF_PORT_HANDLE_NOT_VALID:
 		zfcp_erp_adapter_reopen(zfcp_sdev->port->adapter, 0, "fscuh_1");
@@ -1950,7 +1986,7 @@ static void zfcp_fsf_req_trace(struct zfcp_fsf_req *req, struct scsi_cmnd *scsi)
 {
 	struct fsf_qual_latency_info *lat_in;
 	struct latency_cont *lat = NULL;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(scsi->device);
+	struct zfcp_scsi_dev *zfcp_sdev;
 	struct zfcp_blk_drv_data blktrc;
 	int ticks = req->adapter->timer_ticks;
 
@@ -1965,6 +2001,7 @@ static void zfcp_fsf_req_trace(struct zfcp_fsf_req *req, struct scsi_cmnd *scsi)
 
 	if (req->adapter->adapter_features & FSF_FEATURE_MEASUREMENT_DATA &&
 	    !(req->status & ZFCP_STATUS_FSFREQ_ERROR)) {
+		zfcp_sdev = sdev_to_zfcp(scsi->device);
 		blktrc.flags |= ZFCP_BLK_LAT_VALID;
 		blktrc.channel_lat = lat_in->channel_lat * ticks;
 		blktrc.fabric_lat = lat_in->fabric_lat * ticks;
@@ -2002,12 +2039,14 @@ static void zfcp_fsf_fcp_handler_common(struct zfcp_fsf_req *req)
 {
 	struct scsi_cmnd *scmnd = req->data;
 	struct scsi_device *sdev = scmnd->device;
-	struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev);
+	struct zfcp_scsi_dev *zfcp_sdev;
 	struct fsf_qtcb_header *header = &req->qtcb->header;
 
 	if (unlikely(req->status & ZFCP_STATUS_FSFREQ_ERROR))
 		return;
 
+	zfcp_sdev = sdev_to_zfcp(sdev);
+
 	switch (header->fsf_status) {
 	case FSF_HANDLE_MISMATCH:
 	case FSF_PORT_HANDLE_NOT_VALID:
diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c
index b9fffc8d94a..50b5615848f 100644
--- a/drivers/s390/scsi/zfcp_qdio.c
+++ b/drivers/s390/scsi/zfcp_qdio.c
@@ -102,18 +102,22 @@ static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err,
 {
 	struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
 	struct zfcp_adapter *adapter = qdio->adapter;
-	struct qdio_buffer_element *sbale;
 	int sbal_no, sbal_idx;
-	void *pl[ZFCP_QDIO_MAX_SBALS_PER_REQ + 1];
-	u64 req_id;
-	u8 scount;
 
 	if (unlikely(qdio_err)) {
-		memset(pl, 0, ZFCP_QDIO_MAX_SBALS_PER_REQ * sizeof(void *));
 		if (zfcp_adapter_multi_buffer_active(adapter)) {
+			void *pl[ZFCP_QDIO_MAX_SBALS_PER_REQ + 1];
+			struct qdio_buffer_element *sbale;
+			u64 req_id;
+			u8 scount;
+
+			memset(pl, 0,
+			       ZFCP_QDIO_MAX_SBALS_PER_REQ * sizeof(void *));
 			sbale = qdio->res_q[idx]->element;
 			req_id = (u64) sbale->addr;
-			scount = sbale->scount + 1; /* incl. signaling SBAL */
+			scount = min(sbale->scount + 1,
+				     ZFCP_QDIO_MAX_SBALS_PER_REQ + 1);
+				     /* incl. signaling SBAL */
 
 			for (sbal_no = 0; sbal_no < scount; sbal_no++) {
 				sbal_idx = (idx + sbal_no) %
diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c
index c66af27b230..1e0eb089dfb 100644
--- a/drivers/s390/scsi/zfcp_sysfs.c
+++ b/drivers/s390/scsi/zfcp_sysfs.c
@@ -227,6 +227,8 @@ static ssize_t zfcp_sysfs_port_rescan_store(struct device *dev,
 static ZFCP_DEV_ATTR(adapter, port_rescan, S_IWUSR, NULL,
 		     zfcp_sysfs_port_rescan_store);
 
+DEFINE_MUTEX(zfcp_sysfs_port_units_mutex);
+
 static ssize_t zfcp_sysfs_port_remove_store(struct device *dev,
 					    struct device_attribute *attr,
 					    const char *buf, size_t count)
@@ -249,6 +251,16 @@ static ssize_t zfcp_sysfs_port_remove_store(struct device *dev,
 	else
 		retval = 0;
 
+	mutex_lock(&zfcp_sysfs_port_units_mutex);
+	if (atomic_read(&port->units) > 0) {
+		retval = -EBUSY;
+		mutex_unlock(&zfcp_sysfs_port_units_mutex);
+		goto out;
+	}
+	/* port is about to be removed, so no more unit_add */
+	atomic_set(&port->units, -1);
+	mutex_unlock(&zfcp_sysfs_port_units_mutex);
+
 	write_lock_irq(&adapter->port_list_lock);
 	list_del(&port->list);
 	write_unlock_irq(&adapter->port_list_lock);
@@ -289,12 +301,14 @@ static ssize_t zfcp_sysfs_unit_add_store(struct device *dev,
 {
 	struct zfcp_port *port = container_of(dev, struct zfcp_port, dev);
 	u64 fcp_lun;
+	int retval;
 
 	if (strict_strtoull(buf, 0, (unsigned long long *) &fcp_lun))
 		return -EINVAL;
 
-	if (zfcp_unit_add(port, fcp_lun))
-		return -EINVAL;
+	retval = zfcp_unit_add(port, fcp_lun);
+	if (retval)
+		return retval;
 
 	return count;
 }
diff --git a/drivers/s390/scsi/zfcp_unit.c b/drivers/s390/scsi/zfcp_unit.c
index 3f2bff0d3aa..1cd2b99ab25 100644
--- a/drivers/s390/scsi/zfcp_unit.c
+++ b/drivers/s390/scsi/zfcp_unit.c
@@ -104,7 +104,7 @@ static void zfcp_unit_release(struct device *dev)
 {
 	struct zfcp_unit *unit = container_of(dev, struct zfcp_unit, dev);
 
-	put_device(&unit->port->dev);
+	atomic_dec(&unit->port->units);
 	kfree(unit);
 }
 
@@ -119,16 +119,27 @@ static void zfcp_unit_release(struct device *dev)
 int zfcp_unit_add(struct zfcp_port *port, u64 fcp_lun)
 {
 	struct zfcp_unit *unit;
+	int retval = 0;
+
+	mutex_lock(&zfcp_sysfs_port_units_mutex);
+	if (atomic_read(&port->units) == -1) {
+		/* port is already gone */
+		retval = -ENODEV;
+		goto out;
+	}
 
 	unit = zfcp_unit_find(port, fcp_lun);
 	if (unit) {
 		put_device(&unit->dev);
-		return -EEXIST;
+		retval = -EEXIST;
+		goto out;
 	}
 
 	unit = kzalloc(sizeof(struct zfcp_unit), GFP_KERNEL);
-	if (!unit)
-		return -ENOMEM;
+	if (!unit) {
+		retval = -ENOMEM;
+		goto out;
+	}
 
 	unit->port = port;
 	unit->fcp_lun = fcp_lun;
@@ -139,28 +150,33 @@ int zfcp_unit_add(struct zfcp_port *port, u64 fcp_lun)
 	if (dev_set_name(&unit->dev, "0x%016llx",
 			 (unsigned long long) fcp_lun)) {
 		kfree(unit);
-		return -ENOMEM;
+		retval = -ENOMEM;
+		goto out;
 	}
 
-	get_device(&port->dev);
-
 	if (device_register(&unit->dev)) {
 		put_device(&unit->dev);
-		return -ENOMEM;
+		retval = -ENOMEM;
+		goto out;
 	}
 
 	if (sysfs_create_group(&unit->dev.kobj, &zfcp_sysfs_unit_attrs)) {
 		device_unregister(&unit->dev);
-		return -EINVAL;
+		retval = -EINVAL;
+		goto out;
 	}
 
+	atomic_inc(&port->units); /* under zfcp_sysfs_port_units_mutex ! */
+
 	write_lock_irq(&port->unit_list_lock);
 	list_add_tail(&unit->list, &port->unit_list);
 	write_unlock_irq(&port->unit_list_lock);
 
 	zfcp_unit_scsi_scan(unit);
 
-	return 0;
+out:
+	mutex_unlock(&zfcp_sysfs_port_units_mutex);
+	return retval;
 }
 
 /**
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index 7199534cd07..cb7f1582a6d 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -93,7 +93,7 @@ static DECLARE_PCI_DEVICE_TABLE(aac_pci_tbl) = {
 #elif defined(__devinitconst)
 static const struct pci_device_id aac_pci_tbl[] __devinitconst = {
 #else
-static const struct pci_device_id aac_pci_tbl[] __devinitdata = {
+static const struct pci_device_id aac_pci_tbl[] __devinitconst = {
 #endif
 	{ 0x1028, 0x0001, 0x1028, 0x0001, 0, 0, 0 }, /* PERC 2/Si (Iguana/PERC2Si) */
 	{ 0x1028, 0x0002, 0x1028, 0x0002, 0, 0, 1 }, /* PERC 3/Di (Opal/PERC3Di) */
diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c
index ff80552ead8..1c4120c3db4 100644
--- a/drivers/scsi/aic94xx/aic94xx_init.c
+++ b/drivers/scsi/aic94xx/aic94xx_init.c
@@ -1012,7 +1012,7 @@ static struct sas_domain_function_template aic94xx_transport_functions = {
 	.lldd_ata_set_dmamode	= asd_set_dmamode,
 };
 
-static const struct pci_device_id aic94xx_pci_table[] __devinitdata = {
+static const struct pci_device_id aic94xx_pci_table[] __devinitconst = {
 	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, 0x410),0, 0, 1},
 	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, 0x412),0, 0, 1},
 	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, 0x416),0, 0, 1},
diff --git a/drivers/scsi/atp870u.c b/drivers/scsi/atp870u.c
index 68ce08552f6..a540162ac59 100644
--- a/drivers/scsi/atp870u.c
+++ b/drivers/scsi/atp870u.c
@@ -1173,7 +1173,16 @@ wait_io1:
 	outw(val, tmport);
 	outb(2, 0x80);
 TCM_SYNC:
-	udelay(0x800);
+	/*
+	 * The funny division into multiple delays is to accomodate
+	 * arches like ARM where udelay() multiplies its argument by
+	 * a large number to initialize a loop counter.  To avoid
+	 * overflow, the maximum supported udelay is 2000 microseconds.
+	 *
+	 * XXX it would be more polite to find a way to use msleep()
+	 */
+	mdelay(2);
+	udelay(48);
 	if ((inb(tmport) & 0x80) == 0x00) {	/* bsy ? */
 		outw(0, tmport--);
 		outb(0, tmport);
diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c
index d2e9e933f7a..07d2cb126d9 100644
--- a/drivers/scsi/be2iscsi/be_cmds.c
+++ b/drivers/scsi/be2iscsi/be_cmds.c
@@ -48,7 +48,8 @@ int beiscsi_pci_soft_reset(struct beiscsi_hba *phba)
 	}
 
 	if (sreset & BE2_SET_RESET) {
-		printk(KERN_ERR "Soft Reset  did not deassert\n");
+		printk(KERN_ERR DRV_NAME
+		       " Soft Reset  did not deassert\n");
 		return -EIO;
 	}
 	pconline1 = BE2_MPU_IRAM_ONLINE;
@@ -67,7 +68,8 @@ int beiscsi_pci_soft_reset(struct beiscsi_hba *phba)
 		i++;
 	}
 	if (sreset & BE2_SET_RESET) {
-		printk(KERN_ERR "MPU Online Soft Reset did not deassert\n");
+		printk(KERN_ERR DRV_NAME
+		       " MPU Online Soft Reset did not deassert\n");
 		return -EIO;
 	}
 	return 0;
@@ -93,8 +95,9 @@ int be_chk_reset_complete(struct beiscsi_hba *phba)
 	}
 
 	if ((status & 0x80000000) || (!num_loop)) {
-		printk(KERN_ERR "Failed in be_chk_reset_complete"
-		"status = 0x%x\n", status);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BC_%d : Failed in be_chk_reset_complete"
+			    "status = 0x%x\n", status);
 		return -EIO;
 	}
 
@@ -169,6 +172,7 @@ static int be_mcc_compl_process(struct be_ctrl_info *ctrl,
 				struct be_mcc_compl *compl)
 {
 	u16 compl_status, extd_status;
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 
 	be_dws_le_to_cpu(compl, 4);
 
@@ -177,9 +181,12 @@ static int be_mcc_compl_process(struct be_ctrl_info *ctrl,
 	if (compl_status != MCC_STATUS_SUCCESS) {
 		extd_status = (compl->status >> CQE_STATUS_EXTD_SHIFT) &
 						CQE_STATUS_EXTD_MASK;
-		dev_err(&ctrl->pdev->dev,
-			"error in cmd completion: status(compl/extd)=%d/%d\n",
-			compl_status, extd_status);
+
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : error in cmd completion: status(compl/extd)=%d/%d\n",
+			    compl_status, extd_status);
+
 		return -EBUSY;
 	}
 	return 0;
@@ -233,22 +240,29 @@ void beiscsi_async_link_state_process(struct beiscsi_hba *phba,
 {
 	switch (evt->port_link_status) {
 	case ASYNC_EVENT_LINK_DOWN:
-		SE_DEBUG(DBG_LVL_1, "Link Down on Physical Port %d\n",
-				     evt->physical_port);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
+			    "BC_%d : Link Down on Physical Port %d\n",
+			    evt->physical_port);
+
 		phba->state |= BE_ADAPTER_LINK_DOWN;
 		iscsi_host_for_each_session(phba->shost,
 					    be2iscsi_fail_session);
 		break;
 	case ASYNC_EVENT_LINK_UP:
 		phba->state = BE_ADAPTER_UP;
-		SE_DEBUG(DBG_LVL_1, "Link UP on Physical Port %d\n",
-						evt->physical_port);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
+			    "BC_%d : Link UP on Physical Port %d\n",
+			    evt->physical_port);
 		break;
 	default:
-		SE_DEBUG(DBG_LVL_1, "Unexpected Async Notification %d on"
-				    "Physical Port %d\n",
-				     evt->port_link_status,
-				     evt->physical_port);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
+			    "BC_%d : Unexpected Async Notification %d on"
+			    "Physical Port %d\n",
+			    evt->port_link_status,
+			    evt->physical_port);
 	}
 }
 
@@ -279,9 +293,11 @@ int beiscsi_process_mcc(struct beiscsi_hba *phba)
 				beiscsi_async_link_state_process(phba,
 				   (struct be_async_event_link_state *) compl);
 			else
-				SE_DEBUG(DBG_LVL_1,
-					 " Unsupported Async Event, flags"
-					 " = 0x%08x\n", compl->flags);
+				beiscsi_log(phba, KERN_ERR,
+					    BEISCSI_LOG_CONFIG |
+					    BEISCSI_LOG_MBOX,
+					    "BC_%d : Unsupported Async Event, flags"
+					    " = 0x%08x\n", compl->flags);
 
 		} else if (compl->flags & CQE_FLAGS_COMPLETED_MASK) {
 				status = be_mcc_compl_process(ctrl, compl);
@@ -312,7 +328,10 @@ static int be_mcc_wait_compl(struct beiscsi_hba *phba)
 		udelay(100);
 	}
 	if (i == mcc_timeout) {
-		dev_err(&phba->pcidev->dev, "mccq poll timed out\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : mccq poll timed out\n");
+
 		return -EBUSY;
 	}
 	return 0;
@@ -338,7 +357,11 @@ static int be_mbox_db_ready_wait(struct be_ctrl_info *ctrl)
 			break;
 
 		if (cnt > 12000000) {
-			dev_err(&ctrl->pdev->dev, "mbox_db poll timed out\n");
+			struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+				    "BC_%d : mbox_db poll timed out\n");
+
 			return -EBUSY;
 		}
 
@@ -360,6 +383,7 @@ int be_mbox_notify(struct be_ctrl_info *ctrl)
 	struct be_dma_mem *mbox_mem = &ctrl->mbox_mem;
 	struct be_mcc_mailbox *mbox = mbox_mem->va;
 	struct be_mcc_compl *compl = &mbox->compl;
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 
 	val &= ~MPU_MAILBOX_DB_RDY_MASK;
 	val |= MPU_MAILBOX_DB_HI_MASK;
@@ -368,7 +392,10 @@ int be_mbox_notify(struct be_ctrl_info *ctrl)
 
 	status = be_mbox_db_ready_wait(ctrl);
 	if (status != 0) {
-		SE_DEBUG(DBG_LVL_1, " be_mbox_db_ready_wait failed\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : be_mbox_db_ready_wait failed\n");
+
 		return status;
 	}
 	val = 0;
@@ -379,18 +406,27 @@ int be_mbox_notify(struct be_ctrl_info *ctrl)
 
 	status = be_mbox_db_ready_wait(ctrl);
 	if (status != 0) {
-		SE_DEBUG(DBG_LVL_1, " be_mbox_db_ready_wait failed\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : be_mbox_db_ready_wait failed\n");
+
 		return status;
 	}
 	if (be_mcc_compl_is_new(compl)) {
 		status = be_mcc_compl_process(ctrl, &mbox->compl);
 		be_mcc_compl_use(compl);
 		if (status) {
-			SE_DEBUG(DBG_LVL_1, "After be_mcc_compl_process\n");
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+				    "BC_%d : After be_mcc_compl_process\n");
+
 			return status;
 		}
 	} else {
-		dev_err(&ctrl->pdev->dev, "invalid mailbox completion\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : Invalid Mailbox Completion\n");
+
 		return -EBUSY;
 	}
 	return 0;
@@ -436,7 +472,10 @@ static int be_mbox_notify_wait(struct beiscsi_hba *phba)
 		if (status)
 			return status;
 	} else {
-		dev_err(&phba->pcidev->dev, "invalid mailbox completion\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BC_%d : invalid mailbox completion\n");
+
 		return -EBUSY;
 	}
 	return 0;
@@ -528,7 +567,6 @@ int beiscsi_cmd_eq_create(struct be_ctrl_info *ctrl,
 	struct be_dma_mem *q_mem = &eq->dma_mem;
 	int status;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_cmd_eq_create\n");
 	spin_lock(&ctrl->mbox_lock);
 	memset(wrb, 0, sizeof(*wrb));
 
@@ -563,10 +601,10 @@ int beiscsi_cmd_eq_create(struct be_ctrl_info *ctrl,
 int be_cmd_fw_initialize(struct be_ctrl_info *ctrl)
 {
 	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	int status;
 	u8 *endian_check;
 
-	SE_DEBUG(DBG_LVL_8, "In be_cmd_fw_initialize\n");
 	spin_lock(&ctrl->mbox_lock);
 	memset(wrb, 0, sizeof(*wrb));
 
@@ -583,7 +621,8 @@ int be_cmd_fw_initialize(struct be_ctrl_info *ctrl)
 
 	status = be_mbox_notify(ctrl);
 	if (status)
-		SE_DEBUG(DBG_LVL_1, "be_cmd_fw_initialize Failed\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BC_%d : be_cmd_fw_initialize Failed\n");
 
 	spin_unlock(&ctrl->mbox_lock);
 	return status;
@@ -596,11 +635,11 @@ int beiscsi_cmd_cq_create(struct be_ctrl_info *ctrl,
 	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
 	struct be_cmd_req_cq_create *req = embedded_payload(wrb);
 	struct be_cmd_resp_cq_create *resp = embedded_payload(wrb);
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	struct be_dma_mem *q_mem = &cq->dma_mem;
 	void *ctxt = &req->context;
 	int status;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_cmd_cq_create\n");
 	spin_lock(&ctrl->mbox_lock);
 	memset(wrb, 0, sizeof(*wrb));
 
@@ -608,8 +647,6 @@ int beiscsi_cmd_cq_create(struct be_ctrl_info *ctrl,
 
 	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_COMMON,
 			OPCODE_COMMON_CQ_CREATE, sizeof(*req));
-	if (!q_mem->va)
-		SE_DEBUG(DBG_LVL_1, "uninitialized q_mem->va\n");
 
 	req->num_pages = cpu_to_le16(PAGES_4K_SPANNED(q_mem->va, q_mem->size));
 
@@ -633,8 +670,10 @@ int beiscsi_cmd_cq_create(struct be_ctrl_info *ctrl,
 		cq->id = le16_to_cpu(resp->cq_id);
 		cq->created = true;
 	} else
-		SE_DEBUG(DBG_LVL_1, "In be_cmd_cq_create, status=ox%08x\n",
-			status);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BC_%d : In be_cmd_cq_create, status=ox%08x\n",
+			    status);
+
 	spin_unlock(&ctrl->mbox_lock);
 
 	return status;
@@ -700,10 +739,14 @@ int beiscsi_cmd_q_destroy(struct be_ctrl_info *ctrl, struct be_queue_info *q,
 {
 	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
 	struct be_cmd_req_q_destroy *req = embedded_payload(wrb);
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	u8 subsys = 0, opcode = 0;
 	int status;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_cmd_q_destroy\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BC_%d : In beiscsi_cmd_q_destroy "
+		    "queue_type : %d\n", queue_type);
+
 	spin_lock(&ctrl->mbox_lock);
 	memset(wrb, 0, sizeof(*wrb));
 	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
@@ -759,7 +802,6 @@ int be_cmd_create_default_pdu_queue(struct be_ctrl_info *ctrl,
 	void *ctxt = &req->context;
 	int status;
 
-	SE_DEBUG(DBG_LVL_8, "In be_cmd_create_default_pdu_queue\n");
 	spin_lock(&ctrl->mbox_lock);
 	memset(wrb, 0, sizeof(*wrb));
 
@@ -830,6 +872,7 @@ int be_cmd_iscsi_post_sgl_pages(struct be_ctrl_info *ctrl,
 {
 	struct be_mcc_wrb *wrb = wrb_from_mbox(&ctrl->mbox_mem);
 	struct be_post_sgl_pages_req *req = embedded_payload(wrb);
+	struct beiscsi_hba *phba = pci_get_drvdata(ctrl->pdev);
 	int status;
 	unsigned int curr_pages;
 	u32 internal_page_offset = 0;
@@ -860,8 +903,9 @@ int be_cmd_iscsi_post_sgl_pages(struct be_ctrl_info *ctrl,
 
 		status = be_mbox_notify(ctrl);
 		if (status) {
-			SE_DEBUG(DBG_LVL_1,
-				 "FW CMD to map iscsi frags failed.\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BC_%d : FW CMD to map iscsi frags failed.\n");
+
 			goto error;
 		}
 	} while (num_pages > 0);
@@ -890,3 +934,45 @@ int beiscsi_cmd_reset_function(struct beiscsi_hba  *phba)
 	spin_unlock(&ctrl->mbox_lock);
 	return status;
 }
+
+/**
+ * be_cmd_set_vlan()- Configure VLAN paramters on the adapter
+ * @phba: device priv structure instance
+ * @vlan_tag: TAG to be set
+ *
+ * Set the VLAN_TAG for the adapter or Disable VLAN on adapter
+ *
+ * returns
+ *	TAG for the MBX Cmd
+ * **/
+int be_cmd_set_vlan(struct beiscsi_hba *phba,
+		     uint16_t vlan_tag)
+{
+	unsigned int tag = 0;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_set_vlan_req *req;
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+
+	spin_lock(&ctrl->mbox_lock);
+	tag = alloc_mcc_tag(phba);
+	if (!tag) {
+		spin_unlock(&ctrl->mbox_lock);
+		return tag;
+	}
+
+	wrb = wrb_from_mccq(phba);
+	req = embedded_payload(wrb);
+	wrb->tag0 |= tag;
+	be_wrb_hdr_prepare(wrb, sizeof(*wrb), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI,
+			   OPCODE_COMMON_ISCSI_NTWK_SET_VLAN,
+			   sizeof(*req));
+
+	req->interface_hndl = phba->interface_handle;
+	req->vlan_priority = vlan_tag;
+
+	be_mcc_notify(phba);
+	spin_unlock(&ctrl->mbox_lock);
+
+	return tag;
+}
diff --git a/drivers/scsi/be2iscsi/be_cmds.h b/drivers/scsi/be2iscsi/be_cmds.h
index b0b36c6a145..2c8f98df128 100644
--- a/drivers/scsi/be2iscsi/be_cmds.h
+++ b/drivers/scsi/be2iscsi/be_cmds.h
@@ -348,6 +348,23 @@ struct be_cmd_get_boot_target_resp {
 	int  boot_session_handle;
 };
 
+struct be_cmd_reopen_session_req {
+	struct be_cmd_req_hdr hdr;
+#define BE_REOPEN_ALL_SESSIONS  0x00
+#define BE_REOPEN_BOOT_SESSIONS 0x01
+#define BE_REOPEN_A_SESSION     0x02
+	u16 reopen_type;
+	u16 rsvd;
+	u32 session_handle;
+} __packed;
+
+struct be_cmd_reopen_session_resp {
+	struct be_cmd_resp_hdr hdr;
+	u32 rsvd;
+	u32 session_handle;
+} __packed;
+
+
 struct be_cmd_mac_query_req {
 	struct be_cmd_req_hdr hdr;
 	u8 type;
@@ -432,6 +449,12 @@ struct be_cmd_get_def_gateway_resp {
 	struct ip_addr_format ip_addr;
 } __packed;
 
+#define BEISCSI_VLAN_DISABLE	0xFFFF
+struct be_cmd_set_vlan_req {
+	struct be_cmd_req_hdr hdr;
+	u32 interface_hndl;
+	u32 vlan_priority;
+} __packed;
 /******************** Create CQ ***************************/
 /**
  * Pseudo amap definition in which each bit of the actual structure is defined
@@ -671,6 +694,9 @@ int be_cmd_wrbq_create(struct be_ctrl_info *ctrl, struct be_dma_mem *q_mem,
 
 bool is_link_state_evt(u32 trailer);
 
+/* Configuration Functions */
+int be_cmd_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag);
+
 struct be_default_pdu_context {
 	u32 dw[4];
 } __packed;
@@ -911,6 +937,7 @@ struct be_cmd_get_all_if_id_req {
 #define OPCODE_ISCSI_INI_CFG_GET_HBA_NAME	6
 #define OPCODE_ISCSI_INI_CFG_SET_HBA_NAME	7
 #define OPCODE_ISCSI_INI_SESSION_GET_A_SESSION  14
+#define OPCODE_ISCSI_INI_DRIVER_REOPEN_ALL_SESSIONS 36
 #define OPCODE_ISCSI_INI_DRIVER_OFFLOAD_SESSION 41
 #define OPCODE_ISCSI_INI_DRIVER_INVALIDATE_CONNECTION 42
 #define OPCODE_ISCSI_INI_BOOT_GET_BOOT_TARGET	52
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index 43f35034585..aedb0d9a9da 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -50,21 +50,27 @@ struct iscsi_cls_session *beiscsi_session_create(struct iscsi_endpoint *ep,
 	struct beiscsi_session *beiscsi_sess;
 	struct beiscsi_io_task *io_task;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_session_create\n");
 
 	if (!ep) {
-		SE_DEBUG(DBG_LVL_1, "beiscsi_session_create: invalid ep\n");
+		printk(KERN_ERR
+		       "beiscsi_session_create: invalid ep\n");
 		return NULL;
 	}
 	beiscsi_ep = ep->dd_data;
 	phba = beiscsi_ep->phba;
 	shost = phba->shost;
+
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_session_create\n");
+
 	if (cmds_max > beiscsi_ep->phba->params.wrbs_per_cxn) {
-		shost_printk(KERN_ERR, shost, "Cannot handle %d cmds."
-			     "Max cmds per session supported is %d. Using %d. "
-			     "\n", cmds_max,
-			      beiscsi_ep->phba->params.wrbs_per_cxn,
-			      beiscsi_ep->phba->params.wrbs_per_cxn);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Cannot handle %d cmds."
+			    "Max cmds per session supported is %d. Using %d."
+			    "\n", cmds_max,
+			    beiscsi_ep->phba->params.wrbs_per_cxn,
+			    beiscsi_ep->phba->params.wrbs_per_cxn);
+
 		cmds_max = beiscsi_ep->phba->params.wrbs_per_cxn;
 	}
 
@@ -102,7 +108,7 @@ void beiscsi_session_destroy(struct iscsi_cls_session *cls_session)
 	struct iscsi_session *sess = cls_session->dd_data;
 	struct beiscsi_session *beiscsi_sess = sess->dd_data;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_session_destroy\n");
+	printk(KERN_INFO "In beiscsi_session_destroy\n");
 	pci_pool_destroy(beiscsi_sess->bhs_pool);
 	iscsi_session_teardown(cls_session);
 }
@@ -123,11 +129,13 @@ beiscsi_conn_create(struct iscsi_cls_session *cls_session, u32 cid)
 	struct iscsi_session *sess;
 	struct beiscsi_session *beiscsi_sess;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_create ,cid"
-		 "from iscsi layer=%d\n", cid);
 	shost = iscsi_session_to_shost(cls_session);
 	phba = iscsi_host_priv(shost);
 
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_conn_create ,cid"
+		    "from iscsi layer=%d\n", cid);
+
 	cls_conn = iscsi_conn_setup(cls_session, sizeof(*beiscsi_conn), cid);
 	if (!cls_conn)
 		return NULL;
@@ -154,12 +162,15 @@ static int beiscsi_bindconn_cid(struct beiscsi_hba *phba,
 				unsigned int cid)
 {
 	if (phba->conn_table[cid]) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Connection table already occupied. Detected clash\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Connection table already occupied. Detected clash\n");
+
 		return -EINVAL;
 	} else {
-		SE_DEBUG(DBG_LVL_8, "phba->conn_table[%d]=%p(beiscsi_conn)\n",
-			 cid, beiscsi_conn);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : phba->conn_table[%d]=%p(beiscsi_conn)\n",
+			    cid, beiscsi_conn);
+
 		phba->conn_table[cid] = beiscsi_conn;
 	}
 	return 0;
@@ -184,7 +195,6 @@ int beiscsi_conn_bind(struct iscsi_cls_session *cls_session,
 	struct beiscsi_endpoint *beiscsi_ep;
 	struct iscsi_endpoint *ep;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_bind\n");
 	ep = iscsi_lookup_endpoint(transport_fd);
 	if (!ep)
 		return -EINVAL;
@@ -195,17 +205,21 @@ int beiscsi_conn_bind(struct iscsi_cls_session *cls_session,
 		return -EINVAL;
 
 	if (beiscsi_ep->phba != phba) {
-		SE_DEBUG(DBG_LVL_8,
-			 "beiscsi_ep->hba=%p not equal to phba=%p\n",
-			 beiscsi_ep->phba, phba);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : beiscsi_ep->hba=%p not equal to phba=%p\n",
+			    beiscsi_ep->phba, phba);
+
 		return -EEXIST;
 	}
 
 	beiscsi_conn->beiscsi_conn_cid = beiscsi_ep->ep_cid;
 	beiscsi_conn->ep = beiscsi_ep;
 	beiscsi_ep->conn = beiscsi_conn;
-	SE_DEBUG(DBG_LVL_8, "beiscsi_conn=%p conn=%p ep_cid=%d\n",
-		 beiscsi_conn, conn, beiscsi_ep->ep_cid);
+
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : beiscsi_conn=%p conn=%p ep_cid=%d\n",
+		    beiscsi_conn, conn, beiscsi_ep->ep_cid);
+
 	return beiscsi_bindconn_cid(phba, beiscsi_conn, beiscsi_ep->ep_cid);
 }
 
@@ -219,8 +233,9 @@ static int beiscsi_create_ipv4_iface(struct beiscsi_hba *phba)
 					      ISCSI_IFACE_TYPE_IPV4,
 					      0, 0);
 	if (!phba->ipv4_iface) {
-		shost_printk(KERN_ERR, phba->shost, "Could not "
-			     "create default IPv4 address.\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Could not "
+			    "create default IPv4 address.\n");
 		return -ENODEV;
 	}
 
@@ -237,8 +252,9 @@ static int beiscsi_create_ipv6_iface(struct beiscsi_hba *phba)
 					      ISCSI_IFACE_TYPE_IPV6,
 					      0, 0);
 	if (!phba->ipv6_iface) {
-		shost_printk(KERN_ERR, phba->shost, "Could not "
-			     "create default IPv6 address.\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Could not "
+			    "create default IPv6 address.\n");
 		return -ENODEV;
 	}
 
@@ -299,12 +315,14 @@ beiscsi_set_static_ip(struct Scsi_Host *shost,
 			iface_ip = nla_data(nla);
 		break;
 	default:
-		shost_printk(KERN_ERR, shost, "Unsupported param %d\n",
-			     iface_param->param);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Unsupported param %d\n",
+			    iface_param->param);
 	}
 
 	if (!iface_ip || !iface_subnet) {
-		shost_printk(KERN_ERR, shost, "IP and Subnet Mask required\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : IP and Subnet Mask required\n");
 		return -EINVAL;
 	}
 
@@ -314,6 +332,51 @@ beiscsi_set_static_ip(struct Scsi_Host *shost,
 	return ret;
 }
 
+/**
+ * beiscsi_set_vlan_tag()- Set the VLAN TAG
+ * @shost: Scsi Host for the driver instance
+ * @iface_param: Interface paramters
+ *
+ * Set the VLAN TAG for the adapter or disable
+ * the VLAN config
+ *
+ * returns
+ *	Success: 0
+ *	Failure: Non-Zero Value
+ **/
+static int
+beiscsi_set_vlan_tag(struct Scsi_Host *shost,
+		      struct iscsi_iface_param_info *iface_param)
+{
+	struct beiscsi_hba *phba = iscsi_host_priv(shost);
+	int ret = 0;
+
+	/* Get the Interface Handle */
+	if (mgmt_get_all_if_id(phba)) {
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Getting Interface Handle Failed\n");
+		return -EIO;
+	}
+
+	switch (iface_param->param) {
+	case ISCSI_NET_PARAM_VLAN_ENABLED:
+		if (iface_param->value[0] != ISCSI_VLAN_ENABLE)
+			ret = mgmt_set_vlan(phba, BEISCSI_VLAN_DISABLE);
+		break;
+	case ISCSI_NET_PARAM_VLAN_TAG:
+		ret = mgmt_set_vlan(phba,
+				    *((uint16_t *)iface_param->value));
+		break;
+	default:
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Unkown Param Type : %d\n",
+			    iface_param->param);
+		return -ENOSYS;
+	}
+	return ret;
+}
+
+
 static int
 beiscsi_set_ipv4(struct Scsi_Host *shost,
 		struct iscsi_iface_param_info *iface_param,
@@ -335,8 +398,9 @@ beiscsi_set_ipv4(struct Scsi_Host *shost,
 			ret = beiscsi_set_static_ip(shost, iface_param,
 						    data, dt_len);
 		else
-			shost_printk(KERN_ERR, shost, "Invalid BOOTPROTO: %d\n",
-					iface_param->value[0]);
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : Invalid BOOTPROTO: %d\n",
+				    iface_param->value[0]);
 		break;
 	case ISCSI_NET_PARAM_IFACE_ENABLE:
 		if (iface_param->value[0] == ISCSI_IFACE_ENABLE)
@@ -349,9 +413,14 @@ beiscsi_set_ipv4(struct Scsi_Host *shost,
 		ret = beiscsi_set_static_ip(shost, iface_param,
 					    data, dt_len);
 		break;
+	case ISCSI_NET_PARAM_VLAN_ENABLED:
+	case ISCSI_NET_PARAM_VLAN_TAG:
+		ret = beiscsi_set_vlan_tag(shost, iface_param);
+		break;
 	default:
-		shost_printk(KERN_ERR, shost, "Param %d not supported\n",
-			     iface_param->param);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Param %d not supported\n",
+			    iface_param->param);
 	}
 
 	return ret;
@@ -379,8 +448,9 @@ beiscsi_set_ipv6(struct Scsi_Host *shost,
 				  ISCSI_BOOTPROTO_STATIC);
 		break;
 	default:
-		shost_printk(KERN_ERR, shost, "Param %d not supported\n",
-			     iface_param->param);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Param %d not supported\n",
+			    iface_param->param);
 	}
 
 	return ret;
@@ -390,6 +460,7 @@ int be2iscsi_iface_set_param(struct Scsi_Host *shost,
 		void *data, uint32_t dt_len)
 {
 	struct iscsi_iface_param_info *iface_param = NULL;
+	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	struct nlattr *attrib;
 	uint32_t rm_len = dt_len;
 	int ret = 0 ;
@@ -404,9 +475,11 @@ int be2iscsi_iface_set_param(struct Scsi_Host *shost,
 		 * BE2ISCSI only supports 1 interface
 		 */
 		if (iface_param->iface_num) {
-			shost_printk(KERN_ERR, shost, "Invalid iface_num %d."
-				     "Only iface_num 0 is supported.\n",
-				     iface_param->iface_num);
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : Invalid iface_num %d."
+				    "Only iface_num 0 is supported.\n",
+				    iface_param->iface_num);
+
 			return -EINVAL;
 		}
 
@@ -420,9 +493,9 @@ int be2iscsi_iface_set_param(struct Scsi_Host *shost,
 					       data, dt_len);
 			break;
 		default:
-			shost_printk(KERN_ERR, shost,
-				     "Invalid iface type :%d passed\n",
-				     iface_param->iface_type);
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : Invalid iface type :%d passed\n",
+				    iface_param->iface_type);
 			break;
 		}
 
@@ -465,6 +538,27 @@ static int be2iscsi_get_if_param(struct beiscsi_hba *phba,
 	case ISCSI_NET_PARAM_IPV4_SUBNET:
 		len = sprintf(buf, "%pI4\n", &if_info.ip_addr.subnet_mask);
 		break;
+	case ISCSI_NET_PARAM_VLAN_ENABLED:
+		len = sprintf(buf, "%s\n",
+			     (if_info.vlan_priority == BEISCSI_VLAN_DISABLE)
+			     ? "Disabled" : "Enabled");
+		break;
+	case ISCSI_NET_PARAM_VLAN_ID:
+		if (if_info.vlan_priority == BEISCSI_VLAN_DISABLE)
+			return -EINVAL;
+		else
+			len = sprintf(buf, "%d\n",
+				     (if_info.vlan_priority &
+				     ISCSI_MAX_VLAN_ID));
+		break;
+	case ISCSI_NET_PARAM_VLAN_PRIORITY:
+		if (if_info.vlan_priority == BEISCSI_VLAN_DISABLE)
+			return -EINVAL;
+		else
+			len = sprintf(buf, "%d\n",
+				     ((if_info.vlan_priority >> 13) &
+				     ISCSI_MAX_VLAN_PRIORITY));
+		break;
 	default:
 		WARN_ON(1);
 	}
@@ -486,6 +580,9 @@ int be2iscsi_iface_get_param(struct iscsi_iface *iface,
 	case ISCSI_NET_PARAM_IPV4_SUBNET:
 	case ISCSI_NET_PARAM_IPV4_BOOTPROTO:
 	case ISCSI_NET_PARAM_IPV6_ADDR:
+	case ISCSI_NET_PARAM_VLAN_ENABLED:
+	case ISCSI_NET_PARAM_VLAN_ID:
+	case ISCSI_NET_PARAM_VLAN_PRIORITY:
 		len = be2iscsi_get_if_param(phba, iface, param, buf);
 		break;
 	case ISCSI_NET_PARAM_IFACE_ENABLE:
@@ -518,7 +615,10 @@ int beiscsi_ep_get_param(struct iscsi_endpoint *ep,
 	struct beiscsi_endpoint *beiscsi_ep = ep->dd_data;
 	int len = 0;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_ep_get_param, param= %d\n", param);
+	beiscsi_log(beiscsi_ep->phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_ep_get_param,"
+		    " param= %d\n", param);
 
 	switch (param) {
 	case ISCSI_PARAM_CONN_PORT:
@@ -541,9 +641,14 @@ int beiscsi_set_param(struct iscsi_cls_conn *cls_conn,
 {
 	struct iscsi_conn *conn = cls_conn->dd_data;
 	struct iscsi_session *session = conn->session;
+	struct beiscsi_hba *phba = NULL;
 	int ret;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_set_param, param= %d\n", param);
+	phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_conn_set_param,"
+		    " param= %d\n", param);
+
 	ret = iscsi_set_param(cls_conn, param, buf, buflen);
 	if (ret)
 		return ret;
@@ -593,7 +698,9 @@ static int beiscsi_get_initname(char *buf, struct beiscsi_hba *phba)
 
 	tag = be_cmd_get_initname(phba);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_1, "Getting Initiator Name Failed\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Getting Initiator Name Failed\n");
+
 		return -EBUSY;
 	} else
 		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
@@ -604,9 +711,12 @@ static int beiscsi_get_initname(char *buf, struct beiscsi_hba *phba)
 	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
 
 	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1, "MailBox Command Failed with "
-				"status = %d extd_status = %d\n",
-				status, extd_status);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BS_%d : MailBox Command Failed with "
+			    "status = %d extd_status = %d\n",
+			    status, extd_status);
+
 		free_mcc_tag(&phba->ctrl, tag);
 		return -EAGAIN;
 	}
@@ -650,7 +760,9 @@ static int beiscsi_get_port_speed(struct Scsi_Host *shost)
 
 	tag = be_cmd_get_port_speed(phba);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_1, "Getting Port Speed Failed\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Getting Port Speed Failed\n");
+
 		 return -EBUSY;
 	 } else
 		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
@@ -661,9 +773,12 @@ static int beiscsi_get_port_speed(struct Scsi_Host *shost)
 	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
 
 	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1, "MailBox Command Failed with "
-				"status = %d extd_status = %d\n",
-				status, extd_status);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BS_%d : MailBox Command Failed with "
+			    "status = %d extd_status = %d\n",
+			    status, extd_status);
+
 		free_mcc_tag(&phba->ctrl, tag);
 		return -EAGAIN;
 	}
@@ -704,20 +819,24 @@ int beiscsi_get_host_param(struct Scsi_Host *shost,
 	struct beiscsi_hba *phba = iscsi_host_priv(shost);
 	int status = 0;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_get_host_param, param= %d\n", param);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_get_host_param,"
+		    " param= %d\n", param);
+
 	switch (param) {
 	case ISCSI_HOST_PARAM_HWADDRESS:
 		status = beiscsi_get_macaddr(buf, phba);
 		if (status < 0) {
-			SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : beiscsi_get_macaddr Failed\n");
 			return status;
 		}
 		break;
 	case ISCSI_HOST_PARAM_INITIATOR_NAME:
 		status = beiscsi_get_initname(buf, phba);
 		if (status < 0) {
-			SE_DEBUG(DBG_LVL_1,
-					"Retreiving Initiator Name Failed\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : Retreiving Initiator Name Failed\n");
 			return status;
 		}
 		break;
@@ -728,8 +847,8 @@ int beiscsi_get_host_param(struct Scsi_Host *shost,
 	case ISCSI_HOST_PARAM_PORT_SPEED:
 		status = beiscsi_get_port_speed(shost);
 		if (status) {
-			SE_DEBUG(DBG_LVL_1,
-					"Retreiving Port Speed Failed\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BS_%d : Retreiving Port Speed Failed\n");
 			return status;
 		}
 		status = sprintf(buf, "%s\n", iscsi_get_port_speed_name(shost));
@@ -746,7 +865,7 @@ int beiscsi_get_macaddr(char *buf, struct beiscsi_hba *phba)
 	int rc;
 
 	if (strlen(phba->mac_address))
-		return strlcpy(buf, phba->mac_address, PAGE_SIZE);
+		return sysfs_format_mac(buf, phba->mac_address, ETH_ALEN);
 
 	memset(&resp, 0, sizeof(resp));
 	rc = mgmt_get_nic_conf(phba, &resp);
@@ -768,8 +887,12 @@ void beiscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn,
 			    struct iscsi_stats *stats)
 {
 	struct iscsi_conn *conn = cls_conn->dd_data;
+	struct beiscsi_hba *phba = NULL;
+
+	phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_conn_get_stats\n");
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_get_stats\n");
 	stats->txdata_octets = conn->txdata_octets;
 	stats->rxdata_octets = conn->rxdata_octets;
 	stats->dataout_pdus = conn->dataout_pdus_cnt;
@@ -829,11 +952,16 @@ int beiscsi_conn_start(struct iscsi_cls_conn *cls_conn)
 	struct beiscsi_endpoint *beiscsi_ep;
 	struct beiscsi_offload_params params;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_conn_start\n");
+	beiscsi_log(beiscsi_conn->phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_conn_start\n");
+
 	memset(&params, 0, sizeof(struct beiscsi_offload_params));
 	beiscsi_ep = beiscsi_conn->ep;
 	if (!beiscsi_ep)
-		SE_DEBUG(DBG_LVL_1, "In beiscsi_conn_start , no beiscsi_ep\n");
+		beiscsi_log(beiscsi_conn->phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG,
+			    "BS_%d : In beiscsi_conn_start , no beiscsi_ep\n");
 
 	beiscsi_conn->login_in_progress = 0;
 	beiscsi_set_params_for_offld(beiscsi_conn, &params);
@@ -907,19 +1035,27 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep,
 	unsigned int tag, wrb_num;
 	int ret = -ENOMEM;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_open_conn\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_open_conn\n");
+
 	beiscsi_ep->ep_cid = beiscsi_get_cid(phba);
 	if (beiscsi_ep->ep_cid == 0xFFFF) {
-		SE_DEBUG(DBG_LVL_1, "No free cid available\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : No free cid available\n");
 		return ret;
 	}
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_open_conn, ep_cid=%d\n",
-		 beiscsi_ep->ep_cid);
+
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_open_conn, ep_cid=%d\n",
+		    beiscsi_ep->ep_cid);
+
 	phba->ep_array[beiscsi_ep->ep_cid -
 		       phba->fw_config.iscsi_cid_start] = ep;
 	if (beiscsi_ep->ep_cid > (phba->fw_config.iscsi_cid_start +
 				  phba->params.cxns_per_ctrl * 2)) {
-		SE_DEBUG(DBG_LVL_1, "Failed in allocate iscsi cid\n");
+
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Failed in allocate iscsi cid\n");
 		goto free_ep;
 	}
 
@@ -928,9 +1064,11 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep,
 				sizeof(struct tcp_connect_and_offload_in),
 				&nonemb_cmd.dma);
 	if (nonemb_cmd.va == NULL) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Failed to allocate memory for mgmt_open_connection"
-			 "\n");
+
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Failed to allocate memory for"
+			    " mgmt_open_connection\n");
+
 		beiscsi_put_cid(phba, beiscsi_ep->ep_cid);
 		return -ENOMEM;
 	}
@@ -938,9 +1076,10 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep,
 	memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 	tag = mgmt_open_connection(phba, dst_addr, beiscsi_ep, &nonemb_cmd);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_1,
-			 "mgmt_open_connection Failed for cid=%d\n",
-			 beiscsi_ep->ep_cid);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : mgmt_open_connection Failed for cid=%d\n",
+			    beiscsi_ep->ep_cid);
+
 		beiscsi_put_cid(phba, beiscsi_ep->ep_cid);
 		pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 				    nonemb_cmd.va, nonemb_cmd.dma);
@@ -953,9 +1092,12 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep,
 	extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
 	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
 	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1, "mgmt_open_connection Failed"
-				    " status = %d extd_status = %d\n",
-				    status, extd_status);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BS_%d : mgmt_open_connection Failed"
+			    " status = %d extd_status = %d\n",
+			    status, extd_status);
+
 		free_mcc_tag(&phba->ctrl, tag);
 		pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 			    nonemb_cmd.va, nonemb_cmd.dma);
@@ -968,7 +1110,8 @@ static int beiscsi_open_conn(struct iscsi_endpoint *ep,
 		beiscsi_ep = ep->dd_data;
 		beiscsi_ep->fw_handle = ptcpcnct_out->connection_handle;
 		beiscsi_ep->cid_vld = 1;
-		SE_DEBUG(DBG_LVL_8, "mgmt_open_connection Success\n");
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : mgmt_open_connection Success\n");
 	}
 	pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 			    nonemb_cmd.va, nonemb_cmd.dma);
@@ -996,18 +1139,19 @@ beiscsi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
 	struct iscsi_endpoint *ep;
 	int ret;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_ep_connect\n");
 	if (shost)
 		phba = iscsi_host_priv(shost);
 	else {
 		ret = -ENXIO;
-		SE_DEBUG(DBG_LVL_1, "shost is NULL\n");
+		printk(KERN_ERR
+		       "beiscsi_ep_connect shost is NULL\n");
 		return ERR_PTR(ret);
 	}
 
 	if (phba->state != BE_ADAPTER_UP) {
 		ret = -EBUSY;
-		SE_DEBUG(DBG_LVL_1, "The Adapter state is Not UP\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : The Adapter state is Not UP\n");
 		return ERR_PTR(ret);
 	}
 
@@ -1022,7 +1166,8 @@ beiscsi_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
 	beiscsi_ep->openiscsi_ep = ep;
 	ret = beiscsi_open_conn(ep, NULL, dst_addr, non_blocking);
 	if (ret) {
-		SE_DEBUG(DBG_LVL_1, "Failed in beiscsi_open_conn\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Failed in beiscsi_open_conn\n");
 		goto free_ep;
 	}
 
@@ -1044,7 +1189,9 @@ int beiscsi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
 {
 	struct beiscsi_endpoint *beiscsi_ep = ep->dd_data;
 
-	SE_DEBUG(DBG_LVL_8, "In  beiscsi_ep_poll\n");
+	beiscsi_log(beiscsi_ep->phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In  beiscsi_ep_poll\n");
+
 	if (beiscsi_ep->cid_vld == 1)
 		return 1;
 	else
@@ -1064,8 +1211,10 @@ static int beiscsi_close_conn(struct  beiscsi_endpoint *beiscsi_ep, int flag)
 
 	tag = mgmt_upload_connection(phba, beiscsi_ep->ep_cid, flag);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_8, "upload failed for cid 0x%x\n",
-			 beiscsi_ep->ep_cid);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : upload failed for cid 0x%x\n",
+			    beiscsi_ep->ep_cid);
+
 		ret = -EAGAIN;
 	} else {
 		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
@@ -1086,7 +1235,8 @@ static int beiscsi_unbind_conn_to_cid(struct beiscsi_hba *phba,
 	if (phba->conn_table[cid])
 		phba->conn_table[cid] = NULL;
 	else {
-		SE_DEBUG(DBG_LVL_8, "Connection table Not occupied.\n");
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BS_%d : Connection table Not occupied.\n");
 		return -EINVAL;
 	}
 	return 0;
@@ -1104,38 +1254,40 @@ void beiscsi_ep_disconnect(struct iscsi_endpoint *ep)
 	struct beiscsi_endpoint *beiscsi_ep;
 	struct beiscsi_hba *phba;
 	unsigned int tag;
+	uint8_t mgmt_invalidate_flag, tcp_upload_flag;
 	unsigned short savecfg_flag = CMD_ISCSI_SESSION_SAVE_CFG_ON_FLASH;
 
 	beiscsi_ep = ep->dd_data;
 	phba = beiscsi_ep->phba;
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_ep_disconnect for ep_cid = %d\n",
-			     beiscsi_ep->ep_cid);
-
-	if (!beiscsi_ep->conn) {
-		SE_DEBUG(DBG_LVL_8, "In beiscsi_ep_disconnect, no "
-			 "beiscsi_ep\n");
-		return;
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BS_%d : In beiscsi_ep_disconnect for ep_cid = %d\n",
+		    beiscsi_ep->ep_cid);
+
+	if (beiscsi_ep->conn) {
+		beiscsi_conn = beiscsi_ep->conn;
+		iscsi_suspend_queue(beiscsi_conn->conn);
+		mgmt_invalidate_flag = ~BEISCSI_NO_RST_ISSUE;
+		tcp_upload_flag = CONNECTION_UPLOAD_GRACEFUL;
+	} else {
+		mgmt_invalidate_flag = BEISCSI_NO_RST_ISSUE;
+		tcp_upload_flag = CONNECTION_UPLOAD_ABORT;
 	}
-	beiscsi_conn = beiscsi_ep->conn;
-	iscsi_suspend_queue(beiscsi_conn->conn);
-
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_ep_disconnect ep_cid = %d\n",
-		 beiscsi_ep->ep_cid);
 
 	tag = mgmt_invalidate_connection(phba, beiscsi_ep,
-					    beiscsi_ep->ep_cid, 1,
-					    savecfg_flag);
+					  beiscsi_ep->ep_cid,
+					  mgmt_invalidate_flag,
+					  savecfg_flag);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_1,
-			 "mgmt_invalidate_connection Failed for cid=%d\n",
-			  beiscsi_ep->ep_cid);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BS_%d : mgmt_invalidate_connection Failed for cid=%d\n",
+			    beiscsi_ep->ep_cid);
 	} else {
 		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 					 phba->ctrl.mcc_numtag[tag]);
 		free_mcc_tag(&phba->ctrl, tag);
 	}
 
-	beiscsi_close_conn(beiscsi_ep, CONNECTION_UPLOAD_GRACEFUL);
+	beiscsi_close_conn(beiscsi_ep, tcp_upload_flag);
 	beiscsi_free_ep(beiscsi_ep);
 	beiscsi_unbind_conn_to_cid(phba, beiscsi_ep->ep_cid);
 	iscsi_destroy_endpoint(beiscsi_ep->openiscsi_ep);
@@ -1152,6 +1304,9 @@ umode_t be2iscsi_attr_is_visible(int param_type, int param)
 		case ISCSI_NET_PARAM_IPV4_BOOTPROTO:
 		case ISCSI_NET_PARAM_IPV4_GW:
 		case ISCSI_NET_PARAM_IPV6_ADDR:
+		case ISCSI_NET_PARAM_VLAN_ID:
+		case ISCSI_NET_PARAM_VLAN_PRIORITY:
+		case ISCSI_NET_PARAM_VLAN_ENABLED:
 			return S_IRUGO;
 		default:
 			return 0;
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index 0b1d99c99fd..ff73f9500b0 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -42,6 +42,7 @@
 #include "be_main.h"
 #include "be_iscsi.h"
 #include "be_mgmt.h"
+#include "be_cmds.h"
 
 static unsigned int be_iopoll_budget = 10;
 static unsigned int be_max_phys_size = 64;
@@ -57,9 +58,105 @@ MODULE_LICENSE("GPL");
 module_param(be_iopoll_budget, int, 0);
 module_param(enable_msix, int, 0);
 module_param(be_max_phys_size, uint, S_IRUGO);
-MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
-				   "contiguous memory that can be allocated."
-				   "Range is 16 - 128");
+MODULE_PARM_DESC(be_max_phys_size,
+		"Maximum Size (In Kilobytes) of physically contiguous "
+		"memory that can be allocated. Range is 16 - 128");
+
+#define beiscsi_disp_param(_name)\
+ssize_t	\
+beiscsi_##_name##_disp(struct device *dev,\
+			struct device_attribute *attrib, char *buf)	\
+{	\
+	struct Scsi_Host *shost = class_to_shost(dev);\
+	struct beiscsi_hba *phba = iscsi_host_priv(shost); \
+	uint32_t param_val = 0;	\
+	param_val = phba->attr_##_name;\
+	return snprintf(buf, PAGE_SIZE, "%d\n",\
+			phba->attr_##_name);\
+}
+
+#define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
+int \
+beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
+{\
+	if (val >= _minval && val <= _maxval) {\
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
+			    "BA_%d : beiscsi_"#_name" updated "\
+			    "from 0x%x ==> 0x%x\n",\
+			    phba->attr_##_name, val); \
+		phba->attr_##_name = val;\
+		return 0;\
+	} \
+	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
+		    "BA_%d beiscsi_"#_name" attribute "\
+		    "cannot be updated to 0x%x, "\
+		    "range allowed is ["#_minval" - "#_maxval"]\n", val);\
+		return -EINVAL;\
+}
+
+#define beiscsi_store_param(_name)  \
+ssize_t \
+beiscsi_##_name##_store(struct device *dev,\
+			 struct device_attribute *attr, const char *buf,\
+			 size_t count) \
+{ \
+	struct Scsi_Host  *shost = class_to_shost(dev);\
+	struct beiscsi_hba *phba = iscsi_host_priv(shost);\
+	uint32_t param_val = 0;\
+	if (!isdigit(buf[0]))\
+		return -EINVAL;\
+	if (sscanf(buf, "%i", &param_val) != 1)\
+		return -EINVAL;\
+	if (beiscsi_##_name##_change(phba, param_val) == 0) \
+		return strlen(buf);\
+	else \
+		return -EINVAL;\
+}
+
+#define beiscsi_init_param(_name, _minval, _maxval, _defval) \
+int \
+beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
+{ \
+	if (val >= _minval && val <= _maxval) {\
+		phba->attr_##_name = val;\
+		return 0;\
+	} \
+	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
+		    "BA_%d beiscsi_"#_name" attribute " \
+		    "cannot be updated to 0x%x, "\
+		    "range allowed is ["#_minval" - "#_maxval"]\n", val);\
+	phba->attr_##_name = _defval;\
+	return -EINVAL;\
+}
+
+#define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
+static uint beiscsi_##_name = _defval;\
+module_param(beiscsi_##_name, uint, S_IRUGO);\
+MODULE_PARM_DESC(beiscsi_##_name, _descp);\
+beiscsi_disp_param(_name)\
+beiscsi_change_param(_name, _minval, _maxval, _defval)\
+beiscsi_store_param(_name)\
+beiscsi_init_param(_name, _minval, _maxval, _defval)\
+DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
+	      beiscsi_##_name##_disp, beiscsi_##_name##_store)
+
+/*
+ * When new log level added update the
+ * the MAX allowed value for log_enable
+ */
+BEISCSI_RW_ATTR(log_enable, 0x00,
+		0xFF, 0x00, "Enable logging Bit Mask\n"
+		"\t\t\t\tInitialization Events	: 0x01\n"
+		"\t\t\t\tMailbox Events		: 0x02\n"
+		"\t\t\t\tMiscellaneous Events	: 0x04\n"
+		"\t\t\t\tError Handling		: 0x08\n"
+		"\t\t\t\tIO Path Events		: 0x10\n"
+		"\t\t\t\tConfiguration Path	: 0x20\n");
+
+struct device_attribute *beiscsi_attrs[] = {
+	&dev_attr_beiscsi_log_enable,
+	NULL,
+};
 
 static int beiscsi_slave_configure(struct scsi_device *sdev)
 {
@@ -112,9 +209,9 @@ static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 				sizeof(struct invalidate_commands_params_in),
 				&nonemb_cmd.dma);
 	if (nonemb_cmd.va == NULL) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Failed to allocate memory for"
-			 "mgmt_invalidate_icds\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
+			    "BM_%d : Failed to allocate memory for"
+			    "mgmt_invalidate_icds\n");
 		return FAILED;
 	}
 	nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
@@ -122,9 +219,9 @@ static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 	tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 				   cid, &nonemb_cmd);
 	if (!tag) {
-		shost_printk(KERN_WARNING, phba->shost,
-			     "mgmt_invalidate_icds could not be"
-			     " submitted\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
+			    "BM_%d : mgmt_invalidate_icds could not be"
+			    "submitted\n");
 		pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 				    nonemb_cmd.va, nonemb_cmd.dma);
 
@@ -188,9 +285,9 @@ static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 				sizeof(struct invalidate_commands_params_in),
 				&nonemb_cmd.dma);
 	if (nonemb_cmd.va == NULL) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Failed to allocate memory for"
-			 "mgmt_invalidate_icds\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
+			    "BM_%d : Failed to allocate memory for"
+			    "mgmt_invalidate_icds\n");
 		return FAILED;
 	}
 	nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
@@ -198,9 +295,9 @@ static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 	tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 				   cid, &nonemb_cmd);
 	if (!tag) {
-		shost_printk(KERN_WARNING, phba->shost,
-			     "mgmt_invalidate_icds could not be"
-			     " submitted\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
+			    "BM_%d : mgmt_invalidate_icds could not be"
+			    " submitted\n");
 		pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 				    nonemb_cmd.va, nonemb_cmd.dma);
 		return FAILED;
@@ -389,6 +486,7 @@ static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 };
 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 
+
 static struct scsi_host_template beiscsi_sht = {
 	.module = THIS_MODULE,
 	.name = "Emulex 10Gbe open-iscsi Initiator Driver",
@@ -400,6 +498,7 @@ static struct scsi_host_template beiscsi_sht = {
 	.eh_abort_handler = beiscsi_eh_abort,
 	.eh_device_reset_handler = beiscsi_eh_device_reset,
 	.eh_target_reset_handler = iscsi_eh_session_reset,
+	.shost_attrs = beiscsi_attrs,
 	.sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 	.can_queue = BE2_IO_DEPTH,
 	.this_id = -1,
@@ -419,8 +518,8 @@ static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 
 	shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 	if (!shost) {
-		dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
-			"iscsi_host_alloc failed\n");
+		dev_err(&pcidev->dev,
+			"beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 		return NULL;
 	}
 	shost->dma_boundary = pcidev->dma_mask;
@@ -510,8 +609,8 @@ static int beiscsi_enable_pci(struct pci_dev *pcidev)
 
 	ret = pci_enable_device(pcidev);
 	if (ret) {
-		dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
-			"failed. Returning -ENODEV\n");
+		dev_err(&pcidev->dev,
+			"beiscsi_enable_pci - enable device failed\n");
 		return ret;
 	}
 
@@ -576,8 +675,9 @@ static void beiscsi_get_params(struct beiscsi_hba *phba)
 				    + BE2_TMFS) / 512) + 1) * 512;
 	phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 				? 1024 : phba->params.num_eq_entries;
-	SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
-			     phba->params.num_eq_entries);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : phba->params.num_eq_entries=%d\n",
+		    phba->params.num_eq_entries);
 	phba->params.num_cq_entries =
 	    (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 				    + BE2_TMFS) / 512) + 1) * 512;
@@ -621,8 +721,6 @@ static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 	phba =  pbe_eq->phba;
 	mcc = &phba->ctrl.mcc_obj.cq;
 	eqe = queue_tail_node(eq);
-	if (!eqe)
-		SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 
 	num_eq_processed = 0;
 
@@ -667,8 +765,6 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
 	eq = &pbe_eq->q;
 	cq = pbe_eq->cq;
 	eqe = queue_tail_node(eq);
-	if (!eqe)
-		SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 
 	phba = pbe_eq->phba;
 	num_eq_processed = 0;
@@ -743,8 +839,6 @@ static irqreturn_t be_isr(int irq, void *dev_id)
 	mcc = &phba->ctrl.mcc_obj.cq;
 	index = 0;
 	eqe = queue_tail_node(eq);
-	if (!eqe)
-		SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 
 	num_ioeq_processed = 0;
 	num_mcceq_processed = 0;
@@ -842,9 +936,10 @@ static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 					  phba->msi_name[i],
 					  &phwi_context->be_eq[i]);
 			if (ret) {
-				shost_printk(KERN_ERR, phba->shost,
-					     "beiscsi_init_irqs-Failed to"
-					     "register msix for i = %d\n", i);
+				beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+					    "BM_%d : beiscsi_init_irqs-Failed to"
+					    "register msix for i = %d\n",
+					    i);
 				kfree(phba->msi_name[i]);
 				goto free_msix_irqs;
 			}
@@ -860,8 +955,9 @@ static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 		ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 				  &phwi_context->be_eq[i]);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
-				     "Failed to register beiscsi_msix_mcc\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
+				    "BM_%d : beiscsi_init_irqs-"
+				    "Failed to register beiscsi_msix_mcc\n");
 			kfree(phba->msi_name[i]);
 			goto free_msix_irqs;
 		}
@@ -870,8 +966,9 @@ static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 		ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 				  "beiscsi", phba);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
-				     "Failed to register irq\\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : beiscsi_init_irqs-"
+				    "Failed to register irq\\n");
 			return ret;
 		}
 	}
@@ -922,7 +1019,9 @@ beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 	case ISCSI_OP_REJECT:
 		WARN_ON(!pbuffer);
 		WARN_ON(!(buf_len == 48));
-		SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d : In ISCSI_OP_REJECT\n");
 		break;
 	case ISCSI_OP_LOGIN_RSP:
 	case ISCSI_OP_TEXT_RSP:
@@ -932,11 +1031,12 @@ beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 		login_hdr->itt = io_task->libiscsi_itt;
 		break;
 	default:
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Unrecognized opcode 0x%x in async msg\n",
-			     (ppdu->
+		beiscsi_log(phba, KERN_WARNING,
+			    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+			    "BM_%d : Unrecognized opcode 0x%x in async msg\n",
+			    (ppdu->
 			     dw[offsetof(struct amap_pdu_base, opcode) / 32]
-						& PDUBASE_OPCODE_MASK));
+			     & PDUBASE_OPCODE_MASK));
 		return 1;
 	}
 
@@ -951,9 +1051,11 @@ static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 	struct sgl_handle *psgl_handle;
 
 	if (phba->io_sgl_hndl_avbl) {
-		SE_DEBUG(DBG_LVL_8,
-			 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
-			 phba->io_sgl_alloc_index);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
+			    "BM_%d : In alloc_io_sgl_handle,"
+			    " io_sgl_alloc_index=%d\n",
+			    phba->io_sgl_alloc_index);
+
 		psgl_handle = phba->io_sgl_hndl_base[phba->
 						io_sgl_alloc_index];
 		phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
@@ -971,17 +1073,20 @@ static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 static void
 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 {
-	SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
-		 phba->io_sgl_free_index);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
+		    "BM_%d : In free_,io_sgl_free_index=%d\n",
+		    phba->io_sgl_free_index);
+
 	if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 		/*
 		 * this can happen if clean_task is called on a task that
 		 * failed in xmit_task or alloc_pdu.
 		 */
-		 SE_DEBUG(DBG_LVL_8,
-			 "Double Free in IO SGL io_sgl_free_index=%d,"
-			 "value there=%p\n", phba->io_sgl_free_index,
-			 phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
+		 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
+			     "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
+			     "value there=%p\n", phba->io_sgl_free_index,
+			     phba->io_sgl_hndl_base
+			     [phba->io_sgl_free_index]);
 		return;
 	}
 	phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
@@ -1043,11 +1148,12 @@ free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
 	else
 		pwrb_context->free_index++;
 
-	SE_DEBUG(DBG_LVL_8,
-		 "FREE WRB: pwrb_handle=%p free_index=0x%x"
-		 "wrb_handles_available=%d\n",
-		 pwrb_handle, pwrb_context->free_index,
-		 pwrb_context->wrb_handles_available);
+	beiscsi_log(phba, KERN_INFO,
+		    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+		    "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
+		    "wrb_handles_available=%d\n",
+		    pwrb_handle, pwrb_context->free_index,
+		    pwrb_context->wrb_handles_available);
 }
 
 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
@@ -1057,8 +1163,11 @@ static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
 	if (phba->eh_sgl_hndl_avbl) {
 		psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
 		phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
-		SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
-			 phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+			    "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
+			    phba->eh_sgl_alloc_index,
+			    phba->eh_sgl_alloc_index);
+
 		phba->eh_sgl_hndl_avbl--;
 		if (phba->eh_sgl_alloc_index ==
 		    (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
@@ -1075,16 +1184,20 @@ void
 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 {
 
-	SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
-			     phba->eh_sgl_free_index);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BM_%d : In  free_mgmt_sgl_handle,"
+		    "eh_sgl_free_index=%d\n",
+		    phba->eh_sgl_free_index);
+
 	if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
 		/*
 		 * this can happen if clean_task is called on a task that
 		 * failed in xmit_task or alloc_pdu.
 		 */
-		SE_DEBUG(DBG_LVL_8,
-			 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
-			 phba->eh_sgl_free_index);
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BM_%d : Double Free in eh SGL ,"
+			    "eh_sgl_free_index=%d\n",
+			    phba->eh_sgl_free_index);
 		return;
 	}
 	phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
@@ -1326,9 +1439,10 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
 		break;
 
 	case HWH_TYPE_LOGIN:
-		SE_DEBUG(DBG_LVL_1,
-			 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
-			 "- Solicited path\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
+			    " hwi_complete_cmd- Solicited path\n");
 		break;
 
 	case HWH_TYPE_NOP:
@@ -1336,13 +1450,14 @@ static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
 		break;
 
 	default:
-		shost_printk(KERN_WARNING, phba->shost,
-				"In hwi_complete_cmd, unknown type = %d"
-				"wrb_index 0x%x CID 0x%x\n", type,
-				((psol->dw[offsetof(struct amap_iscsi_wrb,
-				type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
-				((psol->dw[offsetof(struct amap_sol_cqe,
-				cid) / 32] & SOL_CID_MASK) >> 6));
+		beiscsi_log(phba, KERN_WARNING,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d : In hwi_complete_cmd, unknown type = %d"
+			    "wrb_index 0x%x CID 0x%x\n", type,
+			    ((psol->dw[offsetof(struct amap_iscsi_wrb,
+			    type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
+			    ((psol->dw[offsetof(struct amap_sol_cqe,
+			    cid) / 32] & SOL_CID_MASK) >> 6));
 		break;
 	}
 
@@ -1397,10 +1512,11 @@ hwi_get_async_handle(struct beiscsi_hba *phba,
 		break;
 	default:
 		pbusy_list = NULL;
-		shost_printk(KERN_WARNING, phba->shost,
-			"Unexpected code=%d\n",
-			 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
-					code) / 32] & PDUCQE_CODE_MASK);
+		beiscsi_log(phba, KERN_WARNING,
+			    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+			    "BM_%d : Unexpected code=%d\n",
+			    pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
+			    code) / 32] & PDUCQE_CODE_MASK);
 		return NULL;
 	}
 
@@ -1425,8 +1541,9 @@ hwi_get_async_handle(struct beiscsi_hba *phba,
 }
 
 static unsigned int
-hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
-			   unsigned int is_header, unsigned int cq_index)
+hwi_update_async_writables(struct beiscsi_hba *phba,
+			    struct hwi_async_pdu_context *pasync_ctx,
+			    unsigned int is_header, unsigned int cq_index)
 {
 	struct list_head *pbusy_list;
 	struct async_pdu_handle *pasync_handle;
@@ -1463,9 +1580,10 @@ hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
 	}
 
 	if (!writables) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Duplicate notification received - index 0x%x!!\n",
-			 cq_index);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d : Duplicate notification received - index 0x%x!!\n",
+			    cq_index);
 		WARN_ON(1);
 	}
 
@@ -1616,8 +1734,8 @@ static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
 					     pdpdu_cqe, &cq_index);
 	BUG_ON(pasync_handle->is_header != 0);
 	if (pasync_handle->consumed == 0)
-		hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
-					   cq_index);
+		hwi_update_async_writables(phba, pasync_ctx,
+					   pasync_handle->is_header, cq_index);
 
 	hwi_free_async_msg(phba, pasync_handle->cri);
 	hwi_post_async_buffers(phba, pasync_handle->is_header);
@@ -1745,8 +1863,9 @@ static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
 					     pdpdu_cqe, &cq_index);
 
 	if (pasync_handle->consumed == 0)
-		hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
-					   cq_index);
+		hwi_update_async_writables(phba, pasync_ctx,
+					   pasync_handle->is_header, cq_index);
+
 	hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
 	hwi_post_async_buffers(phba, pasync_handle->is_header);
 }
@@ -1774,9 +1893,10 @@ static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
 				beiscsi_async_link_state_process(phba,
 				(struct be_async_event_link_state *) mcc_compl);
 			else
-				SE_DEBUG(DBG_LVL_1,
-					" Unsupported Async Event, flags"
-					" = 0x%08x\n", mcc_compl->flags);
+				beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
+					    "BM_%d :  Unsupported Async Event, flags"
+					    " = 0x%08x\n",
+					    mcc_compl->flags);
 		} else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
 			be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
 			atomic_dec(&phba->ctrl.mcc_obj.q.used);
@@ -1801,6 +1921,7 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
 	struct dmsg_cqe *dmsg;
 	unsigned int num_processed = 0;
 	unsigned int tot_nump = 0;
+	unsigned short code = 0, cid = 0;
 	struct beiscsi_conn *beiscsi_conn;
 	struct beiscsi_endpoint *beiscsi_ep;
 	struct iscsi_endpoint *ep;
@@ -1814,10 +1935,11 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
 	       CQE_VALID_MASK) {
 		be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
 
-		ep = phba->ep_array[(u32) ((sol->
-				   dw[offsetof(struct amap_sol_cqe, cid) / 32] &
-				   SOL_CID_MASK) >> 6) -
-				   phba->fw_config.iscsi_cid_start];
+		cid = ((sol->dw[offsetof(struct amap_sol_cqe, cid)/32] &
+		      CQE_CID_MASK) >> 6);
+		code = (sol->dw[offsetof(struct amap_sol_cqe, code)/32] &
+		       CQE_CODE_MASK);
+		ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start];
 
 		beiscsi_ep = ep->dd_data;
 		beiscsi_conn = beiscsi_ep->conn;
@@ -1829,32 +1951,41 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
 			num_processed = 0;
 		}
 
-		switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
-			32] & CQE_CODE_MASK) {
+		switch (code) {
 		case SOL_CMD_COMPLETE:
 			hwi_complete_cmd(beiscsi_conn, phba, sol);
 			break;
 		case DRIVERMSG_NOTIFY:
-			SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
+			beiscsi_log(phba, KERN_INFO,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : Received DRIVERMSG_NOTIFY\n");
+
 			dmsg = (struct dmsg_cqe *)sol;
 			hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
 			break;
 		case UNSOL_HDR_NOTIFY:
-			SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
+			beiscsi_log(phba, KERN_INFO,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : Received UNSOL_HDR_ NOTIFY\n");
+
 			hwi_process_default_pdu_ring(beiscsi_conn, phba,
 					     (struct i_t_dpdu_cqe *)sol);
 			break;
 		case UNSOL_DATA_NOTIFY:
-			SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
+			beiscsi_log(phba, KERN_INFO,
+				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+				    "BM_%d : Received UNSOL_DATA_NOTIFY\n");
+
 			hwi_process_default_pdu_ring(beiscsi_conn, phba,
 					     (struct i_t_dpdu_cqe *)sol);
 			break;
 		case CXN_INVALIDATE_INDEX_NOTIFY:
 		case CMD_INVALIDATED_NOTIFY:
 		case CXN_INVALIDATE_NOTIFY:
-			SE_DEBUG(DBG_LVL_1,
-				 "Ignoring CQ Error notification for cmd/cxn"
-				 "invalidate\n");
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : Ignoring CQ Error notification for"
+				    " cmd/cxn invalidate\n");
 			break;
 		case SOL_CMD_KILLED_DATA_DIGEST_ERR:
 		case CMD_KILLED_INVALID_STATSN_RCVD:
@@ -1864,17 +1995,16 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
 		case CMD_CXN_KILLED_ITT_INVALID:
 		case CMD_CXN_KILLED_SEQ_OUTOFORDER:
 		case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
-			SE_DEBUG(DBG_LVL_1,
-				 "CQ Error notification for cmd.. "
-				 "code %d cid 0x%x\n",
-				 sol->dw[offsetof(struct amap_sol_cqe, code) /
-				 32] & CQE_CODE_MASK,
-				 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
-				 32] & SOL_CID_MASK));
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+				    "BM_%d : CQ Error notification for cmd.. "
+				    "code %d cid 0x%x\n", code, cid);
 			break;
 		case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
-			SE_DEBUG(DBG_LVL_1,
-				 "Digest error on def pdu ring, dropping..\n");
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : Digest error on def pdu ring,"
+				    " dropping..\n");
 			hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
 					     (struct i_t_dpdu_cqe *) sol);
 			break;
@@ -1892,33 +2022,31 @@ static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
 		case CXN_KILLED_OVER_RUN_RESIDUAL:
 		case CXN_KILLED_UNDER_RUN_RESIDUAL:
 		case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
-			SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
-				 "0x%x...\n",
-				 sol->dw[offsetof(struct amap_sol_cqe, code) /
-				 32] & CQE_CODE_MASK,
-				 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
-				 32] & CQE_CID_MASK));
-			iscsi_conn_failure(beiscsi_conn->conn,
-					   ISCSI_ERR_CONN_FAILED);
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : CQ Error %d, reset CID 0x%x...\n",
+				    code, cid);
+			if (beiscsi_conn)
+				iscsi_conn_failure(beiscsi_conn->conn,
+						   ISCSI_ERR_CONN_FAILED);
 			break;
 		case CXN_KILLED_RST_SENT:
 		case CXN_KILLED_RST_RCVD:
-			SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
-				"received/sent on CID 0x%x...\n",
-				 sol->dw[offsetof(struct amap_sol_cqe, code) /
-				 32] & CQE_CODE_MASK,
-				 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
-				 32] & CQE_CID_MASK));
-			iscsi_conn_failure(beiscsi_conn->conn,
-					   ISCSI_ERR_CONN_FAILED);
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : CQ Error %d, reset"
+				    "received/sent on CID 0x%x...\n",
+				    code, cid);
+			if (beiscsi_conn)
+				iscsi_conn_failure(beiscsi_conn->conn,
+						   ISCSI_ERR_CONN_FAILED);
 			break;
 		default:
-			SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
-				 "received on CID 0x%x...\n",
-				 sol->dw[offsetof(struct amap_sol_cqe, code) /
-				 32] & CQE_CODE_MASK,
-				 (sol->dw[offsetof(struct amap_sol_cqe, cid) /
-				 32] & CQE_CID_MASK));
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+				    "BM_%d : CQ Error Invalid code= %d "
+				    "received on CID 0x%x...\n",
+				    code, cid);
 			break;
 		}
 
@@ -1977,7 +2105,10 @@ static int be_iopoll(struct blk_iopoll *iop, int budget)
 	if (ret < budget) {
 		phba = pbe_eq->phba;
 		blk_iopoll_complete(iop);
-		SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
+		beiscsi_log(phba, KERN_INFO,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
+			    "BM_%d : rearm pbe_eq->q.id =%d\n",
+			    pbe_eq->q.id);
 		hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
 	}
 	return ret;
@@ -2348,16 +2479,16 @@ static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
 				kzalloc(sizeof(struct wrb_handle *) *
 					phba->params.wrbs_per_cxn, GFP_KERNEL);
 		if (!pwrb_context->pwrb_handle_base) {
-			shost_printk(KERN_ERR, phba->shost,
-					"Mem Alloc Failed. Failing to load\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : Mem Alloc Failed. Failing to load\n");
 			goto init_wrb_hndl_failed;
 		}
 		pwrb_context->pwrb_handle_basestd =
 				kzalloc(sizeof(struct wrb_handle *) *
 					phba->params.wrbs_per_cxn, GFP_KERNEL);
 		if (!pwrb_context->pwrb_handle_basestd) {
-			shost_printk(KERN_ERR, phba->shost,
-					"Mem Alloc Failed. Failing to load\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : Mem Alloc Failed. Failing to load\n");
 			goto init_wrb_hndl_failed;
 		}
 		if (!num_cxn_wrbh) {
@@ -2438,12 +2569,13 @@ static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
 	if (mem_descr->mem_array[0].virtual_address) {
-		SE_DEBUG(DBG_LVL_8,
-			 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
-			 "va=%p\n", mem_descr->mem_array[0].virtual_address);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_async_pdu_ctx"
+			    " HWI_MEM_ASYNC_HEADER_BUF va=%p\n",
+			    mem_descr->mem_array[0].virtual_address);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			     "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
 
 	pasync_ctx->async_header.va_base =
 			mem_descr->mem_array[0].virtual_address;
@@ -2454,24 +2586,27 @@ static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_HEADER_RING;
 	if (mem_descr->mem_array[0].virtual_address) {
-		SE_DEBUG(DBG_LVL_8,
-			 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
-			 "va=%p\n", mem_descr->mem_array[0].virtual_address);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_async_pdu_ctx"
+			    " HWI_MEM_ASYNC_HEADER_RING va=%p\n",
+			    mem_descr->mem_array[0].virtual_address);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			    "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
+
 	pasync_ctx->async_header.ring_base =
 			mem_descr->mem_array[0].virtual_address;
 
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
 	if (mem_descr->mem_array[0].virtual_address) {
-		SE_DEBUG(DBG_LVL_8,
-			 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
-			 "va=%p\n", mem_descr->mem_array[0].virtual_address);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_async_pdu_ctx"
+			    " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n",
+			    mem_descr->mem_array[0].virtual_address);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			    "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
 
 	pasync_ctx->async_header.handle_base =
 			mem_descr->mem_array[0].virtual_address;
@@ -2482,12 +2617,13 @@ static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_DATA_RING;
 	if (mem_descr->mem_array[0].virtual_address) {
-		SE_DEBUG(DBG_LVL_8,
-			 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
-			 "va=%p\n", mem_descr->mem_array[0].virtual_address);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_async_pdu_ctx"
+			    " HWI_MEM_ASYNC_DATA_RING va=%p\n",
+			    mem_descr->mem_array[0].virtual_address);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			     "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
 
 	pasync_ctx->async_data.ring_base =
 			mem_descr->mem_array[0].virtual_address;
@@ -2495,8 +2631,8 @@ static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
 	if (!mem_descr->mem_array[0].virtual_address)
-		shost_printk(KERN_WARNING, phba->shost,
-			    "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
 
 	pasync_ctx->async_data.handle_base =
 			mem_descr->mem_array[0].virtual_address;
@@ -2511,12 +2647,14 @@ static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
 	mem_descr = (struct be_mem_descriptor *)phba->init_mem;
 	mem_descr += HWI_MEM_ASYNC_DATA_BUF;
 	if (mem_descr->mem_array[0].virtual_address) {
-		SE_DEBUG(DBG_LVL_8,
-			 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
-			 "va=%p\n", mem_descr->mem_array[0].virtual_address);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_async_pdu_ctx"
+			    " HWI_MEM_ASYNC_DATA_BUF va=%p\n",
+			    mem_descr->mem_array[0].virtual_address);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			    "No Virtual address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : No Virtual address\n");
+
 	idx = 0;
 	pasync_ctx->async_data.va_base =
 			mem_descr->mem_array[idx].virtual_address;
@@ -2657,7 +2795,7 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 			     struct hwi_context_memory *phwi_context)
 {
 	unsigned int i, num_eq_pages;
-	int ret, eq_for_mcc;
+	int ret = 0, eq_for_mcc;
 	struct be_queue_info *eq;
 	struct be_dma_mem *mem;
 	void *eq_vaddress;
@@ -2684,8 +2822,8 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 		ret = be_fill_queue(eq, phba->params.num_eq_entries,
 				    sizeof(struct be_eq_entry), eq_vaddress);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "be_fill_queue Failed for EQ\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : be_fill_queue Failed for EQ\n");
 			goto create_eq_error;
 		}
 
@@ -2693,12 +2831,15 @@ static int beiscsi_create_eqs(struct beiscsi_hba *phba,
 		ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
 					    phwi_context->cur_eqd);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "beiscsi_cmd_eq_create"
-				     "Failedfor EQ\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : beiscsi_cmd_eq_create"
+				    "Failed for EQ\n");
 			goto create_eq_error;
 		}
-		SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
+
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : eqid = %d\n",
+			    phwi_context->be_eq[i].q.id);
 	}
 	return 0;
 create_eq_error:
@@ -2717,7 +2858,7 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 			     struct hwi_context_memory *phwi_context)
 {
 	unsigned int i, num_cq_pages;
-	int ret;
+	int ret = 0;
 	struct be_queue_info *cq, *eq;
 	struct be_dma_mem *mem;
 	struct be_eq_obj *pbe_eq;
@@ -2742,8 +2883,9 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 		ret = be_fill_queue(cq, phba->params.num_cq_entries,
 				    sizeof(struct sol_cqe), cq_vaddress);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "be_fill_queue Failed for ISCSI CQ\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : be_fill_queue Failed "
+				    "for ISCSI CQ\n");
 			goto create_cq_error;
 		}
 
@@ -2751,14 +2893,14 @@ static int beiscsi_create_cqs(struct beiscsi_hba *phba,
 		ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
 					    false, 0);
 		if (ret) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "beiscsi_cmd_eq_create"
-				     "Failed for ISCSI CQ\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : beiscsi_cmd_eq_create"
+				    "Failed for ISCSI CQ\n");
 			goto create_cq_error;
 		}
-		SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
-						 cq->id, eq->id);
-		SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : iscsi cq_id is %d for eq_id %d\n"
+			    "iSCSI CQ CREATED\n", cq->id, eq->id);
 	}
 	return 0;
 
@@ -2799,8 +2941,8 @@ beiscsi_create_def_hdr(struct beiscsi_hba *phba,
 			    sizeof(struct phys_addr),
 			    sizeof(struct phys_addr), dq_vaddress);
 	if (ret) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "be_fill_queue Failed for DEF PDU HDR\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : be_fill_queue Failed for DEF PDU HDR\n");
 		return ret;
 	}
 	mem->dma = (unsigned long)mem_descr->mem_array[idx].
@@ -2809,13 +2951,15 @@ beiscsi_create_def_hdr(struct beiscsi_hba *phba,
 					      def_pdu_ring_sz,
 					      phba->params.defpdu_hdr_sz);
 	if (ret) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n");
 		return ret;
 	}
 	phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
-	SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
-		 phwi_context->be_def_hdrq.id);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : iscsi def pdu id is %d\n",
+		    phwi_context->be_def_hdrq.id);
+
 	hwi_post_async_buffers(phba, 1);
 	return 0;
 }
@@ -2844,8 +2988,8 @@ beiscsi_create_def_data(struct beiscsi_hba *phba,
 			    sizeof(struct phys_addr),
 			    sizeof(struct phys_addr), dq_vaddress);
 	if (ret) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "be_fill_queue Failed for DEF PDU DATA\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : be_fill_queue Failed for DEF PDU DATA\n");
 		return ret;
 	}
 	mem->dma = (unsigned long)mem_descr->mem_array[idx].
@@ -2854,16 +2998,20 @@ beiscsi_create_def_data(struct beiscsi_hba *phba,
 					      def_pdu_ring_sz,
 					      phba->params.defpdu_data_sz);
 	if (ret) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "be_cmd_create_default_pdu_queue Failed"
-			     " for DEF PDU DATA\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d be_cmd_create_default_pdu_queue"
+			    " Failed for DEF PDU DATA\n");
 		return ret;
 	}
 	phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
-	SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
-		 phwi_context->be_def_dataq.id);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : iscsi def data id is %d\n",
+		    phwi_context->be_def_dataq.id);
+
 	hwi_post_async_buffers(phba, 0);
-	SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : DEFAULT PDU DATA RING CREATED\n");
+
 	return 0;
 }
 
@@ -2889,13 +3037,14 @@ beiscsi_post_pages(struct beiscsi_hba *phba)
 						(pm_arr->size / PAGE_SIZE));
 		page_offset += pm_arr->size / PAGE_SIZE;
 		if (status != 0) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "post sgl failed.\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : post sgl failed.\n");
 			return status;
 		}
 		pm_arr++;
 	}
-	SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : POSTED PAGES\n");
 	return 0;
 }
 
@@ -2945,8 +3094,8 @@ beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
 	pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
 			   GFP_KERNEL);
 	if (!pwrb_arr) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Memory alloc failed in create wrb ring.\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Memory alloc failed in create wrb ring.\n");
 		return -ENOMEM;
 	}
 	wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
@@ -2990,8 +3139,8 @@ beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
 		status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
 					    &phwi_context->be_wrbq[i]);
 		if (status != 0) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "wrbq create failed.");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : wrbq create failed.");
 			kfree(pwrb_arr);
 			return status;
 		}
@@ -3127,7 +3276,6 @@ static int find_num_cpus(void)
 	if (num_cpus >= MAX_CPUS)
 		num_cpus = MAX_CPUS - 1;
 
-	SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
 	return num_cpus;
 }
 
@@ -3150,7 +3298,8 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 
 	status = beiscsi_create_eqs(phba, phwi_context);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : EQ not created\n");
 		goto error;
 	}
 
@@ -3160,51 +3309,55 @@ static int hwi_init_port(struct beiscsi_hba *phba)
 
 	status = mgmt_check_supported_fw(ctrl, phba);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Unsupported fw version\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Unsupported fw version\n");
 		goto error;
 	}
 
 	status = beiscsi_create_cqs(phba, phwi_context);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : CQ not created\n");
 		goto error;
 	}
 
 	status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
 					def_pdu_ring_sz);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Default Header not created\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Default Header not created\n");
 		goto error;
 	}
 
 	status = beiscsi_create_def_data(phba, phwi_context,
 					 phwi_ctrlr, def_pdu_ring_sz);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Default Data not created\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Default Data not created\n");
 		goto error;
 	}
 
 	status = beiscsi_post_pages(phba);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Post SGL Pages Failed\n");
 		goto error;
 	}
 
 	status = beiscsi_create_wrb_rings(phba,	phwi_context, phwi_ctrlr);
 	if (status != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "WRB Rings not created\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : WRB Rings not created\n");
 		goto error;
 	}
 
-	SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : hwi_init_port success\n");
 	return 0;
 
 error:
-	shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
+	beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+		    "BM_%d : hwi_init_port failed");
 	hwi_cleanup(phba);
 	return status;
 }
@@ -3217,12 +3370,13 @@ static int hwi_init_controller(struct beiscsi_hba *phba)
 	if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
 		phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
 		    init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
-		SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
-			 phwi_ctrlr->phwi_ctxt);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
+			    phwi_ctrlr->phwi_ctxt);
 	} else {
-		shost_printk(KERN_ERR, phba->shost,
-			     "HWI_MEM_ADDN_CONTEXT is more than one element."
-			     "Failing to load\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
+			    "than one element.Failing to load\n");
 		return -ENOMEM;
 	}
 
@@ -3232,8 +3386,9 @@ static int hwi_init_controller(struct beiscsi_hba *phba)
 
 	hwi_init_async_pdu_ctx(phba);
 	if (hwi_init_port(phba) != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "hwi_init_controller failed\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : hwi_init_controller failed\n");
+
 		return -ENOMEM;
 	}
 	return 0;
@@ -3268,15 +3423,18 @@ static int beiscsi_init_controller(struct beiscsi_hba *phba)
 
 	ret = beiscsi_get_memory(phba);
 	if (ret < 0) {
-		shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
-			     "Failed in beiscsi_alloc_memory\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe -"
+			    "Failed in beiscsi_alloc_memory\n");
 		return ret;
 	}
 
 	ret = hwi_init_controller(phba);
 	if (ret)
 		goto free_init;
-	SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : Return success from beiscsi_init_controller");
+
 	return 0;
 
 free_init:
@@ -3301,8 +3459,8 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
 						 phba->params.ios_per_ctrl,
 						 GFP_KERNEL);
 		if (!phba->io_sgl_hndl_base) {
-			shost_printk(KERN_ERR, phba->shost,
-				     "Mem Alloc Failed. Failing to load\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : Mem Alloc Failed. Failing to load\n");
 			return -ENOMEM;
 		}
 		phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
@@ -3311,14 +3469,14 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
 						 GFP_KERNEL);
 		if (!phba->eh_sgl_hndl_base) {
 			kfree(phba->io_sgl_hndl_base);
-			shost_printk(KERN_ERR, phba->shost,
-				     "Mem Alloc Failed. Failing to load\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : Mem Alloc Failed. Failing to load\n");
 			return -ENOMEM;
 		}
 	} else {
-		shost_printk(KERN_ERR, phba->shost,
-			     "HWI_MEM_SGLH is more than one element."
-			     "Failing to load\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : HWI_MEM_SGLH is more than one element."
+			    "Failing to load\n");
 		return -ENOMEM;
 	}
 
@@ -3344,15 +3502,18 @@ static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
 		}
 		idx++;
 	}
-	SE_DEBUG(DBG_LVL_8,
-		 "phba->io_sgl_hndl_avbl=%d"
-		 "phba->eh_sgl_hndl_avbl=%d\n",
-		 phba->io_sgl_hndl_avbl,
-		 phba->eh_sgl_hndl_avbl);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : phba->io_sgl_hndl_avbl=%d"
+		    "phba->eh_sgl_hndl_avbl=%d\n",
+		    phba->io_sgl_hndl_avbl,
+		    phba->eh_sgl_hndl_avbl);
+
 	mem_descr_sg = phba->init_mem;
 	mem_descr_sg += HWI_MEM_SGE;
-	SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
-		 mem_descr_sg->num_elements);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "\n BM_%d : mem_descr_sg->num_elements=%d\n",
+		    mem_descr_sg->num_elements);
+
 	arr_index = 0;
 	idx = 0;
 	while (idx < mem_descr_sg->num_elements) {
@@ -3390,17 +3551,17 @@ static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
 	phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
 				  GFP_KERNEL);
 	if (!phba->cid_array) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Failed to allocate memory in "
-			     "hba_setup_cid_tbls\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Failed to allocate memory in "
+			    "hba_setup_cid_tbls\n");
 		return -ENOMEM;
 	}
 	phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
 				 phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
 	if (!phba->ep_array) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Failed to allocate memory in "
-			     "hba_setup_cid_tbls\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Failed to allocate memory in "
+			    "hba_setup_cid_tbls\n");
 		kfree(phba->cid_array);
 		return -ENOMEM;
 	}
@@ -3433,18 +3594,22 @@ static void hwi_enable_intr(struct beiscsi_hba *phba)
 	enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 	if (!enabled) {
 		reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
-		SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : reg =x%08x addr=%p\n", reg, addr);
 		iowrite32(reg, addr);
 	}
 
 	if (!phba->msix_enabled) {
 		eq = &phwi_context->be_eq[0].q;
-		SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : eq->id=%d\n", eq->id);
+
 		hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
 	} else {
 		for (i = 0; i <= phba->num_cpus; i++) {
 			eq = &phwi_context->be_eq[i].q;
-			SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
+			beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+				    "BM_%d : eq->id=%d\n", eq->id);
 			hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
 		}
 	}
@@ -3462,64 +3627,60 @@ static void hwi_disable_intr(struct beiscsi_hba *phba)
 		reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 		iowrite32(reg, addr);
 	} else
-		shost_printk(KERN_WARNING, phba->shost,
-			     "In hwi_disable_intr, Already Disabled\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : In hwi_disable_intr, Already Disabled\n");
 }
 
+/**
+ * beiscsi_get_boot_info()- Get the boot session info
+ * @phba: The device priv structure instance
+ *
+ * Get the boot target info and store in driver priv structure
+ *
+ * return values
+ *	Success: 0
+ *	Failure: Non-Zero Value
+ **/
 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
 {
-	struct be_cmd_get_boot_target_resp *boot_resp;
 	struct be_cmd_get_session_resp *session_resp;
 	struct be_mcc_wrb *wrb;
 	struct be_dma_mem nonemb_cmd;
 	unsigned int tag, wrb_num;
 	unsigned short status, extd_status;
+	unsigned int s_handle;
 	struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
 	int ret = -ENOMEM;
 
-	tag = mgmt_get_boot_target(phba);
-	if (!tag) {
-		SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed\n");
-		return -EAGAIN;
-	} else
-		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
-					 phba->ctrl.mcc_numtag[tag]);
-
-	wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
-	extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
-	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
-	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1, "beiscsi_get_boot_info Failed"
-				    " status = %d extd_status = %d\n",
-				    status, extd_status);
-		free_mcc_tag(&phba->ctrl, tag);
-		return -EBUSY;
-	}
-	wrb = queue_get_wrb(mccq, wrb_num);
-	free_mcc_tag(&phba->ctrl, tag);
-	boot_resp = embedded_payload(wrb);
-
-	if (boot_resp->boot_session_handle < 0) {
-		shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
-		return -ENXIO;
+	/* Get the session handle of the boot target */
+	ret = be_mgmt_get_boot_shandle(phba, &s_handle);
+	if (ret) {
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+			    "BM_%d : No boot session\n");
+		return ret;
 	}
-
 	nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 				sizeof(*session_resp),
 				&nonemb_cmd.dma);
 	if (nonemb_cmd.va == NULL) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Failed to allocate memory for"
-			 "beiscsi_get_session_info\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+			    "BM_%d : Failed to allocate memory for"
+			    "beiscsi_get_session_info\n");
+
 		return -ENOMEM;
 	}
 
 	memset(nonemb_cmd.va, 0, sizeof(*session_resp));
-	tag = mgmt_get_session_info(phba, boot_resp->boot_session_handle,
+	tag = mgmt_get_session_info(phba, s_handle,
 				    &nonemb_cmd);
 	if (!tag) {
-		SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
-			" Failed\n");
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+			    "BM_%d : beiscsi_get_session_info"
+			    " Failed\n");
+
 		goto boot_freemem;
 	} else
 		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
@@ -3529,9 +3690,12 @@ static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
 	extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
 	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
 	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
-				    " status = %d extd_status = %d\n",
-				    status, extd_status);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+			    "BM_%d : beiscsi_get_session_info Failed"
+			    " status = %d extd_status = %d\n",
+			    status, extd_status);
+
 		free_mcc_tag(&phba->ctrl, tag);
 		goto boot_freemem;
 	}
@@ -3611,22 +3775,22 @@ static int beiscsi_init_port(struct beiscsi_hba *phba)
 
 	ret = beiscsi_init_controller(phba);
 	if (ret < 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "beiscsi_dev_probe - Failed in"
-			     "beiscsi_init_controller\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe - Failed in"
+			    "beiscsi_init_controller\n");
 		return ret;
 	}
 	ret = beiscsi_init_sgl_handle(phba);
 	if (ret < 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "beiscsi_dev_probe - Failed in"
-			     "beiscsi_init_sgl_handle\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe - Failed in"
+			    "beiscsi_init_sgl_handle\n");
 		goto do_cleanup_ctrlr;
 	}
 
 	if (hba_setup_cid_tbls(phba)) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Failed in hba_setup_cid_tbls\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Failed in hba_setup_cid_tbls\n");
 		kfree(phba->io_sgl_hndl_base);
 		kfree(phba->eh_sgl_hndl_base);
 		goto do_cleanup_ctrlr;
@@ -3678,8 +3842,8 @@ static void beiscsi_clean_port(struct beiscsi_hba *phba)
 
 	mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
 	if (mgmt_status)
-		shost_printk(KERN_WARNING, phba->shost,
-			     "mgmt_epfw_cleanup FAILED\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BM_%d : mgmt_epfw_cleanup FAILED\n");
 
 	hwi_purge_eq(phba);
 	hwi_cleanup(phba);
@@ -3960,7 +4124,9 @@ free_hndls:
 	pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
 		      io_task->bhs_pa.u.a64.address);
 	io_task->cmd_bhs = NULL;
-	SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
+	beiscsi_log(phba, KERN_ERR,
+		    BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
+		    "BM_%d : Alloc of SGL_ICD Failed\n");
 	return -ENOMEM;
 }
 
@@ -3981,15 +4147,6 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
 	io_task->bhs_len = sizeof(struct be_cmd_bhs);
 
 	if (writedir) {
-		memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
-		AMAP_SET_BITS(struct amap_pdu_data_out, itt,
-			      &io_task->cmd_bhs->iscsi_data_pdu,
-			      (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
-		AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
-			      &io_task->cmd_bhs->iscsi_data_pdu,
-			      ISCSI_OPCODE_SCSI_DATA_OUT);
-		AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
-			      &io_task->cmd_bhs->iscsi_data_pdu, 1);
 		AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
 			      INI_WR_CMD);
 		AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
@@ -3998,9 +4155,6 @@ static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
 			      INI_RD_CMD);
 		AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
 	}
-	memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
-	       dw[offsetof(struct amap_pdu_data_out, lun) / 32],
-	       &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
 
 	AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
 		      cpu_to_be16(*(unsigned short *)
@@ -4090,8 +4244,10 @@ static int beiscsi_mtask(struct iscsi_task *task)
 		break;
 
 	default:
-		SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
-			 task->hdr->opcode & ISCSI_OPCODE_MASK);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BM_%d : opcode =%d Not supported\n",
+			    task->hdr->opcode & ISCSI_OPCODE_MASK);
+
 		return -EINVAL;
 	}
 
@@ -4123,17 +4279,22 @@ static int beiscsi_task_xmit(struct iscsi_task *task)
 	io_task->scsi_cmnd = sc;
 	num_sg = scsi_dma_map(sc);
 	if (num_sg < 0) {
-		SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
+		struct iscsi_conn *conn = task->conn;
+		struct beiscsi_hba *phba = NULL;
+
+		phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO,
+			    "BM_%d : scsi_dma_map Failed\n");
+
 		return num_sg;
 	}
 	xferlen = scsi_bufflen(sc);
 	sg = scsi_sglist(sc);
-	if (sc->sc_data_direction == DMA_TO_DEVICE) {
+	if (sc->sc_data_direction == DMA_TO_DEVICE)
 		writedir = 1;
-		SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
-			 task->imm_count);
-	} else
+	 else
 		writedir = 0;
+
 	return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
 }
 
@@ -4162,14 +4323,17 @@ static int beiscsi_bsg_request(struct bsg_job *job)
 					job->request_payload.payload_len,
 					&nonemb_cmd.dma);
 		if (nonemb_cmd.va == NULL) {
-			SE_DEBUG(DBG_LVL_1, "Failed to allocate memory for "
-				 "beiscsi_bsg_request\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BM_%d : Failed to allocate memory for "
+				    "beiscsi_bsg_request\n");
 			return -EIO;
 		}
 		tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
 						  &nonemb_cmd);
 		if (!tag) {
-			SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BM_%d : be_cmd_get_mac_addr Failed\n");
+
 			pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 					    nonemb_cmd.va, nonemb_cmd.dma);
 			return -EAGAIN;
@@ -4191,22 +4355,31 @@ static int beiscsi_bsg_request(struct bsg_job *job)
 		pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 				    nonemb_cmd.va, nonemb_cmd.dma);
 		if (status || extd_status) {
-			SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
-				 " status = %d extd_status = %d\n",
-				 status, extd_status);
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				    "BM_%d : be_cmd_get_mac_addr Failed"
+				    " status = %d extd_status = %d\n",
+				    status, extd_status);
+
 			return -EIO;
 		}
 		break;
 
 	default:
-		SE_DEBUG(DBG_LVL_1, "Unsupported bsg command: 0x%x\n",
-			 bsg_req->msgcode);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+				"BM_%d : Unsupported bsg command: 0x%x\n",
+				bsg_req->msgcode);
 		break;
 	}
 
 	return rc;
 }
 
+void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
+{
+	/* Set the logging parameter */
+	beiscsi_log_enable_init(phba, beiscsi_log_enable);
+}
+
 static void beiscsi_quiesce(struct beiscsi_hba *phba)
 {
 	struct hwi_controller *phwi_ctrlr;
@@ -4316,18 +4489,21 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 
 	ret = beiscsi_enable_pci(pcidev);
 	if (ret < 0) {
-		dev_err(&pcidev->dev, "beiscsi_dev_probe-"
-			" Failed to enable pci device\n");
+		dev_err(&pcidev->dev,
+			"beiscsi_dev_probe - Failed to enable pci device\n");
 		return ret;
 	}
 
 	phba = beiscsi_hba_alloc(pcidev);
 	if (!phba) {
-		dev_err(&pcidev->dev, "beiscsi_dev_probe-"
-			" Failed in beiscsi_hba_alloc\n");
+		dev_err(&pcidev->dev,
+			"beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
 		goto disable_pci;
 	}
 
+	/* Initialize Driver configuration Paramters */
+	beiscsi_hba_attrs_init(phba);
+
 	switch (pcidev->device) {
 	case BE_DEVICE_ID1:
 	case OC_DEVICE_ID1:
@@ -4347,7 +4523,9 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 	else
 		num_cpus = 1;
 	phba->num_cpus = num_cpus;
-	SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "BM_%d : num_cpus = %d\n",
+		    phba->num_cpus);
 
 	if (enable_msix) {
 		beiscsi_msix_enable(phba);
@@ -4356,8 +4534,9 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 	}
 	ret = be_ctrl_init(phba, pcidev);
 	if (ret) {
-		shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
-				"Failed in be_ctrl_init\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe-"
+			    "Failed in be_ctrl_init\n");
 		goto hba_free;
 	}
 
@@ -4366,19 +4545,19 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 		value = readl((void *)real_offset);
 		if (value & 0x00010000) {
 			gcrashmode++;
-			shost_printk(KERN_ERR, phba->shost,
-				"Loading Driver in crashdump mode\n");
+			beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+				    "BM_%d : Loading Driver in crashdump mode\n");
 			ret = beiscsi_cmd_reset_function(phba);
 			if (ret) {
-				shost_printk(KERN_ERR, phba->shost,
-					"Reset Failed. Aborting Crashdump\n");
+				beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+					    "BM_%d : Reset Failed. Aborting Crashdump\n");
 				goto hba_free;
 			}
 			ret = be_chk_reset_complete(phba);
 			if (ret) {
-				shost_printk(KERN_ERR, phba->shost,
-					"Failed to get out of reset."
-					"Aborting Crashdump\n");
+				beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+					    "BM_%d : Failed to get out of reset."
+					    "Aborting Crashdump\n");
 				goto hba_free;
 			}
 		} else {
@@ -4393,8 +4572,8 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 	spin_lock_init(&phba->isr_lock);
 	ret = mgmt_get_fw_config(&phba->ctrl, phba);
 	if (ret != 0) {
-		shost_printk(KERN_ERR, phba->shost,
-			     "Error getting fw config\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Error getting fw config\n");
 		goto free_port;
 	}
 	phba->shost->max_id = phba->fw_config.iscsi_cid_count;
@@ -4402,8 +4581,9 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 	phba->shost->can_queue = phba->params.ios_per_ctrl;
 	ret = beiscsi_init_port(phba);
 	if (ret < 0) {
-		shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
-			     "Failed in beiscsi_init_port\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe-"
+			    "Failed in beiscsi_init_port\n");
 		goto free_port;
 	}
 
@@ -4420,8 +4600,9 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 		 phba->shost->host_no);
 	phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
 	if (!phba->wq) {
-		shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
-				"Failed to allocate work queue\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe-"
+			    "Failed to allocate work queue\n");
 		goto free_twq;
 	}
 
@@ -4439,8 +4620,9 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 	}
 	ret = beiscsi_init_irqs(phba);
 	if (ret < 0) {
-		shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
-			     "Failed to beiscsi_init_irqs\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : beiscsi_dev_probe-"
+			    "Failed to beiscsi_init_irqs\n");
 		goto free_blkenbld;
 	}
 	hwi_enable_intr(phba);
@@ -4450,11 +4632,13 @@ static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
 		 * log error but continue, because we may not be using
 		 * iscsi boot.
 		 */
-		shost_printk(KERN_ERR, phba->shost, "Could not set up "
-			     "iSCSI boot info.\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BM_%d : Could not set up "
+			    "iSCSI boot info.\n");
 
 	beiscsi_create_def_ifaces(phba);
-	SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+		    "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
 	return 0;
 
 free_blkenbld:
@@ -4542,19 +4726,17 @@ static int __init beiscsi_module_init(void)
 	beiscsi_scsi_transport =
 			iscsi_register_transport(&beiscsi_iscsi_transport);
 	if (!beiscsi_scsi_transport) {
-		SE_DEBUG(DBG_LVL_1,
-			 "beiscsi_module_init - Unable to  register beiscsi"
-			 "transport.\n");
+		printk(KERN_ERR
+		       "beiscsi_module_init - Unable to  register beiscsi transport.\n");
 		return -ENOMEM;
 	}
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
-		 &beiscsi_iscsi_transport);
+	printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
+	       &beiscsi_iscsi_transport);
 
 	ret = pci_register_driver(&beiscsi_pci_driver);
 	if (ret) {
-		SE_DEBUG(DBG_LVL_1,
-			 "beiscsi_module_init - Unable to  register"
-			 "beiscsi pci driver.\n");
+		printk(KERN_ERR
+		       "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
 		goto unregister_iscsi_transport;
 	}
 	return 0;
diff --git a/drivers/scsi/be2iscsi/be_main.h b/drivers/scsi/be2iscsi/be_main.h
index 40fea6ec879..b8912263ef4 100644
--- a/drivers/scsi/be2iscsi/be_main.h
+++ b/drivers/scsi/be2iscsi/be_main.h
@@ -24,6 +24,8 @@
 #include <linux/pci.h>
 #include <linux/if_ether.h>
 #include <linux/in.h>
+#include <linux/ctype.h>
+#include <linux/module.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
@@ -34,7 +36,7 @@
 
 #include "be.h"
 #define DRV_NAME		"be2iscsi"
-#define BUILD_STR		"4.2.162.0"
+#define BUILD_STR		"4.4.58.0"
 #define BE_NAME			"Emulex OneConnect" \
 				"Open-iSCSI Driver version" BUILD_STR
 #define DRV_DESC		BE_NAME " " "Driver"
@@ -84,23 +86,7 @@
 #define MAX_CMD_SZ			65536
 #define IIOC_SCSI_DATA                  0x05	/* Write Operation */
 
-#define DBG_LVL				0x00000001
-#define DBG_LVL_1			0x00000001
-#define DBG_LVL_2			0x00000002
-#define DBG_LVL_3			0x00000004
-#define DBG_LVL_4			0x00000008
-#define DBG_LVL_5			0x00000010
-#define DBG_LVL_6			0x00000020
-#define DBG_LVL_7			0x00000040
-#define DBG_LVL_8			0x00000080
-
-#define SE_DEBUG(debug_mask, fmt, args...)		\
-do {							\
-	if (debug_mask & DBG_LVL) {			\
-		printk(KERN_ERR "(%s():%d):", __func__, __LINE__);\
-		printk(fmt, ##args);			\
-	}						\
-} while (0);
+#define INVALID_SESS_HANDLE	0xFFFFFFFF
 
 #define BE_ADAPTER_UP		0x00000000
 #define BE_ADAPTER_LINK_DOWN	0x00000001
@@ -351,6 +337,8 @@ struct beiscsi_hba {
 	struct mgmt_session_info boot_sess;
 	struct invalidate_command_table inv_tbl[128];
 
+	unsigned int attr_log_enable;
+
 };
 
 struct beiscsi_session {
@@ -860,4 +848,20 @@ struct hwi_context_memory {
 	struct hwi_async_pdu_context *pasync_ctx;
 };
 
+/* Logging related definitions */
+#define BEISCSI_LOG_INIT	0x0001	/* Initialization events */
+#define BEISCSI_LOG_MBOX	0x0002	/* Mailbox Events */
+#define BEISCSI_LOG_MISC	0x0004	/* Miscllaneous Events */
+#define BEISCSI_LOG_EH		0x0008	/* Error Handler */
+#define BEISCSI_LOG_IO		0x0010	/* IO Code Path */
+#define BEISCSI_LOG_CONFIG	0x0020	/* CONFIG Code Path */
+
+#define beiscsi_log(phba, level, mask, fmt, arg...) \
+do { \
+	uint32_t log_value = phba->attr_log_enable; \
+		if (((mask) & log_value) || (level[1] <= '3')) \
+			shost_printk(level, phba->shost, \
+				     fmt, __LINE__, ##arg); \
+} while (0)
+
 #endif
diff --git a/drivers/scsi/be2iscsi/be_mgmt.c b/drivers/scsi/be2iscsi/be_mgmt.c
index 2a096795b9a..aab5dd359e2 100644
--- a/drivers/scsi/be2iscsi/be_mgmt.c
+++ b/drivers/scsi/be2iscsi/be_mgmt.c
@@ -23,6 +23,53 @@
 #include "be_mgmt.h"
 #include "be_iscsi.h"
 
+/**
+ * mgmt_reopen_session()- Reopen a session based on reopen_type
+ * @phba: Device priv structure instance
+ * @reopen_type: Type of reopen_session FW should do.
+ * @sess_handle: Session Handle of the session to be re-opened
+ *
+ * return
+ *	the TAG used for MBOX Command
+ *
+ **/
+unsigned int mgmt_reopen_session(struct beiscsi_hba *phba,
+				  unsigned int reopen_type,
+				  unsigned int sess_handle)
+{
+	struct be_ctrl_info *ctrl = &phba->ctrl;
+	struct be_mcc_wrb *wrb;
+	struct be_cmd_reopen_session_req *req;
+	unsigned int tag = 0;
+
+	beiscsi_log(phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+		    "BG_%d : In bescsi_get_boot_target\n");
+
+	spin_lock(&ctrl->mbox_lock);
+	tag = alloc_mcc_tag(phba);
+	if (!tag) {
+		spin_unlock(&ctrl->mbox_lock);
+		return tag;
+	}
+
+	wrb = wrb_from_mccq(phba);
+	req = embedded_payload(wrb);
+	wrb->tag0 |= tag;
+	be_wrb_hdr_prepare(wrb, sizeof(*req), true, 0);
+	be_cmd_hdr_prepare(&req->hdr, CMD_SUBSYSTEM_ISCSI_INI,
+			   OPCODE_ISCSI_INI_DRIVER_REOPEN_ALL_SESSIONS,
+			   sizeof(struct be_cmd_reopen_session_resp));
+
+	/* set the reopen_type,sess_handle */
+	req->reopen_type = reopen_type;
+	req->session_handle = sess_handle;
+
+	be_mcc_notify(phba);
+	spin_unlock(&ctrl->mbox_lock);
+	return tag;
+}
+
 unsigned int mgmt_get_boot_target(struct beiscsi_hba *phba)
 {
 	struct be_ctrl_info *ctrl = &phba->ctrl;
@@ -30,7 +77,10 @@ unsigned int mgmt_get_boot_target(struct beiscsi_hba *phba)
 	struct be_cmd_get_boot_target_req *req;
 	unsigned int tag = 0;
 
-	SE_DEBUG(DBG_LVL_8, "In bescsi_get_boot_target\n");
+	beiscsi_log(phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+		    "BG_%d : In bescsi_get_boot_target\n");
+
 	spin_lock(&ctrl->mbox_lock);
 	tag = alloc_mcc_tag(phba);
 	if (!tag) {
@@ -62,7 +112,10 @@ unsigned int mgmt_get_session_info(struct beiscsi_hba *phba,
 	struct be_cmd_get_session_resp *resp;
 	struct be_sge *sge;
 
-	SE_DEBUG(DBG_LVL_8, "In beiscsi_get_session_info\n");
+	beiscsi_log(phba, KERN_INFO,
+		    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+		    "BG_%d : In beiscsi_get_session_info\n");
+
 	spin_lock(&ctrl->mbox_lock);
 	tag = alloc_mcc_tag(phba);
 	if (!tag) {
@@ -121,16 +174,16 @@ int mgmt_get_fw_config(struct be_ctrl_info *ctrl,
 		phba->fw_config.iscsi_cid_count =
 					pfw_cfg->ulp[0].sq_count;
 		if (phba->fw_config.iscsi_cid_count > (BE2_MAX_SESSIONS / 2)) {
-			SE_DEBUG(DBG_LVL_8,
-				"FW reported MAX CXNS as %d\t"
-				"Max Supported = %d.\n",
-				phba->fw_config.iscsi_cid_count,
-				BE2_MAX_SESSIONS);
+			beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+				    "BG_%d : FW reported MAX CXNS as %d\t"
+				    "Max Supported = %d.\n",
+				    phba->fw_config.iscsi_cid_count,
+				    BE2_MAX_SESSIONS);
 			phba->fw_config.iscsi_cid_count = BE2_MAX_SESSIONS / 2;
 		}
 	} else {
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Failed in mgmt_get_fw_config\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BG_%d : Failed in mgmt_get_fw_config\n");
 	}
 
 	spin_unlock(&ctrl->mbox_lock);
@@ -150,9 +203,9 @@ int mgmt_check_supported_fw(struct be_ctrl_info *ctrl,
 				sizeof(struct be_mgmt_controller_attributes),
 				&nonemb_cmd.dma);
 	if (nonemb_cmd.va == NULL) {
-		SE_DEBUG(DBG_LVL_1,
-			 "Failed to allocate memory for mgmt_check_supported_fw"
-			 "\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d : Failed to allocate memory for "
+			    "mgmt_check_supported_fw\n");
 		return -ENOMEM;
 	}
 	nonemb_cmd.size = sizeof(struct be_mgmt_controller_attributes);
@@ -169,18 +222,23 @@ int mgmt_check_supported_fw(struct be_ctrl_info *ctrl,
 	status = be_mbox_notify(ctrl);
 	if (!status) {
 		struct be_mgmt_controller_attributes_resp *resp = nonemb_cmd.va;
-		SE_DEBUG(DBG_LVL_8, "Firmware version of CMD: %s\n",
-			resp->params.hba_attribs.flashrom_version_string);
-		SE_DEBUG(DBG_LVL_8, "Firmware version is : %s\n",
-			resp->params.hba_attribs.firmware_version_string);
-		SE_DEBUG(DBG_LVL_8,
-			"Developer Build, not performing version check...\n");
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BG_%d : Firmware Version of CMD : %s\n"
+			    "Firmware Version is : %s\n"
+			    "Developer Build, not performing version check...\n",
+			    resp->params.hba_attribs
+			    .flashrom_version_string,
+			    resp->params.hba_attribs.
+			    firmware_version_string);
+
 		phba->fw_config.iscsi_features =
 				resp->params.hba_attribs.iscsi_features;
-		SE_DEBUG(DBG_LVL_8, " phba->fw_config.iscsi_features = %d\n",
-				      phba->fw_config.iscsi_features);
+		beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
+			    "BM_%d : phba->fw_config.iscsi_features = %d\n",
+			    phba->fw_config.iscsi_features);
 	} else
-		SE_DEBUG(DBG_LVL_1, " Failed in mgmt_check_supported_fw\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
+			    "BG_%d :  Failed in mgmt_check_supported_fw\n");
 	spin_unlock(&ctrl->mbox_lock);
 	if (nonemb_cmd.va)
 		pci_free_consistent(ctrl->pdev, nonemb_cmd.size,
@@ -229,9 +287,10 @@ unsigned int mgmt_vendor_specific_fw_cmd(struct be_ctrl_info *ctrl,
 			   OPCODE_COMMON_READ_FLASH, sizeof(*req));
 		break;
 	default:
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Unsupported cmd = 0x%x\n\n", bsg_req->rqst_data.
-			     h_vendor.vendor_cmd[0]);
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Unsupported cmd = 0x%x\n\n",
+			    bsg_req->rqst_data.h_vendor.vendor_cmd[0]);
+
 		spin_unlock(&ctrl->mbox_lock);
 		return -ENOSYS;
 	}
@@ -275,8 +334,8 @@ int mgmt_epfw_cleanup(struct beiscsi_hba *phba, unsigned short chute)
 
 	status =  be_mcc_notify_wait(phba);
 	if (status)
-		shost_printk(KERN_WARNING, phba->shost,
-			     " mgmt_epfw_cleanup , FAILED\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
+			    "BG_%d : mgmt_epfw_cleanup , FAILED\n");
 	spin_unlock(&ctrl->mbox_lock);
 	return status;
 }
@@ -459,8 +518,9 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 		       &daddr_in6->sin6_addr.in6_u.u6_addr8, 16);
 		beiscsi_ep->ip_type = BE2_IPV6;
 	} else{
-		shost_printk(KERN_ERR, phba->shost, "unknown addr family %d\n",
-			     dst_addr->sa_family);
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BG_%d : unknown addr family %d\n",
+			    dst_addr->sa_family);
 		spin_unlock(&ctrl->mbox_lock);
 		free_mcc_tag(&phba->ctrl, tag);
 		return -EINVAL;
@@ -471,7 +531,8 @@ int mgmt_open_connection(struct beiscsi_hba *phba,
 	if (phba->nxt_cqid == phba->num_cpus)
 		phba->nxt_cqid = 0;
 	req->cq_id = phwi_context->be_cq[i].id;
-	SE_DEBUG(DBG_LVL_8, "i=%d cq_id=%d\n", i, req->cq_id);
+	beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
+		    "BG_%d : i=%d cq_id=%d\n", i, req->cq_id);
 	req->defq_id = def_hdr_id;
 	req->hdr_ring_id = def_hdr_id;
 	req->data_ring_id = def_data_id;
@@ -506,8 +567,8 @@ unsigned int mgmt_get_all_if_id(struct beiscsi_hba *phba)
 	if (!status)
 		phba->interface_handle = pbe_allid->if_hndl_list[0];
 	else {
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Failed in mgmt_get_all_if_id\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed in mgmt_get_all_if_id\n");
 	}
 	spin_unlock(&ctrl->mbox_lock);
 
@@ -550,9 +611,10 @@ static int mgmt_exec_nonemb_cmd(struct beiscsi_hba *phba,
 	extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
 	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
 	if (status || extd_status) {
-		SE_DEBUG(DBG_LVL_1,
-			 "mgmt_exec_nonemb_cmd Failed status = %d"
-			 "extd_status = %d\n", status, extd_status);
+		beiscsi_log(phba, KERN_ERR,
+			    BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
+			    "BG_%d : mgmt_exec_nonemb_cmd Failed status = %d"
+			    "extd_status = %d\n", status, extd_status);
 		rc = -EIO;
 		goto free_tag;
 	}
@@ -573,7 +635,8 @@ static int mgmt_alloc_cmd_data(struct beiscsi_hba *phba, struct be_dma_mem *cmd,
 {
 	cmd->va = pci_alloc_consistent(phba->ctrl.pdev, size, &cmd->dma);
 	if (!cmd->va) {
-		SE_DEBUG(DBG_LVL_1, "Failed to allocate memory for if info\n");
+		beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to allocate memory for if info\n");
 		return -ENOMEM;
 	}
 	memset(cmd->va, 0, size);
@@ -629,8 +692,8 @@ mgmt_static_ip_modify(struct beiscsi_hba *phba,
 
 	rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
 	if (rc < 0)
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Failed to Modify existing IP Address\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Modify existing IP Address\n");
 	return rc;
 }
 
@@ -684,8 +747,8 @@ int mgmt_set_ip(struct beiscsi_hba *phba,
 
 	if (boot_proto == ISCSI_BOOTPROTO_DHCP) {
 		if (if_info.dhcp_state) {
-			shost_printk(KERN_WARNING, phba->shost,
-				     "DHCP Already Enabled\n");
+			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+				    "BG_%d : DHCP Already Enabled\n");
 			return 0;
 		}
 		/* The ip_param->len is 1 in DHCP case. Setting
@@ -712,8 +775,9 @@ int mgmt_set_ip(struct beiscsi_hba *phba,
 
 			rc = mgmt_exec_nonemb_cmd(phba, &nonemb_cmd, NULL, 0);
 			if (rc < 0) {
-				shost_printk(KERN_WARNING, phba->shost,
-					     "Failed to Delete existing dhcp\n");
+				beiscsi_log(phba, KERN_WARNING,
+					    BEISCSI_LOG_CONFIG,
+					    "BG_%d : Failed to Delete existing dhcp\n");
 				return rc;
 			}
 		}
@@ -732,8 +796,8 @@ int mgmt_set_ip(struct beiscsi_hba *phba,
 		memset(&gtway_addr_set, 0, sizeof(gtway_addr_set));
 		rc = mgmt_get_gateway(phba, BE2_IPV4, &gtway_addr_set);
 		if (rc) {
-			shost_printk(KERN_WARNING, phba->shost,
-				     "Failed to Get Gateway Addr\n");
+			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+				    "BG_%d : Failed to Get Gateway Addr\n");
 			return rc;
 		}
 
@@ -743,8 +807,9 @@ int mgmt_set_ip(struct beiscsi_hba *phba,
 						 IP_ACTION_DEL, IP_V4_LEN);
 
 			if (rc) {
-				shost_printk(KERN_WARNING, phba->shost,
-					     "Failed to clear Gateway Addr Set\n");
+				beiscsi_log(phba, KERN_WARNING,
+					    BEISCSI_LOG_CONFIG,
+					    "BG_%d : Failed to clear Gateway Addr Set\n");
 				return rc;
 			}
 		}
@@ -783,8 +848,8 @@ int mgmt_set_gateway(struct beiscsi_hba *phba,
 	memset(&gtway_addr_set, 0, sizeof(gtway_addr_set));
 	rt_val = mgmt_get_gateway(phba, BE2_IPV4, &gtway_addr_set);
 	if (rt_val) {
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Failed to Get Gateway Addr\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Get Gateway Addr\n");
 		return rt_val;
 	}
 
@@ -793,8 +858,8 @@ int mgmt_set_gateway(struct beiscsi_hba *phba,
 		rt_val = mgmt_modify_gateway(phba, gtway_addr, IP_ACTION_DEL,
 					     gateway_param->len);
 		if (rt_val) {
-			shost_printk(KERN_WARNING, phba->shost,
-				     "Failed to clear Gateway Addr Set\n");
+			beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+				    "BG_%d : Failed to clear Gateway Addr Set\n");
 			return rt_val;
 		}
 	}
@@ -804,8 +869,8 @@ int mgmt_set_gateway(struct beiscsi_hba *phba,
 				     gateway_param->len);
 
 	if (rt_val)
-		shost_printk(KERN_WARNING, phba->shost,
-			     "Failed to Set Gateway Addr\n");
+		beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
+			    "BG_%d : Failed to Set Gateway Addr\n");
 
 	return rt_val;
 }
@@ -924,3 +989,150 @@ unsigned int be_cmd_get_port_speed(struct beiscsi_hba *phba)
 	spin_unlock(&ctrl->mbox_lock);
 	return tag;
 }
+
+/**
+ * be_mgmt_get_boot_shandle()- Get the session handle
+ * @phba: device priv structure instance
+ * @s_handle: session handle returned for boot session.
+ *
+ * Get the boot target session handle. In case of
+ * crashdump mode driver has to issue and MBX Cmd
+ * for FW to login to boot target
+ *
+ * return
+ *	Success: 0
+ *	Failure: Non-Zero value
+ *
+ **/
+int be_mgmt_get_boot_shandle(struct beiscsi_hba *phba,
+			      unsigned int *s_handle)
+{
+	struct be_cmd_get_boot_target_resp *boot_resp;
+	struct be_mcc_wrb *wrb;
+	unsigned int tag, wrb_num;
+	uint8_t boot_retry = 3;
+	unsigned short status, extd_status;
+	struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
+
+	do {
+		/* Get the Boot Target Session Handle and Count*/
+		tag = mgmt_get_boot_target(phba);
+		if (!tag) {
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_CONFIG | BEISCSI_LOG_INIT,
+				    "BG_%d : Getting Boot Target Info Failed\n");
+			return -EAGAIN;
+		} else
+			wait_event_interruptible(phba->ctrl.mcc_wait[tag],
+						 phba->ctrl.mcc_numtag[tag]);
+
+		wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
+		extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
+		status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
+		if (status || extd_status) {
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+				    "BG_%d : mgmt_get_boot_target Failed"
+				    " status = %d extd_status = %d\n",
+				    status, extd_status);
+			free_mcc_tag(&phba->ctrl, tag);
+			return -EBUSY;
+		}
+		wrb = queue_get_wrb(mccq, wrb_num);
+		free_mcc_tag(&phba->ctrl, tag);
+		boot_resp = embedded_payload(wrb);
+
+		/* Check if the there are any Boot targets configured */
+		if (!boot_resp->boot_session_count) {
+			beiscsi_log(phba, KERN_INFO,
+				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+				    "BG_%d  ;No boot targets configured\n");
+			return -ENXIO;
+		}
+
+		/* FW returns the session handle of the boot session */
+		if (boot_resp->boot_session_handle != INVALID_SESS_HANDLE) {
+			*s_handle = boot_resp->boot_session_handle;
+			return 0;
+		}
+
+		/* Issue MBX Cmd to FW to login to the boot target */
+		tag = mgmt_reopen_session(phba, BE_REOPEN_BOOT_SESSIONS,
+					  INVALID_SESS_HANDLE);
+		if (!tag) {
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+				    "BG_%d : mgmt_reopen_session Failed\n");
+			return -EAGAIN;
+		} else
+			wait_event_interruptible(phba->ctrl.mcc_wait[tag],
+						 phba->ctrl.mcc_numtag[tag]);
+
+		wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
+		extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
+		status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
+		if (status || extd_status) {
+			beiscsi_log(phba, KERN_ERR,
+				    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+				    "BG_%d : mgmt_reopen_session Failed"
+				    " status = %d extd_status = %d\n",
+				    status, extd_status);
+			free_mcc_tag(&phba->ctrl, tag);
+			return -EBUSY;
+		}
+		free_mcc_tag(&phba->ctrl, tag);
+
+	} while (--boot_retry);
+
+	/* Couldn't log into the boot target */
+	beiscsi_log(phba, KERN_ERR,
+		    BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
+		    "BG_%d : Login to Boot Target Failed\n");
+	return -ENXIO;
+}
+
+/**
+ * mgmt_set_vlan()- Issue and wait for CMD completion
+ * @phba: device private structure instance
+ * @vlan_tag: VLAN tag
+ *
+ * Issue the MBX Cmd and wait for the completion of the
+ * command.
+ *
+ * returns
+ *	Success: 0
+ *	Failure: Non-Xero Value
+ **/
+int mgmt_set_vlan(struct beiscsi_hba *phba,
+		   uint16_t vlan_tag)
+{
+	unsigned int tag, wrb_num;
+	unsigned short status, extd_status;
+
+	tag = be_cmd_set_vlan(phba, vlan_tag);
+	if (!tag) {
+		beiscsi_log(phba, KERN_ERR,
+			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
+			    "BG_%d : VLAN Setting Failed\n");
+		return -EBUSY;
+	} else
+		wait_event_interruptible(phba->ctrl.mcc_wait[tag],
+					 phba->ctrl.mcc_numtag[tag]);
+
+	wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
+	extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
+	status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
+
+	if (status || extd_status) {
+		beiscsi_log(phba, KERN_ERR,
+			    (BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX),
+			    "BS_%d : status : %d extd_status : %d\n",
+			    status, extd_status);
+
+		free_mcc_tag(&phba->ctrl, tag);
+		return -EAGAIN;
+	}
+
+	free_mcc_tag(&phba->ctrl, tag);
+	return 0;
+}
diff --git a/drivers/scsi/be2iscsi/be_mgmt.h b/drivers/scsi/be2iscsi/be_mgmt.h
index 5c2e37693ca..c50cef6fec0 100644
--- a/drivers/scsi/be2iscsi/be_mgmt.h
+++ b/drivers/scsi/be2iscsi/be_mgmt.h
@@ -108,6 +108,7 @@ unsigned int mgmt_vendor_specific_fw_cmd(struct be_ctrl_info *ctrl,
 					 struct bsg_job *job,
 					 struct be_dma_mem *nonemb_cmd);
 
+#define BEISCSI_NO_RST_ISSUE	0
 struct iscsi_invalidate_connection_params_in {
 	struct be_cmd_req_hdr hdr;
 	unsigned int session_handle;
@@ -274,6 +275,10 @@ int mgmt_set_ip(struct beiscsi_hba *phba,
 
 unsigned int mgmt_get_boot_target(struct beiscsi_hba *phba);
 
+unsigned int mgmt_reopen_session(struct beiscsi_hba *phba,
+				  unsigned int reopen_type,
+				  unsigned sess_handle);
+
 unsigned int mgmt_get_session_info(struct beiscsi_hba *phba,
 				   u32 boot_session_handle,
 				   struct be_dma_mem *nonemb_cmd);
@@ -290,4 +295,10 @@ int mgmt_get_gateway(struct beiscsi_hba *phba, int ip_type,
 int mgmt_set_gateway(struct beiscsi_hba *phba,
 		     struct iscsi_iface_param_info *gateway_param);
 
+int be_mgmt_get_boot_shandle(struct beiscsi_hba *phba,
+			      unsigned int *s_handle);
+
+unsigned int mgmt_get_all_if_id(struct beiscsi_hba *phba);
+
+int mgmt_set_vlan(struct beiscsi_hba *phba, uint16_t vlan_tag);
 #endif
diff --git a/drivers/scsi/bfa/bfa_core.c b/drivers/scsi/bfa/bfa_core.c
index 456e5762977..b7c326f7a6d 100644
--- a/drivers/scsi/bfa/bfa_core.c
+++ b/drivers/scsi/bfa/bfa_core.c
@@ -775,7 +775,8 @@ bfa_intx(struct bfa_s *bfa)
 	if (!intr)
 		return BFA_TRUE;
 
-	bfa_msix_lpu_err(bfa, intr);
+	if (bfa->intr_enabled)
+		bfa_msix_lpu_err(bfa, intr);
 
 	return BFA_TRUE;
 }
@@ -803,11 +804,17 @@ bfa_isr_enable(struct bfa_s *bfa)
 	writel(~umsk, bfa->iocfc.bfa_regs.intr_mask);
 	bfa->iocfc.intr_mask = ~umsk;
 	bfa_isr_mode_set(bfa, bfa->msix.nvecs != 0);
+
+	/*
+	 * Set the flag indicating successful enabling of interrupts
+	 */
+	bfa->intr_enabled = BFA_TRUE;
 }
 
 void
 bfa_isr_disable(struct bfa_s *bfa)
 {
+	bfa->intr_enabled = BFA_FALSE;
 	bfa_isr_mode_set(bfa, BFA_FALSE);
 	writel(-1L, bfa->iocfc.bfa_regs.intr_mask);
 	bfa_msix_uninstall(bfa);
@@ -1022,7 +1029,7 @@ bfa_iocfc_mem_claim(struct bfa_s *bfa, struct bfa_iocfc_cfg_s *cfg)
 {
 	u8	*dm_kva = NULL;
 	u64	dm_pa = 0;
-	int	i, per_reqq_sz, per_rspq_sz, dbgsz;
+	int	i, per_reqq_sz, per_rspq_sz;
 	struct bfa_iocfc_s  *iocfc = &bfa->iocfc;
 	struct bfa_mem_dma_s *ioc_dma = BFA_MEM_IOC_DMA(bfa);
 	struct bfa_mem_dma_s *iocfc_dma = BFA_MEM_IOCFC_DMA(bfa);
@@ -1083,11 +1090,8 @@ bfa_iocfc_mem_claim(struct bfa_s *bfa, struct bfa_iocfc_cfg_s *cfg)
 			BFA_CACHELINE_SZ);
 
 	/* Claim IOCFC kva memory */
-	dbgsz = (bfa_auto_recover) ? BFA_DBG_FWTRC_LEN : 0;
-	if (dbgsz > 0) {
-		bfa_ioc_debug_memclaim(&bfa->ioc, bfa_mem_kva_curp(iocfc));
-		bfa_mem_kva_curp(iocfc) += dbgsz;
-	}
+	bfa_ioc_debug_memclaim(&bfa->ioc, bfa_mem_kva_curp(iocfc));
+	bfa_mem_kva_curp(iocfc) += BFA_DBG_FWTRC_LEN;
 }
 
 /*
@@ -1429,8 +1433,7 @@ bfa_iocfc_meminfo(struct bfa_iocfc_cfg_s *cfg, struct bfa_meminfo_s *meminfo,
 	bfa_mem_dma_setup(meminfo, iocfc_dma, dm_len);
 
 	/* kva memory setup for IOCFC */
-	bfa_mem_kva_setup(meminfo, iocfc_kva,
-			((bfa_auto_recover) ? BFA_DBG_FWTRC_LEN : 0));
+	bfa_mem_kva_setup(meminfo, iocfc_kva, BFA_DBG_FWTRC_LEN);
 }
 
 /*
diff --git a/drivers/scsi/bfa/bfa_cs.h b/drivers/scsi/bfa/bfa_cs.h
index 12bfeed268e..91a8aa394db 100644
--- a/drivers/scsi/bfa/bfa_cs.h
+++ b/drivers/scsi/bfa/bfa_cs.h
@@ -168,7 +168,7 @@ __bfa_trc32(struct bfa_trc_mod_s *trcm, int fileno, int line, u32 data)
 /*
  * bfa_q_deq - dequeue an element from head of the queue
  */
-#define bfa_q_deq(_q, _qe) {						\
+#define bfa_q_deq(_q, _qe) do {						\
 	if (!list_empty(_q)) {						\
 		(*((struct list_head **) (_qe))) = bfa_q_next(_q);	\
 		bfa_q_prev(bfa_q_next(*((struct list_head **) _qe))) =	\
@@ -177,7 +177,7 @@ __bfa_trc32(struct bfa_trc_mod_s *trcm, int fileno, int line, u32 data)
 	} else {							\
 		*((struct list_head **) (_qe)) = (struct list_head *) NULL;\
 	}								\
-}
+} while (0)
 
 /*
  * bfa_q_deq_tail - dequeue an element from tail of the queue
diff --git a/drivers/scsi/bfa/bfa_defs_fcs.h b/drivers/scsi/bfa/bfa_defs_fcs.h
index 3bbc583f65c..06f0a163ca3 100644
--- a/drivers/scsi/bfa/bfa_defs_fcs.h
+++ b/drivers/scsi/bfa/bfa_defs_fcs.h
@@ -93,6 +93,7 @@ struct bfa_lport_cfg_s {
 	wwn_t	       pwwn;       /*  port wwn */
 	wwn_t	       nwwn;       /*  node wwn */
 	struct bfa_lport_symname_s  sym_name;   /*  vm port symbolic name */
+	struct bfa_lport_symname_s node_sym_name; /* Node symbolic name */
 	enum bfa_lport_role roles;      /* FCS port roles */
 	u32     rsvd;
 	bfa_boolean_t   preboot_vp;  /*  vport created from PBC */
@@ -192,6 +193,18 @@ struct bfa_lport_stats_s {
 	u32	ns_gidft_unknown_rsp;
 	u32	ns_gidft_alloc_wait;
 
+	u32	ns_rnnid_sent;
+	u32	ns_rnnid_accepts;
+	u32	ns_rnnid_rsp_err;
+	u32	ns_rnnid_rejects;
+	u32	ns_rnnid_alloc_wait;
+
+	u32	ns_rsnn_nn_sent;
+	u32	ns_rsnn_nn_accepts;
+	u32	ns_rsnn_nn_rsp_err;
+	u32	ns_rsnn_nn_rejects;
+	u32	ns_rsnn_nn_alloc_wait;
+
 	/*
 	 * Mgmt Server stats
 	 */
@@ -410,6 +423,11 @@ struct bfa_rport_remote_link_stats_s {
 	u32 icc; /*  Invalid CRC Count */
 };
 
+struct bfa_rport_qualifier_s {
+	wwn_t	pwwn;	/* Port WWN */
+	u32	pid;	/* port ID */
+	u32	rsvd;
+};
 
 #define BFA_MAX_IO_INDEX 7
 #define BFA_NO_IO_INDEX 9
diff --git a/drivers/scsi/bfa/bfa_fc.h b/drivers/scsi/bfa/bfa_fc.h
index 8d0b88f67a3..e0beb4d7e26 100644
--- a/drivers/scsi/bfa/bfa_fc.h
+++ b/drivers/scsi/bfa/bfa_fc.h
@@ -1279,6 +1279,7 @@ enum {
 	GS_GSPN_ID	= 0x0118,	/* Get symbolic PN on ID */
 	GS_RFT_ID	= 0x0217,	/* Register fc4type on ID */
 	GS_RSPN_ID	= 0x0218,	/* Register symbolic PN on ID */
+	GS_RSNN_NN	= 0x0239,	/* Register symbolic NN on NN */
 	GS_RPN_ID	= 0x0212,	/* Register port name */
 	GS_RNN_ID	= 0x0213,	/* Register node name */
 	GS_RCS_ID	= 0x0214,	/* Register class of service */
@@ -1357,6 +1358,15 @@ struct fcgs_rspnid_req_s {
 };
 
 /*
+ * RSNN_NN
+ */
+struct fcgs_rsnn_nn_req_s {
+	wwn_t	node_name;	/* Node name */
+	u8	snn_len;	/* symbolic node name length */
+	u8	snn[256];	/* symbolic node name */
+};
+
+/*
  * RPN_ID
  */
 struct fcgs_rpnid_req_s {
diff --git a/drivers/scsi/bfa/bfa_fcbuild.c b/drivers/scsi/bfa/bfa_fcbuild.c
index 17b59b8b564..273cee90b3b 100644
--- a/drivers/scsi/bfa/bfa_fcbuild.c
+++ b/drivers/scsi/bfa/bfa_fcbuild.c
@@ -1252,6 +1252,27 @@ fc_rspnid_build(struct fchs_s *fchs, void *pyld, u32 s_id, u16 ox_id,
 }
 
 u16
+fc_rsnn_nn_build(struct fchs_s *fchs, void *pyld, u32 s_id,
+			wwn_t node_name, u8 *name)
+{
+	struct ct_hdr_s *cthdr = (struct ct_hdr_s *) pyld;
+	struct fcgs_rsnn_nn_req_s *rsnn_nn =
+		(struct fcgs_rsnn_nn_req_s *) (cthdr + 1);
+	u32	d_id = bfa_hton3b(FC_NAME_SERVER);
+
+	fc_gs_fchdr_build(fchs, d_id, s_id, 0);
+	fc_gs_cthdr_build(cthdr, s_id, GS_RSNN_NN);
+
+	memset(rsnn_nn, 0, sizeof(struct fcgs_rsnn_nn_req_s));
+
+	rsnn_nn->node_name = node_name;
+	rsnn_nn->snn_len = (u8) strlen((char *)name);
+	strncpy((char *)rsnn_nn->snn, (char *)name, rsnn_nn->snn_len);
+
+	return sizeof(struct fcgs_rsnn_nn_req_s) + sizeof(struct ct_hdr_s);
+}
+
+u16
 fc_gid_ft_build(struct fchs_s *fchs, void *pyld, u32 s_id, u8 fc4_type)
 {
 
diff --git a/drivers/scsi/bfa/bfa_fcbuild.h b/drivers/scsi/bfa/bfa_fcbuild.h
index 42cd9d4da69..03c753d1e54 100644
--- a/drivers/scsi/bfa/bfa_fcbuild.h
+++ b/drivers/scsi/bfa/bfa_fcbuild.h
@@ -166,6 +166,8 @@ enum fc_parse_status fc_rrq_rsp_parse(struct fchs_s *buf, int len);
 
 u16        fc_rspnid_build(struct fchs_s *fchs, void *pld, u32 s_id,
 				u16 ox_id, u8 *name);
+u16	fc_rsnn_nn_build(struct fchs_s *fchs, void *pld, u32 s_id,
+				wwn_t node_name, u8 *name);
 
 u16        fc_rftid_build(struct fchs_s *fchs, void *pld, u32 s_id,
 			       u16 ox_id, enum bfa_lport_role role);
diff --git a/drivers/scsi/bfa/bfa_fcpim.c b/drivers/scsi/bfa/bfa_fcpim.c
index f0f80e282e3..1633963c66c 100644
--- a/drivers/scsi/bfa/bfa_fcpim.c
+++ b/drivers/scsi/bfa/bfa_fcpim.c
@@ -1466,7 +1466,13 @@ bfa_status_t
 bfa_itnim_get_ioprofile(struct bfa_itnim_s *itnim,
 			struct bfa_itnim_ioprofile_s *ioprofile)
 {
-	struct bfa_fcpim_s *fcpim = BFA_FCPIM(itnim->bfa);
+	struct bfa_fcpim_s *fcpim;
+
+	if (!itnim)
+		return BFA_STATUS_NO_FCPIM_NEXUS;
+
+	fcpim = BFA_FCPIM(itnim->bfa);
+
 	if (!fcpim->io_profile)
 		return BFA_STATUS_IOPROFILE_OFF;
 
@@ -1484,6 +1490,10 @@ void
 bfa_itnim_clear_stats(struct bfa_itnim_s *itnim)
 {
 	int j;
+
+	if (!itnim)
+		return;
+
 	memset(&itnim->stats, 0, sizeof(itnim->stats));
 	memset(&itnim->ioprofile, 0, sizeof(itnim->ioprofile));
 	for (j = 0; j < BFA_IOBUCKET_MAX; j++)
diff --git a/drivers/scsi/bfa/bfa_fcs.c b/drivers/scsi/bfa/bfa_fcs.c
index eaac57e1dde..fd3e84d32bd 100644
--- a/drivers/scsi/bfa/bfa_fcs.c
+++ b/drivers/scsi/bfa/bfa_fcs.c
@@ -76,6 +76,7 @@ bfa_fcs_attach(struct bfa_fcs_s *fcs, struct bfa_s *bfa, struct bfad_s *bfad,
 	fcs->bfa = bfa;
 	fcs->bfad = bfad;
 	fcs->min_cfg = min_cfg;
+	fcs->num_rport_logins = 0;
 
 	bfa->fcs = BFA_TRUE;
 	fcbuild_init();
@@ -119,6 +120,18 @@ bfa_fcs_update_cfg(struct bfa_fcs_s *fcs)
 }
 
 /*
+ * Stop FCS operations.
+ */
+void
+bfa_fcs_stop(struct bfa_fcs_s *fcs)
+{
+	bfa_wc_init(&fcs->wc, bfa_fcs_exit_comp, fcs);
+	bfa_wc_up(&fcs->wc);
+	bfa_fcs_fabric_modstop(fcs);
+	bfa_wc_wait(&fcs->wc);
+}
+
+/*
  * fcs pbc vport initialization
  */
 void
@@ -153,6 +166,7 @@ bfa_fcs_driver_info_init(struct bfa_fcs_s *fcs,
 	fcs->driver_info = *driver_info;
 
 	bfa_fcs_fabric_psymb_init(&fcs->fabric);
+	bfa_fcs_fabric_nsymb_init(&fcs->fabric);
 }
 
 /*
@@ -213,6 +227,8 @@ static void bfa_fcs_fabric_notify_offline(struct bfa_fcs_fabric_s *fabric);
 static void bfa_fcs_fabric_delay(void *cbarg);
 static void bfa_fcs_fabric_delete(struct bfa_fcs_fabric_s *fabric);
 static void bfa_fcs_fabric_delete_comp(void *cbarg);
+static void bfa_fcs_fabric_stop(struct bfa_fcs_fabric_s *fabric);
+static void bfa_fcs_fabric_stop_comp(void *cbarg);
 static void bfa_fcs_fabric_process_uf(struct bfa_fcs_fabric_s *fabric,
 				      struct fchs_s *fchs, u16 len);
 static void bfa_fcs_fabric_process_flogi(struct bfa_fcs_fabric_s *fabric,
@@ -250,6 +266,10 @@ static void	bfa_fcs_fabric_sm_isolated(struct bfa_fcs_fabric_s *fabric,
 					   enum bfa_fcs_fabric_event event);
 static void	bfa_fcs_fabric_sm_deleting(struct bfa_fcs_fabric_s *fabric,
 					   enum bfa_fcs_fabric_event event);
+static void	bfa_fcs_fabric_sm_stopping(struct bfa_fcs_fabric_s *fabric,
+					   enum bfa_fcs_fabric_event event);
+static void	bfa_fcs_fabric_sm_cleanup(struct bfa_fcs_fabric_s *fabric,
+					  enum bfa_fcs_fabric_event event);
 /*
  *   Beginning state before fabric creation.
  */
@@ -334,6 +354,11 @@ bfa_fcs_fabric_sm_linkdown(struct bfa_fcs_fabric_s *fabric,
 		bfa_fcs_fabric_delete(fabric);
 		break;
 
+	case BFA_FCS_FABRIC_SM_STOP:
+		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_cleanup);
+		bfa_fcs_fabric_stop(fabric);
+		break;
+
 	default:
 		bfa_sm_fault(fabric->fcs, event);
 	}
@@ -585,6 +610,11 @@ bfa_fcs_fabric_sm_online(struct bfa_fcs_fabric_s *fabric,
 		bfa_fcs_fabric_delete(fabric);
 		break;
 
+	case BFA_FCS_FABRIC_SM_STOP:
+		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_stopping);
+		bfa_fcs_fabric_stop(fabric);
+		break;
+
 	case BFA_FCS_FABRIC_SM_AUTH_FAILED:
 		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_auth_failed);
 		bfa_sm_send_event(fabric->lps, BFA_LPS_SM_OFFLINE);
@@ -682,7 +712,62 @@ bfa_fcs_fabric_sm_deleting(struct bfa_fcs_fabric_s *fabric,
 	}
 }
 
+/*
+ * Fabric is being stopped, awaiting vport stop completions.
+ */
+static void
+bfa_fcs_fabric_sm_stopping(struct bfa_fcs_fabric_s *fabric,
+			   enum bfa_fcs_fabric_event event)
+{
+	bfa_trc(fabric->fcs, fabric->bport.port_cfg.pwwn);
+	bfa_trc(fabric->fcs, event);
+
+	switch (event) {
+	case BFA_FCS_FABRIC_SM_STOPCOMP:
+		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_cleanup);
+		bfa_sm_send_event(fabric->lps, BFA_LPS_SM_LOGOUT);
+		break;
+
+	case BFA_FCS_FABRIC_SM_LINK_UP:
+		break;
+
+	case BFA_FCS_FABRIC_SM_LINK_DOWN:
+		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_cleanup);
+		break;
+
+	default:
+		bfa_sm_fault(fabric->fcs, event);
+	}
+}
+
+/*
+ * Fabric is being stopped, cleanup without FLOGO
+ */
+static void
+bfa_fcs_fabric_sm_cleanup(struct bfa_fcs_fabric_s *fabric,
+			  enum bfa_fcs_fabric_event event)
+{
+	bfa_trc(fabric->fcs, fabric->bport.port_cfg.pwwn);
+	bfa_trc(fabric->fcs, event);
 
+	switch (event) {
+	case BFA_FCS_FABRIC_SM_STOPCOMP:
+	case BFA_FCS_FABRIC_SM_LOGOCOMP:
+		bfa_sm_set_state(fabric, bfa_fcs_fabric_sm_created);
+		bfa_wc_down(&(fabric->fcs)->wc);
+		break;
+
+	case BFA_FCS_FABRIC_SM_LINK_DOWN:
+		/*
+		 * Ignore - can get this event if we get notified about IOC down
+		 * before the fabric completion callbk is done.
+		 */
+		break;
+
+	default:
+		bfa_sm_fault(fabric->fcs, event);
+	}
+}
 
 /*
  *  fcs_fabric_private fabric private functions
@@ -760,6 +845,44 @@ bfa_fcs_fabric_psymb_init(struct bfa_fcs_fabric_s *fabric)
 }
 
 /*
+ * Node Symbolic Name Creation for base port and all vports
+ */
+void
+bfa_fcs_fabric_nsymb_init(struct bfa_fcs_fabric_s *fabric)
+{
+	struct bfa_lport_cfg_s *port_cfg = &fabric->bport.port_cfg;
+	char model[BFA_ADAPTER_MODEL_NAME_LEN] = {0};
+	struct bfa_fcs_driver_info_s *driver_info = &fabric->fcs->driver_info;
+
+	bfa_ioc_get_adapter_model(&fabric->fcs->bfa->ioc, model);
+
+	/* Model name/number */
+	strncpy((char *)&port_cfg->node_sym_name, model,
+		BFA_FCS_PORT_SYMBNAME_MODEL_SZ);
+	strncat((char *)&port_cfg->node_sym_name,
+			BFA_FCS_PORT_SYMBNAME_SEPARATOR,
+			sizeof(BFA_FCS_PORT_SYMBNAME_SEPARATOR));
+
+	/* Driver Version */
+	strncat((char *)&port_cfg->node_sym_name, (char *)driver_info->version,
+		BFA_FCS_PORT_SYMBNAME_VERSION_SZ);
+	strncat((char *)&port_cfg->node_sym_name,
+			BFA_FCS_PORT_SYMBNAME_SEPARATOR,
+			sizeof(BFA_FCS_PORT_SYMBNAME_SEPARATOR));
+
+	/* Host machine name */
+	strncat((char *)&port_cfg->node_sym_name,
+		(char *)driver_info->host_machine_name,
+		BFA_FCS_PORT_SYMBNAME_MACHINENAME_SZ);
+	strncat((char *)&port_cfg->node_sym_name,
+			BFA_FCS_PORT_SYMBNAME_SEPARATOR,
+			sizeof(BFA_FCS_PORT_SYMBNAME_SEPARATOR));
+
+	/* null terminate */
+	port_cfg->node_sym_name.symname[BFA_SYMNAME_MAXLEN - 1] = 0;
+}
+
+/*
  * bfa lps login completion callback
  */
 void
@@ -919,6 +1042,28 @@ bfa_fcs_fabric_delay(void *cbarg)
 }
 
 /*
+ * Stop all vports and wait for vport stop completions.
+ */
+static void
+bfa_fcs_fabric_stop(struct bfa_fcs_fabric_s *fabric)
+{
+	struct bfa_fcs_vport_s *vport;
+	struct list_head	*qe, *qen;
+
+	bfa_wc_init(&fabric->stop_wc, bfa_fcs_fabric_stop_comp, fabric);
+
+	list_for_each_safe(qe, qen, &fabric->vport_q) {
+		vport = (struct bfa_fcs_vport_s *) qe;
+		bfa_wc_up(&fabric->stop_wc);
+		bfa_fcs_vport_fcs_stop(vport);
+	}
+
+	bfa_wc_up(&fabric->stop_wc);
+	bfa_fcs_lport_stop(&fabric->bport);
+	bfa_wc_wait(&fabric->stop_wc);
+}
+
+/*
  * Computes operating BB_SCN value
  */
 static u8
@@ -978,6 +1123,14 @@ bfa_fcs_fabric_delete_comp(void *cbarg)
 	bfa_sm_send_event(fabric, BFA_FCS_FABRIC_SM_DELCOMP);
 }
 
+static void
+bfa_fcs_fabric_stop_comp(void *cbarg)
+{
+	struct bfa_fcs_fabric_s *fabric = cbarg;
+
+	bfa_sm_send_event(fabric, BFA_FCS_FABRIC_SM_STOPCOMP);
+}
+
 /*
  *  fcs_fabric_public fabric public functions
  */
@@ -1039,6 +1192,19 @@ bfa_fcs_fabric_modexit(struct bfa_fcs_s *fcs)
 }
 
 /*
+ * Fabric module stop -- stop FCS actions
+ */
+void
+bfa_fcs_fabric_modstop(struct bfa_fcs_s *fcs)
+{
+	struct bfa_fcs_fabric_s *fabric;
+
+	bfa_trc(fcs, 0);
+	fabric = &fcs->fabric;
+	bfa_sm_send_event(fabric, BFA_FCS_FABRIC_SM_STOP);
+}
+
+/*
  * Fabric module start -- kick starts FCS actions
  */
 void
@@ -1219,8 +1385,11 @@ bfa_fcs_fabric_uf_recv(struct bfa_fcs_fabric_s *fabric, struct fchs_s *fchs,
 			return;
 		}
 	}
-	bfa_trc(fabric->fcs, els_cmd->els_code);
-	bfa_fcs_lport_uf_recv(&fabric->bport, fchs, len);
+
+	if (!bfa_fcs_fabric_is_switched(fabric))
+		bfa_fcs_lport_uf_recv(&fabric->bport, fchs, len);
+
+	bfa_trc(fabric->fcs, fchs->type);
 }
 
 /*
@@ -1294,7 +1463,7 @@ bfa_fcs_fabric_send_flogi_acc(struct bfa_fcs_fabric_s *fabric)
 	u16	reqlen;
 	struct fchs_s	fchs;
 
-	fcxp = bfa_fcs_fcxp_alloc(fabric->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(fabric->fcs, BFA_FALSE);
 	/*
 	 * Do not expect this failure -- expect remote node to retry
 	 */
@@ -1387,6 +1556,13 @@ bfa_fcs_fabric_set_fabric_name(struct bfa_fcs_fabric_s *fabric,
 	}
 }
 
+void
+bfa_cb_lps_flogo_comp(void *bfad, void *uarg)
+{
+	struct bfa_fcs_fabric_s *fabric = uarg;
+	bfa_sm_send_event(fabric, BFA_FCS_FABRIC_SM_LOGOCOMP);
+}
+
 /*
  *	Returns FCS vf structure for a given vf_id.
  *
diff --git a/drivers/scsi/bfa/bfa_fcs.h b/drivers/scsi/bfa/bfa_fcs.h
index 51c9e134571..6c4377cb287 100644
--- a/drivers/scsi/bfa/bfa_fcs.h
+++ b/drivers/scsi/bfa/bfa_fcs.h
@@ -62,9 +62,9 @@ struct bfa_fcs_s;
 #define N2N_LOCAL_PID	    0x010000
 #define N2N_REMOTE_PID		0x020000
 #define	BFA_FCS_RETRY_TIMEOUT 2000
+#define BFA_FCS_MAX_NS_RETRIES 5
 #define BFA_FCS_PID_IS_WKA(pid)  ((bfa_ntoh3b(pid) > 0xFFF000) ?  1 : 0)
-
-
+#define BFA_FCS_MAX_RPORT_LOGINS 1024
 
 struct bfa_fcs_lport_ns_s {
 	bfa_sm_t        sm;		/*  state machine */
@@ -72,6 +72,8 @@ struct bfa_fcs_lport_ns_s {
 	struct bfa_fcs_lport_s *port;	/*  parent port */
 	struct bfa_fcxp_s *fcxp;
 	struct bfa_fcxp_wqe_s fcxp_wqe;
+	u8	num_rnnid_retries;
+	u8	num_rsnn_nn_retries;
 };
 
 
@@ -205,6 +207,7 @@ struct bfa_fcs_fabric_s {
 	struct bfa_lps_s	*lps;	/*  lport login services	*/
 	u8	fabric_ip_addr[BFA_FCS_FABRIC_IPADDR_SZ];
 					/*  attached fabric's ip addr  */
+	struct bfa_wc_s stop_wc;	/*  wait counter for stop */
 };
 
 #define bfa_fcs_fabric_npiv_capable(__f)    ((__f)->is_npiv)
@@ -264,6 +267,7 @@ struct bfa_fcs_fabric_s;
 #define bfa_fcs_lport_get_pwwn(_lport)	((_lport)->port_cfg.pwwn)
 #define bfa_fcs_lport_get_nwwn(_lport)	((_lport)->port_cfg.nwwn)
 #define bfa_fcs_lport_get_psym_name(_lport)	((_lport)->port_cfg.sym_name)
+#define bfa_fcs_lport_get_nsym_name(_lport) ((_lport)->port_cfg.node_sym_name)
 #define bfa_fcs_lport_is_initiator(_lport)			\
 	((_lport)->port_cfg.roles & BFA_LPORT_ROLE_FCP_IM)
 #define bfa_fcs_lport_get_nrports(_lport)	\
@@ -286,9 +290,8 @@ bfa_fcs_lport_get_drvport(struct bfa_fcs_lport_s *port)
 
 bfa_boolean_t   bfa_fcs_lport_is_online(struct bfa_fcs_lport_s *port);
 struct bfa_fcs_lport_s *bfa_fcs_get_base_port(struct bfa_fcs_s *fcs);
-void bfa_fcs_lport_get_rports(struct bfa_fcs_lport_s *port,
-			      wwn_t rport_wwns[], int *nrports);
-
+void bfa_fcs_lport_get_rport_quals(struct bfa_fcs_lport_s *port,
+			struct bfa_rport_qualifier_s rport[], int *nrports);
 wwn_t bfa_fcs_lport_get_rport(struct bfa_fcs_lport_s *port, wwn_t wwn,
 			      int index, int nrports, bfa_boolean_t bwwn);
 
@@ -324,12 +327,17 @@ void bfa_fcs_lport_init(struct bfa_fcs_lport_s *lport,
 void            bfa_fcs_lport_online(struct bfa_fcs_lport_s *port);
 void            bfa_fcs_lport_offline(struct bfa_fcs_lport_s *port);
 void            bfa_fcs_lport_delete(struct bfa_fcs_lport_s *port);
+void		bfa_fcs_lport_stop(struct bfa_fcs_lport_s *port);
 struct bfa_fcs_rport_s *bfa_fcs_lport_get_rport_by_pid(
 		struct bfa_fcs_lport_s *port, u32 pid);
+struct bfa_fcs_rport_s *bfa_fcs_lport_get_rport_by_old_pid(
+		struct bfa_fcs_lport_s *port, u32 pid);
 struct bfa_fcs_rport_s *bfa_fcs_lport_get_rport_by_pwwn(
 		struct bfa_fcs_lport_s *port, wwn_t pwwn);
 struct bfa_fcs_rport_s *bfa_fcs_lport_get_rport_by_nwwn(
 		struct bfa_fcs_lport_s *port, wwn_t nwwn);
+struct bfa_fcs_rport_s *bfa_fcs_lport_get_rport_by_qualifier(
+		struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 pid);
 void            bfa_fcs_lport_add_rport(struct bfa_fcs_lport_s *port,
 				       struct bfa_fcs_rport_s *rport);
 void            bfa_fcs_lport_del_rport(struct bfa_fcs_lport_s *port,
@@ -338,6 +346,8 @@ void            bfa_fcs_lport_ns_init(struct bfa_fcs_lport_s *vport);
 void            bfa_fcs_lport_ns_offline(struct bfa_fcs_lport_s *vport);
 void            bfa_fcs_lport_ns_online(struct bfa_fcs_lport_s *vport);
 void            bfa_fcs_lport_ns_query(struct bfa_fcs_lport_s *port);
+void		bfa_fcs_lport_ns_util_send_rspn_id(void *cbarg,
+				struct bfa_fcxp_s *fcxp_alloced);
 void            bfa_fcs_lport_scn_init(struct bfa_fcs_lport_s *vport);
 void            bfa_fcs_lport_scn_offline(struct bfa_fcs_lport_s *vport);
 void            bfa_fcs_lport_scn_online(struct bfa_fcs_lport_s *vport);
@@ -382,6 +392,7 @@ void bfa_fcs_vport_online(struct bfa_fcs_vport_s *vport);
 void bfa_fcs_vport_offline(struct bfa_fcs_vport_s *vport);
 void bfa_fcs_vport_delete_comp(struct bfa_fcs_vport_s *vport);
 void bfa_fcs_vport_fcs_delete(struct bfa_fcs_vport_s *vport);
+void bfa_fcs_vport_fcs_stop(struct bfa_fcs_vport_s *vport);
 void bfa_fcs_vport_stop_comp(struct bfa_fcs_vport_s *vport);
 
 #define BFA_FCS_RPORT_DEF_DEL_TIMEOUT	90	/* in secs */
@@ -419,6 +430,7 @@ struct bfa_fcs_rport_s {
 	struct bfa_fcs_s	*fcs;	/*  fcs instance */
 	struct bfad_rport_s	*rp_drv;	/*  driver peer instance */
 	u32	pid;	/*  port ID of rport */
+	u32	old_pid;	/* PID before rport goes offline */
 	u16	maxfrsize;	/*  maximum frame size */
 	__be16	reply_oxid;	/*  OX_ID of inbound requests */
 	enum fc_cos	fc_cos;	/*  FC classes of service supp */
@@ -459,7 +471,7 @@ struct bfa_fcs_rport_s *bfa_fcs_rport_lookup(struct bfa_fcs_lport_s *port,
 struct bfa_fcs_rport_s *bfa_fcs_rport_lookup_by_nwwn(
 	struct bfa_fcs_lport_s *port, wwn_t rnwwn);
 void bfa_fcs_rport_set_del_timeout(u8 rport_tmo);
-
+void bfa_fcs_rport_set_max_logins(u32 max_logins);
 void bfa_fcs_rport_uf_recv(struct bfa_fcs_rport_s *rport,
 	 struct fchs_s *fchs, u16 len);
 void bfa_fcs_rport_scn(struct bfa_fcs_rport_s *rport);
@@ -505,12 +517,13 @@ struct bfa_fcs_itnim_s {
 	struct bfa_fcxp_s	*fcxp;		/*  FCXP in use	*/
 	struct bfa_itnim_stats_s	stats;	/*  itn statistics	*/
 };
-#define bfa_fcs_fcxp_alloc(__fcs)	\
-	bfa_fcxp_alloc(NULL, (__fcs)->bfa, 0, 0, NULL, NULL, NULL, NULL)
-
-#define bfa_fcs_fcxp_alloc_wait(__bfa, __wqe, __alloc_cbfn, __alloc_cbarg) \
-	bfa_fcxp_alloc_wait(__bfa, __wqe, __alloc_cbfn, __alloc_cbarg, \
-					NULL, 0, 0, NULL, NULL, NULL, NULL)
+#define bfa_fcs_fcxp_alloc(__fcs, __req)				\
+	bfa_fcxp_req_rsp_alloc(NULL, (__fcs)->bfa, 0, 0,		\
+			       NULL, NULL, NULL, NULL, __req)
+#define bfa_fcs_fcxp_alloc_wait(__bfa, __wqe, __alloc_cbfn,		\
+				__alloc_cbarg, __req)			\
+	bfa_fcxp_req_rsp_alloc_wait(__bfa, __wqe, __alloc_cbfn,		\
+		__alloc_cbarg, NULL, 0, 0, NULL, NULL, NULL, NULL, __req)
 
 static inline struct bfad_port_s *
 bfa_fcs_itnim_get_drvport(struct bfa_fcs_itnim_s *itnim)
@@ -592,7 +605,7 @@ bfa_status_t bfa_fcs_itnim_stats_clear(struct bfa_fcs_lport_s *port,
 struct bfa_fcs_itnim_s *bfa_fcs_itnim_create(struct bfa_fcs_rport_s *rport);
 void bfa_fcs_itnim_delete(struct bfa_fcs_itnim_s *itnim);
 void bfa_fcs_itnim_rport_offline(struct bfa_fcs_itnim_s *itnim);
-void bfa_fcs_itnim_rport_online(struct bfa_fcs_itnim_s *itnim);
+void bfa_fcs_itnim_brp_online(struct bfa_fcs_itnim_s *itnim);
 bfa_status_t bfa_fcs_itnim_get_online_state(struct bfa_fcs_itnim_s *itnim);
 void bfa_fcs_itnim_is_initiator(struct bfa_fcs_itnim_s *itnim);
 void bfa_fcs_fcpim_uf_recv(struct bfa_fcs_itnim_s *itnim,
@@ -676,6 +689,7 @@ struct bfa_fcs_s {
 	struct bfa_fcs_stats_s	stats;	/*  FCS statistics */
 	struct bfa_wc_s		wc;	/*  waiting counter */
 	int			fcs_aen_seq;
+	u32		num_rport_logins;
 };
 
 /*
@@ -702,6 +716,9 @@ enum bfa_fcs_fabric_event {
 	BFA_FCS_FABRIC_SM_DELCOMP       = 14,   /*  all vports deleted event */
 	BFA_FCS_FABRIC_SM_LOOPBACK      = 15,   /*  Received our own FLOGI   */
 	BFA_FCS_FABRIC_SM_START         = 16,   /*  from driver       */
+	BFA_FCS_FABRIC_SM_STOP		= 17,	/*  Stop from driver	*/
+	BFA_FCS_FABRIC_SM_STOPCOMP	= 18,	/*  Stop completion	*/
+	BFA_FCS_FABRIC_SM_LOGOCOMP	= 19,	/*  FLOGO completion	*/
 };
 
 /*
@@ -727,6 +744,26 @@ enum rport_event {
 	RPSM_EVENT_ADDRESS_DISC = 16,   /*  Need to Discover rport's PID */
 	RPSM_EVENT_PRLO_RCVD   = 17,    /*  PRLO from remote device     */
 	RPSM_EVENT_PLOGI_RETRY = 18,    /*  Retry PLOGI continuously */
+	RPSM_EVENT_FC4_FCS_ONLINE = 19, /*!< FC-4 FCS online complete */
+};
+
+/*
+ * fcs_itnim_sm FCS itnim state machine events
+ */
+enum bfa_fcs_itnim_event {
+	BFA_FCS_ITNIM_SM_FCS_ONLINE = 1,        /*  rport online event */
+	BFA_FCS_ITNIM_SM_OFFLINE = 2,   /*  rport offline */
+	BFA_FCS_ITNIM_SM_FRMSENT = 3,   /*  prli frame is sent */
+	BFA_FCS_ITNIM_SM_RSP_OK = 4,    /*  good response */
+	BFA_FCS_ITNIM_SM_RSP_ERROR = 5, /*  error response */
+	BFA_FCS_ITNIM_SM_TIMEOUT = 6,   /*  delay timeout */
+	BFA_FCS_ITNIM_SM_HCB_OFFLINE = 7, /*  BFA online callback */
+	BFA_FCS_ITNIM_SM_HCB_ONLINE = 8, /*  BFA offline callback */
+	BFA_FCS_ITNIM_SM_INITIATOR = 9, /*  rport is initiator */
+	BFA_FCS_ITNIM_SM_DELETE = 10,   /*  delete event from rport */
+	BFA_FCS_ITNIM_SM_PRLO = 11,     /*  delete event from rport */
+	BFA_FCS_ITNIM_SM_RSP_NOT_SUPP = 12, /* cmd not supported rsp */
+	BFA_FCS_ITNIM_SM_HAL_ONLINE = 13, /*!< bfa rport online event */
 };
 
 /*
@@ -741,6 +778,7 @@ void bfa_fcs_update_cfg(struct bfa_fcs_s *fcs);
 void bfa_fcs_driver_info_init(struct bfa_fcs_s *fcs,
 			      struct bfa_fcs_driver_info_s *driver_info);
 void bfa_fcs_exit(struct bfa_fcs_s *fcs);
+void bfa_fcs_stop(struct bfa_fcs_s *fcs);
 
 /*
  * bfa fcs vf public functions
@@ -766,11 +804,13 @@ void bfa_fcs_fabric_modstart(struct bfa_fcs_s *fcs);
 void bfa_fcs_fabric_uf_recv(struct bfa_fcs_fabric_s *fabric,
 		struct fchs_s *fchs, u16 len);
 void	bfa_fcs_fabric_psymb_init(struct bfa_fcs_fabric_s *fabric);
+void	bfa_fcs_fabric_nsymb_init(struct bfa_fcs_fabric_s *fabric);
 void bfa_fcs_fabric_set_fabric_name(struct bfa_fcs_fabric_s *fabric,
 	       wwn_t fabric_name);
 u16 bfa_fcs_fabric_get_switch_oui(struct bfa_fcs_fabric_s *fabric);
 void bfa_fcs_uf_attach(struct bfa_fcs_s *fcs);
 void bfa_fcs_port_attach(struct bfa_fcs_s *fcs);
+void bfa_fcs_fabric_modstop(struct bfa_fcs_s *fcs);
 void bfa_fcs_fabric_sm_online(struct bfa_fcs_fabric_s *fabric,
 			enum bfa_fcs_fabric_event event);
 void bfa_fcs_fabric_sm_loopback(struct bfa_fcs_fabric_s *fabric,
diff --git a/drivers/scsi/bfa/bfa_fcs_fcpim.c b/drivers/scsi/bfa/bfa_fcs_fcpim.c
index 9272840a240..6dc7926a3ed 100644
--- a/drivers/scsi/bfa/bfa_fcs_fcpim.c
+++ b/drivers/scsi/bfa/bfa_fcs_fcpim.c
@@ -40,25 +40,6 @@ static void	bfa_fcs_itnim_prli_response(void *fcsarg,
 static void	bfa_fcs_itnim_aen_post(struct bfa_fcs_itnim_s *itnim,
 			enum bfa_itnim_aen_event event);
 
-/*
- *  fcs_itnim_sm FCS itnim state machine events
- */
-
-enum bfa_fcs_itnim_event {
-	BFA_FCS_ITNIM_SM_ONLINE = 1,	/*  rport online event */
-	BFA_FCS_ITNIM_SM_OFFLINE = 2,	/*  rport offline */
-	BFA_FCS_ITNIM_SM_FRMSENT = 3,	/*  prli frame is sent */
-	BFA_FCS_ITNIM_SM_RSP_OK = 4,	/*  good response */
-	BFA_FCS_ITNIM_SM_RSP_ERROR = 5,	/*  error response */
-	BFA_FCS_ITNIM_SM_TIMEOUT = 6,	/*  delay timeout */
-	BFA_FCS_ITNIM_SM_HCB_OFFLINE = 7, /*  BFA online callback */
-	BFA_FCS_ITNIM_SM_HCB_ONLINE = 8, /*  BFA offline callback */
-	BFA_FCS_ITNIM_SM_INITIATOR = 9,	/*  rport is initiator */
-	BFA_FCS_ITNIM_SM_DELETE = 10,	/*  delete event from rport */
-	BFA_FCS_ITNIM_SM_PRLO = 11,	/*  delete event from rport */
-	BFA_FCS_ITNIM_SM_RSP_NOT_SUPP = 12, /* cmd not supported rsp */
-};
-
 static void	bfa_fcs_itnim_sm_offline(struct bfa_fcs_itnim_s *itnim,
 					 enum bfa_fcs_itnim_event event);
 static void	bfa_fcs_itnim_sm_prli_send(struct bfa_fcs_itnim_s *itnim,
@@ -69,6 +50,8 @@ static void	bfa_fcs_itnim_sm_prli_retry(struct bfa_fcs_itnim_s *itnim,
 					    enum bfa_fcs_itnim_event event);
 static void	bfa_fcs_itnim_sm_hcb_online(struct bfa_fcs_itnim_s *itnim,
 					    enum bfa_fcs_itnim_event event);
+static void	bfa_fcs_itnim_sm_hal_rport_online(struct bfa_fcs_itnim_s *itnim,
+					enum bfa_fcs_itnim_event event);
 static void	bfa_fcs_itnim_sm_online(struct bfa_fcs_itnim_s *itnim,
 					enum bfa_fcs_itnim_event event);
 static void	bfa_fcs_itnim_sm_hcb_offline(struct bfa_fcs_itnim_s *itnim,
@@ -99,7 +82,7 @@ bfa_fcs_itnim_sm_offline(struct bfa_fcs_itnim_s *itnim,
 	bfa_trc(itnim->fcs, event);
 
 	switch (event) {
-	case BFA_FCS_ITNIM_SM_ONLINE:
+	case BFA_FCS_ITNIM_SM_FCS_ONLINE:
 		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_prli_send);
 		itnim->prli_retries = 0;
 		bfa_fcs_itnim_send_prli(itnim, NULL);
@@ -138,6 +121,7 @@ bfa_fcs_itnim_sm_prli_send(struct bfa_fcs_itnim_s *itnim,
 	case BFA_FCS_ITNIM_SM_INITIATOR:
 		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_initiator);
 		bfa_fcxp_walloc_cancel(itnim->fcs->bfa, &itnim->fcxp_wqe);
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_FCS_ONLINE);
 		break;
 
 	case BFA_FCS_ITNIM_SM_OFFLINE:
@@ -166,12 +150,13 @@ bfa_fcs_itnim_sm_prli(struct bfa_fcs_itnim_s *itnim,
 
 	switch (event) {
 	case BFA_FCS_ITNIM_SM_RSP_OK:
-		if (itnim->rport->scsi_function == BFA_RPORT_INITIATOR) {
+		if (itnim->rport->scsi_function == BFA_RPORT_INITIATOR)
 			bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_initiator);
-		} else {
-			bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_hcb_online);
-			bfa_itnim_online(itnim->bfa_itnim, itnim->seq_rec);
-		}
+		else
+			bfa_sm_set_state(itnim,
+				bfa_fcs_itnim_sm_hal_rport_online);
+
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_FCS_ONLINE);
 		break;
 
 	case BFA_FCS_ITNIM_SM_RSP_ERROR:
@@ -194,6 +179,7 @@ bfa_fcs_itnim_sm_prli(struct bfa_fcs_itnim_s *itnim,
 	case BFA_FCS_ITNIM_SM_INITIATOR:
 		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_initiator);
 		bfa_fcxp_discard(itnim->fcxp);
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_FCS_ONLINE);
 		break;
 
 	case BFA_FCS_ITNIM_SM_DELETE:
@@ -208,6 +194,44 @@ bfa_fcs_itnim_sm_prli(struct bfa_fcs_itnim_s *itnim,
 }
 
 static void
+bfa_fcs_itnim_sm_hal_rport_online(struct bfa_fcs_itnim_s *itnim,
+				enum bfa_fcs_itnim_event event)
+{
+	bfa_trc(itnim->fcs, itnim->rport->pwwn);
+	bfa_trc(itnim->fcs, event);
+
+	switch (event) {
+	case BFA_FCS_ITNIM_SM_HAL_ONLINE:
+		if (!itnim->bfa_itnim)
+			itnim->bfa_itnim = bfa_itnim_create(itnim->fcs->bfa,
+					itnim->rport->bfa_rport, itnim);
+
+		if (itnim->bfa_itnim) {
+			bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_hcb_online);
+			bfa_itnim_online(itnim->bfa_itnim, itnim->seq_rec);
+		} else {
+			bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_offline);
+			bfa_sm_send_event(itnim->rport, RPSM_EVENT_DELETE);
+		}
+
+		break;
+
+	case BFA_FCS_ITNIM_SM_OFFLINE:
+		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_offline);
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_OFFLINE);
+		break;
+
+	case BFA_FCS_ITNIM_SM_DELETE:
+		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_offline);
+		bfa_fcs_itnim_free(itnim);
+		break;
+
+	default:
+		bfa_sm_fault(itnim->fcs, event);
+	}
+}
+
+static void
 bfa_fcs_itnim_sm_prli_retry(struct bfa_fcs_itnim_s *itnim,
 			    enum bfa_fcs_itnim_event event)
 {
@@ -238,6 +262,7 @@ bfa_fcs_itnim_sm_prli_retry(struct bfa_fcs_itnim_s *itnim,
 	case BFA_FCS_ITNIM_SM_INITIATOR:
 		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_initiator);
 		bfa_timer_stop(&itnim->timer);
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_FCS_ONLINE);
 		break;
 
 	case BFA_FCS_ITNIM_SM_DELETE:
@@ -275,9 +300,8 @@ bfa_fcs_itnim_sm_hcb_online(struct bfa_fcs_itnim_s *itnim,
 		break;
 
 	case BFA_FCS_ITNIM_SM_OFFLINE:
-		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_offline);
+		bfa_sm_set_state(itnim, bfa_fcs_itnim_sm_hcb_offline);
 		bfa_itnim_offline(itnim->bfa_itnim);
-		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_OFFLINE);
 		break;
 
 	case BFA_FCS_ITNIM_SM_DELETE:
@@ -372,8 +396,14 @@ bfa_fcs_itnim_sm_initiator(struct bfa_fcs_itnim_s *itnim,
 		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_OFFLINE);
 		break;
 
+	/*
+	 * fcs_online is expected here for well known initiator ports
+	 */
+	case BFA_FCS_ITNIM_SM_FCS_ONLINE:
+		bfa_sm_send_event(itnim->rport, RPSM_EVENT_FC4_FCS_ONLINE);
+		break;
+
 	case BFA_FCS_ITNIM_SM_RSP_ERROR:
-	case BFA_FCS_ITNIM_SM_ONLINE:
 	case BFA_FCS_ITNIM_SM_INITIATOR:
 		break;
 
@@ -426,11 +456,12 @@ bfa_fcs_itnim_send_prli(void *itnim_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(itnim->fcs, itnim->rport->pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		itnim->stats.fcxp_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &itnim->fcxp_wqe,
-				    bfa_fcs_itnim_send_prli, itnim);
+				bfa_fcs_itnim_send_prli, itnim, BFA_TRUE);
 		return;
 	}
 	itnim->fcxp = fcxp;
@@ -483,7 +514,7 @@ bfa_fcs_itnim_prli_response(void *fcsarg, struct bfa_fcxp_s *fcxp, void *cbarg,
 			if (prli_resp->parampage.servparams.initiator) {
 				bfa_trc(itnim->fcs, prli_resp->parampage.type);
 				itnim->rport->scsi_function =
-					 BFA_RPORT_INITIATOR;
+						BFA_RPORT_INITIATOR;
 				itnim->stats.prli_rsp_acc++;
 				itnim->stats.initiator++;
 				bfa_sm_send_event(itnim,
@@ -531,7 +562,11 @@ bfa_fcs_itnim_timeout(void *arg)
 static void
 bfa_fcs_itnim_free(struct bfa_fcs_itnim_s *itnim)
 {
-	bfa_itnim_delete(itnim->bfa_itnim);
+	if (itnim->bfa_itnim) {
+		bfa_itnim_delete(itnim->bfa_itnim);
+		itnim->bfa_itnim = NULL;
+	}
+
 	bfa_fcb_itnim_free(itnim->fcs->bfad, itnim->itnim_drv);
 }
 
@@ -552,7 +587,6 @@ bfa_fcs_itnim_create(struct bfa_fcs_rport_s *rport)
 	struct bfa_fcs_lport_s *port = rport->port;
 	struct bfa_fcs_itnim_s *itnim;
 	struct bfad_itnim_s   *itnim_drv;
-	struct bfa_itnim_s *bfa_itnim;
 
 	/*
 	 * call bfad to allocate the itnim
@@ -570,20 +604,7 @@ bfa_fcs_itnim_create(struct bfa_fcs_rport_s *rport)
 	itnim->fcs = rport->fcs;
 	itnim->itnim_drv = itnim_drv;
 
-	/*
-	 * call BFA to create the itnim
-	 */
-	bfa_itnim =
-		bfa_itnim_create(port->fcs->bfa, rport->bfa_rport, itnim);
-
-	if (bfa_itnim == NULL) {
-		bfa_trc(port->fcs, rport->pwwn);
-		bfa_fcb_itnim_free(port->fcs->bfad, itnim_drv);
-		WARN_ON(1);
-		return NULL;
-	}
-
-	itnim->bfa_itnim     = bfa_itnim;
+	itnim->bfa_itnim     = NULL;
 	itnim->seq_rec	     = BFA_FALSE;
 	itnim->rec_support   = BFA_FALSE;
 	itnim->conf_comp     = BFA_FALSE;
@@ -613,20 +634,12 @@ bfa_fcs_itnim_delete(struct bfa_fcs_itnim_s *itnim)
  * Notification from rport that PLOGI is complete to initiate FC-4 session.
  */
 void
-bfa_fcs_itnim_rport_online(struct bfa_fcs_itnim_s *itnim)
+bfa_fcs_itnim_brp_online(struct bfa_fcs_itnim_s *itnim)
 {
 	itnim->stats.onlines++;
 
-	if (!BFA_FCS_PID_IS_WKA(itnim->rport->pid)) {
-		bfa_sm_send_event(itnim, BFA_FCS_ITNIM_SM_ONLINE);
-	} else {
-		/*
-		 *  For well known addresses, we set the itnim to initiator
-		 *  state
-		 */
-		itnim->stats.initiator++;
-		bfa_sm_send_event(itnim, BFA_FCS_ITNIM_SM_INITIATOR);
-	}
+	if (!BFA_FCS_PID_IS_WKA(itnim->rport->pid))
+		bfa_sm_send_event(itnim, BFA_FCS_ITNIM_SM_HAL_ONLINE);
 }
 
 /*
diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c
index bcc4966e8ba..3b75f6fb2de 100644
--- a/drivers/scsi/bfa/bfa_fcs_lport.c
+++ b/drivers/scsi/bfa/bfa_fcs_lport.c
@@ -131,6 +131,8 @@ bfa_fcs_lport_sm_init(struct bfa_fcs_lport_s *port,
 		/* If vport - send completion call back */
 		if (port->vport)
 			bfa_fcs_vport_stop_comp(port->vport);
+		else
+			bfa_wc_down(&(port->fabric->stop_wc));
 		break;
 
 	case BFA_FCS_PORT_SM_OFFLINE:
@@ -166,6 +168,8 @@ bfa_fcs_lport_sm_online(
 			/* If vport - send completion call back */
 			if (port->vport)
 				bfa_fcs_vport_stop_comp(port->vport);
+			else
+				bfa_wc_down(&(port->fabric->stop_wc));
 		} else {
 			bfa_sm_set_state(port, bfa_fcs_lport_sm_stopping);
 			list_for_each_safe(qe, qen, &port->rport_q) {
@@ -222,6 +226,8 @@ bfa_fcs_lport_sm_offline(
 			/* If vport - send completion call back */
 			if (port->vport)
 				bfa_fcs_vport_stop_comp(port->vport);
+			else
+				bfa_wc_down(&(port->fabric->stop_wc));
 		} else {
 			bfa_sm_set_state(port, bfa_fcs_lport_sm_stopping);
 			list_for_each_safe(qe, qen, &port->rport_q) {
@@ -267,6 +273,8 @@ bfa_fcs_lport_sm_stopping(struct bfa_fcs_lport_s *port,
 			/* If vport - send completion call back */
 			if (port->vport)
 				bfa_fcs_vport_stop_comp(port->vport);
+			else
+				bfa_wc_down(&(port->fabric->stop_wc));
 		}
 		break;
 
@@ -340,7 +348,7 @@ bfa_fcs_lport_send_ls_rjt(struct bfa_fcs_lport_s *port, struct fchs_s *rx_fchs,
 	bfa_trc(port->fcs, rx_fchs->d_id);
 	bfa_trc(port->fcs, rx_fchs->s_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -370,7 +378,7 @@ bfa_fcs_lport_send_fcgs_rjt(struct bfa_fcs_lport_s *port,
 	bfa_trc(port->fcs, rx_fchs->d_id);
 	bfa_trc(port->fcs, rx_fchs->s_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -507,7 +515,7 @@ bfa_fcs_lport_echo(struct bfa_fcs_lport_s *port, struct fchs_s *rx_fchs,
 	bfa_trc(port->fcs, rx_fchs->s_id);
 	bfa_trc(port->fcs, rx_fchs->d_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -552,7 +560,7 @@ bfa_fcs_lport_rnid(struct bfa_fcs_lport_s *port, struct fchs_s *rx_fchs,
 	bfa_trc(port->fcs, rx_fchs->d_id);
 	bfa_trc(port->fcs, rx_len);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -684,7 +692,7 @@ bfa_fcs_lport_abts_acc(struct bfa_fcs_lport_s *port, struct fchs_s *rx_fchs)
 	bfa_trc(port->fcs, rx_fchs->d_id);
 	bfa_trc(port->fcs, rx_fchs->s_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -854,6 +862,25 @@ bfa_fcs_lport_get_rport_by_pid(struct bfa_fcs_lport_s *port, u32 pid)
 }
 
 /*
+ * OLD_PID based Lookup for a R-Port in the Port R-Port Queue
+ */
+struct bfa_fcs_rport_s *
+bfa_fcs_lport_get_rport_by_old_pid(struct bfa_fcs_lport_s *port, u32 pid)
+{
+	struct bfa_fcs_rport_s *rport;
+	struct list_head	*qe;
+
+	list_for_each(qe, &port->rport_q) {
+		rport = (struct bfa_fcs_rport_s *) qe;
+		if (rport->old_pid == pid)
+			return rport;
+	}
+
+	bfa_trc(port->fcs, pid);
+	return NULL;
+}
+
+/*
  *   PWWN based Lookup for a R-Port in the Port R-Port Queue
  */
 struct bfa_fcs_rport_s *
@@ -892,6 +919,26 @@ bfa_fcs_lport_get_rport_by_nwwn(struct bfa_fcs_lport_s *port, wwn_t nwwn)
 }
 
 /*
+ * PWWN & PID based Lookup for a R-Port in the Port R-Port Queue
+ */
+struct bfa_fcs_rport_s *
+bfa_fcs_lport_get_rport_by_qualifier(struct bfa_fcs_lport_s *port,
+				     wwn_t pwwn, u32 pid)
+{
+	struct bfa_fcs_rport_s *rport;
+	struct list_head	*qe;
+
+	list_for_each(qe, &port->rport_q) {
+		rport = (struct bfa_fcs_rport_s *) qe;
+		if (wwn_is_equal(rport->pwwn, pwwn) && rport->pid == pid)
+			return rport;
+	}
+
+	bfa_trc(port->fcs, pwwn);
+	return NULL;
+}
+
+/*
  * Called by rport module when new rports are discovered.
  */
 void
@@ -939,6 +986,16 @@ bfa_fcs_lport_offline(struct bfa_fcs_lport_s *port)
 }
 
 /*
+ * Called by fabric for base port and by vport for virtual ports
+ * when target mode driver is unloaded.
+ */
+void
+bfa_fcs_lport_stop(struct bfa_fcs_lport_s *port)
+{
+	bfa_sm_send_event(port, BFA_FCS_PORT_SM_STOP);
+}
+
+/*
  * Called by fabric to delete base lport and associated resources.
  *
  * Called by vport to delete lport and associated resources. Should call
@@ -1657,10 +1714,11 @@ bfa_fcs_lport_fdmi_send_rhba(void *fdmi_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &fdmi->fcxp_wqe,
-					bfa_fcs_lport_fdmi_send_rhba, fdmi);
+				bfa_fcs_lport_fdmi_send_rhba, fdmi, BFA_TRUE);
 		return;
 	}
 	fdmi->fcxp = fcxp;
@@ -1931,10 +1989,11 @@ bfa_fcs_lport_fdmi_send_rprt(void *fdmi_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &fdmi->fcxp_wqe,
-					bfa_fcs_lport_fdmi_send_rprt, fdmi);
+				bfa_fcs_lport_fdmi_send_rprt, fdmi, BFA_TRUE);
 		return;
 	}
 	fdmi->fcxp = fcxp;
@@ -2146,10 +2205,11 @@ bfa_fcs_lport_fdmi_send_rpa(void *fdmi_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &fdmi->fcxp_wqe,
-					bfa_fcs_lport_fdmi_send_rpa, fdmi);
+				bfa_fcs_lport_fdmi_send_rpa, fdmi, BFA_TRUE);
 		return;
 	}
 	fdmi->fcxp = fcxp;
@@ -2736,10 +2796,11 @@ bfa_fcs_lport_ms_send_gmal(void *ms_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ms->fcxp_wqe,
-					bfa_fcs_lport_ms_send_gmal, ms);
+				bfa_fcs_lport_ms_send_gmal, ms, BFA_TRUE);
 		return;
 	}
 	ms->fcxp = fcxp;
@@ -2936,10 +2997,11 @@ bfa_fcs_lport_ms_send_gfn(void *ms_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ms->fcxp_wqe,
-					bfa_fcs_lport_ms_send_gfn, ms);
+				bfa_fcs_lport_ms_send_gfn, ms, BFA_TRUE);
 		return;
 	}
 	ms->fcxp = fcxp;
@@ -3012,11 +3074,12 @@ bfa_fcs_lport_ms_send_plogi(void *ms_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ms_plogi_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ms->fcxp_wqe,
-					bfa_fcs_lport_ms_send_plogi, ms);
+				bfa_fcs_lport_ms_send_plogi, ms, BFA_TRUE);
 		return;
 	}
 	ms->fcxp = fcxp;
@@ -3166,6 +3229,10 @@ static void     bfa_fcs_lport_ns_send_rff_id(void *ns_cbarg,
 					    struct bfa_fcxp_s *fcxp_alloced);
 static void     bfa_fcs_lport_ns_send_gid_ft(void *ns_cbarg,
 					    struct bfa_fcxp_s *fcxp_alloced);
+static void	bfa_fcs_lport_ns_send_rnn_id(void *ns_cbarg,
+					struct bfa_fcxp_s *fcxp_alloced);
+static void	bfa_fcs_lport_ns_send_rsnn_nn(void *ns_cbarg,
+					struct bfa_fcxp_s *fcxp_alloced);
 static void     bfa_fcs_lport_ns_timeout(void *arg);
 static void     bfa_fcs_lport_ns_plogi_response(void *fcsarg,
 					       struct bfa_fcxp_s *fcxp,
@@ -3202,6 +3269,20 @@ static void     bfa_fcs_lport_ns_gid_ft_response(void *fcsarg,
 						u32 rsp_len,
 						u32 resid_len,
 						struct fchs_s *rsp_fchs);
+static void     bfa_fcs_lport_ns_rnn_id_response(void *fcsarg,
+						struct bfa_fcxp_s *fcxp,
+						void *cbarg,
+						bfa_status_t req_status,
+						u32 rsp_len,
+						u32 resid_len,
+						struct fchs_s *rsp_fchs);
+static void     bfa_fcs_lport_ns_rsnn_nn_response(void *fcsarg,
+						struct bfa_fcxp_s *fcxp,
+						void *cbarg,
+						bfa_status_t req_status,
+						u32 rsp_len,
+						u32 resid_len,
+						struct fchs_s *rsp_fchs);
 static void     bfa_fcs_lport_ns_process_gidft_pids(
 				struct bfa_fcs_lport_s *port,
 				u32 *pid_buf, u32 n_pids);
@@ -3226,6 +3307,8 @@ enum vport_ns_event {
 	NSSM_EVENT_RFTID_SENT = 9,
 	NSSM_EVENT_RFFID_SENT = 10,
 	NSSM_EVENT_GIDFT_SENT = 11,
+	NSSM_EVENT_RNNID_SENT = 12,
+	NSSM_EVENT_RSNN_NN_SENT = 13,
 };
 
 static void     bfa_fcs_lport_ns_sm_offline(struct bfa_fcs_lport_ns_s *ns,
@@ -3266,6 +3349,21 @@ static void     bfa_fcs_lport_ns_sm_gid_ft_retry(struct bfa_fcs_lport_ns_s *ns,
 						enum vport_ns_event event);
 static void     bfa_fcs_lport_ns_sm_online(struct bfa_fcs_lport_ns_s *ns,
 					  enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_sending_rnn_id(
+					struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_rnn_id(struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_rnn_id_retry(struct bfa_fcs_lport_ns_s *ns,
+						enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_sending_rsnn_nn(
+					struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_rsnn_nn(struct bfa_fcs_lport_ns_s *ns,
+						enum vport_ns_event event);
+static void     bfa_fcs_lport_ns_sm_rsnn_nn_retry(
+					struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event);
 /*
  *	Start in offline state - awaiting linkup
  */
@@ -3333,8 +3431,9 @@ bfa_fcs_lport_ns_sm_plogi(struct bfa_fcs_lport_ns_s *ns,
 		break;
 
 	case NSSM_EVENT_RSP_OK:
-		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rspn_id);
-		bfa_fcs_lport_ns_send_rspn_id(ns, NULL);
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rnn_id);
+		ns->num_rnnid_retries = 0;
+		bfa_fcs_lport_ns_send_rnn_id(ns, NULL);
 		break;
 
 	case NSSM_EVENT_PORT_OFFLINE:
@@ -3374,6 +3473,176 @@ bfa_fcs_lport_ns_sm_plogi_retry(struct bfa_fcs_lport_ns_s *ns,
 }
 
 static void
+bfa_fcs_lport_ns_sm_sending_rnn_id(struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_RNNID_SENT:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_rnn_id);
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		bfa_fcxp_walloc_cancel(BFA_FCS_GET_HAL_FROM_PORT(ns->port),
+						&ns->fcxp_wqe);
+		break;
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
+bfa_fcs_lport_ns_sm_rnn_id(struct bfa_fcs_lport_ns_s *ns,
+				enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_RSP_OK:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rsnn_nn);
+		ns->num_rnnid_retries = 0;
+		ns->num_rsnn_nn_retries = 0;
+		bfa_fcs_lport_ns_send_rsnn_nn(ns, NULL);
+		break;
+
+	case NSSM_EVENT_RSP_ERROR:
+		if (ns->num_rnnid_retries < BFA_FCS_MAX_NS_RETRIES) {
+			bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_rnn_id_retry);
+			ns->port->stats.ns_retries++;
+			ns->num_rnnid_retries++;
+			bfa_timer_start(BFA_FCS_GET_HAL_FROM_PORT(ns->port),
+				&ns->timer, bfa_fcs_lport_ns_timeout, ns,
+				BFA_FCS_RETRY_TIMEOUT);
+		} else {
+			bfa_sm_set_state(ns,
+				bfa_fcs_lport_ns_sm_sending_rspn_id);
+			bfa_fcs_lport_ns_send_rspn_id(ns, NULL);
+		}
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_fcxp_discard(ns->fcxp);
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		break;
+
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
+bfa_fcs_lport_ns_sm_rnn_id_retry(struct bfa_fcs_lport_ns_s *ns,
+				enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_TIMEOUT:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rnn_id);
+		bfa_fcs_lport_ns_send_rnn_id(ns, NULL);
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		bfa_timer_stop(&ns->timer);
+		break;
+
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
+bfa_fcs_lport_ns_sm_sending_rsnn_nn(struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_RSNN_NN_SENT:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_rsnn_nn);
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		bfa_fcxp_walloc_cancel(BFA_FCS_GET_HAL_FROM_PORT(ns->port),
+			&ns->fcxp_wqe);
+		break;
+
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
+bfa_fcs_lport_ns_sm_rsnn_nn(struct bfa_fcs_lport_ns_s *ns,
+				enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_RSP_OK:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rspn_id);
+		ns->num_rsnn_nn_retries = 0;
+		bfa_fcs_lport_ns_send_rspn_id(ns, NULL);
+		break;
+
+	case NSSM_EVENT_RSP_ERROR:
+		if (ns->num_rsnn_nn_retries < BFA_FCS_MAX_NS_RETRIES) {
+			bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_rsnn_nn_retry);
+			ns->port->stats.ns_retries++;
+			ns->num_rsnn_nn_retries++;
+			bfa_timer_start(BFA_FCS_GET_HAL_FROM_PORT(ns->port),
+					&ns->timer, bfa_fcs_lport_ns_timeout,
+					ns, BFA_FCS_RETRY_TIMEOUT);
+		} else {
+			bfa_sm_set_state(ns,
+				bfa_fcs_lport_ns_sm_sending_rspn_id);
+			bfa_fcs_lport_ns_send_rspn_id(ns, NULL);
+		}
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		bfa_fcxp_discard(ns->fcxp);
+		break;
+
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
+bfa_fcs_lport_ns_sm_rsnn_nn_retry(struct bfa_fcs_lport_ns_s *ns,
+					enum vport_ns_event event)
+{
+	bfa_trc(ns->port->fcs, ns->port->port_cfg.pwwn);
+	bfa_trc(ns->port->fcs, event);
+
+	switch (event) {
+	case NSSM_EVENT_TIMEOUT:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_sending_rsnn_nn);
+		bfa_fcs_lport_ns_send_rsnn_nn(ns, NULL);
+		break;
+
+	case NSSM_EVENT_PORT_OFFLINE:
+		bfa_sm_set_state(ns, bfa_fcs_lport_ns_sm_offline);
+		bfa_timer_stop(&ns->timer);
+		break;
+
+	default:
+		bfa_sm_fault(ns->port->fcs, event);
+	}
+}
+
+static void
 bfa_fcs_lport_ns_sm_sending_rspn_id(struct bfa_fcs_lport_ns_s *ns,
 				   enum vport_ns_event event)
 {
@@ -3770,11 +4039,12 @@ bfa_fcs_lport_ns_send_plogi(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ns_plogi_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
-					bfa_fcs_lport_ns_send_plogi, ns);
+				bfa_fcs_lport_ns_send_plogi, ns, BFA_TRUE);
 		return;
 	}
 	ns->fcxp = fcxp;
@@ -3853,6 +4123,162 @@ bfa_fcs_lport_ns_plogi_response(void *fcsarg, struct bfa_fcxp_s *fcxp,
 }
 
 /*
+ * Register node name for port_id
+ */
+static void
+bfa_fcs_lport_ns_send_rnn_id(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
+{
+	struct bfa_fcs_lport_ns_s *ns = ns_cbarg;
+	struct bfa_fcs_lport_s *port = ns->port;
+	struct fchs_s  fchs;
+	int	len;
+	struct bfa_fcxp_s *fcxp;
+
+	bfa_trc(port->fcs, port->port_cfg.pwwn);
+
+	fcxp = fcxp_alloced ? fcxp_alloced :
+			bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
+	if (!fcxp) {
+		port->stats.ns_rnnid_alloc_wait++;
+		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
+				bfa_fcs_lport_ns_send_rnn_id, ns, BFA_TRUE);
+		return;
+	}
+
+	ns->fcxp = fcxp;
+
+	len = fc_rnnid_build(&fchs, bfa_fcxp_get_reqbuf(fcxp),
+				bfa_fcs_lport_get_fcid(port),
+				bfa_fcs_lport_get_fcid(port),
+				bfa_fcs_lport_get_nwwn(port));
+
+	bfa_fcxp_send(fcxp, NULL, port->fabric->vf_id, port->lp_tag, BFA_FALSE,
+			  FC_CLASS_3, len, &fchs,
+			  bfa_fcs_lport_ns_rnn_id_response, (void *)ns,
+			  FC_MAX_PDUSZ, FC_FCCT_TOV);
+
+	port->stats.ns_rnnid_sent++;
+	bfa_sm_send_event(ns, NSSM_EVENT_RNNID_SENT);
+}
+
+static void
+bfa_fcs_lport_ns_rnn_id_response(void *fcsarg, struct bfa_fcxp_s *fcxp,
+				void *cbarg, bfa_status_t req_status,
+				u32 rsp_len, u32 resid_len,
+				struct fchs_s *rsp_fchs)
+
+{
+	struct bfa_fcs_lport_ns_s *ns = (struct bfa_fcs_lport_ns_s *) cbarg;
+	struct bfa_fcs_lport_s *port = ns->port;
+	struct ct_hdr_s	*cthdr = NULL;
+
+	bfa_trc(port->fcs, port->port_cfg.pwwn);
+
+	/*
+	 * Sanity Checks
+	 */
+	if (req_status != BFA_STATUS_OK) {
+		bfa_trc(port->fcs, req_status);
+		port->stats.ns_rnnid_rsp_err++;
+		bfa_sm_send_event(ns, NSSM_EVENT_RSP_ERROR);
+		return;
+	}
+
+	cthdr = (struct ct_hdr_s *) BFA_FCXP_RSP_PLD(fcxp);
+	cthdr->cmd_rsp_code = be16_to_cpu(cthdr->cmd_rsp_code);
+
+	if (cthdr->cmd_rsp_code == CT_RSP_ACCEPT) {
+		port->stats.ns_rnnid_accepts++;
+		bfa_sm_send_event(ns, NSSM_EVENT_RSP_OK);
+		return;
+	}
+
+	port->stats.ns_rnnid_rejects++;
+	bfa_trc(port->fcs, cthdr->reason_code);
+	bfa_trc(port->fcs, cthdr->exp_code);
+	bfa_sm_send_event(ns, NSSM_EVENT_RSP_ERROR);
+}
+
+/*
+ * Register the symbolic node name for a given node name.
+ */
+static void
+bfa_fcs_lport_ns_send_rsnn_nn(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
+{
+	struct bfa_fcs_lport_ns_s *ns = ns_cbarg;
+	struct bfa_fcs_lport_s *port = ns->port;
+	struct fchs_s  fchs;
+	int     len;
+	struct bfa_fcxp_s *fcxp;
+	u8 *nsymbl;
+
+	bfa_trc(port->fcs, port->port_cfg.pwwn);
+
+	fcxp = fcxp_alloced ? fcxp_alloced :
+			bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
+	if (!fcxp) {
+		port->stats.ns_rsnn_nn_alloc_wait++;
+		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
+				bfa_fcs_lport_ns_send_rsnn_nn, ns, BFA_TRUE);
+		return;
+	}
+	ns->fcxp = fcxp;
+
+	nsymbl = (u8 *) &(bfa_fcs_lport_get_nsym_name(
+					bfa_fcs_get_base_port(port->fcs)));
+
+	len = fc_rsnn_nn_build(&fchs, bfa_fcxp_get_reqbuf(fcxp),
+				bfa_fcs_lport_get_fcid(port),
+				bfa_fcs_lport_get_nwwn(port), nsymbl);
+
+	bfa_fcxp_send(fcxp, NULL, port->fabric->vf_id, port->lp_tag, BFA_FALSE,
+			  FC_CLASS_3, len, &fchs,
+			  bfa_fcs_lport_ns_rsnn_nn_response, (void *)ns,
+			  FC_MAX_PDUSZ, FC_FCCT_TOV);
+
+	port->stats.ns_rsnn_nn_sent++;
+
+	bfa_sm_send_event(ns, NSSM_EVENT_RSNN_NN_SENT);
+}
+
+static void
+bfa_fcs_lport_ns_rsnn_nn_response(void *fcsarg, struct bfa_fcxp_s *fcxp,
+				void *cbarg, bfa_status_t req_status,
+				u32 rsp_len, u32 resid_len,
+				struct fchs_s *rsp_fchs)
+{
+	struct bfa_fcs_lport_ns_s *ns = (struct bfa_fcs_lport_ns_s *) cbarg;
+	struct bfa_fcs_lport_s *port = ns->port;
+	struct ct_hdr_s	*cthdr = NULL;
+
+	bfa_trc(port->fcs, port->port_cfg.pwwn);
+
+	/*
+	 * Sanity Checks
+	 */
+	if (req_status != BFA_STATUS_OK) {
+		bfa_trc(port->fcs, req_status);
+		port->stats.ns_rsnn_nn_rsp_err++;
+		bfa_sm_send_event(ns, NSSM_EVENT_RSP_ERROR);
+		return;
+	}
+
+	cthdr = (struct ct_hdr_s *) BFA_FCXP_RSP_PLD(fcxp);
+	cthdr->cmd_rsp_code = be16_to_cpu(cthdr->cmd_rsp_code);
+
+	if (cthdr->cmd_rsp_code == CT_RSP_ACCEPT) {
+		port->stats.ns_rsnn_nn_accepts++;
+		bfa_sm_send_event(ns, NSSM_EVENT_RSP_OK);
+		return;
+	}
+
+	port->stats.ns_rsnn_nn_rejects++;
+	bfa_trc(port->fcs, cthdr->reason_code);
+	bfa_trc(port->fcs, cthdr->exp_code);
+	bfa_sm_send_event(ns, NSSM_EVENT_RSP_ERROR);
+}
+
+/*
  * Register the symbolic port name.
  */
 static void
@@ -3870,11 +4296,12 @@ bfa_fcs_lport_ns_send_rspn_id(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ns_rspnid_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
-					bfa_fcs_lport_ns_send_rspn_id, ns);
+				bfa_fcs_lport_ns_send_rspn_id, ns, BFA_TRUE);
 		return;
 	}
 	ns->fcxp = fcxp;
@@ -3971,11 +4398,12 @@ bfa_fcs_lport_ns_send_rft_id(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ns_rftid_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
-					bfa_fcs_lport_ns_send_rft_id, ns);
+				bfa_fcs_lport_ns_send_rft_id, ns, BFA_TRUE);
 		return;
 	}
 	ns->fcxp = fcxp;
@@ -4044,11 +4472,12 @@ bfa_fcs_lport_ns_send_rff_id(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ns_rffid_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
-					bfa_fcs_lport_ns_send_rff_id, ns);
+				bfa_fcs_lport_ns_send_rff_id, ns, BFA_TRUE);
 		return;
 	}
 	ns->fcxp = fcxp;
@@ -4127,11 +4556,12 @@ bfa_fcs_lport_ns_send_gid_ft(void *ns_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(port->fcs, port->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		port->stats.ns_gidft_alloc_wait++;
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
-					bfa_fcs_lport_ns_send_gid_ft, ns);
+				bfa_fcs_lport_ns_send_gid_ft, ns, BFA_TRUE);
 		return;
 	}
 	ns->fcxp = fcxp;
@@ -4261,6 +4691,10 @@ bfa_fcs_lport_ns_process_gidft_pids(struct bfa_fcs_lport_s *port, u32 *pid_buf,
 	struct fcgs_gidft_resp_s *gidft_entry;
 	struct bfa_fcs_rport_s *rport;
 	u32        ii;
+	struct bfa_fcs_fabric_s *fabric = port->fabric;
+	struct bfa_fcs_vport_s *vport;
+	struct list_head *qe;
+	u8 found = 0;
 
 	for (ii = 0; ii < n_pids; ii++) {
 		gidft_entry = (struct fcgs_gidft_resp_s *) &pid_buf[ii];
@@ -4269,6 +4703,29 @@ bfa_fcs_lport_ns_process_gidft_pids(struct bfa_fcs_lport_s *port, u32 *pid_buf,
 			continue;
 
 		/*
+		 * Ignore PID if it is of base port
+		 * (Avoid vports discovering base port as remote port)
+		 */
+		if (gidft_entry->pid == fabric->bport.pid)
+			continue;
+
+		/*
+		 * Ignore PID if it is of vport created on the same base port
+		 * (Avoid vport discovering every other vport created on the
+		 * same port as remote port)
+		 */
+		list_for_each(qe, &fabric->vport_q) {
+			vport = (struct bfa_fcs_vport_s *) qe;
+			if (vport->lport.pid == gidft_entry->pid)
+				found = 1;
+		}
+
+		if (found) {
+			found = 0;
+			continue;
+		}
+
+		/*
 		 * Check if this rport already exists
 		 */
 		rport = bfa_fcs_lport_get_rport_by_pid(port, gidft_entry->pid);
@@ -4335,7 +4792,8 @@ bfa_fcs_lport_ns_query(struct bfa_fcs_lport_s *port)
 	struct bfa_fcs_lport_ns_s *ns = BFA_FCS_GET_NS_FROM_PORT(port);
 
 	bfa_trc(port->fcs, port->pid);
-	bfa_sm_send_event(ns, NSSM_EVENT_NS_QUERY);
+	if (bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_online))
+		bfa_sm_send_event(ns, NSSM_EVENT_NS_QUERY);
 }
 
 static void
@@ -4355,6 +4813,70 @@ bfa_fcs_lport_ns_boot_target_disc(bfa_fcs_lport_t *port)
 	}
 }
 
+void
+bfa_fcs_lport_ns_util_send_rspn_id(void *cbarg, struct bfa_fcxp_s *fcxp_alloced)
+{
+	struct bfa_fcs_lport_ns_s *ns = cbarg;
+	struct bfa_fcs_lport_s *port = ns->port;
+	struct fchs_s fchs;
+	struct bfa_fcxp_s *fcxp;
+	u8 symbl[256];
+	u8 *psymbl = &symbl[0];
+	int len;
+
+	if (!bfa_sm_cmp_state(port, bfa_fcs_lport_sm_online))
+		return;
+
+	/* Avoid sending RSPN in the following states. */
+	if (bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_offline) ||
+	    bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_plogi_sending) ||
+	    bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_plogi) ||
+	    bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_plogi_retry) ||
+	    bfa_sm_cmp_state(ns, bfa_fcs_lport_ns_sm_rspn_id_retry))
+		return;
+
+	memset(symbl, 0, sizeof(symbl));
+	bfa_trc(port->fcs, port->port_cfg.pwwn);
+
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
+	if (!fcxp) {
+		port->stats.ns_rspnid_alloc_wait++;
+		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &ns->fcxp_wqe,
+			bfa_fcs_lport_ns_util_send_rspn_id, ns, BFA_FALSE);
+		return;
+	}
+
+	ns->fcxp = fcxp;
+
+	if (port->vport) {
+		/*
+		 * For Vports, we append the vport's port symbolic name
+		 * to that of the base port.
+		 */
+		strncpy((char *)psymbl, (char *)&(bfa_fcs_lport_get_psym_name
+			(bfa_fcs_get_base_port(port->fcs))),
+			strlen((char *)&bfa_fcs_lport_get_psym_name(
+			bfa_fcs_get_base_port(port->fcs))));
+
+		/* Ensure we have a null terminating string. */
+		((char *)psymbl)[strlen((char *)&bfa_fcs_lport_get_psym_name(
+		 bfa_fcs_get_base_port(port->fcs)))] = 0;
+
+		strncat((char *)psymbl,
+			(char *)&(bfa_fcs_lport_get_psym_name(port)),
+			strlen((char *)&bfa_fcs_lport_get_psym_name(port)));
+	}
+
+	len = fc_rspnid_build(&fchs, bfa_fcxp_get_reqbuf(fcxp),
+			      bfa_fcs_lport_get_fcid(port), 0, psymbl);
+
+	bfa_fcxp_send(fcxp, NULL, port->fabric->vf_id, port->lp_tag, BFA_FALSE,
+		      FC_CLASS_3, len, &fchs, NULL, NULL, FC_MAX_PDUSZ, 0);
+
+	port->stats.ns_rspnid_sent++;
+}
+
 /*
  * FCS SCN
  */
@@ -4529,10 +5051,11 @@ bfa_fcs_lport_scn_send_scr(void *scn_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 	bfa_trc(port->fcs, port->pid);
 	bfa_trc(port->fcs, port->port_cfg.pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &scn->fcxp_wqe,
-					bfa_fcs_lport_scn_send_scr, scn);
+				bfa_fcs_lport_scn_send_scr, scn, BFA_TRUE);
 		return;
 	}
 	scn->fcxp = fcxp;
@@ -4614,7 +5137,7 @@ bfa_fcs_lport_scn_send_ls_acc(struct bfa_fcs_lport_s *port,
 
 	bfa_trc(port->fcs, rx_fchs->s_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -4688,14 +5211,33 @@ static void
 bfa_fcs_lport_scn_portid_rscn(struct bfa_fcs_lport_s *port, u32 rpid)
 {
 	struct bfa_fcs_rport_s *rport;
+	struct bfa_fcs_fabric_s *fabric = port->fabric;
+	struct bfa_fcs_vport_s *vport;
+	struct list_head *qe;
 
 	bfa_trc(port->fcs, rpid);
 
 	/*
+	 * Ignore PID if it is of base port or of vports created on the
+	 * same base port. It is to avoid vports discovering base port or
+	 * other vports created on same base port as remote port
+	 */
+	if (rpid == fabric->bport.pid)
+		return;
+
+	list_for_each(qe, &fabric->vport_q) {
+		vport = (struct bfa_fcs_vport_s *) qe;
+		if (vport->lport.pid == rpid)
+			return;
+	}
+	/*
 	 * If this is an unknown device, then it just came online.
 	 * Otherwise let rport handle the RSCN event.
 	 */
 	rport = bfa_fcs_lport_get_rport_by_pid(port, rpid);
+	if (!rport)
+		rport = bfa_fcs_lport_get_rport_by_old_pid(port, rpid);
+
 	if (rport == NULL) {
 		/*
 		 * If min cfg mode is enabled, we donot need to
@@ -4888,15 +5430,15 @@ bfa_fcs_lport_get_rport(struct bfa_fcs_lport_s *port, wwn_t wwn, int index,
 }
 
 void
-bfa_fcs_lport_get_rports(struct bfa_fcs_lport_s *port,
-	 wwn_t rport_wwns[], int *nrports)
+bfa_fcs_lport_get_rport_quals(struct bfa_fcs_lport_s *port,
+		struct bfa_rport_qualifier_s rports[], int *nrports)
 {
 	struct list_head	*qh, *qe;
 	struct bfa_fcs_rport_s *rport = NULL;
 	int	i;
 	struct bfa_fcs_s	*fcs;
 
-	if (port == NULL || rport_wwns == NULL || *nrports == 0)
+	if (port == NULL || rports == NULL || *nrports == 0)
 		return;
 
 	fcs = port->fcs;
@@ -4916,7 +5458,13 @@ bfa_fcs_lport_get_rports(struct bfa_fcs_lport_s *port,
 			continue;
 		}
 
-		rport_wwns[i] = rport->pwwn;
+		if (!rport->pwwn && !rport->pid) {
+			qe = bfa_q_next(qe);
+			continue;
+		}
+
+		rports[i].pwwn = rport->pwwn;
+		rports[i].pid = rport->pid;
 
 		i++;
 		qe = bfa_q_next(qe);
@@ -5760,6 +6308,16 @@ bfa_fcs_vport_cleanup(struct bfa_fcs_vport_s *vport)
 {
 	vport->vport_stats.fab_cleanup++;
 }
+
+/*
+ * Stop notification from fabric SM. To be invoked from within FCS.
+ */
+void
+bfa_fcs_vport_fcs_stop(struct bfa_fcs_vport_s *vport)
+{
+	bfa_sm_send_event(vport, BFA_FCS_VPORT_SM_STOP);
+}
+
 /*
  * delete notification from fabric SM. To be invoked from within FCS.
  */
diff --git a/drivers/scsi/bfa/bfa_fcs_rport.c b/drivers/scsi/bfa/bfa_fcs_rport.c
index fe0463a1db0..cc43b2a58ce 100644
--- a/drivers/scsi/bfa/bfa_fcs_rport.c
+++ b/drivers/scsi/bfa/bfa_fcs_rport.c
@@ -30,14 +30,22 @@ static u32
 bfa_fcs_rport_del_timeout = BFA_FCS_RPORT_DEF_DEL_TIMEOUT * 1000;
 	 /* In millisecs */
 /*
+ * bfa_fcs_rport_max_logins is max count of bfa_fcs_rports
+ * whereas DEF_CFG_NUM_RPORTS is max count of bfa_rports
+ */
+static u32 bfa_fcs_rport_max_logins = BFA_FCS_MAX_RPORT_LOGINS;
+
+/*
  * forward declarations
  */
 static struct bfa_fcs_rport_s *bfa_fcs_rport_alloc(
 		struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 rpid);
 static void	bfa_fcs_rport_free(struct bfa_fcs_rport_s *rport);
 static void	bfa_fcs_rport_hal_online(struct bfa_fcs_rport_s *rport);
-static void	bfa_fcs_rport_online_action(struct bfa_fcs_rport_s *rport);
-static void	bfa_fcs_rport_offline_action(struct bfa_fcs_rport_s *rport);
+static void	bfa_fcs_rport_fcs_online_action(struct bfa_fcs_rport_s *rport);
+static void	bfa_fcs_rport_hal_online_action(struct bfa_fcs_rport_s *rport);
+static void	bfa_fcs_rport_fcs_offline_action(struct bfa_fcs_rport_s *rport);
+static void	bfa_fcs_rport_hal_offline_action(struct bfa_fcs_rport_s *rport);
 static void	bfa_fcs_rport_update(struct bfa_fcs_rport_s *rport,
 					struct fc_logi_s *plogi);
 static void	bfa_fcs_rport_timeout(void *arg);
@@ -76,6 +84,7 @@ static void	bfa_fcs_rport_send_ls_rjt(struct bfa_fcs_rport_s *rport,
 static void	bfa_fcs_rport_process_adisc(struct bfa_fcs_rport_s *rport,
 				struct fchs_s *rx_fchs, u16 len);
 static void bfa_fcs_rport_send_prlo_acc(struct bfa_fcs_rport_s *rport);
+static void	bfa_fcs_rport_hal_offline(struct bfa_fcs_rport_s *rport);
 
 static void	bfa_fcs_rport_sm_uninit(struct bfa_fcs_rport_s *rport,
 					enum rport_event event);
@@ -87,6 +96,8 @@ static void	bfa_fcs_rport_sm_plogi_retry(struct bfa_fcs_rport_s *rport,
 						enum rport_event event);
 static void	bfa_fcs_rport_sm_plogi(struct bfa_fcs_rport_s *rport,
 					enum rport_event event);
+static void	bfa_fcs_rport_sm_fc4_fcs_online(struct bfa_fcs_rport_s *rport,
+					enum rport_event event);
 static void	bfa_fcs_rport_sm_hal_online(struct bfa_fcs_rport_s *rport,
 						enum rport_event event);
 static void	bfa_fcs_rport_sm_online(struct bfa_fcs_rport_s *rport,
@@ -123,6 +134,10 @@ static void	bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 						enum rport_event event);
 static void	bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 						enum rport_event event);
+static void	bfa_fcs_rport_sm_fc4_off_delete(struct bfa_fcs_rport_s *rport,
+						enum rport_event event);
+static void	bfa_fcs_rport_sm_delete_pending(struct bfa_fcs_rport_s *rport,
+						enum rport_event event);
 
 static struct bfa_sm_table_s rport_sm_table[] = {
 	{BFA_SM(bfa_fcs_rport_sm_uninit), BFA_RPORT_UNINIT},
@@ -130,6 +145,7 @@ static struct bfa_sm_table_s rport_sm_table[] = {
 	{BFA_SM(bfa_fcs_rport_sm_plogiacc_sending), BFA_RPORT_ONLINE},
 	{BFA_SM(bfa_fcs_rport_sm_plogi_retry), BFA_RPORT_PLOGI_RETRY},
 	{BFA_SM(bfa_fcs_rport_sm_plogi), BFA_RPORT_PLOGI},
+	{BFA_SM(bfa_fcs_rport_sm_fc4_fcs_online), BFA_RPORT_ONLINE},
 	{BFA_SM(bfa_fcs_rport_sm_hal_online), BFA_RPORT_ONLINE},
 	{BFA_SM(bfa_fcs_rport_sm_online), BFA_RPORT_ONLINE},
 	{BFA_SM(bfa_fcs_rport_sm_nsquery_sending), BFA_RPORT_NSQUERY},
@@ -167,8 +183,8 @@ bfa_fcs_rport_sm_uninit(struct bfa_fcs_rport_s *rport, enum rport_event event)
 		break;
 
 	case RPSM_EVENT_PLOGI_RCVD:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_plogiacc_sending);
-		bfa_fcs_rport_send_plogiacc(rport, NULL);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
@@ -252,8 +268,8 @@ bfa_fcs_rport_sm_plogiacc_sending(struct bfa_fcs_rport_s *rport,
 
 	switch (event) {
 	case RPSM_EVENT_FCXP_SENT:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	case RPSM_EVENT_DELETE:
@@ -348,9 +364,9 @@ bfa_fcs_rport_sm_plogi_retry(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_timer_stop(&rport->timer);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	default:
@@ -370,9 +386,9 @@ bfa_fcs_rport_sm_plogi(struct bfa_fcs_rport_s *rport, enum rport_event event)
 
 	switch (event) {
 	case RPSM_EVENT_ACCEPTED:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		rport->plogi_retries = 0;
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
@@ -397,6 +413,7 @@ bfa_fcs_rport_sm_plogi(struct bfa_fcs_rport_s *rport, enum rport_event event)
 					BFA_FCS_RETRY_TIMEOUT);
 		} else {
 			bfa_stats(rport->port, rport_del_max_plogi_retry);
+			rport->old_pid = rport->pid;
 			rport->pid = 0;
 			bfa_sm_set_state(rport, bfa_fcs_rport_sm_offline);
 			bfa_timer_start(rport->fcs->bfa, &rport->timer,
@@ -443,13 +460,77 @@ bfa_fcs_rport_sm_plogi(struct bfa_fcs_rport_s *rport, enum rport_event event)
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
+		break;
+
+	default:
+		bfa_sm_fault(rport->fcs, event);
+	}
+}
+
+/*
+ * PLOGI is done. Await bfa_fcs_itnim to ascertain the scsi function
+ */
+static void
+bfa_fcs_rport_sm_fc4_fcs_online(struct bfa_fcs_rport_s *rport,
+				enum rport_event event)
+{
+	bfa_trc(rport->fcs, rport->pwwn);
+	bfa_trc(rport->fcs, rport->pid);
+	bfa_trc(rport->fcs, event);
+
+	switch (event) {
+	case RPSM_EVENT_FC4_FCS_ONLINE:
+		if (rport->scsi_function == BFA_RPORT_INITIATOR) {
+			if (!BFA_FCS_PID_IS_WKA(rport->pid))
+				bfa_fcs_rpf_rport_online(rport);
+			bfa_sm_set_state(rport, bfa_fcs_rport_sm_online);
+			break;
+		}
+
+		if (!rport->bfa_rport)
+			rport->bfa_rport =
+				bfa_rport_create(rport->fcs->bfa, rport);
+
+		if (rport->bfa_rport) {
+			bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+			bfa_fcs_rport_hal_online(rport);
+		} else {
+			bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
+			bfa_fcs_rport_fcs_offline_action(rport);
+		}
+		break;
+
+	case RPSM_EVENT_PLOGI_RCVD:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
+		rport->plogi_pending = BFA_TRUE;
+		bfa_fcs_rport_fcs_offline_action(rport);
+		break;
+
+	case RPSM_EVENT_PLOGI_COMP:
+	case RPSM_EVENT_LOGO_IMP:
+	case RPSM_EVENT_ADDRESS_CHANGE:
+	case RPSM_EVENT_SCN:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
+		bfa_fcs_rport_fcs_offline_action(rport);
+		break;
+
+	case RPSM_EVENT_LOGO_RCVD:
+	case RPSM_EVENT_PRLO_RCVD:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
+		bfa_fcs_rport_fcs_offline_action(rport);
+		break;
+
+	case RPSM_EVENT_DELETE:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
+		bfa_fcs_rport_fcs_offline_action(rport);
 		break;
 
 	default:
 		bfa_sm_fault(rport->fcs, event);
+		break;
 	}
 }
 
@@ -468,41 +549,34 @@ bfa_fcs_rport_sm_hal_online(struct bfa_fcs_rport_s *rport,
 	switch (event) {
 	case RPSM_EVENT_HCB_ONLINE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_online);
-		bfa_fcs_rport_online_action(rport);
+		bfa_fcs_rport_hal_online_action(rport);
 		break;
 
-	case RPSM_EVENT_PRLO_RCVD:
 	case RPSM_EVENT_PLOGI_COMP:
 		break;
 
+	case RPSM_EVENT_PRLO_RCVD:
 	case RPSM_EVENT_LOGO_RCVD:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_logorcv);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
+		bfa_fcs_rport_fcs_offline_action(rport);
 		break;
 
+	case RPSM_EVENT_SCN:
 	case RPSM_EVENT_LOGO_IMP:
 	case RPSM_EVENT_ADDRESS_CHANGE:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_offline);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
+		bfa_fcs_rport_fcs_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_PLOGI_RCVD:
 		rport->plogi_pending = BFA_TRUE;
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_offline);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
+		bfa_fcs_rport_fcs_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_DELETE:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_logosend);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
-		break;
-
-	case RPSM_EVENT_SCN:
-		/*
-		 * @todo
-		 * Ignore SCN - PLOGI just completed, FC-4 login should detect
-		 * device failures.
-		 */
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
+		bfa_fcs_rport_fcs_offline_action(rport);
 		break;
 
 	default:
@@ -537,18 +611,18 @@ bfa_fcs_rport_sm_online(struct bfa_fcs_rport_s *rport, enum rport_event event)
 	case RPSM_EVENT_LOGO_IMP:
 	case RPSM_EVENT_ADDRESS_CHANGE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_DELETE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
 	case RPSM_EVENT_PRLO_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
@@ -579,7 +653,7 @@ bfa_fcs_rport_sm_nsquery_sending(struct bfa_fcs_rport_s *rport,
 	case RPSM_EVENT_DELETE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_SCN:
@@ -592,24 +666,16 @@ bfa_fcs_rport_sm_nsquery_sending(struct bfa_fcs_rport_s *rport,
 	case RPSM_EVENT_PRLO_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_IMP:
-		rport->pid = 0;
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_offline);
-		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_timer_start(rport->fcs->bfa, &rport->timer,
-				bfa_fcs_rport_timeout, rport,
-				bfa_fcs_rport_del_timeout);
-		break;
-
 	case RPSM_EVENT_PLOGI_RCVD:
 	case RPSM_EVENT_ADDRESS_CHANGE:
 	case RPSM_EVENT_PLOGI_COMP:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	default:
@@ -642,14 +708,14 @@ bfa_fcs_rport_sm_nsquery(struct bfa_fcs_rport_s *rport, enum rport_event event)
 			bfa_fcs_rport_send_nsdisc(rport, NULL);
 		} else {
 			bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
-			bfa_fcs_rport_offline_action(rport);
+			bfa_fcs_rport_hal_offline_action(rport);
 		}
 		break;
 
 	case RPSM_EVENT_DELETE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_SCN:
@@ -659,7 +725,7 @@ bfa_fcs_rport_sm_nsquery(struct bfa_fcs_rport_s *rport, enum rport_event event)
 	case RPSM_EVENT_PRLO_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
@@ -668,7 +734,7 @@ bfa_fcs_rport_sm_nsquery(struct bfa_fcs_rport_s *rport, enum rport_event event)
 	case RPSM_EVENT_LOGO_IMP:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	default:
@@ -696,21 +762,21 @@ bfa_fcs_rport_sm_adisc_sending(struct bfa_fcs_rport_s *rport,
 	case RPSM_EVENT_DELETE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_IMP:
 	case RPSM_EVENT_ADDRESS_CHANGE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
 	case RPSM_EVENT_PRLO_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_SCN:
@@ -719,7 +785,7 @@ bfa_fcs_rport_sm_adisc_sending(struct bfa_fcs_rport_s *rport,
 	case RPSM_EVENT_PLOGI_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	default:
@@ -756,13 +822,13 @@ bfa_fcs_rport_sm_adisc(struct bfa_fcs_rport_s *rport, enum rport_event event)
 	case RPSM_EVENT_FAILED:
 	case RPSM_EVENT_ADDRESS_CHANGE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_DELETE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_SCN:
@@ -774,14 +840,14 @@ bfa_fcs_rport_sm_adisc(struct bfa_fcs_rport_s *rport, enum rport_event event)
 	case RPSM_EVENT_LOGO_IMP:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_offline);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
 	case RPSM_EVENT_PRLO_RCVD:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logorcv);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_offline_action(rport);
+		bfa_fcs_rport_hal_offline_action(rport);
 		break;
 
 	default:
@@ -803,13 +869,19 @@ bfa_fcs_rport_sm_fc4_logorcv(struct bfa_fcs_rport_s *rport,
 	switch (event) {
 	case RPSM_EVENT_FC4_OFFLINE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_logorcv);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_fcs_rport_hal_offline(rport);
 		break;
 
 	case RPSM_EVENT_DELETE:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_logosend);
+		if (rport->pid && (rport->prlo == BFA_TRUE))
+			bfa_fcs_rport_send_prlo_acc(rport);
+		if (rport->pid && (rport->prlo == BFA_FALSE))
+			bfa_fcs_rport_send_logo_acc(rport);
+
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_off_delete);
 		break;
 
+	case RPSM_EVENT_HCB_ONLINE:
 	case RPSM_EVENT_LOGO_RCVD:
 	case RPSM_EVENT_PRLO_RCVD:
 	case RPSM_EVENT_ADDRESS_CHANGE:
@@ -835,7 +907,20 @@ bfa_fcs_rport_sm_fc4_logosend(struct bfa_fcs_rport_s *rport,
 	switch (event) {
 	case RPSM_EVENT_FC4_OFFLINE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_logosend);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_fcs_rport_hal_offline(rport);
+		break;
+
+	case RPSM_EVENT_LOGO_RCVD:
+		bfa_fcs_rport_send_logo_acc(rport);
+	case RPSM_EVENT_PRLO_RCVD:
+		if (rport->prlo == BFA_TRUE)
+			bfa_fcs_rport_send_prlo_acc(rport);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_off_delete);
+		break;
+
+	case RPSM_EVENT_HCB_ONLINE:
+	case RPSM_EVENT_DELETE:
+		/* Rport is being deleted */
 		break;
 
 	default:
@@ -857,13 +942,23 @@ bfa_fcs_rport_sm_fc4_offline(struct bfa_fcs_rport_s *rport,
 	switch (event) {
 	case RPSM_EVENT_FC4_OFFLINE:
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_offline);
-		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+		bfa_fcs_rport_hal_offline(rport);
 		break;
 
-	case RPSM_EVENT_SCN:
-	case RPSM_EVENT_LOGO_IMP:
 	case RPSM_EVENT_LOGO_RCVD:
+		/*
+		 * Rport is going offline. Just ack the logo
+		 */
+		bfa_fcs_rport_send_logo_acc(rport);
+		break;
+
 	case RPSM_EVENT_PRLO_RCVD:
+		bfa_fcs_rport_send_prlo_acc(rport);
+		break;
+
+	case RPSM_EVENT_HCB_ONLINE:
+	case RPSM_EVENT_SCN:
+	case RPSM_EVENT_LOGO_IMP:
 	case RPSM_EVENT_ADDRESS_CHANGE:
 		/*
 		 * rport is already going offline.
@@ -907,24 +1002,23 @@ bfa_fcs_rport_sm_hcb_offline(struct bfa_fcs_rport_s *rport,
 		 */
 
 	case RPSM_EVENT_ADDRESS_CHANGE:
-		if (bfa_fcs_lport_is_online(rport->port)) {
-			if (bfa_fcs_fabric_is_switched(rport->port->fabric)) {
-				bfa_sm_set_state(rport,
-					bfa_fcs_rport_sm_nsdisc_sending);
-				rport->ns_retries = 0;
-				bfa_fcs_rport_send_nsdisc(rport, NULL);
-			} else {
-				bfa_sm_set_state(rport,
-					bfa_fcs_rport_sm_plogi_sending);
-				rport->plogi_retries = 0;
-				bfa_fcs_rport_send_plogi(rport, NULL);
-			}
-		} else {
+		if (!bfa_fcs_lport_is_online(rport->port)) {
 			rport->pid = 0;
 			bfa_sm_set_state(rport, bfa_fcs_rport_sm_offline);
 			bfa_timer_start(rport->fcs->bfa, &rport->timer,
 					bfa_fcs_rport_timeout, rport,
 					bfa_fcs_rport_del_timeout);
+			break;
+		}
+		if (bfa_fcs_fabric_is_switched(rport->port->fabric)) {
+			bfa_sm_set_state(rport,
+				bfa_fcs_rport_sm_nsdisc_sending);
+			rport->ns_retries = 0;
+			bfa_fcs_rport_send_nsdisc(rport, NULL);
+		} else {
+			bfa_sm_set_state(rport, bfa_fcs_rport_sm_plogi_sending);
+			rport->plogi_retries = 0;
+			bfa_fcs_rport_send_plogi(rport, NULL);
 		}
 		break;
 
@@ -1001,7 +1095,11 @@ bfa_fcs_rport_sm_hcb_logorcv(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_DELETE:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hcb_logosend);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_delete_pending);
+		if (rport->pid && (rport->prlo == BFA_TRUE))
+			bfa_fcs_rport_send_prlo_acc(rport);
+		if (rport->pid && (rport->prlo == BFA_FALSE))
+			bfa_fcs_rport_send_logo_acc(rport);
 		break;
 
 	case RPSM_EVENT_LOGO_IMP:
@@ -1040,7 +1138,14 @@ bfa_fcs_rport_sm_hcb_logosend(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
+		bfa_fcs_rport_send_logo_acc(rport);
 	case RPSM_EVENT_PRLO_RCVD:
+		if (rport->prlo == BFA_TRUE)
+			bfa_fcs_rport_send_prlo_acc(rport);
+
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_delete_pending);
+		break;
+
 	case RPSM_EVENT_ADDRESS_CHANGE:
 		break;
 
@@ -1072,7 +1177,11 @@ bfa_fcs_rport_sm_logo_sending(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_LOGO_RCVD:
+		bfa_fcs_rport_send_logo_acc(rport);
 	case RPSM_EVENT_PRLO_RCVD:
+		if (rport->prlo == BFA_TRUE)
+			bfa_fcs_rport_send_prlo_acc(rport);
+
 		bfa_sm_set_state(rport, bfa_fcs_rport_sm_uninit);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
 		bfa_fcs_rport_free(rport);
@@ -1126,9 +1235,9 @@ bfa_fcs_rport_sm_offline(struct bfa_fcs_rport_s *rport, enum rport_event event)
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_timer_stop(&rport->timer);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	case RPSM_EVENT_PLOGI_SEND:
@@ -1190,9 +1299,9 @@ bfa_fcs_rport_sm_nsdisc_sending(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_fcxp_walloc_cancel(rport->fcs->bfa, &rport->fcxp_wqe);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	default:
@@ -1254,9 +1363,9 @@ bfa_fcs_rport_sm_nsdisc_retry(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_timer_stop(&rport->timer);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	default:
@@ -1296,6 +1405,7 @@ bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 				 bfa_fcs_rport_sm_nsdisc_sending);
 			bfa_fcs_rport_send_nsdisc(rport, NULL);
 		} else {
+			rport->old_pid = rport->pid;
 			rport->pid = 0;
 			bfa_sm_set_state(rport, bfa_fcs_rport_sm_offline);
 			bfa_timer_start(rport->fcs->bfa, &rport->timer,
@@ -1343,9 +1453,9 @@ bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 		break;
 
 	case RPSM_EVENT_PLOGI_COMP:
-		bfa_sm_set_state(rport, bfa_fcs_rport_sm_hal_online);
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_fc4_fcs_online);
 		bfa_fcxp_discard(rport->fcxp);
-		bfa_fcs_rport_hal_online(rport);
+		bfa_fcs_rport_fcs_online_action(rport);
 		break;
 
 	default:
@@ -1353,7 +1463,63 @@ bfa_fcs_rport_sm_nsdisc_sent(struct bfa_fcs_rport_s *rport,
 	}
 }
 
+/*
+ * Rport needs to be deleted
+ * waiting for ITNIM clean up to finish
+ */
+static void
+bfa_fcs_rport_sm_fc4_off_delete(struct bfa_fcs_rport_s *rport,
+				enum rport_event event)
+{
+	bfa_trc(rport->fcs, rport->pwwn);
+	bfa_trc(rport->fcs, rport->pid);
+	bfa_trc(rport->fcs, event);
 
+	switch (event) {
+	case RPSM_EVENT_FC4_OFFLINE:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_delete_pending);
+		bfa_fcs_rport_hal_offline(rport);
+		break;
+
+	case RPSM_EVENT_DELETE:
+	case RPSM_EVENT_PLOGI_RCVD:
+		/* Ignore these events */
+		break;
+
+	default:
+		bfa_sm_fault(rport->fcs, event);
+		break;
+	}
+}
+
+/*
+ * RPort needs to be deleted
+ * waiting for BFA/FW to finish current processing
+ */
+static void
+bfa_fcs_rport_sm_delete_pending(struct bfa_fcs_rport_s *rport,
+				enum rport_event event)
+{
+	bfa_trc(rport->fcs, rport->pwwn);
+	bfa_trc(rport->fcs, rport->pid);
+	bfa_trc(rport->fcs, event);
+
+	switch (event) {
+	case RPSM_EVENT_HCB_OFFLINE:
+		bfa_sm_set_state(rport, bfa_fcs_rport_sm_uninit);
+		bfa_fcs_rport_free(rport);
+		break;
+
+	case RPSM_EVENT_DELETE:
+	case RPSM_EVENT_LOGO_IMP:
+	case RPSM_EVENT_PLOGI_RCVD:
+		/* Ignore these events */
+		break;
+
+	default:
+		bfa_sm_fault(rport->fcs, event);
+	}
+}
 
 /*
  *  fcs_rport_private FCS RPORT provate functions
@@ -1370,10 +1536,11 @@ bfa_fcs_rport_send_plogi(void *rport_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(rport->fcs, rport->pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rport->fcxp_wqe,
-					bfa_fcs_rport_send_plogi, rport);
+				bfa_fcs_rport_send_plogi, rport, BFA_TRUE);
 		return;
 	}
 	rport->fcxp = fcxp;
@@ -1490,10 +1657,11 @@ bfa_fcs_rport_send_plogiacc(void *rport_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 	bfa_trc(rport->fcs, rport->pwwn);
 	bfa_trc(rport->fcs, rport->reply_oxid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rport->fcxp_wqe,
-					bfa_fcs_rport_send_plogiacc, rport);
+				bfa_fcs_rport_send_plogiacc, rport, BFA_FALSE);
 		return;
 	}
 	rport->fcxp = fcxp;
@@ -1522,10 +1690,11 @@ bfa_fcs_rport_send_adisc(void *rport_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(rport->fcs, rport->pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rport->fcxp_wqe,
-					bfa_fcs_rport_send_adisc, rport);
+				bfa_fcs_rport_send_adisc, rport, BFA_TRUE);
 		return;
 	}
 	rport->fcxp = fcxp;
@@ -1585,10 +1754,11 @@ bfa_fcs_rport_send_nsdisc(void *rport_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(rport->fcs, rport->pid);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rport->fcxp_wqe,
-					bfa_fcs_rport_send_nsdisc, rport);
+				bfa_fcs_rport_send_nsdisc, rport, BFA_TRUE);
 		return;
 	}
 	rport->fcxp = fcxp;
@@ -1741,10 +1911,11 @@ bfa_fcs_rport_send_logo(void *rport_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	port = rport->port;
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rport->fcxp_wqe,
-					bfa_fcs_rport_send_logo, rport);
+				bfa_fcs_rport_send_logo, rport, BFA_FALSE);
 		return;
 	}
 	rport->fcxp = fcxp;
@@ -1778,7 +1949,7 @@ bfa_fcs_rport_send_logo_acc(void *rport_cbarg)
 
 	port = rport->port;
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -1849,7 +2020,7 @@ bfa_fcs_rport_process_prli(struct bfa_fcs_rport_s *rport,
 		bfa_fcs_itnim_is_initiator(rport->itnim);
 	}
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -1886,7 +2057,7 @@ bfa_fcs_rport_process_rpsc(struct bfa_fcs_rport_s *rport,
 
 	speeds.port_op_speed = fc_bfa_speed_to_rpsc_operspeed(pport_attr.speed);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -1920,7 +2091,7 @@ bfa_fcs_rport_process_adisc(struct bfa_fcs_rport_s *rport,
 	 */
 	if (bfa_fcs_itnim_get_online_state(rport->itnim) == BFA_STATUS_OK) {
 
-		fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+		fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 		if (!fcxp)
 			return;
 
@@ -1957,6 +2128,15 @@ bfa_fcs_rport_hal_online(struct bfa_fcs_rport_s *rport)
 	bfa_rport_online(rport->bfa_rport, &rport_info);
 }
 
+static void
+bfa_fcs_rport_hal_offline(struct bfa_fcs_rport_s *rport)
+{
+	if (rport->bfa_rport)
+		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_OFFLINE);
+	else
+		bfa_cb_rport_offline(rport);
+}
+
 static struct bfa_fcs_rport_s *
 bfa_fcs_rport_alloc(struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 rpid)
 {
@@ -1967,6 +2147,11 @@ bfa_fcs_rport_alloc(struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 rpid)
 	/*
 	 * allocate rport
 	 */
+	if (fcs->num_rport_logins >= bfa_fcs_rport_max_logins) {
+		bfa_trc(fcs, rpid);
+		return NULL;
+	}
+
 	if (bfa_fcb_rport_alloc(fcs->bfad, &rport, &rport_drv)
 		!= BFA_STATUS_OK) {
 		bfa_trc(fcs, rpid);
@@ -1981,16 +2166,9 @@ bfa_fcs_rport_alloc(struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 rpid)
 	rport->rp_drv = rport_drv;
 	rport->pid = rpid;
 	rport->pwwn = pwwn;
+	rport->old_pid = 0;
 
-	/*
-	 * allocate BFA rport
-	 */
-	rport->bfa_rport = bfa_rport_create(port->fcs->bfa, rport);
-	if (!rport->bfa_rport) {
-		bfa_trc(fcs, rpid);
-		kfree(rport_drv);
-		return NULL;
-	}
+	rport->bfa_rport = NULL;
 
 	/*
 	 * allocate FC-4s
@@ -2001,14 +2179,13 @@ bfa_fcs_rport_alloc(struct bfa_fcs_lport_s *port, wwn_t pwwn, u32 rpid)
 		rport->itnim = bfa_fcs_itnim_create(rport);
 		if (!rport->itnim) {
 			bfa_trc(fcs, rpid);
-			bfa_sm_send_event(rport->bfa_rport,
-						BFA_RPORT_SM_DELETE);
 			kfree(rport_drv);
 			return NULL;
 		}
 	}
 
 	bfa_fcs_lport_add_rport(port, rport);
+	fcs->num_rport_logins++;
 
 	bfa_sm_set_state(rport, bfa_fcs_rport_sm_uninit);
 
@@ -2024,20 +2201,28 @@ static void
 bfa_fcs_rport_free(struct bfa_fcs_rport_s *rport)
 {
 	struct bfa_fcs_lport_s *port = rport->port;
+	struct bfa_fcs_s *fcs = port->fcs;
 
 	/*
 	 * - delete FC-4s
 	 * - delete BFA rport
 	 * - remove from queue of rports
 	 */
+	rport->plogi_pending = BFA_FALSE;
+
 	if (bfa_fcs_lport_is_initiator(port)) {
 		bfa_fcs_itnim_delete(rport->itnim);
 		if (rport->pid != 0 && !BFA_FCS_PID_IS_WKA(rport->pid))
 			bfa_fcs_rpf_rport_offline(rport);
 	}
 
-	bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_DELETE);
+	if (rport->bfa_rport) {
+		bfa_sm_send_event(rport->bfa_rport, BFA_RPORT_SM_DELETE);
+		rport->bfa_rport = NULL;
+	}
+
 	bfa_fcs_lport_del_rport(port, rport);
+	fcs->num_rport_logins--;
 	kfree(rport->rp_drv);
 }
 
@@ -2071,7 +2256,18 @@ bfa_fcs_rport_aen_post(struct bfa_fcs_rport_s *rport,
 }
 
 static void
-bfa_fcs_rport_online_action(struct bfa_fcs_rport_s *rport)
+bfa_fcs_rport_fcs_online_action(struct bfa_fcs_rport_s *rport)
+{
+	if ((!rport->pid) || (!rport->pwwn)) {
+		bfa_trc(rport->fcs, rport->pid);
+		bfa_sm_fault(rport->fcs, rport->pid);
+	}
+
+	bfa_sm_send_event(rport->itnim, BFA_FCS_ITNIM_SM_FCS_ONLINE);
+}
+
+static void
+bfa_fcs_rport_hal_online_action(struct bfa_fcs_rport_s *rport)
 {
 	struct bfa_fcs_lport_s *port = rport->port;
 	struct bfad_s *bfad = (struct bfad_s *)port->fcs->bfad;
@@ -2086,7 +2282,7 @@ bfa_fcs_rport_online_action(struct bfa_fcs_rport_s *rport)
 	}
 
 	if (bfa_fcs_lport_is_initiator(port)) {
-		bfa_fcs_itnim_rport_online(rport->itnim);
+		bfa_fcs_itnim_brp_online(rport->itnim);
 		if (!BFA_FCS_PID_IS_WKA(rport->pid))
 			bfa_fcs_rpf_rport_online(rport);
 	};
@@ -2102,15 +2298,28 @@ bfa_fcs_rport_online_action(struct bfa_fcs_rport_s *rport)
 }
 
 static void
-bfa_fcs_rport_offline_action(struct bfa_fcs_rport_s *rport)
+bfa_fcs_rport_fcs_offline_action(struct bfa_fcs_rport_s *rport)
+{
+	if (!BFA_FCS_PID_IS_WKA(rport->pid))
+		bfa_fcs_rpf_rport_offline(rport);
+
+	bfa_fcs_itnim_rport_offline(rport->itnim);
+}
+
+static void
+bfa_fcs_rport_hal_offline_action(struct bfa_fcs_rport_s *rport)
 {
 	struct bfa_fcs_lport_s *port = rport->port;
 	struct bfad_s *bfad = (struct bfad_s *)port->fcs->bfad;
 	char	lpwwn_buf[BFA_STRING_32];
 	char	rpwwn_buf[BFA_STRING_32];
 
+	if (!rport->bfa_rport) {
+		bfa_fcs_rport_fcs_offline_action(rport);
+		return;
+	}
+
 	rport->stats.offlines++;
-	rport->plogi_pending = BFA_FALSE;
 
 	wwn2str(lpwwn_buf, bfa_fcs_lport_get_pwwn(port));
 	wwn2str(rpwwn_buf, rport->pwwn);
@@ -2340,7 +2549,6 @@ bfa_fcs_rport_scn(struct bfa_fcs_rport_s *rport)
 	bfa_sm_send_event(rport, RPSM_EVENT_SCN);
 }
 
-
 /*
  *	brief
  *	This routine BFA callback for bfa_rport_online() call.
@@ -2508,7 +2716,7 @@ bfa_fcs_rport_send_prlo_acc(struct bfa_fcs_rport_s *rport)
 
 	bfa_trc(rport->fcs, rport->pid);
 
-	fcxp = bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(port->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 	len = fc_prlo_acc_build(&fchs, bfa_fcxp_get_reqbuf(fcxp),
@@ -2534,7 +2742,7 @@ bfa_fcs_rport_send_ls_rjt(struct bfa_fcs_rport_s *rport, struct fchs_s *rx_fchs,
 
 	bfa_trc(rport->fcs, rx_fchs->s_id);
 
-	fcxp = bfa_fcs_fcxp_alloc(rport->fcs);
+	fcxp = bfa_fcs_fcxp_alloc(rport->fcs, BFA_FALSE);
 	if (!fcxp)
 		return;
 
@@ -2582,6 +2790,17 @@ bfa_fcs_rport_prlo(struct bfa_fcs_rport_s *rport, __be16 ox_id)
 	bfa_sm_send_event(rport, RPSM_EVENT_PRLO_RCVD);
 }
 
+/*
+ * Called by BFAD to set the max limit on number of bfa_fcs_rport allocation
+ * which limits number of concurrent logins to remote ports
+ */
+void
+bfa_fcs_rport_set_max_logins(u32 max_logins)
+{
+	if (max_logins > 0)
+		bfa_fcs_rport_max_logins = max_logins;
+}
+
 void
 bfa_fcs_rport_get_attr(struct bfa_fcs_rport_s *rport,
 		struct bfa_rport_attr_s *rport_attr)
@@ -2605,9 +2824,11 @@ bfa_fcs_rport_get_attr(struct bfa_fcs_rport_s *rport,
 	rport_attr->curr_speed  = rport->rpf.rpsc_speed;
 	rport_attr->assigned_speed  = rport->rpf.assigned_speed;
 
-	qos_attr.qos_priority = rport->bfa_rport->qos_attr.qos_priority;
-	qos_attr.qos_flow_id =
-		cpu_to_be32(rport->bfa_rport->qos_attr.qos_flow_id);
+	if (rport->bfa_rport) {
+		qos_attr.qos_priority = rport->bfa_rport->qos_attr.qos_priority;
+		qos_attr.qos_flow_id =
+			cpu_to_be32(rport->bfa_rport->qos_attr.qos_flow_id);
+	}
 	rport_attr->qos_attr = qos_attr;
 
 	rport_attr->trl_enforced = BFA_FALSE;
@@ -2940,10 +3161,11 @@ bfa_fcs_rpf_send_rpsc2(void *rpf_cbarg, struct bfa_fcxp_s *fcxp_alloced)
 
 	bfa_trc(rport->fcs, rport->pwwn);
 
-	fcxp = fcxp_alloced ? fcxp_alloced : bfa_fcs_fcxp_alloc(port->fcs);
+	fcxp = fcxp_alloced ? fcxp_alloced :
+	       bfa_fcs_fcxp_alloc(port->fcs, BFA_TRUE);
 	if (!fcxp) {
 		bfa_fcs_fcxp_alloc_wait(port->fcs->bfa, &rpf->fcxp_wqe,
-					bfa_fcs_rpf_send_rpsc2, rpf);
+				bfa_fcs_rpf_send_rpsc2, rpf, BFA_TRUE);
 		return;
 	}
 	rpf->fcxp = fcxp;
diff --git a/drivers/scsi/bfa/bfa_ioc.c b/drivers/scsi/bfa/bfa_ioc.c
index 21ad2902e5c..75ca8752b9f 100644
--- a/drivers/scsi/bfa/bfa_ioc.c
+++ b/drivers/scsi/bfa/bfa_ioc.c
@@ -92,7 +92,6 @@ static void bfa_ioc_event_notify(struct bfa_ioc_s *ioc ,
 				enum bfa_ioc_event_e event);
 static void bfa_ioc_disable_comp(struct bfa_ioc_s *ioc);
 static void bfa_ioc_lpu_stop(struct bfa_ioc_s *ioc);
-static void bfa_ioc_debug_save_ftrc(struct bfa_ioc_s *ioc);
 static void bfa_ioc_fail_notify(struct bfa_ioc_s *ioc);
 static void bfa_ioc_pf_fwmismatch(struct bfa_ioc_s *ioc);
 
@@ -599,8 +598,9 @@ bfa_ioc_sm_fail(struct bfa_ioc_s *ioc, enum ioc_event event)
 		break;
 
 	case IOC_E_HWERROR:
+	case IOC_E_HWFAILED:
 		/*
-		 * HB failure notification, ignore.
+		 * HB failure / HW error notification, ignore.
 		 */
 		break;
 	default:
@@ -632,6 +632,10 @@ bfa_ioc_sm_hwfail(struct bfa_ioc_s *ioc, enum ioc_event event)
 		bfa_fsm_set_state(ioc, bfa_ioc_sm_uninit);
 		break;
 
+	case IOC_E_HWERROR:
+		/* Ignore - already in hwfail state */
+		break;
+
 	default:
 		bfa_sm_fault(ioc, event);
 	}
@@ -1455,7 +1459,7 @@ bfa_ioc_fwver_cmp(struct bfa_ioc_s *ioc, struct bfi_ioc_image_hdr_s *fwhdr)
 		bfa_cb_image_get_chunk(bfa_ioc_asic_gen(ioc), 0);
 
 	for (i = 0; i < BFI_IOC_MD5SUM_SZ; i++) {
-		if (fwhdr->md5sum[i] != drv_fwhdr->md5sum[i]) {
+		if (fwhdr->md5sum[i] != cpu_to_le32(drv_fwhdr->md5sum[i])) {
 			bfa_trc(ioc, i);
 			bfa_trc(ioc, fwhdr->md5sum[i]);
 			bfa_trc(ioc, drv_fwhdr->md5sum[i]);
@@ -1480,7 +1484,7 @@ bfa_ioc_fwver_valid(struct bfa_ioc_s *ioc, u32 boot_env)
 	drv_fwhdr = (struct bfi_ioc_image_hdr_s *)
 		bfa_cb_image_get_chunk(bfa_ioc_asic_gen(ioc), 0);
 
-	if (fwhdr.signature != drv_fwhdr->signature) {
+	if (fwhdr.signature != cpu_to_le32(drv_fwhdr->signature)) {
 		bfa_trc(ioc, fwhdr.signature);
 		bfa_trc(ioc, drv_fwhdr->signature);
 		return BFA_FALSE;
@@ -1704,7 +1708,7 @@ bfa_ioc_download_fw(struct bfa_ioc_s *ioc, u32 boot_type,
 		 * write smem
 		 */
 		bfa_mem_write(ioc->ioc_regs.smem_page_start, loff,
-			      fwimg[BFA_IOC_FLASH_OFFSET_IN_CHUNK(i)]);
+			cpu_to_le32(fwimg[BFA_IOC_FLASH_OFFSET_IN_CHUNK(i)]));
 
 		loff += sizeof(u32);
 
@@ -2260,6 +2264,12 @@ bfa_ioc_disable(struct bfa_ioc_s *ioc)
 	bfa_fsm_send_event(ioc, IOC_E_DISABLE);
 }
 
+void
+bfa_ioc_suspend(struct bfa_ioc_s *ioc)
+{
+	ioc->dbg_fwsave_once = BFA_TRUE;
+	bfa_fsm_send_event(ioc, IOC_E_HWERROR);
+}
 
 /*
  * Initialize memory for saving firmware trace. Driver must initialize
@@ -2269,7 +2279,7 @@ void
 bfa_ioc_debug_memclaim(struct bfa_ioc_s *ioc, void *dbg_fwsave)
 {
 	ioc->dbg_fwsave	    = dbg_fwsave;
-	ioc->dbg_fwsave_len = (ioc->iocpf.auto_recover) ? BFA_DBG_FWTRC_LEN : 0;
+	ioc->dbg_fwsave_len = BFA_DBG_FWTRC_LEN;
 }
 
 /*
@@ -2856,7 +2866,7 @@ bfa_ioc_fw_stats_clear(struct bfa_ioc_s *ioc)
 /*
  * Save firmware trace if configured.
  */
-static void
+void
 bfa_ioc_debug_save_ftrc(struct bfa_ioc_s *ioc)
 {
 	int		tlen;
diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h
index 7b916e04ca5..b2856f96567 100644
--- a/drivers/scsi/bfa/bfa_ioc.h
+++ b/drivers/scsi/bfa/bfa_ioc.h
@@ -820,6 +820,7 @@ void bfa_ioc_attach(struct bfa_ioc_s *ioc, void *bfa,
 		struct bfa_ioc_cbfn_s *cbfn, struct bfa_timer_mod_s *timer_mod);
 void bfa_ioc_auto_recover(bfa_boolean_t auto_recover);
 void bfa_ioc_detach(struct bfa_ioc_s *ioc);
+void bfa_ioc_suspend(struct bfa_ioc_s *ioc);
 void bfa_ioc_pci_init(struct bfa_ioc_s *ioc, struct bfa_pcidev_s *pcidev,
 		enum bfi_pcifn_class clscode);
 void bfa_ioc_mem_claim(struct bfa_ioc_s *ioc,  u8 *dm_kva, u64 dm_pa);
@@ -866,6 +867,7 @@ bfa_boolean_t bfa_ioc_fwver_cmp(struct bfa_ioc_s *ioc,
 void bfa_ioc_aen_post(struct bfa_ioc_s *ioc, enum bfa_ioc_aen_event event);
 bfa_status_t bfa_ioc_fw_stats_get(struct bfa_ioc_s *ioc, void *stats);
 bfa_status_t bfa_ioc_fw_stats_clear(struct bfa_ioc_s *ioc);
+void bfa_ioc_debug_save_ftrc(struct bfa_ioc_s *ioc);
 
 /*
  * asic block configuration related APIs
diff --git a/drivers/scsi/bfa/bfa_modules.h b/drivers/scsi/bfa/bfa_modules.h
index 2d36e482383..189fff71e3c 100644
--- a/drivers/scsi/bfa/bfa_modules.h
+++ b/drivers/scsi/bfa/bfa_modules.h
@@ -121,6 +121,7 @@ struct bfa_s {
 	bfa_boolean_t		fcs;		/*  FCS is attached to BFA */
 	struct bfa_msix_s	msix;
 	int			bfa_aen_seq;
+	bfa_boolean_t		intr_enabled;	/*  Status of interrupts */
 };
 
 extern bfa_boolean_t bfa_auto_recover;
diff --git a/drivers/scsi/bfa/bfa_svc.c b/drivers/scsi/bfa/bfa_svc.c
index 2e856e6710f..b2538d60db3 100644
--- a/drivers/scsi/bfa/bfa_svc.c
+++ b/drivers/scsi/bfa/bfa_svc.c
@@ -440,9 +440,11 @@ claim_fcxps_mem(struct bfa_fcxp_mod_s *mod)
 	fcxp = (struct bfa_fcxp_s *) bfa_mem_kva_curp(mod);
 	memset(fcxp, 0, sizeof(struct bfa_fcxp_s) * mod->num_fcxps);
 
-	INIT_LIST_HEAD(&mod->fcxp_free_q);
+	INIT_LIST_HEAD(&mod->fcxp_req_free_q);
+	INIT_LIST_HEAD(&mod->fcxp_rsp_free_q);
 	INIT_LIST_HEAD(&mod->fcxp_active_q);
-	INIT_LIST_HEAD(&mod->fcxp_unused_q);
+	INIT_LIST_HEAD(&mod->fcxp_req_unused_q);
+	INIT_LIST_HEAD(&mod->fcxp_rsp_unused_q);
 
 	mod->fcxp_list = fcxp;
 
@@ -450,7 +452,14 @@ claim_fcxps_mem(struct bfa_fcxp_mod_s *mod)
 		fcxp->fcxp_mod = mod;
 		fcxp->fcxp_tag = i;
 
-		list_add_tail(&fcxp->qe, &mod->fcxp_free_q);
+		if (i < (mod->num_fcxps / 2)) {
+			list_add_tail(&fcxp->qe, &mod->fcxp_req_free_q);
+			fcxp->req_rsp = BFA_TRUE;
+		} else {
+			list_add_tail(&fcxp->qe, &mod->fcxp_rsp_free_q);
+			fcxp->req_rsp = BFA_FALSE;
+		}
+
 		bfa_reqq_winit(&fcxp->reqq_wqe, bfa_fcxp_qresume, fcxp);
 		fcxp->reqq_waiting = BFA_FALSE;
 
@@ -514,7 +523,8 @@ bfa_fcxp_attach(struct bfa_s *bfa, void *bfad, struct bfa_iocfc_cfg_s *cfg,
 	if (!cfg->drvcfg.min_cfg)
 		mod->rsp_pld_sz = BFA_FCXP_MAX_LBUF_SZ;
 
-	INIT_LIST_HEAD(&mod->wait_q);
+	INIT_LIST_HEAD(&mod->req_wait_q);
+	INIT_LIST_HEAD(&mod->rsp_wait_q);
 
 	claim_fcxps_mem(mod);
 }
@@ -542,7 +552,8 @@ bfa_fcxp_iocdisable(struct bfa_s *bfa)
 	struct list_head	      *qe, *qen;
 
 	/* Enqueue unused fcxp resources to free_q */
-	list_splice_tail_init(&mod->fcxp_unused_q, &mod->fcxp_free_q);
+	list_splice_tail_init(&mod->fcxp_req_unused_q, &mod->fcxp_req_free_q);
+	list_splice_tail_init(&mod->fcxp_rsp_unused_q, &mod->fcxp_rsp_free_q);
 
 	list_for_each_safe(qe, qen, &mod->fcxp_active_q) {
 		fcxp = (struct bfa_fcxp_s *) qe;
@@ -559,11 +570,14 @@ bfa_fcxp_iocdisable(struct bfa_s *bfa)
 }
 
 static struct bfa_fcxp_s *
-bfa_fcxp_get(struct bfa_fcxp_mod_s *fm)
+bfa_fcxp_get(struct bfa_fcxp_mod_s *fm, bfa_boolean_t req)
 {
 	struct bfa_fcxp_s *fcxp;
 
-	bfa_q_deq(&fm->fcxp_free_q, &fcxp);
+	if (req)
+		bfa_q_deq(&fm->fcxp_req_free_q, &fcxp);
+	else
+		bfa_q_deq(&fm->fcxp_rsp_free_q, &fcxp);
 
 	if (fcxp)
 		list_add_tail(&fcxp->qe, &fm->fcxp_active_q);
@@ -642,7 +656,11 @@ bfa_fcxp_put(struct bfa_fcxp_s *fcxp)
 	struct bfa_fcxp_mod_s *mod = fcxp->fcxp_mod;
 	struct bfa_fcxp_wqe_s *wqe;
 
-	bfa_q_deq(&mod->wait_q, &wqe);
+	if (fcxp->req_rsp)
+		bfa_q_deq(&mod->req_wait_q, &wqe);
+	else
+		bfa_q_deq(&mod->rsp_wait_q, &wqe);
+
 	if (wqe) {
 		bfa_trc(mod->bfa, fcxp->fcxp_tag);
 
@@ -657,7 +675,11 @@ bfa_fcxp_put(struct bfa_fcxp_s *fcxp)
 
 	WARN_ON(!bfa_q_is_on_q(&mod->fcxp_active_q, fcxp));
 	list_del(&fcxp->qe);
-	list_add_tail(&fcxp->qe, &mod->fcxp_free_q);
+
+	if (fcxp->req_rsp)
+		list_add_tail(&fcxp->qe, &mod->fcxp_req_free_q);
+	else
+		list_add_tail(&fcxp->qe, &mod->fcxp_rsp_free_q);
 }
 
 static void
@@ -900,21 +922,23 @@ bfa_fcxp_queue(struct bfa_fcxp_s *fcxp, struct bfi_fcxp_send_req_s *send_req)
  *				Address (given the sge index).
  * @param[in]	get_rsp_sglen	function ptr to be called to get a response SG
  *				len (given the sge index).
+ * @param[in]	req		Allocated FCXP is used to send req or rsp?
+ *				request - BFA_TRUE, response - BFA_FALSE
  *
  * @return FCXP instance. NULL on failure.
  */
 struct bfa_fcxp_s *
-bfa_fcxp_alloc(void *caller, struct bfa_s *bfa, int nreq_sgles,
-	       int nrsp_sgles, bfa_fcxp_get_sgaddr_t req_sga_cbfn,
-	       bfa_fcxp_get_sglen_t req_sglen_cbfn,
-	       bfa_fcxp_get_sgaddr_t rsp_sga_cbfn,
-	       bfa_fcxp_get_sglen_t rsp_sglen_cbfn)
+bfa_fcxp_req_rsp_alloc(void *caller, struct bfa_s *bfa, int nreq_sgles,
+		int nrsp_sgles, bfa_fcxp_get_sgaddr_t req_sga_cbfn,
+		bfa_fcxp_get_sglen_t req_sglen_cbfn,
+		bfa_fcxp_get_sgaddr_t rsp_sga_cbfn,
+		bfa_fcxp_get_sglen_t rsp_sglen_cbfn, bfa_boolean_t req)
 {
 	struct bfa_fcxp_s *fcxp = NULL;
 
 	WARN_ON(bfa == NULL);
 
-	fcxp = bfa_fcxp_get(BFA_FCXP_MOD(bfa));
+	fcxp = bfa_fcxp_get(BFA_FCXP_MOD(bfa), req);
 	if (fcxp == NULL)
 		return NULL;
 
@@ -1071,17 +1095,20 @@ bfa_fcxp_abort(struct bfa_fcxp_s *fcxp)
 }
 
 void
-bfa_fcxp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe,
+bfa_fcxp_req_rsp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe,
 	       bfa_fcxp_alloc_cbfn_t alloc_cbfn, void *alloc_cbarg,
 	       void *caller, int nreq_sgles,
 	       int nrsp_sgles, bfa_fcxp_get_sgaddr_t req_sga_cbfn,
 	       bfa_fcxp_get_sglen_t req_sglen_cbfn,
 	       bfa_fcxp_get_sgaddr_t rsp_sga_cbfn,
-	       bfa_fcxp_get_sglen_t rsp_sglen_cbfn)
+	       bfa_fcxp_get_sglen_t rsp_sglen_cbfn, bfa_boolean_t req)
 {
 	struct bfa_fcxp_mod_s *mod = BFA_FCXP_MOD(bfa);
 
-	WARN_ON(!list_empty(&mod->fcxp_free_q));
+	if (req)
+		WARN_ON(!list_empty(&mod->fcxp_req_free_q));
+	else
+		WARN_ON(!list_empty(&mod->fcxp_rsp_free_q));
 
 	wqe->alloc_cbfn = alloc_cbfn;
 	wqe->alloc_cbarg = alloc_cbarg;
@@ -1094,7 +1121,10 @@ bfa_fcxp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe,
 	wqe->rsp_sga_cbfn = rsp_sga_cbfn;
 	wqe->rsp_sglen_cbfn = rsp_sglen_cbfn;
 
-	list_add_tail(&wqe->qe, &mod->wait_q);
+	if (req)
+		list_add_tail(&wqe->qe, &mod->req_wait_q);
+	else
+		list_add_tail(&wqe->qe, &mod->rsp_wait_q);
 }
 
 void
@@ -1102,7 +1132,8 @@ bfa_fcxp_walloc_cancel(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe)
 {
 	struct bfa_fcxp_mod_s *mod = BFA_FCXP_MOD(bfa);
 
-	WARN_ON(!bfa_q_is_on_q(&mod->wait_q, wqe));
+	WARN_ON(!bfa_q_is_on_q(&mod->req_wait_q, wqe) ||
+		!bfa_q_is_on_q(&mod->rsp_wait_q, wqe));
 	list_del(&wqe->qe);
 }
 
@@ -1153,8 +1184,13 @@ bfa_fcxp_res_recfg(struct bfa_s *bfa, u16 num_fcxp_fw)
 	int	i;
 
 	for (i = 0; i < (mod->num_fcxps - num_fcxp_fw); i++) {
-		bfa_q_deq_tail(&mod->fcxp_free_q, &qe);
-		list_add_tail(qe, &mod->fcxp_unused_q);
+		if (i < ((mod->num_fcxps - num_fcxp_fw) / 2)) {
+			bfa_q_deq_tail(&mod->fcxp_req_free_q, &qe);
+			list_add_tail(qe, &mod->fcxp_req_unused_q);
+		} else {
+			bfa_q_deq_tail(&mod->fcxp_rsp_free_q, &qe);
+			list_add_tail(qe, &mod->fcxp_rsp_unused_q);
+		}
 	}
 }
 
@@ -1404,11 +1440,11 @@ bfa_lps_sm_logout(struct bfa_lps_s *lps, enum bfa_lps_event event)
 
 	switch (event) {
 	case BFA_LPS_SM_FWRSP:
+	case BFA_LPS_SM_OFFLINE:
 		bfa_sm_set_state(lps, bfa_lps_sm_init);
 		bfa_lps_logout_comp(lps);
 		break;
 
-	case BFA_LPS_SM_OFFLINE:
 	case BFA_LPS_SM_DELETE:
 		bfa_sm_set_state(lps, bfa_lps_sm_init);
 		break;
@@ -1786,6 +1822,8 @@ bfa_lps_logout_comp_cb(void *arg, bfa_boolean_t complete)
 
 	if (lps->fdisc)
 		bfa_cb_lps_fdisclogo_comp(lps->bfa->bfad, lps->uarg);
+	else
+		bfa_cb_lps_flogo_comp(lps->bfa->bfad, lps->uarg);
 }
 
 /*
@@ -4237,6 +4275,10 @@ bfa_rport_sm_offline(struct bfa_rport_s *rp, enum bfa_rport_event event)
 		bfa_sm_set_state(rp, bfa_rport_sm_iocdisable);
 		break;
 
+	case BFA_RPORT_SM_OFFLINE:
+		bfa_rport_offline_cb(rp);
+		break;
+
 	default:
 		bfa_stats(rp, sm_off_unexp);
 		bfa_sm_fault(rp->bfa, event);
@@ -4353,6 +4395,7 @@ bfa_rport_sm_offline_pending(struct bfa_rport_s *rp,
 	case BFA_RPORT_SM_HWFAIL:
 		bfa_stats(rp, sm_offp_hwf);
 		bfa_sm_set_state(rp, bfa_rport_sm_iocdisable);
+		bfa_rport_offline_cb(rp);
 		break;
 
 	default:
@@ -4731,8 +4774,10 @@ bfa_rport_speed(struct bfa_rport_s *rport, enum bfa_port_speed speed)
 	WARN_ON(speed == 0);
 	WARN_ON(speed == BFA_PORT_SPEED_AUTO);
 
-	rport->rport_info.speed = speed;
-	bfa_sm_send_event(rport, BFA_RPORT_SM_SET_SPEED);
+	if (rport) {
+		rport->rport_info.speed = speed;
+		bfa_sm_send_event(rport, BFA_RPORT_SM_SET_SPEED);
+	}
 }
 
 /* Set Rport LUN Mask */
diff --git a/drivers/scsi/bfa/bfa_svc.h b/drivers/scsi/bfa/bfa_svc.h
index f3006756463..1abcf7c5166 100644
--- a/drivers/scsi/bfa/bfa_svc.h
+++ b/drivers/scsi/bfa/bfa_svc.h
@@ -97,10 +97,13 @@ struct bfa_fcxp_mod_s {
 	struct bfa_s      *bfa;		/* backpointer to BFA */
 	struct bfa_fcxp_s *fcxp_list;	/* array of FCXPs */
 	u16	num_fcxps;	/* max num FCXP requests */
-	struct list_head  fcxp_free_q;	/* free FCXPs */
-	struct list_head  fcxp_active_q;	/* active FCXPs */
-	struct list_head  wait_q;		/* wait queue for free fcxp */
-	struct list_head fcxp_unused_q; /* unused fcxps */
+	struct list_head fcxp_req_free_q; /* free FCXPs used for sending req */
+	struct list_head fcxp_rsp_free_q; /* free FCXPs used for sending req */
+	struct list_head fcxp_active_q;	/* active FCXPs */
+	struct list_head req_wait_q;	/* wait queue for free req_fcxp */
+	struct list_head rsp_wait_q;	/* wait queue for free rsp_fcxp */
+	struct list_head fcxp_req_unused_q;	/* unused req_fcxps */
+	struct list_head fcxp_rsp_unused_q;	/* unused rsp_fcxps */
 	u32	req_pld_sz;
 	u32	rsp_pld_sz;
 	struct bfa_mem_dma_s dma_seg[BFA_FCXP_DMA_SEGS];
@@ -197,6 +200,7 @@ struct bfa_fcxp_s {
 	struct bfa_cb_qe_s    hcb_qe;	/*  comp: callback qelem */
 	struct bfa_reqq_wait_s	reqq_wqe;
 	bfa_boolean_t	reqq_waiting;
+	bfa_boolean_t	req_rsp;	/* Used to track req/rsp fcxp */
 };
 
 struct bfa_fcxp_wqe_s {
@@ -586,20 +590,22 @@ void	bfa_rport_unset_lunmask(struct bfa_s *bfa, struct bfa_rport_s *rp);
 /*
  * bfa fcxp API functions
  */
-struct bfa_fcxp_s *bfa_fcxp_alloc(void *bfad_fcxp, struct bfa_s *bfa,
+struct bfa_fcxp_s *bfa_fcxp_req_rsp_alloc(void *bfad_fcxp, struct bfa_s *bfa,
 				  int nreq_sgles, int nrsp_sgles,
 				  bfa_fcxp_get_sgaddr_t get_req_sga,
 				  bfa_fcxp_get_sglen_t get_req_sglen,
 				  bfa_fcxp_get_sgaddr_t get_rsp_sga,
-				  bfa_fcxp_get_sglen_t get_rsp_sglen);
-void bfa_fcxp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe,
+				  bfa_fcxp_get_sglen_t get_rsp_sglen,
+				  bfa_boolean_t req);
+void bfa_fcxp_req_rsp_alloc_wait(struct bfa_s *bfa, struct bfa_fcxp_wqe_s *wqe,
 				bfa_fcxp_alloc_cbfn_t alloc_cbfn,
 				void *cbarg, void *bfad_fcxp,
 				int nreq_sgles, int nrsp_sgles,
 				bfa_fcxp_get_sgaddr_t get_req_sga,
 				bfa_fcxp_get_sglen_t get_req_sglen,
 				bfa_fcxp_get_sgaddr_t get_rsp_sga,
-				bfa_fcxp_get_sglen_t get_rsp_sglen);
+				bfa_fcxp_get_sglen_t get_rsp_sglen,
+				bfa_boolean_t req);
 void bfa_fcxp_walloc_cancel(struct bfa_s *bfa,
 			    struct bfa_fcxp_wqe_s *wqe);
 void bfa_fcxp_discard(struct bfa_fcxp_s *fcxp);
@@ -658,6 +664,7 @@ u8 bfa_lps_get_fwtag(struct bfa_s *bfa, u8 lp_tag);
 u32 bfa_lps_get_base_pid(struct bfa_s *bfa);
 u8 bfa_lps_get_tag_from_pid(struct bfa_s *bfa, u32 pid);
 void bfa_cb_lps_flogi_comp(void *bfad, void *uarg, bfa_status_t status);
+void bfa_cb_lps_flogo_comp(void *bfad, void *uarg);
 void bfa_cb_lps_fdisc_comp(void *bfad, void *uarg, bfa_status_t status);
 void bfa_cb_lps_fdisclogo_comp(void *bfad, void *uarg);
 void bfa_cb_lps_cvl_event(void *bfad, void *uarg);
diff --git a/drivers/scsi/bfa/bfad.c b/drivers/scsi/bfa/bfad.c
index 2c8f0c71307..c37494916a1 100644
--- a/drivers/scsi/bfa/bfad.c
+++ b/drivers/scsi/bfa/bfad.c
@@ -57,6 +57,7 @@ int		pcie_max_read_reqsz;
 int		bfa_debugfs_enable = 1;
 int		msix_disable_cb = 0, msix_disable_ct = 0;
 int		max_xfer_size = BFAD_MAX_SECTORS >> 1;
+int		max_rport_logins = BFA_FCS_MAX_RPORT_LOGINS;
 
 /* Firmware releated */
 u32	bfi_image_cb_size, bfi_image_ct_size, bfi_image_ct2_size;
@@ -148,6 +149,8 @@ MODULE_PARM_DESC(bfa_debugfs_enable, "Enables debugfs feature, default=1,"
 module_param(max_xfer_size, int, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(max_xfer_size, "default=32MB,"
 		" Range[64k|128k|256k|512k|1024k|2048k]");
+module_param(max_rport_logins, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(max_rport_logins, "Max number of logins to initiator and target rports on a port (physical/logical), default=1024");
 
 static void
 bfad_sm_uninit(struct bfad_s *bfad, enum bfad_sm_event event);
@@ -736,6 +739,9 @@ bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad)
 		}
 	}
 
+	/* Enable PCIE Advanced Error Recovery (AER) if kernel supports */
+	pci_enable_pcie_error_reporting(pdev);
+
 	bfad->pci_bar0_kva = pci_iomap(pdev, 0, pci_resource_len(pdev, 0));
 	bfad->pci_bar2_kva = pci_iomap(pdev, 2, pci_resource_len(pdev, 2));
 
@@ -806,6 +812,8 @@ bfad_pci_init(struct pci_dev *pdev, struct bfad_s *bfad)
 		}
 	}
 
+	pci_save_state(pdev);
+
 	return 0;
 
 out_release_region:
@@ -822,6 +830,8 @@ bfad_pci_uninit(struct pci_dev *pdev, struct bfad_s *bfad)
 	pci_iounmap(pdev, bfad->pci_bar0_kva);
 	pci_iounmap(pdev, bfad->pci_bar2_kva);
 	pci_release_regions(pdev);
+	/* Disable PCIE Advanced Error Recovery (AER) */
+	pci_disable_pcie_error_reporting(pdev);
 	pci_disable_device(pdev);
 	pci_set_drvdata(pdev, NULL);
 }
@@ -1258,6 +1268,16 @@ bfad_setup_intr(struct bfad_s *bfad)
 
 		error = pci_enable_msix(bfad->pcidev, msix_entries, bfad->nvec);
 		if (error) {
+			/* In CT1 & CT2, try to allocate just one vector */
+			if (bfa_asic_id_ctc(pdev->device)) {
+				printk(KERN_WARNING "bfa %s: trying one msix "
+				       "vector failed to allocate %d[%d]\n",
+				       bfad->pci_name, bfad->nvec, error);
+				bfad->nvec = 1;
+				error = pci_enable_msix(bfad->pcidev,
+						msix_entries, bfad->nvec);
+			}
+
 			/*
 			 * Only error number of vector is available.
 			 * We don't have a mechanism to map multiple
@@ -1267,12 +1287,13 @@ bfad_setup_intr(struct bfad_s *bfad)
 			 *  vectors. Linux doesn't duplicate vectors
 			 * in the MSIX table for this case.
 			 */
-
-			printk(KERN_WARNING "bfad%d: "
-				"pci_enable_msix failed (%d),"
-				" use line based.\n", bfad->inst_no, error);
-
-			goto line_based;
+			if (error) {
+				printk(KERN_WARNING "bfad%d: "
+				       "pci_enable_msix failed (%d), "
+				       "use line based.\n",
+					bfad->inst_no, error);
+				goto line_based;
+			}
 		}
 
 		/* Disable INTX in MSI-X mode */
@@ -1470,6 +1491,197 @@ bfad_pci_remove(struct pci_dev *pdev)
 	kfree(bfad);
 }
 
+/*
+ * PCI Error Recovery entry, error detected.
+ */
+static pci_ers_result_t
+bfad_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
+{
+	struct bfad_s *bfad = pci_get_drvdata(pdev);
+	unsigned long	flags;
+	pci_ers_result_t ret = PCI_ERS_RESULT_NONE;
+
+	dev_printk(KERN_ERR, &pdev->dev,
+		   "error detected state: %d - flags: 0x%x\n",
+		   state, bfad->bfad_flags);
+
+	switch (state) {
+	case pci_channel_io_normal: /* non-fatal error */
+		spin_lock_irqsave(&bfad->bfad_lock, flags);
+		bfad->bfad_flags &= ~BFAD_EEH_BUSY;
+		/* Suspend/fail all bfa operations */
+		bfa_ioc_suspend(&bfad->bfa.ioc);
+		spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+		del_timer_sync(&bfad->hal_tmo);
+		ret = PCI_ERS_RESULT_CAN_RECOVER;
+		break;
+	case pci_channel_io_frozen: /* fatal error */
+		init_completion(&bfad->comp);
+		spin_lock_irqsave(&bfad->bfad_lock, flags);
+		bfad->bfad_flags |= BFAD_EEH_BUSY;
+		/* Suspend/fail all bfa operations */
+		bfa_ioc_suspend(&bfad->bfa.ioc);
+		bfa_fcs_stop(&bfad->bfa_fcs);
+		spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+		wait_for_completion(&bfad->comp);
+
+		bfad_remove_intr(bfad);
+		del_timer_sync(&bfad->hal_tmo);
+		pci_disable_device(pdev);
+		ret = PCI_ERS_RESULT_NEED_RESET;
+		break;
+	case pci_channel_io_perm_failure: /* PCI Card is DEAD */
+		spin_lock_irqsave(&bfad->bfad_lock, flags);
+		bfad->bfad_flags |= BFAD_EEH_BUSY |
+				    BFAD_EEH_PCI_CHANNEL_IO_PERM_FAILURE;
+		spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+
+		/* If the error_detected handler is called with the reason
+		 * pci_channel_io_perm_failure - it will subsequently call
+		 * pci_remove() entry point to remove the pci device from the
+		 * system - So defer the cleanup to pci_remove(); cleaning up
+		 * here causes inconsistent state during pci_remove().
+		 */
+		ret = PCI_ERS_RESULT_DISCONNECT;
+		break;
+	default:
+		WARN_ON(1);
+	}
+
+	return ret;
+}
+
+int
+restart_bfa(struct bfad_s *bfad)
+{
+	unsigned long flags;
+	struct pci_dev *pdev = bfad->pcidev;
+
+	bfa_attach(&bfad->bfa, bfad, &bfad->ioc_cfg,
+		   &bfad->meminfo, &bfad->hal_pcidev);
+
+	/* Enable Interrupt and wait bfa_init completion */
+	if (bfad_setup_intr(bfad)) {
+		dev_printk(KERN_WARNING, &pdev->dev,
+			   "%s: bfad_setup_intr failed\n", bfad->pci_name);
+		bfa_sm_send_event(bfad, BFAD_E_INTR_INIT_FAILED);
+		return -1;
+	}
+
+	init_completion(&bfad->comp);
+	spin_lock_irqsave(&bfad->bfad_lock, flags);
+	bfa_iocfc_init(&bfad->bfa);
+	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+
+	/* Set up interrupt handler for each vectors */
+	if ((bfad->bfad_flags & BFAD_MSIX_ON) &&
+	    bfad_install_msix_handler(bfad))
+		dev_printk(KERN_WARNING, &pdev->dev,
+			   "%s: install_msix failed.\n", bfad->pci_name);
+
+	bfad_init_timer(bfad);
+	wait_for_completion(&bfad->comp);
+	bfad_drv_start(bfad);
+
+	return 0;
+}
+
+/*
+ * PCI Error Recovery entry, re-initialize the chip.
+ */
+static pci_ers_result_t
+bfad_pci_slot_reset(struct pci_dev *pdev)
+{
+	struct bfad_s *bfad = pci_get_drvdata(pdev);
+	u8 byte;
+
+	dev_printk(KERN_ERR, &pdev->dev,
+		   "bfad_pci_slot_reset flags: 0x%x\n", bfad->bfad_flags);
+
+	if (pci_enable_device(pdev)) {
+		dev_printk(KERN_ERR, &pdev->dev, "Cannot re-enable "
+			   "PCI device after reset.\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	pci_restore_state(pdev);
+
+	/*
+	 * Read some byte (e.g. DMA max. payload size which can't
+	 * be 0xff any time) to make sure - we did not hit another PCI error
+	 * in the middle of recovery. If we did, then declare permanent failure.
+	 */
+	pci_read_config_byte(pdev, 0x68, &byte);
+	if (byte == 0xff) {
+		dev_printk(KERN_ERR, &pdev->dev,
+			   "slot_reset failed ... got another PCI error !\n");
+		goto out_disable_device;
+	}
+
+	pci_save_state(pdev);
+	pci_set_master(pdev);
+
+	if (pci_set_dma_mask(bfad->pcidev, DMA_BIT_MASK(64)) != 0)
+		if (pci_set_dma_mask(bfad->pcidev, DMA_BIT_MASK(32)) != 0)
+			goto out_disable_device;
+
+	pci_cleanup_aer_uncorrect_error_status(pdev);
+
+	if (restart_bfa(bfad) == -1)
+		goto out_disable_device;
+
+	pci_enable_pcie_error_reporting(pdev);
+	dev_printk(KERN_WARNING, &pdev->dev,
+		   "slot_reset completed  flags: 0x%x!\n", bfad->bfad_flags);
+
+	return PCI_ERS_RESULT_RECOVERED;
+
+out_disable_device:
+	pci_disable_device(pdev);
+	return PCI_ERS_RESULT_DISCONNECT;
+}
+
+static pci_ers_result_t
+bfad_pci_mmio_enabled(struct pci_dev *pdev)
+{
+	unsigned long	flags;
+	struct bfad_s *bfad = pci_get_drvdata(pdev);
+
+	dev_printk(KERN_INFO, &pdev->dev, "mmio_enabled\n");
+
+	/* Fetch FW diagnostic information */
+	bfa_ioc_debug_save_ftrc(&bfad->bfa.ioc);
+
+	/* Cancel all pending IOs */
+	spin_lock_irqsave(&bfad->bfad_lock, flags);
+	init_completion(&bfad->comp);
+	bfa_fcs_stop(&bfad->bfa_fcs);
+	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+	wait_for_completion(&bfad->comp);
+
+	bfad_remove_intr(bfad);
+	del_timer_sync(&bfad->hal_tmo);
+	pci_disable_device(pdev);
+
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static void
+bfad_pci_resume(struct pci_dev *pdev)
+{
+	unsigned long	flags;
+	struct bfad_s *bfad = pci_get_drvdata(pdev);
+
+	dev_printk(KERN_WARNING, &pdev->dev, "resume\n");
+
+	/* wait until the link is online */
+	bfad_rport_online_wait(bfad);
+
+	spin_lock_irqsave(&bfad->bfad_lock, flags);
+	bfad->bfad_flags &= ~BFAD_EEH_BUSY;
+	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+}
+
 struct pci_device_id bfad_id_table[] = {
 	{
 		.vendor = BFA_PCI_VENDOR_ID_BROCADE,
@@ -1513,11 +1725,22 @@ struct pci_device_id bfad_id_table[] = {
 
 MODULE_DEVICE_TABLE(pci, bfad_id_table);
 
+/*
+ * PCI error recovery handlers.
+ */
+static struct pci_error_handlers bfad_err_handler = {
+	.error_detected = bfad_pci_error_detected,
+	.slot_reset = bfad_pci_slot_reset,
+	.mmio_enabled = bfad_pci_mmio_enabled,
+	.resume = bfad_pci_resume,
+};
+
 static struct pci_driver bfad_pci_driver = {
 	.name = BFAD_DRIVER_NAME,
 	.id_table = bfad_id_table,
 	.probe = bfad_pci_probe,
 	.remove = __devexit_p(bfad_pci_remove),
+	.err_handler = &bfad_err_handler,
 };
 
 /*
@@ -1546,6 +1769,7 @@ bfad_init(void)
 
 	bfa_auto_recover = ioc_auto_recover;
 	bfa_fcs_rport_set_del_timeout(rport_del_timeout);
+	bfa_fcs_rport_set_max_logins(max_rport_logins);
 
 	error = pci_register_driver(&bfad_pci_driver);
 	if (error) {
diff --git a/drivers/scsi/bfa/bfad_attr.c b/drivers/scsi/bfa/bfad_attr.c
index b8392744017..72f5dc32cc1 100644
--- a/drivers/scsi/bfa/bfad_attr.c
+++ b/drivers/scsi/bfa/bfad_attr.c
@@ -587,6 +587,37 @@ bfad_im_vport_disable(struct fc_vport *fc_vport, bool disable)
 	return 0;
 }
 
+void
+bfad_im_vport_set_symbolic_name(struct fc_vport *fc_vport)
+{
+	struct bfad_vport_s *vport = (struct bfad_vport_s *)fc_vport->dd_data;
+	struct bfad_im_port_s *im_port =
+			(struct bfad_im_port_s *)vport->drv_port.im_port;
+	struct bfad_s *bfad = im_port->bfad;
+	struct Scsi_Host *vshost = vport->drv_port.im_port->shost;
+	char *sym_name = fc_vport->symbolic_name;
+	struct bfa_fcs_vport_s *fcs_vport;
+	wwn_t	pwwn;
+	unsigned long flags;
+
+	u64_to_wwn(fc_host_port_name(vshost), (u8 *)&pwwn);
+
+	spin_lock_irqsave(&bfad->bfad_lock, flags);
+	fcs_vport = bfa_fcs_vport_lookup(&bfad->bfa_fcs, 0, pwwn);
+	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+
+	if (fcs_vport == NULL)
+		return;
+
+	spin_lock_irqsave(&bfad->bfad_lock, flags);
+	if (strlen(sym_name) > 0) {
+		strcpy(fcs_vport->lport.port_cfg.sym_name.symname, sym_name);
+		bfa_fcs_lport_ns_util_send_rspn_id(
+			BFA_FCS_GET_NS_FROM_PORT((&fcs_vport->lport)), NULL);
+	}
+	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
+}
+
 struct fc_function_template bfad_im_fc_function_template = {
 
 	/* Target dynamic attributes */
@@ -640,6 +671,7 @@ struct fc_function_template bfad_im_fc_function_template = {
 	.vport_create = bfad_im_vport_create,
 	.vport_delete = bfad_im_vport_delete,
 	.vport_disable = bfad_im_vport_disable,
+	.set_vport_symbolic_name = bfad_im_vport_set_symbolic_name,
 	.bsg_request = bfad_im_bsg_request,
 	.bsg_timeout = bfad_im_bsg_timeout,
 };
@@ -792,6 +824,13 @@ bfad_im_model_desc_show(struct device *dev, struct device_attribute *attr,
 		else if (nports == 2 && !bfa_ioc_is_cna(&bfad->bfa.ioc))
 			snprintf(model_descr, BFA_ADAPTER_MODEL_DESCR_LEN,
 				"Brocade 16Gbps PCIe dual port FC HBA");
+	} else if (!strcmp(model, "Brocade-1867")) {
+		if (nports == 1 && !bfa_ioc_is_cna(&bfad->bfa.ioc))
+			snprintf(model_descr, BFA_ADAPTER_MODEL_DESCR_LEN,
+				"Brocade 16Gbps PCIe single port FC HBA for IBM");
+		else if (nports == 2 && !bfa_ioc_is_cna(&bfad->bfa.ioc))
+			snprintf(model_descr, BFA_ADAPTER_MODEL_DESCR_LEN,
+				"Brocade 16Gbps PCIe dual port FC HBA for IBM");
 	} else
 		snprintf(model_descr, BFA_ADAPTER_MODEL_DESCR_LEN,
 			"Invalid Model");
@@ -909,15 +948,16 @@ bfad_im_num_of_discovered_ports_show(struct device *dev,
 	struct bfad_port_s    *port = im_port->port;
 	struct bfad_s         *bfad = im_port->bfad;
 	int        nrports = 2048;
-	wwn_t          *rports = NULL;
+	struct bfa_rport_qualifier_s *rports = NULL;
 	unsigned long   flags;
 
-	rports = kzalloc(sizeof(wwn_t) * nrports , GFP_ATOMIC);
+	rports = kzalloc(sizeof(struct bfa_rport_qualifier_s) * nrports,
+			 GFP_ATOMIC);
 	if (rports == NULL)
 		return snprintf(buf, PAGE_SIZE, "Failed\n");
 
 	spin_lock_irqsave(&bfad->bfad_lock, flags);
-	bfa_fcs_lport_get_rports(port->fcs_port, rports, &nrports);
+	bfa_fcs_lport_get_rport_quals(port->fcs_port, rports, &nrports);
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 	kfree(rports);
 
diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c
index 9c1495b321d..0afa39076ce 100644
--- a/drivers/scsi/bfa/bfad_bsg.c
+++ b/drivers/scsi/bfa/bfad_bsg.c
@@ -535,7 +535,8 @@ bfad_iocmd_lport_get_rports(struct bfad_s *bfad, void *cmd,
 
 	if (bfad_chk_iocmd_sz(payload_len,
 			sizeof(struct bfa_bsg_lport_get_rports_s),
-			sizeof(wwn_t) * iocmd->nrports) != BFA_STATUS_OK) {
+			sizeof(struct bfa_rport_qualifier_s) * iocmd->nrports)
+			!= BFA_STATUS_OK) {
 		iocmd->status = BFA_STATUS_VERSION_FAIL;
 		return 0;
 	}
@@ -552,8 +553,9 @@ bfad_iocmd_lport_get_rports(struct bfad_s *bfad, void *cmd,
 		goto out;
 	}
 
-	bfa_fcs_lport_get_rports(fcs_port, (wwn_t *)iocmd_bufptr,
-				&iocmd->nrports);
+	bfa_fcs_lport_get_rport_quals(fcs_port,
+			(struct bfa_rport_qualifier_s *)iocmd_bufptr,
+			&iocmd->nrports);
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 	iocmd->status = BFA_STATUS_OK;
 out:
@@ -578,7 +580,11 @@ bfad_iocmd_rport_get_attr(struct bfad_s *bfad, void *cmd)
 		goto out;
 	}
 
-	fcs_rport = bfa_fcs_rport_lookup(fcs_port, iocmd->rpwwn);
+	if (iocmd->pid)
+		fcs_rport = bfa_fcs_lport_get_rport_by_qualifier(fcs_port,
+						iocmd->rpwwn, iocmd->pid);
+	else
+		fcs_rport = bfa_fcs_rport_lookup(fcs_port, iocmd->rpwwn);
 	if (fcs_rport == NULL) {
 		bfa_trc(bfad, 0);
 		spin_unlock_irqrestore(&bfad->bfad_lock, flags);
@@ -671,9 +677,11 @@ bfad_iocmd_rport_get_stats(struct bfad_s *bfad, void *cmd)
 
 	memcpy((void *)&iocmd->stats, (void *)&fcs_rport->stats,
 		sizeof(struct bfa_rport_stats_s));
-	memcpy((void *)&iocmd->stats.hal_stats,
-	       (void *)&(bfa_fcs_rport_get_halrport(fcs_rport)->stats),
-	       sizeof(struct bfa_rport_hal_stats_s));
+	if (bfa_fcs_rport_get_halrport(fcs_rport)) {
+		memcpy((void *)&iocmd->stats.hal_stats,
+		       (void *)&(bfa_fcs_rport_get_halrport(fcs_rport)->stats),
+			sizeof(struct bfa_rport_hal_stats_s));
+	}
 
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 	iocmd->status = BFA_STATUS_OK;
@@ -709,7 +717,8 @@ bfad_iocmd_rport_clr_stats(struct bfad_s *bfad, void *cmd)
 
 	memset((char *)&fcs_rport->stats, 0, sizeof(struct bfa_rport_stats_s));
 	rport = bfa_fcs_rport_get_halrport(fcs_rport);
-	memset(&rport->stats, 0, sizeof(rport->stats));
+	if (rport)
+		memset(&rport->stats, 0, sizeof(rport->stats));
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 	iocmd->status = BFA_STATUS_OK;
 out:
@@ -744,7 +753,8 @@ bfad_iocmd_rport_set_speed(struct bfad_s *bfad, void *cmd)
 	fcs_rport->rpf.assigned_speed  = iocmd->speed;
 	/* Set this speed in f/w only if the RPSC speed is not available */
 	if (fcs_rport->rpf.rpsc_speed == BFA_PORT_SPEED_UNKNOWN)
-		bfa_rport_speed(fcs_rport->bfa_rport, iocmd->speed);
+		if (fcs_rport->bfa_rport)
+			bfa_rport_speed(fcs_rport->bfa_rport, iocmd->speed);
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
 	iocmd->status = BFA_STATUS_OK;
 out:
@@ -1030,9 +1040,10 @@ bfad_iocmd_itnim_get_iostats(struct bfad_s *bfad, void *cmd)
 			iocmd->status = BFA_STATUS_UNKNOWN_RWWN;
 		else {
 			iocmd->status = BFA_STATUS_OK;
-			memcpy((void *)&iocmd->iostats, (void *)
-			       &(bfa_fcs_itnim_get_halitn(itnim)->stats),
-			       sizeof(struct bfa_itnim_iostats_s));
+			if (bfa_fcs_itnim_get_halitn(itnim))
+				memcpy((void *)&iocmd->iostats, (void *)
+				&(bfa_fcs_itnim_get_halitn(itnim)->stats),
+				       sizeof(struct bfa_itnim_iostats_s));
 		}
 	}
 	spin_unlock_irqrestore(&bfad->bfad_lock, flags);
@@ -2949,13 +2960,13 @@ bfad_fcxp_bsg_send(struct fc_bsg_job *job, struct bfad_fcxp *drv_fcxp,
 	spin_lock_irqsave(&bfad->bfad_lock, flags);
 
 	/* Allocate bfa_fcxp structure */
-	hal_fcxp = bfa_fcxp_alloc(drv_fcxp, &bfad->bfa,
+	hal_fcxp = bfa_fcxp_req_rsp_alloc(drv_fcxp, &bfad->bfa,
 				  drv_fcxp->num_req_sgles,
 				  drv_fcxp->num_rsp_sgles,
 				  bfad_fcxp_get_req_sgaddr_cb,
 				  bfad_fcxp_get_req_sglen_cb,
 				  bfad_fcxp_get_rsp_sgaddr_cb,
-				  bfad_fcxp_get_rsp_sglen_cb);
+				  bfad_fcxp_get_rsp_sglen_cb, BFA_TRUE);
 	if (!hal_fcxp) {
 		bfa_trc(bfad, 0);
 		spin_unlock_irqrestore(&bfad->bfad_lock, flags);
diff --git a/drivers/scsi/bfa/bfad_bsg.h b/drivers/scsi/bfa/bfad_bsg.h
index 17ad6728313..8c569ddb750 100644
--- a/drivers/scsi/bfa/bfad_bsg.h
+++ b/drivers/scsi/bfa/bfad_bsg.h
@@ -319,6 +319,8 @@ struct bfa_bsg_rport_attr_s {
 	u16		vf_id;
 	wwn_t		pwwn;
 	wwn_t		rpwwn;
+	u32		pid;
+	u32		rsvd;
 	struct bfa_rport_attr_s attr;
 };
 
diff --git a/drivers/scsi/bfa/bfad_drv.h b/drivers/scsi/bfa/bfad_drv.h
index 7f74f1d1912..1840651ce1d 100644
--- a/drivers/scsi/bfa/bfad_drv.h
+++ b/drivers/scsi/bfa/bfad_drv.h
@@ -37,6 +37,7 @@
 #include <linux/vmalloc.h>
 #include <linux/workqueue.h>
 #include <linux/bitops.h>
+#include <linux/aer.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_tcq.h>
@@ -56,7 +57,7 @@
 #ifdef BFA_DRIVER_VERSION
 #define BFAD_DRIVER_VERSION    BFA_DRIVER_VERSION
 #else
-#define BFAD_DRIVER_VERSION    "3.0.23.0"
+#define BFAD_DRIVER_VERSION    "3.1.2.0"
 #endif
 
 #define BFAD_PROTO_NAME FCPI_NAME
@@ -81,6 +82,8 @@
 #define BFAD_FC4_PROBE_DONE			0x00000200
 #define BFAD_PORT_DELETE			0x00000001
 #define BFAD_INTX_ON				0x00000400
+#define BFAD_EEH_BUSY				0x00000800
+#define BFAD_EEH_PCI_CHANNEL_IO_PERM_FAILURE	0x00001000
 /*
  * BFAD related definition
  */
diff --git a/drivers/scsi/bfa/bfad_im.c b/drivers/scsi/bfa/bfad_im.c
index 2eebf8d4d58..8f92732655c 100644
--- a/drivers/scsi/bfa/bfad_im.c
+++ b/drivers/scsi/bfa/bfad_im.c
@@ -1216,6 +1216,15 @@ bfad_im_queuecommand_lck(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd
 		return 0;
 	}
 
+	if (bfad->bfad_flags & BFAD_EEH_BUSY) {
+		if (bfad->bfad_flags & BFAD_EEH_PCI_CHANNEL_IO_PERM_FAILURE)
+			cmnd->result = DID_NO_CONNECT << 16;
+		else
+			cmnd->result = DID_REQUEUE << 16;
+		done(cmnd);
+		return 0;
+	}
+
 	sg_cnt = scsi_dma_map(cmnd);
 	if (sg_cnt < 0)
 		return SCSI_MLQUEUE_HOST_BUSY;
diff --git a/drivers/scsi/bnx2fc/bnx2fc_io.c b/drivers/scsi/bnx2fc/bnx2fc_io.c
index 73f231ccd45..8d4626c07a1 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_io.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_io.c
@@ -1807,7 +1807,7 @@ static void bnx2fc_parse_fcp_rsp(struct bnx2fc_cmd *io_req,
 			fcp_sns_len = SCSI_SENSE_BUFFERSIZE;
 		}
 
-		memset(sc_cmd->sense_buffer, 0, sizeof(sc_cmd->sense_buffer));
+		memset(sc_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
 		if (fcp_sns_len)
 			memcpy(sc_cmd->sense_buffer, rq_data, fcp_sns_len);
 
diff --git a/drivers/scsi/constants.c b/drivers/scsi/constants.c
index 450e011f981..76e4c039f0d 100644
--- a/drivers/scsi/constants.c
+++ b/drivers/scsi/constants.c
@@ -1422,7 +1422,8 @@ static const char * const hostbyte_table[]={
 "DID_OK", "DID_NO_CONNECT", "DID_BUS_BUSY", "DID_TIME_OUT", "DID_BAD_TARGET",
 "DID_ABORT", "DID_PARITY", "DID_ERROR", "DID_RESET", "DID_BAD_INTR",
 "DID_PASSTHROUGH", "DID_SOFT_ERROR", "DID_IMM_RETRY", "DID_REQUEUE",
-"DID_TRANSPORT_DISRUPTED", "DID_TRANSPORT_FAILFAST" };
+"DID_TRANSPORT_DISRUPTED", "DID_TRANSPORT_FAILFAST", "DID_TARGET_FAILURE",
+"DID_NEXUS_FAILURE" };
 #define NUM_HOSTBYTE_STRS ARRAY_SIZE(hostbyte_table)
 
 static const char * const driverbyte_table[]={
diff --git a/drivers/scsi/device_handler/scsi_dh_alua.c b/drivers/scsi/device_handler/scsi_dh_alua.c
index 08d80a6d272..6f4d8e6f32f 100644
--- a/drivers/scsi/device_handler/scsi_dh_alua.c
+++ b/drivers/scsi/device_handler/scsi_dh_alua.c
@@ -641,8 +641,7 @@ static int alua_rtpg(struct scsi_device *sdev, struct alua_dh_data *h)
 		h->state = TPGS_STATE_STANDBY;
 		break;
 	case TPGS_STATE_OFFLINE:
-	case TPGS_STATE_UNAVAILABLE:
-		/* Path unusable for unavailable/offline */
+		/* Path unusable */
 		err = SCSI_DH_DEV_OFFLINED;
 		break;
 	default:
diff --git a/drivers/scsi/device_handler/scsi_dh_rdac.c b/drivers/scsi/device_handler/scsi_dh_rdac.c
index 20c4557f5ab..69c915aa77c 100644
--- a/drivers/scsi/device_handler/scsi_dh_rdac.c
+++ b/drivers/scsi/device_handler/scsi_dh_rdac.c
@@ -790,29 +790,19 @@ static const struct scsi_dh_devlist rdac_dev_list[] = {
 	{"IBM", "1815"},
 	{"IBM", "1818"},
 	{"IBM", "3526"},
-	{"SGI", "TP9400"},
-	{"SGI", "TP9500"},
-	{"SGI", "TP9700"},
+	{"SGI", "TP9"},
 	{"SGI", "IS"},
 	{"STK", "OPENstorage D280"},
-	{"SUN", "CSM200_R"},
-	{"SUN", "LCSM100_I"},
-	{"SUN", "LCSM100_S"},
-	{"SUN", "LCSM100_E"},
-	{"SUN", "LCSM100_F"},
-	{"DELL", "MD3000"},
-	{"DELL", "MD3000i"},
-	{"DELL", "MD32xx"},
-	{"DELL", "MD32xxi"},
-	{"DELL", "MD36xxi"},
-	{"DELL", "MD36xxf"},
-	{"LSI", "INF-01-00"},
-	{"ENGENIO", "INF-01-00"},
 	{"STK", "FLEXLINE 380"},
-	{"SUN", "CSM100_R_FC"},
+	{"SUN", "CSM"},
+	{"SUN", "LCSM100"},
 	{"SUN", "STK6580_6780"},
 	{"SUN", "SUN_6180"},
 	{"SUN", "ArrayStorage"},
+	{"DELL", "MD3"},
+	{"NETAPP", "INF-01-00"},
+	{"LSI", "INF-01-00"},
+	{"ENGENIO", "INF-01-00"},
 	{NULL, NULL},
 };
 
@@ -863,7 +853,7 @@ static int rdac_bus_attach(struct scsi_device *sdev)
 	if (!scsi_dh_data) {
 		sdev_printk(KERN_ERR, sdev, "%s: Attach failed\n",
 			    RDAC_NAME);
-		return 0;
+		return -ENOMEM;
 	}
 
 	scsi_dh_data->scsi_dh = &rdac_dh;
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index 2b4261cb774..4217e49aea4 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -99,6 +99,15 @@ static const struct pci_device_id hpsa_pci_device_id[] = {
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3354},
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3355},
 	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x3356},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1920},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1921},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1922},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1923},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1924},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1925},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1926},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSH,     0x103C, 0x1928},
+	{PCI_VENDOR_ID_HP,     PCI_DEVICE_ID_HP_CISSF,     0x103C, 0x334d},
 	{PCI_VENDOR_ID_HP,     PCI_ANY_ID,	PCI_ANY_ID, PCI_ANY_ID,
 		PCI_CLASS_STORAGE_RAID << 8, 0xffff << 8, 0},
 	{0,}
@@ -118,13 +127,22 @@ static struct board_type products[] = {
 	{0x3249103C, "Smart Array P812", &SA5_access},
 	{0x324a103C, "Smart Array P712m", &SA5_access},
 	{0x324b103C, "Smart Array P711m", &SA5_access},
-	{0x3350103C, "Smart Array", &SA5_access},
-	{0x3351103C, "Smart Array", &SA5_access},
-	{0x3352103C, "Smart Array", &SA5_access},
-	{0x3353103C, "Smart Array", &SA5_access},
-	{0x3354103C, "Smart Array", &SA5_access},
-	{0x3355103C, "Smart Array", &SA5_access},
-	{0x3356103C, "Smart Array", &SA5_access},
+	{0x3350103C, "Smart Array P222", &SA5_access},
+	{0x3351103C, "Smart Array P420", &SA5_access},
+	{0x3352103C, "Smart Array P421", &SA5_access},
+	{0x3353103C, "Smart Array P822", &SA5_access},
+	{0x3354103C, "Smart Array P420i", &SA5_access},
+	{0x3355103C, "Smart Array P220i", &SA5_access},
+	{0x3356103C, "Smart Array P721m", &SA5_access},
+	{0x1920103C, "Smart Array", &SA5_access},
+	{0x1921103C, "Smart Array", &SA5_access},
+	{0x1922103C, "Smart Array", &SA5_access},
+	{0x1923103C, "Smart Array", &SA5_access},
+	{0x1924103C, "Smart Array", &SA5_access},
+	{0x1925103C, "Smart Array", &SA5_access},
+	{0x1926103C, "Smart Array", &SA5_access},
+	{0x1928103C, "Smart Array", &SA5_access},
+	{0x334d103C, "Smart Array P822se", &SA5_access},
 	{0xFFFF103C, "Unknown Smart Array", &SA5_access},
 };
 
@@ -2610,7 +2628,7 @@ static int hpsa_eh_abort_handler(struct scsi_cmnd *sc)
 	/* not in reqQ, if also not in cmpQ, must have already completed */
 	found = hpsa_find_cmd_in_queue(h, sc, &h->cmpQ);
 	if (!found)  {
-		dev_dbg(&h->pdev->dev, "%s Request FAILED (not known to driver).\n",
+		dev_dbg(&h->pdev->dev, "%s Request SUCCEEDED (not known to driver).\n",
 				msg);
 		return SUCCESS;
 	}
@@ -3266,7 +3284,7 @@ static void fill_cmd(struct CommandList *c, u8 cmd, struct ctlr_info *h,
 			c->Request.Timeout = 0; /* Don't time out */
 			memset(&c->Request.CDB[0], 0, sizeof(c->Request.CDB));
 			c->Request.CDB[0] =  cmd;
-			c->Request.CDB[1] = 0x03;  /* Reset target above */
+			c->Request.CDB[1] = HPSA_RESET_TYPE_LUN;
 			/* If bytes 4-7 are zero, it means reset the */
 			/* LunID device */
 			c->Request.CDB[4] = 0x00;
@@ -3338,7 +3356,8 @@ static void __iomem *remap_pci_mem(ulong base, ulong size)
 {
 	ulong page_base = ((ulong) base) & PAGE_MASK;
 	ulong page_offs = ((ulong) base) - page_base;
-	void __iomem *page_remapped = ioremap(page_base, page_offs + size);
+	void __iomem *page_remapped = ioremap_nocache(page_base,
+		page_offs + size);
 
 	return page_remapped ? (page_remapped + page_offs) : NULL;
 }
diff --git a/drivers/scsi/ibmvscsi/Makefile b/drivers/scsi/ibmvscsi/Makefile
index ff5b5c5538e..cb150d1e585 100644
--- a/drivers/scsi/ibmvscsi/Makefile
+++ b/drivers/scsi/ibmvscsi/Makefile
@@ -1,7 +1,3 @@
-obj-$(CONFIG_SCSI_IBMVSCSI)	+= ibmvscsic.o
-
-ibmvscsic-y			+= ibmvscsi.o
-ibmvscsic-$(CONFIG_PPC_PSERIES)	+= rpa_vscsi.o 
-
+obj-$(CONFIG_SCSI_IBMVSCSI)	+= ibmvscsi.o
 obj-$(CONFIG_SCSI_IBMVSCSIS)	+= ibmvstgt.o
 obj-$(CONFIG_SCSI_IBMVFC)	+= ibmvfc.o
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c
index 134a0ae85bb..5e8d51bd03d 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.c
+++ b/drivers/scsi/ibmvscsi/ibmvfc.c
@@ -2242,6 +2242,21 @@ static int ibmvfc_match_key(struct ibmvfc_event *evt, void *key)
 }
 
 /**
+ * ibmvfc_match_evt - Match function for specified event
+ * @evt:	ibmvfc event struct
+ * @match:	event to match
+ *
+ * Returns:
+ *	1 if event matches key / 0 if event does not match key
+ **/
+static int ibmvfc_match_evt(struct ibmvfc_event *evt, void *match)
+{
+	if (evt == match)
+		return 1;
+	return 0;
+}
+
+/**
  * ibmvfc_abort_task_set - Abort outstanding commands to the device
  * @sdev:	scsi device to abort commands
  *
@@ -2322,7 +2337,20 @@ static int ibmvfc_abort_task_set(struct scsi_device *sdev)
 		if (rc) {
 			sdev_printk(KERN_INFO, sdev, "Cancel failed, resetting host\n");
 			ibmvfc_reset_host(vhost);
-			rsp_rc = 0;
+			rsp_rc = -EIO;
+			rc = ibmvfc_wait_for_ops(vhost, sdev->hostdata, ibmvfc_match_key);
+
+			if (rc == SUCCESS)
+				rsp_rc = 0;
+
+			rc = ibmvfc_wait_for_ops(vhost, evt, ibmvfc_match_evt);
+			if (rc != SUCCESS) {
+				spin_lock_irqsave(vhost->host->host_lock, flags);
+				ibmvfc_hard_reset_host(vhost);
+				spin_unlock_irqrestore(vhost->host->host_lock, flags);
+				rsp_rc = 0;
+			}
+
 			goto out;
 		}
 	}
@@ -2597,8 +2625,10 @@ static void ibmvfc_handle_async(struct ibmvfc_async_crq *crq,
 	case IBMVFC_AE_SCN_FABRIC:
 	case IBMVFC_AE_SCN_DOMAIN:
 		vhost->events_to_log |= IBMVFC_AE_RSCN;
-		vhost->delay_init = 1;
-		__ibmvfc_reset_host(vhost);
+		if (vhost->state < IBMVFC_HALTED) {
+			vhost->delay_init = 1;
+			__ibmvfc_reset_host(vhost);
+		}
 		break;
 	case IBMVFC_AE_SCN_NPORT:
 	case IBMVFC_AE_SCN_GROUP:
diff --git a/drivers/scsi/ibmvscsi/ibmvfc.h b/drivers/scsi/ibmvscsi/ibmvfc.h
index 834c37fc7ce..3be8af624e6 100644
--- a/drivers/scsi/ibmvscsi/ibmvfc.h
+++ b/drivers/scsi/ibmvscsi/ibmvfc.h
@@ -29,8 +29,8 @@
 #include "viosrp.h"
 
 #define IBMVFC_NAME	"ibmvfc"
-#define IBMVFC_DRIVER_VERSION		"1.0.9"
-#define IBMVFC_DRIVER_DATE		"(August 5, 2010)"
+#define IBMVFC_DRIVER_VERSION		"1.0.10"
+#define IBMVFC_DRIVER_DATE		"(August 24, 2012)"
 
 #define IBMVFC_DEFAULT_TIMEOUT	60
 #define IBMVFC_ADISC_CANCEL_TIMEOUT	45
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c
index 3a6c4742951..ef9a54c7da6 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.c
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.c
@@ -93,13 +93,13 @@ static int max_requests = IBMVSCSI_MAX_REQUESTS_DEFAULT;
 static int max_events = IBMVSCSI_MAX_REQUESTS_DEFAULT + 2;
 static int fast_fail = 1;
 static int client_reserve = 1;
+static char partition_name[97] = "UNKNOWN";
+static unsigned int partition_number = -1;
 
 static struct scsi_transport_template *ibmvscsi_transport_template;
 
 #define IBMVSCSI_VERSION "1.5.9"
 
-static struct ibmvscsi_ops *ibmvscsi_ops;
-
 MODULE_DESCRIPTION("IBM Virtual SCSI");
 MODULE_AUTHOR("Dave Boutcher");
 MODULE_LICENSE("GPL");
@@ -118,6 +118,316 @@ MODULE_PARM_DESC(fast_fail, "Enable fast fail. [Default=1]");
 module_param_named(client_reserve, client_reserve, int, S_IRUGO );
 MODULE_PARM_DESC(client_reserve, "Attempt client managed reserve/release");
 
+static void ibmvscsi_handle_crq(struct viosrp_crq *crq,
+				struct ibmvscsi_host_data *hostdata);
+
+/* ------------------------------------------------------------
+ * Routines for managing the command/response queue
+ */
+/**
+ * ibmvscsi_handle_event: - Interrupt handler for crq events
+ * @irq:	number of irq to handle, not used
+ * @dev_instance: ibmvscsi_host_data of host that received interrupt
+ *
+ * Disables interrupts and schedules srp_task
+ * Always returns IRQ_HANDLED
+ */
+static irqreturn_t ibmvscsi_handle_event(int irq, void *dev_instance)
+{
+	struct ibmvscsi_host_data *hostdata =
+	    (struct ibmvscsi_host_data *)dev_instance;
+	vio_disable_interrupts(to_vio_dev(hostdata->dev));
+	tasklet_schedule(&hostdata->srp_task);
+	return IRQ_HANDLED;
+}
+
+/**
+ * release_crq_queue: - Deallocates data and unregisters CRQ
+ * @queue:	crq_queue to initialize and register
+ * @host_data:	ibmvscsi_host_data of host
+ *
+ * Frees irq, deallocates a page for messages, unmaps dma, and unregisters
+ * the crq with the hypervisor.
+ */
+static void ibmvscsi_release_crq_queue(struct crq_queue *queue,
+				       struct ibmvscsi_host_data *hostdata,
+				       int max_requests)
+{
+	long rc = 0;
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+	free_irq(vdev->irq, (void *)hostdata);
+	tasklet_kill(&hostdata->srp_task);
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
+	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+	dma_unmap_single(hostdata->dev,
+			 queue->msg_token,
+			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
+	free_page((unsigned long)queue->msgs);
+}
+
+/**
+ * crq_queue_next_crq: - Returns the next entry in message queue
+ * @queue:	crq_queue to use
+ *
+ * Returns pointer to next entry in queue, or NULL if there are no new
+ * entried in the CRQ.
+ */
+static struct viosrp_crq *crq_queue_next_crq(struct crq_queue *queue)
+{
+	struct viosrp_crq *crq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&queue->lock, flags);
+	crq = &queue->msgs[queue->cur];
+	if (crq->valid & 0x80) {
+		if (++queue->cur == queue->size)
+			queue->cur = 0;
+	} else
+		crq = NULL;
+	spin_unlock_irqrestore(&queue->lock, flags);
+
+	return crq;
+}
+
+/**
+ * ibmvscsi_send_crq: - Send a CRQ
+ * @hostdata:	the adapter
+ * @word1:	the first 64 bits of the data
+ * @word2:	the second 64 bits of the data
+ */
+static int ibmvscsi_send_crq(struct ibmvscsi_host_data *hostdata,
+			     u64 word1, u64 word2)
+{
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+
+	return plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2);
+}
+
+/**
+ * ibmvscsi_task: - Process srps asynchronously
+ * @data:	ibmvscsi_host_data of host
+ */
+static void ibmvscsi_task(void *data)
+{
+	struct ibmvscsi_host_data *hostdata = (struct ibmvscsi_host_data *)data;
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+	struct viosrp_crq *crq;
+	int done = 0;
+
+	while (!done) {
+		/* Pull all the valid messages off the CRQ */
+		while ((crq = crq_queue_next_crq(&hostdata->queue)) != NULL) {
+			ibmvscsi_handle_crq(crq, hostdata);
+			crq->valid = 0x00;
+		}
+
+		vio_enable_interrupts(vdev);
+		crq = crq_queue_next_crq(&hostdata->queue);
+		if (crq != NULL) {
+			vio_disable_interrupts(vdev);
+			ibmvscsi_handle_crq(crq, hostdata);
+			crq->valid = 0x00;
+		} else {
+			done = 1;
+		}
+	}
+}
+
+static void gather_partition_info(void)
+{
+	struct device_node *rootdn;
+
+	const char *ppartition_name;
+	const unsigned int *p_number_ptr;
+
+	/* Retrieve information about this partition */
+	rootdn = of_find_node_by_path("/");
+	if (!rootdn) {
+		return;
+	}
+
+	ppartition_name = of_get_property(rootdn, "ibm,partition-name", NULL);
+	if (ppartition_name)
+		strncpy(partition_name, ppartition_name,
+				sizeof(partition_name));
+	p_number_ptr = of_get_property(rootdn, "ibm,partition-no", NULL);
+	if (p_number_ptr)
+		partition_number = *p_number_ptr;
+	of_node_put(rootdn);
+}
+
+static void set_adapter_info(struct ibmvscsi_host_data *hostdata)
+{
+	memset(&hostdata->madapter_info, 0x00,
+			sizeof(hostdata->madapter_info));
+
+	dev_info(hostdata->dev, "SRP_VERSION: %s\n", SRP_VERSION);
+	strcpy(hostdata->madapter_info.srp_version, SRP_VERSION);
+
+	strncpy(hostdata->madapter_info.partition_name, partition_name,
+			sizeof(hostdata->madapter_info.partition_name));
+
+	hostdata->madapter_info.partition_number = partition_number;
+
+	hostdata->madapter_info.mad_version = 1;
+	hostdata->madapter_info.os_type = 2;
+}
+
+/**
+ * reset_crq_queue: - resets a crq after a failure
+ * @queue:	crq_queue to initialize and register
+ * @hostdata:	ibmvscsi_host_data of host
+ *
+ */
+static int ibmvscsi_reset_crq_queue(struct crq_queue *queue,
+				    struct ibmvscsi_host_data *hostdata)
+{
+	int rc = 0;
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+
+	/* Close the CRQ */
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
+	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+
+	/* Clean out the queue */
+	memset(queue->msgs, 0x00, PAGE_SIZE);
+	queue->cur = 0;
+
+	set_adapter_info(hostdata);
+
+	/* And re-open it again */
+	rc = plpar_hcall_norets(H_REG_CRQ,
+				vdev->unit_address,
+				queue->msg_token, PAGE_SIZE);
+	if (rc == 2) {
+		/* Adapter is good, but other end is not ready */
+		dev_warn(hostdata->dev, "Partner adapter not ready\n");
+	} else if (rc != 0) {
+		dev_warn(hostdata->dev, "couldn't register crq--rc 0x%x\n", rc);
+	}
+	return rc;
+}
+
+/**
+ * initialize_crq_queue: - Initializes and registers CRQ with hypervisor
+ * @queue:	crq_queue to initialize and register
+ * @hostdata:	ibmvscsi_host_data of host
+ *
+ * Allocates a page for messages, maps it for dma, and registers
+ * the crq with the hypervisor.
+ * Returns zero on success.
+ */
+static int ibmvscsi_init_crq_queue(struct crq_queue *queue,
+				   struct ibmvscsi_host_data *hostdata,
+				   int max_requests)
+{
+	int rc;
+	int retrc;
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+
+	queue->msgs = (struct viosrp_crq *)get_zeroed_page(GFP_KERNEL);
+
+	if (!queue->msgs)
+		goto malloc_failed;
+	queue->size = PAGE_SIZE / sizeof(*queue->msgs);
+
+	queue->msg_token = dma_map_single(hostdata->dev, queue->msgs,
+					  queue->size * sizeof(*queue->msgs),
+					  DMA_BIDIRECTIONAL);
+
+	if (dma_mapping_error(hostdata->dev, queue->msg_token))
+		goto map_failed;
+
+	gather_partition_info();
+	set_adapter_info(hostdata);
+
+	retrc = rc = plpar_hcall_norets(H_REG_CRQ,
+				vdev->unit_address,
+				queue->msg_token, PAGE_SIZE);
+	if (rc == H_RESOURCE)
+		/* maybe kexecing and resource is busy. try a reset */
+		rc = ibmvscsi_reset_crq_queue(queue,
+					      hostdata);
+
+	if (rc == 2) {
+		/* Adapter is good, but other end is not ready */
+		dev_warn(hostdata->dev, "Partner adapter not ready\n");
+		retrc = 0;
+	} else if (rc != 0) {
+		dev_warn(hostdata->dev, "Error %d opening adapter\n", rc);
+		goto reg_crq_failed;
+	}
+
+	queue->cur = 0;
+	spin_lock_init(&queue->lock);
+
+	tasklet_init(&hostdata->srp_task, (void *)ibmvscsi_task,
+		     (unsigned long)hostdata);
+
+	if (request_irq(vdev->irq,
+			ibmvscsi_handle_event,
+			0, "ibmvscsi", (void *)hostdata) != 0) {
+		dev_err(hostdata->dev, "couldn't register irq 0x%x\n",
+			vdev->irq);
+		goto req_irq_failed;
+	}
+
+	rc = vio_enable_interrupts(vdev);
+	if (rc != 0) {
+		dev_err(hostdata->dev, "Error %d enabling interrupts!!!\n", rc);
+		goto req_irq_failed;
+	}
+
+	return retrc;
+
+      req_irq_failed:
+	tasklet_kill(&hostdata->srp_task);
+	rc = 0;
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
+	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+      reg_crq_failed:
+	dma_unmap_single(hostdata->dev,
+			 queue->msg_token,
+			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
+      map_failed:
+	free_page((unsigned long)queue->msgs);
+      malloc_failed:
+	return -1;
+}
+
+/**
+ * reenable_crq_queue: - reenables a crq after
+ * @queue:	crq_queue to initialize and register
+ * @hostdata:	ibmvscsi_host_data of host
+ *
+ */
+static int ibmvscsi_reenable_crq_queue(struct crq_queue *queue,
+				       struct ibmvscsi_host_data *hostdata)
+{
+	int rc = 0;
+	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
+
+	/* Re-enable the CRQ */
+	do {
+		if (rc)
+			msleep(100);
+		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
+	} while ((rc == H_IN_PROGRESS) || (rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
+
+	if (rc)
+		dev_err(hostdata->dev, "Error %d enabling adapter\n", rc);
+	return rc;
+}
+
 /* ------------------------------------------------------------
  * Routines for the event pool and event structs
  */
@@ -611,7 +921,7 @@ static int ibmvscsi_send_srp_event(struct srp_event_struct *evt_struct,
 	}
 
 	if ((rc =
-	     ibmvscsi_ops->send_crq(hostdata, crq_as_u64[0], crq_as_u64[1])) != 0) {
+	     ibmvscsi_send_crq(hostdata, crq_as_u64[0], crq_as_u64[1])) != 0) {
 		list_del(&evt_struct->list);
 		del_timer(&evt_struct->timer);
 
@@ -1420,8 +1730,8 @@ static int ibmvscsi_eh_host_reset_handler(struct scsi_cmnd *cmd)
  * @hostdata:	ibmvscsi_host_data of host
  *
 */
-void ibmvscsi_handle_crq(struct viosrp_crq *crq,
-			 struct ibmvscsi_host_data *hostdata)
+static void ibmvscsi_handle_crq(struct viosrp_crq *crq,
+				struct ibmvscsi_host_data *hostdata)
 {
 	long rc;
 	unsigned long flags;
@@ -1433,8 +1743,8 @@ void ibmvscsi_handle_crq(struct viosrp_crq *crq,
 		case 0x01:	/* Initialization message */
 			dev_info(hostdata->dev, "partner initialized\n");
 			/* Send back a response */
-			if ((rc = ibmvscsi_ops->send_crq(hostdata,
-							 0xC002000000000000LL, 0)) == 0) {
+			rc = ibmvscsi_send_crq(hostdata, 0xC002000000000000LL, 0);
+			if (rc == 0) {
 				/* Now login */
 				init_adapter(hostdata);
 			} else {
@@ -1541,6 +1851,9 @@ static int ibmvscsi_do_host_config(struct ibmvscsi_host_data *hostdata,
 
 	host_config = &evt_struct->iu.mad.host_config;
 
+	/* The transport length field is only 16-bit */
+	length = min(0xffff, length);
+
 	/* Set up a lun reset SRP command */
 	memset(host_config, 0x00, sizeof(*host_config));
 	host_config->common.type = VIOSRP_HOST_CONFIG_TYPE;
@@ -1840,17 +2153,17 @@ static void ibmvscsi_do_work(struct ibmvscsi_host_data *hostdata)
 		smp_rmb();
 		hostdata->reset_crq = 0;
 
-		rc = ibmvscsi_ops->reset_crq_queue(&hostdata->queue, hostdata);
+		rc = ibmvscsi_reset_crq_queue(&hostdata->queue, hostdata);
 		if (!rc)
-			rc = ibmvscsi_ops->send_crq(hostdata, 0xC001000000000000LL, 0);
+			rc = ibmvscsi_send_crq(hostdata, 0xC001000000000000LL, 0);
 		vio_enable_interrupts(to_vio_dev(hostdata->dev));
 	} else if (hostdata->reenable_crq) {
 		smp_rmb();
 		action = "enable";
-		rc = ibmvscsi_ops->reenable_crq_queue(&hostdata->queue, hostdata);
+		rc = ibmvscsi_reenable_crq_queue(&hostdata->queue, hostdata);
 		hostdata->reenable_crq = 0;
 		if (!rc)
-			rc = ibmvscsi_ops->send_crq(hostdata, 0xC001000000000000LL, 0);
+			rc = ibmvscsi_send_crq(hostdata, 0xC001000000000000LL, 0);
 	} else
 		return;
 
@@ -1944,7 +2257,7 @@ static int ibmvscsi_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 		goto init_crq_failed;
 	}
 
-	rc = ibmvscsi_ops->init_crq_queue(&hostdata->queue, hostdata, max_events);
+	rc = ibmvscsi_init_crq_queue(&hostdata->queue, hostdata, max_events);
 	if (rc != 0 && rc != H_RESOURCE) {
 		dev_err(&vdev->dev, "couldn't initialize crq. rc=%d\n", rc);
 		goto kill_kthread;
@@ -1974,7 +2287,7 @@ static int ibmvscsi_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 	 * to fail if the other end is not acive.  In that case we don't
 	 * want to scan
 	 */
-	if (ibmvscsi_ops->send_crq(hostdata, 0xC001000000000000LL, 0) == 0
+	if (ibmvscsi_send_crq(hostdata, 0xC001000000000000LL, 0) == 0
 	    || rc == H_RESOURCE) {
 		/*
 		 * Wait around max init_timeout secs for the adapter to finish
@@ -2002,7 +2315,7 @@ static int ibmvscsi_probe(struct vio_dev *vdev, const struct vio_device_id *id)
       add_host_failed:
 	release_event_pool(&hostdata->pool, hostdata);
       init_pool_failed:
-	ibmvscsi_ops->release_crq_queue(&hostdata->queue, hostdata, max_events);
+	ibmvscsi_release_crq_queue(&hostdata->queue, hostdata, max_events);
       kill_kthread:
       kthread_stop(hostdata->work_thread);
       init_crq_failed:
@@ -2018,7 +2331,7 @@ static int ibmvscsi_remove(struct vio_dev *vdev)
 	struct ibmvscsi_host_data *hostdata = dev_get_drvdata(&vdev->dev);
 	unmap_persist_bufs(hostdata);
 	release_event_pool(&hostdata->pool, hostdata);
-	ibmvscsi_ops->release_crq_queue(&hostdata->queue, hostdata,
+	ibmvscsi_release_crq_queue(&hostdata->queue, hostdata,
 					max_events);
 
 	kthread_stop(hostdata->work_thread);
@@ -2039,7 +2352,10 @@ static int ibmvscsi_remove(struct vio_dev *vdev)
 static int ibmvscsi_resume(struct device *dev)
 {
 	struct ibmvscsi_host_data *hostdata = dev_get_drvdata(dev);
-	return ibmvscsi_ops->resume(hostdata);
+	vio_disable_interrupts(to_vio_dev(hostdata->dev));
+	tasklet_schedule(&hostdata->srp_task);
+
+	return 0;
 }
 
 /**
@@ -2076,9 +2392,7 @@ int __init ibmvscsi_module_init(void)
 	driver_template.can_queue = max_requests;
 	max_events = max_requests + 2;
 
-	if (firmware_has_feature(FW_FEATURE_VIO))
-		ibmvscsi_ops = &rpavscsi_ops;
-	else
+	if (!firmware_has_feature(FW_FEATURE_VIO))
 		return -ENODEV;
 
 	ibmvscsi_transport_template =
diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.h b/drivers/scsi/ibmvscsi/ibmvscsi.h
index c503e177601..7d64867c5dd 100644
--- a/drivers/scsi/ibmvscsi/ibmvscsi.h
+++ b/drivers/scsi/ibmvscsi/ibmvscsi.h
@@ -107,26 +107,4 @@ struct ibmvscsi_host_data {
 	dma_addr_t adapter_info_addr;
 };
 
-/* routines for managing a command/response queue */
-void ibmvscsi_handle_crq(struct viosrp_crq *crq,
-			 struct ibmvscsi_host_data *hostdata);
-
-struct ibmvscsi_ops {
-	int (*init_crq_queue)(struct crq_queue *queue,
-			      struct ibmvscsi_host_data *hostdata,
-			      int max_requests);
-	void (*release_crq_queue)(struct crq_queue *queue,
-				  struct ibmvscsi_host_data *hostdata,
-				  int max_requests);
-	int (*reset_crq_queue)(struct crq_queue *queue,
-			       struct ibmvscsi_host_data *hostdata);
-	int (*reenable_crq_queue)(struct crq_queue *queue,
-				  struct ibmvscsi_host_data *hostdata);
-	int (*send_crq)(struct ibmvscsi_host_data *hostdata,
-		       u64 word1, u64 word2);
-	int (*resume) (struct ibmvscsi_host_data *hostdata);
-};
-
-extern struct ibmvscsi_ops rpavscsi_ops;
-
 #endif				/* IBMVSCSI_H */
diff --git a/drivers/scsi/ibmvscsi/rpa_vscsi.c b/drivers/scsi/ibmvscsi/rpa_vscsi.c
deleted file mode 100644
index f48ae0190d9..00000000000
--- a/drivers/scsi/ibmvscsi/rpa_vscsi.c
+++ /dev/null
@@ -1,368 +0,0 @@
-/* ------------------------------------------------------------
- * rpa_vscsi.c
- * (C) Copyright IBM Corporation 1994, 2003
- * Authors: Colin DeVilbiss (devilbis@us.ibm.com)
- *          Santiago Leon (santil@us.ibm.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
- *
- * ------------------------------------------------------------
- * RPA-specific functions of the SCSI host adapter for Virtual I/O devices
- *
- * This driver allows the Linux SCSI peripheral drivers to directly
- * access devices in the hosting partition, either on an iSeries
- * hypervisor system or a converged hypervisor system.
- */
-
-#include <asm/vio.h>
-#include <asm/prom.h>
-#include <asm/iommu.h>
-#include <asm/hvcall.h>
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/gfp.h>
-#include <linux/interrupt.h>
-#include "ibmvscsi.h"
-
-static char partition_name[97] = "UNKNOWN";
-static unsigned int partition_number = -1;
-
-/* ------------------------------------------------------------
- * Routines for managing the command/response queue
- */
-/**
- * rpavscsi_handle_event: - Interrupt handler for crq events
- * @irq:	number of irq to handle, not used
- * @dev_instance: ibmvscsi_host_data of host that received interrupt
- *
- * Disables interrupts and schedules srp_task
- * Always returns IRQ_HANDLED
- */
-static irqreturn_t rpavscsi_handle_event(int irq, void *dev_instance)
-{
-	struct ibmvscsi_host_data *hostdata =
-	    (struct ibmvscsi_host_data *)dev_instance;
-	vio_disable_interrupts(to_vio_dev(hostdata->dev));
-	tasklet_schedule(&hostdata->srp_task);
-	return IRQ_HANDLED;
-}
-
-/**
- * release_crq_queue: - Deallocates data and unregisters CRQ
- * @queue:	crq_queue to initialize and register
- * @host_data:	ibmvscsi_host_data of host
- *
- * Frees irq, deallocates a page for messages, unmaps dma, and unregisters
- * the crq with the hypervisor.
- */
-static void rpavscsi_release_crq_queue(struct crq_queue *queue,
-				       struct ibmvscsi_host_data *hostdata,
-				       int max_requests)
-{
-	long rc = 0;
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-	free_irq(vdev->irq, (void *)hostdata);
-	tasklet_kill(&hostdata->srp_task);
-	do {
-		if (rc)
-			msleep(100);
-		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
-	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
-	dma_unmap_single(hostdata->dev,
-			 queue->msg_token,
-			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
-	free_page((unsigned long)queue->msgs);
-}
-
-/**
- * crq_queue_next_crq: - Returns the next entry in message queue
- * @queue:	crq_queue to use
- *
- * Returns pointer to next entry in queue, or NULL if there are no new 
- * entried in the CRQ.
- */
-static struct viosrp_crq *crq_queue_next_crq(struct crq_queue *queue)
-{
-	struct viosrp_crq *crq;
-	unsigned long flags;
-
-	spin_lock_irqsave(&queue->lock, flags);
-	crq = &queue->msgs[queue->cur];
-	if (crq->valid & 0x80) {
-		if (++queue->cur == queue->size)
-			queue->cur = 0;
-	} else
-		crq = NULL;
-	spin_unlock_irqrestore(&queue->lock, flags);
-
-	return crq;
-}
-
-/**
- * rpavscsi_send_crq: - Send a CRQ
- * @hostdata:	the adapter
- * @word1:	the first 64 bits of the data
- * @word2:	the second 64 bits of the data
- */
-static int rpavscsi_send_crq(struct ibmvscsi_host_data *hostdata,
-			     u64 word1, u64 word2)
-{
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-
-	return plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2);
-}
-
-/**
- * rpavscsi_task: - Process srps asynchronously
- * @data:	ibmvscsi_host_data of host
- */
-static void rpavscsi_task(void *data)
-{
-	struct ibmvscsi_host_data *hostdata = (struct ibmvscsi_host_data *)data;
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-	struct viosrp_crq *crq;
-	int done = 0;
-
-	while (!done) {
-		/* Pull all the valid messages off the CRQ */
-		while ((crq = crq_queue_next_crq(&hostdata->queue)) != NULL) {
-			ibmvscsi_handle_crq(crq, hostdata);
-			crq->valid = 0x00;
-		}
-
-		vio_enable_interrupts(vdev);
-		if ((crq = crq_queue_next_crq(&hostdata->queue)) != NULL) {
-			vio_disable_interrupts(vdev);
-			ibmvscsi_handle_crq(crq, hostdata);
-			crq->valid = 0x00;
-		} else {
-			done = 1;
-		}
-	}
-}
-
-static void gather_partition_info(void)
-{
-	struct device_node *rootdn;
-
-	const char *ppartition_name;
-	const unsigned int *p_number_ptr;
-
-	/* Retrieve information about this partition */
-	rootdn = of_find_node_by_path("/");
-	if (!rootdn) {
-		return;
-	}
-
-	ppartition_name = of_get_property(rootdn, "ibm,partition-name", NULL);
-	if (ppartition_name)
-		strncpy(partition_name, ppartition_name,
-				sizeof(partition_name));
-	p_number_ptr = of_get_property(rootdn, "ibm,partition-no", NULL);
-	if (p_number_ptr)
-		partition_number = *p_number_ptr;
-	of_node_put(rootdn);
-}
-
-static void set_adapter_info(struct ibmvscsi_host_data *hostdata)
-{
-	memset(&hostdata->madapter_info, 0x00,
-			sizeof(hostdata->madapter_info));
-
-	dev_info(hostdata->dev, "SRP_VERSION: %s\n", SRP_VERSION);
-	strcpy(hostdata->madapter_info.srp_version, SRP_VERSION);
-
-	strncpy(hostdata->madapter_info.partition_name, partition_name,
-			sizeof(hostdata->madapter_info.partition_name));
-
-	hostdata->madapter_info.partition_number = partition_number;
-
-	hostdata->madapter_info.mad_version = 1;
-	hostdata->madapter_info.os_type = 2;
-}
-
-/**
- * reset_crq_queue: - resets a crq after a failure
- * @queue:	crq_queue to initialize and register
- * @hostdata:	ibmvscsi_host_data of host
- *
- */
-static int rpavscsi_reset_crq_queue(struct crq_queue *queue,
-				    struct ibmvscsi_host_data *hostdata)
-{
-	int rc = 0;
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-
-	/* Close the CRQ */
-	do {
-		if (rc)
-			msleep(100);
-		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
-	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
-
-	/* Clean out the queue */
-	memset(queue->msgs, 0x00, PAGE_SIZE);
-	queue->cur = 0;
-
-	set_adapter_info(hostdata);
-
-	/* And re-open it again */
-	rc = plpar_hcall_norets(H_REG_CRQ,
-				vdev->unit_address,
-				queue->msg_token, PAGE_SIZE);
-	if (rc == 2) {
-		/* Adapter is good, but other end is not ready */
-		dev_warn(hostdata->dev, "Partner adapter not ready\n");
-	} else if (rc != 0) {
-		dev_warn(hostdata->dev, "couldn't register crq--rc 0x%x\n", rc);
-	}
-	return rc;
-}
-
-/**
- * initialize_crq_queue: - Initializes and registers CRQ with hypervisor
- * @queue:	crq_queue to initialize and register
- * @hostdata:	ibmvscsi_host_data of host
- *
- * Allocates a page for messages, maps it for dma, and registers
- * the crq with the hypervisor.
- * Returns zero on success.
- */
-static int rpavscsi_init_crq_queue(struct crq_queue *queue,
-				   struct ibmvscsi_host_data *hostdata,
-				   int max_requests)
-{
-	int rc;
-	int retrc;
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-
-	queue->msgs = (struct viosrp_crq *)get_zeroed_page(GFP_KERNEL);
-
-	if (!queue->msgs)
-		goto malloc_failed;
-	queue->size = PAGE_SIZE / sizeof(*queue->msgs);
-
-	queue->msg_token = dma_map_single(hostdata->dev, queue->msgs,
-					  queue->size * sizeof(*queue->msgs),
-					  DMA_BIDIRECTIONAL);
-
-	if (dma_mapping_error(hostdata->dev, queue->msg_token))
-		goto map_failed;
-
-	gather_partition_info();
-	set_adapter_info(hostdata);
-
-	retrc = rc = plpar_hcall_norets(H_REG_CRQ,
-				vdev->unit_address,
-				queue->msg_token, PAGE_SIZE);
-	if (rc == H_RESOURCE)
-		/* maybe kexecing and resource is busy. try a reset */
-		rc = rpavscsi_reset_crq_queue(queue,
-					      hostdata);
-
-	if (rc == 2) {
-		/* Adapter is good, but other end is not ready */
-		dev_warn(hostdata->dev, "Partner adapter not ready\n");
-		retrc = 0;
-	} else if (rc != 0) {
-		dev_warn(hostdata->dev, "Error %d opening adapter\n", rc);
-		goto reg_crq_failed;
-	}
-
-	queue->cur = 0;
-	spin_lock_init(&queue->lock);
-
-	tasklet_init(&hostdata->srp_task, (void *)rpavscsi_task,
-		     (unsigned long)hostdata);
-
-	if (request_irq(vdev->irq,
-			rpavscsi_handle_event,
-			0, "ibmvscsi", (void *)hostdata) != 0) {
-		dev_err(hostdata->dev, "couldn't register irq 0x%x\n",
-			vdev->irq);
-		goto req_irq_failed;
-	}
-
-	rc = vio_enable_interrupts(vdev);
-	if (rc != 0) {
-		dev_err(hostdata->dev, "Error %d enabling interrupts!!!\n", rc);
-		goto req_irq_failed;
-	}
-
-	return retrc;
-
-      req_irq_failed:
-	tasklet_kill(&hostdata->srp_task);
-	rc = 0;
-	do {
-		if (rc)
-			msleep(100);
-		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);
-	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
-      reg_crq_failed:
-	dma_unmap_single(hostdata->dev,
-			 queue->msg_token,
-			 queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
-      map_failed:
-	free_page((unsigned long)queue->msgs);
-      malloc_failed:
-	return -1;
-}
-
-/**
- * reenable_crq_queue: - reenables a crq after
- * @queue:	crq_queue to initialize and register
- * @hostdata:	ibmvscsi_host_data of host
- *
- */
-static int rpavscsi_reenable_crq_queue(struct crq_queue *queue,
-				       struct ibmvscsi_host_data *hostdata)
-{
-	int rc = 0;
-	struct vio_dev *vdev = to_vio_dev(hostdata->dev);
-
-	/* Re-enable the CRQ */
-	do {
-		if (rc)
-			msleep(100);
-		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);
-	} while ((rc == H_IN_PROGRESS) || (rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
-
-	if (rc)
-		dev_err(hostdata->dev, "Error %d enabling adapter\n", rc);
-	return rc;
-}
-
-/**
- * rpavscsi_resume: - resume after suspend
- * @hostdata:	ibmvscsi_host_data of host
- *
- */
-static int rpavscsi_resume(struct ibmvscsi_host_data *hostdata)
-{
-	vio_disable_interrupts(to_vio_dev(hostdata->dev));
-	tasklet_schedule(&hostdata->srp_task);
-	return 0;
-}
-
-struct ibmvscsi_ops rpavscsi_ops = {
-	.init_crq_queue = rpavscsi_init_crq_queue,
-	.release_crq_queue = rpavscsi_release_crq_queue,
-	.reset_crq_queue = rpavscsi_reset_crq_queue,
-	.reenable_crq_queue = rpavscsi_reenable_crq_queue,
-	.send_crq = rpavscsi_send_crq,
-	.resume = rpavscsi_resume,
-};
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 45e192a5100..e3f29f61cbc 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -566,6 +566,23 @@ static void ipr_trc_hook(struct ipr_cmnd *ipr_cmd,
 #endif
 
 /**
+ * ipr_lock_and_done - Acquire lock and complete command
+ * @ipr_cmd:	ipr command struct
+ *
+ * Return value:
+ *	none
+ **/
+static void ipr_lock_and_done(struct ipr_cmnd *ipr_cmd)
+{
+	unsigned long lock_flags;
+	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
+
+	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
+	ipr_cmd->done(ipr_cmd);
+	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+}
+
+/**
  * ipr_reinit_ipr_cmnd - Re-initialize an IPR Cmnd block for reuse
  * @ipr_cmd:	ipr command struct
  *
@@ -611,34 +628,50 @@ static void ipr_reinit_ipr_cmnd(struct ipr_cmnd *ipr_cmd)
  * Return value:
  * 	none
  **/
-static void ipr_init_ipr_cmnd(struct ipr_cmnd *ipr_cmd)
+static void ipr_init_ipr_cmnd(struct ipr_cmnd *ipr_cmd,
+			      void (*fast_done) (struct ipr_cmnd *))
 {
 	ipr_reinit_ipr_cmnd(ipr_cmd);
 	ipr_cmd->u.scratch = 0;
 	ipr_cmd->sibling = NULL;
+	ipr_cmd->fast_done = fast_done;
 	init_timer(&ipr_cmd->timer);
 }
 
 /**
- * ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block
+ * __ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block
  * @ioa_cfg:	ioa config struct
  *
  * Return value:
  * 	pointer to ipr command struct
  **/
 static
-struct ipr_cmnd *ipr_get_free_ipr_cmnd(struct ipr_ioa_cfg *ioa_cfg)
+struct ipr_cmnd *__ipr_get_free_ipr_cmnd(struct ipr_ioa_cfg *ioa_cfg)
 {
 	struct ipr_cmnd *ipr_cmd;
 
 	ipr_cmd = list_entry(ioa_cfg->free_q.next, struct ipr_cmnd, queue);
 	list_del(&ipr_cmd->queue);
-	ipr_init_ipr_cmnd(ipr_cmd);
 
 	return ipr_cmd;
 }
 
 /**
+ * ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block and initialize it
+ * @ioa_cfg:	ioa config struct
+ *
+ * Return value:
+ *	pointer to ipr command struct
+ **/
+static
+struct ipr_cmnd *ipr_get_free_ipr_cmnd(struct ipr_ioa_cfg *ioa_cfg)
+{
+	struct ipr_cmnd *ipr_cmd = __ipr_get_free_ipr_cmnd(ioa_cfg);
+	ipr_init_ipr_cmnd(ipr_cmd, ipr_lock_and_done);
+	return ipr_cmd;
+}
+
+/**
  * ipr_mask_and_clear_interrupts - Mask all and clear specified interrupts
  * @ioa_cfg:	ioa config struct
  * @clr_ints:     interrupts to clear
@@ -5116,8 +5149,9 @@ static irqreturn_t ipr_isr(int irq, void *devp)
 	u16 cmd_index;
 	int num_hrrq = 0;
 	int irq_none = 0;
-	struct ipr_cmnd *ipr_cmd;
+	struct ipr_cmnd *ipr_cmd, *temp;
 	irqreturn_t rc = IRQ_NONE;
+	LIST_HEAD(doneq);
 
 	spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
 
@@ -5138,8 +5172,8 @@ static irqreturn_t ipr_isr(int irq, void *devp)
 
 			if (unlikely(cmd_index >= IPR_NUM_CMD_BLKS)) {
 				ipr_isr_eh(ioa_cfg, "Invalid response handle from IOA");
-				spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
-				return IRQ_HANDLED;
+				rc = IRQ_HANDLED;
+				goto unlock_out;
 			}
 
 			ipr_cmd = ioa_cfg->ipr_cmnd_list[cmd_index];
@@ -5148,9 +5182,7 @@ static irqreturn_t ipr_isr(int irq, void *devp)
 
 			ipr_trc_hook(ipr_cmd, IPR_TRACE_FINISH, ioasc);
 
-			list_del(&ipr_cmd->queue);
-			del_timer(&ipr_cmd->timer);
-			ipr_cmd->done(ipr_cmd);
+			list_move_tail(&ipr_cmd->queue, &doneq);
 
 			rc = IRQ_HANDLED;
 
@@ -5180,8 +5212,8 @@ static irqreturn_t ipr_isr(int irq, void *devp)
 		} else if (num_hrrq == IPR_MAX_HRRQ_RETRIES &&
 			   int_reg & IPR_PCII_HRRQ_UPDATED) {
 			ipr_isr_eh(ioa_cfg, "Error clearing HRRQ");
-			spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
-			return IRQ_HANDLED;
+			rc = IRQ_HANDLED;
+			goto unlock_out;
 		} else
 			break;
 	}
@@ -5189,7 +5221,14 @@ static irqreturn_t ipr_isr(int irq, void *devp)
 	if (unlikely(rc == IRQ_NONE))
 		rc = ipr_handle_other_interrupt(ioa_cfg, int_reg);
 
+unlock_out:
 	spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) {
+		list_del(&ipr_cmd->queue);
+		del_timer(&ipr_cmd->timer);
+		ipr_cmd->fast_done(ipr_cmd);
+	}
+
 	return rc;
 }
 
@@ -5770,21 +5809,28 @@ static void ipr_scsi_done(struct ipr_cmnd *ipr_cmd)
 	struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg;
 	struct scsi_cmnd *scsi_cmd = ipr_cmd->scsi_cmd;
 	u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc);
+	unsigned long lock_flags;
 
 	scsi_set_resid(scsi_cmd, be32_to_cpu(ipr_cmd->s.ioasa.hdr.residual_data_len));
 
 	if (likely(IPR_IOASC_SENSE_KEY(ioasc) == 0)) {
-		scsi_dma_unmap(ipr_cmd->scsi_cmd);
+		scsi_dma_unmap(scsi_cmd);
+
+		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
 		list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q);
 		scsi_cmd->scsi_done(scsi_cmd);
-	} else
+		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	} else {
+		spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
 		ipr_erp_start(ioa_cfg, ipr_cmd);
+		spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags);
+	}
 }
 
 /**
  * ipr_queuecommand - Queue a mid-layer request
+ * @shost:		scsi host struct
  * @scsi_cmd:	scsi command struct
- * @done:		done function
  *
  * This function queues a request generated by the mid-layer.
  *
@@ -5793,61 +5839,61 @@ static void ipr_scsi_done(struct ipr_cmnd *ipr_cmd)
  *	SCSI_MLQUEUE_DEVICE_BUSY if device is busy
  *	SCSI_MLQUEUE_HOST_BUSY if host is busy
  **/
-static int ipr_queuecommand_lck(struct scsi_cmnd *scsi_cmd,
-			    void (*done) (struct scsi_cmnd *))
+static int ipr_queuecommand(struct Scsi_Host *shost,
+			    struct scsi_cmnd *scsi_cmd)
 {
 	struct ipr_ioa_cfg *ioa_cfg;
 	struct ipr_resource_entry *res;
 	struct ipr_ioarcb *ioarcb;
 	struct ipr_cmnd *ipr_cmd;
-	int rc = 0;
+	unsigned long lock_flags;
+	int rc;
 
-	scsi_cmd->scsi_done = done;
-	ioa_cfg = (struct ipr_ioa_cfg *)scsi_cmd->device->host->hostdata;
-	res = scsi_cmd->device->hostdata;
+	ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata;
+
+	spin_lock_irqsave(shost->host_lock, lock_flags);
 	scsi_cmd->result = (DID_OK << 16);
+	res = scsi_cmd->device->hostdata;
 
 	/*
 	 * We are currently blocking all devices due to a host reset
 	 * We have told the host to stop giving us new requests, but
 	 * ERP ops don't count. FIXME
 	 */
-	if (unlikely(!ioa_cfg->allow_cmds && !ioa_cfg->ioa_is_dead))
+	if (unlikely(!ioa_cfg->allow_cmds && !ioa_cfg->ioa_is_dead)) {
+		spin_unlock_irqrestore(shost->host_lock, lock_flags);
 		return SCSI_MLQUEUE_HOST_BUSY;
+	}
 
 	/*
 	 * FIXME - Create scsi_set_host_offline interface
 	 *  and the ioa_is_dead check can be removed
 	 */
 	if (unlikely(ioa_cfg->ioa_is_dead || !res)) {
-		memset(scsi_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
-		scsi_cmd->result = (DID_NO_CONNECT << 16);
-		scsi_cmd->scsi_done(scsi_cmd);
-		return 0;
+		spin_unlock_irqrestore(shost->host_lock, lock_flags);
+		goto err_nodev;
+	}
+
+	if (ipr_is_gata(res) && res->sata_port) {
+		rc = ata_sas_queuecmd(scsi_cmd, res->sata_port->ap);
+		spin_unlock_irqrestore(shost->host_lock, lock_flags);
+		return rc;
 	}
 
-	if (ipr_is_gata(res) && res->sata_port)
-		return ata_sas_queuecmd(scsi_cmd, res->sata_port->ap);
+	ipr_cmd = __ipr_get_free_ipr_cmnd(ioa_cfg);
+	spin_unlock_irqrestore(shost->host_lock, lock_flags);
 
-	ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg);
+	ipr_init_ipr_cmnd(ipr_cmd, ipr_scsi_done);
 	ioarcb = &ipr_cmd->ioarcb;
-	list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q);
 
 	memcpy(ioarcb->cmd_pkt.cdb, scsi_cmd->cmnd, scsi_cmd->cmd_len);
 	ipr_cmd->scsi_cmd = scsi_cmd;
-	ioarcb->res_handle = res->res_handle;
-	ipr_cmd->done = ipr_scsi_done;
-	ipr_trc_hook(ipr_cmd, IPR_TRACE_START, IPR_GET_RES_PHYS_LOC(res));
+	ipr_cmd->done = ipr_scsi_eh_done;
 
 	if (ipr_is_gscsi(res) || ipr_is_vset_device(res)) {
 		if (scsi_cmd->underflow == 0)
 			ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_NO_ULEN_CHK;
 
-		if (res->needs_sync_complete) {
-			ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_SYNC_COMPLETE;
-			res->needs_sync_complete = 0;
-		}
-
 		ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_NO_LINK_DESC;
 		if (ipr_is_gscsi(res))
 			ioarcb->cmd_pkt.flags_lo |= IPR_FLAGS_LO_DELAY_AFTER_RST;
@@ -5859,23 +5905,46 @@ static int ipr_queuecommand_lck(struct scsi_cmnd *scsi_cmd,
 	    (!ipr_is_gscsi(res) || scsi_cmd->cmnd[0] == IPR_QUERY_RSRC_STATE))
 		ioarcb->cmd_pkt.request_type = IPR_RQTYPE_IOACMD;
 
-	if (likely(rc == 0)) {
-		if (ioa_cfg->sis64)
-			rc = ipr_build_ioadl64(ioa_cfg, ipr_cmd);
-		else
-			rc = ipr_build_ioadl(ioa_cfg, ipr_cmd);
-	}
+	if (ioa_cfg->sis64)
+		rc = ipr_build_ioadl64(ioa_cfg, ipr_cmd);
+	else
+		rc = ipr_build_ioadl(ioa_cfg, ipr_cmd);
 
-	if (unlikely(rc != 0)) {
-		list_move_tail(&ipr_cmd->queue, &ioa_cfg->free_q);
+	spin_lock_irqsave(shost->host_lock, lock_flags);
+	if (unlikely(rc || (!ioa_cfg->allow_cmds && !ioa_cfg->ioa_is_dead))) {
+		list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q);
+		spin_unlock_irqrestore(shost->host_lock, lock_flags);
+		if (!rc)
+			scsi_dma_unmap(scsi_cmd);
 		return SCSI_MLQUEUE_HOST_BUSY;
 	}
 
+	if (unlikely(ioa_cfg->ioa_is_dead)) {
+		list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q);
+		spin_unlock_irqrestore(shost->host_lock, lock_flags);
+		scsi_dma_unmap(scsi_cmd);
+		goto err_nodev;
+	}
+
+	ioarcb->res_handle = res->res_handle;
+	if (res->needs_sync_complete) {
+		ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_SYNC_COMPLETE;
+		res->needs_sync_complete = 0;
+	}
+	list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q);
+	ipr_trc_hook(ipr_cmd, IPR_TRACE_START, IPR_GET_RES_PHYS_LOC(res));
 	ipr_send_command(ipr_cmd);
+	spin_unlock_irqrestore(shost->host_lock, lock_flags);
 	return 0;
-}
 
-static DEF_SCSI_QCMD(ipr_queuecommand)
+err_nodev:
+	spin_lock_irqsave(shost->host_lock, lock_flags);
+	memset(scsi_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
+	scsi_cmd->result = (DID_NO_CONNECT << 16);
+	scsi_cmd->scsi_done(scsi_cmd);
+	spin_unlock_irqrestore(shost->host_lock, lock_flags);
+	return 0;
+}
 
 /**
  * ipr_ioctl - IOCTL handler
@@ -8775,8 +8844,7 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev,
 
 	ioa_cfg = (struct ipr_ioa_cfg *)host->hostdata;
 	memset(ioa_cfg, 0, sizeof(struct ipr_ioa_cfg));
-	ata_host_init(&ioa_cfg->ata_host, &pdev->dev,
-		      sata_port_info.flags, &ipr_sata_ops);
+	ata_host_init(&ioa_cfg->ata_host, &pdev->dev, &ipr_sata_ops);
 
 	ioa_cfg->ipr_chip = ipr_get_chip_info(dev_id);
 
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index 153b8bd91d1..c8a137f83bb 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -38,8 +38,8 @@
 /*
  * Literals
  */
-#define IPR_DRIVER_VERSION "2.5.3"
-#define IPR_DRIVER_DATE "(March 10, 2012)"
+#define IPR_DRIVER_VERSION "2.5.4"
+#define IPR_DRIVER_DATE "(July 11, 2012)"
 
 /*
  * IPR_MAX_CMD_PER_LUN: This defines the maximum number of outstanding
@@ -1525,6 +1525,7 @@ struct ipr_cmnd {
 	struct ata_queued_cmd *qc;
 	struct completion completion;
 	struct timer_list timer;
+	void (*fast_done) (struct ipr_cmnd *);
 	void (*done) (struct ipr_cmnd *);
 	int (*job_step) (struct ipr_cmnd *);
 	int (*job_step_failed) (struct ipr_cmnd *);
diff --git a/drivers/scsi/isci/host.c b/drivers/scsi/isci/host.c
index b334fdc1726..609dafd661d 100644
--- a/drivers/scsi/isci/host.c
+++ b/drivers/scsi/isci/host.c
@@ -1044,7 +1044,7 @@ static enum sci_status sci_controller_start(struct isci_host *ihost,
 	return SCI_SUCCESS;
 }
 
-void isci_host_scan_start(struct Scsi_Host *shost)
+void isci_host_start(struct Scsi_Host *shost)
 {
 	struct isci_host *ihost = SHOST_TO_SAS_HA(shost)->lldd_ha;
 	unsigned long tmo = sci_controller_get_suggested_start_timeout(ihost);
@@ -1079,7 +1079,6 @@ static void sci_controller_completion_handler(struct isci_host *ihost)
 
 void ireq_done(struct isci_host *ihost, struct isci_request *ireq, struct sas_task *task)
 {
-	task->lldd_task = NULL;
 	if (!test_bit(IREQ_ABORT_PATH_ACTIVE, &ireq->flags) &&
 	    !(task->task_state_flags & SAS_TASK_STATE_ABORTED)) {
 		if (test_bit(IREQ_COMPLETE_IN_TARGET, &ireq->flags)) {
@@ -1087,16 +1086,19 @@ void ireq_done(struct isci_host *ihost, struct isci_request *ireq, struct sas_ta
 			dev_dbg(&ihost->pdev->dev,
 				"%s: Normal - ireq/task = %p/%p\n",
 				__func__, ireq, task);
-
+			task->lldd_task = NULL;
 			task->task_done(task);
 		} else {
 			dev_dbg(&ihost->pdev->dev,
 				"%s: Error - ireq/task = %p/%p\n",
 				__func__, ireq, task);
-
+			if (sas_protocol_ata(task->task_proto))
+				task->lldd_task = NULL;
 			sas_task_abort(task);
 		}
-	}
+	} else
+		task->lldd_task = NULL;
+
 	if (test_and_clear_bit(IREQ_ABORT_PATH_ACTIVE, &ireq->flags))
 		wake_up_all(&ihost->eventq);
 
@@ -1120,10 +1122,16 @@ void isci_host_completion_routine(unsigned long data)
 	sci_controller_completion_handler(ihost);
 	spin_unlock_irq(&ihost->scic_lock);
 
-	/* the coalesence timeout doubles at each encoding step, so
+	/*
+	 * we subtract SCI_MAX_PORTS to account for the number of dummy TCs
+	 * issued for hardware issue workaround
+	 */
+	active = isci_tci_active(ihost) - SCI_MAX_PORTS;
+
+	/*
+	 * the coalesence timeout doubles at each encoding step, so
 	 * update it based on the ilog2 value of the outstanding requests
 	 */
-	active = isci_tci_active(ihost);
 	writel(SMU_ICC_GEN_VAL(NUMBER, active) |
 	       SMU_ICC_GEN_VAL(TIMER, ISCI_COALESCE_BASE + ilog2(active)),
 	       &ihost->smu_registers->interrupt_coalesce_control);
@@ -1973,7 +1981,7 @@ static void sci_controller_afe_initialization(struct isci_host *ihost)
 	}
 
 	for (phy_id = 0; phy_id < SCI_MAX_PHYS; phy_id++) {
-		struct scu_afe_transceiver *xcvr = &afe->scu_afe_xcvr[phy_id];
+		struct scu_afe_transceiver __iomem *xcvr = &afe->scu_afe_xcvr[phy_id];
 		const struct sci_phy_oem_params *oem_phy = &oem->phys[phy_id];
 		int cable_length_long =
 			is_long_cable(phy_id, cable_selection_mask);
diff --git a/drivers/scsi/isci/host.h b/drivers/scsi/isci/host.h
index 9ab58e0540e..4911310a38f 100644
--- a/drivers/scsi/isci/host.h
+++ b/drivers/scsi/isci/host.h
@@ -473,7 +473,7 @@ void sci_controller_remote_device_stopped(struct isci_host *ihost,
 
 enum sci_status sci_controller_continue_io(struct isci_request *ireq);
 int isci_host_scan_finished(struct Scsi_Host *, unsigned long);
-void isci_host_scan_start(struct Scsi_Host *);
+void isci_host_start(struct Scsi_Host *);
 u16 isci_alloc_tag(struct isci_host *ihost);
 enum sci_status isci_free_tag(struct isci_host *ihost, u16 io_tag);
 void isci_tci_free(struct isci_host *ihost, u16 tci);
diff --git a/drivers/scsi/isci/init.c b/drivers/scsi/isci/init.c
index 9be45a2b223..b74050b95d6 100644
--- a/drivers/scsi/isci/init.c
+++ b/drivers/scsi/isci/init.c
@@ -156,7 +156,7 @@ static struct scsi_host_template isci_sht = {
 	.target_alloc			= sas_target_alloc,
 	.slave_configure		= sas_slave_configure,
 	.scan_finished			= isci_host_scan_finished,
-	.scan_start			= isci_host_scan_start,
+	.scan_start			= isci_host_start,
 	.change_queue_depth		= sas_change_queue_depth,
 	.change_queue_type		= sas_change_queue_type,
 	.bios_param			= sas_bios_param,
@@ -644,7 +644,6 @@ static int __devinit isci_pci_probe(struct pci_dev *pdev, const struct pci_devic
 						orom->hdr.version)) {
 			dev_warn(&pdev->dev,
 				 "[%d]: invalid oem parameters detected, falling back to firmware\n", i);
-			devm_kfree(&pdev->dev, orom);
 			orom = NULL;
 			break;
 		}
@@ -722,11 +721,67 @@ static void __devexit isci_pci_remove(struct pci_dev *pdev)
 	}
 }
 
+#ifdef CONFIG_PM
+static int isci_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct isci_host *ihost;
+	int i;
+
+	for_each_isci_host(i, ihost, pdev) {
+		sas_suspend_ha(&ihost->sas_ha);
+		isci_host_deinit(ihost);
+	}
+
+	pci_save_state(pdev);
+	pci_disable_device(pdev);
+	pci_set_power_state(pdev, PCI_D3hot);
+
+	return 0;
+}
+
+static int isci_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct isci_host *ihost;
+	int rc, i;
+
+	pci_set_power_state(pdev, PCI_D0);
+	pci_restore_state(pdev);
+
+	rc = pcim_enable_device(pdev);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"enabling device failure after resume(%d)\n", rc);
+		return rc;
+	}
+
+	pci_set_master(pdev);
+
+	for_each_isci_host(i, ihost, pdev) {
+		sas_prep_resume_ha(&ihost->sas_ha);
+
+		isci_host_init(ihost);
+		isci_host_start(ihost->sas_ha.core.shost);
+		wait_for_start(ihost);
+
+		sas_resume_ha(&ihost->sas_ha);
+	}
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(isci_pm_ops, isci_suspend, isci_resume);
+#endif
+
 static struct pci_driver isci_pci_driver = {
 	.name		= DRV_NAME,
 	.id_table	= isci_id_table,
 	.probe		= isci_pci_probe,
 	.remove		= __devexit_p(isci_pci_remove),
+#ifdef CONFIG_PM
+	.driver.pm      = &isci_pm_ops,
+#endif
 };
 
 static __init int isci_init(void)
diff --git a/drivers/scsi/isci/phy.c b/drivers/scsi/isci/phy.c
index 18f43d4c30b..cb87b2ef7c9 100644
--- a/drivers/scsi/isci/phy.c
+++ b/drivers/scsi/isci/phy.c
@@ -169,7 +169,7 @@ sci_phy_link_layer_initialization(struct isci_phy *iphy,
 	phy_cap.gen1_no_ssc = 1;
 	if (ihost->oem_parameters.controller.do_enable_ssc) {
 		struct scu_afe_registers __iomem *afe = &ihost->scu_registers->afe;
-		struct scu_afe_transceiver *xcvr = &afe->scu_afe_xcvr[phy_idx];
+		struct scu_afe_transceiver __iomem *xcvr = &afe->scu_afe_xcvr[phy_idx];
 		struct isci_pci_info *pci_info = to_pci_info(ihost->pdev);
 		bool en_sas = false;
 		bool en_sata = false;
@@ -1205,6 +1205,7 @@ static void scu_link_layer_start_oob(struct isci_phy *iphy)
 	/** Reset OOB sequence - start */
 	val = readl(&ll->phy_configuration);
 	val &= ~(SCU_SAS_PCFG_GEN_BIT(OOB_RESET) |
+		 SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE) |
 		 SCU_SAS_PCFG_GEN_BIT(HARD_RESET));
 	writel(val, &ll->phy_configuration);
 	readl(&ll->phy_configuration); /* flush */
@@ -1236,6 +1237,7 @@ static void scu_link_layer_tx_hard_reset(
 	 * to the starting state. */
 	phy_configuration_value =
 		readl(&iphy->link_layer_registers->phy_configuration);
+	phy_configuration_value &= ~(SCU_SAS_PCFG_GEN_BIT(OOB_ENABLE));
 	phy_configuration_value |=
 		(SCU_SAS_PCFG_GEN_BIT(HARD_RESET) |
 		 SCU_SAS_PCFG_GEN_BIT(OOB_RESET));
diff --git a/drivers/scsi/isci/probe_roms.c b/drivers/scsi/isci/probe_roms.c
index 4d95654c3fd..8ac646e5edd 100644
--- a/drivers/scsi/isci/probe_roms.c
+++ b/drivers/scsi/isci/probe_roms.c
@@ -104,7 +104,6 @@ struct isci_orom *isci_request_oprom(struct pci_dev *pdev)
 
 	if (i >= len) {
 		dev_err(&pdev->dev, "oprom parse error\n");
-		devm_kfree(&pdev->dev, rom);
 		rom = NULL;
 	}
 	pci_unmap_biosrom(oprom);
diff --git a/drivers/scsi/isci/remote_node_context.h b/drivers/scsi/isci/remote_node_context.h
index a703b9ce0c2..c7ee81d0112 100644
--- a/drivers/scsi/isci/remote_node_context.h
+++ b/drivers/scsi/isci/remote_node_context.h
@@ -212,7 +212,7 @@ enum sci_status sci_remote_node_context_destruct(struct sci_remote_node_context
 						      scics_sds_remote_node_context_callback callback,
 						      void *callback_parameter);
 enum sci_status sci_remote_node_context_suspend(struct sci_remote_node_context *sci_rnc,
-						     u32 suspend_type,
+						     enum sci_remote_node_suspension_reasons reason,
 						     u32 suspension_code);
 enum sci_status sci_remote_node_context_resume(struct sci_remote_node_context *sci_rnc,
 						    scics_sds_remote_node_context_callback cb_fn,
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 922086105b4..1b91ca0dc1e 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -55,7 +55,7 @@ static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
 static struct scsi_host_template iscsi_sw_tcp_sht;
 static struct iscsi_transport iscsi_sw_tcp_transport;
 
-static unsigned int iscsi_max_lun = 512;
+static unsigned int iscsi_max_lun = ~0;
 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
 
 static int iscsi_sw_tcp_dbg;
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index a59fcdc8fd6..bdb81cda840 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -580,10 +580,7 @@ int sas_ata_init(struct domain_device *found_dev)
 	struct ata_port *ap;
 	int rc;
 
-	ata_host_init(&found_dev->sata_dev.ata_host,
-		      ha->dev,
-		      sata_port_info.flags,
-		      &sas_sata_ops);
+	ata_host_init(&found_dev->sata_dev.ata_host, ha->dev, &sas_sata_ops);
 	ap = ata_sas_port_alloc(&found_dev->sata_dev.ata_host,
 				&sata_port_info,
 				shost);
@@ -700,6 +697,92 @@ void sas_probe_sata(struct asd_sas_port *port)
 		if (ata_dev_disabled(sas_to_ata_dev(dev)))
 			sas_fail_probe(dev, __func__, -ENODEV);
 	}
+
+}
+
+static bool sas_ata_flush_pm_eh(struct asd_sas_port *port, const char *func)
+{
+	struct domain_device *dev, *n;
+	bool retry = false;
+
+	list_for_each_entry_safe(dev, n, &port->dev_list, dev_list_node) {
+		int rc;
+
+		if (!dev_is_sata(dev))
+			continue;
+
+		sas_ata_wait_eh(dev);
+		rc = dev->sata_dev.pm_result;
+		if (rc == -EAGAIN)
+			retry = true;
+		else if (rc) {
+			/* since we don't have a
+			 * ->port_{suspend|resume} routine in our
+			 *  ata_port ops, and no entanglements with
+			 *  acpi, suspend should just be mechanical trip
+			 *  through eh, catch cases where these
+			 *  assumptions are invalidated
+			 */
+			WARN_ONCE(1, "failed %s %s error: %d\n", func,
+				 dev_name(&dev->rphy->dev), rc);
+		}
+
+		/* if libata failed to power manage the device, tear it down */
+		if (ata_dev_disabled(sas_to_ata_dev(dev)))
+			sas_fail_probe(dev, func, -ENODEV);
+	}
+
+	return retry;
+}
+
+void sas_suspend_sata(struct asd_sas_port *port)
+{
+	struct domain_device *dev;
+
+ retry:
+	mutex_lock(&port->ha->disco_mutex);
+	list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+		struct sata_device *sata;
+
+		if (!dev_is_sata(dev))
+			continue;
+
+		sata = &dev->sata_dev;
+		if (sata->ap->pm_mesg.event == PM_EVENT_SUSPEND)
+			continue;
+
+		sata->pm_result = -EIO;
+		ata_sas_port_async_suspend(sata->ap, &sata->pm_result);
+	}
+	mutex_unlock(&port->ha->disco_mutex);
+
+	if (sas_ata_flush_pm_eh(port, __func__))
+		goto retry;
+}
+
+void sas_resume_sata(struct asd_sas_port *port)
+{
+	struct domain_device *dev;
+
+ retry:
+	mutex_lock(&port->ha->disco_mutex);
+	list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+		struct sata_device *sata;
+
+		if (!dev_is_sata(dev))
+			continue;
+
+		sata = &dev->sata_dev;
+		if (sata->ap->pm_mesg.event == PM_EVENT_ON)
+			continue;
+
+		sata->pm_result = -EIO;
+		ata_sas_port_async_resume(sata->ap, &sata->pm_result);
+	}
+	mutex_unlock(&port->ha->disco_mutex);
+
+	if (sas_ata_flush_pm_eh(port, __func__))
+		goto retry;
 }
 
 /**
diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
index 3e9dc1a8435..a0c3003e0c7 100644
--- a/drivers/scsi/libsas/sas_discover.c
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -24,6 +24,7 @@
 
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/async.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_eh.h>
 #include "sas_internal.h"
@@ -180,16 +181,18 @@ int sas_notify_lldd_dev_found(struct domain_device *dev)
 	struct Scsi_Host *shost = sas_ha->core.shost;
 	struct sas_internal *i = to_sas_internal(shost->transportt);
 
-	if (i->dft->lldd_dev_found) {
-		res = i->dft->lldd_dev_found(dev);
-		if (res) {
-			printk("sas: driver on pcidev %s cannot handle "
-			       "device %llx, error:%d\n",
-			       dev_name(sas_ha->dev),
-			       SAS_ADDR(dev->sas_addr), res);
-		}
-		kref_get(&dev->kref);
+	if (!i->dft->lldd_dev_found)
+		return 0;
+
+	res = i->dft->lldd_dev_found(dev);
+	if (res) {
+		printk("sas: driver on pcidev %s cannot handle "
+		       "device %llx, error:%d\n",
+		       dev_name(sas_ha->dev),
+		       SAS_ADDR(dev->sas_addr), res);
 	}
+	set_bit(SAS_DEV_FOUND, &dev->state);
+	kref_get(&dev->kref);
 	return res;
 }
 
@@ -200,7 +203,10 @@ void sas_notify_lldd_dev_gone(struct domain_device *dev)
 	struct Scsi_Host *shost = sas_ha->core.shost;
 	struct sas_internal *i = to_sas_internal(shost->transportt);
 
-	if (i->dft->lldd_dev_gone) {
+	if (!i->dft->lldd_dev_gone)
+		return;
+
+	if (test_and_clear_bit(SAS_DEV_FOUND, &dev->state)) {
 		i->dft->lldd_dev_gone(dev);
 		sas_put_device(dev);
 	}
@@ -234,6 +240,47 @@ static void sas_probe_devices(struct work_struct *work)
 	}
 }
 
+static void sas_suspend_devices(struct work_struct *work)
+{
+	struct asd_sas_phy *phy;
+	struct domain_device *dev;
+	struct sas_discovery_event *ev = to_sas_discovery_event(work);
+	struct asd_sas_port *port = ev->port;
+	struct Scsi_Host *shost = port->ha->core.shost;
+	struct sas_internal *si = to_sas_internal(shost->transportt);
+
+	clear_bit(DISCE_SUSPEND, &port->disc.pending);
+
+	sas_suspend_sata(port);
+
+	/* lldd is free to forget the domain_device across the
+	 * suspension, we force the issue here to keep the reference
+	 * counts aligned
+	 */
+	list_for_each_entry(dev, &port->dev_list, dev_list_node)
+		sas_notify_lldd_dev_gone(dev);
+
+	/* we are suspending, so we know events are disabled and
+	 * phy_list is not being mutated
+	 */
+	list_for_each_entry(phy, &port->phy_list, port_phy_el) {
+		if (si->dft->lldd_port_formed)
+			si->dft->lldd_port_deformed(phy);
+		phy->suspended = 1;
+		port->suspended = 1;
+	}
+}
+
+static void sas_resume_devices(struct work_struct *work)
+{
+	struct sas_discovery_event *ev = to_sas_discovery_event(work);
+	struct asd_sas_port *port = ev->port;
+
+	clear_bit(DISCE_RESUME, &port->disc.pending);
+
+	sas_resume_sata(port);
+}
+
 /**
  * sas_discover_end_dev -- discover an end device (SSP, etc)
  * @end: pointer to domain device of interest
@@ -530,6 +577,8 @@ void sas_init_disc(struct sas_discovery *disc, struct asd_sas_port *port)
 		[DISCE_DISCOVER_DOMAIN] = sas_discover_domain,
 		[DISCE_REVALIDATE_DOMAIN] = sas_revalidate_domain,
 		[DISCE_PROBE] = sas_probe_devices,
+		[DISCE_SUSPEND] = sas_suspend_devices,
+		[DISCE_RESUME] = sas_resume_devices,
 		[DISCE_DESTRUCT] = sas_destruct_devices,
 	};
 
diff --git a/drivers/scsi/libsas/sas_dump.c b/drivers/scsi/libsas/sas_dump.c
index fc460933575..cd6f99c1ae7 100644
--- a/drivers/scsi/libsas/sas_dump.c
+++ b/drivers/scsi/libsas/sas_dump.c
@@ -41,6 +41,7 @@ static const char *sas_phye_str[] = {
 	[1] = "PHYE_OOB_DONE",
 	[2] = "PHYE_OOB_ERROR",
 	[3] = "PHYE_SPINUP_HOLD",
+	[4] = "PHYE_RESUME_TIMEOUT",
 };
 
 void sas_dprint_porte(int phyid, enum port_event pe)
diff --git a/drivers/scsi/libsas/sas_event.c b/drivers/scsi/libsas/sas_event.c
index 789c4d8bb7a..aadbd5314c5 100644
--- a/drivers/scsi/libsas/sas_event.c
+++ b/drivers/scsi/libsas/sas_event.c
@@ -134,7 +134,7 @@ static void notify_port_event(struct asd_sas_phy *phy, enum port_event event)
 			&phy->port_events[event].work, ha);
 }
 
-static void notify_phy_event(struct asd_sas_phy *phy, enum phy_event event)
+void sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event)
 {
 	struct sas_ha_struct *ha = phy->ha;
 
@@ -159,7 +159,7 @@ int sas_init_events(struct sas_ha_struct *sas_ha)
 
 	sas_ha->notify_ha_event = notify_ha_event;
 	sas_ha->notify_port_event = notify_port_event;
-	sas_ha->notify_phy_event = notify_phy_event;
+	sas_ha->notify_phy_event = sas_notify_phy_event;
 
 	return 0;
 }
diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c
index 014297c0588..dbc8a793fd8 100644
--- a/drivers/scsi/libsas/sas_init.c
+++ b/drivers/scsi/libsas/sas_init.c
@@ -178,7 +178,7 @@ Undo_phys:
 	return error;
 }
 
-int sas_unregister_ha(struct sas_ha_struct *sas_ha)
+static void sas_disable_events(struct sas_ha_struct *sas_ha)
 {
 	/* Set the state to unregistered to avoid further unchained
 	 * events to be queued, and flush any in-progress drainers
@@ -189,7 +189,11 @@ int sas_unregister_ha(struct sas_ha_struct *sas_ha)
 	spin_unlock_irq(&sas_ha->lock);
 	__sas_drain_work(sas_ha);
 	mutex_unlock(&sas_ha->drain_mutex);
+}
 
+int sas_unregister_ha(struct sas_ha_struct *sas_ha)
+{
+	sas_disable_events(sas_ha);
 	sas_unregister_ports(sas_ha);
 
 	/* flush unregistration work */
@@ -381,6 +385,90 @@ int sas_set_phy_speed(struct sas_phy *phy,
 	return ret;
 }
 
+void sas_prep_resume_ha(struct sas_ha_struct *ha)
+{
+	int i;
+
+	set_bit(SAS_HA_REGISTERED, &ha->state);
+
+	/* clear out any stale link events/data from the suspension path */
+	for (i = 0; i < ha->num_phys; i++) {
+		struct asd_sas_phy *phy = ha->sas_phy[i];
+
+		memset(phy->attached_sas_addr, 0, SAS_ADDR_SIZE);
+		phy->port_events_pending = 0;
+		phy->phy_events_pending = 0;
+		phy->frame_rcvd_size = 0;
+	}
+}
+EXPORT_SYMBOL(sas_prep_resume_ha);
+
+static int phys_suspended(struct sas_ha_struct *ha)
+{
+	int i, rc = 0;
+
+	for (i = 0; i < ha->num_phys; i++) {
+		struct asd_sas_phy *phy = ha->sas_phy[i];
+
+		if (phy->suspended)
+			rc++;
+	}
+
+	return rc;
+}
+
+void sas_resume_ha(struct sas_ha_struct *ha)
+{
+	const unsigned long tmo = msecs_to_jiffies(25000);
+	int i;
+
+	/* deform ports on phys that did not resume
+	 * at this point we may be racing the phy coming back (as posted
+	 * by the lldd).  So we post the event and once we are in the
+	 * libsas context check that the phy remains suspended before
+	 * tearing it down.
+	 */
+	i = phys_suspended(ha);
+	if (i)
+		dev_info(ha->dev, "waiting up to 25 seconds for %d phy%s to resume\n",
+			 i, i > 1 ? "s" : "");
+	wait_event_timeout(ha->eh_wait_q, phys_suspended(ha) == 0, tmo);
+	for (i = 0; i < ha->num_phys; i++) {
+		struct asd_sas_phy *phy = ha->sas_phy[i];
+
+		if (phy->suspended) {
+			dev_warn(&phy->phy->dev, "resume timeout\n");
+			sas_notify_phy_event(phy, PHYE_RESUME_TIMEOUT);
+		}
+	}
+
+	/* all phys are back up or timed out, turn on i/o so we can
+	 * flush out disks that did not return
+	 */
+	scsi_unblock_requests(ha->core.shost);
+	sas_drain_work(ha);
+}
+EXPORT_SYMBOL(sas_resume_ha);
+
+void sas_suspend_ha(struct sas_ha_struct *ha)
+{
+	int i;
+
+	sas_disable_events(ha);
+	scsi_block_requests(ha->core.shost);
+	for (i = 0; i < ha->num_phys; i++) {
+		struct asd_sas_port *port = ha->sas_port[i];
+
+		sas_discover_event(port, DISCE_SUSPEND);
+	}
+
+	/* flush suspend events while unregistered */
+	mutex_lock(&ha->drain_mutex);
+	__sas_drain_work(ha);
+	mutex_unlock(&ha->drain_mutex);
+}
+EXPORT_SYMBOL(sas_suspend_ha);
+
 static void sas_phy_release(struct sas_phy *phy)
 {
 	kfree(phy->hostdata);
diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h
index 507e4cf12e5..1de67964e5a 100644
--- a/drivers/scsi/libsas/sas_internal.h
+++ b/drivers/scsi/libsas/sas_internal.h
@@ -89,6 +89,7 @@ int sas_smp_phy_control(struct domain_device *dev, int phy_id,
 			enum phy_func phy_func, struct sas_phy_linkrates *);
 int sas_smp_get_phy_events(struct sas_phy *phy);
 
+void sas_notify_phy_event(struct asd_sas_phy *phy, enum phy_event event);
 void sas_device_set_phy(struct domain_device *dev, struct sas_port *port);
 struct domain_device *sas_find_dev_by_rphy(struct sas_rphy *rphy);
 struct domain_device *sas_ex_to_ata(struct domain_device *ex_dev, int phy_id);
diff --git a/drivers/scsi/libsas/sas_phy.c b/drivers/scsi/libsas/sas_phy.c
index 521422e857a..cdee446c29e 100644
--- a/drivers/scsi/libsas/sas_phy.c
+++ b/drivers/scsi/libsas/sas_phy.c
@@ -94,6 +94,25 @@ static void sas_phye_spinup_hold(struct work_struct *work)
 	i->dft->lldd_control_phy(phy, PHY_FUNC_RELEASE_SPINUP_HOLD, NULL);
 }
 
+static void sas_phye_resume_timeout(struct work_struct *work)
+{
+	struct asd_sas_event *ev = to_asd_sas_event(work);
+	struct asd_sas_phy *phy = ev->phy;
+
+	clear_bit(PHYE_RESUME_TIMEOUT, &phy->phy_events_pending);
+
+	/* phew, lldd got the phy back in the nick of time */
+	if (!phy->suspended) {
+		dev_info(&phy->phy->dev, "resume timeout cancelled\n");
+		return;
+	}
+
+	phy->error = 0;
+	phy->suspended = 0;
+	sas_deform_port(phy, 1);
+}
+
+
 /* ---------- Phy class registration ---------- */
 
 int sas_register_phys(struct sas_ha_struct *sas_ha)
@@ -105,6 +124,8 @@ int sas_register_phys(struct sas_ha_struct *sas_ha)
 		[PHYE_OOB_DONE] = sas_phye_oob_done,
 		[PHYE_OOB_ERROR] = sas_phye_oob_error,
 		[PHYE_SPINUP_HOLD] = sas_phye_spinup_hold,
+		[PHYE_RESUME_TIMEOUT] = sas_phye_resume_timeout,
+
 	};
 
 	static const work_func_t sas_port_event_fns[PORT_NUM_EVENTS] = {
diff --git a/drivers/scsi/libsas/sas_port.c b/drivers/scsi/libsas/sas_port.c
index e884a8c58a0..1398b714c01 100644
--- a/drivers/scsi/libsas/sas_port.c
+++ b/drivers/scsi/libsas/sas_port.c
@@ -39,6 +39,49 @@ static bool phy_is_wideport_member(struct asd_sas_port *port, struct asd_sas_phy
 	return true;
 }
 
+static void sas_resume_port(struct asd_sas_phy *phy)
+{
+	struct domain_device *dev;
+	struct asd_sas_port *port = phy->port;
+	struct sas_ha_struct *sas_ha = phy->ha;
+	struct sas_internal *si = to_sas_internal(sas_ha->core.shost->transportt);
+
+	if (si->dft->lldd_port_formed)
+		si->dft->lldd_port_formed(phy);
+
+	if (port->suspended)
+		port->suspended = 0;
+	else {
+		/* we only need to handle "link returned" actions once */
+		return;
+	}
+
+	/* if the port came back:
+	 * 1/ presume every device came back
+	 * 2/ force the next revalidation to check all expander phys
+	 */
+	list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+		int i, rc;
+
+		rc = sas_notify_lldd_dev_found(dev);
+		if (rc) {
+			sas_unregister_dev(port, dev);
+			continue;
+		}
+
+		if (dev->dev_type == EDGE_DEV || dev->dev_type == FANOUT_DEV) {
+			dev->ex_dev.ex_change_count = -1;
+			for (i = 0; i < dev->ex_dev.num_phys; i++) {
+				struct ex_phy *phy = &dev->ex_dev.ex_phy[i];
+
+				phy->phy_change_count = -1;
+			}
+		}
+	}
+
+	sas_discover_event(port, DISCE_RESUME);
+}
+
 /**
  * sas_form_port -- add this phy to a port
  * @phy: the phy of interest
@@ -58,7 +101,14 @@ static void sas_form_port(struct asd_sas_phy *phy)
 	if (port) {
 		if (!phy_is_wideport_member(port, phy))
 			sas_deform_port(phy, 0);
-		else {
+		else if (phy->suspended) {
+			phy->suspended = 0;
+			sas_resume_port(phy);
+
+			/* phy came back, try to cancel the timeout */
+			wake_up(&sas_ha->eh_wait_q);
+			return;
+		} else {
 			SAS_DPRINTK("%s: phy%d belongs to port%d already(%d)!\n",
 				    __func__, phy->id, phy->port->id,
 				    phy->port->num_phys);
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index a65c05a8d48..a184c2443a6 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -73,6 +73,8 @@ struct lpfc_sli2_slim;
 #define LPFC_HB_MBOX_INTERVAL   5	/* Heart beat interval in seconds. */
 #define LPFC_HB_MBOX_TIMEOUT    30	/* Heart beat timeout  in seconds. */
 
+#define LPFC_LOOK_AHEAD_OFF	0	/* Look ahead logic is turned off */
+
 /* Error Attention event polling interval */
 #define LPFC_ERATT_POLL_INTERVAL	5 /* EATT poll interval in seconds */
 
@@ -684,6 +686,7 @@ struct lpfc_hba {
 #define LPFC_FCF_FOV 1		/* Fast fcf failover */
 #define LPFC_FCF_PRIORITY 2	/* Priority fcf failover */
 	uint32_t cfg_fcf_failover_policy;
+	uint32_t cfg_fcp_io_sched;
 	uint32_t cfg_cr_delay;
 	uint32_t cfg_cr_count;
 	uint32_t cfg_multi_ring_support;
@@ -695,6 +698,7 @@ struct lpfc_hba {
 	uint32_t cfg_fcp_imax;
 	uint32_t cfg_fcp_wq_count;
 	uint32_t cfg_fcp_eq_count;
+	uint32_t cfg_fcp_io_channel;
 	uint32_t cfg_sg_seg_cnt;
 	uint32_t cfg_prot_sg_seg_cnt;
 	uint32_t cfg_sg_dma_buf_size;
@@ -732,7 +736,7 @@ struct lpfc_hba {
 	uint32_t hbq_count;	        /* Count of configured HBQs */
 	struct hbq_s hbqs[LPFC_MAX_HBQS]; /* local copy of hbq indicies  */
 
-	uint32_t fcp_qidx;		/* next work queue to post work to */
+	atomic_t fcp_qidx;		/* next work queue to post work to */
 
 	unsigned long pci_bar0_map;     /* Physical address for PCI BAR0 */
 	unsigned long pci_bar1_map;     /* Physical address for PCI BAR1 */
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index adef5bb2100..b032562aa0d 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -3643,18 +3643,25 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba *phba = vport->phba;
 	int val = 0, i;
 
+	/* fcp_imax is only valid for SLI4 */
+	if (phba->sli_rev != LPFC_SLI_REV4)
+		return -EINVAL;
+
 	/* Sanity check on user data */
 	if (!isdigit(buf[0]))
 		return -EINVAL;
 	if (sscanf(buf, "%i", &val) != 1)
 		return -EINVAL;
 
-	/* Value range is [636,651042] */
-	if (val < LPFC_MIM_IMAX || val > LPFC_DMULT_CONST)
+	/*
+	 * Value range for the HBA is [5000,5000000]
+	 * The value for each EQ depends on how many EQs are configured.
+	 */
+	if (val < LPFC_MIN_IMAX || val > LPFC_MAX_IMAX)
 		return -EINVAL;
 
 	phba->cfg_fcp_imax = (uint32_t)val;
-	for (i = 0; i < phba->cfg_fcp_eq_count; i += LPFC_MAX_EQ_DELAY)
+	for (i = 0; i < phba->cfg_fcp_io_channel; i += LPFC_MAX_EQ_DELAY)
 		lpfc_modify_fcp_eq_delay(phba, i);
 
 	return strlen(buf);
@@ -3662,13 +3669,14 @@ lpfc_fcp_imax_store(struct device *dev, struct device_attribute *attr,
 
 /*
 # lpfc_fcp_imax: The maximum number of fast-path FCP interrupts per second
+# for the HBA.
 #
-# Value range is [636,651042]. Default value is 10000.
+# Value range is [5,000 to 5,000,000]. Default value is 50,000.
 */
-static int lpfc_fcp_imax = LPFC_FP_DEF_IMAX;
+static int lpfc_fcp_imax = LPFC_DEF_IMAX;
 module_param(lpfc_fcp_imax, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(lpfc_fcp_imax,
-	    "Set the maximum number of fast-path FCP interrupts per second");
+	    "Set the maximum number of FCP interrupts per second per HBA");
 lpfc_param_show(fcp_imax)
 
 /**
@@ -3687,14 +3695,19 @@ lpfc_param_show(fcp_imax)
 static int
 lpfc_fcp_imax_init(struct lpfc_hba *phba, int val)
 {
-	if (val >= LPFC_MIM_IMAX && val <= LPFC_DMULT_CONST) {
+	if (phba->sli_rev != LPFC_SLI_REV4) {
+		phba->cfg_fcp_imax = 0;
+		return 0;
+	}
+
+	if (val >= LPFC_MIN_IMAX && val <= LPFC_MAX_IMAX) {
 		phba->cfg_fcp_imax = val;
 		return 0;
 	}
 
 	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 			"3016 fcp_imax: %d out of range, using default\n", val);
-	phba->cfg_fcp_imax = LPFC_FP_DEF_IMAX;
+	phba->cfg_fcp_imax = LPFC_DEF_IMAX;
 
 	return 0;
 }
@@ -3765,6 +3778,16 @@ static DEVICE_ATTR(lpfc_max_scsicmpl_time, S_IRUGO | S_IWUSR,
 LPFC_ATTR_R(ack0, 0, 0, 1, "Enable ACK0 support");
 
 /*
+# lpfc_fcp_io_sched: Determine scheduling algrithmn for issuing FCP cmds
+# range is [0,1]. Default value is 0.
+# For [0], FCP commands are issued to Work Queues ina round robin fashion.
+# For [1], FCP commands are issued to a Work Queue associated with the
+#          current CPU.
+*/
+LPFC_ATTR_RW(fcp_io_sched, 0, 0, 1, "Determine scheduling algrithmn for "
+		"issuing commands [0] - Round Robin, [1] - Current CPU");
+
+/*
 # lpfc_cr_delay & lpfc_cr_count: Default values for I/O colaesing
 # cr_delay (msec) or cr_count outstanding commands. cr_delay can take
 # value [0,63]. cr_count can take value [1,255]. Default value of cr_delay
@@ -3844,21 +3867,33 @@ LPFC_ATTR_R(use_msi, 2, 0, 2, "Use Message Signaled Interrupts (1) or "
 
 /*
 # lpfc_fcp_wq_count: Set the number of fast-path FCP work queues
+# This parameter is ignored and will eventually be depricated
 #
-# Value range is [1,31]. Default value is 4.
+# Value range is [1,7]. Default value is 4.
 */
-LPFC_ATTR_R(fcp_wq_count, LPFC_FP_WQN_DEF, LPFC_FP_WQN_MIN, LPFC_FP_WQN_MAX,
+LPFC_ATTR_R(fcp_wq_count, LPFC_FCP_IO_CHAN_DEF, LPFC_FCP_IO_CHAN_MIN,
+	    LPFC_FCP_IO_CHAN_MAX,
 	    "Set the number of fast-path FCP work queues, if possible");
 
 /*
-# lpfc_fcp_eq_count: Set the number of fast-path FCP event queues
+# lpfc_fcp_eq_count: Set the number of FCP EQ/CQ/WQ IO channels
 #
-# Value range is [1,7]. Default value is 1.
+# Value range is [1,7]. Default value is 4.
 */
-LPFC_ATTR_R(fcp_eq_count, LPFC_FP_EQN_DEF, LPFC_FP_EQN_MIN, LPFC_FP_EQN_MAX,
+LPFC_ATTR_R(fcp_eq_count, LPFC_FCP_IO_CHAN_DEF, LPFC_FCP_IO_CHAN_MIN,
+	    LPFC_FCP_IO_CHAN_MAX,
 	    "Set the number of fast-path FCP event queues, if possible");
 
 /*
+# lpfc_fcp_io_channel: Set the number of FCP EQ/CQ/WQ IO channels
+#
+# Value range is [1,7]. Default value is 4.
+*/
+LPFC_ATTR_R(fcp_io_channel, LPFC_FCP_IO_CHAN_DEF, LPFC_FCP_IO_CHAN_MIN,
+	    LPFC_FCP_IO_CHAN_MAX,
+	    "Set the number of FCP I/O channels");
+
+/*
 # lpfc_enable_hba_reset: Allow or prevent HBA resets to the hardware.
 #       0  = HBA resets disabled
 #       1  = HBA resets enabled (default)
@@ -3883,6 +3918,17 @@ LPFC_ATTR_R(enable_hba_heartbeat, 0, 0, 1, "Enable HBA Heartbeat.");
 LPFC_ATTR_R(enable_bg, 0, 0, 1, "Enable BlockGuard Support");
 
 /*
+# lpfc_fcp_look_ahead: Look ahead for completions in FCP start routine
+#       0  = disabled (default)
+#       1  = enabled
+# Value range is [0,1]. Default value is 0.
+*/
+unsigned int lpfc_fcp_look_ahead = LPFC_LOOK_AHEAD_OFF;
+
+module_param(lpfc_fcp_look_ahead, uint, S_IRUGO);
+MODULE_PARM_DESC(lpfc_fcp_look_ahead, "Look ahead for completions");
+
+/*
 # lpfc_prot_mask: i
 #	- Bit mask of host protection capabilities used to register with the
 #	  SCSI mid-layer
@@ -3976,6 +4022,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
 	&dev_attr_lpfc_topology,
 	&dev_attr_lpfc_scan_down,
 	&dev_attr_lpfc_link_speed,
+	&dev_attr_lpfc_fcp_io_sched,
 	&dev_attr_lpfc_cr_delay,
 	&dev_attr_lpfc_cr_count,
 	&dev_attr_lpfc_multi_ring_support,
@@ -4002,6 +4049,7 @@ struct device_attribute *lpfc_hba_attrs[] = {
 	&dev_attr_lpfc_fcp_imax,
 	&dev_attr_lpfc_fcp_wq_count,
 	&dev_attr_lpfc_fcp_eq_count,
+	&dev_attr_lpfc_fcp_io_channel,
 	&dev_attr_lpfc_enable_bg,
 	&dev_attr_lpfc_soft_wwnn,
 	&dev_attr_lpfc_soft_wwpn,
@@ -4964,6 +5012,7 @@ struct fc_function_template lpfc_vport_transport_functions = {
 void
 lpfc_get_cfgparam(struct lpfc_hba *phba)
 {
+	lpfc_fcp_io_sched_init(phba, lpfc_fcp_io_sched);
 	lpfc_cr_delay_init(phba, lpfc_cr_delay);
 	lpfc_cr_count_init(phba, lpfc_cr_count);
 	lpfc_multi_ring_support_init(phba, lpfc_multi_ring_support);
@@ -4980,6 +5029,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 	lpfc_fcp_imax_init(phba, lpfc_fcp_imax);
 	lpfc_fcp_wq_count_init(phba, lpfc_fcp_wq_count);
 	lpfc_fcp_eq_count_init(phba, lpfc_fcp_eq_count);
+	lpfc_fcp_io_channel_init(phba, lpfc_fcp_io_channel);
 	lpfc_enable_hba_reset_init(phba, lpfc_enable_hba_reset);
 	lpfc_enable_hba_heartbeat_init(phba, lpfc_enable_hba_heartbeat);
 	lpfc_enable_bg_init(phba, lpfc_enable_bg);
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index 253d9a85734..f7368eb8041 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -195,7 +195,7 @@ lpfc_bsg_send_mgmt_cmd_cmp(struct lpfc_hba *phba,
 
 	if (rsp->ulpStatus) {
 		if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
-			switch (rsp->un.ulpWord[4] & 0xff) {
+			switch (rsp->un.ulpWord[4] & IOERR_PARAM_MASK) {
 			case IOERR_SEQUENCE_TIMEOUT:
 				rc = -ETIMEDOUT;
 				break;
@@ -1234,7 +1234,7 @@ lpfc_issue_ct_rsp_cmp(struct lpfc_hba *phba,
 
 	if (rsp->ulpStatus) {
 		if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
-			switch (rsp->un.ulpWord[4] & 0xff) {
+			switch (rsp->un.ulpWord[4] & IOERR_PARAM_MASK) {
 			case IOERR_SEQUENCE_TIMEOUT:
 				rc = -ETIMEDOUT;
 				break;
@@ -1714,6 +1714,8 @@ lpfc_sli4_bsg_set_link_diag_state(struct lpfc_hba *phba, uint32_t diag)
 			phba->sli4_hba.lnk_info.lnk_no);
 
 	link_diag_state = &pmboxq->u.mqe.un.link_diag_state;
+	bf_set(lpfc_mbx_set_diag_state_diag_bit_valid, &link_diag_state->u.req,
+	       LPFC_DIAG_STATE_DIAG_BIT_VALID_CHANGE);
 	bf_set(lpfc_mbx_set_diag_state_link_num, &link_diag_state->u.req,
 	       phba->sli4_hba.lnk_info.lnk_no);
 	bf_set(lpfc_mbx_set_diag_state_link_type, &link_diag_state->u.req,
@@ -4796,7 +4798,7 @@ lpfc_bsg_menlo_cmd_cmp(struct lpfc_hba *phba,
 	menlo_resp->xri = rsp->ulpContext;
 	if (rsp->ulpStatus) {
 		if (rsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
-			switch (rsp->un.ulpWord[4] & 0xff) {
+			switch (rsp->un.ulpWord[4] & IOERR_PARAM_MASK) {
 			case IOERR_SEQUENCE_TIMEOUT:
 				rc = -ETIMEDOUT;
 				break;
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index 8a2a514a255..e470c489de0 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -196,8 +196,7 @@ irqreturn_t lpfc_sli_intr_handler(int, void *);
 irqreturn_t lpfc_sli_sp_intr_handler(int, void *);
 irqreturn_t lpfc_sli_fp_intr_handler(int, void *);
 irqreturn_t lpfc_sli4_intr_handler(int, void *);
-irqreturn_t lpfc_sli4_sp_intr_handler(int, void *);
-irqreturn_t lpfc_sli4_fp_intr_handler(int, void *);
+irqreturn_t lpfc_sli4_hba_intr_handler(int, void *);
 
 void lpfc_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *);
 void lpfc_sli4_swap_str(struct lpfc_hba *, LPFC_MBOXQ_t *);
@@ -391,6 +390,7 @@ extern spinlock_t pgcnt_lock;
 extern unsigned int pgcnt;
 extern unsigned int lpfc_prot_mask;
 extern unsigned char lpfc_prot_guard;
+extern unsigned int lpfc_fcp_look_ahead;
 
 /* Interface exported by fabric iocb scheduler */
 void lpfc_fabric_abort_nport(struct lpfc_nodelist *);
@@ -457,6 +457,8 @@ int lpfc_sli4_queue_create(struct lpfc_hba *);
 void lpfc_sli4_queue_destroy(struct lpfc_hba *);
 void lpfc_sli4_abts_err_handler(struct lpfc_hba *, struct lpfc_nodelist *,
 				struct sli4_wcqe_xri_aborted *);
+void lpfc_sli_abts_recover_port(struct lpfc_vport *,
+				struct lpfc_nodelist *);
 int lpfc_hba_init_link_fc_topology(struct lpfc_hba *, uint32_t, uint32_t);
 int lpfc_issue_reg_vfi(struct lpfc_vport *);
 int lpfc_issue_unreg_vfi(struct lpfc_vport *);
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 93e96b3c909..7ffabb7e3af 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -104,7 +104,8 @@ lpfc_ct_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	if (unlikely(icmd->ulpStatus == IOSTAT_NEED_BUFFER)) {
 		lpfc_sli_hbqbuf_add_hbqs(phba, LPFC_ELS_HBQ);
 	} else if ((icmd->ulpStatus == IOSTAT_LOCAL_REJECT) &&
-		((icmd->un.ulpWord[4] & 0xff) == IOERR_RCV_BUFFER_WAITING)) {
+		   ((icmd->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+		   IOERR_RCV_BUFFER_WAITING)) {
 		/* Not enough posted buffers; Try posting more buffers */
 		phba->fc_stat.NoRcvBuf++;
 		if (!(phba->sli3_options & LPFC_SLI3_HBQ_ENABLED))
@@ -633,7 +634,8 @@ lpfc_cmpl_ct_cmd_gid_ft(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		/* Check for retry */
 		if (vport->fc_ns_retry < LPFC_MAX_NS_RETRY) {
 			if (irsp->ulpStatus != IOSTAT_LOCAL_REJECT ||
-			    irsp->un.ulpWord[4] != IOERR_NO_RESOURCES)
+			    (irsp->un.ulpWord[4] && IOERR_PARAM_MASK) !=
+			    IOERR_NO_RESOURCES)
 				vport->fc_ns_retry++;
 
 			/* CT command is being retried */
@@ -783,7 +785,9 @@ lpfc_cmpl_ct_cmd_gff_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		if (cmdiocb->retry < LPFC_MAX_NS_RETRY) {
 			retry = 1;
 			if (irsp->ulpStatus == IOSTAT_LOCAL_REJECT) {
-				switch (irsp->un.ulpWord[4]) {
+				switch ((irsp->un.ulpWord[4] &
+					IOERR_PARAM_MASK)) {
+
 				case IOERR_NO_RESOURCES:
 					/* We don't increment the retry
 					 * count for this case.
@@ -908,8 +912,10 @@ lpfc_cmpl_ct(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				 cmdcode, irsp->ulpStatus, irsp->un.ulpWord[4]);
 
 		if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
-			((irsp->un.ulpWord[4] == IOERR_SLI_DOWN) ||
-			 (irsp->un.ulpWord[4] == IOERR_SLI_ABORTED)))
+			(((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+			  IOERR_SLI_DOWN) ||
+			 ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+			  IOERR_SLI_ABORTED)))
 			goto out;
 
 		retry = cmdiocb->retry;
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 3217d63ed28..f63f5ff7f27 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -490,9 +490,11 @@ lpfc_debugfs_dumpHostSlim_data(struct lpfc_hba *phba, char *buf, int size)
 		len +=  snprintf(buf+len, size-len,
 				 "Ring %d: CMD GetInx:%d (Max:%d Next:%d "
 				 "Local:%d flg:x%x)  RSP PutInx:%d Max:%d\n",
-				 i, pgpp->cmdGetInx, pring->numCiocb,
-				 pring->next_cmdidx, pring->local_getidx,
-				 pring->flag, pgpp->rspPutInx, pring->numRiocb);
+				 i, pgpp->cmdGetInx, pring->sli.sli3.numCiocb,
+				 pring->sli.sli3.next_cmdidx,
+				 pring->sli.sli3.local_getidx,
+				 pring->flag, pgpp->rspPutInx,
+				 pring->sli.sli3.numRiocb);
 	}
 
 	if (phba->sli_rev <= LPFC_SLI_REV3) {
@@ -557,6 +559,9 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 		case NLP_STE_PRLI_ISSUE:
 			statep = "PRLI  ";
 			break;
+		case NLP_STE_LOGO_ISSUE:
+			statep = "LOGO  ";
+			break;
 		case NLP_STE_UNMAPPED_NODE:
 			statep = "UNMAP ";
 			break;
@@ -581,8 +586,13 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 			"WWNN %02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x ",
 			*name, *(name+1), *(name+2), *(name+3),
 			*(name+4), *(name+5), *(name+6), *(name+7));
-		len +=  snprintf(buf+len, size-len, "RPI:%03d flag:x%08x ",
-			ndlp->nlp_rpi, ndlp->nlp_flag);
+		if (ndlp->nlp_flag & NLP_RPI_REGISTERED)
+			len +=  snprintf(buf+len, size-len, "RPI:%03d ",
+				ndlp->nlp_rpi);
+		else
+			len +=  snprintf(buf+len, size-len, "RPI:none ");
+		len +=  snprintf(buf+len, size-len, "flag:x%08x ",
+			ndlp->nlp_flag);
 		if (!ndlp->nlp_type)
 			len +=  snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
 		if (ndlp->nlp_type & NLP_FC_NODE)
@@ -1999,207 +2009,298 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes,
 {
 	struct lpfc_debug *debug = file->private_data;
 	struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private;
-	int len = 0, fcp_qidx;
+	int len = 0;
 	char *pbuffer;
+	int x, cnt;
+	int max_cnt;
+	struct lpfc_queue *qp = NULL;
+
 
 	if (!debug->buffer)
 		debug->buffer = kmalloc(LPFC_QUE_INFO_GET_BUF_SIZE, GFP_KERNEL);
 	if (!debug->buffer)
 		return 0;
 	pbuffer = debug->buffer;
+	max_cnt = LPFC_QUE_INFO_GET_BUF_SIZE - 128;
 
 	if (*ppos)
 		return 0;
 
-	/* Get slow-path event queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path EQ information:\n");
-	if (phba->sli4_hba.sp_eq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tEQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n\n",
-			phba->sli4_hba.sp_eq->queue_id,
-			phba->sli4_hba.sp_eq->entry_count,
-			phba->sli4_hba.sp_eq->entry_size,
-			phba->sli4_hba.sp_eq->host_index,
-			phba->sli4_hba.sp_eq->hba_index);
-	}
+	spin_lock_irq(&phba->hbalock);
+
+	/* Fast-path event queue */
+	if (phba->sli4_hba.hba_eq && phba->cfg_fcp_io_channel) {
+		cnt = phba->cfg_fcp_io_channel;
+
+		for (x = 0; x < cnt; x++) {
+
+			/* Fast-path EQ */
+			qp = phba->sli4_hba.hba_eq[x];
+			if (!qp)
+				goto proc_cq;
+
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"\nHBA EQ info: "
+				"EQ-STAT[max:x%x noE:x%x "
+				"bs:x%x proc:x%llx]\n",
+				qp->q_cnt_1, qp->q_cnt_2,
+				qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
+
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"EQID[%02d], "
+				"QE-CNT[%04d], QE-SIZE[%04d], "
+				"HOST-IDX[%04d], PORT-IDX[%04d]",
+				qp->queue_id,
+				qp->entry_count,
+				qp->entry_size,
+				qp->host_index,
+				qp->hba_index);
+
+
+			/* Reset max counter */
+			qp->EQ_max_eqe = 0;
+
+			len +=  snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+			if (len >= max_cnt)
+				goto too_big;
+proc_cq:
+			/* Fast-path FCP CQ */
+			qp = phba->sli4_hba.fcp_cq[x];
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"\tFCP CQ info: ");
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"AssocEQID[%02d]: "
+				"CQ STAT[max:x%x relw:x%x "
+				"xabt:x%x wq:x%llx]\n",
+				qp->assoc_qid,
+				qp->q_cnt_1, qp->q_cnt_2,
+				qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"\tCQID[%02d], "
+				"QE-CNT[%04d], QE-SIZE[%04d], "
+				"HOST-IDX[%04d], PORT-IDX[%04d]",
+				qp->queue_id, qp->entry_count,
+				qp->entry_size, qp->host_index,
+				qp->hba_index);
+
 
-	/* Get fast-path event queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Fast-path EQ information:\n");
-	if (phba->sli4_hba.fp_eq) {
-		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_eq_count;
-		     fcp_qidx++) {
-			if (phba->sli4_hba.fp_eq[fcp_qidx]) {
+			/* Reset max counter */
+			qp->CQ_max_cqe = 0;
+
+			len +=  snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+			if (len >= max_cnt)
+				goto too_big;
+
+			/* Fast-path FCP WQ */
+			qp = phba->sli4_hba.fcp_wq[x];
+
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"\t\tFCP WQ info: ");
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"AssocCQID[%02d]: "
+				"WQ-STAT[oflow:x%x posted:x%llx]\n",
+				qp->assoc_qid,
+				qp->q_cnt_1, (unsigned long long)qp->q_cnt_4);
+			len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+				"\t\tWQID[%02d], "
+				"QE-CNT[%04d], QE-SIZE[%04d], "
+				"HOST-IDX[%04d], PORT-IDX[%04d]",
+				qp->queue_id,
+				qp->entry_count,
+				qp->entry_size,
+				qp->host_index,
+				qp->hba_index);
+
+			len +=  snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+			if (len >= max_cnt)
+				goto too_big;
+
+			if (x)
+				continue;
+
+			/* Only EQ 0 has slow path CQs configured */
+
+			/* Slow-path mailbox CQ */
+			qp = phba->sli4_hba.mbx_cq;
+			if (qp) {
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\tMBX CQ info: ");
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"AssocEQID[%02d]: "
+					"CQ-STAT[mbox:x%x relw:x%x "
+					"xabt:x%x wq:x%llx]\n",
+					qp->assoc_qid,
+					qp->q_cnt_1, qp->q_cnt_2,
+					qp->q_cnt_3,
+					(unsigned long long)qp->q_cnt_4);
 				len += snprintf(pbuffer+len,
 					LPFC_QUE_INFO_GET_BUF_SIZE-len,
-				"\tEQID[%02d], "
-				"QE-COUNT[%04d], QE-SIZE[%04d], "
-				"HOST-INDEX[%04d], PORT-INDEX[%04d]\n",
-				phba->sli4_hba.fp_eq[fcp_qidx]->queue_id,
-				phba->sli4_hba.fp_eq[fcp_qidx]->entry_count,
-				phba->sli4_hba.fp_eq[fcp_qidx]->entry_size,
-				phba->sli4_hba.fp_eq[fcp_qidx]->host_index,
-				phba->sli4_hba.fp_eq[fcp_qidx]->hba_index);
+					"\tCQID[%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]",
+					qp->queue_id, qp->entry_count,
+					qp->entry_size, qp->host_index,
+					qp->hba_index);
+
+				len +=  snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+				if (len >= max_cnt)
+					goto too_big;
 			}
-		}
-	}
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
-
-	/* Get mailbox complete queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path MBX CQ information:\n");
-	if (phba->sli4_hba.mbx_cq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Associated EQID[%02d]:\n",
-			phba->sli4_hba.mbx_cq->assoc_qid);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tCQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n\n",
-			phba->sli4_hba.mbx_cq->queue_id,
-			phba->sli4_hba.mbx_cq->entry_count,
-			phba->sli4_hba.mbx_cq->entry_size,
-			phba->sli4_hba.mbx_cq->host_index,
-			phba->sli4_hba.mbx_cq->hba_index);
-	}
 
-	/* Get slow-path complete queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path ELS CQ information:\n");
-	if (phba->sli4_hba.els_cq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Associated EQID[%02d]:\n",
-			phba->sli4_hba.els_cq->assoc_qid);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tCQID [%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n\n",
-			phba->sli4_hba.els_cq->queue_id,
-			phba->sli4_hba.els_cq->entry_count,
-			phba->sli4_hba.els_cq->entry_size,
-			phba->sli4_hba.els_cq->host_index,
-			phba->sli4_hba.els_cq->hba_index);
-	}
+			/* Slow-path MBOX MQ */
+			qp = phba->sli4_hba.mbx_wq;
+			if (qp) {
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tMBX MQ info: ");
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"AssocCQID[%02d]:\n",
+					phba->sli4_hba.mbx_wq->assoc_qid);
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tWQID[%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]",
+					qp->queue_id, qp->entry_count,
+					qp->entry_size, qp->host_index,
+					qp->hba_index);
+
+				len +=  snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+				if (len >= max_cnt)
+					goto too_big;
+			}
 
-	/* Get fast-path complete queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Fast-path FCP CQ information:\n");
-	fcp_qidx = 0;
-	if (phba->sli4_hba.fcp_cq) {
-		do {
-			if (phba->sli4_hba.fcp_cq[fcp_qidx]) {
+			/* Slow-path ELS response CQ */
+			qp = phba->sli4_hba.els_cq;
+			if (qp) {
 				len += snprintf(pbuffer+len,
 					LPFC_QUE_INFO_GET_BUF_SIZE-len,
-				"Associated EQID[%02d]:\n",
-				phba->sli4_hba.fcp_cq[fcp_qidx]->assoc_qid);
+					"\tELS CQ info: ");
 				len += snprintf(pbuffer+len,
 					LPFC_QUE_INFO_GET_BUF_SIZE-len,
-				"\tCQID[%02d], "
-				"QE-COUNT[%04d], QE-SIZE[%04d], "
-				"HOST-INDEX[%04d], PORT-INDEX[%04d]\n",
-				phba->sli4_hba.fcp_cq[fcp_qidx]->queue_id,
-				phba->sli4_hba.fcp_cq[fcp_qidx]->entry_count,
-				phba->sli4_hba.fcp_cq[fcp_qidx]->entry_size,
-				phba->sli4_hba.fcp_cq[fcp_qidx]->host_index,
-				phba->sli4_hba.fcp_cq[fcp_qidx]->hba_index);
+					"AssocEQID[%02d]: "
+					"CQ-STAT[max:x%x relw:x%x "
+					"xabt:x%x wq:x%llx]\n",
+					qp->assoc_qid,
+					qp->q_cnt_1, qp->q_cnt_2,
+					qp->q_cnt_3,
+					(unsigned long long)qp->q_cnt_4);
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\tCQID [%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]",
+					qp->queue_id, qp->entry_count,
+					qp->entry_size, qp->host_index,
+					qp->hba_index);
+
+				/* Reset max counter */
+				qp->CQ_max_cqe = 0;
+
+				len +=  snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+				if (len >= max_cnt)
+					goto too_big;
 			}
-		} while (++fcp_qidx < phba->cfg_fcp_eq_count);
-		len += snprintf(pbuffer+len,
-				LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
-	}
 
-	/* Get mailbox queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path MBX MQ information:\n");
-	if (phba->sli4_hba.mbx_wq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Associated CQID[%02d]:\n",
-			phba->sli4_hba.mbx_wq->assoc_qid);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tWQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n\n",
-			phba->sli4_hba.mbx_wq->queue_id,
-			phba->sli4_hba.mbx_wq->entry_count,
-			phba->sli4_hba.mbx_wq->entry_size,
-			phba->sli4_hba.mbx_wq->host_index,
-			phba->sli4_hba.mbx_wq->hba_index);
-	}
+			/* Slow-path ELS WQ */
+			qp = phba->sli4_hba.els_wq;
+			if (qp) {
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tELS WQ info: ");
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"AssocCQID[%02d]: "
+					" WQ-STAT[oflow:x%x "
+					"posted:x%llx]\n",
+					qp->assoc_qid,
+					qp->q_cnt_1,
+					(unsigned long long)qp->q_cnt_4);
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tWQID[%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]",
+					qp->queue_id, qp->entry_count,
+					qp->entry_size, qp->host_index,
+					qp->hba_index);
+
+				len +=  snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+				if (len >= max_cnt)
+					goto too_big;
+			}
 
-	/* Get slow-path work queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path ELS WQ information:\n");
-	if (phba->sli4_hba.els_wq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Associated CQID[%02d]:\n",
-			phba->sli4_hba.els_wq->assoc_qid);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tWQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n\n",
-			phba->sli4_hba.els_wq->queue_id,
-			phba->sli4_hba.els_wq->entry_count,
-			phba->sli4_hba.els_wq->entry_size,
-			phba->sli4_hba.els_wq->host_index,
-			phba->sli4_hba.els_wq->hba_index);
-	}
+			if (phba->sli4_hba.hdr_rq && phba->sli4_hba.dat_rq) {
+				/* Slow-path RQ header */
+				qp = phba->sli4_hba.hdr_rq;
 
-	/* Get fast-path work queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Fast-path FCP WQ information:\n");
-	if (phba->sli4_hba.fcp_wq) {
-		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_wq_count;
-		     fcp_qidx++) {
-			if (!phba->sli4_hba.fcp_wq[fcp_qidx])
-				continue;
-			len += snprintf(pbuffer+len,
+				len += snprintf(pbuffer+len,
+				LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tRQ info: ");
+				len += snprintf(pbuffer+len,
 					LPFC_QUE_INFO_GET_BUF_SIZE-len,
-				"Associated CQID[%02d]:\n",
-				phba->sli4_hba.fcp_wq[fcp_qidx]->assoc_qid);
-			len += snprintf(pbuffer+len,
+					"AssocCQID[%02d]: "
+					"RQ-STAT[nopost:x%x nobuf:x%x "
+					"trunc:x%x rcv:x%llx]\n",
+					qp->assoc_qid,
+					qp->q_cnt_1, qp->q_cnt_2,
+					qp->q_cnt_3,
+					(unsigned long long)qp->q_cnt_4);
+				len += snprintf(pbuffer+len,
 					LPFC_QUE_INFO_GET_BUF_SIZE-len,
-				"\tWQID[%02d], "
-				"QE-COUNT[%04d], WQE-SIZE[%04d], "
-				"HOST-INDEX[%04d], PORT-INDEX[%04d]\n",
-				phba->sli4_hba.fcp_wq[fcp_qidx]->queue_id,
-				phba->sli4_hba.fcp_wq[fcp_qidx]->entry_count,
-				phba->sli4_hba.fcp_wq[fcp_qidx]->entry_size,
-				phba->sli4_hba.fcp_wq[fcp_qidx]->host_index,
-				phba->sli4_hba.fcp_wq[fcp_qidx]->hba_index);
+					"\t\tHQID[%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]\n",
+					qp->queue_id,
+					qp->entry_count,
+					qp->entry_size,
+					qp->host_index,
+					qp->hba_index);
+
+				/* Slow-path RQ data */
+				qp = phba->sli4_hba.dat_rq;
+				len += snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len,
+					"\t\tDQID[%02d], "
+					"QE-CNT[%04d], QE-SIZE[%04d], "
+					"HOST-IDX[%04d], PORT-IDX[%04d]\n",
+					qp->queue_id,
+					qp->entry_count,
+					qp->entry_size,
+					qp->host_index,
+					qp->hba_index);
+
+				len +=  snprintf(pbuffer+len,
+					LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
+			}
 		}
-		len += snprintf(pbuffer+len,
-				LPFC_QUE_INFO_GET_BUF_SIZE-len, "\n");
 	}
 
-	/* Get receive queue information */
-	len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Slow-path RQ information:\n");
-	if (phba->sli4_hba.hdr_rq && phba->sli4_hba.dat_rq) {
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"Associated CQID[%02d]:\n",
-			phba->sli4_hba.hdr_rq->assoc_qid);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tHQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n",
-			phba->sli4_hba.hdr_rq->queue_id,
-			phba->sli4_hba.hdr_rq->entry_count,
-			phba->sli4_hba.hdr_rq->entry_size,
-			phba->sli4_hba.hdr_rq->host_index,
-			phba->sli4_hba.hdr_rq->hba_index);
-		len += snprintf(pbuffer+len, LPFC_QUE_INFO_GET_BUF_SIZE-len,
-			"\tDQID[%02d], "
-			"QE-COUNT[%04d], QE-SIZE[%04d], "
-			"HOST-INDEX[%04d], PORT-INDEX[%04d]\n",
-			phba->sli4_hba.dat_rq->queue_id,
-			phba->sli4_hba.dat_rq->entry_count,
-			phba->sli4_hba.dat_rq->entry_size,
-			phba->sli4_hba.dat_rq->host_index,
-			phba->sli4_hba.dat_rq->hba_index);
-	}
+	spin_unlock_irq(&phba->hbalock);
+	return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len);
+
+too_big:
+	len +=  snprintf(pbuffer+len,
+		LPFC_QUE_INFO_GET_BUF_SIZE-len, "Truncated ...\n");
+	spin_unlock_irq(&phba->hbalock);
 	return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len);
 }
 
@@ -2408,31 +2509,21 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf,
 
 	switch (quetp) {
 	case LPFC_IDIAG_EQ:
-		/* Slow-path event queue */
-		if (phba->sli4_hba.sp_eq &&
-		    phba->sli4_hba.sp_eq->queue_id == queid) {
-			/* Sanity check */
-			rc = lpfc_idiag_que_param_check(
-					phba->sli4_hba.sp_eq, index, count);
-			if (rc)
-				goto error_out;
-			idiag.ptr_private = phba->sli4_hba.sp_eq;
-			goto pass_check;
-		}
-		/* Fast-path event queue */
-		if (phba->sli4_hba.fp_eq) {
-			for (qidx = 0; qidx < phba->cfg_fcp_eq_count; qidx++) {
-				if (phba->sli4_hba.fp_eq[qidx] &&
-				    phba->sli4_hba.fp_eq[qidx]->queue_id ==
+		/* HBA event queue */
+		if (phba->sli4_hba.hba_eq) {
+			for (qidx = 0; qidx < phba->cfg_fcp_io_channel;
+				qidx++) {
+				if (phba->sli4_hba.hba_eq[qidx] &&
+				    phba->sli4_hba.hba_eq[qidx]->queue_id ==
 				    queid) {
 					/* Sanity check */
 					rc = lpfc_idiag_que_param_check(
-						phba->sli4_hba.fp_eq[qidx],
+						phba->sli4_hba.hba_eq[qidx],
 						index, count);
 					if (rc)
 						goto error_out;
 					idiag.ptr_private =
-						phba->sli4_hba.fp_eq[qidx];
+						phba->sli4_hba.hba_eq[qidx];
 					goto pass_check;
 				}
 			}
@@ -2479,7 +2570,7 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf,
 						phba->sli4_hba.fcp_cq[qidx];
 					goto pass_check;
 				}
-			} while (++qidx < phba->cfg_fcp_eq_count);
+			} while (++qidx < phba->cfg_fcp_io_channel);
 		}
 		goto error_out;
 		break;
@@ -2511,7 +2602,8 @@ lpfc_idiag_queacc_write(struct file *file, const char __user *buf,
 		}
 		/* FCP work queue */
 		if (phba->sli4_hba.fcp_wq) {
-			for (qidx = 0; qidx < phba->cfg_fcp_wq_count; qidx++) {
+			for (qidx = 0; qidx < phba->cfg_fcp_io_channel;
+				qidx++) {
 				if (!phba->sli4_hba.fcp_wq[qidx])
 					continue;
 				if (phba->sli4_hba.fcp_wq[qidx]->queue_id ==
@@ -4490,7 +4582,7 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba)
 	lpfc_debug_dump_mbx_wq(phba);
 	lpfc_debug_dump_els_wq(phba);
 
-	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_wq_count; fcp_wqidx++)
+	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
 		lpfc_debug_dump_fcp_wq(phba, fcp_wqidx);
 
 	lpfc_debug_dump_hdr_rq(phba);
@@ -4501,14 +4593,12 @@ lpfc_debug_dump_all_queues(struct lpfc_hba *phba)
 	lpfc_debug_dump_mbx_cq(phba);
 	lpfc_debug_dump_els_cq(phba);
 
-	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_wq_count; fcp_wqidx++)
+	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
 		lpfc_debug_dump_fcp_cq(phba, fcp_wqidx);
 
 	/*
 	 * Dump Event Queues (EQs)
 	 */
-	lpfc_debug_dump_sp_eq(phba);
-
-	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_wq_count; fcp_wqidx++)
-		lpfc_debug_dump_fcp_eq(phba, fcp_wqidx);
+	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++)
+		lpfc_debug_dump_hba_eq(phba, fcp_wqidx);
 }
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h
index afe368fd1b9..8b2b6a3bfc2 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.h
+++ b/drivers/scsi/lpfc/lpfc_debugfs.h
@@ -36,6 +36,9 @@
 /* dumpHostSlim output buffer size */
 #define LPFC_DUMPHOSTSLIM_SIZE 4096
 
+/* dumpSLIqinfo output buffer size */
+#define	LPFC_DUMPSLIQINFO_SIZE 4096
+
 /* hbqinfo output buffer size */
 #define LPFC_HBQINFO_SIZE 8192
 
@@ -366,7 +369,7 @@ static inline void
 lpfc_debug_dump_fcp_wq(struct lpfc_hba *phba, int fcp_wqidx)
 {
 	/* sanity check */
-	if (fcp_wqidx >= phba->cfg_fcp_wq_count)
+	if (fcp_wqidx >= phba->cfg_fcp_io_channel)
 		return;
 
 	printk(KERN_ERR "FCP WQ: WQ[Idx:%d|Qid:%d]\n",
@@ -388,15 +391,15 @@ lpfc_debug_dump_fcp_cq(struct lpfc_hba *phba, int fcp_wqidx)
 	int fcp_cqidx, fcp_cqid;
 
 	/* sanity check */
-	if (fcp_wqidx >= phba->cfg_fcp_wq_count)
+	if (fcp_wqidx >= phba->cfg_fcp_io_channel)
 		return;
 
 	fcp_cqid = phba->sli4_hba.fcp_wq[fcp_wqidx]->assoc_qid;
-	for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_eq_count; fcp_cqidx++)
+	for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++)
 		if (phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id == fcp_cqid)
 			break;
 	if (phba->intr_type == MSIX) {
-		if (fcp_cqidx >= phba->cfg_fcp_eq_count)
+		if (fcp_cqidx >= phba->cfg_fcp_io_channel)
 			return;
 	} else {
 		if (fcp_cqidx > 0)
@@ -410,7 +413,7 @@ lpfc_debug_dump_fcp_cq(struct lpfc_hba *phba, int fcp_wqidx)
 }
 
 /**
- * lpfc_debug_dump_fcp_eq - dump all entries from a fcp work queue's evt queue
+ * lpfc_debug_dump_hba_eq - dump all entries from a fcp work queue's evt queue
  * @phba: Pointer to HBA context object.
  * @fcp_wqidx: Index to a FCP work queue.
  *
@@ -418,36 +421,30 @@ lpfc_debug_dump_fcp_cq(struct lpfc_hba *phba, int fcp_wqidx)
  * associated to the FCP work queue specified by the @fcp_wqidx.
  **/
 static inline void
-lpfc_debug_dump_fcp_eq(struct lpfc_hba *phba, int fcp_wqidx)
+lpfc_debug_dump_hba_eq(struct lpfc_hba *phba, int fcp_wqidx)
 {
 	struct lpfc_queue *qdesc;
 	int fcp_eqidx, fcp_eqid;
 	int fcp_cqidx, fcp_cqid;
 
 	/* sanity check */
-	if (fcp_wqidx >= phba->cfg_fcp_wq_count)
+	if (fcp_wqidx >= phba->cfg_fcp_io_channel)
 		return;
 	fcp_cqid = phba->sli4_hba.fcp_wq[fcp_wqidx]->assoc_qid;
-	for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_eq_count; fcp_cqidx++)
+	for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++)
 		if (phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id == fcp_cqid)
 			break;
 	if (phba->intr_type == MSIX) {
-		if (fcp_cqidx >= phba->cfg_fcp_eq_count)
+		if (fcp_cqidx >= phba->cfg_fcp_io_channel)
 			return;
 	} else {
 		if (fcp_cqidx > 0)
 			return;
 	}
 
-	if (phba->cfg_fcp_eq_count == 0) {
-		fcp_eqidx = -1;
-		fcp_eqid = phba->sli4_hba.sp_eq->queue_id;
-		qdesc = phba->sli4_hba.sp_eq;
-	} else {
-		fcp_eqidx = fcp_cqidx;
-		fcp_eqid = phba->sli4_hba.fp_eq[fcp_eqidx]->queue_id;
-		qdesc = phba->sli4_hba.fp_eq[fcp_eqidx];
-	}
+	fcp_eqidx = fcp_cqidx;
+	fcp_eqid = phba->sli4_hba.hba_eq[fcp_eqidx]->queue_id;
+	qdesc = phba->sli4_hba.hba_eq[fcp_eqidx];
 
 	printk(KERN_ERR "FCP EQ: WQ[Idx:%d|Qid:%d]->CQ[Idx:%d|Qid:%d]->"
 		"EQ[Idx:%d|Qid:%d]\n",
@@ -543,25 +540,6 @@ lpfc_debug_dump_mbx_cq(struct lpfc_hba *phba)
 }
 
 /**
- * lpfc_debug_dump_sp_eq - dump all entries from slow-path event queue
- * @phba: Pointer to HBA context object.
- *
- * This function dumps all entries from the slow-path event queue.
- **/
-static inline void
-lpfc_debug_dump_sp_eq(struct lpfc_hba *phba)
-{
-	printk(KERN_ERR "SP EQ: WQ[Qid:%d/Qid:%d]->CQ[Qid:%d/Qid:%d]->"
-		"EQ[Qid:%d]:\n",
-		phba->sli4_hba.mbx_wq->queue_id,
-		phba->sli4_hba.els_wq->queue_id,
-		phba->sli4_hba.mbx_cq->queue_id,
-		phba->sli4_hba.els_cq->queue_id,
-		phba->sli4_hba.sp_eq->queue_id);
-	lpfc_debug_dump_q(phba->sli4_hba.sp_eq);
-}
-
-/**
  * lpfc_debug_dump_wq_by_id - dump all entries from a work queue by queue id
  * @phba: Pointer to HBA context object.
  * @qid: Work queue identifier.
@@ -574,10 +552,10 @@ lpfc_debug_dump_wq_by_id(struct lpfc_hba *phba, int qid)
 {
 	int wq_idx;
 
-	for (wq_idx = 0; wq_idx < phba->cfg_fcp_wq_count; wq_idx++)
+	for (wq_idx = 0; wq_idx < phba->cfg_fcp_io_channel; wq_idx++)
 		if (phba->sli4_hba.fcp_wq[wq_idx]->queue_id == qid)
 			break;
-	if (wq_idx < phba->cfg_fcp_wq_count) {
+	if (wq_idx < phba->cfg_fcp_io_channel) {
 		printk(KERN_ERR "FCP WQ[Idx:%d|Qid:%d]\n", wq_idx, qid);
 		lpfc_debug_dump_q(phba->sli4_hba.fcp_wq[wq_idx]);
 		return;
@@ -644,9 +622,9 @@ lpfc_debug_dump_cq_by_id(struct lpfc_hba *phba, int qid)
 	do {
 		if (phba->sli4_hba.fcp_cq[cq_idx]->queue_id == qid)
 			break;
-	} while (++cq_idx < phba->cfg_fcp_eq_count);
+	} while (++cq_idx < phba->cfg_fcp_io_channel);
 
-	if (cq_idx < phba->cfg_fcp_eq_count) {
+	if (cq_idx < phba->cfg_fcp_io_channel) {
 		printk(KERN_ERR "FCP CQ[Idx:%d|Qid:%d]\n", cq_idx, qid);
 		lpfc_debug_dump_q(phba->sli4_hba.fcp_cq[cq_idx]);
 		return;
@@ -677,21 +655,17 @@ lpfc_debug_dump_eq_by_id(struct lpfc_hba *phba, int qid)
 {
 	int eq_idx;
 
-	for (eq_idx = 0; eq_idx < phba->cfg_fcp_eq_count; eq_idx++) {
-		if (phba->sli4_hba.fp_eq[eq_idx]->queue_id == qid)
+	for (eq_idx = 0; eq_idx < phba->cfg_fcp_io_channel; eq_idx++) {
+		if (phba->sli4_hba.hba_eq[eq_idx]->queue_id == qid)
 			break;
 	}
 
-	if (eq_idx < phba->cfg_fcp_eq_count) {
+	if (eq_idx < phba->cfg_fcp_io_channel) {
 		printk(KERN_ERR "FCP EQ[Idx:%d|Qid:%d]\n", eq_idx, qid);
-		lpfc_debug_dump_q(phba->sli4_hba.fp_eq[eq_idx]);
+		lpfc_debug_dump_q(phba->sli4_hba.hba_eq[eq_idx]);
 		return;
 	}
 
-	if (phba->sli4_hba.sp_eq->queue_id == qid) {
-		printk(KERN_ERR "SP EQ[|Qid:%d]\n", qid);
-		lpfc_debug_dump_q(phba->sli4_hba.sp_eq);
-	}
 }
 
 void lpfc_debug_dump_all_queues(struct lpfc_hba *);
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 1d84b63fcca..af49fb03dbb 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -145,6 +145,7 @@ struct lpfc_node_rrq {
 #define NLP_RCV_PLOGI      0x00080000	/* Rcv'ed PLOGI from remote system */
 #define NLP_LOGO_ACC       0x00100000	/* Process LOGO after ACC completes */
 #define NLP_TGT_NO_SCSIID  0x00200000	/* good PRLI but no binding for scsid */
+#define NLP_ISSUE_LOGO     0x00400000	/* waiting to issue a LOGO */
 #define NLP_ACC_REGLOGIN   0x01000000	/* Issue Reg Login after successful
 					   ACC */
 #define NLP_NPR_ADISC      0x02000000	/* Issue ADISC when dq'ed from
@@ -201,10 +202,11 @@ struct lpfc_node_rrq {
 #define NLP_STE_ADISC_ISSUE       0x2	/* ADISC was sent to NL_PORT */
 #define NLP_STE_REG_LOGIN_ISSUE   0x3	/* REG_LOGIN was issued for NL_PORT */
 #define NLP_STE_PRLI_ISSUE        0x4	/* PRLI was sent to NL_PORT */
-#define NLP_STE_UNMAPPED_NODE     0x5	/* PRLI completed from NL_PORT */
-#define NLP_STE_MAPPED_NODE       0x6	/* Identified as a FCP Target */
-#define NLP_STE_NPR_NODE          0x7	/* NPort disappeared */
-#define NLP_STE_MAX_STATE         0x8
+#define NLP_STE_LOGO_ISSUE	  0x5	/* LOGO was sent to NL_PORT */
+#define NLP_STE_UNMAPPED_NODE     0x6	/* PRLI completed from NL_PORT */
+#define NLP_STE_MAPPED_NODE       0x7	/* Identified as a FCP Target */
+#define NLP_STE_NPR_NODE          0x8	/* NPort disappeared */
+#define NLP_STE_MAX_STATE         0x9
 #define NLP_STE_FREED_NODE        0xff	/* node entry was freed to MEM_NLP */
 
 /* For UNUSED_NODE state, the node has just been allocated.
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index d54ae199979..cfe533bc979 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -962,7 +962,8 @@ lpfc_cmpl_els_flogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 			if ((phba->fcoe_cvl_eventtag_attn ==
 			     phba->fcoe_cvl_eventtag) &&
 			    (irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
-			    (irsp->un.ulpWord[4] == IOERR_SLI_ABORTED))
+			    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+			    IOERR_SLI_ABORTED))
 				goto stop_rr_fcf_flogi;
 			else
 				phba->fcoe_cvl_eventtag_attn =
@@ -1108,8 +1109,10 @@ flogifail:
 		/* Start discovery */
 		lpfc_disc_start(vport);
 	} else if (((irsp->ulpStatus != IOSTAT_LOCAL_REJECT) ||
-			((irsp->un.ulpWord[4] != IOERR_SLI_ABORTED) &&
-			(irsp->un.ulpWord[4] != IOERR_SLI_DOWN))) &&
+			(((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) !=
+			 IOERR_SLI_ABORTED) &&
+			((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) !=
+			 IOERR_SLI_DOWN))) &&
 			(phba->link_state != LPFC_CLEAR_LA)) {
 		/* If FLOGI failed enable link interrupt. */
 		lpfc_issue_clear_la(phba, vport);
@@ -1476,6 +1479,10 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 		return ndlp;
 	memset(&rrq.xri_bitmap, 0, sizeof(new_ndlp->active_rrqs.xri_bitmap));
 
+	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+		 "3178 PLOGI confirm: ndlp %p x%x: new_ndlp %p\n",
+		 ndlp, ndlp->nlp_DID, new_ndlp);
+
 	if (!new_ndlp) {
 		rc = memcmp(&ndlp->nlp_portname, name,
 			    sizeof(struct lpfc_name));
@@ -1527,6 +1534,9 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 		/* The new_ndlp is replacing ndlp totally, so we need
 		 * to put ndlp on UNUSED list and try to free it.
 		 */
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+			 "3179 PLOGI confirm NEW: %x %x\n",
+			 new_ndlp->nlp_DID, keepDID);
 
 		/* Fix up the rport accordingly */
 		rport =  ndlp->rport;
@@ -1559,23 +1569,34 @@ lpfc_plogi_confirm_nport(struct lpfc_hba *phba, uint32_t *prsp,
 		lpfc_drop_node(vport, ndlp);
 	}
 	else {
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+			 "3180 PLOGI confirm SWAP: %x %x\n",
+			 new_ndlp->nlp_DID, keepDID);
+
 		lpfc_unreg_rpi(vport, ndlp);
+
 		/* Two ndlps cannot have the same did */
 		ndlp->nlp_DID = keepDID;
 		if (phba->sli_rev == LPFC_SLI_REV4)
 			memcpy(&ndlp->active_rrqs.xri_bitmap,
 				&rrq.xri_bitmap,
 				sizeof(ndlp->active_rrqs.xri_bitmap));
+
 		/* Since we are swapping the ndlp passed in with the new one
-		 * and the did has already been swapped, copy over the
-		 * state and names.
+		 * and the did has already been swapped, copy over state.
+		 * The new WWNs are already in new_ndlp since thats what
+		 * we looked it up by in the begining of this routine.
 		 */
-		memcpy(&new_ndlp->nlp_portname, &ndlp->nlp_portname,
-			sizeof(struct lpfc_name));
-		memcpy(&new_ndlp->nlp_nodename, &ndlp->nlp_nodename,
-			sizeof(struct lpfc_name));
 		new_ndlp->nlp_state = ndlp->nlp_state;
-		lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+
+		/* Since we are switching over to the new_ndlp, the old
+		 * ndlp should be put in the NPR state, unless we have
+		 * already started re-discovery on it.
+		 */
+		if ((ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) ||
+		    (ndlp->nlp_state == NLP_STE_MAPPED_NODE))
+			lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+
 		/* Fix up the rport accordingly */
 		rport = ndlp->rport;
 		if (rport) {
@@ -2367,6 +2388,8 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	IOCB_t *irsp;
 	struct lpfc_sli *psli;
 	struct lpfcMboxq *mbox;
+	unsigned long flags;
+	uint32_t skip_recovery = 0;
 
 	psli = &phba->sli;
 	/* we pass cmdiocb to state machine which needs rspiocb as well */
@@ -2381,47 +2404,52 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		"LOGO cmpl:       status:x%x/x%x did:x%x",
 		irsp->ulpStatus, irsp->un.ulpWord[4],
 		ndlp->nlp_DID);
+
 	/* LOGO completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0105 LOGO completes to NPort x%x "
 			 "Data: x%x x%x x%x x%x\n",
 			 ndlp->nlp_DID, irsp->ulpStatus, irsp->un.ulpWord[4],
 			 irsp->ulpTimeout, vport->num_disc_nodes);
-	/* Check to see if link went down during discovery */
-	if (lpfc_els_chk_latt(vport))
+
+	if (lpfc_els_chk_latt(vport)) {
+		skip_recovery = 1;
 		goto out;
+	}
 
+	/* Check to see if link went down during discovery */
 	if (ndlp->nlp_flag & NLP_TARGET_REMOVE) {
 	        /* NLP_EVT_DEVICE_RM should unregister the RPI
 		 * which should abort all outstanding IOs.
 		 */
 		lpfc_disc_state_machine(vport, ndlp, cmdiocb,
 					NLP_EVT_DEVICE_RM);
+		skip_recovery = 1;
 		goto out;
 	}
 
 	if (irsp->ulpStatus) {
 		/* Check for retry */
-		if (lpfc_els_retry(phba, cmdiocb, rspiocb))
+		if (lpfc_els_retry(phba, cmdiocb, rspiocb)) {
 			/* ELS command is being retried */
+			skip_recovery = 1;
 			goto out;
+		}
 		/* LOGO failed */
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
 				 "2756 LOGO failure DID:%06X Status:x%x/x%x\n",
 				 ndlp->nlp_DID, irsp->ulpStatus,
 				 irsp->un.ulpWord[4]);
 		/* Do not call DSM for lpfc_els_abort'ed ELS cmds */
-		if (lpfc_error_lost_link(irsp))
+		if (lpfc_error_lost_link(irsp)) {
+			skip_recovery = 1;
 			goto out;
-		else
-			lpfc_disc_state_machine(vport, ndlp, cmdiocb,
-						NLP_EVT_CMPL_LOGO);
-	} else
-		/* Good status, call state machine.
-		 * This will unregister the rpi if needed.
-		 */
-		lpfc_disc_state_machine(vport, ndlp, cmdiocb,
-					NLP_EVT_CMPL_LOGO);
+		}
+	}
+
+	/* Call state machine. This will unregister the rpi if needed. */
+	lpfc_disc_state_machine(vport, ndlp, cmdiocb, NLP_EVT_CMPL_LOGO);
+
 out:
 	lpfc_els_free_iocb(phba, cmdiocb);
 	/* If we are in pt2pt mode, we could rcv new S_ID on PLOGI */
@@ -2436,9 +2464,30 @@ out:
 			if (lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT) ==
 				MBX_NOT_FINISHED) {
 				mempool_free(mbox, phba->mbox_mem_pool);
+				skip_recovery = 1;
 			}
 		}
 	}
+
+	/*
+	 * If the node is a target, the handling attempts to recover the port.
+	 * For any other port type, the rpi is unregistered as an implicit
+	 * LOGO.
+	 */
+	if ((ndlp->nlp_type & NLP_FCP_TARGET) && (skip_recovery == 0)) {
+		lpfc_cancel_retry_delay_tmo(vport, ndlp);
+		spin_lock_irqsave(shost->host_lock, flags);
+		ndlp->nlp_flag |= NLP_NPR_2B_DISC;
+		spin_unlock_irqrestore(shost->host_lock, flags);
+
+		lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
+				 "3187 LOGO completes to NPort x%x: Start "
+				 "Recovery Data: x%x x%x x%x x%x\n",
+				 ndlp->nlp_DID, irsp->ulpStatus,
+				 irsp->un.ulpWord[4], irsp->ulpTimeout,
+				 vport->num_disc_nodes);
+		lpfc_disc_start(vport);
+	}
 	return;
 }
 
@@ -2501,10 +2550,27 @@ lpfc_issue_els_logo(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 		"Issue LOGO:      did:x%x",
 		ndlp->nlp_DID, 0, 0);
 
+	/*
+	 * If we are issuing a LOGO, we may try to recover the remote NPort
+	 * by issuing a PLOGI later. Even though we issue ELS cmds by the
+	 * VPI, if we have a valid RPI, and that RPI gets unreg'ed while
+	 * that ELS command is in-flight, the HBA returns a IOERR_INVALID_RPI
+	 * for that ELS cmd. To avoid this situation, lets get rid of the
+	 * RPI right now, before any ELS cmds are sent.
+	 */
+	spin_lock_irq(shost->host_lock);
+	ndlp->nlp_flag |= NLP_ISSUE_LOGO;
+	spin_unlock_irq(shost->host_lock);
+	if (lpfc_unreg_rpi(vport, ndlp)) {
+		lpfc_els_free_iocb(phba, elsiocb);
+		return 0;
+	}
+
 	phba->fc_stat.elsXmitLOGO++;
 	elsiocb->iocb_cmpl = lpfc_cmpl_els_logo;
 	spin_lock_irq(shost->host_lock);
 	ndlp->nlp_flag |= NLP_LOGO_SND;
+	ndlp->nlp_flag &= ~NLP_ISSUE_LOGO;
 	spin_unlock_irq(shost->host_lock);
 	rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, elsiocb, 0);
 
@@ -2920,7 +2986,7 @@ lpfc_els_retry_delay_handler(struct lpfc_nodelist *ndlp)
 	case ELS_CMD_LOGO:
 		if (!lpfc_issue_els_logo(vport, ndlp, retry)) {
 			ndlp->nlp_prev_state = ndlp->nlp_state;
-			lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+			lpfc_nlp_set_state(vport, ndlp, NLP_STE_LOGO_ISSUE);
 		}
 		break;
 	case ELS_CMD_FDISC:
@@ -3007,7 +3073,7 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		}
 		break;
 	case IOSTAT_LOCAL_REJECT:
-		switch ((irsp->un.ulpWord[4] & 0xff)) {
+		switch ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK)) {
 		case IOERR_LOOP_OPEN_FAILURE:
 			if (cmd == ELS_CMD_FLOGI) {
 				if (PCI_DEVICE_ID_HORNET ==
@@ -3214,7 +3280,8 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 
 		if (((cmd == ELS_CMD_PLOGI) || (cmd == ELS_CMD_ADISC)) &&
 			((irsp->ulpStatus != IOSTAT_LOCAL_REJECT) ||
-			((irsp->un.ulpWord[4] & 0xff) != IOERR_NO_RESOURCES))) {
+			((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) !=
+			IOERR_NO_RESOURCES))) {
 			/* Don't reset timer for no resources */
 
 			/* If discovery / RSCN timer is running, reset it */
@@ -3273,7 +3340,7 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 			return 1;
 		case ELS_CMD_LOGO:
 			ndlp->nlp_prev_state = ndlp->nlp_state;
-			lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+			lpfc_nlp_set_state(vport, ndlp, NLP_STE_LOGO_ISSUE);
 			lpfc_issue_els_logo(vport, ndlp, cmdiocb->retry);
 			return 1;
 		}
@@ -3533,13 +3600,17 @@ lpfc_mbx_cmpl_dflt_rpi(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 	lpfc_mbuf_free(phba, mp->virt, mp->phys);
 	kfree(mp);
 	mempool_free(pmb, phba->mbox_mem_pool);
-	if (ndlp && NLP_CHK_NODE_ACT(ndlp)) {
-		lpfc_nlp_put(ndlp);
-		/* This is the end of the default RPI cleanup logic for this
-		 * ndlp. If no other discovery threads are using this ndlp.
-		 * we should free all resources associated with it.
-		 */
-		lpfc_nlp_not_used(ndlp);
+	if (ndlp) {
+		if (NLP_CHK_NODE_ACT(ndlp)) {
+			lpfc_nlp_put(ndlp);
+			/* This is the end of the default RPI cleanup logic for
+			 * this ndlp. If no other discovery threads are using
+			 * this ndlp, free all resources associated with it.
+			 */
+			lpfc_nlp_not_used(ndlp);
+		} else {
+			lpfc_drop_node(ndlp->vport, ndlp);
+		}
 	}
 
 	return;
@@ -6803,7 +6874,8 @@ lpfc_els_unsol_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	if (icmd->ulpStatus == IOSTAT_NEED_BUFFER) {
 		lpfc_sli_hbqbuf_add_hbqs(phba, LPFC_ELS_HBQ);
 	} else if (icmd->ulpStatus == IOSTAT_LOCAL_REJECT &&
-	    (icmd->un.ulpWord[4] & 0xff) == IOERR_RCV_BUFFER_WAITING) {
+		   (icmd->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+		   IOERR_RCV_BUFFER_WAITING) {
 		phba->fc_stat.NoRcvBuf++;
 		/* Not enough posted buffers; Try posting more buffers */
 		if (!(phba->sli3_options & LPFC_SLI3_HBQ_ENABLED))
@@ -7985,3 +8057,47 @@ lpfc_sli4_els_xri_aborted(struct lpfc_hba *phba,
 	spin_unlock_irqrestore(&phba->hbalock, iflag);
 	return;
 }
+
+/* lpfc_sli_abts_recover_port - Recover a port that failed a BLS_ABORT req.
+ * @vport: pointer to virtual port object.
+ * @ndlp: nodelist pointer for the impacted node.
+ *
+ * The driver calls this routine in response to an SLI4 XRI ABORT CQE
+ * or an SLI3 ASYNC_STATUS_CN event from the port.  For either event,
+ * the driver is required to send a LOGO to the remote node before it
+ * attempts to recover its login to the remote node.
+ */
+void
+lpfc_sli_abts_recover_port(struct lpfc_vport *vport,
+			   struct lpfc_nodelist *ndlp)
+{
+	struct Scsi_Host *shost;
+	struct lpfc_hba *phba;
+	unsigned long flags = 0;
+
+	shost = lpfc_shost_from_vport(vport);
+	phba = vport->phba;
+	if (ndlp->nlp_state != NLP_STE_MAPPED_NODE) {
+		lpfc_printf_log(phba, KERN_INFO,
+				LOG_SLI, "3093 No rport recovery needed. "
+				"rport in state 0x%x\n", ndlp->nlp_state);
+		return;
+	}
+	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
+			"3094 Start rport recovery on shost id 0x%x "
+			"fc_id 0x%06x vpi 0x%x rpi 0x%x state 0x%x "
+			"flags 0x%x\n",
+			shost->host_no, ndlp->nlp_DID,
+			vport->vpi, ndlp->nlp_rpi, ndlp->nlp_state,
+			ndlp->nlp_flag);
+	/*
+	 * The rport is not responding.  Remove the FCP-2 flag to prevent
+	 * an ADISC in the follow-up recovery code.
+	 */
+	spin_lock_irqsave(shost->host_lock, flags);
+	ndlp->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
+	spin_unlock_irqrestore(shost->host_lock, flags);
+	lpfc_issue_els_logo(vport, ndlp, 0);
+	lpfc_nlp_set_state(vport, ndlp, NLP_STE_LOGO_ISSUE);
+}
+
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 9b4f92941dc..e9845d2ecf1 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -123,6 +123,10 @@ lpfc_dev_loss_tmo_callbk(struct fc_rport *rport)
 		"rport devlosscb: sid:x%x did:x%x flg:x%x",
 		ndlp->nlp_sid, ndlp->nlp_DID, ndlp->nlp_flag);
 
+	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+			 "3181 dev_loss_callbk x%06x, rport %p flg x%x\n",
+			 ndlp->nlp_DID, ndlp->rport, ndlp->nlp_flag);
+
 	/* Don't defer this if we are in the process of deleting the vport
 	 * or unloading the driver. The unload will cleanup the node
 	 * appropriately we just need to cleanup the ndlp rport info here.
@@ -142,6 +146,15 @@ lpfc_dev_loss_tmo_callbk(struct fc_rport *rport)
 	if (ndlp->nlp_state == NLP_STE_MAPPED_NODE)
 		return;
 
+	if (ndlp->nlp_type & NLP_FABRIC) {
+
+		/* If the WWPN of the rport and ndlp don't match, ignore it */
+		if (rport->port_name != wwn_to_u64(ndlp->nlp_portname.u.wwn)) {
+			put_device(&rport->dev);
+			return;
+		}
+	}
+
 	evtp = &ndlp->dev_loss_evt;
 
 	if (!list_empty(&evtp->evt_listp))
@@ -202,6 +215,10 @@ lpfc_dev_loss_tmo_handler(struct lpfc_nodelist *ndlp)
 		"rport devlosstmo:did:x%x type:x%x id:x%x",
 		ndlp->nlp_DID, ndlp->nlp_type, rport->scsi_target_id);
 
+	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+			 "3182 dev_loss_tmo_handler x%06x, rport %p flg x%x\n",
+			 ndlp->nlp_DID, ndlp->rport, ndlp->nlp_flag);
+
 	/* Don't defer this if we are in the process of deleting the vport
 	 * or unloading the driver. The unload will cleanup the node
 	 * appropriately we just need to cleanup the ndlp rport info here.
@@ -3492,7 +3509,7 @@ lpfc_create_static_vport(struct lpfc_hba *phba)
 	LPFC_MBOXQ_t *pmb = NULL;
 	MAILBOX_t *mb;
 	struct static_vport_info *vport_info;
-	int rc = 0, i;
+	int mbx_wait_rc = 0, i;
 	struct fc_vport_identifiers vport_id;
 	struct fc_vport *new_fc_vport;
 	struct Scsi_Host *shost;
@@ -3509,7 +3526,7 @@ lpfc_create_static_vport(struct lpfc_hba *phba)
 				" allocate mailbox memory\n");
 		return;
 	}
-
+	memset(pmb, 0, sizeof(LPFC_MBOXQ_t));
 	mb = &pmb->u.mb;
 
 	vport_info = kzalloc(sizeof(struct static_vport_info), GFP_KERNEL);
@@ -3523,24 +3540,31 @@ lpfc_create_static_vport(struct lpfc_hba *phba)
 
 	vport_buff = (uint8_t *) vport_info;
 	do {
+		/* free dma buffer from previous round */
+		if (pmb->context1) {
+			mp = (struct lpfc_dmabuf *)pmb->context1;
+			lpfc_mbuf_free(phba, mp->virt, mp->phys);
+			kfree(mp);
+		}
 		if (lpfc_dump_static_vport(phba, pmb, offset))
 			goto out;
 
 		pmb->vport = phba->pport;
-		rc = lpfc_sli_issue_mbox_wait(phba, pmb, LPFC_MBOX_TMO);
+		mbx_wait_rc = lpfc_sli_issue_mbox_wait(phba, pmb,
+							LPFC_MBOX_TMO);
 
-		if ((rc != MBX_SUCCESS) || mb->mbxStatus) {
+		if ((mbx_wait_rc != MBX_SUCCESS) || mb->mbxStatus) {
 			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
 				"0544 lpfc_create_static_vport failed to"
 				" issue dump mailbox command ret 0x%x "
 				"status 0x%x\n",
-				rc, mb->mbxStatus);
+				mbx_wait_rc, mb->mbxStatus);
 			goto out;
 		}
 
 		if (phba->sli_rev == LPFC_SLI_REV4) {
 			byte_count = pmb->u.mqe.un.mb_words[5];
-			mp = (struct lpfc_dmabuf *) pmb->context2;
+			mp = (struct lpfc_dmabuf *)pmb->context1;
 			if (byte_count > sizeof(struct static_vport_info) -
 					offset)
 				byte_count = sizeof(struct static_vport_info)
@@ -3604,9 +3628,9 @@ lpfc_create_static_vport(struct lpfc_hba *phba)
 
 out:
 	kfree(vport_info);
-	if (rc != MBX_TIMEOUT) {
-		if (pmb->context2) {
-			mp = (struct lpfc_dmabuf *) pmb->context2;
+	if (mbx_wait_rc != MBX_TIMEOUT) {
+		if (pmb->context1) {
+			mp = (struct lpfc_dmabuf *)pmb->context1;
 			lpfc_mbuf_free(phba, mp->virt, mp->phys);
 			kfree(mp);
 		}
@@ -3834,6 +3858,10 @@ lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	if (rport_ids.roles !=  FC_RPORT_ROLE_UNKNOWN)
 		fc_remote_port_rolechg(rport, rport_ids.roles);
 
+	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+			 "3183 rport register x%06x, rport %p role x%x\n",
+			 ndlp->nlp_DID, rport, rport_ids.roles);
+
 	if ((rport->scsi_target_id != -1) &&
 	    (rport->scsi_target_id < LPFC_MAX_TARGET)) {
 		ndlp->nlp_sid = rport->scsi_target_id;
@@ -3850,6 +3878,10 @@ lpfc_unregister_remote_port(struct lpfc_nodelist *ndlp)
 		"rport delete:    did:x%x flg:x%x type x%x",
 		ndlp->nlp_DID, ndlp->nlp_flag, ndlp->nlp_type);
 
+	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NODE,
+			 "3184 rport unregister x%06x, rport %p\n",
+			 ndlp->nlp_DID, rport);
+
 	fc_remote_port_delete(rport);
 
 	return;
@@ -3964,6 +3996,7 @@ lpfc_nlp_state_name(char *buffer, size_t size, int state)
 		[NLP_STE_ADISC_ISSUE] = "ADISC",
 		[NLP_STE_REG_LOGIN_ISSUE] = "REGLOGIN",
 		[NLP_STE_PRLI_ISSUE] = "PRLI",
+		[NLP_STE_LOGO_ISSUE] = "LOGO",
 		[NLP_STE_UNMAPPED_NODE] = "UNMAPPED",
 		[NLP_STE_MAPPED_NODE] = "MAPPED",
 		[NLP_STE_NPR_NODE] = "NPR",
@@ -4330,6 +4363,26 @@ lpfc_no_rpi(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 	return 0;
 }
 
+/**
+ * lpfc_nlp_logo_unreg - Unreg mailbox completion handler before LOGO
+ * @phba: Pointer to HBA context object.
+ * @pmb: Pointer to mailbox object.
+ *
+ * This function will issue an ELS LOGO command after completing
+ * the UNREG_RPI.
+ **/
+void
+lpfc_nlp_logo_unreg(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
+{
+	struct lpfc_vport  *vport = pmb->vport;
+	struct lpfc_nodelist *ndlp;
+
+	ndlp = (struct lpfc_nodelist *)(pmb->context1);
+	if (!ndlp)
+		return;
+	lpfc_issue_els_logo(vport, ndlp, 0);
+}
+
 /*
  * Free rpi associated with LPFC_NODELIST entry.
  * This routine is called from lpfc_freenode(), when we are removing
@@ -4354,9 +4407,16 @@ lpfc_unreg_rpi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 			rpi = ndlp->nlp_rpi;
 			if (phba->sli_rev == LPFC_SLI_REV4)
 				rpi = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi];
+
 			lpfc_unreg_login(phba, vport->vpi, rpi, mbox);
 			mbox->vport = vport;
-			mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
+			if (ndlp->nlp_flag & NLP_ISSUE_LOGO) {
+				mbox->context1 = ndlp;
+				mbox->mbox_cmpl = lpfc_nlp_logo_unreg;
+			} else {
+				mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
+			}
+
 			rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
 			if (rc == MBX_NOT_FINISHED)
 				mempool_free(mbox, phba->mbox_mem_pool);
@@ -4499,9 +4559,13 @@ lpfc_cleanup_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 		lpfc_disable_node(vport, ndlp);
 	}
 
+
+	/* Don't need to clean up REG_LOGIN64 cmds for Default RPI cleanup */
+
 	/* cleanup any ndlp on mbox q waiting for reglogin cmpl */
 	if ((mb = phba->sli.mbox_active)) {
 		if ((mb->u.mb.mbxCommand == MBX_REG_LOGIN64) &&
+		   !(mb->mbox_flag & LPFC_MBX_IMED_UNREG) &&
 		   (ndlp == (struct lpfc_nodelist *) mb->context2)) {
 			mb->context2 = NULL;
 			mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl;
@@ -4512,6 +4576,7 @@ lpfc_cleanup_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	/* Cleanup REG_LOGIN completions which are not yet processed */
 	list_for_each_entry(mb, &phba->sli.mboxq_cmpl, list) {
 		if ((mb->u.mb.mbxCommand != MBX_REG_LOGIN64) ||
+			(mb->mbox_flag & LPFC_MBX_IMED_UNREG) ||
 			(ndlp != (struct lpfc_nodelist *) mb->context2))
 			continue;
 
@@ -4521,6 +4586,7 @@ lpfc_cleanup_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 
 	list_for_each_entry_safe(mb, nextmb, &phba->sli.mboxq, list) {
 		if ((mb->u.mb.mbxCommand == MBX_REG_LOGIN64) &&
+		   !(mb->mbox_flag & LPFC_MBX_IMED_UNREG) &&
 		    (ndlp == (struct lpfc_nodelist *) mb->context2)) {
 			mp = (struct lpfc_dmabuf *) (mb->context1);
 			if (mp) {
@@ -4585,7 +4651,7 @@ lpfc_nlp_remove(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 				mbox->mbox_flag |= LPFC_MBX_IMED_UNREG;
 				mbox->mbox_cmpl = lpfc_mbx_cmpl_dflt_rpi;
 				mbox->vport = vport;
-				mbox->context2 = NULL;
+				mbox->context2 = ndlp;
 				rc =lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT);
 				if (rc == MBX_NOT_FINISHED) {
 					mempool_free(mbox, phba->mbox_mem_pool);
@@ -5365,9 +5431,17 @@ __lpfc_find_node(struct lpfc_vport *vport, node_filter filter, void *param)
 	struct lpfc_nodelist *ndlp;
 
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
-		if (filter(ndlp, param))
+		if (filter(ndlp, param)) {
+			lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,
+					 "3185 FIND node filter %p DID "
+					 "Data: x%p x%x x%x\n",
+					 filter, ndlp, ndlp->nlp_DID,
+					 ndlp->nlp_flag);
 			return ndlp;
+		}
 	}
+	lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE,
+			 "3186 FIND node filter %p NOT FOUND.\n", filter);
 	return NULL;
 }
 
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index 41bb1d2fb62..7398ca862e9 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -1188,8 +1188,8 @@ typedef struct {
  */
 
 /* Number of rings currently used and available. */
-#define MAX_CONFIGURED_RINGS     3
-#define MAX_RINGS                4
+#define MAX_SLI3_CONFIGURED_RINGS     3
+#define MAX_SLI3_RINGS                4
 
 /* IOCB / Mailbox is owned by FireFly */
 #define OWN_CHIP        1
@@ -1251,6 +1251,8 @@ typedef struct {
 #define PCI_VENDOR_ID_SERVERENGINE  0x19a2
 #define PCI_DEVICE_ID_TIGERSHARK    0x0704
 #define PCI_DEVICE_ID_TOMCAT        0x0714
+#define PCI_DEVICE_ID_SKYHAWK       0x0724
+#define PCI_DEVICE_ID_SKYHAWK_VF    0x072c
 
 #define JEDEC_ID_ADDRESS            0x0080001c
 #define FIREFLY_JEDEC_ID            0x1ACC
@@ -1458,6 +1460,7 @@ typedef struct {		/* FireFly BIU registers */
 #define MBX_UNREG_FCFI	    0xA2
 #define MBX_INIT_VFI        0xA3
 #define MBX_INIT_VPI        0xA4
+#define MBX_ACCESS_VDATA    0xA5
 
 #define MBX_AUTH_PORT       0xF8
 #define MBX_SECURITY_MGMT   0xF9
@@ -2991,7 +2994,7 @@ typedef struct _PCB {
 
 	uint32_t pgpAddrLow;
 	uint32_t pgpAddrHigh;
-	SLI2_RDSC rdsc[MAX_RINGS];
+	SLI2_RDSC rdsc[MAX_SLI3_RINGS];
 } PCB_t;
 
 /* NEW_FEATURE */
@@ -3101,18 +3104,18 @@ struct lpfc_pgp {
 
 struct sli2_desc {
 	uint32_t unused1[16];
-	struct lpfc_hgp host[MAX_RINGS];
-	struct lpfc_pgp port[MAX_RINGS];
+	struct lpfc_hgp host[MAX_SLI3_RINGS];
+	struct lpfc_pgp port[MAX_SLI3_RINGS];
 };
 
 struct sli3_desc {
-	struct lpfc_hgp host[MAX_RINGS];
+	struct lpfc_hgp host[MAX_SLI3_RINGS];
 	uint32_t reserved[8];
 	uint32_t hbq_put[16];
 };
 
 struct sli3_pgp {
-	struct lpfc_pgp port[MAX_RINGS];
+	struct lpfc_pgp port[MAX_SLI3_RINGS];
 	uint32_t hbq_get[16];
 };
 
@@ -3242,6 +3245,7 @@ typedef struct {
 #define IOERR_SLI_DOWN                0x101  /* ulpStatus  - Driver defined */
 #define IOERR_SLI_BRESET              0x102
 #define IOERR_SLI_ABORTED             0x103
+#define IOERR_PARAM_MASK              0x1ff
 } PARM_ERR;
 
 typedef union {
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index 953603a7a43..834b699cac7 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -187,11 +187,17 @@ struct lpfc_sli_intf {
 /* Active interrupt test count */
 #define LPFC_ACT_INTR_CNT	4
 
+/* Algrithmns for scheduling FCP commands to WQs */
+#define	LPFC_FCP_SCHED_ROUND_ROBIN	0
+#define	LPFC_FCP_SCHED_BY_CPU		1
+
 /* Delay Multiplier constant */
 #define LPFC_DMULT_CONST       651042
-#define LPFC_MIM_IMAX          636
-#define LPFC_FP_DEF_IMAX       10000
-#define LPFC_SP_DEF_IMAX       10000
+
+/* Configuration of Interrupts / sec for entire HBA port */
+#define LPFC_MIN_IMAX          5000
+#define LPFC_MAX_IMAX          5000000
+#define LPFC_DEF_IMAX          50000
 
 /* PORT_CAPABILITIES constants. */
 #define LPFC_MAX_SUPPORTED_PAGES	8
@@ -338,7 +344,7 @@ struct lpfc_cqe {
  * Define mask value for xri_aborted and wcqe completed CQE extended status.
  * Currently, extended status is limited to 9 bits (0x0 -> 0x103) .
  */
-#define WCQE_PARAM_MASK		0x1FF;
+#define WCQE_PARAM_MASK		0x1FF
 
 /* completion queue entry for wqe completions */
 struct lpfc_wcqe_complete {
@@ -880,13 +886,19 @@ struct mbox_header {
 #define LPFC_MBOX_OPCODE_EQ_DESTROY			0x37
 #define LPFC_MBOX_OPCODE_QUERY_FW_CFG			0x3A
 #define LPFC_MBOX_OPCODE_FUNCTION_RESET			0x3D
+#define LPFC_MBOX_OPCODE_SET_PHYSICAL_LINK_CONFIG	0x3E
+#define LPFC_MBOX_OPCODE_SET_BOOT_CONFIG		0x43
 #define LPFC_MBOX_OPCODE_GET_PORT_NAME			0x4D
 #define LPFC_MBOX_OPCODE_MQ_CREATE_EXT			0x5A
+#define LPFC_MBOX_OPCODE_GET_VPD_DATA			0x5B
+#define LPFC_MBOX_OPCODE_SEND_ACTIVATION		0x73
+#define LPFC_MBOX_OPCODE_RESET_LICENSES			0x74
 #define LPFC_MBOX_OPCODE_GET_RSRC_EXTENT_INFO		0x9A
 #define LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT		0x9B
 #define LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT		0x9C
 #define LPFC_MBOX_OPCODE_DEALLOC_RSRC_EXTENT		0x9D
 #define LPFC_MBOX_OPCODE_GET_FUNCTION_CONFIG		0xA0
+#define LPFC_MBOX_OPCODE_GET_PROFILE_CAPACITIES		0xA1
 #define LPFC_MBOX_OPCODE_GET_PROFILE_CONFIG		0xA4
 #define LPFC_MBOX_OPCODE_SET_PROFILE_CONFIG		0xA5
 #define LPFC_MBOX_OPCODE_GET_PROFILE_LIST		0xA6
@@ -1382,6 +1394,11 @@ struct lpfc_mbx_set_link_diag_state {
 #define lpfc_mbx_set_diag_state_diag_SHIFT	0
 #define lpfc_mbx_set_diag_state_diag_MASK	0x00000001
 #define lpfc_mbx_set_diag_state_diag_WORD	word0
+#define lpfc_mbx_set_diag_state_diag_bit_valid_SHIFT	2
+#define lpfc_mbx_set_diag_state_diag_bit_valid_MASK	0x00000001
+#define lpfc_mbx_set_diag_state_diag_bit_valid_WORD	word0
+#define LPFC_DIAG_STATE_DIAG_BIT_VALID_NO_CHANGE	0
+#define LPFC_DIAG_STATE_DIAG_BIT_VALID_CHANGE		1
 #define lpfc_mbx_set_diag_state_link_num_SHIFT	16
 #define lpfc_mbx_set_diag_state_link_num_MASK	0x0000003F
 #define lpfc_mbx_set_diag_state_link_num_WORD	word0
@@ -2556,7 +2573,7 @@ struct lpfc_mbx_get_sli4_parameters {
 };
 
 struct lpfc_rscr_desc_generic {
-#define LPFC_RSRC_DESC_WSIZE			18
+#define LPFC_RSRC_DESC_WSIZE			22
 	uint32_t desc[LPFC_RSRC_DESC_WSIZE];
 };
 
@@ -2566,6 +2583,9 @@ struct lpfc_rsrc_desc_pcie {
 #define lpfc_rsrc_desc_pcie_type_MASK		0x000000ff
 #define lpfc_rsrc_desc_pcie_type_WORD		word0
 #define LPFC_RSRC_DESC_TYPE_PCIE		0x40
+#define lpfc_rsrc_desc_pcie_length_SHIFT	8
+#define lpfc_rsrc_desc_pcie_length_MASK		0x000000ff
+#define lpfc_rsrc_desc_pcie_length_WORD		word0
 	uint32_t word1;
 #define lpfc_rsrc_desc_pcie_pfnum_SHIFT		0
 #define lpfc_rsrc_desc_pcie_pfnum_MASK		0x000000ff
@@ -2593,6 +2613,12 @@ struct lpfc_rsrc_desc_fcfcoe {
 #define lpfc_rsrc_desc_fcfcoe_type_MASK		0x000000ff
 #define lpfc_rsrc_desc_fcfcoe_type_WORD		word0
 #define LPFC_RSRC_DESC_TYPE_FCFCOE		0x43
+#define lpfc_rsrc_desc_fcfcoe_length_SHIFT	8
+#define lpfc_rsrc_desc_fcfcoe_length_MASK	0x000000ff
+#define lpfc_rsrc_desc_fcfcoe_length_WORD	word0
+#define LPFC_RSRC_DESC_TYPE_FCFCOE_V0_RSVD	0
+#define LPFC_RSRC_DESC_TYPE_FCFCOE_V0_LENGTH	72
+#define LPFC_RSRC_DESC_TYPE_FCFCOE_V1_LENGTH	88
 	uint32_t word1;
 #define lpfc_rsrc_desc_fcfcoe_vfnum_SHIFT	0
 #define lpfc_rsrc_desc_fcfcoe_vfnum_MASK	0x000000ff
@@ -2651,6 +2677,12 @@ struct lpfc_rsrc_desc_fcfcoe {
 #define lpfc_rsrc_desc_fcfcoe_eq_cnt_SHIFT	16
 #define lpfc_rsrc_desc_fcfcoe_eq_cnt_MASK	0x0000ffff
 #define lpfc_rsrc_desc_fcfcoe_eq_cnt_WORD	word13
+/* extended FC/FCoE Resource Descriptor when length = 88 bytes */
+	uint32_t bw_min;
+	uint32_t bw_max;
+	uint32_t iops_min;
+	uint32_t iops_max;
+	uint32_t reserved[4];
 };
 
 struct lpfc_func_cfg {
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 628a703abdd..8a55a586dd6 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -480,11 +480,11 @@ lpfc_config_port_post(struct lpfc_hba *phba)
 	phba->link_state = LPFC_LINK_DOWN;
 
 	/* Only process IOCBs on ELS ring till hba_state is READY */
-	if (psli->ring[psli->extra_ring].cmdringaddr)
+	if (psli->ring[psli->extra_ring].sli.sli3.cmdringaddr)
 		psli->ring[psli->extra_ring].flag |= LPFC_STOP_IOCB_EVENT;
-	if (psli->ring[psli->fcp_ring].cmdringaddr)
+	if (psli->ring[psli->fcp_ring].sli.sli3.cmdringaddr)
 		psli->ring[psli->fcp_ring].flag |= LPFC_STOP_IOCB_EVENT;
-	if (psli->ring[psli->next_ring].cmdringaddr)
+	if (psli->ring[psli->next_ring].sli.sli3.cmdringaddr)
 		psli->ring[psli->next_ring].flag |= LPFC_STOP_IOCB_EVENT;
 
 	/* Post receive buffers for desired rings */
@@ -2059,6 +2059,11 @@ lpfc_get_hba_model_desc(struct lpfc_hba *phba, uint8_t *mdp, uint8_t *descp)
 		oneConnect = 1;
 		m = (typeof(m)){"OCe15100", "PCIe", "FCoE"};
 		break;
+	case PCI_DEVICE_ID_SKYHAWK:
+	case PCI_DEVICE_ID_SKYHAWK_VF:
+		oneConnect = 1;
+		m = (typeof(m)){"OCe14000", "PCIe", "FCoE"};
+		break;
 	default:
 		m = (typeof(m)){"Unknown", "", ""};
 		break;
@@ -4546,6 +4551,13 @@ lpfc_sli_driver_resource_setup(struct lpfc_hba *phba)
 			phba->cfg_sg_seg_cnt = LPFC_DEFAULT_MENLO_SG_SEG_CNT;
 	}
 
+	if (!phba->sli.ring)
+		phba->sli.ring = (struct lpfc_sli_ring *)
+			kzalloc(LPFC_SLI3_MAX_RING *
+			sizeof(struct lpfc_sli_ring), GFP_KERNEL);
+	if (!phba->sli.ring)
+		return -ENOMEM;
+
 	/*
 	 * Since the sg_tablesize is module parameter, the sg_dma_buf_size
 	 * used to create the sg_dma_buf_pool must be dynamically calculated.
@@ -4690,6 +4702,10 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 	/* Get all the module params for configuring this host */
 	lpfc_get_cfgparam(phba);
 	phba->max_vpi = LPFC_MAX_VPI;
+
+	/* Eventually cfg_fcp_eq_count / cfg_fcp_wq_count will be depricated */
+	phba->cfg_fcp_io_channel = phba->cfg_fcp_eq_count;
+
 	/* This will be set to correct value after the read_config mbox */
 	phba->max_vports = 0;
 
@@ -4705,6 +4721,16 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 		sges_per_segment = 2;
 
 	/*
+	 * For SLI4, instead of using ring 0 (LPFC_FCP_RING) for FCP commands
+	 * we will associate a new ring, for each FCP fastpath EQ/CQ/WQ tuple.
+	 */
+	if (!phba->sli.ring)
+		phba->sli.ring = kzalloc(
+			(LPFC_SLI3_MAX_RING + phba->cfg_fcp_io_channel) *
+			sizeof(struct lpfc_sli_ring), GFP_KERNEL);
+	if (!phba->sli.ring)
+		return -ENOMEM;
+	/*
 	 * Since the sg_tablesize is module parameter, the sg_dma_buf_size
 	 * used to create the sg_dma_buf_pool must be dynamically calculated.
 	 * 2 segments are added since the IOCB needs a command and response bde.
@@ -4909,21 +4935,15 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba)
 		goto out_remove_rpi_hdrs;
 	}
 
-	/*
-	 * The cfg_fcp_eq_count can be zero whenever there is exactly one
-	 * interrupt vector.  This is not an error
-	 */
-	if (phba->cfg_fcp_eq_count) {
-		phba->sli4_hba.fcp_eq_hdl =
-				kzalloc((sizeof(struct lpfc_fcp_eq_hdl) *
-				    phba->cfg_fcp_eq_count), GFP_KERNEL);
-		if (!phba->sli4_hba.fcp_eq_hdl) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"2572 Failed allocate memory for "
-					"fast-path per-EQ handle array\n");
-			rc = -ENOMEM;
-			goto out_free_fcf_rr_bmask;
-		}
+	phba->sli4_hba.fcp_eq_hdl =
+			kzalloc((sizeof(struct lpfc_fcp_eq_hdl) *
+			    phba->cfg_fcp_io_channel), GFP_KERNEL);
+	if (!phba->sli4_hba.fcp_eq_hdl) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"2572 Failed allocate memory for "
+				"fast-path per-EQ handle array\n");
+		rc = -ENOMEM;
+		goto out_free_fcf_rr_bmask;
 	}
 
 	phba->sli4_hba.msix_entries = kzalloc((sizeof(struct msix_entry) *
@@ -5550,6 +5570,10 @@ lpfc_hba_free(struct lpfc_hba *phba)
 	/* Release the driver assigned board number */
 	idr_remove(&lpfc_hba_index, phba->brd_no);
 
+	/* Free memory allocated with sli rings */
+	kfree(phba->sli.ring);
+	phba->sli.ring = NULL;
+
 	kfree(phba);
 	return;
 }
@@ -6275,8 +6299,9 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 	uint32_t shdr_status, shdr_add_status;
 	struct lpfc_mbx_get_func_cfg *get_func_cfg;
 	struct lpfc_rsrc_desc_fcfcoe *desc;
+	char *pdesc_0;
 	uint32_t desc_count;
-	int length, i, rc = 0;
+	int length, i, rc = 0, rc2;
 
 	pmb = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
 	if (!pmb) {
@@ -6388,18 +6413,17 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 			 LPFC_MBOX_OPCODE_GET_FUNCTION_CONFIG,
 			 length, LPFC_SLI4_MBX_EMBED);
 
-	rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
+	rc2 = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL);
 	shdr = (union lpfc_sli4_cfg_shdr *)
 				&pmb->u.mqe.un.sli4_config.header.cfg_shdr;
 	shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response);
 	shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response);
-	if (rc || shdr_status || shdr_add_status) {
+	if (rc2 || shdr_status || shdr_add_status) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"3026 Mailbox failed , mbxCmd x%x "
 				"GET_FUNCTION_CONFIG, mbxStatus x%x\n",
 				bf_get(lpfc_mqe_command, &pmb->u.mqe),
 				bf_get(lpfc_mqe_status, &pmb->u.mqe));
-		rc = -EIO;
 		goto read_cfg_out;
 	}
 
@@ -6407,11 +6431,18 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 	get_func_cfg = &pmb->u.mqe.un.get_func_cfg;
 	desc_count = get_func_cfg->func_cfg.rsrc_desc_count;
 
+	pdesc_0 = (char *)&get_func_cfg->func_cfg.desc[0];
+	desc = (struct lpfc_rsrc_desc_fcfcoe *)pdesc_0;
+	length = bf_get(lpfc_rsrc_desc_fcfcoe_length, desc);
+	if (length == LPFC_RSRC_DESC_TYPE_FCFCOE_V0_RSVD)
+		length = LPFC_RSRC_DESC_TYPE_FCFCOE_V0_LENGTH;
+	else if (length != LPFC_RSRC_DESC_TYPE_FCFCOE_V1_LENGTH)
+		goto read_cfg_out;
+
 	for (i = 0; i < LPFC_RSRC_DESC_MAX_NUM; i++) {
-		desc = (struct lpfc_rsrc_desc_fcfcoe *)
-			&get_func_cfg->func_cfg.desc[i];
+		desc = (struct lpfc_rsrc_desc_fcfcoe *)(pdesc_0 + length * i);
 		if (LPFC_RSRC_DESC_TYPE_FCFCOE ==
-		    bf_get(lpfc_rsrc_desc_pcie_type, desc)) {
+		    bf_get(lpfc_rsrc_desc_fcfcoe_type, desc)) {
 			phba->sli4_hba.iov.pf_number =
 				bf_get(lpfc_rsrc_desc_fcfcoe_pfnum, desc);
 			phba->sli4_hba.iov.vf_number =
@@ -6425,13 +6456,11 @@ lpfc_sli4_read_config(struct lpfc_hba *phba)
 				"3027 GET_FUNCTION_CONFIG: pf_number:%d, "
 				"vf_number:%d\n", phba->sli4_hba.iov.pf_number,
 				phba->sli4_hba.iov.vf_number);
-	else {
+	else
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"3028 GET_FUNCTION_CONFIG: failed to find "
 				"Resrouce Descriptor:x%x\n",
 				LPFC_RSRC_DESC_TYPE_FCFCOE);
-		rc = -EIO;
-	}
 
 read_cfg_out:
 	mempool_free(pmb, phba->mbox_mem_pool);
@@ -6512,53 +6541,40 @@ lpfc_setup_endian_order(struct lpfc_hba *phba)
 static int
 lpfc_sli4_queue_verify(struct lpfc_hba *phba)
 {
-	int cfg_fcp_wq_count;
-	int cfg_fcp_eq_count;
+	int cfg_fcp_io_channel;
+	uint32_t cpu;
+	uint32_t i = 0;
+
 
 	/*
-	 * Sanity check for confiugred queue parameters against the run-time
+	 * Sanity check for configured queue parameters against the run-time
 	 * device parameters
 	 */
 
-	/* Sanity check on FCP fast-path WQ parameters */
-	cfg_fcp_wq_count = phba->cfg_fcp_wq_count;
-	if (cfg_fcp_wq_count >
-	    (phba->sli4_hba.max_cfg_param.max_wq - LPFC_SP_WQN_DEF)) {
-		cfg_fcp_wq_count = phba->sli4_hba.max_cfg_param.max_wq -
-				   LPFC_SP_WQN_DEF;
-		if (cfg_fcp_wq_count < LPFC_FP_WQN_MIN) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"2581 Not enough WQs (%d) from "
-					"the pci function for supporting "
-					"FCP WQs (%d)\n",
-					phba->sli4_hba.max_cfg_param.max_wq,
-					phba->cfg_fcp_wq_count);
-			goto out_error;
-		}
+	/* Sanity check on HBA EQ parameters */
+	cfg_fcp_io_channel = phba->cfg_fcp_io_channel;
+
+	/* It doesn't make sense to have more io channels then CPUs */
+	for_each_online_cpu(cpu) {
+		i++;
+	}
+	if (i < cfg_fcp_io_channel) {
 		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
-				"2582 Not enough WQs (%d) from the pci "
-				"function for supporting the requested "
-				"FCP WQs (%d), the actual FCP WQs can "
-				"be supported: %d\n",
-				phba->sli4_hba.max_cfg_param.max_wq,
-				phba->cfg_fcp_wq_count, cfg_fcp_wq_count);
-	}
-	/* The actual number of FCP work queues adopted */
-	phba->cfg_fcp_wq_count = cfg_fcp_wq_count;
-
-	/* Sanity check on FCP fast-path EQ parameters */
-	cfg_fcp_eq_count = phba->cfg_fcp_eq_count;
-	if (cfg_fcp_eq_count >
-	    (phba->sli4_hba.max_cfg_param.max_eq - LPFC_SP_EQN_DEF)) {
-		cfg_fcp_eq_count = phba->sli4_hba.max_cfg_param.max_eq -
-				   LPFC_SP_EQN_DEF;
-		if (cfg_fcp_eq_count < LPFC_FP_EQN_MIN) {
+				"3188 Reducing IO channels to match number of "
+				"CPUs: from %d to %d\n", cfg_fcp_io_channel, i);
+		cfg_fcp_io_channel = i;
+	}
+
+	if (cfg_fcp_io_channel >
+	    phba->sli4_hba.max_cfg_param.max_eq) {
+		cfg_fcp_io_channel = phba->sli4_hba.max_cfg_param.max_eq;
+		if (cfg_fcp_io_channel < LPFC_FCP_IO_CHAN_MIN) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"2574 Not enough EQs (%d) from the "
 					"pci function for supporting FCP "
 					"EQs (%d)\n",
 					phba->sli4_hba.max_cfg_param.max_eq,
-					phba->cfg_fcp_eq_count);
+					phba->cfg_fcp_io_channel);
 			goto out_error;
 		}
 		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
@@ -6567,22 +6583,16 @@ lpfc_sli4_queue_verify(struct lpfc_hba *phba)
 				"FCP EQs (%d), the actual FCP EQs can "
 				"be supported: %d\n",
 				phba->sli4_hba.max_cfg_param.max_eq,
-				phba->cfg_fcp_eq_count, cfg_fcp_eq_count);
-	}
-	/* It does not make sense to have more EQs than WQs */
-	if (cfg_fcp_eq_count > phba->cfg_fcp_wq_count) {
-		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
-				"2593 The FCP EQ count(%d) cannot be greater "
-				"than the FCP WQ count(%d), limiting the "
-				"FCP EQ count to %d\n", cfg_fcp_eq_count,
-				phba->cfg_fcp_wq_count,
-				phba->cfg_fcp_wq_count);
-		cfg_fcp_eq_count = phba->cfg_fcp_wq_count;
+				phba->cfg_fcp_io_channel, cfg_fcp_io_channel);
 	}
+
+	/* Eventually cfg_fcp_eq_count / cfg_fcp_wq_count will be depricated */
+
 	/* The actual number of FCP event queues adopted */
-	phba->cfg_fcp_eq_count = cfg_fcp_eq_count;
-	/* The overall number of event queues used */
-	phba->sli4_hba.cfg_eqn = phba->cfg_fcp_eq_count + LPFC_SP_EQN_DEF;
+	phba->cfg_fcp_eq_count = cfg_fcp_io_channel;
+	phba->cfg_fcp_wq_count = cfg_fcp_io_channel;
+	phba->cfg_fcp_io_channel = cfg_fcp_io_channel;
+	phba->sli4_hba.cfg_eqn = cfg_fcp_io_channel;
 
 	/* Get EQ depth from module parameter, fake the default for now */
 	phba->sli4_hba.eq_esize = LPFC_EQE_SIZE_4B;
@@ -6615,50 +6625,104 @@ int
 lpfc_sli4_queue_create(struct lpfc_hba *phba)
 {
 	struct lpfc_queue *qdesc;
-	int fcp_eqidx, fcp_cqidx, fcp_wqidx;
+	int idx;
 
 	/*
-	 * Create Event Queues (EQs)
+	 * Create HBA Record arrays.
 	 */
+	if (!phba->cfg_fcp_io_channel)
+		return -ERANGE;
 
-	/* Create slow path event queue */
-	qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.eq_esize,
-				      phba->sli4_hba.eq_ecount);
-	if (!qdesc) {
+	phba->sli4_hba.mq_esize = LPFC_MQE_SIZE;
+	phba->sli4_hba.mq_ecount = LPFC_MQE_DEF_COUNT;
+	phba->sli4_hba.wq_esize = LPFC_WQE_SIZE;
+	phba->sli4_hba.wq_ecount = LPFC_WQE_DEF_COUNT;
+	phba->sli4_hba.rq_esize = LPFC_RQE_SIZE;
+	phba->sli4_hba.rq_ecount = LPFC_RQE_DEF_COUNT;
+
+	phba->sli4_hba.hba_eq =  kzalloc((sizeof(struct lpfc_queue *) *
+				phba->cfg_fcp_io_channel), GFP_KERNEL);
+	if (!phba->sli4_hba.hba_eq) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"2576 Failed allocate memory for "
+			"fast-path EQ record array\n");
+		goto out_error;
+	}
+
+	phba->sli4_hba.fcp_cq = kzalloc((sizeof(struct lpfc_queue *) *
+				phba->cfg_fcp_io_channel), GFP_KERNEL);
+	if (!phba->sli4_hba.fcp_cq) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"0496 Failed allocate slow-path EQ\n");
+				"2577 Failed allocate memory for fast-path "
+				"CQ record array\n");
+		goto out_error;
+	}
+
+	phba->sli4_hba.fcp_wq = kzalloc((sizeof(struct lpfc_queue *) *
+				phba->cfg_fcp_io_channel), GFP_KERNEL);
+	if (!phba->sli4_hba.fcp_wq) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"2578 Failed allocate memory for fast-path "
+				"WQ record array\n");
 		goto out_error;
 	}
-	phba->sli4_hba.sp_eq = qdesc;
 
 	/*
-	 * Create fast-path FCP Event Queue(s).  The cfg_fcp_eq_count can be
-	 * zero whenever there is exactly one interrupt vector.  This is not
-	 * an error.
+	 * Since the first EQ can have multiple CQs associated with it,
+	 * this array is used to quickly see if we have a FCP fast-path
+	 * CQ match.
 	 */
-	if (phba->cfg_fcp_eq_count) {
-		phba->sli4_hba.fp_eq = kzalloc((sizeof(struct lpfc_queue *) *
-				       phba->cfg_fcp_eq_count), GFP_KERNEL);
-		if (!phba->sli4_hba.fp_eq) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"2576 Failed allocate memory for "
-					"fast-path EQ record array\n");
-			goto out_free_sp_eq;
-		}
+	phba->sli4_hba.fcp_cq_map = kzalloc((sizeof(uint16_t) *
+					 phba->cfg_fcp_io_channel), GFP_KERNEL);
+	if (!phba->sli4_hba.fcp_cq_map) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"2545 Failed allocate memory for fast-path "
+				"CQ map\n");
+		goto out_error;
 	}
-	for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_eq_count; fcp_eqidx++) {
+
+	/*
+	 * Create HBA Event Queues (EQs).  The cfg_fcp_io_channel specifies
+	 * how many EQs to create.
+	 */
+	for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
+
+		/* Create EQs */
 		qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.eq_esize,
 					      phba->sli4_hba.eq_ecount);
 		if (!qdesc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0497 Failed allocate fast-path EQ\n");
-			goto out_free_fp_eq;
+					"0497 Failed allocate EQ (%d)\n", idx);
+			goto out_error;
 		}
-		phba->sli4_hba.fp_eq[fcp_eqidx] = qdesc;
+		phba->sli4_hba.hba_eq[idx] = qdesc;
+
+		/* Create Fast Path FCP CQs */
+		qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
+					      phba->sli4_hba.cq_ecount);
+		if (!qdesc) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0499 Failed allocate fast-path FCP "
+					"CQ (%d)\n", idx);
+			goto out_error;
+		}
+		phba->sli4_hba.fcp_cq[idx] = qdesc;
+
+		/* Create Fast Path FCP WQs */
+		qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.wq_esize,
+					      phba->sli4_hba.wq_ecount);
+		if (!qdesc) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0503 Failed allocate fast-path FCP "
+					"WQ (%d)\n", idx);
+			goto out_error;
+		}
+		phba->sli4_hba.fcp_wq[idx] = qdesc;
 	}
 
+
 	/*
-	 * Create Complete Queues (CQs)
+	 * Create Slow Path Completion Queues (CQs)
 	 */
 
 	/* Create slow-path Mailbox Command Complete Queue */
@@ -6667,7 +6731,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0500 Failed allocate slow-path mailbox CQ\n");
-		goto out_free_fp_eq;
+		goto out_error;
 	}
 	phba->sli4_hba.mbx_cq = qdesc;
 
@@ -6677,59 +6741,29 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0501 Failed allocate slow-path ELS CQ\n");
-		goto out_free_mbx_cq;
+		goto out_error;
 	}
 	phba->sli4_hba.els_cq = qdesc;
 
 
 	/*
-	 * Create fast-path FCP Completion Queue(s), one-to-one with FCP EQs.
-	 * If there are no FCP EQs then create exactly one FCP CQ.
+	 * Create Slow Path Work Queues (WQs)
 	 */
-	if (phba->cfg_fcp_eq_count)
-		phba->sli4_hba.fcp_cq = kzalloc((sizeof(struct lpfc_queue *) *
-						 phba->cfg_fcp_eq_count),
-						GFP_KERNEL);
-	else
-		phba->sli4_hba.fcp_cq = kzalloc(sizeof(struct lpfc_queue *),
-						GFP_KERNEL);
-	if (!phba->sli4_hba.fcp_cq) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"2577 Failed allocate memory for fast-path "
-				"CQ record array\n");
-		goto out_free_els_cq;
-	}
-	fcp_cqidx = 0;
-	do {
-		qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.cq_esize,
-					      phba->sli4_hba.cq_ecount);
-		if (!qdesc) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0499 Failed allocate fast-path FCP "
-					"CQ (%d)\n", fcp_cqidx);
-			goto out_free_fcp_cq;
-		}
-		phba->sli4_hba.fcp_cq[fcp_cqidx] = qdesc;
-	} while (++fcp_cqidx < phba->cfg_fcp_eq_count);
 
 	/* Create Mailbox Command Queue */
-	phba->sli4_hba.mq_esize = LPFC_MQE_SIZE;
-	phba->sli4_hba.mq_ecount = LPFC_MQE_DEF_COUNT;
 
 	qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.mq_esize,
 				      phba->sli4_hba.mq_ecount);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0505 Failed allocate slow-path MQ\n");
-		goto out_free_fcp_cq;
+		goto out_error;
 	}
 	phba->sli4_hba.mbx_wq = qdesc;
 
 	/*
-	 * Create all the Work Queues (WQs)
+	 * Create ELS Work Queues
 	 */
-	phba->sli4_hba.wq_esize = LPFC_WQE_SIZE;
-	phba->sli4_hba.wq_ecount = LPFC_WQE_DEF_COUNT;
 
 	/* Create slow-path ELS Work Queue */
 	qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.wq_esize,
@@ -6737,36 +6771,13 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0504 Failed allocate slow-path ELS WQ\n");
-		goto out_free_mbx_wq;
+		goto out_error;
 	}
 	phba->sli4_hba.els_wq = qdesc;
 
-	/* Create fast-path FCP Work Queue(s) */
-	phba->sli4_hba.fcp_wq = kzalloc((sizeof(struct lpfc_queue *) *
-				phba->cfg_fcp_wq_count), GFP_KERNEL);
-	if (!phba->sli4_hba.fcp_wq) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"2578 Failed allocate memory for fast-path "
-				"WQ record array\n");
-		goto out_free_els_wq;
-	}
-	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_wq_count; fcp_wqidx++) {
-		qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.wq_esize,
-					      phba->sli4_hba.wq_ecount);
-		if (!qdesc) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0503 Failed allocate fast-path FCP "
-					"WQ (%d)\n", fcp_wqidx);
-			goto out_free_fcp_wq;
-		}
-		phba->sli4_hba.fcp_wq[fcp_wqidx] = qdesc;
-	}
-
 	/*
 	 * Create Receive Queue (RQ)
 	 */
-	phba->sli4_hba.rq_esize = LPFC_RQE_SIZE;
-	phba->sli4_hba.rq_ecount = LPFC_RQE_DEF_COUNT;
 
 	/* Create Receive Queue for header */
 	qdesc = lpfc_sli4_queue_alloc(phba, phba->sli4_hba.rq_esize,
@@ -6774,7 +6785,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0506 Failed allocate receive HRQ\n");
-		goto out_free_fcp_wq;
+		goto out_error;
 	}
 	phba->sli4_hba.hdr_rq = qdesc;
 
@@ -6784,52 +6795,14 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0507 Failed allocate receive DRQ\n");
-		goto out_free_hdr_rq;
+		goto out_error;
 	}
 	phba->sli4_hba.dat_rq = qdesc;
 
 	return 0;
 
-out_free_hdr_rq:
-	lpfc_sli4_queue_free(phba->sli4_hba.hdr_rq);
-	phba->sli4_hba.hdr_rq = NULL;
-out_free_fcp_wq:
-	for (--fcp_wqidx; fcp_wqidx >= 0; fcp_wqidx--) {
-		lpfc_sli4_queue_free(phba->sli4_hba.fcp_wq[fcp_wqidx]);
-		phba->sli4_hba.fcp_wq[fcp_wqidx] = NULL;
-	}
-	kfree(phba->sli4_hba.fcp_wq);
-	phba->sli4_hba.fcp_wq = NULL;
-out_free_els_wq:
-	lpfc_sli4_queue_free(phba->sli4_hba.els_wq);
-	phba->sli4_hba.els_wq = NULL;
-out_free_mbx_wq:
-	lpfc_sli4_queue_free(phba->sli4_hba.mbx_wq);
-	phba->sli4_hba.mbx_wq = NULL;
-out_free_fcp_cq:
-	for (--fcp_cqidx; fcp_cqidx >= 0; fcp_cqidx--) {
-		lpfc_sli4_queue_free(phba->sli4_hba.fcp_cq[fcp_cqidx]);
-		phba->sli4_hba.fcp_cq[fcp_cqidx] = NULL;
-	}
-	kfree(phba->sli4_hba.fcp_cq);
-	phba->sli4_hba.fcp_cq = NULL;
-out_free_els_cq:
-	lpfc_sli4_queue_free(phba->sli4_hba.els_cq);
-	phba->sli4_hba.els_cq = NULL;
-out_free_mbx_cq:
-	lpfc_sli4_queue_free(phba->sli4_hba.mbx_cq);
-	phba->sli4_hba.mbx_cq = NULL;
-out_free_fp_eq:
-	for (--fcp_eqidx; fcp_eqidx >= 0; fcp_eqidx--) {
-		lpfc_sli4_queue_free(phba->sli4_hba.fp_eq[fcp_eqidx]);
-		phba->sli4_hba.fp_eq[fcp_eqidx] = NULL;
-	}
-	kfree(phba->sli4_hba.fp_eq);
-	phba->sli4_hba.fp_eq = NULL;
-out_free_sp_eq:
-	lpfc_sli4_queue_free(phba->sli4_hba.sp_eq);
-	phba->sli4_hba.sp_eq = NULL;
 out_error:
+	lpfc_sli4_queue_destroy(phba);
 	return -ENOMEM;
 }
 
@@ -6848,58 +6821,86 @@ out_error:
 void
 lpfc_sli4_queue_destroy(struct lpfc_hba *phba)
 {
-	int fcp_qidx;
+	int idx;
+
+	if (phba->sli4_hba.hba_eq != NULL) {
+		/* Release HBA event queue */
+		for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
+			if (phba->sli4_hba.hba_eq[idx] != NULL) {
+				lpfc_sli4_queue_free(
+					phba->sli4_hba.hba_eq[idx]);
+				phba->sli4_hba.hba_eq[idx] = NULL;
+			}
+		}
+		kfree(phba->sli4_hba.hba_eq);
+		phba->sli4_hba.hba_eq = NULL;
+	}
+
+	if (phba->sli4_hba.fcp_cq != NULL) {
+		/* Release FCP completion queue */
+		for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
+			if (phba->sli4_hba.fcp_cq[idx] != NULL) {
+				lpfc_sli4_queue_free(
+					phba->sli4_hba.fcp_cq[idx]);
+				phba->sli4_hba.fcp_cq[idx] = NULL;
+			}
+		}
+		kfree(phba->sli4_hba.fcp_cq);
+		phba->sli4_hba.fcp_cq = NULL;
+	}
+
+	if (phba->sli4_hba.fcp_wq != NULL) {
+		/* Release FCP work queue */
+		for (idx = 0; idx < phba->cfg_fcp_io_channel; idx++) {
+			if (phba->sli4_hba.fcp_wq[idx] != NULL) {
+				lpfc_sli4_queue_free(
+					phba->sli4_hba.fcp_wq[idx]);
+				phba->sli4_hba.fcp_wq[idx] = NULL;
+			}
+		}
+		kfree(phba->sli4_hba.fcp_wq);
+		phba->sli4_hba.fcp_wq = NULL;
+	}
+
+	/* Release FCP CQ mapping array */
+	if (phba->sli4_hba.fcp_cq_map != NULL) {
+		kfree(phba->sli4_hba.fcp_cq_map);
+		phba->sli4_hba.fcp_cq_map = NULL;
+	}
 
 	/* Release mailbox command work queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.mbx_wq);
-	phba->sli4_hba.mbx_wq = NULL;
+	if (phba->sli4_hba.mbx_wq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.mbx_wq);
+		phba->sli4_hba.mbx_wq = NULL;
+	}
 
 	/* Release ELS work queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.els_wq);
-	phba->sli4_hba.els_wq = NULL;
-
-	/* Release FCP work queue */
-	if (phba->sli4_hba.fcp_wq != NULL)
-		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_wq_count;
-		     fcp_qidx++)
-			lpfc_sli4_queue_free(phba->sli4_hba.fcp_wq[fcp_qidx]);
-	kfree(phba->sli4_hba.fcp_wq);
-	phba->sli4_hba.fcp_wq = NULL;
+	if (phba->sli4_hba.els_wq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.els_wq);
+		phba->sli4_hba.els_wq = NULL;
+	}
 
 	/* Release unsolicited receive queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.hdr_rq);
-	phba->sli4_hba.hdr_rq = NULL;
-	lpfc_sli4_queue_free(phba->sli4_hba.dat_rq);
-	phba->sli4_hba.dat_rq = NULL;
+	if (phba->sli4_hba.hdr_rq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.hdr_rq);
+		phba->sli4_hba.hdr_rq = NULL;
+	}
+	if (phba->sli4_hba.dat_rq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.dat_rq);
+		phba->sli4_hba.dat_rq = NULL;
+	}
 
 	/* Release ELS complete queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.els_cq);
-	phba->sli4_hba.els_cq = NULL;
+	if (phba->sli4_hba.els_cq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.els_cq);
+		phba->sli4_hba.els_cq = NULL;
+	}
 
 	/* Release mailbox command complete queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.mbx_cq);
-	phba->sli4_hba.mbx_cq = NULL;
-
-	/* Release FCP response complete queue */
-	fcp_qidx = 0;
-	if (phba->sli4_hba.fcp_cq != NULL)
-		do
-			lpfc_sli4_queue_free(phba->sli4_hba.fcp_cq[fcp_qidx]);
-		while (++fcp_qidx < phba->cfg_fcp_eq_count);
-	kfree(phba->sli4_hba.fcp_cq);
-	phba->sli4_hba.fcp_cq = NULL;
-
-	/* Release fast-path event queue */
-	if (phba->sli4_hba.fp_eq != NULL)
-		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_eq_count;
-		     fcp_qidx++)
-			lpfc_sli4_queue_free(phba->sli4_hba.fp_eq[fcp_qidx]);
-	kfree(phba->sli4_hba.fp_eq);
-	phba->sli4_hba.fp_eq = NULL;
-
-	/* Release slow-path event queue */
-	lpfc_sli4_queue_free(phba->sli4_hba.sp_eq);
-	phba->sli4_hba.sp_eq = NULL;
+	if (phba->sli4_hba.mbx_cq != NULL) {
+		lpfc_sli4_queue_free(phba->sli4_hba.mbx_cq);
+		phba->sli4_hba.mbx_cq = NULL;
+	}
 
 	return;
 }
@@ -6919,61 +6920,124 @@ lpfc_sli4_queue_destroy(struct lpfc_hba *phba)
 int
 lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 {
+	struct lpfc_sli *psli = &phba->sli;
+	struct lpfc_sli_ring *pring;
 	int rc = -ENOMEM;
 	int fcp_eqidx, fcp_cqidx, fcp_wqidx;
 	int fcp_cq_index = 0;
 
 	/*
-	 * Set up Event Queues (EQs)
+	 * Set up HBA Event Queues (EQs)
 	 */
 
-	/* Set up slow-path event queue */
-	if (!phba->sli4_hba.sp_eq) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"0520 Slow-path EQ not allocated\n");
-		goto out_error;
-	}
-	rc = lpfc_eq_create(phba, phba->sli4_hba.sp_eq,
-			    LPFC_SP_DEF_IMAX);
-	if (rc) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"0521 Failed setup of slow-path EQ: "
-				"rc = 0x%x\n", rc);
-		goto out_error;
-	}
-	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-			"2583 Slow-path EQ setup: queue-id=%d\n",
-			phba->sli4_hba.sp_eq->queue_id);
-
-	/* Set up fast-path event queue */
-	if (phba->cfg_fcp_eq_count && !phba->sli4_hba.fp_eq) {
+	/* Set up HBA event queue */
+	if (phba->cfg_fcp_io_channel && !phba->sli4_hba.hba_eq) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"3147 Fast-path EQs not allocated\n");
 		rc = -ENOMEM;
-		goto out_destroy_sp_eq;
+		goto out_error;
 	}
-	for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_eq_count; fcp_eqidx++) {
-		if (!phba->sli4_hba.fp_eq[fcp_eqidx]) {
+	for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel; fcp_eqidx++) {
+		if (!phba->sli4_hba.hba_eq[fcp_eqidx]) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"0522 Fast-path EQ (%d) not "
 					"allocated\n", fcp_eqidx);
 			rc = -ENOMEM;
-			goto out_destroy_fp_eq;
+			goto out_destroy_hba_eq;
 		}
-		rc = lpfc_eq_create(phba, phba->sli4_hba.fp_eq[fcp_eqidx],
-				    phba->cfg_fcp_imax);
+		rc = lpfc_eq_create(phba, phba->sli4_hba.hba_eq[fcp_eqidx],
+			 (phba->cfg_fcp_imax / phba->cfg_fcp_io_channel));
 		if (rc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"0523 Failed setup of fast-path EQ "
 					"(%d), rc = 0x%x\n", fcp_eqidx, rc);
-			goto out_destroy_fp_eq;
+			goto out_destroy_hba_eq;
 		}
 		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-				"2584 Fast-path EQ setup: "
+				"2584 HBA EQ setup: "
 				"queue[%d]-id=%d\n", fcp_eqidx,
-				phba->sli4_hba.fp_eq[fcp_eqidx]->queue_id);
+				phba->sli4_hba.hba_eq[fcp_eqidx]->queue_id);
 	}
 
+	/* Set up fast-path FCP Response Complete Queue */
+	if (!phba->sli4_hba.fcp_cq) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"3148 Fast-path FCP CQ array not "
+				"allocated\n");
+		rc = -ENOMEM;
+		goto out_destroy_hba_eq;
+	}
+
+	for (fcp_cqidx = 0; fcp_cqidx < phba->cfg_fcp_io_channel; fcp_cqidx++) {
+		if (!phba->sli4_hba.fcp_cq[fcp_cqidx]) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0526 Fast-path FCP CQ (%d) not "
+					"allocated\n", fcp_cqidx);
+			rc = -ENOMEM;
+			goto out_destroy_fcp_cq;
+		}
+		rc = lpfc_cq_create(phba, phba->sli4_hba.fcp_cq[fcp_cqidx],
+			phba->sli4_hba.hba_eq[fcp_cqidx], LPFC_WCQ, LPFC_FCP);
+		if (rc) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0527 Failed setup of fast-path FCP "
+					"CQ (%d), rc = 0x%x\n", fcp_cqidx, rc);
+			goto out_destroy_fcp_cq;
+		}
+
+		/* Setup fcp_cq_map for fast lookup */
+		phba->sli4_hba.fcp_cq_map[fcp_cqidx] =
+				phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id;
+
+		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+				"2588 FCP CQ setup: cq[%d]-id=%d, "
+				"parent seq[%d]-id=%d\n",
+				fcp_cqidx,
+				phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id,
+				fcp_cqidx,
+				phba->sli4_hba.hba_eq[fcp_cqidx]->queue_id);
+	}
+
+	/* Set up fast-path FCP Work Queue */
+	if (!phba->sli4_hba.fcp_wq) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"3149 Fast-path FCP WQ array not "
+				"allocated\n");
+		rc = -ENOMEM;
+		goto out_destroy_fcp_cq;
+	}
+
+	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_io_channel; fcp_wqidx++) {
+		if (!phba->sli4_hba.fcp_wq[fcp_wqidx]) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0534 Fast-path FCP WQ (%d) not "
+					"allocated\n", fcp_wqidx);
+			rc = -ENOMEM;
+			goto out_destroy_fcp_wq;
+		}
+		rc = lpfc_wq_create(phba, phba->sli4_hba.fcp_wq[fcp_wqidx],
+				    phba->sli4_hba.fcp_cq[fcp_wqidx],
+				    LPFC_FCP);
+		if (rc) {
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"0535 Failed setup of fast-path FCP "
+					"WQ (%d), rc = 0x%x\n", fcp_wqidx, rc);
+			goto out_destroy_fcp_wq;
+		}
+
+		/* Bind this WQ to the next FCP ring */
+		pring = &psli->ring[MAX_SLI3_CONFIGURED_RINGS + fcp_wqidx];
+		pring->sli.sli4.wqp = (void *)phba->sli4_hba.fcp_wq[fcp_wqidx];
+		phba->sli4_hba.fcp_cq[fcp_wqidx]->pring = pring;
+
+		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+				"2591 FCP WQ setup: wq[%d]-id=%d, "
+				"parent cq[%d]-id=%d\n",
+				fcp_wqidx,
+				phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id,
+				fcp_cq_index,
+				phba->sli4_hba.fcp_cq[fcp_wqidx]->queue_id);
+	}
 	/*
 	 * Set up Complete Queues (CQs)
 	 */
@@ -6983,20 +7047,20 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0528 Mailbox CQ not allocated\n");
 		rc = -ENOMEM;
-		goto out_destroy_fp_eq;
+		goto out_destroy_fcp_wq;
 	}
-	rc = lpfc_cq_create(phba, phba->sli4_hba.mbx_cq, phba->sli4_hba.sp_eq,
-			    LPFC_MCQ, LPFC_MBOX);
+	rc = lpfc_cq_create(phba, phba->sli4_hba.mbx_cq,
+			phba->sli4_hba.hba_eq[0], LPFC_MCQ, LPFC_MBOX);
 	if (rc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0529 Failed setup of slow-path mailbox CQ: "
 				"rc = 0x%x\n", rc);
-		goto out_destroy_fp_eq;
+		goto out_destroy_fcp_wq;
 	}
 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 			"2585 MBX CQ setup: cq-id=%d, parent eq-id=%d\n",
 			phba->sli4_hba.mbx_cq->queue_id,
-			phba->sli4_hba.sp_eq->queue_id);
+			phba->sli4_hba.hba_eq[0]->queue_id);
 
 	/* Set up slow-path ELS Complete Queue */
 	if (!phba->sli4_hba.els_cq) {
@@ -7005,8 +7069,8 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 		rc = -ENOMEM;
 		goto out_destroy_mbx_cq;
 	}
-	rc = lpfc_cq_create(phba, phba->sli4_hba.els_cq, phba->sli4_hba.sp_eq,
-			    LPFC_WCQ, LPFC_ELS);
+	rc = lpfc_cq_create(phba, phba->sli4_hba.els_cq,
+			phba->sli4_hba.hba_eq[0], LPFC_WCQ, LPFC_ELS);
 	if (rc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0531 Failed setup of slow-path ELS CQ: "
@@ -7016,52 +7080,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 			"2586 ELS CQ setup: cq-id=%d, parent eq-id=%d\n",
 			phba->sli4_hba.els_cq->queue_id,
-			phba->sli4_hba.sp_eq->queue_id);
-
-	/* Set up fast-path FCP Response Complete Queue */
-	if (!phba->sli4_hba.fcp_cq) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"3148 Fast-path FCP CQ array not "
-				"allocated\n");
-		rc = -ENOMEM;
-		goto out_destroy_els_cq;
-	}
-	fcp_cqidx = 0;
-	do {
-		if (!phba->sli4_hba.fcp_cq[fcp_cqidx]) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0526 Fast-path FCP CQ (%d) not "
-					"allocated\n", fcp_cqidx);
-			rc = -ENOMEM;
-			goto out_destroy_fcp_cq;
-		}
-		if (phba->cfg_fcp_eq_count)
-			rc = lpfc_cq_create(phba,
-					    phba->sli4_hba.fcp_cq[fcp_cqidx],
-					    phba->sli4_hba.fp_eq[fcp_cqidx],
-					    LPFC_WCQ, LPFC_FCP);
-		else
-			rc = lpfc_cq_create(phba,
-					    phba->sli4_hba.fcp_cq[fcp_cqidx],
-					    phba->sli4_hba.sp_eq,
-					    LPFC_WCQ, LPFC_FCP);
-		if (rc) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0527 Failed setup of fast-path FCP "
-					"CQ (%d), rc = 0x%x\n", fcp_cqidx, rc);
-			goto out_destroy_fcp_cq;
-		}
-		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-				"2588 FCP CQ setup: cq[%d]-id=%d, "
-				"parent %seq[%d]-id=%d\n",
-				fcp_cqidx,
-				phba->sli4_hba.fcp_cq[fcp_cqidx]->queue_id,
-				(phba->cfg_fcp_eq_count) ? "" : "sp_",
-				fcp_cqidx,
-				(phba->cfg_fcp_eq_count) ?
-				   phba->sli4_hba.fp_eq[fcp_cqidx]->queue_id :
-				   phba->sli4_hba.sp_eq->queue_id);
-	} while (++fcp_cqidx < phba->cfg_fcp_eq_count);
+			phba->sli4_hba.hba_eq[0]->queue_id);
 
 	/*
 	 * Set up all the Work Queues (WQs)
@@ -7072,7 +7091,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0538 Slow-path MQ not allocated\n");
 		rc = -ENOMEM;
-		goto out_destroy_fcp_cq;
+		goto out_destroy_els_cq;
 	}
 	rc = lpfc_mq_create(phba, phba->sli4_hba.mbx_wq,
 			    phba->sli4_hba.mbx_cq, LPFC_MBOX);
@@ -7080,7 +7099,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0539 Failed setup of slow-path MQ: "
 				"rc = 0x%x\n", rc);
-		goto out_destroy_fcp_cq;
+		goto out_destroy_els_cq;
 	}
 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 			"2589 MBX MQ setup: wq-id=%d, parent cq-id=%d\n",
@@ -7102,49 +7121,17 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 				"rc = 0x%x\n", rc);
 		goto out_destroy_mbx_wq;
 	}
+
+	/* Bind this WQ to the ELS ring */
+	pring = &psli->ring[LPFC_ELS_RING];
+	pring->sli.sli4.wqp = (void *)phba->sli4_hba.els_wq;
+	phba->sli4_hba.els_cq->pring = pring;
+
 	lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
 			"2590 ELS WQ setup: wq-id=%d, parent cq-id=%d\n",
 			phba->sli4_hba.els_wq->queue_id,
 			phba->sli4_hba.els_cq->queue_id);
 
-	/* Set up fast-path FCP Work Queue */
-	if (!phba->sli4_hba.fcp_wq) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"3149 Fast-path FCP WQ array not "
-				"allocated\n");
-		rc = -ENOMEM;
-		goto out_destroy_els_wq;
-	}
-	for (fcp_wqidx = 0; fcp_wqidx < phba->cfg_fcp_wq_count; fcp_wqidx++) {
-		if (!phba->sli4_hba.fcp_wq[fcp_wqidx]) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0534 Fast-path FCP WQ (%d) not "
-					"allocated\n", fcp_wqidx);
-			rc = -ENOMEM;
-			goto out_destroy_fcp_wq;
-		}
-		rc = lpfc_wq_create(phba, phba->sli4_hba.fcp_wq[fcp_wqidx],
-				    phba->sli4_hba.fcp_cq[fcp_cq_index],
-				    LPFC_FCP);
-		if (rc) {
-			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-					"0535 Failed setup of fast-path FCP "
-					"WQ (%d), rc = 0x%x\n", fcp_wqidx, rc);
-			goto out_destroy_fcp_wq;
-		}
-		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
-				"2591 FCP WQ setup: wq[%d]-id=%d, "
-				"parent cq[%d]-id=%d\n",
-				fcp_wqidx,
-				phba->sli4_hba.fcp_wq[fcp_wqidx]->queue_id,
-				fcp_cq_index,
-				phba->sli4_hba.fcp_cq[fcp_cq_index]->queue_id);
-		/* Round robin FCP Work Queue's Completion Queue assignment */
-		if (phba->cfg_fcp_eq_count)
-			fcp_cq_index = ((fcp_cq_index + 1) %
-					phba->cfg_fcp_eq_count);
-	}
-
 	/*
 	 * Create Receive Queue (RQ)
 	 */
@@ -7152,7 +7139,7 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0540 Receive Queue not allocated\n");
 		rc = -ENOMEM;
-		goto out_destroy_fcp_wq;
+		goto out_destroy_els_wq;
 	}
 
 	lpfc_rq_adjust_repost(phba, phba->sli4_hba.hdr_rq, LPFC_ELS_HBQ);
@@ -7175,25 +7162,23 @@ lpfc_sli4_queue_setup(struct lpfc_hba *phba)
 			phba->sli4_hba.els_cq->queue_id);
 	return 0;
 
-out_destroy_fcp_wq:
-	for (--fcp_wqidx; fcp_wqidx >= 0; fcp_wqidx--)
-		lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_wqidx]);
 out_destroy_els_wq:
 	lpfc_wq_destroy(phba, phba->sli4_hba.els_wq);
 out_destroy_mbx_wq:
 	lpfc_mq_destroy(phba, phba->sli4_hba.mbx_wq);
-out_destroy_fcp_cq:
-	for (--fcp_cqidx; fcp_cqidx >= 0; fcp_cqidx--)
-		lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[fcp_cqidx]);
 out_destroy_els_cq:
 	lpfc_cq_destroy(phba, phba->sli4_hba.els_cq);
 out_destroy_mbx_cq:
 	lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq);
-out_destroy_fp_eq:
+out_destroy_fcp_wq:
+	for (--fcp_wqidx; fcp_wqidx >= 0; fcp_wqidx--)
+		lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_wqidx]);
+out_destroy_fcp_cq:
+	for (--fcp_cqidx; fcp_cqidx >= 0; fcp_cqidx--)
+		lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[fcp_cqidx]);
+out_destroy_hba_eq:
 	for (--fcp_eqidx; fcp_eqidx >= 0; fcp_eqidx--)
-		lpfc_eq_destroy(phba, phba->sli4_hba.fp_eq[fcp_eqidx]);
-out_destroy_sp_eq:
-	lpfc_eq_destroy(phba, phba->sli4_hba.sp_eq);
+		lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[fcp_eqidx]);
 out_error:
 	return rc;
 }
@@ -7222,27 +7207,27 @@ lpfc_sli4_queue_unset(struct lpfc_hba *phba)
 	/* Unset unsolicited receive queue */
 	lpfc_rq_destroy(phba, phba->sli4_hba.hdr_rq, phba->sli4_hba.dat_rq);
 	/* Unset FCP work queue */
-	for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_wq_count; fcp_qidx++)
-		lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_qidx]);
+	if (phba->sli4_hba.fcp_wq) {
+		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
+		     fcp_qidx++)
+			lpfc_wq_destroy(phba, phba->sli4_hba.fcp_wq[fcp_qidx]);
+	}
 	/* Unset mailbox command complete queue */
 	lpfc_cq_destroy(phba, phba->sli4_hba.mbx_cq);
 	/* Unset ELS complete queue */
 	lpfc_cq_destroy(phba, phba->sli4_hba.els_cq);
 	/* Unset FCP response complete queue */
 	if (phba->sli4_hba.fcp_cq) {
-		fcp_qidx = 0;
-		do {
+		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
+		     fcp_qidx++)
 			lpfc_cq_destroy(phba, phba->sli4_hba.fcp_cq[fcp_qidx]);
-		} while (++fcp_qidx < phba->cfg_fcp_eq_count);
 	}
 	/* Unset fast-path event queue */
-	if (phba->sli4_hba.fp_eq) {
-		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_eq_count;
+	if (phba->sli4_hba.hba_eq) {
+		for (fcp_qidx = 0; fcp_qidx < phba->cfg_fcp_io_channel;
 		     fcp_qidx++)
-			lpfc_eq_destroy(phba, phba->sli4_hba.fp_eq[fcp_qidx]);
+			lpfc_eq_destroy(phba, phba->sli4_hba.hba_eq[fcp_qidx]);
 	}
-	/* Unset slow-path event queue */
-	lpfc_eq_destroy(phba, phba->sli4_hba.sp_eq);
 }
 
 /**
@@ -7590,10 +7575,11 @@ lpfc_sli4_send_nop_mbox_cmds(struct lpfc_hba *phba, uint32_t cnt)
 	/* Set up NOP SLI4_CONFIG mailbox-ioctl command */
 	length = (sizeof(struct lpfc_mbx_nop) -
 		  sizeof(struct lpfc_sli4_cfg_mhdr));
-	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
-			 LPFC_MBOX_OPCODE_NOP, length, LPFC_SLI4_MBX_EMBED);
 
 	for (cmdsent = 0; cmdsent < cnt; cmdsent++) {
+		lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
+				 LPFC_MBOX_OPCODE_NOP, length,
+				 LPFC_SLI4_MBX_EMBED);
 		if (!phba->sli4_hba.intr_enable)
 			rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL);
 		else {
@@ -8133,33 +8119,22 @@ enable_msix_vectors:
 				"message=%d\n", index,
 				phba->sli4_hba.msix_entries[index].vector,
 				phba->sli4_hba.msix_entries[index].entry);
+
 	/*
 	 * Assign MSI-X vectors to interrupt handlers
 	 */
-	if (vectors > 1)
-		rc = request_irq(phba->sli4_hba.msix_entries[0].vector,
-				 &lpfc_sli4_sp_intr_handler, IRQF_SHARED,
-				 LPFC_SP_DRIVER_HANDLER_NAME, phba);
-	else
-		/* All Interrupts need to be handled by one EQ */
-		rc = request_irq(phba->sli4_hba.msix_entries[0].vector,
-				 &lpfc_sli4_intr_handler, IRQF_SHARED,
-				 LPFC_DRIVER_NAME, phba);
-	if (rc) {
-		lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
-				"0485 MSI-X slow-path request_irq failed "
-				"(%d)\n", rc);
-		goto msi_fail_out;
-	}
+	for (index = 0; index < vectors; index++) {
+		memset(&phba->sli4_hba.handler_name[index], 0, 16);
+		sprintf((char *)&phba->sli4_hba.handler_name[index],
+			 LPFC_DRIVER_HANDLER_NAME"%d", index);
 
-	/* The rest of the vector(s) are associated to fast-path handler(s) */
-	for (index = 1; index < vectors; index++) {
-		phba->sli4_hba.fcp_eq_hdl[index - 1].idx = index - 1;
-		phba->sli4_hba.fcp_eq_hdl[index - 1].phba = phba;
+		phba->sli4_hba.fcp_eq_hdl[index].idx = index;
+		phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
+		atomic_set(&phba->sli4_hba.fcp_eq_hdl[index].fcp_eq_in_use, 1);
 		rc = request_irq(phba->sli4_hba.msix_entries[index].vector,
-				 &lpfc_sli4_fp_intr_handler, IRQF_SHARED,
-				 LPFC_FP_DRIVER_HANDLER_NAME,
-				 &phba->sli4_hba.fcp_eq_hdl[index - 1]);
+				 &lpfc_sli4_hba_intr_handler, IRQF_SHARED,
+				 (char *)&phba->sli4_hba.handler_name[index],
+				 &phba->sli4_hba.fcp_eq_hdl[index]);
 		if (rc) {
 			lpfc_printf_log(phba, KERN_WARNING, LOG_INIT,
 					"0486 MSI-X fast-path (%d) "
@@ -8173,12 +8148,9 @@ enable_msix_vectors:
 
 cfg_fail_out:
 	/* free the irq already requested */
-	for (--index; index >= 1; index--)
-		free_irq(phba->sli4_hba.msix_entries[index - 1].vector,
-			 &phba->sli4_hba.fcp_eq_hdl[index - 1]);
-
-	/* free the irq already requested */
-	free_irq(phba->sli4_hba.msix_entries[0].vector, phba);
+	for (--index; index >= 0; index--)
+		free_irq(phba->sli4_hba.msix_entries[index].vector,
+			 &phba->sli4_hba.fcp_eq_hdl[index]);
 
 msi_fail_out:
 	/* Unconfigure MSI-X capability structure */
@@ -8199,11 +8171,9 @@ lpfc_sli4_disable_msix(struct lpfc_hba *phba)
 	int index;
 
 	/* Free up MSI-X multi-message vectors */
-	free_irq(phba->sli4_hba.msix_entries[0].vector, phba);
-
-	for (index = 1; index < phba->sli4_hba.msix_vec_nr; index++)
+	for (index = 0; index < phba->sli4_hba.msix_vec_nr; index++)
 		free_irq(phba->sli4_hba.msix_entries[index].vector,
-			 &phba->sli4_hba.fcp_eq_hdl[index - 1]);
+			 &phba->sli4_hba.fcp_eq_hdl[index]);
 
 	/* Disable MSI-X */
 	pci_disable_msix(phba->pcidev);
@@ -8249,7 +8219,7 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba)
 		return rc;
 	}
 
-	for (index = 0; index < phba->cfg_fcp_eq_count; index++) {
+	for (index = 0; index < phba->cfg_fcp_io_channel; index++) {
 		phba->sli4_hba.fcp_eq_hdl[index].idx = index;
 		phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
 	}
@@ -8329,10 +8299,12 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode)
 			/* Indicate initialization to INTx mode */
 			phba->intr_type = INTx;
 			intr_mode = 0;
-			for (index = 0; index < phba->cfg_fcp_eq_count;
+			for (index = 0; index < phba->cfg_fcp_io_channel;
 			     index++) {
 				phba->sli4_hba.fcp_eq_hdl[index].idx = index;
 				phba->sli4_hba.fcp_eq_hdl[index].phba = phba;
+				atomic_set(&phba->sli4_hba.fcp_eq_hdl[index].
+					fcp_eq_in_use, 1);
 			}
 		}
 	}
@@ -9449,7 +9421,7 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid)
 	int error;
 	uint32_t cfg_mode, intr_mode;
 	int mcnt;
-	int adjusted_fcp_eq_count;
+	int adjusted_fcp_io_channel;
 	const struct firmware *fw;
 	uint8_t file_name[16];
 
@@ -9552,13 +9524,13 @@ lpfc_pci_probe_one_s4(struct pci_dev *pdev, const struct pci_device_id *pid)
 		}
 		/* Default to single EQ for non-MSI-X */
 		if (phba->intr_type != MSIX)
-			adjusted_fcp_eq_count = 0;
+			adjusted_fcp_io_channel = 1;
 		else if (phba->sli4_hba.msix_vec_nr <
-					phba->cfg_fcp_eq_count + 1)
-			adjusted_fcp_eq_count = phba->sli4_hba.msix_vec_nr - 1;
+					phba->cfg_fcp_io_channel)
+			adjusted_fcp_io_channel = phba->sli4_hba.msix_vec_nr;
 		else
-			adjusted_fcp_eq_count = phba->cfg_fcp_eq_count;
-		phba->cfg_fcp_eq_count = adjusted_fcp_eq_count;
+			adjusted_fcp_io_channel = phba->cfg_fcp_io_channel;
+		phba->cfg_fcp_io_channel = adjusted_fcp_io_channel;
 		/* Set up SLI-4 HBA */
 		if (lpfc_sli4_hba_setup(phba)) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -9694,6 +9666,7 @@ lpfc_pci_remove_one_s4(struct pci_dev *pdev)
 	 * buffers are released to their corresponding pools here.
 	 */
 	lpfc_scsi_free(phba);
+
 	lpfc_sli4_driver_resource_unset(phba);
 
 	/* Unmap adapter Control and Doorbell registers */
@@ -10420,6 +10393,10 @@ static struct pci_device_id lpfc_id_table[] = {
 		PCI_ANY_ID, PCI_ANY_ID, },
 	{PCI_VENDOR_ID_EMULEX, PCI_DEVICE_ID_LANCER_FCOE_VF,
 		PCI_ANY_ID, PCI_ANY_ID, },
+	{PCI_VENDOR_ID_EMULEX, PCI_DEVICE_ID_SKYHAWK,
+		PCI_ANY_ID, PCI_ANY_ID, },
+	{PCI_VENDOR_ID_EMULEX, PCI_DEVICE_ID_SKYHAWK_VF,
+		PCI_ANY_ID, PCI_ANY_ID, },
 	{ 0 }
 };
 
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index 20336f09fb3..efc9cd9def8 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -92,7 +92,7 @@ lpfc_dump_static_vport(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb,
 	memset(mp->virt, 0, LPFC_BPL_SIZE);
 	INIT_LIST_HEAD(&mp->list);
 	/* save address for completion */
-	pmb->context2 = (uint8_t *) mp;
+	pmb->context1 = (uint8_t *)mp;
 	mb->un.varWords[3] = putPaddrLow(mp->phys);
 	mb->un.varWords[4] = putPaddrHigh(mp->phys);
 	mb->un.varDmp.sli4_length = sizeof(struct static_vport_info);
@@ -950,44 +950,47 @@ lpfc_config_pcb_setup(struct lpfc_hba * phba)
 	for (i = 0; i < psli->num_rings; i++) {
 		pring = &psli->ring[i];
 
-		pring->sizeCiocb = phba->sli_rev == 3 ? SLI3_IOCB_CMD_SIZE:
+		pring->sli.sli3.sizeCiocb =
+			phba->sli_rev == 3 ? SLI3_IOCB_CMD_SIZE :
 							SLI2_IOCB_CMD_SIZE;
-		pring->sizeRiocb = phba->sli_rev == 3 ? SLI3_IOCB_RSP_SIZE:
+		pring->sli.sli3.sizeRiocb =
+			phba->sli_rev == 3 ? SLI3_IOCB_RSP_SIZE :
 							SLI2_IOCB_RSP_SIZE;
 		/* A ring MUST have both cmd and rsp entries defined to be
 		   valid */
-		if ((pring->numCiocb == 0) || (pring->numRiocb == 0)) {
+		if ((pring->sli.sli3.numCiocb == 0) ||
+			(pring->sli.sli3.numRiocb == 0)) {
 			pcbp->rdsc[i].cmdEntries = 0;
 			pcbp->rdsc[i].rspEntries = 0;
 			pcbp->rdsc[i].cmdAddrHigh = 0;
 			pcbp->rdsc[i].rspAddrHigh = 0;
 			pcbp->rdsc[i].cmdAddrLow = 0;
 			pcbp->rdsc[i].rspAddrLow = 0;
-			pring->cmdringaddr = NULL;
-			pring->rspringaddr = NULL;
+			pring->sli.sli3.cmdringaddr = NULL;
+			pring->sli.sli3.rspringaddr = NULL;
 			continue;
 		}
 		/* Command ring setup for ring */
-		pring->cmdringaddr = (void *)&phba->IOCBs[iocbCnt];
-		pcbp->rdsc[i].cmdEntries = pring->numCiocb;
+		pring->sli.sli3.cmdringaddr = (void *)&phba->IOCBs[iocbCnt];
+		pcbp->rdsc[i].cmdEntries = pring->sli.sli3.numCiocb;
 
 		offset = (uint8_t *) &phba->IOCBs[iocbCnt] -
 			 (uint8_t *) phba->slim2p.virt;
 		pdma_addr = phba->slim2p.phys + offset;
 		pcbp->rdsc[i].cmdAddrHigh = putPaddrHigh(pdma_addr);
 		pcbp->rdsc[i].cmdAddrLow = putPaddrLow(pdma_addr);
-		iocbCnt += pring->numCiocb;
+		iocbCnt += pring->sli.sli3.numCiocb;
 
 		/* Response ring setup for ring */
-		pring->rspringaddr = (void *) &phba->IOCBs[iocbCnt];
+		pring->sli.sli3.rspringaddr = (void *) &phba->IOCBs[iocbCnt];
 
-		pcbp->rdsc[i].rspEntries = pring->numRiocb;
+		pcbp->rdsc[i].rspEntries = pring->sli.sli3.numRiocb;
 		offset = (uint8_t *)&phba->IOCBs[iocbCnt] -
 			 (uint8_t *)phba->slim2p.virt;
 		pdma_addr = phba->slim2p.phys + offset;
 		pcbp->rdsc[i].rspAddrHigh = putPaddrHigh(pdma_addr);
 		pcbp->rdsc[i].rspAddrLow = putPaddrLow(pdma_addr);
-		iocbCnt += pring->numRiocb;
+		iocbCnt += pring->sli.sli3.numRiocb;
 	}
 }
 
@@ -1609,12 +1612,15 @@ lpfc_mbox_tmo_val(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 
 	switch (mbox->mbxCommand) {
 	case MBX_WRITE_NV:	/* 0x03 */
+	case MBX_DUMP_MEMORY:	/* 0x17 */
 	case MBX_UPDATE_CFG:	/* 0x1B */
 	case MBX_DOWN_LOAD:	/* 0x1C */
 	case MBX_DEL_LD_ENTRY:	/* 0x1D */
+	case MBX_WRITE_VPARMS:	/* 0x32 */
 	case MBX_LOAD_AREA:	/* 0x81 */
 	case MBX_WRITE_WWN:     /* 0x98 */
 	case MBX_LOAD_EXP_ROM:	/* 0x9C */
+	case MBX_ACCESS_VDATA:	/* 0xA5 */
 		return LPFC_MBOX_TMO_FLASH_CMD;
 	case MBX_SLI4_CONFIG:	/* 0x9b */
 		subsys = lpfc_sli_config_mbox_subsys_get(phba, mboxq);
@@ -1625,11 +1631,17 @@ lpfc_mbox_tmo_val(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 			case LPFC_MBOX_OPCODE_WRITE_OBJECT:
 			case LPFC_MBOX_OPCODE_READ_OBJECT_LIST:
 			case LPFC_MBOX_OPCODE_DELETE_OBJECT:
-			case LPFC_MBOX_OPCODE_GET_FUNCTION_CONFIG:
 			case LPFC_MBOX_OPCODE_GET_PROFILE_LIST:
 			case LPFC_MBOX_OPCODE_SET_ACT_PROFILE:
+			case LPFC_MBOX_OPCODE_GET_PROFILE_CONFIG:
 			case LPFC_MBOX_OPCODE_SET_PROFILE_CONFIG:
 			case LPFC_MBOX_OPCODE_GET_FACTORY_PROFILE_CONFIG:
+			case LPFC_MBOX_OPCODE_GET_PROFILE_CAPACITIES:
+			case LPFC_MBOX_OPCODE_SEND_ACTIVATION:
+			case LPFC_MBOX_OPCODE_RESET_LICENSES:
+			case LPFC_MBOX_OPCODE_SET_BOOT_CONFIG:
+			case LPFC_MBOX_OPCODE_GET_VPD_DATA:
+			case LPFC_MBOX_OPCODE_SET_PHYSICAL_LINK_CONFIG:
 				return LPFC_MBOX_SLI4_CONFIG_EXTENDED_TMO;
 			}
 		}
diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c
index ade763d3930..cd86069a0ba 100644
--- a/drivers/scsi/lpfc/lpfc_mem.c
+++ b/drivers/scsi/lpfc/lpfc_mem.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2004-2009 Emulex.  All rights reserved.           *
+ * Copyright (C) 2004-2012 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
  * www.emulex.com                                                  *
  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
@@ -194,6 +194,10 @@ lpfc_mem_free(struct lpfc_hba *phba)
 		pci_pool_destroy(phba->lpfc_hbq_pool);
 	phba->lpfc_hbq_pool = NULL;
 
+	if (phba->rrq_pool)
+		mempool_destroy(phba->rrq_pool);
+	phba->rrq_pool = NULL;
+
 	/* Free NLP memory pool */
 	mempool_destroy(phba->nlp_mem_pool);
 	phba->nlp_mem_pool = NULL;
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index 9133a97f045..d8fadcb2db7 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1778,6 +1778,117 @@ lpfc_device_recov_prli_issue(struct lpfc_vport *vport,
 }
 
 static uint32_t
+lpfc_rcv_plogi_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			  void *arg, uint32_t evt)
+{
+	struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *)arg;
+	struct ls_rjt     stat;
+
+	memset(&stat, 0, sizeof(struct ls_rjt));
+	stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
+	stat.un.b.lsRjtRsnCodeExp = LSEXP_NOTHING_MORE;
+	lpfc_els_rsp_reject(vport, stat.un.lsRjtError, cmdiocb, ndlp, NULL);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_rcv_prli_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			 void *arg, uint32_t evt)
+{
+	struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *)arg;
+	struct ls_rjt     stat;
+
+	memset(&stat, 0, sizeof(struct ls_rjt));
+	stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
+	stat.un.b.lsRjtRsnCodeExp = LSEXP_NOTHING_MORE;
+	lpfc_els_rsp_reject(vport, stat.un.lsRjtError, cmdiocb, ndlp, NULL);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_rcv_logo_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			 void *arg, uint32_t evt)
+{
+	struct Scsi_Host  *shost = lpfc_shost_from_vport(vport);
+	struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *)arg;
+
+	spin_lock_irq(shost->host_lock);
+	ndlp->nlp_flag &= NLP_LOGO_ACC;
+	spin_unlock_irq(shost->host_lock);
+	lpfc_els_rsp_acc(vport, ELS_CMD_ACC, cmdiocb, ndlp, NULL);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_rcv_padisc_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			   void *arg, uint32_t evt)
+{
+	struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *)arg;
+	struct ls_rjt     stat;
+
+	memset(&stat, 0, sizeof(struct ls_rjt));
+	stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
+	stat.un.b.lsRjtRsnCodeExp = LSEXP_NOTHING_MORE;
+	lpfc_els_rsp_reject(vport, stat.un.lsRjtError, cmdiocb, ndlp, NULL);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_rcv_prlo_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			 void *arg, uint32_t evt)
+{
+	struct lpfc_iocbq *cmdiocb = (struct lpfc_iocbq *)arg;
+	struct ls_rjt     stat;
+
+	memset(&stat, 0, sizeof(struct ls_rjt));
+	stat.un.b.lsRjtRsnCode = LSRJT_UNABLE_TPC;
+	stat.un.b.lsRjtRsnCodeExp = LSEXP_NOTHING_MORE;
+	lpfc_els_rsp_reject(vport, stat.un.lsRjtError, cmdiocb, ndlp, NULL);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_cmpl_logo_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			  void *arg, uint32_t evt)
+{
+	struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+
+	ndlp->nlp_prev_state = NLP_STE_LOGO_ISSUE;
+	lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE);
+	spin_lock_irq(shost->host_lock);
+	ndlp->nlp_flag &= ~(NLP_NODEV_REMOVE | NLP_NPR_2B_DISC);
+	spin_unlock_irq(shost->host_lock);
+	lpfc_disc_set_adisc(vport, ndlp);
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_device_rm_logo_issue(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
+			  void *arg, uint32_t evt)
+{
+	/*
+	 * Take no action.  If a LOGO is outstanding, then possibly DevLoss has
+	 * timed out and is calling for Device Remove.  In this case, the LOGO
+	 * must be allowed to complete in state LOGO_ISSUE so that the rpi
+	 * and other NLP flags are correctly cleaned up.
+	 */
+	return ndlp->nlp_state;
+}
+
+static uint32_t
+lpfc_device_recov_logo_issue(struct lpfc_vport *vport,
+			     struct lpfc_nodelist *ndlp,
+			     void *arg, uint32_t evt)
+{
+	/*
+	 * Device Recovery events have no meaning for a node with a LOGO
+	 * outstanding.  The LOGO has to complete first and handle the
+	 * node from that point.
+	 */
+	return ndlp->nlp_state;
+}
+
+static uint32_t
 lpfc_rcv_plogi_unmap_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			  void *arg, uint32_t evt)
 {
@@ -2083,6 +2194,8 @@ lpfc_cmpl_logo_npr_node(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
 			void *arg, uint32_t evt)
 {
 	struct Scsi_Host *shost = lpfc_shost_from_vport(vport);
+
+	/* For the fabric port just clear the fc flags. */
 	if (ndlp->nlp_DID == Fabric_DID) {
 		spin_lock_irq(shost->host_lock);
 		vport->fc_flag &= ~(FC_FABRIC | FC_PUBLIC_LOOP);
@@ -2297,6 +2410,20 @@ static uint32_t (*lpfc_disc_action[NLP_STE_MAX_STATE * NLP_EVT_MAX_EVENT])
 	lpfc_device_rm_prli_issue,	/* DEVICE_RM       */
 	lpfc_device_recov_prli_issue,	/* DEVICE_RECOVERY */
 
+	lpfc_rcv_plogi_logo_issue,	/* RCV_PLOGI   LOGO_ISSUE     */
+	lpfc_rcv_prli_logo_issue,	/* RCV_PRLI        */
+	lpfc_rcv_logo_logo_issue,	/* RCV_LOGO        */
+	lpfc_rcv_padisc_logo_issue,	/* RCV_ADISC       */
+	lpfc_rcv_padisc_logo_issue,	/* RCV_PDISC       */
+	lpfc_rcv_prlo_logo_issue,	/* RCV_PRLO        */
+	lpfc_cmpl_plogi_illegal,	/* CMPL_PLOGI      */
+	lpfc_disc_illegal,		/* CMPL_PRLI       */
+	lpfc_cmpl_logo_logo_issue,	/* CMPL_LOGO       */
+	lpfc_disc_illegal,		/* CMPL_ADISC      */
+	lpfc_disc_illegal,		/* CMPL_REG_LOGIN  */
+	lpfc_device_rm_logo_issue,	/* DEVICE_RM       */
+	lpfc_device_recov_logo_issue,	/* DEVICE_RECOVERY */
+
 	lpfc_rcv_plogi_unmap_node,	/* RCV_PLOGI   UNMAPPED_NODE  */
 	lpfc_rcv_prli_unmap_node,	/* RCV_PRLI        */
 	lpfc_rcv_logo_unmap_node,	/* RCV_LOGO        */
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 925975d2d76..64013f3097a 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -60,12 +60,6 @@ static char *dif_op_str[] = {
 	"PROT_WRITE_PASS",
 };
 
-static char *dif_grd_str[] = {
-	"NO_GUARD",
-	"DIF_CRC",
-	"DIX_IP",
-};
-
 struct scsi_dif_tuple {
 	__be16 guard_tag;       /* Checksum */
 	__be16 app_tag;         /* Opaque storage */
@@ -3482,9 +3476,15 @@ lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	}
 	lp = (uint32_t *)cmnd->sense_buffer;
 
-	if (!scsi_status && (resp_info & RESID_UNDER) &&
-		vport->cfg_log_verbose & LOG_FCP_UNDER)
-		logit = LOG_FCP_UNDER;
+	/* special handling for under run conditions */
+	if (!scsi_status && (resp_info & RESID_UNDER)) {
+		/* don't log under runs if fcp set... */
+		if (vport->cfg_log_verbose & LOG_FCP)
+			logit = LOG_FCP_ERROR;
+		/* unless operator says so */
+		if (vport->cfg_log_verbose & LOG_FCP_UNDER)
+			logit = LOG_FCP_UNDER;
+	}
 
 	lpfc_printf_vlog(vport, KERN_WARNING, logit,
 			 "9024 FCP command x%x failed: x%x SNS x%x x%x "
@@ -3552,11 +3552,11 @@ lpfc_handle_fcp_err(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 
 	/*
 	 * Check SLI validation that all the transfer was actually done
-	 * (fcpi_parm should be zero). Apply check only to reads.
+	 * (fcpi_parm should be zero).
 	 */
-	} else if (fcpi_parm && (cmnd->sc_data_direction == DMA_FROM_DEVICE)) {
+	} else if (fcpi_parm) {
 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP | LOG_FCP_ERROR,
-				 "9029 FCP Read Check Error Data: "
+				 "9029 FCP Data Transfer Check Error: "
 				 "x%x x%x x%x x%x x%x\n",
 				 be32_to_cpu(fcpcmd->fcpDl),
 				 be32_to_cpu(fcprsp->rspResId),
@@ -3615,7 +3615,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 	cmd = lpfc_cmd->pCmd;
 	shost = cmd->device->host;
 
-	lpfc_cmd->result = pIocbOut->iocb.un.ulpWord[4];
+	lpfc_cmd->result = (pIocbOut->iocb.un.ulpWord[4] & IOERR_PARAM_MASK);
 	lpfc_cmd->status = pIocbOut->iocb.ulpStatus;
 	/* pick up SLI4 exhange busy status from HBA */
 	lpfc_cmd->exch_busy = pIocbOut->iocb_flag & LPFC_EXCHANGE_BUSY;
@@ -3660,10 +3660,10 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 			lpfc_cmd->status = IOSTAT_DRIVER_REJECT;
 		else if (lpfc_cmd->status >= IOSTAT_CNT)
 			lpfc_cmd->status = IOSTAT_DEFAULT;
-		if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR
-			&& !lpfc_cmd->fcp_rsp->rspStatus3
-			&& (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER)
-			&& !(phba->cfg_log_verbose & LOG_FCP_UNDER))
+		if (lpfc_cmd->status == IOSTAT_FCP_RSP_ERROR &&
+		    !lpfc_cmd->fcp_rsp->rspStatus3 &&
+		    (lpfc_cmd->fcp_rsp->rspStatus2 & RESID_UNDER) &&
+		    !(vport->cfg_log_verbose & LOG_FCP_UNDER))
 			logit = 0;
 		else
 			logit = LOG_FCP | LOG_FCP_UNDER;
@@ -3829,12 +3829,15 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 	cmd->scsi_done(cmd);
 
 	if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
+		spin_lock_irq(&phba->hbalock);
+		lpfc_cmd->pCmd = NULL;
+		spin_unlock_irq(&phba->hbalock);
+
 		/*
 		 * If there is a thread waiting for command completion
 		 * wake up the thread.
 		 */
 		spin_lock_irqsave(shost->host_lock, flags);
-		lpfc_cmd->pCmd = NULL;
 		if (lpfc_cmd->waitq)
 			wake_up(lpfc_cmd->waitq);
 		spin_unlock_irqrestore(shost->host_lock, flags);
@@ -3868,12 +3871,15 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 		}
 	}
 
+	spin_lock_irq(&phba->hbalock);
+	lpfc_cmd->pCmd = NULL;
+	spin_unlock_irq(&phba->hbalock);
+
 	/*
 	 * If there is a thread waiting for command completion
 	 * wake up the thread.
 	 */
 	spin_lock_irqsave(shost->host_lock, flags);
-	lpfc_cmd->pCmd = NULL;
 	if (lpfc_cmd->waitq)
 		wake_up(lpfc_cmd->waitq);
 	spin_unlock_irqrestore(shost->host_lock, flags);
@@ -3919,6 +3925,8 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	struct lpfc_iocbq *piocbq = &(lpfc_cmd->cur_iocbq);
 	int datadir = scsi_cmnd->sc_data_direction;
 	char tag[2];
+	uint8_t *ptr;
+	bool sli4;
 
 	if (!pnode || !NLP_CHK_NODE_ACT(pnode))
 		return;
@@ -3930,8 +3938,13 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	int_to_scsilun(lpfc_cmd->pCmd->device->lun,
 			&lpfc_cmd->fcp_cmnd->fcp_lun);
 
-	memset(&fcp_cmnd->fcpCdb[0], 0, LPFC_FCP_CDB_LEN);
-	memcpy(&fcp_cmnd->fcpCdb[0], scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
+	ptr = &fcp_cmnd->fcpCdb[0];
+	memcpy(ptr, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
+	if (scsi_cmnd->cmd_len < LPFC_FCP_CDB_LEN) {
+		ptr += scsi_cmnd->cmd_len;
+		memset(ptr, 0, (LPFC_FCP_CDB_LEN - scsi_cmnd->cmd_len));
+	}
+
 	if (scsi_populate_tag_msg(scsi_cmnd, tag)) {
 		switch (tag[0]) {
 		case HEAD_OF_QUEUE_TAG:
@@ -3947,6 +3960,8 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	} else
 		fcp_cmnd->fcpCntl1 = 0;
 
+	sli4 = (phba->sli_rev == LPFC_SLI_REV4);
+
 	/*
 	 * There are three possibilities here - use scatter-gather segment, use
 	 * the single mapping, or neither.  Start the lpfc command prep by
@@ -3956,11 +3971,12 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	if (scsi_sg_count(scsi_cmnd)) {
 		if (datadir == DMA_TO_DEVICE) {
 			iocb_cmd->ulpCommand = CMD_FCP_IWRITE64_CR;
-			if (phba->sli_rev < LPFC_SLI_REV4) {
+			if (sli4)
+				iocb_cmd->ulpPU = PARM_READ_CHECK;
+			else {
 				iocb_cmd->un.fcpi.fcpi_parm = 0;
 				iocb_cmd->ulpPU = 0;
-			} else
-				iocb_cmd->ulpPU = PARM_READ_CHECK;
+			}
 			fcp_cmnd->fcpCntl3 = WRITE_DATA;
 			phba->fc4OutputRequests++;
 		} else {
@@ -3984,7 +4000,7 @@ lpfc_scsi_prep_cmnd(struct lpfc_vport *vport, struct lpfc_scsi_buf *lpfc_cmd,
 	 * of the scsi_cmnd request_buffer
 	 */
 	piocbq->iocb.ulpContext = pnode->nlp_rpi;
-	if (phba->sli_rev == LPFC_SLI_REV4)
+	if (sli4)
 		piocbq->iocb.ulpContext =
 		  phba->sli4_hba.rpi_ids[pnode->nlp_rpi];
 	if (pnode->nlp_fcp_info & NLP_FCP_2_DEVICE)
@@ -4241,9 +4257,8 @@ void lpfc_poll_timeout(unsigned long ptr)
  *   SCSI_MLQUEUE_HOST_BUSY - Block all devices served by this host temporarily.
  **/
 static int
-lpfc_queuecommand_lck(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
+lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 {
-	struct Scsi_Host  *shost = cmnd->device->host;
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 	struct lpfc_rport_data *rdata = cmnd->device->hostdata;
@@ -4299,53 +4314,28 @@ lpfc_queuecommand_lck(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 	lpfc_cmd->timeout = 0;
 	lpfc_cmd->start_time = jiffies;
 	cmnd->host_scribble = (unsigned char *)lpfc_cmd;
-	cmnd->scsi_done = done;
 
 	if (scsi_get_prot_op(cmnd) != SCSI_PROT_NORMAL) {
 		if (vport->phba->cfg_enable_bg) {
-			lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-				"9033 BLKGRD: rcvd protected cmd:%02x op=%s "
-				"guard=%s\n", cmnd->cmnd[0],
-				dif_op_str[scsi_get_prot_op(cmnd)],
-				dif_grd_str[scsi_host_get_guard(shost)]);
-			if (cmnd->cmnd[0] == READ_10)
-				lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-					"9035 BLKGRD: READ @ sector %llu, "
-					"cnt %u, rpt %d\n",
-					(unsigned long long)scsi_get_lba(cmnd),
-					blk_rq_sectors(cmnd->request),
-					(cmnd->cmnd[1]>>5));
-			else if (cmnd->cmnd[0] == WRITE_10)
-				lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-					"9036 BLKGRD: WRITE @ sector %llu, "
-					"cnt %u, wpt %d\n",
-					(unsigned long long)scsi_get_lba(cmnd),
-					blk_rq_sectors(cmnd->request),
-					(cmnd->cmnd[1]>>5));
+			lpfc_printf_vlog(vport, KERN_INFO, LOG_BG,
+					 "9033 BLKGRD: rcvd %s cmd:x%x "
+					 "sector x%llx cnt %u pt %x\n",
+					 dif_op_str[scsi_get_prot_op(cmnd)],
+					 cmnd->cmnd[0],
+					 (unsigned long long)scsi_get_lba(cmnd),
+					 blk_rq_sectors(cmnd->request),
+					 (cmnd->cmnd[1]>>5));
 		}
-
 		err = lpfc_bg_scsi_prep_dma_buf(phba, lpfc_cmd);
 	} else {
 		if (vport->phba->cfg_enable_bg) {
-			lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-				"9038 BLKGRD: rcvd unprotected cmd:"
-				"%02x op=%s guard=%s\n", cmnd->cmnd[0],
-				dif_op_str[scsi_get_prot_op(cmnd)],
-				dif_grd_str[scsi_host_get_guard(shost)]);
-			if (cmnd->cmnd[0] == READ_10)
-				lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-					"9040 dbg: READ @ sector %llu, "
-					"cnt %u, rpt %d\n",
-					(unsigned long long)scsi_get_lba(cmnd),
+			lpfc_printf_vlog(vport, KERN_INFO, LOG_BG,
+					 "9038 BLKGRD: rcvd PROT_NORMAL cmd: "
+					 "x%x sector x%llx cnt %u pt %x\n",
+					 cmnd->cmnd[0],
+					 (unsigned long long)scsi_get_lba(cmnd),
 					 blk_rq_sectors(cmnd->request),
-					(cmnd->cmnd[1]>>5));
-			else if (cmnd->cmnd[0] == WRITE_10)
-				lpfc_printf_vlog(vport, KERN_WARNING, LOG_BG,
-					"9041 dbg: WRITE @ sector %llu, "
-					"cnt %u, wpt %d\n",
-					(unsigned long long)scsi_get_lba(cmnd),
-					blk_rq_sectors(cmnd->request),
-					(cmnd->cmnd[1]>>5));
+					 (cmnd->cmnd[1]>>5));
 		}
 		err = lpfc_scsi_prep_dma_buf(phba, lpfc_cmd);
 	}
@@ -4363,11 +4353,9 @@ lpfc_queuecommand_lck(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 		goto out_host_busy_free_buf;
 	}
 	if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
-		spin_unlock(shost->host_lock);
 		lpfc_sli_handle_fast_ring_event(phba,
 			&phba->sli.ring[LPFC_FCP_RING], HA_R0RE_REQ);
 
-		spin_lock(shost->host_lock);
 		if (phba->cfg_poll & DISABLE_FCP_RING_INT)
 			lpfc_poll_rearm_timer(phba);
 	}
@@ -4384,11 +4372,10 @@ lpfc_queuecommand_lck(struct scsi_cmnd *cmnd, void (*done) (struct scsi_cmnd *))
 	return SCSI_MLQUEUE_TARGET_BUSY;
 
  out_fail_command:
-	done(cmnd);
+	cmnd->scsi_done(cmnd);
 	return 0;
 }
 
-static DEF_SCSI_QCMD(lpfc_queuecommand)
 
 /**
  * lpfc_abort_handler - scsi_host_template eh_abort_handler entry point
@@ -4414,7 +4401,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(waitq);
 
 	status = fc_block_scsi_eh(cmnd);
-	if (status)
+	if (status != 0 && status != SUCCESS)
 		return status;
 
 	spin_lock_irq(&phba->hbalock);
@@ -4428,7 +4415,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 	}
 
 	lpfc_cmd = (struct lpfc_scsi_buf *)cmnd->host_scribble;
-	if (!lpfc_cmd) {
+	if (!lpfc_cmd || !lpfc_cmd->pCmd) {
 		spin_unlock_irq(&phba->hbalock);
 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,
 			 "2873 SCSI Layer I/O Abort Request IO CMPL Status "
@@ -4521,9 +4508,10 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 		ret = FAILED;
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
 				 "0748 abort handler timed out waiting "
-				 "for abort to complete: ret %#x, ID %d, "
-				 "LUN %d\n",
-				 ret, cmnd->device->id, cmnd->device->lun);
+				 "for abortng I/O (xri:x%x) to complete: "
+				 "ret %#x, ID %d, LUN %d\n",
+				 iocb->sli4_xritag, ret,
+				 cmnd->device->id, cmnd->device->lun);
 	}
 	goto out;
 
@@ -4769,7 +4757,7 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
 	}
 	pnode = rdata->pnode;
 	status = fc_block_scsi_eh(cmnd);
-	if (status)
+	if (status != 0 && status != SUCCESS)
 		return status;
 
 	status = lpfc_chk_tgt_mapped(vport, cmnd);
@@ -4836,7 +4824,7 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
 	}
 	pnode = rdata->pnode;
 	status = fc_block_scsi_eh(cmnd);
-	if (status)
+	if (status != 0 && status != SUCCESS)
 		return status;
 
 	status = lpfc_chk_tgt_mapped(vport, cmnd);
@@ -4904,7 +4892,7 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd)
 		sizeof(scsi_event), (char *)&scsi_event, LPFC_NL_VENDOR_ID);
 
 	status = fc_block_scsi_eh(cmnd);
-	if (status)
+	if (status != 0 && status != SUCCESS)
 		return status;
 
 	/*
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 0e7e144507b..219bf534ef9 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -69,6 +69,8 @@ static int lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *, struct lpfc_queue *,
 				    struct lpfc_cqe *);
 static int lpfc_sli4_post_els_sgl_list(struct lpfc_hba *, struct list_head *,
 				       int);
+static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *, struct lpfc_eqe *,
+			uint32_t);
 
 static IOCB_t *
 lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
@@ -94,6 +96,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 	union lpfc_wqe *temp_wqe;
 	struct lpfc_register doorbell;
 	uint32_t host_index;
+	uint32_t idx;
 
 	/* sanity check on queue memory */
 	if (unlikely(!q))
@@ -101,8 +104,12 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 	temp_wqe = q->qe[q->host_index].wqe;
 
 	/* If the host has not yet processed the next entry then we are done */
-	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
+	idx = ((q->host_index + 1) % q->entry_count);
+	if (idx == q->hba_index) {
+		q->WQ_overflow++;
 		return -ENOMEM;
+	}
+	q->WQ_posted++;
 	/* set consumption flag every once in a while */
 	if (!((q->host_index + 1) % q->entry_repost))
 		bf_set(wqe_wqec, &wqe->generic.wqe_com, 1);
@@ -112,7 +119,8 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 
 	/* Update the host index before invoking device */
 	host_index = q->host_index;
-	q->host_index = ((q->host_index + 1) % q->entry_count);
+
+	q->host_index = idx;
 
 	/* Ring Doorbell */
 	doorbell.word0 = 0;
@@ -120,7 +128,6 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe *wqe)
 	bf_set(lpfc_wq_doorbell_index, &doorbell, host_index);
 	bf_set(lpfc_wq_doorbell_id, &doorbell, q->queue_id);
 	writel(doorbell.word0, q->phba->sli4_hba.WQDBregaddr);
-	readl(q->phba->sli4_hba.WQDBregaddr); /* Flush */
 
 	return 0;
 }
@@ -194,7 +201,6 @@ lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
 	bf_set(lpfc_mq_doorbell_num_posted, &doorbell, 1);
 	bf_set(lpfc_mq_doorbell_id, &doorbell, q->queue_id);
 	writel(doorbell.word0, q->phba->sli4_hba.MQDBregaddr);
-	readl(q->phba->sli4_hba.MQDBregaddr); /* Flush */
 	return 0;
 }
 
@@ -234,6 +240,7 @@ static struct lpfc_eqe *
 lpfc_sli4_eq_get(struct lpfc_queue *q)
 {
 	struct lpfc_eqe *eqe;
+	uint32_t idx;
 
 	/* sanity check on queue memory */
 	if (unlikely(!q))
@@ -244,14 +251,34 @@ lpfc_sli4_eq_get(struct lpfc_queue *q)
 	if (!bf_get_le32(lpfc_eqe_valid, eqe))
 		return NULL;
 	/* If the host has not yet processed the next entry then we are done */
-	if (((q->hba_index + 1) % q->entry_count) == q->host_index)
+	idx = ((q->hba_index + 1) % q->entry_count);
+	if (idx == q->host_index)
 		return NULL;
 
-	q->hba_index = ((q->hba_index + 1) % q->entry_count);
+	q->hba_index = idx;
 	return eqe;
 }
 
 /**
+ * lpfc_sli4_eq_clr_intr - Turn off interrupts from this EQ
+ * @q: The Event Queue to disable interrupts
+ *
+ **/
+static inline void
+lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
+{
+	struct lpfc_register doorbell;
+
+	doorbell.word0 = 0;
+	bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1);
+	bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT);
+	bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell,
+		(q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT));
+	bf_set(lpfc_eqcq_doorbell_eqid_lo, &doorbell, q->queue_id);
+	writel(doorbell.word0, q->phba->sli4_hba.EQCQDBregaddr);
+}
+
+/**
  * lpfc_sli4_eq_release - Indicates the host has finished processing an EQ
  * @q: The Event Queue that the host has completed processing for.
  * @arm: Indicates whether the host wants to arms this CQ.
@@ -318,6 +345,7 @@ static struct lpfc_cqe *
 lpfc_sli4_cq_get(struct lpfc_queue *q)
 {
 	struct lpfc_cqe *cqe;
+	uint32_t idx;
 
 	/* sanity check on queue memory */
 	if (unlikely(!q))
@@ -327,11 +355,12 @@ lpfc_sli4_cq_get(struct lpfc_queue *q)
 	if (!bf_get_le32(lpfc_cqe_valid, q->qe[q->hba_index].cqe))
 		return NULL;
 	/* If the host has not yet processed the next entry then we are done */
-	if (((q->hba_index + 1) % q->entry_count) == q->host_index)
+	idx = ((q->hba_index + 1) % q->entry_count);
+	if (idx == q->host_index)
 		return NULL;
 
 	cqe = q->qe[q->hba_index].cqe;
-	q->hba_index = ((q->hba_index + 1) % q->entry_count);
+	q->hba_index = idx;
 	return cqe;
 }
 
@@ -472,8 +501,8 @@ lpfc_sli4_rq_release(struct lpfc_queue *hq, struct lpfc_queue *dq)
 static inline IOCB_t *
 lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 {
-	return (IOCB_t *) (((char *) pring->cmdringaddr) +
-			   pring->cmdidx * phba->iocb_cmd_size);
+	return (IOCB_t *) (((char *) pring->sli.sli3.cmdringaddr) +
+			   pring->sli.sli3.cmdidx * phba->iocb_cmd_size);
 }
 
 /**
@@ -489,8 +518,8 @@ lpfc_cmd_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 static inline IOCB_t *
 lpfc_resp_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 {
-	return (IOCB_t *) (((char *) pring->rspringaddr) +
-			   pring->rspidx * phba->iocb_rsp_size);
+	return (IOCB_t *) (((char *) pring->sli.sli3.rspringaddr) +
+			   pring->sli.sli3.rspidx * phba->iocb_rsp_size);
 }
 
 /**
@@ -1320,21 +1349,23 @@ static IOCB_t *
 lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 {
 	struct lpfc_pgp *pgp = &phba->port_gp[pring->ringno];
-	uint32_t  max_cmd_idx = pring->numCiocb;
-	if ((pring->next_cmdidx == pring->cmdidx) &&
-	   (++pring->next_cmdidx >= max_cmd_idx))
-		pring->next_cmdidx = 0;
+	uint32_t  max_cmd_idx = pring->sli.sli3.numCiocb;
+	if ((pring->sli.sli3.next_cmdidx == pring->sli.sli3.cmdidx) &&
+	   (++pring->sli.sli3.next_cmdidx >= max_cmd_idx))
+		pring->sli.sli3.next_cmdidx = 0;
 
-	if (unlikely(pring->local_getidx == pring->next_cmdidx)) {
+	if (unlikely(pring->sli.sli3.local_getidx ==
+		pring->sli.sli3.next_cmdidx)) {
 
-		pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
+		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
 
-		if (unlikely(pring->local_getidx >= max_cmd_idx)) {
+		if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 					"0315 Ring %d issue: portCmdGet %d "
 					"is bigger than cmd ring %d\n",
 					pring->ringno,
-					pring->local_getidx, max_cmd_idx);
+					pring->sli.sli3.local_getidx,
+					max_cmd_idx);
 
 			phba->link_state = LPFC_HBA_ERROR;
 			/*
@@ -1349,7 +1380,7 @@ lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 			return NULL;
 		}
 
-		if (pring->local_getidx == pring->next_cmdidx)
+		if (pring->sli.sli3.local_getidx == pring->sli.sli3.next_cmdidx)
 			return NULL;
 	}
 
@@ -1484,8 +1515,8 @@ lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	 * Let the HBA know what IOCB slot will be the next one the
 	 * driver will put a command into.
 	 */
-	pring->cmdidx = pring->next_cmdidx;
-	writel(pring->cmdidx, &phba->host_gp[pring->ringno].cmdPutInx);
+	pring->sli.sli3.cmdidx = pring->sli.sli3.next_cmdidx;
+	writel(pring->sli.sli3.cmdidx, &phba->host_gp[pring->ringno].cmdPutInx);
 }
 
 /**
@@ -2056,6 +2087,7 @@ lpfc_sli_chk_mbx_command(uint8_t mbxCommand)
 	case MBX_READ_EVENT_LOG:
 	case MBX_SECURITY_MGMT:
 	case MBX_AUTH_PORT:
+	case MBX_ACCESS_VDATA:
 		ret = mbxCommand;
 		break;
 	default:
@@ -2786,7 +2818,7 @@ lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 			"0312 Ring %d handler: portRspPut %d "
 			"is bigger than rsp ring %d\n",
 			pring->ringno, le32_to_cpu(pgp->rspPutInx),
-			pring->numRiocb);
+			pring->sli.sli3.numRiocb);
 
 	phba->link_state = LPFC_HBA_ERROR;
 
@@ -2815,10 +2847,26 @@ lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 void lpfc_poll_eratt(unsigned long ptr)
 {
 	struct lpfc_hba *phba;
-	uint32_t eratt = 0;
+	uint32_t eratt = 0, rem;
+	uint64_t sli_intr, cnt;
 
 	phba = (struct lpfc_hba *)ptr;
 
+	/* Here we will also keep track of interrupts per sec of the hba */
+	sli_intr = phba->sli.slistat.sli_intr;
+
+	if (phba->sli.slistat.sli_prev_intr > sli_intr)
+		cnt = (((uint64_t)(-1) - phba->sli.slistat.sli_prev_intr) +
+			sli_intr);
+	else
+		cnt = (sli_intr - phba->sli.slistat.sli_prev_intr);
+
+	/* 64-bit integer division not supporte on 32-bit x86 - use do_div */
+	rem = do_div(cnt, LPFC_ERATT_POLL_INTERVAL);
+	phba->sli.slistat.sli_ips = cnt;
+
+	phba->sli.slistat.sli_prev_intr = sli_intr;
+
 	/* Check chip HA register for error event */
 	eratt = lpfc_sli_check_eratt(phba);
 
@@ -2873,7 +2921,7 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 	 * The next available response entry should never exceed the maximum
 	 * entries.  If it does, treat it as an adapter hardware error.
 	 */
-	portRspMax = pring->numRiocb;
+	portRspMax = pring->sli.sli3.numRiocb;
 	portRspPut = le32_to_cpu(pgp->rspPutInx);
 	if (unlikely(portRspPut >= portRspMax)) {
 		lpfc_sli_rsp_pointers_error(phba, pring);
@@ -2887,7 +2935,7 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 		phba->fcp_ring_in_use = 1;
 
 	rmb();
-	while (pring->rspidx != portRspPut) {
+	while (pring->sli.sli3.rspidx != portRspPut) {
 		/*
 		 * Fetch an entry off the ring and copy it into a local data
 		 * structure.  The copy involves a byte-swap since the
@@ -2896,8 +2944,8 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 		entry = lpfc_resp_iocb(phba, pring);
 		phba->last_completion_time = jiffies;
 
-		if (++pring->rspidx >= portRspMax)
-			pring->rspidx = 0;
+		if (++pring->sli.sli3.rspidx >= portRspMax)
+			pring->sli.sli3.rspidx = 0;
 
 		lpfc_sli_pcimem_bcopy((uint32_t *) entry,
 				      (uint32_t *) &rspiocbq.iocb,
@@ -2915,7 +2963,8 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 			 * queuedepths of the SCSI device.
 			 */
 			if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
-				(irsp->un.ulpWord[4] == IOERR_NO_RESOURCES)) {
+			    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+			     IOERR_NO_RESOURCES)) {
 				spin_unlock_irqrestore(&phba->hbalock, iflag);
 				phba->lpfc_rampdown_queue_depth(phba);
 				spin_lock_irqsave(&phba->hbalock, iflag);
@@ -2998,9 +3047,10 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 		 * been updated, sync the pgp->rspPutInx and fetch the new port
 		 * response put pointer.
 		 */
-		writel(pring->rspidx, &phba->host_gp[pring->ringno].rspGetInx);
+		writel(pring->sli.sli3.rspidx,
+			&phba->host_gp[pring->ringno].rspGetInx);
 
-		if (pring->rspidx == portRspPut)
+		if (pring->sli.sli3.rspidx == portRspPut)
 			portRspPut = le32_to_cpu(pgp->rspPutInx);
 	}
 
@@ -3015,7 +3065,7 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba,
 		pring->stats.iocb_cmd_empty++;
 
 		/* Force update of the local copy of cmdGetInx */
-		pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
+		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
 		lpfc_sli_resume_iocb(phba, pring);
 
 		if ((pring->lpfc_sli_cmd_available))
@@ -3086,7 +3136,8 @@ lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 		 * queuedepths of the SCSI device.
 		 */
 		if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) &&
-		    (irsp->un.ulpWord[4] == IOERR_NO_RESOURCES)) {
+		    ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) ==
+		     IOERR_NO_RESOURCES)) {
 			spin_unlock_irqrestore(&phba->hbalock, iflag);
 			phba->lpfc_rampdown_queue_depth(phba);
 			spin_lock_irqsave(&phba->hbalock, iflag);
@@ -3247,7 +3298,7 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 	 * The next available response entry should never exceed the maximum
 	 * entries.  If it does, treat it as an adapter hardware error.
 	 */
-	portRspMax = pring->numRiocb;
+	portRspMax = pring->sli.sli3.numRiocb;
 	portRspPut = le32_to_cpu(pgp->rspPutInx);
 	if (portRspPut >= portRspMax) {
 		/*
@@ -3269,7 +3320,7 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 	}
 
 	rmb();
-	while (pring->rspidx != portRspPut) {
+	while (pring->sli.sli3.rspidx != portRspPut) {
 		/*
 		 * Build a completion list and call the appropriate handler.
 		 * The process is to get the next available response iocb, get
@@ -3297,8 +3348,8 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 				      phba->iocb_rsp_size);
 		irsp = &rspiocbp->iocb;
 
-		if (++pring->rspidx >= portRspMax)
-			pring->rspidx = 0;
+		if (++pring->sli.sli3.rspidx >= portRspMax)
+			pring->sli.sli3.rspidx = 0;
 
 		if (pring->ringno == LPFC_ELS_RING) {
 			lpfc_debugfs_slow_ring_trc(phba,
@@ -3308,7 +3359,8 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 				*(((uint32_t *) irsp) + 7));
 		}
 
-		writel(pring->rspidx, &phba->host_gp[pring->ringno].rspGetInx);
+		writel(pring->sli.sli3.rspidx,
+			&phba->host_gp[pring->ringno].rspGetInx);
 
 		spin_unlock_irqrestore(&phba->hbalock, iflag);
 		/* Handle the response IOCB */
@@ -3320,10 +3372,10 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 		 * the pgp->rspPutInx in the MAILBOX_tand fetch the new port
 		 * response put pointer.
 		 */
-		if (pring->rspidx == portRspPut) {
+		if (pring->sli.sli3.rspidx == portRspPut) {
 			portRspPut = le32_to_cpu(pgp->rspPutInx);
 		}
-	} /* while (pring->rspidx != portRspPut) */
+	} /* while (pring->sli.sli3.rspidx != portRspPut) */
 
 	if ((rspiocbp != NULL) && (mask & HA_R0RE_REQ)) {
 		/* At least one response entry has been freed */
@@ -3338,7 +3390,7 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba,
 		pring->stats.iocb_cmd_empty++;
 
 		/* Force update of the local copy of cmdGetInx */
-		pring->local_getidx = le32_to_cpu(pgp->cmdGetInx);
+		pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx);
 		lpfc_sli_resume_iocb(phba, pring);
 
 		if ((pring->lpfc_sli_cmd_available))
@@ -3859,10 +3911,10 @@ lpfc_sli_brdreset(struct lpfc_hba *phba)
 	for (i = 0; i < psli->num_rings; i++) {
 		pring = &psli->ring[i];
 		pring->flag = 0;
-		pring->rspidx = 0;
-		pring->next_cmdidx  = 0;
-		pring->local_getidx = 0;
-		pring->cmdidx = 0;
+		pring->sli.sli3.rspidx = 0;
+		pring->sli.sli3.next_cmdidx  = 0;
+		pring->sli.sli3.local_getidx = 0;
+		pring->sli.sli3.cmdidx = 0;
 		pring->missbufcnt = 0;
 	}
 
@@ -4893,16 +4945,15 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba)
 	lpfc_sli4_cq_release(phba->sli4_hba.els_cq, LPFC_QUEUE_REARM);
 	fcp_eqidx = 0;
 	if (phba->sli4_hba.fcp_cq) {
-		do
+		do {
 			lpfc_sli4_cq_release(phba->sli4_hba.fcp_cq[fcp_eqidx],
 					     LPFC_QUEUE_REARM);
-		while (++fcp_eqidx < phba->cfg_fcp_eq_count);
+		} while (++fcp_eqidx < phba->cfg_fcp_io_channel);
 	}
-	lpfc_sli4_eq_release(phba->sli4_hba.sp_eq, LPFC_QUEUE_REARM);
-	if (phba->sli4_hba.fp_eq) {
-		for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_eq_count;
+	if (phba->sli4_hba.hba_eq) {
+		for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel;
 		     fcp_eqidx++)
-			lpfc_sli4_eq_release(phba->sli4_hba.fp_eq[fcp_eqidx],
+			lpfc_sli4_eq_release(phba->sli4_hba.hba_eq[fcp_eqidx],
 					     LPFC_QUEUE_REARM);
 	}
 }
@@ -7784,14 +7835,18 @@ lpfc_sli4_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq,
  *
  * Return: index into SLI4 fast-path FCP queue index.
  **/
-static uint32_t
+static inline uint32_t
 lpfc_sli4_scmd_to_wqidx_distr(struct lpfc_hba *phba)
 {
-	++phba->fcp_qidx;
-	if (phba->fcp_qidx >= phba->cfg_fcp_wq_count)
-		phba->fcp_qidx = 0;
+	int i;
+
+	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_CPU)
+		i = smp_processor_id();
+	else
+		i = atomic_add_return(1, &phba->fcp_qidx);
 
-	return phba->fcp_qidx;
+	i = (i % phba->cfg_fcp_io_channel);
+	return i;
 }
 
 /**
@@ -8311,16 +8366,6 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number,
 
 	if ((piocb->iocb_flag & LPFC_IO_FCP) ||
 		(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) {
-		/*
-		 * For FCP command IOCB, get a new WQ index to distribute
-		 * WQE across the WQsr. On the other hand, for abort IOCB,
-		 * it carries the same WQ index to the original command
-		 * IOCB.
-		 */
-		if (piocb->iocb_flag & LPFC_IO_FCP)
-			piocb->fcp_wqidx = lpfc_sli4_scmd_to_wqidx_distr(phba);
-		if (unlikely(!phba->sli4_hba.fcp_wq))
-			return IOCB_ERROR;
 		if (lpfc_sli4_wq_put(phba->sli4_hba.fcp_wq[piocb->fcp_wqidx],
 				     &wqe))
 			return IOCB_ERROR;
@@ -8401,13 +8446,68 @@ int
 lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number,
 		    struct lpfc_iocbq *piocb, uint32_t flag)
 {
+	struct lpfc_fcp_eq_hdl *fcp_eq_hdl;
+	struct lpfc_sli_ring *pring;
+	struct lpfc_queue *fpeq;
+	struct lpfc_eqe *eqe;
 	unsigned long iflags;
-	int rc;
+	int rc, idx;
 
-	spin_lock_irqsave(&phba->hbalock, iflags);
-	rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
-	spin_unlock_irqrestore(&phba->hbalock, iflags);
+	if (phba->sli_rev == LPFC_SLI_REV4) {
+		if (piocb->iocb_flag &  LPFC_IO_FCP) {
+			if (unlikely(!phba->sli4_hba.fcp_wq))
+				return IOCB_ERROR;
+			idx = lpfc_sli4_scmd_to_wqidx_distr(phba);
+			piocb->fcp_wqidx = idx;
+			ring_number = MAX_SLI3_CONFIGURED_RINGS + idx;
+
+			pring = &phba->sli.ring[ring_number];
+			spin_lock_irqsave(&pring->ring_lock, iflags);
+			rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb,
+				flag);
+			spin_unlock_irqrestore(&pring->ring_lock, iflags);
+
+			if (lpfc_fcp_look_ahead) {
+				fcp_eq_hdl = &phba->sli4_hba.fcp_eq_hdl[idx];
+
+				if (atomic_dec_and_test(&fcp_eq_hdl->
+					fcp_eq_in_use)) {
 
+					/* Get associated EQ with this index */
+					fpeq = phba->sli4_hba.hba_eq[idx];
+
+					/* Turn off interrupts from this EQ */
+					lpfc_sli4_eq_clr_intr(fpeq);
+
+					/*
+					 * Process all the events on FCP EQ
+					 */
+					while ((eqe = lpfc_sli4_eq_get(fpeq))) {
+						lpfc_sli4_hba_handle_eqe(phba,
+							eqe, idx);
+						fpeq->EQ_processed++;
+					}
+
+					/* Always clear and re-arm the EQ */
+					lpfc_sli4_eq_release(fpeq,
+						LPFC_QUEUE_REARM);
+				}
+				atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+			}
+		} else {
+			pring = &phba->sli.ring[ring_number];
+			spin_lock_irqsave(&pring->ring_lock, iflags);
+			rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb,
+				flag);
+			spin_unlock_irqrestore(&pring->ring_lock, iflags);
+
+		}
+	} else {
+		/* For now, SLI2/3 will still use hbalock */
+		spin_lock_irqsave(&phba->hbalock, iflags);
+		rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag);
+		spin_unlock_irqrestore(&phba->hbalock, iflags);
+	}
 	return rc;
 }
 
@@ -8434,18 +8534,18 @@ lpfc_extra_ring_setup( struct lpfc_hba *phba)
 
 	/* Take some away from the FCP ring */
 	pring = &psli->ring[psli->fcp_ring];
-	pring->numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
-	pring->numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
-	pring->numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
-	pring->numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;
+	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R1XTRA_ENTRIES;
+	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R1XTRA_ENTRIES;
+	pring->sli.sli3.numCiocb -= SLI2_IOCB_CMD_R3XTRA_ENTRIES;
+	pring->sli.sli3.numRiocb -= SLI2_IOCB_RSP_R3XTRA_ENTRIES;
 
 	/* and give them to the extra ring */
 	pring = &psli->ring[psli->extra_ring];
 
-	pring->numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
-	pring->numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
-	pring->numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
-	pring->numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;
+	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
+	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
+	pring->sli.sli3.numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
+	pring->sli.sli3.numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;
 
 	/* Setup default profile for this ring */
 	pring->iotag_max = 4096;
@@ -8457,56 +8557,6 @@ lpfc_extra_ring_setup( struct lpfc_hba *phba)
 	return 0;
 }
 
-/* lpfc_sli_abts_recover_port - Recover a port that failed an ABTS.
- * @vport: pointer to virtual port object.
- * @ndlp: nodelist pointer for the impacted rport.
- *
- * The driver calls this routine in response to a XRI ABORT CQE
- * event from the port.  In this event, the driver is required to
- * recover its login to the rport even though its login may be valid
- * from the driver's perspective.  The failed ABTS notice from the
- * port indicates the rport is not responding.
- */
-static void
-lpfc_sli_abts_recover_port(struct lpfc_vport *vport,
-			   struct lpfc_nodelist *ndlp)
-{
-	struct Scsi_Host *shost;
-	struct lpfc_hba *phba;
-	unsigned long flags = 0;
-
-	shost = lpfc_shost_from_vport(vport);
-	phba = vport->phba;
-	if (ndlp->nlp_state != NLP_STE_MAPPED_NODE) {
-		lpfc_printf_log(phba, KERN_INFO,
-			LOG_SLI, "3093 No rport recovery needed. "
-			"rport in state 0x%x\n",
-			ndlp->nlp_state);
-		return;
-	}
-	lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
-			"3094 Start rport recovery on shost id 0x%x "
-			"fc_id 0x%06x vpi 0x%x rpi 0x%x state 0x%x "
-			"flags 0x%x\n",
-			shost->host_no, ndlp->nlp_DID,
-			vport->vpi, ndlp->nlp_rpi, ndlp->nlp_state,
-			ndlp->nlp_flag);
-	/*
-	 * The rport is not responding.  Don't attempt ADISC recovery.
-	 * Remove the FCP-2 flag to force a PLOGI.
-	 */
-	spin_lock_irqsave(shost->host_lock, flags);
-	ndlp->nlp_fcp_info &= ~NLP_FCP_2_DEVICE;
-	spin_unlock_irqrestore(shost->host_lock, flags);
-	lpfc_disc_state_machine(vport, ndlp, NULL,
-				NLP_EVT_DEVICE_RECOVERY);
-	lpfc_cancel_retry_delay_tmo(vport, ndlp);
-	spin_lock_irqsave(shost->host_lock, flags);
-	ndlp->nlp_flag |= NLP_NPR_2B_DISC;
-	spin_unlock_irqrestore(shost->host_lock, flags);
-	lpfc_disc_start(vport);
-}
-
 /* lpfc_sli_abts_err_handler - handle a failed ABTS request from an SLI3 port.
  * @phba: Pointer to HBA context object.
  * @iocbq: Pointer to iocb object.
@@ -8594,7 +8644,7 @@ lpfc_sli4_abts_err_handler(struct lpfc_hba *phba,
 	 * LOCAL_REJECT and 0 for a failed ABTS exchange and later OCe and
 	 * LPe FW releases returned LOCAL_REJECT and SEQUENCE_TIMEOUT.
 	 */
-	ext_status = axri->parameter & WCQE_PARAM_MASK;
+	ext_status = axri->parameter & IOERR_PARAM_MASK;
 	if ((bf_get(lpfc_wcqe_xa_status, axri) == IOSTAT_LOCAL_REJECT) &&
 	    ((ext_status == IOERR_SEQUENCE_TIMEOUT) || (ext_status == 0)))
 		lpfc_sli_abts_recover_port(vport, ndlp);
@@ -8692,7 +8742,9 @@ lpfc_sli_setup(struct lpfc_hba *phba)
 	struct lpfc_sli *psli = &phba->sli;
 	struct lpfc_sli_ring *pring;
 
-	psli->num_rings = MAX_CONFIGURED_RINGS;
+	psli->num_rings = MAX_SLI3_CONFIGURED_RINGS;
+	if (phba->sli_rev == LPFC_SLI_REV4)
+		psli->num_rings += phba->cfg_fcp_io_channel;
 	psli->sli_flag = 0;
 	psli->fcp_ring = LPFC_FCP_RING;
 	psli->next_ring = LPFC_FCP_NEXT_RING;
@@ -8707,16 +8759,20 @@ lpfc_sli_setup(struct lpfc_hba *phba)
 		switch (i) {
 		case LPFC_FCP_RING:	/* ring 0 - FCP */
 			/* numCiocb and numRiocb are used in config_port */
-			pring->numCiocb = SLI2_IOCB_CMD_R0_ENTRIES;
-			pring->numRiocb = SLI2_IOCB_RSP_R0_ENTRIES;
-			pring->numCiocb += SLI2_IOCB_CMD_R1XTRA_ENTRIES;
-			pring->numRiocb += SLI2_IOCB_RSP_R1XTRA_ENTRIES;
-			pring->numCiocb += SLI2_IOCB_CMD_R3XTRA_ENTRIES;
-			pring->numRiocb += SLI2_IOCB_RSP_R3XTRA_ENTRIES;
-			pring->sizeCiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R0_ENTRIES;
+			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R0_ENTRIES;
+			pring->sli.sli3.numCiocb +=
+				SLI2_IOCB_CMD_R1XTRA_ENTRIES;
+			pring->sli.sli3.numRiocb +=
+				SLI2_IOCB_RSP_R1XTRA_ENTRIES;
+			pring->sli.sli3.numCiocb +=
+				SLI2_IOCB_CMD_R3XTRA_ENTRIES;
+			pring->sli.sli3.numRiocb +=
+				SLI2_IOCB_RSP_R3XTRA_ENTRIES;
+			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_CMD_SIZE :
 							SLI2_IOCB_CMD_SIZE;
-			pring->sizeRiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_RSP_SIZE :
 							SLI2_IOCB_RSP_SIZE;
 			pring->iotag_ctr = 0;
@@ -8727,12 +8783,12 @@ lpfc_sli_setup(struct lpfc_hba *phba)
 			break;
 		case LPFC_EXTRA_RING:	/* ring 1 - EXTRA */
 			/* numCiocb and numRiocb are used in config_port */
-			pring->numCiocb = SLI2_IOCB_CMD_R1_ENTRIES;
-			pring->numRiocb = SLI2_IOCB_RSP_R1_ENTRIES;
-			pring->sizeCiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R1_ENTRIES;
+			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R1_ENTRIES;
+			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_CMD_SIZE :
 							SLI2_IOCB_CMD_SIZE;
-			pring->sizeRiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_RSP_SIZE :
 							SLI2_IOCB_RSP_SIZE;
 			pring->iotag_max = phba->cfg_hba_queue_depth;
@@ -8740,12 +8796,12 @@ lpfc_sli_setup(struct lpfc_hba *phba)
 			break;
 		case LPFC_ELS_RING:	/* ring 2 - ELS / CT */
 			/* numCiocb and numRiocb are used in config_port */
-			pring->numCiocb = SLI2_IOCB_CMD_R2_ENTRIES;
-			pring->numRiocb = SLI2_IOCB_RSP_R2_ENTRIES;
-			pring->sizeCiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.numCiocb = SLI2_IOCB_CMD_R2_ENTRIES;
+			pring->sli.sli3.numRiocb = SLI2_IOCB_RSP_R2_ENTRIES;
+			pring->sli.sli3.sizeCiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_CMD_SIZE :
 							SLI2_IOCB_CMD_SIZE;
-			pring->sizeRiocb = (phba->sli_rev == 3) ?
+			pring->sli.sli3.sizeRiocb = (phba->sli_rev == 3) ?
 							SLI3_IOCB_RSP_SIZE :
 							SLI2_IOCB_RSP_SIZE;
 			pring->fast_iotag = 0;
@@ -8786,8 +8842,9 @@ lpfc_sli_setup(struct lpfc_hba *phba)
 			    lpfc_sli4_ct_abort_unsol_event;
 			break;
 		}
-		totiocbsize += (pring->numCiocb * pring->sizeCiocb) +
-				(pring->numRiocb * pring->sizeRiocb);
+		totiocbsize += (pring->sli.sli3.numCiocb *
+			pring->sli.sli3.sizeCiocb) +
+			(pring->sli.sli3.numRiocb * pring->sli.sli3.sizeRiocb);
 	}
 	if (totiocbsize > MAX_SLIM_IOCB_SIZE) {
 		/* Too many cmd / rsp ring entries in SLI2 SLIM */
@@ -8828,14 +8885,15 @@ lpfc_sli_queue_setup(struct lpfc_hba *phba)
 	for (i = 0; i < psli->num_rings; i++) {
 		pring = &psli->ring[i];
 		pring->ringno = i;
-		pring->next_cmdidx  = 0;
-		pring->local_getidx = 0;
-		pring->cmdidx = 0;
+		pring->sli.sli3.next_cmdidx  = 0;
+		pring->sli.sli3.local_getidx = 0;
+		pring->sli.sli3.cmdidx = 0;
 		INIT_LIST_HEAD(&pring->txq);
 		INIT_LIST_HEAD(&pring->txcmplq);
 		INIT_LIST_HEAD(&pring->iocb_continueq);
 		INIT_LIST_HEAD(&pring->iocb_continue_saveq);
 		INIT_LIST_HEAD(&pring->postbufq);
+		spin_lock_init(&pring->ring_lock);
 	}
 	spin_unlock_irq(&phba->hbalock);
 	return 1;
@@ -9334,6 +9392,7 @@ lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 	IOCB_t *icmd = NULL;
 	IOCB_t *iabt = NULL;
 	int retval;
+	unsigned long iflags;
 
 	/*
 	 * There are certain command types we don't want to abort.  And we
@@ -9386,7 +9445,17 @@ lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 			 iabt->un.acxri.abortIoTag,
 			 iabt->un.acxri.abortContextTag,
 			 abtsiocbp->iotag);
-	retval = __lpfc_sli_issue_iocb(phba, pring->ringno, abtsiocbp, 0);
+
+	if (phba->sli_rev == LPFC_SLI_REV4) {
+		/* Note: both hbalock and ring_lock need to be set here */
+		spin_lock_irqsave(&pring->ring_lock, iflags);
+		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
+			abtsiocbp, 0);
+		spin_unlock_irqrestore(&pring->ring_lock, iflags);
+	} else {
+		retval = __lpfc_sli_issue_iocb(phba, pring->ringno,
+			abtsiocbp, 0);
+	}
 
 	if (retval)
 		__lpfc_sli_release_iocbq(phba, abtsiocbp);
@@ -10947,12 +11016,12 @@ lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba,
 	unsigned long iflags;
 
 	wcqe = &irspiocbq->cq_event.cqe.wcqe_cmpl;
-	spin_lock_irqsave(&phba->hbalock, iflags);
+	spin_lock_irqsave(&pring->ring_lock, iflags);
 	pring->stats.iocb_event++;
 	/* Look up the ELS command IOCB and create pseudo response IOCB */
 	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
-	spin_unlock_irqrestore(&phba->hbalock, iflags);
+	spin_unlock_irqrestore(&pring->ring_lock, iflags);
 
 	if (unlikely(!cmdiocbq)) {
 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
@@ -11154,6 +11223,7 @@ lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe)
 /**
  * lpfc_sli4_sp_handle_els_wcqe - Handle els work-queue completion event
  * @phba: Pointer to HBA context object.
+ * @cq: Pointer to associated CQ
  * @wcqe: Pointer to work-queue completion queue entry.
  *
  * This routine handles an ELS work-queue completion event.
@@ -11161,12 +11231,12 @@ lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe)
  * Return: true if work posted to worker thread, otherwise false.
  **/
 static bool
-lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba,
+lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 			     struct lpfc_wcqe_complete *wcqe)
 {
 	struct lpfc_iocbq *irspiocbq;
 	unsigned long iflags;
-	struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_FCP_RING];
+	struct lpfc_sli_ring *pring = cq->pring;
 
 	/* Get an irspiocbq for later ELS response processing use */
 	irspiocbq = lpfc_sli_get_iocbq(phba);
@@ -11311,14 +11381,17 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
 	case FC_STATUS_RQ_BUF_LEN_EXCEEDED:
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"2537 Receive Frame Truncated!!\n");
+		hrq->RQ_buf_trunc++;
 	case FC_STATUS_RQ_SUCCESS:
 		lpfc_sli4_rq_release(hrq, drq);
 		spin_lock_irqsave(&phba->hbalock, iflags);
 		dma_buf = lpfc_sli_hbqbuf_get(&phba->hbqs[0].hbq_buffer_list);
 		if (!dma_buf) {
+			hrq->RQ_no_buf_found++;
 			spin_unlock_irqrestore(&phba->hbalock, iflags);
 			goto out;
 		}
+		hrq->RQ_rcv_buf++;
 		memcpy(&dma_buf->cq_event.cqe.rcqe_cmpl, rcqe, sizeof(*rcqe));
 		/* save off the frame for the word thread to process */
 		list_add_tail(&dma_buf->cq_event.list,
@@ -11330,6 +11403,7 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe)
 		break;
 	case FC_STATUS_INSUFF_BUF_NEED_BUF:
 	case FC_STATUS_INSUFF_BUF_FRM_DISC:
+		hrq->RQ_no_posted_buf++;
 		/* Post more buffers if possible */
 		spin_lock_irqsave(&phba->hbalock, iflags);
 		phba->hba_flag |= HBA_POST_RECEIVE_BUFFER;
@@ -11367,7 +11441,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 	case CQE_CODE_COMPL_WQE:
 		/* Process the WQ/RQ complete event */
 		phba->last_completion_time = jiffies;
-		workposted = lpfc_sli4_sp_handle_els_wcqe(phba,
+		workposted = lpfc_sli4_sp_handle_els_wcqe(phba, cq,
 				(struct lpfc_wcqe_complete *)&cqevt);
 		break;
 	case CQE_CODE_RELEASE_WQE:
@@ -11411,31 +11485,18 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
  *
  **/
 static void
-lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe)
+lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
+	struct lpfc_queue *speq)
 {
-	struct lpfc_queue *cq = NULL, *childq, *speq;
+	struct lpfc_queue *cq = NULL, *childq;
 	struct lpfc_cqe *cqe;
 	bool workposted = false;
 	int ecount = 0;
 	uint16_t cqid;
 
-	if (bf_get_le32(lpfc_eqe_major_code, eqe) != 0) {
-		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
-				"0359 Not a valid slow-path completion "
-				"event: majorcode=x%x, minorcode=x%x\n",
-				bf_get_le32(lpfc_eqe_major_code, eqe),
-				bf_get_le32(lpfc_eqe_minor_code, eqe));
-		return;
-	}
-
 	/* Get the reference to the corresponding CQ */
 	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
 
-	/* Search for completion queue pointer matching this cqid */
-	speq = phba->sli4_hba.sp_eq;
-	/* sanity check on queue memory */
-	if (unlikely(!speq))
-		return;
 	list_for_each_entry(childq, &speq->child_list, list) {
 		if (childq->queue_id == cqid) {
 			cq = childq;
@@ -11457,6 +11518,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe)
 			workposted |= lpfc_sli4_sp_handle_mcqe(phba, cqe);
 			if (!(++ecount % cq->entry_repost))
 				lpfc_sli4_cq_release(cq, LPFC_QUEUE_NOARM);
+			cq->CQ_mbox++;
 		}
 		break;
 	case LPFC_WCQ:
@@ -11470,6 +11532,10 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe)
 			if (!(++ecount % cq->entry_repost))
 				lpfc_sli4_cq_release(cq, LPFC_QUEUE_NOARM);
 		}
+
+		/* Track the max number of CQEs processed in 1 EQ */
+		if (ecount > cq->CQ_max_cqe)
+			cq->CQ_max_cqe = ecount;
 		break;
 	default:
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
@@ -11494,34 +11560,33 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe)
 
 /**
  * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry
- * @eqe: Pointer to fast-path completion queue entry.
+ * @phba: Pointer to HBA context object.
+ * @cq: Pointer to associated CQ
+ * @wcqe: Pointer to work-queue completion queue entry.
  *
  * This routine process a fast-path work queue completion entry from fast-path
  * event queue for FCP command response completion.
  **/
 static void
-lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba,
+lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 			     struct lpfc_wcqe_complete *wcqe)
 {
-	struct lpfc_sli_ring *pring = &phba->sli.ring[LPFC_FCP_RING];
+	struct lpfc_sli_ring *pring = cq->pring;
 	struct lpfc_iocbq *cmdiocbq;
 	struct lpfc_iocbq irspiocbq;
 	unsigned long iflags;
 
-	spin_lock_irqsave(&phba->hbalock, iflags);
-	pring->stats.iocb_event++;
-	spin_unlock_irqrestore(&phba->hbalock, iflags);
-
 	/* Check for response status */
 	if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) {
 		/* If resource errors reported from HBA, reduce queue
 		 * depth of the SCSI device.
 		 */
-		if ((bf_get(lpfc_wcqe_c_status, wcqe) ==
-		     IOSTAT_LOCAL_REJECT) &&
-		    (wcqe->parameter == IOERR_NO_RESOURCES)) {
+		if (((bf_get(lpfc_wcqe_c_status, wcqe) ==
+		     IOSTAT_LOCAL_REJECT)) &&
+		    ((wcqe->parameter & IOERR_PARAM_MASK) ==
+		     IOERR_NO_RESOURCES))
 			phba->lpfc_rampdown_queue_depth(phba);
-		}
+
 		/* Log the error status */
 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
 				"0373 FCP complete error: status=x%x, "
@@ -11534,10 +11599,11 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba,
 	}
 
 	/* Look up the FCP command IOCB and create pseudo response IOCB */
-	spin_lock_irqsave(&phba->hbalock, iflags);
+	spin_lock_irqsave(&pring->ring_lock, iflags);
+	pring->stats.iocb_event++;
 	cmdiocbq = lpfc_sli_iocbq_lookup_by_tag(phba, pring,
 				bf_get(lpfc_wcqe_c_request_tag, wcqe));
-	spin_unlock_irqrestore(&phba->hbalock, iflags);
+	spin_unlock_irqrestore(&pring->ring_lock, iflags);
 	if (unlikely(!cmdiocbq)) {
 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
 				"0374 FCP complete with no corresponding "
@@ -11621,17 +11687,20 @@ lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 	/* Check and process for different type of WCQE and dispatch */
 	switch (bf_get(lpfc_wcqe_c_code, &wcqe)) {
 	case CQE_CODE_COMPL_WQE:
+		cq->CQ_wq++;
 		/* Process the WQ complete event */
 		phba->last_completion_time = jiffies;
-		lpfc_sli4_fp_handle_fcp_wcqe(phba,
+		lpfc_sli4_fp_handle_fcp_wcqe(phba, cq,
 				(struct lpfc_wcqe_complete *)&wcqe);
 		break;
 	case CQE_CODE_RELEASE_WQE:
+		cq->CQ_release_wqe++;
 		/* Process the WQ release event */
 		lpfc_sli4_fp_handle_rel_wcqe(phba, cq,
 				(struct lpfc_wcqe_release *)&wcqe);
 		break;
 	case CQE_CODE_XRI_ABORTED:
+		cq->CQ_xri_aborted++;
 		/* Process the WQ XRI abort event */
 		phba->last_completion_time = jiffies;
 		workposted = lpfc_sli4_sp_handle_abort_xri_wcqe(phba, cq,
@@ -11647,7 +11716,7 @@ lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 }
 
 /**
- * lpfc_sli4_fp_handle_eqe - Process a fast-path event queue entry
+ * lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry
  * @phba: Pointer to HBA context object.
  * @eqe: Pointer to fast-path event queue entry.
  *
@@ -11659,8 +11728,8 @@ lpfc_sli4_fp_handle_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
  * completion queue, and then return.
  **/
 static void
-lpfc_sli4_fp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
-			uint32_t fcp_cqidx)
+lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
+			uint32_t qidx)
 {
 	struct lpfc_queue *cq;
 	struct lpfc_cqe *cqe;
@@ -11670,30 +11739,38 @@ lpfc_sli4_fp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 
 	if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
-				"0366 Not a valid fast-path completion "
+				"0366 Not a valid completion "
 				"event: majorcode=x%x, minorcode=x%x\n",
 				bf_get_le32(lpfc_eqe_major_code, eqe),
 				bf_get_le32(lpfc_eqe_minor_code, eqe));
 		return;
 	}
 
+	/* Get the reference to the corresponding CQ */
+	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
+
+	/* Check if this is a Slow path event */
+	if (unlikely(cqid != phba->sli4_hba.fcp_cq_map[qidx])) {
+		lpfc_sli4_sp_handle_eqe(phba, eqe,
+			phba->sli4_hba.hba_eq[qidx]);
+		return;
+	}
+
 	if (unlikely(!phba->sli4_hba.fcp_cq)) {
 		lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
 				"3146 Fast-path completion queues "
 				"does not exist\n");
 		return;
 	}
-	cq = phba->sli4_hba.fcp_cq[fcp_cqidx];
+	cq = phba->sli4_hba.fcp_cq[qidx];
 	if (unlikely(!cq)) {
 		if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
 			lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 					"0367 Fast-path completion queue "
-					"(%d) does not exist\n", fcp_cqidx);
+					"(%d) does not exist\n", qidx);
 		return;
 	}
 
-	/* Get the reference to the corresponding CQ */
-	cqid = bf_get_le32(lpfc_eqe_resource_id, eqe);
 	if (unlikely(cqid != cq->queue_id)) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"0368 Miss-matched fast-path completion "
@@ -11709,6 +11786,10 @@ lpfc_sli4_fp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 			lpfc_sli4_cq_release(cq, LPFC_QUEUE_NOARM);
 	}
 
+	/* Track the max number of CQEs processed in 1 EQ */
+	if (ecount > cq->CQ_max_cqe)
+		cq->CQ_max_cqe = ecount;
+
 	/* Catch the no cq entry condition */
 	if (unlikely(ecount == 0))
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
@@ -11737,86 +11818,7 @@ lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq)
 }
 
 /**
- * lpfc_sli4_sp_intr_handler - Slow-path interrupt handler to SLI-4 device
- * @irq: Interrupt number.
- * @dev_id: The device context pointer.
- *
- * This function is directly called from the PCI layer as an interrupt
- * service routine when device with SLI-4 interface spec is enabled with
- * MSI-X multi-message interrupt mode and there are slow-path events in
- * the HBA. However, when the device is enabled with either MSI or Pin-IRQ
- * interrupt mode, this function is called as part of the device-level
- * interrupt handler. When the PCI slot is in error recovery or the HBA is
- * undergoing initialization, the interrupt handler will not process the
- * interrupt. The link attention and ELS ring attention events are handled
- * by the worker thread. The interrupt handler signals the worker thread
- * and returns for these events. This function is called without any lock
- * held. It gets the hbalock to access and update SLI data structures.
- *
- * This function returns IRQ_HANDLED when interrupt is handled else it
- * returns IRQ_NONE.
- **/
-irqreturn_t
-lpfc_sli4_sp_intr_handler(int irq, void *dev_id)
-{
-	struct lpfc_hba *phba;
-	struct lpfc_queue *speq;
-	struct lpfc_eqe *eqe;
-	unsigned long iflag;
-	int ecount = 0;
-
-	/*
-	 * Get the driver's phba structure from the dev_id
-	 */
-	phba = (struct lpfc_hba *)dev_id;
-
-	if (unlikely(!phba))
-		return IRQ_NONE;
-
-	/* Get to the EQ struct associated with this vector */
-	speq = phba->sli4_hba.sp_eq;
-	if (unlikely(!speq))
-		return IRQ_NONE;
-
-	/* Check device state for handling interrupt */
-	if (unlikely(lpfc_intr_state_check(phba))) {
-		/* Check again for link_state with lock held */
-		spin_lock_irqsave(&phba->hbalock, iflag);
-		if (phba->link_state < LPFC_LINK_DOWN)
-			/* Flush, clear interrupt, and rearm the EQ */
-			lpfc_sli4_eq_flush(phba, speq);
-		spin_unlock_irqrestore(&phba->hbalock, iflag);
-		return IRQ_NONE;
-	}
-
-	/*
-	 * Process all the event on FCP slow-path EQ
-	 */
-	while ((eqe = lpfc_sli4_eq_get(speq))) {
-		lpfc_sli4_sp_handle_eqe(phba, eqe);
-		if (!(++ecount % speq->entry_repost))
-			lpfc_sli4_eq_release(speq, LPFC_QUEUE_NOARM);
-	}
-
-	/* Always clear and re-arm the slow-path EQ */
-	lpfc_sli4_eq_release(speq, LPFC_QUEUE_REARM);
-
-	/* Catch the no cq entry condition */
-	if (unlikely(ecount == 0)) {
-		if (phba->intr_type == MSIX)
-			/* MSI-X treated interrupt served as no EQ share INT */
-			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
-					"0357 MSI-X interrupt with no EQE\n");
-		else
-			/* Non MSI-X treated on interrupt as EQ share INT */
-			return IRQ_NONE;
-	}
-
-	return IRQ_HANDLED;
-} /* lpfc_sli4_sp_intr_handler */
-
-/**
- * lpfc_sli4_fp_intr_handler - Fast-path interrupt handler to SLI-4 device
+ * lpfc_sli4_hba_intr_handler - HBA interrupt handler to SLI-4 device
  * @irq: Interrupt number.
  * @dev_id: The device context pointer.
  *
@@ -11833,11 +11835,16 @@ lpfc_sli4_sp_intr_handler(int irq, void *dev_id)
  * the FCP EQ to FCP CQ are one-to-one map such that the FCP EQ index is
  * equal to that of FCP CQ index.
  *
+ * The link attention and ELS ring attention events are handled
+ * by the worker thread. The interrupt handler signals the worker thread
+ * and returns for these events. This function is called without any lock
+ * held. It gets the hbalock to access and update SLI data structures.
+ *
  * This function returns IRQ_HANDLED when interrupt is handled else it
  * returns IRQ_NONE.
  **/
 irqreturn_t
-lpfc_sli4_fp_intr_handler(int irq, void *dev_id)
+lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 {
 	struct lpfc_hba *phba;
 	struct lpfc_fcp_eq_hdl *fcp_eq_hdl;
@@ -11854,22 +11861,34 @@ lpfc_sli4_fp_intr_handler(int irq, void *dev_id)
 
 	if (unlikely(!phba))
 		return IRQ_NONE;
-	if (unlikely(!phba->sli4_hba.fp_eq))
+	if (unlikely(!phba->sli4_hba.hba_eq))
 		return IRQ_NONE;
 
 	/* Get to the EQ struct associated with this vector */
-	fpeq = phba->sli4_hba.fp_eq[fcp_eqidx];
+	fpeq = phba->sli4_hba.hba_eq[fcp_eqidx];
 	if (unlikely(!fpeq))
 		return IRQ_NONE;
 
+	if (lpfc_fcp_look_ahead) {
+		if (atomic_dec_and_test(&fcp_eq_hdl->fcp_eq_in_use))
+			lpfc_sli4_eq_clr_intr(fpeq);
+		else {
+			atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+			return IRQ_NONE;
+		}
+	}
+
 	/* Check device state for handling interrupt */
 	if (unlikely(lpfc_intr_state_check(phba))) {
+		fpeq->EQ_badstate++;
 		/* Check again for link_state with lock held */
 		spin_lock_irqsave(&phba->hbalock, iflag);
 		if (phba->link_state < LPFC_LINK_DOWN)
 			/* Flush, clear interrupt, and rearm the EQ */
 			lpfc_sli4_eq_flush(phba, fpeq);
 		spin_unlock_irqrestore(&phba->hbalock, iflag);
+		if (lpfc_fcp_look_ahead)
+			atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
 		return IRQ_NONE;
 	}
 
@@ -11877,15 +11896,27 @@ lpfc_sli4_fp_intr_handler(int irq, void *dev_id)
 	 * Process all the event on FCP fast-path EQ
 	 */
 	while ((eqe = lpfc_sli4_eq_get(fpeq))) {
-		lpfc_sli4_fp_handle_eqe(phba, eqe, fcp_eqidx);
+		lpfc_sli4_hba_handle_eqe(phba, eqe, fcp_eqidx);
 		if (!(++ecount % fpeq->entry_repost))
 			lpfc_sli4_eq_release(fpeq, LPFC_QUEUE_NOARM);
+		fpeq->EQ_processed++;
 	}
 
+	/* Track the max number of EQEs processed in 1 intr */
+	if (ecount > fpeq->EQ_max_eqe)
+		fpeq->EQ_max_eqe = ecount;
+
 	/* Always clear and re-arm the fast-path EQ */
 	lpfc_sli4_eq_release(fpeq, LPFC_QUEUE_REARM);
 
 	if (unlikely(ecount == 0)) {
+		fpeq->EQ_no_entry++;
+
+		if (lpfc_fcp_look_ahead) {
+			atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
+			return IRQ_NONE;
+		}
+
 		if (phba->intr_type == MSIX)
 			/* MSI-X treated interrupt served as no EQ share INT */
 			lpfc_printf_log(phba, KERN_WARNING, LOG_SLI,
@@ -11895,6 +11926,8 @@ lpfc_sli4_fp_intr_handler(int irq, void *dev_id)
 			return IRQ_NONE;
 	}
 
+	if (lpfc_fcp_look_ahead)
+		atomic_inc(&fcp_eq_hdl->fcp_eq_in_use);
 	return IRQ_HANDLED;
 } /* lpfc_sli4_fp_intr_handler */
 
@@ -11919,8 +11952,8 @@ irqreturn_t
 lpfc_sli4_intr_handler(int irq, void *dev_id)
 {
 	struct lpfc_hba  *phba;
-	irqreturn_t sp_irq_rc, fp_irq_rc;
-	bool fp_handled = false;
+	irqreturn_t hba_irq_rc;
+	bool hba_handled = false;
 	uint32_t fcp_eqidx;
 
 	/* Get the driver's phba structure from the dev_id */
@@ -11930,21 +11963,16 @@ lpfc_sli4_intr_handler(int irq, void *dev_id)
 		return IRQ_NONE;
 
 	/*
-	 * Invokes slow-path host attention interrupt handling as appropriate.
-	 */
-	sp_irq_rc = lpfc_sli4_sp_intr_handler(irq, dev_id);
-
-	/*
 	 * Invoke fast-path host attention interrupt handling as appropriate.
 	 */
-	for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_eq_count; fcp_eqidx++) {
-		fp_irq_rc = lpfc_sli4_fp_intr_handler(irq,
+	for (fcp_eqidx = 0; fcp_eqidx < phba->cfg_fcp_io_channel; fcp_eqidx++) {
+		hba_irq_rc = lpfc_sli4_hba_intr_handler(irq,
 					&phba->sli4_hba.fcp_eq_hdl[fcp_eqidx]);
-		if (fp_irq_rc == IRQ_HANDLED)
-			fp_handled |= true;
+		if (hba_irq_rc == IRQ_HANDLED)
+			hba_handled |= true;
 	}
 
-	return (fp_handled == true) ? IRQ_HANDLED : sp_irq_rc;
+	return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE;
 } /* lpfc_sli4_intr_handler */
 
 /**
@@ -12075,7 +12103,7 @@ lpfc_modify_fcp_eq_delay(struct lpfc_hba *phba, uint16_t startq)
 	union lpfc_sli4_cfg_shdr *shdr;
 	uint16_t dmult;
 
-	if (startq >= phba->cfg_fcp_eq_count)
+	if (startq >= phba->cfg_fcp_io_channel)
 		return 0;
 
 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
@@ -12089,12 +12117,13 @@ lpfc_modify_fcp_eq_delay(struct lpfc_hba *phba, uint16_t startq)
 	eq_delay = &mbox->u.mqe.un.eq_delay;
 
 	/* Calculate delay multiper from maximum interrupt per second */
-	dmult = LPFC_DMULT_CONST/phba->cfg_fcp_imax - 1;
+	dmult = phba->cfg_fcp_imax / phba->cfg_fcp_io_channel;
+	dmult = LPFC_DMULT_CONST/dmult - 1;
 
 	cnt = 0;
-	for (fcp_eqidx = startq; fcp_eqidx < phba->cfg_fcp_eq_count;
+	for (fcp_eqidx = startq; fcp_eqidx < phba->cfg_fcp_io_channel;
 	    fcp_eqidx++) {
-		eq = phba->sli4_hba.fp_eq[fcp_eqidx];
+		eq = phba->sli4_hba.hba_eq[fcp_eqidx];
 		if (!eq)
 			continue;
 		eq_delay->u.request.eq[cnt].eq_id = eq->queue_id;
diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h
index 2626f58c074..2f48d000a3b 100644
--- a/drivers/scsi/lpfc/lpfc_sli.h
+++ b/drivers/scsi/lpfc/lpfc_sli.h
@@ -131,7 +131,9 @@ typedef struct lpfcMboxq {
 
 #define LPFC_MAX_RING_MASK  5	/* max num of rctl/type masks allowed per
 				   ring */
-#define LPFC_MAX_RING       4	/* max num of SLI rings used by driver */
+#define LPFC_SLI3_MAX_RING  4	/* Max num of SLI3 rings used by driver.
+				   For SLI4, an additional ring for each
+				   FCP WQ will be allocated.  */
 
 struct lpfc_sli_ring;
 
@@ -158,6 +160,24 @@ struct lpfc_sli_ring_stat {
 	uint64_t iocb_rsp_full;	 /* IOCB rsp ring full */
 };
 
+struct lpfc_sli3_ring {
+	uint32_t local_getidx;  /* last available cmd index (from cmdGetInx) */
+	uint32_t next_cmdidx;   /* next_cmd index */
+	uint32_t rspidx;	/* current index in response ring */
+	uint32_t cmdidx;	/* current index in command ring */
+	uint16_t numCiocb;	/* number of command iocb's per ring */
+	uint16_t numRiocb;	/* number of rsp iocb's per ring */
+	uint16_t sizeCiocb;	/* Size of command iocb's in this ring */
+	uint16_t sizeRiocb;	/* Size of response iocb's in this ring */
+	uint32_t *cmdringaddr;	/* virtual address for cmd rings */
+	uint32_t *rspringaddr;	/* virtual address for rsp rings */
+};
+
+struct lpfc_sli4_ring {
+	struct lpfc_queue *wqp;	/* Pointer to associated WQ */
+};
+
+
 /* Structure used to hold SLI ring information */
 struct lpfc_sli_ring {
 	uint16_t flag;		/* ring flags */
@@ -166,16 +186,10 @@ struct lpfc_sli_ring {
 #define LPFC_STOP_IOCB_EVENT     0x020	/* Stop processing IOCB cmds event */
 	uint16_t abtsiotag;	/* tracks next iotag to use for ABTS */
 
-	uint32_t local_getidx;   /* last available cmd index (from cmdGetInx) */
-	uint32_t next_cmdidx;    /* next_cmd index */
-	uint32_t rspidx;	/* current index in response ring */
-	uint32_t cmdidx;	/* current index in command ring */
 	uint8_t rsvd;
 	uint8_t ringno;		/* ring number */
-	uint16_t numCiocb;	/* number of command iocb's per ring */
-	uint16_t numRiocb;	/* number of rsp iocb's per ring */
-	uint16_t sizeCiocb;	/* Size of command iocb's in this ring */
-	uint16_t sizeRiocb;	/* Size of response iocb's in this ring */
+
+	spinlock_t ring_lock;	/* lock for issuing commands */
 
 	uint32_t fast_iotag;	/* max fastlookup based iotag           */
 	uint32_t iotag_ctr;	/* keeps track of the next iotag to use */
@@ -186,8 +200,6 @@ struct lpfc_sli_ring {
 	struct list_head txcmplq;
 	uint16_t txcmplq_cnt;	/* current length of queue */
 	uint16_t txcmplq_max;	/* max length */
-	uint32_t *cmdringaddr;	/* virtual address for cmd rings */
-	uint32_t *rspringaddr;	/* virtual address for rsp rings */
 	uint32_t missbufcnt;	/* keep track of buffers to post */
 	struct list_head postbufq;
 	uint16_t postbufq_cnt;	/* current length of queue */
@@ -207,6 +219,10 @@ struct lpfc_sli_ring {
 	/* cmd ring available */
 	void (*lpfc_sli_cmd_available) (struct lpfc_hba *,
 					struct lpfc_sli_ring *);
+	union {
+		struct lpfc_sli3_ring sli3;
+		struct lpfc_sli4_ring sli4;
+	} sli;
 };
 
 /* Structure used for configuring rings to a specific profile or rctl / type */
@@ -239,6 +255,8 @@ struct lpfc_sli_stat {
 	uint64_t mbox_stat_err;  /* Mbox cmds completed status error */
 	uint64_t mbox_cmd;       /* Mailbox commands issued */
 	uint64_t sli_intr;       /* Count of Host Attention interrupts */
+	uint64_t sli_prev_intr;  /* Previous cnt of Host Attention interrupts */
+	uint64_t sli_ips;        /* Host Attention interrupts per sec */
 	uint32_t err_attn_event; /* Error Attn event counters */
 	uint32_t link_event;     /* Link event counters */
 	uint32_t mbox_event;     /* Mailbox event counters */
@@ -270,7 +288,7 @@ struct lpfc_sli {
 #define LPFC_MENLO_MAINT          0x1000 /* need for menl fw download */
 #define LPFC_SLI_ASYNC_MBX_BLK    0x2000 /* Async mailbox is blocked */
 
-	struct lpfc_sli_ring ring[LPFC_MAX_RING];
+	struct lpfc_sli_ring *ring;
 	int fcp_ring;		/* ring used for FCP initiator commands */
 	int next_ring;
 
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index ec756118c5c..bd4bc4342ae 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -34,18 +34,10 @@
 /* Number of SGL entries can be posted in a 4KB nonembedded mbox command */
 #define LPFC_NEMBED_MBOX_SGL_CNT		254
 
-/* Multi-queue arrangement for fast-path FCP work queues */
-#define LPFC_FN_EQN_MAX       8
-#define LPFC_SP_EQN_DEF       1
-#define LPFC_FP_EQN_DEF       4
-#define LPFC_FP_EQN_MIN       1
-#define LPFC_FP_EQN_MAX       (LPFC_FN_EQN_MAX - LPFC_SP_EQN_DEF)
-
-#define LPFC_FN_WQN_MAX       32
-#define LPFC_SP_WQN_DEF       1
-#define LPFC_FP_WQN_DEF       4
-#define LPFC_FP_WQN_MIN       1
-#define LPFC_FP_WQN_MAX       (LPFC_FN_WQN_MAX - LPFC_SP_WQN_DEF)
+/* Multi-queue arrangement for FCP EQ/CQ/WQ tuples */
+#define LPFC_FCP_IO_CHAN_DEF       4
+#define LPFC_FCP_IO_CHAN_MIN       1
+#define LPFC_FCP_IO_CHAN_MAX       8
 
 /*
  * Provide the default FCF Record attributes used by the driver
@@ -141,6 +133,37 @@ struct lpfc_queue {
 	uint32_t page_count;	/* Number of pages allocated for this queue */
 	uint32_t host_index;	/* The host's index for putting or getting */
 	uint32_t hba_index;	/* The last known hba index for get or put */
+
+	struct lpfc_sli_ring *pring; /* ptr to io ring associated with q */
+
+	/* For q stats */
+	uint32_t q_cnt_1;
+	uint32_t q_cnt_2;
+	uint32_t q_cnt_3;
+	uint64_t q_cnt_4;
+/* defines for EQ stats */
+#define	EQ_max_eqe		q_cnt_1
+#define	EQ_no_entry		q_cnt_2
+#define	EQ_badstate		q_cnt_3
+#define	EQ_processed		q_cnt_4
+
+/* defines for CQ stats */
+#define	CQ_mbox			q_cnt_1
+#define	CQ_max_cqe		q_cnt_1
+#define	CQ_release_wqe		q_cnt_2
+#define	CQ_xri_aborted		q_cnt_3
+#define	CQ_wq			q_cnt_4
+
+/* defines for WQ stats */
+#define	WQ_overflow		q_cnt_1
+#define	WQ_posted		q_cnt_4
+
+/* defines for RQ stats */
+#define	RQ_no_posted_buf	q_cnt_1
+#define	RQ_no_buf_found		q_cnt_2
+#define	RQ_buf_trunc		q_cnt_3
+#define	RQ_rcv_buf		q_cnt_4
+
 	union sli4_qe qe[1];	/* array to index entries (must be last) */
 };
 
@@ -350,6 +373,7 @@ struct lpfc_hba;
 struct lpfc_fcp_eq_hdl {
 	uint32_t idx;
 	struct lpfc_hba *phba;
+	atomic_t fcp_eq_in_use;
 };
 
 /* Port Capabilities for SLI4 Parameters */
@@ -407,6 +431,8 @@ struct lpfc_sli4_lnk_info {
 	uint8_t lnk_no;
 };
 
+#define LPFC_SLI4_HANDLER_NAME_SZ	16
+
 /* SLI4 HBA data structure entries */
 struct lpfc_sli4_hba {
 	void __iomem *conf_regs_memmap_p; /* Kernel memory mapped address for
@@ -463,20 +489,23 @@ struct lpfc_sli4_hba {
 	struct lpfc_register sli_intf;
 	struct lpfc_pc_sli4_params pc_sli4_params;
 	struct msix_entry *msix_entries;
+	uint8_t handler_name[LPFC_FCP_IO_CHAN_MAX][LPFC_SLI4_HANDLER_NAME_SZ];
 	uint32_t cfg_eqn;
 	uint32_t msix_vec_nr;
 	struct lpfc_fcp_eq_hdl *fcp_eq_hdl; /* FCP per-WQ handle */
+
 	/* Pointers to the constructed SLI4 queues */
-	struct lpfc_queue **fp_eq; /* Fast-path event queue */
-	struct lpfc_queue *sp_eq;  /* Slow-path event queue */
+	struct lpfc_queue **hba_eq;/* Event queues for HBA */
+	struct lpfc_queue **fcp_cq;/* Fast-path FCP compl queue */
 	struct lpfc_queue **fcp_wq;/* Fast-path FCP work queue */
+	uint16_t *fcp_cq_map;
+
+	struct lpfc_queue *mbx_cq; /* Slow-path mailbox complete queue */
+	struct lpfc_queue *els_cq; /* Slow-path ELS response complete queue */
 	struct lpfc_queue *mbx_wq; /* Slow-path MBOX work queue */
 	struct lpfc_queue *els_wq; /* Slow-path ELS work queue */
 	struct lpfc_queue *hdr_rq; /* Slow-path Header Receive queue */
 	struct lpfc_queue *dat_rq; /* Slow-path Data Receive queue */
-	struct lpfc_queue **fcp_cq;/* Fast-path FCP compl queue */
-	struct lpfc_queue *mbx_cq; /* Slow-path mailbox complete queue */
-	struct lpfc_queue *els_cq; /* Slow-path ELS response complete queue */
 
 	/* Setup information for various queue parameters */
 	int eq_esize;
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index 4704e5b5088..04265a1c4e5 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -18,11 +18,16 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "8.3.32"
+#define LPFC_DRIVER_VERSION "8.3.34"
 #define LPFC_DRIVER_NAME		"lpfc"
+
+/* Used for SLI 2/3 */
 #define LPFC_SP_DRIVER_HANDLER_NAME	"lpfc:sp"
 #define LPFC_FP_DRIVER_HANDLER_NAME	"lpfc:fp"
 
+/* Used for SLI4 */
+#define LPFC_DRIVER_HANDLER_NAME	"lpfc:"
+
 #define LPFC_MODULE_DESC "Emulex LightPulse Fibre Channel SCSI driver " \
 		LPFC_DRIVER_VERSION
 #define LPFC_COPYRIGHT "Copyright(c) 2004-2009 Emulex.  All rights reserved."
diff --git a/drivers/scsi/megaraid/megaraid_sas.h b/drivers/scsi/megaraid/megaraid_sas.h
index e8f89264768..fcb005fa4bd 100644
--- a/drivers/scsi/megaraid/megaraid_sas.h
+++ b/drivers/scsi/megaraid/megaraid_sas.h
@@ -33,9 +33,9 @@
 /*
  * MegaRAID SAS Driver meta data
  */
-#define MEGASAS_VERSION				"00.00.06.15-rc1"
-#define MEGASAS_RELDATE				"Mar. 19, 2012"
-#define MEGASAS_EXT_VERSION			"Mon. Mar. 19 17:00:00 PDT 2012"
+#define MEGASAS_VERSION				"00.00.06.18-rc1"
+#define MEGASAS_RELDATE				"Jun. 17, 2012"
+#define MEGASAS_EXT_VERSION			"Tue. Jun. 17 17:00:00 PDT 2012"
 
 /*
  * Device IDs
@@ -747,6 +747,7 @@ struct megasas_ctrl_info {
 #define	MEGASAS_RESET_NOTICE_INTERVAL		5
 #define MEGASAS_IOCTL_CMD			0
 #define MEGASAS_DEFAULT_CMD_TIMEOUT		90
+#define MEGASAS_THROTTLE_QUEUE_DEPTH		16
 
 /*
  * FW reports the maximum of number of commands that it can accept (maximum
@@ -1364,6 +1365,7 @@ struct megasas_instance {
 	unsigned long bar;
 	long reset_flags;
 	struct mutex reset_mutex;
+	int throttlequeuedepth;
 };
 
 enum {
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index ed38454228c..0393ec478cd 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -18,7 +18,7 @@
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  *
  *  FILE: megaraid_sas_base.c
- *  Version : v00.00.06.15-rc1
+ *  Version : v00.00.06.18-rc1
  *
  *  Authors: LSI Corporation
  *           Sreenivas Bagalkote
@@ -71,6 +71,16 @@ static int msix_disable;
 module_param(msix_disable, int, S_IRUGO);
 MODULE_PARM_DESC(msix_disable, "Disable MSI-X interrupt handling. Default: 0");
 
+static int throttlequeuedepth = MEGASAS_THROTTLE_QUEUE_DEPTH;
+module_param(throttlequeuedepth, int, S_IRUGO);
+MODULE_PARM_DESC(throttlequeuedepth,
+	"Adapter queue depth when throttled due to I/O timeout. Default: 16");
+
+int resetwaittime = MEGASAS_RESET_WAIT_TIME;
+module_param(resetwaittime, int, S_IRUGO);
+MODULE_PARM_DESC(resetwaittime, "Wait time in seconds after I/O timeout "
+		 "before resetting adapter. Default: 180");
+
 MODULE_LICENSE("GPL");
 MODULE_VERSION(MEGASAS_VERSION);
 MODULE_AUTHOR("megaraidlinux@lsi.com");
@@ -1595,8 +1605,9 @@ megasas_check_and_restore_queue_depth(struct megasas_instance *instance)
 {
 	unsigned long flags;
 	if (instance->flag & MEGASAS_FW_BUSY
-		&& time_after(jiffies, instance->last_time + 5 * HZ)
-		&& atomic_read(&instance->fw_outstanding) < 17) {
+	    && time_after(jiffies, instance->last_time + 5 * HZ)
+	    && atomic_read(&instance->fw_outstanding) <
+	    instance->throttlequeuedepth + 1) {
 
 		spin_lock_irqsave(instance->host->host_lock, flags);
 		instance->flag &= ~MEGASAS_FW_BUSY;
@@ -1772,7 +1783,7 @@ static int megasas_wait_for_outstanding(struct megasas_instance *instance)
 		return SUCCESS;
 	}
 
-	for (i = 0; i < wait_time; i++) {
+	for (i = 0; i < resetwaittime; i++) {
 
 		int outstanding = atomic_read(&instance->fw_outstanding);
 
@@ -1914,7 +1925,7 @@ blk_eh_timer_return megasas_reset_timer(struct scsi_cmnd *scmd)
 		/* FW is busy, throttle IO */
 		spin_lock_irqsave(instance->host->host_lock, flags);
 
-		instance->host->can_queue = 16;
+		instance->host->can_queue = instance->throttlequeuedepth;
 		instance->last_time = jiffies;
 		instance->flag |= MEGASAS_FW_BUSY;
 
@@ -3577,6 +3588,24 @@ static int megasas_init_fw(struct megasas_instance *instance)
 
 	kfree(ctrl_info);
 
+	/* Check for valid throttlequeuedepth module parameter */
+	if (instance->pdev->device == PCI_DEVICE_ID_LSI_SAS0073SKINNY ||
+	    instance->pdev->device == PCI_DEVICE_ID_LSI_SAS0071SKINNY) {
+		if (throttlequeuedepth > (instance->max_fw_cmds -
+					  MEGASAS_SKINNY_INT_CMDS))
+			instance->throttlequeuedepth =
+				MEGASAS_THROTTLE_QUEUE_DEPTH;
+		else
+			instance->throttlequeuedepth = throttlequeuedepth;
+	} else {
+		if (throttlequeuedepth > (instance->max_fw_cmds -
+					  MEGASAS_INT_CMDS))
+			instance->throttlequeuedepth =
+				MEGASAS_THROTTLE_QUEUE_DEPTH;
+		else
+			instance->throttlequeuedepth = throttlequeuedepth;
+	}
+
         /*
 	* Setup tasklet for cmd completion
 	*/
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index a610cf1d484..ddf094e7d0a 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -94,6 +94,7 @@ int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
 void megaraid_sas_kill_hba(struct megasas_instance *instance);
 
 extern u32 megasas_dbg_lvl;
+extern int resetwaittime;
 
 /**
  * megasas_enable_intr_fusion -	Enables interrupts
@@ -461,8 +462,8 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance)
 	 * Allocate the dynamic array first and then allocate individual
 	 * commands.
 	 */
-	fusion->cmd_list = kmalloc(sizeof(struct megasas_cmd_fusion *)
-				   *max_cmd, GFP_KERNEL);
+	fusion->cmd_list = kzalloc(sizeof(struct megasas_cmd_fusion *)
+				   * max_cmd, GFP_KERNEL);
 
 	if (!fusion->cmd_list) {
 		printk(KERN_DEBUG "megasas: out of memory. Could not alloc "
@@ -470,9 +471,6 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance)
 		goto fail_cmd_list;
 	}
 
-	memset(fusion->cmd_list, 0, sizeof(struct megasas_cmd_fusion *)
-	       *max_cmd);
-
 	max_cmd = instance->max_fw_cmds;
 	for (i = 0; i < max_cmd; i++) {
 		fusion->cmd_list[i] = kmalloc(sizeof(struct megasas_cmd_fusion),
@@ -2063,9 +2061,9 @@ megasas_check_reset_fusion(struct megasas_instance *instance,
 int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance)
 {
 	int i, outstanding, retval = 0;
-	u32 fw_state, wait_time = MEGASAS_RESET_WAIT_TIME;
+	u32 fw_state;
 
-	for (i = 0; i < wait_time; i++) {
+	for (i = 0; i < resetwaittime; i++) {
 		/* Check if firmware is in fault state */
 		fw_state = instance->instancet->read_fw_status_reg(
 			instance->reg_set) & MFI_STATE_MASK;
diff --git a/drivers/scsi/mpt2sas/Kconfig b/drivers/scsi/mpt2sas/Kconfig
index bbb7e4bf30a..39f08dd2055 100644
--- a/drivers/scsi/mpt2sas/Kconfig
+++ b/drivers/scsi/mpt2sas/Kconfig
@@ -2,7 +2,7 @@
 # Kernel configuration file for the MPT2SAS
 #
 # This code is based on drivers/scsi/mpt2sas/Kconfig
-# Copyright (C) 2007-2010  LSI Corporation
+# Copyright (C) 2007-2012  LSI Corporation
 #  (mailto:DL-MPTFusionLinux@lsi.com)
 
 # This program is free software; you can redistribute it and/or
diff --git a/drivers/scsi/mpt2sas/mpi/mpi2.h b/drivers/scsi/mpt2sas/mpi/mpi2.h
index a80f3220c64..e960f9625c7 100644
--- a/drivers/scsi/mpt2sas/mpi/mpi2.h
+++ b/drivers/scsi/mpt2sas/mpi/mpi2.h
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2000-2011 LSI Corporation.
+ *  Copyright (c) 2000-2012 LSI Corporation.
  *
  *
  *           Name:  mpi2.h
@@ -8,7 +8,7 @@
  *                  scatter/gather formats.
  *  Creation Date:  June 21, 2006
  *
- *  mpi2.h Version:  02.00.23
+ *  mpi2.h Version:  02.00.25
  *
  *  Version History
  *  ---------------
@@ -72,6 +72,9 @@
  *  05-25-11  02.00.21  Bumped MPI2_HEADER_VERSION_UNIT.
  *  08-24-11  02.00.22  Bumped MPI2_HEADER_VERSION_UNIT.
  *  11-18-11  02.00.23  Bumped MPI2_HEADER_VERSION_UNIT.
+ *  02-06-12  02.00.24  Bumped MPI2_HEADER_VERSION_UNIT.
+ *  03-29-12  02.00.25  Bumped MPI2_HEADER_VERSION_UNIT.
+ *                      Added Hard Reset delay timings.
  *  --------------------------------------------------------------------------
  */
 
@@ -97,7 +100,7 @@
 #define MPI2_VERSION_02_00                  (0x0200)
 
 /* versioning for this MPI header set */
-#define MPI2_HEADER_VERSION_UNIT            (0x17)
+#define MPI2_HEADER_VERSION_UNIT            (0x19)
 #define MPI2_HEADER_VERSION_DEV             (0x00)
 #define MPI2_HEADER_VERSION_UNIT_MASK       (0xFF00)
 #define MPI2_HEADER_VERSION_UNIT_SHIFT      (8)
@@ -275,6 +278,11 @@ typedef volatile struct _MPI2_SYSTEM_INTERFACE_REGS
 #define MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET    (0x000000C4)
 
 
+/* Hard Reset delay timings */
+#define MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC     (50000)
+#define MPI2_HARD_RESET_PCIE_RESET_READ_WINDOW_MICRO_SEC    (255000)
+#define MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC    (256000)
+
 /*****************************************************************************
 *
 *        Message Descriptors
diff --git a/drivers/scsi/mpt2sas/mpi/mpi2_init.h b/drivers/scsi/mpt2sas/mpi/mpi2_init.h
index de90162413c..38c5da39814 100644
--- a/drivers/scsi/mpt2sas/mpi/mpi2_init.h
+++ b/drivers/scsi/mpt2sas/mpi/mpi2_init.h
@@ -1,12 +1,12 @@
 /*
- *  Copyright (c) 2000-2010 LSI Corporation.
+ *  Copyright (c) 2000-2012 LSI Corporation.
  *
  *
  *           Name:  mpi2_init.h
  *          Title:  MPI SCSI initiator mode messages and structures
  *  Creation Date:  June 23, 2006
  *
- *    mpi2_init.h Version:  02.00.11
+ *    mpi2_init.h Version:  02.00.13
  *
  *  Version History
  *  ---------------
@@ -34,6 +34,8 @@
  *  02-10-10  02.00.09  Removed unused structure that had "#if 0" around it.
  *  05-12-10  02.00.10  Added optional vendor-unique region to SCSI IO Request.
  *  11-10-10  02.00.11  Added MPI2_SCSIIO_NUM_SGLOFFSETS define.
+ *  02-06-12  02.00.13  Added alternate defines for Task Priority / Command
+ *                      Priority to match SAM-4.
  *  --------------------------------------------------------------------------
  */
 
@@ -194,6 +196,9 @@ typedef struct _MPI2_SCSI_IO_REQUEST
 
 #define MPI2_SCSIIO_CONTROL_TASKPRI_MASK        (0x00007800)
 #define MPI2_SCSIIO_CONTROL_TASKPRI_SHIFT       (11)
+/* alternate name for the previous field; called Command Priority in SAM-4 */
+#define MPI2_SCSIIO_CONTROL_CMDPRI_MASK         (0x00007800)
+#define MPI2_SCSIIO_CONTROL_CMDPRI_SHIFT        (11)
 
 #define MPI2_SCSIIO_CONTROL_TASKATTRIBUTE_MASK  (0x00000700)
 #define MPI2_SCSIIO_CONTROL_SIMPLEQ             (0x00000000)
diff --git a/drivers/scsi/mpt2sas/mpi/mpi2_ioc.h b/drivers/scsi/mpt2sas/mpi/mpi2_ioc.h
index 9a925c07a9e..b0d4760bb17 100644
--- a/drivers/scsi/mpt2sas/mpi/mpi2_ioc.h
+++ b/drivers/scsi/mpt2sas/mpi/mpi2_ioc.h
@@ -1,12 +1,12 @@
 /*
- *  Copyright (c) 2000-2011 LSI Corporation.
+ *  Copyright (c) 2000-2012 LSI Corporation.
  *
  *
  *           Name:  mpi2_ioc.h
  *          Title:  MPI IOC, Port, Event, FW Download, and FW Upload messages
  *  Creation Date:  October 11, 2006
  *
- *  mpi2_ioc.h Version:  02.00.19
+ *  mpi2_ioc.h Version:  02.00.21
  *
  *  Version History
  *  ---------------
@@ -117,6 +117,7 @@
  *  08-24-11  02.00.19  Added PhysicalPort field to
  *                      MPI2_EVENT_DATA_SAS_DEVICE_STATUS_CHANGE structure.
  *                      Marked MPI2_PM_CONTROL_FEATURE_PCIE_LINK as obsolete.
+ *  03-29-12  02.00.21  Added a product specific range to event values.
  *  --------------------------------------------------------------------------
  */
 
@@ -492,7 +493,8 @@ typedef struct _MPI2_EVENT_NOTIFICATION_REPLY
 #define MPI2_EVENT_SAS_NOTIFY_PRIMITIVE             (0x0026)
 #define MPI2_EVENT_TEMP_THRESHOLD                   (0x0027)
 #define MPI2_EVENT_HOST_MESSAGE                     (0x0028)
-
+#define MPI2_EVENT_MIN_PRODUCT_SPECIFIC             (0x006E)
+#define MPI2_EVENT_MAX_PRODUCT_SPECIFIC             (0x007F)
 
 /* Log Entry Added Event data */
 
diff --git a/drivers/scsi/mpt2sas/mpi/mpi2_raid.h b/drivers/scsi/mpt2sas/mpi/mpi2_raid.h
index 0601612b875..2b38af213be 100644
--- a/drivers/scsi/mpt2sas/mpi/mpi2_raid.h
+++ b/drivers/scsi/mpt2sas/mpi/mpi2_raid.h
@@ -1,12 +1,12 @@
 /*
- *  Copyright (c) 2000-2010 LSI Corporation.
+ *  Copyright (c) 2000-2012 LSI Corporation.
  *
  *
  *           Name:  mpi2_raid.h
  *          Title:  MPI Integrated RAID messages and structures
  *  Creation Date:  April 26, 2007
  *
- *    mpi2_raid.h Version:  02.00.06
+ *    mpi2_raid.h Version:  02.00.08
  *
  *  Version History
  *  ---------------
@@ -26,7 +26,7 @@
  *  08-24-10  02.00.06  Added MPI2_RAID_ACTION_COMPATIBILITY_CHECK along with
  *                      related structures and defines.
  *                      Added product-specific range to RAID Action values.
-
+ *  02-06-12  02.00.08  Added MPI2_RAID_ACTION_PHYSDISK_HIDDEN.
  *  --------------------------------------------------------------------------
  */
 
@@ -181,6 +181,7 @@ typedef struct _MPI2_RAID_ACTION_REQUEST
 #define MPI2_RAID_ACTION_START_RAID_FUNCTION        (0x21)
 #define MPI2_RAID_ACTION_STOP_RAID_FUNCTION         (0x22)
 #define MPI2_RAID_ACTION_COMPATIBILITY_CHECK        (0x23)
+#define MPI2_RAID_ACTION_PHYSDISK_HIDDEN            (0x24)
 #define MPI2_RAID_ACTION_MIN_PRODUCT_SPECIFIC       (0x80)
 #define MPI2_RAID_ACTION_MAX_PRODUCT_SPECIFIC       (0xFF)
 
diff --git a/drivers/scsi/mpt2sas/mpt2sas_base.c b/drivers/scsi/mpt2sas/mpt2sas_base.c
index 9d5a56c4b33..ffd85c511c8 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_base.c
+++ b/drivers/scsi/mpt2sas/mpt2sas_base.c
@@ -3,7 +3,7 @@
  * for access to MPT (Message Passing Technology) firmware.
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_base.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
@@ -1978,9 +1978,9 @@ _base_display_intel_branding(struct MPT2SAS_ADAPTER *ioc)
 			printk(MPT2SAS_INFO_FMT "%s\n", ioc->name,
 			    MPT2SAS_INTEL_RMS2LL040_BRANDING);
 			break;
-		case MPT2SAS_INTEL_RAMSDALE_SSDID:
+		case MPT2SAS_INTEL_SSD910_SSDID:
 			printk(MPT2SAS_INFO_FMT "%s\n", ioc->name,
-			    MPT2SAS_INTEL_RAMSDALE_BRANDING);
+			    MPT2SAS_INTEL_SSD910_BRANDING);
 			break;
 		default:
 			break;
diff --git a/drivers/scsi/mpt2sas/mpt2sas_base.h b/drivers/scsi/mpt2sas/mpt2sas_base.h
index b3a1a30055d..543d8d63747 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_base.h
+++ b/drivers/scsi/mpt2sas/mpt2sas_base.h
@@ -3,7 +3,7 @@
  * for access to MPT (Message Passing Technology) firmware.
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_base.h
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
@@ -69,8 +69,8 @@
 #define MPT2SAS_DRIVER_NAME		"mpt2sas"
 #define MPT2SAS_AUTHOR	"LSI Corporation <DL-MPTFusionLinux@lsi.com>"
 #define MPT2SAS_DESCRIPTION	"LSI MPT Fusion SAS 2.0 Device Driver"
-#define MPT2SAS_DRIVER_VERSION		"13.100.00.00"
-#define MPT2SAS_MAJOR_VERSION		13
+#define MPT2SAS_DRIVER_VERSION		"14.100.00.00"
+#define MPT2SAS_MAJOR_VERSION		14
 #define MPT2SAS_MINOR_VERSION		100
 #define MPT2SAS_BUILD_VERSION		00
 #define MPT2SAS_RELEASE_VERSION		00
@@ -171,8 +171,8 @@
 				"Intel Integrated RAID Module RMS2LL040"
 #define MPT2SAS_INTEL_RS25GB008_BRANDING       \
 				"Intel(R) RAID Controller RS25GB008"
-#define MPT2SAS_INTEL_RAMSDALE_BRANDING        \
-				"Intel 720 Series SSD"
+#define MPT2SAS_INTEL_SSD910_BRANDING          \
+				"Intel(R) SSD 910 Series"
 /*
  * Intel HBA SSDIDs
  */
@@ -183,7 +183,7 @@
 #define MPT2SAS_INTEL_RMS2LL080_SSDID          0x350E
 #define MPT2SAS_INTEL_RMS2LL040_SSDID          0x350F
 #define MPT2SAS_INTEL_RS25GB008_SSDID          0x3000
-#define MPT2SAS_INTEL_RAMSDALE_SSDID           0x3700
+#define MPT2SAS_INTEL_SSD910_SSDID             0x3700
 
 /*
  * HP HBA branding
@@ -1096,6 +1096,8 @@ int mpt2sas_config_get_iounit_pg1(struct MPT2SAS_ADAPTER *ioc, Mpi2ConfigReply_t
     *mpi_reply, Mpi2IOUnitPage1_t *config_page);
 int mpt2sas_config_set_iounit_pg1(struct MPT2SAS_ADAPTER *ioc, Mpi2ConfigReply_t
     *mpi_reply, Mpi2IOUnitPage1_t *config_page);
+int mpt2sas_config_get_iounit_pg3(struct MPT2SAS_ADAPTER *ioc,
+	Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage3_t *config_page, u16 sz);
 int mpt2sas_config_get_sas_iounit_pg1(struct MPT2SAS_ADAPTER *ioc, Mpi2ConfigReply_t
     *mpi_reply, Mpi2SasIOUnitPage1_t *config_page, u16 sz);
 int mpt2sas_config_set_sas_iounit_pg1(struct MPT2SAS_ADAPTER *ioc,
diff --git a/drivers/scsi/mpt2sas/mpt2sas_config.c b/drivers/scsi/mpt2sas/mpt2sas_config.c
index 2b4d37613d3..863778071a9 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_config.c
+++ b/drivers/scsi/mpt2sas/mpt2sas_config.c
@@ -2,7 +2,7 @@
  * This module provides common API for accessing firmware configuration pages
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_base.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
@@ -683,6 +683,42 @@ mpt2sas_config_set_iounit_pg1(struct MPT2SAS_ADAPTER *ioc,
 }
 
 /**
+ * mpt2sas_config_get_iounit_pg3 - obtain iounit page 3
+ * @ioc: per adapter object
+ * @mpi_reply: reply mf payload returned from firmware
+ * @config_page: contents of the config page
+ * @sz: size of buffer passed in config_page
+ * Context: sleep.
+ *
+ * Returns 0 for success, non-zero for failure.
+ */
+int
+mpt2sas_config_get_iounit_pg3(struct MPT2SAS_ADAPTER *ioc,
+	Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage3_t *config_page, u16 sz)
+{
+	Mpi2ConfigRequest_t mpi_request;
+	int r;
+
+	memset(&mpi_request, 0, sizeof(Mpi2ConfigRequest_t));
+	mpi_request.Function = MPI2_FUNCTION_CONFIG;
+	mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_HEADER;
+	mpi_request.Header.PageType = MPI2_CONFIG_PAGETYPE_IO_UNIT;
+	mpi_request.Header.PageNumber = 3;
+	mpi_request.Header.PageVersion = MPI2_IOUNITPAGE3_PAGEVERSION;
+	mpt2sas_base_build_zero_len_sge(ioc, &mpi_request.PageBufferSGE);
+	r = _config_request(ioc, &mpi_request, mpi_reply,
+	    MPT2_CONFIG_PAGE_DEFAULT_TIMEOUT, NULL, 0);
+	if (r)
+		goto out;
+
+	mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT;
+	r = _config_request(ioc, &mpi_request, mpi_reply,
+	    MPT2_CONFIG_PAGE_DEFAULT_TIMEOUT, config_page, sz);
+ out:
+	return r;
+}
+
+/**
  * mpt2sas_config_get_ioc_pg8 - obtain ioc page 8
  * @ioc: per adapter object
  * @mpi_reply: reply mf payload returned from firmware
diff --git a/drivers/scsi/mpt2sas/mpt2sas_ctl.c b/drivers/scsi/mpt2sas/mpt2sas_ctl.c
index 49bdd2dc845..08685c4cf23 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_ctl.c
+++ b/drivers/scsi/mpt2sas/mpt2sas_ctl.c
@@ -3,7 +3,7 @@
  * controllers
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_ctl.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
@@ -2181,10 +2181,12 @@ _ctl_ioctl_main(struct file *file, unsigned int cmd, void __user *arg,
 		return -EAGAIN;
 
 	state = (file->f_flags & O_NONBLOCK) ? NON_BLOCKING : BLOCKING;
-	if (state == NON_BLOCKING && !mutex_trylock(&ioc->ctl_cmds.mutex))
-		return -EAGAIN;
-	else if (mutex_lock_interruptible(&ioc->ctl_cmds.mutex))
+	if (state == NON_BLOCKING) {
+		if (!mutex_trylock(&ioc->ctl_cmds.mutex))
+			return -EAGAIN;
+	} else if (mutex_lock_interruptible(&ioc->ctl_cmds.mutex)) {
 		return -ERESTARTSYS;
+	}
 
 	switch (cmd) {
 	case MPT2IOCINFO:
@@ -2690,6 +2692,75 @@ _ctl_ioc_reply_queue_count_show(struct device *cdev,
 static DEVICE_ATTR(reply_queue_count, S_IRUGO,
 	 _ctl_ioc_reply_queue_count_show, NULL);
 
+/**
+ * _ctl_BRM_status_show - Backup Rail Monitor Status
+ * @cdev - pointer to embedded class device
+ * @buf - the buffer returned
+ *
+ * This is number of reply queues
+ *
+ * A sysfs 'read-only' shost attribute.
+ */
+static ssize_t
+_ctl_BRM_status_show(struct device *cdev, struct device_attribute *attr,
+	char *buf)
+{
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct MPT2SAS_ADAPTER *ioc = shost_priv(shost);
+	Mpi2IOUnitPage3_t *io_unit_pg3 = NULL;
+	Mpi2ConfigReply_t mpi_reply;
+	u16 backup_rail_monitor_status = 0;
+	u16 ioc_status;
+	int sz;
+	ssize_t rc = 0;
+
+	if (!ioc->is_warpdrive) {
+		printk(MPT2SAS_ERR_FMT "%s: BRM attribute is only for"\
+		    "warpdrive\n", ioc->name, __func__);
+		goto out;
+	}
+
+	/* allocate upto GPIOVal 36 entries */
+	sz = offsetof(Mpi2IOUnitPage3_t, GPIOVal) + (sizeof(u16) * 36);
+	io_unit_pg3 = kzalloc(sz, GFP_KERNEL);
+	if (!io_unit_pg3) {
+		printk(MPT2SAS_ERR_FMT "%s: failed allocating memory"\
+		    "for iounit_pg3: (%d) bytes\n", ioc->name, __func__, sz);
+		goto out;
+	}
+
+	if (mpt2sas_config_get_iounit_pg3(ioc, &mpi_reply, io_unit_pg3, sz) !=
+	    0) {
+		printk(MPT2SAS_ERR_FMT
+		    "%s: failed reading iounit_pg3\n", ioc->name,
+		    __func__);
+		goto out;
+	}
+
+	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
+	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
+		printk(MPT2SAS_ERR_FMT "%s: iounit_pg3 failed with"\
+		    "ioc_status(0x%04x)\n", ioc->name, __func__, ioc_status);
+		goto out;
+	}
+
+	if (io_unit_pg3->GPIOCount < 25) {
+		printk(MPT2SAS_ERR_FMT "%s: iounit_pg3->GPIOCount less than"\
+		     "25 entries, detected (%d) entries\n", ioc->name, __func__,
+		    io_unit_pg3->GPIOCount);
+		goto out;
+	}
+
+	/* BRM status is in bit zero of GPIOVal[24] */
+	backup_rail_monitor_status = le16_to_cpu(io_unit_pg3->GPIOVal[24]);
+	rc = snprintf(buf, PAGE_SIZE, "%d\n", (backup_rail_monitor_status & 1));
+
+ out:
+	kfree(io_unit_pg3);
+	return rc;
+}
+static DEVICE_ATTR(BRM_status, S_IRUGO, _ctl_BRM_status_show, NULL);
+
 struct DIAG_BUFFER_START {
 	__le32 Size;
 	__le32 DiagVersion;
@@ -2901,6 +2972,7 @@ struct device_attribute *mpt2sas_host_attrs[] = {
 	&dev_attr_host_trace_buffer,
 	&dev_attr_host_trace_buffer_enable,
 	&dev_attr_reply_queue_count,
+	&dev_attr_BRM_status,
 	NULL,
 };
 
diff --git a/drivers/scsi/mpt2sas/mpt2sas_ctl.h b/drivers/scsi/mpt2sas/mpt2sas_ctl.h
index 11ff1d5fb8f..b5eb0d1b8ea 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_ctl.h
+++ b/drivers/scsi/mpt2sas/mpt2sas_ctl.h
@@ -3,7 +3,7 @@
  * controllers
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_ctl.h
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
diff --git a/drivers/scsi/mpt2sas/mpt2sas_debug.h b/drivers/scsi/mpt2sas/mpt2sas_debug.h
index 9731f8e661b..69cc7d0c112 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_debug.h
+++ b/drivers/scsi/mpt2sas/mpt2sas_debug.h
@@ -2,7 +2,7 @@
  * Logging Support for MPT (Message Passing Technology) based controllers
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_debug.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
diff --git a/drivers/scsi/mpt2sas/mpt2sas_scsih.c b/drivers/scsi/mpt2sas/mpt2sas_scsih.c
index 1ccae45c527..af4e6c451b1 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_scsih.c
+++ b/drivers/scsi/mpt2sas/mpt2sas_scsih.c
@@ -2,7 +2,7 @@
  * Scsi Host Layer for MPT (Message Passing Technology) based controllers
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_scsih.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
@@ -119,6 +119,15 @@ module_param(diag_buffer_enable, int, 0);
 MODULE_PARM_DESC(diag_buffer_enable, " post diag buffers "
 	"(TRACE=1/SNAPSHOT=2/EXTENDED=4/default=0)");
 
+static int disable_discovery = -1;
+module_param(disable_discovery, int, 0);
+MODULE_PARM_DESC(disable_discovery, " disable discovery ");
+
+/* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
+static int prot_mask = 0;
+module_param(prot_mask, int, 0);
+MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
+
 /**
  * struct sense_info - common structure for obtaining sense keys
  * @skey: sense key
@@ -3768,8 +3777,6 @@ static void
 _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
 {
 	u8 ascq;
-	u8 sk;
-	u8 host_byte;
 
 	switch (ioc_status) {
 	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
@@ -3786,16 +3793,8 @@ _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
 		break;
 	}
 
-	if (scmd->sc_data_direction == DMA_TO_DEVICE) {
-		sk = ILLEGAL_REQUEST;
-		host_byte = DID_ABORT;
-	} else {
-		sk = ABORTED_COMMAND;
-		host_byte = DID_OK;
-	}
-
-	scsi_build_sense_buffer(0, scmd->sense_buffer, sk, 0x10, ascq);
-	scmd->result = DRIVER_SENSE << 24 | (host_byte << 16) |
+	scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST, 0x10, ascq);
+	scmd->result = DRIVER_SENSE << 24 | (DID_ABORT << 16) |
 	    SAM_STAT_CHECK_CONDITION;
 }
 
@@ -5973,8 +5972,14 @@ _scsih_sas_discovery_event(struct MPT2SAS_ADAPTER *ioc,
 #endif
 
 	if (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED &&
-	    !ioc->sas_hba.num_phys)
+	    !ioc->sas_hba.num_phys) {
+		if (disable_discovery > 0 && ioc->shost_recovery) {
+			/* Wait for the reset to complete */
+			while (ioc->shost_recovery)
+				ssleep(1);
+		}
 		_scsih_sas_host_add(ioc);
+	}
 }
 
 /**
@@ -7254,7 +7259,8 @@ mpt2sas_scsih_reset_handler(struct MPT2SAS_ADAPTER *ioc, int reset_phase)
 		_scsih_search_responding_sas_devices(ioc);
 		_scsih_search_responding_raid_devices(ioc);
 		_scsih_search_responding_expanders(ioc);
-		if (!ioc->is_driver_loading) {
+		if ((!ioc->is_driver_loading) && !(disable_discovery > 0 &&
+		    !ioc->sas_hba.num_phys)) {
 			_scsih_prep_device_scan(ioc);
 			_scsih_search_responding_sas_devices(ioc);
 			_scsih_search_responding_raid_devices(ioc);
@@ -7929,6 +7935,9 @@ _scsih_scan_start(struct Scsi_Host *shost)
 	if (diag_buffer_enable != -1 && diag_buffer_enable != 0)
 		mpt2sas_enable_diag_buffer(ioc, diag_buffer_enable);
 
+	if (disable_discovery > 0)
+		return;
+
 	ioc->start_scan = 1;
 	rc = mpt2sas_port_enable(ioc);
 
@@ -7950,6 +7959,12 @@ _scsih_scan_finished(struct Scsi_Host *shost, unsigned long time)
 {
 	struct MPT2SAS_ADAPTER *ioc = shost_priv(shost);
 
+	if (disable_discovery > 0) {
+		ioc->is_driver_loading = 0;
+		ioc->wait_for_discovery_to_complete = 0;
+		return 1;
+	}
+
 	if (time >= (300 * HZ)) {
 		ioc->base_cmds.status = MPT2_CMD_NOT_USED;
 		printk(MPT2SAS_INFO_FMT "port enable: FAILED with timeout "
@@ -8055,8 +8070,8 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (max_sectors != 0xFFFF) {
 		if (max_sectors < 64) {
 			shost->max_sectors = 64;
-			printk(MPT2SAS_WARN_FMT "Invalid value %d passed "
-			    "for max_sectors, range is 64 to 8192. Assigning "
+			printk(MPT2SAS_WARN_FMT "Invalid value %d passed "\
+			    "for max_sectors, range is 64 to 32767. Assigning "\
 			    "value of 64.\n", ioc->name, max_sectors);
 		} else if (max_sectors > 32767) {
 			shost->max_sectors = 32767;
@@ -8078,8 +8093,14 @@ _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto out_add_shost_fail;
 	}
 
-	scsi_host_set_prot(shost, SHOST_DIF_TYPE1_PROTECTION
-	    | SHOST_DIF_TYPE2_PROTECTION | SHOST_DIF_TYPE3_PROTECTION);
+	/* register EEDP capabilities with SCSI layer */
+	if (prot_mask)
+		scsi_host_set_prot(shost, prot_mask);
+	else
+		scsi_host_set_prot(shost, SHOST_DIF_TYPE1_PROTECTION
+				   | SHOST_DIF_TYPE2_PROTECTION
+				   | SHOST_DIF_TYPE3_PROTECTION);
+
 	scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
 
 	/* event thread */
diff --git a/drivers/scsi/mpt2sas/mpt2sas_transport.c b/drivers/scsi/mpt2sas/mpt2sas_transport.c
index c6cf20f6072..8c2ffbe6af0 100644
--- a/drivers/scsi/mpt2sas/mpt2sas_transport.c
+++ b/drivers/scsi/mpt2sas/mpt2sas_transport.c
@@ -2,7 +2,7 @@
  * SAS Transport Layer for MPT (Message Passing Technology) based controllers
  *
  * This code is based on drivers/scsi/mpt2sas/mpt2_transport.c
- * Copyright (C) 2007-2010  LSI Corporation
+ * Copyright (C) 2007-2012  LSI Corporation
  *  (mailto:DL-MPTFusionLinux@lsi.com)
  *
  * This program is free software; you can redistribute it and/or
diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c
index 4539d59a085..a3776d6ced6 100644
--- a/drivers/scsi/mvsas/mv_sas.c
+++ b/drivers/scsi/mvsas/mv_sas.c
@@ -1629,7 +1629,7 @@ int mvs_abort_task(struct sas_task *task)
 			mv_dprintk("mvs_abort_task() mvi=%p task=%p "
 				   "slot=%p slot_idx=x%x\n",
 				   mvi, task, slot, slot_idx);
-			mvs_tmf_timedout((unsigned long)task);
+			task->task_state_flags |= SAS_TASK_STATE_ABORTED;
 			mvs_slot_task_free(mvi, task, slot, slot_idx);
 			rc = TMF_RESP_FUNC_COMPLETE;
 			goto out;
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index 5ab953029f8..1c28215f8be 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -26,7 +26,7 @@ qla2x00_sysfs_read_fw_dump(struct file *filp, struct kobject *kobj,
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
 
-	if (ha->fw_dump_reading == 0)
+	if (!(ha->fw_dump_reading || ha->mctp_dump_reading))
 		return 0;
 
 	if (IS_QLA82XX(ha)) {
@@ -39,9 +39,14 @@ qla2x00_sysfs_read_fw_dump(struct file *filp, struct kobject *kobj,
 		rval = memory_read_from_buffer(buf, count,
 		    &off, ha->md_dump, ha->md_dump_size);
 		return rval;
-	} else
+	} else if (ha->mctp_dumped && ha->mctp_dump_reading)
+		return memory_read_from_buffer(buf, count, &off, ha->mctp_dump,
+		    MCTP_DUMP_SIZE);
+	else if (ha->fw_dump_reading)
 		return memory_read_from_buffer(buf, count, &off, ha->fw_dump,
 					ha->fw_dump_len);
+	else
+		return 0;
 }
 
 static ssize_t
@@ -107,6 +112,22 @@ qla2x00_sysfs_write_fw_dump(struct file *filp, struct kobject *kobj,
 		if (IS_QLA82XX(ha))
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
 		break;
+	case 6:
+		if (!ha->mctp_dump_reading)
+			break;
+		ql_log(ql_log_info, vha, 0x70c1,
+		    "MCTP dump cleared on (%ld).\n", vha->host_no);
+		ha->mctp_dump_reading = 0;
+		ha->mctp_dumped = 0;
+		break;
+	case 7:
+		if (ha->mctp_dumped && !ha->mctp_dump_reading) {
+			ha->mctp_dump_reading = 1;
+			ql_log(ql_log_info, vha, 0x70c2,
+			    "Raw mctp dump ready for read on (%ld).\n",
+			    vha->host_no);
+		}
+		break;
 	}
 	return count;
 }
@@ -564,6 +585,7 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 	struct qla_hw_data *ha = vha->hw;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
 	int type;
+	uint32_t idc_control;
 
 	if (off != 0)
 		return -EINVAL;
@@ -587,22 +609,36 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 		scsi_unblock_requests(vha->host);
 		break;
 	case 0x2025d:
-		if (!IS_QLA81XX(ha) || !IS_QLA8031(ha))
+		if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha))
 			return -EPERM;
 
 		ql_log(ql_log_info, vha, 0x706f,
 		    "Issuing MPI reset.\n");
 
-		/* Make sure FC side is not in reset */
-		qla2x00_wait_for_hba_online(vha);
-
-		/* Issue MPI reset */
-		scsi_block_requests(vha->host);
-		if (qla81xx_restart_mpi_firmware(vha) != QLA_SUCCESS)
-			ql_log(ql_log_warn, vha, 0x7070,
-			    "MPI reset failed.\n");
-		scsi_unblock_requests(vha->host);
-		break;
+		if (IS_QLA83XX(ha)) {
+			uint32_t idc_control;
+
+			qla83xx_idc_lock(vha, 0);
+			__qla83xx_get_idc_control(vha, &idc_control);
+			idc_control |= QLA83XX_IDC_GRACEFUL_RESET;
+			__qla83xx_set_idc_control(vha, idc_control);
+			qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE,
+			    QLA8XXX_DEV_NEED_RESET);
+			qla83xx_idc_audit(vha, IDC_AUDIT_TIMESTAMP);
+			qla83xx_idc_unlock(vha, 0);
+			break;
+		} else {
+			/* Make sure FC side is not in reset */
+			qla2x00_wait_for_hba_online(vha);
+
+			/* Issue MPI reset */
+			scsi_block_requests(vha->host);
+			if (qla81xx_restart_mpi_firmware(vha) != QLA_SUCCESS)
+				ql_log(ql_log_warn, vha, 0x7070,
+				    "MPI reset failed.\n");
+			scsi_unblock_requests(vha->host);
+			break;
+		}
 	case 0x2025e:
 		if (!IS_QLA82XX(ha) || vha != base_vha) {
 			ql_log(ql_log_info, vha, 0x7071,
@@ -616,6 +652,29 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 		qla2xxx_wake_dpc(vha);
 		qla2x00_wait_for_fcoe_ctx_reset(vha);
 		break;
+	case 0x2025f:
+		if (!IS_QLA8031(ha))
+			return -EPERM;
+		ql_log(ql_log_info, vha, 0x70bc,
+		    "Disabling Reset by IDC control\n");
+		qla83xx_idc_lock(vha, 0);
+		__qla83xx_get_idc_control(vha, &idc_control);
+		idc_control |= QLA83XX_IDC_RESET_DISABLED;
+		__qla83xx_set_idc_control(vha, idc_control);
+		qla83xx_idc_unlock(vha, 0);
+		break;
+	case 0x20260:
+		if (!IS_QLA8031(ha))
+			return -EPERM;
+		ql_log(ql_log_info, vha, 0x70bd,
+		    "Enabling Reset by IDC control\n");
+		qla83xx_idc_lock(vha, 0);
+		__qla83xx_get_idc_control(vha, &idc_control);
+		idc_control &= ~QLA83XX_IDC_RESET_DISABLED;
+		__qla83xx_set_idc_control(vha, idc_control);
+		qla83xx_idc_unlock(vha, 0);
+		break;
+
 	}
 	return count;
 }
@@ -1251,6 +1310,49 @@ qla2x00_fw_state_show(struct device *dev, struct device_attribute *attr,
 	    state[1], state[2], state[3], state[4]);
 }
 
+static ssize_t
+qla2x00_diag_requests_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
+	if (!IS_BIDI_CAPABLE(vha->hw))
+		return snprintf(buf, PAGE_SIZE, "\n");
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n", vha->bidi_stats.io_count);
+}
+
+static ssize_t
+qla2x00_diag_megabytes_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
+	if (!IS_BIDI_CAPABLE(vha->hw))
+		return snprintf(buf, PAGE_SIZE, "\n");
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n",
+	    vha->bidi_stats.transfer_bytes >> 20);
+}
+
+static ssize_t
+qla2x00_fw_dump_size_show(struct device *dev, struct device_attribute *attr,
+	char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t size;
+
+	if (!ha->fw_dumped)
+		size = 0;
+	else if (IS_QLA82XX(ha))
+		size = ha->md_template_size + ha->md_dump_size;
+	else
+		size = ha->fw_dump_len;
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", size);
+}
+
 static DEVICE_ATTR(driver_version, S_IRUGO, qla2x00_drvr_version_show, NULL);
 static DEVICE_ATTR(fw_version, S_IRUGO, qla2x00_fw_version_show, NULL);
 static DEVICE_ATTR(serial_num, S_IRUGO, qla2x00_serial_num_show, NULL);
@@ -1289,6 +1391,9 @@ static DEVICE_ATTR(vn_port_mac_address, S_IRUGO,
 static DEVICE_ATTR(fabric_param, S_IRUGO, qla2x00_fabric_param_show, NULL);
 static DEVICE_ATTR(fw_state, S_IRUGO, qla2x00_fw_state_show, NULL);
 static DEVICE_ATTR(thermal_temp, S_IRUGO, qla2x00_thermal_temp_show, NULL);
+static DEVICE_ATTR(diag_requests, S_IRUGO, qla2x00_diag_requests_show, NULL);
+static DEVICE_ATTR(diag_megabytes, S_IRUGO, qla2x00_diag_megabytes_show, NULL);
+static DEVICE_ATTR(fw_dump_size, S_IRUGO, qla2x00_fw_dump_size_show, NULL);
 
 struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_driver_version,
@@ -1318,6 +1423,9 @@ struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_fw_state,
 	&dev_attr_optrom_gold_fw_version,
 	&dev_attr_thermal_temp,
+	&dev_attr_diag_requests,
+	&dev_attr_diag_megabytes,
+	&dev_attr_fw_dump_size,
 	NULL,
 };
 
@@ -1704,7 +1812,7 @@ qla24xx_vport_create(struct fc_vport *fc_vport, bool disable)
 
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif) {
 		if (ha->fw_attributes & BIT_4) {
-			int prot = 0;
+			int prot = 0, guard;
 			vha->flags.difdix_supported = 1;
 			ql_dbg(ql_dbg_user, vha, 0x7082,
 			    "Registered for DIF/DIX type 1 and 3 protection.\n");
@@ -1717,7 +1825,14 @@ qla24xx_vport_create(struct fc_vport *fc_vport, bool disable)
 			    | SHOST_DIX_TYPE1_PROTECTION
 			    | SHOST_DIX_TYPE2_PROTECTION
 			    | SHOST_DIX_TYPE3_PROTECTION);
-			scsi_host_set_guard(vha->host, SHOST_DIX_GUARD_CRC);
+
+			guard = SHOST_DIX_GUARD_CRC;
+
+			if (IS_PI_IPGUARD_CAPABLE(ha) &&
+			    (ql2xenabledif > 1 || IS_PI_DIFB_DIX0_CAPABLE(ha)))
+				guard |= SHOST_DIX_GUARD_IP;
+
+			scsi_host_set_guard(vha->host, guard);
 		} else
 			vha->flags.difdix_supported = 0;
 	}
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index c68883806c5..2f9bddd3c61 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -530,13 +530,13 @@ done_unmap_sg:
 done:
 	return rval;
 }
-
-/* Set the port configuration to enable the
- * internal loopback on ISP81XX
+/*
+ * Set the port configuration to enable the internal or external loopback
+ * depending on the loopback mode.
  */
 static inline int
-qla81xx_set_internal_loopback(scsi_qla_host_t *vha, uint16_t *config,
-    uint16_t *new_config)
+qla81xx_set_loopback_mode(scsi_qla_host_t *vha, uint16_t *config,
+	uint16_t *new_config, uint16_t mode)
 {
 	int ret = 0;
 	int rval = 0;
@@ -545,8 +545,14 @@ qla81xx_set_internal_loopback(scsi_qla_host_t *vha, uint16_t *config,
 	if (!IS_QLA81XX(ha) && !IS_QLA8031(ha))
 		goto done_set_internal;
 
-	new_config[0] = config[0] | (ENABLE_INTERNAL_LOOPBACK << 1);
-	memcpy(&new_config[1], &config[1], sizeof(uint16_t) * 3) ;
+	if (mode == INTERNAL_LOOPBACK)
+		new_config[0] = config[0] | (ENABLE_INTERNAL_LOOPBACK << 1);
+	else if (mode == EXTERNAL_LOOPBACK)
+		new_config[0] = config[0] | (ENABLE_EXTERNAL_LOOPBACK << 1);
+	ql_dbg(ql_dbg_user, vha, 0x70be,
+	     "new_config[0]=%02x\n", (new_config[0] & INTERNAL_LOOPBACK_MASK));
+
+	memcpy(&new_config[1], &config[1], sizeof(uint16_t) * 3);
 
 	ha->notify_dcbx_comp = 1;
 	ret = qla81xx_set_port_config(vha, new_config);
@@ -562,9 +568,17 @@ qla81xx_set_internal_loopback(scsi_qla_host_t *vha, uint16_t *config,
 	if (!wait_for_completion_timeout(&ha->dcbx_comp, (20 * HZ))) {
 		ql_dbg(ql_dbg_user, vha, 0x7022,
 		    "State change notification not received.\n");
-	} else
-		ql_dbg(ql_dbg_user, vha, 0x7023,
-		    "State change received.\n");
+		rval = -EINVAL;
+	} else {
+		if (ha->flags.idc_compl_status) {
+			ql_dbg(ql_dbg_user, vha, 0x70c3,
+			    "Bad status in IDC Completion AEN\n");
+			rval = -EINVAL;
+			ha->flags.idc_compl_status = 0;
+		} else
+			ql_dbg(ql_dbg_user, vha, 0x7023,
+			    "State change received.\n");
+	}
 
 	ha->notify_dcbx_comp = 0;
 
@@ -572,11 +586,9 @@ done_set_internal:
 	return rval;
 }
 
-/* Set the port configuration to disable the
- * internal loopback on ISP81XX
- */
+/* Disable loopback mode */
 static inline int
-qla81xx_reset_internal_loopback(scsi_qla_host_t *vha, uint16_t *config,
+qla81xx_reset_loopback_mode(scsi_qla_host_t *vha, uint16_t *config,
     int wait)
 {
 	int ret = 0;
@@ -589,8 +601,12 @@ qla81xx_reset_internal_loopback(scsi_qla_host_t *vha, uint16_t *config,
 
 	memset(new_config, 0 , sizeof(new_config));
 	if ((config[0] & INTERNAL_LOOPBACK_MASK) >> 1 ==
-			ENABLE_INTERNAL_LOOPBACK) {
+	    ENABLE_INTERNAL_LOOPBACK ||
+	    (config[0] & INTERNAL_LOOPBACK_MASK) >> 1 ==
+	    ENABLE_EXTERNAL_LOOPBACK) {
 		new_config[0] = config[0] & ~INTERNAL_LOOPBACK_MASK;
+		ql_dbg(ql_dbg_user, vha, 0x70bf, "new_config[0]=%02x\n",
+		    (new_config[0] & INTERNAL_LOOPBACK_MASK));
 		memcpy(&new_config[1], &config[1], sizeof(uint16_t) * 3) ;
 
 		ha->notify_dcbx_comp = wait;
@@ -707,7 +723,8 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
 
 	elreq.options = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
 
-	if ((ha->current_topology == ISP_CFG_F ||
+	if (atomic_read(&vha->loop_state) == LOOP_READY &&
+	    (ha->current_topology == ISP_CFG_F ||
 	    ((IS_QLA81XX(ha) || IS_QLA8031(ha)) &&
 	    le32_to_cpu(*(uint32_t *)req_data) == ELS_OPCODE_BYTE
 	    && req_data_len == MAX_ELS_FRAME_PAYLOAD)) &&
@@ -729,30 +746,24 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
 				goto done_free_dma_req;
 			}
 
-			if (elreq.options != EXTERNAL_LOOPBACK) {
-				ql_dbg(ql_dbg_user, vha, 0x7020,
-				    "Internal: current port config = %x\n",
-				    config[0]);
-				if (qla81xx_set_internal_loopback(vha, config,
-					new_config)) {
-					ql_log(ql_log_warn, vha, 0x7024,
-					    "Internal loopback failed.\n");
-					bsg_job->reply->result =
-						(DID_ERROR << 16);
-					rval = -EPERM;
-					goto done_free_dma_req;
-				}
-			} else {
-				/* For external loopback to work
-				 * ensure internal loopback is disabled
-				 */
-				if (qla81xx_reset_internal_loopback(vha,
-					config, 1)) {
-					bsg_job->reply->result =
-						(DID_ERROR << 16);
-					rval = -EPERM;
-					goto done_free_dma_req;
-				}
+			ql_dbg(ql_dbg_user, vha, 0x70c0,
+			    "elreq.options=%04x\n", elreq.options);
+
+			if (elreq.options == EXTERNAL_LOOPBACK)
+				if (IS_QLA8031(ha))
+					rval = qla81xx_set_loopback_mode(vha,
+					    config, new_config, elreq.options);
+				else
+					rval = qla81xx_reset_loopback_mode(vha,
+					    config, 1);
+			else
+				rval = qla81xx_set_loopback_mode(vha, config,
+				    new_config, elreq.options);
+
+			if (rval) {
+				bsg_job->reply->result = (DID_ERROR << 16);
+				rval = -EPERM;
+				goto done_free_dma_req;
 			}
 
 			type = "FC_BSG_HST_VENDOR_LOOPBACK";
@@ -766,7 +777,7 @@ qla2x00_process_loopback(struct fc_bsg_job *bsg_job)
 				/* Revert back to original port config
 				 * Also clear internal loopback
 				 */
-				qla81xx_reset_internal_loopback(vha,
+				qla81xx_reset_loopback_mode(vha,
 				    new_config, 0);
 			}
 
@@ -1364,7 +1375,7 @@ qla2x00_read_optrom(struct fc_bsg_job *bsg_job)
 	struct qla_hw_data *ha = vha->hw;
 	int rval = 0;
 
-	if (ha->flags.isp82xx_reset_hdlr_active)
+	if (ha->flags.nic_core_reset_hdlr_active)
 		return -EBUSY;
 
 	rval = qla2x00_optrom_setup(bsg_job, vha, 0);
@@ -1560,6 +1571,276 @@ done:
 }
 
 static int
+qla2x00_write_i2c(struct fc_bsg_job *bsg_job)
+{
+	struct Scsi_Host *host = bsg_job->shost;
+	scsi_qla_host_t *vha = shost_priv(host);
+	struct qla_hw_data *ha = vha->hw;
+	int rval = 0;
+	uint8_t bsg[DMA_POOL_SIZE];
+	struct qla_i2c_access *i2c = (void *)bsg;
+	dma_addr_t sfp_dma;
+	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+	if (!sfp) {
+		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		    EXT_STATUS_NO_MEMORY;
+		goto done;
+	}
+
+	sg_copy_to_buffer(bsg_job->request_payload.sg_list,
+	    bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c));
+
+	memcpy(sfp, i2c->buffer, i2c->length);
+	rval = qla2x00_write_sfp(vha, sfp_dma, sfp,
+	    i2c->device, i2c->offset, i2c->length, i2c->option);
+
+	if (rval) {
+		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		    EXT_STATUS_MAILBOX;
+		goto dealloc;
+	}
+
+	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+
+dealloc:
+	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
+
+done:
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	bsg_job->reply->result = DID_OK << 16;
+	bsg_job->job_done(bsg_job);
+
+	return 0;
+}
+
+static int
+qla2x00_read_i2c(struct fc_bsg_job *bsg_job)
+{
+	struct Scsi_Host *host = bsg_job->shost;
+	scsi_qla_host_t *vha = shost_priv(host);
+	struct qla_hw_data *ha = vha->hw;
+	int rval = 0;
+	uint8_t bsg[DMA_POOL_SIZE];
+	struct qla_i2c_access *i2c = (void *)bsg;
+	dma_addr_t sfp_dma;
+	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+	if (!sfp) {
+		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		    EXT_STATUS_NO_MEMORY;
+		goto done;
+	}
+
+	sg_copy_to_buffer(bsg_job->request_payload.sg_list,
+	    bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c));
+
+	rval = qla2x00_read_sfp(vha, sfp_dma, sfp,
+		i2c->device, i2c->offset, i2c->length, i2c->option);
+
+	if (rval) {
+		bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] =
+		    EXT_STATUS_MAILBOX;
+		goto dealloc;
+	}
+
+	memcpy(i2c->buffer, sfp, i2c->length);
+	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+	    bsg_job->reply_payload.sg_cnt, i2c, sizeof(*i2c));
+
+	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = 0;
+
+dealloc:
+	dma_pool_free(ha->s_dma_pool, sfp, sfp_dma);
+
+done:
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	bsg_job->reply->reply_payload_rcv_len = sizeof(*i2c);
+	bsg_job->reply->result = DID_OK << 16;
+	bsg_job->job_done(bsg_job);
+
+	return 0;
+}
+
+static int
+qla24xx_process_bidir_cmd(struct fc_bsg_job *bsg_job)
+{
+	struct Scsi_Host *host = bsg_job->shost;
+	scsi_qla_host_t *vha = shost_priv(host);
+	struct qla_hw_data *ha = vha->hw;
+	uint16_t thread_id;
+	uint32_t rval = EXT_STATUS_OK;
+	uint16_t req_sg_cnt = 0;
+	uint16_t rsp_sg_cnt = 0;
+	uint16_t nextlid = 0;
+	uint32_t tot_dsds;
+	srb_t *sp = NULL;
+	uint32_t req_data_len = 0;
+	uint32_t rsp_data_len = 0;
+
+	/* Check the type of the adapter */
+	if (!IS_BIDI_CAPABLE(ha)) {
+		ql_log(ql_log_warn, vha, 0x70a0,
+			"This adapter is not supported\n");
+		rval = EXT_STATUS_NOT_SUPPORTED;
+		goto done;
+	}
+
+	if (test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags) ||
+		test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) ||
+		test_bit(ISP_ABORT_RETRY, &vha->dpc_flags)) {
+		rval =  EXT_STATUS_BUSY;
+		goto done;
+	}
+
+	/* Check if host is online */
+	if (!vha->flags.online) {
+		ql_log(ql_log_warn, vha, 0x70a1,
+			"Host is not online\n");
+		rval = EXT_STATUS_DEVICE_OFFLINE;
+		goto done;
+	}
+
+	/* Check if cable is plugged in or not */
+	if (vha->device_flags & DFLG_NO_CABLE) {
+		ql_log(ql_log_warn, vha, 0x70a2,
+			"Cable is unplugged...\n");
+		rval = EXT_STATUS_INVALID_CFG;
+		goto done;
+	}
+
+	/* Check if the switch is connected or not */
+	if (ha->current_topology != ISP_CFG_F) {
+		ql_log(ql_log_warn, vha, 0x70a3,
+			"Host is not connected to the switch\n");
+		rval = EXT_STATUS_INVALID_CFG;
+		goto done;
+	}
+
+	/* Check if operating mode is P2P */
+	if (ha->operating_mode != P2P) {
+		ql_log(ql_log_warn, vha, 0x70a4,
+		    "Host is operating mode is not P2p\n");
+		rval = EXT_STATUS_INVALID_CFG;
+		goto done;
+	}
+
+	thread_id = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+
+	mutex_lock(&ha->selflogin_lock);
+	if (vha->self_login_loop_id == 0) {
+		/* Initialize all required  fields of fcport */
+		vha->bidir_fcport.vha = vha;
+		vha->bidir_fcport.d_id.b.al_pa = vha->d_id.b.al_pa;
+		vha->bidir_fcport.d_id.b.area = vha->d_id.b.area;
+		vha->bidir_fcport.d_id.b.domain = vha->d_id.b.domain;
+		vha->bidir_fcport.loop_id = vha->loop_id;
+
+		if (qla2x00_fabric_login(vha, &(vha->bidir_fcport), &nextlid)) {
+			ql_log(ql_log_warn, vha, 0x70a7,
+			    "Failed to login port %06X for bidirectional IOCB\n",
+			    vha->bidir_fcport.d_id.b24);
+			mutex_unlock(&ha->selflogin_lock);
+			rval = EXT_STATUS_MAILBOX;
+			goto done;
+		}
+		vha->self_login_loop_id = nextlid - 1;
+
+	}
+	/* Assign the self login loop id to fcport */
+	mutex_unlock(&ha->selflogin_lock);
+
+	vha->bidir_fcport.loop_id = vha->self_login_loop_id;
+
+	req_sg_cnt = dma_map_sg(&ha->pdev->dev,
+		bsg_job->request_payload.sg_list,
+		bsg_job->request_payload.sg_cnt,
+		DMA_TO_DEVICE);
+
+	if (!req_sg_cnt) {
+		rval = EXT_STATUS_NO_MEMORY;
+		goto done;
+	}
+
+	rsp_sg_cnt = dma_map_sg(&ha->pdev->dev,
+		bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt,
+		DMA_FROM_DEVICE);
+
+	if (!rsp_sg_cnt) {
+		rval = EXT_STATUS_NO_MEMORY;
+		goto done_unmap_req_sg;
+	}
+
+	if ((req_sg_cnt !=  bsg_job->request_payload.sg_cnt) ||
+		(rsp_sg_cnt != bsg_job->reply_payload.sg_cnt)) {
+		ql_dbg(ql_dbg_user, vha, 0x70a9,
+		    "Dma mapping resulted in different sg counts "
+		    "[request_sg_cnt: %x dma_request_sg_cnt: %x reply_sg_cnt: "
+		    "%x dma_reply_sg_cnt: %x]\n",
+		    bsg_job->request_payload.sg_cnt, req_sg_cnt,
+		    bsg_job->reply_payload.sg_cnt, rsp_sg_cnt);
+		rval = EXT_STATUS_NO_MEMORY;
+		goto done_unmap_sg;
+	}
+
+	if (req_data_len != rsp_data_len) {
+		rval = EXT_STATUS_BUSY;
+		ql_log(ql_log_warn, vha, 0x70aa,
+		    "req_data_len != rsp_data_len\n");
+		goto done_unmap_sg;
+	}
+
+	req_data_len = bsg_job->request_payload.payload_len;
+	rsp_data_len = bsg_job->reply_payload.payload_len;
+
+
+	/* Alloc SRB structure */
+	sp = qla2x00_get_sp(vha, &(vha->bidir_fcport), GFP_KERNEL);
+	if (!sp) {
+		ql_dbg(ql_dbg_user, vha, 0x70ac,
+		    "Alloc SRB structure failed\n");
+		rval = EXT_STATUS_NO_MEMORY;
+		goto done_unmap_sg;
+	}
+
+	/*Populate srb->ctx with bidir ctx*/
+	sp->u.bsg_job = bsg_job;
+	sp->free = qla2x00_bsg_sp_free;
+	sp->type = SRB_BIDI_CMD;
+	sp->done = qla2x00_bsg_job_done;
+
+	/* Add the read and write sg count */
+	tot_dsds = rsp_sg_cnt + req_sg_cnt;
+
+	rval = qla2x00_start_bidir(sp, vha, tot_dsds);
+	if (rval != EXT_STATUS_OK)
+		goto done_free_srb;
+	/* the bsg request  will be completed in the interrupt handler */
+	return rval;
+
+done_free_srb:
+	mempool_free(sp, ha->srb_mempool);
+done_unmap_sg:
+	dma_unmap_sg(&ha->pdev->dev,
+	    bsg_job->reply_payload.sg_list,
+	    bsg_job->reply_payload.sg_cnt, DMA_FROM_DEVICE);
+done_unmap_req_sg:
+	dma_unmap_sg(&ha->pdev->dev,
+	    bsg_job->request_payload.sg_list,
+	    bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE);
+done:
+
+	/* Return an error vendor specific response
+	 * and complete the bsg request
+	 */
+	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	bsg_job->reply->reply_payload_rcv_len = 0;
+	bsg_job->reply->result = (DID_OK) << 16;
+	bsg_job->job_done(bsg_job);
+	/* Always retrun success, vendor rsp carries correct status */
+	return 0;
+}
+
+static int
 qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job)
 {
 	switch (bsg_job->request->rqst_data.h_vendor.vendor_cmd[0]) {
@@ -1596,6 +1877,15 @@ qla2x00_process_vendor_specific(struct fc_bsg_job *bsg_job)
 	case QL_VND_WRITE_FRU_STATUS:
 		return qla2x00_write_fru_status(bsg_job);
 
+	case QL_VND_WRITE_I2C:
+		return qla2x00_write_i2c(bsg_job);
+
+	case QL_VND_READ_I2C:
+		return qla2x00_read_i2c(bsg_job);
+
+	case QL_VND_DIAG_IO_CMD:
+		return qla24xx_process_bidir_cmd(bsg_job);
+
 	default:
 		bsg_job->reply->result = (DID_ERROR << 16);
 		bsg_job->job_done(bsg_job);
diff --git a/drivers/scsi/qla2xxx/qla_bsg.h b/drivers/scsi/qla2xxx/qla_bsg.h
index 70caa63a893..37b8b7ba742 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.h
+++ b/drivers/scsi/qla2xxx/qla_bsg.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -19,21 +19,41 @@
 #define QL_VND_SET_FRU_VERSION	0x0B
 #define QL_VND_READ_FRU_STATUS	0x0C
 #define QL_VND_WRITE_FRU_STATUS	0x0D
+#define QL_VND_DIAG_IO_CMD	0x0A
+#define QL_VND_WRITE_I2C	0x10
+#define QL_VND_READ_I2C		0x11
 
 /* BSG Vendor specific subcode returns */
 #define EXT_STATUS_OK			0
 #define EXT_STATUS_ERR			1
+#define EXT_STATUS_BUSY			2
 #define EXT_STATUS_INVALID_PARAM	6
+#define EXT_STATUS_DATA_OVERRUN		7
+#define EXT_STATUS_DATA_UNDERRUN	8
 #define EXT_STATUS_MAILBOX		11
 #define EXT_STATUS_NO_MEMORY		17
+#define EXT_STATUS_DEVICE_OFFLINE	22
+
+/*
+ * To support bidirectional iocb
+ * BSG Vendor specific returns
+ */
+#define EXT_STATUS_NOT_SUPPORTED	27
+#define EXT_STATUS_INVALID_CFG		28
+#define EXT_STATUS_DMA_ERR		29
+#define EXT_STATUS_TIMEOUT		30
+#define EXT_STATUS_THREAD_FAILED	31
+#define EXT_STATUS_DATA_CMP_FAILED	32
 
 /* BSG definations for interpreting CommandSent field */
 #define INT_DEF_LB_LOOPBACK_CMD         0
 #define INT_DEF_LB_ECHO_CMD             1
 
 /* Loopback related definations */
+#define INTERNAL_LOOPBACK		0xF1
 #define EXTERNAL_LOOPBACK		0xF2
 #define ENABLE_INTERNAL_LOOPBACK	0x02
+#define ENABLE_EXTERNAL_LOOPBACK	0x04
 #define INTERNAL_LOOPBACK_MASK		0x000E
 #define MAX_ELS_FRAME_PAYLOAD		252
 #define ELS_OPCODE_BYTE			0x10
@@ -183,4 +203,12 @@ struct qla_status_reg {
 	uint8_t reserved[7];
 } __packed;
 
+struct qla_i2c_access {
+	uint16_t device;
+	uint16_t offset;
+	uint16_t option;
+	uint16_t length;
+	uint8_t  buffer[0x40];
+} __packed;
+
 #endif
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index fdee5611f3e..44efe3cc79e 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -11,26 +11,31 @@
  * ----------------------------------------------------------------------
  * |             Level            |   Last Value Used  |     Holes	|
  * ----------------------------------------------------------------------
- * | Module Init and Probe        |       0x0122       | 0x4b,0xba,0xfa |
- * | Mailbox commands             |       0x1140       | 0x111a-0x111b  |
+ * | Module Init and Probe        |       0x0124       | 0x4b,0xba,0xfa |
+ * | Mailbox commands             |       0x114f       | 0x111a-0x111b  |
  * |                              |                    | 0x112c-0x112e  |
  * |                              |                    | 0x113a         |
- * | Device Discovery             |       0x2086       | 0x2020-0x2022  |
- * | Queue Command and IO tracing |       0x3030       | 0x3006,0x3008  |
+ * | Device Discovery             |       0x2087       | 0x2020-0x2022, |
+ * |                              |                    | 0x2016         |
+ * | Queue Command and IO tracing |       0x3030       | 0x3006-0x300b  |
+ * |                              |                    | 0x3027-0x3028  |
  * |                              |                    | 0x302d-0x302e  |
- * | DPC Thread                   |       0x401c       | 0x4002,0x4013  |
- * | Async Events                 |       0x505f       | 0x502b-0x502f  |
+ * | DPC Thread                   |       0x401d       | 0x4002,0x4013  |
+ * | Async Events                 |       0x5071       | 0x502b-0x502f  |
  * |                              |                    | 0x5047,0x5052  |
  * | Timer Routines               |       0x6011       |                |
- * | User Space Interactions      |       0x709f       | 0x7018,0x702e, |
+ * | User Space Interactions      |       0x70c3       | 0x7018,0x702e, |
  * |                              |                    | 0x7039,0x7045, |
  * |                              |                    | 0x7073-0x7075, |
- * |                              |                    | 0x708c         |
+ * |                              |                    | 0x708c,        |
+ * |                              |                    | 0x70a5,0x70a6, |
+ * |                              |                    | 0x70a8,0x70ab, |
+ * |                              |                    | 0x70ad-0x70ae  |
  * | Task Management              |       0x803c       | 0x8025-0x8026  |
  * |                              |                    | 0x800b,0x8039  |
  * | AER/EEH                      |       0x9011       |		|
  * | Virtual Port                 |       0xa007       |		|
- * | ISP82XX Specific             |       0xb054       | 0xb024         |
+ * | ISP82XX Specific             |       0xb084       | 0xb002,0xb024  |
  * | MultiQ                       |       0xc00c       |		|
  * | Misc                         |       0xd010       |		|
  * | Target Mode		  |	  0xe06f       |		|
@@ -2357,7 +2362,7 @@ ql_dbg(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
 
 /*
  * This function is for formatting and logging debug information.
- * It is to be used when vha is not available and pci is availble,
+ * It is to be used when vha is not available and pci is available,
  * i.e., before host allocation. It formats the message and logs it
  * to the messages file.
  * parameters:
@@ -2452,7 +2457,7 @@ ql_log(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
 
 /*
  * This function is for formatting and logging log messages.
- * It is to be used when vha is not available and pci is availble,
+ * It is to be used when vha is not available and pci is available,
  * i.e., before host allocation. It formats the message and logs
  * it to the messages file. All the messages are logged irrespective
  * of the value of ql2xextended_error_logging.
diff --git a/drivers/scsi/qla2xxx/qla_dbg.h b/drivers/scsi/qla2xxx/qla_dbg.h
index f278df8cce0..8f911c0b1e7 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.h
+++ b/drivers/scsi/qla2xxx/qla_dbg.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 39007f53aec..a9725bf5527 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -115,6 +115,82 @@
 #define WRT_REG_DWORD(addr, data)	writel(data,addr)
 
 /*
+ * ISP83XX specific remote register addresses
+ */
+#define QLA83XX_LED_PORT0			0x00201320
+#define QLA83XX_LED_PORT1			0x00201328
+#define QLA83XX_IDC_DEV_STATE		0x22102384
+#define QLA83XX_IDC_MAJOR_VERSION	0x22102380
+#define QLA83XX_IDC_MINOR_VERSION	0x22102398
+#define QLA83XX_IDC_DRV_PRESENCE	0x22102388
+#define QLA83XX_IDC_DRIVER_ACK		0x2210238c
+#define QLA83XX_IDC_CONTROL			0x22102390
+#define QLA83XX_IDC_AUDIT			0x22102394
+#define QLA83XX_IDC_LOCK_RECOVERY	0x2210239c
+#define QLA83XX_DRIVER_LOCKID		0x22102104
+#define QLA83XX_DRIVER_LOCK			0x8111c028
+#define QLA83XX_DRIVER_UNLOCK		0x8111c02c
+#define QLA83XX_FLASH_LOCKID		0x22102100
+#define QLA83XX_FLASH_LOCK			0x8111c010
+#define QLA83XX_FLASH_UNLOCK		0x8111c014
+#define QLA83XX_DEV_PARTINFO1		0x221023e0
+#define QLA83XX_DEV_PARTINFO2		0x221023e4
+#define QLA83XX_FW_HEARTBEAT		0x221020b0
+#define QLA83XX_PEG_HALT_STATUS1	0x221020a8
+#define QLA83XX_PEG_HALT_STATUS2	0x221020ac
+
+/* 83XX: Macros defining 8200 AEN Reason codes */
+#define IDC_DEVICE_STATE_CHANGE BIT_0
+#define IDC_PEG_HALT_STATUS_CHANGE BIT_1
+#define IDC_NIC_FW_REPORTED_FAILURE BIT_2
+#define IDC_HEARTBEAT_FAILURE BIT_3
+
+/* 83XX: Macros defining 8200 AEN Error-levels */
+#define ERR_LEVEL_NON_FATAL 0x1
+#define ERR_LEVEL_RECOVERABLE_FATAL 0x2
+#define ERR_LEVEL_UNRECOVERABLE_FATAL 0x4
+
+/* 83XX: Macros for IDC Version */
+#define QLA83XX_SUPP_IDC_MAJOR_VERSION 0x01
+#define QLA83XX_SUPP_IDC_MINOR_VERSION 0x0
+
+/* 83XX: Macros for scheduling dpc tasks */
+#define QLA83XX_NIC_CORE_RESET 0x1
+#define QLA83XX_IDC_STATE_HANDLER 0x2
+#define QLA83XX_NIC_CORE_UNRECOVERABLE 0x3
+
+/* 83XX: Macros for defining IDC-Control bits */
+#define QLA83XX_IDC_RESET_DISABLED BIT_0
+#define QLA83XX_IDC_GRACEFUL_RESET BIT_1
+
+/* 83XX: Macros for different timeouts */
+#define QLA83XX_IDC_INITIALIZATION_TIMEOUT 30
+#define QLA83XX_IDC_RESET_ACK_TIMEOUT 10
+#define QLA83XX_MAX_LOCK_RECOVERY_WAIT (2 * HZ)
+
+/* 83XX: Macros for defining class in DEV-Partition Info register */
+#define QLA83XX_CLASS_TYPE_NONE		0x0
+#define QLA83XX_CLASS_TYPE_NIC		0x1
+#define QLA83XX_CLASS_TYPE_FCOE		0x2
+#define QLA83XX_CLASS_TYPE_ISCSI	0x3
+
+/* 83XX: Macros for IDC Lock-Recovery stages */
+#define IDC_LOCK_RECOVERY_STAGE1	0x1 /* Stage1: Intent for
+					     * lock-recovery
+					     */
+#define IDC_LOCK_RECOVERY_STAGE2	0x2 /* Stage2: Perform lock-recovery */
+
+/* 83XX: Macros for IDC Audit type */
+#define IDC_AUDIT_TIMESTAMP		0x0 /* IDC-AUDIT: Record timestamp of
+					     * dev-state change to NEED-RESET
+					     * or NEED-QUIESCENT
+					     */
+#define IDC_AUDIT_COMPLETION		0x1 /* IDC-AUDIT: Record duration of
+					     * reset-recovery completion is
+					     * second
+					     */
+
+/*
  * The ISP2312 v2 chip cannot access the FLASH/GPIO registers via MMIO in an
  * 133Mhz slot.
  */
@@ -129,6 +205,7 @@
 #define MAX_FIBRE_DEVICES_2400	2048
 #define MAX_FIBRE_DEVICES_LOOP	128
 #define MAX_FIBRE_DEVICES_MAX	MAX_FIBRE_DEVICES_2400
+#define LOOPID_MAP_SIZE		(ha->max_fibre_devices)
 #define MAX_FIBRE_LUNS  	0xFFFF
 #define	MAX_HOST_COUNT		16
 
@@ -259,6 +336,7 @@ struct srb_iocb {
 #define SRB_ADISC_CMD	6
 #define SRB_TM_CMD	7
 #define SRB_SCSI_CMD	8
+#define SRB_BIDI_CMD	9
 
 typedef struct srb {
 	atomic_t ref_count;
@@ -594,6 +672,20 @@ typedef struct {
 #define MBA_DISCARD_RND_FRAME	0x8048	/* discard RND frame due to error. */
 #define MBA_REJECTED_FCP_CMD	0x8049	/* rejected FCP_CMD. */
 
+/* 83XX FCoE specific */
+#define MBA_IDC_AEN		0x8200  /* FCoE: NIC Core state change AEN */
+
+/* Interrupt type codes */
+#define INTR_ROM_MB_SUCCESS		0x1
+#define INTR_ROM_MB_FAILED		0x2
+#define INTR_MB_SUCCESS			0x10
+#define INTR_MB_FAILED			0x11
+#define INTR_ASYNC_EVENT		0x12
+#define INTR_RSP_QUE_UPDATE		0x13
+#define INTR_RSP_QUE_UPDATE_83XX	0x14
+#define INTR_ATIO_QUE_UPDATE		0x1C
+#define INTR_ATIO_RSP_QUE_UPDATE	0x1D
+
 /* ISP mailbox loopback echo diagnostic error code */
 #define MBS_LB_RESET	0x17
 /*
@@ -718,6 +810,7 @@ typedef struct {
 #define MBC_SEND_RNFT_ELS		0x5e	/* Send RNFT ELS request */
 #define MBC_GET_LINK_PRIV_STATS		0x6d	/* Get link & private data. */
 #define MBC_SET_VENDOR_ID		0x76	/* Set Vendor ID. */
+#define MBC_PORT_RESET			0x120	/* Port Reset */
 #define MBC_SET_PORT_CONFIG		0x122	/* Set port configuration */
 #define MBC_GET_PORT_CONFIG		0x123	/* Get port configuration */
 
@@ -1375,9 +1468,10 @@ typedef struct {
 } cont_a64_entry_t;
 
 #define PO_MODE_DIF_INSERT	0
-#define PO_MODE_DIF_REMOVE	BIT_0
-#define PO_MODE_DIF_PASS	BIT_1
-#define PO_MODE_DIF_REPLACE	(BIT_0 + BIT_1)
+#define PO_MODE_DIF_REMOVE	1
+#define PO_MODE_DIF_PASS	2
+#define PO_MODE_DIF_REPLACE	3
+#define PO_MODE_DIF_TCP_CKSUM	6
 #define PO_ENABLE_DIF_BUNDLING	BIT_8
 #define PO_ENABLE_INCR_GUARD_SEED	BIT_3
 #define PO_DISABLE_INCR_REF_TAG	BIT_5
@@ -1509,6 +1603,13 @@ typedef struct {
 #define CS_RETRY		0x82	/* Driver defined */
 #define CS_LOOP_DOWN_ABORT	0x83	/* Driver defined */
 
+#define CS_BIDIR_RD_OVERRUN			0x700
+#define CS_BIDIR_RD_WR_OVERRUN			0x707
+#define CS_BIDIR_RD_OVERRUN_WR_UNDERRUN		0x715
+#define CS_BIDIR_RD_UNDERRUN			0x1500
+#define CS_BIDIR_RD_UNDERRUN_WR_OVERRUN		0x1507
+#define CS_BIDIR_RD_WR_UNDERRUN			0x1515
+#define CS_BIDIR_DMA				0x200
 /*
  * Status entry status flags
  */
@@ -2373,6 +2474,11 @@ struct qla_statistics {
 	uint64_t output_bytes;
 };
 
+struct bidi_statistics {
+	unsigned long long io_count;
+	unsigned long long transfer_bytes;
+};
+
 /* Multi queue support */
 #define MBC_INITIALIZE_MULTIQ 0x1f
 #define QLA_QUE_PAGE 0X1000
@@ -2509,14 +2615,16 @@ struct qla_hw_data {
 		uint32_t	disable_msix_handshake	:1;
 		uint32_t	fcp_prio_enabled	:1;
 		uint32_t	isp82xx_fw_hung:1;
+		uint32_t	nic_core_hung:1;
 
 		uint32_t	quiesce_owner:1;
 		uint32_t	thermal_supported:1;
-		uint32_t	isp82xx_reset_hdlr_active:1;
-		uint32_t	isp82xx_reset_owner:1;
+		uint32_t	nic_core_reset_hdlr_active:1;
+		uint32_t	nic_core_reset_owner:1;
 		uint32_t	isp82xx_no_md_cap:1;
 		uint32_t	host_shutting_down:1;
-		/* 30 bits */
+		uint32_t	idc_compl_status:1;
+		/* 32 bits */
 	} flags;
 
 	/* This spinlock is used to protect "io transactions", you must
@@ -2670,6 +2778,16 @@ struct qla_hw_data {
 #define HAS_EXTENDED_IDS(ha)    ((ha)->device_type & DT_EXTENDED_IDS)
 #define IS_CT6_SUPPORTED(ha)	((ha)->device_type & DT_CT6_SUPPORTED)
 #define IS_MQUE_CAPABLE(ha)	((ha)->mqenable || IS_QLA83XX(ha))
+#define IS_BIDI_CAPABLE(ha)	((IS_QLA25XX(ha) || IS_QLA2031(ha)))
+/* Bit 21 of fw_attributes decides the MCTP capabilities */
+#define IS_MCTP_CAPABLE(ha)	(IS_QLA2031(ha) && \
+				((ha)->fw_attributes_ext[0] & BIT_0))
+#define IS_PI_UNINIT_CAPABLE(ha)	(IS_QLA83XX(ha))
+#define IS_PI_IPGUARD_CAPABLE(ha)	(IS_QLA83XX(ha))
+#define IS_PI_DIFB_DIX0_CAPABLE(ha)	(0)
+#define IS_PI_SPLIT_DET_CAPABLE_HBA(ha)	(IS_QLA83XX(ha))
+#define IS_PI_SPLIT_DET_CAPABLE(ha)	(IS_PI_SPLIT_DET_CAPABLE_HBA(ha) && \
+    (((ha)->fw_attributes_h << 16 | (ha)->fw_attributes) & BIT_22))
 
 	/* HBA serial number */
 	uint8_t		serial0;
@@ -2753,6 +2871,7 @@ struct qla_hw_data {
 	struct completion mbx_intr_comp;  /* Used for completion notification */
 	struct completion dcbx_comp;	/* For set port config notification */
 	int notify_dcbx_comp;
+	struct mutex selflogin_lock;
 
 	/* Basic firmware related information. */
 	uint16_t	fw_major_version;
@@ -2784,7 +2903,12 @@ struct qla_hw_data {
 	int		fw_dump_reading;
 	dma_addr_t	eft_dma;
 	void		*eft;
-
+/* Current size of mctp dump is 0x086064 bytes */
+#define MCTP_DUMP_SIZE  0x086064
+	dma_addr_t	mctp_dump_dma;
+	void		*mctp_dump;
+	int		mctp_dumped;
+	int		mctp_dump_reading;
 	uint32_t	chain_offset;
 	struct dentry *dfs_dir;
 	struct dentry *dfs_fce;
@@ -2896,8 +3020,8 @@ struct qla_hw_data {
 	unsigned long	mn_win_crb;
 	unsigned long	ms_win_crb;
 	int		qdr_sn_window;
-	uint32_t	nx_dev_init_timeout;
-	uint32_t	nx_reset_timeout;
+	uint32_t	fcoe_dev_init_timeout;
+	uint32_t	fcoe_reset_timeout;
 	rwlock_t	hw_lock;
 	uint16_t	portnum;		/* port number */
 	int		link_width;
@@ -2918,6 +3042,20 @@ struct qla_hw_data {
 	void            *md_dump;
 	uint32_t	md_dump_size;
 
+	void		*loop_id_map;
+
+	/* QLA83XX IDC specific fields */
+	uint32_t	idc_audit_ts;
+
+	/* DPC low-priority workqueue */
+	struct workqueue_struct *dpc_lp_wq;
+	struct work_struct idc_aen;
+	/* DPC high-priority workqueue */
+	struct workqueue_struct *dpc_hp_wq;
+	struct work_struct nic_core_reset;
+	struct work_struct idc_state_handler;
+	struct work_struct nic_core_unrecoverable;
+
 	struct qlt_hw_data tgt;
 };
 
@@ -2985,6 +3123,13 @@ typedef struct scsi_qla_host {
 
 	/* ISP configuration data. */
 	uint16_t	loop_id;		/* Host adapter loop id */
+	uint16_t        self_login_loop_id;     /* host adapter loop id
+						 * get it on self login
+						 */
+	fc_port_t       bidir_fcport;		/* fcport used for bidir cmnds
+						 * no need of allocating it for
+						 * each command
+						 */
 
 	port_id_t	d_id;			/* Host adapter port id */
 	uint8_t		marker_needed;
@@ -3038,6 +3183,7 @@ typedef struct scsi_qla_host {
 	int		seconds_since_last_heartbeat;
 	struct fc_host_statistics fc_host_stat;
 	struct qla_statistics qla_stats;
+	struct bidi_statistics bidi_stats;
 
 	atomic_t	vref_count;
 } scsi_qla_host_t;
diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c
index 499c74e39ee..706c4f7bc7c 100644
--- a/drivers/scsi/qla2xxx/qla_dfs.c
+++ b/drivers/scsi/qla2xxx/qla_dfs.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h
index 6d7d7758c79..59524aa0ab3 100644
--- a/drivers/scsi/qla2xxx/qla_fw.h
+++ b/drivers/scsi/qla2xxx/qla_fw.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -381,6 +381,44 @@ struct init_cb_24xx {
 /*
  * ISP queue - command entry structure definition.
  */
+#define COMMAND_BIDIRECTIONAL 0x75
+struct cmd_bidir {
+	uint8_t entry_type;		/* Entry type. */
+	uint8_t entry_count;		/* Entry count. */
+	uint8_t sys_define;		/* System defined */
+	uint8_t entry_status;		/* Entry status. */
+
+	uint32_t handle;		/* System handle. */
+
+	uint16_t nport_handle;		/* N_PORT hanlde. */
+
+	uint16_t timeout;		/* Commnad timeout. */
+
+	uint16_t wr_dseg_count;		/* Write Data segment count. */
+	uint16_t rd_dseg_count;		/* Read Data segment count. */
+
+	struct scsi_lun lun;		/* FCP LUN (BE). */
+
+	uint16_t control_flags;		/* Control flags. */
+#define BD_WRAP_BACK			BIT_3
+#define BD_READ_DATA			BIT_1
+#define BD_WRITE_DATA			BIT_0
+
+	uint16_t fcp_cmnd_dseg_len;		/* Data segment length. */
+	uint32_t fcp_cmnd_dseg_address[2];	/* Data segment address. */
+
+	uint16_t reserved[2];			/* Reserved */
+
+	uint32_t rd_byte_count;			/* Total Byte count Read. */
+	uint32_t wr_byte_count;			/* Total Byte count write. */
+
+	uint8_t port_id[3];			/* PortID of destination port.*/
+	uint8_t vp_index;
+
+	uint32_t fcp_data_dseg_address[2];	/* Data segment address. */
+	uint16_t fcp_data_dseg_len;		/* Data segment length. */
+};
+
 #define COMMAND_TYPE_6	0x48		/* Command Type 6 entry */
 struct cmd_type_6 {
 	uint8_t entry_type;		/* Entry type. */
@@ -1130,7 +1168,7 @@ struct mid_db_entry_24xx {
 /*
  * Virtual Port Control IOCB
  */
-#define VP_CTRL_IOCB_TYPE	0x30	/* Vitual Port Control entry. */
+#define VP_CTRL_IOCB_TYPE	0x30	/* Virtual Port Control entry. */
 struct vp_ctrl_entry_24xx {
 	uint8_t entry_type;		/* Entry type. */
 	uint8_t entry_count;		/* Entry count. */
@@ -1166,7 +1204,7 @@ struct vp_ctrl_entry_24xx {
 /*
  * Modify Virtual Port Configuration IOCB
  */
-#define VP_CONFIG_IOCB_TYPE	0x31	/* Vitual Port Config entry. */
+#define VP_CONFIG_IOCB_TYPE	0x31	/* Virtual Port Config entry. */
 struct vp_config_entry_24xx {
 	uint8_t entry_type;		/* Entry type. */
 	uint8_t entry_count;		/* Entry count. */
@@ -1502,7 +1540,10 @@ struct access_chip_rsp_84xx {
 /*
  * ISP83xx mailbox commands
  */
-#define MBC_WRITE_REMOTE_REG 0x0001 /* Write remote register */
+#define MBC_WRITE_REMOTE_REG		0x0001 /* Write remote register */
+#define MBC_READ_REMOTE_REG		0x0009 /* Read remote register */
+#define MBC_RESTART_NIC_FIRMWARE	0x003d /* Restart NIC firmware */
+#define MBC_SET_ACCESS_CONTROL		0x003e /* Access control command */
 
 /* Flash access control option field bit definitions */
 #define FAC_OPT_FORCE_SEMAPHORE		BIT_15
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 9eacd2df111..6acb39785a4 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -48,7 +48,7 @@ extern void qla2x00_update_fcports(scsi_qla_host_t *);
 
 extern int qla2x00_abort_isp(scsi_qla_host_t *);
 extern void qla2x00_abort_isp_cleanup(scsi_qla_host_t *);
-extern void qla82xx_quiescent_state_cleanup(scsi_qla_host_t *);
+extern void qla2x00_quiesce_io(scsi_qla_host_t *);
 
 extern void qla2x00_update_fcport(scsi_qla_host_t *, fc_port_t *);
 
@@ -76,6 +76,14 @@ extern int qla24xx_update_fcport_fcp_prio(scsi_qla_host_t *, fc_port_t *);
 
 extern fc_port_t *
 qla2x00_alloc_fcport(scsi_qla_host_t *, gfp_t );
+
+extern int __qla83xx_set_idc_control(scsi_qla_host_t *, uint32_t);
+extern int __qla83xx_get_idc_control(scsi_qla_host_t *, uint32_t *);
+extern void qla83xx_idc_audit(scsi_qla_host_t *, int);
+extern int qla83xx_nic_core_reset(scsi_qla_host_t *);
+extern void qla83xx_reset_ownership(scsi_qla_host_t *);
+extern int qla2xxx_mctp_dump(scsi_qla_host_t *);
+
 /*
  * Global Data in qla_os.c source file.
  */
@@ -133,6 +141,20 @@ extern void qla2x00_relogin(struct scsi_qla_host *);
 extern void qla2x00_do_work(struct scsi_qla_host *);
 extern void qla2x00_free_fcports(struct scsi_qla_host *);
 
+extern void qla83xx_schedule_work(scsi_qla_host_t *, int);
+extern void qla83xx_service_idc_aen(struct work_struct *);
+extern void qla83xx_nic_core_unrecoverable_work(struct work_struct *);
+extern void qla83xx_idc_state_handler_work(struct work_struct *);
+extern void qla83xx_nic_core_reset_work(struct work_struct *);
+
+extern void qla83xx_idc_lock(scsi_qla_host_t *, uint16_t);
+extern void qla83xx_idc_unlock(scsi_qla_host_t *, uint16_t);
+extern int qla83xx_idc_state_handler(scsi_qla_host_t *);
+extern int qla83xx_set_drv_presence(scsi_qla_host_t *vha);
+extern int __qla83xx_set_drv_presence(scsi_qla_host_t *vha);
+extern int qla83xx_clear_drv_presence(scsi_qla_host_t *vha);
+extern int __qla83xx_clear_drv_presence(scsi_qla_host_t *vha);
+
 /*
  * Global Functions in qla_mid.c source file.
  */
@@ -188,6 +210,8 @@ extern int qla2x00_start_sp(srb_t *);
 extern uint16_t qla24xx_calc_iocbs(scsi_qla_host_t *, uint16_t);
 extern void qla24xx_build_scsi_iocbs(srb_t *, struct cmd_type_7 *, uint16_t);
 extern int qla24xx_dif_start_scsi(srb_t *);
+extern int qla2x00_start_bidir(srb_t *, struct scsi_qla_host *, uint32_t);
+extern unsigned long qla2x00_get_async_timeout(struct scsi_qla_host *);
 
 extern void *qla2x00_alloc_iocbs(scsi_qla_host_t *, srb_t *);
 extern int qla2x00_issue_marker(scsi_qla_host_t *, int);
@@ -376,6 +400,9 @@ qla81xx_set_port_config(scsi_qla_host_t *, uint16_t *);
 extern int
 qla2x00_port_logout(scsi_qla_host_t *, struct fc_port *);
 
+extern int
+qla2x00_dump_mctp_data(scsi_qla_host_t *, dma_addr_t, uint32_t, uint32_t);
+
 /*
  * Global Function Prototypes in qla_isr.c source file.
  */
@@ -419,7 +446,11 @@ extern void qla24xx_beacon_blink(struct scsi_qla_host *);
 extern void qla83xx_beacon_blink(struct scsi_qla_host *);
 extern int qla82xx_beacon_on(struct scsi_qla_host *);
 extern int qla82xx_beacon_off(struct scsi_qla_host *);
-extern int qla83xx_write_remote_reg(struct scsi_qla_host *, uint32_t, uint32_t);
+extern int qla83xx_wr_reg(scsi_qla_host_t *, uint32_t, uint32_t);
+extern int qla83xx_rd_reg(scsi_qla_host_t *, uint32_t, uint32_t *);
+extern int qla83xx_restart_nic_firmware(scsi_qla_host_t *);
+extern int qla83xx_access_control(scsi_qla_host_t *, uint16_t, uint32_t,
+    uint32_t, uint16_t *);
 
 extern uint8_t *qla2x00_read_optrom_data(struct scsi_qla_host *, uint8_t *,
     uint32_t, uint32_t);
@@ -527,7 +558,6 @@ extern void qla24xx_wrt_rsp_reg(struct qla_hw_data *, uint16_t, uint16_t);
 /* PCI related functions */
 extern int qla82xx_pci_config(struct scsi_qla_host *);
 extern int qla82xx_pci_mem_read_2M(struct qla_hw_data *, u64, void *, int);
-extern char *qla82xx_pci_info_str(struct scsi_qla_host *, char *);
 extern int qla82xx_pci_region_offset(struct pci_dev *, int);
 extern int qla82xx_iospace_config(struct qla_hw_data *);
 
@@ -580,6 +610,7 @@ extern uint32_t  qla82xx_wait_for_state_change(scsi_qla_host_t *, uint32_t);
 extern int qla82xx_idc_lock(struct qla_hw_data *);
 extern void qla82xx_idc_unlock(struct qla_hw_data *);
 extern int qla82xx_device_state_handler(scsi_qla_host_t *);
+extern void qla8xxx_dev_failed_handler(scsi_qla_host_t *);
 extern void qla82xx_clear_qsnt_ready(scsi_qla_host_t *);
 
 extern void qla2x00_set_model_info(scsi_qla_host_t *, uint8_t *,
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index 05260d25fe4..f4e4bd7c3f4 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -1131,7 +1131,7 @@ qla2x00_mgmt_svr_login(scsi_qla_host_t *vha)
 		return ret;
 
 	rval = ha->isp_ops->fabric_login(vha, vha->mgmt_svr_loop_id, 0xff, 0xff,
-	    0xfa, mb, BIT_1|BIT_0);
+	    0xfa, mb, BIT_1);
 	if (rval != QLA_SUCCESS || mb[0] != MBS_COMMAND_COMPLETE) {
 		if (rval == QLA_MEMORY_ALLOC_FAILED)
 			ql_dbg(ql_dbg_disc, vha, 0x2085,
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index a44653b4216..799a58bb985 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -77,7 +77,7 @@ qla2x00_sp_free(void *data, void *ptr)
 
 /* Asynchronous Login/Logout Routines -------------------------------------- */
 
-static inline unsigned long
+unsigned long
 qla2x00_get_async_timeout(struct scsi_qla_host *vha)
 {
 	unsigned long tmo;
@@ -429,6 +429,79 @@ qla2x00_async_adisc_done(struct scsi_qla_host *vha, fc_port_t *fcport,
 /*                QLogic ISP2x00 Hardware Support Functions.                */
 /****************************************************************************/
 
+int
+qla83xx_nic_core_fw_load(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t idc_major_ver, idc_minor_ver;
+	uint16_t config[4];
+
+	qla83xx_idc_lock(vha, 0);
+
+	/* SV: TODO: Assign initialization timeout from
+	 * flash-info / other param
+	 */
+	ha->fcoe_dev_init_timeout = QLA83XX_IDC_INITIALIZATION_TIMEOUT;
+	ha->fcoe_reset_timeout = QLA83XX_IDC_RESET_ACK_TIMEOUT;
+
+	/* Set our fcoe function presence */
+	if (__qla83xx_set_drv_presence(vha) != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_p3p, vha, 0xb077,
+		    "Error while setting DRV-Presence.\n");
+		rval = QLA_FUNCTION_FAILED;
+		goto exit;
+	}
+
+	/* Decide the reset ownership */
+	qla83xx_reset_ownership(vha);
+
+	/*
+	 * On first protocol driver load:
+	 * Init-Owner: Set IDC-Major-Version and Clear IDC-Lock-Recovery
+	 * register.
+	 * Others: Check compatibility with current IDC Major version.
+	 */
+	qla83xx_rd_reg(vha, QLA83XX_IDC_MAJOR_VERSION, &idc_major_ver);
+	if (ha->flags.nic_core_reset_owner) {
+		/* Set IDC Major version */
+		idc_major_ver = QLA83XX_SUPP_IDC_MAJOR_VERSION;
+		qla83xx_wr_reg(vha, QLA83XX_IDC_MAJOR_VERSION, idc_major_ver);
+
+		/* Clearing IDC-Lock-Recovery register */
+		qla83xx_wr_reg(vha, QLA83XX_IDC_LOCK_RECOVERY, 0);
+	} else if (idc_major_ver != QLA83XX_SUPP_IDC_MAJOR_VERSION) {
+		/*
+		 * Clear further IDC participation if we are not compatible with
+		 * the current IDC Major Version.
+		 */
+		ql_log(ql_log_warn, vha, 0xb07d,
+		    "Failing load, idc_major_ver=%d, expected_major_ver=%d.\n",
+		    idc_major_ver, QLA83XX_SUPP_IDC_MAJOR_VERSION);
+		__qla83xx_clear_drv_presence(vha);
+		rval = QLA_FUNCTION_FAILED;
+		goto exit;
+	}
+	/* Each function sets its supported Minor version. */
+	qla83xx_rd_reg(vha, QLA83XX_IDC_MINOR_VERSION, &idc_minor_ver);
+	idc_minor_ver |= (QLA83XX_SUPP_IDC_MINOR_VERSION << (ha->portnum * 2));
+	qla83xx_wr_reg(vha, QLA83XX_IDC_MINOR_VERSION, idc_minor_ver);
+
+	if (ha->flags.nic_core_reset_owner) {
+		memset(config, 0, sizeof(config));
+		if (!qla81xx_get_port_config(vha, config))
+			qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE,
+			    QLA8XXX_DEV_READY);
+	}
+
+	rval = qla83xx_idc_state_handler(vha);
+
+exit:
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
 /*
 * qla2x00_initialize_adapter
 *      Initialize board.
@@ -537,6 +610,14 @@ qla2x00_initialize_adapter(scsi_qla_host_t *vha)
 		}
 	}
 
+	/* Load the NIC Core f/w if we are the first protocol driver. */
+	if (IS_QLA8031(ha)) {
+		rval = qla83xx_nic_core_fw_load(vha);
+		if (rval)
+			ql_log(ql_log_warn, vha, 0x0124,
+			    "Error in initializing NIC Core f/w.\n");
+	}
+
 	if (IS_QLA24XX_TYPE(ha) || IS_QLA25XX(ha))
 		qla24xx_read_fcp_prio_cfg(vha);
 
@@ -686,7 +767,7 @@ qla24xx_pci_config(scsi_qla_host_t *vha)
 
 	/* PCIe -- adjust Maximum Read Request Size (2048). */
 	if (pci_is_pcie(ha->pdev))
-		pcie_set_readrq(ha->pdev, 2048);
+		pcie_set_readrq(ha->pdev, 4096);
 
 	pci_disable_rom(ha->pdev);
 
@@ -722,7 +803,7 @@ qla25xx_pci_config(scsi_qla_host_t *vha)
 
 	/* PCIe -- adjust Maximum Read Request Size (2048). */
 	if (pci_is_pcie(ha->pdev))
-		pcie_set_readrq(ha->pdev, 2048);
+		pcie_set_readrq(ha->pdev, 4096);
 
 	pci_disable_rom(ha->pdev);
 
@@ -1480,7 +1561,8 @@ enable_82xx_npiv:
 			    "ISP Firmware failed checksum.\n");
 			goto failed;
 		}
-	}
+	} else
+		goto failed;
 
 	if (!IS_FWI2_CAPABLE(ha) && !IS_QLA2100(ha) && !IS_QLA2200(ha)) {
 		/* Enable proper parity. */
@@ -1825,7 +1907,7 @@ qla2x00_init_rings(scsi_qla_host_t *vha)
 	ql_dbg(ql_dbg_init, vha, 0x00d1, "Issue init firmware.\n");
 
 	if (ha->flags.npiv_supported) {
-		if (ha->operating_mode == LOOP)
+		if (ha->operating_mode == LOOP && !IS_CNA_CAPABLE(ha))
 			ha->max_npiv_vports = MIN_MULTI_ID_FABRIC - 1;
 		mid_init_cb->count = cpu_to_le16(ha->max_npiv_vports);
 	}
@@ -2682,11 +2764,6 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
 	new_fcport = NULL;
 	entries = MAX_FIBRE_DEVICES_LOOP;
 
-	ql_dbg(ql_dbg_disc, vha, 0x2016,
-	    "Getting FCAL position map.\n");
-	if (ql2xextended_error_logging & ql_dbg_disc)
-		qla2x00_get_fcal_position_map(vha, NULL);
-
 	/* Get list of logged in devices. */
 	memset(ha->gid_list, 0, qla2x00_gid_list_size(ha));
 	rval = qla2x00_get_id_list(vha, ha->gid_list, ha->gid_list_dma,
@@ -2753,6 +2830,8 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
 		if (loop_id > LAST_LOCAL_LOOP_ID)
 			continue;
 
+		memset(new_fcport, 0, sizeof(fc_port_t));
+
 		/* Fill in member data. */
 		new_fcport->d_id.b.domain = domain;
 		new_fcport->d_id.b.area = area;
@@ -3285,7 +3364,7 @@ qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha,
 			 */
 			if ((fcport->flags & FCF_FABRIC_DEVICE) == 0) {
 				fcport->d_id.b24 = new_fcport->d_id.b24;
-				fcport->loop_id = FC_NO_LOOP_ID;
+				qla2x00_clear_loop_id(fcport);
 				fcport->flags |= (FCF_FABRIC_DEVICE |
 				    FCF_LOGIN_NEEDED);
 				break;
@@ -3306,7 +3385,7 @@ qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha,
 				ha->isp_ops->fabric_logout(vha, fcport->loop_id,
 				    fcport->d_id.b.domain, fcport->d_id.b.area,
 				    fcport->d_id.b.al_pa);
-				fcport->loop_id = FC_NO_LOOP_ID;
+				qla2x00_clear_loop_id(fcport);
 			}
 
 			break;
@@ -3352,71 +3431,32 @@ int
 qla2x00_find_new_loop_id(scsi_qla_host_t *vha, fc_port_t *dev)
 {
 	int	rval;
-	int	found;
-	fc_port_t *fcport;
-	uint16_t first_loop_id;
 	struct qla_hw_data *ha = vha->hw;
-	struct scsi_qla_host *vp;
-	struct scsi_qla_host *tvp;
 	unsigned long flags = 0;
 
 	rval = QLA_SUCCESS;
 
-	/* Save starting loop ID. */
-	first_loop_id = dev->loop_id;
-
-	for (;;) {
-		/* Skip loop ID if already used by adapter. */
-		if (dev->loop_id == vha->loop_id)
-			dev->loop_id++;
-
-		/* Skip reserved loop IDs. */
-		while (qla2x00_is_reserved_id(vha, dev->loop_id))
-			dev->loop_id++;
-
-		/* Reset loop ID if passed the end. */
-		if (dev->loop_id > ha->max_loop_id) {
-			/* first loop ID. */
-			dev->loop_id = ha->min_external_loopid;
-		}
-
-		/* Check for loop ID being already in use. */
-		found = 0;
-		fcport = NULL;
-
-		spin_lock_irqsave(&ha->vport_slock, flags);
-		list_for_each_entry_safe(vp, tvp, &ha->vp_list, list) {
-			list_for_each_entry(fcport, &vp->vp_fcports, list) {
-				if (fcport->loop_id == dev->loop_id &&
-								fcport != dev) {
-					/* ID possibly in use */
-					found++;
-					break;
-				}
-			}
-			if (found)
-				break;
-		}
-		spin_unlock_irqrestore(&ha->vport_slock, flags);
+	spin_lock_irqsave(&ha->vport_slock, flags);
 
-		/* If not in use then it is free to use. */
-		if (!found) {
-			ql_dbg(ql_dbg_disc, dev->vha, 0x2086,
-			    "Assigning new loopid=%x, portid=%x.\n",
-			    dev->loop_id, dev->d_id.b24);
-			break;
-		}
+	dev->loop_id = find_first_zero_bit(ha->loop_id_map,
+	    LOOPID_MAP_SIZE);
+	if (dev->loop_id >= LOOPID_MAP_SIZE ||
+	    qla2x00_is_reserved_id(vha, dev->loop_id)) {
+		dev->loop_id = FC_NO_LOOP_ID;
+		rval = QLA_FUNCTION_FAILED;
+	} else
+		set_bit(dev->loop_id, ha->loop_id_map);
 
-		/* ID in use. Try next value. */
-		dev->loop_id++;
+	spin_unlock_irqrestore(&ha->vport_slock, flags);
 
-		/* If wrap around. No free ID to use. */
-		if (dev->loop_id == first_loop_id) {
-			dev->loop_id = FC_NO_LOOP_ID;
-			rval = QLA_FUNCTION_FAILED;
-			break;
-		}
-	}
+	if (rval == QLA_SUCCESS)
+		ql_dbg(ql_dbg_disc, dev->vha, 0x2086,
+		    "Assigning new loopid=%x, portid=%x.\n",
+		    dev->loop_id, dev->d_id.b24);
+	else
+		ql_log(ql_log_warn, dev->vha, 0x2087,
+		    "No loop_id's available, portid=%x.\n",
+		    dev->d_id.b24);
 
 	return (rval);
 }
@@ -3616,7 +3656,7 @@ qla2x00_fabric_login(scsi_qla_host_t *vha, fc_port_t *fcport,
 			ha->isp_ops->fabric_logout(vha, fcport->loop_id,
 			    fcport->d_id.b.domain, fcport->d_id.b.area,
 			    fcport->d_id.b.al_pa);
-			fcport->loop_id = FC_NO_LOOP_ID;
+			qla2x00_clear_loop_id(fcport);
 			fcport->login_retry = 0;
 
 			rval = 3;
@@ -3775,8 +3815,363 @@ qla2x00_update_fcports(scsi_qla_host_t *base_vha)
 	spin_unlock_irqrestore(&ha->vport_slock, flags);
 }
 
+/* Assumes idc_lock always held on entry */
+void
+qla83xx_reset_ownership(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_presence, drv_presence_mask;
+	uint32_t dev_part_info1, dev_part_info2, class_type;
+	uint32_t class_type_mask = 0x3;
+	uint16_t fcoe_other_function = 0xffff, i;
+
+	qla83xx_rd_reg(vha, QLA83XX_IDC_DRV_PRESENCE, &drv_presence);
+
+	qla83xx_rd_reg(vha, QLA83XX_DEV_PARTINFO1, &dev_part_info1);
+	qla83xx_rd_reg(vha, QLA83XX_DEV_PARTINFO2, &dev_part_info2);
+	for (i = 0; i < 8; i++) {
+		class_type = ((dev_part_info1 >> (i * 4)) & class_type_mask);
+		if ((class_type == QLA83XX_CLASS_TYPE_FCOE) &&
+		    (i != ha->portnum)) {
+			fcoe_other_function = i;
+			break;
+		}
+	}
+	if (fcoe_other_function == 0xffff) {
+		for (i = 0; i < 8; i++) {
+			class_type = ((dev_part_info2 >> (i * 4)) &
+			    class_type_mask);
+			if ((class_type == QLA83XX_CLASS_TYPE_FCOE) &&
+			    ((i + 8) != ha->portnum)) {
+				fcoe_other_function = i + 8;
+				break;
+			}
+		}
+	}
+	/*
+	 * Prepare drv-presence mask based on fcoe functions present.
+	 * However consider only valid physical fcoe function numbers (0-15).
+	 */
+	drv_presence_mask = ~((1 << (ha->portnum)) |
+			((fcoe_other_function == 0xffff) ?
+			 0 : (1 << (fcoe_other_function))));
+
+	/* We are the reset owner iff:
+	 *    - No other protocol drivers present.
+	 *    - This is the lowest among fcoe functions. */
+	if (!(drv_presence & drv_presence_mask) &&
+			(ha->portnum < fcoe_other_function)) {
+		ql_dbg(ql_dbg_p3p, vha, 0xb07f,
+		    "This host is Reset owner.\n");
+		ha->flags.nic_core_reset_owner = 1;
+	}
+}
+
+int
+__qla83xx_set_drv_ack(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_ack;
+
+	rval = qla83xx_rd_reg(vha, QLA83XX_IDC_DRIVER_ACK, &drv_ack);
+	if (rval == QLA_SUCCESS) {
+		drv_ack |= (1 << ha->portnum);
+		rval = qla83xx_wr_reg(vha, QLA83XX_IDC_DRIVER_ACK, drv_ack);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_set_drv_ack(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_set_drv_ack(vha);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+int
+__qla83xx_clear_drv_ack(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_ack;
+
+	rval = qla83xx_rd_reg(vha, QLA83XX_IDC_DRIVER_ACK, &drv_ack);
+	if (rval == QLA_SUCCESS) {
+		drv_ack &= ~(1 << ha->portnum);
+		rval = qla83xx_wr_reg(vha, QLA83XX_IDC_DRIVER_ACK, drv_ack);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_clear_drv_ack(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_clear_drv_ack(vha);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+const char *
+qla83xx_dev_state_to_string(uint32_t dev_state)
+{
+	switch (dev_state) {
+	case QLA8XXX_DEV_COLD:
+		return "COLD/RE-INIT";
+	case QLA8XXX_DEV_INITIALIZING:
+		return "INITIALIZING";
+	case QLA8XXX_DEV_READY:
+		return "READY";
+	case QLA8XXX_DEV_NEED_RESET:
+		return "NEED RESET";
+	case QLA8XXX_DEV_NEED_QUIESCENT:
+		return "NEED QUIESCENT";
+	case QLA8XXX_DEV_FAILED:
+		return "FAILED";
+	case QLA8XXX_DEV_QUIESCENT:
+		return "QUIESCENT";
+	default:
+		return "Unknown";
+	}
+}
+
+/* Assumes idc-lock always held on entry */
+void
+qla83xx_idc_audit(scsi_qla_host_t *vha, int audit_type)
+{
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t idc_audit_reg = 0, duration_secs = 0;
+
+	switch (audit_type) {
+	case IDC_AUDIT_TIMESTAMP:
+		ha->idc_audit_ts = (jiffies_to_msecs(jiffies) / 1000);
+		idc_audit_reg = (ha->portnum) |
+		    (IDC_AUDIT_TIMESTAMP << 7) | (ha->idc_audit_ts << 8);
+		qla83xx_wr_reg(vha, QLA83XX_IDC_AUDIT, idc_audit_reg);
+		break;
+
+	case IDC_AUDIT_COMPLETION:
+		duration_secs = ((jiffies_to_msecs(jiffies) -
+		    jiffies_to_msecs(ha->idc_audit_ts)) / 1000);
+		idc_audit_reg = (ha->portnum) |
+		    (IDC_AUDIT_COMPLETION << 7) | (duration_secs << 8);
+		qla83xx_wr_reg(vha, QLA83XX_IDC_AUDIT, idc_audit_reg);
+		break;
+
+	default:
+		ql_log(ql_log_warn, vha, 0xb078,
+		    "Invalid audit type specified.\n");
+		break;
+	}
+}
+
+/* Assumes idc_lock always held on entry */
+int
+qla83xx_initiating_reset(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t  idc_control, dev_state;
+
+	__qla83xx_get_idc_control(vha, &idc_control);
+	if ((idc_control & QLA83XX_IDC_RESET_DISABLED)) {
+		ql_log(ql_log_info, vha, 0xb080,
+		    "NIC Core reset has been disabled. idc-control=0x%x\n",
+		    idc_control);
+		return QLA_FUNCTION_FAILED;
+	}
+
+	/* Set NEED-RESET iff in READY state and we are the reset-owner */
+	qla83xx_rd_reg(vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+	if (ha->flags.nic_core_reset_owner && dev_state == QLA8XXX_DEV_READY) {
+		qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE,
+		    QLA8XXX_DEV_NEED_RESET);
+		ql_log(ql_log_info, vha, 0xb056, "HW State: NEED RESET.\n");
+		qla83xx_idc_audit(vha, IDC_AUDIT_TIMESTAMP);
+	} else {
+		const char *state = qla83xx_dev_state_to_string(dev_state);
+		ql_log(ql_log_info, vha, 0xb057, "HW State: %s.\n", state);
+
+		/* SV: XXX: Is timeout required here? */
+		/* Wait for IDC state change READY -> NEED_RESET */
+		while (dev_state == QLA8XXX_DEV_READY) {
+			qla83xx_idc_unlock(vha, 0);
+			msleep(200);
+			qla83xx_idc_lock(vha, 0);
+			qla83xx_rd_reg(vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+		}
+	}
+
+	/* Send IDC ack by writing to drv-ack register */
+	__qla83xx_set_drv_ack(vha);
+
+	return QLA_SUCCESS;
+}
+
+int
+__qla83xx_set_idc_control(scsi_qla_host_t *vha, uint32_t idc_control)
+{
+	return qla83xx_wr_reg(vha, QLA83XX_IDC_CONTROL, idc_control);
+}
+
+int
+qla83xx_set_idc_control(scsi_qla_host_t *vha, uint32_t idc_control)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_set_idc_control(vha, idc_control);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+int
+__qla83xx_get_idc_control(scsi_qla_host_t *vha, uint32_t *idc_control)
+{
+	return qla83xx_rd_reg(vha, QLA83XX_IDC_CONTROL, idc_control);
+}
+
+int
+qla83xx_get_idc_control(scsi_qla_host_t *vha, uint32_t *idc_control)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_get_idc_control(vha, idc_control);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+int
+qla83xx_check_driver_presence(scsi_qla_host_t *vha)
+{
+	uint32_t drv_presence = 0;
+	struct qla_hw_data *ha = vha->hw;
+
+	qla83xx_rd_reg(vha, QLA83XX_IDC_DRV_PRESENCE, &drv_presence);
+	if (drv_presence & (1 << ha->portnum))
+		return QLA_SUCCESS;
+	else
+		return QLA_TEST_FAILED;
+}
+
+int
+qla83xx_nic_core_reset(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+
+	ql_dbg(ql_dbg_p3p, vha, 0xb058,
+	    "Entered  %s().\n", __func__);
+
+	if (vha->device_flags & DFLG_DEV_FAILED) {
+		ql_log(ql_log_warn, vha, 0xb059,
+		    "Device in unrecoverable FAILED state.\n");
+		return QLA_FUNCTION_FAILED;
+	}
+
+	qla83xx_idc_lock(vha, 0);
+
+	if (qla83xx_check_driver_presence(vha) != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xb05a,
+		    "Function=0x%x has been removed from IDC participation.\n",
+		    ha->portnum);
+		rval = QLA_FUNCTION_FAILED;
+		goto exit;
+	}
+
+	qla83xx_reset_ownership(vha);
+
+	rval = qla83xx_initiating_reset(vha);
+
+	/*
+	 * Perform reset if we are the reset-owner,
+	 * else wait till IDC state changes to READY/FAILED.
+	 */
+	if (rval == QLA_SUCCESS) {
+		rval = qla83xx_idc_state_handler(vha);
+
+		if (rval == QLA_SUCCESS)
+			ha->flags.nic_core_hung = 0;
+		__qla83xx_clear_drv_ack(vha);
+	}
+
+exit:
+	qla83xx_idc_unlock(vha, 0);
+
+	ql_dbg(ql_dbg_p3p, vha, 0xb05b, "Exiting %s.\n", __func__);
+
+	return rval;
+}
+
+int
+qla2xxx_mctp_dump(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	int rval = QLA_FUNCTION_FAILED;
+
+	if (!IS_MCTP_CAPABLE(ha)) {
+		/* This message can be removed from the final version */
+		ql_log(ql_log_info, vha, 0x506d,
+		    "This board is not MCTP capable\n");
+		return rval;
+	}
+
+	if (!ha->mctp_dump) {
+		ha->mctp_dump = dma_alloc_coherent(&ha->pdev->dev,
+		    MCTP_DUMP_SIZE, &ha->mctp_dump_dma, GFP_KERNEL);
+
+		if (!ha->mctp_dump) {
+			ql_log(ql_log_warn, vha, 0x506e,
+			    "Failed to allocate memory for mctp dump\n");
+			return rval;
+		}
+	}
+
+#define MCTP_DUMP_STR_ADDR	0x00000000
+	rval = qla2x00_dump_mctp_data(vha, ha->mctp_dump_dma,
+	    MCTP_DUMP_STR_ADDR, MCTP_DUMP_SIZE/4);
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0x506f,
+		    "Failed to capture mctp dump\n");
+	} else {
+		ql_log(ql_log_info, vha, 0x5070,
+		    "Mctp dump capture for host (%ld/%p).\n",
+		    vha->host_no, ha->mctp_dump);
+		ha->mctp_dumped = 1;
+	}
+
+	if (!ha->flags.nic_core_reset_hdlr_active && !ha->portnum) {
+		ha->flags.nic_core_reset_hdlr_active = 1;
+		rval = qla83xx_restart_nic_firmware(vha);
+		if (rval)
+			/* NIC Core reset failed. */
+			ql_log(ql_log_warn, vha, 0x5071,
+			    "Failed to restart nic firmware\n");
+		else
+			ql_dbg(ql_dbg_p3p, vha, 0xb084,
+			    "Restarted NIC firmware successfully.\n");
+		ha->flags.nic_core_reset_hdlr_active = 0;
+	}
+
+	return rval;
+
+}
+
 /*
-* qla82xx_quiescent_state_cleanup
+* qla2x00_quiesce_io
 * Description: This function will block the new I/Os
 *              Its not aborting any I/Os as context
 *              is not destroyed during quiescence
@@ -3784,20 +4179,20 @@ qla2x00_update_fcports(scsi_qla_host_t *base_vha)
 * return   : void
 */
 void
-qla82xx_quiescent_state_cleanup(scsi_qla_host_t *vha)
+qla2x00_quiesce_io(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
 	struct scsi_qla_host *vp;
 
-	ql_dbg(ql_dbg_p3p, vha, 0xb002,
-	    "Performing ISP error recovery - ha=%p.\n", ha);
+	ql_dbg(ql_dbg_dpc, vha, 0x401d,
+	    "Quiescing I/O - ha=%p.\n", ha);
 
 	atomic_set(&ha->loop_down_timer, LOOP_DOWN_TIME);
 	if (atomic_read(&vha->loop_state) != LOOP_DOWN) {
 		atomic_set(&vha->loop_state, LOOP_DOWN);
 		qla2x00_mark_all_devices_lost(vha, 0);
 		list_for_each_entry(vp, &ha->vp_list, list)
-			qla2x00_mark_all_devices_lost(vha, 0);
+			qla2x00_mark_all_devices_lost(vp, 0);
 	} else {
 		if (!atomic_read(&vha->loop_down_timer))
 			atomic_set(&vha->loop_down_timer,
@@ -3913,6 +4308,14 @@ qla2x00_abort_isp(scsi_qla_host_t *vha)
 	if (vha->flags.online) {
 		qla2x00_abort_isp_cleanup(vha);
 
+		if (IS_QLA8031(ha)) {
+			ql_dbg(ql_dbg_p3p, vha, 0xb05c,
+			    "Clearing fcoe driver presence.\n");
+			if (qla83xx_clear_drv_presence(vha) != QLA_SUCCESS)
+				ql_dbg(ql_dbg_p3p, vha, 0xb073,
+				    "Error while clearing DRV-Presence.\n");
+		}
+
 		if (unlikely(pci_channel_offline(ha->pdev) &&
 		    ha->flags.pci_channel_io_perm_failure)) {
 			clear_bit(ISP_ABORT_RETRY, &vha->dpc_flags);
@@ -4021,6 +4424,13 @@ qla2x00_abort_isp(scsi_qla_host_t *vha)
 		}
 		spin_unlock_irqrestore(&ha->vport_slock, flags);
 
+		if (IS_QLA8031(ha)) {
+			ql_dbg(ql_dbg_p3p, vha, 0xb05d,
+			    "Setting back fcoe driver presence.\n");
+			if (qla83xx_set_drv_presence(vha) != QLA_SUCCESS)
+				ql_dbg(ql_dbg_p3p, vha, 0xb074,
+				    "Error while setting DRV-Presence.\n");
+		}
 	} else {
 		ql_log(ql_log_warn, vha, 0x8023, "%s **** FAILED ****.\n",
 		       __func__);
@@ -5088,6 +5498,9 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 		rval = 1;
 	}
 
+	if (IS_T10_PI_CAPABLE(ha))
+		nv->frame_payload_size &= ~7;
+
 	/* Reset Initialization control block */
 	memset(icb, 0, ha->init_cb_size);
 
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index 6e457643c63..c0462c04c88 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -57,6 +57,20 @@ host_to_fcp_swap(uint8_t *fcp, uint32_t bsize)
        return fcp;
 }
 
+static inline void
+qla2x00_set_reserved_loop_ids(struct qla_hw_data *ha)
+{
+	int i;
+
+	if (IS_FWI2_CAPABLE(ha))
+		return;
+
+	for (i = 0; i < SNS_FIRST_LOOP_ID; i++)
+		set_bit(i, ha->loop_id_map);
+	set_bit(MANAGEMENT_SERVER, ha->loop_id_map);
+	set_bit(BROADCAST, ha->loop_id_map);
+}
+
 static inline int
 qla2x00_is_reserved_id(scsi_qla_host_t *vha, uint16_t loop_id)
 {
@@ -69,6 +83,18 @@ qla2x00_is_reserved_id(scsi_qla_host_t *vha, uint16_t loop_id)
 }
 
 static inline void
+qla2x00_clear_loop_id(fc_port_t *fcport) {
+	struct qla_hw_data *ha = fcport->vha->hw;
+
+	if (fcport->loop_id == FC_NO_LOOP_ID ||
+	    qla2x00_is_reserved_id(fcport->vha, fcport->loop_id))
+		return;
+
+	clear_bit(fcport->loop_id, ha->loop_id_map);
+	fcport->loop_id = FC_NO_LOOP_ID;
+}
+
+static inline void
 qla2x00_clean_dsd_pool(struct qla_hw_data *ha, srb_t *sp)
 {
 	struct dsd_dma *dsd_ptr, *tdsd_ptr;
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index 70dbf53d9e0..03b75263283 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -147,13 +147,6 @@ qla24xx_configure_prot_mode(srb_t *sp, uint16_t *fw_prot_opts)
 	struct scsi_cmnd *cmd = GET_CMD_SP(sp);
 	uint8_t	guard = scsi_host_get_guard(cmd->device->host);
 
-	/* We only support T10 DIF right now */
-	if (guard != SHOST_DIX_GUARD_CRC) {
-		ql_dbg(ql_dbg_io, sp->fcport->vha, 0x3007,
-		    "Unsupported guard: %d for cmd=%p.\n", guard, cmd);
-		return 0;
-	}
-
 	/* We always use DIFF Bundling for best performance */
 	*fw_prot_opts = 0;
 
@@ -172,10 +165,11 @@ qla24xx_configure_prot_mode(srb_t *sp, uint16_t *fw_prot_opts)
 		*fw_prot_opts |= PO_MODE_DIF_REMOVE;
 		break;
 	case SCSI_PROT_READ_PASS:
-		*fw_prot_opts |= PO_MODE_DIF_PASS;
-		break;
 	case SCSI_PROT_WRITE_PASS:
-		*fw_prot_opts |= PO_MODE_DIF_PASS;
+		if (guard & SHOST_DIX_GUARD_IP)
+			*fw_prot_opts |= PO_MODE_DIF_TCP_CKSUM;
+		else
+			*fw_prot_opts |= PO_MODE_DIF_PASS;
 		break;
 	default:	/* Normal Request */
 		*fw_prot_opts |= PO_MODE_DIF_PASS;
@@ -821,7 +815,6 @@ qla24xx_set_t10dif_tags(srb_t *sp, struct fw_dif_context *pkt,
     unsigned int protcnt)
 {
 	struct scsi_cmnd *cmd = GET_CMD_SP(sp);
-	scsi_qla_host_t *vha = shost_priv(cmd->device->host);
 
 	switch (scsi_get_prot_type(cmd)) {
 	case SCSI_PROT_DIF_TYPE0:
@@ -891,12 +884,6 @@ qla24xx_set_t10dif_tags(srb_t *sp, struct fw_dif_context *pkt,
 		pkt->ref_tag_mask[3] = 0xff;
 		break;
 	}
-
-	ql_dbg(ql_dbg_io, vha, 0x3009,
-	    "Setting protection Tags: (BIG) ref tag = 0x%x, app tag = 0x%x, "
-	    "prot SG count %d, cmd lba 0x%x, prot_type=%u cmd=%p.\n",
-	    pkt->ref_tag, pkt->app_tag, protcnt, (int)scsi_get_lba(cmd),
-	    scsi_get_prot_type(cmd), cmd);
 }
 
 struct qla2_sgx {
@@ -1068,9 +1055,6 @@ qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd,
 	int	i;
 	uint16_t	used_dsds = tot_dsds;
 	struct scsi_cmnd *cmd = GET_CMD_SP(sp);
-	scsi_qla_host_t *vha = shost_priv(cmd->device->host);
-
-	uint8_t		*cp;
 
 	scsi_for_each_sg(cmd, sg, tot_dsds, i) {
 		dma_addr_t	sle_dma;
@@ -1113,19 +1097,12 @@ qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd,
 			cur_dsd = (uint32_t *)next_dsd;
 		}
 		sle_dma = sg_dma_address(sg);
-		ql_dbg(ql_dbg_io, vha, 0x300a,
-		    "sg entry %d - addr=0x%x 0x%x, " "len=%d for cmd=%p.\n",
-		    i, LSD(sle_dma), MSD(sle_dma), sg_dma_len(sg), cmd);
+
 		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
 		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
 		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
 		avail_dsds--;
 
-		if (scsi_get_prot_op(cmd) == SCSI_PROT_WRITE_PASS) {
-			cp = page_address(sg_page(sg)) + sg->offset;
-			ql_dbg(ql_dbg_io, vha, 0x300b,
-			    "User data buffer=%p for cmd=%p.\n", cp, cmd);
-		}
 	}
 	/* Null termination */
 	*cur_dsd++ = 0;
@@ -1148,8 +1125,6 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 	struct scsi_cmnd *cmd;
 	uint32_t *cur_dsd = dsd;
 	uint16_t	used_dsds = tot_dsds;
-	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
-	uint8_t		*cp;
 
 	cmd = GET_CMD_SP(sp);
 	scsi_for_each_prot_sg(cmd, sg, tot_dsds, i) {
@@ -1193,23 +1168,11 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 			cur_dsd = (uint32_t *)next_dsd;
 		}
 		sle_dma = sg_dma_address(sg);
-		if (scsi_get_prot_op(cmd) == SCSI_PROT_WRITE_PASS) {
-			ql_dbg(ql_dbg_io, vha, 0x3027,
-			    "%s(): %p, sg_entry %d - "
-			    "addr=0x%x0x%x, len=%d.\n",
-			    __func__, cur_dsd, i,
-			    LSD(sle_dma), MSD(sle_dma), sg_dma_len(sg));
-		}
+
 		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
 		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
 		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
 
-		if (scsi_get_prot_op(cmd) == SCSI_PROT_WRITE_PASS) {
-			cp = page_address(sg_page(sg)) + sg->offset;
-			ql_dbg(ql_dbg_io, vha, 0x3028,
-			    "%s(): Protection Data buffer = %p.\n", __func__,
-			    cp);
-		}
 		avail_dsds--;
 	}
 	/* Null termination */
@@ -1386,6 +1349,16 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 
 	if (!qla2x00_hba_err_chk_enabled(sp))
 		fw_prot_opts |= 0x10; /* Disable Guard tag checking */
+	/* HBA error checking enabled */
+	else if (IS_PI_UNINIT_CAPABLE(ha)) {
+		if ((scsi_get_prot_type(GET_CMD_SP(sp)) == SCSI_PROT_DIF_TYPE1)
+		    || (scsi_get_prot_type(GET_CMD_SP(sp)) ==
+			SCSI_PROT_DIF_TYPE2))
+			fw_prot_opts |= BIT_10;
+		else if (scsi_get_prot_type(GET_CMD_SP(sp)) ==
+		    SCSI_PROT_DIF_TYPE3)
+			fw_prot_opts |= BIT_11;
+	}
 
 	if (!bundling) {
 		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.nobundling.data_address;
@@ -1858,7 +1831,7 @@ qla2x00_alloc_iocbs(scsi_qla_host_t *vha, srb_t *sp)
 	}
 	if (index == MAX_OUTSTANDING_COMMANDS) {
 		ql_log(ql_log_warn, vha, 0x700b,
-		    "No room on oustanding cmd array.\n");
+		    "No room on outstanding cmd array.\n");
 		goto queuing_error;
 	}
 
@@ -2665,3 +2638,201 @@ done:
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	return rval;
 }
+
+static void
+qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
+				struct cmd_bidir *cmd_pkt, uint32_t tot_dsds)
+{
+	uint16_t avail_dsds;
+	uint32_t *cur_dsd;
+	uint32_t req_data_len = 0;
+	uint32_t rsp_data_len = 0;
+	struct scatterlist *sg;
+	int index;
+	int entry_count = 1;
+	struct fc_bsg_job *bsg_job = sp->u.bsg_job;
+
+	/*Update entry type to indicate bidir command */
+	*((uint32_t *)(&cmd_pkt->entry_type)) =
+		__constant_cpu_to_le32(COMMAND_BIDIRECTIONAL);
+
+	/* Set the transfer direction, in this set both flags
+	 * Also set the BD_WRAP_BACK flag, firmware will take care
+	 * assigning DID=SID for outgoing pkts.
+	 */
+	cmd_pkt->wr_dseg_count = cpu_to_le16(bsg_job->request_payload.sg_cnt);
+	cmd_pkt->rd_dseg_count = cpu_to_le16(bsg_job->reply_payload.sg_cnt);
+	cmd_pkt->control_flags =
+			__constant_cpu_to_le16(BD_WRITE_DATA | BD_READ_DATA |
+							BD_WRAP_BACK);
+
+	req_data_len = rsp_data_len = bsg_job->request_payload.payload_len;
+	cmd_pkt->wr_byte_count = cpu_to_le32(req_data_len);
+	cmd_pkt->rd_byte_count = cpu_to_le32(rsp_data_len);
+	cmd_pkt->timeout = cpu_to_le16(qla2x00_get_async_timeout(vha) + 2);
+
+	vha->bidi_stats.transfer_bytes += req_data_len;
+	vha->bidi_stats.io_count++;
+
+	/* Only one dsd is available for bidirectional IOCB, remaining dsds
+	 * are bundled in continuation iocb
+	 */
+	avail_dsds = 1;
+	cur_dsd = (uint32_t *)&cmd_pkt->fcp_data_dseg_address;
+
+	index = 0;
+
+	for_each_sg(bsg_job->request_payload.sg_list, sg,
+				bsg_job->request_payload.sg_cnt, index) {
+		dma_addr_t sle_dma;
+		cont_a64_entry_t *cont_pkt;
+
+		/* Allocate additional continuation packets */
+		if (avail_dsds == 0) {
+			/* Continuation type 1 IOCB can accomodate
+			 * 5 DSDS
+			 */
+			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, vha->req);
+			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			avail_dsds = 5;
+			entry_count++;
+		}
+		sle_dma = sg_dma_address(sg);
+		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		avail_dsds--;
+	}
+	/* For read request DSD will always goes to continuation IOCB
+	 * and follow the write DSD. If there is room on the current IOCB
+	 * then it is added to that IOCB else new continuation IOCB is
+	 * allocated.
+	 */
+	for_each_sg(bsg_job->reply_payload.sg_list, sg,
+				bsg_job->reply_payload.sg_cnt, index) {
+		dma_addr_t sle_dma;
+		cont_a64_entry_t *cont_pkt;
+
+		/* Allocate additional continuation packets */
+		if (avail_dsds == 0) {
+			/* Continuation type 1 IOCB can accomodate
+			 * 5 DSDS
+			 */
+			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, vha->req);
+			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			avail_dsds = 5;
+			entry_count++;
+		}
+		sle_dma = sg_dma_address(sg);
+		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
+		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		avail_dsds--;
+	}
+	/* This value should be same as number of IOCB required for this cmd */
+	cmd_pkt->entry_count = entry_count;
+}
+
+int
+qla2x00_start_bidir(srb_t *sp, struct scsi_qla_host *vha, uint32_t tot_dsds)
+{
+
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long flags;
+	uint32_t handle;
+	uint32_t index;
+	uint16_t req_cnt;
+	uint16_t cnt;
+	uint32_t *clr_ptr;
+	struct cmd_bidir *cmd_pkt = NULL;
+	struct rsp_que *rsp;
+	struct req_que *req;
+	int rval = EXT_STATUS_OK;
+	device_reg_t __iomem *reg = ISP_QUE_REG(ha, vha->req->id);
+
+	rval = QLA_SUCCESS;
+
+	rsp = ha->rsp_q_map[0];
+	req = vha->req;
+
+	/* Send marker if required */
+	if (vha->marker_needed != 0) {
+		if (qla2x00_marker(vha, req,
+			rsp, 0, 0, MK_SYNC_ALL) != QLA_SUCCESS)
+			return EXT_STATUS_MAILBOX;
+		vha->marker_needed = 0;
+	}
+
+	/* Acquire ring specific lock */
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+
+	/* Check for room in outstanding command list. */
+	handle = req->current_outstanding_cmd;
+	for (index = 1; index < MAX_OUTSTANDING_COMMANDS; index++) {
+		handle++;
+	if (handle == MAX_OUTSTANDING_COMMANDS)
+		handle = 1;
+	if (!req->outstanding_cmds[handle])
+		break;
+	}
+
+	if (index == MAX_OUTSTANDING_COMMANDS) {
+		rval = EXT_STATUS_BUSY;
+		goto queuing_error;
+	}
+
+	/* Calculate number of IOCB required */
+	req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
+
+	/* Check for room on request queue. */
+	if (req->cnt < req_cnt + 2) {
+		if (ha->mqenable)
+			cnt = RD_REG_DWORD(&reg->isp25mq.req_q_out);
+		else if (IS_QLA82XX(ha))
+			cnt = RD_REG_DWORD(&reg->isp82.req_q_out);
+		else if (IS_FWI2_CAPABLE(ha))
+			cnt = RD_REG_DWORD(&reg->isp24.req_q_out);
+		else
+			cnt = qla2x00_debounce_register(
+					ISP_REQ_Q_OUT(ha, &reg->isp));
+
+		if  (req->ring_index < cnt)
+			req->cnt = cnt - req->ring_index;
+		else
+			req->cnt = req->length -
+				(req->ring_index - cnt);
+	}
+	if (req->cnt < req_cnt + 2) {
+		rval = EXT_STATUS_BUSY;
+		goto queuing_error;
+	}
+
+	cmd_pkt = (struct cmd_bidir *)req->ring_ptr;
+	cmd_pkt->handle = MAKE_HANDLE(req->id, handle);
+
+	/* Zero out remaining portion of packet. */
+	/* tagged queuing modifier -- default is TSK_SIMPLE (0).*/
+	clr_ptr = (uint32_t *)cmd_pkt + 2;
+	memset(clr_ptr, 0, REQUEST_ENTRY_SIZE - 8);
+
+	/* Set NPORT-ID  (of vha)*/
+	cmd_pkt->nport_handle = cpu_to_le16(vha->self_login_loop_id);
+	cmd_pkt->port_id[0] = vha->d_id.b.al_pa;
+	cmd_pkt->port_id[1] = vha->d_id.b.area;
+	cmd_pkt->port_id[2] = vha->d_id.b.domain;
+
+	qla25xx_build_bidir_iocb(sp, vha, cmd_pkt, tot_dsds);
+	cmd_pkt->entry_status = (uint8_t) rsp->id;
+	/* Build command packet. */
+	req->current_outstanding_cmd = handle;
+	req->outstanding_cmds[handle] = sp;
+	sp->handle = handle;
+	req->cnt -= req_cnt;
+
+	/* Send the command to the firmware */
+	wmb();
+	qla2x00_start_iocbs(vha, req);
+queuing_error:
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 6f67a9d4998..5733811ce8e 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -294,6 +294,11 @@ qla81xx_idc_event(scsi_qla_host_t *vha, uint16_t aen, uint16_t descr)
 	    "%04x %04x %04x %04x %04x %04x %04x.\n",
 	    event[aen & 0xff], mb[0], mb[1], mb[2], mb[3],
 	    mb[4], mb[5], mb[6]);
+	if ((aen == MBA_IDC_COMPLETE && mb[1] >> 15)) {
+		vha->hw->flags.idc_compl_status = 1;
+		if (vha->hw->notify_dcbx_comp)
+			complete(&vha->hw->dcbx_comp);
+	}
 
 	/* Acknowledgement needed? [Notify && non-zero timeout]. */
 	timeout = (descr >> 8) & 0xf;
@@ -332,6 +337,166 @@ qla2x00_get_link_speed_str(struct qla_hw_data *ha)
 	return link_speed;
 }
 
+void
+qla83xx_handle_8200_aen(scsi_qla_host_t *vha, uint16_t *mb)
+{
+	struct qla_hw_data *ha = vha->hw;
+
+	/*
+	 * 8200 AEN Interpretation:
+	 * mb[0] = AEN code
+	 * mb[1] = AEN Reason code
+	 * mb[2] = LSW of Peg-Halt Status-1 Register
+	 * mb[6] = MSW of Peg-Halt Status-1 Register
+	 * mb[3] = LSW of Peg-Halt Status-2 register
+	 * mb[7] = MSW of Peg-Halt Status-2 register
+	 * mb[4] = IDC Device-State Register value
+	 * mb[5] = IDC Driver-Presence Register value
+	 */
+	ql_dbg(ql_dbg_async, vha, 0x506b, "AEN Code: mb[0] = 0x%x AEN reason: "
+	    "mb[1] = 0x%x PH-status1: mb[2] = 0x%x PH-status1: mb[6] = 0x%x.\n",
+	    mb[0], mb[1], mb[2], mb[6]);
+	ql_dbg(ql_dbg_async, vha, 0x506c, "PH-status2: mb[3] = 0x%x "
+	    "PH-status2: mb[7] = 0x%x Device-State: mb[4] = 0x%x "
+	    "Drv-Presence: mb[5] = 0x%x.\n", mb[3], mb[7], mb[4], mb[5]);
+
+	if (mb[1] & (IDC_PEG_HALT_STATUS_CHANGE | IDC_NIC_FW_REPORTED_FAILURE |
+				IDC_HEARTBEAT_FAILURE)) {
+		ha->flags.nic_core_hung = 1;
+		ql_log(ql_log_warn, vha, 0x5060,
+		    "83XX: F/W Error Reported: Check if reset required.\n");
+
+		if (mb[1] & IDC_PEG_HALT_STATUS_CHANGE) {
+			uint32_t protocol_engine_id, fw_err_code, err_level;
+
+			/*
+			 * IDC_PEG_HALT_STATUS_CHANGE interpretation:
+			 *  - PEG-Halt Status-1 Register:
+			 *	(LSW = mb[2], MSW = mb[6])
+			 *	Bits 0-7   = protocol-engine ID
+			 *	Bits 8-28  = f/w error code
+			 *	Bits 29-31 = Error-level
+			 *	    Error-level 0x1 = Non-Fatal error
+			 *	    Error-level 0x2 = Recoverable Fatal error
+			 *	    Error-level 0x4 = UnRecoverable Fatal error
+			 *  - PEG-Halt Status-2 Register:
+			 *	(LSW = mb[3], MSW = mb[7])
+			 */
+			protocol_engine_id = (mb[2] & 0xff);
+			fw_err_code = (((mb[2] & 0xff00) >> 8) |
+			    ((mb[6] & 0x1fff) << 8));
+			err_level = ((mb[6] & 0xe000) >> 13);
+			ql_log(ql_log_warn, vha, 0x5061, "PegHalt Status-1 "
+			    "Register: protocol_engine_id=0x%x "
+			    "fw_err_code=0x%x err_level=0x%x.\n",
+			    protocol_engine_id, fw_err_code, err_level);
+			ql_log(ql_log_warn, vha, 0x5062, "PegHalt Status-2 "
+			    "Register: 0x%x%x.\n", mb[7], mb[3]);
+			if (err_level == ERR_LEVEL_NON_FATAL) {
+				ql_log(ql_log_warn, vha, 0x5063,
+				    "Not a fatal error, f/w has recovered "
+				    "iteself.\n");
+			} else if (err_level == ERR_LEVEL_RECOVERABLE_FATAL) {
+				ql_log(ql_log_fatal, vha, 0x5064,
+				    "Recoverable Fatal error: Chip reset "
+				    "required.\n");
+				qla83xx_schedule_work(vha,
+				    QLA83XX_NIC_CORE_RESET);
+			} else if (err_level == ERR_LEVEL_UNRECOVERABLE_FATAL) {
+				ql_log(ql_log_fatal, vha, 0x5065,
+				    "Unrecoverable Fatal error: Set FAILED "
+				    "state, reboot required.\n");
+				qla83xx_schedule_work(vha,
+				    QLA83XX_NIC_CORE_UNRECOVERABLE);
+			}
+		}
+
+		if (mb[1] & IDC_NIC_FW_REPORTED_FAILURE) {
+			uint16_t peg_fw_state, nw_interface_link_up;
+			uint16_t nw_interface_signal_detect, sfp_status;
+			uint16_t htbt_counter, htbt_monitor_enable;
+			uint16_t sfp_additonal_info, sfp_multirate;
+			uint16_t sfp_tx_fault, link_speed, dcbx_status;
+
+			/*
+			 * IDC_NIC_FW_REPORTED_FAILURE interpretation:
+			 *  - PEG-to-FC Status Register:
+			 *	(LSW = mb[2], MSW = mb[6])
+			 *	Bits 0-7   = Peg-Firmware state
+			 *	Bit 8      = N/W Interface Link-up
+			 *	Bit 9      = N/W Interface signal detected
+			 *	Bits 10-11 = SFP Status
+			 *	  SFP Status 0x0 = SFP+ transceiver not expected
+			 *	  SFP Status 0x1 = SFP+ transceiver not present
+			 *	  SFP Status 0x2 = SFP+ transceiver invalid
+			 *	  SFP Status 0x3 = SFP+ transceiver present and
+			 *	  valid
+			 *	Bits 12-14 = Heartbeat Counter
+			 *	Bit 15     = Heartbeat Monitor Enable
+			 *	Bits 16-17 = SFP Additional Info
+			 *	  SFP info 0x0 = Unregocnized transceiver for
+			 *	  Ethernet
+			 *	  SFP info 0x1 = SFP+ brand validation failed
+			 *	  SFP info 0x2 = SFP+ speed validation failed
+			 *	  SFP info 0x3 = SFP+ access error
+			 *	Bit 18     = SFP Multirate
+			 *	Bit 19     = SFP Tx Fault
+			 *	Bits 20-22 = Link Speed
+			 *	Bits 23-27 = Reserved
+			 *	Bits 28-30 = DCBX Status
+			 *	  DCBX Status 0x0 = DCBX Disabled
+			 *	  DCBX Status 0x1 = DCBX Enabled
+			 *	  DCBX Status 0x2 = DCBX Exchange error
+			 *	Bit 31     = Reserved
+			 */
+			peg_fw_state = (mb[2] & 0x00ff);
+			nw_interface_link_up = ((mb[2] & 0x0100) >> 8);
+			nw_interface_signal_detect = ((mb[2] & 0x0200) >> 9);
+			sfp_status = ((mb[2] & 0x0c00) >> 10);
+			htbt_counter = ((mb[2] & 0x7000) >> 12);
+			htbt_monitor_enable = ((mb[2] & 0x8000) >> 15);
+			sfp_additonal_info = (mb[6] & 0x0003);
+			sfp_multirate = ((mb[6] & 0x0004) >> 2);
+			sfp_tx_fault = ((mb[6] & 0x0008) >> 3);
+			link_speed = ((mb[6] & 0x0070) >> 4);
+			dcbx_status = ((mb[6] & 0x7000) >> 12);
+
+			ql_log(ql_log_warn, vha, 0x5066,
+			    "Peg-to-Fc Status Register:\n"
+			    "peg_fw_state=0x%x, nw_interface_link_up=0x%x, "
+			    "nw_interface_signal_detect=0x%x"
+			    "\nsfp_statis=0x%x.\n ", peg_fw_state,
+			    nw_interface_link_up, nw_interface_signal_detect,
+			    sfp_status);
+			ql_log(ql_log_warn, vha, 0x5067,
+			    "htbt_counter=0x%x, htbt_monitor_enable=0x%x, "
+			    "sfp_additonal_info=0x%x, sfp_multirate=0x%x.\n ",
+			    htbt_counter, htbt_monitor_enable,
+			    sfp_additonal_info, sfp_multirate);
+			ql_log(ql_log_warn, vha, 0x5068,
+			    "sfp_tx_fault=0x%x, link_state=0x%x, "
+			    "dcbx_status=0x%x.\n", sfp_tx_fault, link_speed,
+			    dcbx_status);
+
+			qla83xx_schedule_work(vha, QLA83XX_NIC_CORE_RESET);
+		}
+
+		if (mb[1] & IDC_HEARTBEAT_FAILURE) {
+			ql_log(ql_log_warn, vha, 0x5069,
+			    "Heartbeat Failure encountered, chip reset "
+			    "required.\n");
+
+			qla83xx_schedule_work(vha, QLA83XX_NIC_CORE_RESET);
+		}
+	}
+
+	if (mb[1] & IDC_DEVICE_STATE_CHANGE) {
+		ql_log(ql_log_info, vha, 0x506a,
+		    "IDC Device-State changed = 0x%x.\n", mb[4]);
+		qla83xx_schedule_work(vha, MBA_IDC_AEN);
+	}
+}
+
 /**
  * qla2x00_async_event() - Process aynchronous events.
  * @ha: SCSI driver HA context
@@ -681,8 +846,7 @@ skip_rio:
 		 * it.  Otherwise ignore it and Wait for RSCN to come in.
 		 */
 		atomic_set(&vha->loop_down_timer, 0);
-		if (atomic_read(&vha->loop_state) != LOOP_DOWN &&
-		    atomic_read(&vha->loop_state) != LOOP_DEAD) {
+		if (mb[1] != 0xffff || (mb[2] != 0x6 && mb[2] != 0x4)) {
 			ql_dbg(ql_dbg_async, vha, 0x5011,
 			    "Asynchronous PORT UPDATE ignored %04x/%04x/%04x.\n",
 			    mb[1], mb[2], mb[3]);
@@ -822,11 +986,28 @@ skip_rio:
 		    "FCF Configuration Error -- %04x %04x %04x.\n",
 		    mb[1], mb[2], mb[3]);
 		break;
-	case MBA_IDC_COMPLETE:
 	case MBA_IDC_NOTIFY:
+		/* See if we need to quiesce any I/O */
+		if (IS_QLA8031(vha->hw))
+			if ((mb[2] & 0x7fff) == MBC_PORT_RESET ||
+			    (mb[2] & 0x7fff) == MBC_SET_PORT_CONFIG) {
+				set_bit(ISP_QUIESCE_NEEDED, &vha->dpc_flags);
+				qla2xxx_wake_dpc(vha);
+			}
+	case MBA_IDC_COMPLETE:
 	case MBA_IDC_TIME_EXT:
-		qla81xx_idc_event(vha, mb[0], mb[1]);
+		if (IS_QLA81XX(vha->hw) || IS_QLA8031(vha->hw))
+			qla81xx_idc_event(vha, mb[0], mb[1]);
 		break;
+
+	case MBA_IDC_AEN:
+		mb[4] = RD_REG_WORD(&reg24->mailbox4);
+		mb[5] = RD_REG_WORD(&reg24->mailbox5);
+		mb[6] = RD_REG_WORD(&reg24->mailbox6);
+		mb[7] = RD_REG_WORD(&reg24->mailbox7);
+		qla83xx_handle_8200_aen(vha, mb);
+		break;
+
 	default:
 		ql_dbg(ql_dbg_async, vha, 0x5057,
 		    "Unknown AEN:%04x %04x %04x %04x\n",
@@ -1414,7 +1595,7 @@ qla2x00_handle_sense(srb_t *sp, uint8_t *sense_data, uint32_t par_sense_len,
 
 struct scsi_dif_tuple {
 	__be16 guard;       /* Checksum */
-	__be16 app_tag;         /* APPL identifer */
+	__be16 app_tag;         /* APPL identifier */
 	__be32 ref_tag;         /* Target LBA or indirect LBA */
 };
 
@@ -1546,6 +1727,149 @@ qla2x00_handle_dif_error(srb_t *sp, struct sts_entry_24xx *sts24)
 	return 1;
 }
 
+static void
+qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
+				  struct req_que *req, uint32_t index)
+{
+	struct qla_hw_data *ha = vha->hw;
+	srb_t *sp;
+	uint16_t	comp_status;
+	uint16_t	scsi_status;
+	uint16_t thread_id;
+	uint32_t rval = EXT_STATUS_OK;
+	struct fc_bsg_job *bsg_job = NULL;
+	sts_entry_t *sts;
+	struct sts_entry_24xx *sts24;
+	sts = (sts_entry_t *) pkt;
+	sts24 = (struct sts_entry_24xx *) pkt;
+
+	/* Validate handle. */
+	if (index >= MAX_OUTSTANDING_COMMANDS) {
+		ql_log(ql_log_warn, vha, 0x70af,
+		    "Invalid SCSI completion handle 0x%x.\n", index);
+		set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+		return;
+	}
+
+	sp = req->outstanding_cmds[index];
+	if (sp) {
+		/* Free outstanding command slot. */
+		req->outstanding_cmds[index] = NULL;
+		bsg_job = sp->u.bsg_job;
+	} else {
+		ql_log(ql_log_warn, vha, 0x70b0,
+		    "Req:%d: Invalid ISP SCSI completion handle(0x%x)\n",
+		    req->id, index);
+
+		set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+		return;
+	}
+
+	if (IS_FWI2_CAPABLE(ha)) {
+		comp_status = le16_to_cpu(sts24->comp_status);
+		scsi_status = le16_to_cpu(sts24->scsi_status) & SS_MASK;
+	} else {
+		comp_status = le16_to_cpu(sts->comp_status);
+		scsi_status = le16_to_cpu(sts->scsi_status) & SS_MASK;
+	}
+
+	thread_id = bsg_job->request->rqst_data.h_vendor.vendor_cmd[1];
+	switch (comp_status) {
+	case CS_COMPLETE:
+		if (scsi_status == 0) {
+			bsg_job->reply->reply_payload_rcv_len =
+					bsg_job->reply_payload.payload_len;
+			rval = EXT_STATUS_OK;
+		}
+		goto done;
+
+	case CS_DATA_OVERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b1,
+		    "Command completed with date overrun thread_id=%d\n",
+		    thread_id);
+		rval = EXT_STATUS_DATA_OVERRUN;
+		break;
+
+	case CS_DATA_UNDERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b2,
+		    "Command completed with date underrun thread_id=%d\n",
+		    thread_id);
+		rval = EXT_STATUS_DATA_UNDERRUN;
+		break;
+	case CS_BIDIR_RD_OVERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b3,
+		    "Command completed with read data overrun thread_id=%d\n",
+		    thread_id);
+		rval = EXT_STATUS_DATA_OVERRUN;
+		break;
+
+	case CS_BIDIR_RD_WR_OVERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b4,
+		    "Command completed with read and write data overrun "
+		    "thread_id=%d\n", thread_id);
+		rval = EXT_STATUS_DATA_OVERRUN;
+		break;
+
+	case CS_BIDIR_RD_OVERRUN_WR_UNDERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b5,
+		    "Command completed with read data over and write data "
+		    "underrun thread_id=%d\n", thread_id);
+		rval = EXT_STATUS_DATA_OVERRUN;
+		break;
+
+	case CS_BIDIR_RD_UNDERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b6,
+		    "Command completed with read data data underrun "
+		    "thread_id=%d\n", thread_id);
+		rval = EXT_STATUS_DATA_UNDERRUN;
+		break;
+
+	case CS_BIDIR_RD_UNDERRUN_WR_OVERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b7,
+		    "Command completed with read data under and write data "
+		    "overrun thread_id=%d\n", thread_id);
+		rval = EXT_STATUS_DATA_UNDERRUN;
+		break;
+
+	case CS_BIDIR_RD_WR_UNDERRUN:
+		ql_dbg(ql_dbg_user, vha, 0x70b8,
+		    "Command completed with read and write data underrun "
+		    "thread_id=%d\n", thread_id);
+		rval = EXT_STATUS_DATA_UNDERRUN;
+		break;
+
+	case CS_BIDIR_DMA:
+		ql_dbg(ql_dbg_user, vha, 0x70b9,
+		    "Command completed with data DMA error thread_id=%d\n",
+		    thread_id);
+		rval = EXT_STATUS_DMA_ERR;
+		break;
+
+	case CS_TIMEOUT:
+		ql_dbg(ql_dbg_user, vha, 0x70ba,
+		    "Command completed with timeout thread_id=%d\n",
+		    thread_id);
+		rval = EXT_STATUS_TIMEOUT;
+		break;
+	default:
+		ql_dbg(ql_dbg_user, vha, 0x70bb,
+		    "Command completed with completion status=0x%x "
+		    "thread_id=%d\n", comp_status, thread_id);
+		rval = EXT_STATUS_ERR;
+		break;
+	}
+		bsg_job->reply->reply_payload_rcv_len = 0;
+
+done:
+	/* Return the vendor specific reply to API */
+	bsg_job->reply->reply_data.vendor_reply.vendor_rsp[0] = rval;
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	/* Always return DID_OK, bsg will send the vendor specific response
+	 * in this case only */
+	sp->done(vha, sp, (DID_OK << 6));
+
+}
+
 /**
  * qla2x00_status_entry() - Process a Status IOCB entry.
  * @ha: SCSI driver HA context
@@ -1573,12 +1897,14 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 	struct req_que *req;
 	int logit = 1;
 	int res = 0;
+	uint16_t state_flags = 0;
 
 	sts = (sts_entry_t *) pkt;
 	sts24 = (struct sts_entry_24xx *) pkt;
 	if (IS_FWI2_CAPABLE(ha)) {
 		comp_status = le16_to_cpu(sts24->comp_status);
 		scsi_status = le16_to_cpu(sts24->scsi_status) & SS_MASK;
+		state_flags = le16_to_cpu(sts24->state_flags);
 	} else {
 		comp_status = le16_to_cpu(sts->comp_status);
 		scsi_status = le16_to_cpu(sts->scsi_status) & SS_MASK;
@@ -1587,17 +1913,9 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 	que = MSW(sts->handle);
 	req = ha->req_q_map[que];
 
-	/* Fast path completion. */
-	if (comp_status == CS_COMPLETE && scsi_status == 0) {
-		qla2x00_process_completed_request(vha, req, handle);
-
-		return;
-	}
-
 	/* Validate handle. */
 	if (handle < MAX_OUTSTANDING_COMMANDS) {
 		sp = req->outstanding_cmds[handle];
-		req->outstanding_cmds[handle] = NULL;
 	} else
 		sp = NULL;
 
@@ -1612,6 +1930,20 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt)
 		qla2xxx_wake_dpc(vha);
 		return;
 	}
+
+	if (unlikely((state_flags & BIT_1) && (sp->type == SRB_BIDI_CMD))) {
+		qla25xx_process_bidir_status_iocb(vha, pkt, req, handle);
+		return;
+	}
+
+	/* Fast path completion. */
+	if (comp_status == CS_COMPLETE && scsi_status == 0) {
+		qla2x00_process_completed_request(vha, req, handle);
+
+		return;
+	}
+
+	req->outstanding_cmds[handle] = NULL;
 	cp = GET_CMD_SP(sp);
 	if (cp == NULL) {
 		ql_dbg(ql_dbg_io, vha, 0x3018,
@@ -1830,7 +2162,21 @@ check_scsi_status:
 
 	case CS_DIF_ERROR:
 		logit = qla2x00_handle_dif_error(sp, sts24);
+		res = cp->result;
 		break;
+
+	case CS_TRANSPORT:
+		res = DID_ERROR << 16;
+
+		if (!IS_PI_SPLIT_DET_CAPABLE(ha))
+			break;
+
+		if (state_flags & BIT_4)
+			scmd_printk(KERN_WARNING, cp,
+			    "Unsupported device '%s' found.\n",
+			    cp->device->vendor);
+		break;
+
 	default:
 		res = DID_ERROR << 16;
 		break;
@@ -2150,7 +2496,7 @@ qla24xx_intr_handler(int irq, void *dev_id)
 	unsigned long	iter;
 	uint32_t	stat;
 	uint32_t	hccr;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 	struct rsp_que *rsp;
 	unsigned long	flags;
 
@@ -2191,29 +2537,29 @@ qla24xx_intr_handler(int irq, void *dev_id)
 			break;
 
 		switch (stat & 0xff) {
-		case 0x1:
-		case 0x2:
-		case 0x10:
-		case 0x11:
+		case INTR_ROM_MB_SUCCESS:
+		case INTR_ROM_MB_FAILED:
+		case INTR_MB_SUCCESS:
+		case INTR_MB_FAILED:
 			qla24xx_mbx_completion(vha, MSW(stat));
 			status |= MBX_INTERRUPT;
 
 			break;
-		case 0x12:
+		case INTR_ASYNC_EVENT:
 			mb[0] = MSW(stat);
 			mb[1] = RD_REG_WORD(&reg->mailbox1);
 			mb[2] = RD_REG_WORD(&reg->mailbox2);
 			mb[3] = RD_REG_WORD(&reg->mailbox3);
 			qla2x00_async_event(vha, rsp, mb);
 			break;
-		case 0x13:
-		case 0x14:
+		case INTR_RSP_QUE_UPDATE:
+		case INTR_RSP_QUE_UPDATE_83XX:
 			qla24xx_process_response_queue(vha, rsp);
 			break;
-		case 0x1C: /* ATIO queue updated */
+		case INTR_ATIO_QUE_UPDATE:
 			qlt_24xx_process_atio_queue(vha);
 			break;
-		case 0x1D: /* ATIO and response queues updated */
+		case INTR_ATIO_RSP_QUE_UPDATE:
 			qlt_24xx_process_atio_queue(vha);
 			qla24xx_process_response_queue(vha, rsp);
 			break;
@@ -2224,6 +2570,8 @@ qla24xx_intr_handler(int irq, void *dev_id)
 		}
 		WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
 		RD_REG_DWORD_RELAXED(&reg->hccr);
+		if (unlikely(IS_QLA83XX(ha) && (ha->pdev->revision == 1)))
+			ndelay(3500);
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -2306,7 +2654,7 @@ qla24xx_msix_default(int irq, void *dev_id)
 	int		status;
 	uint32_t	stat;
 	uint32_t	hccr;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 	unsigned long flags;
 
 	rsp = (struct rsp_que *) dev_id;
@@ -2342,29 +2690,29 @@ qla24xx_msix_default(int irq, void *dev_id)
 			break;
 
 		switch (stat & 0xff) {
-		case 0x1:
-		case 0x2:
-		case 0x10:
-		case 0x11:
+		case INTR_ROM_MB_SUCCESS:
+		case INTR_ROM_MB_FAILED:
+		case INTR_MB_SUCCESS:
+		case INTR_MB_FAILED:
 			qla24xx_mbx_completion(vha, MSW(stat));
 			status |= MBX_INTERRUPT;
 
 			break;
-		case 0x12:
+		case INTR_ASYNC_EVENT:
 			mb[0] = MSW(stat);
 			mb[1] = RD_REG_WORD(&reg->mailbox1);
 			mb[2] = RD_REG_WORD(&reg->mailbox2);
 			mb[3] = RD_REG_WORD(&reg->mailbox3);
 			qla2x00_async_event(vha, rsp, mb);
 			break;
-		case 0x13:
-		case 0x14:
+		case INTR_RSP_QUE_UPDATE:
+		case INTR_RSP_QUE_UPDATE_83XX:
 			qla24xx_process_response_queue(vha, rsp);
 			break;
-		case 0x1C: /* ATIO queue updated */
+		case INTR_ATIO_QUE_UPDATE:
 			qlt_24xx_process_atio_queue(vha);
 			break;
-		case 0x1D: /* ATIO and response queues updated */
+		case INTR_ATIO_RSP_QUE_UPDATE:
 			qlt_24xx_process_atio_queue(vha);
 			qla24xx_process_response_queue(vha, rsp);
 			break;
@@ -2570,7 +2918,7 @@ qla2x00_request_irqs(struct qla_hw_data *ha, struct rsp_que *rsp)
 skip_msix:
 
 	if (!IS_QLA24XX(ha) && !IS_QLA2532(ha) && !IS_QLA8432(ha) &&
-	    !IS_QLA8001(ha))
+	    !IS_QLA8001(ha) && !IS_QLA82XX(ha))
 		goto skip_msi;
 
 	ret = pci_enable_msi(ha->pdev);
@@ -2581,6 +2929,11 @@ skip_msix:
 	} else
 		ql_log(ql_log_warn, vha, 0x0039,
 		    "MSI-X; Falling back-to INTa mode -- %d.\n", ret);
+
+	/* Skip INTx on ISP82xx. */
+	if (!ha->flags.msi_enabled && IS_QLA82XX(ha))
+		return QLA_FUNCTION_FAILED;
+
 skip_msi:
 
 	ret = request_irq(ha->pdev->irq, ha->isp_ops->intr_handler,
@@ -2595,21 +2948,9 @@ skip_msi:
 
 clear_risc_ints:
 
-	/*
-	 * FIXME: Noted that 8014s were being dropped during NK testing.
-	 * Timing deltas during MSI-X/INTa transitions?
-	 */
-	if (IS_QLA81XX(ha) || IS_QLA82XX(ha) || IS_QLA83XX(ha))
-		goto fail;
 	spin_lock_irq(&ha->hardware_lock);
-	if (IS_FWI2_CAPABLE(ha)) {
-		WRT_REG_DWORD(&reg->isp24.hccr, HCCRX_CLR_HOST_INT);
-		WRT_REG_DWORD(&reg->isp24.hccr, HCCRX_CLR_RISC_INT);
-	} else {
+	if (!IS_FWI2_CAPABLE(ha))
 		WRT_REG_WORD(&reg->isp.semaphore, 0);
-		WRT_REG_WORD(&reg->isp.hccr, HCCR_CLR_RISC_INT);
-		WRT_REG_WORD(&reg->isp.hccr, HCCR_CLR_HOST_INT);
-	}
 	spin_unlock_irq(&ha->hardware_lock);
 
 fail:
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index d5ce92c0a8f..18c509fae55 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -75,7 +75,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 		return QLA_FUNCTION_TIMEOUT;
 	}
 
-	if (ha->flags.isp82xx_fw_hung) {
+	if (IS_QLA82XX(ha) && ha->flags.isp82xx_fw_hung) {
 		/* Setting Link-Down error */
 		mcp->mb[0] = MBS_LINK_DOWN_ERROR;
 		ql_log(ql_log_warn, vha, 0x1004,
@@ -232,7 +232,7 @@ qla2x00_mailbox_command(scsi_qla_host_t *vha, mbx_cmd_t *mcp)
 		ha->flags.mbox_int = 0;
 		clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
 
-		if (ha->flags.isp82xx_fw_hung) {
+		if ((IS_QLA82XX(ha) && ha->flags.isp82xx_fw_hung)) {
 			ha->flags.mbox_busy = 0;
 			/* Setting Link-Down error */
 			mcp->mb[0] = MBS_LINK_DOWN_ERROR;
@@ -369,7 +369,7 @@ premature_exit:
 
 mbx_done:
 	if (rval) {
-		ql_dbg(ql_dbg_mbx, base_vha, 0x1020,
+		ql_log(ql_log_warn, base_vha, 0x1020,
 		    "**** Failed mbx[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x, cmd=%x ****.\n",
 		    mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3], command);
 	} else {
@@ -533,7 +533,7 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 	mcp->in_mb = MBX_6|MBX_5|MBX_4|MBX_3|MBX_2|MBX_1|MBX_0;
 	if (IS_QLA81XX(vha->hw) || IS_QLA8031(ha))
 		mcp->in_mb |= MBX_13|MBX_12|MBX_11|MBX_10|MBX_9|MBX_8;
-	if (IS_QLA83XX(vha->hw))
+	if (IS_FWI2_CAPABLE(ha))
 		mcp->in_mb |= MBX_17|MBX_16|MBX_15;
 	mcp->flags = 0;
 	mcp->tov = MBX_TOV_SECONDS;
@@ -559,18 +559,16 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 		ha->phy_version[1] = mcp->mb[9] >> 8;
 		ha->phy_version[2] = mcp->mb[9] & 0xff;
 	}
-	if (IS_QLA83XX(ha)) {
-		if (mcp->mb[6] & BIT_15) {
-			ha->fw_attributes_h = mcp->mb[15];
-			ha->fw_attributes_ext[0] = mcp->mb[16];
-			ha->fw_attributes_ext[1] = mcp->mb[17];
-			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1139,
-			    "%s: FW_attributes Upper: 0x%x, Lower: 0x%x.\n",
-			    __func__, mcp->mb[15], mcp->mb[6]);
-		} else
-			ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x112f,
-			    "%s: FwAttributes [Upper]  invalid, MB6:%04x\n",
-			    __func__, mcp->mb[6]);
+	if (IS_FWI2_CAPABLE(ha)) {
+		ha->fw_attributes_h = mcp->mb[15];
+		ha->fw_attributes_ext[0] = mcp->mb[16];
+		ha->fw_attributes_ext[1] = mcp->mb[17];
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1139,
+		    "%s: FW_attributes Upper: 0x%x, Lower: 0x%x.\n",
+		    __func__, mcp->mb[15], mcp->mb[6]);
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x112f,
+		    "%s: Ext_FwAttributes Upper: 0x%x, Lower: 0x%x.\n",
+		    __func__, mcp->mb[17], mcp->mb[16]);
 	}
 
 failed:
@@ -3408,7 +3406,6 @@ qla2x00_dump_ram(scsi_qla_host_t *vha, dma_addr_t req_dma, uint32_t addr,
 
 	return rval;
 }
-
 /* 84XX Support **************************************************************/
 
 struct cs84xx_mgmt_cmd {
@@ -4428,7 +4425,8 @@ qla2x00_get_thermal_temp(scsi_qla_host_t *vha, uint16_t *temp, uint16_t *frac)
 	    "Entered %s.\n", __func__);
 
 	/* Integer part */
-	rval = qla2x00_read_sfp(vha, 0, &byte, 0x98, 0x01, 1, BIT_13|BIT_0);
+	rval = qla2x00_read_sfp(vha, 0, &byte, 0x98, 0x01, 1,
+		BIT_13|BIT_12|BIT_0);
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x10c9, "Failed=%x.\n", rval);
 		ha->flags.thermal_supported = 0;
@@ -4437,7 +4435,8 @@ qla2x00_get_thermal_temp(scsi_qla_host_t *vha, uint16_t *temp, uint16_t *frac)
 	*temp = byte;
 
 	/* Fraction part */
-	rval = qla2x00_read_sfp(vha, 0, &byte, 0x98, 0x10, 1, BIT_13|BIT_0);
+	rval = qla2x00_read_sfp(vha, 0, &byte, 0x98, 0x10, 1,
+		BIT_13|BIT_12|BIT_0);
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x1019, "Failed=%x.\n", rval);
 		ha->flags.thermal_supported = 0;
@@ -4741,7 +4740,7 @@ qla82xx_mbx_beacon_ctl(scsi_qla_host_t *vha, int enable)
 }
 
 int
-qla83xx_write_remote_reg(scsi_qla_host_t *vha, uint32_t reg, uint32_t data)
+qla83xx_wr_reg(scsi_qla_host_t *vha, uint32_t reg, uint32_t data)
 {
 	int rval;
 	struct qla_hw_data *ha = vha->hw;
@@ -4814,3 +4813,186 @@ qla2x00_port_logout(scsi_qla_host_t *vha, struct fc_port *fcport)
 	return rval;
 }
 
+int
+qla83xx_rd_reg(scsi_qla_host_t *vha, uint32_t reg, uint32_t *data)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long retry_max_time = jiffies + (2 * HZ);
+
+	if (!IS_QLA83XX(ha))
+		return QLA_FUNCTION_FAILED;
+
+	ql_dbg(ql_dbg_mbx, vha, 0x114b, "Entered %s.\n", __func__);
+
+retry_rd_reg:
+	mcp->mb[0] = MBC_READ_REMOTE_REG;
+	mcp->mb[1] = LSW(reg);
+	mcp->mb[2] = MSW(reg);
+	mcp->out_mb = MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_4|MBX_3|MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x114c,
+		    "Failed=%x mb[0]=%x mb[1]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1]);
+	} else {
+		*data = (mcp->mb[3] | (mcp->mb[4] << 16));
+		if (*data == QLA8XXX_BAD_VALUE) {
+			/*
+			 * During soft-reset CAMRAM register reads might
+			 * return 0xbad0bad0. So retry for MAX of 2 sec
+			 * while reading camram registers.
+			 */
+			if (time_after(jiffies, retry_max_time)) {
+				ql_dbg(ql_dbg_mbx, vha, 0x1141,
+				    "Failure to read CAMRAM register. "
+				    "data=0x%x.\n", *data);
+				return QLA_FUNCTION_FAILED;
+			}
+			msleep(100);
+			goto retry_rd_reg;
+		}
+		ql_dbg(ql_dbg_mbx, vha, 0x1142, "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_restart_nic_firmware(scsi_qla_host_t *vha)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA83XX(ha))
+		return QLA_FUNCTION_FAILED;
+
+	ql_dbg(ql_dbg_mbx, vha, 0x1143, "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_RESTART_NIC_FIRMWARE;
+	mcp->out_mb = MBX_0;
+	mcp->in_mb = MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1144,
+		    "Failed=%x mb[0]=%x mb[1]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1]);
+		ha->isp_ops->fw_dump(vha, 0);
+	} else {
+		ql_dbg(ql_dbg_mbx, vha, 0x1145, "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_access_control(scsi_qla_host_t *vha, uint16_t options,
+	uint32_t start_addr, uint32_t end_addr, uint16_t *sector_size)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+	uint8_t subcode = (uint8_t)options;
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA8031(ha))
+		return QLA_FUNCTION_FAILED;
+
+	ql_dbg(ql_dbg_mbx, vha, 0x1146, "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_SET_ACCESS_CONTROL;
+	mcp->mb[1] = options;
+	mcp->out_mb = MBX_1|MBX_0;
+	if (subcode & BIT_2) {
+		mcp->mb[2] = LSW(start_addr);
+		mcp->mb[3] = MSW(start_addr);
+		mcp->mb[4] = LSW(end_addr);
+		mcp->mb[5] = MSW(end_addr);
+		mcp->out_mb |= MBX_5|MBX_4|MBX_3|MBX_2;
+	}
+	mcp->in_mb = MBX_2|MBX_1|MBX_0;
+	if (!(subcode & (BIT_2 | BIT_5)))
+		mcp->in_mb |= MBX_4|MBX_3;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x1147,
+		    "Failed=%x mb[0]=%x mb[1]=%x mb[2]=%x mb[3]=%x mb[4]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3],
+		    mcp->mb[4]);
+		ha->isp_ops->fw_dump(vha, 0);
+	} else {
+		if (subcode & BIT_5)
+			*sector_size = mcp->mb[1];
+		else if (subcode & (BIT_6 | BIT_7)) {
+			ql_dbg(ql_dbg_mbx, vha, 0x1148,
+			    "Driver-lock id=%x%x", mcp->mb[4], mcp->mb[3]);
+		} else if (subcode & (BIT_3 | BIT_4)) {
+			ql_dbg(ql_dbg_mbx, vha, 0x1149,
+			    "Flash-lock id=%x%x", mcp->mb[4], mcp->mb[3]);
+		}
+		ql_dbg(ql_dbg_mbx, vha, 0x114a, "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int
+qla2x00_dump_mctp_data(scsi_qla_host_t *vha, dma_addr_t req_dma, uint32_t addr,
+	uint32_t size)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+
+	if (!IS_MCTP_CAPABLE(vha->hw))
+		return QLA_FUNCTION_FAILED;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x114f,
+	    "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_DUMP_RISC_RAM_EXTENDED;
+	mcp->mb[1] = LSW(addr);
+	mcp->mb[2] = MSW(req_dma);
+	mcp->mb[3] = LSW(req_dma);
+	mcp->mb[4] = MSW(size);
+	mcp->mb[5] = LSW(size);
+	mcp->mb[6] = MSW(MSD(req_dma));
+	mcp->mb[7] = LSW(MSD(req_dma));
+	mcp->mb[8] = MSW(addr);
+	/* Setting RAM ID to valid */
+	mcp->mb[10] |= BIT_7;
+	/* For MCTP RAM ID is 0x40 */
+	mcp->mb[10] |= 0x40;
+
+	mcp->out_mb |= MBX_10|MBX_8|MBX_7|MBX_6|MBX_5|MBX_4|MBX_3|MBX_2|MBX_1|
+	    MBX_0;
+
+	mcp->in_mb = MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x114e,
+		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x114d,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
index 3e8b32419e6..bd4708a422c 100644
--- a/drivers/scsi/qla2xxx/qla_mid.c
+++ b/drivers/scsi/qla2xxx/qla_mid.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -476,7 +476,6 @@ qla24xx_create_vhost(struct fc_vport *fc_vport)
 
 	vha->req = base_vha->req;
 	host->can_queue = base_vha->req->length + 128;
-	host->this_id = 255;
 	host->cmd_per_lun = 3;
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif)
 		host->max_cmd_len = 32;
@@ -643,7 +642,7 @@ qla25xx_create_req_que(struct qla_hw_data *ha, uint16_t options,
 			&req->dma, GFP_KERNEL);
 	if (req->ring == NULL) {
 		ql_log(ql_log_fatal, base_vha, 0x00da,
-		    "Failed to allocte memory for request_ring.\n");
+		    "Failed to allocate memory for request_ring.\n");
 		goto que_failed;
 	}
 
diff --git a/drivers/scsi/qla2xxx/qla_nx.c b/drivers/scsi/qla2xxx/qla_nx.c
index 7cfdf2bd8ed..14cd361742f 100644
--- a/drivers/scsi/qla2xxx/qla_nx.c
+++ b/drivers/scsi/qla2xxx/qla_nx.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -1612,23 +1612,6 @@ qla82xx_get_fw_offs(struct qla_hw_data *ha)
 }
 
 /* PCI related functions */
-char *
-qla82xx_pci_info_str(struct scsi_qla_host *vha, char *str)
-{
-	struct qla_hw_data *ha = vha->hw;
-	char lwstr[6];
-	uint16_t lnk;
-
-	pcie_capability_read_word(ha->pdev, PCI_EXP_LNKSTA, &lnk);
-	ha->link_width = (lnk >> 4) & 0x3f;
-
-	strcpy(str, "PCIe (");
-	strcat(str, "2.5Gb/s ");
-	snprintf(lwstr, sizeof(lwstr), "x%d)", ha->link_width);
-	strcat(str, lwstr);
-	return str;
-}
-
 int qla82xx_pci_region_offset(struct pci_dev *pdev, int region)
 {
 	unsigned long val = 0;
@@ -2320,6 +2303,29 @@ void qla82xx_init_flags(struct qla_hw_data *ha)
 }
 
 inline void
+qla82xx_set_idc_version(scsi_qla_host_t *vha)
+{
+	int idc_ver;
+	uint32_t drv_active;
+	struct qla_hw_data *ha = vha->hw;
+
+	drv_active = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
+	if (drv_active == (QLA82XX_DRV_ACTIVE << (ha->portnum * 4))) {
+		qla82xx_wr_32(ha, QLA82XX_CRB_DRV_IDC_VERSION,
+		    QLA82XX_IDC_VERSION);
+		ql_log(ql_log_info, vha, 0xb082,
+		    "IDC version updated to %d\n", QLA82XX_IDC_VERSION);
+	} else {
+		idc_ver = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_IDC_VERSION);
+		if (idc_ver != QLA82XX_IDC_VERSION)
+			ql_log(ql_log_info, vha, 0xb083,
+			    "qla2xxx driver IDC version %d is not compatible "
+			    "with IDC version %d of the other drivers\n",
+			    QLA82XX_IDC_VERSION, idc_ver);
+	}
+}
+
+inline void
 qla82xx_set_drv_active(scsi_qla_host_t *vha)
 {
 	uint32_t drv_active;
@@ -2353,7 +2359,7 @@ qla82xx_need_reset(struct qla_hw_data *ha)
 	uint32_t drv_state;
 	int rval;
 
-	if (ha->flags.isp82xx_reset_owner)
+	if (ha->flags.nic_core_reset_owner)
 		return 1;
 	else {
 		drv_state = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
@@ -2860,7 +2866,7 @@ qla82xx_device_bootstrap(scsi_qla_host_t *vha)
 		timeout = msleep_interruptible(200);
 		if (timeout) {
 			qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-				QLA82XX_DEV_FAILED);
+				QLA8XXX_DEV_FAILED);
 			return QLA_FUNCTION_FAILED;
 		}
 
@@ -2891,10 +2897,7 @@ dev_initialize:
 	/* set to DEV_INITIALIZING */
 	ql_log(ql_log_info, vha, 0x009e,
 	    "HW State: INITIALIZING.\n");
-	qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_INITIALIZING);
-
-	/* Driver that sets device state to initializating sets IDC version */
-	qla82xx_wr_32(ha, QLA82XX_CRB_DRV_IDC_VERSION, QLA82XX_IDC_VERSION);
+	qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_INITIALIZING);
 
 	qla82xx_idc_unlock(ha);
 	rval = qla82xx_start_firmware(vha);
@@ -2904,14 +2907,14 @@ dev_initialize:
 		ql_log(ql_log_fatal, vha, 0x00ad,
 		    "HW State: FAILED.\n");
 		qla82xx_clear_drv_active(ha);
-		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_FAILED);
+		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_FAILED);
 		return rval;
 	}
 
 dev_ready:
 	ql_log(ql_log_info, vha, 0x00ae,
 	    "HW State: READY.\n");
-	qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_READY);
+	qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_READY);
 
 	return QLA_SUCCESS;
 }
@@ -2935,7 +2938,7 @@ qla82xx_need_qsnt_handler(scsi_qla_host_t *vha)
 
 	if (vha->flags.online) {
 		/*Block any further I/O and wait for pending cmnds to complete*/
-		qla82xx_quiescent_state_cleanup(vha);
+		qla2x00_quiesce_io(vha);
 	}
 
 	/* Set the quiescence ready bit */
@@ -2960,7 +2963,7 @@ qla82xx_need_qsnt_handler(scsi_qla_host_t *vha)
 			    "DRV_STATE:%d.\n", QLA2XXX_DRIVER_NAME,
 			    drv_active, drv_state);
 			qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_READY);
+			    QLA8XXX_DEV_READY);
 			ql_log(ql_log_info, vha, 0xb025,
 			    "HW State: DEV_READY.\n");
 			qla82xx_idc_unlock(ha);
@@ -2981,10 +2984,10 @@ qla82xx_need_qsnt_handler(scsi_qla_host_t *vha)
 	}
 	dev_state = qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 	/* everyone acked so set the state to DEV_QUIESCENCE */
-	if (dev_state == QLA82XX_DEV_NEED_QUIESCENT) {
+	if (dev_state == QLA8XXX_DEV_NEED_QUIESCENT) {
 		ql_log(ql_log_info, vha, 0xb026,
 		    "HW State: DEV_QUIESCENT.\n");
-		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_QUIESCENT);
+		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_QUIESCENT);
 	}
 }
 
@@ -3014,8 +3017,8 @@ qla82xx_wait_for_state_change(scsi_qla_host_t *vha, uint32_t curr_state)
 	return dev_state;
 }
 
-static void
-qla82xx_dev_failed_handler(scsi_qla_host_t *vha)
+void
+qla8xxx_dev_failed_handler(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
 
@@ -3023,9 +3026,10 @@ qla82xx_dev_failed_handler(scsi_qla_host_t *vha)
 	ql_log(ql_log_fatal, vha, 0x00b8,
 	    "Disabling the board.\n");
 
-	qla82xx_idc_lock(ha);
-	qla82xx_clear_drv_active(ha);
-	qla82xx_idc_unlock(ha);
+	if (IS_QLA82XX(ha)) {
+		qla82xx_clear_drv_active(ha);
+		qla82xx_idc_unlock(ha);
+	}
 
 	/* Set DEV_FAILED flag to disable timer */
 	vha->device_flags |= DFLG_DEV_FAILED;
@@ -3064,7 +3068,7 @@ qla82xx_need_reset_handler(scsi_qla_host_t *vha)
 	}
 
 	drv_active = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
-	if (!ha->flags.isp82xx_reset_owner) {
+	if (!ha->flags.nic_core_reset_owner) {
 		ql_dbg(ql_dbg_p3p, vha, 0xb028,
 		    "reset_acknowledged by 0x%x\n", ha->portnum);
 		qla82xx_set_rst_ready(ha);
@@ -3076,7 +3080,7 @@ qla82xx_need_reset_handler(scsi_qla_host_t *vha)
 	}
 
 	/* wait for 10 seconds for reset ack from all functions */
-	reset_timeout = jiffies + (ha->nx_reset_timeout * HZ);
+	reset_timeout = jiffies + (ha->fcoe_reset_timeout * HZ);
 
 	drv_state = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
 	drv_active = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
@@ -3088,7 +3092,7 @@ qla82xx_need_reset_handler(scsi_qla_host_t *vha)
 	    drv_state, drv_active, dev_state, active_mask);
 
 	while (drv_state != drv_active &&
-	    dev_state != QLA82XX_DEV_INITIALIZING) {
+	    dev_state != QLA8XXX_DEV_INITIALIZING) {
 		if (time_after_eq(jiffies, reset_timeout)) {
 			ql_log(ql_log_warn, vha, 0x00b5,
 			    "Reset timeout.\n");
@@ -3099,7 +3103,7 @@ qla82xx_need_reset_handler(scsi_qla_host_t *vha)
 		qla82xx_idc_lock(ha);
 		drv_state = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
 		drv_active = qla82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
-		if (ha->flags.isp82xx_reset_owner)
+		if (ha->flags.nic_core_reset_owner)
 			drv_active &= active_mask;
 		dev_state = qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 	}
@@ -3115,11 +3119,11 @@ qla82xx_need_reset_handler(scsi_qla_host_t *vha)
 	    dev_state < MAX_STATES ? qdev_state(dev_state) : "Unknown");
 
 	/* Force to DEV_COLD unless someone else is starting a reset */
-	if (dev_state != QLA82XX_DEV_INITIALIZING &&
-	    dev_state != QLA82XX_DEV_COLD) {
+	if (dev_state != QLA8XXX_DEV_INITIALIZING &&
+	    dev_state != QLA8XXX_DEV_COLD) {
 		ql_log(ql_log_info, vha, 0x00b7,
 		    "HW State: COLD/RE-INIT.\n");
-		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_COLD);
+		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_COLD);
 		qla82xx_set_rst_ready(ha);
 		if (ql2xmdenable) {
 			if (qla82xx_md_collect(vha))
@@ -3226,8 +3230,10 @@ qla82xx_device_state_handler(scsi_qla_host_t *vha)
 	int loopcount = 0;
 
 	qla82xx_idc_lock(ha);
-	if (!vha->flags.init_done)
+	if (!vha->flags.init_done) {
 		qla82xx_set_drv_active(vha);
+		qla82xx_set_idc_version(vha);
+	}
 
 	dev_state = qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 	old_dev_state = dev_state;
@@ -3237,7 +3243,7 @@ qla82xx_device_state_handler(scsi_qla_host_t *vha)
 	    dev_state < MAX_STATES ? qdev_state(dev_state) : "Unknown");
 
 	/* wait for 30 seconds for device to go ready */
-	dev_init_timeout = jiffies + (ha->nx_dev_init_timeout * HZ);
+	dev_init_timeout = jiffies + (ha->fcoe_dev_init_timeout * HZ);
 
 	while (1) {
 
@@ -3261,18 +3267,18 @@ qla82xx_device_state_handler(scsi_qla_host_t *vha)
 		}
 
 		switch (dev_state) {
-		case QLA82XX_DEV_READY:
-			ha->flags.isp82xx_reset_owner = 0;
-			goto exit;
-		case QLA82XX_DEV_COLD:
+		case QLA8XXX_DEV_READY:
+			ha->flags.nic_core_reset_owner = 0;
+			goto rel_lock;
+		case QLA8XXX_DEV_COLD:
 			rval = qla82xx_device_bootstrap(vha);
 			break;
-		case QLA82XX_DEV_INITIALIZING:
+		case QLA8XXX_DEV_INITIALIZING:
 			qla82xx_idc_unlock(ha);
 			msleep(1000);
 			qla82xx_idc_lock(ha);
 			break;
-		case QLA82XX_DEV_NEED_RESET:
+		case QLA8XXX_DEV_NEED_RESET:
 			if (!ql2xdontresethba)
 				qla82xx_need_reset_handler(vha);
 			else {
@@ -3281,31 +3287,31 @@ qla82xx_device_state_handler(scsi_qla_host_t *vha)
 				qla82xx_idc_lock(ha);
 			}
 			dev_init_timeout = jiffies +
-			    (ha->nx_dev_init_timeout * HZ);
+			    (ha->fcoe_dev_init_timeout * HZ);
 			break;
-		case QLA82XX_DEV_NEED_QUIESCENT:
+		case QLA8XXX_DEV_NEED_QUIESCENT:
 			qla82xx_need_qsnt_handler(vha);
 			/* Reset timeout value after quiescence handler */
-			dev_init_timeout = jiffies + (ha->nx_dev_init_timeout\
+			dev_init_timeout = jiffies + (ha->fcoe_dev_init_timeout\
 							 * HZ);
 			break;
-		case QLA82XX_DEV_QUIESCENT:
+		case QLA8XXX_DEV_QUIESCENT:
 			/* Owner will exit and other will wait for the state
 			 * to get changed
 			 */
 			if (ha->flags.quiesce_owner)
-				goto exit;
+				goto rel_lock;
 
 			qla82xx_idc_unlock(ha);
 			msleep(1000);
 			qla82xx_idc_lock(ha);
 
 			/* Reset timeout value after quiescence handler */
-			dev_init_timeout = jiffies + (ha->nx_dev_init_timeout\
+			dev_init_timeout = jiffies + (ha->fcoe_dev_init_timeout\
 							 * HZ);
 			break;
-		case QLA82XX_DEV_FAILED:
-			qla82xx_dev_failed_handler(vha);
+		case QLA8XXX_DEV_FAILED:
+			qla8xxx_dev_failed_handler(vha);
 			rval = QLA_FUNCTION_FAILED;
 			goto exit;
 		default:
@@ -3315,8 +3321,9 @@ qla82xx_device_state_handler(scsi_qla_host_t *vha)
 		}
 		loopcount++;
 	}
-exit:
+rel_lock:
 	qla82xx_idc_unlock(ha);
+exit:
 	return rval;
 }
 
@@ -3364,22 +3371,30 @@ void qla82xx_watchdog(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 
 	/* don't poll if reset is going on */
-	if (!ha->flags.isp82xx_reset_hdlr_active) {
+	if (!ha->flags.nic_core_reset_hdlr_active) {
 		dev_state = qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 		if (qla82xx_check_temp(vha)) {
 			set_bit(ISP_UNRECOVERABLE, &vha->dpc_flags);
 			ha->flags.isp82xx_fw_hung = 1;
 			qla82xx_clear_pending_mbx(vha);
-		} else if (dev_state == QLA82XX_DEV_NEED_RESET &&
+		} else if (dev_state == QLA8XXX_DEV_NEED_RESET &&
 		    !test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags)) {
 			ql_log(ql_log_warn, vha, 0x6001,
 			    "Adapter reset needed.\n");
 			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
-		} else if (dev_state == QLA82XX_DEV_NEED_QUIESCENT &&
+		} else if (dev_state == QLA8XXX_DEV_NEED_QUIESCENT &&
 			!test_bit(ISP_QUIESCE_NEEDED, &vha->dpc_flags)) {
 			ql_log(ql_log_warn, vha, 0x6002,
 			    "Quiescent needed.\n");
 			set_bit(ISP_QUIESCE_NEEDED, &vha->dpc_flags);
+		} else if (dev_state == QLA8XXX_DEV_FAILED &&
+			!test_bit(ISP_UNRECOVERABLE, &vha->dpc_flags) &&
+			vha->flags.online == 1) {
+			ql_log(ql_log_warn, vha, 0xb055,
+			    "Adapter state is failed. Offlining.\n");
+			set_bit(ISP_UNRECOVERABLE, &vha->dpc_flags);
+			ha->flags.isp82xx_fw_hung = 1;
+			qla82xx_clear_pending_mbx(vha);
 		} else {
 			if (qla82xx_check_fw_alive(vha)) {
 				ql_dbg(ql_dbg_timer, vha, 0x6011,
@@ -3441,12 +3456,12 @@ qla82xx_set_reset_owner(scsi_qla_host_t *vha)
 	uint32_t dev_state;
 
 	dev_state = qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
-	if (dev_state == QLA82XX_DEV_READY) {
+	if (dev_state == QLA8XXX_DEV_READY) {
 		ql_log(ql_log_info, vha, 0xb02f,
 		    "HW State: NEED RESET\n");
 		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			QLA82XX_DEV_NEED_RESET);
-		ha->flags.isp82xx_reset_owner = 1;
+			QLA8XXX_DEV_NEED_RESET);
+		ha->flags.nic_core_reset_owner = 1;
 		ql_dbg(ql_dbg_p3p, vha, 0xb030,
 		    "reset_owner is 0x%x\n", ha->portnum);
 	} else
@@ -3477,7 +3492,7 @@ qla82xx_abort_isp(scsi_qla_host_t *vha)
 		    "Device in failed state, exiting.\n");
 		return QLA_SUCCESS;
 	}
-	ha->flags.isp82xx_reset_hdlr_active = 1;
+	ha->flags.nic_core_reset_hdlr_active = 1;
 
 	qla82xx_idc_lock(ha);
 	qla82xx_set_reset_owner(vha);
@@ -3491,7 +3506,7 @@ qla82xx_abort_isp(scsi_qla_host_t *vha)
 
 	if (rval == QLA_SUCCESS) {
 		ha->flags.isp82xx_fw_hung = 0;
-		ha->flags.isp82xx_reset_hdlr_active = 0;
+		ha->flags.nic_core_reset_hdlr_active = 0;
 		qla82xx_restart_isp(vha);
 	}
 
@@ -4026,7 +4041,7 @@ qla82xx_minidump_process_rdmem(scsi_qla_host_t *vha,
 
 	if (r_addr & 0xf) {
 		ql_log(ql_log_warn, vha, 0xb033,
-		    "Read addr 0x%x not 16 bytes alligned\n", r_addr);
+		    "Read addr 0x%x not 16 bytes aligned\n", r_addr);
 		return rval;
 	}
 
diff --git a/drivers/scsi/qla2xxx/qla_nx.h b/drivers/scsi/qla2xxx/qla_nx.h
index 6eb210e3cc6..6c953e8c08f 100644
--- a/drivers/scsi/qla2xxx/qla_nx.h
+++ b/drivers/scsi/qla2xxx/qla_nx.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -542,14 +542,15 @@
 #define QLA82XX_CRB_DRV_IDC_VERSION  (QLA82XX_CAM_RAM(0x174))
 
 /* Every driver should use these Device State */
-#define QLA82XX_DEV_COLD		1
-#define QLA82XX_DEV_INITIALIZING	2
-#define QLA82XX_DEV_READY		3
-#define QLA82XX_DEV_NEED_RESET		4
-#define QLA82XX_DEV_NEED_QUIESCENT	5
-#define QLA82XX_DEV_FAILED		6
-#define QLA82XX_DEV_QUIESCENT		7
+#define QLA8XXX_DEV_COLD		1
+#define QLA8XXX_DEV_INITIALIZING	2
+#define QLA8XXX_DEV_READY		3
+#define QLA8XXX_DEV_NEED_RESET		4
+#define QLA8XXX_DEV_NEED_QUIESCENT	5
+#define QLA8XXX_DEV_FAILED		6
+#define QLA8XXX_DEV_QUIESCENT		7
 #define	MAX_STATES			8 /* Increment if new state added */
+#define QLA8XXX_BAD_VALUE		0xbad0bad0
 
 #define QLA82XX_IDC_VERSION			1
 #define QLA82XX_ROM_DEV_INIT_TIMEOUT		30
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index d3052622e77..d501bf5f806 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -113,11 +113,11 @@ MODULE_PARM_DESC(ql2xfdmienable,
 static int ql2xmaxqdepth = MAX_Q_DEPTH;
 module_param(ql2xmaxqdepth, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(ql2xmaxqdepth,
-		"Maximum queue depth to report for target devices.");
+		"Maximum queue depth to set for each LUN. "
+		"Default is 32.");
 
-/* Do not change the value of this after module load */
-int ql2xenabledif = 0;
-module_param(ql2xenabledif, int, S_IRUGO|S_IWUSR);
+int ql2xenabledif = 2;
+module_param(ql2xenabledif, int, S_IRUGO);
 MODULE_PARM_DESC(ql2xenabledif,
 		" Enable T10-CRC-DIF "
 		" Default is 0 - No DIF Support. 1 - Enable it"
@@ -1078,7 +1078,7 @@ __qla2xxx_eh_generic_reset(char *name, enum nexus_wait_type type,
 	if (qla2x00_eh_wait_for_pending_commands(vha, cmd->device->id,
 	    cmd->device->lun, type) != QLA_SUCCESS) {
 		ql_log(ql_log_warn, vha, 0x800d,
-		    "wait for peding cmds failed for cmd=%p.\n", cmd);
+		    "wait for pending cmds failed for cmd=%p.\n", cmd);
 		goto eh_reset_failed;
 	}
 
@@ -1177,7 +1177,7 @@ qla2xxx_eh_bus_reset(struct scsi_cmnd *cmd)
 eh_bus_reset_done:
 	ql_log(ql_log_warn, vha, 0x802b,
 	    "BUS RESET %s nexus=%ld:%d:%d.\n",
-	    (ret == FAILED) ? "FAILED" : "SUCCEDED", vha->host_no, id, lun);
+	    (ret == FAILED) ? "FAILED" : "SUCCEEDED", vha->host_no, id, lun);
 
 	return ret;
 }
@@ -1357,6 +1357,9 @@ qla2xxx_slave_configure(struct scsi_device *sdev)
 	scsi_qla_host_t *vha = shost_priv(sdev->host);
 	struct req_que *req = vha->req;
 
+	if (IS_T10_PI_CAPABLE(vha->hw))
+		blk_queue_update_dma_alignment(sdev->request_queue, 0x7);
+
 	if (sdev->tagged_supported)
 		scsi_activate_tcq(sdev, req->max_q_depth);
 	else
@@ -1919,7 +1922,7 @@ static struct isp_operations qla82xx_isp_ops = {
 	.nvram_config		= qla81xx_nvram_config,
 	.update_fw_options	= qla24xx_update_fw_options,
 	.load_risc		= qla82xx_load_risc,
-	.pci_info_str		= qla82xx_pci_info_str,
+	.pci_info_str		= qla24xx_pci_info_str,
 	.fw_version_str		= qla24xx_fw_version_str,
 	.intr_handler		= qla82xx_intr_handler,
 	.enable_intrs		= qla82xx_enable_intrs,
@@ -2149,7 +2152,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	scsi_qla_host_t *base_vha = NULL;
 	struct qla_hw_data *ha;
 	char pci_info[30];
-	char fw_str[30];
+	char fw_str[30], wq_name[30];
 	struct scsi_host_template *sht;
 	int bars, mem_only = 0;
 	uint16_t req_length = 0, rsp_length = 0;
@@ -2203,12 +2206,14 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	ha->mem_only = mem_only;
 	spin_lock_init(&ha->hardware_lock);
 	spin_lock_init(&ha->vport_slock);
+	mutex_init(&ha->selflogin_lock);
 
 	/* Set ISP-type information. */
 	qla2x00_set_isp_flags(ha);
 
 	/* Set EEH reset type to fundamental if required by hba */
-	if (IS_QLA24XX(ha) || IS_QLA25XX(ha) || IS_QLA81XX(ha))
+	if (IS_QLA24XX(ha) || IS_QLA25XX(ha) || IS_QLA81XX(ha) ||
+	    IS_QLA83XX(ha))
 		pdev->needs_freset = 1;
 
 	ha->prev_topology = 0;
@@ -2318,6 +2323,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		ha->nvram_conf_off = FARX_ACCESS_NVRAM_CONF;
 		ha->nvram_data_off = FARX_ACCESS_NVRAM_DATA;
 	} else if (IS_QLA83XX(ha)) {
+		ha->portnum = PCI_FUNC(ha->pdev->devfn);
 		ha->max_fibre_devices = MAX_FIBRE_DEVICES_2400;
 		ha->mbx_count = MAILBOX_REGISTER_COUNT;
 		req_length = REQUEST_ENTRY_CNT_24XX;
@@ -2416,7 +2422,6 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	    host->can_queue, base_vha->req,
 	    base_vha->mgmt_svr_loop_id, host->sg_tablesize);
 	host->max_id = ha->max_fibre_devices;
-	host->this_id = 255;
 	host->cmd_per_lun = 3;
 	host->unique_id = host->host_no;
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif)
@@ -2499,7 +2504,7 @@ que_init:
 		if (IS_QLA82XX(ha)) {
 			qla82xx_idc_lock(ha);
 			qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-				QLA82XX_DEV_FAILED);
+				QLA8XXX_DEV_FAILED);
 			qla82xx_idc_unlock(ha);
 			ql_log(ql_log_fatal, base_vha, 0x00d7,
 			    "HW State: FAILED.\n");
@@ -2542,6 +2547,20 @@ que_init:
 	 */
 	qla2xxx_wake_dpc(base_vha);
 
+	if (IS_QLA8031(ha) || IS_MCTP_CAPABLE(ha)) {
+		sprintf(wq_name, "qla2xxx_%lu_dpc_lp_wq", base_vha->host_no);
+		ha->dpc_lp_wq = create_singlethread_workqueue(wq_name);
+		INIT_WORK(&ha->idc_aen, qla83xx_service_idc_aen);
+
+		sprintf(wq_name, "qla2xxx_%lu_dpc_hp_wq", base_vha->host_no);
+		ha->dpc_hp_wq = create_singlethread_workqueue(wq_name);
+		INIT_WORK(&ha->nic_core_reset, qla83xx_nic_core_reset_work);
+		INIT_WORK(&ha->idc_state_handler,
+		    qla83xx_idc_state_handler_work);
+		INIT_WORK(&ha->nic_core_unrecoverable,
+		    qla83xx_nic_core_unrecoverable_work);
+	}
+
 skip_dpc:
 	list_add_tail(&base_vha->list, &ha->vp_list);
 	base_vha->host->irq = ha->pdev->irq;
@@ -2557,7 +2576,7 @@ skip_dpc:
 
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif) {
 		if (ha->fw_attributes & BIT_4) {
-			int prot = 0;
+			int prot = 0, guard;
 			base_vha->flags.difdix_supported = 1;
 			ql_dbg(ql_dbg_init, base_vha, 0x00f1,
 			    "Registering for DIF/DIX type 1 and 3 protection.\n");
@@ -2570,7 +2589,14 @@ skip_dpc:
 			    | SHOST_DIX_TYPE1_PROTECTION
 			    | SHOST_DIX_TYPE2_PROTECTION
 			    | SHOST_DIX_TYPE3_PROTECTION);
-			scsi_host_set_guard(host, SHOST_DIX_GUARD_CRC);
+
+			guard = SHOST_DIX_GUARD_CRC;
+
+			if (IS_PI_IPGUARD_CAPABLE(ha) &&
+			    (ql2xenabledif > 1 || IS_PI_DIFB_DIX0_CAPABLE(ha)))
+				guard |= SHOST_DIX_GUARD_IP;
+
+			scsi_host_set_guard(host, guard);
 		} else
 			base_vha->flags.difdix_supported = 0;
 	}
@@ -2750,6 +2776,14 @@ qla2x00_remove_one(struct pci_dev *pdev)
 	}
 	mutex_unlock(&ha->vport_lock);
 
+	if (IS_QLA8031(ha)) {
+		ql_dbg(ql_dbg_p3p, base_vha, 0xb07e,
+		    "Clearing fcoe driver presence.\n");
+		if (qla83xx_clear_drv_presence(base_vha) != QLA_SUCCESS)
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb079,
+			    "Error while clearing DRV-Presence.\n");
+	}
+
 	set_bit(UNLOADING, &base_vha->dpc_flags);
 
 	qla2x00_abort_all_cmds(base_vha, DID_NO_CONNECT << 16);
@@ -2771,6 +2805,21 @@ qla2x00_remove_one(struct pci_dev *pdev)
 		ha->wq = NULL;
 	}
 
+	/* Cancel all work and destroy DPC workqueues */
+	if (ha->dpc_lp_wq) {
+		cancel_work_sync(&ha->idc_aen);
+		destroy_workqueue(ha->dpc_lp_wq);
+		ha->dpc_lp_wq = NULL;
+	}
+
+	if (ha->dpc_hp_wq) {
+		cancel_work_sync(&ha->nic_core_reset);
+		cancel_work_sync(&ha->idc_state_handler);
+		cancel_work_sync(&ha->nic_core_unrecoverable);
+		destroy_workqueue(ha->dpc_hp_wq);
+		ha->dpc_hp_wq = NULL;
+	}
+
 	/* Kill the kernel thread for this host */
 	if (ha->dpc_thread) {
 		struct task_struct *t = ha->dpc_thread;
@@ -2837,7 +2886,6 @@ qla2x00_free_device(scsi_qla_host_t *vha)
 	qla2x00_stop_dpc_thread(vha);
 
 	qla25xx_delete_queues(vha);
-
 	if (ha->flags.fce_enabled)
 		qla2x00_disable_fce_trace(vha, NULL, NULL);
 
@@ -2872,6 +2920,7 @@ void qla2x00_free_fcports(struct scsi_qla_host *vha)
 
 	list_for_each_entry_safe(fcport, tfcport, &vha->vp_fcports, list) {
 		list_del(&fcport->list);
+		qla2x00_clear_loop_id(fcport);
 		kfree(fcport);
 		fcport = NULL;
 	}
@@ -3169,6 +3218,18 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len,
 	}
 
 	INIT_LIST_HEAD(&ha->vp_list);
+
+	/* Allocate memory for our loop_id bitmap */
+	ha->loop_id_map = kzalloc(BITS_TO_LONGS(LOOPID_MAP_SIZE) * sizeof(long),
+	    GFP_KERNEL);
+	if (!ha->loop_id_map)
+		goto fail_async_pd;
+	else {
+		qla2x00_set_reserved_loop_ids(ha);
+		ql_dbg_pci(ql_dbg_init, ha->pdev, 0x0123,
+		    "loop_id_map=%p. \n", ha->loop_id_map);
+	}
+
 	return 1;
 
 fail_async_pd:
@@ -3280,6 +3341,10 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 {
 	qla2x00_free_fw_dump(ha);
 
+	if (ha->mctp_dump)
+		dma_free_coherent(&ha->pdev->dev, MCTP_DUMP_SIZE, ha->mctp_dump,
+		    ha->mctp_dump_dma);
+
 	if (ha->srb_mempool)
 		mempool_destroy(ha->srb_mempool);
 
@@ -3352,6 +3417,7 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 	kfree(ha->nvram);
 	kfree(ha->npiv_info);
 	kfree(ha->swl);
+	kfree(ha->loop_id_map);
 
 	ha->srb_mempool = NULL;
 	ha->ctx_mempool = NULL;
@@ -3687,13 +3753,651 @@ void qla2x00_relogin(struct scsi_qla_host *vha)
 			}
 
 			if (fcport->login_retry == 0 && status != QLA_SUCCESS)
-				fcport->loop_id = FC_NO_LOOP_ID;
+				qla2x00_clear_loop_id(fcport);
 		}
 		if (test_bit(LOOP_RESYNC_NEEDED, &vha->dpc_flags))
 			break;
 	}
 }
 
+/* Schedule work on any of the dpc-workqueues */
+void
+qla83xx_schedule_work(scsi_qla_host_t *base_vha, int work_code)
+{
+	struct qla_hw_data *ha = base_vha->hw;
+
+	switch (work_code) {
+	case MBA_IDC_AEN: /* 0x8200 */
+		if (ha->dpc_lp_wq)
+			queue_work(ha->dpc_lp_wq, &ha->idc_aen);
+		break;
+
+	case QLA83XX_NIC_CORE_RESET: /* 0x1 */
+		if (!ha->flags.nic_core_reset_hdlr_active) {
+			if (ha->dpc_hp_wq)
+				queue_work(ha->dpc_hp_wq, &ha->nic_core_reset);
+		} else
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb05e,
+			    "NIC Core reset is already active. Skip "
+			    "scheduling it again.\n");
+		break;
+	case QLA83XX_IDC_STATE_HANDLER: /* 0x2 */
+		if (ha->dpc_hp_wq)
+			queue_work(ha->dpc_hp_wq, &ha->idc_state_handler);
+		break;
+	case QLA83XX_NIC_CORE_UNRECOVERABLE: /* 0x3 */
+		if (ha->dpc_hp_wq)
+			queue_work(ha->dpc_hp_wq, &ha->nic_core_unrecoverable);
+		break;
+	default:
+		ql_log(ql_log_warn, base_vha, 0xb05f,
+		    "Unknow work-code=0x%x.\n", work_code);
+	}
+
+	return;
+}
+
+/* Work: Perform NIC Core Unrecoverable state handling */
+void
+qla83xx_nic_core_unrecoverable_work(struct work_struct *work)
+{
+	struct qla_hw_data *ha =
+		container_of(work, struct qla_hw_data, nic_core_unrecoverable);
+	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
+	uint32_t dev_state = 0;
+
+	qla83xx_idc_lock(base_vha, 0);
+	qla83xx_rd_reg(base_vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+	qla83xx_reset_ownership(base_vha);
+	if (ha->flags.nic_core_reset_owner) {
+		ha->flags.nic_core_reset_owner = 0;
+		qla83xx_wr_reg(base_vha, QLA83XX_IDC_DEV_STATE,
+		    QLA8XXX_DEV_FAILED);
+		ql_log(ql_log_info, base_vha, 0xb060, "HW State: FAILED.\n");
+		qla83xx_schedule_work(base_vha, QLA83XX_IDC_STATE_HANDLER);
+	}
+	qla83xx_idc_unlock(base_vha, 0);
+}
+
+/* Work: Execute IDC state handler */
+void
+qla83xx_idc_state_handler_work(struct work_struct *work)
+{
+	struct qla_hw_data *ha =
+		container_of(work, struct qla_hw_data, idc_state_handler);
+	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
+	uint32_t dev_state = 0;
+
+	qla83xx_idc_lock(base_vha, 0);
+	qla83xx_rd_reg(base_vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+	if (dev_state == QLA8XXX_DEV_FAILED ||
+			dev_state == QLA8XXX_DEV_NEED_QUIESCENT)
+		qla83xx_idc_state_handler(base_vha);
+	qla83xx_idc_unlock(base_vha, 0);
+}
+
+int
+qla83xx_check_nic_core_fw_alive(scsi_qla_host_t *base_vha)
+{
+	int rval = QLA_SUCCESS;
+	unsigned long heart_beat_wait = jiffies + (1 * HZ);
+	uint32_t heart_beat_counter1, heart_beat_counter2;
+
+	do {
+		if (time_after(jiffies, heart_beat_wait)) {
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb07c,
+			    "Nic Core f/w is not alive.\n");
+			rval = QLA_FUNCTION_FAILED;
+			break;
+		}
+
+		qla83xx_idc_lock(base_vha, 0);
+		qla83xx_rd_reg(base_vha, QLA83XX_FW_HEARTBEAT,
+		    &heart_beat_counter1);
+		qla83xx_idc_unlock(base_vha, 0);
+		msleep(100);
+		qla83xx_idc_lock(base_vha, 0);
+		qla83xx_rd_reg(base_vha, QLA83XX_FW_HEARTBEAT,
+		    &heart_beat_counter2);
+		qla83xx_idc_unlock(base_vha, 0);
+	} while (heart_beat_counter1 == heart_beat_counter2);
+
+	return rval;
+}
+
+/* Work: Perform NIC Core Reset handling */
+void
+qla83xx_nic_core_reset_work(struct work_struct *work)
+{
+	struct qla_hw_data *ha =
+		container_of(work, struct qla_hw_data, nic_core_reset);
+	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
+	uint32_t dev_state = 0;
+
+	if (IS_QLA2031(ha)) {
+		if (qla2xxx_mctp_dump(base_vha) != QLA_SUCCESS)
+			ql_log(ql_log_warn, base_vha, 0xb081,
+			    "Failed to dump mctp\n");
+		return;
+	}
+
+	if (!ha->flags.nic_core_reset_hdlr_active) {
+		if (qla83xx_check_nic_core_fw_alive(base_vha) == QLA_SUCCESS) {
+			qla83xx_idc_lock(base_vha, 0);
+			qla83xx_rd_reg(base_vha, QLA83XX_IDC_DEV_STATE,
+			    &dev_state);
+			qla83xx_idc_unlock(base_vha, 0);
+			if (dev_state != QLA8XXX_DEV_NEED_RESET) {
+				ql_dbg(ql_dbg_p3p, base_vha, 0xb07a,
+				    "Nic Core f/w is alive.\n");
+				return;
+			}
+		}
+
+		ha->flags.nic_core_reset_hdlr_active = 1;
+		if (qla83xx_nic_core_reset(base_vha)) {
+			/* NIC Core reset failed. */
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb061,
+			    "NIC Core reset failed.\n");
+		}
+		ha->flags.nic_core_reset_hdlr_active = 0;
+	}
+}
+
+/* Work: Handle 8200 IDC aens */
+void
+qla83xx_service_idc_aen(struct work_struct *work)
+{
+	struct qla_hw_data *ha =
+		container_of(work, struct qla_hw_data, idc_aen);
+	scsi_qla_host_t *base_vha = pci_get_drvdata(ha->pdev);
+	uint32_t dev_state, idc_control;
+
+	qla83xx_idc_lock(base_vha, 0);
+	qla83xx_rd_reg(base_vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+	qla83xx_rd_reg(base_vha, QLA83XX_IDC_CONTROL, &idc_control);
+	qla83xx_idc_unlock(base_vha, 0);
+	if (dev_state == QLA8XXX_DEV_NEED_RESET) {
+		if (idc_control & QLA83XX_IDC_GRACEFUL_RESET) {
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb062,
+			    "Application requested NIC Core Reset.\n");
+			qla83xx_schedule_work(base_vha, QLA83XX_NIC_CORE_RESET);
+		} else if (qla83xx_check_nic_core_fw_alive(base_vha) ==
+		    QLA_SUCCESS) {
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb07b,
+			    "Other protocol driver requested NIC Core Reset.\n");
+			qla83xx_schedule_work(base_vha, QLA83XX_NIC_CORE_RESET);
+		}
+	} else if (dev_state == QLA8XXX_DEV_FAILED ||
+			dev_state == QLA8XXX_DEV_NEED_QUIESCENT) {
+		qla83xx_schedule_work(base_vha, QLA83XX_IDC_STATE_HANDLER);
+	}
+}
+
+static void
+qla83xx_wait_logic(void)
+{
+	int i;
+
+	/* Yield CPU */
+	if (!in_interrupt()) {
+		/*
+		 * Wait about 200ms before retrying again.
+		 * This controls the number of retries for single
+		 * lock operation.
+		 */
+		msleep(100);
+		schedule();
+	} else {
+		for (i = 0; i < 20; i++)
+			cpu_relax(); /* This a nop instr on i386 */
+	}
+}
+
+int
+qla83xx_force_lock_recovery(scsi_qla_host_t *base_vha)
+{
+	int rval;
+	uint32_t data;
+	uint32_t idc_lck_rcvry_stage_mask = 0x3;
+	uint32_t idc_lck_rcvry_owner_mask = 0x3c;
+	struct qla_hw_data *ha = base_vha->hw;
+
+	rval = qla83xx_rd_reg(base_vha, QLA83XX_IDC_LOCK_RECOVERY, &data);
+	if (rval)
+		return rval;
+
+	if ((data & idc_lck_rcvry_stage_mask) > 0) {
+		return QLA_SUCCESS;
+	} else {
+		data = (IDC_LOCK_RECOVERY_STAGE1) | (ha->portnum << 2);
+		rval = qla83xx_wr_reg(base_vha, QLA83XX_IDC_LOCK_RECOVERY,
+		    data);
+		if (rval)
+			return rval;
+
+		msleep(200);
+
+		rval = qla83xx_rd_reg(base_vha, QLA83XX_IDC_LOCK_RECOVERY,
+		    &data);
+		if (rval)
+			return rval;
+
+		if (((data & idc_lck_rcvry_owner_mask) >> 2) == ha->portnum) {
+			data &= (IDC_LOCK_RECOVERY_STAGE2 |
+					~(idc_lck_rcvry_stage_mask));
+			rval = qla83xx_wr_reg(base_vha,
+			    QLA83XX_IDC_LOCK_RECOVERY, data);
+			if (rval)
+				return rval;
+
+			/* Forcefully perform IDC UnLock */
+			rval = qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_UNLOCK,
+			    &data);
+			if (rval)
+				return rval;
+			/* Clear lock-id by setting 0xff */
+			rval = qla83xx_wr_reg(base_vha, QLA83XX_DRIVER_LOCKID,
+			    0xff);
+			if (rval)
+				return rval;
+			/* Clear lock-recovery by setting 0x0 */
+			rval = qla83xx_wr_reg(base_vha,
+			    QLA83XX_IDC_LOCK_RECOVERY, 0x0);
+			if (rval)
+				return rval;
+		} else
+			return QLA_SUCCESS;
+	}
+
+	return rval;
+}
+
+int
+qla83xx_idc_lock_recovery(scsi_qla_host_t *base_vha)
+{
+	int rval = QLA_SUCCESS;
+	uint32_t o_drv_lockid, n_drv_lockid;
+	unsigned long lock_recovery_timeout;
+
+	lock_recovery_timeout = jiffies + QLA83XX_MAX_LOCK_RECOVERY_WAIT;
+retry_lockid:
+	rval = qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_LOCKID, &o_drv_lockid);
+	if (rval)
+		goto exit;
+
+	/* MAX wait time before forcing IDC Lock recovery = 2 secs */
+	if (time_after_eq(jiffies, lock_recovery_timeout)) {
+		if (qla83xx_force_lock_recovery(base_vha) == QLA_SUCCESS)
+			return QLA_SUCCESS;
+		else
+			return QLA_FUNCTION_FAILED;
+	}
+
+	rval = qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_LOCKID, &n_drv_lockid);
+	if (rval)
+		goto exit;
+
+	if (o_drv_lockid == n_drv_lockid) {
+		qla83xx_wait_logic();
+		goto retry_lockid;
+	} else
+		return QLA_SUCCESS;
+
+exit:
+	return rval;
+}
+
+void
+qla83xx_idc_lock(scsi_qla_host_t *base_vha, uint16_t requester_id)
+{
+	uint16_t options = (requester_id << 15) | BIT_6;
+	uint32_t data;
+	struct qla_hw_data *ha = base_vha->hw;
+
+	/* IDC-lock implementation using driver-lock/lock-id remote registers */
+retry_lock:
+	if (qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_LOCK, &data)
+	    == QLA_SUCCESS) {
+		if (data) {
+			/* Setting lock-id to our function-number */
+			qla83xx_wr_reg(base_vha, QLA83XX_DRIVER_LOCKID,
+			    ha->portnum);
+		} else {
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb063,
+			    "Failed to acquire IDC lock. retrying...\n");
+
+			/* Retry/Perform IDC-Lock recovery */
+			if (qla83xx_idc_lock_recovery(base_vha)
+			    == QLA_SUCCESS) {
+				qla83xx_wait_logic();
+				goto retry_lock;
+			} else
+				ql_log(ql_log_warn, base_vha, 0xb075,
+				    "IDC Lock recovery FAILED.\n");
+		}
+
+	}
+
+	return;
+
+	/* XXX: IDC-lock implementation using access-control mbx */
+retry_lock2:
+	if (qla83xx_access_control(base_vha, options, 0, 0, NULL)) {
+		ql_dbg(ql_dbg_p3p, base_vha, 0xb072,
+		    "Failed to acquire IDC lock. retrying...\n");
+		/* Retry/Perform IDC-Lock recovery */
+		if (qla83xx_idc_lock_recovery(base_vha) == QLA_SUCCESS) {
+			qla83xx_wait_logic();
+			goto retry_lock2;
+		} else
+			ql_log(ql_log_warn, base_vha, 0xb076,
+			    "IDC Lock recovery FAILED.\n");
+	}
+
+	return;
+}
+
+void
+qla83xx_idc_unlock(scsi_qla_host_t *base_vha, uint16_t requester_id)
+{
+	uint16_t options = (requester_id << 15) | BIT_7, retry;
+	uint32_t data;
+	struct qla_hw_data *ha = base_vha->hw;
+
+	/* IDC-unlock implementation using driver-unlock/lock-id
+	 * remote registers
+	 */
+	retry = 0;
+retry_unlock:
+	if (qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_LOCKID, &data)
+	    == QLA_SUCCESS) {
+		if (data == ha->portnum) {
+			qla83xx_rd_reg(base_vha, QLA83XX_DRIVER_UNLOCK, &data);
+			/* Clearing lock-id by setting 0xff */
+			qla83xx_wr_reg(base_vha, QLA83XX_DRIVER_LOCKID, 0xff);
+		} else if (retry < 10) {
+			/* SV: XXX: IDC unlock retrying needed here? */
+
+			/* Retry for IDC-unlock */
+			qla83xx_wait_logic();
+			retry++;
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb064,
+			    "Failed to release IDC lock, retyring=%d\n", retry);
+			goto retry_unlock;
+		}
+	} else if (retry < 10) {
+		/* Retry for IDC-unlock */
+		qla83xx_wait_logic();
+		retry++;
+		ql_dbg(ql_dbg_p3p, base_vha, 0xb065,
+		    "Failed to read drv-lockid, retyring=%d\n", retry);
+		goto retry_unlock;
+	}
+
+	return;
+
+	/* XXX: IDC-unlock implementation using access-control mbx */
+	retry = 0;
+retry_unlock2:
+	if (qla83xx_access_control(base_vha, options, 0, 0, NULL)) {
+		if (retry < 10) {
+			/* Retry for IDC-unlock */
+			qla83xx_wait_logic();
+			retry++;
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb066,
+			    "Failed to release IDC lock, retyring=%d\n", retry);
+			goto retry_unlock2;
+		}
+	}
+
+	return;
+}
+
+int
+__qla83xx_set_drv_presence(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_presence;
+
+	rval = qla83xx_rd_reg(vha, QLA83XX_IDC_DRV_PRESENCE, &drv_presence);
+	if (rval == QLA_SUCCESS) {
+		drv_presence |= (1 << ha->portnum);
+		rval = qla83xx_wr_reg(vha, QLA83XX_IDC_DRV_PRESENCE,
+		    drv_presence);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_set_drv_presence(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_set_drv_presence(vha);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+int
+__qla83xx_clear_drv_presence(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_presence;
+
+	rval = qla83xx_rd_reg(vha, QLA83XX_IDC_DRV_PRESENCE, &drv_presence);
+	if (rval == QLA_SUCCESS) {
+		drv_presence &= ~(1 << ha->portnum);
+		rval = qla83xx_wr_reg(vha, QLA83XX_IDC_DRV_PRESENCE,
+		    drv_presence);
+	}
+
+	return rval;
+}
+
+int
+qla83xx_clear_drv_presence(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+
+	qla83xx_idc_lock(vha, 0);
+	rval = __qla83xx_clear_drv_presence(vha);
+	qla83xx_idc_unlock(vha, 0);
+
+	return rval;
+}
+
+void
+qla83xx_need_reset_handler(scsi_qla_host_t *vha)
+{
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t drv_ack, drv_presence;
+	unsigned long ack_timeout;
+
+	/* Wait for IDC ACK from all functions (DRV-ACK == DRV-PRESENCE) */
+	ack_timeout = jiffies + (ha->fcoe_reset_timeout * HZ);
+	while (1) {
+		qla83xx_rd_reg(vha, QLA83XX_IDC_DRIVER_ACK, &drv_ack);
+		qla83xx_rd_reg(vha, QLA83XX_IDC_DRV_PRESENCE, &drv_presence);
+		if (drv_ack == drv_presence)
+			break;
+
+		if (time_after_eq(jiffies, ack_timeout)) {
+			ql_log(ql_log_warn, vha, 0xb067,
+			    "RESET ACK TIMEOUT! drv_presence=0x%x "
+			    "drv_ack=0x%x\n", drv_presence, drv_ack);
+			/*
+			 * The function(s) which did not ack in time are forced
+			 * to withdraw any further participation in the IDC
+			 * reset.
+			 */
+			if (drv_ack != drv_presence)
+				qla83xx_wr_reg(vha, QLA83XX_IDC_DRV_PRESENCE,
+				    drv_ack);
+			break;
+		}
+
+		qla83xx_idc_unlock(vha, 0);
+		msleep(1000);
+		qla83xx_idc_lock(vha, 0);
+	}
+
+	qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE, QLA8XXX_DEV_COLD);
+	ql_log(ql_log_info, vha, 0xb068, "HW State: COLD/RE-INIT.\n");
+}
+
+int
+qla83xx_device_bootstrap(scsi_qla_host_t *vha)
+{
+	int rval = QLA_SUCCESS;
+	uint32_t idc_control;
+
+	qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE, QLA8XXX_DEV_INITIALIZING);
+	ql_log(ql_log_info, vha, 0xb069, "HW State: INITIALIZING.\n");
+
+	/* Clearing IDC-Control Graceful-Reset Bit before resetting f/w */
+	__qla83xx_get_idc_control(vha, &idc_control);
+	idc_control &= ~QLA83XX_IDC_GRACEFUL_RESET;
+	__qla83xx_set_idc_control(vha, 0);
+
+	qla83xx_idc_unlock(vha, 0);
+	rval = qla83xx_restart_nic_firmware(vha);
+	qla83xx_idc_lock(vha, 0);
+
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_fatal, vha, 0xb06a,
+		    "Failed to restart NIC f/w.\n");
+		qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE, QLA8XXX_DEV_FAILED);
+		ql_log(ql_log_info, vha, 0xb06b, "HW State: FAILED.\n");
+	} else {
+		ql_dbg(ql_dbg_p3p, vha, 0xb06c,
+		    "Success in restarting nic f/w.\n");
+		qla83xx_wr_reg(vha, QLA83XX_IDC_DEV_STATE, QLA8XXX_DEV_READY);
+		ql_log(ql_log_info, vha, 0xb06d, "HW State: READY.\n");
+	}
+
+	return rval;
+}
+
+/* Assumes idc_lock always held on entry */
+int
+qla83xx_idc_state_handler(scsi_qla_host_t *base_vha)
+{
+	struct qla_hw_data *ha = base_vha->hw;
+	int rval = QLA_SUCCESS;
+	unsigned long dev_init_timeout;
+	uint32_t dev_state;
+
+	/* Wait for MAX-INIT-TIMEOUT for the device to go ready */
+	dev_init_timeout = jiffies + (ha->fcoe_dev_init_timeout * HZ);
+
+	while (1) {
+
+		if (time_after_eq(jiffies, dev_init_timeout)) {
+			ql_log(ql_log_warn, base_vha, 0xb06e,
+			    "Initialization TIMEOUT!\n");
+			/* Init timeout. Disable further NIC Core
+			 * communication.
+			 */
+			qla83xx_wr_reg(base_vha, QLA83XX_IDC_DEV_STATE,
+				QLA8XXX_DEV_FAILED);
+			ql_log(ql_log_info, base_vha, 0xb06f,
+			    "HW State: FAILED.\n");
+		}
+
+		qla83xx_rd_reg(base_vha, QLA83XX_IDC_DEV_STATE, &dev_state);
+		switch (dev_state) {
+		case QLA8XXX_DEV_READY:
+			if (ha->flags.nic_core_reset_owner)
+				qla83xx_idc_audit(base_vha,
+				    IDC_AUDIT_COMPLETION);
+			ha->flags.nic_core_reset_owner = 0;
+			ql_dbg(ql_dbg_p3p, base_vha, 0xb070,
+			    "Reset_owner reset by 0x%x.\n",
+			    ha->portnum);
+			goto exit;
+		case QLA8XXX_DEV_COLD:
+			if (ha->flags.nic_core_reset_owner)
+				rval = qla83xx_device_bootstrap(base_vha);
+			else {
+			/* Wait for AEN to change device-state */
+				qla83xx_idc_unlock(base_vha, 0);
+				msleep(1000);
+				qla83xx_idc_lock(base_vha, 0);
+			}
+			break;
+		case QLA8XXX_DEV_INITIALIZING:
+			/* Wait for AEN to change device-state */
+			qla83xx_idc_unlock(base_vha, 0);
+			msleep(1000);
+			qla83xx_idc_lock(base_vha, 0);
+			break;
+		case QLA8XXX_DEV_NEED_RESET:
+			if (!ql2xdontresethba && ha->flags.nic_core_reset_owner)
+				qla83xx_need_reset_handler(base_vha);
+			else {
+				/* Wait for AEN to change device-state */
+				qla83xx_idc_unlock(base_vha, 0);
+				msleep(1000);
+				qla83xx_idc_lock(base_vha, 0);
+			}
+			/* reset timeout value after need reset handler */
+			dev_init_timeout = jiffies +
+			    (ha->fcoe_dev_init_timeout * HZ);
+			break;
+		case QLA8XXX_DEV_NEED_QUIESCENT:
+			/* XXX: DEBUG for now */
+			qla83xx_idc_unlock(base_vha, 0);
+			msleep(1000);
+			qla83xx_idc_lock(base_vha, 0);
+			break;
+		case QLA8XXX_DEV_QUIESCENT:
+			/* XXX: DEBUG for now */
+			if (ha->flags.quiesce_owner)
+				goto exit;
+
+			qla83xx_idc_unlock(base_vha, 0);
+			msleep(1000);
+			qla83xx_idc_lock(base_vha, 0);
+			dev_init_timeout = jiffies +
+			    (ha->fcoe_dev_init_timeout * HZ);
+			break;
+		case QLA8XXX_DEV_FAILED:
+			if (ha->flags.nic_core_reset_owner)
+				qla83xx_idc_audit(base_vha,
+				    IDC_AUDIT_COMPLETION);
+			ha->flags.nic_core_reset_owner = 0;
+			__qla83xx_clear_drv_presence(base_vha);
+			qla83xx_idc_unlock(base_vha, 0);
+			qla8xxx_dev_failed_handler(base_vha);
+			rval = QLA_FUNCTION_FAILED;
+			qla83xx_idc_lock(base_vha, 0);
+			goto exit;
+		case QLA8XXX_BAD_VALUE:
+			qla83xx_idc_unlock(base_vha, 0);
+			msleep(1000);
+			qla83xx_idc_lock(base_vha, 0);
+			break;
+		default:
+			ql_log(ql_log_warn, base_vha, 0xb071,
+			    "Unknow Device State: %x.\n", dev_state);
+			qla83xx_idc_unlock(base_vha, 0);
+			qla8xxx_dev_failed_handler(base_vha);
+			rval = QLA_FUNCTION_FAILED;
+			qla83xx_idc_lock(base_vha, 0);
+			goto exit;
+		}
+	}
+
+exit:
+	return rval;
+}
+
 /**************************************************************************
 * qla2x00_do_dpc
 *   This kernel thread is a task that is schedule by the interrupt handler
@@ -3749,7 +4453,7 @@ qla2x00_do_dpc(void *data)
 				&base_vha->dpc_flags)) {
 				qla82xx_idc_lock(ha);
 				qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-					QLA82XX_DEV_FAILED);
+					QLA8XXX_DEV_FAILED);
 				qla82xx_idc_unlock(ha);
 				ql_log(ql_log_info, base_vha, 0x4004,
 				    "HW State: FAILED.\n");
@@ -3819,14 +4523,21 @@ qla2x00_do_dpc(void *data)
 		if (test_bit(ISP_QUIESCE_NEEDED, &base_vha->dpc_flags)) {
 			ql_dbg(ql_dbg_dpc, base_vha, 0x4009,
 			    "Quiescence mode scheduled.\n");
-			qla82xx_device_state_handler(base_vha);
-			clear_bit(ISP_QUIESCE_NEEDED, &base_vha->dpc_flags);
-			if (!ha->flags.quiesce_owner) {
-				qla2x00_perform_loop_resync(base_vha);
-
-				qla82xx_idc_lock(ha);
-				qla82xx_clear_qsnt_ready(base_vha);
-				qla82xx_idc_unlock(ha);
+			if (IS_QLA82XX(ha)) {
+				qla82xx_device_state_handler(base_vha);
+				clear_bit(ISP_QUIESCE_NEEDED,
+				    &base_vha->dpc_flags);
+				if (!ha->flags.quiesce_owner) {
+					qla2x00_perform_loop_resync(base_vha);
+
+					qla82xx_idc_lock(ha);
+					qla82xx_clear_qsnt_ready(base_vha);
+					qla82xx_idc_unlock(ha);
+				}
+			} else {
+				clear_bit(ISP_QUIESCE_NEEDED,
+				    &base_vha->dpc_flags);
+				qla2x00_quiesce_io(base_vha);
 			}
 			ql_dbg(ql_dbg_dpc, base_vha, 0x400a,
 			    "Quiescence mode end.\n");
@@ -4326,7 +5037,7 @@ uint32_t qla82xx_error_recovery(scsi_qla_host_t *base_vha)
 		qla82xx_idc_lock(ha);
 
 		qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-		    QLA82XX_DEV_INITIALIZING);
+		    QLA8XXX_DEV_INITIALIZING);
 
 		qla82xx_wr_32(ha, QLA82XX_CRB_DRV_IDC_VERSION,
 		    QLA82XX_IDC_VERSION);
@@ -4350,12 +5061,12 @@ uint32_t qla82xx_error_recovery(scsi_qla_host_t *base_vha)
 			    "HW State: FAILED.\n");
 			qla82xx_clear_drv_active(ha);
 			qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_FAILED);
+			    QLA8XXX_DEV_FAILED);
 		} else {
 			ql_log(ql_log_info, base_vha, 0x900c,
 			    "HW State: READY.\n");
 			qla82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_READY);
+			    QLA8XXX_DEV_READY);
 			qla82xx_idc_unlock(ha);
 			ha->flags.isp82xx_fw_hung = 0;
 			rval = qla82xx_restart_isp(base_vha);
@@ -4370,7 +5081,7 @@ uint32_t qla82xx_error_recovery(scsi_qla_host_t *base_vha)
 		    "This devfn is not reset owner = 0x%x.\n",
 		    ha->pdev->devfn);
 		if ((qla82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE) ==
-		    QLA82XX_DEV_READY)) {
+		    QLA8XXX_DEV_READY)) {
 			ha->flags.isp82xx_fw_hung = 0;
 			rval = qla82xx_restart_isp(base_vha);
 			qla82xx_idc_lock(ha);
@@ -4495,6 +5206,7 @@ static struct pci_device_id qla2xxx_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2031) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP8001) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP8021) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP8031) },
 	{ 0 },
 };
 MODULE_DEVICE_TABLE(pci, qla2xxx_pci_tbl);
diff --git a/drivers/scsi/qla2xxx/qla_settings.h b/drivers/scsi/qla2xxx/qla_settings.h
index d70f0300898..892a81e457b 100644
--- a/drivers/scsi/qla2xxx/qla_settings.h
+++ b/drivers/scsi/qla2xxx/qla_settings.h
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla2xxx/qla_sup.c b/drivers/scsi/qla2xxx/qla_sup.c
index a683e766d1a..32fdc2a66dd 100644
--- a/drivers/scsi/qla2xxx/qla_sup.c
+++ b/drivers/scsi/qla2xxx/qla_sup.c
@@ -1,6 +1,6 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
@@ -966,16 +966,16 @@ qla2xxx_get_idc_param(scsi_qla_host_t *vha)
 		QLA82XX_IDC_PARAM_ADDR , 8);
 
 	if (*wptr == __constant_cpu_to_le32(0xffffffff)) {
-		ha->nx_dev_init_timeout = QLA82XX_ROM_DEV_INIT_TIMEOUT;
-		ha->nx_reset_timeout = QLA82XX_ROM_DRV_RESET_ACK_TIMEOUT;
+		ha->fcoe_dev_init_timeout = QLA82XX_ROM_DEV_INIT_TIMEOUT;
+		ha->fcoe_reset_timeout = QLA82XX_ROM_DRV_RESET_ACK_TIMEOUT;
 	} else {
-		ha->nx_dev_init_timeout = le32_to_cpu(*wptr++);
-		ha->nx_reset_timeout = le32_to_cpu(*wptr);
+		ha->fcoe_dev_init_timeout = le32_to_cpu(*wptr++);
+		ha->fcoe_reset_timeout = le32_to_cpu(*wptr);
 	}
 	ql_dbg(ql_dbg_init, vha, 0x004e,
-	    "nx_dev_init_timeout=%d "
-	    "nx_reset_timeout=%d.\n", ha->nx_dev_init_timeout,
-	    ha->nx_reset_timeout);
+	    "fcoe_dev_init_timeout=%d "
+	    "fcoe_reset_timeout=%d.\n", ha->fcoe_dev_init_timeout,
+	    ha->fcoe_reset_timeout);
 	return;
 }
 
@@ -1017,7 +1017,7 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 	    !IS_CNA_CAPABLE(ha) && !IS_QLA2031(ha))
 		return;
 
-	if (ha->flags.isp82xx_reset_hdlr_active)
+	if (ha->flags.nic_core_reset_hdlr_active)
 		return;
 
 	ha->isp_ops->read_optrom(vha, (uint8_t *)&hdr,
@@ -1662,6 +1662,23 @@ qla24xx_beacon_blink(struct scsi_qla_host *vha)
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 }
 
+static uint32_t
+qla83xx_select_led_port(struct qla_hw_data *ha)
+{
+	uint32_t led_select_value = 0;
+
+	if (!IS_QLA83XX(ha))
+		goto out;
+
+	if (ha->flags.port0)
+		led_select_value = QLA83XX_LED_PORT0;
+	else
+		led_select_value = QLA83XX_LED_PORT1;
+
+out:
+	return led_select_value;
+}
+
 void
 qla83xx_beacon_blink(struct scsi_qla_host *vha)
 {
@@ -1669,22 +1686,34 @@ qla83xx_beacon_blink(struct scsi_qla_host *vha)
 	struct qla_hw_data *ha = vha->hw;
 	uint16_t led_cfg[6];
 	uint16_t orig_led_cfg[6];
+	uint32_t led_10_value, led_43_value;
 
 	if (!IS_QLA83XX(ha) && !IS_QLA81XX(ha))
 		return;
 
-	if (IS_QLA2031(ha) && ha->beacon_blink_led) {
-		if (ha->flags.port0)
-			led_select_value = 0x00201320;
-		else
-			led_select_value = 0x00201328;
+	if (!ha->beacon_blink_led)
+		return;
+
+	if (IS_QLA2031(ha)) {
+		led_select_value = qla83xx_select_led_port(ha);
 
-		qla83xx_write_remote_reg(vha, led_select_value, 0x40002000);
-		qla83xx_write_remote_reg(vha, led_select_value + 4, 0x40002000);
+		qla83xx_wr_reg(vha, led_select_value, 0x40002000);
+		qla83xx_wr_reg(vha, led_select_value + 4, 0x40002000);
+		msleep(1000);
+		qla83xx_wr_reg(vha, led_select_value, 0x40004000);
+		qla83xx_wr_reg(vha, led_select_value + 4, 0x40004000);
+	} else if (IS_QLA8031(ha)) {
+		led_select_value = qla83xx_select_led_port(ha);
+
+		qla83xx_rd_reg(vha, led_select_value, &led_10_value);
+		qla83xx_rd_reg(vha, led_select_value + 0x10, &led_43_value);
+		qla83xx_wr_reg(vha, led_select_value, 0x01f44000);
+		msleep(500);
+		qla83xx_wr_reg(vha, led_select_value, 0x400001f4);
 		msleep(1000);
-		qla83xx_write_remote_reg(vha, led_select_value, 0x40004000);
-		qla83xx_write_remote_reg(vha, led_select_value + 4, 0x40004000);
-	} else if ((IS_QLA8031(ha) || IS_QLA81XX(ha)) && ha->beacon_blink_led) {
+		qla83xx_wr_reg(vha, led_select_value, led_10_value);
+		qla83xx_wr_reg(vha, led_select_value + 0x10, led_43_value);
+	} else if (IS_QLA81XX(ha)) {
 		int rval;
 
 		/* Save Current */
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index f5fdb16bec9..cfe934e1af4 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -1,15 +1,15 @@
 /*
  * QLogic Fibre Channel HBA Driver
- * Copyright (c)  2003-2011 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla2xxx for copyright and licensing details.
  */
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "8.04.00.03-k"
+#define QLA2XXX_VERSION      "8.04.00.07-k"
 
 #define QLA_DRIVER_MAJOR_VER	8
 #define QLA_DRIVER_MINOR_VER	4
 #define QLA_DRIVER_PATCH_VER	0
-#define QLA_DRIVER_BETA_VER	3
+#define QLA_DRIVER_BETA_VER	0
diff --git a/drivers/scsi/qla4xxx/Kconfig b/drivers/scsi/qla4xxx/Kconfig
index f1ad02ea212..e4dc7c733c2 100644
--- a/drivers/scsi/qla4xxx/Kconfig
+++ b/drivers/scsi/qla4xxx/Kconfig
@@ -4,5 +4,5 @@ config SCSI_QLA_ISCSI
 	select SCSI_ISCSI_ATTRS
 	select ISCSI_BOOT_SYSFS
 	---help---
-	This driver supports the QLogic 40xx (ISP4XXX) and 8022 (ISP82XX)
-	iSCSI host adapter family.
+	This driver supports the QLogic 40xx (ISP4XXX), 8022 (ISP82XX)
+	and 8032 (ISP83XX) iSCSI host adapter family.
diff --git a/drivers/scsi/qla4xxx/Makefile b/drivers/scsi/qla4xxx/Makefile
index 5b44139ff43..4230977748c 100644
--- a/drivers/scsi/qla4xxx/Makefile
+++ b/drivers/scsi/qla4xxx/Makefile
@@ -1,5 +1,5 @@
 qla4xxx-y := ql4_os.o ql4_init.o ql4_mbx.o ql4_iocb.o ql4_isr.o \
-		ql4_nx.o ql4_nvram.o ql4_dbg.o ql4_attr.o ql4_bsg.o
+		ql4_nx.o ql4_nvram.o ql4_dbg.o ql4_attr.o ql4_bsg.o ql4_83xx.o
 
 obj-$(CONFIG_SCSI_QLA_ISCSI) += qla4xxx.o
 
diff --git a/drivers/scsi/qla4xxx/ql4_83xx.c b/drivers/scsi/qla4xxx/ql4_83xx.c
new file mode 100644
index 00000000000..6e9af20be12
--- /dev/null
+++ b/drivers/scsi/qla4xxx/ql4_83xx.c
@@ -0,0 +1,1611 @@
+/*
+ * QLogic iSCSI HBA Driver
+ * Copyright (c)   2003-2012 QLogic Corporation
+ *
+ * See LICENSE.qla4xxx for copyright and licensing details.
+ */
+
+#include <linux/ratelimit.h>
+
+#include "ql4_def.h"
+#include "ql4_version.h"
+#include "ql4_glbl.h"
+#include "ql4_dbg.h"
+#include "ql4_inline.h"
+
+uint32_t qla4_83xx_rd_reg(struct scsi_qla_host *ha, ulong addr)
+{
+	return readl((void __iomem *)(ha->nx_pcibase + addr));
+}
+
+void qla4_83xx_wr_reg(struct scsi_qla_host *ha, ulong addr, uint32_t val)
+{
+	writel(val, (void __iomem *)(ha->nx_pcibase + addr));
+}
+
+static int qla4_83xx_set_win_base(struct scsi_qla_host *ha, uint32_t addr)
+{
+	uint32_t val;
+	int ret_val = QLA_SUCCESS;
+
+	qla4_83xx_wr_reg(ha, QLA83XX_CRB_WIN_FUNC(ha->func_num), addr);
+	val = qla4_83xx_rd_reg(ha, QLA83XX_CRB_WIN_FUNC(ha->func_num));
+	if (val != addr) {
+		ql4_printk(KERN_ERR, ha, "%s: Failed to set register window : addr written 0x%x, read 0x%x!\n",
+			   __func__, addr, val);
+		ret_val = QLA_ERROR;
+	}
+
+	return ret_val;
+}
+
+int qla4_83xx_rd_reg_indirect(struct scsi_qla_host *ha, uint32_t addr,
+			      uint32_t *data)
+{
+	int ret_val;
+
+	ret_val = qla4_83xx_set_win_base(ha, addr);
+
+	if (ret_val == QLA_SUCCESS)
+		*data = qla4_83xx_rd_reg(ha, QLA83XX_WILDCARD);
+	else
+		ql4_printk(KERN_ERR, ha, "%s: failed read of addr 0x%x!\n",
+			   __func__, addr);
+
+	return ret_val;
+}
+
+int qla4_83xx_wr_reg_indirect(struct scsi_qla_host *ha, uint32_t addr,
+			      uint32_t data)
+{
+	int ret_val;
+
+	ret_val = qla4_83xx_set_win_base(ha, addr);
+
+	if (ret_val == QLA_SUCCESS)
+		qla4_83xx_wr_reg(ha, QLA83XX_WILDCARD, data);
+	else
+		ql4_printk(KERN_ERR, ha, "%s: failed wrt to addr 0x%x, data 0x%x\n",
+			   __func__, addr, data);
+
+	return ret_val;
+}
+
+static int qla4_83xx_flash_lock(struct scsi_qla_host *ha)
+{
+	int lock_owner;
+	int timeout = 0;
+	uint32_t lock_status = 0;
+	int ret_val = QLA_SUCCESS;
+
+	while (lock_status == 0) {
+		lock_status = qla4_83xx_rd_reg(ha, QLA83XX_FLASH_LOCK);
+		if (lock_status)
+			break;
+
+		if (++timeout >= QLA83XX_FLASH_LOCK_TIMEOUT / 20) {
+			lock_owner = qla4_83xx_rd_reg(ha,
+						      QLA83XX_FLASH_LOCK_ID);
+			ql4_printk(KERN_ERR, ha, "%s: flash lock by func %d failed, held by func %d\n",
+				   __func__, ha->func_num, lock_owner);
+			ret_val = QLA_ERROR;
+			break;
+		}
+		msleep(20);
+	}
+
+	qla4_83xx_wr_reg(ha, QLA83XX_FLASH_LOCK_ID, ha->func_num);
+	return ret_val;
+}
+
+static void qla4_83xx_flash_unlock(struct scsi_qla_host *ha)
+{
+	/* Reading FLASH_UNLOCK register unlocks the Flash */
+	qla4_83xx_wr_reg(ha, QLA83XX_FLASH_LOCK_ID, 0xFF);
+	qla4_83xx_rd_reg(ha, QLA83XX_FLASH_UNLOCK);
+}
+
+int qla4_83xx_flash_read_u32(struct scsi_qla_host *ha, uint32_t flash_addr,
+			     uint8_t *p_data, int u32_word_count)
+{
+	int i;
+	uint32_t u32_word;
+	uint32_t addr = flash_addr;
+	int ret_val = QLA_SUCCESS;
+
+	ret_val = qla4_83xx_flash_lock(ha);
+	if (ret_val == QLA_ERROR)
+		goto exit_lock_error;
+
+	if (addr & 0x03) {
+		ql4_printk(KERN_ERR, ha, "%s: Illegal addr = 0x%x\n",
+			   __func__, addr);
+		ret_val = QLA_ERROR;
+		goto exit_flash_read;
+	}
+
+	for (i = 0; i < u32_word_count; i++) {
+		ret_val = qla4_83xx_wr_reg_indirect(ha,
+						    QLA83XX_FLASH_DIRECT_WINDOW,
+						    (addr & 0xFFFF0000));
+		if (ret_val == QLA_ERROR) {
+			ql4_printk(KERN_ERR, ha, "%s: failed to write addr 0x%x to FLASH_DIRECT_WINDOW\n!",
+				   __func__, addr);
+			goto exit_flash_read;
+		}
+
+		ret_val = qla4_83xx_rd_reg_indirect(ha,
+						QLA83XX_FLASH_DIRECT_DATA(addr),
+						&u32_word);
+		if (ret_val == QLA_ERROR) {
+			ql4_printk(KERN_ERR, ha, "%s: failed to read addr 0x%x!\n",
+				   __func__, addr);
+			goto exit_flash_read;
+		}
+
+		*(__le32 *)p_data = le32_to_cpu(u32_word);
+		p_data = p_data + 4;
+		addr = addr + 4;
+	}
+
+exit_flash_read:
+	qla4_83xx_flash_unlock(ha);
+
+exit_lock_error:
+	return ret_val;
+}
+
+int qla4_83xx_lockless_flash_read_u32(struct scsi_qla_host *ha,
+				      uint32_t flash_addr, uint8_t *p_data,
+				      int u32_word_count)
+{
+	uint32_t i;
+	uint32_t u32_word;
+	uint32_t flash_offset;
+	uint32_t addr = flash_addr;
+	int ret_val = QLA_SUCCESS;
+
+	flash_offset = addr & (QLA83XX_FLASH_SECTOR_SIZE - 1);
+
+	if (addr & 0x3) {
+		ql4_printk(KERN_ERR, ha, "%s: Illegal addr = 0x%x\n",
+			   __func__, addr);
+		ret_val = QLA_ERROR;
+		goto exit_lockless_read;
+	}
+
+	ret_val = qla4_83xx_wr_reg_indirect(ha, QLA83XX_FLASH_DIRECT_WINDOW,
+					    addr);
+	if (ret_val == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: failed to write addr 0x%x to FLASH_DIRECT_WINDOW!\n",
+			   __func__, addr);
+		goto exit_lockless_read;
+	}
+
+	/* Check if data is spread across multiple sectors  */
+	if ((flash_offset + (u32_word_count * sizeof(uint32_t))) >
+	    (QLA83XX_FLASH_SECTOR_SIZE - 1)) {
+
+		/* Multi sector read */
+		for (i = 0; i < u32_word_count; i++) {
+			ret_val = qla4_83xx_rd_reg_indirect(ha,
+						QLA83XX_FLASH_DIRECT_DATA(addr),
+						&u32_word);
+			if (ret_val == QLA_ERROR) {
+				ql4_printk(KERN_ERR, ha, "%s: failed to read addr 0x%x!\n",
+					   __func__, addr);
+				goto exit_lockless_read;
+			}
+
+			*(__le32 *)p_data  = le32_to_cpu(u32_word);
+			p_data = p_data + 4;
+			addr = addr + 4;
+			flash_offset = flash_offset + 4;
+
+			if (flash_offset > (QLA83XX_FLASH_SECTOR_SIZE - 1)) {
+				/* This write is needed once for each sector */
+				ret_val = qla4_83xx_wr_reg_indirect(ha,
+						   QLA83XX_FLASH_DIRECT_WINDOW,
+						   addr);
+				if (ret_val == QLA_ERROR) {
+					ql4_printk(KERN_ERR, ha, "%s: failed to write addr 0x%x to FLASH_DIRECT_WINDOW!\n",
+						   __func__, addr);
+					goto exit_lockless_read;
+				}
+				flash_offset = 0;
+			}
+		}
+	} else {
+		/* Single sector read */
+		for (i = 0; i < u32_word_count; i++) {
+			ret_val = qla4_83xx_rd_reg_indirect(ha,
+						QLA83XX_FLASH_DIRECT_DATA(addr),
+						&u32_word);
+			if (ret_val == QLA_ERROR) {
+				ql4_printk(KERN_ERR, ha, "%s: failed to read addr 0x%x!\n",
+					   __func__, addr);
+				goto exit_lockless_read;
+			}
+
+			*(__le32 *)p_data = le32_to_cpu(u32_word);
+			p_data = p_data + 4;
+			addr = addr + 4;
+		}
+	}
+
+exit_lockless_read:
+	return ret_val;
+}
+
+void qla4_83xx_rom_lock_recovery(struct scsi_qla_host *ha)
+{
+	if (qla4_83xx_flash_lock(ha))
+		ql4_printk(KERN_INFO, ha, "%s: Resetting rom lock\n", __func__);
+
+	/*
+	 * We got the lock, or someone else is holding the lock
+	 * since we are restting, forcefully unlock
+	 */
+	qla4_83xx_flash_unlock(ha);
+}
+
+/**
+ * qla4_83xx_ms_mem_write_128b - Writes data to MS/off-chip memory
+ * @ha: Pointer to adapter structure
+ * @addr: Flash address to write to
+ * @data: Data to be written
+ * @count: word_count to be written
+ *
+ * Return: On success return QLA_SUCCESS
+ *	   On error return QLA_ERROR
+ **/
+static int qla4_83xx_ms_mem_write_128b(struct scsi_qla_host *ha, uint64_t addr,
+				       uint32_t *data, uint32_t count)
+{
+	int i, j;
+	uint32_t agt_ctrl;
+	unsigned long flags;
+	int ret_val = QLA_SUCCESS;
+
+	/* Only 128-bit aligned access */
+	if (addr & 0xF) {
+		ret_val = QLA_ERROR;
+		goto exit_ms_mem_write;
+	}
+
+	write_lock_irqsave(&ha->hw_lock, flags);
+
+	/* Write address */
+	ret_val = qla4_83xx_wr_reg_indirect(ha, MD_MIU_TEST_AGT_ADDR_HI, 0);
+	if (ret_val == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: write to AGT_ADDR_HI failed\n",
+			   __func__);
+		goto exit_ms_mem_write_unlock;
+	}
+
+	for (i = 0; i < count; i++, addr += 16) {
+		if (!((QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_QDR_NET,
+					     QLA8XXX_ADDR_QDR_NET_MAX)) ||
+		      (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_DDR_NET,
+					     QLA8XXX_ADDR_DDR_NET_MAX)))) {
+			ret_val = QLA_ERROR;
+			goto exit_ms_mem_write_unlock;
+		}
+
+		ret_val = qla4_83xx_wr_reg_indirect(ha, MD_MIU_TEST_AGT_ADDR_LO,
+						    addr);
+		/* Write data */
+		ret_val |= qla4_83xx_wr_reg_indirect(ha,
+						     MD_MIU_TEST_AGT_WRDATA_LO,
+						     *data++);
+		ret_val |= qla4_83xx_wr_reg_indirect(ha,
+						     MD_MIU_TEST_AGT_WRDATA_HI,
+						     *data++);
+		ret_val |= qla4_83xx_wr_reg_indirect(ha,
+						     MD_MIU_TEST_AGT_WRDATA_ULO,
+						     *data++);
+		ret_val |= qla4_83xx_wr_reg_indirect(ha,
+						     MD_MIU_TEST_AGT_WRDATA_UHI,
+						     *data++);
+		if (ret_val == QLA_ERROR) {
+			ql4_printk(KERN_ERR, ha, "%s: write to AGT_WRDATA failed\n",
+				   __func__);
+			goto exit_ms_mem_write_unlock;
+		}
+
+		/* Check write status */
+		ret_val = qla4_83xx_wr_reg_indirect(ha, MD_MIU_TEST_AGT_CTRL,
+						    MIU_TA_CTL_WRITE_ENABLE);
+		ret_val |= qla4_83xx_wr_reg_indirect(ha, MD_MIU_TEST_AGT_CTRL,
+						     MIU_TA_CTL_WRITE_START);
+		if (ret_val == QLA_ERROR) {
+			ql4_printk(KERN_ERR, ha, "%s: write to AGT_CTRL failed\n",
+				   __func__);
+			goto exit_ms_mem_write_unlock;
+		}
+
+		for (j = 0; j < MAX_CTL_CHECK; j++) {
+			ret_val = qla4_83xx_rd_reg_indirect(ha,
+							MD_MIU_TEST_AGT_CTRL,
+							&agt_ctrl);
+			if (ret_val == QLA_ERROR) {
+				ql4_printk(KERN_ERR, ha, "%s: failed to read MD_MIU_TEST_AGT_CTRL\n",
+					   __func__);
+				goto exit_ms_mem_write_unlock;
+			}
+			if ((agt_ctrl & MIU_TA_CTL_BUSY) == 0)
+				break;
+		}
+
+		/* Status check failed */
+		if (j >= MAX_CTL_CHECK) {
+			printk_ratelimited(KERN_ERR "%s: MS memory write failed!\n",
+					   __func__);
+			ret_val = QLA_ERROR;
+			goto exit_ms_mem_write_unlock;
+		}
+	}
+
+exit_ms_mem_write_unlock:
+	write_unlock_irqrestore(&ha->hw_lock, flags);
+
+exit_ms_mem_write:
+	return ret_val;
+}
+
+#define INTENT_TO_RECOVER	0x01
+#define PROCEED_TO_RECOVER	0x02
+
+static int qla4_83xx_lock_recovery(struct scsi_qla_host *ha)
+{
+
+	uint32_t lock = 0, lockid;
+	int ret_val = QLA_ERROR;
+
+	lockid = ha->isp_ops->rd_reg_direct(ha, QLA83XX_DRV_LOCKRECOVERY);
+
+	/* Check for other Recovery in progress, go wait */
+	if ((lockid & 0x3) != 0)
+		goto exit_lock_recovery;
+
+	/* Intent to Recover */
+	ha->isp_ops->wr_reg_direct(ha, QLA83XX_DRV_LOCKRECOVERY,
+				   (ha->func_num << 2) | INTENT_TO_RECOVER);
+
+	msleep(200);
+
+	/* Check Intent to Recover is advertised */
+	lockid = ha->isp_ops->rd_reg_direct(ha, QLA83XX_DRV_LOCKRECOVERY);
+	if ((lockid & 0x3C) != (ha->func_num << 2))
+		goto exit_lock_recovery;
+
+	ql4_printk(KERN_INFO, ha, "%s: IDC Lock recovery initiated for func %d\n",
+		   __func__, ha->func_num);
+
+	/* Proceed to Recover */
+	ha->isp_ops->wr_reg_direct(ha, QLA83XX_DRV_LOCKRECOVERY,
+				   (ha->func_num << 2) | PROCEED_TO_RECOVER);
+
+	/* Force Unlock */
+	ha->isp_ops->wr_reg_direct(ha, QLA83XX_DRV_LOCK_ID, 0xFF);
+	ha->isp_ops->rd_reg_direct(ha, QLA83XX_DRV_UNLOCK);
+
+	/* Clear bits 0-5 in IDC_RECOVERY register*/
+	ha->isp_ops->wr_reg_direct(ha, QLA83XX_DRV_LOCKRECOVERY, 0);
+
+	/* Get lock */
+	lock = ha->isp_ops->rd_reg_direct(ha, QLA83XX_DRV_LOCK);
+	if (lock) {
+		lockid = ha->isp_ops->rd_reg_direct(ha, QLA83XX_DRV_LOCK_ID);
+		lockid = ((lockid + (1 << 8)) & ~0xFF) | ha->func_num;
+		ha->isp_ops->wr_reg_direct(ha, QLA83XX_DRV_LOCK_ID, lockid);
+		ret_val = QLA_SUCCESS;
+	}
+
+exit_lock_recovery:
+	return ret_val;
+}
+
+#define	QLA83XX_DRV_LOCK_MSLEEP		200
+
+int qla4_83xx_drv_lock(struct scsi_qla_host *ha)
+{
+	int timeout = 0;
+	uint32_t status = 0;
+	int ret_val = QLA_SUCCESS;
+	uint32_t first_owner = 0;
+	uint32_t tmo_owner = 0;
+	uint32_t lock_id;
+	uint32_t func_num;
+	uint32_t lock_cnt;
+
+	while (status == 0) {
+		status = qla4_83xx_rd_reg(ha, QLA83XX_DRV_LOCK);
+		if (status) {
+			/* Increment Counter (8-31) and update func_num (0-7) on
+			 * getting a successful lock  */
+			lock_id = qla4_83xx_rd_reg(ha, QLA83XX_DRV_LOCK_ID);
+			lock_id = ((lock_id + (1 << 8)) & ~0xFF) | ha->func_num;
+			qla4_83xx_wr_reg(ha, QLA83XX_DRV_LOCK_ID, lock_id);
+			break;
+		}
+
+		if (timeout == 0)
+			/* Save counter + ID of function holding the lock for
+			 * first failure */
+			first_owner = ha->isp_ops->rd_reg_direct(ha,
+							  QLA83XX_DRV_LOCK_ID);
+
+		if (++timeout >=
+		    (QLA83XX_DRV_LOCK_TIMEOUT / QLA83XX_DRV_LOCK_MSLEEP)) {
+			tmo_owner = qla4_83xx_rd_reg(ha, QLA83XX_DRV_LOCK_ID);
+			func_num = tmo_owner & 0xFF;
+			lock_cnt = tmo_owner >> 8;
+			ql4_printk(KERN_INFO, ha, "%s: Lock by func %d failed after 2s, lock held by func %d, lock count %d, first_owner %d\n",
+				   __func__, ha->func_num, func_num, lock_cnt,
+				   (first_owner & 0xFF));
+
+			if (first_owner != tmo_owner) {
+				/* Some other driver got lock, OR same driver
+				 * got lock again (counter value changed), when
+				 * we were waiting for lock.
+				 * Retry for another 2 sec */
+				ql4_printk(KERN_INFO, ha, "%s: IDC lock failed for func %d\n",
+					   __func__, ha->func_num);
+				timeout = 0;
+			} else {
+				/* Same driver holding lock > 2sec.
+				 * Force Recovery */
+				ret_val = qla4_83xx_lock_recovery(ha);
+				if (ret_val == QLA_SUCCESS) {
+					/* Recovered and got lock */
+					ql4_printk(KERN_INFO, ha, "%s: IDC lock Recovery by %d successful\n",
+						   __func__, ha->func_num);
+					break;
+				}
+				/* Recovery Failed, some other function
+				 * has the lock, wait for 2secs and retry */
+				ql4_printk(KERN_INFO, ha, "%s: IDC lock Recovery by %d failed, Retrying timout\n",
+					   __func__, ha->func_num);
+				timeout = 0;
+			}
+		}
+		msleep(QLA83XX_DRV_LOCK_MSLEEP);
+	}
+
+	return ret_val;
+}
+
+void qla4_83xx_drv_unlock(struct scsi_qla_host *ha)
+{
+	int id;
+
+	id = qla4_83xx_rd_reg(ha, QLA83XX_DRV_LOCK_ID);
+
+	if ((id & 0xFF) != ha->func_num) {
+		ql4_printk(KERN_ERR, ha, "%s: IDC Unlock by %d failed, lock owner is %d\n",
+			   __func__, ha->func_num, (id & 0xFF));
+		return;
+	}
+
+	/* Keep lock counter value, update the ha->func_num to 0xFF */
+	qla4_83xx_wr_reg(ha, QLA83XX_DRV_LOCK_ID, (id | 0xFF));
+	qla4_83xx_rd_reg(ha, QLA83XX_DRV_UNLOCK);
+}
+
+void qla4_83xx_set_idc_dontreset(struct scsi_qla_host *ha)
+{
+	uint32_t idc_ctrl;
+
+	idc_ctrl = qla4_83xx_rd_reg(ha, QLA83XX_IDC_DRV_CTRL);
+	idc_ctrl |= DONTRESET_BIT0;
+	qla4_83xx_wr_reg(ha, QLA83XX_IDC_DRV_CTRL, idc_ctrl);
+	DEBUG2(ql4_printk(KERN_INFO, ha, "%s: idc_ctrl = %d\n", __func__,
+			  idc_ctrl));
+}
+
+void qla4_83xx_clear_idc_dontreset(struct scsi_qla_host *ha)
+{
+	uint32_t idc_ctrl;
+
+	idc_ctrl = qla4_83xx_rd_reg(ha, QLA83XX_IDC_DRV_CTRL);
+	idc_ctrl &= ~DONTRESET_BIT0;
+	qla4_83xx_wr_reg(ha, QLA83XX_IDC_DRV_CTRL, idc_ctrl);
+	DEBUG2(ql4_printk(KERN_INFO, ha, "%s: idc_ctrl = %d\n", __func__,
+			  idc_ctrl));
+}
+
+int qla4_83xx_idc_dontreset(struct scsi_qla_host *ha)
+{
+	uint32_t idc_ctrl;
+
+	idc_ctrl = qla4_83xx_rd_reg(ha, QLA83XX_IDC_DRV_CTRL);
+	return idc_ctrl & DONTRESET_BIT0;
+}
+
+/*-------------------------IDC State Machine ---------------------*/
+
+enum {
+	UNKNOWN_CLASS = 0,
+	NIC_CLASS,
+	FCOE_CLASS,
+	ISCSI_CLASS
+};
+
+struct device_info {
+	int func_num;
+	int device_type;
+	int port_num;
+};
+
+static int qla4_83xx_can_perform_reset(struct scsi_qla_host *ha)
+{
+	uint32_t drv_active;
+	uint32_t dev_part, dev_part1, dev_part2;
+	int i;
+	struct device_info device_map[16];
+	int func_nibble;
+	int nibble;
+	int nic_present = 0;
+	int iscsi_present = 0;
+	int iscsi_func_low = 0;
+
+	/* Use the dev_partition register to determine the PCI function number
+	 * and then check drv_active register to see which driver is loaded */
+	dev_part1 = qla4_83xx_rd_reg(ha,
+				     ha->reg_tbl[QLA8XXX_CRB_DEV_PART_INFO]);
+	dev_part2 = qla4_83xx_rd_reg(ha, QLA83XX_CRB_DEV_PART_INFO2);
+	drv_active = qla4_83xx_rd_reg(ha, ha->reg_tbl[QLA8XXX_CRB_DRV_ACTIVE]);
+
+	/* Each function has 4 bits in dev_partition Info register,
+	 * Lower 2 bits - device type, Upper 2 bits - physical port number */
+	dev_part = dev_part1;
+	for (i = nibble = 0; i <= 15; i++, nibble++) {
+		func_nibble = dev_part & (0xF << (nibble * 4));
+		func_nibble >>= (nibble * 4);
+		device_map[i].func_num = i;
+		device_map[i].device_type = func_nibble & 0x3;
+		device_map[i].port_num = func_nibble & 0xC;
+
+		if (device_map[i].device_type == NIC_CLASS) {
+			if (drv_active & (1 << device_map[i].func_num)) {
+				nic_present++;
+				break;
+			}
+		} else if (device_map[i].device_type == ISCSI_CLASS) {
+			if (drv_active & (1 << device_map[i].func_num)) {
+				if (!iscsi_present ||
+				    (iscsi_present &&
+				     (iscsi_func_low > device_map[i].func_num)))
+					iscsi_func_low = device_map[i].func_num;
+
+				iscsi_present++;
+			}
+		}
+
+		/* For function_num[8..15] get info from dev_part2 register */
+		if (nibble == 7) {
+			nibble = 0;
+			dev_part = dev_part2;
+		}
+	}
+
+	/* NIC, iSCSI and FCOE are the Reset owners based on order, NIC gets
+	 * precedence over iSCSI and FCOE and iSCSI over FCOE, based on drivers
+	 * present. */
+	if (!nic_present && (ha->func_num == iscsi_func_low)) {
+		DEBUG2(ql4_printk(KERN_INFO, ha,
+				  "%s: can reset - NIC not present and lower iSCSI function is %d\n",
+				  __func__, ha->func_num));
+		return 1;
+	}
+
+	return 0;
+}
+
+/**
+ * qla4_83xx_need_reset_handler - Code to start reset sequence
+ * @ha: pointer to adapter structure
+ *
+ * Note: IDC lock must be held upon entry
+ **/
+void qla4_83xx_need_reset_handler(struct scsi_qla_host *ha)
+{
+	uint32_t dev_state, drv_state, drv_active;
+	unsigned long reset_timeout, dev_init_timeout;
+
+	ql4_printk(KERN_INFO, ha, "%s: Performing ISP error recovery\n",
+		   __func__);
+
+	if (!test_bit(AF_8XXX_RST_OWNER, &ha->flags)) {
+		DEBUG2(ql4_printk(KERN_INFO, ha, "%s: reset acknowledged\n",
+				  __func__));
+		qla4_8xxx_set_rst_ready(ha);
+
+		/* Non-reset owners ACK Reset and wait for device INIT state
+		 * as part of Reset Recovery by Reset Owner */
+		dev_init_timeout = jiffies + (ha->nx_dev_init_timeout * HZ);
+
+		do {
+			if (time_after_eq(jiffies, dev_init_timeout)) {
+				ql4_printk(KERN_INFO, ha, "%s: Non Reset owner dev init timeout\n",
+					   __func__);
+				break;
+			}
+
+			ha->isp_ops->idc_unlock(ha);
+			msleep(1000);
+			ha->isp_ops->idc_lock(ha);
+
+			dev_state = qla4_8xxx_rd_direct(ha,
+							QLA8XXX_CRB_DEV_STATE);
+		} while (dev_state == QLA8XXX_DEV_NEED_RESET);
+	} else {
+		qla4_8xxx_set_rst_ready(ha);
+		reset_timeout = jiffies + (ha->nx_reset_timeout * HZ);
+		drv_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_STATE);
+		drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+
+		ql4_printk(KERN_INFO, ha, "%s: drv_state = 0x%x, drv_active = 0x%x\n",
+			   __func__, drv_state, drv_active);
+
+		while (drv_state != drv_active) {
+			if (time_after_eq(jiffies, reset_timeout)) {
+				ql4_printk(KERN_INFO, ha, "%s: %s: RESET TIMEOUT! drv_state: 0x%08x, drv_active: 0x%08x\n",
+					   __func__, DRIVER_NAME, drv_state,
+					   drv_active);
+				break;
+			}
+
+			ha->isp_ops->idc_unlock(ha);
+			msleep(1000);
+			ha->isp_ops->idc_lock(ha);
+
+			drv_state = qla4_8xxx_rd_direct(ha,
+							QLA8XXX_CRB_DRV_STATE);
+			drv_active = qla4_8xxx_rd_direct(ha,
+							QLA8XXX_CRB_DRV_ACTIVE);
+		}
+
+		if (drv_state != drv_active) {
+			ql4_printk(KERN_INFO, ha, "%s: Reset_owner turning off drv_active of non-acking function 0x%x\n",
+				   __func__, (drv_active ^ drv_state));
+			drv_active = drv_active & drv_state;
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_ACTIVE,
+					    drv_active);
+		}
+
+		clear_bit(AF_8XXX_RST_OWNER, &ha->flags);
+		/* Start Reset Recovery */
+		qla4_8xxx_device_bootstrap(ha);
+	}
+}
+
+void qla4_83xx_get_idc_param(struct scsi_qla_host *ha)
+{
+	uint32_t idc_params, ret_val;
+
+	ret_val = qla4_83xx_flash_read_u32(ha, QLA83XX_IDC_PARAM_ADDR,
+					   (uint8_t *)&idc_params, 1);
+	if (ret_val == QLA_SUCCESS) {
+		ha->nx_dev_init_timeout = idc_params & 0xFFFF;
+		ha->nx_reset_timeout = (idc_params >> 16) & 0xFFFF;
+	} else {
+		ha->nx_dev_init_timeout = ROM_DEV_INIT_TIMEOUT;
+		ha->nx_reset_timeout = ROM_DRV_RESET_ACK_TIMEOUT;
+	}
+
+	DEBUG2(ql4_printk(KERN_DEBUG, ha,
+			  "%s: ha->nx_dev_init_timeout = %d, ha->nx_reset_timeout = %d\n",
+			  __func__, ha->nx_dev_init_timeout,
+			  ha->nx_reset_timeout));
+}
+
+/*-------------------------Reset Sequence Functions-----------------------*/
+
+static void qla4_83xx_dump_reset_seq_hdr(struct scsi_qla_host *ha)
+{
+	uint8_t *phdr;
+
+	if (!ha->reset_tmplt.buff) {
+		ql4_printk(KERN_ERR, ha, "%s: Error: Invalid reset_seq_template\n",
+			   __func__);
+		return;
+	}
+
+	phdr = ha->reset_tmplt.buff;
+
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "Reset Template: 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X 0x%X\n",
+			  *phdr, *(phdr+1), *(phdr+2), *(phdr+3), *(phdr+4),
+			  *(phdr+5), *(phdr+6), *(phdr+7), *(phdr + 8),
+			  *(phdr+9), *(phdr+10), *(phdr+11), *(phdr+12),
+			  *(phdr+13), *(phdr+14), *(phdr+15)));
+}
+
+static int qla4_83xx_copy_bootloader(struct scsi_qla_host *ha)
+{
+	uint8_t *p_cache;
+	uint32_t src, count, size;
+	uint64_t dest;
+	int ret_val = QLA_SUCCESS;
+
+	src = QLA83XX_BOOTLOADER_FLASH_ADDR;
+	dest = qla4_83xx_rd_reg(ha, QLA83XX_BOOTLOADER_ADDR);
+	size = qla4_83xx_rd_reg(ha, QLA83XX_BOOTLOADER_SIZE);
+
+	/* 128 bit alignment check */
+	if (size & 0xF)
+		size = (size + 16) & ~0xF;
+
+	/* 16 byte count */
+	count = size/16;
+
+	p_cache = vmalloc(size);
+	if (p_cache == NULL) {
+		ql4_printk(KERN_ERR, ha, "%s: Failed to allocate memory for boot loader cache\n",
+			   __func__);
+		ret_val = QLA_ERROR;
+		goto exit_copy_bootloader;
+	}
+
+	ret_val = qla4_83xx_lockless_flash_read_u32(ha, src, p_cache,
+						    size / sizeof(uint32_t));
+	if (ret_val == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: Error reading firmware from flash\n",
+			   __func__);
+		goto exit_copy_error;
+	}
+	DEBUG2(ql4_printk(KERN_INFO, ha, "%s: Read firmware from flash\n",
+			  __func__));
+
+	/* 128 bit/16 byte write to MS memory */
+	ret_val = qla4_83xx_ms_mem_write_128b(ha, dest, (uint32_t *)p_cache,
+					      count);
+	if (ret_val == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: Error writing firmware to MS\n",
+			   __func__);
+		goto exit_copy_error;
+	}
+
+	DEBUG2(ql4_printk(KERN_INFO, ha, "%s: Wrote firmware size %d to MS\n",
+			  __func__, size));
+
+exit_copy_error:
+	vfree(p_cache);
+
+exit_copy_bootloader:
+	return ret_val;
+}
+
+static int qla4_83xx_check_cmd_peg_status(struct scsi_qla_host *ha)
+{
+	uint32_t val, ret_val = QLA_ERROR;
+	int retries = CRB_CMDPEG_CHECK_RETRY_COUNT;
+
+	do {
+		val = qla4_83xx_rd_reg(ha, QLA83XX_CMDPEG_STATE);
+		if (val == PHAN_INITIALIZE_COMPLETE) {
+			DEBUG2(ql4_printk(KERN_INFO, ha,
+					  "%s: Command Peg initialization complete. State=0x%x\n",
+					  __func__, val));
+			ret_val = QLA_SUCCESS;
+			break;
+		}
+		msleep(CRB_CMDPEG_CHECK_DELAY);
+	} while (--retries);
+
+	return ret_val;
+}
+
+/**
+ * qla4_83xx_poll_reg - Poll the given CRB addr for duration msecs till
+ * value read ANDed with test_mask is equal to test_result.
+ *
+ * @ha : Pointer to adapter structure
+ * @addr : CRB register address
+ * @duration : Poll for total of "duration" msecs
+ * @test_mask : Mask value read with "test_mask"
+ * @test_result : Compare (value&test_mask) with test_result.
+ **/
+static int qla4_83xx_poll_reg(struct scsi_qla_host *ha, uint32_t addr,
+			      int duration, uint32_t test_mask,
+			      uint32_t test_result)
+{
+	uint32_t value;
+	uint8_t retries;
+	int ret_val = QLA_SUCCESS;
+
+	ret_val = qla4_83xx_rd_reg_indirect(ha, addr, &value);
+	if (ret_val == QLA_ERROR)
+		goto exit_poll_reg;
+
+	retries = duration / 10;
+	do {
+		if ((value & test_mask) != test_result) {
+			msleep(duration / 10);
+			ret_val = qla4_83xx_rd_reg_indirect(ha, addr, &value);
+			if (ret_val == QLA_ERROR)
+				goto exit_poll_reg;
+
+			ret_val = QLA_ERROR;
+		} else {
+			ret_val = QLA_SUCCESS;
+			break;
+		}
+	} while (retries--);
+
+exit_poll_reg:
+	if (ret_val == QLA_ERROR) {
+		ha->reset_tmplt.seq_error++;
+		ql4_printk(KERN_ERR, ha, "%s: Poll Failed:  0x%08x 0x%08x 0x%08x\n",
+			   __func__, value, test_mask, test_result);
+	}
+
+	return ret_val;
+}
+
+static int qla4_83xx_reset_seq_checksum_test(struct scsi_qla_host *ha)
+{
+	uint32_t sum =  0;
+	uint16_t *buff = (uint16_t *)ha->reset_tmplt.buff;
+	int u16_count =  ha->reset_tmplt.hdr->size / sizeof(uint16_t);
+	int ret_val;
+
+	while (u16_count-- > 0)
+		sum += *buff++;
+
+	while (sum >> 16)
+		sum = (sum & 0xFFFF) +  (sum >> 16);
+
+	/* checksum of 0 indicates a valid template */
+	if (~sum) {
+		ret_val = QLA_SUCCESS;
+	} else {
+		ql4_printk(KERN_ERR, ha, "%s: Reset seq checksum failed\n",
+			   __func__);
+		ret_val = QLA_ERROR;
+	}
+
+	return ret_val;
+}
+
+/**
+ * qla4_83xx_read_reset_template - Read Reset Template from Flash
+ * @ha: Pointer to adapter structure
+ **/
+void qla4_83xx_read_reset_template(struct scsi_qla_host *ha)
+{
+	uint8_t *p_buff;
+	uint32_t addr, tmplt_hdr_def_size, tmplt_hdr_size;
+	uint32_t ret_val;
+
+	ha->reset_tmplt.seq_error = 0;
+	ha->reset_tmplt.buff = vmalloc(QLA83XX_RESTART_TEMPLATE_SIZE);
+	if (ha->reset_tmplt.buff == NULL) {
+		ql4_printk(KERN_ERR, ha, "%s: Failed to allocate reset template resources\n",
+			   __func__);
+		goto exit_read_reset_template;
+	}
+
+	p_buff = ha->reset_tmplt.buff;
+	addr = QLA83XX_RESET_TEMPLATE_ADDR;
+
+	tmplt_hdr_def_size = sizeof(struct qla4_83xx_reset_template_hdr) /
+				    sizeof(uint32_t);
+
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "%s: Read template hdr size %d from Flash\n",
+			  __func__, tmplt_hdr_def_size));
+
+	/* Copy template header from flash */
+	ret_val = qla4_83xx_flash_read_u32(ha, addr, p_buff,
+					   tmplt_hdr_def_size);
+	if (ret_val != QLA_SUCCESS) {
+		ql4_printk(KERN_ERR, ha, "%s: Failed to read reset template\n",
+			   __func__);
+		goto exit_read_template_error;
+	}
+
+	ha->reset_tmplt.hdr =
+		(struct qla4_83xx_reset_template_hdr *)ha->reset_tmplt.buff;
+
+	/* Validate the template header size and signature */
+	tmplt_hdr_size = ha->reset_tmplt.hdr->hdr_size/sizeof(uint32_t);
+	if ((tmplt_hdr_size != tmplt_hdr_def_size) ||
+	    (ha->reset_tmplt.hdr->signature != RESET_TMPLT_HDR_SIGNATURE)) {
+		ql4_printk(KERN_ERR, ha, "%s: Template Header size %d is invalid, tmplt_hdr_def_size %d\n",
+			   __func__, tmplt_hdr_size, tmplt_hdr_def_size);
+		goto exit_read_template_error;
+	}
+
+	addr = QLA83XX_RESET_TEMPLATE_ADDR + ha->reset_tmplt.hdr->hdr_size;
+	p_buff = ha->reset_tmplt.buff + ha->reset_tmplt.hdr->hdr_size;
+	tmplt_hdr_def_size = (ha->reset_tmplt.hdr->size -
+			      ha->reset_tmplt.hdr->hdr_size) / sizeof(uint32_t);
+
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "%s: Read rest of the template size %d\n",
+			  __func__, ha->reset_tmplt.hdr->size));
+
+	/* Copy rest of the template */
+	ret_val = qla4_83xx_flash_read_u32(ha, addr, p_buff,
+					   tmplt_hdr_def_size);
+	if (ret_val != QLA_SUCCESS) {
+		ql4_printk(KERN_ERR, ha, "%s: Failed to read reset tempelate\n",
+			   __func__);
+		goto exit_read_template_error;
+	}
+
+	/* Integrity check */
+	if (qla4_83xx_reset_seq_checksum_test(ha)) {
+		ql4_printk(KERN_ERR, ha, "%s: Reset Seq checksum failed!\n",
+			   __func__);
+		goto exit_read_template_error;
+	}
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "%s: Reset Seq checksum passed, Get stop, start and init seq offsets\n",
+			  __func__));
+
+	/* Get STOP, START, INIT sequence offsets */
+	ha->reset_tmplt.init_offset = ha->reset_tmplt.buff +
+				      ha->reset_tmplt.hdr->init_seq_offset;
+	ha->reset_tmplt.start_offset = ha->reset_tmplt.buff +
+				       ha->reset_tmplt.hdr->start_seq_offset;
+	ha->reset_tmplt.stop_offset = ha->reset_tmplt.buff +
+				      ha->reset_tmplt.hdr->hdr_size;
+	qla4_83xx_dump_reset_seq_hdr(ha);
+
+	goto exit_read_reset_template;
+
+exit_read_template_error:
+	vfree(ha->reset_tmplt.buff);
+
+exit_read_reset_template:
+	return;
+}
+
+/**
+ * qla4_83xx_read_write_crb_reg - Read from raddr and write value to waddr.
+ *
+ * @ha : Pointer to adapter structure
+ * @raddr : CRB address to read from
+ * @waddr : CRB address to write to
+ **/
+static void qla4_83xx_read_write_crb_reg(struct scsi_qla_host *ha,
+					 uint32_t raddr, uint32_t waddr)
+{
+	uint32_t value;
+
+	qla4_83xx_rd_reg_indirect(ha, raddr, &value);
+	qla4_83xx_wr_reg_indirect(ha, waddr, value);
+}
+
+/**
+ * qla4_83xx_rmw_crb_reg - Read Modify Write crb register
+ *
+ * This function read value from raddr, AND with test_mask,
+ * Shift Left,Right/OR/XOR with values RMW header and write value to waddr.
+ *
+ * @ha : Pointer to adapter structure
+ * @raddr : CRB address to read from
+ * @waddr : CRB address to write to
+ * @p_rmw_hdr : header with shift/or/xor values.
+ **/
+static void qla4_83xx_rmw_crb_reg(struct scsi_qla_host *ha, uint32_t raddr,
+				  uint32_t waddr,
+				  struct qla4_83xx_rmw *p_rmw_hdr)
+{
+	uint32_t value;
+
+	if (p_rmw_hdr->index_a)
+		value = ha->reset_tmplt.array[p_rmw_hdr->index_a];
+	else
+		qla4_83xx_rd_reg_indirect(ha, raddr, &value);
+
+	value &= p_rmw_hdr->test_mask;
+	value <<= p_rmw_hdr->shl;
+	value >>= p_rmw_hdr->shr;
+	value |= p_rmw_hdr->or_value;
+	value ^= p_rmw_hdr->xor_value;
+
+	qla4_83xx_wr_reg_indirect(ha, waddr, value);
+
+	return;
+}
+
+static void qla4_83xx_write_list(struct scsi_qla_host *ha,
+				 struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	struct qla4_83xx_entry *p_entry;
+	uint32_t i;
+
+	p_entry = (struct qla4_83xx_entry *)
+		  ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+
+	for (i = 0; i < p_hdr->count; i++, p_entry++) {
+		qla4_83xx_wr_reg_indirect(ha, p_entry->arg1, p_entry->arg2);
+		if (p_hdr->delay)
+			udelay((uint32_t)(p_hdr->delay));
+	}
+}
+
+static void qla4_83xx_read_write_list(struct scsi_qla_host *ha,
+				      struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	struct qla4_83xx_entry *p_entry;
+	uint32_t i;
+
+	p_entry = (struct qla4_83xx_entry *)
+		  ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+
+	for (i = 0; i < p_hdr->count; i++, p_entry++) {
+		qla4_83xx_read_write_crb_reg(ha, p_entry->arg1, p_entry->arg2);
+		if (p_hdr->delay)
+			udelay((uint32_t)(p_hdr->delay));
+	}
+}
+
+static void qla4_83xx_poll_list(struct scsi_qla_host *ha,
+				struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	long delay;
+	struct qla4_83xx_entry *p_entry;
+	struct qla4_83xx_poll *p_poll;
+	uint32_t i;
+	uint32_t value;
+
+	p_poll = (struct qla4_83xx_poll *)
+		 ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+
+	/* Entries start after 8 byte qla4_83xx_poll, poll header contains
+	 * the test_mask, test_value. */
+	p_entry = (struct qla4_83xx_entry *)((char *)p_poll +
+					     sizeof(struct qla4_83xx_poll));
+
+	delay = (long)p_hdr->delay;
+	if (!delay) {
+		for (i = 0; i < p_hdr->count; i++, p_entry++) {
+			qla4_83xx_poll_reg(ha, p_entry->arg1, delay,
+					   p_poll->test_mask,
+					   p_poll->test_value);
+		}
+	} else {
+		for (i = 0; i < p_hdr->count; i++, p_entry++) {
+			if (qla4_83xx_poll_reg(ha, p_entry->arg1, delay,
+					       p_poll->test_mask,
+					       p_poll->test_value)) {
+				qla4_83xx_rd_reg_indirect(ha, p_entry->arg1,
+							  &value);
+				qla4_83xx_rd_reg_indirect(ha, p_entry->arg2,
+							  &value);
+			}
+		}
+	}
+}
+
+static void qla4_83xx_poll_write_list(struct scsi_qla_host *ha,
+				      struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	long delay;
+	struct qla4_83xx_quad_entry *p_entry;
+	struct qla4_83xx_poll *p_poll;
+	uint32_t i;
+
+	p_poll = (struct qla4_83xx_poll *)
+		 ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+	p_entry = (struct qla4_83xx_quad_entry *)
+		  ((char *)p_poll + sizeof(struct qla4_83xx_poll));
+	delay = (long)p_hdr->delay;
+
+	for (i = 0; i < p_hdr->count; i++, p_entry++) {
+		qla4_83xx_wr_reg_indirect(ha, p_entry->dr_addr,
+					  p_entry->dr_value);
+		qla4_83xx_wr_reg_indirect(ha, p_entry->ar_addr,
+					  p_entry->ar_value);
+		if (delay) {
+			if (qla4_83xx_poll_reg(ha, p_entry->ar_addr, delay,
+					       p_poll->test_mask,
+					       p_poll->test_value)) {
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "%s: Timeout Error: poll list, item_num %d, entry_num %d\n",
+						  __func__, i,
+						  ha->reset_tmplt.seq_index));
+			}
+		}
+	}
+}
+
+static void qla4_83xx_read_modify_write(struct scsi_qla_host *ha,
+					struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	struct qla4_83xx_entry *p_entry;
+	struct qla4_83xx_rmw *p_rmw_hdr;
+	uint32_t i;
+
+	p_rmw_hdr = (struct qla4_83xx_rmw *)
+		    ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+	p_entry = (struct qla4_83xx_entry *)
+		  ((char *)p_rmw_hdr + sizeof(struct qla4_83xx_rmw));
+
+	for (i = 0; i < p_hdr->count; i++, p_entry++) {
+		qla4_83xx_rmw_crb_reg(ha, p_entry->arg1, p_entry->arg2,
+				      p_rmw_hdr);
+		if (p_hdr->delay)
+			udelay((uint32_t)(p_hdr->delay));
+	}
+}
+
+static void qla4_83xx_pause(struct scsi_qla_host *ha,
+			    struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	if (p_hdr->delay)
+		mdelay((uint32_t)((long)p_hdr->delay));
+}
+
+static void qla4_83xx_poll_read_list(struct scsi_qla_host *ha,
+				     struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	long delay;
+	int index;
+	struct qla4_83xx_quad_entry *p_entry;
+	struct qla4_83xx_poll *p_poll;
+	uint32_t i;
+	uint32_t value;
+
+	p_poll = (struct qla4_83xx_poll *)
+		 ((char *)p_hdr + sizeof(struct qla4_83xx_reset_entry_hdr));
+	p_entry = (struct qla4_83xx_quad_entry *)
+		  ((char *)p_poll + sizeof(struct qla4_83xx_poll));
+	delay = (long)p_hdr->delay;
+
+	for (i = 0; i < p_hdr->count; i++, p_entry++) {
+		qla4_83xx_wr_reg_indirect(ha, p_entry->ar_addr,
+					  p_entry->ar_value);
+		if (delay) {
+			if (qla4_83xx_poll_reg(ha, p_entry->ar_addr, delay,
+					       p_poll->test_mask,
+					       p_poll->test_value)) {
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "%s: Timeout Error: poll list, Item_num %d, entry_num %d\n",
+						  __func__, i,
+						  ha->reset_tmplt.seq_index));
+			} else {
+				index = ha->reset_tmplt.array_index;
+				qla4_83xx_rd_reg_indirect(ha, p_entry->dr_addr,
+							  &value);
+				ha->reset_tmplt.array[index++] = value;
+
+				if (index == QLA83XX_MAX_RESET_SEQ_ENTRIES)
+					ha->reset_tmplt.array_index = 1;
+			}
+		}
+	}
+}
+
+static void qla4_83xx_seq_end(struct scsi_qla_host *ha,
+			      struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	ha->reset_tmplt.seq_end = 1;
+}
+
+static void qla4_83xx_template_end(struct scsi_qla_host *ha,
+				   struct qla4_83xx_reset_entry_hdr *p_hdr)
+{
+	ha->reset_tmplt.template_end = 1;
+
+	if (ha->reset_tmplt.seq_error == 0) {
+		DEBUG2(ql4_printk(KERN_INFO, ha,
+				  "%s: Reset sequence completed SUCCESSFULLY.\n",
+				  __func__));
+	} else {
+		ql4_printk(KERN_ERR, ha, "%s: Reset sequence completed with some timeout errors.\n",
+			   __func__);
+	}
+}
+
+/**
+ * qla4_83xx_process_reset_template - Process reset template.
+ *
+ * Process all entries in reset template till entry with SEQ_END opcode,
+ * which indicates end of the reset template processing. Each entry has a
+ * Reset Entry header, entry opcode/command, with size of the entry, number
+ * of entries in sub-sequence and delay in microsecs or timeout in millisecs.
+ *
+ * @ha : Pointer to adapter structure
+ * @p_buff : Common reset entry header.
+ **/
+static void qla4_83xx_process_reset_template(struct scsi_qla_host *ha,
+					     char *p_buff)
+{
+	int index, entries;
+	struct qla4_83xx_reset_entry_hdr *p_hdr;
+	char *p_entry = p_buff;
+
+	ha->reset_tmplt.seq_end = 0;
+	ha->reset_tmplt.template_end = 0;
+	entries = ha->reset_tmplt.hdr->entries;
+	index = ha->reset_tmplt.seq_index;
+
+	for (; (!ha->reset_tmplt.seq_end) && (index  < entries); index++) {
+
+		p_hdr = (struct qla4_83xx_reset_entry_hdr *)p_entry;
+		switch (p_hdr->cmd) {
+		case OPCODE_NOP:
+			break;
+		case OPCODE_WRITE_LIST:
+			qla4_83xx_write_list(ha, p_hdr);
+			break;
+		case OPCODE_READ_WRITE_LIST:
+			qla4_83xx_read_write_list(ha, p_hdr);
+			break;
+		case OPCODE_POLL_LIST:
+			qla4_83xx_poll_list(ha, p_hdr);
+			break;
+		case OPCODE_POLL_WRITE_LIST:
+			qla4_83xx_poll_write_list(ha, p_hdr);
+			break;
+		case OPCODE_READ_MODIFY_WRITE:
+			qla4_83xx_read_modify_write(ha, p_hdr);
+			break;
+		case OPCODE_SEQ_PAUSE:
+			qla4_83xx_pause(ha, p_hdr);
+			break;
+		case OPCODE_SEQ_END:
+			qla4_83xx_seq_end(ha, p_hdr);
+			break;
+		case OPCODE_TMPL_END:
+			qla4_83xx_template_end(ha, p_hdr);
+			break;
+		case OPCODE_POLL_READ_LIST:
+			qla4_83xx_poll_read_list(ha, p_hdr);
+			break;
+		default:
+			ql4_printk(KERN_ERR, ha, "%s: Unknown command ==> 0x%04x on entry = %d\n",
+				   __func__, p_hdr->cmd, index);
+			break;
+		}
+
+		/* Set pointer to next entry in the sequence. */
+		p_entry += p_hdr->size;
+	}
+
+	ha->reset_tmplt.seq_index = index;
+}
+
+static void qla4_83xx_process_stop_seq(struct scsi_qla_host *ha)
+{
+	ha->reset_tmplt.seq_index = 0;
+	qla4_83xx_process_reset_template(ha, ha->reset_tmplt.stop_offset);
+
+	if (ha->reset_tmplt.seq_end != 1)
+		ql4_printk(KERN_ERR, ha, "%s: Abrupt STOP Sub-Sequence end.\n",
+			   __func__);
+}
+
+static void qla4_83xx_process_start_seq(struct scsi_qla_host *ha)
+{
+	qla4_83xx_process_reset_template(ha, ha->reset_tmplt.start_offset);
+
+	if (ha->reset_tmplt.template_end != 1)
+		ql4_printk(KERN_ERR, ha, "%s: Abrupt START Sub-Sequence end.\n",
+			   __func__);
+}
+
+static void qla4_83xx_process_init_seq(struct scsi_qla_host *ha)
+{
+	qla4_83xx_process_reset_template(ha, ha->reset_tmplt.init_offset);
+
+	if (ha->reset_tmplt.seq_end != 1)
+		ql4_printk(KERN_ERR, ha, "%s: Abrupt INIT Sub-Sequence end.\n",
+			   __func__);
+}
+
+static int qla4_83xx_restart(struct scsi_qla_host *ha)
+{
+	int ret_val = QLA_SUCCESS;
+
+	qla4_83xx_process_stop_seq(ha);
+
+	/* Collect minidump*/
+	if (!test_and_clear_bit(AF_83XX_NO_FW_DUMP, &ha->flags))
+		qla4_8xxx_get_minidump(ha);
+
+	qla4_83xx_process_init_seq(ha);
+
+	if (qla4_83xx_copy_bootloader(ha)) {
+		ql4_printk(KERN_ERR, ha, "%s: Copy bootloader, firmware restart failed!\n",
+			   __func__);
+		ret_val = QLA_ERROR;
+		goto exit_restart;
+	}
+
+	qla4_83xx_wr_reg(ha, QLA83XX_FW_IMAGE_VALID, QLA83XX_BOOT_FROM_FLASH);
+	qla4_83xx_process_start_seq(ha);
+
+exit_restart:
+	return ret_val;
+}
+
+int qla4_83xx_start_firmware(struct scsi_qla_host *ha)
+{
+	int ret_val = QLA_SUCCESS;
+
+	ret_val = qla4_83xx_restart(ha);
+	if (ret_val == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: Restart error\n", __func__);
+		goto exit_start_fw;
+	} else {
+		DEBUG2(ql4_printk(KERN_INFO, ha, "%s: Restart done\n",
+				  __func__));
+	}
+
+	ret_val = qla4_83xx_check_cmd_peg_status(ha);
+	if (ret_val == QLA_ERROR)
+		ql4_printk(KERN_ERR, ha, "%s: Peg not initialized\n",
+			   __func__);
+
+exit_start_fw:
+	return ret_val;
+}
+
+/*----------------------Interrupt Related functions ---------------------*/
+
+void qla4_83xx_disable_intrs(struct scsi_qla_host *ha)
+{
+	uint32_t mb_int, ret;
+
+	if (test_and_clear_bit(AF_INTERRUPTS_ON, &ha->flags))
+		qla4_8xxx_mbx_intr_disable(ha);
+
+	ret = readl(&ha->qla4_83xx_reg->mbox_int);
+	mb_int = ret & ~INT_ENABLE_FW_MB;
+	writel(mb_int, &ha->qla4_83xx_reg->mbox_int);
+	writel(1, &ha->qla4_83xx_reg->leg_int_mask);
+}
+
+void qla4_83xx_enable_intrs(struct scsi_qla_host *ha)
+{
+	uint32_t mb_int;
+
+	qla4_8xxx_mbx_intr_enable(ha);
+	mb_int = INT_ENABLE_FW_MB;
+	writel(mb_int, &ha->qla4_83xx_reg->mbox_int);
+	writel(0, &ha->qla4_83xx_reg->leg_int_mask);
+
+	set_bit(AF_INTERRUPTS_ON, &ha->flags);
+}
+
+void qla4_83xx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			      int incount)
+{
+	int i;
+
+	/* Load all mailbox registers, except mailbox 0. */
+	for (i = 1; i < incount; i++)
+		writel(mbx_cmd[i], &ha->qla4_83xx_reg->mailbox_in[i]);
+
+	writel(mbx_cmd[0], &ha->qla4_83xx_reg->mailbox_in[0]);
+
+	/* Set Host Interrupt register to 1, to tell the firmware that
+	 * a mailbox command is pending. Firmware after reading the
+	 * mailbox command, clears the host interrupt register */
+	writel(HINT_MBX_INT_PENDING, &ha->qla4_83xx_reg->host_intr);
+}
+
+void qla4_83xx_process_mbox_intr(struct scsi_qla_host *ha, int outcount)
+{
+	int intr_status;
+
+	intr_status = readl(&ha->qla4_83xx_reg->risc_intr);
+	if (intr_status) {
+		ha->mbox_status_count = outcount;
+		ha->isp_ops->interrupt_service_routine(ha, intr_status);
+	}
+}
+
+/**
+ * qla4_83xx_isp_reset - Resets ISP and aborts all outstanding commands.
+ * @ha: pointer to host adapter structure.
+ **/
+int qla4_83xx_isp_reset(struct scsi_qla_host *ha)
+{
+	int rval;
+	uint32_t dev_state;
+
+	ha->isp_ops->idc_lock(ha);
+	dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
+
+	if (ql4xdontresethba)
+		qla4_83xx_set_idc_dontreset(ha);
+
+	if (dev_state == QLA8XXX_DEV_READY) {
+		/* If IDC_CTRL DONTRESETHBA_BIT0 is set dont do reset
+		 * recovery */
+		if (qla4_83xx_idc_dontreset(ha) == DONTRESET_BIT0) {
+			ql4_printk(KERN_ERR, ha, "%s: Reset recovery disabled\n",
+				   __func__);
+			rval = QLA_ERROR;
+			goto exit_isp_reset;
+		}
+
+		DEBUG2(ql4_printk(KERN_INFO, ha, "%s: HW State: NEED RESET\n",
+				  __func__));
+		qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+				    QLA8XXX_DEV_NEED_RESET);
+
+	} else {
+		/* If device_state is NEED_RESET, go ahead with
+		 * Reset,irrespective of ql4xdontresethba. This is to allow a
+		 * non-reset-owner to force a reset. Non-reset-owner sets
+		 * the IDC_CTRL BIT0 to prevent Reset-owner from doing a Reset
+		 * and then forces a Reset by setting device_state to
+		 * NEED_RESET. */
+		DEBUG2(ql4_printk(KERN_INFO, ha,
+				  "%s: HW state already set to NEED_RESET\n",
+				  __func__));
+	}
+
+	/* For ISP8324, Reset owner is NIC, iSCSI or FCOE based on priority
+	 * and which drivers are present. Unlike ISP8022, the function setting
+	 * NEED_RESET, may not be the Reset owner. */
+	if (qla4_83xx_can_perform_reset(ha))
+		set_bit(AF_8XXX_RST_OWNER, &ha->flags);
+
+	ha->isp_ops->idc_unlock(ha);
+	rval = qla4_8xxx_device_state_handler(ha);
+
+	ha->isp_ops->idc_lock(ha);
+	qla4_8xxx_clear_rst_ready(ha);
+exit_isp_reset:
+	ha->isp_ops->idc_unlock(ha);
+
+	if (rval == QLA_SUCCESS)
+		clear_bit(AF_FW_RECOVERY, &ha->flags);
+
+	return rval;
+}
+
+static void qla4_83xx_dump_pause_control_regs(struct scsi_qla_host *ha)
+{
+	u32 val = 0, val1 = 0;
+	int i, status = QLA_SUCCESS;
+
+	status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_SRE_SHIM_CONTROL, &val);
+	DEBUG2(ql4_printk(KERN_INFO, ha, "SRE-Shim Ctrl:0x%x\n", val));
+
+	/* Port 0 Rx Buffer Pause Threshold Registers. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+		"Port 0 Rx Buffer Pause Threshold Registers[TC7..TC0]:"));
+	for (i = 0; i < 8; i++) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+				QLA83XX_PORT0_RXB_PAUSE_THRS + (i * 0x4), &val);
+		DEBUG2(pr_info("0x%x ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	/* Port 1 Rx Buffer Pause Threshold Registers. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+		"Port 1 Rx Buffer Pause Threshold Registers[TC7..TC0]:"));
+	for (i = 0; i < 8; i++) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+				QLA83XX_PORT1_RXB_PAUSE_THRS + (i * 0x4), &val);
+		DEBUG2(pr_info("0x%x  ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	/* Port 0 RxB Traffic Class Max Cell Registers. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+		"Port 0 RxB Traffic Class Max Cell Registers[3..0]:"));
+	for (i = 0; i < 4; i++) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+			       QLA83XX_PORT0_RXB_TC_MAX_CELL + (i * 0x4), &val);
+		DEBUG2(pr_info("0x%x  ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	/* Port 1 RxB Traffic Class Max Cell Registers. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+		"Port 1 RxB Traffic Class Max Cell Registers[3..0]:"));
+	for (i = 0; i < 4; i++) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+			       QLA83XX_PORT1_RXB_TC_MAX_CELL + (i * 0x4), &val);
+		DEBUG2(pr_info("0x%x  ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	/* Port 0 RxB Rx Traffic Class Stats. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "Port 0 RxB Rx Traffic Class Stats [TC7..TC0]"));
+	for (i = 7; i >= 0; i--) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+						   QLA83XX_PORT0_RXB_TC_STATS,
+						   &val);
+		val &= ~(0x7 << 29);    /* Reset bits 29 to 31 */
+		qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT0_RXB_TC_STATS,
+					  (val | (i << 29)));
+		status = qla4_83xx_rd_reg_indirect(ha,
+						   QLA83XX_PORT0_RXB_TC_STATS,
+						   &val);
+		DEBUG2(pr_info("0x%x  ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	/* Port 1 RxB Rx Traffic Class Stats. */
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "Port 1 RxB Rx Traffic Class Stats [TC7..TC0]"));
+	for (i = 7; i >= 0; i--) {
+		status = qla4_83xx_rd_reg_indirect(ha,
+						   QLA83XX_PORT1_RXB_TC_STATS,
+						   &val);
+		val &= ~(0x7 << 29);    /* Reset bits 29 to 31 */
+		qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT1_RXB_TC_STATS,
+					  (val | (i << 29)));
+		status = qla4_83xx_rd_reg_indirect(ha,
+						   QLA83XX_PORT1_RXB_TC_STATS,
+						   &val);
+		DEBUG2(pr_info("0x%x  ", val));
+	}
+
+	DEBUG2(pr_info("\n"));
+
+	status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT2_IFB_PAUSE_THRS,
+					   &val);
+	status = qla4_83xx_rd_reg_indirect(ha, QLA83XX_PORT3_IFB_PAUSE_THRS,
+					   &val1);
+
+	DEBUG2(ql4_printk(KERN_INFO, ha,
+			  "IFB-Pause Thresholds: Port 2:0x%x, Port 3:0x%x\n",
+			  val, val1));
+}
+
+static void __qla4_83xx_disable_pause(struct scsi_qla_host *ha)
+{
+	int i;
+
+	/* set SRE-Shim Control Register */
+	qla4_83xx_wr_reg_indirect(ha, QLA83XX_SRE_SHIM_CONTROL,
+				  QLA83XX_SET_PAUSE_VAL);
+
+	for (i = 0; i < 8; i++) {
+		/* Port 0 Rx Buffer Pause Threshold Registers. */
+		qla4_83xx_wr_reg_indirect(ha,
+				      QLA83XX_PORT0_RXB_PAUSE_THRS + (i * 0x4),
+				      QLA83XX_SET_PAUSE_VAL);
+		/* Port 1 Rx Buffer Pause Threshold Registers. */
+		qla4_83xx_wr_reg_indirect(ha,
+				      QLA83XX_PORT1_RXB_PAUSE_THRS + (i * 0x4),
+				      QLA83XX_SET_PAUSE_VAL);
+	}
+
+	for (i = 0; i < 4; i++) {
+		/* Port 0 RxB Traffic Class Max Cell Registers. */
+		qla4_83xx_wr_reg_indirect(ha,
+				     QLA83XX_PORT0_RXB_TC_MAX_CELL + (i * 0x4),
+				     QLA83XX_SET_TC_MAX_CELL_VAL);
+		/* Port 1 RxB Traffic Class Max Cell Registers. */
+		qla4_83xx_wr_reg_indirect(ha,
+				     QLA83XX_PORT1_RXB_TC_MAX_CELL + (i * 0x4),
+				     QLA83XX_SET_TC_MAX_CELL_VAL);
+	}
+
+	qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT2_IFB_PAUSE_THRS,
+				  QLA83XX_SET_PAUSE_VAL);
+	qla4_83xx_wr_reg_indirect(ha, QLA83XX_PORT3_IFB_PAUSE_THRS,
+				  QLA83XX_SET_PAUSE_VAL);
+
+	ql4_printk(KERN_INFO, ha, "Disabled pause frames successfully.\n");
+}
+
+void qla4_83xx_disable_pause(struct scsi_qla_host *ha)
+{
+	ha->isp_ops->idc_lock(ha);
+	qla4_83xx_dump_pause_control_regs(ha);
+	__qla4_83xx_disable_pause(ha);
+	ha->isp_ops->idc_unlock(ha);
+}
diff --git a/drivers/scsi/qla4xxx/ql4_83xx.h b/drivers/scsi/qla4xxx/ql4_83xx.h
new file mode 100644
index 00000000000..6a00f903f2a
--- /dev/null
+++ b/drivers/scsi/qla4xxx/ql4_83xx.h
@@ -0,0 +1,283 @@
+/*
+ * QLogic iSCSI HBA Driver
+ * Copyright (c)  2003-2012 QLogic Corporation
+ *
+ * See LICENSE.qla4xxx for copyright and licensing details.
+ */
+
+#ifndef __QL483XX_H
+#define __QL483XX_H
+
+/* Indirectly Mapped Registers */
+#define QLA83XX_FLASH_SPI_STATUS	0x2808E010
+#define QLA83XX_FLASH_SPI_CONTROL	0x2808E014
+#define QLA83XX_FLASH_STATUS		0x42100004
+#define QLA83XX_FLASH_CONTROL		0x42110004
+#define QLA83XX_FLASH_ADDR		0x42110008
+#define QLA83XX_FLASH_WRDATA		0x4211000C
+#define QLA83XX_FLASH_RDDATA		0x42110018
+#define QLA83XX_FLASH_DIRECT_WINDOW	0x42110030
+#define QLA83XX_FLASH_DIRECT_DATA(DATA) (0x42150000 | (0x0000FFFF&DATA))
+
+/* Directly Mapped Registers in 83xx register table */
+
+/* Flash access regs */
+#define QLA83XX_FLASH_LOCK		0x3850
+#define QLA83XX_FLASH_UNLOCK		0x3854
+#define QLA83XX_FLASH_LOCK_ID		0x3500
+
+/* Driver Lock regs */
+#define QLA83XX_DRV_LOCK		0x3868
+#define QLA83XX_DRV_UNLOCK		0x386C
+#define QLA83XX_DRV_LOCK_ID		0x3504
+#define QLA83XX_DRV_LOCKRECOVERY	0x379C
+
+/* IDC version */
+#define QLA83XX_IDC_VER_MAJ_VALUE       0x1
+#define QLA83XX_IDC_VER_MIN_VALUE       0x0
+
+/* IDC Registers : Driver Coexistence Defines */
+#define QLA83XX_CRB_IDC_VER_MAJOR	0x3780
+#define QLA83XX_CRB_IDC_VER_MINOR	0x3798
+#define QLA83XX_IDC_DRV_CTRL		0x3790
+#define QLA83XX_IDC_DRV_AUDIT		0x3794
+#define QLA83XX_SRE_SHIM_CONTROL	0x0D200284
+#define QLA83XX_PORT0_RXB_PAUSE_THRS	0x0B2003A4
+#define QLA83XX_PORT1_RXB_PAUSE_THRS	0x0B2013A4
+#define QLA83XX_PORT0_RXB_TC_MAX_CELL	0x0B200388
+#define QLA83XX_PORT1_RXB_TC_MAX_CELL	0x0B201388
+#define QLA83XX_PORT0_RXB_TC_STATS	0x0B20039C
+#define QLA83XX_PORT1_RXB_TC_STATS	0x0B20139C
+#define QLA83XX_PORT2_IFB_PAUSE_THRS	0x0B200704
+#define QLA83XX_PORT3_IFB_PAUSE_THRS	0x0B201704
+
+/* set value to pause threshold value */
+#define QLA83XX_SET_PAUSE_VAL		0x0
+#define QLA83XX_SET_TC_MAX_CELL_VAL	0x03FF03FF
+
+/* qla_83xx_reg_tbl registers */
+#define QLA83XX_PEG_HALT_STATUS1	0x34A8
+#define QLA83XX_PEG_HALT_STATUS2	0x34AC
+#define QLA83XX_PEG_ALIVE_COUNTER	0x34B0 /* FW_HEARTBEAT */
+#define QLA83XX_FW_CAPABILITIES		0x3528
+#define QLA83XX_CRB_DRV_ACTIVE		0x3788 /* IDC_DRV_PRESENCE */
+#define QLA83XX_CRB_DEV_STATE		0x3784 /* IDC_DEV_STATE */
+#define QLA83XX_CRB_DRV_STATE		0x378C /* IDC_DRV_ACK */
+#define QLA83XX_CRB_DRV_SCRATCH		0x3548
+#define QLA83XX_CRB_DEV_PART_INFO1	0x37E0
+#define QLA83XX_CRB_DEV_PART_INFO2	0x37E4
+
+#define QLA83XX_FW_VER_MAJOR		0x3550
+#define QLA83XX_FW_VER_MINOR		0x3554
+#define QLA83XX_FW_VER_SUB		0x3558
+#define QLA83XX_NPAR_STATE		0x359C
+#define QLA83XX_FW_IMAGE_VALID		0x35FC
+#define QLA83XX_CMDPEG_STATE		0x3650
+#define QLA83XX_ASIC_TEMP		0x37B4
+#define QLA83XX_FW_API			0x356C
+#define QLA83XX_DRV_OP_MODE		0x3570
+
+static const uint32_t qla4_83xx_reg_tbl[] = {
+	QLA83XX_PEG_HALT_STATUS1,
+	QLA83XX_PEG_HALT_STATUS2,
+	QLA83XX_PEG_ALIVE_COUNTER,
+	QLA83XX_CRB_DRV_ACTIVE,
+	QLA83XX_CRB_DEV_STATE,
+	QLA83XX_CRB_DRV_STATE,
+	QLA83XX_CRB_DRV_SCRATCH,
+	QLA83XX_CRB_DEV_PART_INFO1,
+	QLA83XX_CRB_IDC_VER_MAJOR,
+	QLA83XX_FW_VER_MAJOR,
+	QLA83XX_FW_VER_MINOR,
+	QLA83XX_FW_VER_SUB,
+	QLA83XX_CMDPEG_STATE,
+	QLA83XX_ASIC_TEMP,
+};
+
+#define QLA83XX_CRB_WIN_BASE		0x3800
+#define QLA83XX_CRB_WIN_FUNC(f)		(QLA83XX_CRB_WIN_BASE+((f)*4))
+#define QLA83XX_SEM_LOCK_BASE		0x3840
+#define QLA83XX_SEM_UNLOCK_BASE		0x3844
+#define QLA83XX_SEM_LOCK_FUNC(f)	(QLA83XX_SEM_LOCK_BASE+((f)*8))
+#define QLA83XX_SEM_UNLOCK_FUNC(f)	(QLA83XX_SEM_UNLOCK_BASE+((f)*8))
+#define QLA83XX_LINK_STATE(f)		(0x3698+((f) > 7 ? 4 : 0))
+#define QLA83XX_LINK_SPEED(f)		(0x36E0+(((f) >> 2) * 4))
+#define QLA83XX_MAX_LINK_SPEED(f)       (0x36F0+(((f) / 4) * 4))
+#define QLA83XX_LINK_SPEED_FACTOR	10
+
+/* FLASH API Defines */
+#define QLA83xx_FLASH_MAX_WAIT_USEC	100
+#define QLA83XX_FLASH_LOCK_TIMEOUT	10000
+#define QLA83XX_FLASH_SECTOR_SIZE	65536
+#define QLA83XX_DRV_LOCK_TIMEOUT	2000
+#define QLA83XX_FLASH_SECTOR_ERASE_CMD	0xdeadbeef
+#define QLA83XX_FLASH_WRITE_CMD		0xdacdacda
+#define QLA83XX_FLASH_BUFFER_WRITE_CMD	0xcadcadca
+#define QLA83XX_FLASH_READ_RETRY_COUNT	2000
+#define QLA83XX_FLASH_STATUS_READY	0x6
+#define QLA83XX_FLASH_BUFFER_WRITE_MIN	2
+#define QLA83XX_FLASH_BUFFER_WRITE_MAX	64
+#define QLA83XX_FLASH_STATUS_REG_POLL_DELAY 1
+#define QLA83XX_ERASE_MODE		1
+#define QLA83XX_WRITE_MODE		2
+#define QLA83XX_DWORD_WRITE_MODE	3
+
+#define QLA83XX_GLOBAL_RESET		0x38CC
+#define QLA83XX_WILDCARD		0x38F0
+#define QLA83XX_INFORMANT		0x38FC
+#define QLA83XX_HOST_MBX_CTRL		0x3038
+#define QLA83XX_FW_MBX_CTRL		0x303C
+#define QLA83XX_BOOTLOADER_ADDR		0x355C
+#define QLA83XX_BOOTLOADER_SIZE		0x3560
+#define QLA83XX_FW_IMAGE_ADDR		0x3564
+#define QLA83XX_MBX_INTR_ENABLE		0x1000
+#define QLA83XX_MBX_INTR_MASK		0x1200
+
+/* IDC Control Register bit defines */
+#define DONTRESET_BIT0		0x1
+#define GRACEFUL_RESET_BIT1	0x2
+
+#define QLA83XX_HALT_STATUS_INFORMATIONAL	(0x1 << 29)
+#define QLA83XX_HALT_STATUS_FW_RESET		(0x2 << 29)
+#define QLA83XX_HALT_STATUS_UNRECOVERABLE	(0x4 << 29)
+
+/* Firmware image definitions */
+#define QLA83XX_BOOTLOADER_FLASH_ADDR	0x10000
+#define QLA83XX_BOOT_FROM_FLASH		0
+
+#define QLA83XX_IDC_PARAM_ADDR		0x3e8020
+/* Reset template definitions */
+#define QLA83XX_MAX_RESET_SEQ_ENTRIES	16
+#define QLA83XX_RESTART_TEMPLATE_SIZE	0x2000
+#define QLA83XX_RESET_TEMPLATE_ADDR	0x4F0000
+#define QLA83XX_RESET_SEQ_VERSION	0x0101
+
+/* Reset template entry opcodes */
+#define OPCODE_NOP			0x0000
+#define OPCODE_WRITE_LIST		0x0001
+#define OPCODE_READ_WRITE_LIST		0x0002
+#define OPCODE_POLL_LIST		0x0004
+#define OPCODE_POLL_WRITE_LIST		0x0008
+#define OPCODE_READ_MODIFY_WRITE	0x0010
+#define OPCODE_SEQ_PAUSE		0x0020
+#define OPCODE_SEQ_END			0x0040
+#define OPCODE_TMPL_END			0x0080
+#define OPCODE_POLL_READ_LIST		0x0100
+
+/* Template Header */
+#define RESET_TMPLT_HDR_SIGNATURE	0xCAFE
+struct qla4_83xx_reset_template_hdr {
+	__le16	version;
+	__le16	signature;
+	__le16	size;
+	__le16	entries;
+	__le16	hdr_size;
+	__le16	checksum;
+	__le16	init_seq_offset;
+	__le16	start_seq_offset;
+} __packed;
+
+/* Common Entry Header. */
+struct qla4_83xx_reset_entry_hdr {
+	__le16 cmd;
+	__le16 size;
+	__le16 count;
+	__le16 delay;
+} __packed;
+
+/* Generic poll entry type. */
+struct qla4_83xx_poll {
+	__le32  test_mask;
+	__le32  test_value;
+} __packed;
+
+/* Read modify write entry type. */
+struct qla4_83xx_rmw {
+	__le32  test_mask;
+	__le32  xor_value;
+	__le32  or_value;
+	uint8_t shl;
+	uint8_t shr;
+	uint8_t index_a;
+	uint8_t rsvd;
+} __packed;
+
+/* Generic Entry Item with 2 DWords. */
+struct qla4_83xx_entry {
+	__le32 arg1;
+	__le32 arg2;
+} __packed;
+
+/* Generic Entry Item with 4 DWords.*/
+struct qla4_83xx_quad_entry {
+	__le32 dr_addr;
+	__le32 dr_value;
+	__le32 ar_addr;
+	__le32 ar_value;
+} __packed;
+
+struct qla4_83xx_reset_template {
+	int seq_index;
+	int seq_error;
+	int array_index;
+	uint32_t array[QLA83XX_MAX_RESET_SEQ_ENTRIES];
+	uint8_t *buff;
+	uint8_t *stop_offset;
+	uint8_t *start_offset;
+	uint8_t *init_offset;
+	struct qla4_83xx_reset_template_hdr *hdr;
+	uint8_t seq_end;
+	uint8_t template_end;
+};
+
+/* POLLRD Entry */
+struct qla83xx_minidump_entry_pollrd {
+	struct qla8xxx_minidump_entry_hdr h;
+	uint32_t select_addr;
+	uint32_t read_addr;
+	uint32_t select_value;
+	uint16_t select_value_stride;
+	uint16_t op_count;
+	uint32_t poll_wait;
+	uint32_t poll_mask;
+	uint32_t data_size;
+	uint32_t rsvd_1;
+};
+
+/* RDMUX2 Entry */
+struct qla83xx_minidump_entry_rdmux2 {
+	struct qla8xxx_minidump_entry_hdr h;
+	uint32_t select_addr_1;
+	uint32_t select_addr_2;
+	uint32_t select_value_1;
+	uint32_t select_value_2;
+	uint32_t op_count;
+	uint32_t select_value_mask;
+	uint32_t read_addr;
+	uint8_t select_value_stride;
+	uint8_t data_size;
+	uint8_t rsvd[2];
+};
+
+/* POLLRDMWR Entry */
+struct qla83xx_minidump_entry_pollrdmwr {
+	struct qla8xxx_minidump_entry_hdr h;
+	uint32_t addr_1;
+	uint32_t addr_2;
+	uint32_t value_1;
+	uint32_t value_2;
+	uint32_t poll_wait;
+	uint32_t poll_mask;
+	uint32_t modify_mask;
+	uint32_t data_size;
+};
+
+/* IDC additional information */
+struct qla4_83xx_idc_information {
+	uint32_t request_desc;  /* IDC request descriptor */
+	uint32_t info1; /* IDC additional info */
+	uint32_t info2; /* IDC additional info */
+	uint32_t info3; /* IDC additional info */
+};
+
+#endif
diff --git a/drivers/scsi/qla4xxx/ql4_attr.c b/drivers/scsi/qla4xxx/ql4_attr.c
index c681b2a355e..76819b71ada 100644
--- a/drivers/scsi/qla4xxx/ql4_attr.c
+++ b/drivers/scsi/qla4xxx/ql4_attr.c
@@ -17,7 +17,7 @@ qla4_8xxx_sysfs_read_fw_dump(struct file *filep, struct kobject *kobj,
 	struct scsi_qla_host *ha = to_qla_host(dev_to_shost(container_of(kobj,
 					       struct device, kobj)));
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		return -EINVAL;
 
 	if (!test_bit(AF_82XX_DUMP_READING, &ha->flags))
@@ -38,7 +38,7 @@ qla4_8xxx_sysfs_write_fw_dump(struct file *filep, struct kobject *kobj,
 	long reading;
 	int ret = 0;
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		return -EINVAL;
 
 	if (off != 0)
@@ -75,21 +75,21 @@ qla4_8xxx_sysfs_write_fw_dump(struct file *filep, struct kobject *kobj,
 		break;
 	case 2:
 		/* Reset HBA */
-		qla4_8xxx_idc_lock(ha);
-		dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
-		if (dev_state == QLA82XX_DEV_READY) {
+		ha->isp_ops->idc_lock(ha);
+		dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
+		if (dev_state == QLA8XXX_DEV_READY) {
 			ql4_printk(KERN_INFO, ha,
 				   "%s: Setting Need reset, reset_owner is 0x%x.\n",
 				   __func__, ha->func_num);
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-					QLA82XX_DEV_NEED_RESET);
-			set_bit(AF_82XX_RST_OWNER, &ha->flags);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_NEED_RESET);
+			set_bit(AF_8XXX_RST_OWNER, &ha->flags);
 		} else
 			ql4_printk(KERN_INFO, ha,
 				   "%s: Reset not performed as device state is 0x%x\n",
 				   __func__, dev_state);
 
-		qla4_8xxx_idc_unlock(ha);
+		ha->isp_ops->idc_unlock(ha);
 		break;
 	default:
 		/* do nothing */
@@ -150,7 +150,7 @@ qla4xxx_fw_version_show(struct device *dev,
 {
 	struct scsi_qla_host *ha = to_qla_host(class_to_shost(dev));
 
-	if (is_qla8022(ha))
+	if (is_qla80XX(ha))
 		return snprintf(buf, PAGE_SIZE, "%d.%02d.%02d (%x)\n",
 				ha->firmware_version[0],
 				ha->firmware_version[1],
@@ -214,7 +214,7 @@ qla4xxx_phy_port_cnt_show(struct device *dev, struct device_attribute *attr,
 {
 	struct scsi_qla_host *ha = to_qla_host(class_to_shost(dev));
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		return -ENOSYS;
 
 	return snprintf(buf, PAGE_SIZE, "0x%04X\n", ha->phy_port_cnt);
@@ -226,7 +226,7 @@ qla4xxx_phy_port_num_show(struct device *dev, struct device_attribute *attr,
 {
 	struct scsi_qla_host *ha = to_qla_host(class_to_shost(dev));
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		return -ENOSYS;
 
 	return snprintf(buf, PAGE_SIZE, "0x%04X\n", ha->phy_port_num);
@@ -238,7 +238,7 @@ qla4xxx_iscsi_func_cnt_show(struct device *dev, struct device_attribute *attr,
 {
 	struct scsi_qla_host *ha = to_qla_host(class_to_shost(dev));
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		return -ENOSYS;
 
 	return snprintf(buf, PAGE_SIZE, "0x%04X\n", ha->iscsi_pci_func_cnt);
diff --git a/drivers/scsi/qla4xxx/ql4_dbg.c b/drivers/scsi/qla4xxx/ql4_dbg.c
index 8d58ae27482..77b7c594010 100644
--- a/drivers/scsi/qla4xxx/ql4_dbg.c
+++ b/drivers/scsi/qla4xxx/ql4_dbg.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -37,7 +37,7 @@ void qla4xxx_dump_registers(struct scsi_qla_host *ha)
 	if (is_qla8022(ha)) {
 		for (i = 1; i < MBOX_REG_COUNT; i++)
 			printk(KERN_INFO "mailbox[%d]     = 0x%08X\n",
-			    i, readl(&ha->qla4_8xxx_reg->mailbox_in[i]));
+			    i, readl(&ha->qla4_82xx_reg->mailbox_in[i]));
 		return;
 	}
 
@@ -131,3 +131,31 @@ void qla4xxx_dump_registers(struct scsi_qla_host *ha)
 		    &ha->reg->ctrl_status);
 	}
 }
+
+void qla4_8xxx_dump_peg_reg(struct scsi_qla_host *ha)
+{
+	uint32_t halt_status1, halt_status2;
+
+	halt_status1 = qla4_8xxx_rd_direct(ha, QLA8XXX_PEG_HALT_STATUS1);
+	halt_status2 = qla4_8xxx_rd_direct(ha, QLA8XXX_PEG_HALT_STATUS2);
+
+	if (is_qla8022(ha)) {
+		ql4_printk(KERN_INFO, ha,
+			   "scsi(%ld): %s, ISP8022 Dumping hw/fw registers:\n"
+			   " PEG_HALT_STATUS1: 0x%x, PEG_HALT_STATUS2: 0x%x,\n"
+			   " PEG_NET_0_PC: 0x%x, PEG_NET_1_PC: 0x%x,\n"
+			   " PEG_NET_2_PC: 0x%x, PEG_NET_3_PC: 0x%x,\n"
+			   " PEG_NET_4_PC: 0x%x\n", ha->host_no,
+			   __func__, halt_status1, halt_status2,
+			   qla4_82xx_rd_32(ha, QLA82XX_CRB_PEG_NET_0 + 0x3c),
+			   qla4_82xx_rd_32(ha, QLA82XX_CRB_PEG_NET_1 + 0x3c),
+			   qla4_82xx_rd_32(ha, QLA82XX_CRB_PEG_NET_2 + 0x3c),
+			   qla4_82xx_rd_32(ha, QLA82XX_CRB_PEG_NET_3 + 0x3c),
+			   qla4_82xx_rd_32(ha, QLA82XX_CRB_PEG_NET_4 + 0x3c));
+	} else if (is_qla8032(ha)) {
+		ql4_printk(KERN_INFO, ha,
+			   "scsi(%ld): %s, ISP8324 Dumping hw/fw registers:\n"
+			   " PEG_HALT_STATUS1: 0x%x, PEG_HALT_STATUS2: 0x%x,\n",
+			   ha->host_no, __func__, halt_status1, halt_status2);
+	}
+}
diff --git a/drivers/scsi/qla4xxx/ql4_dbg.h b/drivers/scsi/qla4xxx/ql4_dbg.h
index abd83602cdd..5b0afc18ef1 100644
--- a/drivers/scsi/qla4xxx/ql4_dbg.h
+++ b/drivers/scsi/qla4xxx/ql4_dbg.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla4xxx/ql4_def.h b/drivers/scsi/qla4xxx/ql4_def.h
index 7fdba7f1ffb..329d553eae9 100644
--- a/drivers/scsi/qla4xxx/ql4_def.h
+++ b/drivers/scsi/qla4xxx/ql4_def.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -42,6 +42,7 @@
 #include "ql4_nx.h"
 #include "ql4_fw.h"
 #include "ql4_nvram.h"
+#include "ql4_83xx.h"
 
 #ifndef PCI_DEVICE_ID_QLOGIC_ISP4010
 #define PCI_DEVICE_ID_QLOGIC_ISP4010	0x4010
@@ -59,6 +60,10 @@
 #define PCI_DEVICE_ID_QLOGIC_ISP8022	0x8022
 #endif
 
+#ifndef PCI_DEVICE_ID_QLOGIC_ISP8324
+#define PCI_DEVICE_ID_QLOGIC_ISP8324	0x8032
+#endif
+
 #define ISP4XXX_PCI_FN_1	0x1
 #define ISP4XXX_PCI_FN_2	0x3
 
@@ -388,8 +393,10 @@ struct isp_operations {
 	void (*disable_intrs) (struct scsi_qla_host *);
 	void (*enable_intrs) (struct scsi_qla_host *);
 	int (*start_firmware) (struct scsi_qla_host *);
+	int (*restart_firmware) (struct scsi_qla_host *);
 	irqreturn_t (*intr_handler) (int , void *);
 	void (*interrupt_service_routine) (struct scsi_qla_host *, uint32_t);
+	int (*need_reset) (struct scsi_qla_host *);
 	int (*reset_chip) (struct scsi_qla_host *);
 	int (*reset_firmware) (struct scsi_qla_host *);
 	void (*queue_iocb) (struct scsi_qla_host *);
@@ -397,6 +404,15 @@ struct isp_operations {
 	uint16_t (*rd_shdw_req_q_out) (struct scsi_qla_host *);
 	uint16_t (*rd_shdw_rsp_q_in) (struct scsi_qla_host *);
 	int (*get_sys_info) (struct scsi_qla_host *);
+	uint32_t (*rd_reg_direct) (struct scsi_qla_host *, ulong);
+	void (*wr_reg_direct) (struct scsi_qla_host *, ulong, uint32_t);
+	int (*rd_reg_indirect) (struct scsi_qla_host *, uint32_t, uint32_t *);
+	int (*wr_reg_indirect) (struct scsi_qla_host *, uint32_t, uint32_t);
+	int (*idc_lock) (struct scsi_qla_host *);
+	void (*idc_unlock) (struct scsi_qla_host *);
+	void (*rom_lock_recovery) (struct scsi_qla_host *);
+	void (*queue_mailbox_command) (struct scsi_qla_host *, uint32_t *, int);
+	void (*process_mailbox_interrupt) (struct scsi_qla_host *, int);
 };
 
 struct ql4_mdump_size_table {
@@ -497,8 +513,9 @@ struct scsi_qla_host {
 #define AF_PCI_CHANNEL_IO_PERM_FAILURE	21 /* 0x00200000 */
 #define AF_BUILD_DDB_LIST		22 /* 0x00400000 */
 #define AF_82XX_FW_DUMPED		24 /* 0x01000000 */
-#define AF_82XX_RST_OWNER		25 /* 0x02000000 */
+#define AF_8XXX_RST_OWNER		25 /* 0x02000000 */
 #define AF_82XX_DUMP_READING		26 /* 0x04000000 */
+#define AF_83XX_NO_FW_DUMP		27 /* 0x08000000 */
 
 	unsigned long dpc_flags;
 
@@ -514,7 +531,7 @@ struct scsi_qla_host {
 #define DPC_RESET_ACTIVE		20 /* 0x00040000 */
 #define DPC_HA_UNRECOVERABLE		21 /* 0x00080000 ISP-82xx only*/
 #define DPC_HA_NEED_QUIESCENT		22 /* 0x00100000 ISP-82xx only*/
-
+#define DPC_POST_IDC_ACK		23 /* 0x00200000 */
 
 	struct Scsi_Host *host; /* pointer to host data */
 	uint32_t tot_ddbs;
@@ -647,7 +664,7 @@ struct scsi_qla_host {
 	uint8_t acb_version;
 
 	/* qla82xx specific fields */
-	struct device_reg_82xx  __iomem *qla4_8xxx_reg; /* Base I/O address */
+	struct device_reg_82xx  __iomem *qla4_82xx_reg; /* Base I/O address */
 	unsigned long nx_pcibase;	/* Base I/O address */
 	uint8_t *nx_db_rd_ptr;		/* Doorbell read pointer */
 	unsigned long nx_db_wr_ptr;	/* Door bell write pointer */
@@ -733,6 +750,13 @@ struct scsi_qla_host {
 #define MAX_MRB		128
 	struct mrb *active_mrb_array[MAX_MRB];
 	uint32_t mrb_index;
+
+	uint32_t *reg_tbl;
+	struct qla4_83xx_reset_template reset_tmplt;
+	struct device_reg_83xx  __iomem *qla4_83xx_reg; /* Base I/O address
+							   for ISP8324 */
+	uint32_t pf_bit;
+	struct qla4_83xx_idc_information idc_info;
 };
 
 struct ql4_task_data {
@@ -752,7 +776,7 @@ struct ql4_task_data {
 
 struct qla_endpoint {
 	struct Scsi_Host *host;
-	struct sockaddr dst_addr;
+	struct sockaddr_storage dst_addr;
 };
 
 struct qla_conn {
@@ -795,13 +819,20 @@ static inline int is_qla8022(struct scsi_qla_host *ha)
 	return ha->pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8022;
 }
 
-/* Note: Currently AER/EEH is now supported only for 8022 cards
- * This function needs to be updated when AER/EEH is enabled
- * for other cards.
- */
+static inline int is_qla8032(struct scsi_qla_host *ha)
+{
+	return ha->pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8324;
+}
+
+static inline int is_qla80XX(struct scsi_qla_host *ha)
+{
+	return is_qla8022(ha) || is_qla8032(ha);
+}
+
 static inline int is_aer_supported(struct scsi_qla_host *ha)
 {
-	return ha->pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8022;
+	return ((ha->pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8022) ||
+		(ha->pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8324));
 }
 
 static inline int adapter_up(struct scsi_qla_host *ha)
@@ -942,6 +973,20 @@ static inline int ql4xxx_reset_active(struct scsi_qla_host *ha)
 	       test_bit(DPC_HA_UNRECOVERABLE, &ha->dpc_flags);
 
 }
+
+static inline int qla4_8xxx_rd_direct(struct scsi_qla_host *ha,
+				      const uint32_t crb_reg)
+{
+	return ha->isp_ops->rd_reg_direct(ha, ha->reg_tbl[crb_reg]);
+}
+
+static inline void qla4_8xxx_wr_direct(struct scsi_qla_host *ha,
+				       const uint32_t crb_reg,
+				       const uint32_t value)
+{
+	ha->isp_ops->wr_reg_direct(ha, ha->reg_tbl[crb_reg], value);
+}
+
 /*---------------------------------------------------------------------------*/
 
 /* Defines for qla4xxx_initialize_adapter() and qla4xxx_recover_adapter() */
diff --git a/drivers/scsi/qla4xxx/ql4_fw.h b/drivers/scsi/qla4xxx/ql4_fw.h
index 7240948fb92..1c479502035 100644
--- a/drivers/scsi/qla4xxx/ql4_fw.h
+++ b/drivers/scsi/qla4xxx/ql4_fw.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -65,6 +65,40 @@ struct device_reg_82xx {
 #define ISRX_82XX_RISC_INT	BIT_0 /* RISC interrupt. */
 };
 
+/* ISP 83xx I/O Register Set structure */
+struct device_reg_83xx {
+	__le32 mailbox_in[16];	/* 0x0000 */
+	__le32 reserve1[496];	/* 0x0040 */
+	__le32 mailbox_out[16];	/* 0x0800 */
+	__le32 reserve2[496];
+	__le32 mbox_int;	/* 0x1000 */
+	__le32 reserve3[63];
+	__le32 req_q_out;	/* 0x1100 */
+	__le32 reserve4[63];
+
+	__le32 rsp_q_in;	/* 0x1200 */
+	__le32 reserve5[1919];
+
+	__le32 req_q_in;	/* 0x3000 */
+	__le32 reserve6[3];
+	__le32 iocb_int_mask;	/* 0x3010 */
+	__le32 reserve7[3];
+	__le32 rsp_q_out;	/* 0x3020 */
+	__le32 reserve8[3];
+	__le32 anonymousbuff;	/* 0x3030 */
+	__le32 mb_int_mask;	/* 0x3034 */
+
+	__le32 host_intr;	/* 0x3038 - Host Interrupt Register */
+	__le32 risc_intr;	/* 0x303C - RISC Interrupt Register */
+	__le32 reserve9[544];
+	__le32 leg_int_ptr;	/* 0x38C0 - Legacy Interrupt Pointer Register */
+	__le32 leg_int_trig;	/* 0x38C4 - Legacy Interrupt Trigger Control */
+	__le32 leg_int_mask;	/* 0x38C8 - Legacy Interrupt Mask Register */
+};
+
+#define INT_ENABLE_FW_MB	(1 << 2)
+#define INT_MASK_FW_MB		(1 << 2)
+
 /*  remote register set (access via PCI memory read/write) */
 struct isp_reg {
 #define MBOX_REG_COUNT 8
@@ -356,6 +390,9 @@ struct qla_flt_region {
 #define LOGOUT_OPTION_CLOSE_SESSION		0x0002
 #define LOGOUT_OPTION_RELOGIN			0x0004
 #define LOGOUT_OPTION_FREE_DDB			0x0008
+#define MBOX_CMD_SET_PARAM			0x0059
+#define SET_DRVR_VERSION			0x200
+#define MAX_DRVR_VER_LEN			24
 #define MBOX_CMD_EXECUTE_IOCB_A64		0x005A
 #define MBOX_CMD_INITIALIZE_FIRMWARE		0x0060
 #define MBOX_CMD_GET_INIT_FW_CTRL_BLOCK		0x0061
@@ -417,6 +454,10 @@ struct qla_flt_region {
 #define MBOX_CMD_GET_CRASH_RECORD		0x0076	/* 4010 only */
 #define MBOX_CMD_GET_CONN_EVENT_LOG		0x0077
 
+#define MBOX_CMD_IDC_ACK			0x0101
+#define MBOX_CMD_PORT_RESET			0x0120
+#define MBOX_CMD_SET_PORT_CONFIG		0x0122
+
 /*  Mailbox status definitions */
 #define MBOX_COMPLETION_STATUS			4
 #define MBOX_STS_BUSY				0x0007
@@ -453,6 +494,8 @@ struct qla_flt_region {
 #define MBOX_ASTS_IPV6_ND_PREFIX_IGNORED	0x802C
 #define MBOX_ASTS_IPV6_LCL_PREFIX_IGNORED	0x802D
 #define MBOX_ASTS_ICMPV6_ERROR_MSG_RCVD		0x802E
+#define MBOX_ASTS_IDC_COMPLETE			0x8100
+#define MBOX_ASTS_IDC_NOTIFY			0x8101
 #define MBOX_ASTS_TXSCVR_INSERTED		0x8130
 #define MBOX_ASTS_TXSCVR_REMOVED		0x8131
 
@@ -1195,9 +1238,12 @@ struct ql_iscsi_stats {
 	uint8_t reserved2[264]; /* 0x0308 - 0x040F */
 };
 
-#define QLA82XX_DBG_STATE_ARRAY_LEN		16
-#define QLA82XX_DBG_CAP_SIZE_ARRAY_LEN		8
-#define QLA82XX_DBG_RSVD_ARRAY_LEN		8
+#define QLA8XXX_DBG_STATE_ARRAY_LEN		16
+#define QLA8XXX_DBG_CAP_SIZE_ARRAY_LEN		8
+#define QLA8XXX_DBG_RSVD_ARRAY_LEN		8
+#define QLA83XX_DBG_OCM_WNDREG_ARRAY_LEN	16
+#define QLA83XX_SS_OCM_WNDREG_INDEX		3
+#define QLA83XX_SS_PCI_INDEX			0
 
 struct qla4_8xxx_minidump_template_hdr {
 	uint32_t entry_type;
@@ -1214,8 +1260,9 @@ struct qla4_8xxx_minidump_template_hdr {
 	uint32_t driver_info_word3;
 	uint32_t driver_info_word4;
 
-	uint32_t saved_state_array[QLA82XX_DBG_STATE_ARRAY_LEN];
-	uint32_t capture_size_array[QLA82XX_DBG_CAP_SIZE_ARRAY_LEN];
+	uint32_t saved_state_array[QLA8XXX_DBG_STATE_ARRAY_LEN];
+	uint32_t capture_size_array[QLA8XXX_DBG_CAP_SIZE_ARRAY_LEN];
+	uint32_t ocm_window_reg[QLA83XX_DBG_OCM_WNDREG_ARRAY_LEN];
 };
 
 #endif /*  _QLA4X_FW_H */
diff --git a/drivers/scsi/qla4xxx/ql4_glbl.h b/drivers/scsi/qla4xxx/ql4_glbl.h
index 5b2525c4139..57a5a3cf577 100644
--- a/drivers/scsi/qla4xxx/ql4_glbl.h
+++ b/drivers/scsi/qla4xxx/ql4_glbl.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -109,28 +109,28 @@ uint8_t qla4xxx_update_local_ifcb(struct scsi_qla_host *ha,
 void qla4_8xxx_pci_config(struct scsi_qla_host *);
 int qla4_8xxx_iospace_config(struct scsi_qla_host *ha);
 int qla4_8xxx_load_risc(struct scsi_qla_host *);
-irqreturn_t qla4_8xxx_intr_handler(int irq, void *dev_id);
-void qla4_8xxx_queue_iocb(struct scsi_qla_host *ha);
-void qla4_8xxx_complete_iocb(struct scsi_qla_host *ha);
+irqreturn_t qla4_82xx_intr_handler(int irq, void *dev_id);
+void qla4_82xx_queue_iocb(struct scsi_qla_host *ha);
+void qla4_82xx_complete_iocb(struct scsi_qla_host *ha);
 
-int qla4_8xxx_crb_win_lock(struct scsi_qla_host *);
-void qla4_8xxx_crb_win_unlock(struct scsi_qla_host *);
-int qla4_8xxx_pci_get_crb_addr_2M(struct scsi_qla_host *, ulong *);
-void qla4_8xxx_wr_32(struct scsi_qla_host *, ulong, u32);
-int qla4_8xxx_rd_32(struct scsi_qla_host *, ulong);
-int qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *, u64, void *, int);
-int qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha, u64, void *, int);
-int qla4_8xxx_isp_reset(struct scsi_qla_host *ha);
-void qla4_8xxx_interrupt_service_routine(struct scsi_qla_host *ha,
+int qla4_82xx_crb_win_lock(struct scsi_qla_host *);
+void qla4_82xx_crb_win_unlock(struct scsi_qla_host *);
+int qla4_82xx_pci_get_crb_addr_2M(struct scsi_qla_host *, ulong *);
+void qla4_82xx_wr_32(struct scsi_qla_host *, ulong, u32);
+uint32_t qla4_82xx_rd_32(struct scsi_qla_host *, ulong);
+int qla4_82xx_pci_mem_read_2M(struct scsi_qla_host *, u64, void *, int);
+int qla4_82xx_pci_mem_write_2M(struct scsi_qla_host *ha, u64, void *, int);
+int qla4_82xx_isp_reset(struct scsi_qla_host *ha);
+void qla4_82xx_interrupt_service_routine(struct scsi_qla_host *ha,
 		uint32_t intr_status);
-uint16_t qla4_8xxx_rd_shdw_req_q_out(struct scsi_qla_host *ha);
-uint16_t qla4_8xxx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha);
+uint16_t qla4_82xx_rd_shdw_req_q_out(struct scsi_qla_host *ha);
+uint16_t qla4_82xx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha);
 int qla4_8xxx_get_sys_info(struct scsi_qla_host *ha);
 void qla4_8xxx_watchdog(struct scsi_qla_host *ha);
 int qla4_8xxx_stop_firmware(struct scsi_qla_host *ha);
 int qla4_8xxx_get_flash_info(struct scsi_qla_host *ha);
-void qla4_8xxx_enable_intrs(struct scsi_qla_host *ha);
-void qla4_8xxx_disable_intrs(struct scsi_qla_host *ha);
+void qla4_82xx_enable_intrs(struct scsi_qla_host *ha);
+void qla4_82xx_disable_intrs(struct scsi_qla_host *ha);
 int qla4_8xxx_enable_msix(struct scsi_qla_host *ha);
 void qla4_8xxx_disable_msix(struct scsi_qla_host *ha);
 irqreturn_t qla4_8xxx_msi_handler(int irq, void *dev_id);
@@ -138,8 +138,8 @@ irqreturn_t qla4_8xxx_default_intr_handler(int irq, void *dev_id);
 irqreturn_t qla4_8xxx_msix_rsp_q(int irq, void *dev_id);
 void qla4xxx_mark_all_devices_missing(struct scsi_qla_host *ha);
 void qla4xxx_dead_adapter_cleanup(struct scsi_qla_host *ha);
-int qla4_8xxx_idc_lock(struct scsi_qla_host *ha);
-void qla4_8xxx_idc_unlock(struct scsi_qla_host *ha);
+int qla4_82xx_idc_lock(struct scsi_qla_host *ha);
+void qla4_82xx_idc_unlock(struct scsi_qla_host *ha);
 int qla4_8xxx_device_state_handler(struct scsi_qla_host *ha);
 void qla4_8xxx_need_qsnt_handler(struct scsi_qla_host *ha);
 void qla4_8xxx_clear_drv_active(struct scsi_qla_host *ha);
@@ -203,6 +203,62 @@ int qla4xxx_req_template_size(struct scsi_qla_host *ha);
 void qla4_8xxx_alloc_sysfs_attr(struct scsi_qla_host *ha);
 void qla4_8xxx_free_sysfs_attr(struct scsi_qla_host *ha);
 void qla4xxx_alloc_fw_dump(struct scsi_qla_host *ha);
+int qla4_82xx_try_start_fw(struct scsi_qla_host *ha);
+int qla4_8xxx_need_reset(struct scsi_qla_host *ha);
+int qla4_82xx_md_rd_32(struct scsi_qla_host *ha, uint32_t off, uint32_t *data);
+int qla4_82xx_md_wr_32(struct scsi_qla_host *ha, uint32_t off, uint32_t data);
+void qla4_82xx_rom_lock_recovery(struct scsi_qla_host *ha);
+void qla4_82xx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			      int incount);
+void qla4_82xx_process_mbox_intr(struct scsi_qla_host *ha, int outcount);
+void qla4xxx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			    int incount);
+void qla4xxx_process_mbox_intr(struct scsi_qla_host *ha, int outcount);
+void qla4_8xxx_dump_peg_reg(struct scsi_qla_host *ha);
+void qla4_83xx_disable_intrs(struct scsi_qla_host *ha);
+void qla4_83xx_enable_intrs(struct scsi_qla_host *ha);
+int qla4_83xx_start_firmware(struct scsi_qla_host *ha);
+irqreturn_t qla4_83xx_intr_handler(int irq, void *dev_id);
+void qla4_83xx_interrupt_service_routine(struct scsi_qla_host *ha,
+					 uint32_t intr_status);
+int qla4_83xx_isp_reset(struct scsi_qla_host *ha);
+void qla4_83xx_queue_iocb(struct scsi_qla_host *ha);
+void qla4_83xx_complete_iocb(struct scsi_qla_host *ha);
+uint16_t qla4_83xx_rd_shdw_req_q_out(struct scsi_qla_host *ha);
+uint16_t qla4_83xx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha);
+uint32_t qla4_83xx_rd_reg(struct scsi_qla_host *ha, ulong addr);
+void qla4_83xx_wr_reg(struct scsi_qla_host *ha, ulong addr, uint32_t val);
+int qla4_83xx_rd_reg_indirect(struct scsi_qla_host *ha, uint32_t addr,
+			      uint32_t *data);
+int qla4_83xx_wr_reg_indirect(struct scsi_qla_host *ha, uint32_t addr,
+			      uint32_t data);
+int qla4_83xx_drv_lock(struct scsi_qla_host *ha);
+void qla4_83xx_drv_unlock(struct scsi_qla_host *ha);
+void qla4_83xx_rom_lock_recovery(struct scsi_qla_host *ha);
+void qla4_83xx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			      int incount);
+void qla4_83xx_process_mbox_intr(struct scsi_qla_host *ha, int outcount);
+void qla4_83xx_read_reset_template(struct scsi_qla_host *ha);
+void qla4_83xx_set_idc_dontreset(struct scsi_qla_host *ha);
+int qla4_83xx_idc_dontreset(struct scsi_qla_host *ha);
+int qla4_83xx_lockless_flash_read_u32(struct scsi_qla_host *ha,
+				      uint32_t flash_addr, uint8_t *p_data,
+				      int u32_word_count);
+void qla4_83xx_clear_idc_dontreset(struct scsi_qla_host *ha);
+void qla4_83xx_need_reset_handler(struct scsi_qla_host *ha);
+int qla4_83xx_flash_read_u32(struct scsi_qla_host *ha, uint32_t flash_addr,
+			     uint8_t *p_data, int u32_word_count);
+void qla4_83xx_get_idc_param(struct scsi_qla_host *ha);
+void qla4_8xxx_set_rst_ready(struct scsi_qla_host *ha);
+void qla4_8xxx_clear_rst_ready(struct scsi_qla_host *ha);
+int qla4_8xxx_device_bootstrap(struct scsi_qla_host *ha);
+void qla4_8xxx_get_minidump(struct scsi_qla_host *ha);
+int qla4_8xxx_mbx_intr_disable(struct scsi_qla_host *ha);
+int qla4_8xxx_mbx_intr_enable(struct scsi_qla_host *ha);
+int qla4_8xxx_set_param(struct scsi_qla_host *ha, int param);
+int qla4_8xxx_update_idc_reg(struct scsi_qla_host *ha);
+int qla4_83xx_post_idc_ack(struct scsi_qla_host *ha);
+void qla4_83xx_disable_pause(struct scsi_qla_host *ha);
 
 extern int ql4xextended_error_logging;
 extern int ql4xdontresethba;
diff --git a/drivers/scsi/qla4xxx/ql4_init.c b/drivers/scsi/qla4xxx/ql4_init.c
index ddd9472066c..1aca1b4f70b 100644
--- a/drivers/scsi/qla4xxx/ql4_init.c
+++ b/drivers/scsi/qla4xxx/ql4_init.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -102,11 +102,18 @@ int qla4xxx_init_rings(struct scsi_qla_host *ha)
 
 	if (is_qla8022(ha)) {
 		writel(0,
-		    (unsigned long  __iomem *)&ha->qla4_8xxx_reg->req_q_out);
+		    (unsigned long  __iomem *)&ha->qla4_82xx_reg->req_q_out);
 		writel(0,
-		    (unsigned long  __iomem *)&ha->qla4_8xxx_reg->rsp_q_in);
+		    (unsigned long  __iomem *)&ha->qla4_82xx_reg->rsp_q_in);
 		writel(0,
-		    (unsigned long  __iomem *)&ha->qla4_8xxx_reg->rsp_q_out);
+		    (unsigned long  __iomem *)&ha->qla4_82xx_reg->rsp_q_out);
+	} else if (is_qla8032(ha)) {
+		writel(0,
+		       (unsigned long __iomem *)&ha->qla4_83xx_reg->req_q_in);
+		writel(0,
+		       (unsigned long __iomem *)&ha->qla4_83xx_reg->rsp_q_in);
+		writel(0,
+		       (unsigned long __iomem *)&ha->qla4_83xx_reg->rsp_q_out);
 	} else {
 		/*
 		 * Initialize DMA Shadow registers.  The firmware is really
@@ -524,7 +531,7 @@ static int qla4xxx_init_firmware(struct scsi_qla_host *ha)
 	/* For 82xx, stop firmware before initializing because if BIOS
 	 * has previously initialized firmware, then driver's initialize
 	 * firmware will fail. */
-	if (is_qla8022(ha))
+	if (is_qla80XX(ha))
 		qla4_8xxx_stop_firmware(ha);
 
 	ql4_printk(KERN_INFO, ha, "Initializing firmware..\n");
@@ -537,7 +544,7 @@ static int qla4xxx_init_firmware(struct scsi_qla_host *ha)
 	if (!qla4xxx_fw_ready(ha))
 		return status;
 
-	if (is_qla8022(ha) && !test_bit(AF_INIT_DONE, &ha->flags))
+	if (is_qla80XX(ha) && !test_bit(AF_INIT_DONE, &ha->flags))
 		qla4xxx_alloc_fw_dump(ha);
 
 	return qla4xxx_get_firmware_status(ha);
@@ -946,9 +953,9 @@ int qla4xxx_initialize_adapter(struct scsi_qla_host *ha, int is_reset)
 
 	set_bit(AF_ONLINE, &ha->flags);
 exit_init_hba:
-	if (is_qla8022(ha) && (status == QLA_ERROR)) {
+	if (is_qla80XX(ha) && (status == QLA_ERROR)) {
 		/* Since interrupts are registered in start_firmware for
-		 * 82xx, release them here if initialize_adapter fails */
+		 * 80XX, release them here if initialize_adapter fails */
 		qla4xxx_free_irqs(ha);
 	}
 
diff --git a/drivers/scsi/qla4xxx/ql4_inline.h b/drivers/scsi/qla4xxx/ql4_inline.h
index 62f90bdec5d..6f4decd44c6 100644
--- a/drivers/scsi/qla4xxx/ql4_inline.h
+++ b/drivers/scsi/qla4xxx/ql4_inline.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla4xxx/ql4_iocb.c b/drivers/scsi/qla4xxx/ql4_iocb.c
index 2a2022a6bb9..f48f37a281d 100644
--- a/drivers/scsi/qla4xxx/ql4_iocb.c
+++ b/drivers/scsi/qla4xxx/ql4_iocb.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -192,35 +192,47 @@ static void qla4xxx_build_scsi_iocbs(struct srb *srb,
 	}
 }
 
+void qla4_83xx_queue_iocb(struct scsi_qla_host *ha)
+{
+	writel(ha->request_in, &ha->qla4_83xx_reg->req_q_in);
+	readl(&ha->qla4_83xx_reg->req_q_in);
+}
+
+void qla4_83xx_complete_iocb(struct scsi_qla_host *ha)
+{
+	writel(ha->response_out, &ha->qla4_83xx_reg->rsp_q_out);
+	readl(&ha->qla4_83xx_reg->rsp_q_out);
+}
+
 /**
- * qla4_8xxx_queue_iocb - Tell ISP it's got new request(s)
+ * qla4_82xx_queue_iocb - Tell ISP it's got new request(s)
  * @ha: pointer to host adapter structure.
  *
  * This routine notifies the ISP that one or more new request
  * queue entries have been placed on the request queue.
  **/
-void qla4_8xxx_queue_iocb(struct scsi_qla_host *ha)
+void qla4_82xx_queue_iocb(struct scsi_qla_host *ha)
 {
 	uint32_t dbval = 0;
 
 	dbval = 0x14 | (ha->func_num << 5);
 	dbval = dbval | (0 << 8) | (ha->request_in << 16);
 
-	qla4_8xxx_wr_32(ha, ha->nx_db_wr_ptr, ha->request_in);
+	qla4_82xx_wr_32(ha, ha->nx_db_wr_ptr, ha->request_in);
 }
 
 /**
- * qla4_8xxx_complete_iocb - Tell ISP we're done with response(s)
+ * qla4_82xx_complete_iocb - Tell ISP we're done with response(s)
  * @ha: pointer to host adapter structure.
  *
  * This routine notifies the ISP that one or more response/completion
  * queue entries have been processed by the driver.
  * This also clears the interrupt.
  **/
-void qla4_8xxx_complete_iocb(struct scsi_qla_host *ha)
+void qla4_82xx_complete_iocb(struct scsi_qla_host *ha)
 {
-	writel(ha->response_out, &ha->qla4_8xxx_reg->rsp_q_out);
-	readl(&ha->qla4_8xxx_reg->rsp_q_out);
+	writel(ha->response_out, &ha->qla4_82xx_reg->rsp_q_out);
+	readl(&ha->qla4_82xx_reg->rsp_q_out);
 }
 
 /**
diff --git a/drivers/scsi/qla4xxx/ql4_isr.c b/drivers/scsi/qla4xxx/ql4_isr.c
index fc542a9bb10..15ea81465ce 100644
--- a/drivers/scsi/qla4xxx/ql4_isr.c
+++ b/drivers/scsi/qla4xxx/ql4_isr.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -126,7 +126,7 @@ static void qla4xxx_status_entry(struct scsi_qla_host *ha,
 		ql4_printk(KERN_WARNING, ha, "%s invalid status entry: "
 			   "handle=0x%0x, srb=%p\n", __func__,
 			   sts_entry->handle, srb);
-		if (is_qla8022(ha))
+		if (is_qla80XX(ha))
 			set_bit(DPC_RESET_HA_FW_CONTEXT, &ha->dpc_flags);
 		else
 			set_bit(DPC_RESET_HA, &ha->dpc_flags);
@@ -243,56 +243,72 @@ static void qla4xxx_status_entry(struct scsi_qla_host *ha,
 
 		scsi_set_resid(cmd, residual);
 
-		/*
-		 * If there is scsi_status, it takes precedense over
-		 * underflow condition.
-		 */
-		if (scsi_status != 0) {
-			cmd->result = DID_OK << 16 | scsi_status;
+		if (sts_entry->iscsiFlags & ISCSI_FLAG_RESIDUAL_UNDER) {
+
+			/* Both the firmware and target reported UNDERRUN:
+			 *
+			 * MID-LAYER UNDERFLOW case:
+			 * Some kernels do not properly detect midlayer
+			 * underflow, so we manually check it and return
+			 * ERROR if the minimum required data was not
+			 * received.
+			 *
+			 * ALL OTHER cases:
+			 * Fall thru to check scsi_status
+			 */
+			if (!scsi_status && (scsi_bufflen(cmd) - residual) <
+			    cmd->underflow) {
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "scsi%ld:%d:%d:%d: %s: Mid-layer Data underrun, xferlen = 0x%x,residual = 0x%x\n",
+						   ha->host_no,
+						   cmd->device->channel,
+						   cmd->device->id,
+						   cmd->device->lun, __func__,
+						   scsi_bufflen(cmd),
+						   residual));
 
-			if (scsi_status != SCSI_CHECK_CONDITION)
+				cmd->result = DID_ERROR << 16;
 				break;
+			}
+
+		} else if (scsi_status != SAM_STAT_TASK_SET_FULL &&
+			   scsi_status != SAM_STAT_BUSY) {
 
-			/* Copy Sense Data into sense buffer. */
-			qla4xxx_copy_sense(ha, sts_entry, srb);
-		} else {
 			/*
-			 * If RISC reports underrun and target does not
-			 * report it then we must have a lost frame, so
-			 * tell upper layer to retry it by reporting a
-			 * bus busy.
+			 * The firmware reports UNDERRUN, but the target does
+			 * not report it:
+			 *
+			 *   scsi_status     |    host_byte       device_byte
+			 *                   |     (19:16)          (7:0)
+			 *   =============   |    =========       ===========
+			 *   TASK_SET_FULL   |    DID_OK          scsi_status
+			 *   BUSY            |    DID_OK          scsi_status
+			 *   ALL OTHERS      |    DID_ERROR       scsi_status
+			 *
+			 *   Note: If scsi_status is task set full or busy,
+			 *   then this else if would fall thru to check the
+			 *   scsi_status and return DID_OK.
 			 */
-			if ((sts_entry->iscsiFlags &
-			     ISCSI_FLAG_RESIDUAL_UNDER) == 0) {
-				cmd->result = DID_BUS_BUSY << 16;
-			} else if ((scsi_bufflen(cmd) - residual) <
-				   cmd->underflow) {
-				/*
-				 * Handle mid-layer underflow???
-				 *
-				 * For kernels less than 2.4, the driver must
-				 * return an error if an underflow is detected.
-				 * For kernels equal-to and above 2.4, the
-				 * mid-layer will appearantly handle the
-				 * underflow by detecting the residual count --
-				 * unfortunately, we do not see where this is
-				 * actually being done.	 In the interim, we
-				 * will return DID_ERROR.
-				 */
-				DEBUG2(printk("scsi%ld:%d:%d:%d: %s: "
-					"Mid-layer Data underrun1, "
-					"xferlen = 0x%x, "
-					"residual = 0x%x\n", ha->host_no,
-					cmd->device->channel,
-					cmd->device->id,
-					cmd->device->lun, __func__,
-					scsi_bufflen(cmd), residual));
 
-				cmd->result = DID_ERROR << 16;
-			} else {
-				cmd->result = DID_OK << 16;
-			}
+			DEBUG2(ql4_printk(KERN_INFO, ha,
+					  "scsi%ld:%d:%d:%d: %s: Dropped frame(s) detected (0x%x of 0x%x bytes).\n",
+					  ha->host_no,
+					  cmd->device->channel,
+					  cmd->device->id,
+					  cmd->device->lun, __func__,
+					  residual,
+					  scsi_bufflen(cmd)));
+
+			cmd->result = DID_ERROR << 16 | scsi_status;
+			goto check_scsi_status;
 		}
+
+		cmd->result = DID_OK << 16 | scsi_status;
+
+check_scsi_status:
+		if (scsi_status == SAM_STAT_CHECK_CONDITION)
+			qla4xxx_copy_sense(ha, sts_entry, srb);
+
 		break;
 
 	case SCS_DEVICE_LOGGED_OUT:
@@ -578,6 +594,14 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 {
 	int i;
 	uint32_t mbox_sts[MBOX_AEN_REG_COUNT];
+	__le32 __iomem *mailbox_out;
+
+	if (is_qla8032(ha))
+		mailbox_out = &ha->qla4_83xx_reg->mailbox_out[0];
+	else if (is_qla8022(ha))
+		mailbox_out = &ha->qla4_82xx_reg->mailbox_out[0];
+	else
+		mailbox_out = &ha->reg->mailbox[0];
 
 	if ((mbox_status == MBOX_STS_BUSY) ||
 	    (mbox_status == MBOX_STS_INTERMEDIATE_COMPLETION) ||
@@ -590,9 +614,7 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 			 * location and set mailbox command done flag
 			 */
 			for (i = 0; i < ha->mbox_status_count; i++)
-				ha->mbox_status[i] = is_qla8022(ha)
-				    ? readl(&ha->qla4_8xxx_reg->mailbox_out[i])
-				    : readl(&ha->reg->mailbox[i]);
+				ha->mbox_status[i] = readl(&mailbox_out[i]);
 
 			set_bit(AF_MBOX_COMMAND_DONE, &ha->flags);
 
@@ -601,9 +623,7 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 		}
 	} else if (mbox_status >> 12 == MBOX_ASYNC_EVENT_STATUS) {
 		for (i = 0; i < MBOX_AEN_REG_COUNT; i++)
-			mbox_sts[i] = is_qla8022(ha)
-			    ? readl(&ha->qla4_8xxx_reg->mailbox_out[i])
-			    : readl(&ha->reg->mailbox[i]);
+			mbox_sts[i] = readl(&mailbox_out[i]);
 
 		/* Immediately process the AENs that don't require much work.
 		 * Only queue the database_changed AENs */
@@ -619,7 +639,8 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 			ql4_printk(KERN_INFO, ha, "%s: System Err\n", __func__);
 			qla4xxx_dump_registers(ha);
 
-			if (ql4xdontresethba) {
+			if ((is_qla8022(ha) && ql4xdontresethba) ||
+			    (is_qla8032(ha) && qla4_83xx_idc_dontreset(ha))) {
 				DEBUG2(printk("scsi%ld: %s:Don't Reset HBA\n",
 				    ha->host_no, __func__));
 			} else {
@@ -635,7 +656,7 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 		case MBOX_ASTS_DHCP_LEASE_EXPIRED:
 			DEBUG2(printk("scsi%ld: AEN %04x, ERROR Status, "
 				      "Reset HA\n", ha->host_no, mbox_status));
-			if (is_qla8022(ha))
+			if (is_qla80XX(ha))
 				set_bit(DPC_RESET_HA_FW_CONTEXT,
 					&ha->dpc_flags);
 			else
@@ -700,7 +721,7 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 				set_bit(DPC_GET_DHCP_IP_ADDR, &ha->dpc_flags);
 			else if ((mbox_sts[3] == ACB_STATE_ACQUIRING) &&
 				 (mbox_sts[2] == ACB_STATE_VALID)) {
-				if (is_qla8022(ha))
+				if (is_qla80XX(ha))
 					set_bit(DPC_RESET_HA_FW_CONTEXT,
 						&ha->dpc_flags);
 				else
@@ -785,6 +806,43 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 			    " removed\n",  ha->host_no, mbox_sts[0]));
 			break;
 
+		case MBOX_ASTS_IDC_NOTIFY:
+		{
+			uint32_t opcode;
+			if (is_qla8032(ha)) {
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "scsi%ld: AEN %04x, mbox_sts[1]=%08x, mbox_sts[2]=%08x, mbox_sts[3]=%08x, mbox_sts[4]=%08x\n",
+						  ha->host_no, mbox_sts[0],
+						  mbox_sts[1], mbox_sts[2],
+						  mbox_sts[3], mbox_sts[4]));
+				opcode = mbox_sts[1] >> 16;
+				if ((opcode == MBOX_CMD_SET_PORT_CONFIG) ||
+				    (opcode == MBOX_CMD_PORT_RESET)) {
+					set_bit(DPC_POST_IDC_ACK,
+						&ha->dpc_flags);
+					ha->idc_info.request_desc = mbox_sts[1];
+					ha->idc_info.info1 = mbox_sts[2];
+					ha->idc_info.info2 = mbox_sts[3];
+					ha->idc_info.info3 = mbox_sts[4];
+					qla4xxx_wake_dpc(ha);
+				}
+			}
+			break;
+		}
+
+		case MBOX_ASTS_IDC_COMPLETE:
+			if (is_qla8032(ha)) {
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "scsi%ld: AEN %04x, mbox_sts[1]=%08x, mbox_sts[2]=%08x, mbox_sts[3]=%08x, mbox_sts[4]=%08x\n",
+						  ha->host_no, mbox_sts[0],
+						  mbox_sts[1], mbox_sts[2],
+						  mbox_sts[3], mbox_sts[4]));
+				DEBUG2(ql4_printk(KERN_INFO, ha,
+						  "scsi:%ld: AEN %04x IDC Complete notification\n",
+						  ha->host_no, mbox_sts[0]));
+			}
+			break;
+
 		default:
 			DEBUG2(printk(KERN_WARNING
 				      "scsi%ld: AEN %04x UNKNOWN\n",
@@ -799,14 +857,31 @@ static void qla4xxx_isr_decode_mailbox(struct scsi_qla_host * ha,
 	}
 }
 
+void qla4_83xx_interrupt_service_routine(struct scsi_qla_host *ha,
+					 uint32_t intr_status)
+{
+	/* Process mailbox/asynch event interrupt.*/
+	if (intr_status) {
+		qla4xxx_isr_decode_mailbox(ha,
+				readl(&ha->qla4_83xx_reg->mailbox_out[0]));
+		/* clear the interrupt */
+		writel(0, &ha->qla4_83xx_reg->risc_intr);
+	} else {
+		qla4xxx_process_response_queue(ha);
+	}
+
+	/* clear the interrupt */
+	writel(0, &ha->qla4_83xx_reg->mb_int_mask);
+}
+
 /**
- * qla4_8xxx_interrupt_service_routine - isr
+ * qla4_82xx_interrupt_service_routine - isr
  * @ha: pointer to host adapter structure.
  *
  * This is the main interrupt service routine.
  * hardware_lock locked upon entry. runs in interrupt context.
  **/
-void qla4_8xxx_interrupt_service_routine(struct scsi_qla_host *ha,
+void qla4_82xx_interrupt_service_routine(struct scsi_qla_host *ha,
     uint32_t intr_status)
 {
 	/* Process response queue interrupt. */
@@ -816,11 +891,11 @@ void qla4_8xxx_interrupt_service_routine(struct scsi_qla_host *ha,
 	/* Process mailbox/asynch event interrupt.*/
 	if (intr_status & HSRX_RISC_MB_INT)
 		qla4xxx_isr_decode_mailbox(ha,
-		    readl(&ha->qla4_8xxx_reg->mailbox_out[0]));
+		    readl(&ha->qla4_82xx_reg->mailbox_out[0]));
 
 	/* clear the interrupt */
-	writel(0, &ha->qla4_8xxx_reg->host_int);
-	readl(&ha->qla4_8xxx_reg->host_int);
+	writel(0, &ha->qla4_82xx_reg->host_int);
+	readl(&ha->qla4_82xx_reg->host_int);
 }
 
 /**
@@ -850,12 +925,12 @@ void qla4xxx_interrupt_service_routine(struct scsi_qla_host * ha,
 }
 
 /**
- * qla4_8xxx_spurious_interrupt - processes spurious interrupt
+ * qla4_82xx_spurious_interrupt - processes spurious interrupt
  * @ha: pointer to host adapter structure.
  * @reqs_count: .
  *
  **/
-static void qla4_8xxx_spurious_interrupt(struct scsi_qla_host *ha,
+static void qla4_82xx_spurious_interrupt(struct scsi_qla_host *ha,
     uint8_t reqs_count)
 {
 	if (reqs_count)
@@ -863,9 +938,9 @@ static void qla4_8xxx_spurious_interrupt(struct scsi_qla_host *ha,
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Spurious Interrupt\n"));
 	if (is_qla8022(ha)) {
-		writel(0, &ha->qla4_8xxx_reg->host_int);
+		writel(0, &ha->qla4_82xx_reg->host_int);
 		if (test_bit(AF_INTx_ENABLED, &ha->flags))
-			qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg,
+			qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg,
 			    0xfbff);
 	}
 	ha->spurious_int_count++;
@@ -968,11 +1043,11 @@ irqreturn_t qla4xxx_intr_handler(int irq, void *dev_id)
 }
 
 /**
- * qla4_8xxx_intr_handler - hardware interrupt handler.
+ * qla4_82xx_intr_handler - hardware interrupt handler.
  * @irq: Unused
  * @dev_id: Pointer to host adapter structure
  **/
-irqreturn_t qla4_8xxx_intr_handler(int irq, void *dev_id)
+irqreturn_t qla4_82xx_intr_handler(int irq, void *dev_id)
 {
 	struct scsi_qla_host *ha = dev_id;
 	uint32_t intr_status;
@@ -984,11 +1059,11 @@ irqreturn_t qla4_8xxx_intr_handler(int irq, void *dev_id)
 		return IRQ_HANDLED;
 
 	ha->isr_count++;
-	status = qla4_8xxx_rd_32(ha, ISR_INT_VECTOR);
+	status = qla4_82xx_rd_32(ha, ISR_INT_VECTOR);
 	if (!(status & ha->nx_legacy_intr.int_vec_bit))
 		return IRQ_NONE;
 
-	status = qla4_8xxx_rd_32(ha, ISR_INT_STATE_REG);
+	status = qla4_82xx_rd_32(ha, ISR_INT_STATE_REG);
 	if (!ISR_IS_LEGACY_INTR_TRIGGERED(status)) {
 		DEBUG2(ql4_printk(KERN_INFO, ha,
 		    "%s legacy Int not triggered\n", __func__));
@@ -996,30 +1071,30 @@ irqreturn_t qla4_8xxx_intr_handler(int irq, void *dev_id)
 	}
 
 	/* clear the interrupt */
-	qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_status_reg, 0xffffffff);
+	qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_status_reg, 0xffffffff);
 
 	/* read twice to ensure write is flushed */
-	qla4_8xxx_rd_32(ha, ISR_INT_VECTOR);
-	qla4_8xxx_rd_32(ha, ISR_INT_VECTOR);
+	qla4_82xx_rd_32(ha, ISR_INT_VECTOR);
+	qla4_82xx_rd_32(ha, ISR_INT_VECTOR);
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	while (1) {
-		if (!(readl(&ha->qla4_8xxx_reg->host_int) &
+		if (!(readl(&ha->qla4_82xx_reg->host_int) &
 		    ISRX_82XX_RISC_INT)) {
-			qla4_8xxx_spurious_interrupt(ha, reqs_count);
+			qla4_82xx_spurious_interrupt(ha, reqs_count);
 			break;
 		}
-		intr_status =  readl(&ha->qla4_8xxx_reg->host_status);
+		intr_status =  readl(&ha->qla4_82xx_reg->host_status);
 		if ((intr_status &
 		    (HSRX_RISC_MB_INT | HSRX_RISC_IOCB_INT)) == 0)  {
-			qla4_8xxx_spurious_interrupt(ha, reqs_count);
+			qla4_82xx_spurious_interrupt(ha, reqs_count);
 			break;
 		}
 
 		ha->isp_ops->interrupt_service_routine(ha, intr_status);
 
 		/* Enable Interrupt */
-		qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff);
+		qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff);
 
 		if (++reqs_count == MAX_REQS_SERVICED_PER_INTR)
 			break;
@@ -1029,6 +1104,59 @@ irqreturn_t qla4_8xxx_intr_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+#define LEG_INT_PTR_B31		(1 << 31)
+#define LEG_INT_PTR_B30		(1 << 30)
+#define PF_BITS_MASK		(0xF << 16)
+
+/**
+ * qla4_83xx_intr_handler - hardware interrupt handler.
+ * @irq: Unused
+ * @dev_id: Pointer to host adapter structure
+ **/
+irqreturn_t qla4_83xx_intr_handler(int irq, void *dev_id)
+{
+	struct scsi_qla_host *ha = dev_id;
+	uint32_t leg_int_ptr = 0;
+	unsigned long flags = 0;
+
+	ha->isr_count++;
+	leg_int_ptr = readl(&ha->qla4_83xx_reg->leg_int_ptr);
+
+	/* Legacy interrupt is valid if bit31 of leg_int_ptr is set */
+	if (!(leg_int_ptr & LEG_INT_PTR_B31)) {
+		ql4_printk(KERN_ERR, ha,
+			   "%s: Legacy Interrupt Bit 31 not set, spurious interrupt!\n",
+			   __func__);
+		return IRQ_NONE;
+	}
+
+	/* Validate the PCIE function ID set in leg_int_ptr bits [19..16] */
+	if ((leg_int_ptr & PF_BITS_MASK) != ha->pf_bit) {
+		ql4_printk(KERN_ERR, ha,
+			   "%s: Incorrect function ID 0x%x in legacy interrupt register, ha->pf_bit = 0x%x\n",
+			   __func__, (leg_int_ptr & PF_BITS_MASK), ha->pf_bit);
+		return IRQ_NONE;
+	}
+
+	/* To de-assert legacy interrupt, write 0 to Legacy Interrupt Trigger
+	 * Control register and poll till Legacy Interrupt Pointer register
+	 * bit30 is 0.
+	 */
+	writel(0, &ha->qla4_83xx_reg->leg_int_trig);
+	do {
+		leg_int_ptr = readl(&ha->qla4_83xx_reg->leg_int_ptr);
+		if ((leg_int_ptr & PF_BITS_MASK) != ha->pf_bit)
+			break;
+	} while (leg_int_ptr & LEG_INT_PTR_B30);
+
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+	leg_int_ptr = readl(&ha->qla4_83xx_reg->risc_intr);
+	ha->isp_ops->interrupt_service_routine(ha, leg_int_ptr);
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
+	return IRQ_HANDLED;
+}
+
 irqreturn_t
 qla4_8xxx_msi_handler(int irq, void *dev_id)
 {
@@ -1043,15 +1171,46 @@ qla4_8xxx_msi_handler(int irq, void *dev_id)
 
 	ha->isr_count++;
 	/* clear the interrupt */
-	qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_status_reg, 0xffffffff);
+	qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_status_reg, 0xffffffff);
 
 	/* read twice to ensure write is flushed */
-	qla4_8xxx_rd_32(ha, ISR_INT_VECTOR);
-	qla4_8xxx_rd_32(ha, ISR_INT_VECTOR);
+	qla4_82xx_rd_32(ha, ISR_INT_VECTOR);
+	qla4_82xx_rd_32(ha, ISR_INT_VECTOR);
 
 	return qla4_8xxx_default_intr_handler(irq, dev_id);
 }
 
+static irqreturn_t qla4_83xx_mailbox_intr_handler(int irq, void *dev_id)
+{
+	struct scsi_qla_host *ha = dev_id;
+	unsigned long flags;
+	uint32_t ival = 0;
+
+	spin_lock_irqsave(&ha->hardware_lock, flags);
+
+	ival = readl(&ha->qla4_83xx_reg->risc_intr);
+	if (ival == 0) {
+		ql4_printk(KERN_INFO, ha,
+			   "%s: It is a spurious mailbox interrupt!\n",
+			   __func__);
+		ival = readl(&ha->qla4_83xx_reg->mb_int_mask);
+		ival &= ~INT_MASK_FW_MB;
+		writel(ival, &ha->qla4_83xx_reg->mb_int_mask);
+		goto exit;
+	}
+
+	qla4xxx_isr_decode_mailbox(ha,
+				   readl(&ha->qla4_83xx_reg->mailbox_out[0]));
+	writel(0, &ha->qla4_83xx_reg->risc_intr);
+	ival = readl(&ha->qla4_83xx_reg->mb_int_mask);
+	ival &= ~INT_MASK_FW_MB;
+	writel(ival, &ha->qla4_83xx_reg->mb_int_mask);
+	ha->isr_count++;
+exit:
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+	return IRQ_HANDLED;
+}
+
 /**
  * qla4_8xxx_default_intr_handler - hardware interrupt handler.
  * @irq: Unused
@@ -1068,29 +1227,32 @@ qla4_8xxx_default_intr_handler(int irq, void *dev_id)
 	uint32_t intr_status;
 	uint8_t reqs_count = 0;
 
-	spin_lock_irqsave(&ha->hardware_lock, flags);
-	while (1) {
-		if (!(readl(&ha->qla4_8xxx_reg->host_int) &
-		    ISRX_82XX_RISC_INT)) {
-			qla4_8xxx_spurious_interrupt(ha, reqs_count);
-			break;
-		}
+	if (is_qla8032(ha)) {
+		qla4_83xx_mailbox_intr_handler(irq, dev_id);
+	} else {
+		spin_lock_irqsave(&ha->hardware_lock, flags);
+		while (1) {
+			if (!(readl(&ha->qla4_82xx_reg->host_int) &
+			    ISRX_82XX_RISC_INT)) {
+				qla4_82xx_spurious_interrupt(ha, reqs_count);
+				break;
+			}
 
-		intr_status =  readl(&ha->qla4_8xxx_reg->host_status);
-		if ((intr_status &
-		    (HSRX_RISC_MB_INT | HSRX_RISC_IOCB_INT)) == 0) {
-			qla4_8xxx_spurious_interrupt(ha, reqs_count);
-			break;
-		}
+			intr_status =  readl(&ha->qla4_82xx_reg->host_status);
+			if ((intr_status &
+			    (HSRX_RISC_MB_INT | HSRX_RISC_IOCB_INT)) == 0) {
+				qla4_82xx_spurious_interrupt(ha, reqs_count);
+				break;
+			}
 
-		ha->isp_ops->interrupt_service_routine(ha, intr_status);
+			ha->isp_ops->interrupt_service_routine(ha, intr_status);
 
-		if (++reqs_count == MAX_REQS_SERVICED_PER_INTR)
-			break;
+			if (++reqs_count == MAX_REQS_SERVICED_PER_INTR)
+				break;
+		}
+		ha->isr_count++;
+		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	}
-
-	ha->isr_count++;
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -1099,13 +1261,25 @@ qla4_8xxx_msix_rsp_q(int irq, void *dev_id)
 {
 	struct scsi_qla_host *ha = dev_id;
 	unsigned long flags;
+	uint32_t ival = 0;
 
 	spin_lock_irqsave(&ha->hardware_lock, flags);
-	qla4xxx_process_response_queue(ha);
-	writel(0, &ha->qla4_8xxx_reg->host_int);
-	spin_unlock_irqrestore(&ha->hardware_lock, flags);
-
+	if (is_qla8032(ha)) {
+		ival = readl(&ha->qla4_83xx_reg->iocb_int_mask);
+		if (ival == 0) {
+			ql4_printk(KERN_INFO, ha, "%s: It is a spurious iocb interrupt!\n",
+				   __func__);
+			goto exit_msix_rsp_q;
+		}
+		qla4xxx_process_response_queue(ha);
+		writel(0, &ha->qla4_83xx_reg->iocb_int_mask);
+	} else {
+		qla4xxx_process_response_queue(ha);
+		writel(0, &ha->qla4_82xx_reg->host_int);
+	}
 	ha->isr_count++;
+exit_msix_rsp_q:
+	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	return IRQ_HANDLED;
 }
 
@@ -1177,11 +1351,18 @@ int qla4xxx_request_irqs(struct scsi_qla_host *ha)
 {
 	int ret;
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		goto try_intx;
 
-	if (ql4xenablemsix == 2)
+	if (ql4xenablemsix == 2) {
+		/* Note: MSI Interrupts not supported for ISP8324 */
+		if (is_qla8032(ha)) {
+			ql4_printk(KERN_INFO, ha, "%s: MSI Interrupts not supported for ISP8324, Falling back-to INTx mode\n",
+				   __func__);
+			goto try_intx;
+		}
 		goto try_msi;
+	}
 
 	if (ql4xenablemsix == 0 || ql4xenablemsix != 1)
 		goto try_intx;
@@ -1192,6 +1373,12 @@ int qla4xxx_request_irqs(struct scsi_qla_host *ha)
 		DEBUG2(ql4_printk(KERN_INFO, ha,
 		    "MSI-X: Enabled (0x%X).\n", ha->revision_id));
 		goto irq_attached;
+	} else {
+		if (is_qla8032(ha)) {
+			ql4_printk(KERN_INFO, ha, "%s: ISP8324: MSI-X: Falling back-to INTx mode. ret = %d\n",
+				   __func__, ret);
+			goto try_intx;
+		}
 	}
 
 	ql4_printk(KERN_WARNING, ha,
@@ -1214,9 +1401,15 @@ try_msi:
 			pci_disable_msi(ha->pdev);
 		}
 	}
-	ql4_printk(KERN_WARNING, ha,
-	    "MSI: Falling back-to INTx mode -- %d.\n", ret);
 
+	/*
+	 * Prevent interrupts from falling back to INTx mode in cases where
+	 * interrupts cannot get acquired through MSI-X or MSI mode.
+	 */
+	if (is_qla8022(ha)) {
+		ql4_printk(KERN_WARNING, ha, "IRQ not attached -- %d.\n", ret);
+		goto irq_not_attached;
+	}
 try_intx:
 	/* Trying INTx */
 	ret = request_irq(ha->pdev->irq, ha->isp_ops->intr_handler,
@@ -1230,7 +1423,7 @@ try_intx:
 		ql4_printk(KERN_WARNING, ha,
 		    "INTx: Failed to reserve interrupt %d already in"
 		    " use.\n", ha->pdev->irq);
-		return ret;
+		goto irq_not_attached;
 	}
 
 irq_attached:
@@ -1238,6 +1431,7 @@ irq_attached:
 	ha->host->irq = ha->pdev->irq;
 	ql4_printk(KERN_INFO, ha, "%s: irq %d attached\n",
 	    __func__, ha->pdev->irq);
+irq_not_attached:
 	return ret;
 }
 
diff --git a/drivers/scsi/qla4xxx/ql4_mbx.c b/drivers/scsi/qla4xxx/ql4_mbx.c
index cab8f665a41..3d41034191f 100644
--- a/drivers/scsi/qla4xxx/ql4_mbx.c
+++ b/drivers/scsi/qla4xxx/ql4_mbx.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -9,7 +9,39 @@
 #include "ql4_glbl.h"
 #include "ql4_dbg.h"
 #include "ql4_inline.h"
+#include "ql4_version.h"
 
+void qla4xxx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			    int in_count)
+{
+	int i;
+
+	/* Load all mailbox registers, except mailbox 0. */
+	for (i = 1; i < in_count; i++)
+		writel(mbx_cmd[i], &ha->reg->mailbox[i]);
+
+	/* Wakeup firmware  */
+	writel(mbx_cmd[0], &ha->reg->mailbox[0]);
+	readl(&ha->reg->mailbox[0]);
+	writel(set_rmask(CSR_INTR_RISC), &ha->reg->ctrl_status);
+	readl(&ha->reg->ctrl_status);
+}
+
+void qla4xxx_process_mbox_intr(struct scsi_qla_host *ha, int out_count)
+{
+	int intr_status;
+
+	intr_status = readl(&ha->reg->ctrl_status);
+	if (intr_status & INTR_PENDING) {
+		/*
+		 * Service the interrupt.
+		 * The ISR will save the mailbox status registers
+		 * to a temporary storage location in the adapter structure.
+		 */
+		ha->mbox_status_count = out_count;
+		ha->isp_ops->interrupt_service_routine(ha, intr_status);
+	}
+}
 
 /**
  * qla4xxx_mailbox_command - issues mailbox commands
@@ -30,7 +62,6 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 	int status = QLA_ERROR;
 	uint8_t i;
 	u_long wait_count;
-	uint32_t intr_status;
 	unsigned long flags = 0;
 	uint32_t dev_state;
 
@@ -77,7 +108,7 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 		msleep(10);
 	}
 
-	if (is_qla8022(ha)) {
+	if (is_qla80XX(ha)) {
 		if (test_bit(AF_FW_RECOVERY, &ha->flags)) {
 			DEBUG2(ql4_printk(KERN_WARNING, ha,
 					  "scsi%ld: %s: prematurely completing mbx cmd as firmware recovery detected\n",
@@ -85,10 +116,10 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 			goto mbox_exit;
 		}
 		/* Do not send any mbx cmd if h/w is in failed state*/
-		qla4_8xxx_idc_lock(ha);
-		dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
-		qla4_8xxx_idc_unlock(ha);
-		if (dev_state == QLA82XX_DEV_FAILED) {
+		ha->isp_ops->idc_lock(ha);
+		dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
+		ha->isp_ops->idc_unlock(ha);
+		if (dev_state == QLA8XXX_DEV_FAILED) {
 			ql4_printk(KERN_WARNING, ha,
 				   "scsi%ld: %s: H/W is in failed state, do not send any mailbox commands\n",
 				   ha->host_no, __func__);
@@ -102,30 +133,8 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 	for (i = 0; i < outCount; i++)
 		ha->mbox_status[i] = 0;
 
-	if (is_qla8022(ha)) {
-		/* Load all mailbox registers, except mailbox 0. */
-		DEBUG5(
-		    printk("scsi%ld: %s: Cmd ", ha->host_no, __func__);
-		    for (i = 0; i < inCount; i++)
-			printk("mb%d=%04x ", i, mbx_cmd[i]);
-		    printk("\n"));
-
-		for (i = 1; i < inCount; i++)
-			writel(mbx_cmd[i], &ha->qla4_8xxx_reg->mailbox_in[i]);
-		writel(mbx_cmd[0], &ha->qla4_8xxx_reg->mailbox_in[0]);
-		readl(&ha->qla4_8xxx_reg->mailbox_in[0]);
-		writel(HINT_MBX_INT_PENDING, &ha->qla4_8xxx_reg->hint);
-	} else {
-		/* Load all mailbox registers, except mailbox 0. */
-		for (i = 1; i < inCount; i++)
-			writel(mbx_cmd[i], &ha->reg->mailbox[i]);
-
-		/* Wakeup firmware  */
-		writel(mbx_cmd[0], &ha->reg->mailbox[0]);
-		readl(&ha->reg->mailbox[0]);
-		writel(set_rmask(CSR_INTR_RISC), &ha->reg->ctrl_status);
-		readl(&ha->reg->ctrl_status);
-	}
+	/* Queue the mailbox command to the firmware */
+	ha->isp_ops->queue_mailbox_command(ha, mbx_cmd, inCount);
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
 
@@ -167,37 +176,7 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 			 */
 
 			spin_lock_irqsave(&ha->hardware_lock, flags);
-			if (is_qla8022(ha)) {
-				intr_status =
-				    readl(&ha->qla4_8xxx_reg->host_int);
-				if (intr_status & ISRX_82XX_RISC_INT) {
-					ha->mbox_status_count = outCount;
-					intr_status =
-					 readl(&ha->qla4_8xxx_reg->host_status);
-					ha->isp_ops->interrupt_service_routine(
-					    ha, intr_status);
-					if (test_bit(AF_INTERRUPTS_ON,
-					    &ha->flags) &&
-					    test_bit(AF_INTx_ENABLED,
-					    &ha->flags))
-						qla4_8xxx_wr_32(ha,
-						ha->nx_legacy_intr.tgt_mask_reg,
-						0xfbff);
-				}
-			} else {
-				intr_status = readl(&ha->reg->ctrl_status);
-				if (intr_status & INTR_PENDING) {
-					/*
-					 * Service the interrupt.
-					 * The ISR will save the mailbox status
-					 * registers to a temporary storage
-					 * location in the adapter structure.
-					 */
-					ha->mbox_status_count = outCount;
-					ha->isp_ops->interrupt_service_routine(
-					    ha, intr_status);
-				}
-			}
+			ha->isp_ops->process_mailbox_interrupt(ha, outCount);
 			spin_unlock_irqrestore(&ha->hardware_lock, flags);
 			msleep(10);
 		}
@@ -205,7 +184,7 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 
 	/* Check for mailbox timeout. */
 	if (!test_bit(AF_MBOX_COMMAND_DONE, &ha->flags)) {
-		if (is_qla8022(ha) &&
+		if (is_qla80XX(ha) &&
 		    test_bit(AF_FW_RECOVERY, &ha->flags)) {
 			DEBUG2(ql4_printk(KERN_INFO, ha,
 			    "scsi%ld: %s: prematurely completing mbx cmd as "
@@ -222,9 +201,13 @@ int qla4xxx_mailbox_command(struct scsi_qla_host *ha, uint8_t inCount,
 		if (is_qla8022(ha)) {
 			ql4_printk(KERN_INFO, ha,
 				   "disabling pause transmit on port 0 & 1.\n");
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
+			qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
 					CRB_NIU_XG_PAUSE_CTL_P0 |
 					CRB_NIU_XG_PAUSE_CTL_P1);
+		} else if (is_qla8032(ha)) {
+			ql4_printk(KERN_INFO, ha, " %s: disabling pause transmit on port 0 & 1.\n",
+				   __func__);
+			qla4_83xx_disable_pause(ha);
 		}
 		goto mbox_exit;
 	}
@@ -373,7 +356,7 @@ qla4xxx_set_ifcb(struct scsi_qla_host *ha, uint32_t *mbox_cmd,
 	memset(mbox_sts, 0, sizeof(mbox_sts[0]) * MBOX_REG_COUNT);
 
 	if (is_qla8022(ha))
-		qla4_8xxx_wr_32(ha, ha->nx_db_wr_ptr, 0);
+		qla4_82xx_wr_32(ha, ha->nx_db_wr_ptr, 0);
 
 	mbox_cmd[0] = MBOX_CMD_INITIALIZE_FIRMWARE;
 	mbox_cmd[1] = 0;
@@ -566,7 +549,7 @@ int qla4xxx_initialize_fw_cb(struct scsi_qla_host * ha)
 		__constant_cpu_to_le16(FWOPT_SESSION_MODE |
 				       FWOPT_INITIATOR_MODE);
 
-	if (is_qla8022(ha))
+	if (is_qla80XX(ha))
 		init_fw_cb->fw_options |=
 		    __constant_cpu_to_le16(FWOPT_ENABLE_CRBDB);
 
@@ -1695,7 +1678,7 @@ int qla4xxx_set_param_ddbentry(struct scsi_qla_host *ha,
 	conn = cls_conn->dd_data;
 	qla_conn = conn->dd_data;
 	sess = conn->session;
-	dst_addr = &qla_conn->qla_ep->dst_addr;
+	dst_addr = (struct sockaddr *)&qla_conn->qla_ep->dst_addr;
 
 	if (dst_addr->sa_family == AF_INET6)
 		options |= IPV6_DEFAULT_DDB_ENTRY;
@@ -1953,3 +1936,72 @@ int qla4xxx_restore_factory_defaults(struct scsi_qla_host *ha,
 	}
 	return status;
 }
+
+/**
+ * qla4_8xxx_set_param - set driver version in firmware.
+ * @ha: Pointer to host adapter structure.
+ * @param: Parameter to set i.e driver version
+ **/
+int qla4_8xxx_set_param(struct scsi_qla_host *ha, int param)
+{
+	uint32_t mbox_cmd[MBOX_REG_COUNT];
+	uint32_t mbox_sts[MBOX_REG_COUNT];
+	uint32_t status;
+
+	memset(&mbox_cmd, 0, sizeof(mbox_cmd));
+	memset(&mbox_sts, 0, sizeof(mbox_sts));
+
+	mbox_cmd[0] = MBOX_CMD_SET_PARAM;
+	if (param == SET_DRVR_VERSION) {
+		mbox_cmd[1] = SET_DRVR_VERSION;
+		strncpy((char *)&mbox_cmd[2], QLA4XXX_DRIVER_VERSION,
+			MAX_DRVR_VER_LEN);
+	} else {
+		ql4_printk(KERN_ERR, ha, "%s: invalid parameter 0x%x\n",
+			   __func__, param);
+		status = QLA_ERROR;
+		goto exit_set_param;
+	}
+
+	status = qla4xxx_mailbox_command(ha, MBOX_REG_COUNT, 2, mbox_cmd,
+					 mbox_sts);
+	if (status == QLA_ERROR)
+		ql4_printk(KERN_ERR, ha, "%s: failed status %04X\n",
+			   __func__, mbox_sts[0]);
+
+exit_set_param:
+	return status;
+}
+
+/**
+ * qla4_83xx_post_idc_ack - post IDC ACK
+ * @ha: Pointer to host adapter structure.
+ *
+ * Posts IDC ACK for IDC Request Notification AEN.
+ **/
+int qla4_83xx_post_idc_ack(struct scsi_qla_host *ha)
+{
+	uint32_t mbox_cmd[MBOX_REG_COUNT];
+	uint32_t mbox_sts[MBOX_REG_COUNT];
+	int status;
+
+	memset(&mbox_cmd, 0, sizeof(mbox_cmd));
+	memset(&mbox_sts, 0, sizeof(mbox_sts));
+
+	mbox_cmd[0] = MBOX_CMD_IDC_ACK;
+	mbox_cmd[1] = ha->idc_info.request_desc;
+	mbox_cmd[2] = ha->idc_info.info1;
+	mbox_cmd[3] = ha->idc_info.info2;
+	mbox_cmd[4] = ha->idc_info.info3;
+
+	status = qla4xxx_mailbox_command(ha, MBOX_REG_COUNT, MBOX_REG_COUNT,
+					 mbox_cmd, mbox_sts);
+	if (status == QLA_ERROR)
+		ql4_printk(KERN_ERR, ha, "%s: failed status %04X\n", __func__,
+			   mbox_sts[0]);
+	else
+	       DEBUG2(ql4_printk(KERN_INFO, ha, "%s: IDC ACK posted\n",
+				 __func__));
+
+	return status;
+}
diff --git a/drivers/scsi/qla4xxx/ql4_nvram.c b/drivers/scsi/qla4xxx/ql4_nvram.c
index 7851f314ba9..325db1f2c09 100644
--- a/drivers/scsi/qla4xxx/ql4_nvram.c
+++ b/drivers/scsi/qla4xxx/ql4_nvram.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla4xxx/ql4_nvram.h b/drivers/scsi/qla4xxx/ql4_nvram.h
index 945cc328f57..dba0514d1c7 100644
--- a/drivers/scsi/qla4xxx/ql4_nvram.h
+++ b/drivers/scsi/qla4xxx/ql4_nvram.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c
index 807bf76f1b6..499a92db1cf 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.c
+++ b/drivers/scsi/qla4xxx/ql4_nx.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -10,6 +10,7 @@
 #include <linux/ratelimit.h>
 #include "ql4_def.h"
 #include "ql4_glbl.h"
+#include "ql4_inline.h"
 
 #include <asm-generic/io-64-nonatomic-lo-hi.h>
 
@@ -27,7 +28,7 @@
 #define CRB_BLK(off)	((off >> 20) & 0x3f)
 #define CRB_SUBBLK(off)	((off >> 16) & 0xf)
 #define CRB_WINDOW_2M	(0x130060)
-#define CRB_HI(off)	((qla4_8xxx_crb_hub_agt[CRB_BLK(off)] << 20) | \
+#define CRB_HI(off)	((qla4_82xx_crb_hub_agt[CRB_BLK(off)] << 20) | \
 			((off) & 0xf0000))
 #define QLA82XX_PCI_CAMQM_2M_END	(0x04800800UL)
 #define QLA82XX_PCI_CAMQM_2M_BASE	(0x000ff800UL)
@@ -51,7 +52,7 @@ static int qla4_8xxx_crb_table_initialized;
 	(crb_addr_xform[QLA82XX_HW_PX_MAP_CRB_##name] = \
 	 QLA82XX_HW_CRB_HUB_AGT_ADR_##name << 20)
 static void
-qla4_8xxx_crb_addr_transform_setup(void)
+qla4_82xx_crb_addr_transform_setup(void)
 {
 	qla4_8xxx_crb_addr_transform(XDMA);
 	qla4_8xxx_crb_addr_transform(TIMR);
@@ -268,7 +269,7 @@ static struct crb_128M_2M_block_map crb_128M_2M_map[64] = {
 /*
  * top 12 bits of crb internal address (hub, agent)
  */
-static unsigned qla4_8xxx_crb_hub_agt[64] = {
+static unsigned qla4_82xx_crb_hub_agt[64] = {
 	0,
 	QLA82XX_HW_CRB_HUB_AGT_ADR_PS,
 	QLA82XX_HW_CRB_HUB_AGT_ADR_MN,
@@ -353,7 +354,7 @@ static char *qdev_state[] = {
  * side effect: lock crb window
  */
 static void
-qla4_8xxx_pci_set_crbwindow_2M(struct scsi_qla_host *ha, ulong *off)
+qla4_82xx_pci_set_crbwindow_2M(struct scsi_qla_host *ha, ulong *off)
 {
 	u32 win_read;
 
@@ -373,96 +374,115 @@ qla4_8xxx_pci_set_crbwindow_2M(struct scsi_qla_host *ha, ulong *off)
 }
 
 void
-qla4_8xxx_wr_32(struct scsi_qla_host *ha, ulong off, u32 data)
+qla4_82xx_wr_32(struct scsi_qla_host *ha, ulong off, u32 data)
 {
 	unsigned long flags = 0;
 	int rv;
 
-	rv = qla4_8xxx_pci_get_crb_addr_2M(ha, &off);
+	rv = qla4_82xx_pci_get_crb_addr_2M(ha, &off);
 
 	BUG_ON(rv == -1);
 
 	if (rv == 1) {
 		write_lock_irqsave(&ha->hw_lock, flags);
-		qla4_8xxx_crb_win_lock(ha);
-		qla4_8xxx_pci_set_crbwindow_2M(ha, &off);
+		qla4_82xx_crb_win_lock(ha);
+		qla4_82xx_pci_set_crbwindow_2M(ha, &off);
 	}
 
 	writel(data, (void __iomem *)off);
 
 	if (rv == 1) {
-		qla4_8xxx_crb_win_unlock(ha);
+		qla4_82xx_crb_win_unlock(ha);
 		write_unlock_irqrestore(&ha->hw_lock, flags);
 	}
 }
 
-int
-qla4_8xxx_rd_32(struct scsi_qla_host *ha, ulong off)
+uint32_t qla4_82xx_rd_32(struct scsi_qla_host *ha, ulong off)
 {
 	unsigned long flags = 0;
 	int rv;
 	u32 data;
 
-	rv = qla4_8xxx_pci_get_crb_addr_2M(ha, &off);
+	rv = qla4_82xx_pci_get_crb_addr_2M(ha, &off);
 
 	BUG_ON(rv == -1);
 
 	if (rv == 1) {
 		write_lock_irqsave(&ha->hw_lock, flags);
-		qla4_8xxx_crb_win_lock(ha);
-		qla4_8xxx_pci_set_crbwindow_2M(ha, &off);
+		qla4_82xx_crb_win_lock(ha);
+		qla4_82xx_pci_set_crbwindow_2M(ha, &off);
 	}
 	data = readl((void __iomem *)off);
 
 	if (rv == 1) {
-		qla4_8xxx_crb_win_unlock(ha);
+		qla4_82xx_crb_win_unlock(ha);
 		write_unlock_irqrestore(&ha->hw_lock, flags);
 	}
 	return data;
 }
 
 /* Minidump related functions */
-static int qla4_8xxx_md_rw_32(struct scsi_qla_host *ha, uint32_t off,
-			      u32 data, uint8_t flag)
+int qla4_82xx_md_rd_32(struct scsi_qla_host *ha, uint32_t off, uint32_t *data)
 {
-	uint32_t win_read, off_value, rval = QLA_SUCCESS;
+	uint32_t win_read, off_value;
+	int rval = QLA_SUCCESS;
 
 	off_value  = off & 0xFFFF0000;
 	writel(off_value, (void __iomem *)(CRB_WINDOW_2M + ha->nx_pcibase));
 
-	/* Read back value to make sure write has gone through before trying
+	/*
+	 * Read back value to make sure write has gone through before trying
 	 * to use it.
 	 */
 	win_read = readl((void __iomem *)(CRB_WINDOW_2M + ha->nx_pcibase));
 	if (win_read != off_value) {
 		DEBUG2(ql4_printk(KERN_INFO, ha,
 				  "%s: Written (0x%x) != Read (0x%x), off=0x%x\n",
-				   __func__, off_value, win_read, off));
-		return QLA_ERROR;
+				  __func__, off_value, win_read, off));
+		rval = QLA_ERROR;
+	} else {
+		off_value  = off & 0x0000FFFF;
+		*data = readl((void __iomem *)(off_value + CRB_INDIRECT_2M +
+					       ha->nx_pcibase));
 	}
+	return rval;
+}
 
-	off_value  = off & 0x0000FFFF;
+int qla4_82xx_md_wr_32(struct scsi_qla_host *ha, uint32_t off, uint32_t data)
+{
+	uint32_t win_read, off_value;
+	int rval = QLA_SUCCESS;
+
+	off_value  = off & 0xFFFF0000;
+	writel(off_value, (void __iomem *)(CRB_WINDOW_2M + ha->nx_pcibase));
 
-	if (flag)
+	/* Read back value to make sure write has gone through before trying
+	 * to use it.
+	 */
+	win_read = readl((void __iomem *)(CRB_WINDOW_2M + ha->nx_pcibase));
+	if (win_read != off_value) {
+		DEBUG2(ql4_printk(KERN_INFO, ha,
+				  "%s: Written (0x%x) != Read (0x%x), off=0x%x\n",
+				  __func__, off_value, win_read, off));
+		rval = QLA_ERROR;
+	} else {
+		off_value  = off & 0x0000FFFF;
 		writel(data, (void __iomem *)(off_value + CRB_INDIRECT_2M +
 					      ha->nx_pcibase));
-	else
-		rval = readl((void __iomem *)(off_value + CRB_INDIRECT_2M +
-					      ha->nx_pcibase));
-
+	}
 	return rval;
 }
 
 #define CRB_WIN_LOCK_TIMEOUT 100000000
 
-int qla4_8xxx_crb_win_lock(struct scsi_qla_host *ha)
+int qla4_82xx_crb_win_lock(struct scsi_qla_host *ha)
 {
 	int i;
 	int done = 0, timeout = 0;
 
 	while (!done) {
 		/* acquire semaphore3 from PCI HW block */
-		done = qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM7_LOCK));
+		done = qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM7_LOCK));
 		if (done == 1)
 			break;
 		if (timeout >= CRB_WIN_LOCK_TIMEOUT)
@@ -478,32 +498,32 @@ int qla4_8xxx_crb_win_lock(struct scsi_qla_host *ha)
 				cpu_relax();    /*This a nop instr on i386*/
 		}
 	}
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_WIN_LOCK_ID, ha->func_num);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_WIN_LOCK_ID, ha->func_num);
 	return 0;
 }
 
-void qla4_8xxx_crb_win_unlock(struct scsi_qla_host *ha)
+void qla4_82xx_crb_win_unlock(struct scsi_qla_host *ha)
 {
-	qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM7_UNLOCK));
+	qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM7_UNLOCK));
 }
 
 #define IDC_LOCK_TIMEOUT 100000000
 
 /**
- * qla4_8xxx_idc_lock - hw_lock
+ * qla4_82xx_idc_lock - hw_lock
  * @ha: pointer to adapter structure
  *
  * General purpose lock used to synchronize access to
  * CRB_DEV_STATE, CRB_DEV_REF_COUNT, etc.
  **/
-int qla4_8xxx_idc_lock(struct scsi_qla_host *ha)
+int qla4_82xx_idc_lock(struct scsi_qla_host *ha)
 {
 	int i;
 	int done = 0, timeout = 0;
 
 	while (!done) {
 		/* acquire semaphore5 from PCI HW block */
-		done = qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM5_LOCK));
+		done = qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM5_LOCK));
 		if (done == 1)
 			break;
 		if (timeout >= IDC_LOCK_TIMEOUT)
@@ -522,13 +542,13 @@ int qla4_8xxx_idc_lock(struct scsi_qla_host *ha)
 	return 0;
 }
 
-void qla4_8xxx_idc_unlock(struct scsi_qla_host *ha)
+void qla4_82xx_idc_unlock(struct scsi_qla_host *ha)
 {
-	qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM5_UNLOCK));
+	qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM5_UNLOCK));
 }
 
 int
-qla4_8xxx_pci_get_crb_addr_2M(struct scsi_qla_host *ha, ulong *off)
+qla4_82xx_pci_get_crb_addr_2M(struct scsi_qla_host *ha, ulong *off)
 {
 	struct crb_128M_2M_sub_block_map *m;
 
@@ -562,44 +582,40 @@ qla4_8xxx_pci_get_crb_addr_2M(struct scsi_qla_host *ha, ulong *off)
 	return 1;
 }
 
-/*  PCI Windowing for DDR regions.  */
-#define QLA82XX_ADDR_IN_RANGE(addr, low, high)            \
-	(((addr) <= (high)) && ((addr) >= (low)))
-
 /*
 * check memory access boundary.
 * used by test agent. support ddr access only for now
 */
 static unsigned long
-qla4_8xxx_pci_mem_bound_check(struct scsi_qla_host *ha,
+qla4_82xx_pci_mem_bound_check(struct scsi_qla_host *ha,
 		unsigned long long addr, int size)
 {
-	if (!QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_DDR_NET,
-	    QLA82XX_ADDR_DDR_NET_MAX) ||
-	    !QLA82XX_ADDR_IN_RANGE(addr + size - 1,
-	    QLA82XX_ADDR_DDR_NET, QLA82XX_ADDR_DDR_NET_MAX) ||
+	if (!QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_DDR_NET,
+	    QLA8XXX_ADDR_DDR_NET_MAX) ||
+	    !QLA8XXX_ADDR_IN_RANGE(addr + size - 1,
+	    QLA8XXX_ADDR_DDR_NET, QLA8XXX_ADDR_DDR_NET_MAX) ||
 	    ((size != 1) && (size != 2) && (size != 4) && (size != 8))) {
 		return 0;
 	}
 	return 1;
 }
 
-static int qla4_8xxx_pci_set_window_warning_count;
+static int qla4_82xx_pci_set_window_warning_count;
 
 static unsigned long
-qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
+qla4_82xx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 {
 	int window;
 	u32 win_read;
 
-	if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_DDR_NET,
-	    QLA82XX_ADDR_DDR_NET_MAX)) {
+	if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_DDR_NET,
+	    QLA8XXX_ADDR_DDR_NET_MAX)) {
 		/* DDR network side */
 		window = MN_WIN(addr);
 		ha->ddr_mn_window = window;
-		qla4_8xxx_wr_32(ha, ha->mn_win_crb |
+		qla4_82xx_wr_32(ha, ha->mn_win_crb |
 		    QLA82XX_PCI_CRBSPACE, window);
-		win_read = qla4_8xxx_rd_32(ha, ha->mn_win_crb |
+		win_read = qla4_82xx_rd_32(ha, ha->mn_win_crb |
 		    QLA82XX_PCI_CRBSPACE);
 		if ((win_read << 17) != window) {
 			ql4_printk(KERN_WARNING, ha,
@@ -607,8 +623,8 @@ qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 			__func__, window, win_read);
 		}
 		addr = GET_MEM_OFFS_2M(addr) + QLA82XX_PCI_DDR_NET;
-	} else if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_OCM0,
-				QLA82XX_ADDR_OCM0_MAX)) {
+	} else if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_OCM0,
+				QLA8XXX_ADDR_OCM0_MAX)) {
 		unsigned int temp1;
 		/* if bits 19:18&17:11 are on */
 		if ((addr & 0x00ff800) == 0xff800) {
@@ -618,9 +634,9 @@ qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 
 		window = OCM_WIN(addr);
 		ha->ddr_mn_window = window;
-		qla4_8xxx_wr_32(ha, ha->mn_win_crb |
+		qla4_82xx_wr_32(ha, ha->mn_win_crb |
 		    QLA82XX_PCI_CRBSPACE, window);
-		win_read = qla4_8xxx_rd_32(ha, ha->mn_win_crb |
+		win_read = qla4_82xx_rd_32(ha, ha->mn_win_crb |
 		    QLA82XX_PCI_CRBSPACE);
 		temp1 = ((window & 0x1FF) << 7) |
 		    ((window & 0x0FFFE0000) >> 17);
@@ -630,14 +646,14 @@ qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 		}
 		addr = GET_MEM_OFFS_2M(addr) + QLA82XX_PCI_OCM0_2M;
 
-	} else if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_QDR_NET,
+	} else if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_QDR_NET,
 				QLA82XX_P3_ADDR_QDR_NET_MAX)) {
 		/* QDR network side */
 		window = MS_WIN(addr);
 		ha->qdr_sn_window = window;
-		qla4_8xxx_wr_32(ha, ha->ms_win_crb |
+		qla4_82xx_wr_32(ha, ha->ms_win_crb |
 		    QLA82XX_PCI_CRBSPACE, window);
-		win_read = qla4_8xxx_rd_32(ha,
+		win_read = qla4_82xx_rd_32(ha,
 		     ha->ms_win_crb | QLA82XX_PCI_CRBSPACE);
 		if (win_read != window) {
 			printk("%s: Written MSwin (0x%x) != Read "
@@ -650,8 +666,8 @@ qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 		 * peg gdb frequently accesses memory that doesn't exist,
 		 * this limits the chit chat so debugging isn't slowed down.
 		 */
-		if ((qla4_8xxx_pci_set_window_warning_count++ < 8) ||
-		    (qla4_8xxx_pci_set_window_warning_count%64 == 0)) {
+		if ((qla4_82xx_pci_set_window_warning_count++ < 8) ||
+		    (qla4_82xx_pci_set_window_warning_count%64 == 0)) {
 			printk("%s: Warning:%s Unknown address range!\n",
 			    __func__, DRIVER_NAME);
 		}
@@ -661,7 +677,7 @@ qla4_8xxx_pci_set_window(struct scsi_qla_host *ha, unsigned long long addr)
 }
 
 /* check if address is in the same windows as the previous access */
-static int qla4_8xxx_pci_is_same_window(struct scsi_qla_host *ha,
+static int qla4_82xx_pci_is_same_window(struct scsi_qla_host *ha,
 		unsigned long long addr)
 {
 	int window;
@@ -669,20 +685,20 @@ static int qla4_8xxx_pci_is_same_window(struct scsi_qla_host *ha,
 
 	qdr_max = QLA82XX_P3_ADDR_QDR_NET_MAX;
 
-	if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_DDR_NET,
-	    QLA82XX_ADDR_DDR_NET_MAX)) {
+	if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_DDR_NET,
+	    QLA8XXX_ADDR_DDR_NET_MAX)) {
 		/* DDR network side */
 		BUG();	/* MN access can not come here */
-	} else if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_OCM0,
-	     QLA82XX_ADDR_OCM0_MAX)) {
+	} else if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_OCM0,
+	     QLA8XXX_ADDR_OCM0_MAX)) {
 		return 1;
-	} else if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_OCM1,
-	     QLA82XX_ADDR_OCM1_MAX)) {
+	} else if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_OCM1,
+	     QLA8XXX_ADDR_OCM1_MAX)) {
 		return 1;
-	} else if (QLA82XX_ADDR_IN_RANGE(addr, QLA82XX_ADDR_QDR_NET,
+	} else if (QLA8XXX_ADDR_IN_RANGE(addr, QLA8XXX_ADDR_QDR_NET,
 	    qdr_max)) {
 		/* QDR network side */
-		window = ((addr - QLA82XX_ADDR_QDR_NET) >> 22) & 0x3f;
+		window = ((addr - QLA8XXX_ADDR_QDR_NET) >> 22) & 0x3f;
 		if (ha->qdr_sn_window == window)
 			return 1;
 	}
@@ -690,7 +706,7 @@ static int qla4_8xxx_pci_is_same_window(struct scsi_qla_host *ha,
 	return 0;
 }
 
-static int qla4_8xxx_pci_mem_read_direct(struct scsi_qla_host *ha,
+static int qla4_82xx_pci_mem_read_direct(struct scsi_qla_host *ha,
 		u64 off, void *data, int size)
 {
 	unsigned long flags;
@@ -707,9 +723,9 @@ static int qla4_8xxx_pci_mem_read_direct(struct scsi_qla_host *ha,
 	 * If attempting to access unknown address or straddle hw windows,
 	 * do not access.
 	 */
-	start = qla4_8xxx_pci_set_window(ha, off);
+	start = qla4_82xx_pci_set_window(ha, off);
 	if ((start == -1UL) ||
-	    (qla4_8xxx_pci_is_same_window(ha, off + size - 1) == 0)) {
+	    (qla4_82xx_pci_is_same_window(ha, off + size - 1) == 0)) {
 		write_unlock_irqrestore(&ha->hw_lock, flags);
 		printk(KERN_ERR"%s out of bound pci memory access. "
 				"offset is 0x%llx\n", DRIVER_NAME, off);
@@ -763,7 +779,7 @@ static int qla4_8xxx_pci_mem_read_direct(struct scsi_qla_host *ha,
 }
 
 static int
-qla4_8xxx_pci_mem_write_direct(struct scsi_qla_host *ha, u64 off,
+qla4_82xx_pci_mem_write_direct(struct scsi_qla_host *ha, u64 off,
 		void *data, int size)
 {
 	unsigned long flags;
@@ -780,9 +796,9 @@ qla4_8xxx_pci_mem_write_direct(struct scsi_qla_host *ha, u64 off,
 	 * If attempting to access unknown address or straddle hw windows,
 	 * do not access.
 	 */
-	start = qla4_8xxx_pci_set_window(ha, off);
+	start = qla4_82xx_pci_set_window(ha, off);
 	if ((start == -1UL) ||
-	    (qla4_8xxx_pci_is_same_window(ha, off + size - 1) == 0)) {
+	    (qla4_82xx_pci_is_same_window(ha, off + size - 1) == 0)) {
 		write_unlock_irqrestore(&ha->hw_lock, flags);
 		printk(KERN_ERR"%s out of bound pci memory access. "
 				"offset is 0x%llx\n", DRIVER_NAME, off);
@@ -835,13 +851,13 @@ qla4_8xxx_pci_mem_write_direct(struct scsi_qla_host *ha, u64 off,
 #define MTU_FUDGE_FACTOR 100
 
 static unsigned long
-qla4_8xxx_decode_crb_addr(unsigned long addr)
+qla4_82xx_decode_crb_addr(unsigned long addr)
 {
 	int i;
 	unsigned long base_addr, offset, pci_base;
 
 	if (!qla4_8xxx_crb_table_initialized)
-		qla4_8xxx_crb_addr_transform_setup();
+		qla4_82xx_crb_addr_transform_setup();
 
 	pci_base = ADDR_ERROR;
 	base_addr = addr & 0xfff00000;
@@ -860,10 +876,10 @@ qla4_8xxx_decode_crb_addr(unsigned long addr)
 }
 
 static long rom_max_timeout = 100;
-static long qla4_8xxx_rom_lock_timeout = 100;
+static long qla4_82xx_rom_lock_timeout = 100;
 
 static int
-qla4_8xxx_rom_lock(struct scsi_qla_host *ha)
+qla4_82xx_rom_lock(struct scsi_qla_host *ha)
 {
 	int i;
 	int done = 0, timeout = 0;
@@ -871,10 +887,10 @@ qla4_8xxx_rom_lock(struct scsi_qla_host *ha)
 	while (!done) {
 		/* acquire semaphore2 from PCI HW block */
 
-		done = qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM2_LOCK));
+		done = qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM2_LOCK));
 		if (done == 1)
 			break;
-		if (timeout >= qla4_8xxx_rom_lock_timeout)
+		if (timeout >= qla4_82xx_rom_lock_timeout)
 			return -1;
 
 		timeout++;
@@ -887,24 +903,24 @@ qla4_8xxx_rom_lock(struct scsi_qla_host *ha)
 				cpu_relax();    /*This a nop instr on i386*/
 		}
 	}
-	qla4_8xxx_wr_32(ha, QLA82XX_ROM_LOCK_ID, ROM_LOCK_DRIVER);
+	qla4_82xx_wr_32(ha, QLA82XX_ROM_LOCK_ID, ROM_LOCK_DRIVER);
 	return 0;
 }
 
 static void
-qla4_8xxx_rom_unlock(struct scsi_qla_host *ha)
+qla4_82xx_rom_unlock(struct scsi_qla_host *ha)
 {
-	qla4_8xxx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM2_UNLOCK));
+	qla4_82xx_rd_32(ha, QLA82XX_PCIE_REG(PCIE_SEM2_UNLOCK));
 }
 
 static int
-qla4_8xxx_wait_rom_done(struct scsi_qla_host *ha)
+qla4_82xx_wait_rom_done(struct scsi_qla_host *ha)
 {
 	long timeout = 0;
 	long done = 0 ;
 
 	while (done == 0) {
-		done = qla4_8xxx_rd_32(ha, QLA82XX_ROMUSB_GLB_STATUS);
+		done = qla4_82xx_rd_32(ha, QLA82XX_ROMUSB_GLB_STATUS);
 		done &= 2;
 		timeout++;
 		if (timeout >= rom_max_timeout) {
@@ -917,40 +933,41 @@ qla4_8xxx_wait_rom_done(struct scsi_qla_host *ha)
 }
 
 static int
-qla4_8xxx_do_rom_fast_read(struct scsi_qla_host *ha, int addr, int *valp)
+qla4_82xx_do_rom_fast_read(struct scsi_qla_host *ha, int addr, int *valp)
 {
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_ADDRESS, addr);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_DUMMY_BYTE_CNT, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 3);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_INSTR_OPCODE, 0xb);
-	if (qla4_8xxx_wait_rom_done(ha)) {
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_ADDRESS, addr);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_DUMMY_BYTE_CNT, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 3);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_INSTR_OPCODE, 0xb);
+	if (qla4_82xx_wait_rom_done(ha)) {
 		printk("%s: Error waiting for rom done\n", DRIVER_NAME);
 		return -1;
 	}
 	/* reset abyte_cnt and dummy_byte_cnt */
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_DUMMY_BYTE_CNT, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_DUMMY_BYTE_CNT, 0);
 	udelay(10);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 0);
 
-	*valp = qla4_8xxx_rd_32(ha, QLA82XX_ROMUSB_ROM_RDATA);
+	*valp = qla4_82xx_rd_32(ha, QLA82XX_ROMUSB_ROM_RDATA);
 	return 0;
 }
 
 static int
-qla4_8xxx_rom_fast_read(struct scsi_qla_host *ha, int addr, int *valp)
+qla4_82xx_rom_fast_read(struct scsi_qla_host *ha, int addr, int *valp)
 {
 	int ret, loops = 0;
 
-	while ((qla4_8xxx_rom_lock(ha) != 0) && (loops < 50000)) {
+	while ((qla4_82xx_rom_lock(ha) != 0) && (loops < 50000)) {
 		udelay(100);
 		loops++;
 	}
 	if (loops >= 50000) {
-		printk("%s: qla4_8xxx_rom_lock failed\n", DRIVER_NAME);
+		ql4_printk(KERN_WARNING, ha, "%s: qla4_82xx_rom_lock failed\n",
+			   DRIVER_NAME);
 		return -1;
 	}
-	ret = qla4_8xxx_do_rom_fast_read(ha, addr, valp);
-	qla4_8xxx_rom_unlock(ha);
+	ret = qla4_82xx_do_rom_fast_read(ha, addr, valp);
+	qla4_82xx_rom_unlock(ha);
 	return ret;
 }
 
@@ -959,7 +976,7 @@ qla4_8xxx_rom_fast_read(struct scsi_qla_host *ha, int addr, int *valp)
  * to put the ISP into operational state
  **/
 static int
-qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
+qla4_82xx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 {
 	int addr, val;
 	int i ;
@@ -973,68 +990,68 @@ qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 	};
 
 	/* Halt all the indiviual PEGs and other blocks of the ISP */
-	qla4_8xxx_rom_lock(ha);
+	qla4_82xx_rom_lock(ha);
 
 	/* disable all I2Q */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x10, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x14, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x18, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x1c, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x20, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_I2Q + 0x24, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x10, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x14, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x18, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x1c, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x20, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_I2Q + 0x24, 0x0);
 
 	/* disable all niu interrupts */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x40, 0xff);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x40, 0xff);
 	/* disable xge rx/tx */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x70000, 0x00);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x70000, 0x00);
 	/* disable xg1 rx/tx */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x80000, 0x00);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x80000, 0x00);
 	/* disable sideband mac */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x90000, 0x00);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x90000, 0x00);
 	/* disable ap0 mac */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0xa0000, 0x00);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0xa0000, 0x00);
 	/* disable ap1 mac */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0xb0000, 0x00);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0xb0000, 0x00);
 
 	/* halt sre */
-	val = qla4_8xxx_rd_32(ha, QLA82XX_CRB_SRE + 0x1000);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_SRE + 0x1000, val & (~(0x1)));
+	val = qla4_82xx_rd_32(ha, QLA82XX_CRB_SRE + 0x1000);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_SRE + 0x1000, val & (~(0x1)));
 
 	/* halt epg */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_EPG + 0x1300, 0x1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_EPG + 0x1300, 0x1);
 
 	/* halt timers */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x0, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x8, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x10, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x18, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x100, 0x0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_TIMER + 0x200, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x0, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x8, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x10, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x18, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x100, 0x0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_TIMER + 0x200, 0x0);
 
 	/* halt pegs */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_0 + 0x3c, 1);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_1 + 0x3c, 1);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_2 + 0x3c, 1);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_3 + 0x3c, 1);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_4 + 0x3c, 1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_0 + 0x3c, 1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_1 + 0x3c, 1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_2 + 0x3c, 1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_3 + 0x3c, 1);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_4 + 0x3c, 1);
 	msleep(5);
 
 	/* big hammer */
 	if (test_bit(DPC_RESET_HA, &ha->dpc_flags))
 		/* don't reset CAM block on reset */
-		qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0xfeffffff);
+		qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0xfeffffff);
 	else
-		qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0xffffffff);
+		qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0xffffffff);
 
-	qla4_8xxx_rom_unlock(ha);
+	qla4_82xx_rom_unlock(ha);
 
 	/* Read the signature value from the flash.
 	 * Offset 0: Contain signature (0xcafecafe)
 	 * Offset 4: Offset and number of addr/value pairs
 	 * that present in CRB initialize sequence
 	 */
-	if (qla4_8xxx_rom_fast_read(ha, 0, &n) != 0 || n != 0xcafecafeUL ||
-	    qla4_8xxx_rom_fast_read(ha, 4, &n) != 0) {
+	if (qla4_82xx_rom_fast_read(ha, 0, &n) != 0 || n != 0xcafecafeUL ||
+	    qla4_82xx_rom_fast_read(ha, 4, &n) != 0) {
 		ql4_printk(KERN_WARNING, ha,
 			"[ERROR] Reading crb_init area: n: %08x\n", n);
 		return -1;
@@ -1065,8 +1082,8 @@ qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 	}
 
 	for (i = 0; i < n; i++) {
-		if (qla4_8xxx_rom_fast_read(ha, 8*i + 4*offset, &val) != 0 ||
-		    qla4_8xxx_rom_fast_read(ha, 8*i + 4*offset + 4, &addr) !=
+		if (qla4_82xx_rom_fast_read(ha, 8*i + 4*offset, &val) != 0 ||
+		    qla4_82xx_rom_fast_read(ha, 8*i + 4*offset + 4, &addr) !=
 		    0) {
 			kfree(buf);
 			return -1;
@@ -1080,7 +1097,7 @@ qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 		/* Translate internal CRB initialization
 		 * address to PCI bus address
 		 */
-		off = qla4_8xxx_decode_crb_addr((unsigned long)buf[i].addr) +
+		off = qla4_82xx_decode_crb_addr((unsigned long)buf[i].addr) +
 		    QLA82XX_PCI_CRBSPACE;
 		/* Not all CRB  addr/value pair to be written,
 		 * some of them are skipped
@@ -1125,7 +1142,7 @@ qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 			continue;
 		}
 
-		qla4_8xxx_wr_32(ha, off, buf[i].data);
+		qla4_82xx_wr_32(ha, off, buf[i].data);
 
 		/* ISP requires much bigger delay to settle down,
 		 * else crb_window returns 0xffffffff
@@ -1142,25 +1159,25 @@ qla4_8xxx_pinit_from_rom(struct scsi_qla_host *ha, int verbose)
 	kfree(buf);
 
 	/* Resetting the data and instruction cache */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_D+0xec, 0x1e);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_D+0x4c, 8);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_I+0x4c, 8);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_D+0xec, 0x1e);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_D+0x4c, 8);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_I+0x4c, 8);
 
 	/* Clear all protocol processing engines */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_0+0x8, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_0+0xc, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_1+0x8, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_1+0xc, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_2+0x8, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_2+0xc, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_3+0x8, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_3+0xc, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_0+0x8, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_0+0xc, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_1+0x8, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_1+0xc, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_2+0x8, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_2+0xc, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_3+0x8, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_3+0xc, 0);
 
 	return 0;
 }
 
 static int
-qla4_8xxx_load_from_flash(struct scsi_qla_host *ha, uint32_t image_start)
+qla4_82xx_load_from_flash(struct scsi_qla_host *ha, uint32_t image_start)
 {
 	int  i, rval = 0;
 	long size = 0;
@@ -1175,14 +1192,14 @@ qla4_8xxx_load_from_flash(struct scsi_qla_host *ha, uint32_t image_start)
 	    ha->host_no, __func__, flashaddr, image_start));
 
 	for (i = 0; i < size; i++) {
-		if ((qla4_8xxx_rom_fast_read(ha, flashaddr, (int *)&low)) ||
-		    (qla4_8xxx_rom_fast_read(ha, flashaddr + 4,
+		if ((qla4_82xx_rom_fast_read(ha, flashaddr, (int *)&low)) ||
+		    (qla4_82xx_rom_fast_read(ha, flashaddr + 4,
 		    (int *)&high))) {
 			rval = -1;
 			goto exit_load_from_flash;
 		}
 		data = ((u64)high << 32) | low ;
-		rval = qla4_8xxx_pci_mem_write_2M(ha, memaddr, &data, 8);
+		rval = qla4_82xx_pci_mem_write_2M(ha, memaddr, &data, 8);
 		if (rval)
 			goto exit_load_from_flash;
 
@@ -1197,20 +1214,20 @@ qla4_8xxx_load_from_flash(struct scsi_qla_host *ha, uint32_t image_start)
 	udelay(100);
 
 	read_lock(&ha->hw_lock);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_PEG_NET_0 + 0x18, 0x1020);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0x80001e);
+	qla4_82xx_wr_32(ha, QLA82XX_CRB_PEG_NET_0 + 0x18, 0x1020);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, 0x80001e);
 	read_unlock(&ha->hw_lock);
 
 exit_load_from_flash:
 	return rval;
 }
 
-static int qla4_8xxx_load_fw(struct scsi_qla_host *ha, uint32_t image_start)
+static int qla4_82xx_load_fw(struct scsi_qla_host *ha, uint32_t image_start)
 {
 	u32 rst;
 
-	qla4_8xxx_wr_32(ha, CRB_CMDPEG_STATE, 0);
-	if (qla4_8xxx_pinit_from_rom(ha, 0) != QLA_SUCCESS) {
+	qla4_82xx_wr_32(ha, CRB_CMDPEG_STATE, 0);
+	if (qla4_82xx_pinit_from_rom(ha, 0) != QLA_SUCCESS) {
 		printk(KERN_WARNING "%s: Error during CRB Initialization\n",
 		    __func__);
 		return QLA_ERROR;
@@ -1223,12 +1240,12 @@ static int qla4_8xxx_load_fw(struct scsi_qla_host *ha, uint32_t image_start)
 	 * To get around this, QM is brought out of reset.
 	 */
 
-	rst = qla4_8xxx_rd_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET);
+	rst = qla4_82xx_rd_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET);
 	/* unreset qm */
 	rst &= ~(1 << 28);
-	qla4_8xxx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, rst);
+	qla4_82xx_wr_32(ha, QLA82XX_ROMUSB_GLB_SW_RESET, rst);
 
-	if (qla4_8xxx_load_from_flash(ha, image_start)) {
+	if (qla4_82xx_load_from_flash(ha, image_start)) {
 		printk("%s: Error trying to load fw from flash!\n", __func__);
 		return QLA_ERROR;
 	}
@@ -1237,7 +1254,7 @@ static int qla4_8xxx_load_fw(struct scsi_qla_host *ha, uint32_t image_start)
 }
 
 int
-qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *ha,
+qla4_82xx_pci_mem_read_2M(struct scsi_qla_host *ha,
 		u64 off, void *data, int size)
 {
 	int i, j = 0, k, start, end, loop, sz[2], off0[2];
@@ -1249,12 +1266,12 @@ qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *ha,
 	 * If not MN, go check for MS or invalid.
 	 */
 
-	if (off >= QLA82XX_ADDR_QDR_NET && off <= QLA82XX_P3_ADDR_QDR_NET_MAX)
+	if (off >= QLA8XXX_ADDR_QDR_NET && off <= QLA82XX_P3_ADDR_QDR_NET_MAX)
 		mem_crb = QLA82XX_CRB_QDR_NET;
 	else {
 		mem_crb = QLA82XX_CRB_DDR_NET;
-		if (qla4_8xxx_pci_mem_bound_check(ha, off, size) == 0)
-			return qla4_8xxx_pci_mem_read_direct(ha,
+		if (qla4_82xx_pci_mem_bound_check(ha, off, size) == 0)
+			return qla4_82xx_pci_mem_read_direct(ha,
 					off, data, size);
 	}
 
@@ -1270,16 +1287,16 @@ qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *ha,
 
 	for (i = 0; i < loop; i++) {
 		temp = off8 + (i << shift_amount);
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_ADDR_LO, temp);
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_ADDR_LO, temp);
 		temp = 0;
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_ADDR_HI, temp);
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_ADDR_HI, temp);
 		temp = MIU_TA_CTL_ENABLE;
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_CTRL, temp);
-		temp = MIU_TA_CTL_START | MIU_TA_CTL_ENABLE;
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_CTRL, temp);
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_CTRL, temp);
+		temp = MIU_TA_CTL_START_ENABLE;
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_CTRL, temp);
 
 		for (j = 0; j < MAX_CTL_CHECK; j++) {
-			temp = qla4_8xxx_rd_32(ha, mem_crb + MIU_TEST_AGT_CTRL);
+			temp = qla4_82xx_rd_32(ha, mem_crb + MIU_TEST_AGT_CTRL);
 			if ((temp & MIU_TA_CTL_BUSY) == 0)
 				break;
 		}
@@ -1294,7 +1311,7 @@ qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *ha,
 		start = off0[i] >> 2;
 		end   = (off0[i] + sz[i] - 1) >> 2;
 		for (k = start; k <= end; k++) {
-			temp = qla4_8xxx_rd_32(ha,
+			temp = qla4_82xx_rd_32(ha,
 				mem_crb + MIU_TEST_AGT_RDDATA(k));
 			word[i] |= ((uint64_t)temp << (32 * (k & 1)));
 		}
@@ -1328,7 +1345,7 @@ qla4_8xxx_pci_mem_read_2M(struct scsi_qla_host *ha,
 }
 
 int
-qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha,
+qla4_82xx_pci_mem_write_2M(struct scsi_qla_host *ha,
 		u64 off, void *data, int size)
 {
 	int i, j, ret = 0, loop, sz[2], off0;
@@ -1339,12 +1356,12 @@ qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha,
 	/*
 	 * If not MN, go check for MS or invalid.
 	 */
-	if (off >= QLA82XX_ADDR_QDR_NET && off <= QLA82XX_P3_ADDR_QDR_NET_MAX)
+	if (off >= QLA8XXX_ADDR_QDR_NET && off <= QLA82XX_P3_ADDR_QDR_NET_MAX)
 		mem_crb = QLA82XX_CRB_QDR_NET;
 	else {
 		mem_crb = QLA82XX_CRB_DDR_NET;
-		if (qla4_8xxx_pci_mem_bound_check(ha, off, size) == 0)
-			return qla4_8xxx_pci_mem_write_direct(ha,
+		if (qla4_82xx_pci_mem_bound_check(ha, off, size) == 0)
+			return qla4_82xx_pci_mem_write_direct(ha,
 					off, data, size);
 	}
 
@@ -1359,7 +1376,7 @@ qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha,
 	startword = (off & 0xf)/8;
 
 	for (i = 0; i < loop; i++) {
-		if (qla4_8xxx_pci_mem_read_2M(ha, off8 +
+		if (qla4_82xx_pci_mem_read_2M(ha, off8 +
 		    (i << shift_amount), &word[i * scale], 8))
 			return -1;
 	}
@@ -1395,27 +1412,27 @@ qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha,
 
 	for (i = 0; i < loop; i++) {
 		temp = off8 + (i << shift_amount);
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_ADDR_LO, temp);
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_ADDR_LO, temp);
 		temp = 0;
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_ADDR_HI, temp);
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_ADDR_HI, temp);
 		temp = word[i * scale] & 0xffffffff;
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_WRDATA_LO, temp);
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_WRDATA_LO, temp);
 		temp = (word[i * scale] >> 32) & 0xffffffff;
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_WRDATA_HI, temp);
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_WRDATA_HI, temp);
 		temp = word[i*scale + 1] & 0xffffffff;
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_WRDATA_UPPER_LO,
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_WRDATA_UPPER_LO,
 		    temp);
 		temp = (word[i*scale + 1] >> 32) & 0xffffffff;
-		qla4_8xxx_wr_32(ha, mem_crb + MIU_TEST_AGT_WRDATA_UPPER_HI,
+		qla4_82xx_wr_32(ha, mem_crb + MIU_TEST_AGT_WRDATA_UPPER_HI,
 		    temp);
 
-		temp = MIU_TA_CTL_ENABLE | MIU_TA_CTL_WRITE;
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_CTRL, temp);
-		temp = MIU_TA_CTL_START | MIU_TA_CTL_ENABLE | MIU_TA_CTL_WRITE;
-		qla4_8xxx_wr_32(ha, mem_crb+MIU_TEST_AGT_CTRL, temp);
+		temp = MIU_TA_CTL_WRITE_ENABLE;
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_CTRL, temp);
+		temp = MIU_TA_CTL_WRITE_START;
+		qla4_82xx_wr_32(ha, mem_crb+MIU_TEST_AGT_CTRL, temp);
 
 		for (j = 0; j < MAX_CTL_CHECK; j++) {
-			temp = qla4_8xxx_rd_32(ha, mem_crb + MIU_TEST_AGT_CTRL);
+			temp = qla4_82xx_rd_32(ha, mem_crb + MIU_TEST_AGT_CTRL);
 			if ((temp & MIU_TA_CTL_BUSY) == 0)
 				break;
 		}
@@ -1433,14 +1450,14 @@ qla4_8xxx_pci_mem_write_2M(struct scsi_qla_host *ha,
 	return ret;
 }
 
-static int qla4_8xxx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val)
+static int qla4_82xx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val)
 {
 	u32 val = 0;
 	int retries = 60;
 
 	if (!pegtune_val) {
 		do {
-			val = qla4_8xxx_rd_32(ha, CRB_CMDPEG_STATE);
+			val = qla4_82xx_rd_32(ha, CRB_CMDPEG_STATE);
 			if ((val == PHAN_INITIALIZE_COMPLETE) ||
 			    (val == PHAN_INITIALIZE_ACK))
 				return 0;
@@ -1450,7 +1467,7 @@ static int qla4_8xxx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val)
 		} while (--retries);
 
 		if (!retries) {
-			pegtune_val = qla4_8xxx_rd_32(ha,
+			pegtune_val = qla4_82xx_rd_32(ha,
 				QLA82XX_ROMUSB_GLB_PEGTUNE_DONE);
 			printk(KERN_WARNING "%s: init failed, "
 				"pegtune_val = %x\n", __func__, pegtune_val);
@@ -1460,21 +1477,21 @@ static int qla4_8xxx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val)
 	return 0;
 }
 
-static int qla4_8xxx_rcvpeg_ready(struct scsi_qla_host *ha)
+static int qla4_82xx_rcvpeg_ready(struct scsi_qla_host *ha)
 {
 	uint32_t state = 0;
 	int loops = 0;
 
 	/* Window 1 call */
 	read_lock(&ha->hw_lock);
-	state = qla4_8xxx_rd_32(ha, CRB_RCVPEG_STATE);
+	state = qla4_82xx_rd_32(ha, CRB_RCVPEG_STATE);
 	read_unlock(&ha->hw_lock);
 
 	while ((state != PHAN_PEG_RCV_INITIALIZED) && (loops < 30000)) {
 		udelay(100);
 		/* Window 1 call */
 		read_lock(&ha->hw_lock);
-		state = qla4_8xxx_rd_32(ha, CRB_RCVPEG_STATE);
+		state = qla4_82xx_rd_32(ha, CRB_RCVPEG_STATE);
 		read_unlock(&ha->hw_lock);
 
 		loops++;
@@ -1494,11 +1511,21 @@ qla4_8xxx_set_drv_active(struct scsi_qla_host *ha)
 {
 	uint32_t drv_active;
 
-	drv_active = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
-	drv_active |= (1 << (ha->func_num * 4));
+	drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function
+	 */
+	if (is_qla8032(ha))
+		drv_active |= (1 << ha->func_num);
+	else
+		drv_active |= (1 << (ha->func_num * 4));
+
 	ql4_printk(KERN_INFO, ha, "%s(%ld): drv_active: 0x%08x\n",
 		   __func__, ha->host_no, drv_active);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_ACTIVE, drv_active);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_ACTIVE, drv_active);
 }
 
 void
@@ -1506,50 +1533,87 @@ qla4_8xxx_clear_drv_active(struct scsi_qla_host *ha)
 {
 	uint32_t drv_active;
 
-	drv_active = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
-	drv_active &= ~(1 << (ha->func_num * 4));
+	drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function
+	 */
+	if (is_qla8032(ha))
+		drv_active &= ~(1 << (ha->func_num));
+	else
+		drv_active &= ~(1 << (ha->func_num * 4));
+
 	ql4_printk(KERN_INFO, ha, "%s(%ld): drv_active: 0x%08x\n",
 		   __func__, ha->host_no, drv_active);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_ACTIVE, drv_active);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_ACTIVE, drv_active);
 }
 
-static inline int
-qla4_8xxx_need_reset(struct scsi_qla_host *ha)
+inline int qla4_8xxx_need_reset(struct scsi_qla_host *ha)
 {
 	uint32_t drv_state, drv_active;
 	int rval;
 
-	drv_active = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
-	drv_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-	rval = drv_state & (1 << (ha->func_num * 4));
+	drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+	drv_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_STATE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function
+	 */
+	if (is_qla8032(ha))
+		rval = drv_state & (1 << ha->func_num);
+	else
+		rval = drv_state & (1 << (ha->func_num * 4));
+
 	if ((test_bit(AF_EEH_BUSY, &ha->flags)) && drv_active)
 		rval = 1;
 
 	return rval;
 }
 
-static inline void
-qla4_8xxx_set_rst_ready(struct scsi_qla_host *ha)
+void qla4_8xxx_set_rst_ready(struct scsi_qla_host *ha)
 {
 	uint32_t drv_state;
 
-	drv_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-	drv_state |= (1 << (ha->func_num * 4));
+	drv_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_STATE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function
+	 */
+	if (is_qla8032(ha))
+		drv_state |= (1 << ha->func_num);
+	else
+		drv_state |= (1 << (ha->func_num * 4));
+
 	ql4_printk(KERN_INFO, ha, "%s(%ld): drv_state: 0x%08x\n",
 		   __func__, ha->host_no, drv_state);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_STATE, drv_state);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_STATE, drv_state);
 }
 
-static inline void
-qla4_8xxx_clear_rst_ready(struct scsi_qla_host *ha)
+void qla4_8xxx_clear_rst_ready(struct scsi_qla_host *ha)
 {
 	uint32_t drv_state;
 
-	drv_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-	drv_state &= ~(1 << (ha->func_num * 4));
+	drv_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_STATE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function
+	 */
+	if (is_qla8032(ha))
+		drv_state &= ~(1 << ha->func_num);
+	else
+		drv_state &= ~(1 << (ha->func_num * 4));
+
 	ql4_printk(KERN_INFO, ha, "%s(%ld): drv_state: 0x%08x\n",
 		   __func__, ha->host_no, drv_state);
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_STATE, drv_state);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_STATE, drv_state);
 }
 
 static inline void
@@ -1557,33 +1621,43 @@ qla4_8xxx_set_qsnt_ready(struct scsi_qla_host *ha)
 {
 	uint32_t qsnt_state;
 
-	qsnt_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-	qsnt_state |= (2 << (ha->func_num * 4));
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_STATE, qsnt_state);
+	qsnt_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_STATE);
+
+	/*
+	 * For ISP8324, drv_active register has 1 bit per function,
+	 * shift 1 by func_num to set a bit for the function.
+	 * For ISP8022, drv_active has 4 bits per function.
+	 */
+	if (is_qla8032(ha))
+		qsnt_state |= (1 << ha->func_num);
+	else
+		qsnt_state |= (2 << (ha->func_num * 4));
+
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_STATE, qsnt_state);
 }
 
 
 static int
-qla4_8xxx_start_firmware(struct scsi_qla_host *ha, uint32_t image_start)
+qla4_82xx_start_firmware(struct scsi_qla_host *ha, uint32_t image_start)
 {
 	uint16_t lnk;
 
 	/* scrub dma mask expansion register */
-	qla4_8xxx_wr_32(ha, CRB_DMA_SHIFT, 0x55555555);
+	qla4_82xx_wr_32(ha, CRB_DMA_SHIFT, 0x55555555);
 
 	/* Overwrite stale initialization register values */
-	qla4_8xxx_wr_32(ha, CRB_CMDPEG_STATE, 0);
-	qla4_8xxx_wr_32(ha, CRB_RCVPEG_STATE, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_PEG_HALT_STATUS1, 0);
-	qla4_8xxx_wr_32(ha, QLA82XX_PEG_HALT_STATUS2, 0);
+	qla4_82xx_wr_32(ha, CRB_CMDPEG_STATE, 0);
+	qla4_82xx_wr_32(ha, CRB_RCVPEG_STATE, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_PEG_HALT_STATUS1, 0);
+	qla4_82xx_wr_32(ha, QLA82XX_PEG_HALT_STATUS2, 0);
 
-	if (qla4_8xxx_load_fw(ha, image_start) != QLA_SUCCESS) {
+	if (qla4_82xx_load_fw(ha, image_start) != QLA_SUCCESS) {
 		printk("%s: Error trying to start fw!\n", __func__);
 		return QLA_ERROR;
 	}
 
 	/* Handshake with the card before we register the devices. */
-	if (qla4_8xxx_cmdpeg_ready(ha, 0) != QLA_SUCCESS) {
+	if (qla4_82xx_cmdpeg_ready(ha, 0) != QLA_SUCCESS) {
 		printk("%s: Error during card handshake!\n", __func__);
 		return QLA_ERROR;
 	}
@@ -1593,11 +1667,10 @@ qla4_8xxx_start_firmware(struct scsi_qla_host *ha, uint32_t image_start)
 	ha->link_width = (lnk >> 4) & 0x3f;
 
 	/* Synchronize with Receive peg */
-	return qla4_8xxx_rcvpeg_ready(ha);
+	return qla4_82xx_rcvpeg_ready(ha);
 }
 
-static int
-qla4_8xxx_try_start_fw(struct scsi_qla_host *ha)
+int qla4_82xx_try_start_fw(struct scsi_qla_host *ha)
 {
 	int rval = QLA_ERROR;
 
@@ -1615,7 +1688,7 @@ qla4_8xxx_try_start_fw(struct scsi_qla_host *ha)
 
 	ql4_printk(KERN_INFO, ha,
 	    "FW: Attempting to load firmware from flash...\n");
-	rval = qla4_8xxx_start_firmware(ha, ha->hw.flt_region_fw);
+	rval = qla4_82xx_start_firmware(ha, ha->hw.flt_region_fw);
 
 	if (rval != QLA_SUCCESS) {
 		ql4_printk(KERN_ERR, ha, "FW: Load firmware from flash"
@@ -1626,9 +1699,9 @@ qla4_8xxx_try_start_fw(struct scsi_qla_host *ha)
 	return rval;
 }
 
-static void qla4_8xxx_rom_lock_recovery(struct scsi_qla_host *ha)
+void qla4_82xx_rom_lock_recovery(struct scsi_qla_host *ha)
 {
-	if (qla4_8xxx_rom_lock(ha)) {
+	if (qla4_82xx_rom_lock(ha)) {
 		/* Someone else is holding the lock. */
 		dev_info(&ha->pdev->dev, "Resetting rom_lock\n");
 	}
@@ -1638,25 +1711,25 @@ static void qla4_8xxx_rom_lock_recovery(struct scsi_qla_host *ha)
 	 * else died while holding it.
 	 * In either case, unlock.
 	 */
-	qla4_8xxx_rom_unlock(ha);
+	qla4_82xx_rom_unlock(ha);
 }
 
 static void qla4_8xxx_minidump_process_rdcrb(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t r_addr, r_stride, loop_cnt, i, r_value;
-	struct qla82xx_minidump_entry_crb *crb_hdr;
+	struct qla8xxx_minidump_entry_crb *crb_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	crb_hdr = (struct qla82xx_minidump_entry_crb *)entry_hdr;
+	crb_hdr = (struct qla8xxx_minidump_entry_crb *)entry_hdr;
 	r_addr = crb_hdr->addr;
 	r_stride = crb_hdr->crb_strd.addr_stride;
 	loop_cnt = crb_hdr->op_count;
 
 	for (i = 0; i < loop_cnt; i++) {
-		r_value = qla4_8xxx_md_rw_32(ha, r_addr, 0, 0);
+		ha->isp_ops->rd_reg_indirect(ha, r_addr, &r_value);
 		*data_ptr++ = cpu_to_le32(r_addr);
 		*data_ptr++ = cpu_to_le32(r_value);
 		r_addr += r_stride;
@@ -1665,19 +1738,19 @@ static void qla4_8xxx_minidump_process_rdcrb(struct scsi_qla_host *ha,
 }
 
 static int qla4_8xxx_minidump_process_l2tag(struct scsi_qla_host *ha,
-				 struct qla82xx_minidump_entry_hdr *entry_hdr,
+				 struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				 uint32_t **d_ptr)
 {
 	uint32_t addr, r_addr, c_addr, t_r_addr;
 	uint32_t i, k, loop_count, t_value, r_cnt, r_value;
 	unsigned long p_wait, w_time, p_mask;
 	uint32_t c_value_w, c_value_r;
-	struct qla82xx_minidump_entry_cache *cache_hdr;
+	struct qla8xxx_minidump_entry_cache *cache_hdr;
 	int rval = QLA_ERROR;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	cache_hdr = (struct qla82xx_minidump_entry_cache *)entry_hdr;
+	cache_hdr = (struct qla8xxx_minidump_entry_cache *)entry_hdr;
 
 	loop_count = cache_hdr->op_count;
 	r_addr = cache_hdr->read_addr;
@@ -1691,16 +1764,16 @@ static int qla4_8xxx_minidump_process_l2tag(struct scsi_qla_host *ha,
 	p_mask = cache_hdr->cache_ctrl.poll_mask;
 
 	for (i = 0; i < loop_count; i++) {
-		qla4_8xxx_md_rw_32(ha, t_r_addr, t_value, 1);
+		ha->isp_ops->wr_reg_indirect(ha, t_r_addr, t_value);
 
 		if (c_value_w)
-			qla4_8xxx_md_rw_32(ha, c_addr, c_value_w, 1);
+			ha->isp_ops->wr_reg_indirect(ha, c_addr, c_value_w);
 
 		if (p_mask) {
 			w_time = jiffies + p_wait;
 			do {
-				c_value_r = qla4_8xxx_md_rw_32(ha, c_addr,
-								0, 0);
+				ha->isp_ops->rd_reg_indirect(ha, c_addr,
+							     &c_value_r);
 				if ((c_value_r & p_mask) == 0) {
 					break;
 				} else if (time_after_eq(jiffies, w_time)) {
@@ -1712,7 +1785,7 @@ static int qla4_8xxx_minidump_process_l2tag(struct scsi_qla_host *ha,
 
 		addr = r_addr;
 		for (k = 0; k < r_cnt; k++) {
-			r_value = qla4_8xxx_md_rw_32(ha, addr, 0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, addr, &r_value);
 			*data_ptr++ = cpu_to_le32(r_value);
 			addr += cache_hdr->read_ctrl.read_addr_stride;
 		}
@@ -1724,9 +1797,9 @@ static int qla4_8xxx_minidump_process_l2tag(struct scsi_qla_host *ha,
 }
 
 static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr)
+				struct qla8xxx_minidump_entry_hdr *entry_hdr)
 {
-	struct qla82xx_minidump_entry_crb *crb_entry;
+	struct qla8xxx_minidump_entry_crb *crb_entry;
 	uint32_t read_value, opcode, poll_time, addr, index, rval = QLA_SUCCESS;
 	uint32_t crb_addr;
 	unsigned long wtime;
@@ -1736,58 +1809,59 @@ static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
 	tmplt_hdr = (struct qla4_8xxx_minidump_template_hdr *)
 						ha->fw_dump_tmplt_hdr;
-	crb_entry = (struct qla82xx_minidump_entry_crb *)entry_hdr;
+	crb_entry = (struct qla8xxx_minidump_entry_crb *)entry_hdr;
 
 	crb_addr = crb_entry->addr;
 	for (i = 0; i < crb_entry->op_count; i++) {
 		opcode = crb_entry->crb_ctrl.opcode;
-		if (opcode & QLA82XX_DBG_OPCODE_WR) {
-			qla4_8xxx_md_rw_32(ha, crb_addr,
-					   crb_entry->value_1, 1);
-			opcode &= ~QLA82XX_DBG_OPCODE_WR;
+		if (opcode & QLA8XXX_DBG_OPCODE_WR) {
+			ha->isp_ops->wr_reg_indirect(ha, crb_addr,
+						     crb_entry->value_1);
+			opcode &= ~QLA8XXX_DBG_OPCODE_WR;
 		}
-		if (opcode & QLA82XX_DBG_OPCODE_RW) {
-			read_value = qla4_8xxx_md_rw_32(ha, crb_addr, 0, 0);
-			qla4_8xxx_md_rw_32(ha, crb_addr, read_value, 1);
-			opcode &= ~QLA82XX_DBG_OPCODE_RW;
+		if (opcode & QLA8XXX_DBG_OPCODE_RW) {
+			ha->isp_ops->rd_reg_indirect(ha, crb_addr, &read_value);
+			ha->isp_ops->wr_reg_indirect(ha, crb_addr, read_value);
+			opcode &= ~QLA8XXX_DBG_OPCODE_RW;
 		}
-		if (opcode & QLA82XX_DBG_OPCODE_AND) {
-			read_value = qla4_8xxx_md_rw_32(ha, crb_addr, 0, 0);
+		if (opcode & QLA8XXX_DBG_OPCODE_AND) {
+			ha->isp_ops->rd_reg_indirect(ha, crb_addr, &read_value);
 			read_value &= crb_entry->value_2;
-			opcode &= ~QLA82XX_DBG_OPCODE_AND;
-			if (opcode & QLA82XX_DBG_OPCODE_OR) {
+			opcode &= ~QLA8XXX_DBG_OPCODE_AND;
+			if (opcode & QLA8XXX_DBG_OPCODE_OR) {
 				read_value |= crb_entry->value_3;
-				opcode &= ~QLA82XX_DBG_OPCODE_OR;
+				opcode &= ~QLA8XXX_DBG_OPCODE_OR;
 			}
-			qla4_8xxx_md_rw_32(ha, crb_addr, read_value, 1);
+			ha->isp_ops->wr_reg_indirect(ha, crb_addr, read_value);
 		}
-		if (opcode & QLA82XX_DBG_OPCODE_OR) {
-			read_value = qla4_8xxx_md_rw_32(ha, crb_addr, 0, 0);
+		if (opcode & QLA8XXX_DBG_OPCODE_OR) {
+			ha->isp_ops->rd_reg_indirect(ha, crb_addr, &read_value);
 			read_value |= crb_entry->value_3;
-			qla4_8xxx_md_rw_32(ha, crb_addr, read_value, 1);
-			opcode &= ~QLA82XX_DBG_OPCODE_OR;
+			ha->isp_ops->wr_reg_indirect(ha, crb_addr, read_value);
+			opcode &= ~QLA8XXX_DBG_OPCODE_OR;
 		}
-		if (opcode & QLA82XX_DBG_OPCODE_POLL) {
+		if (opcode & QLA8XXX_DBG_OPCODE_POLL) {
 			poll_time = crb_entry->crb_strd.poll_timeout;
 			wtime = jiffies + poll_time;
-			read_value = qla4_8xxx_md_rw_32(ha, crb_addr, 0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, crb_addr, &read_value);
 
 			do {
 				if ((read_value & crb_entry->value_2) ==
-				    crb_entry->value_1)
+				    crb_entry->value_1) {
 					break;
-				else if (time_after_eq(jiffies, wtime)) {
+				} else if (time_after_eq(jiffies, wtime)) {
 					/* capturing dump failed */
 					rval = QLA_ERROR;
 					break;
-				} else
-					read_value = qla4_8xxx_md_rw_32(ha,
-								crb_addr, 0, 0);
+				} else {
+					ha->isp_ops->rd_reg_indirect(ha,
+							crb_addr, &read_value);
+				}
 			} while (1);
-			opcode &= ~QLA82XX_DBG_OPCODE_POLL;
+			opcode &= ~QLA8XXX_DBG_OPCODE_POLL;
 		}
 
-		if (opcode & QLA82XX_DBG_OPCODE_RDSTATE) {
+		if (opcode & QLA8XXX_DBG_OPCODE_RDSTATE) {
 			if (crb_entry->crb_strd.state_index_a) {
 				index = crb_entry->crb_strd.state_index_a;
 				addr = tmplt_hdr->saved_state_array[index];
@@ -1795,13 +1869,13 @@ static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
 				addr = crb_addr;
 			}
 
-			read_value = qla4_8xxx_md_rw_32(ha, addr, 0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, addr, &read_value);
 			index = crb_entry->crb_ctrl.state_index_v;
 			tmplt_hdr->saved_state_array[index] = read_value;
-			opcode &= ~QLA82XX_DBG_OPCODE_RDSTATE;
+			opcode &= ~QLA8XXX_DBG_OPCODE_RDSTATE;
 		}
 
-		if (opcode & QLA82XX_DBG_OPCODE_WRSTATE) {
+		if (opcode & QLA8XXX_DBG_OPCODE_WRSTATE) {
 			if (crb_entry->crb_strd.state_index_a) {
 				index = crb_entry->crb_strd.state_index_a;
 				addr = tmplt_hdr->saved_state_array[index];
@@ -1817,11 +1891,11 @@ static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
 				read_value = crb_entry->value_1;
 			}
 
-			qla4_8xxx_md_rw_32(ha, addr, read_value, 1);
-			opcode &= ~QLA82XX_DBG_OPCODE_WRSTATE;
+			ha->isp_ops->wr_reg_indirect(ha, addr, read_value);
+			opcode &= ~QLA8XXX_DBG_OPCODE_WRSTATE;
 		}
 
-		if (opcode & QLA82XX_DBG_OPCODE_MDSTATE) {
+		if (opcode & QLA8XXX_DBG_OPCODE_MDSTATE) {
 			index = crb_entry->crb_ctrl.state_index_v;
 			read_value = tmplt_hdr->saved_state_array[index];
 			read_value <<= crb_entry->crb_ctrl.shl;
@@ -1831,7 +1905,7 @@ static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
 			read_value |= crb_entry->value_3;
 			read_value += crb_entry->value_1;
 			tmplt_hdr->saved_state_array[index] = read_value;
-			opcode &= ~QLA82XX_DBG_OPCODE_MDSTATE;
+			opcode &= ~QLA8XXX_DBG_OPCODE_MDSTATE;
 		}
 		crb_addr += crb_entry->crb_strd.addr_stride;
 	}
@@ -1840,15 +1914,15 @@ static int qla4_8xxx_minidump_process_control(struct scsi_qla_host *ha,
 }
 
 static void qla4_8xxx_minidump_process_rdocm(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t r_addr, r_stride, loop_cnt, i, r_value;
-	struct qla82xx_minidump_entry_rdocm *ocm_hdr;
+	struct qla8xxx_minidump_entry_rdocm *ocm_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	ocm_hdr = (struct qla82xx_minidump_entry_rdocm *)entry_hdr;
+	ocm_hdr = (struct qla8xxx_minidump_entry_rdocm *)entry_hdr;
 	r_addr = ocm_hdr->read_addr;
 	r_stride = ocm_hdr->read_addr_stride;
 	loop_cnt = ocm_hdr->op_count;
@@ -1863,20 +1937,20 @@ static void qla4_8xxx_minidump_process_rdocm(struct scsi_qla_host *ha,
 		r_addr += r_stride;
 	}
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Leaving fn: %s datacount: 0x%lx\n",
-			  __func__, (loop_cnt * sizeof(uint32_t))));
+		__func__, (long unsigned int) (loop_cnt * sizeof(uint32_t))));
 	*d_ptr = data_ptr;
 }
 
 static void qla4_8xxx_minidump_process_rdmux(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t r_addr, s_stride, s_addr, s_value, loop_cnt, i, r_value;
-	struct qla82xx_minidump_entry_mux *mux_hdr;
+	struct qla8xxx_minidump_entry_mux *mux_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	mux_hdr = (struct qla82xx_minidump_entry_mux *)entry_hdr;
+	mux_hdr = (struct qla8xxx_minidump_entry_mux *)entry_hdr;
 	r_addr = mux_hdr->read_addr;
 	s_addr = mux_hdr->select_addr;
 	s_stride = mux_hdr->select_value_stride;
@@ -1884,8 +1958,8 @@ static void qla4_8xxx_minidump_process_rdmux(struct scsi_qla_host *ha,
 	loop_cnt = mux_hdr->op_count;
 
 	for (i = 0; i < loop_cnt; i++) {
-		qla4_8xxx_md_rw_32(ha, s_addr, s_value, 1);
-		r_value = qla4_8xxx_md_rw_32(ha, r_addr, 0, 0);
+		ha->isp_ops->wr_reg_indirect(ha, s_addr, s_value);
+		ha->isp_ops->rd_reg_indirect(ha, r_addr, &r_value);
 		*data_ptr++ = cpu_to_le32(s_value);
 		*data_ptr++ = cpu_to_le32(r_value);
 		s_value += s_stride;
@@ -1894,16 +1968,16 @@ static void qla4_8xxx_minidump_process_rdmux(struct scsi_qla_host *ha,
 }
 
 static void qla4_8xxx_minidump_process_l1cache(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t addr, r_addr, c_addr, t_r_addr;
 	uint32_t i, k, loop_count, t_value, r_cnt, r_value;
 	uint32_t c_value_w;
-	struct qla82xx_minidump_entry_cache *cache_hdr;
+	struct qla8xxx_minidump_entry_cache *cache_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
-	cache_hdr = (struct qla82xx_minidump_entry_cache *)entry_hdr;
+	cache_hdr = (struct qla8xxx_minidump_entry_cache *)entry_hdr;
 	loop_count = cache_hdr->op_count;
 	r_addr = cache_hdr->read_addr;
 	c_addr = cache_hdr->control_addr;
@@ -1914,11 +1988,11 @@ static void qla4_8xxx_minidump_process_l1cache(struct scsi_qla_host *ha,
 	r_cnt = cache_hdr->read_ctrl.read_addr_cnt;
 
 	for (i = 0; i < loop_count; i++) {
-		qla4_8xxx_md_rw_32(ha, t_r_addr, t_value, 1);
-		qla4_8xxx_md_rw_32(ha, c_addr, c_value_w, 1);
+		ha->isp_ops->wr_reg_indirect(ha, t_r_addr, t_value);
+		ha->isp_ops->wr_reg_indirect(ha, c_addr, c_value_w);
 		addr = r_addr;
 		for (k = 0; k < r_cnt; k++) {
-			r_value = qla4_8xxx_md_rw_32(ha, addr, 0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, addr, &r_value);
 			*data_ptr++ = cpu_to_le32(r_value);
 			addr += cache_hdr->read_ctrl.read_addr_stride;
 		}
@@ -1928,27 +2002,27 @@ static void qla4_8xxx_minidump_process_l1cache(struct scsi_qla_host *ha,
 }
 
 static void qla4_8xxx_minidump_process_queue(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t s_addr, r_addr;
 	uint32_t r_stride, r_value, r_cnt, qid = 0;
 	uint32_t i, k, loop_cnt;
-	struct qla82xx_minidump_entry_queue *q_hdr;
+	struct qla8xxx_minidump_entry_queue *q_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	q_hdr = (struct qla82xx_minidump_entry_queue *)entry_hdr;
+	q_hdr = (struct qla8xxx_minidump_entry_queue *)entry_hdr;
 	s_addr = q_hdr->select_addr;
 	r_cnt = q_hdr->rd_strd.read_addr_cnt;
 	r_stride = q_hdr->rd_strd.read_addr_stride;
 	loop_cnt = q_hdr->op_count;
 
 	for (i = 0; i < loop_cnt; i++) {
-		qla4_8xxx_md_rw_32(ha, s_addr, qid, 1);
+		ha->isp_ops->wr_reg_indirect(ha, s_addr, qid);
 		r_addr = q_hdr->read_addr;
 		for (k = 0; k < r_cnt; k++) {
-			r_value = qla4_8xxx_md_rw_32(ha, r_addr, 0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, r_addr, &r_value);
 			*data_ptr++ = cpu_to_le32(r_value);
 			r_addr += r_stride;
 		}
@@ -1960,17 +2034,17 @@ static void qla4_8xxx_minidump_process_queue(struct scsi_qla_host *ha,
 #define MD_DIRECT_ROM_WINDOW		0x42110030
 #define MD_DIRECT_ROM_READ_BASE		0x42150000
 
-static void qla4_8xxx_minidump_process_rdrom(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+static void qla4_82xx_minidump_process_rdrom(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t r_addr, r_value;
 	uint32_t i, loop_cnt;
-	struct qla82xx_minidump_entry_rdrom *rom_hdr;
+	struct qla8xxx_minidump_entry_rdrom *rom_hdr;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	rom_hdr = (struct qla82xx_minidump_entry_rdrom *)entry_hdr;
+	rom_hdr = (struct qla8xxx_minidump_entry_rdrom *)entry_hdr;
 	r_addr = rom_hdr->read_addr;
 	loop_cnt = rom_hdr->read_data_size/sizeof(uint32_t);
 
@@ -1979,11 +2053,11 @@ static void qla4_8xxx_minidump_process_rdrom(struct scsi_qla_host *ha,
 			   __func__, r_addr, loop_cnt));
 
 	for (i = 0; i < loop_cnt; i++) {
-		qla4_8xxx_md_rw_32(ha, MD_DIRECT_ROM_WINDOW,
-				   (r_addr & 0xFFFF0000), 1);
-		r_value = qla4_8xxx_md_rw_32(ha,
-					     MD_DIRECT_ROM_READ_BASE +
-					     (r_addr & 0x0000FFFF), 0, 0);
+		ha->isp_ops->wr_reg_indirect(ha, MD_DIRECT_ROM_WINDOW,
+					     (r_addr & 0xFFFF0000));
+		ha->isp_ops->rd_reg_indirect(ha,
+				MD_DIRECT_ROM_READ_BASE + (r_addr & 0x0000FFFF),
+				&r_value);
 		*data_ptr++ = cpu_to_le32(r_value);
 		r_addr += sizeof(uint32_t);
 	}
@@ -1995,17 +2069,17 @@ static void qla4_8xxx_minidump_process_rdrom(struct scsi_qla_host *ha,
 #define MD_MIU_TEST_AGT_ADDR_HI		0x41000098
 
 static int qla4_8xxx_minidump_process_rdmem(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				uint32_t **d_ptr)
 {
 	uint32_t r_addr, r_value, r_data;
 	uint32_t i, j, loop_cnt;
-	struct qla82xx_minidump_entry_rdmem *m_hdr;
+	struct qla8xxx_minidump_entry_rdmem *m_hdr;
 	unsigned long flags;
 	uint32_t *data_ptr = *d_ptr;
 
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Entering fn: %s\n", __func__));
-	m_hdr = (struct qla82xx_minidump_entry_rdmem *)entry_hdr;
+	m_hdr = (struct qla8xxx_minidump_entry_rdmem *)entry_hdr;
 	r_addr = m_hdr->read_addr;
 	loop_cnt = m_hdr->read_data_size/16;
 
@@ -2033,17 +2107,19 @@ static int qla4_8xxx_minidump_process_rdmem(struct scsi_qla_host *ha,
 
 	write_lock_irqsave(&ha->hw_lock, flags);
 	for (i = 0; i < loop_cnt; i++) {
-		qla4_8xxx_md_rw_32(ha, MD_MIU_TEST_AGT_ADDR_LO, r_addr, 1);
+		ha->isp_ops->wr_reg_indirect(ha, MD_MIU_TEST_AGT_ADDR_LO,
+					     r_addr);
 		r_value = 0;
-		qla4_8xxx_md_rw_32(ha, MD_MIU_TEST_AGT_ADDR_HI, r_value, 1);
+		ha->isp_ops->wr_reg_indirect(ha, MD_MIU_TEST_AGT_ADDR_HI,
+					     r_value);
 		r_value = MIU_TA_CTL_ENABLE;
-		qla4_8xxx_md_rw_32(ha, MD_MIU_TEST_AGT_CTRL, r_value, 1);
-		r_value = MIU_TA_CTL_START | MIU_TA_CTL_ENABLE;
-		qla4_8xxx_md_rw_32(ha, MD_MIU_TEST_AGT_CTRL, r_value, 1);
+		ha->isp_ops->wr_reg_indirect(ha, MD_MIU_TEST_AGT_CTRL, r_value);
+		r_value = MIU_TA_CTL_START_ENABLE;
+		ha->isp_ops->wr_reg_indirect(ha, MD_MIU_TEST_AGT_CTRL, r_value);
 
 		for (j = 0; j < MAX_CTL_CHECK; j++) {
-			r_value = qla4_8xxx_md_rw_32(ha, MD_MIU_TEST_AGT_CTRL,
-						     0, 0);
+			ha->isp_ops->rd_reg_indirect(ha, MD_MIU_TEST_AGT_CTRL,
+						     &r_value);
 			if ((r_value & MIU_TA_CTL_BUSY) == 0)
 				break;
 		}
@@ -2057,9 +2133,9 @@ static int qla4_8xxx_minidump_process_rdmem(struct scsi_qla_host *ha,
 		}
 
 		for (j = 0; j < 4; j++) {
-			r_data = qla4_8xxx_md_rw_32(ha,
-						    MD_MIU_TEST_AGT_RDDATA[j],
-						    0, 0);
+			ha->isp_ops->rd_reg_indirect(ha,
+						     MD_MIU_TEST_AGT_RDDATA[j],
+						     &r_data);
 			*data_ptr++ = cpu_to_le32(r_data);
 		}
 
@@ -2074,25 +2150,215 @@ static int qla4_8xxx_minidump_process_rdmem(struct scsi_qla_host *ha,
 	return QLA_SUCCESS;
 }
 
-static void ql4_8xxx_mark_entry_skipped(struct scsi_qla_host *ha,
-				struct qla82xx_minidump_entry_hdr *entry_hdr,
+static void qla4_8xxx_mark_entry_skipped(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
 				int index)
 {
-	entry_hdr->d_ctrl.driver_flags |= QLA82XX_DBG_SKIPPED_FLAG;
+	entry_hdr->d_ctrl.driver_flags |= QLA8XXX_DBG_SKIPPED_FLAG;
 	DEBUG2(ql4_printk(KERN_INFO, ha,
 			  "scsi(%ld): Skipping entry[%d]: ETYPE[0x%x]-ELEVEL[0x%x]\n",
 			  ha->host_no, index, entry_hdr->entry_type,
 			  entry_hdr->d_ctrl.entry_capture_mask));
 }
 
+/* ISP83xx functions to process new minidump entries... */
+static uint32_t qla83xx_minidump_process_pollrd(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
+				uint32_t **d_ptr)
+{
+	uint32_t r_addr, s_addr, s_value, r_value, poll_wait, poll_mask;
+	uint16_t s_stride, i;
+	uint32_t *data_ptr = *d_ptr;
+	uint32_t rval = QLA_SUCCESS;
+	struct qla83xx_minidump_entry_pollrd *pollrd_hdr;
+
+	pollrd_hdr = (struct qla83xx_minidump_entry_pollrd *)entry_hdr;
+	s_addr = le32_to_cpu(pollrd_hdr->select_addr);
+	r_addr = le32_to_cpu(pollrd_hdr->read_addr);
+	s_value = le32_to_cpu(pollrd_hdr->select_value);
+	s_stride = le32_to_cpu(pollrd_hdr->select_value_stride);
+
+	poll_wait = le32_to_cpu(pollrd_hdr->poll_wait);
+	poll_mask = le32_to_cpu(pollrd_hdr->poll_mask);
+
+	for (i = 0; i < le32_to_cpu(pollrd_hdr->op_count); i++) {
+		ha->isp_ops->wr_reg_indirect(ha, s_addr, s_value);
+		poll_wait = le32_to_cpu(pollrd_hdr->poll_wait);
+		while (1) {
+			ha->isp_ops->rd_reg_indirect(ha, s_addr, &r_value);
+
+			if ((r_value & poll_mask) != 0) {
+				break;
+			} else {
+				msleep(1);
+				if (--poll_wait == 0) {
+					ql4_printk(KERN_ERR, ha, "%s: TIMEOUT\n",
+						   __func__);
+					rval = QLA_ERROR;
+					goto exit_process_pollrd;
+				}
+			}
+		}
+		ha->isp_ops->rd_reg_indirect(ha, r_addr, &r_value);
+		*data_ptr++ = cpu_to_le32(s_value);
+		*data_ptr++ = cpu_to_le32(r_value);
+		s_value += s_stride;
+	}
+
+	*d_ptr = data_ptr;
+
+exit_process_pollrd:
+	return rval;
+}
+
+static void qla83xx_minidump_process_rdmux2(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
+				uint32_t **d_ptr)
+{
+	uint32_t sel_val1, sel_val2, t_sel_val, data, i;
+	uint32_t sel_addr1, sel_addr2, sel_val_mask, read_addr;
+	struct qla83xx_minidump_entry_rdmux2 *rdmux2_hdr;
+	uint32_t *data_ptr = *d_ptr;
+
+	rdmux2_hdr = (struct qla83xx_minidump_entry_rdmux2 *)entry_hdr;
+	sel_val1 = le32_to_cpu(rdmux2_hdr->select_value_1);
+	sel_val2 = le32_to_cpu(rdmux2_hdr->select_value_2);
+	sel_addr1 = le32_to_cpu(rdmux2_hdr->select_addr_1);
+	sel_addr2 = le32_to_cpu(rdmux2_hdr->select_addr_2);
+	sel_val_mask = le32_to_cpu(rdmux2_hdr->select_value_mask);
+	read_addr = le32_to_cpu(rdmux2_hdr->read_addr);
+
+	for (i = 0; i < rdmux2_hdr->op_count; i++) {
+		ha->isp_ops->wr_reg_indirect(ha, sel_addr1, sel_val1);
+		t_sel_val = sel_val1 & sel_val_mask;
+		*data_ptr++ = cpu_to_le32(t_sel_val);
+
+		ha->isp_ops->wr_reg_indirect(ha, sel_addr2, t_sel_val);
+		ha->isp_ops->rd_reg_indirect(ha, read_addr, &data);
+
+		*data_ptr++ = cpu_to_le32(data);
+
+		ha->isp_ops->wr_reg_indirect(ha, sel_addr1, sel_val2);
+		t_sel_val = sel_val2 & sel_val_mask;
+		*data_ptr++ = cpu_to_le32(t_sel_val);
+
+		ha->isp_ops->wr_reg_indirect(ha, sel_addr2, t_sel_val);
+		ha->isp_ops->rd_reg_indirect(ha, read_addr, &data);
+
+		*data_ptr++ = cpu_to_le32(data);
+
+		sel_val1 += rdmux2_hdr->select_value_stride;
+		sel_val2 += rdmux2_hdr->select_value_stride;
+	}
+
+	*d_ptr = data_ptr;
+}
+
+static uint32_t qla83xx_minidump_process_pollrdmwr(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
+				uint32_t **d_ptr)
+{
+	uint32_t poll_wait, poll_mask, r_value, data;
+	uint32_t addr_1, addr_2, value_1, value_2;
+	uint32_t *data_ptr = *d_ptr;
+	uint32_t rval = QLA_SUCCESS;
+	struct qla83xx_minidump_entry_pollrdmwr *poll_hdr;
+
+	poll_hdr = (struct qla83xx_minidump_entry_pollrdmwr *)entry_hdr;
+	addr_1 = le32_to_cpu(poll_hdr->addr_1);
+	addr_2 = le32_to_cpu(poll_hdr->addr_2);
+	value_1 = le32_to_cpu(poll_hdr->value_1);
+	value_2 = le32_to_cpu(poll_hdr->value_2);
+	poll_mask = le32_to_cpu(poll_hdr->poll_mask);
+
+	ha->isp_ops->wr_reg_indirect(ha, addr_1, value_1);
+
+	poll_wait = le32_to_cpu(poll_hdr->poll_wait);
+	while (1) {
+		ha->isp_ops->rd_reg_indirect(ha, addr_1, &r_value);
+
+		if ((r_value & poll_mask) != 0) {
+			break;
+		} else {
+			msleep(1);
+			if (--poll_wait == 0) {
+				ql4_printk(KERN_ERR, ha, "%s: TIMEOUT_1\n",
+					   __func__);
+				rval = QLA_ERROR;
+				goto exit_process_pollrdmwr;
+			}
+		}
+	}
+
+	ha->isp_ops->rd_reg_indirect(ha, addr_2, &data);
+	data &= le32_to_cpu(poll_hdr->modify_mask);
+	ha->isp_ops->wr_reg_indirect(ha, addr_2, data);
+	ha->isp_ops->wr_reg_indirect(ha, addr_1, value_2);
+
+	poll_wait = le32_to_cpu(poll_hdr->poll_wait);
+	while (1) {
+		ha->isp_ops->rd_reg_indirect(ha, addr_1, &r_value);
+
+		if ((r_value & poll_mask) != 0) {
+			break;
+		} else {
+			msleep(1);
+			if (--poll_wait == 0) {
+				ql4_printk(KERN_ERR, ha, "%s: TIMEOUT_2\n",
+					   __func__);
+				rval = QLA_ERROR;
+				goto exit_process_pollrdmwr;
+			}
+		}
+	}
+
+	*data_ptr++ = cpu_to_le32(addr_2);
+	*data_ptr++ = cpu_to_le32(data);
+	*d_ptr = data_ptr;
+
+exit_process_pollrdmwr:
+	return rval;
+}
+
+static uint32_t qla4_83xx_minidump_process_rdrom(struct scsi_qla_host *ha,
+				struct qla8xxx_minidump_entry_hdr *entry_hdr,
+				uint32_t **d_ptr)
+{
+	uint32_t fl_addr, u32_count, rval;
+	struct qla8xxx_minidump_entry_rdrom *rom_hdr;
+	uint32_t *data_ptr = *d_ptr;
+
+	rom_hdr = (struct qla8xxx_minidump_entry_rdrom *)entry_hdr;
+	fl_addr = le32_to_cpu(rom_hdr->read_addr);
+	u32_count = le32_to_cpu(rom_hdr->read_data_size)/sizeof(uint32_t);
+
+	DEBUG2(ql4_printk(KERN_INFO, ha, "[%s]: fl_addr: 0x%x, count: 0x%x\n",
+			  __func__, fl_addr, u32_count));
+
+	rval = qla4_83xx_lockless_flash_read_u32(ha, fl_addr,
+						 (u8 *)(data_ptr), u32_count);
+
+	if (rval == QLA_ERROR) {
+		ql4_printk(KERN_ERR, ha, "%s: Flash Read Error,Count=%d\n",
+			   __func__, u32_count);
+		goto exit_process_rdrom;
+	}
+
+	data_ptr += u32_count;
+	*d_ptr = data_ptr;
+
+exit_process_rdrom:
+	return rval;
+}
+
 /**
- * qla82xx_collect_md_data - Retrieve firmware minidump data.
+ * qla4_8xxx_collect_md_data - Retrieve firmware minidump data.
  * @ha: pointer to adapter structure
  **/
 static int qla4_8xxx_collect_md_data(struct scsi_qla_host *ha)
 {
 	int num_entry_hdr = 0;
-	struct qla82xx_minidump_entry_hdr *entry_hdr;
+	struct qla8xxx_minidump_entry_hdr *entry_hdr;
 	struct qla4_8xxx_minidump_template_hdr *tmplt_hdr;
 	uint32_t *data_ptr;
 	uint32_t data_collected = 0;
@@ -2128,10 +2394,14 @@ static int qla4_8xxx_collect_md_data(struct scsi_qla_host *ha)
 	timestamp = (u32)(jiffies_to_msecs(now) / 1000);
 	tmplt_hdr->driver_timestamp = timestamp;
 
-	entry_hdr = (struct qla82xx_minidump_entry_hdr *)
+	entry_hdr = (struct qla8xxx_minidump_entry_hdr *)
 					(((uint8_t *)ha->fw_dump_tmplt_hdr) +
 					 tmplt_hdr->first_entry_offset);
 
+	if (is_qla8032(ha))
+		tmplt_hdr->saved_state_array[QLA83XX_SS_OCM_WNDREG_INDEX] =
+					tmplt_hdr->ocm_window_reg[ha->func_num];
+
 	/* Walk through the entry headers - validate/perform required action */
 	for (i = 0; i < num_entry_hdr; i++) {
 		if (data_collected >= ha->fw_dump_size) {
@@ -2144,7 +2414,7 @@ static int qla4_8xxx_collect_md_data(struct scsi_qla_host *ha)
 		if (!(entry_hdr->d_ctrl.entry_capture_mask &
 		      ha->fw_dump_capture_mask)) {
 			entry_hdr->d_ctrl.driver_flags |=
-						QLA82XX_DBG_SKIPPED_FLAG;
+						QLA8XXX_DBG_SKIPPED_FLAG;
 			goto skip_nxt_entry;
 		}
 
@@ -2157,65 +2427,105 @@ static int qla4_8xxx_collect_md_data(struct scsi_qla_host *ha)
 		 * debug data
 		 */
 		switch (entry_hdr->entry_type) {
-		case QLA82XX_RDEND:
-			ql4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+		case QLA8XXX_RDEND:
+			qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
 			break;
-		case QLA82XX_CNTRL:
+		case QLA8XXX_CNTRL:
 			rval = qla4_8xxx_minidump_process_control(ha,
 								  entry_hdr);
 			if (rval != QLA_SUCCESS) {
-				ql4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
 				goto md_failed;
 			}
 			break;
-		case QLA82XX_RDCRB:
+		case QLA8XXX_RDCRB:
 			qla4_8xxx_minidump_process_rdcrb(ha, entry_hdr,
 							 &data_ptr);
 			break;
-		case QLA82XX_RDMEM:
+		case QLA8XXX_RDMEM:
 			rval = qla4_8xxx_minidump_process_rdmem(ha, entry_hdr,
 								&data_ptr);
 			if (rval != QLA_SUCCESS) {
-				ql4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
 				goto md_failed;
 			}
 			break;
-		case QLA82XX_BOARD:
-		case QLA82XX_RDROM:
-			qla4_8xxx_minidump_process_rdrom(ha, entry_hdr,
-							 &data_ptr);
+		case QLA8XXX_BOARD:
+		case QLA8XXX_RDROM:
+			if (is_qla8022(ha)) {
+				qla4_82xx_minidump_process_rdrom(ha, entry_hdr,
+								 &data_ptr);
+			} else if (is_qla8032(ha)) {
+				rval = qla4_83xx_minidump_process_rdrom(ha,
+								    entry_hdr,
+								    &data_ptr);
+				if (rval != QLA_SUCCESS)
+					qla4_8xxx_mark_entry_skipped(ha,
+								     entry_hdr,
+								     i);
+			}
 			break;
-		case QLA82XX_L2DTG:
-		case QLA82XX_L2ITG:
-		case QLA82XX_L2DAT:
-		case QLA82XX_L2INS:
+		case QLA8XXX_L2DTG:
+		case QLA8XXX_L2ITG:
+		case QLA8XXX_L2DAT:
+		case QLA8XXX_L2INS:
 			rval = qla4_8xxx_minidump_process_l2tag(ha, entry_hdr,
 								&data_ptr);
 			if (rval != QLA_SUCCESS) {
-				ql4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
 				goto md_failed;
 			}
 			break;
-		case QLA82XX_L1DAT:
-		case QLA82XX_L1INS:
+		case QLA8XXX_L1DTG:
+		case QLA8XXX_L1ITG:
+		case QLA8XXX_L1DAT:
+		case QLA8XXX_L1INS:
 			qla4_8xxx_minidump_process_l1cache(ha, entry_hdr,
 							   &data_ptr);
 			break;
-		case QLA82XX_RDOCM:
+		case QLA8XXX_RDOCM:
 			qla4_8xxx_minidump_process_rdocm(ha, entry_hdr,
 							 &data_ptr);
 			break;
-		case QLA82XX_RDMUX:
+		case QLA8XXX_RDMUX:
 			qla4_8xxx_minidump_process_rdmux(ha, entry_hdr,
 							 &data_ptr);
 			break;
-		case QLA82XX_QUEUE:
+		case QLA8XXX_QUEUE:
 			qla4_8xxx_minidump_process_queue(ha, entry_hdr,
 							 &data_ptr);
 			break;
-		case QLA82XX_RDNOP:
+		case QLA83XX_POLLRD:
+			if (!is_qla8032(ha)) {
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				break;
+			}
+			rval = qla83xx_minidump_process_pollrd(ha, entry_hdr,
+							       &data_ptr);
+			if (rval != QLA_SUCCESS)
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+			break;
+		case QLA83XX_RDMUX2:
+			if (!is_qla8032(ha)) {
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				break;
+			}
+			qla83xx_minidump_process_rdmux2(ha, entry_hdr,
+							&data_ptr);
+			break;
+		case QLA83XX_POLLRDMWR:
+			if (!is_qla8032(ha)) {
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+				break;
+			}
+			rval = qla83xx_minidump_process_pollrdmwr(ha, entry_hdr,
+								  &data_ptr);
+			if (rval != QLA_SUCCESS)
+				qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+			break;
+		case QLA8XXX_RDNOP:
 		default:
-			ql4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
+			qla4_8xxx_mark_entry_skipped(ha, entry_hdr, i);
 			break;
 		}
 
@@ -2224,7 +2534,7 @@ static int qla4_8xxx_collect_md_data(struct scsi_qla_host *ha)
 						ha->fw_dump_tmplt_size));
 skip_nxt_entry:
 		/*  next entry in the template */
-		entry_hdr = (struct qla82xx_minidump_entry_hdr *)
+		entry_hdr = (struct qla8xxx_minidump_entry_hdr *)
 				(((uint8_t *)entry_hdr) +
 				 entry_hdr->entry_size);
 	}
@@ -2264,33 +2574,45 @@ static void qla4_8xxx_uevent_emit(struct scsi_qla_host *ha, u32 code)
 	kobject_uevent_env(&(&ha->pdev->dev)->kobj, KOBJ_CHANGE, envp);
 }
 
+void qla4_8xxx_get_minidump(struct scsi_qla_host *ha)
+{
+	if (ql4xenablemd && test_bit(AF_FW_RECOVERY, &ha->flags) &&
+	    !test_bit(AF_82XX_FW_DUMPED, &ha->flags)) {
+		if (!qla4_8xxx_collect_md_data(ha)) {
+			qla4_8xxx_uevent_emit(ha, QL4_UEVENT_CODE_FW_DUMP);
+			set_bit(AF_82XX_FW_DUMPED, &ha->flags);
+		} else {
+			ql4_printk(KERN_INFO, ha, "%s: Unable to collect minidump\n",
+				   __func__);
+		}
+	}
+}
+
 /**
  * qla4_8xxx_device_bootstrap - Initialize device, set DEV_READY, start fw
  * @ha: pointer to adapter structure
  *
  * Note: IDC lock must be held upon entry
  **/
-static int
-qla4_8xxx_device_bootstrap(struct scsi_qla_host *ha)
+int qla4_8xxx_device_bootstrap(struct scsi_qla_host *ha)
 {
 	int rval = QLA_ERROR;
 	int i, timeout;
-	uint32_t old_count, count;
+	uint32_t old_count, count, idc_ctrl;
 	int need_reset = 0, peg_stuck = 1;
 
-	need_reset = qla4_8xxx_need_reset(ha);
-
-	old_count = qla4_8xxx_rd_32(ha, QLA82XX_PEG_ALIVE_COUNTER);
+	need_reset = ha->isp_ops->need_reset(ha);
+	old_count = qla4_8xxx_rd_direct(ha, QLA8XXX_PEG_ALIVE_COUNTER);
 
 	for (i = 0; i < 10; i++) {
 		timeout = msleep_interruptible(200);
 		if (timeout) {
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			   QLA82XX_DEV_FAILED);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
 			return rval;
 		}
 
-		count = qla4_8xxx_rd_32(ha, QLA82XX_PEG_ALIVE_COUNTER);
+		count = qla4_8xxx_rd_direct(ha, QLA8XXX_PEG_ALIVE_COUNTER);
 		if (count != old_count)
 			peg_stuck = 0;
 	}
@@ -2298,13 +2620,13 @@ qla4_8xxx_device_bootstrap(struct scsi_qla_host *ha)
 	if (need_reset) {
 		/* We are trying to perform a recovery here. */
 		if (peg_stuck)
-			qla4_8xxx_rom_lock_recovery(ha);
+			ha->isp_ops->rom_lock_recovery(ha);
 		goto dev_initialize;
 	} else  {
 		/* Start of day for this ha context. */
 		if (peg_stuck) {
 			/* Either we are the first or recovery in progress. */
-			qla4_8xxx_rom_lock_recovery(ha);
+			ha->isp_ops->rom_lock_recovery(ha);
 			goto dev_initialize;
 		} else {
 			/* Firmware already running. */
@@ -2316,46 +2638,53 @@ qla4_8xxx_device_bootstrap(struct scsi_qla_host *ha)
 dev_initialize:
 	/* set to DEV_INITIALIZING */
 	ql4_printk(KERN_INFO, ha, "HW State: INITIALIZING\n");
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_INITIALIZING);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+			    QLA8XXX_DEV_INITIALIZING);
 
-	/* Driver that sets device state to initializating sets IDC version */
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_IDC_VERSION, QLA82XX_IDC_VERSION);
-
-	qla4_8xxx_idc_unlock(ha);
-	if (ql4xenablemd && test_bit(AF_FW_RECOVERY, &ha->flags) &&
-	    !test_and_set_bit(AF_82XX_FW_DUMPED, &ha->flags)) {
-		if (!qla4_8xxx_collect_md_data(ha)) {
-			qla4_8xxx_uevent_emit(ha, QL4_UEVENT_CODE_FW_DUMP);
-		} else {
-			ql4_printk(KERN_INFO, ha, "Unable to collect minidump\n");
-			clear_bit(AF_82XX_FW_DUMPED, &ha->flags);
+	/*
+	 * For ISP8324, if IDC_CTRL GRACEFUL_RESET_BIT1 is set, reset it after
+	 * device goes to INIT state.
+	 */
+	if (is_qla8032(ha)) {
+		idc_ctrl = qla4_83xx_rd_reg(ha, QLA83XX_IDC_DRV_CTRL);
+		if (idc_ctrl & GRACEFUL_RESET_BIT1) {
+			qla4_83xx_wr_reg(ha, QLA83XX_IDC_DRV_CTRL,
+					 (idc_ctrl & ~GRACEFUL_RESET_BIT1));
+			set_bit(AF_83XX_NO_FW_DUMP, &ha->flags);
 		}
 	}
-	rval = qla4_8xxx_try_start_fw(ha);
-	qla4_8xxx_idc_lock(ha);
+
+	ha->isp_ops->idc_unlock(ha);
+
+	if (is_qla8022(ha))
+		qla4_8xxx_get_minidump(ha);
+
+	rval = ha->isp_ops->restart_firmware(ha);
+	ha->isp_ops->idc_lock(ha);
 
 	if (rval != QLA_SUCCESS) {
 		ql4_printk(KERN_INFO, ha, "HW State: FAILED\n");
 		qla4_8xxx_clear_drv_active(ha);
-		qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_FAILED);
+		qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+				    QLA8XXX_DEV_FAILED);
 		return rval;
 	}
 
 dev_ready:
 	ql4_printk(KERN_INFO, ha, "HW State: READY\n");
-	qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_READY);
+	qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE, QLA8XXX_DEV_READY);
 
 	return rval;
 }
 
 /**
- * qla4_8xxx_need_reset_handler - Code to start reset sequence
+ * qla4_82xx_need_reset_handler - Code to start reset sequence
  * @ha: pointer to adapter structure
  *
  * Note: IDC lock must be held upon entry
  **/
 static void
-qla4_8xxx_need_reset_handler(struct scsi_qla_host *ha)
+qla4_82xx_need_reset_handler(struct scsi_qla_host *ha)
 {
 	uint32_t dev_state, drv_state, drv_active;
 	uint32_t active_mask = 0xFFFFFFFF;
@@ -2365,12 +2694,12 @@ qla4_8xxx_need_reset_handler(struct scsi_qla_host *ha)
 		"Performing ISP error recovery\n");
 
 	if (test_and_clear_bit(AF_ONLINE, &ha->flags)) {
-		qla4_8xxx_idc_unlock(ha);
+		qla4_82xx_idc_unlock(ha);
 		ha->isp_ops->disable_intrs(ha);
-		qla4_8xxx_idc_lock(ha);
+		qla4_82xx_idc_lock(ha);
 	}
 
-	if (!test_bit(AF_82XX_RST_OWNER, &ha->flags)) {
+	if (!test_bit(AF_8XXX_RST_OWNER, &ha->flags)) {
 		DEBUG2(ql4_printk(KERN_INFO, ha,
 				  "%s(%ld): reset acknowledged\n",
 				  __func__, ha->host_no));
@@ -2382,8 +2711,8 @@ qla4_8xxx_need_reset_handler(struct scsi_qla_host *ha)
 	/* wait for 10 seconds for reset ack from all functions */
 	reset_timeout = jiffies + (ha->nx_reset_timeout * HZ);
 
-	drv_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-	drv_active = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
+	drv_state = qla4_82xx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
+	drv_active = qla4_82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
 
 	ql4_printk(KERN_INFO, ha,
 		"%s(%ld): drv_state = 0x%x, drv_active = 0x%x\n",
@@ -2401,31 +2730,31 @@ qla4_8xxx_need_reset_handler(struct scsi_qla_host *ha)
 		 * When reset_owner times out, check which functions
 		 * acked/did not ack
 		 */
-		if (test_bit(AF_82XX_RST_OWNER, &ha->flags)) {
+		if (test_bit(AF_8XXX_RST_OWNER, &ha->flags)) {
 			ql4_printk(KERN_INFO, ha,
 				   "%s(%ld): drv_state = 0x%x, drv_active = 0x%x\n",
 				   __func__, ha->host_no, drv_state,
 				   drv_active);
 		}
-		qla4_8xxx_idc_unlock(ha);
+		qla4_82xx_idc_unlock(ha);
 		msleep(1000);
-		qla4_8xxx_idc_lock(ha);
+		qla4_82xx_idc_lock(ha);
 
-		drv_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
-		drv_active = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
+		drv_state = qla4_82xx_rd_32(ha, QLA82XX_CRB_DRV_STATE);
+		drv_active = qla4_82xx_rd_32(ha, QLA82XX_CRB_DRV_ACTIVE);
 	}
 
 	/* Clear RESET OWNER as we are not going to use it any further */
-	clear_bit(AF_82XX_RST_OWNER, &ha->flags);
+	clear_bit(AF_8XXX_RST_OWNER, &ha->flags);
 
-	dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
+	dev_state = qla4_82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 	ql4_printk(KERN_INFO, ha, "Device state is 0x%x = %s\n", dev_state,
 		   dev_state < MAX_STATES ? qdev_state[dev_state] : "Unknown");
 
 	/* Force to DEV_COLD unless someone else is starting a reset */
-	if (dev_state != QLA82XX_DEV_INITIALIZING) {
+	if (dev_state != QLA8XXX_DEV_INITIALIZING) {
 		ql4_printk(KERN_INFO, ha, "HW State: COLD/RE-INIT\n");
-		qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA82XX_DEV_COLD);
+		qla4_82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE, QLA8XXX_DEV_COLD);
 		qla4_8xxx_set_rst_ready(ha);
 	}
 }
@@ -2437,9 +2766,104 @@ qla4_8xxx_need_reset_handler(struct scsi_qla_host *ha)
 void
 qla4_8xxx_need_qsnt_handler(struct scsi_qla_host *ha)
 {
-	qla4_8xxx_idc_lock(ha);
+	ha->isp_ops->idc_lock(ha);
 	qla4_8xxx_set_qsnt_ready(ha);
-	qla4_8xxx_idc_unlock(ha);
+	ha->isp_ops->idc_unlock(ha);
+}
+
+static void qla4_82xx_set_idc_ver(struct scsi_qla_host *ha)
+{
+	int idc_ver;
+	uint32_t drv_active;
+
+	drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+	if (drv_active == (1 << (ha->func_num * 4))) {
+		qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_IDC_VERSION,
+				    QLA82XX_IDC_VERSION);
+		ql4_printk(KERN_INFO, ha,
+			   "%s: IDC version updated to %d\n", __func__,
+			   QLA82XX_IDC_VERSION);
+	} else {
+		idc_ver = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_IDC_VERSION);
+		if (QLA82XX_IDC_VERSION != idc_ver) {
+			ql4_printk(KERN_INFO, ha,
+				   "%s: qla4xxx driver IDC version %d is not compatible with IDC version %d of other drivers!\n",
+				   __func__, QLA82XX_IDC_VERSION, idc_ver);
+		}
+	}
+}
+
+static int qla4_83xx_set_idc_ver(struct scsi_qla_host *ha)
+{
+	int idc_ver;
+	uint32_t drv_active;
+	int rval = QLA_SUCCESS;
+
+	drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+	if (drv_active == (1 << ha->func_num)) {
+		idc_ver = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_IDC_VERSION);
+		idc_ver &= (~0xFF);
+		idc_ver |= QLA83XX_IDC_VER_MAJ_VALUE;
+		qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_IDC_VERSION, idc_ver);
+		ql4_printk(KERN_INFO, ha,
+			   "%s: IDC version updated to %d\n", __func__,
+			   idc_ver);
+	} else {
+		idc_ver = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_IDC_VERSION);
+		idc_ver &= 0xFF;
+		if (QLA83XX_IDC_VER_MAJ_VALUE != idc_ver) {
+			ql4_printk(KERN_INFO, ha,
+				   "%s: qla4xxx driver IDC version %d is not compatible with IDC version %d of other drivers!\n",
+				   __func__, QLA83XX_IDC_VER_MAJ_VALUE,
+				   idc_ver);
+			rval = QLA_ERROR;
+			goto exit_set_idc_ver;
+		}
+	}
+
+	/* Update IDC_MINOR_VERSION */
+	idc_ver = qla4_83xx_rd_reg(ha, QLA83XX_CRB_IDC_VER_MINOR);
+	idc_ver &= ~(0x03 << (ha->func_num * 2));
+	idc_ver |= (QLA83XX_IDC_VER_MIN_VALUE << (ha->func_num * 2));
+	qla4_83xx_wr_reg(ha, QLA83XX_CRB_IDC_VER_MINOR, idc_ver);
+
+exit_set_idc_ver:
+	return rval;
+}
+
+int qla4_8xxx_update_idc_reg(struct scsi_qla_host *ha)
+{
+	uint32_t drv_active;
+	int rval = QLA_SUCCESS;
+
+	if (test_bit(AF_INIT_DONE, &ha->flags))
+		goto exit_update_idc_reg;
+
+	ha->isp_ops->idc_lock(ha);
+	qla4_8xxx_set_drv_active(ha);
+
+	/*
+	 * If we are the first driver to load and
+	 * ql4xdontresethba is not set, clear IDC_CTRL BIT0.
+	 */
+	if (is_qla8032(ha)) {
+		drv_active = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DRV_ACTIVE);
+		if ((drv_active == (1 << ha->func_num)) && !ql4xdontresethba)
+			qla4_83xx_clear_idc_dontreset(ha);
+	}
+
+	if (is_qla8022(ha)) {
+		qla4_82xx_set_idc_ver(ha);
+	} else if (is_qla8032(ha)) {
+		rval = qla4_83xx_set_idc_ver(ha);
+		if (rval == QLA_ERROR)
+			qla4_8xxx_clear_drv_active(ha);
+	}
+
+	ha->isp_ops->idc_unlock(ha);
+
+exit_update_idc_reg:
+	return rval;
 }
 
 /**
@@ -2454,13 +2878,11 @@ int qla4_8xxx_device_state_handler(struct scsi_qla_host *ha)
 	int rval = QLA_SUCCESS;
 	unsigned long dev_init_timeout;
 
-	if (!test_bit(AF_INIT_DONE, &ha->flags)) {
-		qla4_8xxx_idc_lock(ha);
-		qla4_8xxx_set_drv_active(ha);
-		qla4_8xxx_idc_unlock(ha);
-	}
+	rval = qla4_8xxx_update_idc_reg(ha);
+	if (rval == QLA_ERROR)
+		goto exit_state_handler;
 
-	dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
+	dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
 	DEBUG2(ql4_printk(KERN_INFO, ha, "Device state is 0x%x = %s\n",
 			  dev_state, dev_state < MAX_STATES ?
 			  qdev_state[dev_state] : "Unknown"));
@@ -2468,7 +2890,7 @@ int qla4_8xxx_device_state_handler(struct scsi_qla_host *ha)
 	/* wait for 30 seconds for device to go ready */
 	dev_init_timeout = jiffies + (ha->nx_dev_init_timeout * HZ);
 
-	qla4_8xxx_idc_lock(ha);
+	ha->isp_ops->idc_lock(ha);
 	while (1) {
 
 		if (time_after_eq(jiffies, dev_init_timeout)) {
@@ -2477,65 +2899,75 @@ int qla4_8xxx_device_state_handler(struct scsi_qla_host *ha)
 				   DRIVER_NAME,
 				   dev_state, dev_state < MAX_STATES ?
 				   qdev_state[dev_state] : "Unknown");
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-				QLA82XX_DEV_FAILED);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
 		}
 
-		dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
+		dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
 		ql4_printk(KERN_INFO, ha, "Device state is 0x%x = %s\n",
 			   dev_state, dev_state < MAX_STATES ?
 			   qdev_state[dev_state] : "Unknown");
 
 		/* NOTE: Make sure idc unlocked upon exit of switch statement */
 		switch (dev_state) {
-		case QLA82XX_DEV_READY:
+		case QLA8XXX_DEV_READY:
 			goto exit;
-		case QLA82XX_DEV_COLD:
+		case QLA8XXX_DEV_COLD:
 			rval = qla4_8xxx_device_bootstrap(ha);
 			goto exit;
-		case QLA82XX_DEV_INITIALIZING:
-			qla4_8xxx_idc_unlock(ha);
+		case QLA8XXX_DEV_INITIALIZING:
+			ha->isp_ops->idc_unlock(ha);
 			msleep(1000);
-			qla4_8xxx_idc_lock(ha);
+			ha->isp_ops->idc_lock(ha);
 			break;
-		case QLA82XX_DEV_NEED_RESET:
-			if (!ql4xdontresethba) {
-				qla4_8xxx_need_reset_handler(ha);
-				/* Update timeout value after need
-				 * reset handler */
-				dev_init_timeout = jiffies +
-					(ha->nx_dev_init_timeout * HZ);
-			} else {
-				qla4_8xxx_idc_unlock(ha);
-				msleep(1000);
-				qla4_8xxx_idc_lock(ha);
+		case QLA8XXX_DEV_NEED_RESET:
+			/*
+			 * For ISP8324, if NEED_RESET is set by any driver,
+			 * it should be honored, irrespective of IDC_CTRL
+			 * DONTRESET_BIT0
+			 */
+			if (is_qla8032(ha)) {
+				qla4_83xx_need_reset_handler(ha);
+			} else if (is_qla8022(ha)) {
+				if (!ql4xdontresethba) {
+					qla4_82xx_need_reset_handler(ha);
+					/* Update timeout value after need
+					 * reset handler */
+					dev_init_timeout = jiffies +
+						(ha->nx_dev_init_timeout * HZ);
+				} else {
+					ha->isp_ops->idc_unlock(ha);
+					msleep(1000);
+					ha->isp_ops->idc_lock(ha);
+				}
 			}
 			break;
-		case QLA82XX_DEV_NEED_QUIESCENT:
+		case QLA8XXX_DEV_NEED_QUIESCENT:
 			/* idc locked/unlocked in handler */
 			qla4_8xxx_need_qsnt_handler(ha);
 			break;
-		case QLA82XX_DEV_QUIESCENT:
-			qla4_8xxx_idc_unlock(ha);
+		case QLA8XXX_DEV_QUIESCENT:
+			ha->isp_ops->idc_unlock(ha);
 			msleep(1000);
-			qla4_8xxx_idc_lock(ha);
+			ha->isp_ops->idc_lock(ha);
 			break;
-		case QLA82XX_DEV_FAILED:
-			qla4_8xxx_idc_unlock(ha);
+		case QLA8XXX_DEV_FAILED:
+			ha->isp_ops->idc_unlock(ha);
 			qla4xxx_dead_adapter_cleanup(ha);
 			rval = QLA_ERROR;
-			qla4_8xxx_idc_lock(ha);
+			ha->isp_ops->idc_lock(ha);
 			goto exit;
 		default:
-			qla4_8xxx_idc_unlock(ha);
+			ha->isp_ops->idc_unlock(ha);
 			qla4xxx_dead_adapter_cleanup(ha);
 			rval = QLA_ERROR;
-			qla4_8xxx_idc_lock(ha);
+			ha->isp_ops->idc_lock(ha);
 			goto exit;
 		}
 	}
 exit:
-	qla4_8xxx_idc_unlock(ha);
+	ha->isp_ops->idc_unlock(ha);
+exit_state_handler:
 	return rval;
 }
 
@@ -2544,8 +2976,13 @@ int qla4_8xxx_load_risc(struct scsi_qla_host *ha)
 	int retval;
 
 	/* clear the interrupt */
-	writel(0, &ha->qla4_8xxx_reg->host_int);
-	readl(&ha->qla4_8xxx_reg->host_int);
+	if (is_qla8032(ha)) {
+		writel(0, &ha->qla4_83xx_reg->risc_intr);
+		readl(&ha->qla4_83xx_reg->risc_intr);
+	} else if (is_qla8022(ha)) {
+		writel(0, &ha->qla4_82xx_reg->host_int);
+		readl(&ha->qla4_82xx_reg->host_int);
+	}
 
 	retval = qla4_8xxx_device_state_handler(ha);
 
@@ -2579,13 +3016,13 @@ flash_data_addr(struct ql82xx_hw_data *hw, uint32_t faddr)
 }
 
 static uint32_t *
-qla4_8xxx_read_flash_data(struct scsi_qla_host *ha, uint32_t *dwptr,
+qla4_82xx_read_flash_data(struct scsi_qla_host *ha, uint32_t *dwptr,
     uint32_t faddr, uint32_t length)
 {
 	uint32_t i;
 	uint32_t val;
 	int loops = 0;
-	while ((qla4_8xxx_rom_lock(ha) != 0) && (loops < 50000)) {
+	while ((qla4_82xx_rom_lock(ha) != 0) && (loops < 50000)) {
 		udelay(100);
 		cond_resched();
 		loops++;
@@ -2597,7 +3034,7 @@ qla4_8xxx_read_flash_data(struct scsi_qla_host *ha, uint32_t *dwptr,
 
 	/* Dword reads to flash. */
 	for (i = 0; i < length/4; i++, faddr += 4) {
-		if (qla4_8xxx_do_rom_fast_read(ha, faddr, &val)) {
+		if (qla4_82xx_do_rom_fast_read(ha, faddr, &val)) {
 			ql4_printk(KERN_WARNING, ha,
 			    "Do ROM fast read failed\n");
 			goto done_read;
@@ -2606,7 +3043,7 @@ qla4_8xxx_read_flash_data(struct scsi_qla_host *ha, uint32_t *dwptr,
 	}
 
 done_read:
-	qla4_8xxx_rom_unlock(ha);
+	qla4_82xx_rom_unlock(ha);
 	return dwptr;
 }
 
@@ -2614,10 +3051,10 @@ done_read:
  * Address and length are byte address
  **/
 static uint8_t *
-qla4_8xxx_read_optrom_data(struct scsi_qla_host *ha, uint8_t *buf,
+qla4_82xx_read_optrom_data(struct scsi_qla_host *ha, uint8_t *buf,
 		uint32_t offset, uint32_t length)
 {
-	qla4_8xxx_read_flash_data(ha, (uint32_t *)buf, offset, length);
+	qla4_82xx_read_flash_data(ha, (uint32_t *)buf, offset, length);
 	return buf;
 }
 
@@ -2644,7 +3081,7 @@ qla4_8xxx_get_flt_info(struct scsi_qla_host *ha, uint32_t flt_addr)
 	const char *loc, *locations[] = { "DEF", "FLT" };
 	uint16_t *wptr;
 	uint16_t cnt, chksum;
-	uint32_t start;
+	uint32_t start, status;
 	struct qla_flt_header *flt;
 	struct qla_flt_region *region;
 	struct ql82xx_hw_data *hw = &ha->hw;
@@ -2653,8 +3090,18 @@ qla4_8xxx_get_flt_info(struct scsi_qla_host *ha, uint32_t flt_addr)
 	wptr = (uint16_t *)ha->request_ring;
 	flt = (struct qla_flt_header *)ha->request_ring;
 	region = (struct qla_flt_region *)&flt[1];
-	qla4_8xxx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
-			flt_addr << 2, OPTROM_BURST_SIZE);
+
+	if (is_qla8022(ha)) {
+		qla4_82xx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
+					   flt_addr << 2, OPTROM_BURST_SIZE);
+	} else if (is_qla8032(ha)) {
+		status = qla4_83xx_flash_read_u32(ha, flt_addr << 2,
+						  (uint8_t *)ha->request_ring,
+						  0x400);
+		if (status != QLA_SUCCESS)
+			goto no_flash_data;
+	}
+
 	if (*wptr == __constant_cpu_to_le16(0xffff))
 		goto no_flash_data;
 	if (flt->version != __constant_cpu_to_le16(1)) {
@@ -2730,7 +3177,7 @@ done:
 }
 
 static void
-qla4_8xxx_get_fdt_info(struct scsi_qla_host *ha)
+qla4_82xx_get_fdt_info(struct scsi_qla_host *ha)
 {
 #define FLASH_BLK_SIZE_4K       0x1000
 #define FLASH_BLK_SIZE_32K      0x8000
@@ -2748,7 +3195,7 @@ qla4_8xxx_get_fdt_info(struct scsi_qla_host *ha)
 
 	wptr = (uint16_t *)ha->request_ring;
 	fdt = (struct qla_fdt_layout *)ha->request_ring;
-	qla4_8xxx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
+	qla4_82xx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
 	    hw->flt_region_fdt << 2, OPTROM_BURST_SIZE);
 
 	if (*wptr == __constant_cpu_to_le16(0xffff))
@@ -2797,7 +3244,7 @@ done:
 }
 
 static void
-qla4_8xxx_get_idc_param(struct scsi_qla_host *ha)
+qla4_82xx_get_idc_param(struct scsi_qla_host *ha)
 {
 #define QLA82XX_IDC_PARAM_ADDR      0x003e885c
 	uint32_t *wptr;
@@ -2805,7 +3252,7 @@ qla4_8xxx_get_idc_param(struct scsi_qla_host *ha)
 	if (!is_qla8022(ha))
 		return;
 	wptr = (uint32_t *)ha->request_ring;
-	qla4_8xxx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
+	qla4_82xx_read_optrom_data(ha, (uint8_t *)ha->request_ring,
 			QLA82XX_IDC_PARAM_ADDR , 8);
 
 	if (*wptr == __constant_cpu_to_le32(0xffffffff)) {
@@ -2823,6 +3270,39 @@ qla4_8xxx_get_idc_param(struct scsi_qla_host *ha)
 	return;
 }
 
+void qla4_82xx_queue_mbox_cmd(struct scsi_qla_host *ha, uint32_t *mbx_cmd,
+			      int in_count)
+{
+	int i;
+
+	/* Load all mailbox registers, except mailbox 0. */
+	for (i = 1; i < in_count; i++)
+		writel(mbx_cmd[i], &ha->qla4_82xx_reg->mailbox_in[i]);
+
+	/* Wakeup firmware  */
+	writel(mbx_cmd[0], &ha->qla4_82xx_reg->mailbox_in[0]);
+	readl(&ha->qla4_82xx_reg->mailbox_in[0]);
+	writel(HINT_MBX_INT_PENDING, &ha->qla4_82xx_reg->hint);
+	readl(&ha->qla4_82xx_reg->hint);
+}
+
+void qla4_82xx_process_mbox_intr(struct scsi_qla_host *ha, int out_count)
+{
+	int intr_status;
+
+	intr_status = readl(&ha->qla4_82xx_reg->host_int);
+	if (intr_status & ISRX_82XX_RISC_INT) {
+		ha->mbox_status_count = out_count;
+		intr_status = readl(&ha->qla4_82xx_reg->host_status);
+		ha->isp_ops->interrupt_service_routine(ha, intr_status);
+
+		if (test_bit(AF_INTERRUPTS_ON, &ha->flags) &&
+		    test_bit(AF_INTx_ENABLED, &ha->flags))
+			qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg,
+					0xfbff);
+	}
+}
+
 int
 qla4_8xxx_get_flash_info(struct scsi_qla_host *ha)
 {
@@ -2834,8 +3314,12 @@ qla4_8xxx_get_flash_info(struct scsi_qla_host *ha)
 		return ret;
 
 	qla4_8xxx_get_flt_info(ha, flt_addr);
-	qla4_8xxx_get_fdt_info(ha);
-	qla4_8xxx_get_idc_param(ha);
+	if (is_qla8022(ha)) {
+		qla4_82xx_get_fdt_info(ha);
+		qla4_82xx_get_idc_param(ha);
+	} else if (is_qla8032(ha)) {
+		qla4_83xx_get_idc_param(ha);
+	}
 
 	return QLA_SUCCESS;
 }
@@ -2869,36 +3353,36 @@ qla4_8xxx_stop_firmware(struct scsi_qla_host *ha)
 }
 
 /**
- * qla4_8xxx_isp_reset - Resets ISP and aborts all outstanding commands.
+ * qla4_82xx_isp_reset - Resets ISP and aborts all outstanding commands.
  * @ha: pointer to host adapter structure.
  **/
 int
-qla4_8xxx_isp_reset(struct scsi_qla_host *ha)
+qla4_82xx_isp_reset(struct scsi_qla_host *ha)
 {
 	int rval;
 	uint32_t dev_state;
 
-	qla4_8xxx_idc_lock(ha);
-	dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
+	qla4_82xx_idc_lock(ha);
+	dev_state = qla4_82xx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
 
-	if (dev_state == QLA82XX_DEV_READY) {
+	if (dev_state == QLA8XXX_DEV_READY) {
 		ql4_printk(KERN_INFO, ha, "HW State: NEED RESET\n");
-		qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-		    QLA82XX_DEV_NEED_RESET);
-		set_bit(AF_82XX_RST_OWNER, &ha->flags);
+		qla4_82xx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
+		    QLA8XXX_DEV_NEED_RESET);
+		set_bit(AF_8XXX_RST_OWNER, &ha->flags);
 	} else
 		ql4_printk(KERN_INFO, ha, "HW State: DEVICE INITIALIZING\n");
 
-	qla4_8xxx_idc_unlock(ha);
+	qla4_82xx_idc_unlock(ha);
 
 	rval = qla4_8xxx_device_state_handler(ha);
 
-	qla4_8xxx_idc_lock(ha);
+	qla4_82xx_idc_lock(ha);
 	qla4_8xxx_clear_rst_ready(ha);
-	qla4_8xxx_idc_unlock(ha);
+	qla4_82xx_idc_unlock(ha);
 
 	if (rval == QLA_SUCCESS) {
-		ql4_printk(KERN_INFO, ha, "Clearing AF_RECOVERY in qla4_8xxx_isp_reset\n");
+		ql4_printk(KERN_INFO, ha, "Clearing AF_RECOVERY in qla4_82xx_isp_reset\n");
 		clear_bit(AF_FW_RECOVERY, &ha->flags);
 	}
 
@@ -2979,8 +3463,7 @@ exit_validate_mac82:
 
 /* Interrupt handling helpers. */
 
-static int
-qla4_8xxx_mbx_intr_enable(struct scsi_qla_host *ha)
+int qla4_8xxx_mbx_intr_enable(struct scsi_qla_host *ha)
 {
 	uint32_t mbox_cmd[MBOX_REG_COUNT];
 	uint32_t mbox_sts[MBOX_REG_COUNT];
@@ -3001,8 +3484,7 @@ qla4_8xxx_mbx_intr_enable(struct scsi_qla_host *ha)
 	return QLA_SUCCESS;
 }
 
-static int
-qla4_8xxx_mbx_intr_disable(struct scsi_qla_host *ha)
+int qla4_8xxx_mbx_intr_disable(struct scsi_qla_host *ha)
 {
 	uint32_t mbox_cmd[MBOX_REG_COUNT];
 	uint32_t mbox_sts[MBOX_REG_COUNT];
@@ -3025,26 +3507,26 @@ qla4_8xxx_mbx_intr_disable(struct scsi_qla_host *ha)
 }
 
 void
-qla4_8xxx_enable_intrs(struct scsi_qla_host *ha)
+qla4_82xx_enable_intrs(struct scsi_qla_host *ha)
 {
 	qla4_8xxx_mbx_intr_enable(ha);
 
 	spin_lock_irq(&ha->hardware_lock);
 	/* BIT 10 - reset */
-	qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff);
+	qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0xfbff);
 	spin_unlock_irq(&ha->hardware_lock);
 	set_bit(AF_INTERRUPTS_ON, &ha->flags);
 }
 
 void
-qla4_8xxx_disable_intrs(struct scsi_qla_host *ha)
+qla4_82xx_disable_intrs(struct scsi_qla_host *ha)
 {
 	if (test_and_clear_bit(AF_INTERRUPTS_ON, &ha->flags))
 		qla4_8xxx_mbx_intr_disable(ha);
 
 	spin_lock_irq(&ha->hardware_lock);
 	/* BIT 10 - set */
-	qla4_8xxx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0x0400);
+	qla4_82xx_wr_32(ha, ha->nx_legacy_intr.tgt_mask_reg, 0x0400);
 	spin_unlock_irq(&ha->hardware_lock);
 }
 
diff --git a/drivers/scsi/qla4xxx/ql4_nx.h b/drivers/scsi/qla4xxx/ql4_nx.h
index 30258479f10..9dc0bbfe50d 100644
--- a/drivers/scsi/qla4xxx/ql4_nx.h
+++ b/drivers/scsi/qla4xxx/ql4_nx.h
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -25,6 +25,8 @@
 #define CRB_RCVPEG_STATE		QLA82XX_REG(0x13c)
 #define CRB_DMA_SHIFT			QLA82XX_REG(0xcc)
 #define CRB_TEMP_STATE			QLA82XX_REG(0x1b4)
+#define CRB_CMDPEG_CHECK_RETRY_COUNT	60
+#define CRB_CMDPEG_CHECK_DELAY		500
 
 #define qla82xx_get_temp_val(x)		((x) >> 16)
 #define qla82xx_get_temp_state(x)	((x) & 0xffff)
@@ -490,8 +492,8 @@ enum {
  * Base addresses of major components on-chip.
  * ====================== BASE ADDRESSES ON-CHIP ======================
  */
-#define QLA82XX_ADDR_DDR_NET		(0x0000000000000000ULL)
-#define QLA82XX_ADDR_DDR_NET_MAX	(0x000000000fffffffULL)
+#define QLA8XXX_ADDR_DDR_NET		(0x0000000000000000ULL)
+#define QLA8XXX_ADDR_DDR_NET_MAX	(0x000000000fffffffULL)
 
 /* Imbus address bit used to indicate a host address. This bit is
  * eliminated by the pcie bar and bar select before presentation
@@ -500,14 +502,15 @@ enum {
 #define QLA82XX_P2_ADDR_PCIE	(0x0000000800000000ULL)
 #define QLA82XX_P3_ADDR_PCIE	(0x0000008000000000ULL)
 #define QLA82XX_ADDR_PCIE_MAX	(0x0000000FFFFFFFFFULL)
-#define QLA82XX_ADDR_OCM0	(0x0000000200000000ULL)
-#define QLA82XX_ADDR_OCM0_MAX	(0x00000002000fffffULL)
-#define QLA82XX_ADDR_OCM1	(0x0000000200400000ULL)
-#define QLA82XX_ADDR_OCM1_MAX	(0x00000002004fffffULL)
-#define QLA82XX_ADDR_QDR_NET	(0x0000000300000000ULL)
+#define QLA8XXX_ADDR_OCM0	(0x0000000200000000ULL)
+#define QLA8XXX_ADDR_OCM0_MAX	(0x00000002000fffffULL)
+#define QLA8XXX_ADDR_OCM1	(0x0000000200400000ULL)
+#define QLA8XXX_ADDR_OCM1_MAX	(0x00000002004fffffULL)
+#define QLA8XXX_ADDR_QDR_NET	(0x0000000300000000ULL)
 
 #define QLA82XX_P2_ADDR_QDR_NET_MAX	(0x00000003001fffffULL)
 #define QLA82XX_P3_ADDR_QDR_NET_MAX	(0x0000000303ffffffULL)
+#define QLA8XXX_ADDR_QDR_NET_MAX	(0x0000000307ffffffULL)
 
 #define QLA82XX_PCI_CRBSPACE		(unsigned long)0x06000000
 #define QLA82XX_PCI_DIRECT_CRB		(unsigned long)0x04400000
@@ -517,6 +520,10 @@ enum {
 #define QLA82XX_PCI_QDR_NET		(unsigned long)0x04000000
 #define QLA82XX_PCI_QDR_NET_MAX		(unsigned long)0x043fffff
 
+/*  PCI Windowing for DDR regions.  */
+#define QLA8XXX_ADDR_IN_RANGE(addr, low, high)            \
+	(((addr) <= (high)) && ((addr) >= (low)))
+
 /*
  *   Register offsets for MN
  */
@@ -540,6 +547,11 @@ enum {
 #define MIU_TA_CTL_WRITE	4
 #define MIU_TA_CTL_BUSY		8
 
+#define MIU_TA_CTL_WRITE_ENABLE		(MIU_TA_CTL_WRITE | MIU_TA_CTL_ENABLE)
+#define MIU_TA_CTL_WRITE_START		(MIU_TA_CTL_WRITE | MIU_TA_CTL_ENABLE |\
+					 MIU_TA_CTL_START)
+#define MIU_TA_CTL_START_ENABLE		(MIU_TA_CTL_START | MIU_TA_CTL_ENABLE)
+
 /*CAM RAM */
 # define QLA82XX_CAM_RAM_BASE	(QLA82XX_CRB_CAM + 0x02000)
 # define QLA82XX_CAM_RAM(reg)	(QLA82XX_CAM_RAM_BASE + (reg))
@@ -565,20 +577,53 @@ enum {
 /* Driver Coexistence Defines */
 #define QLA82XX_CRB_DRV_ACTIVE		(QLA82XX_CAM_RAM(0x138))
 #define QLA82XX_CRB_DEV_STATE		(QLA82XX_CAM_RAM(0x140))
-#define QLA82XX_CRB_DEV_PART_INFO	(QLA82XX_CAM_RAM(0x14c))
-#define QLA82XX_CRB_DRV_IDC_VERSION	(QLA82XX_CAM_RAM(0x174))
 #define QLA82XX_CRB_DRV_STATE		(QLA82XX_CAM_RAM(0x144))
 #define QLA82XX_CRB_DRV_SCRATCH		(QLA82XX_CAM_RAM(0x148))
 #define QLA82XX_CRB_DEV_PART_INFO	(QLA82XX_CAM_RAM(0x14c))
+#define QLA82XX_CRB_DRV_IDC_VERSION	(QLA82XX_CAM_RAM(0x174))
+
+enum qla_regs {
+	QLA8XXX_PEG_HALT_STATUS1 = 0,
+	QLA8XXX_PEG_HALT_STATUS2,
+	QLA8XXX_PEG_ALIVE_COUNTER,
+	QLA8XXX_CRB_DRV_ACTIVE,
+	QLA8XXX_CRB_DEV_STATE,
+	QLA8XXX_CRB_DRV_STATE,
+	QLA8XXX_CRB_DRV_SCRATCH,
+	QLA8XXX_CRB_DEV_PART_INFO,
+	QLA8XXX_CRB_DRV_IDC_VERSION,
+	QLA8XXX_FW_VERSION_MAJOR,
+	QLA8XXX_FW_VERSION_MINOR,
+	QLA8XXX_FW_VERSION_SUB,
+	QLA8XXX_CRB_CMDPEG_STATE,
+	QLA8XXX_CRB_TEMP_STATE,
+};
+
+static const uint32_t qla4_82xx_reg_tbl[] = {
+	QLA82XX_PEG_HALT_STATUS1,
+	QLA82XX_PEG_HALT_STATUS2,
+	QLA82XX_PEG_ALIVE_COUNTER,
+	QLA82XX_CRB_DRV_ACTIVE,
+	QLA82XX_CRB_DEV_STATE,
+	QLA82XX_CRB_DRV_STATE,
+	QLA82XX_CRB_DRV_SCRATCH,
+	QLA82XX_CRB_DEV_PART_INFO,
+	QLA82XX_CRB_DRV_IDC_VERSION,
+	QLA82XX_FW_VERSION_MAJOR,
+	QLA82XX_FW_VERSION_MINOR,
+	QLA82XX_FW_VERSION_SUB,
+	CRB_CMDPEG_STATE,
+	CRB_TEMP_STATE,
+};
 
 /* Every driver should use these Device State */
-#define QLA82XX_DEV_COLD		1
-#define QLA82XX_DEV_INITIALIZING	2
-#define QLA82XX_DEV_READY		3
-#define QLA82XX_DEV_NEED_RESET		4
-#define QLA82XX_DEV_NEED_QUIESCENT	5
-#define QLA82XX_DEV_FAILED		6
-#define QLA82XX_DEV_QUIESCENT		7
+#define QLA8XXX_DEV_COLD		1
+#define QLA8XXX_DEV_INITIALIZING	2
+#define QLA8XXX_DEV_READY		3
+#define QLA8XXX_DEV_NEED_RESET		4
+#define QLA8XXX_DEV_NEED_QUIESCENT	5
+#define QLA8XXX_DEV_FAILED		6
+#define QLA8XXX_DEV_QUIESCENT		7
 #define MAX_STATES			8 /* Increment if new state added */
 
 #define QLA82XX_IDC_VERSION		0x1
@@ -795,47 +840,51 @@ struct crb_addr_pair {
 /* Minidump related */
 
 /* Entry Type Defines */
-#define QLA82XX_RDNOP	0
-#define QLA82XX_RDCRB	1
-#define QLA82XX_RDMUX	2
-#define QLA82XX_QUEUE	3
-#define QLA82XX_BOARD	4
-#define QLA82XX_RDOCM	6
-#define QLA82XX_PREGS	7
-#define QLA82XX_L1DTG	8
-#define QLA82XX_L1ITG	9
-#define QLA82XX_L1DAT	11
-#define QLA82XX_L1INS	12
-#define QLA82XX_L2DTG	21
-#define QLA82XX_L2ITG	22
-#define QLA82XX_L2DAT	23
-#define QLA82XX_L2INS	24
-#define QLA82XX_RDROM	71
-#define QLA82XX_RDMEM	72
-#define QLA82XX_CNTRL	98
-#define QLA82XX_RDEND	255
+#define QLA8XXX_RDNOP	0
+#define QLA8XXX_RDCRB	1
+#define QLA8XXX_RDMUX	2
+#define QLA8XXX_QUEUE	3
+#define QLA8XXX_BOARD	4
+#define QLA8XXX_RDOCM	6
+#define QLA8XXX_PREGS	7
+#define QLA8XXX_L1DTG	8
+#define QLA8XXX_L1ITG	9
+#define QLA8XXX_L1DAT	11
+#define QLA8XXX_L1INS	12
+#define QLA8XXX_L2DTG	21
+#define QLA8XXX_L2ITG	22
+#define QLA8XXX_L2DAT	23
+#define QLA8XXX_L2INS	24
+#define QLA83XX_POLLRD	35
+#define QLA83XX_RDMUX2	36
+#define QLA83XX_POLLRDMWR  37
+#define QLA8XXX_RDROM	71
+#define QLA8XXX_RDMEM	72
+#define QLA8XXX_CNTRL	98
+#define QLA83XX_TLHDR	99
+#define QLA8XXX_RDEND	255
 
 /* Opcodes for Control Entries.
  * These Flags are bit fields.
  */
-#define QLA82XX_DBG_OPCODE_WR		0x01
-#define QLA82XX_DBG_OPCODE_RW		0x02
-#define QLA82XX_DBG_OPCODE_AND		0x04
-#define QLA82XX_DBG_OPCODE_OR		0x08
-#define QLA82XX_DBG_OPCODE_POLL		0x10
-#define QLA82XX_DBG_OPCODE_RDSTATE	0x20
-#define QLA82XX_DBG_OPCODE_WRSTATE	0x40
-#define QLA82XX_DBG_OPCODE_MDSTATE	0x80
+#define QLA8XXX_DBG_OPCODE_WR		0x01
+#define QLA8XXX_DBG_OPCODE_RW		0x02
+#define QLA8XXX_DBG_OPCODE_AND		0x04
+#define QLA8XXX_DBG_OPCODE_OR		0x08
+#define QLA8XXX_DBG_OPCODE_POLL		0x10
+#define QLA8XXX_DBG_OPCODE_RDSTATE	0x20
+#define QLA8XXX_DBG_OPCODE_WRSTATE	0x40
+#define QLA8XXX_DBG_OPCODE_MDSTATE	0x80
 
 /* Driver Flags */
-#define QLA82XX_DBG_SKIPPED_FLAG	0x80 /* driver skipped this entry  */
-#define QLA82XX_DBG_SIZE_ERR_FLAG	0x40 /* Entry vs Capture size
+#define QLA8XXX_DBG_SKIPPED_FLAG	0x80 /* driver skipped this entry  */
+#define QLA8XXX_DBG_SIZE_ERR_FLAG	0x40 /* Entry vs Capture size
 					      * mismatch */
 
 /* Driver_code is for driver to write some info about the entry
  * currently not used.
  */
-struct qla82xx_minidump_entry_hdr {
+struct qla8xxx_minidump_entry_hdr {
 	uint32_t entry_type;
 	uint32_t entry_size;
 	uint32_t entry_capture_size;
@@ -848,8 +897,8 @@ struct qla82xx_minidump_entry_hdr {
 };
 
 /*  Read CRB entry header */
-struct qla82xx_minidump_entry_crb {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_crb {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t addr;
 	struct {
 		uint8_t addr_stride;
@@ -871,8 +920,8 @@ struct qla82xx_minidump_entry_crb {
 	uint32_t value_3;
 };
 
-struct qla82xx_minidump_entry_cache {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_cache {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t tag_reg_addr;
 	struct {
 		uint16_t tag_value_stride;
@@ -895,8 +944,8 @@ struct qla82xx_minidump_entry_cache {
 };
 
 /* Read OCM */
-struct qla82xx_minidump_entry_rdocm {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_rdocm {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t rsvd_0;
 	uint32_t rsvd_1;
 	uint32_t data_size;
@@ -908,24 +957,24 @@ struct qla82xx_minidump_entry_rdocm {
 };
 
 /* Read Memory */
-struct qla82xx_minidump_entry_rdmem {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_rdmem {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t rsvd[6];
 	uint32_t read_addr;
 	uint32_t read_data_size;
 };
 
 /* Read ROM */
-struct qla82xx_minidump_entry_rdrom {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_rdrom {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t rsvd[6];
 	uint32_t read_addr;
 	uint32_t read_data_size;
 };
 
 /* Mux entry */
-struct qla82xx_minidump_entry_mux {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_mux {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t select_addr;
 	uint32_t rsvd_0;
 	uint32_t data_size;
@@ -937,8 +986,8 @@ struct qla82xx_minidump_entry_mux {
 };
 
 /* Queue entry */
-struct qla82xx_minidump_entry_queue {
-	struct qla82xx_minidump_entry_hdr h;
+struct qla8xxx_minidump_entry_queue {
+	struct qla8xxx_minidump_entry_hdr h;
 	uint32_t select_addr;
 	struct {
 		uint16_t queue_id_stride;
@@ -956,23 +1005,6 @@ struct qla82xx_minidump_entry_queue {
 	} rd_strd;
 };
 
-#define QLA82XX_MINIDUMP_OCM0_SIZE		(256 * 1024)
-#define QLA82XX_MINIDUMP_L1C_SIZE		(256 * 1024)
-#define QLA82XX_MINIDUMP_L2C_SIZE		1572864
-#define QLA82XX_MINIDUMP_COMMON_STR_SIZE	0
-#define QLA82XX_MINIDUMP_FCOE_STR_SIZE		0
-#define QLA82XX_MINIDUMP_MEM_SIZE		0
-#define QLA82XX_MAX_ENTRY_HDR			4
-
-struct qla82xx_minidump {
-	uint32_t md_ocm0_data[QLA82XX_MINIDUMP_OCM0_SIZE];
-	uint32_t md_l1c_data[QLA82XX_MINIDUMP_L1C_SIZE];
-	uint32_t md_l2c_data[QLA82XX_MINIDUMP_L2C_SIZE];
-	uint32_t md_cs_data[QLA82XX_MINIDUMP_COMMON_STR_SIZE];
-	uint32_t md_fcoes_data[QLA82XX_MINIDUMP_FCOE_STR_SIZE];
-	uint32_t md_mem_data[QLA82XX_MINIDUMP_MEM_SIZE];
-};
-
 #define MBC_DIAGNOSTIC_MINIDUMP_TEMPLATE	0x129
 #define RQST_TMPLT_SIZE				0x0
 #define RQST_TMPLT				0x1
@@ -982,6 +1014,16 @@ struct qla82xx_minidump {
 #define MD_MIU_TEST_AGT_ADDR_LO			0x41000094
 #define MD_MIU_TEST_AGT_ADDR_HI			0x41000098
 
+#define MD_MIU_TEST_AGT_WRDATA_LO		0x410000A0
+#define MD_MIU_TEST_AGT_WRDATA_HI		0x410000A4
+#define MD_MIU_TEST_AGT_WRDATA_ULO		0x410000B0
+#define MD_MIU_TEST_AGT_WRDATA_UHI		0x410000B4
+
+#define MD_MIU_TEST_AGT_RDDATA_LO		0x410000A8
+#define MD_MIU_TEST_AGT_RDDATA_HI		0x410000AC
+#define MD_MIU_TEST_AGT_RDDATA_ULO		0x410000B8
+#define MD_MIU_TEST_AGT_RDDATA_UHI		0x410000BC
+
 static const int MD_MIU_TEST_AGT_RDDATA[] = { 0x410000A8,
 				0x410000AC, 0x410000B8, 0x410000BC };
 #endif
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 79243b76d17..fbc546e893a 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -1,6 +1,6 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
@@ -18,6 +18,7 @@
 #include "ql4_glbl.h"
 #include "ql4_dbg.h"
 #include "ql4_inline.h"
+#include "ql4_83xx.h"
 
 /*
  * Driver version
@@ -160,7 +161,7 @@ static int qla4xxx_eh_host_reset(struct scsi_cmnd *cmd);
 static int qla4xxx_slave_alloc(struct scsi_device *device);
 static int qla4xxx_slave_configure(struct scsi_device *device);
 static void qla4xxx_slave_destroy(struct scsi_device *sdev);
-static umode_t ql4_attr_is_visible(int param_type, int param);
+static umode_t qla4_attr_is_visible(int param_type, int param);
 static int qla4xxx_host_reset(struct Scsi_Host *shost, int reset_type);
 static int qla4xxx_change_queue_depth(struct scsi_device *sdev, int qdepth,
 				      int reason);
@@ -203,7 +204,7 @@ static struct iscsi_transport qla4xxx_iscsi_transport = {
 				  CAP_DATA_PATH_OFFLOAD | CAP_HDRDGST |
 				  CAP_DATADGST | CAP_LOGIN_OFFLOAD |
 				  CAP_MULTI_R2T,
-	.attr_is_visible	= ql4_attr_is_visible,
+	.attr_is_visible	= qla4_attr_is_visible,
 	.create_session         = qla4xxx_session_create,
 	.destroy_session        = qla4xxx_session_destroy,
 	.start_conn             = qla4xxx_conn_start,
@@ -315,7 +316,7 @@ exit_send_ping:
 	return rval;
 }
 
-static umode_t ql4_attr_is_visible(int param_type, int param)
+static umode_t qla4_attr_is_visible(int param_type, int param)
 {
 	switch (param_type) {
 	case ISCSI_HOST_PARAM:
@@ -1366,7 +1367,7 @@ static int qla4xxx_conn_get_param(struct iscsi_cls_conn *cls_conn,
 
 	conn = cls_conn->dd_data;
 	qla_conn = conn->dd_data;
-	dst_addr = &qla_conn->qla_ep->dst_addr;
+	dst_addr = (struct sockaddr *)&qla_conn->qla_ep->dst_addr;
 
 	switch (param) {
 	case ISCSI_PARAM_CONN_PORT:
@@ -2315,8 +2316,17 @@ static void qla4xxx_mem_free(struct scsi_qla_host *ha)
 		if (ha->nx_pcibase)
 			iounmap(
 			    (struct device_reg_82xx __iomem *)ha->nx_pcibase);
-	} else if (ha->reg)
+	} else if (is_qla8032(ha)) {
+		if (ha->nx_pcibase)
+			iounmap(
+			    (struct device_reg_83xx __iomem *)ha->nx_pcibase);
+	} else if (ha->reg) {
 		iounmap(ha->reg);
+	}
+
+	if (ha->reset_tmplt.buff)
+		vfree(ha->reset_tmplt.buff);
+
 	pci_release_regions(ha->pdev);
 }
 
@@ -2420,7 +2430,7 @@ static int qla4_8xxx_check_temp(struct scsi_qla_host *ha)
 	uint32_t temp, temp_state, temp_val;
 	int status = QLA_SUCCESS;
 
-	temp = qla4_8xxx_rd_32(ha, CRB_TEMP_STATE);
+	temp = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_TEMP_STATE);
 
 	temp_state = qla82xx_get_temp_state(temp);
 	temp_val = qla82xx_get_temp_val(temp);
@@ -2456,7 +2466,8 @@ static int qla4_8xxx_check_fw_alive(struct scsi_qla_host *ha)
 	uint32_t fw_heartbeat_counter;
 	int status = QLA_SUCCESS;
 
-	fw_heartbeat_counter = qla4_8xxx_rd_32(ha, QLA82XX_PEG_ALIVE_COUNTER);
+	fw_heartbeat_counter = qla4_8xxx_rd_direct(ha,
+						   QLA8XXX_PEG_ALIVE_COUNTER);
 	/* If PEG_ALIVE_COUNTER is 0xffffffff, AER/EEH is in progress, ignore */
 	if (fw_heartbeat_counter == 0xffffffff) {
 		DEBUG2(printk(KERN_WARNING "scsi%ld: %s: Device in frozen "
@@ -2470,28 +2481,7 @@ static int qla4_8xxx_check_fw_alive(struct scsi_qla_host *ha)
 		/* FW not alive after 2 seconds */
 		if (ha->seconds_since_last_heartbeat == 2) {
 			ha->seconds_since_last_heartbeat = 0;
-
-			ql4_printk(KERN_INFO, ha,
-				   "scsi(%ld): %s, Dumping hw/fw registers:\n "
-				   " PEG_HALT_STATUS1: 0x%x, PEG_HALT_STATUS2:"
-				   " 0x%x,\n PEG_NET_0_PC: 0x%x, PEG_NET_1_PC:"
-				   " 0x%x,\n PEG_NET_2_PC: 0x%x, PEG_NET_3_PC:"
-				   " 0x%x,\n PEG_NET_4_PC: 0x%x\n",
-				   ha->host_no, __func__,
-				   qla4_8xxx_rd_32(ha,
-						   QLA82XX_PEG_HALT_STATUS1),
-				   qla4_8xxx_rd_32(ha,
-						   QLA82XX_PEG_HALT_STATUS2),
-				   qla4_8xxx_rd_32(ha, QLA82XX_CRB_PEG_NET_0 +
-						   0x3c),
-				   qla4_8xxx_rd_32(ha, QLA82XX_CRB_PEG_NET_1 +
-						   0x3c),
-				   qla4_8xxx_rd_32(ha, QLA82XX_CRB_PEG_NET_2 +
-						   0x3c),
-				   qla4_8xxx_rd_32(ha, QLA82XX_CRB_PEG_NET_3 +
-						   0x3c),
-				   qla4_8xxx_rd_32(ha, QLA82XX_CRB_PEG_NET_4 +
-						   0x3c));
+			qla4_8xxx_dump_peg_reg(ha);
 			status = QLA_ERROR;
 		}
 	} else
@@ -2501,6 +2491,48 @@ static int qla4_8xxx_check_fw_alive(struct scsi_qla_host *ha)
 	return status;
 }
 
+static void qla4_8xxx_process_fw_error(struct scsi_qla_host *ha)
+{
+	uint32_t halt_status;
+	int halt_status_unrecoverable = 0;
+
+	halt_status = qla4_8xxx_rd_direct(ha, QLA8XXX_PEG_HALT_STATUS1);
+
+	if (is_qla8022(ha)) {
+		ql4_printk(KERN_INFO, ha, "%s: disabling pause transmit on port 0 & 1.\n",
+			   __func__);
+		qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
+				CRB_NIU_XG_PAUSE_CTL_P0 |
+				CRB_NIU_XG_PAUSE_CTL_P1);
+
+		if (QLA82XX_FWERROR_CODE(halt_status) == 0x67)
+			ql4_printk(KERN_ERR, ha, "%s: Firmware aborted with error code 0x00006700. Device is being reset\n",
+				   __func__);
+		if (halt_status & HALT_STATUS_UNRECOVERABLE)
+			halt_status_unrecoverable = 1;
+	} else if (is_qla8032(ha)) {
+		if (halt_status & QLA83XX_HALT_STATUS_FW_RESET)
+			ql4_printk(KERN_ERR, ha, "%s: Firmware error detected device is being reset\n",
+				   __func__);
+		else if (halt_status & QLA83XX_HALT_STATUS_UNRECOVERABLE)
+			halt_status_unrecoverable = 1;
+	}
+
+	/*
+	 * Since we cannot change dev_state in interrupt context,
+	 * set appropriate DPC flag then wakeup DPC
+	 */
+	if (halt_status_unrecoverable) {
+		set_bit(DPC_HA_UNRECOVERABLE, &ha->dpc_flags);
+	} else {
+		ql4_printk(KERN_INFO, ha, "%s: detect abort needed!\n",
+			   __func__);
+		set_bit(DPC_RESET_HA, &ha->dpc_flags);
+	}
+	qla4xxx_mailbox_premature_completion(ha);
+	qla4xxx_wake_dpc(ha);
+}
+
 /**
  * qla4_8xxx_watchdog - Poll dev state
  * @ha: Pointer to host adapter structure.
@@ -2509,31 +2541,33 @@ static int qla4_8xxx_check_fw_alive(struct scsi_qla_host *ha)
  **/
 void qla4_8xxx_watchdog(struct scsi_qla_host *ha)
 {
-	uint32_t dev_state, halt_status;
+	uint32_t dev_state;
 
 	/* don't poll if reset is going on */
 	if (!(test_bit(DPC_RESET_ACTIVE, &ha->dpc_flags) ||
 	    test_bit(DPC_RESET_HA, &ha->dpc_flags) ||
 	    test_bit(DPC_RETRY_RESET_HA, &ha->dpc_flags))) {
-		dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
+		dev_state = qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE);
 
 		if (qla4_8xxx_check_temp(ha)) {
-			ql4_printk(KERN_INFO, ha, "disabling pause"
-				   " transmit on port 0 & 1.\n");
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
-					CRB_NIU_XG_PAUSE_CTL_P0 |
-					CRB_NIU_XG_PAUSE_CTL_P1);
+			if (is_qla8022(ha)) {
+				ql4_printk(KERN_INFO, ha, "disabling pause transmit on port 0 & 1.\n");
+				qla4_82xx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
+						CRB_NIU_XG_PAUSE_CTL_P0 |
+						CRB_NIU_XG_PAUSE_CTL_P1);
+			}
 			set_bit(DPC_HA_UNRECOVERABLE, &ha->dpc_flags);
 			qla4xxx_wake_dpc(ha);
-		} else if (dev_state == QLA82XX_DEV_NEED_RESET &&
-		    !test_bit(DPC_RESET_HA, &ha->dpc_flags)) {
-			if (!ql4xdontresethba) {
+		} else if (dev_state == QLA8XXX_DEV_NEED_RESET &&
+			   !test_bit(DPC_RESET_HA, &ha->dpc_flags)) {
+			if (is_qla8032(ha) ||
+			    (is_qla8022(ha) && !ql4xdontresethba)) {
 				ql4_printk(KERN_INFO, ha, "%s: HW State: "
 				    "NEED RESET!\n", __func__);
 				set_bit(DPC_RESET_HA, &ha->dpc_flags);
 				qla4xxx_wake_dpc(ha);
 			}
-		} else if (dev_state == QLA82XX_DEV_NEED_QUIESCENT &&
+		} else if (dev_state == QLA8XXX_DEV_NEED_QUIESCENT &&
 		    !test_bit(DPC_HA_NEED_QUIESCENT, &ha->dpc_flags)) {
 			ql4_printk(KERN_INFO, ha, "%s: HW State: NEED QUIES!\n",
 			    __func__);
@@ -2541,36 +2575,8 @@ void qla4_8xxx_watchdog(struct scsi_qla_host *ha)
 			qla4xxx_wake_dpc(ha);
 		} else  {
 			/* Check firmware health */
-			if (qla4_8xxx_check_fw_alive(ha)) {
-				ql4_printk(KERN_INFO, ha, "disabling pause"
-					   " transmit on port 0 & 1.\n");
-				qla4_8xxx_wr_32(ha, QLA82XX_CRB_NIU + 0x98,
-						CRB_NIU_XG_PAUSE_CTL_P0 |
-						CRB_NIU_XG_PAUSE_CTL_P1);
-				halt_status = qla4_8xxx_rd_32(ha,
-						QLA82XX_PEG_HALT_STATUS1);
-
-				if (QLA82XX_FWERROR_CODE(halt_status) == 0x67)
-					ql4_printk(KERN_ERR, ha, "%s:"
-						   " Firmware aborted with"
-						   " error code 0x00006700."
-						   " Device is being reset\n",
-						   __func__);
-
-				/* Since we cannot change dev_state in interrupt
-				 * context, set appropriate DPC flag then wakeup
-				 * DPC */
-				if (halt_status & HALT_STATUS_UNRECOVERABLE)
-					set_bit(DPC_HA_UNRECOVERABLE,
-						&ha->dpc_flags);
-				else {
-					ql4_printk(KERN_INFO, ha, "%s: detect "
-						   "abort needed!\n", __func__);
-					set_bit(DPC_RESET_HA, &ha->dpc_flags);
-				}
-				qla4xxx_mailbox_premature_completion(ha);
-				qla4xxx_wake_dpc(ha);
-			}
+			if (qla4_8xxx_check_fw_alive(ha))
+				qla4_8xxx_process_fw_error(ha);
 		}
 	}
 }
@@ -2652,11 +2658,10 @@ static void qla4xxx_timer(struct scsi_qla_host *ha)
 	if (!pci_channel_offline(ha->pdev))
 		pci_read_config_word(ha->pdev, PCI_VENDOR_ID, &w);
 
-	if (is_qla8022(ha)) {
+	if (is_qla80XX(ha))
 		qla4_8xxx_watchdog(ha);
-	}
 
-	if (!is_qla8022(ha)) {
+	if (is_qla40XX(ha)) {
 		/* Check for heartbeat interval. */
 		if (ha->firmware_options & FWOPT_HEARTBEAT_ENABLE &&
 		    ha->heartbeat_interval != 0) {
@@ -2941,6 +2946,14 @@ static int qla4xxx_recover_adapter(struct scsi_qla_host *ha)
 
 	set_bit(DPC_RESET_ACTIVE, &ha->dpc_flags);
 
+	if (is_qla8032(ha) &&
+	    !test_bit(DPC_RESET_HA_FW_CONTEXT, &ha->dpc_flags)) {
+		ql4_printk(KERN_INFO, ha, "%s: disabling pause transmit on port 0 & 1.\n",
+			   __func__);
+		/* disable pause frame for ISP83xx */
+		qla4_83xx_disable_pause(ha);
+	}
+
 	iscsi_host_for_each_session(ha->host, qla4xxx_fail_session);
 
 	if (test_bit(DPC_RESET_HA, &ha->dpc_flags))
@@ -2953,9 +2966,9 @@ static int qla4xxx_recover_adapter(struct scsi_qla_host *ha)
 		goto recover_ha_init_adapter;
 	}
 
-	/* For the ISP-82xx adapter, issue a stop_firmware if invoked
+	/* For the ISP-8xxx adapter, issue a stop_firmware if invoked
 	 * from eh_host_reset or ioctl module */
-	if (is_qla8022(ha) && !reset_chip &&
+	if (is_qla80XX(ha) && !reset_chip &&
 	    test_bit(DPC_RESET_HA_FW_CONTEXT, &ha->dpc_flags)) {
 
 		DEBUG2(ql4_printk(KERN_INFO, ha,
@@ -2978,13 +2991,13 @@ static int qla4xxx_recover_adapter(struct scsi_qla_host *ha)
 	}
 
 	/* Issue full chip reset if recovering from a catastrophic error,
-	 * or if stop_firmware fails for ISP-82xx.
+	 * or if stop_firmware fails for ISP-8xxx.
 	 * This is the default case for ISP-4xxx */
-	if (!is_qla8022(ha) || reset_chip) {
-		if (!is_qla8022(ha))
+	if (is_qla40XX(ha) || reset_chip) {
+		if (is_qla40XX(ha))
 			goto chip_reset;
 
-		/* Check if 82XX firmware is alive or not
+		/* Check if 8XXX firmware is alive or not
 		 * We may have arrived here from NEED_RESET
 		 * detection only */
 		if (test_bit(AF_FW_RECOVERY, &ha->flags))
@@ -3000,10 +3013,10 @@ static int qla4xxx_recover_adapter(struct scsi_qla_host *ha)
 			set_current_state(TASK_UNINTERRUPTIBLE);
 			schedule_timeout(HZ);
 		}
-
+chip_reset:
 		if (!test_bit(AF_FW_RECOVERY, &ha->flags))
 			qla4xxx_cmd_wait(ha);
-chip_reset:
+
 		qla4xxx_process_aen(ha, FLUSH_DDB_CHANGED_AENS);
 		qla4xxx_abort_active_cmds(ha, DID_RESET << 16);
 		DEBUG2(ql4_printk(KERN_INFO, ha,
@@ -3021,7 +3034,7 @@ recover_ha_init_adapter:
 		/* For ISP-4xxx, force function 1 to always initialize
 		 * before function 3 to prevent both funcions from
 		 * stepping on top of the other */
-		if (!is_qla8022(ha) && (ha->mac_index == 3))
+		if (is_qla40XX(ha) && (ha->mac_index == 3))
 			ssleep(6);
 
 		/* NOTE: AF_ONLINE flag set upon successful completion of
@@ -3039,11 +3052,12 @@ recover_ha_init_adapter:
 		 * Since we don't want to block the DPC for too long
 		 * with multiple resets in the same thread,
 		 * utilize DPC to retry */
-		if (is_qla8022(ha)) {
-			qla4_8xxx_idc_lock(ha);
-			dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
-			qla4_8xxx_idc_unlock(ha);
-			if (dev_state == QLA82XX_DEV_FAILED) {
+		if (is_qla80XX(ha)) {
+			ha->isp_ops->idc_lock(ha);
+			dev_state = qla4_8xxx_rd_direct(ha,
+							QLA8XXX_CRB_DEV_STATE);
+			ha->isp_ops->idc_unlock(ha);
+			if (dev_state == QLA8XXX_DEV_FAILED) {
 				ql4_printk(KERN_INFO, ha, "%s: don't retry "
 					   "recover adapter. H/W is in Failed "
 					   "state\n", __func__);
@@ -3168,6 +3182,7 @@ int qla4xxx_unblock_ddb(struct iscsi_cls_session *cls_session)
 	struct iscsi_session *sess;
 	struct ddb_entry *ddb_entry;
 	struct scsi_qla_host *ha;
+	int status = QLA_SUCCESS;
 
 	sess = cls_session->dd_data;
 	ddb_entry = sess->dd_data;
@@ -3175,11 +3190,20 @@ int qla4xxx_unblock_ddb(struct iscsi_cls_session *cls_session)
 	ql4_printk(KERN_INFO, ha, "scsi%ld: %s: ddb[%d]"
 		   " unblock user space session\n", ha->host_no, __func__,
 		   ddb_entry->fw_ddb_index);
-	iscsi_conn_start(ddb_entry->conn);
-	iscsi_conn_login_event(ddb_entry->conn,
-			       ISCSI_CONN_STATE_LOGGED_IN);
 
-	return QLA_SUCCESS;
+	if (!iscsi_is_session_online(cls_session)) {
+		iscsi_conn_start(ddb_entry->conn);
+		iscsi_conn_login_event(ddb_entry->conn,
+				       ISCSI_CONN_STATE_LOGGED_IN);
+	} else {
+		ql4_printk(KERN_INFO, ha,
+			   "scsi%ld: %s: ddb[%d] session [%d] already logged in\n",
+			   ha->host_no, __func__, ddb_entry->fw_ddb_index,
+			   cls_session->sid);
+		status = QLA_ERROR;
+	}
+
+	return status;
 }
 
 static void qla4xxx_relogin_all_devices(struct scsi_qla_host *ha)
@@ -3373,15 +3397,26 @@ static void qla4xxx_do_dpc(struct work_struct *work)
 	/* post events to application */
 	qla4xxx_do_work(ha);
 
-	if (is_qla8022(ha)) {
+	if (is_qla80XX(ha)) {
 		if (test_bit(DPC_HA_UNRECOVERABLE, &ha->dpc_flags)) {
-			qla4_8xxx_idc_lock(ha);
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_FAILED);
-			qla4_8xxx_idc_unlock(ha);
+			if (is_qla8032(ha)) {
+				ql4_printk(KERN_INFO, ha, "%s: disabling pause transmit on port 0 & 1.\n",
+					   __func__);
+				/* disable pause frame for ISP83xx */
+				qla4_83xx_disable_pause(ha);
+			}
+
+			ha->isp_ops->idc_lock(ha);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
+			ha->isp_ops->idc_unlock(ha);
 			ql4_printk(KERN_INFO, ha, "HW State: FAILED\n");
 			qla4_8xxx_device_state_handler(ha);
 		}
+
+		if (test_and_clear_bit(DPC_POST_IDC_ACK, &ha->dpc_flags))
+			qla4_83xx_post_idc_ack(ha);
+
 		if (test_and_clear_bit(DPC_HA_NEED_QUIESCENT, &ha->dpc_flags)) {
 			qla4_8xxx_need_qsnt_handler(ha);
 		}
@@ -3391,7 +3426,8 @@ static void qla4xxx_do_dpc(struct work_struct *work)
 	    (test_bit(DPC_RESET_HA, &ha->dpc_flags) ||
 	    test_bit(DPC_RESET_HA_INTR, &ha->dpc_flags) ||
 	    test_bit(DPC_RESET_HA_FW_CONTEXT, &ha->dpc_flags))) {
-		if (ql4xdontresethba) {
+		if ((is_qla8022(ha) && ql4xdontresethba) ||
+		    (is_qla8032(ha) && qla4_83xx_idc_dontreset(ha))) {
 			DEBUG2(printk("scsi%ld: %s: Don't Reset HBA\n",
 			    ha->host_no, __func__));
 			clear_bit(DPC_RESET_HA, &ha->dpc_flags);
@@ -3477,6 +3513,18 @@ static void qla4xxx_free_adapter(struct scsi_qla_host *ha)
 		ha->isp_ops->disable_intrs(ha);
 	}
 
+	if (is_qla40XX(ha)) {
+		writel(set_rmask(CSR_SCSI_PROCESSOR_INTR),
+		       &ha->reg->ctrl_status);
+		readl(&ha->reg->ctrl_status);
+	} else if (is_qla8022(ha)) {
+		writel(0, &ha->qla4_82xx_reg->host_int);
+		readl(&ha->qla4_82xx_reg->host_int);
+	} else if (is_qla8032(ha)) {
+		writel(0, &ha->qla4_83xx_reg->risc_intr);
+		readl(&ha->qla4_83xx_reg->risc_intr);
+	}
+
 	/* Remove timer thread, if present */
 	if (ha->timer_active)
 		qla4xxx_stop_timer(ha);
@@ -3492,10 +3540,10 @@ static void qla4xxx_free_adapter(struct scsi_qla_host *ha)
 	/* Put firmware in known state */
 	ha->isp_ops->reset_firmware(ha);
 
-	if (is_qla8022(ha)) {
-		qla4_8xxx_idc_lock(ha);
+	if (is_qla80XX(ha)) {
+		ha->isp_ops->idc_lock(ha);
 		qla4_8xxx_clear_drv_active(ha);
-		qla4_8xxx_idc_unlock(ha);
+		ha->isp_ops->idc_unlock(ha);
 	}
 
 	/* Detach interrupts */
@@ -3542,16 +3590,20 @@ int qla4_8xxx_iospace_config(struct scsi_qla_host *ha)
 	/* Mapping of IO base pointer, door bell read and write pointer */
 
 	/* mapping of IO base pointer */
-	ha->qla4_8xxx_reg =
-	    (struct device_reg_82xx  __iomem *)((uint8_t *)ha->nx_pcibase +
-	    0xbc000 + (ha->pdev->devfn << 11));
+	if (is_qla8022(ha)) {
+		ha->qla4_82xx_reg = (struct device_reg_82xx  __iomem *)
+				    ((uint8_t *)ha->nx_pcibase + 0xbc000 +
+				     (ha->pdev->devfn << 11));
+		ha->nx_db_wr_ptr = (ha->pdev->devfn == 4 ? QLA82XX_CAM_RAM_DB1 :
+				    QLA82XX_CAM_RAM_DB2);
+	} else if (is_qla8032(ha)) {
+		ha->qla4_83xx_reg = (struct device_reg_83xx __iomem *)
+				    ((uint8_t *)ha->nx_pcibase);
+	}
 
 	db_base = pci_resource_start(pdev, 4);  /* doorbell is on bar 4 */
 	db_len = pci_resource_len(pdev, 4);
 
-	ha->nx_db_wr_ptr = (ha->pdev->devfn == 4 ? QLA82XX_CAM_RAM_DB1 :
-	    QLA82XX_CAM_RAM_DB2);
-
 	return 0;
 iospace_error_exit:
 	return -ENOMEM;
@@ -3639,23 +3691,64 @@ static struct isp_operations qla4xxx_isp_ops = {
 	.rd_shdw_req_q_out      = qla4xxx_rd_shdw_req_q_out,
 	.rd_shdw_rsp_q_in       = qla4xxx_rd_shdw_rsp_q_in,
 	.get_sys_info           = qla4xxx_get_sys_info,
+	.queue_mailbox_command	= qla4xxx_queue_mbox_cmd,
+	.process_mailbox_interrupt = qla4xxx_process_mbox_intr,
 };
 
-static struct isp_operations qla4_8xxx_isp_ops = {
+static struct isp_operations qla4_82xx_isp_ops = {
 	.iospace_config         = qla4_8xxx_iospace_config,
 	.pci_config             = qla4_8xxx_pci_config,
-	.disable_intrs          = qla4_8xxx_disable_intrs,
-	.enable_intrs           = qla4_8xxx_enable_intrs,
+	.disable_intrs          = qla4_82xx_disable_intrs,
+	.enable_intrs           = qla4_82xx_enable_intrs,
 	.start_firmware         = qla4_8xxx_load_risc,
-	.intr_handler           = qla4_8xxx_intr_handler,
-	.interrupt_service_routine = qla4_8xxx_interrupt_service_routine,
-	.reset_chip             = qla4_8xxx_isp_reset,
+	.restart_firmware	= qla4_82xx_try_start_fw,
+	.intr_handler           = qla4_82xx_intr_handler,
+	.interrupt_service_routine = qla4_82xx_interrupt_service_routine,
+	.need_reset		= qla4_8xxx_need_reset,
+	.reset_chip             = qla4_82xx_isp_reset,
 	.reset_firmware         = qla4_8xxx_stop_firmware,
-	.queue_iocb             = qla4_8xxx_queue_iocb,
-	.complete_iocb          = qla4_8xxx_complete_iocb,
-	.rd_shdw_req_q_out      = qla4_8xxx_rd_shdw_req_q_out,
-	.rd_shdw_rsp_q_in       = qla4_8xxx_rd_shdw_rsp_q_in,
+	.queue_iocb             = qla4_82xx_queue_iocb,
+	.complete_iocb          = qla4_82xx_complete_iocb,
+	.rd_shdw_req_q_out      = qla4_82xx_rd_shdw_req_q_out,
+	.rd_shdw_rsp_q_in       = qla4_82xx_rd_shdw_rsp_q_in,
 	.get_sys_info           = qla4_8xxx_get_sys_info,
+	.rd_reg_direct		= qla4_82xx_rd_32,
+	.wr_reg_direct		= qla4_82xx_wr_32,
+	.rd_reg_indirect	= qla4_82xx_md_rd_32,
+	.wr_reg_indirect	= qla4_82xx_md_wr_32,
+	.idc_lock		= qla4_82xx_idc_lock,
+	.idc_unlock		= qla4_82xx_idc_unlock,
+	.rom_lock_recovery	= qla4_82xx_rom_lock_recovery,
+	.queue_mailbox_command	= qla4_82xx_queue_mbox_cmd,
+	.process_mailbox_interrupt = qla4_82xx_process_mbox_intr,
+};
+
+static struct isp_operations qla4_83xx_isp_ops = {
+	.iospace_config		= qla4_8xxx_iospace_config,
+	.pci_config		= qla4_8xxx_pci_config,
+	.disable_intrs		= qla4_83xx_disable_intrs,
+	.enable_intrs		= qla4_83xx_enable_intrs,
+	.start_firmware		= qla4_8xxx_load_risc,
+	.restart_firmware	= qla4_83xx_start_firmware,
+	.intr_handler		= qla4_83xx_intr_handler,
+	.interrupt_service_routine = qla4_83xx_interrupt_service_routine,
+	.need_reset		= qla4_8xxx_need_reset,
+	.reset_chip		= qla4_83xx_isp_reset,
+	.reset_firmware		= qla4_8xxx_stop_firmware,
+	.queue_iocb		= qla4_83xx_queue_iocb,
+	.complete_iocb		= qla4_83xx_complete_iocb,
+	.rd_shdw_req_q_out	= qla4_83xx_rd_shdw_req_q_out,
+	.rd_shdw_rsp_q_in	= qla4_83xx_rd_shdw_rsp_q_in,
+	.get_sys_info		= qla4_8xxx_get_sys_info,
+	.rd_reg_direct		= qla4_83xx_rd_reg,
+	.wr_reg_direct		= qla4_83xx_wr_reg,
+	.rd_reg_indirect	= qla4_83xx_rd_reg_indirect,
+	.wr_reg_indirect	= qla4_83xx_wr_reg_indirect,
+	.idc_lock		= qla4_83xx_drv_lock,
+	.idc_unlock		= qla4_83xx_drv_unlock,
+	.rom_lock_recovery	= qla4_83xx_rom_lock_recovery,
+	.queue_mailbox_command	= qla4_83xx_queue_mbox_cmd,
+	.process_mailbox_interrupt = qla4_83xx_process_mbox_intr,
 };
 
 uint16_t qla4xxx_rd_shdw_req_q_out(struct scsi_qla_host *ha)
@@ -3663,9 +3756,14 @@ uint16_t qla4xxx_rd_shdw_req_q_out(struct scsi_qla_host *ha)
 	return (uint16_t)le32_to_cpu(ha->shadow_regs->req_q_out);
 }
 
-uint16_t qla4_8xxx_rd_shdw_req_q_out(struct scsi_qla_host *ha)
+uint16_t qla4_82xx_rd_shdw_req_q_out(struct scsi_qla_host *ha)
 {
-	return (uint16_t)le32_to_cpu(readl(&ha->qla4_8xxx_reg->req_q_out));
+	return (uint16_t)le32_to_cpu(readl(&ha->qla4_82xx_reg->req_q_out));
+}
+
+uint16_t qla4_83xx_rd_shdw_req_q_out(struct scsi_qla_host *ha)
+{
+	return (uint16_t)le32_to_cpu(readl(&ha->qla4_83xx_reg->req_q_out));
 }
 
 uint16_t qla4xxx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha)
@@ -3673,9 +3771,14 @@ uint16_t qla4xxx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha)
 	return (uint16_t)le32_to_cpu(ha->shadow_regs->rsp_q_in);
 }
 
-uint16_t qla4_8xxx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha)
+uint16_t qla4_82xx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha)
 {
-	return (uint16_t)le32_to_cpu(readl(&ha->qla4_8xxx_reg->rsp_q_in));
+	return (uint16_t)le32_to_cpu(readl(&ha->qla4_82xx_reg->rsp_q_in));
+}
+
+uint16_t qla4_83xx_rd_shdw_rsp_q_in(struct scsi_qla_host *ha)
+{
+	return (uint16_t)le32_to_cpu(readl(&ha->qla4_83xx_reg->rsp_q_in));
 }
 
 static ssize_t qla4xxx_show_boot_eth_info(void *data, int type, char *buf)
@@ -5050,30 +5153,36 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 	ha->pdev = pdev;
 	ha->host = host;
 	ha->host_no = host->host_no;
+	ha->func_num = PCI_FUNC(ha->pdev->devfn);
 
 	pci_enable_pcie_error_reporting(pdev);
 
 	/* Setup Runtime configurable options */
 	if (is_qla8022(ha)) {
-		ha->isp_ops = &qla4_8xxx_isp_ops;
-		rwlock_init(&ha->hw_lock);
+		ha->isp_ops = &qla4_82xx_isp_ops;
+		ha->reg_tbl = (uint32_t *) qla4_82xx_reg_tbl;
 		ha->qdr_sn_window = -1;
 		ha->ddr_mn_window = -1;
 		ha->curr_window = 255;
-		ha->func_num = PCI_FUNC(ha->pdev->devfn);
 		nx_legacy_intr = &legacy_intr[ha->func_num];
 		ha->nx_legacy_intr.int_vec_bit = nx_legacy_intr->int_vec_bit;
 		ha->nx_legacy_intr.tgt_status_reg =
 			nx_legacy_intr->tgt_status_reg;
 		ha->nx_legacy_intr.tgt_mask_reg = nx_legacy_intr->tgt_mask_reg;
 		ha->nx_legacy_intr.pci_int_reg = nx_legacy_intr->pci_int_reg;
+	} else if (is_qla8032(ha)) {
+		ha->isp_ops = &qla4_83xx_isp_ops;
+		ha->reg_tbl = (uint32_t *)qla4_83xx_reg_tbl;
 	} else {
 		ha->isp_ops = &qla4xxx_isp_ops;
 	}
 
-	/* Set EEH reset type to fundamental if required by hba */
-	if (is_qla8022(ha))
+	if (is_qla80XX(ha)) {
+		rwlock_init(&ha->hw_lock);
+		ha->pf_bit = ha->func_num << 16;
+		/* Set EEH reset type to fundamental if required by hba */
 		pdev->needs_freset = 1;
+	}
 
 	/* Configure PCI I/O space. */
 	ret = ha->isp_ops->iospace_config(ha);
@@ -5094,6 +5203,7 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 	init_completion(&ha->disable_acb_comp);
 
 	spin_lock_init(&ha->hardware_lock);
+	spin_lock_init(&ha->work_lock);
 
 	/* Initialize work list */
 	INIT_LIST_HEAD(&ha->work_list);
@@ -5128,8 +5238,20 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 	if (ret)
 		goto probe_failed;
 
-	if (is_qla8022(ha))
-		(void) qla4_8xxx_get_flash_info(ha);
+	if (is_qla80XX(ha))
+		qla4_8xxx_get_flash_info(ha);
+
+	if (is_qla8032(ha)) {
+		qla4_83xx_read_reset_template(ha);
+		/*
+		 * NOTE: If ql4dontresethba==1, set IDC_CTRL DONTRESET_BIT0.
+		 * If DONRESET_BIT0 is set, drivers should not set dev_state
+		 * to NEED_RESET. But if NEED_RESET is set, drivers should
+		 * should honor the reset.
+		 */
+		if (ql4xdontresethba == 1)
+			qla4_83xx_set_idc_dontreset(ha);
+	}
 
 	/*
 	 * Initialize the Host adapter request/response queues and
@@ -5137,14 +5259,20 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 	 * NOTE: interrupts enabled upon successful completion
 	 */
 	status = qla4xxx_initialize_adapter(ha, INIT_ADAPTER);
+
+	/* Dont retry adapter initialization if IRQ allocation failed */
+	if (!test_bit(AF_IRQ_ATTACHED, &ha->flags))
+		goto skip_retry_init;
+
 	while ((!test_bit(AF_ONLINE, &ha->flags)) &&
 	    init_retry_count++ < MAX_INIT_RETRIES) {
 
-		if (is_qla8022(ha)) {
-			qla4_8xxx_idc_lock(ha);
-			dev_state = qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE);
-			qla4_8xxx_idc_unlock(ha);
-			if (dev_state == QLA82XX_DEV_FAILED) {
+		if (is_qla80XX(ha)) {
+			ha->isp_ops->idc_lock(ha);
+			dev_state = qla4_8xxx_rd_direct(ha,
+							QLA82XX_CRB_DEV_STATE);
+			ha->isp_ops->idc_unlock(ha);
+			if (dev_state == QLA8XXX_DEV_FAILED) {
 				ql4_printk(KERN_WARNING, ha, "%s: don't retry "
 				    "initialize adapter. H/W is in failed state\n",
 				    __func__);
@@ -5160,16 +5288,18 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 		status = qla4xxx_initialize_adapter(ha, INIT_ADAPTER);
 	}
 
+skip_retry_init:
 	if (!test_bit(AF_ONLINE, &ha->flags)) {
 		ql4_printk(KERN_WARNING, ha, "Failed to initialize adapter\n");
 
-		if (is_qla8022(ha) && ql4xdontresethba) {
+		if ((is_qla8022(ha) && ql4xdontresethba) ||
+		    (is_qla8032(ha) && qla4_83xx_idc_dontreset(ha))) {
 			/* Put the device in failed state. */
 			DEBUG2(printk(KERN_ERR "HW STATE: FAILED\n"));
-			qla4_8xxx_idc_lock(ha);
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_FAILED);
-			qla4_8xxx_idc_unlock(ha);
+			ha->isp_ops->idc_lock(ha);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
+			ha->isp_ops->idc_unlock(ha);
 		}
 		ret = -ENODEV;
 		goto remove_host;
@@ -5195,12 +5325,13 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 		goto remove_host;
 	}
 
-	/* For ISP-82XX, request_irqs is called in qla4_8xxx_load_risc
+	/*
+	 * For ISP-8XXX, request_irqs is called in qla4_8xxx_load_risc
 	 * (which is called indirectly by qla4xxx_initialize_adapter),
 	 * so that irqs will be registered after crbinit but before
 	 * mbx_intr_enable.
 	 */
-	if (!is_qla8022(ha)) {
+	if (is_qla40XX(ha)) {
 		ret = qla4xxx_request_irqs(ha);
 		if (ret) {
 			ql4_printk(KERN_WARNING, ha, "Failed to reserve "
@@ -5226,6 +5357,10 @@ static int __devinit qla4xxx_probe_adapter(struct pci_dev *pdev,
 	       ha->host_no, ha->firmware_version[0], ha->firmware_version[1],
 	       ha->patch_number, ha->build_number);
 
+	/* Set the driver version */
+	if (is_qla80XX(ha))
+		qla4_8xxx_set_param(ha, SET_DRVR_VERSION);
+
 	if (qla4xxx_setup_boot_info(ha))
 		ql4_printk(KERN_ERR, ha,
 			   "%s: No iSCSI boot target configured\n", __func__);
@@ -5333,9 +5468,16 @@ static void __devexit qla4xxx_remove_adapter(struct pci_dev *pdev)
 {
 	struct scsi_qla_host *ha;
 
+	/*
+	 * If the PCI device is disabled then it means probe_adapter had
+	 * failed and resources already cleaned up on probe_adapter exit.
+	 */
+	if (!pci_is_enabled(pdev))
+		return;
+
 	ha = pci_get_drvdata(pdev);
 
-	if (!is_qla8022(ha))
+	if (is_qla40XX(ha))
 		qla4xxx_prevent_other_port_reinit(ha);
 
 	/* destroy iface from sysfs */
@@ -5755,7 +5897,16 @@ static int qla4xxx_eh_host_reset(struct scsi_cmnd *cmd)
 
 	ha = to_qla_host(cmd->device->host);
 
-	if (ql4xdontresethba) {
+	if (is_qla8032(ha) && ql4xdontresethba)
+		qla4_83xx_set_idc_dontreset(ha);
+
+	/*
+	 * For ISP8324, if IDC_CTRL DONTRESET_BIT0 is set by other
+	 * protocol drivers, we should not set device_state to
+	 * NEED_RESET
+	 */
+	if (ql4xdontresethba ||
+	    (is_qla8032(ha) && qla4_83xx_idc_dontreset(ha))) {
 		DEBUG2(printk("scsi%ld: %s: Don't Reset HBA\n",
 		     ha->host_no, __func__));
 
@@ -5779,7 +5930,7 @@ static int qla4xxx_eh_host_reset(struct scsi_cmnd *cmd)
 	}
 
 	if (!test_bit(DPC_RESET_HA, &ha->dpc_flags)) {
-		if (is_qla8022(ha))
+		if (is_qla80XX(ha))
 			set_bit(DPC_RESET_HA_FW_CONTEXT, &ha->dpc_flags);
 		else
 			set_bit(DPC_RESET_HA, &ha->dpc_flags);
@@ -5874,7 +6025,7 @@ static int qla4xxx_host_reset(struct Scsi_Host *shost, int reset_type)
 		break;
 	case SCSI_FIRMWARE_RESET:
 		if (!test_bit(DPC_RESET_HA, &ha->dpc_flags)) {
-			if (is_qla8022(ha))
+			if (is_qla80XX(ha))
 				/* set firmware context reset */
 				set_bit(DPC_RESET_HA_FW_CONTEXT,
 					&ha->dpc_flags);
@@ -6013,32 +6164,43 @@ static uint32_t qla4_8xxx_error_recovery(struct scsi_qla_host *ha)
 		    "0x%x is the owner\n", ha->host_no, __func__,
 		    ha->pdev->devfn);
 
-		qla4_8xxx_idc_lock(ha);
-		qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-		    QLA82XX_DEV_COLD);
-
-		qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_IDC_VERSION,
-		    QLA82XX_IDC_VERSION);
+		ha->isp_ops->idc_lock(ha);
+		qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+				    QLA8XXX_DEV_COLD);
+		ha->isp_ops->idc_unlock(ha);
+
+		rval = qla4_8xxx_update_idc_reg(ha);
+		if (rval == QLA_ERROR) {
+			ql4_printk(KERN_INFO, ha, "scsi%ld: %s: HW State: FAILED\n",
+				   ha->host_no, __func__);
+			ha->isp_ops->idc_lock(ha);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
+			ha->isp_ops->idc_unlock(ha);
+			goto exit_error_recovery;
+		}
 
-		qla4_8xxx_idc_unlock(ha);
 		clear_bit(AF_FW_RECOVERY, &ha->flags);
 		rval = qla4xxx_initialize_adapter(ha, RESET_ADAPTER);
-		qla4_8xxx_idc_lock(ha);
 
 		if (rval != QLA_SUCCESS) {
 			ql4_printk(KERN_INFO, ha, "scsi%ld: %s: HW State: "
 			    "FAILED\n", ha->host_no, __func__);
+			ha->isp_ops->idc_lock(ha);
 			qla4_8xxx_clear_drv_active(ha);
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_FAILED);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_FAILED);
+			ha->isp_ops->idc_unlock(ha);
 		} else {
 			ql4_printk(KERN_INFO, ha, "scsi%ld: %s: HW State: "
 			    "READY\n", ha->host_no, __func__);
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DEV_STATE,
-			    QLA82XX_DEV_READY);
+			ha->isp_ops->idc_lock(ha);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DEV_STATE,
+					    QLA8XXX_DEV_READY);
 			/* Clear driver state register */
-			qla4_8xxx_wr_32(ha, QLA82XX_CRB_DRV_STATE, 0);
+			qla4_8xxx_wr_direct(ha, QLA8XXX_CRB_DRV_STATE, 0);
 			qla4_8xxx_set_drv_active(ha);
+			ha->isp_ops->idc_unlock(ha);
 			ret = qla4xxx_request_irqs(ha);
 			if (ret) {
 				ql4_printk(KERN_WARNING, ha, "Failed to "
@@ -6050,13 +6212,12 @@ static uint32_t qla4_8xxx_error_recovery(struct scsi_qla_host *ha)
 				rval = QLA_SUCCESS;
 			}
 		}
-		qla4_8xxx_idc_unlock(ha);
 	} else {
 		ql4_printk(KERN_INFO, ha, "scsi%ld: %s: devfn 0x%x is not "
 		    "the reset owner\n", ha->host_no, __func__,
 		    ha->pdev->devfn);
-		if ((qla4_8xxx_rd_32(ha, QLA82XX_CRB_DEV_STATE) ==
-		    QLA82XX_DEV_READY)) {
+		if ((qla4_8xxx_rd_direct(ha, QLA8XXX_CRB_DEV_STATE) ==
+		     QLA8XXX_DEV_READY)) {
 			clear_bit(AF_FW_RECOVERY, &ha->flags);
 			rval = qla4xxx_initialize_adapter(ha, RESET_ADAPTER);
 			if (rval == QLA_SUCCESS) {
@@ -6071,11 +6232,12 @@ static uint32_t qla4_8xxx_error_recovery(struct scsi_qla_host *ha)
 					rval = QLA_SUCCESS;
 				}
 			}
-			qla4_8xxx_idc_lock(ha);
+			ha->isp_ops->idc_lock(ha);
 			qla4_8xxx_set_drv_active(ha);
-			qla4_8xxx_idc_unlock(ha);
+			ha->isp_ops->idc_unlock(ha);
 		}
 	}
+exit_error_recovery:
 	clear_bit(DPC_RESET_ACTIVE, &ha->dpc_flags);
 	return rval;
 }
@@ -6114,7 +6276,7 @@ qla4xxx_pci_slot_reset(struct pci_dev *pdev)
 
 	ha->isp_ops->disable_intrs(ha);
 
-	if (is_qla8022(ha)) {
+	if (is_qla80XX(ha)) {
 		if (qla4_8xxx_error_recovery(ha) == QLA_SUCCESS) {
 			ret = PCI_ERS_RESULT_RECOVERED;
 			goto exit_slot_reset;
@@ -6180,6 +6342,12 @@ static struct pci_device_id qla4xxx_pci_tbl[] = {
 		.subvendor      = PCI_ANY_ID,
 		.subdevice      = PCI_ANY_ID,
 	},
+	{
+		.vendor		= PCI_VENDOR_ID_QLOGIC,
+		.device		= PCI_DEVICE_ID_QLOGIC_ISP8324,
+		.subvendor	= PCI_ANY_ID,
+		.subdevice	= PCI_ANY_ID,
+	},
 	{0, 0},
 };
 MODULE_DEVICE_TABLE(pci, qla4xxx_pci_tbl);
diff --git a/drivers/scsi/qla4xxx/ql4_version.h b/drivers/scsi/qla4xxx/ql4_version.h
index 725034f4252..f6df2ea91ab 100644
--- a/drivers/scsi/qla4xxx/ql4_version.h
+++ b/drivers/scsi/qla4xxx/ql4_version.h
@@ -1,8 +1,8 @@
 /*
  * QLogic iSCSI HBA Driver
- * Copyright (c)  2003-2010 QLogic Corporation
+ * Copyright (c)  2003-2012 QLogic Corporation
  *
  * See LICENSE.qla4xxx for copyright and licensing details.
  */
 
-#define QLA4XXX_DRIVER_VERSION	"5.02.00-k18"
+#define QLA4XXX_DRIVER_VERSION	"5.03.00-k1"
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index 182d5a57ab7..57fbd5a3d4e 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -109,6 +109,7 @@ static const char * scsi_debug_version_date = "20100324";
 #define DEF_OPT_BLKS 64
 #define DEF_PHYSBLK_EXP 0
 #define DEF_PTYPE   0
+#define DEF_REMOVABLE false
 #define DEF_SCSI_LEVEL   5    /* INQUIRY, byte2 [5->SPC-3] */
 #define DEF_SECTOR_SIZE 512
 #define DEF_UNMAP_ALIGNMENT 0
@@ -193,11 +194,11 @@ static unsigned int scsi_debug_unmap_granularity = DEF_UNMAP_GRANULARITY;
 static unsigned int scsi_debug_unmap_max_blocks = DEF_UNMAP_MAX_BLOCKS;
 static unsigned int scsi_debug_unmap_max_desc = DEF_UNMAP_MAX_DESC;
 static unsigned int scsi_debug_write_same_length = DEF_WRITESAME_LENGTH;
+static bool scsi_debug_removable = DEF_REMOVABLE;
 
 static int scsi_debug_cmnd_count = 0;
 
 #define DEV_READONLY(TGT)      (0)
-#define DEV_REMOVEABLE(TGT)    (0)
 
 static unsigned int sdebug_store_sectors;
 static sector_t sdebug_capacity;	/* in sectors */
@@ -919,7 +920,7 @@ static int resp_inquiry(struct scsi_cmnd * scp, int target,
 		return ret;
 	}
 	/* drops through here for a standard inquiry */
-	arr[1] = DEV_REMOVEABLE(target) ? 0x80 : 0;	/* Removable disk */
+	arr[1] = scsi_debug_removable ? 0x80 : 0;	/* Removable disk */
 	arr[2] = scsi_debug_scsi_level;
 	arr[3] = 2;    /* response_data_format==2 */
 	arr[4] = SDEBUG_LONG_INQ_SZ - 5;
@@ -1211,7 +1212,7 @@ static int resp_format_pg(unsigned char * p, int pcontrol, int target)
 	p[11] = sdebug_sectors_per & 0xff;
 	p[12] = (scsi_debug_sector_size >> 8) & 0xff;
 	p[13] = scsi_debug_sector_size & 0xff;
-	if (DEV_REMOVEABLE(target))
+	if (scsi_debug_removable)
 		p[20] |= 0x20; /* should agree with INQUIRY */
 	if (1 == pcontrol)
 		memset(p + 2, 0, sizeof(format_pg) - 2);
@@ -2754,6 +2755,7 @@ module_param_named(opt_blks, scsi_debug_opt_blks, int, S_IRUGO);
 module_param_named(opts, scsi_debug_opts, int, S_IRUGO | S_IWUSR);
 module_param_named(physblk_exp, scsi_debug_physblk_exp, int, S_IRUGO);
 module_param_named(ptype, scsi_debug_ptype, int, S_IRUGO | S_IWUSR);
+module_param_named(removable, scsi_debug_removable, bool, S_IRUGO | S_IWUSR);
 module_param_named(scsi_level, scsi_debug_scsi_level, int, S_IRUGO);
 module_param_named(sector_size, scsi_debug_sector_size, int, S_IRUGO);
 module_param_named(unmap_alignment, scsi_debug_unmap_alignment, int, S_IRUGO);
@@ -2796,6 +2798,7 @@ MODULE_PARM_DESC(opt_blks, "optimal transfer length in block (def=64)");
 MODULE_PARM_DESC(opts, "1->noise, 2->medium_err, 4->timeout, 8->recovered_err... (def=0)");
 MODULE_PARM_DESC(physblk_exp, "physical block exponent (def=0)");
 MODULE_PARM_DESC(ptype, "SCSI peripheral type(def=0[disk])");
+MODULE_PARM_DESC(removable, "claim to have removable media (def=0)");
 MODULE_PARM_DESC(scsi_level, "SCSI level to simulate(def=5[SPC-3])");
 MODULE_PARM_DESC(sector_size, "logical block size in bytes (def=512)");
 MODULE_PARM_DESC(unmap_alignment, "lowest aligned thin provisioning lba (def=0)");
@@ -3205,6 +3208,25 @@ static ssize_t sdebug_map_show(struct device_driver *ddp, char *buf)
 }
 DRIVER_ATTR(map, S_IRUGO, sdebug_map_show, NULL);
 
+static ssize_t sdebug_removable_show(struct device_driver *ddp,
+				     char *buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", scsi_debug_removable ? 1 : 0);
+}
+static ssize_t sdebug_removable_store(struct device_driver *ddp,
+				      const char *buf, size_t count)
+{
+	int n;
+
+	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
+		scsi_debug_removable = (n > 0);
+		return count;
+	}
+	return -EINVAL;
+}
+DRIVER_ATTR(removable, S_IRUGO | S_IWUSR, sdebug_removable_show,
+	    sdebug_removable_store);
+
 
 /* Note: The following function creates attribute files in the
    /sys/bus/pseudo/drivers/scsi_debug directory. The advantage of these
@@ -3230,6 +3252,7 @@ static int do_create_driverfs_files(void)
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_num_tgts);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_ptype);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_opts);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_removable);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_scsi_level);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_virtual_gb);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_vpd_use_hostno);
@@ -3255,6 +3278,7 @@ static void do_remove_driverfs_files(void)
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_scsi_level);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_opts);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_ptype);
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_removable);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_num_tgts);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_num_parts);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_no_uld);
diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c
index cf8dfab9489..43fca9170bf 100644
--- a/drivers/scsi/scsi_devinfo.c
+++ b/drivers/scsi/scsi_devinfo.c
@@ -172,6 +172,7 @@ static struct {
 	{"HITACHI", "DF400", "*", BLIST_REPORTLUN2},
 	{"HITACHI", "DF500", "*", BLIST_REPORTLUN2},
 	{"HITACHI", "DISK-SUBSYSTEM", "*", BLIST_REPORTLUN2},
+	{"HITACHI", "HUS1530", "*", BLIST_NO_DIF},
 	{"HITACHI", "OPEN-", "*", BLIST_REPORTLUN2},
 	{"HITACHI", "OP-C-", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
 	{"HITACHI", "3380-", "*", BLIST_SPARSELUN | BLIST_LARGELUN},
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index faa790fba13..da36a3a81a9 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -2473,7 +2473,8 @@ scsi_internal_device_unblock(struct scsi_device *sdev,
 	 * Try to transition the scsi device to SDEV_RUNNING or one of the
 	 * offlined states and goose the device queue if successful.
 	 */
-	if (sdev->sdev_state == SDEV_BLOCK)
+	if ((sdev->sdev_state == SDEV_BLOCK) ||
+	    (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
 		sdev->sdev_state = new_state;
 	else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
 		if (new_state == SDEV_TRANSPORT_OFFLINE ||
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index d947ffc20ce..3e58b2245f1 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -921,6 +921,9 @@ static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result,
 	if (*bflags & BLIST_RETRY_HWERROR)
 		sdev->retry_hwerror = 1;
 
+	if (*bflags & BLIST_NO_DIF)
+		sdev->no_dif = 1;
+
 	transport_configure_device(&sdev->sdev_gendev);
 
 	if (sdev->host->hostt->slave_configure) {
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index 093d4f6a54d..ce5224c92ed 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -1031,33 +1031,31 @@ static void __scsi_remove_target(struct scsi_target *starget)
 void scsi_remove_target(struct device *dev)
 {
 	struct Scsi_Host *shost = dev_to_shost(dev->parent);
-	struct scsi_target *starget, *found;
+	struct scsi_target *starget, *last = NULL;
 	unsigned long flags;
 
- restart:
-	found = NULL;
+	/* remove targets being careful to lookup next entry before
+	 * deleting the last
+	 */
 	spin_lock_irqsave(shost->host_lock, flags);
 	list_for_each_entry(starget, &shost->__targets, siblings) {
 		if (starget->state == STARGET_DEL)
 			continue;
 		if (starget->dev.parent == dev || &starget->dev == dev) {
-			found = starget;
-			found->reap_ref++;
-			break;
+			/* assuming new targets arrive at the end */
+			starget->reap_ref++;
+			spin_unlock_irqrestore(shost->host_lock, flags);
+			if (last)
+				scsi_target_reap(last);
+			last = starget;
+			__scsi_remove_target(starget);
+			spin_lock_irqsave(shost->host_lock, flags);
 		}
 	}
 	spin_unlock_irqrestore(shost->host_lock, flags);
 
-	if (found) {
-		__scsi_remove_target(found);
-		scsi_target_reap(found);
-		/* in the case where @dev has multiple starget children,
-		 * continue removing.
-		 *
-		 * FIXME: does such a case exist?
-		 */
-		goto restart;
-	}
+	if (last)
+		scsi_target_reap(last);
 }
 EXPORT_SYMBOL(scsi_remove_target);
 
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 4df73e52a4f..12f6fdfc114 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -262,6 +262,28 @@ sd_show_protection_type(struct device *dev, struct device_attribute *attr,
 }
 
 static ssize_t
+sd_store_protection_type(struct device *dev, struct device_attribute *attr,
+			 const char *buf, size_t count)
+{
+	struct scsi_disk *sdkp = to_scsi_disk(dev);
+	unsigned int val;
+	int err;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	err = kstrtouint(buf, 10, &val);
+
+	if (err)
+		return err;
+
+	if (val >= 0 && val <= SD_DIF_TYPE3_PROTECTION)
+		sdkp->protection_type = val;
+
+	return count;
+}
+
+static ssize_t
 sd_show_protection_mode(struct device *dev, struct device_attribute *attr,
 			char *buf)
 {
@@ -381,7 +403,8 @@ static struct device_attribute sd_disk_attrs[] = {
 	       sd_store_allow_restart),
 	__ATTR(manage_start_stop, S_IRUGO|S_IWUSR, sd_show_manage_start_stop,
 	       sd_store_manage_start_stop),
-	__ATTR(protection_type, S_IRUGO, sd_show_protection_type, NULL),
+	__ATTR(protection_type, S_IRUGO|S_IWUSR, sd_show_protection_type,
+	       sd_store_protection_type),
 	__ATTR(protection_mode, S_IRUGO, sd_show_protection_mode, NULL),
 	__ATTR(app_tag_own, S_IRUGO, sd_show_app_tag_own, NULL),
 	__ATTR(thin_provisioning, S_IRUGO, sd_show_thin_provisioning, NULL),
@@ -804,9 +827,8 @@ static int sd_prep_fn(struct request_queue *q, struct request *rq)
 		SCpnt->cmnd[0] = WRITE_6;
 		SCpnt->sc_data_direction = DMA_TO_DEVICE;
 
-		if (blk_integrity_rq(rq) &&
-		    sd_dif_prepare(rq, block, sdp->sector_size) == -EIO)
-			goto out;
+		if (blk_integrity_rq(rq))
+			sd_dif_prepare(rq, block, sdp->sector_size);
 
 	} else if (rq_data_dir(rq) == READ) {
 		SCpnt->cmnd[0] = READ_6;
@@ -1671,34 +1693,42 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 /*
  * Determine whether disk supports Data Integrity Field.
  */
-static void sd_read_protection_type(struct scsi_disk *sdkp, unsigned char *buffer)
+static int sd_read_protection_type(struct scsi_disk *sdkp, unsigned char *buffer)
 {
 	struct scsi_device *sdp = sdkp->device;
 	u8 type;
+	int ret = 0;
 
 	if (scsi_device_protection(sdp) == 0 || (buffer[12] & 1) == 0)
-		return;
+		return ret;
 
 	type = ((buffer[12] >> 1) & 7) + 1; /* P_TYPE 0 = Type 1 */
 
-	if (type == sdkp->protection_type || !sdkp->first_scan)
-		return;
+	if (type > SD_DIF_TYPE3_PROTECTION)
+		ret = -ENODEV;
+	else if (scsi_host_dif_capable(sdp->host, type))
+		ret = 1;
+
+	if (sdkp->first_scan || type != sdkp->protection_type)
+		switch (ret) {
+		case -ENODEV:
+			sd_printk(KERN_ERR, sdkp, "formatted with unsupported" \
+				  " protection type %u. Disabling disk!\n",
+				  type);
+			break;
+		case 1:
+			sd_printk(KERN_NOTICE, sdkp,
+				  "Enabling DIF Type %u protection\n", type);
+			break;
+		case 0:
+			sd_printk(KERN_NOTICE, sdkp,
+				  "Disabling DIF Type %u protection\n", type);
+			break;
+		}
 
 	sdkp->protection_type = type;
 
-	if (type > SD_DIF_TYPE3_PROTECTION) {
-		sd_printk(KERN_ERR, sdkp, "formatted with unsupported "	\
-			  "protection type %u. Disabling disk!\n", type);
-		sdkp->capacity = 0;
-		return;
-	}
-
-	if (scsi_host_dif_capable(sdp->host, type))
-		sd_printk(KERN_NOTICE, sdkp,
-			  "Enabling DIF Type %u protection\n", type);
-	else
-		sd_printk(KERN_NOTICE, sdkp,
-			  "Disabling DIF Type %u protection\n", type);
+	return ret;
 }
 
 static void read_capacity_error(struct scsi_disk *sdkp, struct scsi_device *sdp,
@@ -1794,7 +1824,10 @@ static int read_capacity_16(struct scsi_disk *sdkp, struct scsi_device *sdp,
 	sector_size = get_unaligned_be32(&buffer[8]);
 	lba = get_unaligned_be64(&buffer[0]);
 
-	sd_read_protection_type(sdkp, buffer);
+	if (sd_read_protection_type(sdkp, buffer) < 0) {
+		sdkp->capacity = 0;
+		return -ENODEV;
+	}
 
 	if ((sizeof(sdkp->capacity) == 4) && (lba >= 0xffffffffULL)) {
 		sd_printk(KERN_ERR, sdkp, "Too big for this kernel. Use a "
@@ -2632,7 +2665,8 @@ static void sd_probe_async(void *data, async_cookie_t cookie)
 	}
 
 	add_disk(gd);
-	sd_dif_config_host(sdkp);
+	if (sdkp->capacity)
+		sd_dif_config_host(sdkp);
 
 	sd_revalidate_disk(gd);
 
diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
index f703f4827b6..47c52a6d733 100644
--- a/drivers/scsi/sd.h
+++ b/drivers/scsi/sd.h
@@ -156,7 +156,7 @@ struct sd_dif_tuple {
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 
 extern void sd_dif_config_host(struct scsi_disk *);
-extern int sd_dif_prepare(struct request *rq, sector_t, unsigned int);
+extern void sd_dif_prepare(struct request *rq, sector_t, unsigned int);
 extern void sd_dif_complete(struct scsi_cmnd *, unsigned int);
 
 #else /* CONFIG_BLK_DEV_INTEGRITY */
diff --git a/drivers/scsi/sd_dif.c b/drivers/scsi/sd_dif.c
index e52d5bc42bc..04998f36e50 100644
--- a/drivers/scsi/sd_dif.c
+++ b/drivers/scsi/sd_dif.c
@@ -366,7 +366,8 @@ void sd_dif_config_host(struct scsi_disk *sdkp)
  *
  * Type 3 does not have a reference tag so no remapping is required.
  */
-int sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int sector_sz)
+void sd_dif_prepare(struct request *rq, sector_t hw_sector,
+		    unsigned int sector_sz)
 {
 	const int tuple_sz = sizeof(struct sd_dif_tuple);
 	struct bio *bio;
@@ -378,7 +379,7 @@ int sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int sector_s
 	sdkp = rq->bio->bi_bdev->bd_disk->private_data;
 
 	if (sdkp->protection_type == SD_DIF_TYPE3_PROTECTION)
-		return 0;
+		return;
 
 	phys = hw_sector & 0xffffffff;
 
@@ -397,10 +398,9 @@ int sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int sector_s
 
 			for (j = 0 ; j < iv->bv_len ; j += tuple_sz, sdt++) {
 
-				if (be32_to_cpu(sdt->ref_tag) != virt)
-					goto error;
+				if (be32_to_cpu(sdt->ref_tag) == virt)
+					sdt->ref_tag = cpu_to_be32(phys);
 
-				sdt->ref_tag = cpu_to_be32(phys);
 				virt++;
 				phys++;
 			}
@@ -410,16 +410,6 @@ int sd_dif_prepare(struct request *rq, sector_t hw_sector, unsigned int sector_s
 
 		bio->bi_flags |= (1 << BIO_MAPPED_INTEGRITY);
 	}
-
-	return 0;
-
-error:
-	kunmap_atomic(sdt);
-	sd_printk(KERN_ERR, sdkp, "%s: virt %u, phys %u, ref %u, app %4x\n",
-		  __func__, virt, phys, be32_to_cpu(sdt->ref_tag),
-		  be16_to_cpu(sdt->app_tag));
-
-	return -EILSEQ;
 }
 
 /*
@@ -463,10 +453,7 @@ void sd_dif_complete(struct scsi_cmnd *scmd, unsigned int good_bytes)
 					return;
 				}
 
-				if (be32_to_cpu(sdt->ref_tag) != phys &&
-				    sdt->app_tag != 0xffff)
-					sdt->ref_tag = 0xffffffff; /* Bad ref */
-				else
+				if (be32_to_cpu(sdt->ref_tag) == phys)
 					sdt->ref_tag = cpu_to_be32(virt);
 
 				virt++;
diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c
index e41998cb098..98156a97c47 100644
--- a/drivers/scsi/st.c
+++ b/drivers/scsi/st.c
@@ -37,6 +37,7 @@ static const char *verstr = "20101219";
 #include <linux/blkdev.h>
 #include <linux/moduleparam.h>
 #include <linux/cdev.h>
+#include <linux/idr.h>
 #include <linux/delay.h>
 #include <linux/mutex.h>
 
@@ -74,17 +75,14 @@ static const char *verstr = "20101219";
 #include "st_options.h"
 #include "st.h"
 
-static DEFINE_MUTEX(st_mutex);
 static int buffer_kbs;
 static int max_sg_segs;
 static int try_direct_io = TRY_DIRECT_IO;
 static int try_rdio = 1;
 static int try_wdio = 1;
 
-static int st_dev_max;
-static int st_nr_dev;
-
-static struct class *st_sysfs_class;
+static struct class st_sysfs_class;
+static struct device_attribute st_dev_attrs[];
 
 MODULE_AUTHOR("Kai Makisara");
 MODULE_DESCRIPTION("SCSI tape (st) driver");
@@ -173,13 +171,9 @@ static int debugging = DEBUG;
    24 bits) */
 #define SET_DENS_AND_BLK 0x10001
 
-static DEFINE_RWLOCK(st_dev_arr_lock);
-
 static int st_fixed_buffer_size = ST_FIXED_BUFFER_SIZE;
 static int st_max_sg_segs = ST_MAX_SG;
 
-static struct scsi_tape **scsi_tapes = NULL;
-
 static int modes_defined;
 
 static int enlarge_buffer(struct st_buffer *, int, int);
@@ -198,7 +192,6 @@ static int st_remove(struct device *);
 
 static int do_create_sysfs_files(void);
 static void do_remove_sysfs_files(void);
-static int do_create_class_files(struct scsi_tape *, int, int);
 
 static struct scsi_driver st_template = {
 	.owner			= THIS_MODULE,
@@ -221,6 +214,10 @@ static void scsi_tape_release(struct kref *);
 #define to_scsi_tape(obj) container_of(obj, struct scsi_tape, kref)
 
 static DEFINE_MUTEX(st_ref_mutex);
+static DEFINE_SPINLOCK(st_index_lock);
+static DEFINE_SPINLOCK(st_use_lock);
+static DEFINE_IDR(st_index_idr);
+
 
 
 #include "osst_detect.h"
@@ -238,10 +235,9 @@ static struct scsi_tape *scsi_tape_get(int dev)
 	struct scsi_tape *STp = NULL;
 
 	mutex_lock(&st_ref_mutex);
-	write_lock(&st_dev_arr_lock);
+	spin_lock(&st_index_lock);
 
-	if (dev < st_dev_max && scsi_tapes != NULL)
-		STp = scsi_tapes[dev];
+	STp = idr_find(&st_index_idr, dev);
 	if (!STp) goto out;
 
 	kref_get(&STp->kref);
@@ -258,7 +254,7 @@ out_put:
 	kref_put(&STp->kref, scsi_tape_release);
 	STp = NULL;
 out:
-	write_unlock(&st_dev_arr_lock);
+	spin_unlock(&st_index_lock);
 	mutex_unlock(&st_ref_mutex);
 	return STp;
 }
@@ -1188,7 +1184,6 @@ static int st_open(struct inode *inode, struct file *filp)
 	int dev = TAPE_NR(inode);
 	char *name;
 
-	mutex_lock(&st_mutex);
 	/*
 	 * We really want to do nonseekable_open(inode, filp); here, but some
 	 * versions of tar incorrectly call lseek on tapes and bail out if that
@@ -1197,24 +1192,22 @@ static int st_open(struct inode *inode, struct file *filp)
 	filp->f_mode &= ~(FMODE_PREAD | FMODE_PWRITE);
 
 	if (!(STp = scsi_tape_get(dev))) {
-		mutex_unlock(&st_mutex);
 		return -ENXIO;
 	}
 
-	write_lock(&st_dev_arr_lock);
 	filp->private_data = STp;
 	name = tape_name(STp);
 
+	spin_lock(&st_use_lock);
 	if (STp->in_use) {
-		write_unlock(&st_dev_arr_lock);
+		spin_unlock(&st_use_lock);
 		scsi_tape_put(STp);
-		mutex_unlock(&st_mutex);
 		DEB( printk(ST_DEB_MSG "%s: Device already in use.\n", name); )
 		return (-EBUSY);
 	}
 
 	STp->in_use = 1;
-	write_unlock(&st_dev_arr_lock);
+	spin_unlock(&st_use_lock);
 	STp->rew_at_close = STp->autorew_dev = (iminor(inode) & 0x80) == 0;
 
 	if (scsi_autopm_get_device(STp->device) < 0) {
@@ -1262,16 +1255,16 @@ static int st_open(struct inode *inode, struct file *filp)
 			retval = (-EIO);
 		goto err_out;
 	}
-	mutex_unlock(&st_mutex);
 	return 0;
 
  err_out:
 	normalize_buffer(STp->buffer);
+	spin_lock(&st_use_lock);
 	STp->in_use = 0;
+	spin_unlock(&st_use_lock);
 	scsi_tape_put(STp);
 	if (resumed)
 		scsi_autopm_put_device(STp->device);
-	mutex_unlock(&st_mutex);
 	return retval;
 
 }
@@ -1403,9 +1396,9 @@ static int st_release(struct inode *inode, struct file *filp)
 		do_door_lock(STp, 0);
 
 	normalize_buffer(STp->buffer);
-	write_lock(&st_dev_arr_lock);
+	spin_lock(&st_use_lock);
 	STp->in_use = 0;
-	write_unlock(&st_dev_arr_lock);
+	spin_unlock(&st_use_lock);
 	scsi_autopm_put_device(STp->device);
 	scsi_tape_put(STp);
 
@@ -3992,16 +3985,98 @@ static const struct file_operations st_fops =
 	.llseek =	noop_llseek,
 };
 
+static int create_one_cdev(struct scsi_tape *tape, int mode, int rew)
+{
+	int i, error;
+	dev_t cdev_devno;
+	struct cdev *cdev;
+	struct device *dev;
+	struct st_modedef *STm = &(tape->modes[mode]);
+	char name[10];
+	int dev_num = tape->index;
+
+	cdev_devno = MKDEV(SCSI_TAPE_MAJOR, TAPE_MINOR(dev_num, mode, rew));
+
+	cdev = cdev_alloc();
+	if (!cdev) {
+		pr_err("st%d: out of memory. Device not attached.\n", dev_num);
+		error = -ENOMEM;
+		goto out;
+	}
+	cdev->owner = THIS_MODULE;
+	cdev->ops = &st_fops;
+
+	error = cdev_add(cdev, cdev_devno, 1);
+	if (error) {
+		pr_err("st%d: Can't add %s-rewind mode %d\n", dev_num,
+		       rew ? "non" : "auto", mode);
+		pr_err("st%d: Device not attached.\n", dev_num);
+		goto out_free;
+	}
+	STm->cdevs[rew] = cdev;
+
+	i = mode << (4 - ST_NBR_MODE_BITS);
+	snprintf(name, 10, "%s%s%s", rew ? "n" : "",
+		 tape->disk->disk_name, st_formats[i]);
+
+	dev = device_create(&st_sysfs_class, &tape->device->sdev_gendev,
+			    cdev_devno, &tape->modes[mode], "%s", name);
+	if (IS_ERR(dev)) {
+		pr_err("st%d: device_create failed\n", dev_num);
+		error = PTR_ERR(dev);
+		goto out_free;
+	}
+
+	STm->devs[rew] = dev;
+
+	return 0;
+out_free:
+	cdev_del(STm->cdevs[rew]);
+	STm->cdevs[rew] = NULL;
+out:
+	return error;
+}
+
+static int create_cdevs(struct scsi_tape *tape)
+{
+	int mode, error;
+	for (mode = 0; mode < ST_NBR_MODES; ++mode) {
+		error = create_one_cdev(tape, mode, 0);
+		if (error)
+			return error;
+		error = create_one_cdev(tape, mode, 1);
+		if (error)
+			return error;
+	}
+
+	return sysfs_create_link(&tape->device->sdev_gendev.kobj,
+				 &tape->modes[0].devs[0]->kobj, "tape");
+}
+
+static void remove_cdevs(struct scsi_tape *tape)
+{
+	int mode, rew;
+	sysfs_remove_link(&tape->device->sdev_gendev.kobj, "tape");
+	for (mode = 0; mode < ST_NBR_MODES; mode++) {
+		struct st_modedef *STm = &(tape->modes[mode]);
+		for (rew = 0; rew < 2; rew++) {
+			if (STm->cdevs[rew])
+				cdev_del(STm->cdevs[rew]);
+			if (STm->devs[rew])
+				device_unregister(STm->devs[rew]);
+		}
+	}
+}
+
 static int st_probe(struct device *dev)
 {
 	struct scsi_device *SDp = to_scsi_device(dev);
 	struct gendisk *disk = NULL;
-	struct cdev *cdev = NULL;
 	struct scsi_tape *tpnt = NULL;
 	struct st_modedef *STm;
 	struct st_partstat *STps;
 	struct st_buffer *buffer;
-	int i, j, mode, dev_num, error;
+	int i, dev_num, error;
 	char *stp;
 
 	if (SDp->type != TYPE_TAPE)
@@ -4028,58 +4103,16 @@ static int st_probe(struct device *dev)
 		goto out_buffer_free;
 	}
 
-	write_lock(&st_dev_arr_lock);
-	if (st_nr_dev >= st_dev_max) {
-		struct scsi_tape **tmp_da;
-		int tmp_dev_max;
-
-		tmp_dev_max = max(st_nr_dev * 2, 8);
-		if (tmp_dev_max > ST_MAX_TAPES)
-			tmp_dev_max = ST_MAX_TAPES;
-		if (tmp_dev_max <= st_nr_dev) {
-			write_unlock(&st_dev_arr_lock);
-			printk(KERN_ERR "st: Too many tape devices (max. %d).\n",
-			       ST_MAX_TAPES);
-			goto out_put_disk;
-		}
-
-		tmp_da = kzalloc(tmp_dev_max * sizeof(struct scsi_tape *), GFP_ATOMIC);
-		if (tmp_da == NULL) {
-			write_unlock(&st_dev_arr_lock);
-			printk(KERN_ERR "st: Can't extend device array.\n");
-			goto out_put_disk;
-		}
-
-		if (scsi_tapes != NULL) {
-			memcpy(tmp_da, scsi_tapes,
-			       st_dev_max * sizeof(struct scsi_tape *));
-			kfree(scsi_tapes);
-		}
-		scsi_tapes = tmp_da;
-
-		st_dev_max = tmp_dev_max;
-	}
-
-	for (i = 0; i < st_dev_max; i++)
-		if (scsi_tapes[i] == NULL)
-			break;
-	if (i >= st_dev_max)
-		panic("scsi_devices corrupt (st)");
-
 	tpnt = kzalloc(sizeof(struct scsi_tape), GFP_ATOMIC);
 	if (tpnt == NULL) {
-		write_unlock(&st_dev_arr_lock);
 		printk(KERN_ERR "st: Can't allocate device descriptor.\n");
 		goto out_put_disk;
 	}
 	kref_init(&tpnt->kref);
 	tpnt->disk = disk;
-	sprintf(disk->disk_name, "st%d", i);
 	disk->private_data = &tpnt->driver;
 	disk->queue = SDp->request_queue;
 	tpnt->driver = &st_template;
-	scsi_tapes[i] = tpnt;
-	dev_num = i;
 
 	tpnt->device = SDp;
 	if (SDp->scsi_level <= 2)
@@ -4125,6 +4158,7 @@ static int st_probe(struct device *dev)
 		STm->default_compression = ST_DONT_TOUCH;
 		STm->default_blksize = (-1);	/* No forced size */
 		STm->default_density = (-1);	/* No forced density */
+		STm->tape = tpnt;
 	}
 
 	for (i = 0; i < ST_NBR_PARTITIONS; i++) {
@@ -4144,38 +4178,34 @@ static int st_probe(struct device *dev)
 	    tpnt->blksize_changed = 0;
 	mutex_init(&tpnt->lock);
 
-	st_nr_dev++;
-	write_unlock(&st_dev_arr_lock);
+	if (!idr_pre_get(&st_index_idr, GFP_KERNEL)) {
+		pr_warn("st: idr expansion failed\n");
+		error = -ENOMEM;
+		goto out_put_disk;
+	}
 
-	for (mode = 0; mode < ST_NBR_MODES; ++mode) {
-		STm = &(tpnt->modes[mode]);
-		for (j=0; j < 2; j++) {
-			cdev = cdev_alloc();
-			if (!cdev) {
-				printk(KERN_ERR
-				       "st%d: out of memory. Device not attached.\n",
-				       dev_num);
-				goto out_free_tape;
-			}
-			cdev->owner = THIS_MODULE;
-			cdev->ops = &st_fops;
-
-			error = cdev_add(cdev,
-					 MKDEV(SCSI_TAPE_MAJOR, TAPE_MINOR(dev_num, mode, j)),
-					 1);
-			if (error) {
-				printk(KERN_ERR "st%d: Can't add %s-rewind mode %d\n",
-				       dev_num, j ? "non" : "auto", mode);
-				printk(KERN_ERR "st%d: Device not attached.\n", dev_num);
-				goto out_free_tape;
-			}
-			STm->cdevs[j] = cdev;
+	spin_lock(&st_index_lock);
+	error = idr_get_new(&st_index_idr, tpnt, &dev_num);
+	spin_unlock(&st_index_lock);
+	if (error) {
+		pr_warn("st: idr allocation failed: %d\n", error);
+		goto out_put_disk;
+	}
 
-		}
-		error = do_create_class_files(tpnt, dev_num, mode);
-		if (error)
-			goto out_free_tape;
+	if (dev_num > ST_MAX_TAPES) {
+		pr_err("st: Too many tape devices (max. %d).\n", ST_MAX_TAPES);
+		goto out_put_index;
 	}
+
+	tpnt->index = dev_num;
+	sprintf(disk->disk_name, "st%d", dev_num);
+
+	dev_set_drvdata(dev, tpnt);
+
+
+	error = create_cdevs(tpnt);
+	if (error)
+		goto out_remove_devs;
 	scsi_autopm_put_device(SDp);
 
 	sdev_printk(KERN_NOTICE, SDp,
@@ -4186,28 +4216,12 @@ static int st_probe(struct device *dev)
 
 	return 0;
 
-out_free_tape:
-	for (mode=0; mode < ST_NBR_MODES; mode++) {
-		STm = &(tpnt->modes[mode]);
-		sysfs_remove_link(&tpnt->device->sdev_gendev.kobj,
-				  "tape");
-		for (j=0; j < 2; j++) {
-			if (STm->cdevs[j]) {
-				if (cdev == STm->cdevs[j])
-					cdev = NULL;
-					device_destroy(st_sysfs_class,
-						       MKDEV(SCSI_TAPE_MAJOR,
-							     TAPE_MINOR(i, mode, j)));
-				cdev_del(STm->cdevs[j]);
-			}
-		}
-	}
-	if (cdev)
-		cdev_del(cdev);
-	write_lock(&st_dev_arr_lock);
-	scsi_tapes[dev_num] = NULL;
-	st_nr_dev--;
-	write_unlock(&st_dev_arr_lock);
+out_remove_devs:
+	remove_cdevs(tpnt);
+out_put_index:
+	spin_lock(&st_index_lock);
+	idr_remove(&st_index_idr, dev_num);
+	spin_unlock(&st_index_lock);
 out_put_disk:
 	put_disk(disk);
 	kfree(tpnt);
@@ -4220,38 +4234,18 @@ out:
 
 static int st_remove(struct device *dev)
 {
-	struct scsi_device *SDp = to_scsi_device(dev);
-	struct scsi_tape *tpnt;
-	int i, j, mode;
-
-	scsi_autopm_get_device(SDp);
-	write_lock(&st_dev_arr_lock);
-	for (i = 0; i < st_dev_max; i++) {
-		tpnt = scsi_tapes[i];
-		if (tpnt != NULL && tpnt->device == SDp) {
-			scsi_tapes[i] = NULL;
-			st_nr_dev--;
-			write_unlock(&st_dev_arr_lock);
-			sysfs_remove_link(&tpnt->device->sdev_gendev.kobj,
-					  "tape");
-			for (mode = 0; mode < ST_NBR_MODES; ++mode) {
-				for (j=0; j < 2; j++) {
-					device_destroy(st_sysfs_class,
-						       MKDEV(SCSI_TAPE_MAJOR,
-							     TAPE_MINOR(i, mode, j)));
-					cdev_del(tpnt->modes[mode].cdevs[j]);
-					tpnt->modes[mode].cdevs[j] = NULL;
-				}
-			}
+	struct scsi_tape *tpnt = dev_get_drvdata(dev);
+	int index = tpnt->index;
 
-			mutex_lock(&st_ref_mutex);
-			kref_put(&tpnt->kref, scsi_tape_release);
-			mutex_unlock(&st_ref_mutex);
-			return 0;
-		}
-	}
+	scsi_autopm_get_device(to_scsi_device(dev));
+	remove_cdevs(tpnt);
 
-	write_unlock(&st_dev_arr_lock);
+	mutex_lock(&st_ref_mutex);
+	kref_put(&tpnt->kref, scsi_tape_release);
+	mutex_unlock(&st_ref_mutex);
+	spin_lock(&st_index_lock);
+	idr_remove(&st_index_idr, index);
+	spin_unlock(&st_index_lock);
 	return 0;
 }
 
@@ -4283,6 +4277,11 @@ static void scsi_tape_release(struct kref *kref)
 	return;
 }
 
+static struct class st_sysfs_class = {
+	.name = "scsi_tape",
+	.dev_attrs = st_dev_attrs,
+};
+
 static int __init init_st(void)
 {
 	int err;
@@ -4292,10 +4291,10 @@ static int __init init_st(void)
 	printk(KERN_INFO "st: Version %s, fixed bufsize %d, s/g segs %d\n",
 		verstr, st_fixed_buffer_size, st_max_sg_segs);
 
-	st_sysfs_class = class_create(THIS_MODULE, "scsi_tape");
-	if (IS_ERR(st_sysfs_class)) {
-		printk(KERN_ERR "Unable create sysfs class for SCSI tapes\n");
-		return PTR_ERR(st_sysfs_class);
+	err = class_register(&st_sysfs_class);
+	if (err) {
+		pr_err("Unable register sysfs class for SCSI tapes\n");
+		return err;
 	}
 
 	err = register_chrdev_region(MKDEV(SCSI_TAPE_MAJOR, 0),
@@ -4322,7 +4321,7 @@ err_chrdev:
 	unregister_chrdev_region(MKDEV(SCSI_TAPE_MAJOR, 0),
 				 ST_MAX_TAPE_ENTRIES);
 err_class:
-	class_destroy(st_sysfs_class);
+	class_unregister(&st_sysfs_class);
 	return err;
 }
 
@@ -4332,8 +4331,7 @@ static void __exit exit_st(void)
 	scsi_unregister_driver(&st_template.gendrv);
 	unregister_chrdev_region(MKDEV(SCSI_TAPE_MAJOR, 0),
 				 ST_MAX_TAPE_ENTRIES);
-	class_destroy(st_sysfs_class);
-	kfree(scsi_tapes);
+	class_unregister(&st_sysfs_class);
 	printk(KERN_INFO "st: Unloaded.\n");
 }
 
@@ -4405,10 +4403,9 @@ static void do_remove_sysfs_files(void)
 	driver_remove_file(sysfs, &driver_attr_try_direct_io);
 }
 
-
 /* The sysfs simple class interface */
 static ssize_t
-st_defined_show(struct device *dev, struct device_attribute *attr, char *buf)
+defined_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct st_modedef *STm = dev_get_drvdata(dev);
 	ssize_t l = 0;
@@ -4417,10 +4414,9 @@ st_defined_show(struct device *dev, struct device_attribute *attr, char *buf)
 	return l;
 }
 
-DEVICE_ATTR(defined, S_IRUGO, st_defined_show, NULL);
-
 static ssize_t
-st_defblk_show(struct device *dev, struct device_attribute *attr, char *buf)
+default_blksize_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
 {
 	struct st_modedef *STm = dev_get_drvdata(dev);
 	ssize_t l = 0;
@@ -4429,10 +4425,10 @@ st_defblk_show(struct device *dev, struct device_attribute *attr, char *buf)
 	return l;
 }
 
-DEVICE_ATTR(default_blksize, S_IRUGO, st_defblk_show, NULL);
 
 static ssize_t
-st_defdensity_show(struct device *dev, struct device_attribute *attr, char *buf)
+default_density_show(struct device *dev, struct device_attribute *attr,
+		     char *buf)
 {
 	struct st_modedef *STm = dev_get_drvdata(dev);
 	ssize_t l = 0;
@@ -4443,11 +4439,9 @@ st_defdensity_show(struct device *dev, struct device_attribute *attr, char *buf)
 	return l;
 }
 
-DEVICE_ATTR(default_density, S_IRUGO, st_defdensity_show, NULL);
-
 static ssize_t
-st_defcompression_show(struct device *dev, struct device_attribute *attr,
-		       char *buf)
+default_compression_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
 {
 	struct st_modedef *STm = dev_get_drvdata(dev);
 	ssize_t l = 0;
@@ -4456,28 +4450,14 @@ st_defcompression_show(struct device *dev, struct device_attribute *attr,
 	return l;
 }
 
-DEVICE_ATTR(default_compression, S_IRUGO, st_defcompression_show, NULL);
-
 static ssize_t
-st_options_show(struct device *dev, struct device_attribute *attr, char *buf)
+options_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct st_modedef *STm = dev_get_drvdata(dev);
-	struct scsi_tape *STp;
-	int i, j, options;
+	struct scsi_tape *STp = STm->tape;
+	int options;
 	ssize_t l = 0;
 
-	for (i=0; i < st_dev_max; i++) {
-		for (j=0; j < ST_NBR_MODES; j++)
-			if (&scsi_tapes[i]->modes[j] == STm)
-				break;
-		if (j < ST_NBR_MODES)
-			break;
-	}
-	if (i == st_dev_max)
-		return 0;  /* should never happen */
-
-	STp = scsi_tapes[i];
-
 	options = STm->do_buffer_writes ? MT_ST_BUFFER_WRITES : 0;
 	options |= STm->do_async_writes ? MT_ST_ASYNC_WRITES : 0;
 	options |= STm->do_read_ahead ? MT_ST_READ_AHEAD : 0;
@@ -4498,66 +4478,14 @@ st_options_show(struct device *dev, struct device_attribute *attr, char *buf)
 	return l;
 }
 
-DEVICE_ATTR(options, S_IRUGO, st_options_show, NULL);
-
-static int do_create_class_files(struct scsi_tape *STp, int dev_num, int mode)
-{
-	int i, rew, error;
-	char name[10];
-	struct device *st_class_member;
-
-	for (rew=0; rew < 2; rew++) {
-		/* Make sure that the minor numbers corresponding to the four
-		   first modes always get the same names */
-		i = mode << (4 - ST_NBR_MODE_BITS);
-		snprintf(name, 10, "%s%s%s", rew ? "n" : "",
-			 STp->disk->disk_name, st_formats[i]);
-		st_class_member =
-			device_create(st_sysfs_class, &STp->device->sdev_gendev,
-				      MKDEV(SCSI_TAPE_MAJOR,
-					    TAPE_MINOR(dev_num, mode, rew)),
-				      &STp->modes[mode], "%s", name);
-		if (IS_ERR(st_class_member)) {
-			printk(KERN_WARNING "st%d: device_create failed\n",
-			       dev_num);
-			error = PTR_ERR(st_class_member);
-			goto out;
-		}
-
-		error = device_create_file(st_class_member,
-					   &dev_attr_defined);
-		if (error) goto out;
-		error = device_create_file(st_class_member,
-					   &dev_attr_default_blksize);
-		if (error) goto out;
-		error = device_create_file(st_class_member,
-					   &dev_attr_default_density);
-		if (error) goto out;
-		error = device_create_file(st_class_member,
-					   &dev_attr_default_compression);
-		if (error) goto out;
-		error = device_create_file(st_class_member,
-					   &dev_attr_options);
-		if (error) goto out;
-
-		if (mode == 0 && rew == 0) {
-			error = sysfs_create_link(&STp->device->sdev_gendev.kobj,
-						  &st_class_member->kobj,
-						  "tape");
-			if (error) {
-				printk(KERN_ERR
-				       "st%d: Can't create sysfs link from SCSI device.\n",
-				       dev_num);
-				goto out;
-			}
-		}
-	}
-
-	return 0;
-
-out:
-	return error;
-}
+static struct device_attribute st_dev_attrs[] = {
+	__ATTR_RO(defined),
+	__ATTR_RO(default_blksize),
+	__ATTR_RO(default_density),
+	__ATTR_RO(default_compression),
+	__ATTR_RO(options),
+	__ATTR_NULL,
+};
 
 /* The following functions may be useful for a larger audience. */
 static int sgl_map_user_pages(struct st_buffer *STbp,
diff --git a/drivers/scsi/st.h b/drivers/scsi/st.h
index b548923785e..f3eee0f9f40 100644
--- a/drivers/scsi/st.h
+++ b/drivers/scsi/st.h
@@ -66,6 +66,8 @@ struct st_modedef {
 	unsigned char default_compression;	/* 0 = don't touch, etc */
 	short default_density;	/* Forced density, -1 = no value */
 	int default_blksize;	/* Forced blocksize, -1 = no value */
+	struct scsi_tape *tape;
+	struct device *devs[2];  /* Auto-rewind and non-rewind devices */
 	struct cdev *cdevs[2];  /* Auto-rewind and non-rewind devices */
 };
 
@@ -76,7 +78,7 @@ struct st_modedef {
 #define ST_MODE_SHIFT (7 - ST_NBR_MODE_BITS)
 #define ST_MODE_MASK ((ST_NBR_MODES - 1) << ST_MODE_SHIFT)
 
-#define ST_MAX_TAPES 128
+#define ST_MAX_TAPES (1 << (20 - (ST_NBR_MODE_BITS + 1)))
 #define ST_MAX_TAPE_ENTRIES  (ST_MAX_TAPES << (ST_NBR_MODE_BITS + 1))
 
 /* The status related to each partition */
@@ -99,6 +101,7 @@ struct scsi_tape {
 	struct mutex lock;	/* For serialization */
 	struct completion wait;	/* For SCSI commands */
 	struct st_buffer *buffer;
+	int index;
 
 	/* Drive characteristics */
 	unsigned char omit_blklims;
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 1860c3aca7e..1a81c90a4a7 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -835,9 +835,7 @@ static struct s3c64xx_spi_csinfo *s3c64xx_get_slave_ctrldata(
 		return ERR_PTR(-EINVAL);
 	}
 
-	for_each_child_of_node(slave_np, data_np)
-		if (!strcmp(data_np->name, "controller-data"))
-			break;
+	data_np = of_get_child_by_name(slave_np, "controller-data");
 	if (!data_np) {
 		dev_err(&spi->dev, "child node 'controller-data' not found\n");
 		return ERR_PTR(-EINVAL);
@@ -847,6 +845,7 @@ static struct s3c64xx_spi_csinfo *s3c64xx_get_slave_ctrldata(
 	if (!cs) {
 		dev_err(&spi->dev, "could not allocate memory for controller"
 					" data\n");
+		of_node_put(data_np);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -855,11 +854,13 @@ static struct s3c64xx_spi_csinfo *s3c64xx_get_slave_ctrldata(
 		dev_err(&spi->dev, "chip select gpio is not specified or "
 					"invalid\n");
 		kfree(cs);
+		of_node_put(data_np);
 		return ERR_PTR(-EINVAL);
 	}
 
 	of_property_read_u32(data_np, "samsung,spi-feedback-delay", &fb_delay);
 	cs->fb_delay = fb_delay;
+	of_node_put(data_np);
 	return cs;
 }
 
diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c
index b1937ca1357..7b0ba92e7e4 100644
--- a/drivers/staging/android/binder.c
+++ b/drivers/staging/android/binder.c
@@ -362,71 +362,22 @@ struct binder_transaction {
 static void
 binder_defer_work(struct binder_proc *proc, enum binder_deferred_state defer);
 
-/*
- * copied from get_unused_fd_flags
- */
 static int task_get_unused_fd_flags(struct binder_proc *proc, int flags)
 {
 	struct files_struct *files = proc->files;
-	int fd, error;
-	struct fdtable *fdt;
 	unsigned long rlim_cur;
 	unsigned long irqs;
 
 	if (files == NULL)
 		return -ESRCH;
 
-	error = -EMFILE;
-	spin_lock(&files->file_lock);
-
-repeat:
-	fdt = files_fdtable(files);
-	fd = find_next_zero_bit(fdt->open_fds, fdt->max_fds, files->next_fd);
-
-	/*
-	 * N.B. For clone tasks sharing a files structure, this test
-	 * will limit the total number of files that can be opened.
-	 */
-	rlim_cur = 0;
-	if (lock_task_sighand(proc->tsk, &irqs)) {
-		rlim_cur = proc->tsk->signal->rlim[RLIMIT_NOFILE].rlim_cur;
-		unlock_task_sighand(proc->tsk, &irqs);
-	}
-	if (fd >= rlim_cur)
-		goto out;
-
-	/* Do we need to expand the fd array or fd set?  */
-	error = expand_files(files, fd);
-	if (error < 0)
-		goto out;
-
-	if (error) {
-		/*
-		 * If we needed to expand the fs array we
-		 * might have blocked - try again.
-		 */
-		error = -EMFILE;
-		goto repeat;
-	}
-
-	__set_open_fd(fd, fdt);
-	if (flags & O_CLOEXEC)
-		__set_close_on_exec(fd, fdt);
-	else
-		__clear_close_on_exec(fd, fdt);
-	files->next_fd = fd + 1;
-
-	/* Sanity check */
-	if (fdt->fd[fd] != NULL) {
-		pr_warn("get_unused_fd: slot %d not NULL!\n", fd);
-		fdt->fd[fd] = NULL;
-	}
+	if (!lock_task_sighand(proc->tsk, &irqs))
+		return -EMFILE;
 
-	error = fd;
+	rlim_cur = task_rlimit(proc->tsk, RLIMIT_NOFILE);
+	unlock_task_sighand(proc->tsk, &irqs);
 
-out:
-	spin_unlock(&files->file_lock);
-	return error;
+	return __alloc_fd(files, 0, rlim_cur, flags);
 }
 
 /*
@@ -435,28 +386,8 @@ out:
 static void task_fd_install(
 	struct binder_proc *proc, unsigned int fd, struct file *file)
 {
-	struct files_struct *files = proc->files;
-	struct fdtable *fdt;
-
-	if (files == NULL)
-		return;
-
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	BUG_ON(fdt->fd[fd] != NULL);
-	rcu_assign_pointer(fdt->fd[fd], file);
-	spin_unlock(&files->file_lock);
-}
-
-/*
- * copied from __put_unused_fd in open.c
- */
-static void __put_unused_fd(struct files_struct *files, unsigned int fd)
-{
-	struct fdtable *fdt = files_fdtable(files);
-	__clear_open_fd(fd, fdt);
-	if (fd < files->next_fd)
-		files->next_fd = fd;
+	if (proc->files)
+		__fd_install(proc->files, fd, file);
 }
 
 /*
@@ -464,27 +395,12 @@ static void __put_unused_fd(struct files_struct *files, unsigned int fd)
  */
 static long task_close_fd(struct binder_proc *proc, unsigned int fd)
 {
-	struct file *filp;
-	struct files_struct *files = proc->files;
-	struct fdtable *fdt;
 	int retval;
 
-	if (files == NULL)
+	if (proc->files == NULL)
 		return -ESRCH;
 
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	if (fd >= fdt->max_fds)
-		goto out_unlock;
-	filp = fdt->fd[fd];
-	if (!filp)
-		goto out_unlock;
-	rcu_assign_pointer(fdt->fd[fd], NULL);
-	__clear_close_on_exec(fd, fdt);
-	__put_unused_fd(files, fd);
-	spin_unlock(&files->file_lock);
-	retval = filp_close(filp, files);
-
+	retval = __close_fd(proc->files, fd);
 	/* can't restart close syscall because file table entry was cleared */
 	if (unlikely(retval == -ERESTARTSYS ||
 		     retval == -ERESTARTNOINTR ||
@@ -493,10 +409,6 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd)
 		retval = -EINTR;
 
 	return retval;
-
-out_unlock:
-	spin_unlock(&files->file_lock);
-	return -EBADF;
 }
 
 static void binder_set_nice(long nice)
@@ -2793,6 +2705,9 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 	const char *failure_string;
 	struct binder_buffer *buffer;
 
+	if (proc->tsk != current)
+		return -EINVAL;
+
 	if ((vma->vm_end - vma->vm_start) > SZ_4M)
 		vma->vm_end = vma->vm_start + SZ_4M;
 
@@ -2857,7 +2772,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 	binder_insert_free_buffer(proc, buffer);
 	proc->free_async_space = proc->buffer_size / 2;
 	barrier();
-	proc->files = get_files_struct(proc->tsk);
+	proc->files = get_files_struct(current);
 	proc->vma = vma;
 	proc->vma_vm_mm = vma->vm_mm;
 
diff --git a/drivers/staging/dgrp/dgrp_common.c b/drivers/staging/dgrp/dgrp_common.c
index fce74e7fb83..3553998b72b 100644
--- a/drivers/staging/dgrp/dgrp_common.c
+++ b/drivers/staging/dgrp/dgrp_common.c
@@ -179,9 +179,9 @@ void dgrp_carrier(struct ch_struct *ch)
  */
 int dgrp_chk_perm(int mode, int op)
 {
-	if (!current_euid())
+	if (!uid_eq(GLOBAL_ROOT_UID, current_euid()))
 		mode >>= 6;
-	else if (in_egroup_p(0))
+	else if (in_egroup_p(GLOBAL_ROOT_GID))
 		mode >>= 3;
 
 	if ((mode & op & 0007) == op)
diff --git a/drivers/staging/omapdrm/omap_connector.c b/drivers/staging/omapdrm/omap_connector.c
index 55e9c865585..38be186c249 100644
--- a/drivers/staging/omapdrm/omap_connector.c
+++ b/drivers/staging/omapdrm/omap_connector.c
@@ -200,14 +200,11 @@ static int omap_connector_get_modes(struct drm_connector *connector)
 			drm_mode_connector_update_edid_property(
 					connector, edid);
 			n = drm_add_edid_modes(connector, edid);
-			kfree(connector->display_info.raw_edid);
-			connector->display_info.raw_edid = edid;
 		} else {
 			drm_mode_connector_update_edid_property(
 					connector, NULL);
-			connector->display_info.raw_edid = NULL;
-			kfree(edid);
 		}
+		kfree(edid);
 	} else {
 		struct drm_display_mode *mode = drm_mode_create(dev);
 		struct omap_video_timings timings = {0};
diff --git a/drivers/staging/omapdrm/omap_fbdev.c b/drivers/staging/omapdrm/omap_fbdev.c
index 8c6ed3b0c6f..8a027bb77d9 100644
--- a/drivers/staging/omapdrm/omap_fbdev.c
+++ b/drivers/staging/omapdrm/omap_fbdev.c
@@ -276,7 +276,7 @@ fail:
 		if (fbi)
 			framebuffer_release(fbi);
 		if (fb)
-			fb->funcs->destroy(fb);
+			drm_framebuffer_remove(fb);
 	}
 
 	return ret;
@@ -401,7 +401,7 @@ void omap_fbdev_free(struct drm_device *dev)
 
 	/* this will free the backing object */
 	if (fbdev->fb)
-		fbdev->fb->funcs->destroy(fbdev->fb);
+		drm_framebuffer_remove(fbdev->fb);
 
 	kfree(fbdev);
 
diff --git a/drivers/staging/omapdrm/omap_gem.c b/drivers/staging/omapdrm/omap_gem.c
index c8287438e0d..3434e6ec014 100644
--- a/drivers/staging/omapdrm/omap_gem.c
+++ b/drivers/staging/omapdrm/omap_gem.c
@@ -592,9 +592,8 @@ int omap_gem_mmap_obj(struct drm_gem_object *obj,
 		 * in particular in the case of mmap'd dmabufs)
 		 */
 		fput(vma->vm_file);
-		get_file(obj->filp);
 		vma->vm_pgoff = 0;
-		vma->vm_file  = obj->filp;
+		vma->vm_file  = get_file(obj->filp);
 
 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
 	}
diff --git a/drivers/thermal/thermal_sys.c b/drivers/thermal/thermal_sys.c
index 67789b8345d..efd81bb25e0 100644
--- a/drivers/thermal/thermal_sys.c
+++ b/drivers/thermal/thermal_sys.c
@@ -78,7 +78,7 @@ again:
 	else if (unlikely(err))
 		return err;
 
-	*id = *id & MAX_ID_MASK;
+	*id = *id & MAX_IDR_MASK;
 	return 0;
 }
 
diff --git a/drivers/tty/hvc/hvc_xen.c b/drivers/tty/hvc/hvc_xen.c
index 1e456dca4f6..2944ff88fdc 100644
--- a/drivers/tty/hvc/hvc_xen.c
+++ b/drivers/tty/hvc/hvc_xen.c
@@ -21,6 +21,7 @@
 #include <linux/console.h>
 #include <linux/delay.h>
 #include <linux/err.h>
+#include <linux/irq.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/list.h>
@@ -35,6 +36,7 @@
 #include <xen/page.h>
 #include <xen/events.h>
 #include <xen/interface/io/console.h>
+#include <xen/interface/sched.h>
 #include <xen/hvc-console.h>
 #include <xen/xenbus.h>
 
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 8a5a8b06461..2ea176b2280 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -1166,10 +1166,8 @@ ssize_t redirected_tty_write(struct file *file, const char __user *buf,
 	struct file *p = NULL;
 
 	spin_lock(&redirect_lock);
-	if (redirect) {
-		get_file(redirect);
-		p = redirect;
-	}
+	if (redirect)
+		p = get_file(redirect);
 	spin_unlock(&redirect_lock);
 
 	if (p) {
@@ -2264,8 +2262,7 @@ static int tioccons(struct file *file)
 		spin_unlock(&redirect_lock);
 		return -EBUSY;
 	}
-	get_file(file);
-	redirect = file;
+	redirect = get_file(file);
 	spin_unlock(&redirect_lock);
 	return 0;
 }
@@ -2809,6 +2806,13 @@ static long tty_compat_ioctl(struct file *file, unsigned int cmd,
 }
 #endif
 
+static int this_tty(const void *t, struct file *file, unsigned fd)
+{
+	if (likely(file->f_op->read != tty_read))
+		return 0;
+	return file_tty(file) != t ? 0 : fd + 1;
+}
+	
 /*
  * This implements the "Secure Attention Key" ---  the idea is to
  * prevent trojan horses by killing all processes associated with this
@@ -2836,8 +2840,6 @@ void __do_SAK(struct tty_struct *tty)
 	struct task_struct *g, *p;
 	struct pid *session;
 	int		i;
-	struct file	*filp;
-	struct fdtable *fdt;
 
 	if (!tty)
 		return;
@@ -2867,27 +2869,12 @@ void __do_SAK(struct tty_struct *tty)
 			continue;
 		}
 		task_lock(p);
-		if (p->files) {
-			/*
-			 * We don't take a ref to the file, so we must
-			 * hold ->file_lock instead.
-			 */
-			spin_lock(&p->files->file_lock);
-			fdt = files_fdtable(p->files);
-			for (i = 0; i < fdt->max_fds; i++) {
-				filp = fcheck_files(p->files, i);
-				if (!filp)
-					continue;
-				if (filp->f_op->read == tty_read &&
-				    file_tty(filp) == tty) {
-					printk(KERN_NOTICE "SAK: killed process %d"
-					    " (%s): fd#%d opened to the tty\n",
-					    task_pid_nr(p), p->comm, i);
-					force_sig(SIGKILL, p);
-					break;
-				}
-			}
-			spin_unlock(&p->files->file_lock);
+		i = iterate_fd(p->files, 0, this_tty, tty);
+		if (i != 0) {
+			printk(KERN_NOTICE "SAK: killed process %d"
+			    " (%s): fd#%d opened to the tty\n",
+				    task_pid_nr(p), p->comm, i - 1);
+			force_sig(SIGKILL, p);
 		}
 		task_unlock(p);
 	} while_each_thread(g, p);
diff --git a/drivers/usb/gadget/f_fs.c b/drivers/usb/gadget/f_fs.c
index a26c43a151f..64c4ec10d1f 100644
--- a/drivers/usb/gadget/f_fs.c
+++ b/drivers/usb/gadget/f_fs.c
@@ -340,7 +340,7 @@ ffs_sb_create_file(struct super_block *sb, const char *name, void *data,
 
 static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
 	__attribute__((warn_unused_result, nonnull));
-static char *ffs_prepare_buffer(const char * __user buf, size_t len)
+static char *ffs_prepare_buffer(const char __user *buf, size_t len)
 	__attribute__((warn_unused_result, nonnull));
 
 
@@ -2445,7 +2445,7 @@ static int ffs_mutex_lock(struct mutex *mutex, unsigned nonblock)
 		: mutex_lock_interruptible(mutex);
 }
 
-static char *ffs_prepare_buffer(const char * __user buf, size_t len)
+static char *ffs_prepare_buffer(const char __user *buf, size_t len)
 {
 	char *data;
 
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index 17830c9c7cc..56097c6d072 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1014,7 +1014,7 @@ static void vfio_group_try_dissolve_container(struct vfio_group *group)
 
 static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 {
-	struct file *filep;
+	struct fd f;
 	struct vfio_container *container;
 	struct vfio_iommu_driver *driver;
 	int ret = 0;
@@ -1022,17 +1022,17 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 	if (atomic_read(&group->container_users))
 		return -EINVAL;
 
-	filep = fget(container_fd);
-	if (!filep)
+	f = fdget(container_fd);
+	if (!f.file)
 		return -EBADF;
 
 	/* Sanity check, is this really our fd? */
-	if (filep->f_op != &vfio_fops) {
-		fput(filep);
+	if (f.file->f_op != &vfio_fops) {
+		fdput(f);
 		return -EINVAL;
 	}
 
-	container = filep->private_data;
+	container = f.file->private_data;
 	WARN_ON(!container); /* fget ensures we don't race vfio_release */
 
 	mutex_lock(&container->group_lock);
@@ -1054,8 +1054,7 @@ static int vfio_group_set_container(struct vfio_group *group, int container_fd)
 
 unlock_out:
 	mutex_unlock(&container->group_lock);
-	fput(filep);
-
+	fdput(f);
 	return ret;
 }
 
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index ef82a0d1848..99ac2cb08b4 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -636,8 +636,8 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
 
 static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
 {
-	struct file *eventfp, *filep = NULL,
-		    *pollstart = NULL, *pollstop = NULL;
+	struct file *eventfp, *filep = NULL;
+	bool pollstart = false, pollstop = false;
 	struct eventfd_ctx *ctx = NULL;
 	u32 __user *idxp = argp;
 	struct vhost_virtqueue *vq;
@@ -763,8 +763,8 @@ static long vhost_set_vring(struct vhost_dev *d, int ioctl, void __user *argp)
 			break;
 		}
 		if (eventfp != vq->kick) {
-			pollstop = filep = vq->kick;
-			pollstart = vq->kick = eventfp;
+			pollstop = (filep = vq->kick) != NULL;
+			pollstart = (vq->kick = eventfp) != NULL;
 		} else
 			filep = eventfp;
 		break;
diff --git a/drivers/video/aty/aty128fb.c b/drivers/video/aty/aty128fb.c
index 747442d2c0f..0fefa84ed9a 100644
--- a/drivers/video/aty/aty128fb.c
+++ b/drivers/video/aty/aty128fb.c
@@ -149,7 +149,7 @@ enum {
 };
 
 /* Must match above enum */
-static const char *r128_family[] __devinitdata = {
+static char * const r128_family[] __devinitconst = {
 	"AGP",
 	"PCI",
 	"PRO AGP",
diff --git a/drivers/video/backlight/88pm860x_bl.c b/drivers/video/backlight/88pm860x_bl.c
index f49181c7311..b7ec34c57f4 100644
--- a/drivers/video/backlight/88pm860x_bl.c
+++ b/drivers/video/backlight/88pm860x_bl.c
@@ -11,6 +11,7 @@
 
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/fb.h>
@@ -31,57 +32,26 @@ struct pm860x_backlight_data {
 	int	port;
 	int	pwm;
 	int	iset;
+	int	reg_duty_cycle;
+	int	reg_always_on;
+	int	reg_current;
 };
 
-static inline int wled_a(int port)
-{
-	int ret;
-
-	ret = ((port - PM8606_BACKLIGHT1) << 1) + 2;
-	return ret;
-}
-
-static inline int wled_b(int port)
-{
-	int ret;
-
-	ret = ((port - PM8606_BACKLIGHT1) << 1) + 3;
-	return ret;
-}
-
-/* WLED2 & WLED3 share the same IDC */
-static inline int wled_idc(int port)
-{
-	int ret;
-
-	switch (port) {
-	case PM8606_BACKLIGHT1:
-	case PM8606_BACKLIGHT2:
-		ret = ((port - PM8606_BACKLIGHT1) << 1) + 3;
-		break;
-	case PM8606_BACKLIGHT3:
-	default:
-		ret = ((port - PM8606_BACKLIGHT2) << 1) + 3;
-		break;
-	}
-	return ret;
-}
-
 static int backlight_power_set(struct pm860x_chip *chip, int port,
 		int on)
 {
 	int ret = -EINVAL;
 
 	switch (port) {
-	case PM8606_BACKLIGHT1:
+	case 0:
 		ret = on ? pm8606_osc_enable(chip, WLED1_DUTY) :
 			pm8606_osc_disable(chip, WLED1_DUTY);
 		break;
-	case PM8606_BACKLIGHT2:
+	case 1:
 		ret = on ? pm8606_osc_enable(chip, WLED2_DUTY) :
 			pm8606_osc_disable(chip, WLED2_DUTY);
 		break;
-	case PM8606_BACKLIGHT3:
+	case 2:
 		ret = on ? pm8606_osc_enable(chip, WLED3_DUTY) :
 			pm8606_osc_disable(chip, WLED3_DUTY);
 		break;
@@ -104,13 +74,13 @@ static int pm860x_backlight_set(struct backlight_device *bl, int brightness)
 	if (brightness)
 		backlight_power_set(chip, data->port, 1);
 
-	ret = pm860x_reg_write(data->i2c, wled_a(data->port), value);
+	ret = pm860x_reg_write(data->i2c, data->reg_duty_cycle, value);
 	if (ret < 0)
 		goto out;
 
 	if ((data->current_brightness == 0) && brightness) {
 		if (data->iset) {
-			ret = pm860x_set_bits(data->i2c, wled_idc(data->port),
+			ret = pm860x_set_bits(data->i2c, data->reg_current,
 					      CURRENT_BITMASK, data->iset);
 			if (ret < 0)
 				goto out;
@@ -123,17 +93,17 @@ static int pm860x_backlight_set(struct backlight_device *bl, int brightness)
 		}
 		if (brightness == MAX_BRIGHTNESS) {
 			/* set WLED_ON bit as 100% */
-			ret = pm860x_set_bits(data->i2c, wled_b(data->port),
+			ret = pm860x_set_bits(data->i2c, data->reg_always_on,
 					      PM8606_WLED_ON, PM8606_WLED_ON);
 		}
 	} else {
 		if (brightness == MAX_BRIGHTNESS) {
 			/* set WLED_ON bit as 100% */
-			ret = pm860x_set_bits(data->i2c, wled_b(data->port),
+			ret = pm860x_set_bits(data->i2c, data->reg_always_on,
 					      PM8606_WLED_ON, PM8606_WLED_ON);
 		} else {
 			/* clear WLED_ON bit since it's not 100% */
-			ret = pm860x_set_bits(data->i2c, wled_b(data->port),
+			ret = pm860x_set_bits(data->i2c, data->reg_always_on,
 					      PM8606_WLED_ON, 0);
 		}
 	}
@@ -174,7 +144,7 @@ static int pm860x_backlight_get_brightness(struct backlight_device *bl)
 	struct pm860x_chip *chip = data->chip;
 	int ret;
 
-	ret = pm860x_reg_read(data->i2c, wled_a(data->port));
+	ret = pm860x_reg_read(data->i2c, data->reg_duty_cycle);
 	if (ret < 0)
 		goto out;
 	data->current_brightness = ret;
@@ -190,45 +160,85 @@ static const struct backlight_ops pm860x_backlight_ops = {
 	.get_brightness	= pm860x_backlight_get_brightness,
 };
 
+#ifdef CONFIG_OF
+static int pm860x_backlight_dt_init(struct platform_device *pdev,
+				    struct pm860x_backlight_data *data,
+				    char *name)
+{
+	struct device_node *nproot = pdev->dev.parent->of_node, *np;
+	int iset = 0;
+	if (!nproot)
+		return -ENODEV;
+	nproot = of_find_node_by_name(nproot, "backlights");
+	if (!nproot) {
+		dev_err(&pdev->dev, "failed to find backlights node\n");
+		return -ENODEV;
+	}
+	for_each_child_of_node(nproot, np) {
+		if (!of_node_cmp(np->name, name)) {
+			of_property_read_u32(np, "marvell,88pm860x-iset",
+					     &iset);
+			data->iset = PM8606_WLED_CURRENT(iset);
+			of_property_read_u32(np, "marvell,88pm860x-pwm",
+					     &data->pwm);
+			break;
+		}
+	}
+	return 0;
+}
+#else
+#define pm860x_backlight_dt_init(x, y, z)	(-1)
+#endif
+
 static int pm860x_backlight_probe(struct platform_device *pdev)
 {
 	struct pm860x_chip *chip = dev_get_drvdata(pdev->dev.parent);
-	struct pm860x_backlight_pdata *pdata = NULL;
+	struct pm860x_backlight_pdata *pdata = pdev->dev.platform_data;
 	struct pm860x_backlight_data *data;
 	struct backlight_device *bl;
 	struct resource *res;
 	struct backlight_properties props;
 	char name[MFD_NAME_SIZE];
-	int ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource!\n");
-		return -EINVAL;
-	}
-
-	pdata = pdev->dev.platform_data;
-	if (pdata == NULL) {
-		dev_err(&pdev->dev, "platform data isn't assigned to "
-			"backlight\n");
-		return -EINVAL;
-	}
+	int ret = 0;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(struct pm860x_backlight_data),
 			    GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
-	strncpy(name, res->name, MFD_NAME_SIZE);
+	res = platform_get_resource_byname(pdev, IORESOURCE_REG, "duty cycle");
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for duty cycle\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_duty_cycle = res->start;
+	res = platform_get_resource_byname(pdev, IORESOURCE_REG, "always on");
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resorce for always on\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_always_on = res->start;
+	res = platform_get_resource_byname(pdev, IORESOURCE_REG, "current");
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for current\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_current = res->start;
+
+	memset(name, 0, MFD_NAME_SIZE);
+	sprintf(name, "backlight-%d", pdev->id);
+	data->port = pdev->id;
 	data->chip = chip;
 	data->i2c = (chip->id == CHIP_PM8606) ? chip->client	\
 			: chip->companion;
 	data->current_brightness = MAX_BRIGHTNESS;
-	data->pwm = pdata->pwm;
-	data->iset = pdata->iset;
-	data->port = pdata->flags;
-	if (data->port < 0) {
-		dev_err(&pdev->dev, "wrong platform data is assigned");
-		return -EINVAL;
+	if (pm860x_backlight_dt_init(pdev, data, name)) {
+		if (pdata) {
+			data->pwm = pdata->pwm;
+			data->iset = pdata->iset;
+		}
 	}
 
 	memset(&props, 0, sizeof(struct backlight_properties));
@@ -247,12 +257,14 @@ static int pm860x_backlight_probe(struct platform_device *pdev)
 	/* read current backlight */
 	ret = pm860x_backlight_get_brightness(bl);
 	if (ret < 0)
-		goto out;
+		goto out_brt;
 
 	backlight_update_status(bl);
 	return 0;
-out:
+out_brt:
 	backlight_device_unregister(bl);
+out:
+	devm_kfree(&pdev->dev, data);
 	return ret;
 }
 
diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
index cf282763a8d..c101697a4ba 100644
--- a/drivers/video/backlight/Kconfig
+++ b/drivers/video/backlight/Kconfig
@@ -229,13 +229,6 @@ config BACKLIGHT_HP700
 	  If you have an HP Jornada 700 series,
 	  say Y to include backlight control driver.
 
-config BACKLIGHT_PROGEAR
-	tristate "Frontpath ProGear Backlight Driver"
-	depends on PCI && X86
-	help
-	  If you have a Frontpath ProGear say Y to enable the
-	  backlight driver.
-
 config BACKLIGHT_CARILLO_RANCH
 	tristate "Intel Carillo Ranch Backlight Driver"
 	depends on LCD_CLASS_DEVICE && PCI && X86 && FB_LE80578
@@ -352,6 +345,22 @@ config BACKLIGHT_AAT2870
 	  If you have a AnalogicTech AAT2870 say Y to enable the
 	  backlight driver.
 
+config BACKLIGHT_LM3630
+	tristate "Backlight Driver for LM3630"
+	depends on BACKLIGHT_CLASS_DEVICE && I2C
+	select REGMAP_I2C
+	help
+	  This supports TI LM3630 Backlight Driver
+
+config BACKLIGHT_LM3639
+	tristate "Backlight Driver for LM3639"
+	depends on BACKLIGHT_CLASS_DEVICE && I2C
+	select REGMAP_I2C
+	select NEW_LEDS
+	select LEDS_CLASS
+	help
+	  This supports TI LM3639 Backlight + 1.5A Flash LED Driver
+
 config BACKLIGHT_LP855X
 	tristate "Backlight driver for TI LP855X"
 	depends on BACKLIGHT_CLASS_DEVICE && I2C
@@ -373,6 +382,13 @@ config BACKLIGHT_PANDORA
 	  If you have a Pandora console, say Y to enable the
 	  backlight driver.
 
+config BACKLIGHT_TPS65217
+	tristate "TPS65217 Backlight"
+	depends on BACKLIGHT_CLASS_DEVICE && MFD_TPS65217
+	help
+	  If you have a Texas Instruments TPS65217 say Y to enable the
+	  backlight driver.
+
 endif # BACKLIGHT_CLASS_DEVICE
 
 endif # BACKLIGHT_LCD_SUPPORT
diff --git a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile
index a2ac9cfbaf6..e7ce7291635 100644
--- a/drivers/video/backlight/Makefile
+++ b/drivers/video/backlight/Makefile
@@ -23,10 +23,11 @@ obj-$(CONFIG_BACKLIGHT_HP700)	+= jornada720_bl.o
 obj-$(CONFIG_BACKLIGHT_HP680)	+= hp680_bl.o
 obj-$(CONFIG_BACKLIGHT_LM3533)	+= lm3533_bl.o
 obj-$(CONFIG_BACKLIGHT_LOCOMO)	+= locomolcd.o
+obj-$(CONFIG_BACKLIGHT_LM3630)	+= lm3630_bl.o
+obj-$(CONFIG_BACKLIGHT_LM3639)	+= lm3639_bl.o
 obj-$(CONFIG_BACKLIGHT_LP855X)	+= lp855x_bl.o
 obj-$(CONFIG_BACKLIGHT_OMAP1)	+= omap1_bl.o
 obj-$(CONFIG_BACKLIGHT_PANDORA)	+= pandora_bl.o
-obj-$(CONFIG_BACKLIGHT_PROGEAR) += progear_bl.o
 obj-$(CONFIG_BACKLIGHT_CARILLO_RANCH) += cr_bllcd.o
 obj-$(CONFIG_BACKLIGHT_PWM)	+= pwm_bl.o
 obj-$(CONFIG_BACKLIGHT_DA903X)	+= da903x_bl.o
@@ -43,3 +44,4 @@ obj-$(CONFIG_BACKLIGHT_88PM860X) += 88pm860x_bl.o
 obj-$(CONFIG_BACKLIGHT_PCF50633)	+= pcf50633-backlight.o
 obj-$(CONFIG_BACKLIGHT_AAT2870) += aat2870_bl.o
 obj-$(CONFIG_BACKLIGHT_OT200) += ot200_bl.o
+obj-$(CONFIG_BACKLIGHT_TPS65217) += tps65217_bl.o
diff --git a/drivers/video/backlight/da9052_bl.c b/drivers/video/backlight/da9052_bl.c
index b628d68f516..ac196181fe4 100644
--- a/drivers/video/backlight/da9052_bl.c
+++ b/drivers/video/backlight/da9052_bl.c
@@ -72,7 +72,7 @@ static int da9052_adjust_wled_brightness(struct da9052_bl *wleds)
 	if (ret < 0)
 		return ret;
 
-	msleep(10);
+	usleep_range(10000, 11000);
 
 	if (wleds->brightness) {
 		ret = da9052_reg_write(wleds->da9052, wled_bank[wleds->led_reg],
@@ -129,7 +129,6 @@ static int da9052_backlight_probe(struct platform_device *pdev)
 				       &da9052_backlight_ops, &props);
 	if (IS_ERR(bl)) {
 		dev_err(&pdev->dev, "Failed to register backlight\n");
-		devm_kfree(&pdev->dev, wleds);
 		return PTR_ERR(bl);
 	}
 
@@ -149,7 +148,6 @@ static int da9052_backlight_remove(struct platform_device *pdev)
 	wleds->state = DA9052_WLEDS_OFF;
 	da9052_adjust_wled_brightness(wleds);
 	backlight_device_unregister(bl);
-	devm_kfree(&pdev->dev, wleds);
 
 	return 0;
 }
diff --git a/drivers/video/backlight/kb3886_bl.c b/drivers/video/backlight/kb3886_bl.c
index 72dd5556a35..6c5ed6b242c 100644
--- a/drivers/video/backlight/kb3886_bl.c
+++ b/drivers/video/backlight/kb3886_bl.c
@@ -34,9 +34,9 @@ static void kb3886_bl_set_intensity(int intensity)
 	mutex_lock(&bl_mutex);
 	intensity = intensity&0xff;
 	outb(KB3886_ADC_DAC_PWM, KB3886_PARENT);
-	msleep(10);
+	usleep_range(10000, 11000);
 	outb(KB3886_PWM0_WRITE, KB3886_IO);
-	msleep(10);
+	usleep_range(10000, 11000);
 	outb(intensity, KB3886_IO);
 	mutex_unlock(&bl_mutex);
 }
diff --git a/drivers/video/backlight/lm3630_bl.c b/drivers/video/backlight/lm3630_bl.c
new file mode 100644
index 00000000000..dc191441796
--- /dev/null
+++ b/drivers/video/backlight/lm3630_bl.c
@@ -0,0 +1,475 @@
+/*
+* Simple driver for Texas Instruments LM3630 Backlight driver chip
+* Copyright (C) 2012 Texas Instruments
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+*/
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/backlight.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+#include <linux/interrupt.h>
+#include <linux/regmap.h>
+#include <linux/platform_data/lm3630_bl.h>
+
+#define REG_CTRL	0x00
+#define REG_CONFIG	0x01
+#define REG_BRT_A	0x03
+#define REG_BRT_B	0x04
+#define REG_INT_STATUS	0x09
+#define REG_INT_EN	0x0A
+#define REG_FAULT	0x0B
+#define REG_PWM_OUTLOW	0x12
+#define REG_PWM_OUTHIGH	0x13
+#define REG_MAX		0x1F
+
+#define INT_DEBOUNCE_MSEC	10
+
+enum lm3630_leds {
+	BLED_ALL = 0,
+	BLED_1,
+	BLED_2
+};
+
+static const char *bled_name[] = {
+	[BLED_ALL] = "lm3630_bled",	/*Bank1 controls all string */
+	[BLED_1] = "lm3630_bled1",	/*Bank1 controls bled1 */
+	[BLED_2] = "lm3630_bled2",	/*Bank1 or 2 controls bled2 */
+};
+
+struct lm3630_chip_data {
+	struct device *dev;
+	struct delayed_work work;
+	int irq;
+	struct workqueue_struct *irqthread;
+	struct lm3630_platform_data *pdata;
+	struct backlight_device *bled1;
+	struct backlight_device *bled2;
+	struct regmap *regmap;
+};
+
+/* initialize chip */
+static int __devinit lm3630_chip_init(struct lm3630_chip_data *pchip)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3630_platform_data *pdata = pchip->pdata;
+
+	/*pwm control */
+	reg_val = ((pdata->pwm_active & 0x01) << 2) | (pdata->pwm_ctrl & 0x03);
+	ret = regmap_update_bits(pchip->regmap, REG_CONFIG, 0x07, reg_val);
+	if (ret < 0)
+		goto out;
+
+	/* bank control */
+	reg_val = ((pdata->bank_b_ctrl & 0x01) << 1) |
+			(pdata->bank_a_ctrl & 0x07);
+	ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x07, reg_val);
+	if (ret < 0)
+		goto out;
+
+	ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+	if (ret < 0)
+		goto out;
+
+	/* set initial brightness */
+	if (pdata->bank_a_ctrl != BANK_A_CTRL_DISABLE) {
+		ret = regmap_write(pchip->regmap,
+				   REG_BRT_A, pdata->init_brt_led1);
+		if (ret < 0)
+			goto out;
+	}
+
+	if (pdata->bank_b_ctrl != BANK_B_CTRL_DISABLE) {
+		ret = regmap_write(pchip->regmap,
+				   REG_BRT_B, pdata->init_brt_led2);
+		if (ret < 0)
+			goto out;
+	}
+	return ret;
+
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return ret;
+}
+
+/* interrupt handling */
+static void lm3630_delayed_func(struct work_struct *work)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3630_chip_data *pchip;
+
+	pchip = container_of(work, struct lm3630_chip_data, work.work);
+
+	ret = regmap_read(pchip->regmap, REG_INT_STATUS, &reg_val);
+	if (ret < 0) {
+		dev_err(pchip->dev,
+			"i2c failed to access REG_INT_STATUS Register\n");
+		return;
+	}
+
+	dev_info(pchip->dev, "REG_INT_STATUS Register is 0x%x\n", reg_val);
+}
+
+static irqreturn_t lm3630_isr_func(int irq, void *chip)
+{
+	int ret;
+	struct lm3630_chip_data *pchip = chip;
+	unsigned long delay = msecs_to_jiffies(INT_DEBOUNCE_MSEC);
+
+	queue_delayed_work(pchip->irqthread, &pchip->work, delay);
+
+	ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+	if (ret < 0)
+		goto out;
+
+	return IRQ_HANDLED;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return IRQ_HANDLED;
+}
+
+static int lm3630_intr_config(struct lm3630_chip_data *pchip)
+{
+	INIT_DELAYED_WORK(&pchip->work, lm3630_delayed_func);
+	pchip->irqthread = create_singlethread_workqueue("lm3630-irqthd");
+	if (!pchip->irqthread) {
+		dev_err(pchip->dev, "create irq thread fail...\n");
+		return -1;
+	}
+	if (request_threaded_irq
+	    (pchip->irq, NULL, lm3630_isr_func,
+	     IRQF_TRIGGER_FALLING | IRQF_ONESHOT, "lm3630_irq", pchip)) {
+		dev_err(pchip->dev, "request threaded irq fail..\n");
+		return -1;
+	}
+	return 0;
+}
+
+static bool
+set_intensity(struct backlight_device *bl, struct lm3630_chip_data *pchip)
+{
+	if (!pchip->pdata->pwm_set_intensity)
+		return false;
+	pchip->pdata->pwm_set_intensity(bl->props.brightness - 1,
+					pchip->pdata->pwm_period);
+	return true;
+}
+
+/* update and get brightness */
+static int lm3630_bank_a_update_status(struct backlight_device *bl)
+{
+	int ret;
+	struct lm3630_chip_data *pchip = bl_get_data(bl);
+	enum lm3630_pwm_ctrl pwm_ctrl = pchip->pdata->pwm_ctrl;
+
+	/* brightness 0 means disable */
+	if (!bl->props.brightness) {
+		ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x04, 0x00);
+		if (ret < 0)
+			goto out;
+		return bl->props.brightness;
+	}
+
+	/* pwm control */
+	if (pwm_ctrl == PWM_CTRL_BANK_A || pwm_ctrl == PWM_CTRL_BANK_ALL) {
+		if (!set_intensity(bl, pchip))
+			dev_err(pchip->dev, "No pwm control func. in plat-data\n");
+	} else {
+
+		/* i2c control */
+		ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+		if (ret < 0)
+			goto out;
+		mdelay(1);
+		ret = regmap_write(pchip->regmap,
+				   REG_BRT_A, bl->props.brightness - 1);
+		if (ret < 0)
+			goto out;
+	}
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access REG_CTRL\n");
+	return bl->props.brightness;
+}
+
+static int lm3630_bank_a_get_brightness(struct backlight_device *bl)
+{
+	unsigned int reg_val;
+	int brightness, ret;
+	struct lm3630_chip_data *pchip = bl_get_data(bl);
+	enum lm3630_pwm_ctrl pwm_ctrl = pchip->pdata->pwm_ctrl;
+
+	if (pwm_ctrl == PWM_CTRL_BANK_A || pwm_ctrl == PWM_CTRL_BANK_ALL) {
+		ret = regmap_read(pchip->regmap, REG_PWM_OUTHIGH, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = reg_val & 0x01;
+		ret = regmap_read(pchip->regmap, REG_PWM_OUTLOW, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = ((brightness << 8) | reg_val) + 1;
+	} else {
+		ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+		if (ret < 0)
+			goto out;
+		mdelay(1);
+		ret = regmap_read(pchip->regmap, REG_BRT_A, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = reg_val + 1;
+	}
+	bl->props.brightness = brightness;
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return 0;
+}
+
+static const struct backlight_ops lm3630_bank_a_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = lm3630_bank_a_update_status,
+	.get_brightness = lm3630_bank_a_get_brightness,
+};
+
+static int lm3630_bank_b_update_status(struct backlight_device *bl)
+{
+	int ret;
+	struct lm3630_chip_data *pchip = bl_get_data(bl);
+	enum lm3630_pwm_ctrl pwm_ctrl = pchip->pdata->pwm_ctrl;
+
+	if (pwm_ctrl == PWM_CTRL_BANK_B || pwm_ctrl == PWM_CTRL_BANK_ALL) {
+		if (!set_intensity(bl, pchip))
+			dev_err(pchip->dev,
+				"no pwm control func. in plat-data\n");
+	} else {
+		ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+		if (ret < 0)
+			goto out;
+		mdelay(1);
+		ret = regmap_write(pchip->regmap,
+				   REG_BRT_B, bl->props.brightness - 1);
+	}
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return bl->props.brightness;
+}
+
+static int lm3630_bank_b_get_brightness(struct backlight_device *bl)
+{
+	unsigned int reg_val;
+	int brightness, ret;
+	struct lm3630_chip_data *pchip = bl_get_data(bl);
+	enum lm3630_pwm_ctrl pwm_ctrl = pchip->pdata->pwm_ctrl;
+
+	if (pwm_ctrl == PWM_CTRL_BANK_B || pwm_ctrl == PWM_CTRL_BANK_ALL) {
+		ret = regmap_read(pchip->regmap, REG_PWM_OUTHIGH, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = reg_val & 0x01;
+		ret = regmap_read(pchip->regmap, REG_PWM_OUTLOW, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = ((brightness << 8) | reg_val) + 1;
+	} else {
+		ret = regmap_update_bits(pchip->regmap, REG_CTRL, 0x80, 0x00);
+		if (ret < 0)
+			goto out;
+		mdelay(1);
+		ret = regmap_read(pchip->regmap, REG_BRT_B, &reg_val);
+		if (ret < 0)
+			goto out;
+		brightness = reg_val + 1;
+	}
+	bl->props.brightness = brightness;
+
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return bl->props.brightness;
+}
+
+static const struct backlight_ops lm3630_bank_b_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = lm3630_bank_b_update_status,
+	.get_brightness = lm3630_bank_b_get_brightness,
+};
+
+static int lm3630_backlight_register(struct lm3630_chip_data *pchip,
+				     enum lm3630_leds ledno)
+{
+	const char *name = bled_name[ledno];
+	struct backlight_properties props;
+	struct lm3630_platform_data *pdata = pchip->pdata;
+
+	props.type = BACKLIGHT_RAW;
+	switch (ledno) {
+	case BLED_1:
+	case BLED_ALL:
+		props.brightness = pdata->init_brt_led1;
+		props.max_brightness = pdata->max_brt_led1;
+		pchip->bled1 =
+		    backlight_device_register(name, pchip->dev, pchip,
+					      &lm3630_bank_a_ops, &props);
+		if (IS_ERR(pchip->bled1))
+			return -EIO;
+		break;
+	case BLED_2:
+		props.brightness = pdata->init_brt_led2;
+		props.max_brightness = pdata->max_brt_led2;
+		pchip->bled2 =
+		    backlight_device_register(name, pchip->dev, pchip,
+					      &lm3630_bank_b_ops, &props);
+		if (IS_ERR(pchip->bled2))
+			return -EIO;
+		break;
+	}
+	return 0;
+}
+
+static void lm3630_backlight_unregister(struct lm3630_chip_data *pchip)
+{
+	if (pchip->bled1)
+		backlight_device_unregister(pchip->bled1);
+	if (pchip->bled2)
+		backlight_device_unregister(pchip->bled2);
+}
+
+static const struct regmap_config lm3630_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = REG_MAX,
+};
+
+static int __devinit lm3630_probe(struct i2c_client *client,
+				  const struct i2c_device_id *id)
+{
+	struct lm3630_platform_data *pdata = client->dev.platform_data;
+	struct lm3630_chip_data *pchip;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev, "fail : i2c functionality check...\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (pdata == NULL) {
+		dev_err(&client->dev, "fail : no platform data.\n");
+		return -ENODATA;
+	}
+
+	pchip = devm_kzalloc(&client->dev, sizeof(struct lm3630_chip_data),
+			     GFP_KERNEL);
+	if (!pchip)
+		return -ENOMEM;
+	pchip->pdata = pdata;
+	pchip->dev = &client->dev;
+
+	pchip->regmap = devm_regmap_init_i2c(client, &lm3630_regmap);
+	if (IS_ERR(pchip->regmap)) {
+		ret = PTR_ERR(pchip->regmap);
+		dev_err(&client->dev, "fail : allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+	i2c_set_clientdata(client, pchip);
+
+	/* chip initialize */
+	ret = lm3630_chip_init(pchip);
+	if (ret < 0) {
+		dev_err(&client->dev, "fail : init chip\n");
+		goto err_chip_init;
+	}
+
+	switch (pdata->bank_a_ctrl) {
+	case BANK_A_CTRL_ALL:
+		ret = lm3630_backlight_register(pchip, BLED_ALL);
+		pdata->bank_b_ctrl = BANK_B_CTRL_DISABLE;
+		break;
+	case BANK_A_CTRL_LED1:
+		ret = lm3630_backlight_register(pchip, BLED_1);
+		break;
+	case BANK_A_CTRL_LED2:
+		ret = lm3630_backlight_register(pchip, BLED_2);
+		pdata->bank_b_ctrl = BANK_B_CTRL_DISABLE;
+		break;
+	default:
+		break;
+	}
+
+	if (ret < 0)
+		goto err_bl_reg;
+
+	if (pdata->bank_b_ctrl && pchip->bled2 == NULL) {
+		ret = lm3630_backlight_register(pchip, BLED_2);
+		if (ret < 0)
+			goto err_bl_reg;
+	}
+
+	/* interrupt enable  : irq 0 is not allowed for lm3630 */
+	pchip->irq = client->irq;
+	if (pchip->irq)
+		lm3630_intr_config(pchip);
+
+	dev_info(&client->dev, "LM3630 backlight register OK.\n");
+	return 0;
+
+err_bl_reg:
+	dev_err(&client->dev, "fail : backlight register.\n");
+	lm3630_backlight_unregister(pchip);
+err_chip_init:
+	return ret;
+}
+
+static int __devexit lm3630_remove(struct i2c_client *client)
+{
+	int ret;
+	struct lm3630_chip_data *pchip = i2c_get_clientdata(client);
+
+	ret = regmap_write(pchip->regmap, REG_BRT_A, 0);
+	if (ret < 0)
+		dev_err(pchip->dev, "i2c failed to access register\n");
+
+	ret = regmap_write(pchip->regmap, REG_BRT_B, 0);
+	if (ret < 0)
+		dev_err(pchip->dev, "i2c failed to access register\n");
+
+	lm3630_backlight_unregister(pchip);
+	if (pchip->irq) {
+		free_irq(pchip->irq, pchip);
+		flush_workqueue(pchip->irqthread);
+		destroy_workqueue(pchip->irqthread);
+	}
+	return 0;
+}
+
+static const struct i2c_device_id lm3630_id[] = {
+	{LM3630_NAME, 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, lm3630_id);
+
+static struct i2c_driver lm3630_i2c_driver = {
+	.driver = {
+		   .name = LM3630_NAME,
+		   },
+	.probe = lm3630_probe,
+	.remove = __devexit_p(lm3630_remove),
+	.id_table = lm3630_id,
+};
+
+module_i2c_driver(lm3630_i2c_driver);
+
+MODULE_DESCRIPTION("Texas Instruments Backlight driver for LM3630");
+MODULE_AUTHOR("G.Shark Jeong <gshark.jeong@gmail.com>");
+MODULE_AUTHOR("Daniel Jeong <daniel.jeong@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/video/backlight/lm3639_bl.c b/drivers/video/backlight/lm3639_bl.c
new file mode 100644
index 00000000000..c6915c6c3cd
--- /dev/null
+++ b/drivers/video/backlight/lm3639_bl.c
@@ -0,0 +1,437 @@
+/*
+* Simple driver for Texas Instruments LM3639 Backlight + Flash LED driver chip
+* Copyright (C) 2012 Texas Instruments
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+*/
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/i2c.h>
+#include <linux/leds.h>
+#include <linux/backlight.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <linux/uaccess.h>
+#include <linux/interrupt.h>
+#include <linux/regmap.h>
+#include <linux/platform_data/lm3639_bl.h>
+
+#define REG_DEV_ID	0x00
+#define REG_CHECKSUM	0x01
+#define REG_BL_CONF_1	0x02
+#define REG_BL_CONF_2	0x03
+#define REG_BL_CONF_3	0x04
+#define REG_BL_CONF_4	0x05
+#define REG_FL_CONF_1	0x06
+#define REG_FL_CONF_2	0x07
+#define REG_FL_CONF_3	0x08
+#define REG_IO_CTRL	0x09
+#define REG_ENABLE	0x0A
+#define REG_FLAG	0x0B
+#define REG_MAX		REG_FLAG
+
+struct lm3639_chip_data {
+	struct device *dev;
+	struct lm3639_platform_data *pdata;
+
+	struct backlight_device *bled;
+	struct led_classdev cdev_flash;
+	struct led_classdev cdev_torch;
+	struct regmap *regmap;
+
+	unsigned int bled_mode;
+	unsigned int bled_map;
+	unsigned int last_flag;
+};
+
+/* initialize chip */
+static int __devinit lm3639_chip_init(struct lm3639_chip_data *pchip)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3639_platform_data *pdata = pchip->pdata;
+
+	/* input pins config. */
+	ret =
+	    regmap_update_bits(pchip->regmap, REG_BL_CONF_1, 0x08,
+			       pdata->pin_pwm);
+	if (ret < 0)
+		goto out;
+
+	reg_val = (pdata->pin_pwm & 0x40) | pdata->pin_strobe | pdata->pin_tx;
+	ret = regmap_update_bits(pchip->regmap, REG_IO_CTRL, 0x7C, reg_val);
+	if (ret < 0)
+		goto out;
+
+	/* init brightness */
+	ret = regmap_write(pchip->regmap, REG_BL_CONF_4, pdata->init_brt_led);
+	if (ret < 0)
+		goto out;
+
+	ret = regmap_write(pchip->regmap, REG_BL_CONF_3, pdata->init_brt_led);
+	if (ret < 0)
+		goto out;
+
+	/* output pins config. */
+	if (!pdata->init_brt_led)
+		reg_val = pdata->fled_pins | pdata->bled_pins;
+	else
+		reg_val = pdata->fled_pins | pdata->bled_pins | 0x01;
+
+	ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x79, reg_val);
+	if (ret < 0)
+		goto out;
+
+	return ret;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return ret;
+}
+
+/* update and get brightness */
+static int lm3639_bled_update_status(struct backlight_device *bl)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3639_chip_data *pchip = bl_get_data(bl);
+	struct lm3639_platform_data *pdata = pchip->pdata;
+
+	ret = regmap_read(pchip->regmap, REG_FLAG, &reg_val);
+	if (ret < 0)
+		goto out;
+
+	if (reg_val != 0)
+		dev_info(pchip->dev, "last flag is 0x%x\n", reg_val);
+
+	/* pwm control */
+	if (pdata->pin_pwm) {
+		if (pdata->pwm_set_intensity)
+			pdata->pwm_set_intensity(bl->props.brightness,
+						 pdata->max_brt_led);
+		else
+			dev_err(pchip->dev,
+				"No pwm control func. in plat-data\n");
+		return bl->props.brightness;
+	}
+
+	/* i2c control and set brigtness */
+	ret = regmap_write(pchip->regmap, REG_BL_CONF_4, bl->props.brightness);
+	if (ret < 0)
+		goto out;
+	ret = regmap_write(pchip->regmap, REG_BL_CONF_3, bl->props.brightness);
+	if (ret < 0)
+		goto out;
+
+	if (!bl->props.brightness)
+		ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x01, 0x00);
+	else
+		ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x01, 0x01);
+	if (ret < 0)
+		goto out;
+
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access registers\n");
+	return bl->props.brightness;
+}
+
+static int lm3639_bled_get_brightness(struct backlight_device *bl)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3639_chip_data *pchip = bl_get_data(bl);
+	struct lm3639_platform_data *pdata = pchip->pdata;
+
+	if (pdata->pin_pwm) {
+		if (pdata->pwm_get_intensity)
+			bl->props.brightness = pdata->pwm_get_intensity();
+		else
+			dev_err(pchip->dev,
+				"No pwm control func. in plat-data\n");
+		return bl->props.brightness;
+	}
+
+	ret = regmap_read(pchip->regmap, REG_BL_CONF_1, &reg_val);
+	if (ret < 0)
+		goto out;
+	if (reg_val & 0x10)
+		ret = regmap_read(pchip->regmap, REG_BL_CONF_4, &reg_val);
+	else
+		ret = regmap_read(pchip->regmap, REG_BL_CONF_3, &reg_val);
+	if (ret < 0)
+		goto out;
+	bl->props.brightness = reg_val;
+
+	return bl->props.brightness;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return bl->props.brightness;
+}
+
+static const struct backlight_ops lm3639_bled_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = lm3639_bled_update_status,
+	.get_brightness = lm3639_bled_get_brightness,
+};
+
+/* backlight mapping mode */
+static ssize_t lm3639_bled_mode_store(struct device *dev,
+				      struct device_attribute *devAttr,
+				      const char *buf, size_t size)
+{
+	ssize_t ret;
+	struct lm3639_chip_data *pchip = dev_get_drvdata(dev);
+	unsigned int state;
+
+	ret = kstrtouint(buf, 10, &state);
+	if (ret)
+		goto out_input;
+
+	if (!state)
+		ret =
+		    regmap_update_bits(pchip->regmap, REG_BL_CONF_1, 0x10,
+				       0x00);
+	else
+		ret =
+		    regmap_update_bits(pchip->regmap, REG_BL_CONF_1, 0x10,
+				       0x10);
+
+	if (ret < 0)
+		goto out;
+
+	return size;
+
+out:
+	dev_err(pchip->dev, "%s:i2c access fail to register\n", __func__);
+	return size;
+
+out_input:
+	dev_err(pchip->dev, "%s:input conversion fail\n", __func__);
+	return size;
+
+}
+
+static DEVICE_ATTR(bled_mode, 0666, NULL, lm3639_bled_mode_store);
+
+/* torch */
+static void lm3639_torch_brightness_set(struct led_classdev *cdev,
+					enum led_brightness brightness)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3639_chip_data *pchip;
+
+	pchip = container_of(cdev, struct lm3639_chip_data, cdev_torch);
+
+	ret = regmap_read(pchip->regmap, REG_FLAG, &reg_val);
+	if (ret < 0)
+		goto out;
+	if (reg_val != 0)
+		dev_info(pchip->dev, "last flag is 0x%x\n", reg_val);
+
+	/* brightness 0 means off state */
+	if (!brightness) {
+		ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x06, 0x00);
+		if (ret < 0)
+			goto out;
+		return;
+	}
+
+	ret = regmap_update_bits(pchip->regmap,
+				 REG_FL_CONF_1, 0x70, (brightness - 1) << 4);
+	if (ret < 0)
+		goto out;
+	ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x06, 0x02);
+	if (ret < 0)
+		goto out;
+
+	return;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return;
+}
+
+/* flash */
+static void lm3639_flash_brightness_set(struct led_classdev *cdev,
+					enum led_brightness brightness)
+{
+	int ret;
+	unsigned int reg_val;
+	struct lm3639_chip_data *pchip;
+
+	pchip = container_of(cdev, struct lm3639_chip_data, cdev_flash);
+
+	ret = regmap_read(pchip->regmap, REG_FLAG, &reg_val);
+	if (ret < 0)
+		goto out;
+	if (reg_val != 0)
+		dev_info(pchip->dev, "last flag is 0x%x\n", reg_val);
+
+	/* torch off before flash control */
+	ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x06, 0x00);
+	if (ret < 0)
+		goto out;
+
+	/* brightness 0 means off state */
+	if (!brightness)
+		return;
+
+	ret = regmap_update_bits(pchip->regmap,
+				 REG_FL_CONF_1, 0x0F, brightness - 1);
+	if (ret < 0)
+		goto out;
+	ret = regmap_update_bits(pchip->regmap, REG_ENABLE, 0x06, 0x06);
+	if (ret < 0)
+		goto out;
+
+	return;
+out:
+	dev_err(pchip->dev, "i2c failed to access register\n");
+	return;
+}
+
+static const struct regmap_config lm3639_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = REG_MAX,
+};
+
+static int __devinit lm3639_probe(struct i2c_client *client,
+				  const struct i2c_device_id *id)
+{
+	int ret;
+	struct lm3639_chip_data *pchip;
+	struct lm3639_platform_data *pdata = client->dev.platform_data;
+	struct backlight_properties props;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+		dev_err(&client->dev, "i2c functionality check fail.\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (pdata == NULL) {
+		dev_err(&client->dev, "Needs Platform Data.\n");
+		return -ENODATA;
+	}
+
+	pchip = devm_kzalloc(&client->dev,
+			     sizeof(struct lm3639_chip_data), GFP_KERNEL);
+	if (!pchip)
+		return -ENOMEM;
+
+	pchip->pdata = pdata;
+	pchip->dev = &client->dev;
+
+	pchip->regmap = devm_regmap_init_i2c(client, &lm3639_regmap);
+	if (IS_ERR(pchip->regmap)) {
+		ret = PTR_ERR(pchip->regmap);
+		dev_err(&client->dev, "fail : allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+	i2c_set_clientdata(client, pchip);
+
+	/* chip initialize */
+	ret = lm3639_chip_init(pchip);
+	if (ret < 0) {
+		dev_err(&client->dev, "fail : chip init\n");
+		goto err_out;
+	}
+
+	/* backlight */
+	props.type = BACKLIGHT_RAW;
+	props.brightness = pdata->init_brt_led;
+	props.max_brightness = pdata->max_brt_led;
+	pchip->bled =
+	    backlight_device_register("lm3639_bled", pchip->dev, pchip,
+				      &lm3639_bled_ops, &props);
+	if (IS_ERR(pchip->bled)) {
+		dev_err(&client->dev, "fail : backlight register\n");
+		ret = -EIO;
+		goto err_out;
+	}
+
+	ret = device_create_file(&(pchip->bled->dev), &dev_attr_bled_mode);
+	if (ret < 0) {
+		dev_err(&client->dev, "failed : add sysfs entries\n");
+		ret = -EIO;
+		goto err_bled_mode;
+	}
+
+	/* flash */
+	pchip->cdev_flash.name = "lm3639_flash";
+	pchip->cdev_flash.max_brightness = 16;
+	pchip->cdev_flash.brightness_set = lm3639_flash_brightness_set;
+	ret = led_classdev_register((struct device *)
+				    &client->dev, &pchip->cdev_flash);
+	if (ret < 0) {
+		dev_err(&client->dev, "fail : flash register\n");
+		ret = -EIO;
+		goto err_flash;
+	}
+
+	/* torch */
+	pchip->cdev_torch.name = "lm3639_torch";
+	pchip->cdev_torch.max_brightness = 8;
+	pchip->cdev_torch.brightness_set = lm3639_torch_brightness_set;
+	ret = led_classdev_register((struct device *)
+				    &client->dev, &pchip->cdev_torch);
+	if (ret < 0) {
+		dev_err(&client->dev, "fail : torch register\n");
+		ret = -EIO;
+		goto err_torch;
+	}
+
+	return 0;
+
+err_torch:
+	led_classdev_unregister(&pchip->cdev_flash);
+err_flash:
+	device_remove_file(&(pchip->bled->dev), &dev_attr_bled_mode);
+err_bled_mode:
+	backlight_device_unregister(pchip->bled);
+err_out:
+	return ret;
+}
+
+static int __devexit lm3639_remove(struct i2c_client *client)
+{
+	struct lm3639_chip_data *pchip = i2c_get_clientdata(client);
+
+	regmap_write(pchip->regmap, REG_ENABLE, 0x00);
+
+	if (&pchip->cdev_torch)
+		led_classdev_unregister(&pchip->cdev_torch);
+	if (&pchip->cdev_flash)
+		led_classdev_unregister(&pchip->cdev_flash);
+	if (pchip->bled) {
+		device_remove_file(&(pchip->bled->dev), &dev_attr_bled_mode);
+		backlight_device_unregister(pchip->bled);
+	}
+	return 0;
+}
+
+static const struct i2c_device_id lm3639_id[] = {
+	{LM3639_NAME, 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, lm3639_id);
+static struct i2c_driver lm3639_i2c_driver = {
+	.driver = {
+		   .name = LM3639_NAME,
+		   },
+	.probe = lm3639_probe,
+	.remove = __devexit_p(lm3639_remove),
+	.id_table = lm3639_id,
+};
+
+module_i2c_driver(lm3639_i2c_driver);
+
+MODULE_DESCRIPTION("Texas Instruments Backlight+Flash LED driver for LM3639");
+MODULE_AUTHOR("Daniel Jeong <daniel.jeong@ti.com>");
+MODULE_AUTHOR("G.Shark Jeong <gshark.jeong@gmail.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/video/backlight/ltv350qv.c b/drivers/video/backlight/ltv350qv.c
index 6c0f1ac0d32..4066a5bbd82 100644
--- a/drivers/video/backlight/ltv350qv.c
+++ b/drivers/video/backlight/ltv350qv.c
@@ -75,7 +75,7 @@ static int ltv350qv_power_on(struct ltv350qv *lcd)
 	/* Power On Reset Display off State */
 	if (ltv350qv_write_reg(lcd, LTV_PWRCTL1, 0x0000))
 		goto err;
-	msleep(15);
+	usleep_range(15000, 16000);
 
 	/* Power Setting Function 1 */
 	if (ltv350qv_write_reg(lcd, LTV_PWRCTL1, LTV_VCOM_DISABLE))
@@ -153,7 +153,7 @@ err_settings:
 err_power2:
 err_power1:
 	ltv350qv_write_reg(lcd, LTV_PWRCTL2, 0x0000);
-	msleep(1);
+	usleep_range(1000, 1100);
 err:
 	ltv350qv_write_reg(lcd, LTV_PWRCTL1, LTV_VCOM_DISABLE);
 	return -EIO;
@@ -175,7 +175,7 @@ static int ltv350qv_power_off(struct ltv350qv *lcd)
 	ret |= ltv350qv_write_reg(lcd, LTV_PWRCTL2, 0x0000);
 
 	/* Wait at least 1 ms */
-	msleep(1);
+	usleep_range(1000, 1100);
 
 	/* Power down setting 2 */
 	ret |= ltv350qv_write_reg(lcd, LTV_PWRCTL1, LTV_VCOM_DISABLE);
diff --git a/drivers/video/backlight/max8925_bl.c b/drivers/video/backlight/max8925_bl.c
index e833ac72e06..f72ba54f364 100644
--- a/drivers/video/backlight/max8925_bl.c
+++ b/drivers/video/backlight/max8925_bl.c
@@ -27,7 +27,9 @@
 struct max8925_backlight_data {
 	struct max8925_chip	*chip;
 
-	int		current_brightness;
+	int	current_brightness;
+	int	reg_mode_cntl;
+	int	reg_cntl;
 };
 
 static int max8925_backlight_set(struct backlight_device *bl, int brightness)
@@ -42,16 +44,16 @@ static int max8925_backlight_set(struct backlight_device *bl, int brightness)
 	else
 		value = brightness;
 
-	ret = max8925_reg_write(chip->i2c, MAX8925_WLED_CNTL, value);
+	ret = max8925_reg_write(chip->i2c, data->reg_cntl, value);
 	if (ret < 0)
 		goto out;
 
 	if (!data->current_brightness && brightness)
 		/* enable WLED output */
-		ret = max8925_set_bits(chip->i2c, MAX8925_WLED_MODE_CNTL, 1, 1);
+		ret = max8925_set_bits(chip->i2c, data->reg_mode_cntl, 1, 1);
 	else if (!brightness)
 		/* disable WLED output */
-		ret = max8925_set_bits(chip->i2c, MAX8925_WLED_MODE_CNTL, 1, 0);
+		ret = max8925_set_bits(chip->i2c, data->reg_mode_cntl, 1, 0);
 	if (ret < 0)
 		goto out;
 	dev_dbg(chip->dev, "set brightness %d\n", value);
@@ -85,7 +87,7 @@ static int max8925_backlight_get_brightness(struct backlight_device *bl)
 	struct max8925_chip *chip = data->chip;
 	int ret;
 
-	ret = max8925_reg_read(chip->i2c, MAX8925_WLED_CNTL);
+	ret = max8925_reg_read(chip->i2c, data->reg_cntl);
 	if (ret < 0)
 		return -EINVAL;
 	data->current_brightness = ret;
@@ -102,69 +104,70 @@ static const struct backlight_ops max8925_backlight_ops = {
 static int __devinit max8925_backlight_probe(struct platform_device *pdev)
 {
 	struct max8925_chip *chip = dev_get_drvdata(pdev->dev.parent);
-	struct max8925_platform_data *max8925_pdata;
-	struct max8925_backlight_pdata *pdata = NULL;
+	struct max8925_backlight_pdata *pdata = pdev->dev.platform_data;
 	struct max8925_backlight_data *data;
 	struct backlight_device *bl;
 	struct backlight_properties props;
 	struct resource *res;
-	char name[MAX8925_NAME_SIZE];
 	unsigned char value;
-	int ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "No I/O resource!\n");
-		return -EINVAL;
-	}
-
-	if (pdev->dev.parent->platform_data) {
-		max8925_pdata = pdev->dev.parent->platform_data;
-		pdata = max8925_pdata->backlight;
-	}
-
-	if (!pdata) {
-		dev_err(&pdev->dev, "platform data isn't assigned to "
-			"backlight\n");
-		return -EINVAL;
-	}
+	int ret = 0;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(struct max8925_backlight_data),
 			    GFP_KERNEL);
 	if (data == NULL)
 		return -ENOMEM;
-	strncpy(name, res->name, MAX8925_NAME_SIZE);
+
+	res = platform_get_resource(pdev, IORESOURCE_REG, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for mode control!\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_mode_cntl = res->start;
+	res = platform_get_resource(pdev, IORESOURCE_REG, 1);
+	if (!res) {
+		dev_err(&pdev->dev, "No REG resource for control!\n");
+		ret = -ENXIO;
+		goto out;
+	}
+	data->reg_cntl = res->start;
+
 	data->chip = chip;
 	data->current_brightness = 0;
 
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_RAW;
 	props.max_brightness = MAX_BRIGHTNESS;
-	bl = backlight_device_register(name, &pdev->dev, data,
+	bl = backlight_device_register("max8925-backlight", &pdev->dev, data,
 					&max8925_backlight_ops, &props);
 	if (IS_ERR(bl)) {
 		dev_err(&pdev->dev, "failed to register backlight\n");
-		return PTR_ERR(bl);
+		ret = PTR_ERR(bl);
+		goto out;
 	}
 	bl->props.brightness = MAX_BRIGHTNESS;
 
 	platform_set_drvdata(pdev, bl);
 
 	value = 0;
-	if (pdata->lxw_scl)
-		value |= (1 << 7);
-	if (pdata->lxw_freq)
-		value |= (LWX_FREQ(pdata->lxw_freq) << 4);
-	if (pdata->dual_string)
-		value |= (1 << 1);
-	ret = max8925_set_bits(chip->i2c, MAX8925_WLED_MODE_CNTL, 0xfe, value);
+	if (pdata) {
+		if (pdata->lxw_scl)
+			value |= (1 << 7);
+		if (pdata->lxw_freq)
+			value |= (LWX_FREQ(pdata->lxw_freq) << 4);
+		if (pdata->dual_string)
+			value |= (1 << 1);
+	}
+	ret = max8925_set_bits(chip->i2c, data->reg_mode_cntl, 0xfe, value);
 	if (ret < 0)
-		goto out;
+		goto out_brt;
 
 	backlight_update_status(bl);
 	return 0;
-out:
+out_brt:
 	backlight_device_unregister(bl);
+out:
+	devm_kfree(&pdev->dev, data);
 	return ret;
 }
 
diff --git a/drivers/video/backlight/platform_lcd.c b/drivers/video/backlight/platform_lcd.c
index b6672340d6c..ca4f5d70fe1 100644
--- a/drivers/video/backlight/platform_lcd.c
+++ b/drivers/video/backlight/platform_lcd.c
@@ -16,6 +16,7 @@
 #include <linux/fb.h>
 #include <linux/backlight.h>
 #include <linux/lcd.h>
+#include <linux/of.h>
 #include <linux/slab.h>
 
 #include <video/platform_lcd.h>
@@ -145,6 +146,14 @@ static SIMPLE_DEV_PM_OPS(platform_lcd_pm_ops, platform_lcd_suspend,
 			platform_lcd_resume);
 #endif
 
+#ifdef CONFIG_OF
+static const struct of_device_id platform_lcd_of_match[] = {
+	{ .compatible = "platform-lcd" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, platform_lcd_of_match);
+#endif
+
 static struct platform_driver platform_lcd_driver = {
 	.driver		= {
 		.name	= "platform-lcd",
@@ -152,6 +161,7 @@ static struct platform_driver platform_lcd_driver = {
 #ifdef CONFIG_PM
 		.pm	= &platform_lcd_pm_ops,
 #endif
+		.of_match_table = of_match_ptr(platform_lcd_of_match),
 	},
 	.probe		= platform_lcd_probe,
 	.remove		= __devexit_p(platform_lcd_remove),
diff --git a/drivers/video/backlight/progear_bl.c b/drivers/video/backlight/progear_bl.c
deleted file mode 100644
index 69b35f02929..00000000000
--- a/drivers/video/backlight/progear_bl.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- *  Backlight Driver for Frontpath ProGear HX1050+
- *
- *  Copyright (c) 2006 Marcin Juszkiewicz
- *
- *  Based on Progear LCD driver by M Schacht
- *  <mschacht at alumni dot washington dot edu>
- *
- *  Based on Sharp's Corgi Backlight Driver
- *  Based on Backlight Driver for HP Jornada 680
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- *
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/platform_device.h>
-#include <linux/mutex.h>
-#include <linux/fb.h>
-#include <linux/backlight.h>
-#include <linux/pci.h>
-
-#define PMU_LPCR               0xB0
-#define SB_MPS1                0x61
-#define HW_LEVEL_MAX           0x77
-#define HW_LEVEL_MIN           0x4f
-
-static struct pci_dev *pmu_dev = NULL;
-static struct pci_dev *sb_dev = NULL;
-
-static int progearbl_set_intensity(struct backlight_device *bd)
-{
-	int intensity = bd->props.brightness;
-
-	if (bd->props.power != FB_BLANK_UNBLANK)
-		intensity = 0;
-	if (bd->props.fb_blank != FB_BLANK_UNBLANK)
-		intensity = 0;
-
-	pci_write_config_byte(pmu_dev, PMU_LPCR, intensity + HW_LEVEL_MIN);
-
-	return 0;
-}
-
-static int progearbl_get_intensity(struct backlight_device *bd)
-{
-	u8 intensity;
-	pci_read_config_byte(pmu_dev, PMU_LPCR, &intensity);
-
-	return intensity - HW_LEVEL_MIN;
-}
-
-static const struct backlight_ops progearbl_ops = {
-	.get_brightness = progearbl_get_intensity,
-	.update_status = progearbl_set_intensity,
-};
-
-static int progearbl_probe(struct platform_device *pdev)
-{
-	struct backlight_properties props;
-	u8 temp;
-	struct backlight_device *progear_backlight_device;
-	int ret;
-
-	pmu_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M7101, NULL);
-	if (!pmu_dev) {
-		pr_err("ALI M7101 PMU not found.\n");
-		return -ENODEV;
-	}
-
-	sb_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL);
-	if (!sb_dev) {
-		pr_err("ALI 1533 SB not found.\n");
-		ret = -ENODEV;
-		goto put_pmu;
-	}
-
-	/*     Set SB_MPS1 to enable brightness control. */
-	pci_read_config_byte(sb_dev, SB_MPS1, &temp);
-	pci_write_config_byte(sb_dev, SB_MPS1, temp | 0x20);
-
-	memset(&props, 0, sizeof(struct backlight_properties));
-	props.type = BACKLIGHT_RAW;
-	props.max_brightness = HW_LEVEL_MAX - HW_LEVEL_MIN;
-	progear_backlight_device = backlight_device_register("progear-bl",
-							     &pdev->dev, NULL,
-							     &progearbl_ops,
-							     &props);
-	if (IS_ERR(progear_backlight_device)) {
-		ret = PTR_ERR(progear_backlight_device);
-		goto put_sb;
-	}
-
-	platform_set_drvdata(pdev, progear_backlight_device);
-
-	progear_backlight_device->props.power = FB_BLANK_UNBLANK;
-	progear_backlight_device->props.brightness = HW_LEVEL_MAX - HW_LEVEL_MIN;
-	progearbl_set_intensity(progear_backlight_device);
-
-	return 0;
-put_sb:
-	pci_dev_put(sb_dev);
-put_pmu:
-	pci_dev_put(pmu_dev);
-	return ret;
-}
-
-static int progearbl_remove(struct platform_device *pdev)
-{
-	struct backlight_device *bd = platform_get_drvdata(pdev);
-	backlight_device_unregister(bd);
-
-	return 0;
-}
-
-static struct platform_driver progearbl_driver = {
-	.probe = progearbl_probe,
-	.remove = progearbl_remove,
-	.driver = {
-		   .name = "progear-bl",
-		   },
-};
-
-static struct platform_device *progearbl_device;
-
-static int __init progearbl_init(void)
-{
-	int ret = platform_driver_register(&progearbl_driver);
-
-	if (ret)
-		return ret;
-	progearbl_device = platform_device_register_simple("progear-bl", -1,
-								NULL, 0);
-	if (IS_ERR(progearbl_device)) {
-		platform_driver_unregister(&progearbl_driver);
-		return PTR_ERR(progearbl_device);
-	}
-
-	return 0;
-}
-
-static void __exit progearbl_exit(void)
-{
-	pci_dev_put(pmu_dev);
-	pci_dev_put(sb_dev);
-
-	platform_device_unregister(progearbl_device);
-	platform_driver_unregister(&progearbl_driver);
-}
-
-module_init(progearbl_init);
-module_exit(progearbl_exit);
-
-MODULE_AUTHOR("Marcin Juszkiewicz <linux@hrw.one.pl>");
-MODULE_DESCRIPTION("ProGear Backlight Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/video/backlight/tps65217_bl.c b/drivers/video/backlight/tps65217_bl.c
new file mode 100644
index 00000000000..70881633b45
--- /dev/null
+++ b/drivers/video/backlight/tps65217_bl.c
@@ -0,0 +1,342 @@
+/*
+ * tps65217_bl.c
+ *
+ * TPS65217 backlight driver
+ *
+ * Copyright (C) 2012 Matthias Kaehlcke
+ * Author: Matthias Kaehlcke <matthias@kaehlcke.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/backlight.h>
+#include <linux/err.h>
+#include <linux/fb.h>
+#include <linux/mfd/tps65217.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+struct tps65217_bl {
+	struct tps65217 *tps;
+	struct device *dev;
+	struct backlight_device *bl;
+	bool is_enabled;
+};
+
+static int tps65217_bl_enable(struct tps65217_bl *tps65217_bl)
+{
+	int rc;
+
+	rc = tps65217_set_bits(tps65217_bl->tps, TPS65217_REG_WLEDCTRL1,
+			TPS65217_WLEDCTRL1_ISINK_ENABLE,
+			TPS65217_WLEDCTRL1_ISINK_ENABLE, TPS65217_PROTECT_NONE);
+	if (rc) {
+		dev_err(tps65217_bl->dev,
+			"failed to enable backlight: %d\n", rc);
+		return rc;
+	}
+
+	tps65217_bl->is_enabled = true;
+
+	dev_dbg(tps65217_bl->dev, "backlight enabled\n");
+
+	return 0;
+}
+
+static int tps65217_bl_disable(struct tps65217_bl *tps65217_bl)
+{
+	int rc;
+
+	rc = tps65217_clear_bits(tps65217_bl->tps,
+				TPS65217_REG_WLEDCTRL1,
+				TPS65217_WLEDCTRL1_ISINK_ENABLE,
+				TPS65217_PROTECT_NONE);
+	if (rc) {
+		dev_err(tps65217_bl->dev,
+			"failed to disable backlight: %d\n", rc);
+		return rc;
+	}
+
+	tps65217_bl->is_enabled = false;
+
+	dev_dbg(tps65217_bl->dev, "backlight disabled\n");
+
+	return 0;
+}
+
+static int tps65217_bl_update_status(struct backlight_device *bl)
+{
+	struct tps65217_bl *tps65217_bl = bl_get_data(bl);
+	int rc;
+	int brightness = bl->props.brightness;
+
+	if (bl->props.state & BL_CORE_SUSPENDED)
+		brightness = 0;
+
+	if ((bl->props.power != FB_BLANK_UNBLANK) ||
+		(bl->props.fb_blank != FB_BLANK_UNBLANK))
+		/* framebuffer in low power mode or blanking active */
+		brightness = 0;
+
+	if (brightness > 0) {
+		rc = tps65217_reg_write(tps65217_bl->tps,
+					TPS65217_REG_WLEDCTRL2,
+					brightness - 1,
+					TPS65217_PROTECT_NONE);
+		if (rc) {
+			dev_err(tps65217_bl->dev,
+				"failed to set brightness level: %d\n", rc);
+			return rc;
+		}
+
+		dev_dbg(tps65217_bl->dev, "brightness set to %d\n", brightness);
+
+		if (!tps65217_bl->is_enabled)
+			rc = tps65217_bl_enable(tps65217_bl);
+	} else {
+		rc = tps65217_bl_disable(tps65217_bl);
+	}
+
+	return rc;
+}
+
+static int tps65217_bl_get_brightness(struct backlight_device *bl)
+{
+	return bl->props.brightness;
+}
+
+static const struct backlight_ops tps65217_bl_ops = {
+	.options	= BL_CORE_SUSPENDRESUME,
+	.update_status	= tps65217_bl_update_status,
+	.get_brightness	= tps65217_bl_get_brightness
+};
+
+static int tps65217_bl_hw_init(struct tps65217_bl *tps65217_bl,
+			struct tps65217_bl_pdata *pdata)
+{
+	int rc;
+
+	rc = tps65217_bl_disable(tps65217_bl);
+	if (rc)
+		return rc;
+
+	switch (pdata->isel) {
+	case TPS65217_BL_ISET1:
+		/* select ISET_1 current level */
+		rc = tps65217_clear_bits(tps65217_bl->tps,
+					TPS65217_REG_WLEDCTRL1,
+					TPS65217_WLEDCTRL1_ISEL,
+					TPS65217_PROTECT_NONE);
+		if (rc) {
+			dev_err(tps65217_bl->dev,
+				"failed to select ISET1 current level: %d)\n",
+				rc);
+			return rc;
+		}
+
+		dev_dbg(tps65217_bl->dev, "selected ISET1 current level\n");
+
+		break;
+
+	case TPS65217_BL_ISET2:
+		/* select ISET2 current level */
+		rc = tps65217_set_bits(tps65217_bl->tps, TPS65217_REG_WLEDCTRL1,
+				TPS65217_WLEDCTRL1_ISEL,
+				TPS65217_WLEDCTRL1_ISEL, TPS65217_PROTECT_NONE);
+		if (rc) {
+			dev_err(tps65217_bl->dev,
+				"failed to select ISET2 current level: %d\n",
+				rc);
+			return rc;
+		}
+
+		dev_dbg(tps65217_bl->dev, "selected ISET2 current level\n");
+
+		break;
+
+	default:
+		dev_err(tps65217_bl->dev,
+			"invalid value for current level: %d\n", pdata->isel);
+		return -EINVAL;
+	}
+
+	/* set PWM frequency */
+	rc = tps65217_set_bits(tps65217_bl->tps,
+			TPS65217_REG_WLEDCTRL1,
+			TPS65217_WLEDCTRL1_FDIM_MASK,
+			pdata->fdim,
+			TPS65217_PROTECT_NONE);
+	if (rc) {
+		dev_err(tps65217_bl->dev,
+			"failed to select PWM dimming frequency: %d\n",
+			rc);
+		return rc;
+	}
+
+	return 0;
+}
+
+#ifdef CONFIG_OF
+static struct tps65217_bl_pdata *
+tps65217_bl_parse_dt(struct platform_device *pdev)
+{
+	struct tps65217 *tps = dev_get_drvdata(pdev->dev.parent);
+	struct device_node *node = of_node_get(tps->dev->of_node);
+	struct tps65217_bl_pdata *pdata, *err;
+	u32 val;
+
+	node = of_find_node_by_name(node, "backlight");
+	if (!node)
+		return ERR_PTR(-ENODEV);
+
+	pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata) {
+		dev_err(&pdev->dev, "failed to allocate platform data\n");
+		err = ERR_PTR(-ENOMEM);
+		goto err;
+	}
+
+	pdata->isel = TPS65217_BL_ISET1;
+	if (!of_property_read_u32(node, "isel", &val)) {
+		if (val < TPS65217_BL_ISET1 ||
+			val > TPS65217_BL_ISET2) {
+			dev_err(&pdev->dev,
+				"invalid 'isel' value in the device tree\n");
+			err = ERR_PTR(-EINVAL);
+			goto err;
+		}
+
+		pdata->isel = val;
+	}
+
+	pdata->fdim = TPS65217_BL_FDIM_200HZ;
+	if (!of_property_read_u32(node, "fdim", &val)) {
+		switch (val) {
+		case 100:
+			pdata->fdim = TPS65217_BL_FDIM_100HZ;
+			break;
+
+		case 200:
+			pdata->fdim = TPS65217_BL_FDIM_200HZ;
+			break;
+
+		case 500:
+			pdata->fdim = TPS65217_BL_FDIM_500HZ;
+			break;
+
+		case 1000:
+			pdata->fdim = TPS65217_BL_FDIM_1000HZ;
+			break;
+
+		default:
+			dev_err(&pdev->dev,
+				"invalid 'fdim' value in the device tree\n");
+			err = ERR_PTR(-EINVAL);
+			goto err;
+		}
+	}
+
+	of_node_put(node);
+
+	return pdata;
+
+err:
+	of_node_put(node);
+
+	return err;
+}
+#else
+static struct tps65217_bl_pdata *
+tps65217_bl_parse_dt(struct platform_device *pdev)
+{
+	return NULL;
+}
+#endif
+
+static int tps65217_bl_probe(struct platform_device *pdev)
+{
+	int rc;
+	struct tps65217 *tps = dev_get_drvdata(pdev->dev.parent);
+	struct tps65217_bl *tps65217_bl;
+	struct tps65217_bl_pdata *pdata;
+	struct backlight_properties bl_props;
+
+	if (tps->dev->of_node) {
+		pdata = tps65217_bl_parse_dt(pdev);
+		if (IS_ERR(pdata))
+			return PTR_ERR(pdata);
+	} else {
+		if (!pdev->dev.platform_data) {
+			dev_err(&pdev->dev, "no platform data provided\n");
+			return -EINVAL;
+		}
+
+		pdata = pdev->dev.platform_data;
+	}
+
+	tps65217_bl = devm_kzalloc(&pdev->dev, sizeof(*tps65217_bl),
+				GFP_KERNEL);
+	if (tps65217_bl == NULL) {
+		dev_err(&pdev->dev, "allocation of struct tps65217_bl failed\n");
+		return -ENOMEM;
+	}
+
+	tps65217_bl->tps = tps;
+	tps65217_bl->dev = &pdev->dev;
+	tps65217_bl->is_enabled = false;
+
+	rc = tps65217_bl_hw_init(tps65217_bl, pdata);
+	if (rc)
+		return rc;
+
+	memset(&bl_props, 0, sizeof(struct backlight_properties));
+	bl_props.type = BACKLIGHT_RAW;
+	bl_props.max_brightness = 100;
+
+	tps65217_bl->bl = backlight_device_register(pdev->name,
+						tps65217_bl->dev, tps65217_bl,
+						&tps65217_bl_ops, &bl_props);
+	if (IS_ERR(tps65217_bl->bl)) {
+		dev_err(tps65217_bl->dev,
+			"registration of backlight device failed: %d\n", rc);
+		return PTR_ERR(tps65217_bl->bl);
+	}
+
+	tps65217_bl->bl->props.brightness = 0;
+	platform_set_drvdata(pdev, tps65217_bl);
+
+	return 0;
+}
+
+static int tps65217_bl_remove(struct platform_device *pdev)
+{
+	struct tps65217_bl *tps65217_bl = platform_get_drvdata(pdev);
+
+	backlight_device_unregister(tps65217_bl->bl);
+
+	return 0;
+}
+
+static struct platform_driver tps65217_bl_driver = {
+	.probe		= tps65217_bl_probe,
+	.remove		= tps65217_bl_remove,
+	.driver		= {
+		.owner	= THIS_MODULE,
+		.name	= "tps65217-bl",
+	},
+};
+
+module_platform_driver(tps65217_bl_driver);
+
+MODULE_DESCRIPTION("TPS65217 Backlight driver");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Matthias Kaehlcke <matthias@kaehlcke.net>");
diff --git a/drivers/video/geode/gx1fb_core.c b/drivers/video/geode/gx1fb_core.c
index 5a5d0928df3..265c5ed59ad 100644
--- a/drivers/video/geode/gx1fb_core.c
+++ b/drivers/video/geode/gx1fb_core.c
@@ -29,7 +29,7 @@ static int  crt_option = 1;
 static char panel_option[32] = "";
 
 /* Modes relevant to the GX1 (taken from modedb.c) */
-static const struct fb_videomode __devinitdata gx1_modedb[] = {
+static const struct fb_videomode __devinitconst gx1_modedb[] = {
 	/* 640x480-60 VESA */
 	{ NULL, 60, 640, 480, 39682,  48, 16, 33, 10, 96, 2,
 	  0, FB_VMODE_NONINTERLACED, FB_MODE_IS_VESA },
diff --git a/drivers/video/gxt4500.c b/drivers/video/gxt4500.c
index 0fad23f810a..0e9afa41d16 100644
--- a/drivers/video/gxt4500.c
+++ b/drivers/video/gxt4500.c
@@ -156,7 +156,7 @@ struct gxt4500_par {
 static char *mode_option;
 
 /* default mode: 1280x1024 @ 60 Hz, 8 bpp */
-static const struct fb_videomode defaultmode __devinitdata = {
+static const struct fb_videomode defaultmode __devinitconst = {
 	.refresh = 60,
 	.xres = 1280,
 	.yres = 1024,
@@ -581,7 +581,7 @@ static int gxt4500_blank(int blank, struct fb_info *info)
 	return 0;
 }
 
-static const struct fb_fix_screeninfo gxt4500_fix __devinitdata = {
+static const struct fb_fix_screeninfo gxt4500_fix __devinitconst = {
 	.id = "IBM GXT4500P",
 	.type = FB_TYPE_PACKED_PIXELS,
 	.visual = FB_VISUAL_PSEUDOCOLOR,
diff --git a/drivers/video/i810/i810_main.c b/drivers/video/i810/i810_main.c
index b83f36190ca..5c067816a81 100644
--- a/drivers/video/i810/i810_main.c
+++ b/drivers/video/i810/i810_main.c
@@ -97,7 +97,7 @@ static int i810fb_blank      (int blank_mode, struct fb_info *info);
 static void i810fb_release_resource       (struct fb_info *info, struct i810fb_par *par);
 
 /* PCI */
-static const char *i810_pci_list[] __devinitdata = {
+static const char * const i810_pci_list[] __devinitconst = {
 	"Intel(R) 810 Framebuffer Device"                                 ,
 	"Intel(R) 810-DC100 Framebuffer Device"                           ,
 	"Intel(R) 810E Framebuffer Device"                                ,
diff --git a/drivers/video/jz4740_fb.c b/drivers/video/jz4740_fb.c
index de366937c93..3c63fc24bb1 100644
--- a/drivers/video/jz4740_fb.c
+++ b/drivers/video/jz4740_fb.c
@@ -136,7 +136,7 @@ struct jzfb {
 	uint32_t pseudo_palette[16];
 };
 
-static const struct fb_fix_screeninfo jzfb_fix __devinitdata = {
+static const struct fb_fix_screeninfo jzfb_fix __devinitconst = {
 	.id		= "JZ4740 FB",
 	.type		= FB_TYPE_PACKED_PIXELS,
 	.visual		= FB_VISUAL_TRUECOLOR,
diff --git a/drivers/video/msm/mdp.c b/drivers/video/msm/mdp.c
index d1f881e8030..2e0f3bab611 100644
--- a/drivers/video/msm/mdp.c
+++ b/drivers/video/msm/mdp.c
@@ -257,19 +257,17 @@ int get_img(struct mdp_img *img, struct fb_info *info,
 	    unsigned long *start, unsigned long *len,
 	    struct file **filep)
 {
-	int put_needed, ret = 0;
-	struct file *file;
-
-	file = fget_light(img->memory_id, &put_needed);
-	if (file == NULL)
+	int ret = 0;
+	struct fd f = fdget(img->memory_id);
+	if (f.file == NULL)
 		return -1;
 
-	if (MAJOR(file->f_dentry->d_inode->i_rdev) == FB_MAJOR) {
+	if (MAJOR(f.file->f_dentry->d_inode->i_rdev) == FB_MAJOR) {
 		*start = info->fix.smem_start;
 		*len = info->fix.smem_len;
 	} else
 		ret = -1;
-	fput_light(file, put_needed);
+	fdput(f);
 
 	return ret;
 }
diff --git a/drivers/watchdog/iTCO_wdt.c b/drivers/watchdog/iTCO_wdt.c
index ceed39f2601..545d387de41 100644
--- a/drivers/watchdog/iTCO_wdt.c
+++ b/drivers/watchdog/iTCO_wdt.c
@@ -37,6 +37,7 @@
  *	document number TBD                   : DH89xxCC
  *	document number TBD                   : Panther Point
  *	document number TBD                   : Lynx Point
+ *	document number TBD                   : Lynx Point-LP
  */
 
 /*
diff --git a/drivers/xen/events.c b/drivers/xen/events.c
index 7595581d032..c60d1629c91 100644
--- a/drivers/xen/events.c
+++ b/drivers/xen/events.c
@@ -373,11 +373,22 @@ static void unmask_evtchn(int port)
 {
 	struct shared_info *s = HYPERVISOR_shared_info;
 	unsigned int cpu = get_cpu();
+	int do_hypercall = 0, evtchn_pending = 0;
 
 	BUG_ON(!irqs_disabled());
 
-	/* Slow path (hypercall) if this is a non-local port. */
-	if (unlikely(cpu != cpu_from_evtchn(port))) {
+	if (unlikely((cpu != cpu_from_evtchn(port))))
+		do_hypercall = 1;
+	else
+		evtchn_pending = sync_test_bit(port, &s->evtchn_pending[0]);
+
+	if (unlikely(evtchn_pending && xen_hvm_domain()))
+		do_hypercall = 1;
+
+	/* Slow path (hypercall) if this is a non-local port or if this is
+	 * an hvm domain and an event is pending (hvm domains don't have
+	 * their own implementation of irq_enable). */
+	if (do_hypercall) {
 		struct evtchn_unmask unmask = { .port = port };
 		(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
 	} else {
@@ -390,7 +401,7 @@ static void unmask_evtchn(int port)
 		 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
 		 * the interrupt edge' if the channel is masked.
 		 */
-		if (sync_test_bit(port, &s->evtchn_pending[0]) &&
+		if (evtchn_pending &&
 		    !sync_test_and_set_bit(port / BITS_PER_LONG,
 					   &vcpu_info->evtchn_pending_sel))
 			vcpu_info->evtchn_upcall_pending = 1;
@@ -831,6 +842,7 @@ int bind_evtchn_to_irq(unsigned int evtchn)
 		struct irq_info *info = info_for_irq(irq);
 		WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
 	}
+	irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
 
 out:
 	mutex_unlock(&irq_mapping_update_lock);
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index 7f124160848..5df9fd847b2 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -446,7 +446,7 @@ static void mn_release(struct mmu_notifier *mn,
 	spin_unlock(&priv->lock);
 }
 
-struct mmu_notifier_ops gntdev_mmu_ops = {
+static struct mmu_notifier_ops gntdev_mmu_ops = {
 	.release                = mn_release,
 	.invalidate_page        = mn_invl_page,
 	.invalidate_range_start = mn_invl_range_start,
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 006726688ba..b2b0a375b34 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -38,6 +38,7 @@
 #include <linux/vmalloc.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
+#include <linux/delay.h>
 #include <linux/hardirq.h>
 
 #include <xen/xen.h>
@@ -47,6 +48,7 @@
 #include <xen/interface/memory.h>
 #include <xen/hvc-console.h>
 #include <asm/xen/hypercall.h>
+#include <asm/xen/interface.h>
 
 #include <asm/pgtable.h>
 #include <asm/sync_bitops.h>
@@ -285,10 +287,9 @@ int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 }
 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 
-void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
-				    unsigned long frame, int flags,
-				    unsigned page_off,
-				    unsigned length)
+static void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
+					   unsigned long frame, int flags,
+					   unsigned page_off, unsigned length)
 {
 	gnttab_shared.v2[ref].sub_page.frame = frame;
 	gnttab_shared.v2[ref].sub_page.page_off = page_off;
@@ -345,9 +346,9 @@ bool gnttab_subpage_grants_available(void)
 }
 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 
-void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
-				  int flags, domid_t trans_domid,
-				  grant_ref_t trans_gref)
+static void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
+					 int flags, domid_t trans_domid,
+					 grant_ref_t trans_gref)
 {
 	gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 	gnttab_shared.v2[ref].transitive.gref = trans_gref;
@@ -823,6 +824,52 @@ unsigned int gnttab_max_grant_frames(void)
 }
 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 
+/* Handling of paged out grant targets (GNTST_eagain) */
+#define MAX_DELAY 256
+static inline void
+gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
+						const char *func)
+{
+	unsigned delay = 1;
+
+	do {
+		BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
+		if (*status == GNTST_eagain)
+			msleep(delay++);
+	} while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
+
+	if (delay >= MAX_DELAY) {
+		printk(KERN_ERR "%s: %s eagain grant\n", func, current->comm);
+		*status = GNTST_bad_page;
+	}
+}
+
+void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
+{
+	struct gnttab_map_grant_ref *op;
+
+	if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
+		BUG();
+	for (op = batch; op < batch + count; op++)
+		if (op->status == GNTST_eagain)
+			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
+						&op->status, __func__);
+}
+EXPORT_SYMBOL_GPL(gnttab_batch_map);
+
+void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
+{
+	struct gnttab_copy *op;
+
+	if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
+		BUG();
+	for (op = batch; op < batch + count; op++)
+		if (op->status == GNTST_eagain)
+			gnttab_retry_eagain_gop(GNTTABOP_copy, op,
+						&op->status, __func__);
+}
+EXPORT_SYMBOL_GPL(gnttab_batch_copy);
+
 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 		    struct gnttab_map_grant_ref *kmap_ops,
 		    struct page **pages, unsigned int count)
@@ -836,6 +883,12 @@ int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 	if (ret)
 		return ret;
 
+	/* Retry eagain maps */
+	for (i = 0; i < count; i++)
+		if (map_ops[i].status == GNTST_eagain)
+			gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
+						&map_ops[i].status, __func__);
+
 	if (xen_feature(XENFEAT_auto_translated_physmap))
 		return ret;
 
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index ccee0f16bcf..ef6389580b8 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -76,7 +76,7 @@ static void free_page_list(struct list_head *pages)
  */
 static int gather_array(struct list_head *pagelist,
 			unsigned nelem, size_t size,
-			void __user *data)
+			const void __user *data)
 {
 	unsigned pageidx;
 	void *pagedata;
@@ -246,61 +246,117 @@ struct mmap_batch_state {
 	domid_t domain;
 	unsigned long va;
 	struct vm_area_struct *vma;
-	int err;
-
-	xen_pfn_t __user *user;
+	/* A tristate:
+	 *      0 for no errors
+	 *      1 if at least one error has happened (and no
+	 *          -ENOENT errors have happened)
+	 *      -ENOENT if at least 1 -ENOENT has happened.
+	 */
+	int global_error;
+	/* An array for individual errors */
+	int *err;
+
+	/* User-space mfn array to store errors in the second pass for V1. */
+	xen_pfn_t __user *user_mfn;
 };
 
 static int mmap_batch_fn(void *data, void *state)
 {
 	xen_pfn_t *mfnp = data;
 	struct mmap_batch_state *st = state;
+	int ret;
+
+	ret = xen_remap_domain_mfn_range(st->vma, st->va & PAGE_MASK, *mfnp, 1,
+					 st->vma->vm_page_prot, st->domain);
 
-	if (xen_remap_domain_mfn_range(st->vma, st->va & PAGE_MASK, *mfnp, 1,
-				       st->vma->vm_page_prot, st->domain) < 0) {
-		*mfnp |= 0xf0000000U;
-		st->err++;
+	/* Store error code for second pass. */
+	*(st->err++) = ret;
+
+	/* And see if it affects the global_error. */
+	if (ret < 0) {
+		if (ret == -ENOENT)
+			st->global_error = -ENOENT;
+		else {
+			/* Record that at least one error has happened. */
+			if (st->global_error == 0)
+				st->global_error = 1;
+		}
 	}
 	st->va += PAGE_SIZE;
 
 	return 0;
 }
 
-static int mmap_return_errors(void *data, void *state)
+static int mmap_return_errors_v1(void *data, void *state)
 {
 	xen_pfn_t *mfnp = data;
 	struct mmap_batch_state *st = state;
-
-	return put_user(*mfnp, st->user++);
+	int err = *(st->err++);
+
+	/*
+	 * V1 encodes the error codes in the 32bit top nibble of the
+	 * mfn (with its known limitations vis-a-vis 64 bit callers).
+	 */
+	*mfnp |= (err == -ENOENT) ?
+				PRIVCMD_MMAPBATCH_PAGED_ERROR :
+				PRIVCMD_MMAPBATCH_MFN_ERROR;
+	return __put_user(*mfnp, st->user_mfn++);
 }
 
 static struct vm_operations_struct privcmd_vm_ops;
 
-static long privcmd_ioctl_mmap_batch(void __user *udata)
+static long privcmd_ioctl_mmap_batch(void __user *udata, int version)
 {
 	int ret;
-	struct privcmd_mmapbatch m;
+	struct privcmd_mmapbatch_v2 m;
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 	unsigned long nr_pages;
 	LIST_HEAD(pagelist);
+	int *err_array = NULL;
 	struct mmap_batch_state state;
 
 	if (!xen_initial_domain())
 		return -EPERM;
 
-	if (copy_from_user(&m, udata, sizeof(m)))
-		return -EFAULT;
+	switch (version) {
+	case 1:
+		if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch)))
+			return -EFAULT;
+		/* Returns per-frame error in m.arr. */
+		m.err = NULL;
+		if (!access_ok(VERIFY_WRITE, m.arr, m.num * sizeof(*m.arr)))
+			return -EFAULT;
+		break;
+	case 2:
+		if (copy_from_user(&m, udata, sizeof(struct privcmd_mmapbatch_v2)))
+			return -EFAULT;
+		/* Returns per-frame error code in m.err. */
+		if (!access_ok(VERIFY_WRITE, m.err, m.num * (sizeof(*m.err))))
+			return -EFAULT;
+		break;
+	default:
+		return -EINVAL;
+	}
 
 	nr_pages = m.num;
 	if ((m.num <= 0) || (nr_pages > (LONG_MAX >> PAGE_SHIFT)))
 		return -EINVAL;
 
-	ret = gather_array(&pagelist, m.num, sizeof(xen_pfn_t),
-			   m.arr);
+	ret = gather_array(&pagelist, m.num, sizeof(xen_pfn_t), m.arr);
 
-	if (ret || list_empty(&pagelist))
+	if (ret)
 		goto out;
+	if (list_empty(&pagelist)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	err_array = kcalloc(m.num, sizeof(int), GFP_KERNEL);
+	if (err_array == NULL) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	down_write(&mm->mmap_sem);
 
@@ -315,24 +371,37 @@ static long privcmd_ioctl_mmap_batch(void __user *udata)
 		goto out;
 	}
 
-	state.domain = m.dom;
-	state.vma = vma;
-	state.va = m.addr;
-	state.err = 0;
+	state.domain        = m.dom;
+	state.vma           = vma;
+	state.va            = m.addr;
+	state.global_error  = 0;
+	state.err           = err_array;
 
-	ret = traverse_pages(m.num, sizeof(xen_pfn_t),
-			     &pagelist, mmap_batch_fn, &state);
+	/* mmap_batch_fn guarantees ret == 0 */
+	BUG_ON(traverse_pages(m.num, sizeof(xen_pfn_t),
+			     &pagelist, mmap_batch_fn, &state));
 
 	up_write(&mm->mmap_sem);
 
-	if (state.err > 0) {
-		state.user = m.arr;
+	if (state.global_error && (version == 1)) {
+		/* Write back errors in second pass. */
+		state.user_mfn = (xen_pfn_t *)m.arr;
+		state.err      = err_array;
 		ret = traverse_pages(m.num, sizeof(xen_pfn_t),
-			       &pagelist,
-			       mmap_return_errors, &state);
+				     &pagelist, mmap_return_errors_v1, &state);
+	} else if (version == 2) {
+		ret = __copy_to_user(m.err, err_array, m.num * sizeof(int));
+		if (ret)
+			ret = -EFAULT;
 	}
 
+	/* If we have not had any EFAULT-like global errors then set the global
+	 * error to -ENOENT if necessary. */
+	if ((ret == 0) && (state.global_error == -ENOENT))
+		ret = -ENOENT;
+
 out:
+	kfree(err_array);
 	free_page_list(&pagelist);
 
 	return ret;
@@ -354,7 +423,11 @@ static long privcmd_ioctl(struct file *file,
 		break;
 
 	case IOCTL_PRIVCMD_MMAPBATCH:
-		ret = privcmd_ioctl_mmap_batch(udata);
+		ret = privcmd_ioctl_mmap_batch(udata, 1);
+		break;
+
+	case IOCTL_PRIVCMD_MMAPBATCH_V2:
+		ret = privcmd_ioctl_mmap_batch(udata, 2);
 		break;
 
 	default:
@@ -380,10 +453,6 @@ static struct vm_operations_struct privcmd_vm_ops = {
 
 static int privcmd_mmap(struct file *file, struct vm_area_struct *vma)
 {
-	/* Unsupported for auto-translate guests. */
-	if (xen_feature(XENFEAT_auto_translated_physmap))
-		return -ENOSYS;
-
 	/* DONTCOPY is essential for Xen because copy_page_range doesn't know
 	 * how to recreate these mappings */
 	vma->vm_flags |= VM_RESERVED | VM_IO | VM_DONTCOPY | VM_PFNMAP;
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 4d519488d30..58db6df866e 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -52,7 +52,7 @@ static unsigned long xen_io_tlb_nslabs;
  * Quick lookup value of the bus address of the IOTLB.
  */
 
-u64 start_dma_addr;
+static u64 start_dma_addr;
 
 static dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
 {
@@ -144,31 +144,72 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
 	} while (i < nslabs);
 	return 0;
 }
+static unsigned long xen_set_nslabs(unsigned long nr_tbl)
+{
+	if (!nr_tbl) {
+		xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
+		xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE);
+	} else
+		xen_io_tlb_nslabs = nr_tbl;
 
-void __init xen_swiotlb_init(int verbose)
+	return xen_io_tlb_nslabs << IO_TLB_SHIFT;
+}
+
+enum xen_swiotlb_err {
+	XEN_SWIOTLB_UNKNOWN = 0,
+	XEN_SWIOTLB_ENOMEM,
+	XEN_SWIOTLB_EFIXUP
+};
+
+static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
+{
+	switch (err) {
+	case XEN_SWIOTLB_ENOMEM:
+		return "Cannot allocate Xen-SWIOTLB buffer\n";
+	case XEN_SWIOTLB_EFIXUP:
+		return "Failed to get contiguous memory for DMA from Xen!\n"\
+		    "You either: don't have the permissions, do not have"\
+		    " enough free memory under 4GB, or the hypervisor memory"\
+		    " is too fragmented!";
+	default:
+		break;
+	}
+	return "";
+}
+int __ref xen_swiotlb_init(int verbose, bool early)
 {
-	unsigned long bytes;
+	unsigned long bytes, order;
 	int rc = -ENOMEM;
-	unsigned long nr_tbl;
-	char *m = NULL;
+	enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
 	unsigned int repeat = 3;
 
-	nr_tbl = swiotlb_nr_tbl();
-	if (nr_tbl)
-		xen_io_tlb_nslabs = nr_tbl;
-	else {
-		xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
-		xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE);
-	}
+	xen_io_tlb_nslabs = swiotlb_nr_tbl();
 retry:
-	bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
-
+	bytes = xen_set_nslabs(xen_io_tlb_nslabs);
+	order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
 	/*
 	 * Get IO TLB memory from any location.
 	 */
-	xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
+	if (early)
+		xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
+	else {
+#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
+#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
+		while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
+			xen_io_tlb_start = (void *)__get_free_pages(__GFP_NOWARN, order);
+			if (xen_io_tlb_start)
+				break;
+			order--;
+		}
+		if (order != get_order(bytes)) {
+			pr_warn("Warning: only able to allocate %ld MB "
+				"for software IO TLB\n", (PAGE_SIZE << order) >> 20);
+			xen_io_tlb_nslabs = SLABS_PER_PAGE << order;
+			bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
+		}
+	}
 	if (!xen_io_tlb_start) {
-		m = "Cannot allocate Xen-SWIOTLB buffer!\n";
+		m_ret = XEN_SWIOTLB_ENOMEM;
 		goto error;
 	}
 	xen_io_tlb_end = xen_io_tlb_start + bytes;
@@ -179,17 +220,22 @@ retry:
 			       bytes,
 			       xen_io_tlb_nslabs);
 	if (rc) {
-		free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
-		m = "Failed to get contiguous memory for DMA from Xen!\n"\
-		    "You either: don't have the permissions, do not have"\
-		    " enough free memory under 4GB, or the hypervisor memory"\
-		    "is too fragmented!";
+		if (early)
+			free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
+		else {
+			free_pages((unsigned long)xen_io_tlb_start, order);
+			xen_io_tlb_start = NULL;
+		}
+		m_ret = XEN_SWIOTLB_EFIXUP;
 		goto error;
 	}
 	start_dma_addr = xen_virt_to_bus(xen_io_tlb_start);
-	swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose);
-
-	return;
+	if (early) {
+		swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose);
+		rc = 0;
+	} else
+		rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
+	return rc;
 error:
 	if (repeat--) {
 		xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */
@@ -198,10 +244,13 @@ error:
 		      (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20);
 		goto retry;
 	}
-	xen_raw_printk("%s (rc:%d)", m, rc);
-	panic("%s (rc:%d)", m, rc);
+	pr_err("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
+	if (early)
+		panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
+	else
+		free_pages((unsigned long)xen_io_tlb_start, order);
+	return rc;
 }
-
 void *
 xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
 			   dma_addr_t *dma_handle, gfp_t flags,
@@ -466,14 +515,6 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
 }
 EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg_attrs);
 
-int
-xen_swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-		   enum dma_data_direction dir)
-{
-	return xen_swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
-}
-EXPORT_SYMBOL_GPL(xen_swiotlb_map_sg);
-
 /*
  * Unmap a set of streaming mode DMA translations.  Again, cpu read rules
  * concerning calls here are the same as for swiotlb_unmap_page() above.
@@ -494,14 +535,6 @@ xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
 }
 EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs);
 
-void
-xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
-		     enum dma_data_direction dir)
-{
-	return xen_swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
-}
-EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg);
-
 /*
  * Make physical memory consistent for a set of streaming mode DMA translations
  * after a transfer.
diff --git a/drivers/xen/sys-hypervisor.c b/drivers/xen/sys-hypervisor.c
index fdb6d229c9b..5e5ad7e2885 100644
--- a/drivers/xen/sys-hypervisor.c
+++ b/drivers/xen/sys-hypervisor.c
@@ -114,7 +114,7 @@ static void xen_sysfs_version_destroy(void)
 
 /* UUID */
 
-static ssize_t uuid_show(struct hyp_sysfs_attr *attr, char *buffer)
+static ssize_t uuid_show_fallback(struct hyp_sysfs_attr *attr, char *buffer)
 {
 	char *vm, *val;
 	int ret;
@@ -135,6 +135,17 @@ static ssize_t uuid_show(struct hyp_sysfs_attr *attr, char *buffer)
 	return ret;
 }
 
+static ssize_t uuid_show(struct hyp_sysfs_attr *attr, char *buffer)
+{
+	xen_domain_handle_t uuid;
+	int ret;
+	ret = HYPERVISOR_xen_version(XENVER_guest_handle, uuid);
+	if (ret)
+		return uuid_show_fallback(attr, buffer);
+	ret = sprintf(buffer, "%pU\n", uuid);
+	return ret;
+}
+
 HYPERVISOR_ATTR_RO(uuid);
 
 static int __init xen_sysfs_uuid_init(void)
diff --git a/drivers/xen/tmem.c b/drivers/xen/tmem.c
index 89f264c6742..144564e5eb2 100644
--- a/drivers/xen/tmem.c
+++ b/drivers/xen/tmem.c
@@ -21,6 +21,7 @@
 #include <asm/xen/hypercall.h>
 #include <asm/xen/page.h>
 #include <asm/xen/hypervisor.h>
+#include <xen/tmem.h>
 
 #define TMEM_CONTROL               0
 #define TMEM_NEW_POOL              1
diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c
index b590ee067fc..316df65163c 100644
--- a/drivers/xen/xen-acpi-processor.c
+++ b/drivers/xen/xen-acpi-processor.c
@@ -98,7 +98,6 @@ static int push_cxx_to_hypervisor(struct acpi_processor *_pr)
 
 		dst_cx->type = cx->type;
 		dst_cx->latency = cx->latency;
-		dst_cx->power = cx->power;
 
 		dst_cx->dpcnt = 0;
 		set_xen_guest_handle(dst_cx->dp, NULL);
diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c
index 92ff01dbeb1..961d664e2d2 100644
--- a/drivers/xen/xen-pciback/pci_stub.c
+++ b/drivers/xen/xen-pciback/pci_stub.c
@@ -362,6 +362,7 @@ static int __devinit pcistub_init_device(struct pci_dev *dev)
 	else {
 		dev_dbg(&dev->dev, "reseting (FLR, D3, etc) the device\n");
 		__pci_reset_function_locked(dev);
+		pci_restore_state(dev);
 	}
 	/* Now disable the device (this also ensures some private device
 	 * data is setup before we export)
@@ -681,14 +682,14 @@ static pci_ers_result_t xen_pcibk_slot_reset(struct pci_dev *dev)
 		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 
 	if (!test_bit(_XEN_PCIB_AERHANDLER,
 		(unsigned long *)&psdev->pdev->sh_info->flags)) {
 		dev_err(&dev->dev,
 			"guest with no AER driver should have been killed\n");
-		goto release;
+		goto end;
 	}
 	result = common_process(psdev, 1, XEN_PCI_OP_aer_slotreset, result);
 
@@ -698,9 +699,9 @@ static pci_ers_result_t xen_pcibk_slot_reset(struct pci_dev *dev)
 			"No AER slot_reset service or disconnected!\n");
 		kill_domain_by_device(psdev);
 	}
-release:
-	pcistub_device_put(psdev);
 end:
+	if (psdev)
+		pcistub_device_put(psdev);
 	up_write(&pcistub_sem);
 	return result;
 
@@ -739,14 +740,14 @@ static pci_ers_result_t xen_pcibk_mmio_enabled(struct pci_dev *dev)
 		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 
 	if (!test_bit(_XEN_PCIB_AERHANDLER,
 		(unsigned long *)&psdev->pdev->sh_info->flags)) {
 		dev_err(&dev->dev,
 			"guest with no AER driver should have been killed\n");
-		goto release;
+		goto end;
 	}
 	result = common_process(psdev, 1, XEN_PCI_OP_aer_mmio, result);
 
@@ -756,9 +757,9 @@ static pci_ers_result_t xen_pcibk_mmio_enabled(struct pci_dev *dev)
 			"No AER mmio_enabled service or disconnected!\n");
 		kill_domain_by_device(psdev);
 	}
-release:
-	pcistub_device_put(psdev);
 end:
+	if (psdev)
+		pcistub_device_put(psdev);
 	up_write(&pcistub_sem);
 	return result;
 }
@@ -797,7 +798,7 @@ static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev,
 		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 
 	/*Guest owns the device yet no aer handler regiested, kill guest*/
@@ -805,7 +806,7 @@ static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev,
 		(unsigned long *)&psdev->pdev->sh_info->flags)) {
 		dev_dbg(&dev->dev, "guest may have no aer driver, kill it\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 	result = common_process(psdev, error, XEN_PCI_OP_aer_detected, result);
 
@@ -815,9 +816,9 @@ static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev,
 			"No AER error_detected service or disconnected!\n");
 		kill_domain_by_device(psdev);
 	}
-release:
-	pcistub_device_put(psdev);
 end:
+	if (psdev)
+		pcistub_device_put(psdev);
 	up_write(&pcistub_sem);
 	return result;
 }
@@ -851,7 +852,7 @@ static void xen_pcibk_error_resume(struct pci_dev *dev)
 		dev_err(&dev->dev, DRV_NAME " device is not connected or owned"
 			" by HVM, kill it\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 
 	if (!test_bit(_XEN_PCIB_AERHANDLER,
@@ -859,13 +860,13 @@ static void xen_pcibk_error_resume(struct pci_dev *dev)
 		dev_err(&dev->dev,
 			"guest with no AER driver should have been killed\n");
 		kill_domain_by_device(psdev);
-		goto release;
+		goto end;
 	}
 	common_process(psdev, 1, XEN_PCI_OP_aer_resume,
 		       PCI_ERS_RESULT_RECOVERED);
-release:
-	pcistub_device_put(psdev);
 end:
+	if (psdev)
+		pcistub_device_put(psdev);
 	up_write(&pcistub_sem);
 	return;
 }
@@ -897,17 +898,41 @@ static inline int str_to_slot(const char *buf, int *domain, int *bus,
 			      int *slot, int *func)
 {
 	int err;
+	char wc = '*';
 
 	err = sscanf(buf, " %x:%x:%x.%x", domain, bus, slot, func);
-	if (err == 4)
+	switch (err) {
+	case 3:
+		*func = -1;
+		err = sscanf(buf, " %x:%x:%x.%c", domain, bus, slot, &wc);
+		break;
+	case 2:
+		*slot = *func = -1;
+		err = sscanf(buf, " %x:%x:*.%c", domain, bus, &wc);
+		if (err >= 2)
+			++err;
+		break;
+	}
+	if (err == 4 && wc == '*')
 		return 0;
 	else if (err < 0)
 		return -EINVAL;
 
 	/* try again without domain */
 	*domain = 0;
+	wc = '*';
 	err = sscanf(buf, " %x:%x.%x", bus, slot, func);
-	if (err == 3)
+	switch (err) {
+	case 2:
+		*func = -1;
+		err = sscanf(buf, " %x:%x.%c", bus, slot, &wc);
+		break;
+	case 1:
+		*slot = *func = -1;
+		err = sscanf(buf, " %x:*.%c", bus, &wc) + 1;
+		break;
+	}
+	if (err == 3 && wc == '*')
 		return 0;
 
 	return -EINVAL;
@@ -930,6 +955,19 @@ static int pcistub_device_id_add(int domain, int bus, int slot, int func)
 {
 	struct pcistub_device_id *pci_dev_id;
 	unsigned long flags;
+	int rc = 0;
+
+	if (slot < 0) {
+		for (slot = 0; !rc && slot < 32; ++slot)
+			rc = pcistub_device_id_add(domain, bus, slot, func);
+		return rc;
+	}
+
+	if (func < 0) {
+		for (func = 0; !rc && func < 8; ++func)
+			rc = pcistub_device_id_add(domain, bus, slot, func);
+		return rc;
+	}
 
 	pci_dev_id = kmalloc(sizeof(*pci_dev_id), GFP_KERNEL);
 	if (!pci_dev_id)
@@ -952,15 +990,15 @@ static int pcistub_device_id_add(int domain, int bus, int slot, int func)
 static int pcistub_device_id_remove(int domain, int bus, int slot, int func)
 {
 	struct pcistub_device_id *pci_dev_id, *t;
-	int devfn = PCI_DEVFN(slot, func);
 	int err = -ENOENT;
 	unsigned long flags;
 
 	spin_lock_irqsave(&device_ids_lock, flags);
 	list_for_each_entry_safe(pci_dev_id, t, &pcistub_device_ids,
 				 slot_list) {
-		if (pci_dev_id->domain == domain
-		    && pci_dev_id->bus == bus && pci_dev_id->devfn == devfn) {
+		if (pci_dev_id->domain == domain && pci_dev_id->bus == bus
+		    && (slot < 0 || PCI_SLOT(pci_dev_id->devfn) == slot)
+		    && (func < 0 || PCI_FUNC(pci_dev_id->devfn) == func)) {
 			/* Don't break; here because it's possible the same
 			 * slot could be in the list more than once
 			 */
@@ -987,7 +1025,7 @@ static int pcistub_reg_add(int domain, int bus, int slot, int func, int reg,
 	struct config_field *field;
 
 	psdev = pcistub_device_find(domain, bus, slot, func);
-	if (!psdev || !psdev->dev) {
+	if (!psdev) {
 		err = -ENODEV;
 		goto out;
 	}
@@ -1011,6 +1049,8 @@ static int pcistub_reg_add(int domain, int bus, int slot, int func, int reg,
 	if (err)
 		kfree(field);
 out:
+	if (psdev)
+		pcistub_device_put(psdev);
 	return err;
 }
 
@@ -1115,10 +1155,9 @@ static ssize_t pcistub_irq_handler_switch(struct device_driver *drv,
 
 	err = str_to_slot(buf, &domain, &bus, &slot, &func);
 	if (err)
-		goto out;
+		return err;
 
 	psdev = pcistub_device_find(domain, bus, slot, func);
-
 	if (!psdev)
 		goto out;
 
@@ -1134,6 +1173,8 @@ static ssize_t pcistub_irq_handler_switch(struct device_driver *drv,
 	if (dev_data->isr_on)
 		dev_data->ack_intr = 1;
 out:
+	if (psdev)
+		pcistub_device_put(psdev);
 	if (!err)
 		err = count;
 	return err;
@@ -1216,15 +1257,16 @@ static ssize_t permissive_add(struct device_driver *drv, const char *buf,
 	err = str_to_slot(buf, &domain, &bus, &slot, &func);
 	if (err)
 		goto out;
+	if (slot < 0 || func < 0) {
+		err = -EINVAL;
+		goto out;
+	}
 	psdev = pcistub_device_find(domain, bus, slot, func);
 	if (!psdev) {
 		err = -ENODEV;
 		goto out;
 	}
-	if (!psdev->dev) {
-		err = -ENODEV;
-		goto release;
-	}
+
 	dev_data = pci_get_drvdata(psdev->dev);
 	/* the driver data for a device should never be null at this point */
 	if (!dev_data) {
@@ -1297,17 +1339,51 @@ static int __init pcistub_init(void)
 
 	if (pci_devs_to_hide && *pci_devs_to_hide) {
 		do {
+			char wc = '*';
+
 			parsed = 0;
 
 			err = sscanf(pci_devs_to_hide + pos,
 				     " (%x:%x:%x.%x) %n",
 				     &domain, &bus, &slot, &func, &parsed);
-			if (err != 4) {
+			switch (err) {
+			case 3:
+				func = -1;
+				err = sscanf(pci_devs_to_hide + pos,
+					     " (%x:%x:%x.%c) %n",
+					     &domain, &bus, &slot, &wc,
+					     &parsed);
+				break;
+			case 2:
+				slot = func = -1;
+				err = sscanf(pci_devs_to_hide + pos,
+					     " (%x:%x:*.%c) %n",
+					     &domain, &bus, &wc, &parsed) + 1;
+				break;
+			}
+
+			if (err != 4 || wc != '*') {
 				domain = 0;
+				wc = '*';
 				err = sscanf(pci_devs_to_hide + pos,
 					     " (%x:%x.%x) %n",
 					     &bus, &slot, &func, &parsed);
-				if (err != 3)
+				switch (err) {
+				case 2:
+					func = -1;
+					err = sscanf(pci_devs_to_hide + pos,
+						     " (%x:%x.%c) %n",
+						     &bus, &slot, &wc,
+						     &parsed);
+					break;
+				case 1:
+					slot = func = -1;
+					err = sscanf(pci_devs_to_hide + pos,
+						     " (%x:*.%c) %n",
+						     &bus, &wc, &parsed) + 1;
+					break;
+				}
+				if (err != 3 || wc != '*')
 					goto parse_error;
 			}
 
diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c
index b3e146edb51..bcf3ba4a6ec 100644
--- a/drivers/xen/xenbus/xenbus_client.c
+++ b/drivers/xen/xenbus/xenbus_client.c
@@ -490,8 +490,7 @@ static int xenbus_map_ring_valloc_pv(struct xenbus_device *dev,
 
 	op.host_addr = arbitrary_virt_to_machine(pte).maddr;
 
-	if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &op, 1))
-		BUG();
+	gnttab_batch_map(&op, 1);
 
 	if (op.status != GNTST_okay) {
 		free_vm_area(area);
@@ -572,8 +571,7 @@ int xenbus_map_ring(struct xenbus_device *dev, int gnt_ref,
 	gnttab_set_map_op(&op, (unsigned long)vaddr, GNTMAP_host_map, gnt_ref,
 			  dev->otherend_id);
 
-	if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &op, 1))
-		BUG();
+	gnttab_batch_map(&op, 1);
 
 	if (op.status != GNTST_okay) {
 		xenbus_dev_fatal(dev, op.status,
diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c
index 52fe7ad0766..c5aa55c5d37 100644
--- a/drivers/xen/xenbus/xenbus_comms.c
+++ b/drivers/xen/xenbus/xenbus_comms.c
@@ -224,7 +224,7 @@ int xb_init_comms(void)
 		int err;
 		err = bind_evtchn_to_irqhandler(xen_store_evtchn, wake_waiting,
 						0, "xenbus", &xb_waitq);
-		if (err <= 0) {
+		if (err < 0) {
 			printk(KERN_ERR "XENBUS request irq failed %i\n", err);
 			return err;
 		}
diff --git a/drivers/xen/xenbus/xenbus_dev_backend.c b/drivers/xen/xenbus/xenbus_dev_backend.c
index be738c43104..d7300080076 100644
--- a/drivers/xen/xenbus/xenbus_dev_backend.c
+++ b/drivers/xen/xenbus/xenbus_dev_backend.c
@@ -107,7 +107,7 @@ static int xenbus_backend_mmap(struct file *file, struct vm_area_struct *vma)
 	return 0;
 }
 
-const struct file_operations xenbus_backend_fops = {
+static const struct file_operations xenbus_backend_fops = {
 	.open = xenbus_backend_open,
 	.mmap = xenbus_backend_mmap,
 	.unlocked_ioctl = xenbus_backend_ioctl,
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index b793723e724..038b71dbf03 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -324,8 +324,8 @@ static int cmp_dev(struct device *dev, void *data)
 	return 0;
 }
 
-struct xenbus_device *xenbus_device_find(const char *nodename,
-					 struct bus_type *bus)
+static struct xenbus_device *xenbus_device_find(const char *nodename,
+						struct bus_type *bus)
 {
 	struct xb_find_info info = { .dev = NULL, .nodename = nodename };
 
@@ -719,17 +719,47 @@ static int __init xenstored_local_init(void)
 	return err;
 }
 
+enum xenstore_init {
+	UNKNOWN,
+	PV,
+	HVM,
+	LOCAL,
+};
 static int __init xenbus_init(void)
 {
 	int err = 0;
+	enum xenstore_init usage = UNKNOWN;
+	uint64_t v = 0;
 
 	if (!xen_domain())
 		return -ENODEV;
 
 	xenbus_ring_ops_init();
 
-	if (xen_hvm_domain()) {
-		uint64_t v = 0;
+	if (xen_pv_domain())
+		usage = PV;
+	if (xen_hvm_domain())
+		usage = HVM;
+	if (xen_hvm_domain() && xen_initial_domain())
+		usage = LOCAL;
+	if (xen_pv_domain() && !xen_start_info->store_evtchn)
+		usage = LOCAL;
+	if (xen_pv_domain() && xen_start_info->store_evtchn)
+		xenstored_ready = 1;
+
+	switch (usage) {
+	case LOCAL:
+		err = xenstored_local_init();
+		if (err)
+			goto out_error;
+		xen_store_interface = mfn_to_virt(xen_store_mfn);
+		break;
+	case PV:
+		xen_store_evtchn = xen_start_info->store_evtchn;
+		xen_store_mfn = xen_start_info->store_mfn;
+		xen_store_interface = mfn_to_virt(xen_store_mfn);
+		break;
+	case HVM:
 		err = hvm_get_parameter(HVM_PARAM_STORE_EVTCHN, &v);
 		if (err)
 			goto out_error;
@@ -738,18 +768,12 @@ static int __init xenbus_init(void)
 		if (err)
 			goto out_error;
 		xen_store_mfn = (unsigned long)v;
-		xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT, PAGE_SIZE);
-	} else {
-		xen_store_evtchn = xen_start_info->store_evtchn;
-		xen_store_mfn = xen_start_info->store_mfn;
-		if (xen_store_evtchn)
-			xenstored_ready = 1;
-		else {
-			err = xenstored_local_init();
-			if (err)
-				goto out_error;
-		}
-		xen_store_interface = mfn_to_virt(xen_store_mfn);
+		xen_store_interface =
+			ioremap(xen_store_mfn << PAGE_SHIFT, PAGE_SIZE);
+		break;
+	default:
+		pr_warn("Xenstore state unknown\n");
+		break;
 	}
 
 	/* Initialize the interface to xenstore. */
diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c
index a31b54d4883..3159a37d966 100644
--- a/drivers/xen/xenbus/xenbus_probe_frontend.c
+++ b/drivers/xen/xenbus/xenbus_probe_frontend.c
@@ -21,6 +21,7 @@
 #include <xen/xenbus.h>
 #include <xen/events.h>
 #include <xen/page.h>
+#include <xen/xen.h>
 
 #include <xen/platform_pci.h>
 
diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c
index bce15cf4a8d..131dec04794 100644
--- a/drivers/xen/xenbus/xenbus_xs.c
+++ b/drivers/xen/xenbus/xenbus_xs.c
@@ -44,6 +44,7 @@
 #include <linux/rwsem.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <asm/xen/hypervisor.h>
 #include <xen/xenbus.h>
 #include <xen/xen.h>
 #include "xenbus_comms.h"
@@ -622,7 +623,7 @@ static void xs_reset_watches(void)
 {
 	int err, supported = 0;
 
-	if (!xen_hvm_domain())
+	if (!xen_hvm_domain() || xen_initial_domain())
 		return;
 
 	err = xenbus_scanf(XBT_NIL, "control",
diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c
index b85efa77394..392c5dac198 100644
--- a/fs/9p/v9fs.c
+++ b/fs/9p/v9fs.c
@@ -560,6 +560,11 @@ static int v9fs_init_inode_cache(void)
  */
 static void v9fs_destroy_inode_cache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(v9fs_inode_cache);
 }
 
diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt
index 02257420274..0efd1524b97 100644
--- a/fs/Kconfig.binfmt
+++ b/fs/Kconfig.binfmt
@@ -164,3 +164,11 @@ config BINFMT_MISC
 	  You may say M here for module support and later load the module when
 	  you have use for it; the module is called binfmt_misc. If you
 	  don't know what to answer at this point, say Y.
+
+config COREDUMP
+	bool "Enable core dump support" if EXPERT
+	default y
+	help
+	  This option enables support for performing core dumps. You almost
+	  certainly want to say Y here. Not necessary on systems that never
+	  need debugging or only ever run flawless code.
diff --git a/fs/Makefile b/fs/Makefile
index 2fb97793467..1d7af79288a 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_FS_MBCACHE)	+= mbcache.o
 obj-$(CONFIG_FS_POSIX_ACL)	+= posix_acl.o xattr_acl.o
 obj-$(CONFIG_NFS_COMMON)	+= nfs_common/
 obj-$(CONFIG_GENERIC_ACL)	+= generic_acl.o
+obj-$(CONFIG_COREDUMP)		+= coredump.o
 
 obj-$(CONFIG_FHANDLE)		+= fhandle.o
 
diff --git a/fs/adfs/super.c b/fs/adfs/super.c
index 22a0d7ed5fa..d5712293579 100644
--- a/fs/adfs/super.c
+++ b/fs/adfs/super.c
@@ -280,6 +280,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(adfs_inode_cachep);
 }
 
diff --git a/fs/affs/super.c b/fs/affs/super.c
index 1f030825cd3..b84dc735250 100644
--- a/fs/affs/super.c
+++ b/fs/affs/super.c
@@ -147,6 +147,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(affs_inode_cachep);
 }
 
diff --git a/fs/afs/super.c b/fs/afs/super.c
index df8c6047c2a..43165009428 100644
--- a/fs/afs/super.c
+++ b/fs/afs/super.c
@@ -123,6 +123,11 @@ void __exit afs_fs_exit(void)
 		BUG();
 	}
 
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(afs_inode_cachep);
 	_leave("");
 }
diff --git a/fs/attr.c b/fs/attr.c
index 29e38a1f7f7..cce7df53b69 100644
--- a/fs/attr.c
+++ b/fs/attr.c
@@ -14,6 +14,7 @@
 #include <linux/fcntl.h>
 #include <linux/security.h>
 #include <linux/evm.h>
+#include <linux/ima.h>
 
 /**
  * inode_change_ok - check if attribute changes to an inode are allowed
@@ -247,6 +248,7 @@ int notify_change(struct dentry * dentry, struct iattr * attr)
 
 	if (!error) {
 		fsnotify_change(dentry, ia_valid);
+		ima_inode_post_setattr(dentry);
 		evm_inode_post_setattr(dentry, ia_valid);
 	}
 
diff --git a/fs/autofs4/dev-ioctl.c b/fs/autofs4/dev-ioctl.c
index abf645c1703..a16214109d3 100644
--- a/fs/autofs4/dev-ioctl.c
+++ b/fs/autofs4/dev-ioctl.c
@@ -221,20 +221,6 @@ static int test_by_type(struct path *path, void *p)
 	return ino && ino->sbi->type & *(unsigned *)p;
 }
 
-static void autofs_dev_ioctl_fd_install(unsigned int fd, struct file *file)
-{
-	struct files_struct *files = current->files;
-	struct fdtable *fdt;
-
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	BUG_ON(fdt->fd[fd] != NULL);
-	rcu_assign_pointer(fdt->fd[fd], file);
-	__set_close_on_exec(fd, fdt);
-	spin_unlock(&files->file_lock);
-}
-
-
 /*
  * Open a file descriptor on the autofs mount point corresponding
  * to the given path and device number (aka. new_encode_dev(sb->s_dev)).
@@ -243,7 +229,7 @@ static int autofs_dev_ioctl_open_mountpoint(const char *name, dev_t devid)
 {
 	int err, fd;
 
-	fd = get_unused_fd();
+	fd = get_unused_fd_flags(O_CLOEXEC);
 	if (likely(fd >= 0)) {
 		struct file *filp;
 		struct path path;
@@ -264,7 +250,7 @@ static int autofs_dev_ioctl_open_mountpoint(const char *name, dev_t devid)
 			goto out;
 		}
 
-		autofs_dev_ioctl_fd_install(fd, filp);
+		fd_install(fd, filp);
 	}
 
 	return fd;
diff --git a/fs/autofs4/waitq.c b/fs/autofs4/waitq.c
index da8876d38a7..dce436e595c 100644
--- a/fs/autofs4/waitq.c
+++ b/fs/autofs4/waitq.c
@@ -175,8 +175,7 @@ static void autofs4_notify_daemon(struct autofs_sb_info *sbi,
 		return;
 	}
 
-	pipe = sbi->pipe;
-	get_file(pipe);
+	pipe = get_file(sbi->pipe);
 
 	mutex_unlock(&sbi->wq_mutex);
 
diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c
index 7f73a692bfd..2b3bda8d5e6 100644
--- a/fs/befs/linuxvfs.c
+++ b/fs/befs/linuxvfs.c
@@ -457,6 +457,11 @@ befs_init_inodecache(void)
 static void
 befs_destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(befs_inode_cachep);
 }
 
diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c
index b242beba58e..737aaa3f709 100644
--- a/fs/bfs/inode.c
+++ b/fs/bfs/inode.c
@@ -280,6 +280,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(bfs_inode_cachep);
 }
 
diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c
index d146e181d10..0e7a6f81ae3 100644
--- a/fs/binfmt_aout.c
+++ b/fs/binfmt_aout.c
@@ -32,31 +32,8 @@
 
 static int load_aout_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout_library(struct file*);
-static int aout_core_dump(struct coredump_params *cprm);
-
-static struct linux_binfmt aout_format = {
-	.module		= THIS_MODULE,
-	.load_binary	= load_aout_binary,
-	.load_shlib	= load_aout_library,
-	.core_dump	= aout_core_dump,
-	.min_coredump	= PAGE_SIZE
-};
-
-#define BAD_ADDR(x)	((unsigned long)(x) >= TASK_SIZE)
-
-static int set_brk(unsigned long start, unsigned long end)
-{
-	start = PAGE_ALIGN(start);
-	end = PAGE_ALIGN(end);
-	if (end > start) {
-		unsigned long addr;
-		addr = vm_brk(start, end - start);
-		if (BAD_ADDR(addr))
-			return addr;
-	}
-	return 0;
-}
 
+#ifdef CONFIG_COREDUMP
 /*
  * Routine writes a core dump image in the current directory.
  * Currently only a stub-function.
@@ -66,7 +43,6 @@ static int set_brk(unsigned long start, unsigned long end)
  * field, which also makes sure the core-dumps won't be recursive if the
  * dumping of the process results in another error..
  */
-
 static int aout_core_dump(struct coredump_params *cprm)
 {
 	struct file *file = cprm->file;
@@ -89,7 +65,7 @@ static int aout_core_dump(struct coredump_params *cprm)
 	current->flags |= PF_DUMPCORE;
        	strncpy(dump.u_comm, current->comm, sizeof(dump.u_comm));
 	dump.u_ar0 = offsetof(struct user, regs);
-	dump.signal = cprm->signr;
+	dump.signal = cprm->siginfo->si_signo;
 	aout_dump_thread(cprm->regs, &dump);
 
 /* If the size of the dump file exceeds the rlimit, then see what would happen
@@ -135,6 +111,32 @@ end_coredump:
 	set_fs(fs);
 	return has_dumped;
 }
+#else
+#define aout_core_dump NULL
+#endif
+
+static struct linux_binfmt aout_format = {
+	.module		= THIS_MODULE,
+	.load_binary	= load_aout_binary,
+	.load_shlib	= load_aout_library,
+	.core_dump	= aout_core_dump,
+	.min_coredump	= PAGE_SIZE
+};
+
+#define BAD_ADDR(x)	((unsigned long)(x) >= TASK_SIZE)
+
+static int set_brk(unsigned long start, unsigned long end)
+{
+	start = PAGE_ALIGN(start);
+	end = PAGE_ALIGN(end);
+	if (end > start) {
+		unsigned long addr;
+		addr = vm_brk(start, end - start);
+		if (BAD_ADDR(addr))
+			return addr;
+	}
+	return 0;
+}
 
 /*
  * create_aout_tables() parses the env- and arg-strings in new user
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 1b52956afe3..28a64e76952 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -27,6 +27,7 @@
 #include <linux/compiler.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/vmalloc.h>
 #include <linux/security.h>
 #include <linux/random.h>
 #include <linux/elf.h>
@@ -37,6 +38,13 @@
 #include <asm/page.h>
 #include <asm/exec.h>
 
+#ifndef user_long_t
+#define user_long_t long
+#endif
+#ifndef user_siginfo_t
+#define user_siginfo_t siginfo_t
+#endif
+
 static int load_elf_binary(struct linux_binprm *bprm, struct pt_regs *regs);
 static int load_elf_library(struct file *);
 static unsigned long elf_map(struct file *, unsigned long, struct elf_phdr *,
@@ -881,7 +889,7 @@ static int load_elf_binary(struct linux_binprm *bprm, struct pt_regs *regs)
 	}
 
 	if (elf_interpreter) {
-		unsigned long uninitialized_var(interp_map_addr);
+		unsigned long interp_map_addr = 0;
 
 		elf_entry = load_elf_interp(&loc->interp_elf_ex,
 					    interpreter,
@@ -1372,6 +1380,103 @@ static void fill_auxv_note(struct memelfnote *note, struct mm_struct *mm)
 	fill_note(note, "CORE", NT_AUXV, i * sizeof(elf_addr_t), auxv);
 }
 
+static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata,
+		siginfo_t *siginfo)
+{
+	mm_segment_t old_fs = get_fs();
+	set_fs(KERNEL_DS);
+	copy_siginfo_to_user((user_siginfo_t __user *) csigdata, siginfo);
+	set_fs(old_fs);
+	fill_note(note, "CORE", NT_SIGINFO, sizeof(*csigdata), csigdata);
+}
+
+#define MAX_FILE_NOTE_SIZE (4*1024*1024)
+/*
+ * Format of NT_FILE note:
+ *
+ * long count     -- how many files are mapped
+ * long page_size -- units for file_ofs
+ * array of [COUNT] elements of
+ *   long start
+ *   long end
+ *   long file_ofs
+ * followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL...
+ */
+static void fill_files_note(struct memelfnote *note)
+{
+	struct vm_area_struct *vma;
+	unsigned count, size, names_ofs, remaining, n;
+	user_long_t *data;
+	user_long_t *start_end_ofs;
+	char *name_base, *name_curpos;
+
+	/* *Estimated* file count and total data size needed */
+	count = current->mm->map_count;
+	size = count * 64;
+
+	names_ofs = (2 + 3 * count) * sizeof(data[0]);
+ alloc:
+	if (size >= MAX_FILE_NOTE_SIZE) /* paranoia check */
+		goto err;
+	size = round_up(size, PAGE_SIZE);
+	data = vmalloc(size);
+	if (!data)
+		goto err;
+
+	start_end_ofs = data + 2;
+	name_base = name_curpos = ((char *)data) + names_ofs;
+	remaining = size - names_ofs;
+	count = 0;
+	for (vma = current->mm->mmap; vma != NULL; vma = vma->vm_next) {
+		struct file *file;
+		const char *filename;
+
+		file = vma->vm_file;
+		if (!file)
+			continue;
+		filename = d_path(&file->f_path, name_curpos, remaining);
+		if (IS_ERR(filename)) {
+			if (PTR_ERR(filename) == -ENAMETOOLONG) {
+				vfree(data);
+				size = size * 5 / 4;
+				goto alloc;
+			}
+			continue;
+		}
+
+		/* d_path() fills at the end, move name down */
+		/* n = strlen(filename) + 1: */
+		n = (name_curpos + remaining) - filename;
+		remaining = filename - name_curpos;
+		memmove(name_curpos, filename, n);
+		name_curpos += n;
+
+		*start_end_ofs++ = vma->vm_start;
+		*start_end_ofs++ = vma->vm_end;
+		*start_end_ofs++ = vma->vm_pgoff;
+		count++;
+	}
+
+	/* Now we know exact count of files, can store it */
+	data[0] = count;
+	data[1] = PAGE_SIZE;
+	/*
+	 * Count usually is less than current->mm->map_count,
+	 * we need to move filenames down.
+	 */
+	n = current->mm->map_count - count;
+	if (n != 0) {
+		unsigned shift_bytes = n * 3 * sizeof(data[0]);
+		memmove(name_base - shift_bytes, name_base,
+			name_curpos - name_base);
+		name_curpos -= shift_bytes;
+	}
+
+	size = name_curpos - (char *)data;
+	fill_note(note, "CORE", NT_FILE, size, data);
+ err: ;
+}
+
 #ifdef CORE_DUMP_USE_REGSET
 #include <linux/regset.h>
 
@@ -1385,7 +1490,10 @@ struct elf_thread_core_info {
 struct elf_note_info {
 	struct elf_thread_core_info *thread;
 	struct memelfnote psinfo;
+	struct memelfnote signote;
 	struct memelfnote auxv;
+	struct memelfnote files;
+	user_siginfo_t csigdata;
 	size_t size;
 	int thread_notes;
 };
@@ -1480,7 +1588,7 @@ static int fill_thread_core_info(struct elf_thread_core_info *t,
 
 static int fill_note_info(struct elfhdr *elf, int phdrs,
 			  struct elf_note_info *info,
-			  long signr, struct pt_regs *regs)
+			  siginfo_t *siginfo, struct pt_regs *regs)
 {
 	struct task_struct *dump_task = current;
 	const struct user_regset_view *view = task_user_regset_view(dump_task);
@@ -1550,7 +1658,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
 	 * Now fill in each thread's information.
 	 */
 	for (t = info->thread; t != NULL; t = t->next)
-		if (!fill_thread_core_info(t, view, signr, &info->size))
+		if (!fill_thread_core_info(t, view, siginfo->si_signo, &info->size))
 			return 0;
 
 	/*
@@ -1559,9 +1667,15 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
 	fill_psinfo(psinfo, dump_task->group_leader, dump_task->mm);
 	info->size += notesize(&info->psinfo);
 
+	fill_siginfo_note(&info->signote, &info->csigdata, siginfo);
+	info->size += notesize(&info->signote);
+
 	fill_auxv_note(&info->auxv, current->mm);
 	info->size += notesize(&info->auxv);
 
+	fill_files_note(&info->files);
+	info->size += notesize(&info->files);
+
 	return 1;
 }
 
@@ -1588,8 +1702,12 @@ static int write_note_info(struct elf_note_info *info,
 
 		if (first && !writenote(&info->psinfo, file, foffset))
 			return 0;
+		if (first && !writenote(&info->signote, file, foffset))
+			return 0;
 		if (first && !writenote(&info->auxv, file, foffset))
 			return 0;
+		if (first && !writenote(&info->files, file, foffset))
+			return 0;
 
 		for (i = 1; i < info->thread_notes; ++i)
 			if (t->notes[i].data &&
@@ -1616,6 +1734,7 @@ static void free_note_info(struct elf_note_info *info)
 		kfree(t);
 	}
 	kfree(info->psinfo.data);
+	vfree(info->files.data);
 }
 
 #else
@@ -1681,6 +1800,7 @@ struct elf_note_info {
 #ifdef ELF_CORE_COPY_XFPREGS
 	elf_fpxregset_t *xfpu;
 #endif
+	user_siginfo_t csigdata;
 	int thread_status_size;
 	int numnote;
 };
@@ -1690,48 +1810,37 @@ static int elf_note_info_init(struct elf_note_info *info)
 	memset(info, 0, sizeof(*info));
 	INIT_LIST_HEAD(&info->thread_list);
 
-	/* Allocate space for six ELF notes */
-	info->notes = kmalloc(6 * sizeof(struct memelfnote), GFP_KERNEL);
+	/* Allocate space for ELF notes */
+	info->notes = kmalloc(8 * sizeof(struct memelfnote), GFP_KERNEL);
 	if (!info->notes)
 		return 0;
 	info->psinfo = kmalloc(sizeof(*info->psinfo), GFP_KERNEL);
 	if (!info->psinfo)
-		goto notes_free;
+		return 0;
 	info->prstatus = kmalloc(sizeof(*info->prstatus), GFP_KERNEL);
 	if (!info->prstatus)
-		goto psinfo_free;
+		return 0;
 	info->fpu = kmalloc(sizeof(*info->fpu), GFP_KERNEL);
 	if (!info->fpu)
-		goto prstatus_free;
+		return 0;
 #ifdef ELF_CORE_COPY_XFPREGS
 	info->xfpu = kmalloc(sizeof(*info->xfpu), GFP_KERNEL);
 	if (!info->xfpu)
-		goto fpu_free;
+		return 0;
 #endif
 	return 1;
-#ifdef ELF_CORE_COPY_XFPREGS
- fpu_free:
-	kfree(info->fpu);
-#endif
- prstatus_free:
-	kfree(info->prstatus);
- psinfo_free:
-	kfree(info->psinfo);
- notes_free:
-	kfree(info->notes);
-	return 0;
 }
 
 static int fill_note_info(struct elfhdr *elf, int phdrs,
 			  struct elf_note_info *info,
-			  long signr, struct pt_regs *regs)
+			  siginfo_t *siginfo, struct pt_regs *regs)
 {
 	struct list_head *t;
 
 	if (!elf_note_info_init(info))
 		return 0;
 
-	if (signr) {
+	if (siginfo->si_signo) {
 		struct core_thread *ct;
 		struct elf_thread_status *ets;
 
@@ -1749,13 +1858,13 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
 			int sz;
 
 			ets = list_entry(t, struct elf_thread_status, list);
-			sz = elf_dump_thread_status(signr, ets);
+			sz = elf_dump_thread_status(siginfo->si_signo, ets);
 			info->thread_status_size += sz;
 		}
 	}
 	/* now collect the dump for the current */
 	memset(info->prstatus, 0, sizeof(*info->prstatus));
-	fill_prstatus(info->prstatus, current, signr);
+	fill_prstatus(info->prstatus, current, siginfo->si_signo);
 	elf_core_copy_regs(&info->prstatus->pr_reg, regs);
 
 	/* Set up header */
@@ -1772,9 +1881,11 @@ static int fill_note_info(struct elfhdr *elf, int phdrs,
 	fill_note(info->notes + 1, "CORE", NT_PRPSINFO,
 		  sizeof(*info->psinfo), info->psinfo);
 
-	info->numnote = 2;
+	fill_siginfo_note(info->notes + 2, &info->csigdata, siginfo);
+	fill_auxv_note(info->notes + 3, current->mm);
+	fill_files_note(info->notes + 4);
 
-	fill_auxv_note(&info->notes[info->numnote++], current->mm);
+	info->numnote = 5;
 
 	/* Try to dump the FPU. */
 	info->prstatus->pr_fpvalid = elf_core_copy_task_fpregs(current, regs,
@@ -1836,6 +1947,9 @@ static void free_note_info(struct elf_note_info *info)
 		kfree(list_entry(tmp, struct elf_thread_status, list));
 	}
 
+	/* Free data allocated by fill_files_note(): */
+	vfree(info->notes[4].data);
+
 	kfree(info->prstatus);
 	kfree(info->psinfo);
 	kfree(info->notes);
@@ -1962,7 +2076,7 @@ static int elf_core_dump(struct coredump_params *cprm)
 	 * Collect all the non-memory information about the process for the
 	 * notes.  This also sets up the file header.
 	 */
-	if (!fill_note_info(elf, e_phnum, &info, cprm->signr, cprm->regs))
+	if (!fill_note_info(elf, e_phnum, &info, cprm->siginfo, cprm->regs))
 		goto cleanup;
 
 	has_dumped = 1;
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index 3d77cf81ba3..08d812b3228 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -1642,7 +1642,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 		goto cleanup;
 #endif
 
-	if (cprm->signr) {
+	if (cprm->siginfo->si_signo) {
 		struct core_thread *ct;
 		struct elf_thread_status *tmp;
 
@@ -1661,13 +1661,13 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm)
 			int sz;
 
 			tmp = list_entry(t, struct elf_thread_status, list);
-			sz = elf_dump_thread_status(cprm->signr, tmp);
+			sz = elf_dump_thread_status(cprm->siginfo->si_signo, tmp);
 			thread_status_size += sz;
 		}
 	}
 
 	/* now collect the dump for the current */
-	fill_prstatus(prstatus, current, cprm->signr);
+	fill_prstatus(prstatus, current, cprm->siginfo->si_signo);
 	elf_core_copy_regs(&prstatus->pr_reg, cprm->regs);
 
 	segs = current->mm->map_count;
diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c
index 178cb70acc2..e280352b28f 100644
--- a/fs/binfmt_flat.c
+++ b/fs/binfmt_flat.c
@@ -107,7 +107,7 @@ static struct linux_binfmt flat_format = {
 static int flat_core_dump(struct coredump_params *cprm)
 {
 	printk("Process %s:%d received signr %d and should have core dumped\n",
-			current->comm, current->pid, (int) cprm->signr);
+			current->comm, current->pid, (int) cprm->siginfo->si_signo);
 	return(1);
 }
 
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 4c878476bb9..b08ea4717e9 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -107,6 +107,12 @@ void extent_io_exit(void)
 		list_del(&eb->leak_list);
 		kmem_cache_free(extent_buffer_cache, eb);
 	}
+
+	/*
+	 * Make sure all delayed rcu free are flushed before we
+	 * destroy caches.
+	 */
+	rcu_barrier();
 	if (extent_state_cache)
 		kmem_cache_destroy(extent_state_cache);
 	if (extent_buffer_cache)
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 2a028a58619..a6ed6944e50 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7076,6 +7076,11 @@ static void init_once(void *foo)
 
 void btrfs_destroy_cachep(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	if (btrfs_inode_cachep)
 		kmem_cache_destroy(btrfs_inode_cachep);
 	if (btrfs_trans_handle_cachep)
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 27bfce58da3..47127c1bd29 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -1397,7 +1397,6 @@ static noinline int btrfs_ioctl_snap_create_transid(struct file *file,
 				u64 *transid, bool readonly,
 				struct btrfs_qgroup_inherit **inherit)
 {
-	struct file *src_file;
 	int namelen;
 	int ret = 0;
 
@@ -1421,25 +1420,24 @@ static noinline int btrfs_ioctl_snap_create_transid(struct file *file,
 		ret = btrfs_mksubvol(&file->f_path, name, namelen,
 				     NULL, transid, readonly, inherit);
 	} else {
+		struct fd src = fdget(fd);
 		struct inode *src_inode;
-		src_file = fget(fd);
-		if (!src_file) {
+		if (!src.file) {
 			ret = -EINVAL;
 			goto out_drop_write;
 		}
 
-		src_inode = src_file->f_path.dentry->d_inode;
+		src_inode = src.file->f_path.dentry->d_inode;
 		if (src_inode->i_sb != file->f_path.dentry->d_inode->i_sb) {
 			printk(KERN_INFO "btrfs: Snapshot src from "
 			       "another FS\n");
 			ret = -EINVAL;
-			fput(src_file);
-			goto out_drop_write;
+		} else {
+			ret = btrfs_mksubvol(&file->f_path, name, namelen,
+					     BTRFS_I(src_inode)->root,
+					     transid, readonly, inherit);
 		}
-		ret = btrfs_mksubvol(&file->f_path, name, namelen,
-				     BTRFS_I(src_inode)->root,
-				     transid, readonly, inherit);
-		fput(src_file);
+		fdput(src);
 	}
 out_drop_write:
 	mnt_drop_write_file(file);
@@ -2341,7 +2339,7 @@ static noinline long btrfs_ioctl_clone(struct file *file, unsigned long srcfd,
 {
 	struct inode *inode = fdentry(file)->d_inode;
 	struct btrfs_root *root = BTRFS_I(inode)->root;
-	struct file *src_file;
+	struct fd src_file;
 	struct inode *src;
 	struct btrfs_trans_handle *trans;
 	struct btrfs_path *path;
@@ -2376,24 +2374,24 @@ static noinline long btrfs_ioctl_clone(struct file *file, unsigned long srcfd,
 	if (ret)
 		return ret;
 
-	src_file = fget(srcfd);
-	if (!src_file) {
+	src_file = fdget(srcfd);
+	if (!src_file.file) {
 		ret = -EBADF;
 		goto out_drop_write;
 	}
 
 	ret = -EXDEV;
-	if (src_file->f_path.mnt != file->f_path.mnt)
+	if (src_file.file->f_path.mnt != file->f_path.mnt)
 		goto out_fput;
 
-	src = src_file->f_dentry->d_inode;
+	src = src_file.file->f_dentry->d_inode;
 
 	ret = -EINVAL;
 	if (src == inode)
 		goto out_fput;
 
 	/* the src must be open for reading */
-	if (!(src_file->f_mode & FMODE_READ))
+	if (!(src_file.file->f_mode & FMODE_READ))
 		goto out_fput;
 
 	/* don't make the dst file partly checksummed */
@@ -2724,7 +2722,7 @@ out_unlock:
 	vfree(buf);
 	btrfs_free_path(path);
 out_fput:
-	fput(src_file);
+	fdput(src_file);
 out_drop_write:
 	mnt_drop_write_file(file);
 	return ret;
diff --git a/fs/btrfs/reada.c b/fs/btrfs/reada.c
index 48a4882d8ad..a955669519a 100644
--- a/fs/btrfs/reada.c
+++ b/fs/btrfs/reada.c
@@ -68,7 +68,7 @@ struct reada_extent {
 	u32			blocksize;
 	int			err;
 	struct list_head	extctl;
-	struct kref		refcnt;
+	int 			refcnt;
 	spinlock_t		lock;
 	struct reada_zone	*zones[BTRFS_MAX_MIRRORS];
 	int			nzones;
@@ -126,7 +126,7 @@ static int __readahead_hook(struct btrfs_root *root, struct extent_buffer *eb,
 	spin_lock(&fs_info->reada_lock);
 	re = radix_tree_lookup(&fs_info->reada_tree, index);
 	if (re)
-		kref_get(&re->refcnt);
+		re->refcnt++;
 	spin_unlock(&fs_info->reada_lock);
 
 	if (!re)
@@ -336,7 +336,7 @@ static struct reada_extent *reada_find_extent(struct btrfs_root *root,
 	spin_lock(&fs_info->reada_lock);
 	re = radix_tree_lookup(&fs_info->reada_tree, index);
 	if (re)
-		kref_get(&re->refcnt);
+		re->refcnt++;
 	spin_unlock(&fs_info->reada_lock);
 
 	if (re)
@@ -352,7 +352,7 @@ static struct reada_extent *reada_find_extent(struct btrfs_root *root,
 	re->top = *top;
 	INIT_LIST_HEAD(&re->extctl);
 	spin_lock_init(&re->lock);
-	kref_init(&re->refcnt);
+	re->refcnt = 1;
 
 	/*
 	 * map block
@@ -398,7 +398,7 @@ static struct reada_extent *reada_find_extent(struct btrfs_root *root,
 	if (ret == -EEXIST) {
 		re_exist = radix_tree_lookup(&fs_info->reada_tree, index);
 		BUG_ON(!re_exist);
-		kref_get(&re_exist->refcnt);
+		re_exist->refcnt++;
 		spin_unlock(&fs_info->reada_lock);
 		goto error;
 	}
@@ -465,10 +465,6 @@ error:
 	return re_exist;
 }
 
-static void reada_kref_dummy(struct kref *kr)
-{
-}
-
 static void reada_extent_put(struct btrfs_fs_info *fs_info,
 			     struct reada_extent *re)
 {
@@ -476,7 +472,7 @@ static void reada_extent_put(struct btrfs_fs_info *fs_info,
 	unsigned long index = re->logical >> PAGE_CACHE_SHIFT;
 
 	spin_lock(&fs_info->reada_lock);
-	if (!kref_put(&re->refcnt, reada_kref_dummy)) {
+	if (--re->refcnt) {
 		spin_unlock(&fs_info->reada_lock);
 		return;
 	}
@@ -671,7 +667,7 @@ static int reada_start_machine_dev(struct btrfs_fs_info *fs_info,
 		return 0;
 	}
 	dev->reada_next = re->logical + re->blocksize;
-	kref_get(&re->refcnt);
+	re->refcnt++;
 
 	spin_unlock(&fs_info->reada_lock);
 
diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c
index 4b5762ef7c2..ba95eea201b 100644
--- a/fs/ceph/inode.c
+++ b/fs/ceph/inode.c
@@ -1104,7 +1104,7 @@ int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req,
 				pr_err("fill_trace bad get_inode "
 				       "%llx.%llx\n", vino.ino, vino.snap);
 				err = PTR_ERR(in);
-				d_delete(dn);
+				d_drop(dn);
 				goto done;
 			}
 			dn = splice_dentry(dn, in, &have_lease, true);
@@ -1277,7 +1277,7 @@ retry_lookup:
 			in = ceph_get_inode(parent->d_sb, vino);
 			if (IS_ERR(in)) {
 				dout("new_inode badness\n");
-				d_delete(dn);
+				d_drop(dn);
 				dput(dn);
 				err = PTR_ERR(in);
 				goto out;
diff --git a/fs/ceph/super.c b/fs/ceph/super.c
index b982239f38f..3a42d932637 100644
--- a/fs/ceph/super.c
+++ b/fs/ceph/super.c
@@ -603,6 +603,11 @@ bad_cap:
 
 static void destroy_caches(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ceph_inode_cachep);
 	kmem_cache_destroy(ceph_cap_cachep);
 	kmem_cache_destroy(ceph_dentry_cachep);
diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c
index a41044a3108..e7931cc55d0 100644
--- a/fs/cifs/cifsfs.c
+++ b/fs/cifs/cifsfs.c
@@ -968,6 +968,11 @@ cifs_init_inodecache(void)
 static void
 cifs_destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(cifs_inode_cachep);
 }
 
diff --git a/fs/coda/inode.c b/fs/coda/inode.c
index f1813120d75..be2aa490948 100644
--- a/fs/coda/inode.c
+++ b/fs/coda/inode.c
@@ -85,6 +85,11 @@ int coda_init_inodecache(void)
 
 void coda_destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(coda_inode_cachep);
 }
 
@@ -107,43 +112,41 @@ static const struct super_operations coda_super_operations =
 
 static int get_device_index(struct coda_mount_data *data)
 {
-	struct file *file;
+	struct fd f;
 	struct inode *inode;
 	int idx;
 
-	if(data == NULL) {
+	if (data == NULL) {
 		printk("coda_read_super: Bad mount data\n");
 		return -1;
 	}
 
-	if(data->version != CODA_MOUNT_VERSION) {
+	if (data->version != CODA_MOUNT_VERSION) {
 		printk("coda_read_super: Bad mount version\n");
 		return -1;
 	}
 
-	file = fget(data->fd);
-	inode = NULL;
-	if(file)
-		inode = file->f_path.dentry->d_inode;
-	
-	if(!inode || !S_ISCHR(inode->i_mode) ||
-	   imajor(inode) != CODA_PSDEV_MAJOR) {
-		if(file)
-			fput(file);
-
-		printk("coda_read_super: Bad file\n");
-		return -1;
+	f = fdget(data->fd);
+	if (!f.file)
+		goto Ebadf;
+	inode = f.file->f_path.dentry->d_inode;
+	if (!S_ISCHR(inode->i_mode) || imajor(inode) != CODA_PSDEV_MAJOR) {
+		fdput(f);
+		goto Ebadf;
 	}
 
 	idx = iminor(inode);
-	fput(file);
+	fdput(f);
 
-	if(idx < 0 || idx >= MAX_CODADEVS) {
+	if (idx < 0 || idx >= MAX_CODADEVS) {
 		printk("coda_read_super: Bad minor number\n");
 		return -1;
 	}
 
 	return idx;
+Ebadf:
+	printk("coda_read_super: Bad file\n");
+	return -1;
 }
 
 static int coda_fill_super(struct super_block *sb, void *data, int silent)
diff --git a/fs/compat.c b/fs/compat.c
index 1bdb350ea5d..b7a24d0ca30 100644
--- a/fs/compat.c
+++ b/fs/compat.c
@@ -870,22 +870,20 @@ asmlinkage long compat_sys_old_readdir(unsigned int fd,
 	struct compat_old_linux_dirent __user *dirent, unsigned int count)
 {
 	int error;
-	struct file *file;
-	int fput_needed;
+	struct fd f = fdget(fd);
 	struct compat_readdir_callback buf;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
 
 	buf.result = 0;
 	buf.dirent = dirent;
 
-	error = vfs_readdir(file, compat_fillonedir, &buf);
+	error = vfs_readdir(f.file, compat_fillonedir, &buf);
 	if (buf.result)
 		error = buf.result;
 
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
 
@@ -949,17 +947,16 @@ efault:
 asmlinkage long compat_sys_getdents(unsigned int fd,
 		struct compat_linux_dirent __user *dirent, unsigned int count)
 {
-	struct file * file;
+	struct fd f;
 	struct compat_linux_dirent __user * lastdirent;
 	struct compat_getdents_callback buf;
-	int fput_needed;
 	int error;
 
 	if (!access_ok(VERIFY_WRITE, dirent, count))
 		return -EFAULT;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 
 	buf.current_dir = dirent;
@@ -967,17 +964,17 @@ asmlinkage long compat_sys_getdents(unsigned int fd,
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, compat_filldir, &buf);
+	error = vfs_readdir(f.file, compat_filldir, &buf);
 	if (error >= 0)
 		error = buf.error;
 	lastdirent = buf.previous;
 	if (lastdirent) {
-		if (put_user(file->f_pos, &lastdirent->d_off))
+		if (put_user(f.file->f_pos, &lastdirent->d_off))
 			error = -EFAULT;
 		else
 			error = count - buf.count;
 	}
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
 
@@ -1035,17 +1032,16 @@ efault:
 asmlinkage long compat_sys_getdents64(unsigned int fd,
 		struct linux_dirent64 __user * dirent, unsigned int count)
 {
-	struct file * file;
+	struct fd f;
 	struct linux_dirent64 __user * lastdirent;
 	struct compat_getdents_callback64 buf;
-	int fput_needed;
 	int error;
 
 	if (!access_ok(VERIFY_WRITE, dirent, count))
 		return -EFAULT;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 
 	buf.current_dir = dirent;
@@ -1053,18 +1049,18 @@ asmlinkage long compat_sys_getdents64(unsigned int fd,
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, compat_filldir64, &buf);
+	error = vfs_readdir(f.file, compat_filldir64, &buf);
 	if (error >= 0)
 		error = buf.error;
 	lastdirent = buf.previous;
 	if (lastdirent) {
-		typeof(lastdirent->d_off) d_off = file->f_pos;
+		typeof(lastdirent->d_off) d_off = f.file->f_pos;
 		if (__put_user_unaligned(d_off, &lastdirent->d_off))
 			error = -EFAULT;
 		else
 			error = count - buf.count;
 	}
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
 #endif /* ! __ARCH_OMIT_COMPAT_SYS_GETDENTS64 */
@@ -1152,18 +1148,16 @@ asmlinkage ssize_t
 compat_sys_readv(unsigned long fd, const struct compat_iovec __user *vec,
 		 unsigned long vlen)
 {
-	struct file *file;
-	int fput_needed;
+	struct fd f = fdget(fd);
 	ssize_t ret;
 	loff_t pos;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
-	pos = file->f_pos;
-	ret = compat_readv(file, vec, vlen, &pos);
-	file->f_pos = pos;
-	fput_light(file, fput_needed);
+	pos = f.file->f_pos;
+	ret = compat_readv(f.file, vec, vlen, &pos);
+	f.file->f_pos = pos;
+	fdput(f);
 	return ret;
 }
 
@@ -1171,19 +1165,18 @@ asmlinkage ssize_t
 compat_sys_preadv64(unsigned long fd, const struct compat_iovec __user *vec,
 		    unsigned long vlen, loff_t pos)
 {
-	struct file *file;
-	int fput_needed;
+	struct fd f;
 	ssize_t ret;
 
 	if (pos < 0)
 		return -EINVAL;
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 	ret = -ESPIPE;
-	if (file->f_mode & FMODE_PREAD)
-		ret = compat_readv(file, vec, vlen, &pos);
-	fput_light(file, fput_needed);
+	if (f.file->f_mode & FMODE_PREAD)
+		ret = compat_readv(f.file, vec, vlen, &pos);
+	fdput(f);
 	return ret;
 }
 
@@ -1221,18 +1214,16 @@ asmlinkage ssize_t
 compat_sys_writev(unsigned long fd, const struct compat_iovec __user *vec,
 		  unsigned long vlen)
 {
-	struct file *file;
-	int fput_needed;
+	struct fd f = fdget(fd);
 	ssize_t ret;
 	loff_t pos;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
-	pos = file->f_pos;
-	ret = compat_writev(file, vec, vlen, &pos);
-	file->f_pos = pos;
-	fput_light(file, fput_needed);
+	pos = f.file->f_pos;
+	ret = compat_writev(f.file, vec, vlen, &pos);
+	f.file->f_pos = pos;
+	fdput(f);
 	return ret;
 }
 
@@ -1240,19 +1231,18 @@ asmlinkage ssize_t
 compat_sys_pwritev64(unsigned long fd, const struct compat_iovec __user *vec,
 		     unsigned long vlen, loff_t pos)
 {
-	struct file *file;
-	int fput_needed;
+	struct fd f;
 	ssize_t ret;
 
 	if (pos < 0)
 		return -EINVAL;
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 	ret = -ESPIPE;
-	if (file->f_mode & FMODE_PWRITE)
-		ret = compat_writev(file, vec, vlen, &pos);
-	fput_light(file, fput_needed);
+	if (f.file->f_mode & FMODE_PWRITE)
+		ret = compat_writev(f.file, vec, vlen, &pos);
+	fdput(f);
 	return ret;
 }
 
@@ -1802,3 +1792,25 @@ compat_sys_open_by_handle_at(int mountdirfd,
 	return do_handle_open(mountdirfd, handle, flags);
 }
 #endif
+
+#ifdef __ARCH_WANT_COMPAT_SYS_SENDFILE
+asmlinkage long compat_sys_sendfile(int out_fd, int in_fd,
+				    compat_off_t __user *offset, compat_size_t count)
+{
+	loff_t pos;
+	off_t off;
+	ssize_t ret;
+
+	if (offset) {
+		if (unlikely(get_user(off, offset)))
+			return -EFAULT;
+		pos = off;
+		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
+		if (unlikely(put_user(pos, offset)))
+			return -EFAULT;
+		return ret;
+	}
+
+	return do_sendfile(out_fd, in_fd, NULL, count, 0);
+}
+#endif /* __ARCH_WANT_COMPAT_SYS_SENDFILE */
diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c
index 112e45a17e9..a81147e2e4e 100644
--- a/fs/compat_binfmt_elf.c
+++ b/fs/compat_binfmt_elf.c
@@ -38,6 +38,13 @@
 #define elf_addr_t	Elf32_Addr
 
 /*
+ * Some data types as stored in coredump.
+ */
+#define user_long_t		compat_long_t
+#define user_siginfo_t		compat_siginfo_t
+#define copy_siginfo_to_user	copy_siginfo_to_user32
+
+/*
  * The machine-dependent core note format types are defined in elfcore-compat.h,
  * which requires asm/elf.h to define compat_elf_gregset_t et al.
  */
diff --git a/fs/compat_ioctl.c b/fs/compat_ioctl.c
index 9c03a3ae898..f5054025f9d 100644
--- a/fs/compat_ioctl.c
+++ b/fs/compat_ioctl.c
@@ -1539,16 +1539,13 @@ static int compat_ioctl_check_table(unsigned int xcmd)
 asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
 				unsigned long arg)
 {
-	struct file *filp;
+	struct fd f = fdget(fd);
 	int error = -EBADF;
-	int fput_needed;
-
-	filp = fget_light(fd, &fput_needed);
-	if (!filp)
+	if (!f.file)
 		goto out;
 
 	/* RED-PEN how should LSM module know it's handling 32bit? */
-	error = security_file_ioctl(filp, cmd, arg);
+	error = security_file_ioctl(f.file, cmd, arg);
 	if (error)
 		goto out_fput;
 
@@ -1568,30 +1565,30 @@ asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
 #if defined(CONFIG_IA64) || defined(CONFIG_X86_64)
 	case FS_IOC_RESVSP_32:
 	case FS_IOC_RESVSP64_32:
-		error = compat_ioctl_preallocate(filp, compat_ptr(arg));
+		error = compat_ioctl_preallocate(f.file, compat_ptr(arg));
 		goto out_fput;
 #else
 	case FS_IOC_RESVSP:
 	case FS_IOC_RESVSP64:
-		error = ioctl_preallocate(filp, compat_ptr(arg));
+		error = ioctl_preallocate(f.file, compat_ptr(arg));
 		goto out_fput;
 #endif
 
 	case FIBMAP:
 	case FIGETBSZ:
 	case FIONREAD:
-		if (S_ISREG(filp->f_path.dentry->d_inode->i_mode))
+		if (S_ISREG(f.file->f_path.dentry->d_inode->i_mode))
 			break;
 		/*FALL THROUGH*/
 
 	default:
-		if (filp->f_op && filp->f_op->compat_ioctl) {
-			error = filp->f_op->compat_ioctl(filp, cmd, arg);
+		if (f.file->f_op && f.file->f_op->compat_ioctl) {
+			error = f.file->f_op->compat_ioctl(f.file, cmd, arg);
 			if (error != -ENOIOCTLCMD)
 				goto out_fput;
 		}
 
-		if (!filp->f_op || !filp->f_op->unlocked_ioctl)
+		if (!f.file->f_op || !f.file->f_op->unlocked_ioctl)
 			goto do_ioctl;
 		break;
 	}
@@ -1599,7 +1596,7 @@ asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
 	if (compat_ioctl_check_table(XFORM(cmd)))
 		goto found_handler;
 
-	error = do_ioctl_trans(fd, cmd, arg, filp);
+	error = do_ioctl_trans(fd, cmd, arg, f.file);
 	if (error == -ENOIOCTLCMD)
 		error = -ENOTTY;
 
@@ -1608,9 +1605,9 @@ asmlinkage long compat_sys_ioctl(unsigned int fd, unsigned int cmd,
  found_handler:
 	arg = (unsigned long)compat_ptr(arg);
  do_ioctl:
-	error = do_vfs_ioctl(filp, fd, cmd, arg);
+	error = do_vfs_ioctl(f.file, fd, cmd, arg);
  out_fput:
-	fput_light(filp, fput_needed);
+	fdput(f);
  out:
 	return error;
 }
diff --git a/fs/coredump.c b/fs/coredump.c
new file mode 100644
index 00000000000..fd37facac8d
--- /dev/null
+++ b/fs/coredump.c
@@ -0,0 +1,692 @@
+#include <linux/slab.h>
+#include <linux/file.h>
+#include <linux/fdtable.h>
+#include <linux/mm.h>
+#include <linux/stat.h>
+#include <linux/fcntl.h>
+#include <linux/swap.h>
+#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/pagemap.h>
+#include <linux/perf_event.h>
+#include <linux/highmem.h>
+#include <linux/spinlock.h>
+#include <linux/key.h>
+#include <linux/personality.h>
+#include <linux/binfmts.h>
+#include <linux/coredump.h>
+#include <linux/utsname.h>
+#include <linux/pid_namespace.h>
+#include <linux/module.h>
+#include <linux/namei.h>
+#include <linux/mount.h>
+#include <linux/security.h>
+#include <linux/syscalls.h>
+#include <linux/tsacct_kern.h>
+#include <linux/cn_proc.h>
+#include <linux/audit.h>
+#include <linux/tracehook.h>
+#include <linux/kmod.h>
+#include <linux/fsnotify.h>
+#include <linux/fs_struct.h>
+#include <linux/pipe_fs_i.h>
+#include <linux/oom.h>
+#include <linux/compat.h>
+
+#include <asm/uaccess.h>
+#include <asm/mmu_context.h>
+#include <asm/tlb.h>
+#include <asm/exec.h>
+
+#include <trace/events/task.h>
+#include "internal.h"
+#include "coredump.h"
+
+#include <trace/events/sched.h>
+
+int core_uses_pid;
+char core_pattern[CORENAME_MAX_SIZE] = "core";
+unsigned int core_pipe_limit;
+
+struct core_name {
+	char *corename;
+	int used, size;
+};
+static atomic_t call_count = ATOMIC_INIT(1);
+
+/* The maximal length of core_pattern is also specified in sysctl.c */
+
+static int expand_corename(struct core_name *cn)
+{
+	char *old_corename = cn->corename;
+
+	cn->size = CORENAME_MAX_SIZE * atomic_inc_return(&call_count);
+	cn->corename = krealloc(old_corename, cn->size, GFP_KERNEL);
+
+	if (!cn->corename) {
+		kfree(old_corename);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int cn_printf(struct core_name *cn, const char *fmt, ...)
+{
+	char *cur;
+	int need;
+	int ret;
+	va_list arg;
+
+	va_start(arg, fmt);
+	need = vsnprintf(NULL, 0, fmt, arg);
+	va_end(arg);
+
+	if (likely(need < cn->size - cn->used - 1))
+		goto out_printf;
+
+	ret = expand_corename(cn);
+	if (ret)
+		goto expand_fail;
+
+out_printf:
+	cur = cn->corename + cn->used;
+	va_start(arg, fmt);
+	vsnprintf(cur, need + 1, fmt, arg);
+	va_end(arg);
+	cn->used += need;
+	return 0;
+
+expand_fail:
+	return ret;
+}
+
+static void cn_escape(char *str)
+{
+	for (; *str; str++)
+		if (*str == '/')
+			*str = '!';
+}
+
+static int cn_print_exe_file(struct core_name *cn)
+{
+	struct file *exe_file;
+	char *pathbuf, *path;
+	int ret;
+
+	exe_file = get_mm_exe_file(current->mm);
+	if (!exe_file) {
+		char *commstart = cn->corename + cn->used;
+		ret = cn_printf(cn, "%s (path unknown)", current->comm);
+		cn_escape(commstart);
+		return ret;
+	}
+
+	pathbuf = kmalloc(PATH_MAX, GFP_TEMPORARY);
+	if (!pathbuf) {
+		ret = -ENOMEM;
+		goto put_exe_file;
+	}
+
+	path = d_path(&exe_file->f_path, pathbuf, PATH_MAX);
+	if (IS_ERR(path)) {
+		ret = PTR_ERR(path);
+		goto free_buf;
+	}
+
+	cn_escape(path);
+
+	ret = cn_printf(cn, "%s", path);
+
+free_buf:
+	kfree(pathbuf);
+put_exe_file:
+	fput(exe_file);
+	return ret;
+}
+
+/* format_corename will inspect the pattern parameter, and output a
+ * name into corename, which must have space for at least
+ * CORENAME_MAX_SIZE bytes plus one byte for the zero terminator.
+ */
+static int format_corename(struct core_name *cn, struct coredump_params *cprm)
+{
+	const struct cred *cred = current_cred();
+	const char *pat_ptr = core_pattern;
+	int ispipe = (*pat_ptr == '|');
+	int pid_in_pattern = 0;
+	int err = 0;
+
+	cn->size = CORENAME_MAX_SIZE * atomic_read(&call_count);
+	cn->corename = kmalloc(cn->size, GFP_KERNEL);
+	cn->used = 0;
+
+	if (!cn->corename)
+		return -ENOMEM;
+
+	/* Repeat as long as we have more pattern to process and more output
+	   space */
+	while (*pat_ptr) {
+		if (*pat_ptr != '%') {
+			if (*pat_ptr == 0)
+				goto out;
+			err = cn_printf(cn, "%c", *pat_ptr++);
+		} else {
+			switch (*++pat_ptr) {
+			/* single % at the end, drop that */
+			case 0:
+				goto out;
+			/* Double percent, output one percent */
+			case '%':
+				err = cn_printf(cn, "%c", '%');
+				break;
+			/* pid */
+			case 'p':
+				pid_in_pattern = 1;
+				err = cn_printf(cn, "%d",
+					      task_tgid_vnr(current));
+				break;
+			/* uid */
+			case 'u':
+				err = cn_printf(cn, "%d", cred->uid);
+				break;
+			/* gid */
+			case 'g':
+				err = cn_printf(cn, "%d", cred->gid);
+				break;
+			case 'd':
+				err = cn_printf(cn, "%d",
+					__get_dumpable(cprm->mm_flags));
+				break;
+			/* signal that caused the coredump */
+			case 's':
+				err = cn_printf(cn, "%ld", cprm->siginfo->si_signo);
+				break;
+			/* UNIX time of coredump */
+			case 't': {
+				struct timeval tv;
+				do_gettimeofday(&tv);
+				err = cn_printf(cn, "%lu", tv.tv_sec);
+				break;
+			}
+			/* hostname */
+			case 'h': {
+				char *namestart = cn->corename + cn->used;
+				down_read(&uts_sem);
+				err = cn_printf(cn, "%s",
+					      utsname()->nodename);
+				up_read(&uts_sem);
+				cn_escape(namestart);
+				break;
+			}
+			/* executable */
+			case 'e': {
+				char *commstart = cn->corename + cn->used;
+				err = cn_printf(cn, "%s", current->comm);
+				cn_escape(commstart);
+				break;
+			}
+			case 'E':
+				err = cn_print_exe_file(cn);
+				break;
+			/* core limit size */
+			case 'c':
+				err = cn_printf(cn, "%lu",
+					      rlimit(RLIMIT_CORE));
+				break;
+			default:
+				break;
+			}
+			++pat_ptr;
+		}
+
+		if (err)
+			return err;
+	}
+
+	/* Backward compatibility with core_uses_pid:
+	 *
+	 * If core_pattern does not include a %p (as is the default)
+	 * and core_uses_pid is set, then .%pid will be appended to
+	 * the filename. Do not do this for piped commands. */
+	if (!ispipe && !pid_in_pattern && core_uses_pid) {
+		err = cn_printf(cn, ".%d", task_tgid_vnr(current));
+		if (err)
+			return err;
+	}
+out:
+	return ispipe;
+}
+
+static int zap_process(struct task_struct *start, int exit_code)
+{
+	struct task_struct *t;
+	int nr = 0;
+
+	start->signal->flags = SIGNAL_GROUP_EXIT;
+	start->signal->group_exit_code = exit_code;
+	start->signal->group_stop_count = 0;
+
+	t = start;
+	do {
+		task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
+		if (t != current && t->mm) {
+			sigaddset(&t->pending.signal, SIGKILL);
+			signal_wake_up(t, 1);
+			nr++;
+		}
+	} while_each_thread(start, t);
+
+	return nr;
+}
+
+static inline int zap_threads(struct task_struct *tsk, struct mm_struct *mm,
+				struct core_state *core_state, int exit_code)
+{
+	struct task_struct *g, *p;
+	unsigned long flags;
+	int nr = -EAGAIN;
+
+	spin_lock_irq(&tsk->sighand->siglock);
+	if (!signal_group_exit(tsk->signal)) {
+		mm->core_state = core_state;
+		nr = zap_process(tsk, exit_code);
+	}
+	spin_unlock_irq(&tsk->sighand->siglock);
+	if (unlikely(nr < 0))
+		return nr;
+
+	if (atomic_read(&mm->mm_users) == nr + 1)
+		goto done;
+	/*
+	 * We should find and kill all tasks which use this mm, and we should
+	 * count them correctly into ->nr_threads. We don't take tasklist
+	 * lock, but this is safe wrt:
+	 *
+	 * fork:
+	 *	None of sub-threads can fork after zap_process(leader). All
+	 *	processes which were created before this point should be
+	 *	visible to zap_threads() because copy_process() adds the new
+	 *	process to the tail of init_task.tasks list, and lock/unlock
+	 *	of ->siglock provides a memory barrier.
+	 *
+	 * do_exit:
+	 *	The caller holds mm->mmap_sem. This means that the task which
+	 *	uses this mm can't pass exit_mm(), so it can't exit or clear
+	 *	its ->mm.
+	 *
+	 * de_thread:
+	 *	It does list_replace_rcu(&leader->tasks, &current->tasks),
+	 *	we must see either old or new leader, this does not matter.
+	 *	However, it can change p->sighand, so lock_task_sighand(p)
+	 *	must be used. Since p->mm != NULL and we hold ->mmap_sem
+	 *	it can't fail.
+	 *
+	 *	Note also that "g" can be the old leader with ->mm == NULL
+	 *	and already unhashed and thus removed from ->thread_group.
+	 *	This is OK, __unhash_process()->list_del_rcu() does not
+	 *	clear the ->next pointer, we will find the new leader via
+	 *	next_thread().
+	 */
+	rcu_read_lock();
+	for_each_process(g) {
+		if (g == tsk->group_leader)
+			continue;
+		if (g->flags & PF_KTHREAD)
+			continue;
+		p = g;
+		do {
+			if (p->mm) {
+				if (unlikely(p->mm == mm)) {
+					lock_task_sighand(p, &flags);
+					nr += zap_process(p, exit_code);
+					unlock_task_sighand(p, &flags);
+				}
+				break;
+			}
+		} while_each_thread(g, p);
+	}
+	rcu_read_unlock();
+done:
+	atomic_set(&core_state->nr_threads, nr);
+	return nr;
+}
+
+static int coredump_wait(int exit_code, struct core_state *core_state)
+{
+	struct task_struct *tsk = current;
+	struct mm_struct *mm = tsk->mm;
+	int core_waiters = -EBUSY;
+
+	init_completion(&core_state->startup);
+	core_state->dumper.task = tsk;
+	core_state->dumper.next = NULL;
+
+	down_write(&mm->mmap_sem);
+	if (!mm->core_state)
+		core_waiters = zap_threads(tsk, mm, core_state, exit_code);
+	up_write(&mm->mmap_sem);
+
+	if (core_waiters > 0) {
+		struct core_thread *ptr;
+
+		wait_for_completion(&core_state->startup);
+		/*
+		 * Wait for all the threads to become inactive, so that
+		 * all the thread context (extended register state, like
+		 * fpu etc) gets copied to the memory.
+		 */
+		ptr = core_state->dumper.next;
+		while (ptr != NULL) {
+			wait_task_inactive(ptr->task, 0);
+			ptr = ptr->next;
+		}
+	}
+
+	return core_waiters;
+}
+
+static void coredump_finish(struct mm_struct *mm)
+{
+	struct core_thread *curr, *next;
+	struct task_struct *task;
+
+	next = mm->core_state->dumper.next;
+	while ((curr = next) != NULL) {
+		next = curr->next;
+		task = curr->task;
+		/*
+		 * see exit_mm(), curr->task must not see
+		 * ->task == NULL before we read ->next.
+		 */
+		smp_mb();
+		curr->task = NULL;
+		wake_up_process(task);
+	}
+
+	mm->core_state = NULL;
+}
+
+static void wait_for_dump_helpers(struct file *file)
+{
+	struct pipe_inode_info *pipe;
+
+	pipe = file->f_path.dentry->d_inode->i_pipe;
+
+	pipe_lock(pipe);
+	pipe->readers++;
+	pipe->writers--;
+
+	while ((pipe->readers > 1) && (!signal_pending(current))) {
+		wake_up_interruptible_sync(&pipe->wait);
+		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
+		pipe_wait(pipe);
+	}
+
+	pipe->readers--;
+	pipe->writers++;
+	pipe_unlock(pipe);
+
+}
+
+/*
+ * umh_pipe_setup
+ * helper function to customize the process used
+ * to collect the core in userspace.  Specifically
+ * it sets up a pipe and installs it as fd 0 (stdin)
+ * for the process.  Returns 0 on success, or
+ * PTR_ERR on failure.
+ * Note that it also sets the core limit to 1.  This
+ * is a special value that we use to trap recursive
+ * core dumps
+ */
+static int umh_pipe_setup(struct subprocess_info *info, struct cred *new)
+{
+	struct file *files[2];
+	struct coredump_params *cp = (struct coredump_params *)info->data;
+	int err = create_pipe_files(files, 0);
+	if (err)
+		return err;
+
+	cp->file = files[1];
+
+	replace_fd(0, files[0], 0);
+	/* and disallow core files too */
+	current->signal->rlim[RLIMIT_CORE] = (struct rlimit){1, 1};
+
+	return 0;
+}
+
+void do_coredump(siginfo_t *siginfo, struct pt_regs *regs)
+{
+	struct core_state core_state;
+	struct core_name cn;
+	struct mm_struct *mm = current->mm;
+	struct linux_binfmt * binfmt;
+	const struct cred *old_cred;
+	struct cred *cred;
+	int retval = 0;
+	int flag = 0;
+	int ispipe;
+	struct files_struct *displaced;
+	bool need_nonrelative = false;
+	static atomic_t core_dump_count = ATOMIC_INIT(0);
+	struct coredump_params cprm = {
+		.siginfo = siginfo,
+		.regs = regs,
+		.limit = rlimit(RLIMIT_CORE),
+		/*
+		 * We must use the same mm->flags while dumping core to avoid
+		 * inconsistency of bit flags, since this flag is not protected
+		 * by any locks.
+		 */
+		.mm_flags = mm->flags,
+	};
+
+	audit_core_dumps(siginfo->si_signo);
+
+	binfmt = mm->binfmt;
+	if (!binfmt || !binfmt->core_dump)
+		goto fail;
+	if (!__get_dumpable(cprm.mm_flags))
+		goto fail;
+
+	cred = prepare_creds();
+	if (!cred)
+		goto fail;
+	/*
+	 * We cannot trust fsuid as being the "true" uid of the process
+	 * nor do we know its entire history. We only know it was tainted
+	 * so we dump it as root in mode 2, and only into a controlled
+	 * environment (pipe handler or fully qualified path).
+	 */
+	if (__get_dumpable(cprm.mm_flags) == SUID_DUMPABLE_SAFE) {
+		/* Setuid core dump mode */
+		flag = O_EXCL;		/* Stop rewrite attacks */
+		cred->fsuid = GLOBAL_ROOT_UID;	/* Dump root private */
+		need_nonrelative = true;
+	}
+
+	retval = coredump_wait(siginfo->si_signo, &core_state);
+	if (retval < 0)
+		goto fail_creds;
+
+	old_cred = override_creds(cred);
+
+	/*
+	 * Clear any false indication of pending signals that might
+	 * be seen by the filesystem code called to write the core file.
+	 */
+	clear_thread_flag(TIF_SIGPENDING);
+
+	ispipe = format_corename(&cn, &cprm);
+
+ 	if (ispipe) {
+		int dump_count;
+		char **helper_argv;
+
+		if (ispipe < 0) {
+			printk(KERN_WARNING "format_corename failed\n");
+			printk(KERN_WARNING "Aborting core\n");
+			goto fail_corename;
+		}
+
+		if (cprm.limit == 1) {
+			/* See umh_pipe_setup() which sets RLIMIT_CORE = 1.
+			 *
+			 * Normally core limits are irrelevant to pipes, since
+			 * we're not writing to the file system, but we use
+			 * cprm.limit of 1 here as a speacial value, this is a
+			 * consistent way to catch recursive crashes.
+			 * We can still crash if the core_pattern binary sets
+			 * RLIM_CORE = !1, but it runs as root, and can do
+			 * lots of stupid things.
+			 *
+			 * Note that we use task_tgid_vnr here to grab the pid
+			 * of the process group leader.  That way we get the
+			 * right pid if a thread in a multi-threaded
+			 * core_pattern process dies.
+			 */
+			printk(KERN_WARNING
+				"Process %d(%s) has RLIMIT_CORE set to 1\n",
+				task_tgid_vnr(current), current->comm);
+			printk(KERN_WARNING "Aborting core\n");
+			goto fail_unlock;
+		}
+		cprm.limit = RLIM_INFINITY;
+
+		dump_count = atomic_inc_return(&core_dump_count);
+		if (core_pipe_limit && (core_pipe_limit < dump_count)) {
+			printk(KERN_WARNING "Pid %d(%s) over core_pipe_limit\n",
+			       task_tgid_vnr(current), current->comm);
+			printk(KERN_WARNING "Skipping core dump\n");
+			goto fail_dropcount;
+		}
+
+		helper_argv = argv_split(GFP_KERNEL, cn.corename+1, NULL);
+		if (!helper_argv) {
+			printk(KERN_WARNING "%s failed to allocate memory\n",
+			       __func__);
+			goto fail_dropcount;
+		}
+
+		retval = call_usermodehelper_fns(helper_argv[0], helper_argv,
+					NULL, UMH_WAIT_EXEC, umh_pipe_setup,
+					NULL, &cprm);
+		argv_free(helper_argv);
+		if (retval) {
+ 			printk(KERN_INFO "Core dump to %s pipe failed\n",
+			       cn.corename);
+			goto close_fail;
+ 		}
+	} else {
+		struct inode *inode;
+
+		if (cprm.limit < binfmt->min_coredump)
+			goto fail_unlock;
+
+		if (need_nonrelative && cn.corename[0] != '/') {
+			printk(KERN_WARNING "Pid %d(%s) can only dump core "\
+				"to fully qualified path!\n",
+				task_tgid_vnr(current), current->comm);
+			printk(KERN_WARNING "Skipping core dump\n");
+			goto fail_unlock;
+		}
+
+		cprm.file = filp_open(cn.corename,
+				 O_CREAT | 2 | O_NOFOLLOW | O_LARGEFILE | flag,
+				 0600);
+		if (IS_ERR(cprm.file))
+			goto fail_unlock;
+
+		inode = cprm.file->f_path.dentry->d_inode;
+		if (inode->i_nlink > 1)
+			goto close_fail;
+		if (d_unhashed(cprm.file->f_path.dentry))
+			goto close_fail;
+		/*
+		 * AK: actually i see no reason to not allow this for named
+		 * pipes etc, but keep the previous behaviour for now.
+		 */
+		if (!S_ISREG(inode->i_mode))
+			goto close_fail;
+		/*
+		 * Dont allow local users get cute and trick others to coredump
+		 * into their pre-created files.
+		 */
+		if (!uid_eq(inode->i_uid, current_fsuid()))
+			goto close_fail;
+		if (!cprm.file->f_op || !cprm.file->f_op->write)
+			goto close_fail;
+		if (do_truncate(cprm.file->f_path.dentry, 0, 0, cprm.file))
+			goto close_fail;
+	}
+
+	/* get us an unshared descriptor table; almost always a no-op */
+	retval = unshare_files(&displaced);
+	if (retval)
+		goto close_fail;
+	if (displaced)
+		put_files_struct(displaced);
+	retval = binfmt->core_dump(&cprm);
+	if (retval)
+		current->signal->group_exit_code |= 0x80;
+
+	if (ispipe && core_pipe_limit)
+		wait_for_dump_helpers(cprm.file);
+close_fail:
+	if (cprm.file)
+		filp_close(cprm.file, NULL);
+fail_dropcount:
+	if (ispipe)
+		atomic_dec(&core_dump_count);
+fail_unlock:
+	kfree(cn.corename);
+fail_corename:
+	coredump_finish(mm);
+	revert_creds(old_cred);
+fail_creds:
+	put_cred(cred);
+fail:
+	return;
+}
+
+/*
+ * Core dumping helper functions.  These are the only things you should
+ * do on a core-file: use only these functions to write out all the
+ * necessary info.
+ */
+int dump_write(struct file *file, const void *addr, int nr)
+{
+	return access_ok(VERIFY_READ, addr, nr) && file->f_op->write(file, addr, nr, &file->f_pos) == nr;
+}
+EXPORT_SYMBOL(dump_write);
+
+int dump_seek(struct file *file, loff_t off)
+{
+	int ret = 1;
+
+	if (file->f_op->llseek && file->f_op->llseek != no_llseek) {
+		if (file->f_op->llseek(file, off, SEEK_CUR) < 0)
+			return 0;
+	} else {
+		char *buf = (char *)get_zeroed_page(GFP_KERNEL);
+
+		if (!buf)
+			return 0;
+		while (off > 0) {
+			unsigned long n = off;
+
+			if (n > PAGE_SIZE)
+				n = PAGE_SIZE;
+			if (!dump_write(file, buf, n)) {
+				ret = 0;
+				break;
+			}
+			off -= n;
+		}
+		free_page((unsigned long)buf);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(dump_seek);
diff --git a/fs/coredump.h b/fs/coredump.h
new file mode 100644
index 00000000000..e39ff072110
--- /dev/null
+++ b/fs/coredump.h
@@ -0,0 +1,6 @@
+#ifndef _FS_COREDUMP_H
+#define _FS_COREDUMP_H
+
+extern int __get_dumpable(unsigned long mm_flags);
+
+#endif
diff --git a/fs/dcache.c b/fs/dcache.c
index 693f95bf1ca..3a463d0c4fe 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -2113,7 +2113,7 @@ again:
 	inode = dentry->d_inode;
 	isdir = S_ISDIR(inode->i_mode);
 	if (dentry->d_count == 1) {
-		if (inode && !spin_trylock(&inode->i_lock)) {
+		if (!spin_trylock(&inode->i_lock)) {
 			spin_unlock(&dentry->d_lock);
 			cpu_relax();
 			goto again;
diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c
index 24bb043e50d..4e0886c9e5c 100644
--- a/fs/ecryptfs/main.c
+++ b/fs/ecryptfs/main.c
@@ -711,6 +711,12 @@ static void ecryptfs_free_kmem_caches(void)
 {
 	int i;
 
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
+
 	for (i = 0; i < ARRAY_SIZE(ecryptfs_cache_infos); i++) {
 		struct ecryptfs_cache_info *info;
 
diff --git a/fs/efs/super.c b/fs/efs/super.c
index e755ec746c6..2002431ef9a 100644
--- a/fs/efs/super.c
+++ b/fs/efs/super.c
@@ -96,6 +96,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(efs_inode_cachep);
 }
 
diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index eedec84c180..da72250ddc1 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -346,7 +346,7 @@ static inline struct epitem *ep_item_from_epqueue(poll_table *p)
 /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
 static inline int ep_op_has_event(int op)
 {
-	return op != EPOLL_CTL_DEL;
+	return op == EPOLL_CTL_ADD || op == EPOLL_CTL_MOD;
 }
 
 /* Initialize the poll safe wake up structure */
@@ -676,6 +676,34 @@ static int ep_remove(struct eventpoll *ep, struct epitem *epi)
 	return 0;
 }
 
+/*
+ * Disables a "struct epitem" in the eventpoll set. Returns -EBUSY if the item
+ * had no event flags set, indicating that another thread may be currently
+ * handling that item's events (in the case that EPOLLONESHOT was being
+ * used). Otherwise a zero result indicates that the item has been disabled
+ * from receiving events. A disabled item may be re-enabled via
+ * EPOLL_CTL_MOD. Must be called with "mtx" held.
+ */
+static int ep_disable(struct eventpoll *ep, struct epitem *epi)
+{
+	int result = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ep->lock, flags);
+	if (epi->event.events & ~EP_PRIVATE_BITS) {
+		if (ep_is_linked(&epi->rdllink))
+			list_del_init(&epi->rdllink);
+		/* Ensure ep_poll_callback will not add epi back onto ready
+		   list: */
+		epi->event.events &= EP_PRIVATE_BITS;
+		}
+	else
+		result = -EBUSY;
+	spin_unlock_irqrestore(&ep->lock, flags);
+
+	return result;
+}
+
 static void ep_free(struct eventpoll *ep)
 {
 	struct rb_node *rbp;
@@ -1020,8 +1048,6 @@ static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
 	rb_insert_color(&epi->rbn, &ep->rbr);
 }
 
-
-
 #define PATH_ARR_SIZE 5
 /*
  * These are the number paths of length 1 to 5, that we are allowing to emanate
@@ -1787,6 +1813,12 @@ SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
 		} else
 			error = -ENOENT;
 		break;
+	case EPOLL_CTL_DISABLE:
+		if (epi)
+			error = ep_disable(ep, epi);
+		else
+			error = -ENOENT;
+		break;
 	}
 	mutex_unlock(&ep->mtx);
 
@@ -1810,7 +1842,7 @@ SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
 		int, maxevents, int, timeout)
 {
 	int error;
-	struct file *file;
+	struct fd f;
 	struct eventpoll *ep;
 
 	/* The maximum number of event must be greater than zero */
@@ -1818,38 +1850,33 @@ SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
 		return -EINVAL;
 
 	/* Verify that the area passed by the user is writeable */
-	if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event))) {
-		error = -EFAULT;
-		goto error_return;
-	}
+	if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event)))
+		return -EFAULT;
 
 	/* Get the "struct file *" for the eventpoll file */
-	error = -EBADF;
-	file = fget(epfd);
-	if (!file)
-		goto error_return;
+	f = fdget(epfd);
+	if (!f.file)
+		return -EBADF;
 
 	/*
 	 * We have to check that the file structure underneath the fd
 	 * the user passed to us _is_ an eventpoll file.
 	 */
 	error = -EINVAL;
-	if (!is_file_epoll(file))
+	if (!is_file_epoll(f.file))
 		goto error_fput;
 
 	/*
 	 * At this point it is safe to assume that the "private_data" contains
 	 * our own data structure.
 	 */
-	ep = file->private_data;
+	ep = f.file->private_data;
 
 	/* Time to fish for events ... */
 	error = ep_poll(ep, events, maxevents, timeout);
 
 error_fput:
-	fput(file);
-error_return:
-
+	fdput(f);
 	return error;
 }
 
diff --git a/fs/exec.c b/fs/exec.c
index 574cf4de4ec..9824473a7ec 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -63,22 +63,12 @@
 
 #include <trace/events/task.h>
 #include "internal.h"
+#include "coredump.h"
 
 #include <trace/events/sched.h>
 
-int core_uses_pid;
-char core_pattern[CORENAME_MAX_SIZE] = "core";
-unsigned int core_pipe_limit;
 int suid_dumpable = 0;
 
-struct core_name {
-	char *corename;
-	int used, size;
-};
-static atomic_t call_count = ATOMIC_INIT(1);
-
-/* The maximal length of core_pattern is also specified in sysctl.c */
-
 static LIST_HEAD(formats);
 static DEFINE_RWLOCK(binfmt_lock);
 
@@ -1006,40 +996,6 @@ no_thread_group:
 	return 0;
 }
 
-/*
- * These functions flushes out all traces of the currently running executable
- * so that a new one can be started
- */
-static void flush_old_files(struct files_struct * files)
-{
-	long j = -1;
-	struct fdtable *fdt;
-
-	spin_lock(&files->file_lock);
-	for (;;) {
-		unsigned long set, i;
-
-		j++;
-		i = j * BITS_PER_LONG;
-		fdt = files_fdtable(files);
-		if (i >= fdt->max_fds)
-			break;
-		set = fdt->close_on_exec[j];
-		if (!set)
-			continue;
-		fdt->close_on_exec[j] = 0;
-		spin_unlock(&files->file_lock);
-		for ( ; set ; i++,set >>= 1) {
-			if (set & 1) {
-				sys_close(i);
-			}
-		}
-		spin_lock(&files->file_lock);
-
-	}
-	spin_unlock(&files->file_lock);
-}
-
 char *get_task_comm(char *buf, struct task_struct *tsk)
 {
 	/* buf must be at least sizeof(tsk->comm) in size */
@@ -1050,6 +1006,11 @@ char *get_task_comm(char *buf, struct task_struct *tsk)
 }
 EXPORT_SYMBOL_GPL(get_task_comm);
 
+/*
+ * These functions flushes out all traces of the currently running executable
+ * so that a new one can be started
+ */
+
 void set_task_comm(struct task_struct *tsk, char *buf)
 {
 	task_lock(tsk);
@@ -1136,7 +1097,7 @@ void setup_new_exec(struct linux_binprm * bprm)
 	current->sas_ss_sp = current->sas_ss_size = 0;
 
 	if (uid_eq(current_euid(), current_uid()) && gid_eq(current_egid(), current_gid()))
-		set_dumpable(current->mm, 1);
+		set_dumpable(current->mm, SUID_DUMPABLE_ENABLED);
 	else
 		set_dumpable(current->mm, suid_dumpable);
 
@@ -1171,7 +1132,7 @@ void setup_new_exec(struct linux_binprm * bprm)
 	current->self_exec_id++;
 			
 	flush_signal_handlers(current, 0);
-	flush_old_files(current->files);
+	do_close_on_exec(current->files);
 }
 EXPORT_SYMBOL(setup_new_exec);
 
@@ -1632,353 +1593,6 @@ void set_binfmt(struct linux_binfmt *new)
 
 EXPORT_SYMBOL(set_binfmt);
 
-static int expand_corename(struct core_name *cn)
-{
-	char *old_corename = cn->corename;
-
-	cn->size = CORENAME_MAX_SIZE * atomic_inc_return(&call_count);
-	cn->corename = krealloc(old_corename, cn->size, GFP_KERNEL);
-
-	if (!cn->corename) {
-		kfree(old_corename);
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static int cn_printf(struct core_name *cn, const char *fmt, ...)
-{
-	char *cur;
-	int need;
-	int ret;
-	va_list arg;
-
-	va_start(arg, fmt);
-	need = vsnprintf(NULL, 0, fmt, arg);
-	va_end(arg);
-
-	if (likely(need < cn->size - cn->used - 1))
-		goto out_printf;
-
-	ret = expand_corename(cn);
-	if (ret)
-		goto expand_fail;
-
-out_printf:
-	cur = cn->corename + cn->used;
-	va_start(arg, fmt);
-	vsnprintf(cur, need + 1, fmt, arg);
-	va_end(arg);
-	cn->used += need;
-	return 0;
-
-expand_fail:
-	return ret;
-}
-
-static void cn_escape(char *str)
-{
-	for (; *str; str++)
-		if (*str == '/')
-			*str = '!';
-}
-
-static int cn_print_exe_file(struct core_name *cn)
-{
-	struct file *exe_file;
-	char *pathbuf, *path;
-	int ret;
-
-	exe_file = get_mm_exe_file(current->mm);
-	if (!exe_file) {
-		char *commstart = cn->corename + cn->used;
-		ret = cn_printf(cn, "%s (path unknown)", current->comm);
-		cn_escape(commstart);
-		return ret;
-	}
-
-	pathbuf = kmalloc(PATH_MAX, GFP_TEMPORARY);
-	if (!pathbuf) {
-		ret = -ENOMEM;
-		goto put_exe_file;
-	}
-
-	path = d_path(&exe_file->f_path, pathbuf, PATH_MAX);
-	if (IS_ERR(path)) {
-		ret = PTR_ERR(path);
-		goto free_buf;
-	}
-
-	cn_escape(path);
-
-	ret = cn_printf(cn, "%s", path);
-
-free_buf:
-	kfree(pathbuf);
-put_exe_file:
-	fput(exe_file);
-	return ret;
-}
-
-/* format_corename will inspect the pattern parameter, and output a
- * name into corename, which must have space for at least
- * CORENAME_MAX_SIZE bytes plus one byte for the zero terminator.
- */
-static int format_corename(struct core_name *cn, long signr)
-{
-	const struct cred *cred = current_cred();
-	const char *pat_ptr = core_pattern;
-	int ispipe = (*pat_ptr == '|');
-	int pid_in_pattern = 0;
-	int err = 0;
-
-	cn->size = CORENAME_MAX_SIZE * atomic_read(&call_count);
-	cn->corename = kmalloc(cn->size, GFP_KERNEL);
-	cn->used = 0;
-
-	if (!cn->corename)
-		return -ENOMEM;
-
-	/* Repeat as long as we have more pattern to process and more output
-	   space */
-	while (*pat_ptr) {
-		if (*pat_ptr != '%') {
-			if (*pat_ptr == 0)
-				goto out;
-			err = cn_printf(cn, "%c", *pat_ptr++);
-		} else {
-			switch (*++pat_ptr) {
-			/* single % at the end, drop that */
-			case 0:
-				goto out;
-			/* Double percent, output one percent */
-			case '%':
-				err = cn_printf(cn, "%c", '%');
-				break;
-			/* pid */
-			case 'p':
-				pid_in_pattern = 1;
-				err = cn_printf(cn, "%d",
-					      task_tgid_vnr(current));
-				break;
-			/* uid */
-			case 'u':
-				err = cn_printf(cn, "%d", cred->uid);
-				break;
-			/* gid */
-			case 'g':
-				err = cn_printf(cn, "%d", cred->gid);
-				break;
-			/* signal that caused the coredump */
-			case 's':
-				err = cn_printf(cn, "%ld", signr);
-				break;
-			/* UNIX time of coredump */
-			case 't': {
-				struct timeval tv;
-				do_gettimeofday(&tv);
-				err = cn_printf(cn, "%lu", tv.tv_sec);
-				break;
-			}
-			/* hostname */
-			case 'h': {
-				char *namestart = cn->corename + cn->used;
-				down_read(&uts_sem);
-				err = cn_printf(cn, "%s",
-					      utsname()->nodename);
-				up_read(&uts_sem);
-				cn_escape(namestart);
-				break;
-			}
-			/* executable */
-			case 'e': {
-				char *commstart = cn->corename + cn->used;
-				err = cn_printf(cn, "%s", current->comm);
-				cn_escape(commstart);
-				break;
-			}
-			case 'E':
-				err = cn_print_exe_file(cn);
-				break;
-			/* core limit size */
-			case 'c':
-				err = cn_printf(cn, "%lu",
-					      rlimit(RLIMIT_CORE));
-				break;
-			default:
-				break;
-			}
-			++pat_ptr;
-		}
-
-		if (err)
-			return err;
-	}
-
-	/* Backward compatibility with core_uses_pid:
-	 *
-	 * If core_pattern does not include a %p (as is the default)
-	 * and core_uses_pid is set, then .%pid will be appended to
-	 * the filename. Do not do this for piped commands. */
-	if (!ispipe && !pid_in_pattern && core_uses_pid) {
-		err = cn_printf(cn, ".%d", task_tgid_vnr(current));
-		if (err)
-			return err;
-	}
-out:
-	return ispipe;
-}
-
-static int zap_process(struct task_struct *start, int exit_code)
-{
-	struct task_struct *t;
-	int nr = 0;
-
-	start->signal->flags = SIGNAL_GROUP_EXIT;
-	start->signal->group_exit_code = exit_code;
-	start->signal->group_stop_count = 0;
-
-	t = start;
-	do {
-		task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
-		if (t != current && t->mm) {
-			sigaddset(&t->pending.signal, SIGKILL);
-			signal_wake_up(t, 1);
-			nr++;
-		}
-	} while_each_thread(start, t);
-
-	return nr;
-}
-
-static inline int zap_threads(struct task_struct *tsk, struct mm_struct *mm,
-				struct core_state *core_state, int exit_code)
-{
-	struct task_struct *g, *p;
-	unsigned long flags;
-	int nr = -EAGAIN;
-
-	spin_lock_irq(&tsk->sighand->siglock);
-	if (!signal_group_exit(tsk->signal)) {
-		mm->core_state = core_state;
-		nr = zap_process(tsk, exit_code);
-	}
-	spin_unlock_irq(&tsk->sighand->siglock);
-	if (unlikely(nr < 0))
-		return nr;
-
-	if (atomic_read(&mm->mm_users) == nr + 1)
-		goto done;
-	/*
-	 * We should find and kill all tasks which use this mm, and we should
-	 * count them correctly into ->nr_threads. We don't take tasklist
-	 * lock, but this is safe wrt:
-	 *
-	 * fork:
-	 *	None of sub-threads can fork after zap_process(leader). All
-	 *	processes which were created before this point should be
-	 *	visible to zap_threads() because copy_process() adds the new
-	 *	process to the tail of init_task.tasks list, and lock/unlock
-	 *	of ->siglock provides a memory barrier.
-	 *
-	 * do_exit:
-	 *	The caller holds mm->mmap_sem. This means that the task which
-	 *	uses this mm can't pass exit_mm(), so it can't exit or clear
-	 *	its ->mm.
-	 *
-	 * de_thread:
-	 *	It does list_replace_rcu(&leader->tasks, &current->tasks),
-	 *	we must see either old or new leader, this does not matter.
-	 *	However, it can change p->sighand, so lock_task_sighand(p)
-	 *	must be used. Since p->mm != NULL and we hold ->mmap_sem
-	 *	it can't fail.
-	 *
-	 *	Note also that "g" can be the old leader with ->mm == NULL
-	 *	and already unhashed and thus removed from ->thread_group.
-	 *	This is OK, __unhash_process()->list_del_rcu() does not
-	 *	clear the ->next pointer, we will find the new leader via
-	 *	next_thread().
-	 */
-	rcu_read_lock();
-	for_each_process(g) {
-		if (g == tsk->group_leader)
-			continue;
-		if (g->flags & PF_KTHREAD)
-			continue;
-		p = g;
-		do {
-			if (p->mm) {
-				if (unlikely(p->mm == mm)) {
-					lock_task_sighand(p, &flags);
-					nr += zap_process(p, exit_code);
-					unlock_task_sighand(p, &flags);
-				}
-				break;
-			}
-		} while_each_thread(g, p);
-	}
-	rcu_read_unlock();
-done:
-	atomic_set(&core_state->nr_threads, nr);
-	return nr;
-}
-
-static int coredump_wait(int exit_code, struct core_state *core_state)
-{
-	struct task_struct *tsk = current;
-	struct mm_struct *mm = tsk->mm;
-	int core_waiters = -EBUSY;
-
-	init_completion(&core_state->startup);
-	core_state->dumper.task = tsk;
-	core_state->dumper.next = NULL;
-
-	down_write(&mm->mmap_sem);
-	if (!mm->core_state)
-		core_waiters = zap_threads(tsk, mm, core_state, exit_code);
-	up_write(&mm->mmap_sem);
-
-	if (core_waiters > 0) {
-		struct core_thread *ptr;
-
-		wait_for_completion(&core_state->startup);
-		/*
-		 * Wait for all the threads to become inactive, so that
-		 * all the thread context (extended register state, like
-		 * fpu etc) gets copied to the memory.
-		 */
-		ptr = core_state->dumper.next;
-		while (ptr != NULL) {
-			wait_task_inactive(ptr->task, 0);
-			ptr = ptr->next;
-		}
-	}
-
-	return core_waiters;
-}
-
-static void coredump_finish(struct mm_struct *mm)
-{
-	struct core_thread *curr, *next;
-	struct task_struct *task;
-
-	next = mm->core_state->dumper.next;
-	while ((curr = next) != NULL) {
-		next = curr->next;
-		task = curr->task;
-		/*
-		 * see exit_mm(), curr->task must not see
-		 * ->task == NULL before we read ->next.
-		 */
-		smp_mb();
-		curr->task = NULL;
-		wake_up_process(task);
-	}
-
-	mm->core_state = NULL;
-}
-
 /*
  * set_dumpable converts traditional three-value dumpable to two flags and
  * stores them into mm->flags.  It modifies lower two bits of mm->flags, but
@@ -2020,7 +1634,7 @@ void set_dumpable(struct mm_struct *mm, int value)
 	}
 }
 
-static int __get_dumpable(unsigned long mm_flags)
+int __get_dumpable(unsigned long mm_flags)
 {
 	int ret;
 
@@ -2032,290 +1646,3 @@ int get_dumpable(struct mm_struct *mm)
 {
 	return __get_dumpable(mm->flags);
 }
-
-static void wait_for_dump_helpers(struct file *file)
-{
-	struct pipe_inode_info *pipe;
-
-	pipe = file->f_path.dentry->d_inode->i_pipe;
-
-	pipe_lock(pipe);
-	pipe->readers++;
-	pipe->writers--;
-
-	while ((pipe->readers > 1) && (!signal_pending(current))) {
-		wake_up_interruptible_sync(&pipe->wait);
-		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
-		pipe_wait(pipe);
-	}
-
-	pipe->readers--;
-	pipe->writers++;
-	pipe_unlock(pipe);
-
-}
-
-
-/*
- * umh_pipe_setup
- * helper function to customize the process used
- * to collect the core in userspace.  Specifically
- * it sets up a pipe and installs it as fd 0 (stdin)
- * for the process.  Returns 0 on success, or
- * PTR_ERR on failure.
- * Note that it also sets the core limit to 1.  This
- * is a special value that we use to trap recursive
- * core dumps
- */
-static int umh_pipe_setup(struct subprocess_info *info, struct cred *new)
-{
-	struct file *files[2];
-	struct fdtable *fdt;
-	struct coredump_params *cp = (struct coredump_params *)info->data;
-	struct files_struct *cf = current->files;
-	int err = create_pipe_files(files, 0);
-	if (err)
-		return err;
-
-	cp->file = files[1];
-
-	sys_close(0);
-	fd_install(0, files[0]);
-	spin_lock(&cf->file_lock);
-	fdt = files_fdtable(cf);
-	__set_open_fd(0, fdt);
-	__clear_close_on_exec(0, fdt);
-	spin_unlock(&cf->file_lock);
-
-	/* and disallow core files too */
-	current->signal->rlim[RLIMIT_CORE] = (struct rlimit){1, 1};
-
-	return 0;
-}
-
-void do_coredump(long signr, int exit_code, struct pt_regs *regs)
-{
-	struct core_state core_state;
-	struct core_name cn;
-	struct mm_struct *mm = current->mm;
-	struct linux_binfmt * binfmt;
-	const struct cred *old_cred;
-	struct cred *cred;
-	int retval = 0;
-	int flag = 0;
-	int ispipe;
-	bool need_nonrelative = false;
-	static atomic_t core_dump_count = ATOMIC_INIT(0);
-	struct coredump_params cprm = {
-		.signr = signr,
-		.regs = regs,
-		.limit = rlimit(RLIMIT_CORE),
-		/*
-		 * We must use the same mm->flags while dumping core to avoid
-		 * inconsistency of bit flags, since this flag is not protected
-		 * by any locks.
-		 */
-		.mm_flags = mm->flags,
-	};
-
-	audit_core_dumps(signr);
-
-	binfmt = mm->binfmt;
-	if (!binfmt || !binfmt->core_dump)
-		goto fail;
-	if (!__get_dumpable(cprm.mm_flags))
-		goto fail;
-
-	cred = prepare_creds();
-	if (!cred)
-		goto fail;
-	/*
-	 * We cannot trust fsuid as being the "true" uid of the process
-	 * nor do we know its entire history. We only know it was tainted
-	 * so we dump it as root in mode 2, and only into a controlled
-	 * environment (pipe handler or fully qualified path).
-	 */
-	if (__get_dumpable(cprm.mm_flags) == SUID_DUMPABLE_SAFE) {
-		/* Setuid core dump mode */
-		flag = O_EXCL;		/* Stop rewrite attacks */
-		cred->fsuid = GLOBAL_ROOT_UID;	/* Dump root private */
-		need_nonrelative = true;
-	}
-
-	retval = coredump_wait(exit_code, &core_state);
-	if (retval < 0)
-		goto fail_creds;
-
-	old_cred = override_creds(cred);
-
-	/*
-	 * Clear any false indication of pending signals that might
-	 * be seen by the filesystem code called to write the core file.
-	 */
-	clear_thread_flag(TIF_SIGPENDING);
-
-	ispipe = format_corename(&cn, signr);
-
- 	if (ispipe) {
-		int dump_count;
-		char **helper_argv;
-
-		if (ispipe < 0) {
-			printk(KERN_WARNING "format_corename failed\n");
-			printk(KERN_WARNING "Aborting core\n");
-			goto fail_corename;
-		}
-
-		if (cprm.limit == 1) {
-			/* See umh_pipe_setup() which sets RLIMIT_CORE = 1.
-			 *
-			 * Normally core limits are irrelevant to pipes, since
-			 * we're not writing to the file system, but we use
-			 * cprm.limit of 1 here as a speacial value, this is a
-			 * consistent way to catch recursive crashes.
-			 * We can still crash if the core_pattern binary sets
-			 * RLIM_CORE = !1, but it runs as root, and can do
-			 * lots of stupid things.
-			 *
-			 * Note that we use task_tgid_vnr here to grab the pid
-			 * of the process group leader.  That way we get the
-			 * right pid if a thread in a multi-threaded
-			 * core_pattern process dies.
-			 */
-			printk(KERN_WARNING
-				"Process %d(%s) has RLIMIT_CORE set to 1\n",
-				task_tgid_vnr(current), current->comm);
-			printk(KERN_WARNING "Aborting core\n");
-			goto fail_unlock;
-		}
-		cprm.limit = RLIM_INFINITY;
-
-		dump_count = atomic_inc_return(&core_dump_count);
-		if (core_pipe_limit && (core_pipe_limit < dump_count)) {
-			printk(KERN_WARNING "Pid %d(%s) over core_pipe_limit\n",
-			       task_tgid_vnr(current), current->comm);
-			printk(KERN_WARNING "Skipping core dump\n");
-			goto fail_dropcount;
-		}
-
-		helper_argv = argv_split(GFP_KERNEL, cn.corename+1, NULL);
-		if (!helper_argv) {
-			printk(KERN_WARNING "%s failed to allocate memory\n",
-			       __func__);
-			goto fail_dropcount;
-		}
-
-		retval = call_usermodehelper_fns(helper_argv[0], helper_argv,
-					NULL, UMH_WAIT_EXEC, umh_pipe_setup,
-					NULL, &cprm);
-		argv_free(helper_argv);
-		if (retval) {
- 			printk(KERN_INFO "Core dump to %s pipe failed\n",
-			       cn.corename);
-			goto close_fail;
- 		}
-	} else {
-		struct inode *inode;
-
-		if (cprm.limit < binfmt->min_coredump)
-			goto fail_unlock;
-
-		if (need_nonrelative && cn.corename[0] != '/') {
-			printk(KERN_WARNING "Pid %d(%s) can only dump core "\
-				"to fully qualified path!\n",
-				task_tgid_vnr(current), current->comm);
-			printk(KERN_WARNING "Skipping core dump\n");
-			goto fail_unlock;
-		}
-
-		cprm.file = filp_open(cn.corename,
-				 O_CREAT | 2 | O_NOFOLLOW | O_LARGEFILE | flag,
-				 0600);
-		if (IS_ERR(cprm.file))
-			goto fail_unlock;
-
-		inode = cprm.file->f_path.dentry->d_inode;
-		if (inode->i_nlink > 1)
-			goto close_fail;
-		if (d_unhashed(cprm.file->f_path.dentry))
-			goto close_fail;
-		/*
-		 * AK: actually i see no reason to not allow this for named
-		 * pipes etc, but keep the previous behaviour for now.
-		 */
-		if (!S_ISREG(inode->i_mode))
-			goto close_fail;
-		/*
-		 * Dont allow local users get cute and trick others to coredump
-		 * into their pre-created files.
-		 */
-		if (!uid_eq(inode->i_uid, current_fsuid()))
-			goto close_fail;
-		if (!cprm.file->f_op || !cprm.file->f_op->write)
-			goto close_fail;
-		if (do_truncate(cprm.file->f_path.dentry, 0, 0, cprm.file))
-			goto close_fail;
-	}
-
-	retval = binfmt->core_dump(&cprm);
-	if (retval)
-		current->signal->group_exit_code |= 0x80;
-
-	if (ispipe && core_pipe_limit)
-		wait_for_dump_helpers(cprm.file);
-close_fail:
-	if (cprm.file)
-		filp_close(cprm.file, NULL);
-fail_dropcount:
-	if (ispipe)
-		atomic_dec(&core_dump_count);
-fail_unlock:
-	kfree(cn.corename);
-fail_corename:
-	coredump_finish(mm);
-	revert_creds(old_cred);
-fail_creds:
-	put_cred(cred);
-fail:
-	return;
-}
-
-/*
- * Core dumping helper functions.  These are the only things you should
- * do on a core-file: use only these functions to write out all the
- * necessary info.
- */
-int dump_write(struct file *file, const void *addr, int nr)
-{
-	return access_ok(VERIFY_READ, addr, nr) && file->f_op->write(file, addr, nr, &file->f_pos) == nr;
-}
-EXPORT_SYMBOL(dump_write);
-
-int dump_seek(struct file *file, loff_t off)
-{
-	int ret = 1;
-
-	if (file->f_op->llseek && file->f_op->llseek != no_llseek) {
-		if (file->f_op->llseek(file, off, SEEK_CUR) < 0)
-			return 0;
-	} else {
-		char *buf = (char *)get_zeroed_page(GFP_KERNEL);
-
-		if (!buf)
-			return 0;
-		while (off > 0) {
-			unsigned long n = off;
-
-			if (n > PAGE_SIZE)
-				n = PAGE_SIZE;
-			if (!dump_write(file, buf, n)) {
-				ret = 0;
-				break;
-			}
-			off -= n;
-		}
-		free_page((unsigned long)buf);
-	}
-	return ret;
-}
-EXPORT_SYMBOL(dump_seek);
diff --git a/fs/exofs/super.c b/fs/exofs/super.c
index dde41a75c7c..59e3bbfac0b 100644
--- a/fs/exofs/super.c
+++ b/fs/exofs/super.c
@@ -206,6 +206,11 @@ static int init_inodecache(void)
  */
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(exofs_inode_cachep);
 }
 
diff --git a/fs/ext2/super.c b/fs/ext2/super.c
index af74d9e27b7..6c205d0c565 100644
--- a/fs/ext2/super.c
+++ b/fs/ext2/super.c
@@ -206,6 +206,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ext2_inode_cachep);
 }
 
diff --git a/fs/ext3/super.c b/fs/ext3/super.c
index 09b8455bd7e..17ae5c83d23 100644
--- a/fs/ext3/super.c
+++ b/fs/ext3/super.c
@@ -532,6 +532,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ext3_inode_cachep);
 }
 
@@ -975,7 +980,7 @@ static int parse_options (char *options, struct super_block *sb,
 		 * Initialize args struct so we know whether arg was
 		 * found; some options take optional arguments.
 		 */
-		args[0].to = args[0].from = 0;
+		args[0].to = args[0].from = NULL;
 		token = match_token(p, tokens, args);
 		switch (token) {
 		case Opt_bsd_df:
@@ -1479,10 +1484,12 @@ static void ext3_orphan_cleanup (struct super_block * sb,
 	}
 
 	if (EXT3_SB(sb)->s_mount_state & EXT3_ERROR_FS) {
-		if (es->s_last_orphan)
+		/* don't clear list on RO mount w/ errors */
+		if (es->s_last_orphan && !(s_flags & MS_RDONLY)) {
 			jbd_debug(1, "Errors on filesystem, "
 				  "clearing orphan list.\n");
-		es->s_last_orphan = 0;
+			es->s_last_orphan = 0;
+		}
 		jbd_debug(1, "Skipping orphan recovery on fs with errors.\n");
 		return;
 	}
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index 7f7dad78760..5439d6a56e9 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -233,7 +233,7 @@ group_extend_out:
 
 	case EXT4_IOC_MOVE_EXT: {
 		struct move_extent me;
-		struct file *donor_filp;
+		struct fd donor;
 		int err;
 
 		if (!(filp->f_mode & FMODE_READ) ||
@@ -245,11 +245,11 @@ group_extend_out:
 			return -EFAULT;
 		me.moved_len = 0;
 
-		donor_filp = fget(me.donor_fd);
-		if (!donor_filp)
+		donor = fdget(me.donor_fd);
+		if (!donor.file)
 			return -EBADF;
 
-		if (!(donor_filp->f_mode & FMODE_WRITE)) {
+		if (!(donor.file->f_mode & FMODE_WRITE)) {
 			err = -EBADF;
 			goto mext_out;
 		}
@@ -258,14 +258,15 @@ group_extend_out:
 			       EXT4_FEATURE_RO_COMPAT_BIGALLOC)) {
 			ext4_msg(sb, KERN_ERR,
 				 "Online defrag not supported with bigalloc");
-			return -EOPNOTSUPP;
+			err = -EOPNOTSUPP;
+			goto mext_out;
 		}
 
 		err = mnt_want_write_file(filp);
 		if (err)
 			goto mext_out;
 
-		err = ext4_move_extents(filp, donor_filp, me.orig_start,
+		err = ext4_move_extents(filp, donor.file, me.orig_start,
 					me.donor_start, me.len, &me.moved_len);
 		mnt_drop_write_file(filp);
 
@@ -273,7 +274,7 @@ group_extend_out:
 				 &me, sizeof(me)))
 			err = -EFAULT;
 mext_out:
-		fput(donor_filp);
+		fdput(donor);
 		return err;
 	}
 
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index 1f15cc836fb..69c55d4e462 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -1019,6 +1019,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ext4_inode_cachep);
 }
 
diff --git a/fs/fat/Makefile b/fs/fat/Makefile
index e06190322c1..964b634f666 100644
--- a/fs/fat/Makefile
+++ b/fs/fat/Makefile
@@ -6,6 +6,6 @@ obj-$(CONFIG_FAT_FS) += fat.o
 obj-$(CONFIG_VFAT_FS) += vfat.o
 obj-$(CONFIG_MSDOS_FS) += msdos.o
 
-fat-y := cache.o dir.o fatent.o file.o inode.o misc.o
+fat-y := cache.o dir.o fatent.o file.o inode.o misc.o nfs.o
 vfat-y := namei_vfat.o
 msdos-y := namei_msdos.o
diff --git a/fs/fat/cache.c b/fs/fat/cache.c
index 1cc7038e273..91ad9e1c944 100644
--- a/fs/fat/cache.c
+++ b/fs/fat/cache.c
@@ -190,7 +190,8 @@ static void __fat_cache_inval_inode(struct inode *inode)
 	struct fat_cache *cache;
 
 	while (!list_empty(&i->cache_lru)) {
-		cache = list_entry(i->cache_lru.next, struct fat_cache, cache_list);
+		cache = list_entry(i->cache_lru.next,
+				   struct fat_cache, cache_list);
 		list_del_init(&cache->cache_list);
 		i->nr_caches--;
 		fat_cache_free(cache);
@@ -261,9 +262,10 @@ int fat_get_cluster(struct inode *inode, int cluster, int *fclus, int *dclus)
 		if (nr < 0)
 			goto out;
 		else if (nr == FAT_ENT_FREE) {
-			fat_fs_error_ratelimit(sb, "%s: invalid cluster chain"
-					       " (i_pos %lld)", __func__,
-					       MSDOS_I(inode)->i_pos);
+			fat_fs_error_ratelimit(sb,
+				       "%s: invalid cluster chain (i_pos %lld)",
+				       __func__,
+				       MSDOS_I(inode)->i_pos);
 			nr = -EIO;
 			goto out;
 		} else if (nr == FAT_ENT_EOF) {
diff --git a/fs/fat/dir.c b/fs/fat/dir.c
index dc49ed2cbff..bca6d0a1255 100644
--- a/fs/fat/dir.c
+++ b/fs/fat/dir.c
@@ -18,7 +18,7 @@
 #include <linux/time.h>
 #include <linux/buffer_head.h>
 #include <linux/compat.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <linux/kernel.h>
 #include "fat.h"
 
@@ -123,7 +123,8 @@ static inline int fat_get_entry(struct inode *dir, loff_t *pos,
 {
 	/* Fast stuff first */
 	if (*bh && *de &&
-	    (*de - (struct msdos_dir_entry *)(*bh)->b_data) < MSDOS_SB(dir->i_sb)->dir_per_block - 1) {
+	   (*de - (struct msdos_dir_entry *)(*bh)->b_data) <
+				MSDOS_SB(dir->i_sb)->dir_per_block - 1) {
 		*pos += sizeof(struct msdos_dir_entry);
 		(*de)++;
 		return 0;
@@ -155,7 +156,8 @@ static int uni16_to_x8(struct super_block *sb, unsigned char *ascii,
 
 	while (*ip && ((len - NLS_MAX_CHARSET_SIZE) > 0)) {
 		ec = *ip++;
-		if ((charlen = nls->uni2char(ec, op, NLS_MAX_CHARSET_SIZE)) > 0) {
+		charlen = nls->uni2char(ec, op, NLS_MAX_CHARSET_SIZE);
+		if (charlen > 0) {
 			op += charlen;
 			len -= charlen;
 		} else {
@@ -172,12 +174,12 @@ static int uni16_to_x8(struct super_block *sb, unsigned char *ascii,
 	}
 
 	if (unlikely(*ip)) {
-		fat_msg(sb, KERN_WARNING, "filename was truncated while "
-			"converting.");
+		fat_msg(sb, KERN_WARNING,
+			"filename was truncated while converting.");
 	}
 
 	*op = 0;
-	return (op - ascii);
+	return op - ascii;
 }
 
 static inline int fat_uni_to_x8(struct super_block *sb, const wchar_t *uni,
@@ -205,7 +207,8 @@ fat_short2uni(struct nls_table *t, unsigned char *c, int clen, wchar_t *uni)
 }
 
 static inline int
-fat_short2lower_uni(struct nls_table *t, unsigned char *c, int clen, wchar_t *uni)
+fat_short2lower_uni(struct nls_table *t, unsigned char *c,
+		    int clen, wchar_t *uni)
 {
 	int charlen;
 	wchar_t wc;
@@ -220,7 +223,8 @@ fat_short2lower_uni(struct nls_table *t, unsigned char *c, int clen, wchar_t *un
 		if (!nc)
 			nc = *c;
 
-		if ( (charlen = t->char2uni(&nc, 1, uni)) < 0) {
+		charlen = t->char2uni(&nc, 1, uni);
+		if (charlen < 0) {
 			*uni = 0x003f;	/* a question mark */
 			charlen = 1;
 		}
@@ -537,7 +541,6 @@ end_of_dir:
 
 	return err;
 }
-
 EXPORT_SYMBOL_GPL(fat_search_long);
 
 struct fat_ioctl_filldir_callback {
@@ -574,7 +577,8 @@ static int __fat_readdir(struct inode *inode, struct file *filp, void *dirent,
 	/* Fake . and .. for the root directory. */
 	if (inode->i_ino == MSDOS_ROOT_INO) {
 		while (cpos < 2) {
-			if (filldir(dirent, "..", cpos+1, cpos, MSDOS_ROOT_INO, DT_DIR) < 0)
+			if (filldir(dirent, "..", cpos+1, cpos,
+				    MSDOS_ROOT_INO, DT_DIR) < 0)
 				goto out;
 			cpos++;
 			filp->f_pos++;
@@ -872,25 +876,26 @@ static int fat_get_short_entry(struct inode *dir, loff_t *pos,
 }
 
 /*
- * The ".." entry can not provide the "struct fat_slot_info" informations
- * for inode. So, this function provide the some informations only.
+ * The ".." entry can not provide the "struct fat_slot_info" information
+ * for inode, nor a usable i_pos. So, this function provides some information
+ * only.
+ *
+ * Since this function walks through the on-disk inodes within a directory,
+ * callers are responsible for taking any locks necessary to prevent the
+ * directory from changing.
  */
 int fat_get_dotdot_entry(struct inode *dir, struct buffer_head **bh,
-			 struct msdos_dir_entry **de, loff_t *i_pos)
+			 struct msdos_dir_entry **de)
 {
-	loff_t offset;
+	loff_t offset = 0;
 
-	offset = 0;
-	*bh = NULL;
+	*de = NULL;
 	while (fat_get_short_entry(dir, &offset, bh, de) >= 0) {
-		if (!strncmp((*de)->name, MSDOS_DOTDOT, MSDOS_NAME)) {
-			*i_pos = fat_make_i_pos(dir->i_sb, *bh, *de);
+		if (!strncmp((*de)->name, MSDOS_DOTDOT, MSDOS_NAME))
 			return 0;
-		}
 	}
 	return -ENOENT;
 }
-
 EXPORT_SYMBOL_GPL(fat_get_dotdot_entry);
 
 /* See if directory is empty */
@@ -913,7 +918,6 @@ int fat_dir_empty(struct inode *dir)
 	brelse(bh);
 	return result;
 }
-
 EXPORT_SYMBOL_GPL(fat_dir_empty);
 
 /*
@@ -959,7 +963,6 @@ int fat_scan(struct inode *dir, const unsigned char *name,
 	}
 	return -ENOENT;
 }
-
 EXPORT_SYMBOL_GPL(fat_scan);
 
 static int __fat_remove_entries(struct inode *dir, loff_t pos, int nr_slots)
@@ -1047,7 +1050,6 @@ int fat_remove_entries(struct inode *dir, struct fat_slot_info *sinfo)
 
 	return 0;
 }
-
 EXPORT_SYMBOL_GPL(fat_remove_entries);
 
 static int fat_zeroed_cluster(struct inode *dir, sector_t blknr, int nr_used,
@@ -1141,10 +1143,8 @@ int fat_alloc_new_dir(struct inode *dir, struct timespec *ts)
 		de[0].ctime_cs = de[1].ctime_cs = 0;
 		de[0].adate = de[0].cdate = de[1].adate = de[1].cdate = 0;
 	}
-	de[0].start = cpu_to_le16(cluster);
-	de[0].starthi = cpu_to_le16(cluster >> 16);
-	de[1].start = cpu_to_le16(MSDOS_I(dir)->i_logstart);
-	de[1].starthi = cpu_to_le16(MSDOS_I(dir)->i_logstart >> 16);
+	fat_set_start(&de[0], cluster);
+	fat_set_start(&de[1], MSDOS_I(dir)->i_logstart);
 	de[0].size = de[1].size = 0;
 	memset(de + 2, 0, sb->s_blocksize - 2 * sizeof(*de));
 	set_buffer_uptodate(bhs[0]);
@@ -1161,7 +1161,6 @@ error_free:
 error:
 	return err;
 }
-
 EXPORT_SYMBOL_GPL(fat_alloc_new_dir);
 
 static int fat_add_new_entries(struct inode *dir, void *slots, int nr_slots,
@@ -1377,5 +1376,4 @@ error_remove:
 		__fat_remove_entries(dir, pos, free_slots);
 	return err;
 }
-
 EXPORT_SYMBOL_GPL(fat_add_entries);
diff --git a/fs/fat/fat.h b/fs/fat/fat.h
index 7d8e0dcac5d..ca7e8f8bad7 100644
--- a/fs/fat/fat.h
+++ b/fs/fat/fat.h
@@ -5,6 +5,7 @@
 #include <linux/string.h>
 #include <linux/nls.h>
 #include <linux/fs.h>
+#include <linux/hash.h>
 #include <linux/mutex.h>
 #include <linux/ratelimit.h>
 #include <linux/msdos_fs.h>
@@ -27,26 +28,27 @@ struct fat_mount_options {
 	kgid_t fs_gid;
 	unsigned short fs_fmask;
 	unsigned short fs_dmask;
-	unsigned short codepage;  /* Codepage for shortname conversions */
-	char *iocharset;          /* Charset used for filename input/display */
-	unsigned short shortname; /* flags for shortname display/create rule */
-	unsigned char name_check; /* r = relaxed, n = normal, s = strict */
-	unsigned char errors;	  /* On error: continue, panic, remount-ro */
+	unsigned short codepage;   /* Codepage for shortname conversions */
+	char *iocharset;           /* Charset used for filename input/display */
+	unsigned short shortname;  /* flags for shortname display/create rule */
+	unsigned char name_check;  /* r = relaxed, n = normal, s = strict */
+	unsigned char errors;	   /* On error: continue, panic, remount-ro */
 	unsigned short allow_utime;/* permission for setting the [am]time */
-	unsigned quiet:1,         /* set = fake successful chmods and chowns */
-		 showexec:1,      /* set = only set x bit for com/exe/bat */
-		 sys_immutable:1, /* set = system files are immutable */
-		 dotsOK:1,        /* set = hidden and system files are named '.filename' */
-		 isvfat:1,        /* 0=no vfat long filename support, 1=vfat support */
-		 utf8:1,	  /* Use of UTF-8 character set (Default) */
-		 unicode_xlate:1, /* create escape sequences for unhandled Unicode */
-		 numtail:1,       /* Does first alias have a numeric '~1' type tail? */
-		 flush:1,	  /* write things quickly */
-		 nocase:1,	  /* Does this need case conversion? 0=need case conversion*/
-		 usefree:1,	  /* Use free_clusters for FAT32 */
-		 tz_utc:1,	  /* Filesystem timestamps are in UTC */
-		 rodir:1,	  /* allow ATTR_RO for directory */
-		 discard:1;	  /* Issue discard requests on deletions */
+	unsigned quiet:1,          /* set = fake successful chmods and chowns */
+		 showexec:1,       /* set = only set x bit for com/exe/bat */
+		 sys_immutable:1,  /* set = system files are immutable */
+		 dotsOK:1,         /* set = hidden and system files are named '.filename' */
+		 isvfat:1,         /* 0=no vfat long filename support, 1=vfat support */
+		 utf8:1,	   /* Use of UTF-8 character set (Default) */
+		 unicode_xlate:1,  /* create escape sequences for unhandled Unicode */
+		 numtail:1,        /* Does first alias have a numeric '~1' type tail? */
+		 flush:1,	   /* write things quickly */
+		 nocase:1,	   /* Does this need case conversion? 0=need case conversion*/
+		 usefree:1,	   /* Use free_clusters for FAT32 */
+		 tz_utc:1,	   /* Filesystem timestamps are in UTC */
+		 rodir:1,	   /* allow ATTR_RO for directory */
+		 discard:1,	   /* Issue discard requests on deletions */
+		 nfs:1;		   /* Do extra work needed for NFS export */
 };
 
 #define FAT_HASH_BITS	8
@@ -56,28 +58,28 @@ struct fat_mount_options {
  * MS-DOS file system in-core superblock data
  */
 struct msdos_sb_info {
-	unsigned short sec_per_clus; /* sectors/cluster */
-	unsigned short cluster_bits; /* log2(cluster_size) */
-	unsigned int cluster_size;   /* cluster size */
-	unsigned char fats,fat_bits; /* number of FATs, FAT bits (12 or 16) */
+	unsigned short sec_per_clus;  /* sectors/cluster */
+	unsigned short cluster_bits;  /* log2(cluster_size) */
+	unsigned int cluster_size;    /* cluster size */
+	unsigned char fats, fat_bits; /* number of FATs, FAT bits (12 or 16) */
 	unsigned short fat_start;
-	unsigned long fat_length;    /* FAT start & length (sec.) */
+	unsigned long fat_length;     /* FAT start & length (sec.) */
 	unsigned long dir_start;
-	unsigned short dir_entries;  /* root dir start & entries */
-	unsigned long data_start;    /* first data sector */
-	unsigned long max_cluster;   /* maximum cluster number */
-	unsigned long root_cluster;  /* first cluster of the root directory */
-	unsigned long fsinfo_sector; /* sector number of FAT32 fsinfo */
+	unsigned short dir_entries;   /* root dir start & entries */
+	unsigned long data_start;     /* first data sector */
+	unsigned long max_cluster;    /* maximum cluster number */
+	unsigned long root_cluster;   /* first cluster of the root directory */
+	unsigned long fsinfo_sector;  /* sector number of FAT32 fsinfo */
 	struct mutex fat_lock;
-	unsigned int prev_free;      /* previously allocated cluster number */
-	unsigned int free_clusters;  /* -1 if undefined */
+	unsigned int prev_free;       /* previously allocated cluster number */
+	unsigned int free_clusters;   /* -1 if undefined */
 	unsigned int free_clus_valid; /* is free_clusters valid? */
 	struct fat_mount_options options;
-	struct nls_table *nls_disk;  /* Codepage used on disk */
-	struct nls_table *nls_io;    /* Charset used for input and display */
-	const void *dir_ops;		     /* Opaque; default directory operations */
-	int dir_per_block;	     /* dir entries per block */
-	int dir_per_block_bits;	     /* log2(dir_per_block) */
+	struct nls_table *nls_disk;   /* Codepage used on disk */
+	struct nls_table *nls_io;     /* Charset used for input and display */
+	const void *dir_ops;	      /* Opaque; default directory operations */
+	int dir_per_block;	      /* dir entries per block */
+	int dir_per_block_bits;	      /* log2(dir_per_block) */
 
 	int fatent_shift;
 	struct fatent_operations *fatent_ops;
@@ -88,6 +90,9 @@ struct msdos_sb_info {
 
 	spinlock_t inode_hash_lock;
 	struct hlist_head inode_hashtable[FAT_HASH_SIZE];
+
+	spinlock_t dir_hash_lock;
+	struct hlist_head dir_hashtable[FAT_HASH_SIZE];
 };
 
 #define FAT_CACHE_VALID	0	/* special case for valid cache */
@@ -110,6 +115,7 @@ struct msdos_inode_info {
 	int i_attrs;		/* unused attribute bits */
 	loff_t i_pos;		/* on-disk position of directory entry or 0 */
 	struct hlist_node i_fat_hash;	/* hash by i_location */
+	struct hlist_node i_dir_hash;	/* hash by i_logstart */
 	struct rw_semaphore truncate_lock; /* protect bmap against truncate */
 	struct inode vfs_inode;
 };
@@ -262,7 +268,7 @@ extern int fat_subdirs(struct inode *dir);
 extern int fat_scan(struct inode *dir, const unsigned char *name,
 		    struct fat_slot_info *sinfo);
 extern int fat_get_dotdot_entry(struct inode *dir, struct buffer_head **bh,
-				struct msdos_dir_entry **de, loff_t *i_pos);
+				struct msdos_dir_entry **de);
 extern int fat_alloc_new_dir(struct inode *dir, struct timespec *ts);
 extern int fat_add_entries(struct inode *dir, void *slots, int nr_slots,
 			   struct fat_slot_info *sinfo);
@@ -322,7 +328,7 @@ extern long fat_generic_ioctl(struct file *filp, unsigned int cmd,
 			      unsigned long arg);
 extern const struct file_operations fat_file_operations;
 extern const struct inode_operations fat_file_inode_operations;
-extern int fat_setattr(struct dentry * dentry, struct iattr * attr);
+extern int fat_setattr(struct dentry *dentry, struct iattr *attr);
 extern void fat_truncate_blocks(struct inode *inode, loff_t offset);
 extern int fat_getattr(struct vfsmount *mnt, struct dentry *dentry,
 		       struct kstat *stat);
@@ -340,7 +346,12 @@ extern int fat_fill_super(struct super_block *sb, void *data, int silent,
 			  int isvfat, void (*setup)(struct super_block *));
 
 extern int fat_flush_inodes(struct super_block *sb, struct inode *i1,
-		            struct inode *i2);
+			    struct inode *i2);
+static inline unsigned long fat_dir_hash(int logstart)
+{
+	return hash_32(logstart, FAT_HASH_BITS);
+}
+
 /* fat/misc.c */
 extern __printf(3, 4) __cold
 void __fat_fs_error(struct super_block *sb, int report, const char *fmt, ...);
@@ -366,6 +377,14 @@ extern int fat_sync_bhs(struct buffer_head **bhs, int nr_bhs);
 int fat_cache_init(void);
 void fat_cache_destroy(void);
 
+/* fat/nfs.c */
+struct fid;
+extern struct dentry *fat_fh_to_dentry(struct super_block *sb, struct fid *fid,
+				       int fh_len, int fh_type);
+extern struct dentry *fat_fh_to_parent(struct super_block *sb, struct fid *fid,
+				       int fh_len, int fh_type);
+extern struct dentry *fat_get_parent(struct dentry *child_dir);
+
 /* helper for printk */
 typedef unsigned long long	llu;
 
diff --git a/fs/fat/fatent.c b/fs/fat/fatent.c
index 31f08ab62c5..260705c5806 100644
--- a/fs/fat/fatent.c
+++ b/fs/fat/fatent.c
@@ -186,9 +186,6 @@ static void fat16_ent_put(struct fat_entry *fatent, int new)
 
 static void fat32_ent_put(struct fat_entry *fatent, int new)
 {
-	if (new == FAT_ENT_EOF)
-		new = EOF_FAT32;
-
 	WARN_ON(new & 0xf0000000);
 	new |= le32_to_cpu(*fatent->u.ent32_p) & ~0x0fffffff;
 	*fatent->u.ent32_p = cpu_to_le32(new);
@@ -203,15 +200,18 @@ static int fat12_ent_next(struct fat_entry *fatent)
 
 	fatent->entry++;
 	if (fatent->nr_bhs == 1) {
-		WARN_ON(ent12_p[0] > (u8 *)(bhs[0]->b_data + (bhs[0]->b_size - 2)));
-		WARN_ON(ent12_p[1] > (u8 *)(bhs[0]->b_data + (bhs[0]->b_size - 1)));
+		WARN_ON(ent12_p[0] > (u8 *)(bhs[0]->b_data +
+							(bhs[0]->b_size - 2)));
+		WARN_ON(ent12_p[1] > (u8 *)(bhs[0]->b_data +
+							(bhs[0]->b_size - 1)));
 		if (nextp < (u8 *)(bhs[0]->b_data + (bhs[0]->b_size - 1))) {
 			ent12_p[0] = nextp - 1;
 			ent12_p[1] = nextp;
 			return 1;
 		}
 	} else {
-		WARN_ON(ent12_p[0] != (u8 *)(bhs[0]->b_data + (bhs[0]->b_size - 1)));
+		WARN_ON(ent12_p[0] != (u8 *)(bhs[0]->b_data +
+							(bhs[0]->b_size - 1)));
 		WARN_ON(ent12_p[1] != (u8 *)bhs[1]->b_data);
 		ent12_p[0] = nextp - 1;
 		ent12_p[1] = nextp;
@@ -631,7 +631,6 @@ error:
 
 	return err;
 }
-
 EXPORT_SYMBOL_GPL(fat_free_clusters);
 
 /* 128kb is the whole sectors for FAT12 and FAT16 */
diff --git a/fs/fat/inode.c b/fs/fat/inode.c
index 47d9eb0be88..76f60c642c0 100644
--- a/fs/fat/inode.c
+++ b/fs/fat/inode.c
@@ -281,15 +281,42 @@ static inline unsigned long fat_hash(loff_t i_pos)
 	return hash_32(i_pos, FAT_HASH_BITS);
 }
 
+static void dir_hash_init(struct super_block *sb)
+{
+	struct msdos_sb_info *sbi = MSDOS_SB(sb);
+	int i;
+
+	spin_lock_init(&sbi->dir_hash_lock);
+	for (i = 0; i < FAT_HASH_SIZE; i++)
+		INIT_HLIST_HEAD(&sbi->dir_hashtable[i]);
+}
+
 void fat_attach(struct inode *inode, loff_t i_pos)
 {
 	struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb);
-	struct hlist_head *head = sbi->inode_hashtable + fat_hash(i_pos);
 
-	spin_lock(&sbi->inode_hash_lock);
-	MSDOS_I(inode)->i_pos = i_pos;
-	hlist_add_head(&MSDOS_I(inode)->i_fat_hash, head);
-	spin_unlock(&sbi->inode_hash_lock);
+	if (inode->i_ino != MSDOS_ROOT_INO) {
+		struct hlist_head *head =   sbi->inode_hashtable
+					  + fat_hash(i_pos);
+
+		spin_lock(&sbi->inode_hash_lock);
+		MSDOS_I(inode)->i_pos = i_pos;
+		hlist_add_head(&MSDOS_I(inode)->i_fat_hash, head);
+		spin_unlock(&sbi->inode_hash_lock);
+	}
+
+	/* If NFS support is enabled, cache the mapping of start cluster
+	 * to directory inode. This is used during reconnection of
+	 * dentries to the filesystem root.
+	 */
+	if (S_ISDIR(inode->i_mode) && sbi->options.nfs) {
+		struct hlist_head *d_head = sbi->dir_hashtable;
+		d_head += fat_dir_hash(MSDOS_I(inode)->i_logstart);
+
+		spin_lock(&sbi->dir_hash_lock);
+		hlist_add_head(&MSDOS_I(inode)->i_dir_hash, d_head);
+		spin_unlock(&sbi->dir_hash_lock);
+	}
 }
 EXPORT_SYMBOL_GPL(fat_attach);
 
@@ -300,6 +327,12 @@ void fat_detach(struct inode *inode)
 	MSDOS_I(inode)->i_pos = 0;
 	hlist_del_init(&MSDOS_I(inode)->i_fat_hash);
 	spin_unlock(&sbi->inode_hash_lock);
+
+	if (S_ISDIR(inode->i_mode) && sbi->options.nfs) {
+		spin_lock(&sbi->dir_hash_lock);
+		hlist_del_init(&MSDOS_I(inode)->i_dir_hash);
+		spin_unlock(&sbi->dir_hash_lock);
+	}
 }
 EXPORT_SYMBOL_GPL(fat_detach);
 
@@ -504,6 +537,7 @@ static void init_once(void *foo)
 	ei->cache_valid_id = FAT_CACHE_VALID + 1;
 	INIT_LIST_HEAD(&ei->cache_lru);
 	INIT_HLIST_NODE(&ei->i_fat_hash);
+	INIT_HLIST_NODE(&ei->i_dir_hash);
 	inode_init_once(&ei->vfs_inode);
 }
 
@@ -521,6 +555,11 @@ static int __init fat_init_inodecache(void)
 
 static void __exit fat_destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(fat_inode_cachep);
 }
 
@@ -663,125 +702,9 @@ static const struct super_operations fat_sops = {
 	.show_options	= fat_show_options,
 };
 
-/*
- * a FAT file handle with fhtype 3 is
- *  0/  i_ino - for fast, reliable lookup if still in the cache
- *  1/  i_generation - to see if i_ino is still valid
- *          bit 0 == 0 iff directory
- *  2/  i_pos(8-39) - if ino has changed, but still in cache
- *  3/  i_pos(4-7)|i_logstart - to semi-verify inode found at i_pos
- *  4/  i_pos(0-3)|parent->i_logstart - maybe used to hunt for the file on disc
- *
- * Hack for NFSv2: Maximum FAT entry number is 28bits and maximum
- * i_pos is 40bits (blocknr(32) + dir offset(8)), so two 4bits
- * of i_logstart is used to store the directory entry offset.
- */
-
-static struct dentry *fat_fh_to_dentry(struct super_block *sb,
-		struct fid *fid, int fh_len, int fh_type)
-{
-	struct inode *inode = NULL;
-	u32 *fh = fid->raw;
-
-	if (fh_len < 5 || fh_type != 3)
-		return NULL;
-
-	inode = ilookup(sb, fh[0]);
-	if (!inode || inode->i_generation != fh[1]) {
-		if (inode)
-			iput(inode);
-		inode = NULL;
-	}
-	if (!inode) {
-		loff_t i_pos;
-		int i_logstart = fh[3] & 0x0fffffff;
-
-		i_pos = (loff_t)fh[2] << 8;
-		i_pos |= ((fh[3] >> 24) & 0xf0) | (fh[4] >> 28);
-
-		/* try 2 - see if i_pos is in F-d-c
-		 * require i_logstart to be the same
-		 * Will fail if you truncate and then re-write
-		 */
-
-		inode = fat_iget(sb, i_pos);
-		if (inode && MSDOS_I(inode)->i_logstart != i_logstart) {
-			iput(inode);
-			inode = NULL;
-		}
-	}
-
-	/*
-	 * For now, do nothing if the inode is not found.
-	 *
-	 * What we could do is:
-	 *
-	 *	- follow the file starting at fh[4], and record the ".." entry,
-	 *	  and the name of the fh[2] entry.
-	 *	- then follow the ".." file finding the next step up.
-	 *
-	 * This way we build a path to the root of the tree. If this works, we
-	 * lookup the path and so get this inode into the cache.  Finally try
-	 * the fat_iget lookup again.  If that fails, then we are totally out
-	 * of luck.  But all that is for another day
-	 */
-	return d_obtain_alias(inode);
-}
-
-static int
-fat_encode_fh(struct inode *inode, __u32 *fh, int *lenp, struct inode *parent)
-{
-	int len = *lenp;
-	struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb);
-	loff_t i_pos;
-
-	if (len < 5) {
-		*lenp = 5;
-		return 255; /* no room */
-	}
-
-	i_pos = fat_i_pos_read(sbi, inode);
-	*lenp = 5;
-	fh[0] = inode->i_ino;
-	fh[1] = inode->i_generation;
-	fh[2] = i_pos >> 8;
-	fh[3] = ((i_pos & 0xf0) << 24) | MSDOS_I(inode)->i_logstart;
-	fh[4] = (i_pos & 0x0f) << 28;
-	if (parent)
-		fh[4] |= MSDOS_I(parent)->i_logstart;
-	return 3;
-}
-
-static struct dentry *fat_get_parent(struct dentry *child)
-{
-	struct super_block *sb = child->d_sb;
-	struct buffer_head *bh;
-	struct msdos_dir_entry *de;
-	loff_t i_pos;
-	struct dentry *parent;
-	struct inode *inode;
-	int err;
-
-	lock_super(sb);
-
-	err = fat_get_dotdot_entry(child->d_inode, &bh, &de, &i_pos);
-	if (err) {
-		parent = ERR_PTR(err);
-		goto out;
-	}
-	inode = fat_build_inode(sb, de, i_pos);
-	brelse(bh);
-
-	parent = d_obtain_alias(inode);
-out:
-	unlock_super(sb);
-
-	return parent;
-}
-
 static const struct export_operations fat_export_ops = {
-	.encode_fh	= fat_encode_fh,
 	.fh_to_dentry	= fat_fh_to_dentry,
+	.fh_to_parent	= fat_fh_to_parent,
 	.get_parent	= fat_get_parent,
 };
 
@@ -831,6 +754,8 @@ static int fat_show_options(struct seq_file *m, struct dentry *root)
 		seq_puts(m, ",usefree");
 	if (opts->quiet)
 		seq_puts(m, ",quiet");
+	if (opts->nfs)
+		seq_puts(m, ",nfs");
 	if (opts->showexec)
 		seq_puts(m, ",showexec");
 	if (opts->sys_immutable)
@@ -875,7 +800,7 @@ enum {
 	Opt_shortname_winnt, Opt_shortname_mixed, Opt_utf8_no, Opt_utf8_yes,
 	Opt_uni_xl_no, Opt_uni_xl_yes, Opt_nonumtail_no, Opt_nonumtail_yes,
 	Opt_obsolete, Opt_flush, Opt_tz_utc, Opt_rodir, Opt_err_cont,
-	Opt_err_panic, Opt_err_ro, Opt_discard, Opt_err,
+	Opt_err_panic, Opt_err_ro, Opt_discard, Opt_nfs, Opt_err,
 };
 
 static const match_table_t fat_tokens = {
@@ -904,6 +829,7 @@ static const match_table_t fat_tokens = {
 	{Opt_err_panic, "errors=panic"},
 	{Opt_err_ro, "errors=remount-ro"},
 	{Opt_discard, "discard"},
+	{Opt_nfs, "nfs"},
 	{Opt_obsolete, "conv=binary"},
 	{Opt_obsolete, "conv=text"},
 	{Opt_obsolete, "conv=auto"},
@@ -984,6 +910,7 @@ static int parse_options(struct super_block *sb, char *options, int is_vfat,
 	opts->numtail = 1;
 	opts->usefree = opts->nocase = 0;
 	opts->tz_utc = 0;
+	opts->nfs = 0;
 	opts->errors = FAT_ERRORS_RO;
 	*debug = 0;
 
@@ -1148,6 +1075,9 @@ static int parse_options(struct super_block *sb, char *options, int is_vfat,
 		case Opt_discard:
 			opts->discard = 1;
 			break;
+		case Opt_nfs:
+			opts->nfs = 1;
+			break;
 
 		/* obsolete mount options */
 		case Opt_obsolete:
@@ -1438,6 +1368,7 @@ int fat_fill_super(struct super_block *sb, void *data, int silent, int isvfat,
 
 	/* set up enough so that it can read an inode */
 	fat_hash_init(sb);
+	dir_hash_init(sb);
 	fat_ent_access_init(sb);
 
 	/*
@@ -1492,6 +1423,7 @@ int fat_fill_super(struct super_block *sb, void *data, int silent, int isvfat,
 	}
 	error = -ENOMEM;
 	insert_inode_hash(root_inode);
+	fat_attach(root_inode, 0);
 	sb->s_root = d_make_root(root_inode);
 	if (!sb->s_root) {
 		fat_msg(sb, KERN_ERR, "get root inode failed");
@@ -1531,18 +1463,14 @@ static int writeback_inode(struct inode *inode)
 {
 
 	int ret;
-	struct address_space *mapping = inode->i_mapping;
-	struct writeback_control wbc = {
-	       .sync_mode = WB_SYNC_NONE,
-	      .nr_to_write = 0,
-	};
-	/* if we used WB_SYNC_ALL, sync_inode waits for the io for the
-	* inode to finish.  So WB_SYNC_NONE is sent down to sync_inode
+
+	/* if we used wait=1, sync_inode_metadata waits for the io for the
+	* inode to finish.  So wait=0 is sent down to sync_inode_metadata
 	* and filemap_fdatawrite is used for the data blocks
 	*/
-	ret = sync_inode(inode, &wbc);
+	ret = sync_inode_metadata(inode, 0);
 	if (!ret)
-	       ret = filemap_fdatawrite(mapping);
+		ret = filemap_fdatawrite(inode->i_mapping);
 	return ret;
 }
 
diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c
index b0e12bf9f4a..c1055e778ff 100644
--- a/fs/fat/namei_msdos.c
+++ b/fs/fat/namei_msdos.c
@@ -407,7 +407,7 @@ out:
 static int msdos_unlink(struct inode *dir, struct dentry *dentry)
 {
 	struct inode *inode = dentry->d_inode;
-	struct super_block *sb= inode->i_sb;
+	struct super_block *sb = inode->i_sb;
 	struct fat_slot_info sinfo;
 	int err;
 
@@ -440,7 +440,7 @@ static int do_msdos_rename(struct inode *old_dir, unsigned char *old_name,
 	struct inode *old_inode, *new_inode;
 	struct fat_slot_info old_sinfo, sinfo;
 	struct timespec ts;
-	loff_t dotdot_i_pos, new_i_pos;
+	loff_t new_i_pos;
 	int err, old_attrs, is_dir, update_dotdot, corrupt = 0;
 
 	old_sinfo.bh = sinfo.bh = dotdot_bh = NULL;
@@ -456,8 +456,7 @@ static int do_msdos_rename(struct inode *old_dir, unsigned char *old_name,
 	is_dir = S_ISDIR(old_inode->i_mode);
 	update_dotdot = (is_dir && old_dir != new_dir);
 	if (update_dotdot) {
-		if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de,
-					 &dotdot_i_pos) < 0) {
+		if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de)) {
 			err = -EIO;
 			goto out;
 		}
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index 6a6d8c0715a..e535dd75b98 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -914,7 +914,7 @@ static int vfat_rename(struct inode *old_dir, struct dentry *old_dentry,
 	struct inode *old_inode, *new_inode;
 	struct fat_slot_info old_sinfo, sinfo;
 	struct timespec ts;
-	loff_t dotdot_i_pos, new_i_pos;
+	loff_t new_i_pos;
 	int err, is_dir, update_dotdot, corrupt = 0;
 	struct super_block *sb = old_dir->i_sb;
 
@@ -929,8 +929,7 @@ static int vfat_rename(struct inode *old_dir, struct dentry *old_dentry,
 	is_dir = S_ISDIR(old_inode->i_mode);
 	update_dotdot = (is_dir && old_dir != new_dir);
 	if (update_dotdot) {
-		if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de,
-					 &dotdot_i_pos) < 0) {
+		if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de)) {
 			err = -EIO;
 			goto out;
 		}
diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c
new file mode 100644
index 00000000000..ef4b5faba87
--- /dev/null
+++ b/fs/fat/nfs.c
@@ -0,0 +1,101 @@
+/* fs/fat/nfs.c
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/exportfs.h>
+#include "fat.h"
+
+/**
+ * Look up a directory inode given its starting cluster.
+ */
+static struct inode *fat_dget(struct super_block *sb, int i_logstart)
+{
+	struct msdos_sb_info *sbi = MSDOS_SB(sb);
+	struct hlist_head *head;
+	struct hlist_node *_p;
+	struct msdos_inode_info *i;
+	struct inode *inode = NULL;
+
+	head = sbi->dir_hashtable + fat_dir_hash(i_logstart);
+	spin_lock(&sbi->dir_hash_lock);
+	hlist_for_each_entry(i, _p, head, i_dir_hash) {
+		BUG_ON(i->vfs_inode.i_sb != sb);
+		if (i->i_logstart != i_logstart)
+			continue;
+		inode = igrab(&i->vfs_inode);
+		if (inode)
+			break;
+	}
+	spin_unlock(&sbi->dir_hash_lock);
+	return inode;
+}
+
+static struct inode *fat_nfs_get_inode(struct super_block *sb,
+				       u64 ino, u32 generation)
+{
+	struct inode *inode;
+
+	if ((ino < MSDOS_ROOT_INO) || (ino == MSDOS_FSINFO_INO))
+		return NULL;
+
+	inode = ilookup(sb, ino);
+	if (inode && generation && (inode->i_generation != generation)) {
+		iput(inode);
+		inode = NULL;
+	}
+
+	return inode;
+}
+
+/**
+ * Map a NFS file handle to a corresponding dentry.
+ * The dentry may or may not be connected to the filesystem root.
+ */
+struct dentry *fat_fh_to_dentry(struct super_block *sb, struct fid *fid,
+				int fh_len, int fh_type)
+{
+	return generic_fh_to_dentry(sb, fid, fh_len, fh_type,
+				    fat_nfs_get_inode);
+}
+
+/*
+ * Find the parent for a file specified by NFS handle.
+ * This requires that the handle contain the i_ino of the parent.
+ */
+struct dentry *fat_fh_to_parent(struct super_block *sb, struct fid *fid,
+				int fh_len, int fh_type)
+{
+	return generic_fh_to_parent(sb, fid, fh_len, fh_type,
+				    fat_nfs_get_inode);
+}
+
+/*
+ * Find the parent for a directory that is not currently connected to
+ * the filesystem root.
+ *
+ * On entry, the caller holds child_dir->d_inode->i_mutex.
+ */
+struct dentry *fat_get_parent(struct dentry *child_dir)
+{
+	struct super_block *sb = child_dir->d_sb;
+	struct buffer_head *bh = NULL;
+	struct msdos_dir_entry *de;
+	struct inode *parent_inode = NULL;
+
+	if (!fat_get_dotdot_entry(child_dir->d_inode, &bh, &de)) {
+		int parent_logstart = fat_get_start(MSDOS_SB(sb), de);
+		parent_inode = fat_dget(sb, parent_logstart);
+	}
+	brelse(bh);
+
+	return d_obtain_alias(parent_inode);
+}
diff --git a/fs/fcntl.c b/fs/fcntl.c
index 887b5ba8c9b..8f704291d4e 100644
--- a/fs/fcntl.c
+++ b/fs/fcntl.c
@@ -26,124 +26,6 @@
 #include <asm/siginfo.h>
 #include <asm/uaccess.h>
 
-void set_close_on_exec(unsigned int fd, int flag)
-{
-	struct files_struct *files = current->files;
-	struct fdtable *fdt;
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	if (flag)
-		__set_close_on_exec(fd, fdt);
-	else
-		__clear_close_on_exec(fd, fdt);
-	spin_unlock(&files->file_lock);
-}
-
-static bool get_close_on_exec(unsigned int fd)
-{
-	struct files_struct *files = current->files;
-	struct fdtable *fdt;
-	bool res;
-	rcu_read_lock();
-	fdt = files_fdtable(files);
-	res = close_on_exec(fd, fdt);
-	rcu_read_unlock();
-	return res;
-}
-
-SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
-{
-	int err = -EBADF;
-	struct file * file, *tofree;
-	struct files_struct * files = current->files;
-	struct fdtable *fdt;
-
-	if ((flags & ~O_CLOEXEC) != 0)
-		return -EINVAL;
-
-	if (unlikely(oldfd == newfd))
-		return -EINVAL;
-
-	spin_lock(&files->file_lock);
-	err = expand_files(files, newfd);
-	file = fcheck(oldfd);
-	if (unlikely(!file))
-		goto Ebadf;
-	if (unlikely(err < 0)) {
-		if (err == -EMFILE)
-			goto Ebadf;
-		goto out_unlock;
-	}
-	/*
-	 * We need to detect attempts to do dup2() over allocated but still
-	 * not finished descriptor.  NB: OpenBSD avoids that at the price of
-	 * extra work in their equivalent of fget() - they insert struct
-	 * file immediately after grabbing descriptor, mark it larval if
-	 * more work (e.g. actual opening) is needed and make sure that
-	 * fget() treats larval files as absent.  Potentially interesting,
-	 * but while extra work in fget() is trivial, locking implications
-	 * and amount of surgery on open()-related paths in VFS are not.
-	 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
-	 * deadlocks in rather amusing ways, AFAICS.  All of that is out of
-	 * scope of POSIX or SUS, since neither considers shared descriptor
-	 * tables and this condition does not arise without those.
-	 */
-	err = -EBUSY;
-	fdt = files_fdtable(files);
-	tofree = fdt->fd[newfd];
-	if (!tofree && fd_is_open(newfd, fdt))
-		goto out_unlock;
-	get_file(file);
-	rcu_assign_pointer(fdt->fd[newfd], file);
-	__set_open_fd(newfd, fdt);
-	if (flags & O_CLOEXEC)
-		__set_close_on_exec(newfd, fdt);
-	else
-		__clear_close_on_exec(newfd, fdt);
-	spin_unlock(&files->file_lock);
-
-	if (tofree)
-		filp_close(tofree, files);
-
-	return newfd;
-
-Ebadf:
-	err = -EBADF;
-out_unlock:
-	spin_unlock(&files->file_lock);
-	return err;
-}
-
-SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
-{
-	if (unlikely(newfd == oldfd)) { /* corner case */
-		struct files_struct *files = current->files;
-		int retval = oldfd;
-
-		rcu_read_lock();
-		if (!fcheck_files(files, oldfd))
-			retval = -EBADF;
-		rcu_read_unlock();
-		return retval;
-	}
-	return sys_dup3(oldfd, newfd, 0);
-}
-
-SYSCALL_DEFINE1(dup, unsigned int, fildes)
-{
-	int ret = -EBADF;
-	struct file *file = fget_raw(fildes);
-
-	if (file) {
-		ret = get_unused_fd();
-		if (ret >= 0)
-			fd_install(ret, file);
-		else
-			fput(file);
-	}
-	return ret;
-}
-
 #define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME)
 
 static int setfl(int fd, struct file * filp, unsigned long arg)
@@ -267,7 +149,7 @@ pid_t f_getown(struct file *filp)
 
 static int f_setown_ex(struct file *filp, unsigned long arg)
 {
-	struct f_owner_ex * __user owner_p = (void * __user)arg;
+	struct f_owner_ex __user *owner_p = (void __user *)arg;
 	struct f_owner_ex owner;
 	struct pid *pid;
 	int type;
@@ -307,7 +189,7 @@ static int f_setown_ex(struct file *filp, unsigned long arg)
 
 static int f_getown_ex(struct file *filp, unsigned long arg)
 {
-	struct f_owner_ex * __user owner_p = (void * __user)arg;
+	struct f_owner_ex __user *owner_p = (void __user *)arg;
 	struct f_owner_ex owner;
 	int ret = 0;
 
@@ -345,7 +227,7 @@ static int f_getown_ex(struct file *filp, unsigned long arg)
 static int f_getowner_uids(struct file *filp, unsigned long arg)
 {
 	struct user_namespace *user_ns = current_user_ns();
-	uid_t * __user dst = (void * __user)arg;
+	uid_t __user *dst = (void __user *)arg;
 	uid_t src[2];
 	int err;
 
@@ -373,14 +255,10 @@ static long do_fcntl(int fd, unsigned int cmd, unsigned long arg,
 
 	switch (cmd) {
 	case F_DUPFD:
+		err = f_dupfd(arg, filp, 0);
+		break;
 	case F_DUPFD_CLOEXEC:
-		if (arg >= rlimit(RLIMIT_NOFILE))
-			break;
-		err = alloc_fd(arg, cmd == F_DUPFD_CLOEXEC ? O_CLOEXEC : 0);
-		if (err >= 0) {
-			get_file(filp);
-			fd_install(err, filp);
-		}
+		err = f_dupfd(arg, filp, FD_CLOEXEC);
 		break;
 	case F_GETFD:
 		err = get_close_on_exec(fd) ? FD_CLOEXEC : 0;
@@ -470,25 +348,23 @@ static int check_fcntl_cmd(unsigned cmd)
 
 SYSCALL_DEFINE3(fcntl, unsigned int, fd, unsigned int, cmd, unsigned long, arg)
 {	
-	struct file *filp;
-	int fput_needed;
+	struct fd f = fdget_raw(fd);
 	long err = -EBADF;
 
-	filp = fget_raw_light(fd, &fput_needed);
-	if (!filp)
+	if (!f.file)
 		goto out;
 
-	if (unlikely(filp->f_mode & FMODE_PATH)) {
+	if (unlikely(f.file->f_mode & FMODE_PATH)) {
 		if (!check_fcntl_cmd(cmd))
 			goto out1;
 	}
 
-	err = security_file_fcntl(filp, cmd, arg);
+	err = security_file_fcntl(f.file, cmd, arg);
 	if (!err)
-		err = do_fcntl(fd, cmd, arg, filp);
+		err = do_fcntl(fd, cmd, arg, f.file);
 
 out1:
- 	fput_light(filp, fput_needed);
+ 	fdput(f);
 out:
 	return err;
 }
@@ -497,38 +373,36 @@ out:
 SYSCALL_DEFINE3(fcntl64, unsigned int, fd, unsigned int, cmd,
 		unsigned long, arg)
 {	
-	struct file * filp;
+	struct fd f = fdget_raw(fd);
 	long err = -EBADF;
-	int fput_needed;
 
-	filp = fget_raw_light(fd, &fput_needed);
-	if (!filp)
+	if (!f.file)
 		goto out;
 
-	if (unlikely(filp->f_mode & FMODE_PATH)) {
+	if (unlikely(f.file->f_mode & FMODE_PATH)) {
 		if (!check_fcntl_cmd(cmd))
 			goto out1;
 	}
 
-	err = security_file_fcntl(filp, cmd, arg);
+	err = security_file_fcntl(f.file, cmd, arg);
 	if (err)
 		goto out1;
 	
 	switch (cmd) {
 		case F_GETLK64:
-			err = fcntl_getlk64(filp, (struct flock64 __user *) arg);
+			err = fcntl_getlk64(f.file, (struct flock64 __user *) arg);
 			break;
 		case F_SETLK64:
 		case F_SETLKW64:
-			err = fcntl_setlk64(fd, filp, cmd,
+			err = fcntl_setlk64(fd, f.file, cmd,
 					(struct flock64 __user *) arg);
 			break;
 		default:
-			err = do_fcntl(fd, cmd, arg, filp);
+			err = do_fcntl(fd, cmd, arg, f.file);
 			break;
 	}
 out1:
-	fput_light(filp, fput_needed);
+	fdput(f);
 out:
 	return err;
 }
diff --git a/fs/fhandle.c b/fs/fhandle.c
index a48e4a139be..f775bfdd6e4 100644
--- a/fs/fhandle.c
+++ b/fs/fhandle.c
@@ -113,24 +113,21 @@ SYSCALL_DEFINE5(name_to_handle_at, int, dfd, const char __user *, name,
 
 static struct vfsmount *get_vfsmount_from_fd(int fd)
 {
-	struct path path;
+	struct vfsmount *mnt;
 
 	if (fd == AT_FDCWD) {
 		struct fs_struct *fs = current->fs;
 		spin_lock(&fs->lock);
-		path = fs->pwd;
-		mntget(path.mnt);
+		mnt = mntget(fs->pwd.mnt);
 		spin_unlock(&fs->lock);
 	} else {
-		int fput_needed;
-		struct file *file = fget_light(fd, &fput_needed);
-		if (!file)
+		struct fd f = fdget(fd);
+		if (!f.file)
 			return ERR_PTR(-EBADF);
-		path = file->f_path;
-		mntget(path.mnt);
-		fput_light(file, fput_needed);
+		mnt = mntget(f.file->f_path.mnt);
+		fdput(f);
 	}
-	return path.mnt;
+	return mnt;
 }
 
 static int vfs_dentry_acceptable(void *context, struct dentry *dentry)
diff --git a/fs/file.c b/fs/file.c
index ba3f6053025..0f1bda4bebf 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -6,6 +6,7 @@
  *  Manage the dynamic fd arrays in the process files_struct.
  */
 
+#include <linux/syscalls.h>
 #include <linux/export.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
@@ -84,22 +85,14 @@ static void free_fdtable_work(struct work_struct *work)
 	}
 }
 
-void free_fdtable_rcu(struct rcu_head *rcu)
+static void free_fdtable_rcu(struct rcu_head *rcu)
 {
 	struct fdtable *fdt = container_of(rcu, struct fdtable, rcu);
 	struct fdtable_defer *fddef;
 
 	BUG_ON(!fdt);
+	BUG_ON(fdt->max_fds <= NR_OPEN_DEFAULT);
 
-	if (fdt->max_fds <= NR_OPEN_DEFAULT) {
-		/*
-		 * This fdtable is embedded in the files structure and that
-		 * structure itself is getting destroyed.
-		 */
-		kmem_cache_free(files_cachep,
-				container_of(fdt, struct files_struct, fdtab));
-		return;
-	}
 	if (!is_vmalloc_addr(fdt->fd) && !is_vmalloc_addr(fdt->open_fds)) {
 		kfree(fdt->fd);
 		kfree(fdt->open_fds);
@@ -229,7 +222,7 @@ static int expand_fdtable(struct files_struct *files, int nr)
 		copy_fdtable(new_fdt, cur_fdt);
 		rcu_assign_pointer(files->fdt, new_fdt);
 		if (cur_fdt->max_fds > NR_OPEN_DEFAULT)
-			free_fdtable(cur_fdt);
+			call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
 	} else {
 		/* Somebody else expanded, so undo our attempt */
 		__free_fdtable(new_fdt);
@@ -245,19 +238,12 @@ static int expand_fdtable(struct files_struct *files, int nr)
  * expanded and execution may have blocked.
  * The files->file_lock should be held on entry, and will be held on exit.
  */
-int expand_files(struct files_struct *files, int nr)
+static int expand_files(struct files_struct *files, int nr)
 {
 	struct fdtable *fdt;
 
 	fdt = files_fdtable(files);
 
-	/*
-	 * N.B. For clone tasks sharing a files structure, this test
-	 * will limit the total number of files that can be opened.
-	 */
-	if (nr >= rlimit(RLIMIT_NOFILE))
-		return -EMFILE;
-
 	/* Do we need to expand? */
 	if (nr < fdt->max_fds)
 		return 0;
@@ -270,6 +256,26 @@ int expand_files(struct files_struct *files, int nr)
 	return expand_fdtable(files, nr);
 }
 
+static inline void __set_close_on_exec(int fd, struct fdtable *fdt)
+{
+	__set_bit(fd, fdt->close_on_exec);
+}
+
+static inline void __clear_close_on_exec(int fd, struct fdtable *fdt)
+{
+	__clear_bit(fd, fdt->close_on_exec);
+}
+
+static inline void __set_open_fd(int fd, struct fdtable *fdt)
+{
+	__set_bit(fd, fdt->open_fds);
+}
+
+static inline void __clear_open_fd(int fd, struct fdtable *fdt)
+{
+	__clear_bit(fd, fdt->open_fds);
+}
+
 static int count_open_files(struct fdtable *fdt)
 {
 	int size = fdt->max_fds;
@@ -395,6 +401,95 @@ out:
 	return NULL;
 }
 
+static void close_files(struct files_struct * files)
+{
+	int i, j;
+	struct fdtable *fdt;
+
+	j = 0;
+
+	/*
+	 * It is safe to dereference the fd table without RCU or
+	 * ->file_lock because this is the last reference to the
+	 * files structure.  But use RCU to shut RCU-lockdep up.
+	 */
+	rcu_read_lock();
+	fdt = files_fdtable(files);
+	rcu_read_unlock();
+	for (;;) {
+		unsigned long set;
+		i = j * BITS_PER_LONG;
+		if (i >= fdt->max_fds)
+			break;
+		set = fdt->open_fds[j++];
+		while (set) {
+			if (set & 1) {
+				struct file * file = xchg(&fdt->fd[i], NULL);
+				if (file) {
+					filp_close(file, files);
+					cond_resched();
+				}
+			}
+			i++;
+			set >>= 1;
+		}
+	}
+}
+
+struct files_struct *get_files_struct(struct task_struct *task)
+{
+	struct files_struct *files;
+
+	task_lock(task);
+	files = task->files;
+	if (files)
+		atomic_inc(&files->count);
+	task_unlock(task);
+
+	return files;
+}
+
+void put_files_struct(struct files_struct *files)
+{
+	struct fdtable *fdt;
+
+	if (atomic_dec_and_test(&files->count)) {
+		close_files(files);
+		/* not really needed, since nobody can see us */
+		rcu_read_lock();
+		fdt = files_fdtable(files);
+		rcu_read_unlock();
+		/* free the arrays if they are not embedded */
+		if (fdt != &files->fdtab)
+			__free_fdtable(fdt);
+		kmem_cache_free(files_cachep, files);
+	}
+}
+
+void reset_files_struct(struct files_struct *files)
+{
+	struct task_struct *tsk = current;
+	struct files_struct *old;
+
+	old = tsk->files;
+	task_lock(tsk);
+	tsk->files = files;
+	task_unlock(tsk);
+	put_files_struct(old);
+}
+
+void exit_files(struct task_struct *tsk)
+{
+	struct files_struct * files = tsk->files;
+
+	if (files) {
+		task_lock(tsk);
+		tsk->files = NULL;
+		task_unlock(tsk);
+		put_files_struct(files);
+	}
+}
+
 static void __devinit fdtable_defer_list_init(int cpu)
 {
 	struct fdtable_defer *fddef = &per_cpu(fdtable_defer_list, cpu);
@@ -424,12 +519,18 @@ struct files_struct init_files = {
 	.file_lock	= __SPIN_LOCK_UNLOCKED(init_task.file_lock),
 };
 
+void daemonize_descriptors(void)
+{
+	atomic_inc(&init_files.count);
+	reset_files_struct(&init_files);
+}
+
 /*
  * allocate a file descriptor, mark it busy.
  */
-int alloc_fd(unsigned start, unsigned flags)
+int __alloc_fd(struct files_struct *files,
+	       unsigned start, unsigned end, unsigned flags)
 {
-	struct files_struct *files = current->files;
 	unsigned int fd;
 	int error;
 	struct fdtable *fdt;
@@ -444,6 +545,14 @@ repeat:
 	if (fd < fdt->max_fds)
 		fd = find_next_zero_bit(fdt->open_fds, fdt->max_fds, fd);
 
+	/*
+	 * N.B. For clone tasks sharing a files structure, this test
+	 * will limit the total number of files that can be opened.
+	 */
+	error = -EMFILE;
+	if (fd >= end)
+		goto out;
+
 	error = expand_files(files, fd);
 	if (error < 0)
 		goto out;
@@ -477,8 +586,424 @@ out:
 	return error;
 }
 
-int get_unused_fd(void)
+static int alloc_fd(unsigned start, unsigned flags)
+{
+	return __alloc_fd(current->files, start, rlimit(RLIMIT_NOFILE), flags);
+}
+
+int get_unused_fd_flags(unsigned flags)
+{
+	return __alloc_fd(current->files, 0, rlimit(RLIMIT_NOFILE), flags);
+}
+EXPORT_SYMBOL(get_unused_fd_flags);
+
+static void __put_unused_fd(struct files_struct *files, unsigned int fd)
+{
+	struct fdtable *fdt = files_fdtable(files);
+	__clear_open_fd(fd, fdt);
+	if (fd < files->next_fd)
+		files->next_fd = fd;
+}
+
+void put_unused_fd(unsigned int fd)
+{
+	struct files_struct *files = current->files;
+	spin_lock(&files->file_lock);
+	__put_unused_fd(files, fd);
+	spin_unlock(&files->file_lock);
+}
+
+EXPORT_SYMBOL(put_unused_fd);
+
+/*
+ * Install a file pointer in the fd array.
+ *
+ * The VFS is full of places where we drop the files lock between
+ * setting the open_fds bitmap and installing the file in the file
+ * array.  At any such point, we are vulnerable to a dup2() race
+ * installing a file in the array before us.  We need to detect this and
+ * fput() the struct file we are about to overwrite in this case.
+ *
+ * It should never happen - if we allow dup2() do it, _really_ bad things
+ * will follow.
+ *
+ * NOTE: __fd_install() variant is really, really low-level; don't
+ * use it unless you are forced to by truly lousy API shoved down
+ * your throat.  'files' *MUST* be either current->files or obtained
+ * by get_files_struct(current) done by whoever had given it to you,
+ * or really bad things will happen.  Normally you want to use
+ * fd_install() instead.
+ */
+
+void __fd_install(struct files_struct *files, unsigned int fd,
+		struct file *file)
+{
+	struct fdtable *fdt;
+	spin_lock(&files->file_lock);
+	fdt = files_fdtable(files);
+	BUG_ON(fdt->fd[fd] != NULL);
+	rcu_assign_pointer(fdt->fd[fd], file);
+	spin_unlock(&files->file_lock);
+}
+
+void fd_install(unsigned int fd, struct file *file)
 {
-	return alloc_fd(0, 0);
+	__fd_install(current->files, fd, file);
+}
+
+EXPORT_SYMBOL(fd_install);
+
+/*
+ * The same warnings as for __alloc_fd()/__fd_install() apply here...
+ */
+int __close_fd(struct files_struct *files, unsigned fd)
+{
+	struct file *file;
+	struct fdtable *fdt;
+
+	spin_lock(&files->file_lock);
+	fdt = files_fdtable(files);
+	if (fd >= fdt->max_fds)
+		goto out_unlock;
+	file = fdt->fd[fd];
+	if (!file)
+		goto out_unlock;
+	rcu_assign_pointer(fdt->fd[fd], NULL);
+	__clear_close_on_exec(fd, fdt);
+	__put_unused_fd(files, fd);
+	spin_unlock(&files->file_lock);
+	return filp_close(file, files);
+
+out_unlock:
+	spin_unlock(&files->file_lock);
+	return -EBADF;
+}
+
+void do_close_on_exec(struct files_struct *files)
+{
+	unsigned i;
+	struct fdtable *fdt;
+
+	/* exec unshares first */
+	BUG_ON(atomic_read(&files->count) != 1);
+	spin_lock(&files->file_lock);
+	for (i = 0; ; i++) {
+		unsigned long set;
+		unsigned fd = i * BITS_PER_LONG;
+		fdt = files_fdtable(files);
+		if (fd >= fdt->max_fds)
+			break;
+		set = fdt->close_on_exec[i];
+		if (!set)
+			continue;
+		fdt->close_on_exec[i] = 0;
+		for ( ; set ; fd++, set >>= 1) {
+			struct file *file;
+			if (!(set & 1))
+				continue;
+			file = fdt->fd[fd];
+			if (!file)
+				continue;
+			rcu_assign_pointer(fdt->fd[fd], NULL);
+			__put_unused_fd(files, fd);
+			spin_unlock(&files->file_lock);
+			filp_close(file, files);
+			cond_resched();
+			spin_lock(&files->file_lock);
+		}
+
+	}
+	spin_unlock(&files->file_lock);
+}
+
+struct file *fget(unsigned int fd)
+{
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	rcu_read_lock();
+	file = fcheck_files(files, fd);
+	if (file) {
+		/* File object ref couldn't be taken */
+		if (file->f_mode & FMODE_PATH ||
+		    !atomic_long_inc_not_zero(&file->f_count))
+			file = NULL;
+	}
+	rcu_read_unlock();
+
+	return file;
+}
+
+EXPORT_SYMBOL(fget);
+
+struct file *fget_raw(unsigned int fd)
+{
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	rcu_read_lock();
+	file = fcheck_files(files, fd);
+	if (file) {
+		/* File object ref couldn't be taken */
+		if (!atomic_long_inc_not_zero(&file->f_count))
+			file = NULL;
+	}
+	rcu_read_unlock();
+
+	return file;
+}
+
+EXPORT_SYMBOL(fget_raw);
+
+/*
+ * Lightweight file lookup - no refcnt increment if fd table isn't shared.
+ *
+ * You can use this instead of fget if you satisfy all of the following
+ * conditions:
+ * 1) You must call fput_light before exiting the syscall and returning control
+ *    to userspace (i.e. you cannot remember the returned struct file * after
+ *    returning to userspace).
+ * 2) You must not call filp_close on the returned struct file * in between
+ *    calls to fget_light and fput_light.
+ * 3) You must not clone the current task in between the calls to fget_light
+ *    and fput_light.
+ *
+ * The fput_needed flag returned by fget_light should be passed to the
+ * corresponding fput_light.
+ */
+struct file *fget_light(unsigned int fd, int *fput_needed)
+{
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	*fput_needed = 0;
+	if (atomic_read(&files->count) == 1) {
+		file = fcheck_files(files, fd);
+		if (file && (file->f_mode & FMODE_PATH))
+			file = NULL;
+	} else {
+		rcu_read_lock();
+		file = fcheck_files(files, fd);
+		if (file) {
+			if (!(file->f_mode & FMODE_PATH) &&
+			    atomic_long_inc_not_zero(&file->f_count))
+				*fput_needed = 1;
+			else
+				/* Didn't get the reference, someone's freed */
+				file = NULL;
+		}
+		rcu_read_unlock();
+	}
+
+	return file;
+}
+EXPORT_SYMBOL(fget_light);
+
+struct file *fget_raw_light(unsigned int fd, int *fput_needed)
+{
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	*fput_needed = 0;
+	if (atomic_read(&files->count) == 1) {
+		file = fcheck_files(files, fd);
+	} else {
+		rcu_read_lock();
+		file = fcheck_files(files, fd);
+		if (file) {
+			if (atomic_long_inc_not_zero(&file->f_count))
+				*fput_needed = 1;
+			else
+				/* Didn't get the reference, someone's freed */
+				file = NULL;
+		}
+		rcu_read_unlock();
+	}
+
+	return file;
+}
+
+void set_close_on_exec(unsigned int fd, int flag)
+{
+	struct files_struct *files = current->files;
+	struct fdtable *fdt;
+	spin_lock(&files->file_lock);
+	fdt = files_fdtable(files);
+	if (flag)
+		__set_close_on_exec(fd, fdt);
+	else
+		__clear_close_on_exec(fd, fdt);
+	spin_unlock(&files->file_lock);
+}
+
+bool get_close_on_exec(unsigned int fd)
+{
+	struct files_struct *files = current->files;
+	struct fdtable *fdt;
+	bool res;
+	rcu_read_lock();
+	fdt = files_fdtable(files);
+	res = close_on_exec(fd, fdt);
+	rcu_read_unlock();
+	return res;
+}
+
+static int do_dup2(struct files_struct *files,
+	struct file *file, unsigned fd, unsigned flags)
+{
+	struct file *tofree;
+	struct fdtable *fdt;
+
+	/*
+	 * We need to detect attempts to do dup2() over allocated but still
+	 * not finished descriptor.  NB: OpenBSD avoids that at the price of
+	 * extra work in their equivalent of fget() - they insert struct
+	 * file immediately after grabbing descriptor, mark it larval if
+	 * more work (e.g. actual opening) is needed and make sure that
+	 * fget() treats larval files as absent.  Potentially interesting,
+	 * but while extra work in fget() is trivial, locking implications
+	 * and amount of surgery on open()-related paths in VFS are not.
+	 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
+	 * deadlocks in rather amusing ways, AFAICS.  All of that is out of
+	 * scope of POSIX or SUS, since neither considers shared descriptor
+	 * tables and this condition does not arise without those.
+	 */
+	fdt = files_fdtable(files);
+	tofree = fdt->fd[fd];
+	if (!tofree && fd_is_open(fd, fdt))
+		goto Ebusy;
+	get_file(file);
+	rcu_assign_pointer(fdt->fd[fd], file);
+	__set_open_fd(fd, fdt);
+	if (flags & O_CLOEXEC)
+		__set_close_on_exec(fd, fdt);
+	else
+		__clear_close_on_exec(fd, fdt);
+	spin_unlock(&files->file_lock);
+
+	if (tofree)
+		filp_close(tofree, files);
+
+	return fd;
+
+Ebusy:
+	spin_unlock(&files->file_lock);
+	return -EBUSY;
+}
+
+int replace_fd(unsigned fd, struct file *file, unsigned flags)
+{
+	int err;
+	struct files_struct *files = current->files;
+
+	if (!file)
+		return __close_fd(files, fd);
+
+	if (fd >= rlimit(RLIMIT_NOFILE))
+		return -EMFILE;
+
+	spin_lock(&files->file_lock);
+	err = expand_files(files, fd);
+	if (unlikely(err < 0))
+		goto out_unlock;
+	return do_dup2(files, file, fd, flags);
+
+out_unlock:
+	spin_unlock(&files->file_lock);
+	return err;
+}
+
+SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
+{
+	int err = -EBADF;
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	if ((flags & ~O_CLOEXEC) != 0)
+		return -EINVAL;
+
+	if (newfd >= rlimit(RLIMIT_NOFILE))
+		return -EMFILE;
+
+	spin_lock(&files->file_lock);
+	err = expand_files(files, newfd);
+	file = fcheck(oldfd);
+	if (unlikely(!file))
+		goto Ebadf;
+	if (unlikely(err < 0)) {
+		if (err == -EMFILE)
+			goto Ebadf;
+		goto out_unlock;
+	}
+	return do_dup2(files, file, newfd, flags);
+
+Ebadf:
+	err = -EBADF;
+out_unlock:
+	spin_unlock(&files->file_lock);
+	return err;
+}
+
+SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
+{
+	if (unlikely(newfd == oldfd)) { /* corner case */
+		struct files_struct *files = current->files;
+		int retval = oldfd;
+
+		rcu_read_lock();
+		if (!fcheck_files(files, oldfd))
+			retval = -EBADF;
+		rcu_read_unlock();
+		return retval;
+	}
+	return sys_dup3(oldfd, newfd, 0);
+}
+
+SYSCALL_DEFINE1(dup, unsigned int, fildes)
+{
+	int ret = -EBADF;
+	struct file *file = fget_raw(fildes);
+
+	if (file) {
+		ret = get_unused_fd();
+		if (ret >= 0)
+			fd_install(ret, file);
+		else
+			fput(file);
+	}
+	return ret;
+}
+
+int f_dupfd(unsigned int from, struct file *file, unsigned flags)
+{
+	int err;
+	if (from >= rlimit(RLIMIT_NOFILE))
+		return -EINVAL;
+	err = alloc_fd(from, flags);
+	if (err >= 0) {
+		get_file(file);
+		fd_install(err, file);
+	}
+	return err;
+}
+
+int iterate_fd(struct files_struct *files, unsigned n,
+		int (*f)(const void *, struct file *, unsigned),
+		const void *p)
+{
+	struct fdtable *fdt;
+	struct file *file;
+	int res = 0;
+	if (!files)
+		return 0;
+	spin_lock(&files->file_lock);
+	fdt = files_fdtable(files);
+	while (!res && n < fdt->max_fds) {
+		file = rcu_dereference_check_fdtable(files, fdt->fd[n++]);
+		if (file)
+			res = f(p, file, n);
+	}
+	spin_unlock(&files->file_lock);
+	return res;
 }
-EXPORT_SYMBOL(get_unused_fd);
+EXPORT_SYMBOL(iterate_fd);
diff --git a/fs/file_table.c b/fs/file_table.c
index 701985e4ccd..dac67923330 100644
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -243,10 +243,10 @@ static void __fput(struct file *file)
 		if (file->f_op && file->f_op->fasync)
 			file->f_op->fasync(-1, file, 0);
 	}
+	ima_file_free(file);
 	if (file->f_op && file->f_op->release)
 		file->f_op->release(inode, file);
 	security_file_free(file);
-	ima_file_free(file);
 	if (unlikely(S_ISCHR(inode->i_mode) && inode->i_cdev != NULL &&
 		     !(file->f_mode & FMODE_PATH))) {
 		cdev_put(inode->i_cdev);
@@ -339,112 +339,6 @@ void __fput_sync(struct file *file)
 
 EXPORT_SYMBOL(fput);
 
-struct file *fget(unsigned int fd)
-{
-	struct file *file;
-	struct files_struct *files = current->files;
-
-	rcu_read_lock();
-	file = fcheck_files(files, fd);
-	if (file) {
-		/* File object ref couldn't be taken */
-		if (file->f_mode & FMODE_PATH ||
-		    !atomic_long_inc_not_zero(&file->f_count))
-			file = NULL;
-	}
-	rcu_read_unlock();
-
-	return file;
-}
-
-EXPORT_SYMBOL(fget);
-
-struct file *fget_raw(unsigned int fd)
-{
-	struct file *file;
-	struct files_struct *files = current->files;
-
-	rcu_read_lock();
-	file = fcheck_files(files, fd);
-	if (file) {
-		/* File object ref couldn't be taken */
-		if (!atomic_long_inc_not_zero(&file->f_count))
-			file = NULL;
-	}
-	rcu_read_unlock();
-
-	return file;
-}
-
-EXPORT_SYMBOL(fget_raw);
-
-/*
- * Lightweight file lookup - no refcnt increment if fd table isn't shared.
- *
- * You can use this instead of fget if you satisfy all of the following
- * conditions:
- * 1) You must call fput_light before exiting the syscall and returning control
- *    to userspace (i.e. you cannot remember the returned struct file * after
- *    returning to userspace).
- * 2) You must not call filp_close on the returned struct file * in between
- *    calls to fget_light and fput_light.
- * 3) You must not clone the current task in between the calls to fget_light
- *    and fput_light.
- *
- * The fput_needed flag returned by fget_light should be passed to the
- * corresponding fput_light.
- */
-struct file *fget_light(unsigned int fd, int *fput_needed)
-{
-	struct file *file;
-	struct files_struct *files = current->files;
-
-	*fput_needed = 0;
-	if (atomic_read(&files->count) == 1) {
-		file = fcheck_files(files, fd);
-		if (file && (file->f_mode & FMODE_PATH))
-			file = NULL;
-	} else {
-		rcu_read_lock();
-		file = fcheck_files(files, fd);
-		if (file) {
-			if (!(file->f_mode & FMODE_PATH) &&
-			    atomic_long_inc_not_zero(&file->f_count))
-				*fput_needed = 1;
-			else
-				/* Didn't get the reference, someone's freed */
-				file = NULL;
-		}
-		rcu_read_unlock();
-	}
-
-	return file;
-}
-
-struct file *fget_raw_light(unsigned int fd, int *fput_needed)
-{
-	struct file *file;
-	struct files_struct *files = current->files;
-
-	*fput_needed = 0;
-	if (atomic_read(&files->count) == 1) {
-		file = fcheck_files(files, fd);
-	} else {
-		rcu_read_lock();
-		file = fcheck_files(files, fd);
-		if (file) {
-			if (atomic_long_inc_not_zero(&file->f_count))
-				*fput_needed = 1;
-			else
-				/* Didn't get the reference, someone's freed */
-				file = NULL;
-		}
-		rcu_read_unlock();
-	}
-
-	return file;
-}
-
 void put_filp(struct file *file)
 {
 	if (atomic_long_dec_and_test(&file->f_count)) {
diff --git a/fs/freevxfs/vxfs_super.c b/fs/freevxfs/vxfs_super.c
index d4fabd26084..fed2c8afb3a 100644
--- a/fs/freevxfs/vxfs_super.c
+++ b/fs/freevxfs/vxfs_super.c
@@ -279,6 +279,11 @@ static void __exit
 vxfs_cleanup(void)
 {
 	unregister_filesystem(&vxfs_fs_type);
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(vxfs_inode_cachep);
 }
 
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index f4246cfc8d8..8c23fa7a91e 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -148,8 +148,7 @@ static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
 		if (ff->reserved_req) {
 			req = ff->reserved_req;
 			ff->reserved_req = NULL;
-			get_file(file);
-			req->stolen_file = file;
+			req->stolen_file = get_file(file);
 		}
 		spin_unlock(&fc->lock);
 	} while (!req);
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index fca222dabe3..f0eda124cff 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -1197,6 +1197,12 @@ static void fuse_fs_cleanup(void)
 {
 	unregister_filesystem(&fuse_fs_type);
 	unregister_fuseblk();
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(fuse_inode_cachep);
 }
 
diff --git a/fs/hfs/super.c b/fs/hfs/super.c
index 0b63d135a09..e93ddaadfd1 100644
--- a/fs/hfs/super.c
+++ b/fs/hfs/super.c
@@ -492,6 +492,12 @@ static int __init init_hfs_fs(void)
 static void __exit exit_hfs_fs(void)
 {
 	unregister_filesystem(&hfs_fs_type);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(hfs_inode_cachep);
 }
 
diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c
index fdafb2d7165..811a84d2d96 100644
--- a/fs/hfsplus/super.c
+++ b/fs/hfsplus/super.c
@@ -635,6 +635,12 @@ static int __init init_hfsplus_fs(void)
 static void __exit exit_hfsplus_fs(void)
 {
 	unregister_filesystem(&hfsplus_fs_type);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(hfsplus_inode_cachep);
 }
 
diff --git a/fs/hpfs/anode.c b/fs/hpfs/anode.c
index 4bae4a4a60b..2d5b254ad9e 100644
--- a/fs/hpfs/anode.c
+++ b/fs/hpfs/anode.c
@@ -102,7 +102,7 @@ secno hpfs_add_sector_to_btree(struct super_block *s, secno node, int fnod, unsi
 			return -1;
 		}
 		if (hpfs_alloc_if_possible(s, se = le32_to_cpu(btree->u.external[n].disk_secno) + le32_to_cpu(btree->u.external[n].length))) {
-			btree->u.external[n].length = cpu_to_le32(le32_to_cpu(btree->u.external[n].length) + 1);
+			le32_add_cpu(&btree->u.external[n].length, 1);
 			mark_buffer_dirty(bh);
 			brelse(bh);
 			return se;
@@ -153,7 +153,7 @@ secno hpfs_add_sector_to_btree(struct super_block *s, secno node, int fnod, unsi
 		btree = &anode->btree;
 	}
 	btree->n_free_nodes--; n = btree->n_used_nodes++;
-	btree->first_free = cpu_to_le16(le16_to_cpu(btree->first_free) + 12);
+	le16_add_cpu(&btree->first_free, 12);
 	btree->u.external[n].disk_secno = cpu_to_le32(se);
 	btree->u.external[n].file_secno = cpu_to_le32(fs);
 	btree->u.external[n].length = cpu_to_le32(1);
@@ -174,7 +174,7 @@ secno hpfs_add_sector_to_btree(struct super_block *s, secno node, int fnod, unsi
 		}
 		if (btree->n_free_nodes) {
 			btree->n_free_nodes--; n = btree->n_used_nodes++;
-			btree->first_free = cpu_to_le16(le16_to_cpu(btree->first_free) + 8);
+			le16_add_cpu(&btree->first_free, 8);
 			btree->u.internal[n].file_secno = cpu_to_le32(-1);
 			btree->u.internal[n].down = cpu_to_le32(na);
 			btree->u.internal[n-1].file_secno = cpu_to_le32(fs);
diff --git a/fs/hpfs/dnode.c b/fs/hpfs/dnode.c
index 3228c524ebe..4364b2a02c5 100644
--- a/fs/hpfs/dnode.c
+++ b/fs/hpfs/dnode.c
@@ -145,10 +145,10 @@ static void set_last_pointer(struct super_block *s, struct dnode *d, dnode_secno
 		}
 	}
 	if (ptr) {
-		d->first_free = cpu_to_le32(le32_to_cpu(d->first_free) + 4);
+		le32_add_cpu(&d->first_free, 4);
 		if (le32_to_cpu(d->first_free) > 2048) {
 			hpfs_error(s, "set_last_pointer: too long dnode %08x", le32_to_cpu(d->self));
-			d->first_free = cpu_to_le32(le32_to_cpu(d->first_free) - 4);
+			le32_add_cpu(&d->first_free, -4);
 			return;
 		}
 		de->length = cpu_to_le16(36);
@@ -184,7 +184,7 @@ struct hpfs_dirent *hpfs_add_de(struct super_block *s, struct dnode *d,
 	de->not_8x3 = hpfs_is_name_long(name, namelen);
 	de->namelen = namelen;
 	memcpy(de->name, name, namelen);
-	d->first_free = cpu_to_le32(le32_to_cpu(d->first_free) + d_size);
+	le32_add_cpu(&d->first_free, d_size);
 	return de;
 }
 
@@ -314,7 +314,7 @@ static int hpfs_add_to_dnode(struct inode *i, dnode_secno dno,
 	set_last_pointer(i->i_sb, ad, de->down ? de_down_pointer(de) : 0);
 	de = de_next_de(de);
 	memmove((char *)nd + 20, de, le32_to_cpu(nd->first_free) + (char *)nd - (char *)de);
-	nd->first_free = cpu_to_le32(le32_to_cpu(nd->first_free) - ((char *)de - (char *)nd - 20));
+	le32_add_cpu(&nd->first_free, -((char *)de - (char *)nd - 20));
 	memcpy(d, nd, le32_to_cpu(nd->first_free));
 	for_all_poss(i, hpfs_pos_del, (loff_t)dno << 4, pos);
 	fix_up_ptrs(i->i_sb, ad);
@@ -474,8 +474,8 @@ static secno move_to_top(struct inode *i, dnode_secno from, dnode_secno to)
 			hpfs_brelse4(&qbh);
 			return 0;
 		}
-		dnode->first_free = cpu_to_le32(le32_to_cpu(dnode->first_free) - 4);
-		de->length = cpu_to_le16(le16_to_cpu(de->length) - 4);
+		le32_add_cpu(&dnode->first_free, -4);
+		le16_add_cpu(&de->length, -4);
 		de->down = 0;
 		hpfs_mark_4buffers_dirty(&qbh);
 		dno = up;
@@ -570,8 +570,8 @@ static void delete_empty_dnode(struct inode *i, dnode_secno dno)
 		for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | 1, ((loff_t)up << 4) | p);
 		if (!down) {
 			de->down = 0;
-			de->length = cpu_to_le16(le16_to_cpu(de->length) - 4);
-			dnode->first_free = cpu_to_le32(le32_to_cpu(dnode->first_free) - 4);
+			le16_add_cpu(&de->length, -4);
+			le32_add_cpu(&dnode->first_free, -4);
 			memmove(de_next_de(de), (char *)de_next_de(de) + 4,
 				(char *)dnode + le32_to_cpu(dnode->first_free) - (char *)de_next_de(de));
 		} else {
@@ -647,14 +647,14 @@ static void delete_empty_dnode(struct inode *i, dnode_secno dno)
 					printk("HPFS: warning: unbalanced dnode tree, see hpfs.txt 4 more info\n");
 					printk("HPFS: warning: goin'on\n");
 				}
-				del->length = cpu_to_le16(le16_to_cpu(del->length) + 4);
+				le16_add_cpu(&del->length, 4);
 				del->down = 1;
-				d1->first_free = cpu_to_le32(le32_to_cpu(d1->first_free) + 4);
+				le32_add_cpu(&d1->first_free, 4);
 			}
 			if (dlp && !down) {
-				del->length = cpu_to_le16(le16_to_cpu(del->length) - 4);
+				le16_add_cpu(&del->length, -4);
 				del->down = 0;
-				d1->first_free = cpu_to_le32(le32_to_cpu(d1->first_free) - 4);
+				le32_add_cpu(&d1->first_free, -4);
 			} else if (down)
 				*(__le32 *) ((void *) del + le16_to_cpu(del->length) - 4) = cpu_to_le32(down);
 		} else goto endm;
@@ -668,9 +668,9 @@ static void delete_empty_dnode(struct inode *i, dnode_secno dno)
 		memcpy(de_cp, de_prev, le16_to_cpu(de_prev->length));
 		hpfs_delete_de(i->i_sb, dnode, de_prev);
 		if (!de_prev->down) {
-			de_prev->length = cpu_to_le16(le16_to_cpu(de_prev->length) + 4);
+			le16_add_cpu(&de_prev->length, 4);
 			de_prev->down = 1;
-			dnode->first_free = cpu_to_le32(le32_to_cpu(dnode->first_free) + 4);
+			le32_add_cpu(&dnode->first_free, 4);
 		}
 		*(__le32 *) ((void *) de_prev + le16_to_cpu(de_prev->length) - 4) = cpu_to_le32(ndown);
 		hpfs_mark_4buffers_dirty(&qbh);
diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c
index a152783602d..bc28bf077a6 100644
--- a/fs/hpfs/super.c
+++ b/fs/hpfs/super.c
@@ -210,6 +210,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(hpfs_inode_cachep);
 }
 
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 6e572c4fbf6..9460120a517 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -1048,6 +1048,11 @@ static int __init init_hugetlbfs_fs(void)
 
 static void __exit exit_hugetlbfs_fs(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(hugetlbfs_inode_cachep);
 	kern_unmount(hugetlbfs_vfsmount);
 	unregister_filesystem(&hugetlbfs_fs_type);
diff --git a/fs/ioctl.c b/fs/ioctl.c
index 29167bebe87..3bdad6d1f26 100644
--- a/fs/ioctl.c
+++ b/fs/ioctl.c
@@ -603,21 +603,14 @@ int do_vfs_ioctl(struct file *filp, unsigned int fd, unsigned int cmd,
 
 SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, unsigned long, arg)
 {
-	struct file *filp;
-	int error = -EBADF;
-	int fput_needed;
-
-	filp = fget_light(fd, &fput_needed);
-	if (!filp)
-		goto out;
-
-	error = security_file_ioctl(filp, cmd, arg);
-	if (error)
-		goto out_fput;
-
-	error = do_vfs_ioctl(filp, fd, cmd, arg);
- out_fput:
-	fput_light(filp, fput_needed);
- out:
+	int error;
+	struct fd f = fdget(fd);
+
+	if (!f.file)
+		return -EBADF;
+	error = security_file_ioctl(f.file, cmd, arg);
+	if (!error)
+		error = do_vfs_ioctl(f.file, fd, cmd, arg);
+	fdput(f);
 	return error;
 }
diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c
index a7d8e6cc5e0..67ce52507d7 100644
--- a/fs/isofs/inode.c
+++ b/fs/isofs/inode.c
@@ -115,6 +115,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(isofs_inode_cachep);
 }
 
diff --git a/fs/jbd/commit.c b/fs/jbd/commit.c
index 52c15c77602..86b39b167c2 100644
--- a/fs/jbd/commit.c
+++ b/fs/jbd/commit.c
@@ -86,7 +86,12 @@ nope:
 static void release_data_buffer(struct buffer_head *bh)
 {
 	if (buffer_freed(bh)) {
+		WARN_ON_ONCE(buffer_dirty(bh));
 		clear_buffer_freed(bh);
+		clear_buffer_mapped(bh);
+		clear_buffer_new(bh);
+		clear_buffer_req(bh);
+		bh->b_bdev = NULL;
 		release_buffer_page(bh);
 	} else
 		put_bh(bh);
@@ -866,17 +871,35 @@ restart_loop:
 		 * there's no point in keeping a checkpoint record for
 		 * it. */
 
-		/* A buffer which has been freed while still being
-		 * journaled by a previous transaction may end up still
-		 * being dirty here, but we want to avoid writing back
-		 * that buffer in the future after the "add to orphan"
-		 * operation been committed,  That's not only a performance
-		 * gain, it also stops aliasing problems if the buffer is
-		 * left behind for writeback and gets reallocated for another
-		 * use in a different page. */
-		if (buffer_freed(bh) && !jh->b_next_transaction) {
-			clear_buffer_freed(bh);
-			clear_buffer_jbddirty(bh);
+		/*
+		 * A buffer which has been freed while still being journaled by
+		 * a previous transaction.
+		 */
+		if (buffer_freed(bh)) {
+			/*
+			 * If the running transaction is the one containing
+			 * "add to orphan" operation (b_next_transaction !=
+			 * NULL), we have to wait for that transaction to
+			 * commit before we can really get rid of the buffer.
+			 * So just clear b_modified to not confuse transaction
+			 * credit accounting and refile the buffer to
+			 * BJ_Forget of the running transaction. If the just
+			 * committed transaction contains "add to orphan"
+			 * operation, we can completely invalidate the buffer
+			 * now. We are rather throughout in that since the
+			 * buffer may be still accessible when blocksize <
+			 * pagesize and it is attached to the last partial
+			 * page.
+			 */
+			jh->b_modified = 0;
+			if (!jh->b_next_transaction) {
+				clear_buffer_freed(bh);
+				clear_buffer_jbddirty(bh);
+				clear_buffer_mapped(bh);
+				clear_buffer_new(bh);
+				clear_buffer_req(bh);
+				bh->b_bdev = NULL;
+			}
 		}
 
 		if (buffer_jbddirty(bh)) {
diff --git a/fs/jbd/transaction.c b/fs/jbd/transaction.c
index febc10db5ce..78b7f84241d 100644
--- a/fs/jbd/transaction.c
+++ b/fs/jbd/transaction.c
@@ -1843,15 +1843,16 @@ static int __dispose_buffer(struct journal_head *jh, transaction_t *transaction)
  * We're outside-transaction here.  Either or both of j_running_transaction
  * and j_committing_transaction may be NULL.
  */
-static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
+static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh,
+				int partial_page)
 {
 	transaction_t *transaction;
 	struct journal_head *jh;
 	int may_free = 1;
-	int ret;
 
 	BUFFER_TRACE(bh, "entry");
 
+retry:
 	/*
 	 * It is safe to proceed here without the j_list_lock because the
 	 * buffers cannot be stolen by try_to_free_buffers as long as we are
@@ -1879,10 +1880,18 @@ static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
 	 * clear the buffer dirty bit at latest at the moment when the
 	 * transaction marking the buffer as freed in the filesystem
 	 * structures is committed because from that moment on the
-	 * buffer can be reallocated and used by a different page.
+	 * block can be reallocated and used by a different page.
 	 * Since the block hasn't been freed yet but the inode has
 	 * already been added to orphan list, it is safe for us to add
 	 * the buffer to BJ_Forget list of the newest transaction.
+	 *
+	 * Also we have to clear buffer_mapped flag of a truncated buffer
+	 * because the buffer_head may be attached to the page straddling
+	 * i_size (can happen only when blocksize < pagesize) and thus the
+	 * buffer_head can be reused when the file is extended again. So we end
+	 * up keeping around invalidated buffers attached to transactions'
+	 * BJ_Forget list just to stop checkpointing code from cleaning up
+	 * the transaction this buffer was modified in.
 	 */
 	transaction = jh->b_transaction;
 	if (transaction == NULL) {
@@ -1909,13 +1918,9 @@ static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
 			 * committed, the buffer won't be needed any
 			 * longer. */
 			JBUFFER_TRACE(jh, "checkpointed: add to BJ_Forget");
-			ret = __dispose_buffer(jh,
+			may_free = __dispose_buffer(jh,
 					journal->j_running_transaction);
-			journal_put_journal_head(jh);
-			spin_unlock(&journal->j_list_lock);
-			jbd_unlock_bh_state(bh);
-			spin_unlock(&journal->j_state_lock);
-			return ret;
+			goto zap_buffer;
 		} else {
 			/* There is no currently-running transaction. So the
 			 * orphan record which we wrote for this file must have
@@ -1923,13 +1928,9 @@ static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
 			 * the committing transaction, if it exists. */
 			if (journal->j_committing_transaction) {
 				JBUFFER_TRACE(jh, "give to committing trans");
-				ret = __dispose_buffer(jh,
+				may_free = __dispose_buffer(jh,
 					journal->j_committing_transaction);
-				journal_put_journal_head(jh);
-				spin_unlock(&journal->j_list_lock);
-				jbd_unlock_bh_state(bh);
-				spin_unlock(&journal->j_state_lock);
-				return ret;
+				goto zap_buffer;
 			} else {
 				/* The orphan record's transaction has
 				 * committed.  We can cleanse this buffer */
@@ -1950,10 +1951,24 @@ static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
 		}
 		/*
 		 * The buffer is committing, we simply cannot touch
-		 * it. So we just set j_next_transaction to the
-		 * running transaction (if there is one) and mark
-		 * buffer as freed so that commit code knows it should
-		 * clear dirty bits when it is done with the buffer.
+		 * it. If the page is straddling i_size we have to wait
+		 * for commit and try again.
+		 */
+		if (partial_page) {
+			tid_t tid = journal->j_committing_transaction->t_tid;
+
+			journal_put_journal_head(jh);
+			spin_unlock(&journal->j_list_lock);
+			jbd_unlock_bh_state(bh);
+			spin_unlock(&journal->j_state_lock);
+			log_wait_commit(journal, tid);
+			goto retry;
+		}
+		/*
+		 * OK, buffer won't be reachable after truncate. We just set
+		 * j_next_transaction to the running transaction (if there is
+		 * one) and mark buffer as freed so that commit code knows it
+		 * should clear dirty bits when it is done with the buffer.
 		 */
 		set_buffer_freed(bh);
 		if (journal->j_running_transaction && buffer_jbddirty(bh))
@@ -1976,6 +1991,14 @@ static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh)
 	}
 
 zap_buffer:
+	/*
+	 * This is tricky. Although the buffer is truncated, it may be reused
+	 * if blocksize < pagesize and it is attached to the page straddling
+	 * EOF. Since the buffer might have been added to BJ_Forget list of the
+	 * running transaction, journal_get_write_access() won't clear
+	 * b_modified and credit accounting gets confused. So clear b_modified
+	 * here. */
+	jh->b_modified = 0;
 	journal_put_journal_head(jh);
 zap_buffer_no_jh:
 	spin_unlock(&journal->j_list_lock);
@@ -2024,7 +2047,8 @@ void journal_invalidatepage(journal_t *journal,
 		if (offset <= curr_off) {
 			/* This block is wholly outside the truncation point */
 			lock_buffer(bh);
-			may_free &= journal_unmap_buffer(journal, bh);
+			may_free &= journal_unmap_buffer(journal, bh,
+							 offset > 0);
 			unlock_buffer(bh);
 		}
 		curr_off = next_off;
diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c
index 61ea41389f9..ff487954cd9 100644
--- a/fs/jffs2/super.c
+++ b/fs/jffs2/super.c
@@ -418,6 +418,12 @@ static void __exit exit_jffs2_fs(void)
 	unregister_filesystem(&jffs2_fs_type);
 	jffs2_destroy_slab_caches();
 	jffs2_compressors_exit();
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(jffs2_inode_cachep);
 }
 
diff --git a/fs/jfs/Makefile b/fs/jfs/Makefile
index a58fa72d7e5..d20d4737b3e 100644
--- a/fs/jfs/Makefile
+++ b/fs/jfs/Makefile
@@ -6,7 +6,7 @@ obj-$(CONFIG_JFS_FS) += jfs.o
 
 jfs-y    := super.o file.o inode.o namei.o jfs_mount.o jfs_umount.o \
 	    jfs_xtree.o jfs_imap.o jfs_debug.o jfs_dmap.o \
-	    jfs_unicode.o jfs_dtree.o jfs_inode.o \
+	    jfs_unicode.o jfs_dtree.o jfs_inode.o jfs_discard.o \
 	    jfs_extent.o symlink.o jfs_metapage.o \
 	    jfs_logmgr.o jfs_txnmgr.o jfs_uniupr.o \
 	    resize.o xattr.o ioctl.o
diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c
index f19d1e04a37..bc555ff417e 100644
--- a/fs/jfs/ioctl.c
+++ b/fs/jfs/ioctl.c
@@ -11,13 +11,17 @@
 #include <linux/mount.h>
 #include <linux/time.h>
 #include <linux/sched.h>
+#include <linux/blkdev.h>
 #include <asm/current.h>
 #include <asm/uaccess.h>
 
+#include "jfs_filsys.h"
+#include "jfs_debug.h"
 #include "jfs_incore.h"
 #include "jfs_dinode.h"
 #include "jfs_inode.h"
-
+#include "jfs_dmap.h"
+#include "jfs_discard.h"
 
 static struct {
 	long jfs_flag;
@@ -123,6 +127,40 @@ setflags_out:
 		mnt_drop_write_file(filp);
 		return err;
 	}
+
+	case FITRIM:
+	{
+		struct super_block *sb = inode->i_sb;
+		struct request_queue *q = bdev_get_queue(sb->s_bdev);
+		struct fstrim_range range;
+		s64 ret = 0;
+
+		if (!capable(CAP_SYS_ADMIN))
+			return -EPERM;
+
+		if (!blk_queue_discard(q)) {
+			jfs_warn("FITRIM not supported on device");
+			return -EOPNOTSUPP;
+		}
+
+		if (copy_from_user(&range, (struct fstrim_range __user *)arg,
+		    sizeof(range)))
+			return -EFAULT;
+
+		range.minlen = max_t(unsigned int, range.minlen,
+			q->limits.discard_granularity);
+
+		ret = jfs_ioc_trim(inode, &range);
+		if (ret < 0)
+			return ret;
+
+		if (copy_to_user((struct fstrim_range __user *)arg, &range,
+		    sizeof(range)))
+			return -EFAULT;
+
+		return 0;
+	}
+
 	default:
 		return -ENOTTY;
 	}
@@ -142,6 +180,9 @@ long jfs_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	case JFS_IOC_SETFLAGS32:
 		cmd = JFS_IOC_SETFLAGS;
 		break;
+	case FITRIM:
+		cmd = FITRIM;
+		break;
 	}
 	return jfs_ioctl(filp, cmd, arg);
 }
diff --git a/fs/jfs/jfs_discard.c b/fs/jfs/jfs_discard.c
new file mode 100644
index 00000000000..9947563e417
--- /dev/null
+++ b/fs/jfs/jfs_discard.c
@@ -0,0 +1,117 @@
+/*
+ *   Copyright (C) Tino Reichardt, 2012
+ *
+ *   This program is free software;  you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+ *   the GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program;  if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/blkdev.h>
+
+#include "jfs_incore.h"
+#include "jfs_superblock.h"
+#include "jfs_discard.h"
+#include "jfs_dmap.h"
+#include "jfs_debug.h"
+
+
+/*
+ * NAME:	jfs_issue_discard()
+ *
+ * FUNCTION:	TRIM the specified block range on device, if supported
+ *
+ * PARAMETERS:
+ *	ip	- pointer to in-core inode
+ *	blkno	- starting block number to be trimmed (0..N)
+ *	nblocks	- number of blocks to be trimmed
+ *
+ * RETURN VALUES:
+ *	none
+ *
+ * serialization: IREAD_LOCK(ipbmap) held on entry/exit;
+ */
+void jfs_issue_discard(struct inode *ip, u64 blkno, u64 nblocks)
+{
+	struct super_block *sb = ip->i_sb;
+	int r = 0;
+
+	r = sb_issue_discard(sb, blkno, nblocks, GFP_NOFS, 0);
+	if (unlikely(r != 0)) {
+		jfs_err("JFS: sb_issue_discard" \
+			"(%p, %llu, %llu, GFP_NOFS, 0) = %d => failed!\n",
+			sb, (unsigned long long)blkno,
+			(unsigned long long)nblocks, r);
+	}
+
+	jfs_info("JFS: sb_issue_discard" \
+		"(%p, %llu, %llu, GFP_NOFS, 0) = %d\n",
+		sb, (unsigned long long)blkno,
+		(unsigned long long)nblocks, r);
+
+	return;
+}
+
+/*
+ * NAME:	jfs_ioc_trim()
+ *
+ * FUNCTION:	attempt to discard (TRIM) all free blocks from the
+ *              filesystem.
+ *
+ * PARAMETERS:
+ *	ip	- pointer to in-core inode;
+ *	range	- the range, given by user space
+ *
+ * RETURN VALUES:
+ *	0	- success
+ *	-EIO	- i/o error
+ */
+int jfs_ioc_trim(struct inode *ip, struct fstrim_range *range)
+{
+	struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap;
+	struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap;
+	struct super_block *sb = ipbmap->i_sb;
+	int agno, agno_end;
+	s64 start, end, minlen;
+	u64 trimmed = 0;
+
+	/**
+	 * convert byte values to block size of filesystem:
+	 * start:	First Byte to trim
+	 * len:		number of Bytes to trim from start
+	 * minlen:	minimum extent length in Bytes
+	 */
+	start = range->start >> sb->s_blocksize_bits;
+	if (start < 0)
+		start = 0;
+	end = start + (range->len >> sb->s_blocksize_bits) - 1;
+	if (end >= bmp->db_mapsize)
+		end = bmp->db_mapsize - 1;
+	minlen = range->minlen >> sb->s_blocksize_bits;
+	if (minlen <= 0)
+		minlen = 1;
+
+	/**
+	 * we trim all ag's within the range
+	 */
+	agno = BLKTOAG(start, JFS_SBI(ip->i_sb));
+	agno_end = BLKTOAG(end, JFS_SBI(ip->i_sb));
+	while (agno <= agno_end) {
+		trimmed += dbDiscardAG(ip, agno, minlen);
+		agno++;
+	}
+	range->len = trimmed << sb->s_blocksize_bits;
+
+	return 0;
+}
diff --git a/fs/jfs/jfs_discard.h b/fs/jfs/jfs_discard.h
new file mode 100644
index 00000000000..40d1ee6081a
--- /dev/null
+++ b/fs/jfs/jfs_discard.h
@@ -0,0 +1,26 @@
+/*
+ *   Copyright (C) Tino Reichardt, 2012
+ *
+ *   This program is free software;  you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+ *   the GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program;  if not, write to the Free Software
+ *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#ifndef _H_JFS_DISCARD
+#define _H_JFS_DISCARD
+
+struct fstrim_range;
+
+extern void jfs_issue_discard(struct inode *ip, u64 blkno, u64 nblocks);
+extern int jfs_ioc_trim(struct inode *ip, struct fstrim_range *range);
+
+#endif /* _H_JFS_DISCARD */
diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c
index 9cbd11a3f80..9a55f53be5f 100644
--- a/fs/jfs/jfs_dmap.c
+++ b/fs/jfs/jfs_dmap.c
@@ -1,5 +1,6 @@
 /*
  *   Copyright (C) International Business Machines Corp., 2000-2004
+ *   Portions Copyright (C) Tino Reichardt, 2012
  *
  *   This program is free software;  you can redistribute it and/or modify
  *   it under the terms of the GNU General Public License as published by
@@ -25,6 +26,7 @@
 #include "jfs_lock.h"
 #include "jfs_metapage.h"
 #include "jfs_debug.h"
+#include "jfs_discard.h"
 
 /*
  *	SERIALIZATION of the Block Allocation Map.
@@ -104,7 +106,6 @@ static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 static int dbFreeDmap(struct bmap * bmp, struct dmap * dp, s64 blkno,
 		      int nblocks);
 static int dbMaxBud(u8 * cp);
-s64 dbMapFileSizeToMapSize(struct inode *ipbmap);
 static int blkstol2(s64 nb);
 
 static int cntlz(u32 value);
@@ -145,7 +146,6 @@ static const s8 budtab[256] = {
 	2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, -1
 };
 
-
 /*
  * NAME:	dbMount()
  *
@@ -310,7 +310,6 @@ int dbSync(struct inode *ipbmap)
 	return (0);
 }
 
-
 /*
  * NAME:	dbFree()
  *
@@ -337,6 +336,7 @@ int dbFree(struct inode *ip, s64 blkno, s64 nblocks)
 	s64 lblkno, rem;
 	struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap;
 	struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap;
+	struct super_block *sb = ipbmap->i_sb;
 
 	IREAD_LOCK(ipbmap, RDWRLOCK_DMAP);
 
@@ -351,6 +351,13 @@ int dbFree(struct inode *ip, s64 blkno, s64 nblocks)
 		return -EIO;
 	}
 
+	/**
+	 * TRIM the blocks, when mounted with discard option
+	 */
+	if (JFS_SBI(sb)->flag & JFS_DISCARD)
+		if (JFS_SBI(sb)->minblks_trim <= nblocks)
+			jfs_issue_discard(ipbmap, blkno, nblocks);
+
 	/*
 	 * free the blocks a dmap at a time.
 	 */
@@ -1095,7 +1102,6 @@ static int dbExtend(struct inode *ip, s64 blkno, s64 nblocks, s64 addnblocks)
 		/* we were not successful */
 		release_metapage(mp);
 
-
 	return (rc);
 }
 
@@ -1590,6 +1596,118 @@ static int dbAllocAny(struct bmap * bmp, s64 nblocks, int l2nb, s64 * results)
 
 
 /*
+ * NAME:	dbDiscardAG()
+ *
+ * FUNCTION:	attempt to discard (TRIM) all free blocks of specific AG
+ *
+ *		algorithm:
+ *		1) allocate blocks, as large as possible and save them
+ *		   while holding IWRITE_LOCK on ipbmap
+ *		2) trim all these saved block/length values
+ *		3) mark the blocks free again
+ *
+ *		benefit:
+ *		- we work only on one ag at some time, minimizing how long we
+ *		  need to lock ipbmap
+ *		- reading / writing the fs is possible most time, even on
+ *		  trimming
+ *
+ *		downside:
+ *		- we write two times to the dmapctl and dmap pages
+ *		- but for me, this seems the best way, better ideas?
+ *		/TR 2012
+ *
+ * PARAMETERS:
+ *	ip	- pointer to in-core inode
+ *	agno	- ag to trim
+ *	minlen	- minimum value of contiguous blocks
+ *
+ * RETURN VALUES:
+ *	s64	- actual number of blocks trimmed
+ */
+s64 dbDiscardAG(struct inode *ip, int agno, s64 minlen)
+{
+	struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap;
+	struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap;
+	s64 nblocks, blkno;
+	u64 trimmed = 0;
+	int rc, l2nb;
+	struct super_block *sb = ipbmap->i_sb;
+
+	struct range2trim {
+		u64 blkno;
+		u64 nblocks;
+	} *totrim, *tt;
+
+	/* max blkno / nblocks pairs to trim */
+	int count = 0, range_cnt;
+	u64 max_ranges;
+
+	/* prevent others from writing new stuff here, while trimming */
+	IWRITE_LOCK(ipbmap, RDWRLOCK_DMAP);
+
+	nblocks = bmp->db_agfree[agno];
+	max_ranges = nblocks;
+	do_div(max_ranges, minlen);
+	range_cnt = min_t(u64, max_ranges + 1, 32 * 1024);
+	totrim = kmalloc(sizeof(struct range2trim) * range_cnt, GFP_NOFS);
+	if (totrim == NULL) {
+		jfs_error(bmp->db_ipbmap->i_sb,
+			  "dbDiscardAG: no memory for trim array");
+		IWRITE_UNLOCK(ipbmap);
+		return 0;
+	}
+
+	tt = totrim;
+	while (nblocks >= minlen) {
+		l2nb = BLKSTOL2(nblocks);
+
+		/* 0 = okay, -EIO = fatal, -ENOSPC -> try smaller block */
+		rc = dbAllocAG(bmp, agno, nblocks, l2nb, &blkno);
+		if (rc == 0) {
+			tt->blkno = blkno;
+			tt->nblocks = nblocks;
+			tt++; count++;
+
+			/* the whole ag is free, trim now */
+			if (bmp->db_agfree[agno] == 0)
+				break;
+
+			/* give a hint for the next while */
+			nblocks = bmp->db_agfree[agno];
+			continue;
+		} else if (rc == -ENOSPC) {
+			/* search for next smaller log2 block */
+			l2nb = BLKSTOL2(nblocks) - 1;
+			nblocks = 1 << l2nb;
+		} else {
+			/* Trim any already allocated blocks */
+			jfs_error(bmp->db_ipbmap->i_sb,
+				"dbDiscardAG: -EIO");
+			break;
+		}
+
+		/* check, if our trim array is full */
+		if (unlikely(count >= range_cnt - 1))
+			break;
+	}
+	IWRITE_UNLOCK(ipbmap);
+
+	tt->nblocks = 0; /* mark the current end */
+	for (tt = totrim; tt->nblocks != 0; tt++) {
+		/* when mounted with online discard, dbFree() will
+		 * call jfs_issue_discard() itself */
+		if (!(JFS_SBI(sb)->flag & JFS_DISCARD))
+			jfs_issue_discard(ip, tt->blkno, tt->nblocks);
+		dbFree(ip, tt->blkno, tt->nblocks);
+		trimmed += tt->nblocks;
+	}
+	kfree(totrim);
+
+	return trimmed;
+}
+
+/*
  * NAME:	dbFindCtl()
  *
  * FUNCTION:	starting at a specified dmap control page level and block
diff --git a/fs/jfs/jfs_dmap.h b/fs/jfs/jfs_dmap.h
index 6dcb906c55d..562b9a7e431 100644
--- a/fs/jfs/jfs_dmap.h
+++ b/fs/jfs/jfs_dmap.h
@@ -311,4 +311,6 @@ extern int dbAllocBottomUp(struct inode *ip, s64 blkno, s64 nblocks);
 extern int dbExtendFS(struct inode *ipbmap, s64 blkno, s64 nblocks);
 extern void dbFinalizeBmap(struct inode *ipbmap);
 extern s64 dbMapFileSizeToMapSize(struct inode *ipbmap);
+extern s64 dbDiscardAG(struct inode *ip, int agno, s64 minlen);
+
 #endif				/* _H_JFS_DMAP */
diff --git a/fs/jfs/jfs_filsys.h b/fs/jfs/jfs_filsys.h
index b3f5463fbe5..b67d64671bb 100644
--- a/fs/jfs/jfs_filsys.h
+++ b/fs/jfs/jfs_filsys.h
@@ -45,6 +45,9 @@
 /* mount time flag to disable journaling to disk */
 #define JFS_NOINTEGRITY 0x00000040
 
+/* mount time flag to enable TRIM to ssd disks */
+#define JFS_DISCARD     0x00000080
+
 /* commit option */
 #define	JFS_COMMIT	0x00000f00	/* commit option mask */
 #define	JFS_GROUPCOMMIT	0x00000100	/* group (of 1) commit */
diff --git a/fs/jfs/jfs_incore.h b/fs/jfs/jfs_incore.h
index 680605d7bf1..cf47f09e8ac 100644
--- a/fs/jfs/jfs_incore.h
+++ b/fs/jfs/jfs_incore.h
@@ -195,6 +195,7 @@ struct jfs_sb_info {
 	kuid_t		uid;		/* uid to override on-disk uid */
 	kgid_t		gid;		/* gid to override on-disk gid */
 	uint		umask;		/* umask to override on-disk umask */
+	uint		minblks_trim;	/* minimum blocks, for online trim */
 };
 
 /* jfs_sb_info commit_state */
diff --git a/fs/jfs/jfs_txnmgr.c b/fs/jfs/jfs_txnmgr.c
index bb8b661bcc5..5fcc02eaa64 100644
--- a/fs/jfs/jfs_txnmgr.c
+++ b/fs/jfs/jfs_txnmgr.c
@@ -2977,12 +2977,9 @@ int jfs_sync(void *arg)
 				 * put back on the anon_list.
 				 */
 
-				/* Take off anon_list */
-				list_del(&jfs_ip->anon_inode_list);
-
-				/* Put on anon_list2 */
-				list_add(&jfs_ip->anon_inode_list,
-					 &TxAnchor.anon_list2);
+				/* Move from anon_list to anon_list2 */
+				list_move(&jfs_ip->anon_inode_list,
+					  &TxAnchor.anon_list2);
 
 				TXN_UNLOCK();
 				iput(ip);
diff --git a/fs/jfs/super.c b/fs/jfs/super.c
index 706692f2403..1a543be09c7 100644
--- a/fs/jfs/super.c
+++ b/fs/jfs/super.c
@@ -33,6 +33,7 @@
 #include <linux/slab.h>
 #include <asm/uaccess.h>
 #include <linux/seq_file.h>
+#include <linux/blkdev.h>
 
 #include "jfs_incore.h"
 #include "jfs_filsys.h"
@@ -100,7 +101,7 @@ void jfs_error(struct super_block *sb, const char * function, ...)
 	vsnprintf(error_buf, sizeof(error_buf), function, args);
 	va_end(args);
 
-	printk(KERN_ERR "ERROR: (device %s): %s\n", sb->s_id, error_buf);
+	pr_err("ERROR: (device %s): %s\n", sb->s_id, error_buf);
 
 	jfs_handle_error(sb);
 }
@@ -197,7 +198,8 @@ static void jfs_put_super(struct super_block *sb)
 enum {
 	Opt_integrity, Opt_nointegrity, Opt_iocharset, Opt_resize,
 	Opt_resize_nosize, Opt_errors, Opt_ignore, Opt_err, Opt_quota,
-	Opt_usrquota, Opt_grpquota, Opt_uid, Opt_gid, Opt_umask
+	Opt_usrquota, Opt_grpquota, Opt_uid, Opt_gid, Opt_umask,
+	Opt_discard, Opt_nodiscard, Opt_discard_minblk
 };
 
 static const match_table_t tokens = {
@@ -214,6 +216,9 @@ static const match_table_t tokens = {
 	{Opt_uid, "uid=%u"},
 	{Opt_gid, "gid=%u"},
 	{Opt_umask, "umask=%u"},
+	{Opt_discard, "discard"},
+	{Opt_nodiscard, "nodiscard"},
+	{Opt_discard_minblk, "discard=%u"},
 	{Opt_err, NULL}
 };
 
@@ -255,8 +260,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 			else {
 				nls_map = load_nls(args[0].from);
 				if (!nls_map) {
-					printk(KERN_ERR
-					       "JFS: charset not found\n");
+					pr_err("JFS: charset not found\n");
 					goto cleanup;
 				}
 			}
@@ -272,8 +276,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 			*newLVSize = sb->s_bdev->bd_inode->i_size >>
 				sb->s_blocksize_bits;
 			if (*newLVSize == 0)
-				printk(KERN_ERR
-				       "JFS: Cannot determine volume size\n");
+				pr_err("JFS: Cannot determine volume size\n");
 			break;
 		}
 		case Opt_errors:
@@ -294,8 +297,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 				*flag &= ~JFS_ERR_REMOUNT_RO;
 				*flag |= JFS_ERR_PANIC;
 			} else {
-				printk(KERN_ERR
-				       "JFS: %s is an invalid error handler\n",
+				pr_err("JFS: %s is an invalid error handler\n",
 				       errors);
 				goto cleanup;
 			}
@@ -314,8 +316,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 		case Opt_usrquota:
 		case Opt_grpquota:
 		case Opt_quota:
-			printk(KERN_ERR
-			       "JFS: quota operations not supported\n");
+			pr_err("JFS: quota operations not supported\n");
 			break;
 #endif
 		case Opt_uid:
@@ -327,6 +328,7 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 				goto cleanup;
 			break;
 		}
+
 		case Opt_gid:
 		{
 			char *gid = args[0].from;
@@ -336,17 +338,54 @@ static int parse_options(char *options, struct super_block *sb, s64 *newLVSize,
 				goto cleanup;
 			break;
 		}
+
 		case Opt_umask:
 		{
 			char *umask = args[0].from;
 			sbi->umask = simple_strtoul(umask, &umask, 8);
 			if (sbi->umask & ~0777) {
-				printk(KERN_ERR
-				       "JFS: Invalid value of umask\n");
+				pr_err("JFS: Invalid value of umask\n");
 				goto cleanup;
 			}
 			break;
 		}
+
+		case Opt_discard:
+		{
+			struct request_queue *q = bdev_get_queue(sb->s_bdev);
+			/* if set to 1, even copying files will cause
+			 * trimming :O
+			 * -> user has more control over the online trimming
+			 */
+			sbi->minblks_trim = 64;
+			if (blk_queue_discard(q)) {
+				*flag |= JFS_DISCARD;
+			} else {
+				pr_err("JFS: discard option " \
+					"not supported on device\n");
+			}
+			break;
+		}
+
+		case Opt_nodiscard:
+			*flag &= ~JFS_DISCARD;
+			break;
+
+		case Opt_discard_minblk:
+		{
+			struct request_queue *q = bdev_get_queue(sb->s_bdev);
+			char *minblks_trim = args[0].from;
+			if (blk_queue_discard(q)) {
+				*flag |= JFS_DISCARD;
+				sbi->minblks_trim = simple_strtoull(
+					minblks_trim, &minblks_trim, 0);
+			} else {
+				pr_err("JFS: discard option " \
+					"not supported on device\n");
+			}
+			break;
+		}
+
 		default:
 			printk("jfs: Unrecognized mount option \"%s\" "
 					" or missing value\n", p);
@@ -380,8 +419,8 @@ static int jfs_remount(struct super_block *sb, int *flags, char *data)
 
 	if (newLVSize) {
 		if (sb->s_flags & MS_RDONLY) {
-			printk(KERN_ERR
-		  "JFS: resize requires volume to be mounted read-write\n");
+			pr_err("JFS: resize requires volume" \
+				" to be mounted read-write\n");
 			return -EROFS;
 		}
 		rc = jfs_extendfs(sb, newLVSize, 0);
@@ -465,7 +504,7 @@ static int jfs_fill_super(struct super_block *sb, void *data, int silent)
 #endif
 
 	if (newLVSize) {
-		printk(KERN_ERR "resize option for remount only\n");
+		pr_err("resize option for remount only\n");
 		goto out_kfree;
 	}
 
@@ -633,6 +672,8 @@ static int jfs_show_options(struct seq_file *seq, struct dentry *root)
 		seq_printf(seq, ",umask=%03o", sbi->umask);
 	if (sbi->flag & JFS_NOINTEGRITY)
 		seq_puts(seq, ",nointegrity");
+	if (sbi->flag & JFS_DISCARD)
+		seq_printf(seq, ",discard=%u", sbi->minblks_trim);
 	if (sbi->nls_tab)
 		seq_printf(seq, ",iocharset=%s", sbi->nls_tab->charset);
 	if (sbi->flag & JFS_ERR_CONTINUE)
@@ -911,6 +952,12 @@ static void __exit exit_jfs_fs(void)
 	jfs_proc_clean();
 #endif
 	unregister_filesystem(&jfs_fs_type);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(jfs_inode_cachep);
 }
 
diff --git a/fs/locks.c b/fs/locks.c
index 7e81bfc7516..abc7dc6c490 100644
--- a/fs/locks.c
+++ b/fs/locks.c
@@ -1625,15 +1625,13 @@ EXPORT_SYMBOL(flock_lock_file_wait);
  */
 SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd)
 {
-	struct file *filp;
-	int fput_needed;
+	struct fd f = fdget(fd);
 	struct file_lock *lock;
 	int can_sleep, unlock;
 	int error;
 
 	error = -EBADF;
-	filp = fget_light(fd, &fput_needed);
-	if (!filp)
+	if (!f.file)
 		goto out;
 
 	can_sleep = !(cmd & LOCK_NB);
@@ -1641,31 +1639,31 @@ SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd)
 	unlock = (cmd == LOCK_UN);
 
 	if (!unlock && !(cmd & LOCK_MAND) &&
-	    !(filp->f_mode & (FMODE_READ|FMODE_WRITE)))
+	    !(f.file->f_mode & (FMODE_READ|FMODE_WRITE)))
 		goto out_putf;
 
-	error = flock_make_lock(filp, &lock, cmd);
+	error = flock_make_lock(f.file, &lock, cmd);
 	if (error)
 		goto out_putf;
 	if (can_sleep)
 		lock->fl_flags |= FL_SLEEP;
 
-	error = security_file_lock(filp, lock->fl_type);
+	error = security_file_lock(f.file, lock->fl_type);
 	if (error)
 		goto out_free;
 
-	if (filp->f_op && filp->f_op->flock)
-		error = filp->f_op->flock(filp,
+	if (f.file->f_op && f.file->f_op->flock)
+		error = f.file->f_op->flock(f.file,
 					  (can_sleep) ? F_SETLKW : F_SETLK,
 					  lock);
 	else
-		error = flock_lock_file_wait(filp, lock);
+		error = flock_lock_file_wait(f.file, lock);
 
  out_free:
 	locks_free_lock(lock);
 
  out_putf:
-	fput_light(filp, fput_needed);
+	fdput(f);
  out:
 	return error;
 }
diff --git a/fs/logfs/inode.c b/fs/logfs/inode.c
index bda39085309..adb90116d36 100644
--- a/fs/logfs/inode.c
+++ b/fs/logfs/inode.c
@@ -417,5 +417,10 @@ int logfs_init_inode_cache(void)
 
 void logfs_destroy_inode_cache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(logfs_inode_cache);
 }
diff --git a/fs/minix/inode.c b/fs/minix/inode.c
index d0e42c67892..4fc5f8ab1c4 100644
--- a/fs/minix/inode.c
+++ b/fs/minix/inode.c
@@ -100,6 +100,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(minix_inode_cachep);
 }
 
diff --git a/fs/namei.c b/fs/namei.c
index a856e7f7b6e..aa30d19e9ed 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -1797,8 +1797,6 @@ static int path_init(int dfd, const char *name, unsigned int flags,
 		     struct nameidata *nd, struct file **fp)
 {
 	int retval = 0;
-	int fput_needed;
-	struct file *file;
 
 	nd->last_type = LAST_ROOT; /* if there are only slashes... */
 	nd->flags = flags | LOOKUP_JUMPED;
@@ -1850,44 +1848,41 @@ static int path_init(int dfd, const char *name, unsigned int flags,
 			get_fs_pwd(current->fs, &nd->path);
 		}
 	} else {
+		struct fd f = fdget_raw(dfd);
 		struct dentry *dentry;
 
-		file = fget_raw_light(dfd, &fput_needed);
-		retval = -EBADF;
-		if (!file)
-			goto out_fail;
+		if (!f.file)
+			return -EBADF;
 
-		dentry = file->f_path.dentry;
+		dentry = f.file->f_path.dentry;
 
 		if (*name) {
-			retval = -ENOTDIR;
-			if (!S_ISDIR(dentry->d_inode->i_mode))
-				goto fput_fail;
+			if (!S_ISDIR(dentry->d_inode->i_mode)) {
+				fdput(f);
+				return -ENOTDIR;
+			}
 
 			retval = inode_permission(dentry->d_inode, MAY_EXEC);
-			if (retval)
-				goto fput_fail;
+			if (retval) {
+				fdput(f);
+				return retval;
+			}
 		}
 
-		nd->path = file->f_path;
+		nd->path = f.file->f_path;
 		if (flags & LOOKUP_RCU) {
-			if (fput_needed)
-				*fp = file;
+			if (f.need_put)
+				*fp = f.file;
 			nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
 			lock_rcu_walk();
 		} else {
-			path_get(&file->f_path);
-			fput_light(file, fput_needed);
+			path_get(&nd->path);
+			fdput(f);
 		}
 	}
 
 	nd->inode = nd->path.dentry->d_inode;
 	return 0;
-
-fput_fail:
-	fput_light(file, fput_needed);
-out_fail:
-	return retval;
 }
 
 static inline int lookup_last(struct nameidata *nd, struct path *path)
@@ -3971,7 +3966,7 @@ EXPORT_SYMBOL(user_path_at);
 EXPORT_SYMBOL(follow_down_one);
 EXPORT_SYMBOL(follow_down);
 EXPORT_SYMBOL(follow_up);
-EXPORT_SYMBOL(get_write_access); /* binfmt_aout */
+EXPORT_SYMBOL(get_write_access); /* nfsd */
 EXPORT_SYMBOL(getname);
 EXPORT_SYMBOL(lock_rename);
 EXPORT_SYMBOL(lookup_one_len);
diff --git a/fs/ncpfs/inode.c b/fs/ncpfs/inode.c
index eaa74323663..d7e9fe77188 100644
--- a/fs/ncpfs/inode.c
+++ b/fs/ncpfs/inode.c
@@ -89,6 +89,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ncp_inode_cachep);
 }
 
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 9b47610338f..e4c716d374a 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1571,6 +1571,11 @@ static int __init nfs_init_inodecache(void)
 
 static void nfs_destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(nfs_inode_cachep);
 }
 
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index cc894eda385..48a1bad3733 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -2837,8 +2837,7 @@ static int nfs4_setlease(struct nfs4_delegation *dp, int flag)
 		return -ENOMEM;
 	}
 	fp->fi_lease = fl;
-	fp->fi_deleg_file = fl->fl_file;
-	get_file(fp->fi_deleg_file);
+	fp->fi_deleg_file = get_file(fl->fl_file);
 	atomic_set(&fp->fi_delegees, 1);
 	list_add(&dp->dl_perfile, &fp->fi_delegations);
 	return 0;
diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c
index 6a10812711c..3c991dc84f2 100644
--- a/fs/nilfs2/super.c
+++ b/fs/nilfs2/super.c
@@ -1382,6 +1382,12 @@ static void nilfs_segbuf_init_once(void *obj)
 
 static void nilfs_destroy_cachep(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
+
 	if (nilfs_inode_cachep)
 		kmem_cache_destroy(nilfs_inode_cachep);
 	if (nilfs_transaction_cachep)
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
index d4380366973..721d692fa8d 100644
--- a/fs/notify/fanotify/fanotify_user.c
+++ b/fs/notify/fanotify/fanotify_user.c
@@ -58,7 +58,9 @@ static struct fsnotify_event *get_one_event(struct fsnotify_group *group,
 	return fsnotify_remove_notify_event(group);
 }
 
-static int create_fd(struct fsnotify_group *group, struct fsnotify_event *event)
+static int create_fd(struct fsnotify_group *group,
+			struct fsnotify_event *event,
+			struct file **file)
 {
 	int client_fd;
 	struct file *new_file;
@@ -98,7 +100,7 @@ static int create_fd(struct fsnotify_group *group, struct fsnotify_event *event)
 		put_unused_fd(client_fd);
 		client_fd = PTR_ERR(new_file);
 	} else {
-		fd_install(client_fd, new_file);
+		*file = new_file;
 	}
 
 	return client_fd;
@@ -106,13 +108,15 @@ static int create_fd(struct fsnotify_group *group, struct fsnotify_event *event)
 
 static int fill_event_metadata(struct fsnotify_group *group,
 				   struct fanotify_event_metadata *metadata,
-				   struct fsnotify_event *event)
+				   struct fsnotify_event *event,
+				   struct file **file)
 {
 	int ret = 0;
 
 	pr_debug("%s: group=%p metadata=%p event=%p\n", __func__,
 		 group, metadata, event);
 
+	*file = NULL;
 	metadata->event_len = FAN_EVENT_METADATA_LEN;
 	metadata->metadata_len = FAN_EVENT_METADATA_LEN;
 	metadata->vers = FANOTIFY_METADATA_VERSION;
@@ -121,7 +125,7 @@ static int fill_event_metadata(struct fsnotify_group *group,
 	if (unlikely(event->mask & FAN_Q_OVERFLOW))
 		metadata->fd = FAN_NOFD;
 	else {
-		metadata->fd = create_fd(group, event);
+		metadata->fd = create_fd(group, event, file);
 		if (metadata->fd < 0)
 			ret = metadata->fd;
 	}
@@ -220,25 +224,6 @@ static int prepare_for_access_response(struct fsnotify_group *group,
 	return 0;
 }
 
-static void remove_access_response(struct fsnotify_group *group,
-				   struct fsnotify_event *event,
-				   __s32 fd)
-{
-	struct fanotify_response_event *re;
-
-	if (!(event->mask & FAN_ALL_PERM_EVENTS))
-		return;
-
-	re = dequeue_re(group, fd);
-	if (!re)
-		return;
-
-	BUG_ON(re->event != event);
-
-	kmem_cache_free(fanotify_response_event_cache, re);
-
-	return;
-}
 #else
 static int prepare_for_access_response(struct fsnotify_group *group,
 				       struct fsnotify_event *event,
@@ -247,12 +232,6 @@ static int prepare_for_access_response(struct fsnotify_group *group,
 	return 0;
 }
 
-static void remove_access_response(struct fsnotify_group *group,
-				   struct fsnotify_event *event,
-				   __s32 fd)
-{
-	return;
-}
 #endif
 
 static ssize_t copy_event_to_user(struct fsnotify_group *group,
@@ -260,31 +239,33 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group,
 				  char __user *buf)
 {
 	struct fanotify_event_metadata fanotify_event_metadata;
+	struct file *f;
 	int fd, ret;
 
 	pr_debug("%s: group=%p event=%p\n", __func__, group, event);
 
-	ret = fill_event_metadata(group, &fanotify_event_metadata, event);
+	ret = fill_event_metadata(group, &fanotify_event_metadata, event, &f);
 	if (ret < 0)
 		goto out;
 
 	fd = fanotify_event_metadata.fd;
-	ret = prepare_for_access_response(group, event, fd);
-	if (ret)
-		goto out_close_fd;
-
 	ret = -EFAULT;
 	if (copy_to_user(buf, &fanotify_event_metadata,
 			 fanotify_event_metadata.event_len))
-		goto out_kill_access_response;
+		goto out_close_fd;
 
+	ret = prepare_for_access_response(group, event, fd);
+	if (ret)
+		goto out_close_fd;
+
+	fd_install(fd, f);
 	return fanotify_event_metadata.event_len;
 
-out_kill_access_response:
-	remove_access_response(group, event, fd);
 out_close_fd:
-	if (fd != FAN_NOFD)
-		sys_close(fd);
+	if (fd != FAN_NOFD) {
+		put_unused_fd(fd);
+		fput(f);
+	}
 out:
 #ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
 	if (event->mask & FAN_ALL_PERM_EVENTS) {
@@ -470,24 +451,22 @@ static int fanotify_find_path(int dfd, const char __user *filename,
 		 dfd, filename, flags);
 
 	if (filename == NULL) {
-		struct file *file;
-		int fput_needed;
+		struct fd f = fdget(dfd);
 
 		ret = -EBADF;
-		file = fget_light(dfd, &fput_needed);
-		if (!file)
+		if (!f.file)
 			goto out;
 
 		ret = -ENOTDIR;
 		if ((flags & FAN_MARK_ONLYDIR) &&
-		    !(S_ISDIR(file->f_path.dentry->d_inode->i_mode))) {
-			fput_light(file, fput_needed);
+		    !(S_ISDIR(f.file->f_path.dentry->d_inode->i_mode))) {
+			fdput(f);
 			goto out;
 		}
 
-		*path = file->f_path;
+		*path = f.file->f_path;
 		path_get(path);
-		fput_light(file, fput_needed);
+		fdput(f);
 	} else {
 		unsigned int lookup_flags = 0;
 
@@ -767,9 +746,9 @@ SYSCALL_DEFINE(fanotify_mark)(int fanotify_fd, unsigned int flags,
 	struct inode *inode = NULL;
 	struct vfsmount *mnt = NULL;
 	struct fsnotify_group *group;
-	struct file *filp;
+	struct fd f;
 	struct path path;
-	int ret, fput_needed;
+	int ret;
 
 	pr_debug("%s: fanotify_fd=%d flags=%x dfd=%d pathname=%p mask=%llx\n",
 		 __func__, fanotify_fd, flags, dfd, pathname, mask);
@@ -803,15 +782,15 @@ SYSCALL_DEFINE(fanotify_mark)(int fanotify_fd, unsigned int flags,
 #endif
 		return -EINVAL;
 
-	filp = fget_light(fanotify_fd, &fput_needed);
-	if (unlikely(!filp))
+	f = fdget(fanotify_fd);
+	if (unlikely(!f.file))
 		return -EBADF;
 
 	/* verify that this is indeed an fanotify instance */
 	ret = -EINVAL;
-	if (unlikely(filp->f_op != &fanotify_fops))
+	if (unlikely(f.file->f_op != &fanotify_fops))
 		goto fput_and_out;
-	group = filp->private_data;
+	group = f.file->private_data;
 
 	/*
 	 * group->priority == FS_PRIO_0 == FAN_CLASS_NOTIF.  These are not
@@ -858,7 +837,7 @@ SYSCALL_DEFINE(fanotify_mark)(int fanotify_fd, unsigned int flags,
 
 	path_put(&path);
 fput_and_out:
-	fput_light(filp, fput_needed);
+	fdput(f);
 	return ret;
 }
 
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c
index 8445fbc8985..c311dda054a 100644
--- a/fs/notify/inotify/inotify_user.c
+++ b/fs/notify/inotify/inotify_user.c
@@ -757,16 +757,16 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname,
 	struct fsnotify_group *group;
 	struct inode *inode;
 	struct path path;
-	struct file *filp;
-	int ret, fput_needed;
+	struct fd f;
+	int ret;
 	unsigned flags = 0;
 
-	filp = fget_light(fd, &fput_needed);
-	if (unlikely(!filp))
+	f = fdget(fd);
+	if (unlikely(!f.file))
 		return -EBADF;
 
 	/* verify that this is indeed an inotify instance */
-	if (unlikely(filp->f_op != &inotify_fops)) {
+	if (unlikely(f.file->f_op != &inotify_fops)) {
 		ret = -EINVAL;
 		goto fput_and_out;
 	}
@@ -782,13 +782,13 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname,
 
 	/* inode held in place by reference to path; group by fget on fd */
 	inode = path.dentry->d_inode;
-	group = filp->private_data;
+	group = f.file->private_data;
 
 	/* create/update an inode mark */
 	ret = inotify_update_watch(group, inode, mask);
 	path_put(&path);
 fput_and_out:
-	fput_light(filp, fput_needed);
+	fdput(f);
 	return ret;
 }
 
@@ -796,19 +796,19 @@ SYSCALL_DEFINE2(inotify_rm_watch, int, fd, __s32, wd)
 {
 	struct fsnotify_group *group;
 	struct inotify_inode_mark *i_mark;
-	struct file *filp;
-	int ret = 0, fput_needed;
+	struct fd f;
+	int ret = 0;
 
-	filp = fget_light(fd, &fput_needed);
-	if (unlikely(!filp))
+	f = fdget(fd);
+	if (unlikely(!f.file))
 		return -EBADF;
 
 	/* verify that this is indeed an inotify instance */
 	ret = -EINVAL;
-	if (unlikely(filp->f_op != &inotify_fops))
+	if (unlikely(f.file->f_op != &inotify_fops))
 		goto out;
 
-	group = filp->private_data;
+	group = f.file->private_data;
 
 	ret = -EINVAL;
 	i_mark = inotify_idr_find(group, wd);
@@ -823,7 +823,7 @@ SYSCALL_DEFINE2(inotify_rm_watch, int, fd, __s32, wd)
 	fsnotify_put_mark(&i_mark->fsn_mark);
 
 out:
-	fput_light(filp, fput_needed);
+	fdput(f);
 	return ret;
 }
 
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
index da01c165067..4a8289f8b16 100644
--- a/fs/ntfs/super.c
+++ b/fs/ntfs/super.c
@@ -3193,6 +3193,12 @@ static void __exit exit_ntfs_fs(void)
 	ntfs_debug("Unregistering NTFS driver.");
 
 	unregister_filesystem(&ntfs_fs_type);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ntfs_big_inode_cache);
 	kmem_cache_destroy(ntfs_inode_cache);
 	kmem_cache_destroy(ntfs_name_cache);
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index a4e855e3690..f7c648d7d6b 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -1746,8 +1746,8 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 	long fd;
 	int sectsize;
 	char *p = (char *)page;
-	struct file *filp = NULL;
-	struct inode *inode = NULL;
+	struct fd f;
+	struct inode *inode;
 	ssize_t ret = -EINVAL;
 	int live_threshold;
 
@@ -1766,26 +1766,26 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 	if (fd < 0 || fd >= INT_MAX)
 		goto out;
 
-	filp = fget(fd);
-	if (filp == NULL)
+	f = fdget(fd);
+	if (f.file == NULL)
 		goto out;
 
 	if (reg->hr_blocks == 0 || reg->hr_start_block == 0 ||
 	    reg->hr_block_bytes == 0)
-		goto out;
+		goto out2;
 
-	inode = igrab(filp->f_mapping->host);
+	inode = igrab(f.file->f_mapping->host);
 	if (inode == NULL)
-		goto out;
+		goto out2;
 
 	if (!S_ISBLK(inode->i_mode))
-		goto out;
+		goto out3;
 
-	reg->hr_bdev = I_BDEV(filp->f_mapping->host);
+	reg->hr_bdev = I_BDEV(f.file->f_mapping->host);
 	ret = blkdev_get(reg->hr_bdev, FMODE_WRITE | FMODE_READ, NULL);
 	if (ret) {
 		reg->hr_bdev = NULL;
-		goto out;
+		goto out3;
 	}
 	inode = NULL;
 
@@ -1797,7 +1797,7 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 		     "blocksize %u incorrect for device, expected %d",
 		     reg->hr_block_bytes, sectsize);
 		ret = -EINVAL;
-		goto out;
+		goto out3;
 	}
 
 	o2hb_init_region_params(reg);
@@ -1811,13 +1811,13 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 	ret = o2hb_map_slot_data(reg);
 	if (ret) {
 		mlog_errno(ret);
-		goto out;
+		goto out3;
 	}
 
 	ret = o2hb_populate_slot_data(reg);
 	if (ret) {
 		mlog_errno(ret);
-		goto out;
+		goto out3;
 	}
 
 	INIT_DELAYED_WORK(&reg->hr_write_timeout_work, o2hb_write_timeout);
@@ -1847,7 +1847,7 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 	if (IS_ERR(hb_task)) {
 		ret = PTR_ERR(hb_task);
 		mlog_errno(ret);
-		goto out;
+		goto out3;
 	}
 
 	spin_lock(&o2hb_live_lock);
@@ -1863,7 +1863,7 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 
 	if (reg->hr_aborted_start) {
 		ret = -EIO;
-		goto out;
+		goto out3;
 	}
 
 	/* Ok, we were woken.  Make sure it wasn't by drop_item() */
@@ -1882,11 +1882,11 @@ static ssize_t o2hb_region_dev_write(struct o2hb_region *reg,
 		printk(KERN_NOTICE "o2hb: Heartbeat started on region %s (%s)\n",
 		       config_item_name(&reg->hr_item), reg->hr_dev_name);
 
+out3:
+	iput(inode);
+out2:
+	fdput(f);
 out:
-	if (filp)
-		fput(filp);
-	if (inode)
-		iput(inode);
 	if (ret < 0) {
 		if (reg->hr_bdev) {
 			blkdev_put(reg->hr_bdev, FMODE_READ|FMODE_WRITE);
diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c
index 83b6f98e066..16b712d260d 100644
--- a/fs/ocfs2/dlmfs/dlmfs.c
+++ b/fs/ocfs2/dlmfs/dlmfs.c
@@ -691,6 +691,11 @@ static void __exit exit_dlmfs_fs(void)
 	flush_workqueue(user_dlm_worker);
 	destroy_workqueue(user_dlm_worker);
 
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(dlmfs_inode_cache);
 
 	bdi_destroy(&dlmfs_backing_dev_info);
diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
index 68f4541c2db..0e91ec22a94 100644
--- a/fs/ocfs2/super.c
+++ b/fs/ocfs2/super.c
@@ -1818,6 +1818,11 @@ static int ocfs2_initialize_mem_caches(void)
 
 static void ocfs2_free_mem_caches(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	if (ocfs2_inode_cachep)
 		kmem_cache_destroy(ocfs2_inode_cachep);
 	ocfs2_inode_cachep = NULL;
diff --git a/fs/omfs/file.c b/fs/omfs/file.c
index 2c6d95257a4..77e3cb2962b 100644
--- a/fs/omfs/file.c
+++ b/fs/omfs/file.c
@@ -146,8 +146,7 @@ static int omfs_grow_extent(struct inode *inode, struct omfs_extent *oe,
 			be64_to_cpu(entry->e_blocks);
 
 		if (omfs_allocate_block(inode->i_sb, new_block)) {
-			entry->e_blocks =
-				cpu_to_be64(be64_to_cpu(entry->e_blocks) + 1);
+			be64_add_cpu(&entry->e_blocks, 1);
 			terminator->e_blocks = ~(cpu_to_be64(
 				be64_to_cpu(~terminator->e_blocks) + 1));
 			goto out;
@@ -177,7 +176,7 @@ static int omfs_grow_extent(struct inode *inode, struct omfs_extent *oe,
 		be64_to_cpu(~terminator->e_blocks) + (u64) new_count));
 
 	/* write in new entry */
-	oe->e_extent_count = cpu_to_be32(1 + be32_to_cpu(oe->e_extent_count));
+	be32_add_cpu(&oe->e_extent_count, 1);
 
 out:
 	*ret_block = new_block;
diff --git a/fs/open.c b/fs/open.c
index b0bae3a4182..44da0feeca2 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -132,27 +132,27 @@ SYSCALL_DEFINE2(truncate, const char __user *, path, long, length)
 
 static long do_sys_ftruncate(unsigned int fd, loff_t length, int small)
 {
-	struct inode * inode;
+	struct inode *inode;
 	struct dentry *dentry;
-	struct file * file;
+	struct fd f;
 	int error;
 
 	error = -EINVAL;
 	if (length < 0)
 		goto out;
 	error = -EBADF;
-	file = fget(fd);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		goto out;
 
 	/* explicitly opened as large or we are on 64-bit box */
-	if (file->f_flags & O_LARGEFILE)
+	if (f.file->f_flags & O_LARGEFILE)
 		small = 0;
 
-	dentry = file->f_path.dentry;
+	dentry = f.file->f_path.dentry;
 	inode = dentry->d_inode;
 	error = -EINVAL;
-	if (!S_ISREG(inode->i_mode) || !(file->f_mode & FMODE_WRITE))
+	if (!S_ISREG(inode->i_mode) || !(f.file->f_mode & FMODE_WRITE))
 		goto out_putf;
 
 	error = -EINVAL;
@@ -165,14 +165,14 @@ static long do_sys_ftruncate(unsigned int fd, loff_t length, int small)
 		goto out_putf;
 
 	sb_start_write(inode->i_sb);
-	error = locks_verify_truncate(inode, file, length);
+	error = locks_verify_truncate(inode, f.file, length);
 	if (!error)
-		error = security_path_truncate(&file->f_path);
+		error = security_path_truncate(&f.file->f_path);
 	if (!error)
-		error = do_truncate(dentry, length, ATTR_MTIME|ATTR_CTIME, file);
+		error = do_truncate(dentry, length, ATTR_MTIME|ATTR_CTIME, f.file);
 	sb_end_write(inode->i_sb);
 out_putf:
-	fput(file);
+	fdput(f);
 out:
 	return error;
 }
@@ -276,15 +276,13 @@ int do_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 
 SYSCALL_DEFINE(fallocate)(int fd, int mode, loff_t offset, loff_t len)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	int error = -EBADF;
 
-	file = fget(fd);
-	if (file) {
-		error = do_fallocate(file, mode, offset, len);
-		fput(file);
+	if (f.file) {
+		error = do_fallocate(f.file, mode, offset, len);
+		fdput(f);
 	}
-
 	return error;
 }
 
@@ -400,16 +398,15 @@ out:
 
 SYSCALL_DEFINE1(fchdir, unsigned int, fd)
 {
-	struct file *file;
+	struct fd f = fdget_raw(fd);
 	struct inode *inode;
-	int error, fput_needed;
+	int error = -EBADF;
 
 	error = -EBADF;
-	file = fget_raw_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		goto out;
 
-	inode = file->f_path.dentry->d_inode;
+	inode = f.file->f_path.dentry->d_inode;
 
 	error = -ENOTDIR;
 	if (!S_ISDIR(inode->i_mode))
@@ -417,9 +414,9 @@ SYSCALL_DEFINE1(fchdir, unsigned int, fd)
 
 	error = inode_permission(inode, MAY_EXEC | MAY_CHDIR);
 	if (!error)
-		set_fs_pwd(current->fs, &file->f_path);
+		set_fs_pwd(current->fs, &f.file->f_path);
 out_putf:
-	fput_light(file, fput_needed);
+	fdput(f);
 out:
 	return error;
 }
@@ -582,23 +579,20 @@ SYSCALL_DEFINE3(lchown, const char __user *, filename, uid_t, user, gid_t, group
 
 SYSCALL_DEFINE3(fchown, unsigned int, fd, uid_t, user, gid_t, group)
 {
-	struct file * file;
+	struct fd f = fdget(fd);
 	int error = -EBADF;
-	struct dentry * dentry;
 
-	file = fget(fd);
-	if (!file)
+	if (!f.file)
 		goto out;
 
-	error = mnt_want_write_file(file);
+	error = mnt_want_write_file(f.file);
 	if (error)
 		goto out_fput;
-	dentry = file->f_path.dentry;
-	audit_inode(NULL, dentry);
-	error = chown_common(&file->f_path, user, group);
-	mnt_drop_write_file(file);
+	audit_inode(NULL, f.file->f_path.dentry);
+	error = chown_common(&f.file->f_path, user, group);
+	mnt_drop_write_file(f.file);
 out_fput:
-	fput(file);
+	fdput(f);
 out:
 	return error;
 }
@@ -803,50 +797,6 @@ struct file *dentry_open(const struct path *path, int flags,
 }
 EXPORT_SYMBOL(dentry_open);
 
-static void __put_unused_fd(struct files_struct *files, unsigned int fd)
-{
-	struct fdtable *fdt = files_fdtable(files);
-	__clear_open_fd(fd, fdt);
-	if (fd < files->next_fd)
-		files->next_fd = fd;
-}
-
-void put_unused_fd(unsigned int fd)
-{
-	struct files_struct *files = current->files;
-	spin_lock(&files->file_lock);
-	__put_unused_fd(files, fd);
-	spin_unlock(&files->file_lock);
-}
-
-EXPORT_SYMBOL(put_unused_fd);
-
-/*
- * Install a file pointer in the fd array.
- *
- * The VFS is full of places where we drop the files lock between
- * setting the open_fds bitmap and installing the file in the file
- * array.  At any such point, we are vulnerable to a dup2() race
- * installing a file in the array before us.  We need to detect this and
- * fput() the struct file we are about to overwrite in this case.
- *
- * It should never happen - if we allow dup2() do it, _really_ bad things
- * will follow.
- */
-
-void fd_install(unsigned int fd, struct file *file)
-{
-	struct files_struct *files = current->files;
-	struct fdtable *fdt;
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	BUG_ON(fdt->fd[fd] != NULL);
-	rcu_assign_pointer(fdt->fd[fd], file);
-	spin_unlock(&files->file_lock);
-}
-
-EXPORT_SYMBOL(fd_install);
-
 static inline int build_open_flags(int flags, umode_t mode, struct open_flags *op)
 {
 	int lookup_flags = 0;
@@ -858,7 +808,7 @@ static inline int build_open_flags(int flags, umode_t mode, struct open_flags *o
 		op->mode = 0;
 
 	/* Must never be set by userspace */
-	flags &= ~FMODE_NONOTIFY;
+	flags &= ~FMODE_NONOTIFY & ~O_CLOEXEC;
 
 	/*
 	 * O_SYNC is implemented as __O_SYNC|O_DSYNC.  As many places only
@@ -1038,23 +988,7 @@ EXPORT_SYMBOL(filp_close);
  */
 SYSCALL_DEFINE1(close, unsigned int, fd)
 {
-	struct file * filp;
-	struct files_struct *files = current->files;
-	struct fdtable *fdt;
-	int retval;
-
-	spin_lock(&files->file_lock);
-	fdt = files_fdtable(files);
-	if (fd >= fdt->max_fds)
-		goto out_unlock;
-	filp = fdt->fd[fd];
-	if (!filp)
-		goto out_unlock;
-	rcu_assign_pointer(fdt->fd[fd], NULL);
-	__clear_close_on_exec(fd, fdt);
-	__put_unused_fd(files, fd);
-	spin_unlock(&files->file_lock);
-	retval = filp_close(filp, files);
+	int retval = __close_fd(current->files, fd);
 
 	/* can't restart close syscall because file table entry was cleared */
 	if (unlikely(retval == -ERESTARTSYS ||
@@ -1064,10 +998,6 @@ SYSCALL_DEFINE1(close, unsigned int, fd)
 		retval = -EINTR;
 
 	return retval;
-
-out_unlock:
-	spin_unlock(&files->file_lock);
-	return -EBADF;
 }
 EXPORT_SYMBOL(sys_close);
 
diff --git a/fs/openpromfs/inode.c b/fs/openpromfs/inode.c
index 4a3477949bc..2ad080faca3 100644
--- a/fs/openpromfs/inode.c
+++ b/fs/openpromfs/inode.c
@@ -463,6 +463,11 @@ static int __init init_openprom_fs(void)
 static void __exit exit_openprom_fs(void)
 {
 	unregister_filesystem(&openprom_fs_type);
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(op_inode_cachep);
 }
 
diff --git a/fs/pipe.c b/fs/pipe.c
index 8d85d7068c1..bd3479db4b6 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -1064,9 +1064,8 @@ err_inode:
 	return err;
 }
 
-int do_pipe_flags(int *fd, int flags)
+static int __do_pipe_flags(int *fd, struct file **files, int flags)
 {
-	struct file *files[2];
 	int error;
 	int fdw, fdr;
 
@@ -1088,11 +1087,8 @@ int do_pipe_flags(int *fd, int flags)
 	fdw = error;
 
 	audit_fd_pair(fdr, fdw);
-	fd_install(fdr, files[0]);
-	fd_install(fdw, files[1]);
 	fd[0] = fdr;
 	fd[1] = fdw;
-
 	return 0;
 
  err_fdr:
@@ -1103,21 +1099,38 @@ int do_pipe_flags(int *fd, int flags)
 	return error;
 }
 
+int do_pipe_flags(int *fd, int flags)
+{
+	struct file *files[2];
+	int error = __do_pipe_flags(fd, files, flags);
+	if (!error) {
+		fd_install(fd[0], files[0]);
+		fd_install(fd[1], files[1]);
+	}
+	return error;
+}
+
 /*
  * sys_pipe() is the normal C calling standard for creating
  * a pipe. It's not the way Unix traditionally does this, though.
  */
 SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
 {
+	struct file *files[2];
 	int fd[2];
 	int error;
 
-	error = do_pipe_flags(fd, flags);
+	error = __do_pipe_flags(fd, files, flags);
 	if (!error) {
-		if (copy_to_user(fildes, fd, sizeof(fd))) {
-			sys_close(fd[0]);
-			sys_close(fd[1]);
+		if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
+			fput(files[0]);
+			fput(files[1]);
+			put_unused_fd(fd[0]);
+			put_unused_fd(fd[1]);
 			error = -EFAULT;
+		} else {
+			fd_install(fd[0], files[0]);
+			fd_install(fd[1], files[1]);
 		}
 	}
 	return error;
diff --git a/fs/proc/Makefile b/fs/proc/Makefile
index c1c72933592..99349efbbc2 100644
--- a/fs/proc/Makefile
+++ b/fs/proc/Makefile
@@ -8,7 +8,7 @@ proc-y			:= nommu.o task_nommu.o
 proc-$(CONFIG_MMU)	:= mmu.o task_mmu.o
 
 proc-y       += inode.o root.o base.o generic.o array.o \
-		proc_tty.o
+		proc_tty.o fd.o
 proc-y	+= cmdline.o
 proc-y	+= consoles.o
 proc-y	+= cpuinfo.o
diff --git a/fs/proc/base.c b/fs/proc/base.c
index acd1960c28a..d295af99367 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -90,6 +90,7 @@
 #endif
 #include <trace/events/oom.h>
 #include "internal.h"
+#include "fd.h"
 
 /* NOTE:
  *	Implementing inode permission operations in /proc is almost
@@ -136,8 +137,6 @@ struct pid_entry {
 		NULL, &proc_single_file_operations,	\
 		{ .proc_show = show } )
 
-static int proc_fd_permission(struct inode *inode, int mask);
-
 /*
  * Count the number of hardlinks for the pid_entry table, excluding the .
  * and .. links.
@@ -1500,7 +1499,7 @@ out:
 	return error;
 }
 
-static const struct inode_operations proc_pid_link_inode_operations = {
+const struct inode_operations proc_pid_link_inode_operations = {
 	.readlink	= proc_pid_readlink,
 	.follow_link	= proc_pid_follow_link,
 	.setattr	= proc_setattr,
@@ -1509,21 +1508,6 @@ static const struct inode_operations proc_pid_link_inode_operations = {
 
 /* building an inode */
 
-static int task_dumpable(struct task_struct *task)
-{
-	int dumpable = 0;
-	struct mm_struct *mm;
-
-	task_lock(task);
-	mm = task->mm;
-	if (mm)
-		dumpable = get_dumpable(mm);
-	task_unlock(task);
-	if(dumpable == 1)
-		return 1;
-	return 0;
-}
-
 struct inode *proc_pid_make_inode(struct super_block * sb, struct task_struct *task)
 {
 	struct inode * inode;
@@ -1649,15 +1633,6 @@ int pid_revalidate(struct dentry *dentry, unsigned int flags)
 	return 0;
 }
 
-static int pid_delete_dentry(const struct dentry * dentry)
-{
-	/* Is the task we represent dead?
-	 * If so, then don't put the dentry on the lru list,
-	 * kill it immediately.
-	 */
-	return !proc_pid(dentry->d_inode)->tasks[PIDTYPE_PID].first;
-}
-
 const struct dentry_operations pid_dentry_operations =
 {
 	.d_revalidate	= pid_revalidate,
@@ -1720,289 +1695,6 @@ end_instantiate:
 	return filldir(dirent, name, len, filp->f_pos, ino, type);
 }
 
-static unsigned name_to_int(struct dentry *dentry)
-{
-	const char *name = dentry->d_name.name;
-	int len = dentry->d_name.len;
-	unsigned n = 0;
-
-	if (len > 1 && *name == '0')
-		goto out;
-	while (len-- > 0) {
-		unsigned c = *name++ - '0';
-		if (c > 9)
-			goto out;
-		if (n >= (~0U-9)/10)
-			goto out;
-		n *= 10;
-		n += c;
-	}
-	return n;
-out:
-	return ~0U;
-}
-
-#define PROC_FDINFO_MAX 64
-
-static int proc_fd_info(struct inode *inode, struct path *path, char *info)
-{
-	struct task_struct *task = get_proc_task(inode);
-	struct files_struct *files = NULL;
-	struct file *file;
-	int fd = proc_fd(inode);
-
-	if (task) {
-		files = get_files_struct(task);
-		put_task_struct(task);
-	}
-	if (files) {
-		/*
-		 * We are not taking a ref to the file structure, so we must
-		 * hold ->file_lock.
-		 */
-		spin_lock(&files->file_lock);
-		file = fcheck_files(files, fd);
-		if (file) {
-			unsigned int f_flags;
-			struct fdtable *fdt;
-
-			fdt = files_fdtable(files);
-			f_flags = file->f_flags & ~O_CLOEXEC;
-			if (close_on_exec(fd, fdt))
-				f_flags |= O_CLOEXEC;
-
-			if (path) {
-				*path = file->f_path;
-				path_get(&file->f_path);
-			}
-			if (info)
-				snprintf(info, PROC_FDINFO_MAX,
-					 "pos:\t%lli\n"
-					 "flags:\t0%o\n",
-					 (long long) file->f_pos,
-					 f_flags);
-			spin_unlock(&files->file_lock);
-			put_files_struct(files);
-			return 0;
-		}
-		spin_unlock(&files->file_lock);
-		put_files_struct(files);
-	}
-	return -ENOENT;
-}
-
-static int proc_fd_link(struct dentry *dentry, struct path *path)
-{
-	return proc_fd_info(dentry->d_inode, path, NULL);
-}
-
-static int tid_fd_revalidate(struct dentry *dentry, unsigned int flags)
-{
-	struct inode *inode;
-	struct task_struct *task;
-	int fd;
-	struct files_struct *files;
-	const struct cred *cred;
-
-	if (flags & LOOKUP_RCU)
-		return -ECHILD;
-
-	inode = dentry->d_inode;
-	task = get_proc_task(inode);
-	fd = proc_fd(inode);
-
-	if (task) {
-		files = get_files_struct(task);
-		if (files) {
-			struct file *file;
-			rcu_read_lock();
-			file = fcheck_files(files, fd);
-			if (file) {
-				unsigned f_mode = file->f_mode;
-
-				rcu_read_unlock();
-				put_files_struct(files);
-
-				if (task_dumpable(task)) {
-					rcu_read_lock();
-					cred = __task_cred(task);
-					inode->i_uid = cred->euid;
-					inode->i_gid = cred->egid;
-					rcu_read_unlock();
-				} else {
-					inode->i_uid = GLOBAL_ROOT_UID;
-					inode->i_gid = GLOBAL_ROOT_GID;
-				}
-
-				if (S_ISLNK(inode->i_mode)) {
-					unsigned i_mode = S_IFLNK;
-					if (f_mode & FMODE_READ)
-						i_mode |= S_IRUSR | S_IXUSR;
-					if (f_mode & FMODE_WRITE)
-						i_mode |= S_IWUSR | S_IXUSR;
-					inode->i_mode = i_mode;
-				}
-
-				security_task_to_inode(task, inode);
-				put_task_struct(task);
-				return 1;
-			}
-			rcu_read_unlock();
-			put_files_struct(files);
-		}
-		put_task_struct(task);
-	}
-	d_drop(dentry);
-	return 0;
-}
-
-static const struct dentry_operations tid_fd_dentry_operations =
-{
-	.d_revalidate	= tid_fd_revalidate,
-	.d_delete	= pid_delete_dentry,
-};
-
-static struct dentry *proc_fd_instantiate(struct inode *dir,
-	struct dentry *dentry, struct task_struct *task, const void *ptr)
-{
-	unsigned fd = (unsigned long)ptr;
- 	struct inode *inode;
- 	struct proc_inode *ei;
-	struct dentry *error = ERR_PTR(-ENOENT);
-
-	inode = proc_pid_make_inode(dir->i_sb, task);
-	if (!inode)
-		goto out;
-	ei = PROC_I(inode);
-	ei->fd = fd;
-
-	inode->i_mode = S_IFLNK;
-	inode->i_op = &proc_pid_link_inode_operations;
-	inode->i_size = 64;
-	ei->op.proc_get_link = proc_fd_link;
-	d_set_d_op(dentry, &tid_fd_dentry_operations);
-	d_add(dentry, inode);
-	/* Close the race of the process dying before we return the dentry */
-	if (tid_fd_revalidate(dentry, 0))
-		error = NULL;
-
- out:
-	return error;
-}
-
-static struct dentry *proc_lookupfd_common(struct inode *dir,
-					   struct dentry *dentry,
-					   instantiate_t instantiate)
-{
-	struct task_struct *task = get_proc_task(dir);
-	unsigned fd = name_to_int(dentry);
-	struct dentry *result = ERR_PTR(-ENOENT);
-
-	if (!task)
-		goto out_no_task;
-	if (fd == ~0U)
-		goto out;
-
-	result = instantiate(dir, dentry, task, (void *)(unsigned long)fd);
-out:
-	put_task_struct(task);
-out_no_task:
-	return result;
-}
-
-static int proc_readfd_common(struct file * filp, void * dirent,
-			      filldir_t filldir, instantiate_t instantiate)
-{
-	struct dentry *dentry = filp->f_path.dentry;
-	struct inode *inode = dentry->d_inode;
-	struct task_struct *p = get_proc_task(inode);
-	unsigned int fd, ino;
-	int retval;
-	struct files_struct * files;
-
-	retval = -ENOENT;
-	if (!p)
-		goto out_no_task;
-	retval = 0;
-
-	fd = filp->f_pos;
-	switch (fd) {
-		case 0:
-			if (filldir(dirent, ".", 1, 0, inode->i_ino, DT_DIR) < 0)
-				goto out;
-			filp->f_pos++;
-		case 1:
-			ino = parent_ino(dentry);
-			if (filldir(dirent, "..", 2, 1, ino, DT_DIR) < 0)
-				goto out;
-			filp->f_pos++;
-		default:
-			files = get_files_struct(p);
-			if (!files)
-				goto out;
-			rcu_read_lock();
-			for (fd = filp->f_pos-2;
-			     fd < files_fdtable(files)->max_fds;
-			     fd++, filp->f_pos++) {
-				char name[PROC_NUMBUF];
-				int len;
-				int rv;
-
-				if (!fcheck_files(files, fd))
-					continue;
-				rcu_read_unlock();
-
-				len = snprintf(name, sizeof(name), "%d", fd);
-				rv = proc_fill_cache(filp, dirent, filldir,
-						     name, len, instantiate, p,
-						     (void *)(unsigned long)fd);
-				if (rv < 0)
-					goto out_fd_loop;
-				rcu_read_lock();
-			}
-			rcu_read_unlock();
-out_fd_loop:
-			put_files_struct(files);
-	}
-out:
-	put_task_struct(p);
-out_no_task:
-	return retval;
-}
-
-static struct dentry *proc_lookupfd(struct inode *dir, struct dentry *dentry,
-				    unsigned int flags)
-{
-	return proc_lookupfd_common(dir, dentry, proc_fd_instantiate);
-}
-
-static int proc_readfd(struct file *filp, void *dirent, filldir_t filldir)
-{
-	return proc_readfd_common(filp, dirent, filldir, proc_fd_instantiate);
-}
-
-static ssize_t proc_fdinfo_read(struct file *file, char __user *buf,
-				      size_t len, loff_t *ppos)
-{
-	char tmp[PROC_FDINFO_MAX];
-	int err = proc_fd_info(file->f_path.dentry->d_inode, NULL, tmp);
-	if (!err)
-		err = simple_read_from_buffer(buf, len, ppos, tmp, strlen(tmp));
-	return err;
-}
-
-static const struct file_operations proc_fdinfo_file_operations = {
-	.open           = nonseekable_open,
-	.read		= proc_fdinfo_read,
-	.llseek		= no_llseek,
-};
-
-static const struct file_operations proc_fd_operations = {
-	.read		= generic_read_dir,
-	.readdir	= proc_readfd,
-	.llseek		= default_llseek,
-};
-
 #ifdef CONFIG_CHECKPOINT_RESTORE
 
 /*
@@ -2121,7 +1813,7 @@ out:
 }
 
 struct map_files_info {
-	struct file	*file;
+	fmode_t		mode;
 	unsigned long	len;
 	unsigned char	name[4*sizeof(long)+2]; /* max: %lx-%lx\0 */
 };
@@ -2130,13 +1822,10 @@ static struct dentry *
 proc_map_files_instantiate(struct inode *dir, struct dentry *dentry,
 			   struct task_struct *task, const void *ptr)
 {
-	const struct file *file = ptr;
+	fmode_t mode = (fmode_t)(unsigned long)ptr;
 	struct proc_inode *ei;
 	struct inode *inode;
 
-	if (!file)
-		return ERR_PTR(-ENOENT);
-
 	inode = proc_pid_make_inode(dir->i_sb, task);
 	if (!inode)
 		return ERR_PTR(-ENOENT);
@@ -2148,9 +1837,9 @@ proc_map_files_instantiate(struct inode *dir, struct dentry *dentry,
 	inode->i_size = 64;
 	inode->i_mode = S_IFLNK;
 
-	if (file->f_mode & FMODE_READ)
+	if (mode & FMODE_READ)
 		inode->i_mode |= S_IRUSR;
-	if (file->f_mode & FMODE_WRITE)
+	if (mode & FMODE_WRITE)
 		inode->i_mode |= S_IWUSR;
 
 	d_set_d_op(dentry, &tid_map_files_dentry_operations);
@@ -2194,7 +1883,8 @@ static struct dentry *proc_map_files_lookup(struct inode *dir,
 	if (!vma)
 		goto out_no_vma;
 
-	result = proc_map_files_instantiate(dir, dentry, task, vma->vm_file);
+	result = proc_map_files_instantiate(dir, dentry, task,
+			(void *)(unsigned long)vma->vm_file->f_mode);
 
 out_no_vma:
 	up_read(&mm->mmap_sem);
@@ -2295,8 +1985,7 @@ proc_map_files_readdir(struct file *filp, void *dirent, filldir_t filldir)
 				if (++pos <= filp->f_pos)
 					continue;
 
-				get_file(vma->vm_file);
-				info.file = vma->vm_file;
+				info.mode = vma->vm_file->f_mode;
 				info.len = snprintf(info.name,
 						sizeof(info.name), "%lx-%lx",
 						vma->vm_start, vma->vm_end);
@@ -2311,19 +2000,11 @@ proc_map_files_readdir(struct file *filp, void *dirent, filldir_t filldir)
 			ret = proc_fill_cache(filp, dirent, filldir,
 					      p->name, p->len,
 					      proc_map_files_instantiate,
-					      task, p->file);
+					      task,
+					      (void *)(unsigned long)p->mode);
 			if (ret)
 				break;
 			filp->f_pos++;
-			fput(p->file);
-		}
-		for (; i < nr_files; i++) {
-			/*
-			 * In case of error don't forget
-			 * to put rest of file refs.
-			 */
-			p = flex_array_get(fa, i);
-			fput(p->file);
 		}
 		if (fa)
 			flex_array_free(fa);
@@ -2345,82 +2026,6 @@ static const struct file_operations proc_map_files_operations = {
 
 #endif /* CONFIG_CHECKPOINT_RESTORE */
 
-/*
- * /proc/pid/fd needs a special permission handler so that a process can still
- * access /proc/self/fd after it has executed a setuid().
- */
-static int proc_fd_permission(struct inode *inode, int mask)
-{
-	int rv = generic_permission(inode, mask);
-	if (rv == 0)
-		return 0;
-	if (task_pid(current) == proc_pid(inode))
-		rv = 0;
-	return rv;
-}
-
-/*
- * proc directories can do almost nothing..
- */
-static const struct inode_operations proc_fd_inode_operations = {
-	.lookup		= proc_lookupfd,
-	.permission	= proc_fd_permission,
-	.setattr	= proc_setattr,
-};
-
-static struct dentry *proc_fdinfo_instantiate(struct inode *dir,
-	struct dentry *dentry, struct task_struct *task, const void *ptr)
-{
-	unsigned fd = (unsigned long)ptr;
- 	struct inode *inode;
- 	struct proc_inode *ei;
-	struct dentry *error = ERR_PTR(-ENOENT);
-
-	inode = proc_pid_make_inode(dir->i_sb, task);
-	if (!inode)
-		goto out;
-	ei = PROC_I(inode);
-	ei->fd = fd;
-	inode->i_mode = S_IFREG | S_IRUSR;
-	inode->i_fop = &proc_fdinfo_file_operations;
-	d_set_d_op(dentry, &tid_fd_dentry_operations);
-	d_add(dentry, inode);
-	/* Close the race of the process dying before we return the dentry */
-	if (tid_fd_revalidate(dentry, 0))
-		error = NULL;
-
- out:
-	return error;
-}
-
-static struct dentry *proc_lookupfdinfo(struct inode *dir,
-					struct dentry *dentry,
-					unsigned int flags)
-{
-	return proc_lookupfd_common(dir, dentry, proc_fdinfo_instantiate);
-}
-
-static int proc_readfdinfo(struct file *filp, void *dirent, filldir_t filldir)
-{
-	return proc_readfd_common(filp, dirent, filldir,
-				  proc_fdinfo_instantiate);
-}
-
-static const struct file_operations proc_fdinfo_operations = {
-	.read		= generic_read_dir,
-	.readdir	= proc_readfdinfo,
-	.llseek		= default_llseek,
-};
-
-/*
- * proc directories can do almost nothing..
- */
-static const struct inode_operations proc_fdinfo_inode_operations = {
-	.lookup		= proc_lookupfdinfo,
-	.setattr	= proc_setattr,
-};
-
-
 static struct dentry *proc_pident_instantiate(struct inode *dir,
 	struct dentry *dentry, struct task_struct *task, const void *ptr)
 {
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
new file mode 100644
index 00000000000..f28a875f877
--- /dev/null
+++ b/fs/proc/fd.c
@@ -0,0 +1,367 @@
+#include <linux/sched.h>
+#include <linux/errno.h>
+#include <linux/dcache.h>
+#include <linux/path.h>
+#include <linux/fdtable.h>
+#include <linux/namei.h>
+#include <linux/pid.h>
+#include <linux/security.h>
+#include <linux/file.h>
+#include <linux/seq_file.h>
+
+#include <linux/proc_fs.h>
+
+#include "internal.h"
+#include "fd.h"
+
+static int seq_show(struct seq_file *m, void *v)
+{
+	struct files_struct *files = NULL;
+	int f_flags = 0, ret = -ENOENT;
+	struct file *file = NULL;
+	struct task_struct *task;
+
+	task = get_proc_task(m->private);
+	if (!task)
+		return -ENOENT;
+
+	files = get_files_struct(task);
+	put_task_struct(task);
+
+	if (files) {
+		int fd = proc_fd(m->private);
+
+		spin_lock(&files->file_lock);
+		file = fcheck_files(files, fd);
+		if (file) {
+			struct fdtable *fdt = files_fdtable(files);
+
+			f_flags = file->f_flags;
+			if (close_on_exec(fd, fdt))
+				f_flags |= O_CLOEXEC;
+
+			get_file(file);
+			ret = 0;
+		}
+		spin_unlock(&files->file_lock);
+		put_files_struct(files);
+	}
+
+	if (!ret) {
+                seq_printf(m, "pos:\t%lli\nflags:\t0%o\n",
+			   (long long)file->f_pos, f_flags);
+		fput(file);
+	}
+
+	return ret;
+}
+
+static int seq_fdinfo_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, seq_show, inode);
+}
+
+static const struct file_operations proc_fdinfo_file_operations = {
+	.open		= seq_fdinfo_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+static int tid_fd_revalidate(struct dentry *dentry, unsigned int flags)
+{
+	struct files_struct *files;
+	struct task_struct *task;
+	const struct cred *cred;
+	struct inode *inode;
+	int fd;
+
+	if (flags & LOOKUP_RCU)
+		return -ECHILD;
+
+	inode = dentry->d_inode;
+	task = get_proc_task(inode);
+	fd = proc_fd(inode);
+
+	if (task) {
+		files = get_files_struct(task);
+		if (files) {
+			struct file *file;
+
+			rcu_read_lock();
+			file = fcheck_files(files, fd);
+			if (file) {
+				unsigned f_mode = file->f_mode;
+
+				rcu_read_unlock();
+				put_files_struct(files);
+
+				if (task_dumpable(task)) {
+					rcu_read_lock();
+					cred = __task_cred(task);
+					inode->i_uid = cred->euid;
+					inode->i_gid = cred->egid;
+					rcu_read_unlock();
+				} else {
+					inode->i_uid = GLOBAL_ROOT_UID;
+					inode->i_gid = GLOBAL_ROOT_GID;
+				}
+
+				if (S_ISLNK(inode->i_mode)) {
+					unsigned i_mode = S_IFLNK;
+					if (f_mode & FMODE_READ)
+						i_mode |= S_IRUSR | S_IXUSR;
+					if (f_mode & FMODE_WRITE)
+						i_mode |= S_IWUSR | S_IXUSR;
+					inode->i_mode = i_mode;
+				}
+
+				security_task_to_inode(task, inode);
+				put_task_struct(task);
+				return 1;
+			}
+			rcu_read_unlock();
+			put_files_struct(files);
+		}
+		put_task_struct(task);
+	}
+
+	d_drop(dentry);
+	return 0;
+}
+
+static const struct dentry_operations tid_fd_dentry_operations = {
+	.d_revalidate	= tid_fd_revalidate,
+	.d_delete	= pid_delete_dentry,
+};
+
+static int proc_fd_link(struct dentry *dentry, struct path *path)
+{
+	struct files_struct *files = NULL;
+	struct task_struct *task;
+	int ret = -ENOENT;
+
+	task = get_proc_task(dentry->d_inode);
+	if (task) {
+		files = get_files_struct(task);
+		put_task_struct(task);
+	}
+
+	if (files) {
+		int fd = proc_fd(dentry->d_inode);
+		struct file *fd_file;
+
+		spin_lock(&files->file_lock);
+		fd_file = fcheck_files(files, fd);
+		if (fd_file) {
+			*path = fd_file->f_path;
+			path_get(&fd_file->f_path);
+			ret = 0;
+		}
+		spin_unlock(&files->file_lock);
+		put_files_struct(files);
+	}
+
+	return ret;
+}
+
+static struct dentry *
+proc_fd_instantiate(struct inode *dir, struct dentry *dentry,
+		    struct task_struct *task, const void *ptr)
+{
+	struct dentry *error = ERR_PTR(-ENOENT);
+	unsigned fd = (unsigned long)ptr;
+	struct proc_inode *ei;
+	struct inode *inode;
+
+	inode = proc_pid_make_inode(dir->i_sb, task);
+	if (!inode)
+		goto out;
+
+	ei = PROC_I(inode);
+	ei->fd = fd;
+
+	inode->i_mode = S_IFLNK;
+	inode->i_op = &proc_pid_link_inode_operations;
+	inode->i_size = 64;
+
+	ei->op.proc_get_link = proc_fd_link;
+
+	d_set_d_op(dentry, &tid_fd_dentry_operations);
+	d_add(dentry, inode);
+
+	/* Close the race of the process dying before we return the dentry */
+	if (tid_fd_revalidate(dentry, 0))
+		error = NULL;
+ out:
+	return error;
+}
+
+static struct dentry *proc_lookupfd_common(struct inode *dir,
+					   struct dentry *dentry,
+					   instantiate_t instantiate)
+{
+	struct task_struct *task = get_proc_task(dir);
+	struct dentry *result = ERR_PTR(-ENOENT);
+	unsigned fd = name_to_int(dentry);
+
+	if (!task)
+		goto out_no_task;
+	if (fd == ~0U)
+		goto out;
+
+	result = instantiate(dir, dentry, task, (void *)(unsigned long)fd);
+out:
+	put_task_struct(task);
+out_no_task:
+	return result;
+}
+
+static int proc_readfd_common(struct file * filp, void * dirent,
+			      filldir_t filldir, instantiate_t instantiate)
+{
+	struct dentry *dentry = filp->f_path.dentry;
+	struct inode *inode = dentry->d_inode;
+	struct task_struct *p = get_proc_task(inode);
+	struct files_struct *files;
+	unsigned int fd, ino;
+	int retval;
+
+	retval = -ENOENT;
+	if (!p)
+		goto out_no_task;
+	retval = 0;
+
+	fd = filp->f_pos;
+	switch (fd) {
+		case 0:
+			if (filldir(dirent, ".", 1, 0, inode->i_ino, DT_DIR) < 0)
+				goto out;
+			filp->f_pos++;
+		case 1:
+			ino = parent_ino(dentry);
+			if (filldir(dirent, "..", 2, 1, ino, DT_DIR) < 0)
+				goto out;
+			filp->f_pos++;
+		default:
+			files = get_files_struct(p);
+			if (!files)
+				goto out;
+			rcu_read_lock();
+			for (fd = filp->f_pos - 2;
+			     fd < files_fdtable(files)->max_fds;
+			     fd++, filp->f_pos++) {
+				char name[PROC_NUMBUF];
+				int len;
+				int rv;
+
+				if (!fcheck_files(files, fd))
+					continue;
+				rcu_read_unlock();
+
+				len = snprintf(name, sizeof(name), "%d", fd);
+				rv = proc_fill_cache(filp, dirent, filldir,
+						     name, len, instantiate, p,
+						     (void *)(unsigned long)fd);
+				if (rv < 0)
+					goto out_fd_loop;
+				rcu_read_lock();
+			}
+			rcu_read_unlock();
+out_fd_loop:
+			put_files_struct(files);
+	}
+out:
+	put_task_struct(p);
+out_no_task:
+	return retval;
+}
+
+static int proc_readfd(struct file *filp, void *dirent, filldir_t filldir)
+{
+	return proc_readfd_common(filp, dirent, filldir, proc_fd_instantiate);
+}
+
+const struct file_operations proc_fd_operations = {
+	.read		= generic_read_dir,
+	.readdir	= proc_readfd,
+	.llseek		= default_llseek,
+};
+
+static struct dentry *proc_lookupfd(struct inode *dir, struct dentry *dentry,
+				    unsigned int flags)
+{
+	return proc_lookupfd_common(dir, dentry, proc_fd_instantiate);
+}
+
+/*
+ * /proc/pid/fd needs a special permission handler so that a process can still
+ * access /proc/self/fd after it has executed a setuid().
+ */
+int proc_fd_permission(struct inode *inode, int mask)
+{
+	int rv = generic_permission(inode, mask);
+	if (rv == 0)
+		return 0;
+	if (task_pid(current) == proc_pid(inode))
+		rv = 0;
+	return rv;
+}
+
+const struct inode_operations proc_fd_inode_operations = {
+	.lookup		= proc_lookupfd,
+	.permission	= proc_fd_permission,
+	.setattr	= proc_setattr,
+};
+
+static struct dentry *
+proc_fdinfo_instantiate(struct inode *dir, struct dentry *dentry,
+			struct task_struct *task, const void *ptr)
+{
+	struct dentry *error = ERR_PTR(-ENOENT);
+	unsigned fd = (unsigned long)ptr;
+	struct proc_inode *ei;
+	struct inode *inode;
+
+	inode = proc_pid_make_inode(dir->i_sb, task);
+	if (!inode)
+		goto out;
+
+	ei = PROC_I(inode);
+	ei->fd = fd;
+
+	inode->i_mode = S_IFREG | S_IRUSR;
+	inode->i_fop = &proc_fdinfo_file_operations;
+
+	d_set_d_op(dentry, &tid_fd_dentry_operations);
+	d_add(dentry, inode);
+
+	/* Close the race of the process dying before we return the dentry */
+	if (tid_fd_revalidate(dentry, 0))
+		error = NULL;
+ out:
+	return error;
+}
+
+static struct dentry *
+proc_lookupfdinfo(struct inode *dir, struct dentry *dentry, unsigned int flags)
+{
+	return proc_lookupfd_common(dir, dentry, proc_fdinfo_instantiate);
+}
+
+static int proc_readfdinfo(struct file *filp, void *dirent, filldir_t filldir)
+{
+	return proc_readfd_common(filp, dirent, filldir,
+				  proc_fdinfo_instantiate);
+}
+
+const struct inode_operations proc_fdinfo_inode_operations = {
+	.lookup		= proc_lookupfdinfo,
+	.setattr	= proc_setattr,
+};
+
+const struct file_operations proc_fdinfo_operations = {
+	.read		= generic_read_dir,
+	.readdir	= proc_readfdinfo,
+	.llseek		= default_llseek,
+};
diff --git a/fs/proc/fd.h b/fs/proc/fd.h
new file mode 100644
index 00000000000..cbb1d47deda
--- /dev/null
+++ b/fs/proc/fd.h
@@ -0,0 +1,14 @@
+#ifndef __PROCFS_FD_H__
+#define __PROCFS_FD_H__
+
+#include <linux/fs.h>
+
+extern const struct file_operations proc_fd_operations;
+extern const struct inode_operations proc_fd_inode_operations;
+
+extern const struct file_operations proc_fdinfo_operations;
+extern const struct inode_operations proc_fdinfo_inode_operations;
+
+extern int proc_fd_permission(struct inode *inode, int mask);
+
+#endif /* __PROCFS_FD_H__ */
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index b3647fe6a60..0d80cef4cfb 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -427,7 +427,7 @@ struct dentry *proc_lookup_de(struct proc_dir_entry *de, struct inode *dir,
 		if (!memcmp(dentry->d_name.name, de->name, de->namelen)) {
 			pde_get(de);
 			spin_unlock(&proc_subdir_lock);
-			error = -EINVAL;
+			error = -ENOMEM;
 			inode = proc_get_inode(dir->i_sb, de);
 			goto out_unlock;
 		}
@@ -605,7 +605,8 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent,
 	unsigned int len;
 
 	/* make sure name is valid */
-	if (!name || !strlen(name)) goto out;
+	if (!name || !strlen(name))
+		goto out;
 
 	if (xlate_proc_name(name, parent, &fn) != 0)
 		goto out;
@@ -616,20 +617,18 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent,
 
 	len = strlen(fn);
 
-	ent = kmalloc(sizeof(struct proc_dir_entry) + len + 1, GFP_KERNEL);
-	if (!ent) goto out;
+	ent = kzalloc(sizeof(struct proc_dir_entry) + len + 1, GFP_KERNEL);
+	if (!ent)
+		goto out;
 
-	memset(ent, 0, sizeof(struct proc_dir_entry));
 	memcpy(ent->name, fn, len + 1);
 	ent->namelen = len;
 	ent->mode = mode;
 	ent->nlink = nlink;
 	atomic_set(&ent->count, 1);
-	ent->pde_users = 0;
 	spin_lock_init(&ent->pde_unload_lock);
-	ent->pde_unload_completion = NULL;
 	INIT_LIST_HEAD(&ent->pde_openers);
- out:
+out:
 	return ent;
 }
 
diff --git a/fs/proc/inode.c b/fs/proc/inode.c
index 7ac817b64a7..3b22bbdee9e 100644
--- a/fs/proc/inode.c
+++ b/fs/proc/inode.c
@@ -450,7 +450,6 @@ struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de)
 		return NULL;
 	if (inode->i_state & I_NEW) {
 		inode->i_mtime = inode->i_atime = inode->i_ctime = CURRENT_TIME;
-		PROC_I(inode)->fd = 0;
 		PROC_I(inode)->pde = de;
 
 		if (de->mode) {
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index e1167a1c912..cceaab07ad5 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -9,6 +9,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include <linux/sched.h>
 #include <linux/proc_fs.h>
 struct  ctl_table_header;
 
@@ -65,6 +66,7 @@ extern const struct file_operations proc_clear_refs_operations;
 extern const struct file_operations proc_pagemap_operations;
 extern const struct file_operations proc_net_operations;
 extern const struct inode_operations proc_net_inode_operations;
+extern const struct inode_operations proc_pid_link_inode_operations;
 
 struct proc_maps_private {
 	struct pid *pid;
@@ -91,6 +93,52 @@ static inline int proc_fd(struct inode *inode)
 	return PROC_I(inode)->fd;
 }
 
+static inline int task_dumpable(struct task_struct *task)
+{
+	int dumpable = 0;
+	struct mm_struct *mm;
+
+	task_lock(task);
+	mm = task->mm;
+	if (mm)
+		dumpable = get_dumpable(mm);
+	task_unlock(task);
+	if (dumpable == SUID_DUMPABLE_ENABLED)
+		return 1;
+	return 0;
+}
+
+static inline int pid_delete_dentry(const struct dentry * dentry)
+{
+	/* Is the task we represent dead?
+	 * If so, then don't put the dentry on the lru list,
+	 * kill it immediately.
+	 */
+	return !proc_pid(dentry->d_inode)->tasks[PIDTYPE_PID].first;
+}
+
+static inline unsigned name_to_int(struct dentry *dentry)
+{
+	const char *name = dentry->d_name.name;
+	int len = dentry->d_name.len;
+	unsigned n = 0;
+
+	if (len > 1 && *name == '0')
+		goto out;
+	while (len-- > 0) {
+		unsigned c = *name++ - '0';
+		if (c > 9)
+			goto out;
+		if (n >= (~0U-9)/10)
+			goto out;
+		n *= 10;
+		n += c;
+	}
+	return n;
+out:
+	return ~0U;
+}
+
 struct dentry *proc_lookup_de(struct proc_dir_entry *de, struct inode *ino,
 		struct dentry *dentry);
 int proc_readdir_de(struct proc_dir_entry *de, struct file *filp, void *dirent,
diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c
index eb7cc91b725..dcd56f84db7 100644
--- a/fs/proc/proc_sysctl.c
+++ b/fs/proc/proc_sysctl.c
@@ -266,8 +266,7 @@ void sysctl_head_put(struct ctl_table_header *head)
 
 static struct ctl_table_header *sysctl_head_grab(struct ctl_table_header *head)
 {
-	if (!head)
-		BUG();
+	BUG_ON(!head);
 	spin_lock(&sysctl_lock);
 	if (!use_table(head))
 		head = ERR_PTR(-ENOENT);
diff --git a/fs/proc/root.c b/fs/proc/root.c
index 9a2d9fd7cad..9889a92d2e0 100644
--- a/fs/proc/root.c
+++ b/fs/proc/root.c
@@ -61,7 +61,7 @@ static int proc_parse_options(char *options, struct pid_namespace *pid)
 		if (!*p)
 			continue;
 
-		args[0].to = args[0].from = 0;
+		args[0].to = args[0].from = NULL;
 		token = match_token(p, tokens, args);
 		switch (token) {
 		case Opt_gid:
diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c
index 5c3c7b02e17..43098bb5723 100644
--- a/fs/qnx4/inode.c
+++ b/fs/qnx4/inode.c
@@ -391,6 +391,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(qnx4_inode_cachep);
 }
 
diff --git a/fs/qnx6/inode.c b/fs/qnx6/inode.c
index f4eef0b5e7b..b6addf56048 100644
--- a/fs/qnx6/inode.c
+++ b/fs/qnx6/inode.c
@@ -651,6 +651,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(qnx6_inode_cachep);
 }
 
diff --git a/fs/read_write.c b/fs/read_write.c
index 1adfb691e4f..d06534857e9 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -232,23 +232,18 @@ EXPORT_SYMBOL(vfs_llseek);
 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
 {
 	off_t retval;
-	struct file * file;
-	int fput_needed;
-
-	retval = -EBADF;
-	file = fget_light(fd, &fput_needed);
-	if (!file)
-		goto bad;
+	struct fd f = fdget(fd);
+	if (!f.file)
+		return -EBADF;
 
 	retval = -EINVAL;
 	if (origin <= SEEK_MAX) {
-		loff_t res = vfs_llseek(file, offset, origin);
+		loff_t res = vfs_llseek(f.file, offset, origin);
 		retval = res;
 		if (res != (loff_t)retval)
 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
 	}
-	fput_light(file, fput_needed);
-bad:
+	fdput(f);
 	return retval;
 }
 
@@ -258,20 +253,17 @@ SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
 		unsigned int, origin)
 {
 	int retval;
-	struct file * file;
+	struct fd f = fdget(fd);
 	loff_t offset;
-	int fput_needed;
 
-	retval = -EBADF;
-	file = fget_light(fd, &fput_needed);
-	if (!file)
-		goto bad;
+	if (!f.file)
+		return -EBADF;
 
 	retval = -EINVAL;
 	if (origin > SEEK_MAX)
 		goto out_putf;
 
-	offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
+	offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
 			origin);
 
 	retval = (int)offset;
@@ -281,8 +273,7 @@ SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
 			retval = 0;
 	}
 out_putf:
-	fput_light(file, fput_needed);
-bad:
+	fdput(f);
 	return retval;
 }
 #endif
@@ -461,34 +452,29 @@ static inline void file_pos_write(struct file *file, loff_t pos)
 
 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
-		loff_t pos = file_pos_read(file);
-		ret = vfs_read(file, buf, count, &pos);
-		file_pos_write(file, pos);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		loff_t pos = file_pos_read(f.file);
+		ret = vfs_read(f.file, buf, count, &pos);
+		file_pos_write(f.file, pos);
+		fdput(f);
 	}
-
 	return ret;
 }
 
 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
 		size_t, count)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
-		loff_t pos = file_pos_read(file);
-		ret = vfs_write(file, buf, count, &pos);
-		file_pos_write(file, pos);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		loff_t pos = file_pos_read(f.file);
+		ret = vfs_write(f.file, buf, count, &pos);
+		file_pos_write(f.file, pos);
+		fdput(f);
 	}
 
 	return ret;
@@ -497,19 +483,18 @@ SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
 SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
 			size_t count, loff_t pos)
 {
-	struct file *file;
+	struct fd f;
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
+	f = fdget(fd);
+	if (f.file) {
 		ret = -ESPIPE;
-		if (file->f_mode & FMODE_PREAD)
-			ret = vfs_read(file, buf, count, &pos);
-		fput_light(file, fput_needed);
+		if (f.file->f_mode & FMODE_PREAD)
+			ret = vfs_read(f.file, buf, count, &pos);
+		fdput(f);
 	}
 
 	return ret;
@@ -526,19 +511,18 @@ SYSCALL_ALIAS(sys_pread64, SyS_pread64);
 SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
 			 size_t count, loff_t pos)
 {
-	struct file *file;
+	struct fd f;
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
+	f = fdget(fd);
+	if (f.file) {
 		ret = -ESPIPE;
-		if (file->f_mode & FMODE_PWRITE)  
-			ret = vfs_write(file, buf, count, &pos);
-		fput_light(file, fput_needed);
+		if (f.file->f_mode & FMODE_PWRITE)  
+			ret = vfs_write(f.file, buf, count, &pos);
+		fdput(f);
 	}
 
 	return ret;
@@ -789,16 +773,14 @@ EXPORT_SYMBOL(vfs_writev);
 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
 		unsigned long, vlen)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
-		loff_t pos = file_pos_read(file);
-		ret = vfs_readv(file, vec, vlen, &pos);
-		file_pos_write(file, pos);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		loff_t pos = file_pos_read(f.file);
+		ret = vfs_readv(f.file, vec, vlen, &pos);
+		file_pos_write(f.file, pos);
+		fdput(f);
 	}
 
 	if (ret > 0)
@@ -810,16 +792,14 @@ SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
 		unsigned long, vlen)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
-		loff_t pos = file_pos_read(file);
-		ret = vfs_writev(file, vec, vlen, &pos);
-		file_pos_write(file, pos);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		loff_t pos = file_pos_read(f.file);
+		ret = vfs_writev(f.file, vec, vlen, &pos);
+		file_pos_write(f.file, pos);
+		fdput(f);
 	}
 
 	if (ret > 0)
@@ -838,19 +818,18 @@ SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
 {
 	loff_t pos = pos_from_hilo(pos_h, pos_l);
-	struct file *file;
+	struct fd f;
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
+	f = fdget(fd);
+	if (f.file) {
 		ret = -ESPIPE;
-		if (file->f_mode & FMODE_PREAD)
-			ret = vfs_readv(file, vec, vlen, &pos);
-		fput_light(file, fput_needed);
+		if (f.file->f_mode & FMODE_PREAD)
+			ret = vfs_readv(f.file, vec, vlen, &pos);
+		fdput(f);
 	}
 
 	if (ret > 0)
@@ -863,19 +842,18 @@ SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
 		unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
 {
 	loff_t pos = pos_from_hilo(pos_h, pos_l);
-	struct file *file;
+	struct fd f;
 	ssize_t ret = -EBADF;
-	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
+	f = fdget(fd);
+	if (f.file) {
 		ret = -ESPIPE;
-		if (file->f_mode & FMODE_PWRITE)
-			ret = vfs_writev(file, vec, vlen, &pos);
-		fput_light(file, fput_needed);
+		if (f.file->f_mode & FMODE_PWRITE)
+			ret = vfs_writev(f.file, vec, vlen, &pos);
+		fdput(f);
 	}
 
 	if (ret > 0)
@@ -884,31 +862,31 @@ SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
 	return ret;
 }
 
-static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
-			   size_t count, loff_t max)
+ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, size_t count,
+		    loff_t max)
 {
-	struct file * in_file, * out_file;
-	struct inode * in_inode, * out_inode;
+	struct fd in, out;
+	struct inode *in_inode, *out_inode;
 	loff_t pos;
 	ssize_t retval;
-	int fput_needed_in, fput_needed_out, fl;
+	int fl;
 
 	/*
 	 * Get input file, and verify that it is ok..
 	 */
 	retval = -EBADF;
-	in_file = fget_light(in_fd, &fput_needed_in);
-	if (!in_file)
+	in = fdget(in_fd);
+	if (!in.file)
 		goto out;
-	if (!(in_file->f_mode & FMODE_READ))
+	if (!(in.file->f_mode & FMODE_READ))
 		goto fput_in;
 	retval = -ESPIPE;
 	if (!ppos)
-		ppos = &in_file->f_pos;
+		ppos = &in.file->f_pos;
 	else
-		if (!(in_file->f_mode & FMODE_PREAD))
+		if (!(in.file->f_mode & FMODE_PREAD))
 			goto fput_in;
-	retval = rw_verify_area(READ, in_file, ppos, count);
+	retval = rw_verify_area(READ, in.file, ppos, count);
 	if (retval < 0)
 		goto fput_in;
 	count = retval;
@@ -917,15 +895,15 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 	 * Get output file, and verify that it is ok..
 	 */
 	retval = -EBADF;
-	out_file = fget_light(out_fd, &fput_needed_out);
-	if (!out_file)
+	out = fdget(out_fd);
+	if (!out.file)
 		goto fput_in;
-	if (!(out_file->f_mode & FMODE_WRITE))
+	if (!(out.file->f_mode & FMODE_WRITE))
 		goto fput_out;
 	retval = -EINVAL;
-	in_inode = in_file->f_path.dentry->d_inode;
-	out_inode = out_file->f_path.dentry->d_inode;
-	retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
+	in_inode = in.file->f_path.dentry->d_inode;
+	out_inode = out.file->f_path.dentry->d_inode;
+	retval = rw_verify_area(WRITE, out.file, &out.file->f_pos, count);
 	if (retval < 0)
 		goto fput_out;
 	count = retval;
@@ -949,10 +927,10 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 	 * and the application is arguably buggy if it doesn't expect
 	 * EAGAIN on a non-blocking file descriptor.
 	 */
-	if (in_file->f_flags & O_NONBLOCK)
+	if (in.file->f_flags & O_NONBLOCK)
 		fl = SPLICE_F_NONBLOCK;
 #endif
-	retval = do_splice_direct(in_file, ppos, out_file, count, fl);
+	retval = do_splice_direct(in.file, ppos, out.file, count, fl);
 
 	if (retval > 0) {
 		add_rchar(current, retval);
@@ -965,9 +943,9 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 		retval = -EOVERFLOW;
 
 fput_out:
-	fput_light(out_file, fput_needed_out);
+	fdput(out);
 fput_in:
-	fput_light(in_file, fput_needed_in);
+	fdput(in);
 out:
 	return retval;
 }
diff --git a/fs/read_write.h b/fs/read_write.h
index d07b954c6e0..d3e00ef6742 100644
--- a/fs/read_write.h
+++ b/fs/read_write.h
@@ -12,3 +12,5 @@ ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
 		unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn);
 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
 		unsigned long nr_segs, loff_t *ppos, io_fn_t fn);
+ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, size_t count,
+		    loff_t max);
diff --git a/fs/readdir.c b/fs/readdir.c
index 39e3370d79c..5e69ef533b7 100644
--- a/fs/readdir.c
+++ b/fs/readdir.c
@@ -106,22 +106,20 @@ SYSCALL_DEFINE3(old_readdir, unsigned int, fd,
 		struct old_linux_dirent __user *, dirent, unsigned int, count)
 {
 	int error;
-	struct file * file;
+	struct fd f = fdget(fd);
 	struct readdir_callback buf;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
 
 	buf.result = 0;
 	buf.dirent = dirent;
 
-	error = vfs_readdir(file, fillonedir, &buf);
+	error = vfs_readdir(f.file, fillonedir, &buf);
 	if (buf.result)
 		error = buf.result;
 
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
 
@@ -191,17 +189,16 @@ efault:
 SYSCALL_DEFINE3(getdents, unsigned int, fd,
 		struct linux_dirent __user *, dirent, unsigned int, count)
 {
-	struct file * file;
+	struct fd f;
 	struct linux_dirent __user * lastdirent;
 	struct getdents_callback buf;
-	int fput_needed;
 	int error;
 
 	if (!access_ok(VERIFY_WRITE, dirent, count))
 		return -EFAULT;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 
 	buf.current_dir = dirent;
@@ -209,17 +206,17 @@ SYSCALL_DEFINE3(getdents, unsigned int, fd,
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, filldir, &buf);
+	error = vfs_readdir(f.file, filldir, &buf);
 	if (error >= 0)
 		error = buf.error;
 	lastdirent = buf.previous;
 	if (lastdirent) {
-		if (put_user(file->f_pos, &lastdirent->d_off))
+		if (put_user(f.file->f_pos, &lastdirent->d_off))
 			error = -EFAULT;
 		else
 			error = count - buf.count;
 	}
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
 
@@ -272,17 +269,16 @@ efault:
 SYSCALL_DEFINE3(getdents64, unsigned int, fd,
 		struct linux_dirent64 __user *, dirent, unsigned int, count)
 {
-	struct file * file;
+	struct fd f;
 	struct linux_dirent64 __user * lastdirent;
 	struct getdents_callback64 buf;
-	int fput_needed;
 	int error;
 
 	if (!access_ok(VERIFY_WRITE, dirent, count))
 		return -EFAULT;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return -EBADF;
 
 	buf.current_dir = dirent;
@@ -290,17 +286,17 @@ SYSCALL_DEFINE3(getdents64, unsigned int, fd,
 	buf.count = count;
 	buf.error = 0;
 
-	error = vfs_readdir(file, filldir64, &buf);
+	error = vfs_readdir(f.file, filldir64, &buf);
 	if (error >= 0)
 		error = buf.error;
 	lastdirent = buf.previous;
 	if (lastdirent) {
-		typeof(lastdirent->d_off) d_off = file->f_pos;
+		typeof(lastdirent->d_off) d_off = f.file->f_pos;
 		if (__put_user(d_off, &lastdirent->d_off))
 			error = -EFAULT;
 		else
 			error = count - buf.count;
 	}
-	fput_light(file, fput_needed);
+	fdput(f);
 	return error;
 }
diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c
index 7a37dabf5a9..1078ae17999 100644
--- a/fs/reiserfs/super.c
+++ b/fs/reiserfs/super.c
@@ -608,6 +608,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(reiserfs_inode_cachep);
 }
 
diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c
index d319963aeb1..c196369fe40 100644
--- a/fs/reiserfs/xattr.c
+++ b/fs/reiserfs/xattr.c
@@ -896,7 +896,7 @@ static int create_privroot(struct dentry *dentry) { return 0; }
 #endif
 
 /* Actual operations that are exported to VFS-land */
-const struct xattr_handler *reiserfs_xattr_handlers[] = {
+static const struct xattr_handler *reiserfs_xattr_handlers[] = {
 #ifdef CONFIG_REISERFS_FS_XATTR
 	&reiserfs_xattr_user_handler,
 	&reiserfs_xattr_trusted_handler,
diff --git a/fs/romfs/super.c b/fs/romfs/super.c
index 77c5f217398..fd7c5f60b46 100644
--- a/fs/romfs/super.c
+++ b/fs/romfs/super.c
@@ -648,6 +648,11 @@ error_register:
 static void __exit exit_romfs_fs(void)
 {
 	unregister_filesystem(&romfs_fs_type);
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(romfs_inode_cachep);
 }
 
diff --git a/fs/select.c b/fs/select.c
index db14c781335..2ef72d96503 100644
--- a/fs/select.c
+++ b/fs/select.c
@@ -220,8 +220,7 @@ static void __pollwait(struct file *filp, wait_queue_head_t *wait_address,
 	struct poll_table_entry *entry = poll_get_entry(pwq);
 	if (!entry)
 		return;
-	get_file(filp);
-	entry->filp = filp;
+	entry->filp = get_file(filp);
 	entry->wait_address = wait_address;
 	entry->key = p->_key;
 	init_waitqueue_func_entry(&entry->wait, pollwake);
@@ -429,8 +428,6 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
 		for (i = 0; i < n; ++rinp, ++routp, ++rexp) {
 			unsigned long in, out, ex, all_bits, bit = 1, mask, j;
 			unsigned long res_in = 0, res_out = 0, res_ex = 0;
-			const struct file_operations *f_op = NULL;
-			struct file *file = NULL;
 
 			in = *inp++; out = *outp++; ex = *exp++;
 			all_bits = in | out | ex;
@@ -440,20 +437,21 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
 			}
 
 			for (j = 0; j < BITS_PER_LONG; ++j, ++i, bit <<= 1) {
-				int fput_needed;
+				struct fd f;
 				if (i >= n)
 					break;
 				if (!(bit & all_bits))
 					continue;
-				file = fget_light(i, &fput_needed);
-				if (file) {
-					f_op = file->f_op;
+				f = fdget(i);
+				if (f.file) {
+					const struct file_operations *f_op;
+					f_op = f.file->f_op;
 					mask = DEFAULT_POLLMASK;
 					if (f_op && f_op->poll) {
 						wait_key_set(wait, in, out, bit);
-						mask = (*f_op->poll)(file, wait);
+						mask = (*f_op->poll)(f.file, wait);
 					}
-					fput_light(file, fput_needed);
+					fdput(f);
 					if ((mask & POLLIN_SET) && (in & bit)) {
 						res_in |= bit;
 						retval++;
@@ -726,20 +724,17 @@ static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait)
 	mask = 0;
 	fd = pollfd->fd;
 	if (fd >= 0) {
-		int fput_needed;
-		struct file * file;
-
-		file = fget_light(fd, &fput_needed);
+		struct fd f = fdget(fd);
 		mask = POLLNVAL;
-		if (file != NULL) {
+		if (f.file) {
 			mask = DEFAULT_POLLMASK;
-			if (file->f_op && file->f_op->poll) {
+			if (f.file->f_op && f.file->f_op->poll) {
 				pwait->_key = pollfd->events|POLLERR|POLLHUP;
-				mask = file->f_op->poll(file, pwait);
+				mask = f.file->f_op->poll(f.file, pwait);
 			}
 			/* Mask out unneeded events. */
 			mask &= pollfd->events | POLLERR | POLLHUP;
-			fput_light(file, fput_needed);
+			fdput(f);
 		}
 	}
 	pollfd->revents = mask;
diff --git a/fs/signalfd.c b/fs/signalfd.c
index 9f35a37173d..8bee4e57091 100644
--- a/fs/signalfd.c
+++ b/fs/signalfd.c
@@ -269,13 +269,12 @@ SYSCALL_DEFINE4(signalfd4, int, ufd, sigset_t __user *, user_mask,
 		if (ufd < 0)
 			kfree(ctx);
 	} else {
-		int fput_needed;
-		struct file *file = fget_light(ufd, &fput_needed);
-		if (!file)
+		struct fd f = fdget(ufd);
+		if (!f.file)
 			return -EBADF;
-		ctx = file->private_data;
-		if (file->f_op != &signalfd_fops) {
-			fput_light(file, fput_needed);
+		ctx = f.file->private_data;
+		if (f.file->f_op != &signalfd_fops) {
+			fdput(f);
 			return -EINVAL;
 		}
 		spin_lock_irq(&current->sighand->siglock);
@@ -283,7 +282,7 @@ SYSCALL_DEFINE4(signalfd4, int, ufd, sigset_t __user *, user_mask,
 		spin_unlock_irq(&current->sighand->siglock);
 
 		wake_up(&current->sighand->signalfd_wqh);
-		fput_light(file, fput_needed);
+		fdput(f);
 	}
 
 	return ufd;
diff --git a/fs/splice.c b/fs/splice.c
index 41514dd8946..13e5b4776e7 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -1666,9 +1666,8 @@ static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
 		unsigned long, nr_segs, unsigned int, flags)
 {
-	struct file *file;
+	struct fd f;
 	long error;
-	int fput;
 
 	if (unlikely(nr_segs > UIO_MAXIOV))
 		return -EINVAL;
@@ -1676,14 +1675,14 @@ SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
 		return 0;
 
 	error = -EBADF;
-	file = fget_light(fd, &fput);
-	if (file) {
-		if (file->f_mode & FMODE_WRITE)
-			error = vmsplice_to_pipe(file, iov, nr_segs, flags);
-		else if (file->f_mode & FMODE_READ)
-			error = vmsplice_to_user(file, iov, nr_segs, flags);
-
-		fput_light(file, fput);
+	f = fdget(fd);
+	if (f.file) {
+		if (f.file->f_mode & FMODE_WRITE)
+			error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
+		else if (f.file->f_mode & FMODE_READ)
+			error = vmsplice_to_user(f.file, iov, nr_segs, flags);
+
+		fdput(f);
 	}
 
 	return error;
@@ -1693,30 +1692,27 @@ SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
 		int, fd_out, loff_t __user *, off_out,
 		size_t, len, unsigned int, flags)
 {
+	struct fd in, out;
 	long error;
-	struct file *in, *out;
-	int fput_in, fput_out;
 
 	if (unlikely(!len))
 		return 0;
 
 	error = -EBADF;
-	in = fget_light(fd_in, &fput_in);
-	if (in) {
-		if (in->f_mode & FMODE_READ) {
-			out = fget_light(fd_out, &fput_out);
-			if (out) {
-				if (out->f_mode & FMODE_WRITE)
-					error = do_splice(in, off_in,
-							  out, off_out,
+	in = fdget(fd_in);
+	if (in.file) {
+		if (in.file->f_mode & FMODE_READ) {
+			out = fdget(fd_out);
+			if (out.file) {
+				if (out.file->f_mode & FMODE_WRITE)
+					error = do_splice(in.file, off_in,
+							  out.file, off_out,
 							  len, flags);
-				fput_light(out, fput_out);
+				fdput(out);
 			}
 		}
-
-		fput_light(in, fput_in);
+		fdput(in);
 	}
-
 	return error;
 }
 
@@ -2027,26 +2023,25 @@ static long do_tee(struct file *in, struct file *out, size_t len,
 
 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
 {
-	struct file *in;
-	int error, fput_in;
+	struct fd in;
+	int error;
 
 	if (unlikely(!len))
 		return 0;
 
 	error = -EBADF;
-	in = fget_light(fdin, &fput_in);
-	if (in) {
-		if (in->f_mode & FMODE_READ) {
-			int fput_out;
-			struct file *out = fget_light(fdout, &fput_out);
-
-			if (out) {
-				if (out->f_mode & FMODE_WRITE)
-					error = do_tee(in, out, len, flags);
-				fput_light(out, fput_out);
+	in = fdget(fdin);
+	if (in.file) {
+		if (in.file->f_mode & FMODE_READ) {
+			struct fd out = fdget(fdout);
+			if (out.file) {
+				if (out.file->f_mode & FMODE_WRITE)
+					error = do_tee(in.file, out.file,
+							len, flags);
+				fdput(out);
 			}
 		}
- 		fput_light(in, fput_in);
+ 		fdput(in);
  	}
 
 	return error;
diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c
index 29cd014ed3a..260e3928d4f 100644
--- a/fs/squashfs/super.c
+++ b/fs/squashfs/super.c
@@ -425,6 +425,11 @@ static int __init init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(squashfs_inode_cachep);
 }
 
diff --git a/fs/stat.c b/fs/stat.c
index 208039eec6c..eae494630a3 100644
--- a/fs/stat.c
+++ b/fs/stat.c
@@ -57,13 +57,13 @@ EXPORT_SYMBOL(vfs_getattr);
 
 int vfs_fstat(unsigned int fd, struct kstat *stat)
 {
-	int fput_needed;
-	struct file *f = fget_raw_light(fd, &fput_needed);
+	struct fd f = fdget_raw(fd);
 	int error = -EBADF;
 
-	if (f) {
-		error = vfs_getattr(f->f_path.mnt, f->f_path.dentry, stat);
-		fput_light(f, fput_needed);
+	if (f.file) {
+		error = vfs_getattr(f.file->f_path.mnt, f.file->f_path.dentry,
+				    stat);
+		fdput(f);
 	}
 	return error;
 }
diff --git a/fs/statfs.c b/fs/statfs.c
index 95ad5c0e586..f8e832e6f0a 100644
--- a/fs/statfs.c
+++ b/fs/statfs.c
@@ -87,12 +87,11 @@ int user_statfs(const char __user *pathname, struct kstatfs *st)
 
 int fd_statfs(int fd, struct kstatfs *st)
 {
-	int fput_needed;
-	struct file *file = fget_light(fd, &fput_needed);
+	struct fd f = fdget(fd);
 	int error = -EBADF;
-	if (file) {
-		error = vfs_statfs(&file->f_path, st);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		error = vfs_statfs(&f.file->f_path, st);
+		fdput(f);
 	}
 	return error;
 }
diff --git a/fs/super.c b/fs/super.c
index 0902cfa6a12..a3bc935069d 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -307,12 +307,6 @@ void deactivate_locked_super(struct super_block *s)
 
 		/* caches are now gone, we can safely kill the shrinker now */
 		unregister_shrinker(&s->s_shrink);
-
-		/*
-		 * We need to call rcu_barrier so all the delayed rcu free
-		 * inodes are flushed before we release the fs module.
-		 */
-		rcu_barrier();
 		put_filesystem(fs);
 		put_super(s);
 	} else {
@@ -871,7 +865,7 @@ int get_anon_bdev(dev_t *p)
 	else if (error)
 		return -EAGAIN;
 
-	if ((dev & MAX_ID_MASK) == (1 << MINORBITS)) {
+	if ((dev & MAX_IDR_MASK) == (1 << MINORBITS)) {
 		spin_lock(&unnamed_dev_lock);
 		ida_remove(&unnamed_dev_ida, dev);
 		if (unnamed_dev_start > dev)
diff --git a/fs/sync.c b/fs/sync.c
index eb8722dc556..14eefeb4463 100644
--- a/fs/sync.c
+++ b/fs/sync.c
@@ -148,21 +148,19 @@ void emergency_sync(void)
  */
 SYSCALL_DEFINE1(syncfs, int, fd)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	struct super_block *sb;
 	int ret;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
-	sb = file->f_dentry->d_sb;
+	sb = f.file->f_dentry->d_sb;
 
 	down_read(&sb->s_umount);
 	ret = sync_filesystem(sb);
 	up_read(&sb->s_umount);
 
-	fput_light(file, fput_needed);
+	fdput(f);
 	return ret;
 }
 
@@ -201,14 +199,12 @@ EXPORT_SYMBOL(vfs_fsync);
 
 static int do_fsync(unsigned int fd, int datasync)
 {
-	struct file *file;
+	struct fd f = fdget(fd);
 	int ret = -EBADF;
-	int fput_needed;
 
-	file = fget_light(fd, &fput_needed);
-	if (file) {
-		ret = vfs_fsync(file, datasync);
-		fput_light(file, fput_needed);
+	if (f.file) {
+		ret = vfs_fsync(f.file, datasync);
+		fdput(f);
 	}
 	return ret;
 }
@@ -291,10 +287,9 @@ SYSCALL_DEFINE(sync_file_range)(int fd, loff_t offset, loff_t nbytes,
 				unsigned int flags)
 {
 	int ret;
-	struct file *file;
+	struct fd f;
 	struct address_space *mapping;
 	loff_t endbyte;			/* inclusive */
-	int fput_needed;
 	umode_t i_mode;
 
 	ret = -EINVAL;
@@ -333,17 +328,17 @@ SYSCALL_DEFINE(sync_file_range)(int fd, loff_t offset, loff_t nbytes,
 		endbyte--;		/* inclusive */
 
 	ret = -EBADF;
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		goto out;
 
-	i_mode = file->f_path.dentry->d_inode->i_mode;
+	i_mode = f.file->f_path.dentry->d_inode->i_mode;
 	ret = -ESPIPE;
 	if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode) &&
 			!S_ISLNK(i_mode))
 		goto out_put;
 
-	mapping = file->f_mapping;
+	mapping = f.file->f_mapping;
 	if (!mapping) {
 		ret = -EINVAL;
 		goto out_put;
@@ -366,7 +361,7 @@ SYSCALL_DEFINE(sync_file_range)(int fd, loff_t offset, loff_t nbytes,
 		ret = filemap_fdatawait_range(mapping, offset, endbyte);
 
 out_put:
-	fput_light(file, fput_needed);
+	fdput(f);
 out:
 	return ret;
 }
diff --git a/fs/sysv/inode.c b/fs/sysv/inode.c
index b23ab736685..d33e506c1ea 100644
--- a/fs/sysv/inode.c
+++ b/fs/sysv/inode.c
@@ -360,5 +360,10 @@ int __init sysv_init_icache(void)
 
 void sysv_destroy_icache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(sysv_inode_cachep);
 }
diff --git a/fs/timerfd.c b/fs/timerfd.c
index dffeb3795af..d03822bbf19 100644
--- a/fs/timerfd.c
+++ b/fs/timerfd.c
@@ -234,19 +234,17 @@ static const struct file_operations timerfd_fops = {
 	.llseek		= noop_llseek,
 };
 
-static struct file *timerfd_fget(int fd)
+static int timerfd_fget(int fd, struct fd *p)
 {
-	struct file *file;
-
-	file = fget(fd);
-	if (!file)
-		return ERR_PTR(-EBADF);
-	if (file->f_op != &timerfd_fops) {
-		fput(file);
-		return ERR_PTR(-EINVAL);
+	struct fd f = fdget(fd);
+	if (!f.file)
+		return -EBADF;
+	if (f.file->f_op != &timerfd_fops) {
+		fdput(f);
+		return -EINVAL;
 	}
-
-	return file;
+	*p = f;
+	return 0;
 }
 
 SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags)
@@ -284,7 +282,7 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
 		const struct itimerspec __user *, utmr,
 		struct itimerspec __user *, otmr)
 {
-	struct file *file;
+	struct fd f;
 	struct timerfd_ctx *ctx;
 	struct itimerspec ktmr, kotmr;
 	int ret;
@@ -297,10 +295,10 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
 	    !timespec_valid(&ktmr.it_interval))
 		return -EINVAL;
 
-	file = timerfd_fget(ufd);
-	if (IS_ERR(file))
-		return PTR_ERR(file);
-	ctx = file->private_data;
+	ret = timerfd_fget(ufd, &f);
+	if (ret)
+		return ret;
+	ctx = f.file->private_data;
 
 	timerfd_setup_cancel(ctx, flags);
 
@@ -334,7 +332,7 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
 	ret = timerfd_setup(ctx, flags, &ktmr);
 
 	spin_unlock_irq(&ctx->wqh.lock);
-	fput(file);
+	fdput(f);
 	if (otmr && copy_to_user(otmr, &kotmr, sizeof(kotmr)))
 		return -EFAULT;
 
@@ -343,14 +341,13 @@ SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags,
 
 SYSCALL_DEFINE2(timerfd_gettime, int, ufd, struct itimerspec __user *, otmr)
 {
-	struct file *file;
+	struct fd f;
 	struct timerfd_ctx *ctx;
 	struct itimerspec kotmr;
-
-	file = timerfd_fget(ufd);
-	if (IS_ERR(file))
-		return PTR_ERR(file);
-	ctx = file->private_data;
+	int ret = timerfd_fget(ufd, &f);
+	if (ret)
+		return ret;
+	ctx = f.file->private_data;
 
 	spin_lock_irq(&ctx->wqh.lock);
 	if (ctx->expired && ctx->tintv.tv64) {
@@ -362,7 +359,7 @@ SYSCALL_DEFINE2(timerfd_gettime, int, ufd, struct itimerspec __user *, otmr)
 	kotmr.it_value = ktime_to_timespec(timerfd_get_remaining(ctx));
 	kotmr.it_interval = ktime_to_timespec(ctx->tintv);
 	spin_unlock_irq(&ctx->wqh.lock);
-	fput(file);
+	fdput(f);
 
 	return copy_to_user(otmr, &kotmr, sizeof(kotmr)) ? -EFAULT: 0;
 }
diff --git a/fs/ubifs/budget.c b/fs/ubifs/budget.c
index 969489e478b..e8e01d74dc0 100644
--- a/fs/ubifs/budget.c
+++ b/fs/ubifs/budget.c
@@ -342,9 +342,8 @@ static int do_budget_space(struct ubifs_info *c)
 	lebs = c->lst.empty_lebs + c->freeable_cnt + c->idx_gc_cnt -
 	       c->lst.taken_empty_lebs;
 	if (unlikely(rsvd_idx_lebs > lebs)) {
-		dbg_budg("out of indexing space: min_idx_lebs %d (old %d), "
-			 "rsvd_idx_lebs %d", min_idx_lebs, c->bi.min_idx_lebs,
-			 rsvd_idx_lebs);
+		dbg_budg("out of indexing space: min_idx_lebs %d (old %d), rsvd_idx_lebs %d",
+			 min_idx_lebs, c->bi.min_idx_lebs, rsvd_idx_lebs);
 		return -ENOSPC;
 	}
 
diff --git a/fs/ubifs/commit.c b/fs/ubifs/commit.c
index 8eda717cb99..ff8229340cd 100644
--- a/fs/ubifs/commit.c
+++ b/fs/ubifs/commit.c
@@ -293,8 +293,8 @@ int ubifs_bg_thread(void *info)
 	int err;
 	struct ubifs_info *c = info;
 
-	dbg_msg("background thread \"%s\" started, PID %d",
-		c->bgt_name, current->pid);
+	ubifs_msg("background thread \"%s\" started, PID %d",
+		  c->bgt_name, current->pid);
 	set_freezable();
 
 	while (1) {
@@ -328,7 +328,7 @@ int ubifs_bg_thread(void *info)
 		cond_resched();
 	}
 
-	dbg_msg("background thread \"%s\" stops", c->bgt_name);
+	ubifs_msg("background thread \"%s\" stops", c->bgt_name);
 	return 0;
 }
 
@@ -514,7 +514,7 @@ struct idx_node {
 	struct list_head list;
 	int iip;
 	union ubifs_key upper_key;
-	struct ubifs_idx_node idx __attribute__((aligned(8)));
+	struct ubifs_idx_node idx __aligned(8);
 };
 
 /**
diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c
index 11e4132f314..2bfa0953335 100644
--- a/fs/ubifs/compress.c
+++ b/fs/ubifs/compress.c
@@ -112,8 +112,7 @@ void ubifs_compress(const void *in_buf, int in_len, void *out_buf, int *out_len,
 	if (compr->comp_mutex)
 		mutex_unlock(compr->comp_mutex);
 	if (unlikely(err)) {
-		ubifs_warn("cannot compress %d bytes, compressor %s, "
-			   "error %d, leave data uncompressed",
+		ubifs_warn("cannot compress %d bytes, compressor %s, error %d, leave data uncompressed",
 			   in_len, compr->name, err);
 		 goto no_compr;
 	}
@@ -176,8 +175,8 @@ int ubifs_decompress(const void *in_buf, int in_len, void *out_buf,
 	if (compr->decomp_mutex)
 		mutex_unlock(compr->decomp_mutex);
 	if (err)
-		ubifs_err("cannot decompress %d bytes, compressor %s, "
-			  "error %d", in_len, compr->name, err);
+		ubifs_err("cannot decompress %d bytes, compressor %s, error %d",
+			  in_len, compr->name, err);
 
 	return err;
 }
diff --git a/fs/ubifs/debug.c b/fs/ubifs/debug.c
index 340d1afc130..62911637e12 100644
--- a/fs/ubifs/debug.c
+++ b/fs/ubifs/debug.c
@@ -219,15 +219,15 @@ const char *dbg_jhead(int jhead)
 
 static void dump_ch(const struct ubifs_ch *ch)
 {
-	printk(KERN_ERR "\tmagic          %#x\n", le32_to_cpu(ch->magic));
-	printk(KERN_ERR "\tcrc            %#x\n", le32_to_cpu(ch->crc));
-	printk(KERN_ERR "\tnode_type      %d (%s)\n", ch->node_type,
+	pr_err("\tmagic          %#x\n", le32_to_cpu(ch->magic));
+	pr_err("\tcrc            %#x\n", le32_to_cpu(ch->crc));
+	pr_err("\tnode_type      %d (%s)\n", ch->node_type,
 	       dbg_ntype(ch->node_type));
-	printk(KERN_ERR "\tgroup_type     %d (%s)\n", ch->group_type,
+	pr_err("\tgroup_type     %d (%s)\n", ch->group_type,
 	       dbg_gtype(ch->group_type));
-	printk(KERN_ERR "\tsqnum          %llu\n",
+	pr_err("\tsqnum          %llu\n",
 	       (unsigned long long)le64_to_cpu(ch->sqnum));
-	printk(KERN_ERR "\tlen            %u\n", le32_to_cpu(ch->len));
+	pr_err("\tlen            %u\n", le32_to_cpu(ch->len));
 }
 
 void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
@@ -238,43 +238,43 @@ void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
 	struct ubifs_dent_node *dent, *pdent = NULL;
 	int count = 2;
 
-	printk(KERN_ERR "Dump in-memory inode:");
-	printk(KERN_ERR "\tinode          %lu\n", inode->i_ino);
-	printk(KERN_ERR "\tsize           %llu\n",
+	pr_err("Dump in-memory inode:");
+	pr_err("\tinode          %lu\n", inode->i_ino);
+	pr_err("\tsize           %llu\n",
 	       (unsigned long long)i_size_read(inode));
-	printk(KERN_ERR "\tnlink          %u\n", inode->i_nlink);
-	printk(KERN_ERR "\tuid            %u\n", (unsigned int)i_uid_read(inode));
-	printk(KERN_ERR "\tgid            %u\n", (unsigned int)i_gid_read(inode));
-	printk(KERN_ERR "\tatime          %u.%u\n",
+	pr_err("\tnlink          %u\n", inode->i_nlink);
+	pr_err("\tuid            %u\n", (unsigned int)i_uid_read(inode));
+	pr_err("\tgid            %u\n", (unsigned int)i_gid_read(inode));
+	pr_err("\tatime          %u.%u\n",
 	       (unsigned int)inode->i_atime.tv_sec,
 	       (unsigned int)inode->i_atime.tv_nsec);
-	printk(KERN_ERR "\tmtime          %u.%u\n",
+	pr_err("\tmtime          %u.%u\n",
 	       (unsigned int)inode->i_mtime.tv_sec,
 	       (unsigned int)inode->i_mtime.tv_nsec);
-	printk(KERN_ERR "\tctime          %u.%u\n",
+	pr_err("\tctime          %u.%u\n",
 	       (unsigned int)inode->i_ctime.tv_sec,
 	       (unsigned int)inode->i_ctime.tv_nsec);
-	printk(KERN_ERR "\tcreat_sqnum    %llu\n", ui->creat_sqnum);
-	printk(KERN_ERR "\txattr_size     %u\n", ui->xattr_size);
-	printk(KERN_ERR "\txattr_cnt      %u\n", ui->xattr_cnt);
-	printk(KERN_ERR "\txattr_names    %u\n", ui->xattr_names);
-	printk(KERN_ERR "\tdirty          %u\n", ui->dirty);
-	printk(KERN_ERR "\txattr          %u\n", ui->xattr);
-	printk(KERN_ERR "\tbulk_read      %u\n", ui->xattr);
-	printk(KERN_ERR "\tsynced_i_size  %llu\n",
+	pr_err("\tcreat_sqnum    %llu\n", ui->creat_sqnum);
+	pr_err("\txattr_size     %u\n", ui->xattr_size);
+	pr_err("\txattr_cnt      %u\n", ui->xattr_cnt);
+	pr_err("\txattr_names    %u\n", ui->xattr_names);
+	pr_err("\tdirty          %u\n", ui->dirty);
+	pr_err("\txattr          %u\n", ui->xattr);
+	pr_err("\tbulk_read      %u\n", ui->xattr);
+	pr_err("\tsynced_i_size  %llu\n",
 	       (unsigned long long)ui->synced_i_size);
-	printk(KERN_ERR "\tui_size        %llu\n",
+	pr_err("\tui_size        %llu\n",
 	       (unsigned long long)ui->ui_size);
-	printk(KERN_ERR "\tflags          %d\n", ui->flags);
-	printk(KERN_ERR "\tcompr_type     %d\n", ui->compr_type);
-	printk(KERN_ERR "\tlast_page_read %lu\n", ui->last_page_read);
-	printk(KERN_ERR "\tread_in_a_row  %lu\n", ui->read_in_a_row);
-	printk(KERN_ERR "\tdata_len       %d\n", ui->data_len);
+	pr_err("\tflags          %d\n", ui->flags);
+	pr_err("\tcompr_type     %d\n", ui->compr_type);
+	pr_err("\tlast_page_read %lu\n", ui->last_page_read);
+	pr_err("\tread_in_a_row  %lu\n", ui->read_in_a_row);
+	pr_err("\tdata_len       %d\n", ui->data_len);
 
 	if (!S_ISDIR(inode->i_mode))
 		return;
 
-	printk(KERN_ERR "List of directory entries:\n");
+	pr_err("List of directory entries:\n");
 	ubifs_assert(!mutex_is_locked(&c->tnc_mutex));
 
 	lowest_dent_key(c, &key, inode->i_ino);
@@ -282,11 +282,11 @@ void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode)
 		dent = ubifs_tnc_next_ent(c, &key, &nm);
 		if (IS_ERR(dent)) {
 			if (PTR_ERR(dent) != -ENOENT)
-				printk(KERN_ERR "error %ld\n", PTR_ERR(dent));
+				pr_err("error %ld\n", PTR_ERR(dent));
 			break;
 		}
 
-		printk(KERN_ERR "\t%d: %s (%s)\n",
+		pr_err("\t%d: %s (%s)\n",
 		       count++, dent->name, get_dent_type(dent->type));
 
 		nm.name = dent->name;
@@ -305,12 +305,9 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 	const struct ubifs_ch *ch = node;
 	char key_buf[DBG_KEY_BUF_LEN];
 
-	if (dbg_is_tst_rcvry(c))
-		return;
-
 	/* If the magic is incorrect, just hexdump the first bytes */
 	if (le32_to_cpu(ch->magic) != UBIFS_NODE_MAGIC) {
-		printk(KERN_ERR "Not a node, first %zu bytes:", UBIFS_CH_SZ);
+		pr_err("Not a node, first %zu bytes:", UBIFS_CH_SZ);
 		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 32, 1,
 			       (void *)node, UBIFS_CH_SZ, 1);
 		return;
@@ -324,8 +321,7 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 	{
 		const struct ubifs_pad_node *pad = node;
 
-		printk(KERN_ERR "\tpad_len        %u\n",
-		       le32_to_cpu(pad->pad_len));
+		pr_err("\tpad_len        %u\n", le32_to_cpu(pad->pad_len));
 		break;
 	}
 	case UBIFS_SB_NODE:
@@ -333,112 +329,77 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 		const struct ubifs_sb_node *sup = node;
 		unsigned int sup_flags = le32_to_cpu(sup->flags);
 
-		printk(KERN_ERR "\tkey_hash       %d (%s)\n",
+		pr_err("\tkey_hash       %d (%s)\n",
 		       (int)sup->key_hash, get_key_hash(sup->key_hash));
-		printk(KERN_ERR "\tkey_fmt        %d (%s)\n",
+		pr_err("\tkey_fmt        %d (%s)\n",
 		       (int)sup->key_fmt, get_key_fmt(sup->key_fmt));
-		printk(KERN_ERR "\tflags          %#x\n", sup_flags);
-		printk(KERN_ERR "\t  big_lpt      %u\n",
+		pr_err("\tflags          %#x\n", sup_flags);
+		pr_err("\t  big_lpt      %u\n",
 		       !!(sup_flags & UBIFS_FLG_BIGLPT));
-		printk(KERN_ERR "\t  space_fixup  %u\n",
+		pr_err("\t  space_fixup  %u\n",
 		       !!(sup_flags & UBIFS_FLG_SPACE_FIXUP));
-		printk(KERN_ERR "\tmin_io_size    %u\n",
-		       le32_to_cpu(sup->min_io_size));
-		printk(KERN_ERR "\tleb_size       %u\n",
-		       le32_to_cpu(sup->leb_size));
-		printk(KERN_ERR "\tleb_cnt        %u\n",
-		       le32_to_cpu(sup->leb_cnt));
-		printk(KERN_ERR "\tmax_leb_cnt    %u\n",
-		       le32_to_cpu(sup->max_leb_cnt));
-		printk(KERN_ERR "\tmax_bud_bytes  %llu\n",
+		pr_err("\tmin_io_size    %u\n", le32_to_cpu(sup->min_io_size));
+		pr_err("\tleb_size       %u\n", le32_to_cpu(sup->leb_size));
+		pr_err("\tleb_cnt        %u\n", le32_to_cpu(sup->leb_cnt));
+		pr_err("\tmax_leb_cnt    %u\n", le32_to_cpu(sup->max_leb_cnt));
+		pr_err("\tmax_bud_bytes  %llu\n",
 		       (unsigned long long)le64_to_cpu(sup->max_bud_bytes));
-		printk(KERN_ERR "\tlog_lebs       %u\n",
-		       le32_to_cpu(sup->log_lebs));
-		printk(KERN_ERR "\tlpt_lebs       %u\n",
-		       le32_to_cpu(sup->lpt_lebs));
-		printk(KERN_ERR "\torph_lebs      %u\n",
-		       le32_to_cpu(sup->orph_lebs));
-		printk(KERN_ERR "\tjhead_cnt      %u\n",
-		       le32_to_cpu(sup->jhead_cnt));
-		printk(KERN_ERR "\tfanout         %u\n",
-		       le32_to_cpu(sup->fanout));
-		printk(KERN_ERR "\tlsave_cnt      %u\n",
-		       le32_to_cpu(sup->lsave_cnt));
-		printk(KERN_ERR "\tdefault_compr  %u\n",
+		pr_err("\tlog_lebs       %u\n", le32_to_cpu(sup->log_lebs));
+		pr_err("\tlpt_lebs       %u\n", le32_to_cpu(sup->lpt_lebs));
+		pr_err("\torph_lebs      %u\n", le32_to_cpu(sup->orph_lebs));
+		pr_err("\tjhead_cnt      %u\n", le32_to_cpu(sup->jhead_cnt));
+		pr_err("\tfanout         %u\n", le32_to_cpu(sup->fanout));
+		pr_err("\tlsave_cnt      %u\n", le32_to_cpu(sup->lsave_cnt));
+		pr_err("\tdefault_compr  %u\n",
 		       (int)le16_to_cpu(sup->default_compr));
-		printk(KERN_ERR "\trp_size        %llu\n",
+		pr_err("\trp_size        %llu\n",
 		       (unsigned long long)le64_to_cpu(sup->rp_size));
-		printk(KERN_ERR "\trp_uid         %u\n",
-		       le32_to_cpu(sup->rp_uid));
-		printk(KERN_ERR "\trp_gid         %u\n",
-		       le32_to_cpu(sup->rp_gid));
-		printk(KERN_ERR "\tfmt_version    %u\n",
-		       le32_to_cpu(sup->fmt_version));
-		printk(KERN_ERR "\ttime_gran      %u\n",
-		       le32_to_cpu(sup->time_gran));
-		printk(KERN_ERR "\tUUID           %pUB\n",
-		       sup->uuid);
+		pr_err("\trp_uid         %u\n", le32_to_cpu(sup->rp_uid));
+		pr_err("\trp_gid         %u\n", le32_to_cpu(sup->rp_gid));
+		pr_err("\tfmt_version    %u\n", le32_to_cpu(sup->fmt_version));
+		pr_err("\ttime_gran      %u\n", le32_to_cpu(sup->time_gran));
+		pr_err("\tUUID           %pUB\n", sup->uuid);
 		break;
 	}
 	case UBIFS_MST_NODE:
 	{
 		const struct ubifs_mst_node *mst = node;
 
-		printk(KERN_ERR "\thighest_inum   %llu\n",
+		pr_err("\thighest_inum   %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->highest_inum));
-		printk(KERN_ERR "\tcommit number  %llu\n",
+		pr_err("\tcommit number  %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->cmt_no));
-		printk(KERN_ERR "\tflags          %#x\n",
-		       le32_to_cpu(mst->flags));
-		printk(KERN_ERR "\tlog_lnum       %u\n",
-		       le32_to_cpu(mst->log_lnum));
-		printk(KERN_ERR "\troot_lnum      %u\n",
-		       le32_to_cpu(mst->root_lnum));
-		printk(KERN_ERR "\troot_offs      %u\n",
-		       le32_to_cpu(mst->root_offs));
-		printk(KERN_ERR "\troot_len       %u\n",
-		       le32_to_cpu(mst->root_len));
-		printk(KERN_ERR "\tgc_lnum        %u\n",
-		       le32_to_cpu(mst->gc_lnum));
-		printk(KERN_ERR "\tihead_lnum     %u\n",
-		       le32_to_cpu(mst->ihead_lnum));
-		printk(KERN_ERR "\tihead_offs     %u\n",
-		       le32_to_cpu(mst->ihead_offs));
-		printk(KERN_ERR "\tindex_size     %llu\n",
+		pr_err("\tflags          %#x\n", le32_to_cpu(mst->flags));
+		pr_err("\tlog_lnum       %u\n", le32_to_cpu(mst->log_lnum));
+		pr_err("\troot_lnum      %u\n", le32_to_cpu(mst->root_lnum));
+		pr_err("\troot_offs      %u\n", le32_to_cpu(mst->root_offs));
+		pr_err("\troot_len       %u\n", le32_to_cpu(mst->root_len));
+		pr_err("\tgc_lnum        %u\n", le32_to_cpu(mst->gc_lnum));
+		pr_err("\tihead_lnum     %u\n", le32_to_cpu(mst->ihead_lnum));
+		pr_err("\tihead_offs     %u\n", le32_to_cpu(mst->ihead_offs));
+		pr_err("\tindex_size     %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->index_size));
-		printk(KERN_ERR "\tlpt_lnum       %u\n",
-		       le32_to_cpu(mst->lpt_lnum));
-		printk(KERN_ERR "\tlpt_offs       %u\n",
-		       le32_to_cpu(mst->lpt_offs));
-		printk(KERN_ERR "\tnhead_lnum     %u\n",
-		       le32_to_cpu(mst->nhead_lnum));
-		printk(KERN_ERR "\tnhead_offs     %u\n",
-		       le32_to_cpu(mst->nhead_offs));
-		printk(KERN_ERR "\tltab_lnum      %u\n",
-		       le32_to_cpu(mst->ltab_lnum));
-		printk(KERN_ERR "\tltab_offs      %u\n",
-		       le32_to_cpu(mst->ltab_offs));
-		printk(KERN_ERR "\tlsave_lnum     %u\n",
-		       le32_to_cpu(mst->lsave_lnum));
-		printk(KERN_ERR "\tlsave_offs     %u\n",
-		       le32_to_cpu(mst->lsave_offs));
-		printk(KERN_ERR "\tlscan_lnum     %u\n",
-		       le32_to_cpu(mst->lscan_lnum));
-		printk(KERN_ERR "\tleb_cnt        %u\n",
-		       le32_to_cpu(mst->leb_cnt));
-		printk(KERN_ERR "\tempty_lebs     %u\n",
-		       le32_to_cpu(mst->empty_lebs));
-		printk(KERN_ERR "\tidx_lebs       %u\n",
-		       le32_to_cpu(mst->idx_lebs));
-		printk(KERN_ERR "\ttotal_free     %llu\n",
+		pr_err("\tlpt_lnum       %u\n", le32_to_cpu(mst->lpt_lnum));
+		pr_err("\tlpt_offs       %u\n", le32_to_cpu(mst->lpt_offs));
+		pr_err("\tnhead_lnum     %u\n", le32_to_cpu(mst->nhead_lnum));
+		pr_err("\tnhead_offs     %u\n", le32_to_cpu(mst->nhead_offs));
+		pr_err("\tltab_lnum      %u\n", le32_to_cpu(mst->ltab_lnum));
+		pr_err("\tltab_offs      %u\n", le32_to_cpu(mst->ltab_offs));
+		pr_err("\tlsave_lnum     %u\n", le32_to_cpu(mst->lsave_lnum));
+		pr_err("\tlsave_offs     %u\n", le32_to_cpu(mst->lsave_offs));
+		pr_err("\tlscan_lnum     %u\n", le32_to_cpu(mst->lscan_lnum));
+		pr_err("\tleb_cnt        %u\n", le32_to_cpu(mst->leb_cnt));
+		pr_err("\tempty_lebs     %u\n", le32_to_cpu(mst->empty_lebs));
+		pr_err("\tidx_lebs       %u\n", le32_to_cpu(mst->idx_lebs));
+		pr_err("\ttotal_free     %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->total_free));
-		printk(KERN_ERR "\ttotal_dirty    %llu\n",
+		pr_err("\ttotal_dirty    %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->total_dirty));
-		printk(KERN_ERR "\ttotal_used     %llu\n",
+		pr_err("\ttotal_used     %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->total_used));
-		printk(KERN_ERR "\ttotal_dead     %llu\n",
+		pr_err("\ttotal_dead     %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->total_dead));
-		printk(KERN_ERR "\ttotal_dark     %llu\n",
+		pr_err("\ttotal_dark     %llu\n",
 		       (unsigned long long)le64_to_cpu(mst->total_dark));
 		break;
 	}
@@ -446,12 +407,9 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 	{
 		const struct ubifs_ref_node *ref = node;
 
-		printk(KERN_ERR "\tlnum           %u\n",
-		       le32_to_cpu(ref->lnum));
-		printk(KERN_ERR "\toffs           %u\n",
-		       le32_to_cpu(ref->offs));
-		printk(KERN_ERR "\tjhead          %u\n",
-		       le32_to_cpu(ref->jhead));
+		pr_err("\tlnum           %u\n", le32_to_cpu(ref->lnum));
+		pr_err("\toffs           %u\n", le32_to_cpu(ref->offs));
+		pr_err("\tjhead          %u\n", le32_to_cpu(ref->jhead));
 		break;
 	}
 	case UBIFS_INO_NODE:
@@ -459,41 +417,32 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 		const struct ubifs_ino_node *ino = node;
 
 		key_read(c, &ino->key, &key);
-		printk(KERN_ERR "\tkey            %s\n",
+		pr_err("\tkey            %s\n",
 		       dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN));
-		printk(KERN_ERR "\tcreat_sqnum    %llu\n",
+		pr_err("\tcreat_sqnum    %llu\n",
 		       (unsigned long long)le64_to_cpu(ino->creat_sqnum));
-		printk(KERN_ERR "\tsize           %llu\n",
+		pr_err("\tsize           %llu\n",
 		       (unsigned long long)le64_to_cpu(ino->size));
-		printk(KERN_ERR "\tnlink          %u\n",
-		       le32_to_cpu(ino->nlink));
-		printk(KERN_ERR "\tatime          %lld.%u\n",
+		pr_err("\tnlink          %u\n", le32_to_cpu(ino->nlink));
+		pr_err("\tatime          %lld.%u\n",
 		       (long long)le64_to_cpu(ino->atime_sec),
 		       le32_to_cpu(ino->atime_nsec));
-		printk(KERN_ERR "\tmtime          %lld.%u\n",
+		pr_err("\tmtime          %lld.%u\n",
 		       (long long)le64_to_cpu(ino->mtime_sec),
 		       le32_to_cpu(ino->mtime_nsec));
-		printk(KERN_ERR "\tctime          %lld.%u\n",
+		pr_err("\tctime          %lld.%u\n",
 		       (long long)le64_to_cpu(ino->ctime_sec),
 		       le32_to_cpu(ino->ctime_nsec));
-		printk(KERN_ERR "\tuid            %u\n",
-		       le32_to_cpu(ino->uid));
-		printk(KERN_ERR "\tgid            %u\n",
-		       le32_to_cpu(ino->gid));
-		printk(KERN_ERR "\tmode           %u\n",
-		       le32_to_cpu(ino->mode));
-		printk(KERN_ERR "\tflags          %#x\n",
-		       le32_to_cpu(ino->flags));
-		printk(KERN_ERR "\txattr_cnt      %u\n",
-		       le32_to_cpu(ino->xattr_cnt));
-		printk(KERN_ERR "\txattr_size     %u\n",
-		       le32_to_cpu(ino->xattr_size));
-		printk(KERN_ERR "\txattr_names    %u\n",
-		       le32_to_cpu(ino->xattr_names));
-		printk(KERN_ERR "\tcompr_type     %#x\n",
+		pr_err("\tuid            %u\n", le32_to_cpu(ino->uid));
+		pr_err("\tgid            %u\n", le32_to_cpu(ino->gid));
+		pr_err("\tmode           %u\n", le32_to_cpu(ino->mode));
+		pr_err("\tflags          %#x\n", le32_to_cpu(ino->flags));
+		pr_err("\txattr_cnt      %u\n", le32_to_cpu(ino->xattr_cnt));
+		pr_err("\txattr_size     %u\n", le32_to_cpu(ino->xattr_size));
+		pr_err("\txattr_names    %u\n", le32_to_cpu(ino->xattr_names));
+		pr_err("\tcompr_type     %#x\n",
 		       (int)le16_to_cpu(ino->compr_type));
-		printk(KERN_ERR "\tdata len       %u\n",
-		       le32_to_cpu(ino->data_len));
+		pr_err("\tdata len       %u\n", le32_to_cpu(ino->data_len));
 		break;
 	}
 	case UBIFS_DENT_NODE:
@@ -503,22 +452,21 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 		int nlen = le16_to_cpu(dent->nlen);
 
 		key_read(c, &dent->key, &key);
-		printk(KERN_ERR "\tkey            %s\n",
+		pr_err("\tkey            %s\n",
 		       dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN));
-		printk(KERN_ERR "\tinum           %llu\n",
+		pr_err("\tinum           %llu\n",
 		       (unsigned long long)le64_to_cpu(dent->inum));
-		printk(KERN_ERR "\ttype           %d\n", (int)dent->type);
-		printk(KERN_ERR "\tnlen           %d\n", nlen);
-		printk(KERN_ERR "\tname           ");
+		pr_err("\ttype           %d\n", (int)dent->type);
+		pr_err("\tnlen           %d\n", nlen);
+		pr_err("\tname           ");
 
 		if (nlen > UBIFS_MAX_NLEN)
-			printk(KERN_ERR "(bad name length, not printing, "
-					  "bad or corrupted node)");
+			pr_err("(bad name length, not printing, bad or corrupted node)");
 		else {
 			for (i = 0; i < nlen && dent->name[i]; i++)
-				printk(KERN_CONT "%c", dent->name[i]);
+				pr_cont("%c", dent->name[i]);
 		}
-		printk(KERN_CONT "\n");
+		pr_cont("\n");
 
 		break;
 	}
@@ -528,15 +476,13 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 		int dlen = le32_to_cpu(ch->len) - UBIFS_DATA_NODE_SZ;
 
 		key_read(c, &dn->key, &key);
-		printk(KERN_ERR "\tkey            %s\n",
+		pr_err("\tkey            %s\n",
 		       dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN));
-		printk(KERN_ERR "\tsize           %u\n",
-		       le32_to_cpu(dn->size));
-		printk(KERN_ERR "\tcompr_typ      %d\n",
+		pr_err("\tsize           %u\n", le32_to_cpu(dn->size));
+		pr_err("\tcompr_typ      %d\n",
 		       (int)le16_to_cpu(dn->compr_type));
-		printk(KERN_ERR "\tdata size      %d\n",
-		       dlen);
-		printk(KERN_ERR "\tdata:\n");
+		pr_err("\tdata size      %d\n", dlen);
+		pr_err("\tdata:\n");
 		print_hex_dump(KERN_ERR, "\t", DUMP_PREFIX_OFFSET, 32, 1,
 			       (void *)&dn->data, dlen, 0);
 		break;
@@ -545,11 +491,10 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 	{
 		const struct ubifs_trun_node *trun = node;
 
-		printk(KERN_ERR "\tinum           %u\n",
-		       le32_to_cpu(trun->inum));
-		printk(KERN_ERR "\told_size       %llu\n",
+		pr_err("\tinum           %u\n", le32_to_cpu(trun->inum));
+		pr_err("\told_size       %llu\n",
 		       (unsigned long long)le64_to_cpu(trun->old_size));
-		printk(KERN_ERR "\tnew_size       %llu\n",
+		pr_err("\tnew_size       %llu\n",
 		       (unsigned long long)le64_to_cpu(trun->new_size));
 		break;
 	}
@@ -558,17 +503,16 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 		const struct ubifs_idx_node *idx = node;
 
 		n = le16_to_cpu(idx->child_cnt);
-		printk(KERN_ERR "\tchild_cnt      %d\n", n);
-		printk(KERN_ERR "\tlevel          %d\n",
-		       (int)le16_to_cpu(idx->level));
-		printk(KERN_ERR "\tBranches:\n");
+		pr_err("\tchild_cnt      %d\n", n);
+		pr_err("\tlevel          %d\n", (int)le16_to_cpu(idx->level));
+		pr_err("\tBranches:\n");
 
 		for (i = 0; i < n && i < c->fanout - 1; i++) {
 			const struct ubifs_branch *br;
 
 			br = ubifs_idx_branch(c, idx, i);
 			key_read(c, &br->key, &key);
-			printk(KERN_ERR "\t%d: LEB %d:%d len %d key %s\n",
+			pr_err("\t%d: LEB %d:%d len %d key %s\n",
 			       i, le32_to_cpu(br->lnum), le32_to_cpu(br->offs),
 			       le32_to_cpu(br->len),
 			       dbg_snprintf_key(c, &key, key_buf,
@@ -582,20 +526,20 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 	{
 		const struct ubifs_orph_node *orph = node;
 
-		printk(KERN_ERR "\tcommit number  %llu\n",
+		pr_err("\tcommit number  %llu\n",
 		       (unsigned long long)
 				le64_to_cpu(orph->cmt_no) & LLONG_MAX);
-		printk(KERN_ERR "\tlast node flag %llu\n",
+		pr_err("\tlast node flag %llu\n",
 		       (unsigned long long)(le64_to_cpu(orph->cmt_no)) >> 63);
 		n = (le32_to_cpu(ch->len) - UBIFS_ORPH_NODE_SZ) >> 3;
-		printk(KERN_ERR "\t%d orphan inode numbers:\n", n);
+		pr_err("\t%d orphan inode numbers:\n", n);
 		for (i = 0; i < n; i++)
-			printk(KERN_ERR "\t  ino %llu\n",
+			pr_err("\t  ino %llu\n",
 			       (unsigned long long)le64_to_cpu(orph->inos[i]));
 		break;
 	}
 	default:
-		printk(KERN_ERR "node type %d was not recognized\n",
+		pr_err("node type %d was not recognized\n",
 		       (int)ch->node_type);
 	}
 	spin_unlock(&dbg_lock);
@@ -604,16 +548,16 @@ void ubifs_dump_node(const struct ubifs_info *c, const void *node)
 void ubifs_dump_budget_req(const struct ubifs_budget_req *req)
 {
 	spin_lock(&dbg_lock);
-	printk(KERN_ERR "Budgeting request: new_ino %d, dirtied_ino %d\n",
+	pr_err("Budgeting request: new_ino %d, dirtied_ino %d\n",
 	       req->new_ino, req->dirtied_ino);
-	printk(KERN_ERR "\tnew_ino_d   %d, dirtied_ino_d %d\n",
+	pr_err("\tnew_ino_d   %d, dirtied_ino_d %d\n",
 	       req->new_ino_d, req->dirtied_ino_d);
-	printk(KERN_ERR "\tnew_page    %d, dirtied_page %d\n",
+	pr_err("\tnew_page    %d, dirtied_page %d\n",
 	       req->new_page, req->dirtied_page);
-	printk(KERN_ERR "\tnew_dent    %d, mod_dent     %d\n",
+	pr_err("\tnew_dent    %d, mod_dent     %d\n",
 	       req->new_dent, req->mod_dent);
-	printk(KERN_ERR "\tidx_growth  %d\n", req->idx_growth);
-	printk(KERN_ERR "\tdata_growth %d dd_growth     %d\n",
+	pr_err("\tidx_growth  %d\n", req->idx_growth);
+	pr_err("\tdata_growth %d dd_growth     %d\n",
 	       req->data_growth, req->dd_growth);
 	spin_unlock(&dbg_lock);
 }
@@ -621,14 +565,12 @@ void ubifs_dump_budget_req(const struct ubifs_budget_req *req)
 void ubifs_dump_lstats(const struct ubifs_lp_stats *lst)
 {
 	spin_lock(&dbg_lock);
-	printk(KERN_ERR "(pid %d) Lprops statistics: empty_lebs %d, "
-	       "idx_lebs  %d\n", current->pid, lst->empty_lebs, lst->idx_lebs);
-	printk(KERN_ERR "\ttaken_empty_lebs %d, total_free %lld, "
-	       "total_dirty %lld\n", lst->taken_empty_lebs, lst->total_free,
-	       lst->total_dirty);
-	printk(KERN_ERR "\ttotal_used %lld, total_dark %lld, "
-	       "total_dead %lld\n", lst->total_used, lst->total_dark,
-	       lst->total_dead);
+	pr_err("(pid %d) Lprops statistics: empty_lebs %d, idx_lebs  %d\n",
+	       current->pid, lst->empty_lebs, lst->idx_lebs);
+	pr_err("\ttaken_empty_lebs %d, total_free %lld, total_dirty %lld\n",
+	       lst->taken_empty_lebs, lst->total_free, lst->total_dirty);
+	pr_err("\ttotal_used %lld, total_dark %lld, total_dead %lld\n",
+	       lst->total_used, lst->total_dark, lst->total_dead);
 	spin_unlock(&dbg_lock);
 }
 
@@ -642,21 +584,17 @@ void ubifs_dump_budg(struct ubifs_info *c, const struct ubifs_budg_info *bi)
 
 	spin_lock(&c->space_lock);
 	spin_lock(&dbg_lock);
-	printk(KERN_ERR "(pid %d) Budgeting info: data budget sum %lld, "
-	       "total budget sum %lld\n", current->pid,
-	       bi->data_growth + bi->dd_growth,
+	pr_err("(pid %d) Budgeting info: data budget sum %lld, total budget sum %lld\n",
+	       current->pid, bi->data_growth + bi->dd_growth,
 	       bi->data_growth + bi->dd_growth + bi->idx_growth);
-	printk(KERN_ERR "\tbudg_data_growth %lld, budg_dd_growth %lld, "
-	       "budg_idx_growth %lld\n", bi->data_growth, bi->dd_growth,
-	       bi->idx_growth);
-	printk(KERN_ERR "\tmin_idx_lebs %d, old_idx_sz %llu, "
-	       "uncommitted_idx %lld\n", bi->min_idx_lebs, bi->old_idx_sz,
-	       bi->uncommitted_idx);
-	printk(KERN_ERR "\tpage_budget %d, inode_budget %d, dent_budget %d\n",
+	pr_err("\tbudg_data_growth %lld, budg_dd_growth %lld, budg_idx_growth %lld\n",
+	       bi->data_growth, bi->dd_growth, bi->idx_growth);
+	pr_err("\tmin_idx_lebs %d, old_idx_sz %llu, uncommitted_idx %lld\n",
+	       bi->min_idx_lebs, bi->old_idx_sz, bi->uncommitted_idx);
+	pr_err("\tpage_budget %d, inode_budget %d, dent_budget %d\n",
 	       bi->page_budget, bi->inode_budget, bi->dent_budget);
-	printk(KERN_ERR "\tnospace %u, nospace_rp %u\n",
-	       bi->nospace, bi->nospace_rp);
-	printk(KERN_ERR "\tdark_wm %d, dead_wm %d, max_idx_node_sz %d\n",
+	pr_err("\tnospace %u, nospace_rp %u\n", bi->nospace, bi->nospace_rp);
+	pr_err("\tdark_wm %d, dead_wm %d, max_idx_node_sz %d\n",
 	       c->dark_wm, c->dead_wm, c->max_idx_node_sz);
 
 	if (bi != &c->bi)
@@ -667,38 +605,37 @@ void ubifs_dump_budg(struct ubifs_info *c, const struct ubifs_budg_info *bi)
 		 */
 		goto out_unlock;
 
-	printk(KERN_ERR "\tfreeable_cnt %d, calc_idx_sz %lld, idx_gc_cnt %d\n",
+	pr_err("\tfreeable_cnt %d, calc_idx_sz %lld, idx_gc_cnt %d\n",
 	       c->freeable_cnt, c->calc_idx_sz, c->idx_gc_cnt);
-	printk(KERN_ERR "\tdirty_pg_cnt %ld, dirty_zn_cnt %ld, "
-	       "clean_zn_cnt %ld\n", atomic_long_read(&c->dirty_pg_cnt),
+	pr_err("\tdirty_pg_cnt %ld, dirty_zn_cnt %ld, clean_zn_cnt %ld\n",
+	       atomic_long_read(&c->dirty_pg_cnt),
 	       atomic_long_read(&c->dirty_zn_cnt),
 	       atomic_long_read(&c->clean_zn_cnt));
-	printk(KERN_ERR "\tgc_lnum %d, ihead_lnum %d\n",
-	       c->gc_lnum, c->ihead_lnum);
+	pr_err("\tgc_lnum %d, ihead_lnum %d\n", c->gc_lnum, c->ihead_lnum);
 
 	/* If we are in R/O mode, journal heads do not exist */
 	if (c->jheads)
 		for (i = 0; i < c->jhead_cnt; i++)
-			printk(KERN_ERR "\tjhead %s\t LEB %d\n",
+			pr_err("\tjhead %s\t LEB %d\n",
 			       dbg_jhead(c->jheads[i].wbuf.jhead),
 			       c->jheads[i].wbuf.lnum);
 	for (rb = rb_first(&c->buds); rb; rb = rb_next(rb)) {
 		bud = rb_entry(rb, struct ubifs_bud, rb);
-		printk(KERN_ERR "\tbud LEB %d\n", bud->lnum);
+		pr_err("\tbud LEB %d\n", bud->lnum);
 	}
 	list_for_each_entry(bud, &c->old_buds, list)
-		printk(KERN_ERR "\told bud LEB %d\n", bud->lnum);
+		pr_err("\told bud LEB %d\n", bud->lnum);
 	list_for_each_entry(idx_gc, &c->idx_gc, list)
-		printk(KERN_ERR "\tGC'ed idx LEB %d unmap %d\n",
+		pr_err("\tGC'ed idx LEB %d unmap %d\n",
 		       idx_gc->lnum, idx_gc->unmap);
-	printk(KERN_ERR "\tcommit state %d\n", c->cmt_state);
+	pr_err("\tcommit state %d\n", c->cmt_state);
 
 	/* Print budgeting predictions */
 	available = ubifs_calc_available(c, c->bi.min_idx_lebs);
 	outstanding = c->bi.data_growth + c->bi.dd_growth;
 	free = ubifs_get_free_space_nolock(c);
-	printk(KERN_ERR "Budgeting predictions:\n");
-	printk(KERN_ERR "\tavailable: %lld, outstanding %lld, free %lld\n",
+	pr_err("Budgeting predictions:\n");
+	pr_err("\tavailable: %lld, outstanding %lld, free %lld\n",
 	       available, outstanding, free);
 out_unlock:
 	spin_unlock(&dbg_lock);
@@ -718,21 +655,19 @@ void ubifs_dump_lprop(const struct ubifs_info *c, const struct ubifs_lprops *lp)
 		dark = ubifs_calc_dark(c, spc);
 
 	if (lp->flags & LPROPS_INDEX)
-		printk(KERN_ERR "LEB %-7d free %-8d dirty %-8d used %-8d "
-		       "free + dirty %-8d flags %#x (", lp->lnum, lp->free,
-		       lp->dirty, c->leb_size - spc, spc, lp->flags);
+		pr_err("LEB %-7d free %-8d dirty %-8d used %-8d free + dirty %-8d flags %#x (",
+		       lp->lnum, lp->free, lp->dirty, c->leb_size - spc, spc,
+		       lp->flags);
 	else
-		printk(KERN_ERR "LEB %-7d free %-8d dirty %-8d used %-8d "
-		       "free + dirty %-8d dark %-4d dead %-4d nodes fit %-3d "
-		       "flags %#-4x (", lp->lnum, lp->free, lp->dirty,
-		       c->leb_size - spc, spc, dark, dead,
-		       (int)(spc / UBIFS_MAX_NODE_SZ), lp->flags);
+		pr_err("LEB %-7d free %-8d dirty %-8d used %-8d free + dirty %-8d dark %-4d dead %-4d nodes fit %-3d flags %#-4x (",
+		       lp->lnum, lp->free, lp->dirty, c->leb_size - spc, spc,
+		       dark, dead, (int)(spc / UBIFS_MAX_NODE_SZ), lp->flags);
 
 	if (lp->flags & LPROPS_TAKEN) {
 		if (lp->flags & LPROPS_INDEX)
-			printk(KERN_CONT "index, taken");
+			pr_cont("index, taken");
 		else
-			printk(KERN_CONT "taken");
+			pr_cont("taken");
 	} else {
 		const char *s;
 
@@ -769,7 +704,7 @@ void ubifs_dump_lprop(const struct ubifs_info *c, const struct ubifs_lprops *lp)
 				break;
 			}
 		}
-		printk(KERN_CONT "%s", s);
+		pr_cont("%s", s);
 	}
 
 	for (rb = rb_first((struct rb_root *)&c->buds); rb; rb = rb_next(rb)) {
@@ -784,19 +719,18 @@ void ubifs_dump_lprop(const struct ubifs_info *c, const struct ubifs_lprops *lp)
 				 */
 				if (c->jheads &&
 				    lp->lnum == c->jheads[i].wbuf.lnum) {
-					printk(KERN_CONT ", jhead %s",
-					       dbg_jhead(i));
+					pr_cont(", jhead %s", dbg_jhead(i));
 					head = 1;
 				}
 			}
 			if (!head)
-				printk(KERN_CONT ", bud of jhead %s",
+				pr_cont(", bud of jhead %s",
 				       dbg_jhead(bud->jhead));
 		}
 	}
 	if (lp->lnum == c->gc_lnum)
-		printk(KERN_CONT ", GC LEB");
-	printk(KERN_CONT ")\n");
+		pr_cont(", GC LEB");
+	pr_cont(")\n");
 }
 
 void ubifs_dump_lprops(struct ubifs_info *c)
@@ -805,8 +739,7 @@ void ubifs_dump_lprops(struct ubifs_info *c)
 	struct ubifs_lprops lp;
 	struct ubifs_lp_stats lst;
 
-	printk(KERN_ERR "(pid %d) start dumping LEB properties\n",
-	       current->pid);
+	pr_err("(pid %d) start dumping LEB properties\n", current->pid);
 	ubifs_get_lp_stats(c, &lst);
 	ubifs_dump_lstats(&lst);
 
@@ -817,8 +750,7 @@ void ubifs_dump_lprops(struct ubifs_info *c)
 
 		ubifs_dump_lprop(c, &lp);
 	}
-	printk(KERN_ERR "(pid %d) finish dumping LEB properties\n",
-	       current->pid);
+	pr_err("(pid %d) finish dumping LEB properties\n", current->pid);
 }
 
 void ubifs_dump_lpt_info(struct ubifs_info *c)
@@ -826,37 +758,36 @@ void ubifs_dump_lpt_info(struct ubifs_info *c)
 	int i;
 
 	spin_lock(&dbg_lock);
-	printk(KERN_ERR "(pid %d) dumping LPT information\n", current->pid);
-	printk(KERN_ERR "\tlpt_sz:        %lld\n", c->lpt_sz);
-	printk(KERN_ERR "\tpnode_sz:      %d\n", c->pnode_sz);
-	printk(KERN_ERR "\tnnode_sz:      %d\n", c->nnode_sz);
-	printk(KERN_ERR "\tltab_sz:       %d\n", c->ltab_sz);
-	printk(KERN_ERR "\tlsave_sz:      %d\n", c->lsave_sz);
-	printk(KERN_ERR "\tbig_lpt:       %d\n", c->big_lpt);
-	printk(KERN_ERR "\tlpt_hght:      %d\n", c->lpt_hght);
-	printk(KERN_ERR "\tpnode_cnt:     %d\n", c->pnode_cnt);
-	printk(KERN_ERR "\tnnode_cnt:     %d\n", c->nnode_cnt);
-	printk(KERN_ERR "\tdirty_pn_cnt:  %d\n", c->dirty_pn_cnt);
-	printk(KERN_ERR "\tdirty_nn_cnt:  %d\n", c->dirty_nn_cnt);
-	printk(KERN_ERR "\tlsave_cnt:     %d\n", c->lsave_cnt);
-	printk(KERN_ERR "\tspace_bits:    %d\n", c->space_bits);
-	printk(KERN_ERR "\tlpt_lnum_bits: %d\n", c->lpt_lnum_bits);
-	printk(KERN_ERR "\tlpt_offs_bits: %d\n", c->lpt_offs_bits);
-	printk(KERN_ERR "\tlpt_spc_bits:  %d\n", c->lpt_spc_bits);
-	printk(KERN_ERR "\tpcnt_bits:     %d\n", c->pcnt_bits);
-	printk(KERN_ERR "\tlnum_bits:     %d\n", c->lnum_bits);
-	printk(KERN_ERR "\tLPT root is at %d:%d\n", c->lpt_lnum, c->lpt_offs);
-	printk(KERN_ERR "\tLPT head is at %d:%d\n",
+	pr_err("(pid %d) dumping LPT information\n", current->pid);
+	pr_err("\tlpt_sz:        %lld\n", c->lpt_sz);
+	pr_err("\tpnode_sz:      %d\n", c->pnode_sz);
+	pr_err("\tnnode_sz:      %d\n", c->nnode_sz);
+	pr_err("\tltab_sz:       %d\n", c->ltab_sz);
+	pr_err("\tlsave_sz:      %d\n", c->lsave_sz);
+	pr_err("\tbig_lpt:       %d\n", c->big_lpt);
+	pr_err("\tlpt_hght:      %d\n", c->lpt_hght);
+	pr_err("\tpnode_cnt:     %d\n", c->pnode_cnt);
+	pr_err("\tnnode_cnt:     %d\n", c->nnode_cnt);
+	pr_err("\tdirty_pn_cnt:  %d\n", c->dirty_pn_cnt);
+	pr_err("\tdirty_nn_cnt:  %d\n", c->dirty_nn_cnt);
+	pr_err("\tlsave_cnt:     %d\n", c->lsave_cnt);
+	pr_err("\tspace_bits:    %d\n", c->space_bits);
+	pr_err("\tlpt_lnum_bits: %d\n", c->lpt_lnum_bits);
+	pr_err("\tlpt_offs_bits: %d\n", c->lpt_offs_bits);
+	pr_err("\tlpt_spc_bits:  %d\n", c->lpt_spc_bits);
+	pr_err("\tpcnt_bits:     %d\n", c->pcnt_bits);
+	pr_err("\tlnum_bits:     %d\n", c->lnum_bits);
+	pr_err("\tLPT root is at %d:%d\n", c->lpt_lnum, c->lpt_offs);
+	pr_err("\tLPT head is at %d:%d\n",
 	       c->nhead_lnum, c->nhead_offs);
-	printk(KERN_ERR "\tLPT ltab is at %d:%d\n",
-	       c->ltab_lnum, c->ltab_offs);
+	pr_err("\tLPT ltab is at %d:%d\n", c->ltab_lnum, c->ltab_offs);
 	if (c->big_lpt)
-		printk(KERN_ERR "\tLPT lsave is at %d:%d\n",
+		pr_err("\tLPT lsave is at %d:%d\n",
 		       c->lsave_lnum, c->lsave_offs);
 	for (i = 0; i < c->lpt_lebs; i++)
-		printk(KERN_ERR "\tLPT LEB %d free %d dirty %d tgc %d "
-		       "cmt %d\n", i + c->lpt_first, c->ltab[i].free,
-		       c->ltab[i].dirty, c->ltab[i].tgc, c->ltab[i].cmt);
+		pr_err("\tLPT LEB %d free %d dirty %d tgc %d cmt %d\n",
+		       i + c->lpt_first, c->ltab[i].free, c->ltab[i].dirty,
+		       c->ltab[i].tgc, c->ltab[i].cmt);
 	spin_unlock(&dbg_lock);
 }
 
@@ -865,13 +796,13 @@ void ubifs_dump_sleb(const struct ubifs_info *c,
 {
 	struct ubifs_scan_node *snod;
 
-	printk(KERN_ERR "(pid %d) start dumping scanned data from LEB %d:%d\n",
+	pr_err("(pid %d) start dumping scanned data from LEB %d:%d\n",
 	       current->pid, sleb->lnum, offs);
 
 	list_for_each_entry(snod, &sleb->nodes, list) {
 		cond_resched();
-		printk(KERN_ERR "Dumping node at LEB %d:%d len %d\n", sleb->lnum,
-		       snod->offs, snod->len);
+		pr_err("Dumping node at LEB %d:%d len %d\n",
+		       sleb->lnum, snod->offs, snod->len);
 		ubifs_dump_node(c, snod->node);
 	}
 }
@@ -882,11 +813,7 @@ void ubifs_dump_leb(const struct ubifs_info *c, int lnum)
 	struct ubifs_scan_node *snod;
 	void *buf;
 
-	if (dbg_is_tst_rcvry(c))
-		return;
-
-	printk(KERN_ERR "(pid %d) start dumping LEB %d\n",
-	       current->pid, lnum);
+	pr_err("(pid %d) start dumping LEB %d\n", current->pid, lnum);
 
 	buf = __vmalloc(c->leb_size, GFP_NOFS, PAGE_KERNEL);
 	if (!buf) {
@@ -900,18 +827,17 @@ void ubifs_dump_leb(const struct ubifs_info *c, int lnum)
 		goto out;
 	}
 
-	printk(KERN_ERR "LEB %d has %d nodes ending at %d\n", lnum,
+	pr_err("LEB %d has %d nodes ending at %d\n", lnum,
 	       sleb->nodes_cnt, sleb->endpt);
 
 	list_for_each_entry(snod, &sleb->nodes, list) {
 		cond_resched();
-		printk(KERN_ERR "Dumping node at LEB %d:%d len %d\n", lnum,
+		pr_err("Dumping node at LEB %d:%d len %d\n", lnum,
 		       snod->offs, snod->len);
 		ubifs_dump_node(c, snod->node);
 	}
 
-	printk(KERN_ERR "(pid %d) finish dumping LEB %d\n",
-	       current->pid, lnum);
+	pr_err("(pid %d) finish dumping LEB %d\n", current->pid, lnum);
 	ubifs_scan_destroy(sleb);
 
 out:
@@ -932,33 +858,28 @@ void ubifs_dump_znode(const struct ubifs_info *c,
 	else
 		zbr = &c->zroot;
 
-	printk(KERN_ERR "znode %p, LEB %d:%d len %d parent %p iip %d level %d"
-	       " child_cnt %d flags %lx\n", znode, zbr->lnum, zbr->offs,
-	       zbr->len, znode->parent, znode->iip, znode->level,
-	       znode->child_cnt, znode->flags);
+	pr_err("znode %p, LEB %d:%d len %d parent %p iip %d level %d child_cnt %d flags %lx\n",
+	       znode, zbr->lnum, zbr->offs, zbr->len, znode->parent, znode->iip,
+	       znode->level, znode->child_cnt, znode->flags);
 
 	if (znode->child_cnt <= 0 || znode->child_cnt > c->fanout) {
 		spin_unlock(&dbg_lock);
 		return;
 	}
 
-	printk(KERN_ERR "zbranches:\n");
+	pr_err("zbranches:\n");
 	for (n = 0; n < znode->child_cnt; n++) {
 		zbr = &znode->zbranch[n];
 		if (znode->level > 0)
-			printk(KERN_ERR "\t%d: znode %p LEB %d:%d len %d key "
-					  "%s\n", n, zbr->znode, zbr->lnum,
-					  zbr->offs, zbr->len,
-					  dbg_snprintf_key(c, &zbr->key,
-							   key_buf,
-							   DBG_KEY_BUF_LEN));
+			pr_err("\t%d: znode %p LEB %d:%d len %d key %s\n",
+			       n, zbr->znode, zbr->lnum, zbr->offs, zbr->len,
+			       dbg_snprintf_key(c, &zbr->key, key_buf,
+						DBG_KEY_BUF_LEN));
 		else
-			printk(KERN_ERR "\t%d: LNC %p LEB %d:%d len %d key "
-					  "%s\n", n, zbr->znode, zbr->lnum,
-					  zbr->offs, zbr->len,
-					  dbg_snprintf_key(c, &zbr->key,
-							   key_buf,
-							   DBG_KEY_BUF_LEN));
+			pr_err("\t%d: LNC %p LEB %d:%d len %d key %s\n",
+			       n, zbr->znode, zbr->lnum, zbr->offs, zbr->len,
+			       dbg_snprintf_key(c, &zbr->key, key_buf,
+						DBG_KEY_BUF_LEN));
 	}
 	spin_unlock(&dbg_lock);
 }
@@ -967,16 +888,16 @@ void ubifs_dump_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat)
 {
 	int i;
 
-	printk(KERN_ERR "(pid %d) start dumping heap cat %d (%d elements)\n",
+	pr_err("(pid %d) start dumping heap cat %d (%d elements)\n",
 	       current->pid, cat, heap->cnt);
 	for (i = 0; i < heap->cnt; i++) {
 		struct ubifs_lprops *lprops = heap->arr[i];
 
-		printk(KERN_ERR "\t%d. LEB %d hpos %d free %d dirty %d "
-		       "flags %d\n", i, lprops->lnum, lprops->hpos,
-		       lprops->free, lprops->dirty, lprops->flags);
+		pr_err("\t%d. LEB %d hpos %d free %d dirty %d flags %d\n",
+		       i, lprops->lnum, lprops->hpos, lprops->free,
+		       lprops->dirty, lprops->flags);
 	}
-	printk(KERN_ERR "(pid %d) finish dumping heap\n", current->pid);
+	pr_err("(pid %d) finish dumping heap\n", current->pid);
 }
 
 void ubifs_dump_pnode(struct ubifs_info *c, struct ubifs_pnode *pnode,
@@ -984,15 +905,15 @@ void ubifs_dump_pnode(struct ubifs_info *c, struct ubifs_pnode *pnode,
 {
 	int i;
 
-	printk(KERN_ERR "(pid %d) dumping pnode:\n", current->pid);
-	printk(KERN_ERR "\taddress %zx parent %zx cnext %zx\n",
+	pr_err("(pid %d) dumping pnode:\n", current->pid);
+	pr_err("\taddress %zx parent %zx cnext %zx\n",
 	       (size_t)pnode, (size_t)parent, (size_t)pnode->cnext);
-	printk(KERN_ERR "\tflags %lu iip %d level %d num %d\n",
+	pr_err("\tflags %lu iip %d level %d num %d\n",
 	       pnode->flags, iip, pnode->level, pnode->num);
 	for (i = 0; i < UBIFS_LPT_FANOUT; i++) {
 		struct ubifs_lprops *lp = &pnode->lprops[i];
 
-		printk(KERN_ERR "\t%d: free %d dirty %d flags %d lnum %d\n",
+		pr_err("\t%d: free %d dirty %d flags %d lnum %d\n",
 		       i, lp->free, lp->dirty, lp->flags, lp->lnum);
 	}
 }
@@ -1002,20 +923,20 @@ void ubifs_dump_tnc(struct ubifs_info *c)
 	struct ubifs_znode *znode;
 	int level;
 
-	printk(KERN_ERR "\n");
-	printk(KERN_ERR "(pid %d) start dumping TNC tree\n", current->pid);
+	pr_err("\n");
+	pr_err("(pid %d) start dumping TNC tree\n", current->pid);
 	znode = ubifs_tnc_levelorder_next(c->zroot.znode, NULL);
 	level = znode->level;
-	printk(KERN_ERR "== Level %d ==\n", level);
+	pr_err("== Level %d ==\n", level);
 	while (znode) {
 		if (level != znode->level) {
 			level = znode->level;
-			printk(KERN_ERR "== Level %d ==\n", level);
+			pr_err("== Level %d ==\n", level);
 		}
 		ubifs_dump_znode(c, znode);
 		znode = ubifs_tnc_levelorder_next(c->zroot.znode, znode);
 	}
-	printk(KERN_ERR "(pid %d) finish dumping TNC tree\n", current->pid);
+	pr_err("(pid %d) finish dumping TNC tree\n", current->pid);
 }
 
 static int dump_znode(struct ubifs_info *c, struct ubifs_znode *znode,
@@ -1154,8 +1075,8 @@ int dbg_check_synced_i_size(const struct ubifs_info *c, struct inode *inode)
 	mutex_lock(&ui->ui_mutex);
 	spin_lock(&ui->ui_lock);
 	if (ui->ui_size != ui->synced_i_size && !ui->dirty) {
-		ubifs_err("ui_size is %lld, synced_i_size is %lld, but inode "
-			  "is clean", ui->ui_size, ui->synced_i_size);
+		ubifs_err("ui_size is %lld, synced_i_size is %lld, but inode is clean",
+			  ui->ui_size, ui->synced_i_size);
 		ubifs_err("i_ino %lu, i_mode %#x, i_size %lld", inode->i_ino,
 			  inode->i_mode, i_size_read(inode));
 		dump_stack();
@@ -1217,17 +1138,16 @@ int dbg_check_dir(struct ubifs_info *c, const struct inode *dir)
 	kfree(pdent);
 
 	if (i_size_read(dir) != size) {
-		ubifs_err("directory inode %lu has size %llu, "
-			  "but calculated size is %llu", dir->i_ino,
-			  (unsigned long long)i_size_read(dir),
+		ubifs_err("directory inode %lu has size %llu, but calculated size is %llu",
+			  dir->i_ino, (unsigned long long)i_size_read(dir),
 			  (unsigned long long)size);
 		ubifs_dump_inode(c, dir);
 		dump_stack();
 		return -EINVAL;
 	}
 	if (dir->i_nlink != nlink) {
-		ubifs_err("directory inode %lu has nlink %u, but calculated "
-			  "nlink is %u", dir->i_ino, dir->i_nlink, nlink);
+		ubifs_err("directory inode %lu has nlink %u, but calculated nlink is %u",
+			  dir->i_ino, dir->i_nlink, nlink);
 		ubifs_dump_inode(c, dir);
 		dump_stack();
 		return -EINVAL;
@@ -1686,8 +1606,8 @@ int dbg_walk_index(struct ubifs_info *c, dbg_leaf_callback leaf_cb,
 		if (znode_cb) {
 			err = znode_cb(c, znode, priv);
 			if (err) {
-				ubifs_err("znode checking function returned "
-					  "error %d", err);
+				ubifs_err("znode checking function returned error %d",
+					  err);
 				ubifs_dump_znode(c, znode);
 				goto out_dump;
 			}
@@ -1697,9 +1617,7 @@ int dbg_walk_index(struct ubifs_info *c, dbg_leaf_callback leaf_cb,
 				zbr = &znode->zbranch[idx];
 				err = leaf_cb(c, zbr, priv);
 				if (err) {
-					ubifs_err("leaf checking function "
-						  "returned error %d, for leaf "
-						  "at LEB %d:%d",
+					ubifs_err("leaf checking function returned error %d, for leaf at LEB %d:%d",
 						  err, zbr->lnum, zbr->offs);
 					goto out_dump;
 				}
@@ -1807,8 +1725,8 @@ int dbg_check_idx_size(struct ubifs_info *c, long long idx_size)
 	}
 
 	if (calc != idx_size) {
-		ubifs_err("index size check failed: calculated size is %lld, "
-			  "should be %lld", calc, idx_size);
+		ubifs_err("index size check failed: calculated size is %lld, should be %lld",
+			  calc, idx_size);
 		dump_stack();
 		return -EINVAL;
 	}
@@ -2120,8 +2038,7 @@ static int check_leaf(struct ubifs_info *c, struct ubifs_zbranch *zbr,
 		fscki = read_add_inode(c, priv, inum);
 		if (IS_ERR(fscki)) {
 			err = PTR_ERR(fscki);
-			ubifs_err("error %d while processing data node and "
-				  "trying to find inode node %lu",
+			ubifs_err("error %d while processing data node and trying to find inode node %lu",
 				  err, (unsigned long)inum);
 			goto out_dump;
 		}
@@ -2131,9 +2048,8 @@ static int check_leaf(struct ubifs_info *c, struct ubifs_zbranch *zbr,
 		blk_offs <<= UBIFS_BLOCK_SHIFT;
 		blk_offs += le32_to_cpu(dn->size);
 		if (blk_offs > fscki->size) {
-			ubifs_err("data node at LEB %d:%d is not within inode "
-				  "size %lld", zbr->lnum, zbr->offs,
-				  fscki->size);
+			ubifs_err("data node at LEB %d:%d is not within inode size %lld",
+				  zbr->lnum, zbr->offs, fscki->size);
 			err = -EINVAL;
 			goto out_dump;
 		}
@@ -2154,8 +2070,7 @@ static int check_leaf(struct ubifs_info *c, struct ubifs_zbranch *zbr,
 		fscki = read_add_inode(c, priv, inum);
 		if (IS_ERR(fscki)) {
 			err = PTR_ERR(fscki);
-			ubifs_err("error %d while processing entry node and "
-				  "trying to find inode node %lu",
+			ubifs_err("error %d while processing entry node and trying to find inode node %lu",
 				  err, (unsigned long)inum);
 			goto out_dump;
 		}
@@ -2167,8 +2082,7 @@ static int check_leaf(struct ubifs_info *c, struct ubifs_zbranch *zbr,
 		fscki1 = read_add_inode(c, priv, inum);
 		if (IS_ERR(fscki1)) {
 			err = PTR_ERR(fscki1);
-			ubifs_err("error %d while processing entry node and "
-				  "trying to find parent inode node %lu",
+			ubifs_err("error %d while processing entry node and trying to find parent inode node %lu",
 				  err, (unsigned long)inum);
 			goto out_dump;
 		}
@@ -2258,61 +2172,52 @@ static int check_inodes(struct ubifs_info *c, struct fsck_data *fsckd)
 			 */
 			if (fscki->inum != UBIFS_ROOT_INO &&
 			    fscki->references != 1) {
-				ubifs_err("directory inode %lu has %d "
-					  "direntries which refer it, but "
-					  "should be 1",
+				ubifs_err("directory inode %lu has %d direntries which refer it, but should be 1",
 					  (unsigned long)fscki->inum,
 					  fscki->references);
 				goto out_dump;
 			}
 			if (fscki->inum == UBIFS_ROOT_INO &&
 			    fscki->references != 0) {
-				ubifs_err("root inode %lu has non-zero (%d) "
-					  "direntries which refer it",
+				ubifs_err("root inode %lu has non-zero (%d) direntries which refer it",
 					  (unsigned long)fscki->inum,
 					  fscki->references);
 				goto out_dump;
 			}
 			if (fscki->calc_sz != fscki->size) {
-				ubifs_err("directory inode %lu size is %lld, "
-					  "but calculated size is %lld",
+				ubifs_err("directory inode %lu size is %lld, but calculated size is %lld",
 					  (unsigned long)fscki->inum,
 					  fscki->size, fscki->calc_sz);
 				goto out_dump;
 			}
 			if (fscki->calc_cnt != fscki->nlink) {
-				ubifs_err("directory inode %lu nlink is %d, "
-					  "but calculated nlink is %d",
+				ubifs_err("directory inode %lu nlink is %d, but calculated nlink is %d",
 					  (unsigned long)fscki->inum,
 					  fscki->nlink, fscki->calc_cnt);
 				goto out_dump;
 			}
 		} else {
 			if (fscki->references != fscki->nlink) {
-				ubifs_err("inode %lu nlink is %d, but "
-					  "calculated nlink is %d",
+				ubifs_err("inode %lu nlink is %d, but calculated nlink is %d",
 					  (unsigned long)fscki->inum,
 					  fscki->nlink, fscki->references);
 				goto out_dump;
 			}
 		}
 		if (fscki->xattr_sz != fscki->calc_xsz) {
-			ubifs_err("inode %lu has xattr size %u, but "
-				  "calculated size is %lld",
+			ubifs_err("inode %lu has xattr size %u, but calculated size is %lld",
 				  (unsigned long)fscki->inum, fscki->xattr_sz,
 				  fscki->calc_xsz);
 			goto out_dump;
 		}
 		if (fscki->xattr_cnt != fscki->calc_xcnt) {
-			ubifs_err("inode %lu has %u xattrs, but "
-				  "calculated count is %lld",
+			ubifs_err("inode %lu has %u xattrs, but calculated count is %lld",
 				  (unsigned long)fscki->inum,
 				  fscki->xattr_cnt, fscki->calc_xcnt);
 			goto out_dump;
 		}
 		if (fscki->xattr_nms != fscki->calc_xnms) {
-			ubifs_err("inode %lu has xattr names' size %u, but "
-				  "calculated names' size is %lld",
+			ubifs_err("inode %lu has xattr names' size %u, but calculated names' size is %lld",
 				  (unsigned long)fscki->inum, fscki->xattr_nms,
 				  fscki->calc_xnms);
 			goto out_dump;
@@ -2652,20 +2557,18 @@ static int power_cut_emulated(struct ubifs_info *c, int lnum, int write)
 	return 1;
 }
 
-static void cut_data(const void *buf, unsigned int len)
+static int corrupt_data(const struct ubifs_info *c, const void *buf,
+			unsigned int len)
 {
 	unsigned int from, to, i, ffs = chance(1, 2);
 	unsigned char *p = (void *)buf;
 
 	from = random32() % (len + 1);
-	if (chance(1, 2))
-		to = random32() % (len - from + 1);
-	else
-		to = len;
+	/* Corruption may only span one max. write unit */
+	to = min(len, ALIGN(from, c->max_write_size));
 
-	if (from < to)
-		ubifs_warn("filled bytes %u-%u with %s", from, to - 1,
-			   ffs ? "0xFFs" : "random data");
+	ubifs_warn("filled bytes %u-%u with %s", from, to - 1,
+		   ffs ? "0xFFs" : "random data");
 
 	if (ffs)
 		for (i = from; i < to; i++)
@@ -2673,6 +2576,8 @@ static void cut_data(const void *buf, unsigned int len)
 	else
 		for (i = from; i < to; i++)
 			p[i] = random32() % 0x100;
+
+	return to;
 }
 
 int dbg_leb_write(struct ubifs_info *c, int lnum, const void *buf,
@@ -2685,7 +2590,9 @@ int dbg_leb_write(struct ubifs_info *c, int lnum, const void *buf,
 
 	failing = power_cut_emulated(c, lnum, 1);
 	if (failing)
-		cut_data(buf, len);
+		len = corrupt_data(c, buf, len);
+	ubifs_warn("actually write %d bytes to LEB %d:%d (the buffer was corrupted)",
+		   len, lnum, offs);
 	err = ubi_leb_write(c->ubi, lnum, buf, offs, len);
 	if (err)
 		return err;
diff --git a/fs/ubifs/debug.h b/fs/ubifs/debug.h
index 760de723dad..e03d5179769 100644
--- a/fs/ubifs/debug.h
+++ b/fs/ubifs/debug.h
@@ -150,7 +150,7 @@ struct ubifs_global_debug_info {
 
 #define ubifs_assert(expr) do {                                                \
 	if (unlikely(!(expr))) {                                               \
-		printk(KERN_CRIT "UBIFS assert failed in %s at %u (pid %d)\n", \
+		pr_crit("UBIFS assert failed in %s at %u (pid %d)\n",          \
 		       __func__, __LINE__, current->pid);                      \
 		dump_stack();                                                  \
 	}                                                                      \
@@ -159,26 +159,23 @@ struct ubifs_global_debug_info {
 #define ubifs_assert_cmt_locked(c) do {                                        \
 	if (unlikely(down_write_trylock(&(c)->commit_sem))) {                  \
 		up_write(&(c)->commit_sem);                                    \
-		printk(KERN_CRIT "commit lock is not locked!\n");              \
+		pr_crit("commit lock is not locked!\n");                       \
 		ubifs_assert(0);                                               \
 	}                                                                      \
 } while (0)
 
 #define ubifs_dbg_msg(type, fmt, ...) \
-	pr_debug("UBIFS DBG " type ": " fmt "\n", ##__VA_ARGS__)
+	pr_debug("UBIFS DBG " type " (pid %d): " fmt "\n", current->pid,       \
+		 ##__VA_ARGS__)
 
 #define DBG_KEY_BUF_LEN 48
 #define ubifs_dbg_msg_key(type, key, fmt, ...) do {                            \
 	char __tmp_key_buf[DBG_KEY_BUF_LEN];                                   \
-	pr_debug("UBIFS DBG " type ": " fmt "%s\n", ##__VA_ARGS__,             \
+	pr_debug("UBIFS DBG " type " (pid %d): " fmt "%s\n", current->pid,     \
+		 ##__VA_ARGS__,                                                \
 		 dbg_snprintf_key(c, key, __tmp_key_buf, DBG_KEY_BUF_LEN));    \
 } while (0)
 
-/* Just a debugging messages not related to any specific UBIFS subsystem */
-#define dbg_msg(fmt, ...)                                                      \
-	printk(KERN_DEBUG "UBIFS DBG (pid %d): %s: " fmt "\n", current->pid,   \
-	       __func__, ##__VA_ARGS__)
-
 /* General messages */
 #define dbg_gen(fmt, ...)   ubifs_dbg_msg("gen", fmt, ##__VA_ARGS__)
 /* Additional journal messages */
diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c
index c95681cf1b7..e271fba1651 100644
--- a/fs/ubifs/dir.c
+++ b/fs/ubifs/dir.c
@@ -980,8 +980,8 @@ static int ubifs_rename(struct inode *old_dir, struct dentry *old_dentry,
 	 * separately.
 	 */
 
-	dbg_gen("dent '%.*s' ino %lu in dir ino %lu to dent '%.*s' in "
-		"dir ino %lu", old_dentry->d_name.len, old_dentry->d_name.name,
+	dbg_gen("dent '%.*s' ino %lu in dir ino %lu to dent '%.*s' in dir ino %lu",
+		old_dentry->d_name.len, old_dentry->d_name.name,
 		old_inode->i_ino, old_dir->i_ino, new_dentry->d_name.len,
 		new_dentry->d_name.name, new_dir->i_ino);
 	ubifs_assert(mutex_is_locked(&old_dir->i_mutex));
diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c
index 7bd6e72afd1..ff48c5a8530 100644
--- a/fs/ubifs/file.c
+++ b/fs/ubifs/file.c
@@ -1486,8 +1486,8 @@ static int ubifs_vm_page_mkwrite(struct vm_area_struct *vma,
 	err = ubifs_budget_space(c, &req);
 	if (unlikely(err)) {
 		if (err == -ENOSPC)
-			ubifs_warn("out of space for mmapped file "
-				   "(inode number %lu)", inode->i_ino);
+			ubifs_warn("out of space for mmapped file (inode number %lu)",
+				   inode->i_ino);
 		return VM_FAULT_SIGBUS;
 	}
 
diff --git a/fs/ubifs/gc.c b/fs/ubifs/gc.c
index 04dd6f47635..76ca53cd3ee 100644
--- a/fs/ubifs/gc.c
+++ b/fs/ubifs/gc.c
@@ -714,9 +714,9 @@ int ubifs_garbage_collect(struct ubifs_info *c, int anyway)
 			break;
 		}
 
-		dbg_gc("found LEB %d: free %d, dirty %d, sum %d "
-		       "(min. space %d)", lp.lnum, lp.free, lp.dirty,
-		       lp.free + lp.dirty, min_space);
+		dbg_gc("found LEB %d: free %d, dirty %d, sum %d (min. space %d)",
+		       lp.lnum, lp.free, lp.dirty, lp.free + lp.dirty,
+		       min_space);
 
 		space_before = c->leb_size - wbuf->offs - wbuf->used;
 		if (wbuf->lnum == -1)
diff --git a/fs/ubifs/log.c b/fs/ubifs/log.c
index c80b15d6c8d..36bd4efd081 100644
--- a/fs/ubifs/log.c
+++ b/fs/ubifs/log.c
@@ -315,17 +315,15 @@ static void remove_buds(struct ubifs_info *c)
 			 * heads (non-closed buds).
 			 */
 			c->cmt_bud_bytes += wbuf->offs - bud->start;
-			dbg_log("preserve %d:%d, jhead %s, bud bytes %d, "
-				"cmt_bud_bytes %lld", bud->lnum, bud->start,
-				dbg_jhead(bud->jhead), wbuf->offs - bud->start,
-				c->cmt_bud_bytes);
+			dbg_log("preserve %d:%d, jhead %s, bud bytes %d, cmt_bud_bytes %lld",
+				bud->lnum, bud->start, dbg_jhead(bud->jhead),
+				wbuf->offs - bud->start, c->cmt_bud_bytes);
 			bud->start = wbuf->offs;
 		} else {
 			c->cmt_bud_bytes += c->leb_size - bud->start;
-			dbg_log("remove %d:%d, jhead %s, bud bytes %d, "
-				"cmt_bud_bytes %lld", bud->lnum, bud->start,
-				dbg_jhead(bud->jhead), c->leb_size - bud->start,
-				c->cmt_bud_bytes);
+			dbg_log("remove %d:%d, jhead %s, bud bytes %d, cmt_bud_bytes %lld",
+				bud->lnum, bud->start, dbg_jhead(bud->jhead),
+				c->leb_size - bud->start, c->cmt_bud_bytes);
 			rb_erase(p1, &c->buds);
 			/*
 			 * If the commit does not finish, the recovery will need
diff --git a/fs/ubifs/lprops.c b/fs/ubifs/lprops.c
index 86eb8e53324..e5a2a35a46d 100644
--- a/fs/ubifs/lprops.c
+++ b/fs/ubifs/lprops.c
@@ -867,15 +867,15 @@ int dbg_check_cats(struct ubifs_info *c)
 
 	list_for_each_entry(lprops, &c->empty_list, list) {
 		if (lprops->free != c->leb_size) {
-			ubifs_err("non-empty LEB %d on empty list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("non-empty LEB %d on empty list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 		if (lprops->flags & LPROPS_TAKEN) {
-			ubifs_err("taken LEB %d on empty list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("taken LEB %d on empty list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 	}
@@ -883,15 +883,15 @@ int dbg_check_cats(struct ubifs_info *c)
 	i = 0;
 	list_for_each_entry(lprops, &c->freeable_list, list) {
 		if (lprops->free + lprops->dirty != c->leb_size) {
-			ubifs_err("non-freeable LEB %d on freeable list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("non-freeable LEB %d on freeable list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 		if (lprops->flags & LPROPS_TAKEN) {
-			ubifs_err("taken LEB %d on freeable list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("taken LEB %d on freeable list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 		i += 1;
@@ -913,21 +913,21 @@ int dbg_check_cats(struct ubifs_info *c)
 
 	list_for_each_entry(lprops, &c->frdi_idx_list, list) {
 		if (lprops->free + lprops->dirty != c->leb_size) {
-			ubifs_err("non-freeable LEB %d on frdi_idx list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("non-freeable LEB %d on frdi_idx list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 		if (lprops->flags & LPROPS_TAKEN) {
-			ubifs_err("taken LEB %d on frdi_idx list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("taken LEB %d on frdi_idx list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 		if (!(lprops->flags & LPROPS_INDEX)) {
-			ubifs_err("non-index LEB %d on frdi_idx list "
-				  "(free %d dirty %d flags %d)", lprops->lnum,
-				  lprops->free, lprops->dirty, lprops->flags);
+			ubifs_err("non-index LEB %d on frdi_idx list (free %d dirty %d flags %d)",
+				  lprops->lnum, lprops->free, lprops->dirty,
+				  lprops->flags);
 			return -EINVAL;
 		}
 	}
@@ -982,9 +982,9 @@ void dbg_check_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat,
 			goto out;
 		}
 		if (lprops != lp) {
-			dbg_msg("lprops %zx lp %zx lprops->lnum %d lp->lnum %d",
-				(size_t)lprops, (size_t)lp, lprops->lnum,
-				lp->lnum);
+			ubifs_err("lprops %zx lp %zx lprops->lnum %d lp->lnum %d",
+				  (size_t)lprops, (size_t)lp, lprops->lnum,
+				  lp->lnum);
 			err = 4;
 			goto out;
 		}
@@ -1002,7 +1002,7 @@ void dbg_check_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat,
 	}
 out:
 	if (err) {
-		dbg_msg("failed cat %d hpos %d err %d", cat, i, err);
+		ubifs_err("failed cat %d hpos %d err %d", cat, i, err);
 		dump_stack();
 		ubifs_dump_heap(c, heap, cat);
 	}
@@ -1153,8 +1153,8 @@ static int scan_check_cb(struct ubifs_info *c,
 
 	if (free > c->leb_size || free < 0 || dirty > c->leb_size ||
 	    dirty < 0) {
-		ubifs_err("bad calculated accounting for LEB %d: "
-			  "free %d, dirty %d", lnum, free, dirty);
+		ubifs_err("bad calculated accounting for LEB %d: free %d, dirty %d",
+			  lnum, free, dirty);
 		goto out_destroy;
 	}
 
@@ -1200,8 +1200,7 @@ static int scan_check_cb(struct ubifs_info *c,
 			/* Free but not unmapped LEB, it's fine */
 			is_idx = 0;
 		else {
-			ubifs_err("indexing node without indexing "
-				  "flag");
+			ubifs_err("indexing node without indexing flag");
 			goto out_print;
 		}
 	}
@@ -1236,8 +1235,7 @@ static int scan_check_cb(struct ubifs_info *c,
 	return LPT_SCAN_CONTINUE;
 
 out_print:
-	ubifs_err("bad accounting of LEB %d: free %d, dirty %d flags %#x, "
-		  "should be free %d, dirty %d",
+	ubifs_err("bad accounting of LEB %d: free %d, dirty %d flags %#x, should be free %d, dirty %d",
 		  lnum, lp->free, lp->dirty, lp->flags, free, dirty);
 	ubifs_dump_leb(c, lnum);
 out_destroy:
@@ -1290,12 +1288,10 @@ int dbg_check_lprops(struct ubifs_info *c)
 	    lst.total_dirty != c->lst.total_dirty ||
 	    lst.total_used != c->lst.total_used) {
 		ubifs_err("bad overall accounting");
-		ubifs_err("calculated: empty_lebs %d, idx_lebs %d, "
-			  "total_free %lld, total_dirty %lld, total_used %lld",
+		ubifs_err("calculated: empty_lebs %d, idx_lebs %d, total_free %lld, total_dirty %lld, total_used %lld",
 			  lst.empty_lebs, lst.idx_lebs, lst.total_free,
 			  lst.total_dirty, lst.total_used);
-		ubifs_err("read from lprops: empty_lebs %d, idx_lebs %d, "
-			  "total_free %lld, total_dirty %lld, total_used %lld",
+		ubifs_err("read from lprops: empty_lebs %d, idx_lebs %d, total_free %lld, total_dirty %lld, total_used %lld",
 			  c->lst.empty_lebs, c->lst.idx_lebs, c->lst.total_free,
 			  c->lst.total_dirty, c->lst.total_used);
 		err = -EINVAL;
diff --git a/fs/ubifs/lpt.c b/fs/ubifs/lpt.c
index 8640920766e..d46b19ec181 100644
--- a/fs/ubifs/lpt.c
+++ b/fs/ubifs/lpt.c
@@ -1311,7 +1311,7 @@ out:
 	ubifs_err("error %d reading pnode at %d:%d", err, lnum, offs);
 	ubifs_dump_pnode(c, pnode, parent, iip);
 	dump_stack();
-	dbg_msg("calc num: %d", calc_pnode_num_from_parent(c, parent, iip));
+	ubifs_err("calc num: %d", calc_pnode_num_from_parent(c, parent, iip));
 	kfree(pnode);
 	return err;
 }
@@ -2237,8 +2237,7 @@ int dbg_check_lpt_nodes(struct ubifs_info *c, struct ubifs_cnode *cnode,
 			/* cnode is a nnode */
 			num = calc_nnode_num(row, col);
 			if (cnode->num != num) {
-				ubifs_err("nnode num %d expected %d "
-					  "parent num %d iip %d",
+				ubifs_err("nnode num %d expected %d parent num %d iip %d",
 					  cnode->num, num,
 					  (nnode ? nnode->num : 0), cnode->iip);
 				return -EINVAL;
diff --git a/fs/ubifs/lpt_commit.c b/fs/ubifs/lpt_commit.c
index 4fa70734e6e..9daaeef675d 100644
--- a/fs/ubifs/lpt_commit.c
+++ b/fs/ubifs/lpt_commit.c
@@ -320,8 +320,8 @@ static int layout_cnodes(struct ubifs_info *c)
 	return 0;
 
 no_space:
-	ubifs_err("LPT out of space at LEB %d:%d needing %d, done_ltab %d, "
-		  "done_lsave %d", lnum, offs, len, done_ltab, done_lsave);
+	ubifs_err("LPT out of space at LEB %d:%d needing %d, done_ltab %d, done_lsave %d",
+		  lnum, offs, len, done_ltab, done_lsave);
 	ubifs_dump_lpt_info(c);
 	ubifs_dump_lpt_lebs(c);
 	dump_stack();
@@ -545,8 +545,8 @@ static int write_cnodes(struct ubifs_info *c)
 	return 0;
 
 no_space:
-	ubifs_err("LPT out of space mismatch at LEB %d:%d needing %d, done_ltab "
-		  "%d, done_lsave %d", lnum, offs, len, done_ltab, done_lsave);
+	ubifs_err("LPT out of space mismatch at LEB %d:%d needing %d, done_ltab %d, done_lsave %d",
+		  lnum, offs, len, done_ltab, done_lsave);
 	ubifs_dump_lpt_info(c);
 	ubifs_dump_lpt_lebs(c);
 	dump_stack();
@@ -1662,21 +1662,19 @@ static int dbg_check_ltab_lnum(struct ubifs_info *c, int lnum)
 				continue;
 			}
 			if (!dbg_is_all_ff(p, len)) {
-				dbg_msg("invalid empty space in LEB %d at %d",
-					lnum, c->leb_size - len);
+				ubifs_err("invalid empty space in LEB %d at %d",
+					  lnum, c->leb_size - len);
 				err = -EINVAL;
 			}
 			i = lnum - c->lpt_first;
 			if (len != c->ltab[i].free) {
-				dbg_msg("invalid free space in LEB %d "
-					"(free %d, expected %d)",
-					lnum, len, c->ltab[i].free);
+				ubifs_err("invalid free space in LEB %d (free %d, expected %d)",
+					  lnum, len, c->ltab[i].free);
 				err = -EINVAL;
 			}
 			if (dirty != c->ltab[i].dirty) {
-				dbg_msg("invalid dirty space in LEB %d "
-					"(dirty %d, expected %d)",
-					lnum, dirty, c->ltab[i].dirty);
+				ubifs_err("invalid dirty space in LEB %d (dirty %d, expected %d)",
+					  lnum, dirty, c->ltab[i].dirty);
 				err = -EINVAL;
 			}
 			goto out;
@@ -1888,8 +1886,7 @@ static void dump_lpt_leb(const struct ubifs_info *c, int lnum)
 	int err, len = c->leb_size, node_type, node_num, node_len, offs;
 	void *buf, *p;
 
-	printk(KERN_DEBUG "(pid %d) start dumping LEB %d\n",
-	       current->pid, lnum);
+	pr_err("(pid %d) start dumping LEB %d\n", current->pid, lnum);
 	buf = p = __vmalloc(c->leb_size, GFP_NOFS, PAGE_KERNEL);
 	if (!buf) {
 		ubifs_err("cannot allocate memory to dump LPT");
@@ -1907,14 +1904,14 @@ static void dump_lpt_leb(const struct ubifs_info *c, int lnum)
 
 			pad_len = get_pad_len(c, p, len);
 			if (pad_len) {
-				printk(KERN_DEBUG "LEB %d:%d, pad %d bytes\n",
+				pr_err("LEB %d:%d, pad %d bytes\n",
 				       lnum, offs, pad_len);
 				p += pad_len;
 				len -= pad_len;
 				continue;
 			}
 			if (len)
-				printk(KERN_DEBUG "LEB %d:%d, free %d bytes\n",
+				pr_err("LEB %d:%d, free %d bytes\n",
 				       lnum, offs, len);
 			break;
 		}
@@ -1925,11 +1922,10 @@ static void dump_lpt_leb(const struct ubifs_info *c, int lnum)
 		{
 			node_len = c->pnode_sz;
 			if (c->big_lpt)
-				printk(KERN_DEBUG "LEB %d:%d, pnode num %d\n",
+				pr_err("LEB %d:%d, pnode num %d\n",
 				       lnum, offs, node_num);
 			else
-				printk(KERN_DEBUG "LEB %d:%d, pnode\n",
-				       lnum, offs);
+				pr_err("LEB %d:%d, pnode\n", lnum, offs);
 			break;
 		}
 		case UBIFS_LPT_NNODE:
@@ -1939,29 +1935,28 @@ static void dump_lpt_leb(const struct ubifs_info *c, int lnum)
 
 			node_len = c->nnode_sz;
 			if (c->big_lpt)
-				printk(KERN_DEBUG "LEB %d:%d, nnode num %d, ",
+				pr_err("LEB %d:%d, nnode num %d, ",
 				       lnum, offs, node_num);
 			else
-				printk(KERN_DEBUG "LEB %d:%d, nnode, ",
+				pr_err("LEB %d:%d, nnode, ",
 				       lnum, offs);
 			err = ubifs_unpack_nnode(c, p, &nnode);
 			for (i = 0; i < UBIFS_LPT_FANOUT; i++) {
-				printk(KERN_CONT "%d:%d", nnode.nbranch[i].lnum,
+				pr_cont("%d:%d", nnode.nbranch[i].lnum,
 				       nnode.nbranch[i].offs);
 				if (i != UBIFS_LPT_FANOUT - 1)
-					printk(KERN_CONT ", ");
+					pr_cont(", ");
 			}
-			printk(KERN_CONT "\n");
+			pr_cont("\n");
 			break;
 		}
 		case UBIFS_LPT_LTAB:
 			node_len = c->ltab_sz;
-			printk(KERN_DEBUG "LEB %d:%d, ltab\n",
-			       lnum, offs);
+			pr_err("LEB %d:%d, ltab\n", lnum, offs);
 			break;
 		case UBIFS_LPT_LSAVE:
 			node_len = c->lsave_sz;
-			printk(KERN_DEBUG "LEB %d:%d, lsave len\n", lnum, offs);
+			pr_err("LEB %d:%d, lsave len\n", lnum, offs);
 			break;
 		default:
 			ubifs_err("LPT node type %d not recognized", node_type);
@@ -1972,8 +1967,7 @@ static void dump_lpt_leb(const struct ubifs_info *c, int lnum)
 		len -= node_len;
 	}
 
-	printk(KERN_DEBUG "(pid %d) finish dumping LEB %d\n",
-	       current->pid, lnum);
+	pr_err("(pid %d) finish dumping LEB %d\n", current->pid, lnum);
 out:
 	vfree(buf);
 	return;
@@ -1990,12 +1984,10 @@ void ubifs_dump_lpt_lebs(const struct ubifs_info *c)
 {
 	int i;
 
-	printk(KERN_DEBUG "(pid %d) start dumping all LPT LEBs\n",
-	       current->pid);
+	pr_err("(pid %d) start dumping all LPT LEBs\n", current->pid);
 	for (i = 0; i < c->lpt_lebs; i++)
 		dump_lpt_leb(c, i + c->lpt_first);
-	printk(KERN_DEBUG "(pid %d) finish dumping all LPT LEBs\n",
-	       current->pid);
+	pr_err("(pid %d) finish dumping all LPT LEBs\n", current->pid);
 }
 
 /**
diff --git a/fs/ubifs/orphan.c b/fs/ubifs/orphan.c
index cebf17ea045..769701ccb5c 100644
--- a/fs/ubifs/orphan.c
+++ b/fs/ubifs/orphan.c
@@ -562,8 +562,8 @@ static int do_kill_orphans(struct ubifs_info *c, struct ubifs_scan_leb *sleb,
 
 	list_for_each_entry(snod, &sleb->nodes, list) {
 		if (snod->type != UBIFS_ORPH_NODE) {
-			ubifs_err("invalid node type %d in orphan area at "
-				  "%d:%d", snod->type, sleb->lnum, snod->offs);
+			ubifs_err("invalid node type %d in orphan area at %d:%d",
+				  snod->type, sleb->lnum, snod->offs);
 			ubifs_dump_node(c, snod->node);
 			return -EINVAL;
 		}
@@ -589,8 +589,7 @@ static int do_kill_orphans(struct ubifs_info *c, struct ubifs_scan_leb *sleb,
 			 * number. That makes this orphan node, out of date.
 			 */
 			if (!first) {
-				ubifs_err("out of order commit number %llu in "
-					  "orphan node at %d:%d",
+				ubifs_err("out of order commit number %llu in orphan node at %d:%d",
 					  cmt_no, sleb->lnum, snod->offs);
 				ubifs_dump_node(c, snod->node);
 				return -EINVAL;
diff --git a/fs/ubifs/recovery.c b/fs/ubifs/recovery.c
index edeec499c04..065096e36ed 100644
--- a/fs/ubifs/recovery.c
+++ b/fs/ubifs/recovery.c
@@ -609,7 +609,8 @@ static void drop_last_node(struct ubifs_scan_leb *sleb, int *offs)
 		snod = list_entry(sleb->nodes.prev, struct ubifs_scan_node,
 				  list);
 
-		dbg_rcvry("dropping last node at %d:%d", sleb->lnum, snod->offs);
+		dbg_rcvry("dropping last node at %d:%d",
+			  sleb->lnum, snod->offs);
 		*offs = snod->offs;
 		list_del(&snod->list);
 		kfree(snod);
@@ -702,8 +703,8 @@ struct ubifs_scan_leb *ubifs_recover_leb(struct ubifs_info *c, int lnum,
 			 * See header comment for this file for more
 			 * explanations about the reasons we have this check.
 			 */
-			ubifs_err("corrupt empty space LEB %d:%d, corruption "
-				  "starts at %d", lnum, offs, corruption);
+			ubifs_err("corrupt empty space LEB %d:%d, corruption starts at %d",
+				  lnum, offs, corruption);
 			/* Make sure we dump interesting non-0xFF data */
 			offs += corruption;
 			buf += corruption;
@@ -899,8 +900,8 @@ struct ubifs_scan_leb *ubifs_recover_log_leb(struct ubifs_info *c, int lnum,
 				}
 			}
 			if (snod->sqnum > cs_sqnum) {
-				ubifs_err("unrecoverable log corruption "
-					  "in LEB %d", lnum);
+				ubifs_err("unrecoverable log corruption in LEB %d",
+					  lnum);
 				ubifs_scan_destroy(sleb);
 				return ERR_PTR(-EUCLEAN);
 			}
diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c
index 94d78fc5d4e..3187925e987 100644
--- a/fs/ubifs/replay.c
+++ b/fs/ubifs/replay.c
@@ -141,9 +141,9 @@ static int set_bud_lprops(struct ubifs_info *c, struct bud_entry *b)
 		 * during the replay.
 		 */
 		if (dirty != 0)
-			dbg_msg("LEB %d lp: %d free %d dirty "
-				"replay: %d free %d dirty", b->bud->lnum,
-				lp->free, lp->dirty, b->free, b->dirty);
+			dbg_mnt("LEB %d lp: %d free %d dirty replay: %d free %d dirty",
+				b->bud->lnum, lp->free, lp->dirty, b->free,
+				b->dirty);
 	}
 	lp = ubifs_change_lp(c, lp, b->free, dirty + b->dirty,
 			     lp->flags | LPROPS_TAKEN, 0);
@@ -677,7 +677,8 @@ static int replay_bud(struct ubifs_info *c, struct bud_entry *b)
 
 	b->dirty = sleb->endpt - offs - used;
 	b->free = c->leb_size - sleb->endpt;
-	dbg_mnt("bud LEB %d replied: dirty %d, free %d", lnum, b->dirty, b->free);
+	dbg_mnt("bud LEB %d replied: dirty %d, free %d",
+		lnum, b->dirty, b->free);
 
 out:
 	ubifs_scan_destroy(sleb);
@@ -865,8 +866,7 @@ static int replay_log_leb(struct ubifs_info *c, int lnum, int offs, void *sbuf)
 			goto out_dump;
 		}
 		if (le64_to_cpu(node->cmt_no) != c->cmt_no) {
-			ubifs_err("first CS node at LEB %d:%d has wrong "
-				  "commit number %llu expected %llu",
+			ubifs_err("first CS node at LEB %d:%d has wrong commit number %llu expected %llu",
 				  lnum, offs,
 				  (unsigned long long)le64_to_cpu(node->cmt_no),
 				  c->cmt_no);
@@ -1058,8 +1058,8 @@ int ubifs_replay_journal(struct ubifs_info *c)
 	c->bi.uncommitted_idx *= c->max_idx_node_sz;
 
 	ubifs_assert(c->bud_bytes <= c->max_bud_bytes || c->need_recovery);
-	dbg_mnt("finished, log head LEB %d:%d, max_sqnum %llu, "
-		"highest_inum %lu", c->lhead_lnum, c->lhead_offs, c->max_sqnum,
+	dbg_mnt("finished, log head LEB %d:%d, max_sqnum %llu, highest_inum %lu",
+		c->lhead_lnum, c->lhead_offs, c->max_sqnum,
 		(unsigned long)c->highest_inum);
 out:
 	destroy_replay_list(c);
diff --git a/fs/ubifs/sb.c b/fs/ubifs/sb.c
index 52c21f4190f..4c37607a958 100644
--- a/fs/ubifs/sb.c
+++ b/fs/ubifs/sb.c
@@ -391,9 +391,8 @@ static int validate_sb(struct ubifs_info *c, struct ubifs_sb_node *sup)
 	min_leb_cnt += c->lpt_lebs + c->orph_lebs + c->jhead_cnt + 6;
 
 	if (c->leb_cnt < min_leb_cnt || c->leb_cnt > c->vi.size) {
-		ubifs_err("bad LEB count: %d in superblock, %d on UBI volume, "
-			  "%d minimum required", c->leb_cnt, c->vi.size,
-			  min_leb_cnt);
+		ubifs_err("bad LEB count: %d in superblock, %d on UBI volume, %d minimum required",
+			  c->leb_cnt, c->vi.size, min_leb_cnt);
 		goto failed;
 	}
 
@@ -411,15 +410,14 @@ static int validate_sb(struct ubifs_info *c, struct ubifs_sb_node *sup)
 
 	max_bytes = (long long)c->leb_size * UBIFS_MIN_BUD_LEBS;
 	if (c->max_bud_bytes < max_bytes) {
-		ubifs_err("too small journal (%lld bytes), must be at least "
-			  "%lld bytes",  c->max_bud_bytes, max_bytes);
+		ubifs_err("too small journal (%lld bytes), must be at least %lld bytes",
+			  c->max_bud_bytes, max_bytes);
 		goto failed;
 	}
 
 	max_bytes = (long long)c->leb_size * c->main_lebs;
 	if (c->max_bud_bytes > max_bytes) {
-		ubifs_err("too large journal size (%lld bytes), only %lld bytes"
-			  "available in the main area",
+		ubifs_err("too large journal size (%lld bytes), only %lld bytes available in the main area",
 			  c->max_bud_bytes, max_bytes);
 		goto failed;
 	}
@@ -549,10 +547,9 @@ int ubifs_read_superblock(struct ubifs_info *c)
 		ubifs_assert(!c->ro_media || c->ro_mount);
 		if (!c->ro_mount ||
 		    c->ro_compat_version > UBIFS_RO_COMPAT_VERSION) {
-			ubifs_err("on-flash format version is w%d/r%d, but "
-				  "software only supports up to version "
-				  "w%d/r%d", c->fmt_version,
-				  c->ro_compat_version, UBIFS_FORMAT_VERSION,
+			ubifs_err("on-flash format version is w%d/r%d, but software only supports up to version w%d/r%d",
+				  c->fmt_version, c->ro_compat_version,
+				  UBIFS_FORMAT_VERSION,
 				  UBIFS_RO_COMPAT_VERSION);
 			if (c->ro_compat_version <= UBIFS_RO_COMPAT_VERSION) {
 				ubifs_msg("only R/O mounting is possible");
diff --git a/fs/ubifs/scan.c b/fs/ubifs/scan.c
index 7c40e6025fd..58aa05df2bb 100644
--- a/fs/ubifs/scan.c
+++ b/fs/ubifs/scan.c
@@ -75,7 +75,7 @@ int ubifs_scan_a_node(const struct ubifs_info *c, void *buf, int len, int lnum,
 	magic = le32_to_cpu(ch->magic);
 
 	if (magic == 0xFFFFFFFF) {
-		dbg_scan("hit empty space");
+		dbg_scan("hit empty space at LEB %d:%d", lnum, offs);
 		return SCANNED_EMPTY_SPACE;
 	}
 
@@ -85,7 +85,8 @@ int ubifs_scan_a_node(const struct ubifs_info *c, void *buf, int len, int lnum,
 	if (len < UBIFS_CH_SZ)
 		return SCANNED_GARBAGE;
 
-	dbg_scan("scanning %s", dbg_ntype(ch->node_type));
+	dbg_scan("scanning %s at LEB %d:%d",
+		 dbg_ntype(ch->node_type), lnum, offs);
 
 	if (ubifs_check_node(c, buf, lnum, offs, quiet, 1))
 		return SCANNED_A_CORRUPT_NODE;
@@ -114,8 +115,8 @@ int ubifs_scan_a_node(const struct ubifs_info *c, void *buf, int len, int lnum,
 			return SCANNED_A_BAD_PAD_NODE;
 		}
 
-		dbg_scan("%d bytes padded, offset now %d",
-			 pad_len, ALIGN(offs + node_len + pad_len, 8));
+		dbg_scan("%d bytes padded at LEB %d:%d, offset now %d", pad_len,
+			 lnum, offs, ALIGN(offs + node_len + pad_len, 8));
 
 		return node_len + pad_len;
 	}
@@ -150,8 +151,8 @@ struct ubifs_scan_leb *ubifs_start_scan(const struct ubifs_info *c, int lnum,
 
 	err = ubifs_leb_read(c, lnum, sbuf + offs, offs, c->leb_size - offs, 0);
 	if (err && err != -EBADMSG) {
-		ubifs_err("cannot read %d bytes from LEB %d:%d,"
-			  " error %d", c->leb_size - offs, lnum, offs, err);
+		ubifs_err("cannot read %d bytes from LEB %d:%d, error %d",
+			  c->leb_size - offs, lnum, offs, err);
 		kfree(sleb);
 		return ERR_PTR(err);
 	}
@@ -240,8 +241,6 @@ void ubifs_scanned_corruption(const struct ubifs_info *c, int lnum, int offs,
 	int len;
 
 	ubifs_err("corruption at LEB %d:%d", lnum, offs);
-	if (dbg_is_tst_rcvry(c))
-		return;
 	len = c->leb_size - offs;
 	if (len > 8192)
 		len = 8192;
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 681f3a94244..ddc0f6ae65e 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -89,9 +89,8 @@ static int validate_inode(struct ubifs_info *c, const struct inode *inode)
 		return 5;
 
 	if (!ubifs_compr_present(ui->compr_type)) {
-		ubifs_warn("inode %lu uses '%s' compression, but it was not "
-			   "compiled in", inode->i_ino,
-			   ubifs_compr_name(ui->compr_type));
+		ubifs_warn("inode %lu uses '%s' compression, but it was not compiled in",
+			   inode->i_ino, ubifs_compr_name(ui->compr_type));
 	}
 
 	err = dbg_check_dir(c, inode);
@@ -1061,8 +1060,8 @@ static int ubifs_parse_options(struct ubifs_info *c, char *options,
 
 			flag = parse_standard_option(p);
 			if (!flag) {
-				ubifs_err("unrecognized mount option \"%s\" "
-					  "or missing value", p);
+				ubifs_err("unrecognized mount option \"%s\" or missing value",
+					  p);
 				return -EINVAL;
 			}
 			sb->s_flags |= flag;
@@ -1124,8 +1123,8 @@ again:
 		}
 
 		/* Just disable bulk-read */
-		ubifs_warn("Cannot allocate %d bytes of memory for bulk-read, "
-			   "disabling it", c->max_bu_buf_len);
+		ubifs_warn("cannot allocate %d bytes of memory for bulk-read, disabling it",
+			   c->max_bu_buf_len);
 		c->mount_opts.bulk_read = 1;
 		c->bulk_read = 0;
 		return;
@@ -1161,7 +1160,7 @@ static int check_free_space(struct ubifs_info *c)
 static int mount_ubifs(struct ubifs_info *c)
 {
 	int err;
-	long long x;
+	long long x, y;
 	size_t sz;
 
 	c->ro_mount = !!(c->vfs_sb->s_flags & MS_RDONLY);
@@ -1411,75 +1410,69 @@ static int mount_ubifs(struct ubifs_info *c)
 
 	c->mounting = 0;
 
-	ubifs_msg("mounted UBI device %d, volume %d, name \"%s\"",
-		  c->vi.ubi_num, c->vi.vol_id, c->vi.name);
-	if (c->ro_mount)
-		ubifs_msg("mounted read-only");
+	ubifs_msg("mounted UBI device %d, volume %d, name \"%s\"%s",
+		  c->vi.ubi_num, c->vi.vol_id, c->vi.name,
+		  c->ro_mount ? ", R/O mode" : NULL);
 	x = (long long)c->main_lebs * c->leb_size;
-	ubifs_msg("file system size:   %lld bytes (%lld KiB, %lld MiB, %d "
-		  "LEBs)", x, x >> 10, x >> 20, c->main_lebs);
-	x = (long long)c->log_lebs * c->leb_size + c->max_bud_bytes;
-	ubifs_msg("journal size:       %lld bytes (%lld KiB, %lld MiB, %d "
-		  "LEBs)", x, x >> 10, x >> 20, c->log_lebs + c->max_bud_cnt);
-	ubifs_msg("media format:       w%d/r%d (latest is w%d/r%d)",
+	y = (long long)c->log_lebs * c->leb_size + c->max_bud_bytes;
+	ubifs_msg("LEB size: %d bytes (%d KiB), min./max. I/O unit sizes: %d bytes/%d bytes",
+		  c->leb_size, c->leb_size >> 10, c->min_io_size,
+		  c->max_write_size);
+	ubifs_msg("FS size: %lld bytes (%lld MiB, %d LEBs), journal size %lld bytes (%lld MiB, %d LEBs)",
+		  x, x >> 20, c->main_lebs,
+		  y, y >> 20, c->log_lebs + c->max_bud_cnt);
+	ubifs_msg("reserved for root: %llu bytes (%llu KiB)",
+		  c->report_rp_size, c->report_rp_size >> 10);
+	ubifs_msg("media format: w%d/r%d (latest is w%d/r%d), UUID %pUB%s",
 		  c->fmt_version, c->ro_compat_version,
-		  UBIFS_FORMAT_VERSION, UBIFS_RO_COMPAT_VERSION);
-	ubifs_msg("default compressor: %s", ubifs_compr_name(c->default_compr));
-	ubifs_msg("reserved for root:  %llu bytes (%llu KiB)",
-		c->report_rp_size, c->report_rp_size >> 10);
-
-	dbg_msg("compiled on:         " __DATE__ " at " __TIME__);
-	dbg_msg("min. I/O unit size:  %d bytes", c->min_io_size);
-	dbg_msg("max. write size:     %d bytes", c->max_write_size);
-	dbg_msg("LEB size:            %d bytes (%d KiB)",
-		c->leb_size, c->leb_size >> 10);
-	dbg_msg("data journal heads:  %d",
+		  UBIFS_FORMAT_VERSION, UBIFS_RO_COMPAT_VERSION, c->uuid,
+		  c->big_lpt ? ", big LPT model" : ", small LPT model");
+
+	dbg_gen("default compressor:  %s", ubifs_compr_name(c->default_compr));
+	dbg_gen("data journal heads:  %d",
 		c->jhead_cnt - NONDATA_JHEADS_CNT);
-	dbg_msg("UUID:                %pUB", c->uuid);
-	dbg_msg("big_lpt              %d", c->big_lpt);
-	dbg_msg("log LEBs:            %d (%d - %d)",
+	dbg_gen("log LEBs:            %d (%d - %d)",
 		c->log_lebs, UBIFS_LOG_LNUM, c->log_last);
-	dbg_msg("LPT area LEBs:       %d (%d - %d)",
+	dbg_gen("LPT area LEBs:       %d (%d - %d)",
 		c->lpt_lebs, c->lpt_first, c->lpt_last);
-	dbg_msg("orphan area LEBs:    %d (%d - %d)",
+	dbg_gen("orphan area LEBs:    %d (%d - %d)",
 		c->orph_lebs, c->orph_first, c->orph_last);
-	dbg_msg("main area LEBs:      %d (%d - %d)",
+	dbg_gen("main area LEBs:      %d (%d - %d)",
 		c->main_lebs, c->main_first, c->leb_cnt - 1);
-	dbg_msg("index LEBs:          %d", c->lst.idx_lebs);
-	dbg_msg("total index bytes:   %lld (%lld KiB, %lld MiB)",
+	dbg_gen("index LEBs:          %d", c->lst.idx_lebs);
+	dbg_gen("total index bytes:   %lld (%lld KiB, %lld MiB)",
 		c->bi.old_idx_sz, c->bi.old_idx_sz >> 10,
 		c->bi.old_idx_sz >> 20);
-	dbg_msg("key hash type:       %d", c->key_hash_type);
-	dbg_msg("tree fanout:         %d", c->fanout);
-	dbg_msg("reserved GC LEB:     %d", c->gc_lnum);
-	dbg_msg("first main LEB:      %d", c->main_first);
-	dbg_msg("max. znode size      %d", c->max_znode_sz);
-	dbg_msg("max. index node size %d", c->max_idx_node_sz);
-	dbg_msg("node sizes:          data %zu, inode %zu, dentry %zu",
+	dbg_gen("key hash type:       %d", c->key_hash_type);
+	dbg_gen("tree fanout:         %d", c->fanout);
+	dbg_gen("reserved GC LEB:     %d", c->gc_lnum);
+	dbg_gen("max. znode size      %d", c->max_znode_sz);
+	dbg_gen("max. index node size %d", c->max_idx_node_sz);
+	dbg_gen("node sizes:          data %zu, inode %zu, dentry %zu",
 		UBIFS_DATA_NODE_SZ, UBIFS_INO_NODE_SZ, UBIFS_DENT_NODE_SZ);
-	dbg_msg("node sizes:          trun %zu, sb %zu, master %zu",
+	dbg_gen("node sizes:          trun %zu, sb %zu, master %zu",
 		UBIFS_TRUN_NODE_SZ, UBIFS_SB_NODE_SZ, UBIFS_MST_NODE_SZ);
-	dbg_msg("node sizes:          ref %zu, cmt. start %zu, orph %zu",
+	dbg_gen("node sizes:          ref %zu, cmt. start %zu, orph %zu",
 		UBIFS_REF_NODE_SZ, UBIFS_CS_NODE_SZ, UBIFS_ORPH_NODE_SZ);
-	dbg_msg("max. node sizes:     data %zu, inode %zu dentry %zu, idx %d",
+	dbg_gen("max. node sizes:     data %zu, inode %zu dentry %zu, idx %d",
 		UBIFS_MAX_DATA_NODE_SZ, UBIFS_MAX_INO_NODE_SZ,
 		UBIFS_MAX_DENT_NODE_SZ, ubifs_idx_node_sz(c, c->fanout));
-	dbg_msg("dead watermark:      %d", c->dead_wm);
-	dbg_msg("dark watermark:      %d", c->dark_wm);
-	dbg_msg("LEB overhead:        %d", c->leb_overhead);
+	dbg_gen("dead watermark:      %d", c->dead_wm);
+	dbg_gen("dark watermark:      %d", c->dark_wm);
+	dbg_gen("LEB overhead:        %d", c->leb_overhead);
 	x = (long long)c->main_lebs * c->dark_wm;
-	dbg_msg("max. dark space:     %lld (%lld KiB, %lld MiB)",
+	dbg_gen("max. dark space:     %lld (%lld KiB, %lld MiB)",
 		x, x >> 10, x >> 20);
-	dbg_msg("maximum bud bytes:   %lld (%lld KiB, %lld MiB)",
+	dbg_gen("maximum bud bytes:   %lld (%lld KiB, %lld MiB)",
 		c->max_bud_bytes, c->max_bud_bytes >> 10,
 		c->max_bud_bytes >> 20);
-	dbg_msg("BG commit bud bytes: %lld (%lld KiB, %lld MiB)",
+	dbg_gen("BG commit bud bytes: %lld (%lld KiB, %lld MiB)",
 		c->bg_bud_bytes, c->bg_bud_bytes >> 10,
 		c->bg_bud_bytes >> 20);
-	dbg_msg("current bud bytes    %lld (%lld KiB, %lld MiB)",
+	dbg_gen("current bud bytes    %lld (%lld KiB, %lld MiB)",
 		c->bud_bytes, c->bud_bytes >> 10, c->bud_bytes >> 20);
-	dbg_msg("max. seq. number:    %llu", c->max_sqnum);
-	dbg_msg("commit number:       %llu", c->cmt_no);
+	dbg_gen("max. seq. number:    %llu", c->max_sqnum);
+	dbg_gen("commit number:       %llu", c->cmt_no);
 
 	return 0;
 
@@ -1564,10 +1557,9 @@ static int ubifs_remount_rw(struct ubifs_info *c)
 
 	if (c->rw_incompat) {
 		ubifs_err("the file-system is not R/W-compatible");
-		ubifs_msg("on-flash format version is w%d/r%d, but software "
-			  "only supports up to version w%d/r%d", c->fmt_version,
-			  c->ro_compat_version, UBIFS_FORMAT_VERSION,
-			  UBIFS_RO_COMPAT_VERSION);
+		ubifs_msg("on-flash format version is w%d/r%d, but software only supports up to version w%d/r%d",
+			  c->fmt_version, c->ro_compat_version,
+			  UBIFS_FORMAT_VERSION, UBIFS_RO_COMPAT_VERSION);
 		return -EROFS;
 	}
 
@@ -1828,8 +1820,8 @@ static void ubifs_put_super(struct super_block *sb)
 				 * next mount, so we just print a message and
 				 * continue to unmount normally.
 				 */
-				ubifs_err("failed to write master node, "
-					  "error %d", err);
+				ubifs_err("failed to write master node, error %d",
+					  err);
 		} else {
 			for (i = 0; i < c->jhead_cnt; i++)
 				/* Make sure write-buffer timers are canceled */
@@ -2248,8 +2240,7 @@ static int __init ubifs_init(void)
 	 * UBIFS_BLOCK_SIZE. It is assumed that both are powers of 2.
 	 */
 	if (PAGE_CACHE_SIZE < UBIFS_BLOCK_SIZE) {
-		ubifs_err("VFS page cache size is %u bytes, but UBIFS requires"
-			  " at least 4096 bytes",
+		ubifs_err("VFS page cache size is %u bytes, but UBIFS requires at least 4096 bytes",
 			  (unsigned int)PAGE_CACHE_SIZE);
 		return -EINVAL;
 	}
@@ -2298,6 +2289,12 @@ static void __exit ubifs_exit(void)
 	dbg_debugfs_exit();
 	ubifs_compressors_exit();
 	unregister_shrinker(&ubifs_shrinker_info);
+
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ubifs_inode_slab);
 	unregister_filesystem(&ubifs_fs_type);
 }
diff --git a/fs/ubifs/tnc_misc.c b/fs/ubifs/tnc_misc.c
index d38ac7f9654..f6bf8995c7b 100644
--- a/fs/ubifs/tnc_misc.c
+++ b/fs/ubifs/tnc_misc.c
@@ -328,8 +328,8 @@ static int read_znode(struct ubifs_info *c, int lnum, int offs, int len,
 		case UBIFS_XENT_KEY:
 			break;
 		default:
-			dbg_msg("bad key type at slot %d: %d",
-				i, key_type(c, &zbr->key));
+			ubifs_err("bad key type at slot %d: %d",
+				  i, key_type(c, &zbr->key));
 			err = 3;
 			goto out_dump;
 		}
diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h
index 64f2367c2f4..5486346d0a3 100644
--- a/fs/ubifs/ubifs.h
+++ b/fs/ubifs/ubifs.h
@@ -42,16 +42,15 @@
 #define UBIFS_VERSION 1
 
 /* Normal UBIFS messages */
-#define ubifs_msg(fmt, ...) \
-		printk(KERN_NOTICE "UBIFS: " fmt "\n", ##__VA_ARGS__)
+#define ubifs_msg(fmt, ...) pr_notice("UBIFS: " fmt "\n", ##__VA_ARGS__)
 /* UBIFS error messages */
-#define ubifs_err(fmt, ...)                                                  \
-	printk(KERN_ERR "UBIFS error (pid %d): %s: " fmt "\n", current->pid, \
+#define ubifs_err(fmt, ...)                                         \
+	pr_err("UBIFS error (pid %d): %s: " fmt "\n", current->pid, \
 	       __func__, ##__VA_ARGS__)
 /* UBIFS warning messages */
-#define ubifs_warn(fmt, ...)                                         \
-	printk(KERN_WARNING "UBIFS warning (pid %d): %s: " fmt "\n", \
-	       current->pid, __func__, ##__VA_ARGS__)
+#define ubifs_warn(fmt, ...)                                        \
+	pr_warn("UBIFS warning (pid %d): %s: " fmt "\n",            \
+		current->pid, __func__, ##__VA_ARGS__)
 
 /* UBIFS file system VFS magic number */
 #define UBIFS_SUPER_MAGIC 0x24051905
diff --git a/fs/udf/file.c b/fs/udf/file.c
index d1c6093fd3d..77b5953eaac 100644
--- a/fs/udf/file.c
+++ b/fs/udf/file.c
@@ -118,11 +118,20 @@ static int udf_adinicb_write_end(struct file *file,
 	return simple_write_end(file, mapping, pos, len, copied, page, fsdata);
 }
 
+static ssize_t udf_adinicb_direct_IO(int rw, struct kiocb *iocb,
+				     const struct iovec *iov,
+				     loff_t offset, unsigned long nr_segs)
+{
+	/* Fallback to buffered I/O. */
+	return 0;
+}
+
 const struct address_space_operations udf_adinicb_aops = {
 	.readpage	= udf_adinicb_readpage,
 	.writepage	= udf_adinicb_writepage,
 	.write_begin	= udf_adinicb_write_begin,
 	.write_end	= udf_adinicb_write_end,
+	.direct_IO	= udf_adinicb_direct_IO,
 };
 
 static ssize_t udf_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
diff --git a/fs/udf/inode.c b/fs/udf/inode.c
index 287ef9f587b..df88b957ccf 100644
--- a/fs/udf/inode.c
+++ b/fs/udf/inode.c
@@ -95,11 +95,33 @@ void udf_evict_inode(struct inode *inode)
 	}
 }
 
+static void udf_write_failed(struct address_space *mapping, loff_t to)
+{
+	struct inode *inode = mapping->host;
+	struct udf_inode_info *iinfo = UDF_I(inode);
+	loff_t isize = inode->i_size;
+
+	if (to > isize) {
+		truncate_pagecache(inode, to, isize);
+		if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) {
+			down_write(&iinfo->i_data_sem);
+			udf_truncate_extents(inode);
+			up_write(&iinfo->i_data_sem);
+		}
+	}
+}
+
 static int udf_writepage(struct page *page, struct writeback_control *wbc)
 {
 	return block_write_full_page(page, udf_get_block, wbc);
 }
 
+static int udf_writepages(struct address_space *mapping,
+			struct writeback_control *wbc)
+{
+	return mpage_writepages(mapping, wbc, udf_get_block);
+}
+
 static int udf_readpage(struct file *file, struct page *page)
 {
 	return mpage_readpage(page, udf_get_block);
@@ -118,21 +140,24 @@ static int udf_write_begin(struct file *file, struct address_space *mapping,
 	int ret;
 
 	ret = block_write_begin(mapping, pos, len, flags, pagep, udf_get_block);
-	if (unlikely(ret)) {
-		struct inode *inode = mapping->host;
-		struct udf_inode_info *iinfo = UDF_I(inode);
-		loff_t isize = inode->i_size;
-
-		if (pos + len > isize) {
-			truncate_pagecache(inode, pos + len, isize);
-			if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) {
-				down_write(&iinfo->i_data_sem);
-				udf_truncate_extents(inode);
-				up_write(&iinfo->i_data_sem);
-			}
-		}
-	}
+	if (unlikely(ret))
+		udf_write_failed(mapping, pos + len);
+	return ret;
+}
 
+static ssize_t udf_direct_IO(int rw, struct kiocb *iocb,
+			     const struct iovec *iov,
+			     loff_t offset, unsigned long nr_segs)
+{
+	struct file *file = iocb->ki_filp;
+	struct address_space *mapping = file->f_mapping;
+	struct inode *inode = mapping->host;
+	ssize_t ret;
+
+	ret = blockdev_direct_IO(rw, iocb, inode, iov, offset, nr_segs,
+				  udf_get_block);
+	if (unlikely(ret < 0 && (rw & WRITE)))
+		udf_write_failed(mapping, offset + iov_length(iov, nr_segs));
 	return ret;
 }
 
@@ -145,8 +170,10 @@ const struct address_space_operations udf_aops = {
 	.readpage	= udf_readpage,
 	.readpages	= udf_readpages,
 	.writepage	= udf_writepage,
-	.write_begin		= udf_write_begin,
-	.write_end		= generic_write_end,
+	.writepages	= udf_writepages,
+	.write_begin	= udf_write_begin,
+	.write_end	= generic_write_end,
+	.direct_IO	= udf_direct_IO,
 	.bmap		= udf_bmap,
 };
 
diff --git a/fs/udf/super.c b/fs/udf/super.c
index 862741dddf2..d44fb568abe 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -171,6 +171,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(udf_inode_cachep);
 }
 
diff --git a/fs/ufs/super.c b/fs/ufs/super.c
index 444927e5706..f7cfecfe1ca 100644
--- a/fs/ufs/super.c
+++ b/fs/ufs/super.c
@@ -1466,6 +1466,11 @@ static int init_inodecache(void)
 
 static void destroy_inodecache(void)
 {
+	/*
+	 * Make sure all delayed rcu free inodes are flushed before we
+	 * destroy cache.
+	 */
+	rcu_barrier();
 	kmem_cache_destroy(ufs_inode_cachep);
 }
 
diff --git a/fs/utimes.c b/fs/utimes.c
index fa4dbe451e2..bb0696a4173 100644
--- a/fs/utimes.c
+++ b/fs/utimes.c
@@ -140,19 +140,18 @@ long do_utimes(int dfd, const char __user *filename, struct timespec *times,
 		goto out;
 
 	if (filename == NULL && dfd != AT_FDCWD) {
-		int fput_needed;
-		struct file *file;
+		struct fd f;
 
 		if (flags & AT_SYMLINK_NOFOLLOW)
 			goto out;
 
-		file = fget_light(dfd, &fput_needed);
+		f = fdget(dfd);
 		error = -EBADF;
-		if (!file)
+		if (!f.file)
 			goto out;
 
-		error = utimes_common(&file->f_path, times);
-		fput_light(file, fput_needed);
+		error = utimes_common(&f.file->f_path, times);
+		fdput(f);
 	} else {
 		struct path path;
 		int lookup_flags = 0;
diff --git a/fs/xattr.c b/fs/xattr.c
index f7f7f09b0b4..1780f062dba 100644
--- a/fs/xattr.c
+++ b/fs/xattr.c
@@ -296,11 +296,13 @@ vfs_removexattr(struct dentry *dentry, const char *name)
 	if (error)
 		return error;
 
+	mutex_lock(&inode->i_mutex);
 	error = security_inode_removexattr(dentry, name);
-	if (error)
+	if (error) {
+		mutex_unlock(&inode->i_mutex);
 		return error;
+	}
 
-	mutex_lock(&inode->i_mutex);
 	error = inode->i_op->removexattr(dentry, name);
 	mutex_unlock(&inode->i_mutex);
 
@@ -403,22 +405,20 @@ SYSCALL_DEFINE5(lsetxattr, const char __user *, pathname,
 SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name,
 		const void __user *,value, size_t, size, int, flags)
 {
-	int fput_needed;
-	struct file *f;
+	struct fd f = fdget(fd);
 	struct dentry *dentry;
 	int error = -EBADF;
 
-	f = fget_light(fd, &fput_needed);
-	if (!f)
+	if (!f.file)
 		return error;
-	dentry = f->f_path.dentry;
+	dentry = f.file->f_path.dentry;
 	audit_inode(NULL, dentry);
-	error = mnt_want_write_file(f);
+	error = mnt_want_write_file(f.file);
 	if (!error) {
 		error = setxattr(dentry, name, value, size, flags);
-		mnt_drop_write_file(f);
+		mnt_drop_write_file(f.file);
 	}
-	fput_light(f, fput_needed);
+	fdput(f);
 	return error;
 }
 
@@ -502,16 +502,14 @@ SYSCALL_DEFINE4(lgetxattr, const char __user *, pathname,
 SYSCALL_DEFINE4(fgetxattr, int, fd, const char __user *, name,
 		void __user *, value, size_t, size)
 {
-	int fput_needed;
-	struct file *f;
+	struct fd f = fdget(fd);
 	ssize_t error = -EBADF;
 
-	f = fget_light(fd, &fput_needed);
-	if (!f)
+	if (!f.file)
 		return error;
-	audit_inode(NULL, f->f_path.dentry);
-	error = getxattr(f->f_path.dentry, name, value, size);
-	fput_light(f, fput_needed);
+	audit_inode(NULL, f.file->f_path.dentry);
+	error = getxattr(f.file->f_path.dentry, name, value, size);
+	fdput(f);
 	return error;
 }
 
@@ -583,16 +581,14 @@ SYSCALL_DEFINE3(llistxattr, const char __user *, pathname, char __user *, list,
 
 SYSCALL_DEFINE3(flistxattr, int, fd, char __user *, list, size_t, size)
 {
-	int fput_needed;
-	struct file *f;
+	struct fd f = fdget(fd);
 	ssize_t error = -EBADF;
 
-	f = fget_light(fd, &fput_needed);
-	if (!f)
+	if (!f.file)
 		return error;
-	audit_inode(NULL, f->f_path.dentry);
-	error = listxattr(f->f_path.dentry, list, size);
-	fput_light(f, fput_needed);
+	audit_inode(NULL, f.file->f_path.dentry);
+	error = listxattr(f.file->f_path.dentry, list, size);
+	fdput(f);
 	return error;
 }
 
@@ -652,22 +648,20 @@ SYSCALL_DEFINE2(lremovexattr, const char __user *, pathname,
 
 SYSCALL_DEFINE2(fremovexattr, int, fd, const char __user *, name)
 {
-	int fput_needed;
-	struct file *f;
+	struct fd f = fdget(fd);
 	struct dentry *dentry;
 	int error = -EBADF;
 
-	f = fget_light(fd, &fput_needed);
-	if (!f)
+	if (!f.file)
 		return error;
-	dentry = f->f_path.dentry;
+	dentry = f.file->f_path.dentry;
 	audit_inode(NULL, dentry);
-	error = mnt_want_write_file(f);
+	error = mnt_want_write_file(f.file);
 	if (!error) {
 		error = removexattr(dentry, name);
-		mnt_drop_write_file(f);
+		mnt_drop_write_file(f.file);
 	}
-	fput_light(f, fput_needed);
+	fdput(f);
 	return error;
 }
 
diff --git a/fs/xfs/xfs_dfrag.c b/fs/xfs/xfs_dfrag.c
index e00de08dc8a..b9b8646e62d 100644
--- a/fs/xfs/xfs_dfrag.c
+++ b/fs/xfs/xfs_dfrag.c
@@ -48,44 +48,44 @@ xfs_swapext(
 	xfs_swapext_t	*sxp)
 {
 	xfs_inode_t     *ip, *tip;
-	struct file	*file, *tmp_file;
+	struct fd	f, tmp;
 	int		error = 0;
 
 	/* Pull information for the target fd */
-	file = fget((int)sxp->sx_fdtarget);
-	if (!file) {
+	f = fdget((int)sxp->sx_fdtarget);
+	if (!f.file) {
 		error = XFS_ERROR(EINVAL);
 		goto out;
 	}
 
-	if (!(file->f_mode & FMODE_WRITE) ||
-	    !(file->f_mode & FMODE_READ) ||
-	    (file->f_flags & O_APPEND)) {
+	if (!(f.file->f_mode & FMODE_WRITE) ||
+	    !(f.file->f_mode & FMODE_READ) ||
+	    (f.file->f_flags & O_APPEND)) {
 		error = XFS_ERROR(EBADF);
 		goto out_put_file;
 	}
 
-	tmp_file = fget((int)sxp->sx_fdtmp);
-	if (!tmp_file) {
+	tmp = fdget((int)sxp->sx_fdtmp);
+	if (!tmp.file) {
 		error = XFS_ERROR(EINVAL);
 		goto out_put_file;
 	}
 
-	if (!(tmp_file->f_mode & FMODE_WRITE) ||
-	    !(tmp_file->f_mode & FMODE_READ) ||
-	    (tmp_file->f_flags & O_APPEND)) {
+	if (!(tmp.file->f_mode & FMODE_WRITE) ||
+	    !(tmp.file->f_mode & FMODE_READ) ||
+	    (tmp.file->f_flags & O_APPEND)) {
 		error = XFS_ERROR(EBADF);
 		goto out_put_tmp_file;
 	}
 
-	if (IS_SWAPFILE(file->f_path.dentry->d_inode) ||
-	    IS_SWAPFILE(tmp_file->f_path.dentry->d_inode)) {
+	if (IS_SWAPFILE(f.file->f_path.dentry->d_inode) ||
+	    IS_SWAPFILE(tmp.file->f_path.dentry->d_inode)) {
 		error = XFS_ERROR(EINVAL);
 		goto out_put_tmp_file;
 	}
 
-	ip = XFS_I(file->f_path.dentry->d_inode);
-	tip = XFS_I(tmp_file->f_path.dentry->d_inode);
+	ip = XFS_I(f.file->f_path.dentry->d_inode);
+	tip = XFS_I(tmp.file->f_path.dentry->d_inode);
 
 	if (ip->i_mount != tip->i_mount) {
 		error = XFS_ERROR(EINVAL);
@@ -105,9 +105,9 @@ xfs_swapext(
 	error = xfs_swap_extents(ip, tip, sxp);
 
  out_put_tmp_file:
-	fput(tmp_file);
+	fdput(tmp);
  out_put_file:
-	fput(file);
+	fdput(f);
  out:
 	return error;
 }
diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index 56afcdb2377..1eaeb8be3aa 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -36,6 +36,7 @@
 
 #include <linux/dcache.h>
 #include <linux/falloc.h>
+#include <linux/pagevec.h>
 
 static const struct vm_operations_struct xfs_file_vm_ops;
 
@@ -959,17 +960,232 @@ xfs_vm_page_mkwrite(
 	return block_page_mkwrite(vma, vmf, xfs_get_blocks);
 }
 
+/*
+ * This type is designed to indicate the type of offset we would like
+ * to search from page cache for either xfs_seek_data() or xfs_seek_hole().
+ */
+enum {
+	HOLE_OFF = 0,
+	DATA_OFF,
+};
+
+/*
+ * Lookup the desired type of offset from the given page.
+ *
+ * On success, return true and the offset argument will point to the
+ * start of the region that was found.  Otherwise this function will
+ * return false and keep the offset argument unchanged.
+ */
+STATIC bool
+xfs_lookup_buffer_offset(
+	struct page		*page,
+	loff_t			*offset,
+	unsigned int		type)
+{
+	loff_t			lastoff = page_offset(page);
+	bool			found = false;
+	struct buffer_head	*bh, *head;
+
+	bh = head = page_buffers(page);
+	do {
+		/*
+		 * Unwritten extents that have data in the page
+		 * cache covering them can be identified by the
+		 * BH_Unwritten state flag.  Pages with multiple
+		 * buffers might have a mix of holes, data and
+		 * unwritten extents - any buffer with valid
+		 * data in it should have BH_Uptodate flag set
+		 * on it.
+		 */
+		if (buffer_unwritten(bh) ||
+		    buffer_uptodate(bh)) {
+			if (type == DATA_OFF)
+				found = true;
+		} else {
+			if (type == HOLE_OFF)
+				found = true;
+		}
+
+		if (found) {
+			*offset = lastoff;
+			break;
+		}
+		lastoff += bh->b_size;
+	} while ((bh = bh->b_this_page) != head);
+
+	return found;
+}
+
+/*
+ * This routine is called to find out and return a data or hole offset
+ * from the page cache for unwritten extents according to the desired
+ * type for xfs_seek_data() or xfs_seek_hole().
+ *
+ * The argument offset is used to tell where we start to search from the
+ * page cache.  Map is used to figure out the end points of the range to
+ * lookup pages.
+ *
+ * Return true if the desired type of offset was found, and the argument
+ * offset is filled with that address.  Otherwise, return false and keep
+ * offset unchanged.
+ */
+STATIC bool
+xfs_find_get_desired_pgoff(
+	struct inode		*inode,
+	struct xfs_bmbt_irec	*map,
+	unsigned int		type,
+	loff_t			*offset)
+{
+	struct xfs_inode	*ip = XFS_I(inode);
+	struct xfs_mount	*mp = ip->i_mount;
+	struct pagevec		pvec;
+	pgoff_t			index;
+	pgoff_t			end;
+	loff_t			endoff;
+	loff_t			startoff = *offset;
+	loff_t			lastoff = startoff;
+	bool			found = false;
+
+	pagevec_init(&pvec, 0);
+
+	index = startoff >> PAGE_CACHE_SHIFT;
+	endoff = XFS_FSB_TO_B(mp, map->br_startoff + map->br_blockcount);
+	end = endoff >> PAGE_CACHE_SHIFT;
+	do {
+		int		want;
+		unsigned	nr_pages;
+		unsigned int	i;
+
+		want = min_t(pgoff_t, end - index, PAGEVEC_SIZE);
+		nr_pages = pagevec_lookup(&pvec, inode->i_mapping, index,
+					  want);
+		/*
+		 * No page mapped into given range.  If we are searching holes
+		 * and if this is the first time we got into the loop, it means
+		 * that the given offset is landed in a hole, return it.
+		 *
+		 * If we have already stepped through some block buffers to find
+		 * holes but they all contains data.  In this case, the last
+		 * offset is already updated and pointed to the end of the last
+		 * mapped page, if it does not reach the endpoint to search,
+		 * that means there should be a hole between them.
+		 */
+		if (nr_pages == 0) {
+			/* Data search found nothing */
+			if (type == DATA_OFF)
+				break;
+
+			ASSERT(type == HOLE_OFF);
+			if (lastoff == startoff || lastoff < endoff) {
+				found = true;
+				*offset = lastoff;
+			}
+			break;
+		}
+
+		/*
+		 * At lease we found one page.  If this is the first time we
+		 * step into the loop, and if the first page index offset is
+		 * greater than the given search offset, a hole was found.
+		 */
+		if (type == HOLE_OFF && lastoff == startoff &&
+		    lastoff < page_offset(pvec.pages[0])) {
+			found = true;
+			break;
+		}
+
+		for (i = 0; i < nr_pages; i++) {
+			struct page	*page = pvec.pages[i];
+			loff_t		b_offset;
+
+			/*
+			 * At this point, the page may be truncated or
+			 * invalidated (changing page->mapping to NULL),
+			 * or even swizzled back from swapper_space to tmpfs
+			 * file mapping. However, page->index will not change
+			 * because we have a reference on the page.
+			 *
+			 * Searching done if the page index is out of range.
+			 * If the current offset is not reaches the end of
+			 * the specified search range, there should be a hole
+			 * between them.
+			 */
+			if (page->index > end) {
+				if (type == HOLE_OFF && lastoff < endoff) {
+					*offset = lastoff;
+					found = true;
+				}
+				goto out;
+			}
+
+			lock_page(page);
+			/*
+			 * Page truncated or invalidated(page->mapping == NULL).
+			 * We can freely skip it and proceed to check the next
+			 * page.
+			 */
+			if (unlikely(page->mapping != inode->i_mapping)) {
+				unlock_page(page);
+				continue;
+			}
+
+			if (!page_has_buffers(page)) {
+				unlock_page(page);
+				continue;
+			}
+
+			found = xfs_lookup_buffer_offset(page, &b_offset, type);
+			if (found) {
+				/*
+				 * The found offset may be less than the start
+				 * point to search if this is the first time to
+				 * come here.
+				 */
+				*offset = max_t(loff_t, startoff, b_offset);
+				unlock_page(page);
+				goto out;
+			}
+
+			/*
+			 * We either searching data but nothing was found, or
+			 * searching hole but found a data buffer.  In either
+			 * case, probably the next page contains the desired
+			 * things, update the last offset to it so.
+			 */
+			lastoff = page_offset(page) + PAGE_SIZE;
+			unlock_page(page);
+		}
+
+		/*
+		 * The number of returned pages less than our desired, search
+		 * done.  In this case, nothing was found for searching data,
+		 * but we found a hole behind the last offset.
+		 */
+		if (nr_pages < want) {
+			if (type == HOLE_OFF) {
+				*offset = lastoff;
+				found = true;
+			}
+			break;
+		}
+
+		index = pvec.pages[i - 1]->index + 1;
+		pagevec_release(&pvec);
+	} while (index <= end);
+
+out:
+	pagevec_release(&pvec);
+	return found;
+}
+
 STATIC loff_t
 xfs_seek_data(
 	struct file		*file,
-	loff_t			start,
-	u32			type)
+	loff_t			start)
 {
 	struct inode		*inode = file->f_mapping->host;
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_mount	*mp = ip->i_mount;
-	struct xfs_bmbt_irec	map[2];
-	int			nmap = 2;
 	loff_t			uninitialized_var(offset);
 	xfs_fsize_t		isize;
 	xfs_fileoff_t		fsbno;
@@ -985,36 +1201,74 @@ xfs_seek_data(
 		goto out_unlock;
 	}
 
-	fsbno = XFS_B_TO_FSBT(mp, start);
-
 	/*
 	 * Try to read extents from the first block indicated
 	 * by fsbno to the end block of the file.
 	 */
+	fsbno = XFS_B_TO_FSBT(mp, start);
 	end = XFS_B_TO_FSB(mp, isize);
+	for (;;) {
+		struct xfs_bmbt_irec	map[2];
+		int			nmap = 2;
+		unsigned int		i;
 
-	error = xfs_bmapi_read(ip, fsbno, end - fsbno, map, &nmap,
-			       XFS_BMAPI_ENTIRE);
-	if (error)
-		goto out_unlock;
+		error = xfs_bmapi_read(ip, fsbno, end - fsbno, map, &nmap,
+				       XFS_BMAPI_ENTIRE);
+		if (error)
+			goto out_unlock;
 
-	/*
-	 * Treat unwritten extent as data extent since it might
-	 * contains dirty data in page cache.
-	 */
-	if (map[0].br_startblock != HOLESTARTBLOCK) {
-		offset = max_t(loff_t, start,
-			       XFS_FSB_TO_B(mp, map[0].br_startoff));
-	} else {
+		/* No extents at given offset, must be beyond EOF */
+		if (nmap == 0) {
+			error = ENXIO;
+			goto out_unlock;
+		}
+
+		for (i = 0; i < nmap; i++) {
+			offset = max_t(loff_t, start,
+				       XFS_FSB_TO_B(mp, map[i].br_startoff));
+
+			/* Landed in a data extent */
+			if (map[i].br_startblock == DELAYSTARTBLOCK ||
+			    (map[i].br_state == XFS_EXT_NORM &&
+			     !isnullstartblock(map[i].br_startblock)))
+				goto out;
+
+			/*
+			 * Landed in an unwritten extent, try to search data
+			 * from page cache.
+			 */
+			if (map[i].br_state == XFS_EXT_UNWRITTEN) {
+				if (xfs_find_get_desired_pgoff(inode, &map[i],
+							DATA_OFF, &offset))
+					goto out;
+			}
+		}
+
+		/*
+		 * map[0] is hole or its an unwritten extent but
+		 * without data in page cache.  Probably means that
+		 * we are reading after EOF if nothing in map[1].
+		 */
 		if (nmap == 1) {
 			error = ENXIO;
 			goto out_unlock;
 		}
 
-		offset = max_t(loff_t, start,
-			       XFS_FSB_TO_B(mp, map[1].br_startoff));
+		ASSERT(i > 1);
+
+		/*
+		 * Nothing was found, proceed to the next round of search
+		 * if reading offset not beyond or hit EOF.
+		 */
+		fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
+		start = XFS_FSB_TO_B(mp, fsbno);
+		if (start >= isize) {
+			error = ENXIO;
+			goto out_unlock;
+		}
 	}
 
+out:
 	if (offset != file->f_pos)
 		file->f_pos = offset;
 
@@ -1029,16 +1283,15 @@ out_unlock:
 STATIC loff_t
 xfs_seek_hole(
 	struct file		*file,
-	loff_t			start,
-	u32			type)
+	loff_t			start)
 {
 	struct inode		*inode = file->f_mapping->host;
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_mount	*mp = ip->i_mount;
 	loff_t			uninitialized_var(offset);
-	loff_t			holeoff;
 	xfs_fsize_t		isize;
 	xfs_fileoff_t		fsbno;
+	xfs_filblks_t		end;
 	uint			lock;
 	int			error;
 
@@ -1054,21 +1307,77 @@ xfs_seek_hole(
 	}
 
 	fsbno = XFS_B_TO_FSBT(mp, start);
-	error = xfs_bmap_first_unused(NULL, ip, 1, &fsbno, XFS_DATA_FORK);
-	if (error)
-		goto out_unlock;
+	end = XFS_B_TO_FSB(mp, isize);
+
+	for (;;) {
+		struct xfs_bmbt_irec	map[2];
+		int			nmap = 2;
+		unsigned int		i;
+
+		error = xfs_bmapi_read(ip, fsbno, end - fsbno, map, &nmap,
+				       XFS_BMAPI_ENTIRE);
+		if (error)
+			goto out_unlock;
+
+		/* No extents at given offset, must be beyond EOF */
+		if (nmap == 0) {
+			error = ENXIO;
+			goto out_unlock;
+		}
+
+		for (i = 0; i < nmap; i++) {
+			offset = max_t(loff_t, start,
+				       XFS_FSB_TO_B(mp, map[i].br_startoff));
+
+			/* Landed in a hole */
+			if (map[i].br_startblock == HOLESTARTBLOCK)
+				goto out;
+
+			/*
+			 * Landed in an unwritten extent, try to search hole
+			 * from page cache.
+			 */
+			if (map[i].br_state == XFS_EXT_UNWRITTEN) {
+				if (xfs_find_get_desired_pgoff(inode, &map[i],
+							HOLE_OFF, &offset))
+					goto out;
+			}
+		}
+
+		/*
+		 * map[0] contains data or its unwritten but contains
+		 * data in page cache, probably means that we are
+		 * reading after EOF.  We should fix offset to point
+		 * to the end of the file(i.e., there is an implicit
+		 * hole at the end of any file).
+		 */
+		if (nmap == 1) {
+			offset = isize;
+			break;
+		}
+
+		ASSERT(i > 1);
 
-	holeoff = XFS_FSB_TO_B(mp, fsbno);
-	if (holeoff <= start)
-		offset = start;
-	else {
 		/*
-		 * xfs_bmap_first_unused() could return a value bigger than
-		 * isize if there are no more holes past the supplied offset.
+		 * Both mappings contains data, proceed to the next round of
+		 * search if the current reading offset not beyond or hit EOF.
 		 */
-		offset = min_t(loff_t, holeoff, isize);
+		fsbno = map[i - 1].br_startoff + map[i - 1].br_blockcount;
+		start = XFS_FSB_TO_B(mp, fsbno);
+		if (start >= isize) {
+			offset = isize;
+			break;
+		}
 	}
 
+out:
+	/*
+	 * At this point, we must have found a hole.  However, the returned
+	 * offset may be bigger than the file size as it may be aligned to
+	 * page boundary for unwritten extents, we need to deal with this
+	 * situation in particular.
+	 */
+	offset = min_t(loff_t, offset, isize);
 	if (offset != file->f_pos)
 		file->f_pos = offset;
 
@@ -1092,9 +1401,9 @@ xfs_file_llseek(
 	case SEEK_SET:
 		return generic_file_llseek(file, offset, origin);
 	case SEEK_DATA:
-		return xfs_seek_data(file, offset, origin);
+		return xfs_seek_data(file, offset);
 	case SEEK_HOLE:
-		return xfs_seek_hole(file, offset, origin);
+		return xfs_seek_hole(file, offset);
 	default:
 		return -EINVAL;
 	}
diff --git a/fs/xfs/xfs_ialloc.c b/fs/xfs/xfs_ialloc.c
index 5aceb3f8ecd..445bf1aef31 100644
--- a/fs/xfs/xfs_ialloc.c
+++ b/fs/xfs/xfs_ialloc.c
@@ -431,7 +431,7 @@ xfs_ialloc_next_ag(
 
 	spin_lock(&mp->m_agirotor_lock);
 	agno = mp->m_agirotor;
-	if (++mp->m_agirotor == mp->m_maxagi)
+	if (++mp->m_agirotor >= mp->m_maxagi)
 		mp->m_agirotor = 0;
 	spin_unlock(&mp->m_agirotor_lock);
 
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index 0e0232c3b6d..8305f2ac677 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -70,16 +70,16 @@ xfs_find_handle(
 	int			hsize;
 	xfs_handle_t		handle;
 	struct inode		*inode;
-	struct file		*file = NULL;
+	struct fd		f;
 	struct path		path;
 	int			error;
 	struct xfs_inode	*ip;
 
 	if (cmd == XFS_IOC_FD_TO_HANDLE) {
-		file = fget(hreq->fd);
-		if (!file)
+		f = fdget(hreq->fd);
+		if (!f.file)
 			return -EBADF;
-		inode = file->f_path.dentry->d_inode;
+		inode = f.file->f_path.dentry->d_inode;
 	} else {
 		error = user_lpath((const char __user *)hreq->path, &path);
 		if (error)
@@ -134,7 +134,7 @@ xfs_find_handle(
 
  out_put:
 	if (cmd == XFS_IOC_FD_TO_HANDLE)
-		fput(file);
+		fdput(f);
 	else
 		path_put(&path);
 	return error;
diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
index 29c2f83d414..b2bd3a0e637 100644
--- a/fs/xfs/xfs_mount.c
+++ b/fs/xfs/xfs_mount.c
@@ -440,7 +440,7 @@ xfs_initialize_perag(
 	xfs_agnumber_t	agcount,
 	xfs_agnumber_t	*maxagi)
 {
-	xfs_agnumber_t	index, max_metadata;
+	xfs_agnumber_t	index;
 	xfs_agnumber_t	first_initialised = 0;
 	xfs_perag_t	*pag;
 	xfs_agino_t	agino;
@@ -500,43 +500,10 @@ xfs_initialize_perag(
 	else
 		mp->m_flags &= ~XFS_MOUNT_32BITINODES;
 
-	if (mp->m_flags & XFS_MOUNT_32BITINODES) {
-		/*
-		 * Calculate how much should be reserved for inodes to meet
-		 * the max inode percentage.
-		 */
-		if (mp->m_maxicount) {
-			__uint64_t	icount;
-
-			icount = sbp->sb_dblocks * sbp->sb_imax_pct;
-			do_div(icount, 100);
-			icount += sbp->sb_agblocks - 1;
-			do_div(icount, sbp->sb_agblocks);
-			max_metadata = icount;
-		} else {
-			max_metadata = agcount;
-		}
-
-		for (index = 0; index < agcount; index++) {
-			ino = XFS_AGINO_TO_INO(mp, index, agino);
-			if (ino > XFS_MAXINUMBER_32) {
-				index++;
-				break;
-			}
-
-			pag = xfs_perag_get(mp, index);
-			pag->pagi_inodeok = 1;
-			if (index < max_metadata)
-				pag->pagf_metadata = 1;
-			xfs_perag_put(pag);
-		}
-	} else {
-		for (index = 0; index < agcount; index++) {
-			pag = xfs_perag_get(mp, index);
-			pag->pagi_inodeok = 1;
-			xfs_perag_put(pag);
-		}
-	}
+	if (mp->m_flags & XFS_MOUNT_32BITINODES)
+		index = xfs_set_inode32(mp);
+	else
+		index = xfs_set_inode64(mp);
 
 	if (maxagi)
 		*maxagi = index;
diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h
index 05a05a7b611..deee09e534d 100644
--- a/fs/xfs/xfs_mount.h
+++ b/fs/xfs/xfs_mount.h
@@ -54,12 +54,7 @@ typedef struct xfs_trans_reservations {
 #include "xfs_sync.h"
 
 struct xlog;
-struct xfs_mount_args;
 struct xfs_inode;
-struct xfs_bmbt_irec;
-struct xfs_bmap_free;
-struct xfs_extdelta;
-struct xfs_swapext;
 struct xfs_mru_cache;
 struct xfs_nameops;
 struct xfs_ail;
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index 001537f92ca..26a09bd7f97 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -88,6 +88,8 @@ mempool_t *xfs_ioend_pool;
 					 * unwritten extent conversion */
 #define MNTOPT_NOBARRIER "nobarrier"	/* .. disable */
 #define MNTOPT_64BITINODE   "inode64"	/* inodes can be allocated anywhere */
+#define MNTOPT_32BITINODE   "inode32"	/* inode allocation limited to
+					 * XFS_MAXINUMBER_32 */
 #define MNTOPT_IKEEP	"ikeep"		/* do not free empty inode clusters */
 #define MNTOPT_NOIKEEP	"noikeep"	/* free empty inode clusters */
 #define MNTOPT_LARGEIO	   "largeio"	/* report large I/O sizes in stat() */
@@ -120,12 +122,18 @@ mempool_t *xfs_ioend_pool;
  * in the future, too.
  */
 enum {
-	Opt_barrier, Opt_nobarrier, Opt_err
+	Opt_barrier,
+	Opt_nobarrier,
+	Opt_inode64,
+	Opt_inode32,
+	Opt_err
 };
 
 static const match_table_t tokens = {
 	{Opt_barrier, "barrier"},
 	{Opt_nobarrier, "nobarrier"},
+	{Opt_inode64, "inode64"},
+	{Opt_inode32, "inode32"},
 	{Opt_err, NULL}
 };
 
@@ -197,7 +205,9 @@ xfs_parseargs(
 	 */
 	mp->m_flags |= XFS_MOUNT_BARRIER;
 	mp->m_flags |= XFS_MOUNT_COMPAT_IOSIZE;
+#if !XFS_BIG_INUMS
 	mp->m_flags |= XFS_MOUNT_SMALL_INUMS;
+#endif
 
 	/*
 	 * These can be overridden by the mount option parsing.
@@ -294,6 +304,8 @@ xfs_parseargs(
 				return EINVAL;
 			}
 			dswidth = simple_strtoul(value, &eov, 10);
+		} else if (!strcmp(this_char, MNTOPT_32BITINODE)) {
+			mp->m_flags |= XFS_MOUNT_SMALL_INUMS;
 		} else if (!strcmp(this_char, MNTOPT_64BITINODE)) {
 			mp->m_flags &= ~XFS_MOUNT_SMALL_INUMS;
 #if !XFS_BIG_INUMS
@@ -492,6 +504,7 @@ xfs_showargs(
 		{ XFS_MOUNT_FILESTREAMS,	"," MNTOPT_FILESTREAM },
 		{ XFS_MOUNT_GRPID,		"," MNTOPT_GRPID },
 		{ XFS_MOUNT_DISCARD,		"," MNTOPT_DISCARD },
+		{ XFS_MOUNT_SMALL_INUMS,	"," MNTOPT_32BITINODE },
 		{ 0, NULL }
 	};
 	static struct proc_xfs_info xfs_info_unset[] = {
@@ -591,6 +604,80 @@ xfs_max_file_offset(
 	return (((__uint64_t)pagefactor) << bitshift) - 1;
 }
 
+xfs_agnumber_t
+xfs_set_inode32(struct xfs_mount *mp)
+{
+	xfs_agnumber_t	index = 0;
+	xfs_agnumber_t	maxagi = 0;
+	xfs_sb_t	*sbp = &mp->m_sb;
+	xfs_agnumber_t	max_metadata;
+	xfs_agino_t	agino =	XFS_OFFBNO_TO_AGINO(mp, sbp->sb_agblocks -1, 0);
+	xfs_ino_t	ino = XFS_AGINO_TO_INO(mp, sbp->sb_agcount -1, agino);
+	xfs_perag_t	*pag;
+
+	/* Calculate how much should be reserved for inodes to meet
+	 * the max inode percentage.
+	 */
+	if (mp->m_maxicount) {
+		__uint64_t	icount;
+
+		icount = sbp->sb_dblocks * sbp->sb_imax_pct;
+		do_div(icount, 100);
+		icount += sbp->sb_agblocks - 1;
+		do_div(icount, sbp->sb_agblocks);
+		max_metadata = icount;
+	} else {
+		max_metadata = sbp->sb_agcount;
+	}
+
+	for (index = 0; index < sbp->sb_agcount; index++) {
+		ino = XFS_AGINO_TO_INO(mp, index, agino);
+
+		if (ino > XFS_MAXINUMBER_32) {
+			pag = xfs_perag_get(mp, index);
+			pag->pagi_inodeok = 0;
+			pag->pagf_metadata = 0;
+			xfs_perag_put(pag);
+			continue;
+		}
+
+		pag = xfs_perag_get(mp, index);
+		pag->pagi_inodeok = 1;
+		maxagi++;
+		if (index < max_metadata)
+			pag->pagf_metadata = 1;
+		xfs_perag_put(pag);
+	}
+	mp->m_flags |= (XFS_MOUNT_32BITINODES |
+			XFS_MOUNT_SMALL_INUMS);
+
+	return maxagi;
+}
+
+xfs_agnumber_t
+xfs_set_inode64(struct xfs_mount *mp)
+{
+	xfs_agnumber_t index = 0;
+
+	for (index = 0; index < mp->m_sb.sb_agcount; index++) {
+		struct xfs_perag	*pag;
+
+		pag = xfs_perag_get(mp, index);
+		pag->pagi_inodeok = 1;
+		pag->pagf_metadata = 0;
+		xfs_perag_put(pag);
+	}
+
+	/* There is no need for lock protection on m_flags,
+	 * the rw_semaphore of the VFS superblock is locked
+	 * during mount/umount/remount operations, so this is
+	 * enough to avoid concurency on the m_flags field
+	 */
+	mp->m_flags &= ~(XFS_MOUNT_32BITINODES |
+			 XFS_MOUNT_SMALL_INUMS);
+	return index;
+}
+
 STATIC int
 xfs_blkdev_get(
 	xfs_mount_t		*mp,
@@ -1056,6 +1143,12 @@ xfs_fs_remount(
 		case Opt_nobarrier:
 			mp->m_flags &= ~XFS_MOUNT_BARRIER;
 			break;
+		case Opt_inode64:
+			mp->m_maxagi = xfs_set_inode64(mp);
+			break;
+		case Opt_inode32:
+			mp->m_maxagi = xfs_set_inode32(mp);
+			break;
 		default:
 			/*
 			 * Logically we would return an error here to prevent
@@ -1506,6 +1599,11 @@ xfs_init_zones(void)
 STATIC void
 xfs_destroy_zones(void)
 {
+	/*
+	 * Make sure all delayed rcu free are flushed before we
+	 * destroy caches.
+	 */
+	rcu_barrier();
 	kmem_zone_destroy(xfs_ili_zone);
 	kmem_zone_destroy(xfs_inode_zone);
 	kmem_zone_destroy(xfs_efi_zone);
diff --git a/fs/xfs/xfs_super.h b/fs/xfs/xfs_super.h
index 09b0c26b224..9de4a920ba0 100644
--- a/fs/xfs/xfs_super.h
+++ b/fs/xfs/xfs_super.h
@@ -75,6 +75,8 @@ struct block_device;
 extern __uint64_t xfs_max_file_offset(unsigned int);
 
 extern void xfs_blkdev_issue_flush(struct xfs_buftarg *);
+extern xfs_agnumber_t xfs_set_inode32(struct xfs_mount *);
+extern xfs_agnumber_t xfs_set_inode64(struct xfs_mount *);
 
 extern const struct export_operations xfs_export_operations;
 extern const struct xattr_handler *xfs_xattr_handlers[];
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index e5795dd6013..7d36ccf57f9 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -37,6 +37,7 @@ struct xlog_recover;
 struct xlog_recover_item;
 struct xfs_buf_log_format;
 struct xfs_inode_log_format;
+struct xfs_bmbt_irec;
 
 DECLARE_EVENT_CLASS(xfs_attr_list_class,
 	TP_PROTO(struct xfs_attr_list_context *ctx),
diff --git a/include/acpi/acpi.h b/include/acpi/acpi.h
index c433d5e2767..c1ea8436961 100644
--- a/include/acpi/acpi.h
+++ b/include/acpi/acpi.h
@@ -53,14 +53,14 @@
  *
  * Note: The order of these include files is important.
  */
-#include "platform/acenv.h"	/* Environment-specific items */
-#include "acnames.h"		/* Common ACPI names and strings */
-#include "actypes.h"		/* ACPICA data types and structures */
-#include "acexcep.h"		/* ACPICA exceptions */
-#include "actbl.h"		/* ACPI table definitions */
-#include "acoutput.h"		/* Error output and Debug macros */
-#include "acrestyp.h"		/* Resource Descriptor structs */
-#include "acpiosxf.h"		/* OSL interfaces (ACPICA-to-OS) */
-#include "acpixf.h"		/* ACPI core subsystem external interfaces */
+#include <acpi/platform/acenv.h>	/* Environment-specific items */
+#include <acpi/acnames.h>		/* Common ACPI names and strings */
+#include <acpi/actypes.h>		/* ACPICA data types and structures */
+#include <acpi/acexcep.h>		/* ACPICA exceptions */
+#include <acpi/actbl.h>		/* ACPI table definitions */
+#include <acpi/acoutput.h>		/* Error output and Debug macros */
+#include <acpi/acrestyp.h>		/* Resource Descriptor structs */
+#include <acpi/acpiosxf.h>		/* OSL interfaces (ACPICA-to-OS) */
+#include <acpi/acpixf.h>		/* ACPI core subsystem external interfaces */
 
 #endif				/* __ACPI_H__ */
diff --git a/include/acpi/acpiosxf.h b/include/acpi/acpiosxf.h
index 0650f5fa7ce..1222ba93d80 100644
--- a/include/acpi/acpiosxf.h
+++ b/include/acpi/acpiosxf.h
@@ -47,8 +47,8 @@
 #ifndef __ACPIOSXF_H__
 #define __ACPIOSXF_H__
 
-#include "platform/acenv.h"
-#include "actypes.h"
+#include <acpi/platform/acenv.h>
+#include <acpi/actypes.h>
 
 /* Types for acpi_os_execute */
 
diff --git a/include/acpi/acpixf.h b/include/acpi/acpixf.h
index 26a92fc28a5..51405d32ac6 100644
--- a/include/acpi/acpixf.h
+++ b/include/acpi/acpixf.h
@@ -49,9 +49,9 @@
 
 #define ACPI_CA_VERSION                 0x20120711
 
-#include "acconfig.h"
-#include "actypes.h"
-#include "actbl.h"
+#include <acpi/acconfig.h>
+#include <acpi/actypes.h>
+#include <acpi/actbl.h>
 
 extern u8 acpi_gbl_permanent_mmap;
 
diff --git a/include/acpi/platform/acenv.h b/include/acpi/platform/acenv.h
index 560a9f272f3..89cee88dd2a 100644
--- a/include/acpi/platform/acenv.h
+++ b/include/acpi/platform/acenv.h
@@ -138,7 +138,7 @@
 /*! [Begin] no source code translation */
 
 #if defined(_LINUX) || defined(__linux__)
-#include "aclinux.h"
+#include <acpi/platform/aclinux.h>
 
 #elif defined(_AED_EFI)
 #include "acefi.h"
diff --git a/include/acpi/platform/aclinux.h b/include/acpi/platform/aclinux.h
index 7509be30ca0..85d5d8f3845 100644
--- a/include/acpi/platform/aclinux.h
+++ b/include/acpi/platform/aclinux.h
@@ -106,7 +106,7 @@
 
 /* Linux uses GCC */
 
-#include "acgcc.h"
+#include <acpi/platform/acgcc.h>
 
 
 #ifdef __KERNEL__
diff --git a/include/acpi/processor.h b/include/acpi/processor.h
index 64ec644808b..555d0337ad9 100644
--- a/include/acpi/processor.h
+++ b/include/acpi/processor.h
@@ -3,7 +3,6 @@
 
 #include <linux/kernel.h>
 #include <linux/cpu.h>
-#include <linux/cpuidle.h>
 #include <linux/thermal.h>
 #include <asm/acpi.h>
 
@@ -59,13 +58,11 @@ struct acpi_processor_cx {
 	u8 entry_method;
 	u8 index;
 	u32 latency;
-	u32 power;
 	u8 bm_sts_skip;
 	char desc[ACPI_CX_DESC_LEN];
 };
 
 struct acpi_processor_power {
-	struct cpuidle_device dev;
 	struct acpi_processor_cx *state;
 	unsigned long bm_check_timestamp;
 	u32 default_state;
@@ -325,12 +322,10 @@ extern void acpi_processor_reevaluate_tstate(struct acpi_processor *pr,
 extern const struct file_operations acpi_processor_throttling_fops;
 extern void acpi_processor_throttling_init(void);
 /* in processor_idle.c */
-int acpi_processor_power_init(struct acpi_processor *pr,
-			      struct acpi_device *device);
+int acpi_processor_power_init(struct acpi_processor *pr);
+int acpi_processor_power_exit(struct acpi_processor *pr);
 int acpi_processor_cst_has_changed(struct acpi_processor *pr);
 int acpi_processor_hotplug(struct acpi_processor *pr);
-int acpi_processor_power_exit(struct acpi_processor *pr,
-			      struct acpi_device *device);
 int acpi_processor_suspend(struct device *dev);
 int acpi_processor_resume(struct device *dev);
 extern struct cpuidle_driver acpi_idle_driver;
diff --git a/include/asm-generic/Kbuild.asm b/include/asm-generic/Kbuild.asm
index c5d2e5dd871..d2ee86b4c09 100644
--- a/include/asm-generic/Kbuild.asm
+++ b/include/asm-generic/Kbuild.asm
@@ -1,45 +1 @@
-ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/kvm.h \
-		  $(srctree)/include/asm-$(SRCARCH)/kvm.h),)
-header-y  += kvm.h
-endif
-
-ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/kvm_para.h \
-		  $(srctree)/include/asm-$(SRCARCH)/kvm_para.h),)
-header-y  += kvm_para.h
-endif
-
-ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/a.out.h \
-		  $(srctree)/include/asm-$(SRCARCH)/a.out.h),)
-header-y += a.out.h
-endif
-
-header-y += auxvec.h
-header-y += bitsperlong.h
-header-y += byteorder.h
-header-y += errno.h
-header-y += fcntl.h
-header-y += ioctl.h
-header-y += ioctls.h
-header-y += ipcbuf.h
-header-y += mman.h
-header-y += msgbuf.h
-header-y += param.h
-header-y += poll.h
-header-y += posix_types.h
-header-y += ptrace.h
-header-y += resource.h
-header-y += sembuf.h
-header-y += setup.h
-header-y += shmbuf.h
-header-y += sigcontext.h
-header-y += siginfo.h
-header-y += signal.h
-header-y += socket.h
-header-y += sockios.h
-header-y += stat.h
-header-y += statfs.h
-header-y += swab.h
-header-y += termbits.h
-header-y += termios.h
-header-y += types.h
-header-y += unistd.h
+include include/uapi/asm-generic/Kbuild.asm
diff --git a/include/asm-generic/bitops/le.h b/include/asm-generic/bitops/le.h
index f95c663a6a4..61731543c00 100644
--- a/include/asm-generic/bitops/le.h
+++ b/include/asm-generic/bitops/le.h
@@ -54,6 +54,16 @@ static inline int test_bit_le(int nr, const void *addr)
 	return test_bit(nr ^ BITOP_LE_SWIZZLE, addr);
 }
 
+static inline void set_bit_le(int nr, void *addr)
+{
+	set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
+}
+
+static inline void clear_bit_le(int nr, void *addr)
+{
+	clear_bit(nr ^ BITOP_LE_SWIZZLE, addr);
+}
+
 static inline void __set_bit_le(int nr, void *addr)
 {
 	__set_bit(nr ^ BITOP_LE_SWIZZLE, addr);
diff --git a/include/crypto/cast5.h b/include/crypto/cast5.h
new file mode 100644
index 00000000000..586183a0406
--- /dev/null
+++ b/include/crypto/cast5.h
@@ -0,0 +1,27 @@
+#ifndef _CRYPTO_CAST5_H
+#define _CRYPTO_CAST5_H
+
+#include <linux/types.h>
+#include <linux/crypto.h>
+
+#define CAST5_BLOCK_SIZE 8
+#define CAST5_MIN_KEY_SIZE 5
+#define CAST5_MAX_KEY_SIZE 16
+
+struct cast5_ctx {
+	u32 Km[16];
+	u8 Kr[16];
+	int rr;	/* rr ? rounds = 12 : rounds = 16; (rfc 2144) */
+};
+
+int cast5_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen);
+
+void __cast5_encrypt(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
+void __cast5_decrypt(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
+
+extern const u32 cast5_s1[256];
+extern const u32 cast5_s2[256];
+extern const u32 cast5_s3[256];
+extern const u32 cast5_s4[256];
+
+#endif
diff --git a/include/crypto/cast6.h b/include/crypto/cast6.h
new file mode 100644
index 00000000000..157af6f342c
--- /dev/null
+++ b/include/crypto/cast6.h
@@ -0,0 +1,28 @@
+#ifndef _CRYPTO_CAST6_H
+#define _CRYPTO_CAST6_H
+
+#include <linux/types.h>
+#include <linux/crypto.h>
+
+#define CAST6_BLOCK_SIZE 16
+#define CAST6_MIN_KEY_SIZE 16
+#define CAST6_MAX_KEY_SIZE 32
+
+struct cast6_ctx {
+	u32 Km[12][4];
+	u8 Kr[12][4];
+};
+
+int __cast6_setkey(struct cast6_ctx *ctx, const u8 *key,
+		   unsigned int keylen, u32 *flags);
+int cast6_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen);
+
+void __cast6_encrypt(struct cast6_ctx *ctx, u8 *dst, const u8 *src);
+void __cast6_decrypt(struct cast6_ctx *ctx, u8 *dst, const u8 *src);
+
+extern const u32 cast6_s1[256];
+extern const u32 cast6_s2[256];
+extern const u32 cast6_s3[256];
+extern const u32 cast6_s4[256];
+
+#endif
diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h
index 5bfad8c8059..821eae8cbd8 100644
--- a/include/crypto/internal/hash.h
+++ b/include/crypto/internal/hash.h
@@ -83,6 +83,8 @@ struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask);
 
 int crypto_register_shash(struct shash_alg *alg);
 int crypto_unregister_shash(struct shash_alg *alg);
+int crypto_register_shashes(struct shash_alg *algs, int count);
+int crypto_unregister_shashes(struct shash_alg *algs, int count);
 int shash_register_instance(struct crypto_template *tmpl,
 			    struct shash_instance *inst);
 void shash_free_instance(struct crypto_instance *inst);
diff --git a/include/drm/drm.h b/include/drm/drm.h
index e51035a3757..1e3481edf06 100644
--- a/include/drm/drm.h
+++ b/include/drm/drm.h
@@ -628,7 +628,7 @@ struct drm_prime_handle {
 	__s32 fd;
 };
 
-#include "drm_mode.h"
+#include <drm/drm_mode.h>
 
 #define DRM_IOCTL_BASE			'd'
 #define DRM_IO(nr)			_IO(DRM_IOCTL_BASE,nr)
diff --git a/include/drm/drmP.h b/include/drm/drmP.h
index 9bc5c6a1d52..3fd82809b2d 100644
--- a/include/drm/drmP.h
+++ b/include/drm/drmP.h
@@ -72,7 +72,8 @@
 #include <linux/workqueue.h>
 #include <linux/poll.h>
 #include <asm/pgalloc.h>
-#include "drm.h"
+#include <drm/drm.h>
+#include <drm/drm_sarea.h>
 
 #include <linux/idr.h>
 
@@ -84,9 +85,9 @@ struct module;
 struct drm_file;
 struct drm_device;
 
-#include "drm_os_linux.h"
-#include "drm_hashtab.h"
-#include "drm_mm.h"
+#include <drm/drm_os_linux.h>
+#include <drm/drm_hashtab.h>
+#include <drm/drm_mm.h>
 
 #define DRM_UT_CORE 		0x01
 #define DRM_UT_DRIVER		0x02
@@ -675,7 +676,7 @@ struct drm_gem_object {
 	struct dma_buf_attachment *import_attach;
 };
 
-#include "drm_crtc.h"
+#include <drm/drm_crtc.h>
 
 /* per-master structure */
 struct drm_master {
@@ -1303,7 +1304,7 @@ extern void drm_vm_close_locked(struct drm_device *dev, struct vm_area_struct *v
 extern unsigned int drm_poll(struct file *filp, struct poll_table_struct *wait);
 
 				/* Memory management support (drm_memory.h) */
-#include "drm_memory.h"
+#include <drm/drm_memory.h>
 extern void drm_free_agp(DRM_AGP_MEM * handle, int pages);
 extern int drm_bind_agp(DRM_AGP_MEM * handle, unsigned int start);
 extern DRM_AGP_MEM *drm_agp_bind_pages(struct drm_device *dev,
@@ -1367,6 +1368,7 @@ extern int drm_remove_magic(struct drm_master *master, drm_magic_t magic);
 
 /* Cache management (drm_cache.c) */
 void drm_clflush_pages(struct page *pages[], unsigned long num_pages);
+void drm_clflush_sg(struct sg_table *st);
 void drm_clflush_virt_range(char *addr, unsigned long length);
 
 				/* Locking IOCTL support (drm_lock.h) */
@@ -1612,7 +1614,7 @@ void drm_gem_vm_open(struct vm_area_struct *vma);
 void drm_gem_vm_close(struct vm_area_struct *vma);
 int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
 
-#include "drm_global.h"
+#include <drm/drm_global.h>
 
 static inline void
 drm_gem_object_reference(struct drm_gem_object *obj)
@@ -1721,7 +1723,7 @@ static __inline__ void drm_core_dropmap(struct drm_local_map *map)
 {
 }
 
-#include "drm_mem_util.h"
+#include <drm/drm_mem_util.h>
 
 extern int drm_fill_in_dev(struct drm_device *dev,
 			   const struct pci_device_id *ent,
diff --git a/include/drm/drm_buffer.h b/include/drm/drm_buffer.h
index 322dbff3f86..c80d3a340b9 100644
--- a/include/drm/drm_buffer.h
+++ b/include/drm/drm_buffer.h
@@ -35,7 +35,7 @@
 #ifndef _DRM_BUFFER_H_
 #define _DRM_BUFFER_H_
 
-#include "drmP.h"
+#include <drm/drmP.h>
 
 struct drm_buffer {
 	int iterator;
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index bfacf0d5a22..1816bb31273 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -30,6 +30,7 @@
 #include <linux/types.h>
 #include <linux/idr.h>
 #include <linux/fb.h>
+#include <drm/drm_mode.h>
 
 #include <drm/drm_fourcc.h>
 
@@ -215,11 +216,10 @@ struct drm_display_info {
 	u32 color_formats;
 
 	u8 cea_rev;
-
-	char *raw_edid; /* if any */
 };
 
 struct drm_framebuffer_funcs {
+	/* note: use drm_framebuffer_remove() */
 	void (*destroy)(struct drm_framebuffer *framebuffer);
 	int (*create_handle)(struct drm_framebuffer *fb,
 			     struct drm_file *file_priv,
@@ -244,6 +244,16 @@ struct drm_framebuffer_funcs {
 
 struct drm_framebuffer {
 	struct drm_device *dev;
+	/*
+	 * Note that the fb is refcounted for the benefit of driver internals,
+	 * for example some hw, disabling a CRTC/plane is asynchronous, and
+	 * scanout does not actually complete until the next vblank.  So some
+	 * cleanup (like releasing the reference(s) on the backing GEM bo(s))
+	 * should be deferred.  In cases like this, the driver would like to
+	 * hold a ref to the fb even though it has already been removed from
+	 * userspace perspective.
+	 */
+	struct kref refcount;
 	struct list_head head;
 	struct drm_mode_object base;
 	const struct drm_framebuffer_funcs *funcs;
@@ -359,6 +369,9 @@ struct drm_crtc_funcs {
  * @enabled: is this CRTC enabled?
  * @mode: current mode timings
  * @hwmode: mode timings as programmed to hw regs
+ * @invert_dimensions: for purposes of error checking crtc vs fb sizes,
+ *    invert the width/height of the crtc.  This is used if the driver
+ *    is performing 90 or 270 degree rotated scanout
  * @x: x position on screen
  * @y: y position on screen
  * @funcs: CRTC control functions
@@ -392,6 +405,8 @@ struct drm_crtc {
 	 */
 	struct drm_display_mode hwmode;
 
+	bool invert_dimensions;
+
 	int x, y;
 	const struct drm_crtc_funcs *funcs;
 
@@ -593,6 +608,7 @@ struct drm_connector {
 	int video_latency[2];	/* [0]: progressive, [1]: interlaced */
 	int audio_latency[2];
 	int null_edid_counter; /* needed to workaround some HW bugs where we get all 0s */
+	unsigned bad_edid_counter;
 };
 
 /**
@@ -920,6 +936,9 @@ extern void drm_framebuffer_set_object(struct drm_device *dev,
 extern int drm_framebuffer_init(struct drm_device *dev,
 				struct drm_framebuffer *fb,
 				const struct drm_framebuffer_funcs *funcs);
+extern void drm_framebuffer_unreference(struct drm_framebuffer *fb);
+extern void drm_framebuffer_reference(struct drm_framebuffer *fb);
+extern void drm_framebuffer_remove(struct drm_framebuffer *fb);
 extern void drm_framebuffer_cleanup(struct drm_framebuffer *fb);
 extern int drmfb_probe(struct drm_device *dev, struct drm_crtc *crtc);
 extern int drmfb_remove(struct drm_device *dev, struct drm_framebuffer *fb);
@@ -1035,7 +1054,7 @@ extern int drm_add_modes_noedid(struct drm_connector *connector,
 				int hdisplay, int vdisplay);
 
 extern int drm_edid_header_is_valid(const u8 *raw_edid);
-extern bool drm_edid_block_valid(u8 *raw_edid, int block);
+extern bool drm_edid_block_valid(u8 *raw_edid, int block, bool print_bad_edid);
 extern bool drm_edid_is_valid(struct edid *edid);
 struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
 					   int hsize, int vsize, int fresh,
diff --git a/include/drm/drm_encoder_slave.h b/include/drm/drm_encoder_slave.h
index 7dc38523380..b0c11a7809b 100644
--- a/include/drm/drm_encoder_slave.h
+++ b/include/drm/drm_encoder_slave.h
@@ -27,8 +27,8 @@
 #ifndef __DRM_ENCODER_SLAVE_H__
 #define __DRM_ENCODER_SLAVE_H__
 
-#include "drmP.h"
-#include "drm_crtc.h"
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
 
 /**
  * struct drm_encoder_slave_funcs - Entry points exposed by a slave encoder driver
diff --git a/include/drm/drm_fb_cma_helper.h b/include/drm/drm_fb_cma_helper.h
new file mode 100644
index 00000000000..76c70983754
--- /dev/null
+++ b/include/drm/drm_fb_cma_helper.h
@@ -0,0 +1,27 @@
+#ifndef __DRM_FB_CMA_HELPER_H__
+#define __DRM_FB_CMA_HELPER_H__
+
+struct drm_fbdev_cma;
+struct drm_gem_cma_object;
+
+struct drm_framebuffer;
+struct drm_device;
+struct drm_file;
+struct drm_mode_fb_cmd2;
+
+struct drm_fbdev_cma *drm_fbdev_cma_init(struct drm_device *dev,
+	unsigned int preferred_bpp, unsigned int num_crtc,
+	unsigned int max_conn_count);
+void drm_fbdev_cma_fini(struct drm_fbdev_cma *fbdev_cma);
+
+void drm_fbdev_cma_restore_mode(struct drm_fbdev_cma *fbdev_cma);
+void drm_fbdev_cma_hotplug_event(struct drm_fbdev_cma *fbdev_cma);
+
+struct drm_framebuffer *drm_fb_cma_create(struct drm_device *dev,
+	struct drm_file *file_priv, struct drm_mode_fb_cmd2 *mode_cmd);
+
+struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
+	unsigned int plane);
+
+#endif
+
diff --git a/include/drm/drm_fourcc.h b/include/drm/drm_fourcc.h
index f4621184a9b..646ae5f39f4 100644
--- a/include/drm/drm_fourcc.h
+++ b/include/drm/drm_fourcc.h
@@ -106,6 +106,8 @@
 #define DRM_FORMAT_NV21		fourcc_code('N', 'V', '2', '1') /* 2x2 subsampled Cb:Cr plane */
 #define DRM_FORMAT_NV16		fourcc_code('N', 'V', '1', '6') /* 2x1 subsampled Cr:Cb plane */
 #define DRM_FORMAT_NV61		fourcc_code('N', 'V', '6', '1') /* 2x1 subsampled Cb:Cr plane */
+#define DRM_FORMAT_NV24		fourcc_code('N', 'V', '2', '4') /* non-subsampled Cr:Cb plane */
+#define DRM_FORMAT_NV42		fourcc_code('N', 'V', '4', '2') /* non-subsampled Cb:Cr plane */
 
 /* special NV12 tiled format */
 #define DRM_FORMAT_NV12MT	fourcc_code('T', 'M', '1', '2') /* 2x2 subsampled Cr:Cb plane 64x32 macroblocks */
diff --git a/include/drm/drm_gem_cma_helper.h b/include/drm/drm_gem_cma_helper.h
new file mode 100644
index 00000000000..f0f6b1af25a
--- /dev/null
+++ b/include/drm/drm_gem_cma_helper.h
@@ -0,0 +1,44 @@
+#ifndef __DRM_GEM_CMA_HELPER_H__
+#define __DRM_GEM_CMA_HELPER_H__
+
+struct drm_gem_cma_object {
+	struct drm_gem_object base;
+	dma_addr_t paddr;
+	void *vaddr;
+};
+
+static inline struct drm_gem_cma_object *
+to_drm_gem_cma_obj(struct drm_gem_object *gem_obj)
+{
+	return container_of(gem_obj, struct drm_gem_cma_object, base);
+}
+
+/* free gem object. */
+void drm_gem_cma_free_object(struct drm_gem_object *gem_obj);
+
+/* create memory region for drm framebuffer. */
+int drm_gem_cma_dumb_create(struct drm_file *file_priv,
+		struct drm_device *drm, struct drm_mode_create_dumb *args);
+
+/* map memory region for drm framebuffer to user space. */
+int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv,
+		struct drm_device *drm, uint32_t handle, uint64_t *offset);
+
+/* set vm_flags and we can change the vm attribute to other one at here. */
+int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma);
+
+/*
+ * destroy memory region allocated.
+ *	- a gem handle and physical memory region pointed by a gem object
+ *	would be released by drm_gem_handle_delete().
+ */
+int drm_gem_cma_dumb_destroy(struct drm_file *file_priv,
+		struct drm_device *drm, unsigned int handle);
+
+/* allocate physical memory. */
+struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
+		unsigned int size);
+
+extern const struct vm_operations_struct drm_gem_cma_vm_ops;
+
+#endif /* __DRM_GEM_CMA_HELPER_H__ */
diff --git a/include/drm/drm_memory.h b/include/drm/drm_memory.h
index 15af9b32ae4..4baf57a207e 100644
--- a/include/drm/drm_memory.h
+++ b/include/drm/drm_memory.h
@@ -35,7 +35,7 @@
 
 #include <linux/highmem.h>
 #include <linux/vmalloc.h>
-#include "drmP.h"
+#include <drm/drmP.h>
 
 /**
  * Cut down version of drm_memory_debug.h, which used to be called
diff --git a/include/drm/drm_sarea.h b/include/drm/drm_sarea.h
index ee5389d22c6..413a5642d49 100644
--- a/include/drm/drm_sarea.h
+++ b/include/drm/drm_sarea.h
@@ -32,11 +32,13 @@
 #ifndef _DRM_SAREA_H_
 #define _DRM_SAREA_H_
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /* SAREA area needs to be at least a page */
 #if defined(__alpha__)
 #define SAREA_MAX                       0x2000U
+#elif defined(__mips__)
+#define SAREA_MAX                       0x4000U
 #elif defined(__ia64__)
 #define SAREA_MAX                       0x10000U	/* 64kB */
 #else
diff --git a/include/drm/exynos_drm.h b/include/drm/exynos_drm.h
index c20b0018153..1f2acdfbfd6 100644
--- a/include/drm/exynos_drm.h
+++ b/include/drm/exynos_drm.h
@@ -29,7 +29,7 @@
 #ifndef _EXYNOS_DRM_H_
 #define _EXYNOS_DRM_H_
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /**
  * User-desired buffer creation information structure.
diff --git a/include/drm/i915_drm.h b/include/drm/i915_drm.h
index 8cc70837f92..a940d4e1891 100644
--- a/include/drm/i915_drm.h
+++ b/include/drm/i915_drm.h
@@ -27,7 +27,7 @@
 #ifndef _I915_DRM_H_
 #define _I915_DRM_H_
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /* Please note that modifications to all structs defined here are
  * subject to backwards-compatibility constraints.
@@ -203,6 +203,9 @@ typedef struct _drm_i915_sarea {
 #define DRM_I915_GEM_WAIT	0x2c
 #define DRM_I915_GEM_CONTEXT_CREATE	0x2d
 #define DRM_I915_GEM_CONTEXT_DESTROY	0x2e
+#define DRM_I915_GEM_SET_CACHING	0x2f
+#define DRM_I915_GEM_GET_CACHING	0x30
+#define DRM_I915_REG_READ		0x31
 
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
@@ -227,6 +230,8 @@ typedef struct _drm_i915_sarea {
 #define DRM_IOCTL_I915_GEM_PIN		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_PIN, struct drm_i915_gem_pin)
 #define DRM_IOCTL_I915_GEM_UNPIN	DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_UNPIN, struct drm_i915_gem_unpin)
 #define DRM_IOCTL_I915_GEM_BUSY		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_BUSY, struct drm_i915_gem_busy)
+#define DRM_IOCTL_I915_GEM_SET_CACHING		DRM_IOW(DRM_COMMAND_BASE + DRM_I915_GEM_SET_CACHING, struct drm_i915_gem_caching)
+#define DRM_IOCTL_I915_GEM_GET_CACHING		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_GET_CACHING, struct drm_i915_gem_caching)
 #define DRM_IOCTL_I915_GEM_THROTTLE	DRM_IO ( DRM_COMMAND_BASE + DRM_I915_GEM_THROTTLE)
 #define DRM_IOCTL_I915_GEM_ENTERVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_ENTERVT)
 #define DRM_IOCTL_I915_GEM_LEAVEVT	DRM_IO(DRM_COMMAND_BASE + DRM_I915_GEM_LEAVEVT)
@@ -249,6 +254,7 @@ typedef struct _drm_i915_sarea {
 #define DRM_IOCTL_I915_GEM_WAIT		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_WAIT, struct drm_i915_gem_wait)
 #define DRM_IOCTL_I915_GEM_CONTEXT_CREATE	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_CREATE, struct drm_i915_gem_context_create)
 #define DRM_IOCTL_I915_GEM_CONTEXT_DESTROY	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_DESTROY, struct drm_i915_gem_context_destroy)
+#define DRM_IOCTL_I915_REG_READ			DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_REG_READ, struct drm_i915_reg_read)
 
 /* Allow drivers to submit batchbuffers directly to hardware, relying
  * on the security mechanisms provided by hardware.
@@ -305,6 +311,9 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_HAS_LLC     	 	 17
 #define I915_PARAM_HAS_ALIASING_PPGTT	 18
 #define I915_PARAM_HAS_WAIT_TIMEOUT	 19
+#define I915_PARAM_HAS_SEMAPHORES	 20
+#define I915_PARAM_HAS_PRIME_VMAP_FLUSH	 21
+#define I915_PARAM_RSVD_FOR_FUTURE_USE	 22
 
 typedef struct drm_i915_getparam {
 	int param;
@@ -698,10 +707,31 @@ struct drm_i915_gem_busy {
 	/** Handle of the buffer to check for busy */
 	__u32 handle;
 
-	/** Return busy status (1 if busy, 0 if idle) */
+	/** Return busy status (1 if busy, 0 if idle).
+	 * The high word is used to indicate on which rings the object
+	 * currently resides:
+	 *  16:31 - busy (r or r/w) rings (16 render, 17 bsd, 18 blt, etc)
+	 */
 	__u32 busy;
 };
 
+#define I915_CACHING_NONE		0
+#define I915_CACHING_CACHED		1
+
+struct drm_i915_gem_caching {
+	/**
+	 * Handle of the buffer to set/get the caching level of. */
+	__u32 handle;
+
+	/**
+	 * Cacheing level to apply or return value
+	 *
+	 * bits0-15 are for generic caching control (i.e. the above defined
+	 * values). bits16-31 are reserved for platform-specific variations
+	 * (e.g. l3$ caching on gen7). */
+	__u32 caching;
+};
+
 #define I915_TILING_NONE	0
 #define I915_TILING_X		1
 #define I915_TILING_Y		2
@@ -918,4 +948,8 @@ struct drm_i915_gem_context_destroy {
 	__u32 pad;
 };
 
+struct drm_i915_reg_read {
+	__u64 offset;
+	__u64 val; /* Return value */
+};
 #endif				/* _I915_DRM_H_ */
diff --git a/include/drm/intel-gtt.h b/include/drm/intel-gtt.h
index 8e29d551bb3..2e37e9f02e7 100644
--- a/include/drm/intel-gtt.h
+++ b/include/drm/intel-gtt.h
@@ -30,16 +30,10 @@ void intel_gmch_remove(void);
 bool intel_enable_gtt(void);
 
 void intel_gtt_chipset_flush(void);
-void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg);
-void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries);
-int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
-			 struct scatterlist **sg_list, int *num_sg);
-void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
-				 unsigned int sg_len,
+void intel_gtt_insert_sg_entries(struct sg_table *st,
 				 unsigned int pg_start,
 				 unsigned int flags);
-void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
-			    struct page **pages, unsigned int flags);
+void intel_gtt_clear_range(unsigned int first_entry, unsigned int num_entries);
 
 /* Special gtt memory types */
 #define AGP_DCACHE_MEMORY	1
diff --git a/include/drm/mga_drm.h b/include/drm/mga_drm.h
index fca817009e1..2375bfd6e5e 100644
--- a/include/drm/mga_drm.h
+++ b/include/drm/mga_drm.h
@@ -35,7 +35,7 @@
 #ifndef __MGA_DRM_H__
 #define __MGA_DRM_H__
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /* WARNING: If you change any of these defines, make sure to change the
  * defines in the Xserver file (mga_sarea.h)
diff --git a/include/drm/radeon_drm.h b/include/drm/radeon_drm.h
index dc3a8cd7db8..4766c0f6a83 100644
--- a/include/drm/radeon_drm.h
+++ b/include/drm/radeon_drm.h
@@ -33,7 +33,7 @@
 #ifndef __RADEON_DRM_H__
 #define __RADEON_DRM_H__
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /* WARNING: If you change any of these defines, make sure to change the
  * defines in the X server file (radeon_sarea.h)
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index e15f2a89a27..e8028ade567 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -31,7 +31,7 @@
 #ifndef _TTM_BO_API_H_
 #define _TTM_BO_API_H_
 
-#include "drm_hashtab.h"
+#include <drm/drm_hashtab.h>
 #include <linux/kref.h>
 #include <linux/list.h>
 #include <linux/wait.h>
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 084e8989a6e..d803b92b032 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -30,14 +30,14 @@
 #ifndef _TTM_BO_DRIVER_H_
 #define _TTM_BO_DRIVER_H_
 
-#include "ttm/ttm_bo_api.h"
-#include "ttm/ttm_memory.h"
-#include "ttm/ttm_module.h"
-#include "drm_mm.h"
-#include "drm_global.h"
-#include "linux/workqueue.h"
-#include "linux/fs.h"
-#include "linux/spinlock.h"
+#include <ttm/ttm_bo_api.h>
+#include <ttm/ttm_memory.h>
+#include <ttm/ttm_module.h>
+#include <drm/drm_mm.h>
+#include <drm/drm_global.h>
+#include <linux/workqueue.h>
+#include <linux/fs.h>
+#include <linux/spinlock.h>
 
 struct ttm_backend_func {
 	/**
diff --git a/include/drm/ttm/ttm_execbuf_util.h b/include/drm/ttm/ttm_execbuf_util.h
index 26cc7f9ffa4..1926cae373b 100644
--- a/include/drm/ttm/ttm_execbuf_util.h
+++ b/include/drm/ttm/ttm_execbuf_util.h
@@ -31,7 +31,7 @@
 #ifndef _TTM_EXECBUF_UTIL_H_
 #define _TTM_EXECBUF_UTIL_H_
 
-#include "ttm/ttm_bo_api.h"
+#include <ttm/ttm_bo_api.h>
 #include <linux/list.h>
 
 /**
diff --git a/include/drm/ttm/ttm_lock.h b/include/drm/ttm/ttm_lock.h
index 2e7f0c941b5..2902beb5f68 100644
--- a/include/drm/ttm/ttm_lock.h
+++ b/include/drm/ttm/ttm_lock.h
@@ -49,7 +49,7 @@
 #ifndef _TTM_LOCK_H_
 #define _TTM_LOCK_H_
 
-#include "ttm/ttm_object.h"
+#include <ttm/ttm_object.h>
 #include <linux/wait.h>
 #include <linux/atomic.h>
 
diff --git a/include/drm/ttm/ttm_object.h b/include/drm/ttm/ttm_object.h
index e46054e5255..b01c563b275 100644
--- a/include/drm/ttm/ttm_object.h
+++ b/include/drm/ttm/ttm_object.h
@@ -38,7 +38,7 @@
 #define _TTM_OBJECT_H_
 
 #include <linux/list.h>
-#include "drm_hashtab.h"
+#include <drm/drm_hashtab.h>
 #include <linux/kref.h>
 #include <ttm/ttm_memory.h>
 
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 5fe27400d17..706b962c646 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -26,8 +26,8 @@
 #ifndef TTM_PAGE_ALLOC
 #define TTM_PAGE_ALLOC
 
-#include "ttm_bo_driver.h"
-#include "ttm_memory.h"
+#include <drm/ttm/ttm_bo_driver.h>
+#include <drm/ttm/ttm_memory.h>
 
 /**
  * Initialize pool allocator.
diff --git a/include/drm/via_drm.h b/include/drm/via_drm.h
index 79b3b6e0f6b..8b0533ccbd5 100644
--- a/include/drm/via_drm.h
+++ b/include/drm/via_drm.h
@@ -24,7 +24,7 @@
 #ifndef _VIA_DRM_H_
 #define _VIA_DRM_H_
 
-#include "drm.h"
+#include <drm/drm.h>
 
 /* WARNING: These defines must be the same as what the Xserver uses.
  * if you change them, you must change the defines in the Xserver.
diff --git a/include/linux/Kbuild b/include/linux/Kbuild
index 7f1c0f00db9..e149e8be906 100644
--- a/include/linux/Kbuild
+++ b/include/linux/Kbuild
@@ -20,8 +20,6 @@ header-y += netfilter_ipv6/
 header-y += usb/
 header-y += wimax/
 
-objhdr-y += version.h
-
 ifneq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/a.out.h \
 		  $(srctree)/include/asm-$(SRCARCH)/a.out.h \
 		  $(INSTALL_HDR_PATH)/include/asm-*/a.out.h),)
diff --git a/include/linux/ata.h b/include/linux/ata.h
index 5713d3ac381..408da950217 100644
--- a/include/linux/ata.h
+++ b/include/linux/ata.h
@@ -77,6 +77,9 @@ enum {
 	ATA_ID_EIDE_PIO_IORDY	= 68,
 	ATA_ID_ADDITIONAL_SUPP	= 69,
 	ATA_ID_QUEUE_DEPTH	= 75,
+	ATA_ID_SATA_CAPABILITY	= 76,
+	ATA_ID_SATA_CAPABILITY_2	= 77,
+	ATA_ID_FEATURE_SUPP	= 78,
 	ATA_ID_MAJOR_VER	= 80,
 	ATA_ID_COMMAND_SET_1	= 82,
 	ATA_ID_COMMAND_SET_2	= 83,
@@ -292,6 +295,13 @@ enum {
 
 	/* READ_LOG_EXT pages */
 	ATA_LOG_SATA_NCQ	= 0x10,
+	ATA_LOG_SATA_ID_DEV_DATA  = 0x30,
+	ATA_LOG_SATA_SETTINGS	  = 0x08,
+	ATA_LOG_DEVSLP_MDAT	  = 0x30,
+	ATA_LOG_DEVSLP_MDAT_MASK  = 0x1F,
+	ATA_LOG_DEVSLP_DETO	  = 0x31,
+	ATA_LOG_DEVSLP_VALID	  = 0x37,
+	ATA_LOG_DEVSLP_VALID_MASK = 0x80,
 
 	/* READ/WRITE LONG (obsolete) */
 	ATA_CMD_READ_LONG	= 0x22,
@@ -345,6 +355,7 @@ enum {
 	SATA_FPDMA_IN_ORDER	= 0x04,	/* FPDMA in-order data delivery */
 	SATA_AN			= 0x05,	/* Asynchronous Notification */
 	SATA_SSP		= 0x06,	/* Software Settings Preservation */
+	SATA_DEVSLP		= 0x09,	/* Device Sleep */
 
 	/* feature values for SET_MAX */
 	ATA_SET_MAX_ADDR	= 0x00,
@@ -558,15 +569,17 @@ static inline int ata_is_data(u8 prot)
 #define ata_id_is_ata(id)	(((id)[ATA_ID_CONFIG] & (1 << 15)) == 0)
 #define ata_id_has_lba(id)	((id)[ATA_ID_CAPABILITY] & (1 << 9))
 #define ata_id_has_dma(id)	((id)[ATA_ID_CAPABILITY] & (1 << 8))
-#define ata_id_has_ncq(id)	((id)[76] & (1 << 8))
+#define ata_id_has_ncq(id)	((id)[ATA_ID_SATA_CAPABILITY] & (1 << 8))
 #define ata_id_queue_depth(id)	(((id)[ATA_ID_QUEUE_DEPTH] & 0x1f) + 1)
 #define ata_id_removeable(id)	((id)[ATA_ID_CONFIG] & (1 << 7))
 #define ata_id_has_atapi_AN(id)	\
-	( (((id)[76] != 0x0000) && ((id)[76] != 0xffff)) && \
-	  ((id)[78] & (1 << 5)) )
+	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
+	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
+	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 5)))
 #define ata_id_has_fpdma_aa(id)	\
-	( (((id)[76] != 0x0000) && ((id)[76] != 0xffff)) && \
-	  ((id)[78] & (1 << 2)) )
+	((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \
+	  ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \
+	 ((id)[ATA_ID_FEATURE_SUPP] & (1 << 2)))
 #define ata_id_iordy_disable(id) ((id)[ATA_ID_CAPABILITY] & (1 << 10))
 #define ata_id_has_iordy(id) ((id)[ATA_ID_CAPABILITY] & (1 << 11))
 #define ata_id_u32(id,n)	\
@@ -578,11 +591,12 @@ static inline int ata_is_data(u8 prot)
 	  ((u64) (id)[(n) + 0]) )
 
 #define ata_id_cdb_intr(id)	(((id)[ATA_ID_CONFIG] & 0x60) == 0x20)
-#define ata_id_has_da(id)	((id)[77] & (1 << 4))
+#define ata_id_has_da(id)	((id)[ATA_ID_SATA_CAPABILITY_2] & (1 << 4))
+#define ata_id_has_devslp(id)	((id)[ATA_ID_FEATURE_SUPP] & (1 << 8))
 
 static inline bool ata_id_has_hipm(const u16 *id)
 {
-	u16 val = id[76];
+	u16 val = id[ATA_ID_SATA_CAPABILITY];
 
 	if (val == 0 || val == 0xffff)
 		return false;
@@ -592,7 +606,7 @@ static inline bool ata_id_has_hipm(const u16 *id)
 
 static inline bool ata_id_has_dipm(const u16 *id)
 {
-	u16 val = id[78];
+	u16 val = id[ATA_ID_FEATURE_SUPP];
 
 	if (val == 0 || val == 0xffff)
 		return false;
diff --git a/include/linux/audit.h b/include/linux/audit.h
index 12367cbadfe..2c83e5f7edb 100644
--- a/include/linux/audit.h
+++ b/include/linux/audit.h
@@ -527,10 +527,20 @@ static inline void audit_ptrace(struct task_struct *t)
 extern unsigned int audit_serial(void);
 extern int auditsc_get_stamp(struct audit_context *ctx,
 			      struct timespec *t, unsigned int *serial);
-extern int  audit_set_loginuid(kuid_t loginuid);
-#define audit_get_loginuid(t) ((t)->loginuid)
-#define audit_get_sessionid(t) ((t)->sessionid)
+extern int audit_set_loginuid(kuid_t loginuid);
+
+static inline kuid_t audit_get_loginuid(struct task_struct *tsk)
+{
+	return tsk->loginuid;
+}
+
+static inline int audit_get_sessionid(struct task_struct *tsk)
+{
+	return tsk->sessionid;
+}
+
 extern void audit_log_task_context(struct audit_buffer *ab);
+extern void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk);
 extern void __audit_ipc_obj(struct kern_ipc_perm *ipcp);
 extern void __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, umode_t mode);
 extern int __audit_bprm(struct linux_binprm *bprm);
@@ -625,37 +635,101 @@ static inline void audit_mmap_fd(int fd, int flags)
 extern int audit_n_rules;
 extern int audit_signals;
 #else /* CONFIG_AUDITSYSCALL */
-#define audit_alloc(t) ({ 0; })
-#define audit_free(t) do { ; } while (0)
-#define audit_syscall_entry(ta,a,b,c,d,e) do { ; } while (0)
-#define audit_syscall_exit(r) do { ; } while (0)
-#define audit_dummy_context() 1
-#define audit_getname(n) do { ; } while (0)
-#define audit_putname(n) do { ; } while (0)
-#define __audit_inode(n,d) do { ; } while (0)
-#define __audit_inode_child(i,p) do { ; } while (0)
-#define audit_inode(n,d) do { (void)(d); } while (0)
-#define audit_inode_child(i,p) do { ; } while (0)
-#define audit_core_dumps(i) do { ; } while (0)
-#define audit_seccomp(i,s,c) do { ; } while (0)
-#define auditsc_get_stamp(c,t,s) (0)
-#define audit_get_loginuid(t) (INVALID_UID)
-#define audit_get_sessionid(t) (-1)
-#define audit_log_task_context(b) do { ; } while (0)
-#define audit_ipc_obj(i) ((void)0)
-#define audit_ipc_set_perm(q,u,g,m) ((void)0)
-#define audit_bprm(p) ({ 0; })
-#define audit_socketcall(n,a) ((void)0)
-#define audit_fd_pair(n,a) ((void)0)
-#define audit_sockaddr(len, addr) ({ 0; })
-#define audit_mq_open(o,m,a) ((void)0)
-#define audit_mq_sendrecv(d,l,p,t) ((void)0)
-#define audit_mq_notify(d,n) ((void)0)
-#define audit_mq_getsetattr(d,s) ((void)0)
-#define audit_log_bprm_fcaps(b, ncr, ocr) ({ 0; })
-#define audit_log_capset(pid, ncr, ocr) ((void)0)
-#define audit_mmap_fd(fd, flags) ((void)0)
-#define audit_ptrace(t) ((void)0)
+static inline int audit_alloc(struct task_struct *task)
+{
+	return 0;
+}
+static inline void audit_free(struct task_struct *task)
+{ }
+static inline void audit_syscall_entry(int arch, int major, unsigned long a0,
+				       unsigned long a1, unsigned long a2,
+				       unsigned long a3)
+{ }
+static inline void audit_syscall_exit(void *pt_regs)
+{ }
+static inline int audit_dummy_context(void)
+{
+	return 1;
+}
+static inline void audit_getname(const char *name)
+{ }
+static inline void audit_putname(const char *name)
+{ }
+static inline void __audit_inode(const char *name, const struct dentry *dentry)
+{ }
+static inline void __audit_inode_child(const struct dentry *dentry,
+					const struct inode *parent)
+{ }
+static inline void audit_inode(const char *name, const struct dentry *dentry)
+{ }
+static inline void audit_inode_child(const struct dentry *dentry,
+				     const struct inode *parent)
+{ }
+static inline void audit_core_dumps(long signr)
+{ }
+static inline void __audit_seccomp(unsigned long syscall, long signr, int code)
+{ }
+static inline void audit_seccomp(unsigned long syscall, long signr, int code)
+{ }
+static inline int auditsc_get_stamp(struct audit_context *ctx,
+			      struct timespec *t, unsigned int *serial)
+{
+	return 0;
+}
+static inline kuid_t audit_get_loginuid(struct task_struct *tsk)
+{
+	return INVALID_UID;
+}
+static inline int audit_get_sessionid(struct task_struct *tsk)
+{
+	return -1;
+}
+static inline void audit_log_task_context(struct audit_buffer *ab)
+{ }
+static inline void audit_log_task_info(struct audit_buffer *ab,
+				       struct task_struct *tsk)
+{ }
+static inline void audit_ipc_obj(struct kern_ipc_perm *ipcp)
+{ }
+static inline void audit_ipc_set_perm(unsigned long qbytes, uid_t uid,
+					gid_t gid, umode_t mode)
+{ }
+static inline int audit_bprm(struct linux_binprm *bprm)
+{
+	return 0;
+}
+static inline void audit_socketcall(int nargs, unsigned long *args)
+{ }
+static inline void audit_fd_pair(int fd1, int fd2)
+{ }
+static inline int audit_sockaddr(int len, void *addr)
+{
+	return 0;
+}
+static inline void audit_mq_open(int oflag, umode_t mode, struct mq_attr *attr)
+{ }
+static inline void audit_mq_sendrecv(mqd_t mqdes, size_t msg_len,
+				     unsigned int msg_prio,
+				     const struct timespec *abs_timeout)
+{ }
+static inline void audit_mq_notify(mqd_t mqdes,
+				   const struct sigevent *notification)
+{ }
+static inline void audit_mq_getsetattr(mqd_t mqdes, struct mq_attr *mqstat)
+{ }
+static inline int audit_log_bprm_fcaps(struct linux_binprm *bprm,
+				       const struct cred *new,
+				       const struct cred *old)
+{
+	return 0;
+}
+static inline void audit_log_capset(pid_t pid, const struct cred *new,
+				   const struct cred *old)
+{ }
+static inline void audit_mmap_fd(int fd, int flags)
+{ }
+static inline void audit_ptrace(struct task_struct *t)
+{ }
 #define audit_n_rules 0
 #define audit_signals 0
 #endif /* CONFIG_AUDITSYSCALL */
@@ -679,7 +753,6 @@ extern void		    audit_log_n_hex(struct audit_buffer *ab,
 extern void		    audit_log_n_string(struct audit_buffer *ab,
 					       const char *buf,
 					       size_t n);
-#define audit_log_string(a,b) audit_log_n_string(a, b, strlen(b));
 extern void		    audit_log_n_untrustedstring(struct audit_buffer *ab,
 							const char *string,
 							size_t n);
@@ -696,7 +769,8 @@ extern void		    audit_log_lost(const char *message);
 #ifdef CONFIG_SECURITY
 extern void 		    audit_log_secctx(struct audit_buffer *ab, u32 secid);
 #else
-#define audit_log_secctx(b,s) do { ; } while (0)
+static inline void	    audit_log_secctx(struct audit_buffer *ab, u32 secid)
+{ }
 #endif
 
 extern int		    audit_update_lsm_rules(void);
@@ -708,22 +782,50 @@ extern int  audit_receive_filter(int type, int pid, int seq,
 				void *data, size_t datasz, kuid_t loginuid,
 				u32 sessionid, u32 sid);
 extern int audit_enabled;
-#else
-#define audit_log(c,g,t,f,...) do { ; } while (0)
-#define audit_log_start(c,g,t) ({ NULL; })
-#define audit_log_vformat(b,f,a) do { ; } while (0)
-#define audit_log_format(b,f,...) do { ; } while (0)
-#define audit_log_end(b) do { ; } while (0)
-#define audit_log_n_hex(a,b,l) do { ; } while (0)
-#define audit_log_n_string(a,c,l) do { ; } while (0)
-#define audit_log_string(a,c) do { ; } while (0)
-#define audit_log_n_untrustedstring(a,n,s) do { ; } while (0)
-#define audit_log_untrustedstring(a,s) do { ; } while (0)
-#define audit_log_d_path(b, p, d) do { ; } while (0)
-#define audit_log_key(b, k) do { ; } while (0)
-#define audit_log_link_denied(o, l) do { ; } while (0)
-#define audit_log_secctx(b,s) do { ; } while (0)
+#else /* CONFIG_AUDIT */
+static inline __printf(4, 5)
+void audit_log(struct audit_context *ctx, gfp_t gfp_mask, int type,
+	       const char *fmt, ...)
+{ }
+static inline struct audit_buffer *audit_log_start(struct audit_context *ctx,
+						   gfp_t gfp_mask, int type)
+{
+	return NULL;
+}
+static inline __printf(2, 3)
+void audit_log_format(struct audit_buffer *ab, const char *fmt, ...)
+{ }
+static inline void audit_log_end(struct audit_buffer *ab)
+{ }
+static inline void audit_log_n_hex(struct audit_buffer *ab,
+				   const unsigned char *buf, size_t len)
+{ }
+static inline void audit_log_n_string(struct audit_buffer *ab,
+				      const char *buf, size_t n)
+{ }
+static inline void  audit_log_n_untrustedstring(struct audit_buffer *ab,
+						const char *string, size_t n)
+{ }
+static inline void audit_log_untrustedstring(struct audit_buffer *ab,
+					     const char *string)
+{ }
+static inline void audit_log_d_path(struct audit_buffer *ab,
+				    const char *prefix,
+				    const struct path *path)
+{ }
+static inline void audit_log_key(struct audit_buffer *ab, char *key)
+{ }
+static inline void audit_log_link_denied(const char *string,
+					 const struct path *link)
+{ }
+static inline void audit_log_secctx(struct audit_buffer *ab, u32 secid)
+{ }
 #define audit_enabled 0
-#endif
+#endif /* CONFIG_AUDIT */
+static inline void audit_log_string(struct audit_buffer *ab, const char *buf)
+{
+	audit_log_n_string(ab, buf, strlen(buf));
+}
+
 #endif
 #endif
diff --git a/include/linux/bcma/bcma.h b/include/linux/bcma/bcma.h
index 1954a4e305a..4180eb78d57 100644
--- a/include/linux/bcma/bcma.h
+++ b/include/linux/bcma/bcma.h
@@ -10,7 +10,7 @@
 #include <linux/bcma/bcma_driver_gmac_cmn.h>
 #include <linux/ssb/ssb.h> /* SPROM sharing */
 
-#include "bcma_regs.h"
+#include <linux/bcma/bcma_regs.h>
 
 struct bcma_device;
 struct bcma_bus;
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 366422bc163..37935c2d2e8 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -72,7 +72,7 @@ struct linux_binprm {
 
 /* Function parameter for binfmt->coredump */
 struct coredump_params {
-	long signr;
+	siginfo_t *siginfo;
 	struct pt_regs *regs;
 	struct file *file;
 	unsigned long limit;
@@ -132,7 +132,6 @@ extern int copy_strings_kernel(int argc, const char *const *argv,
 			       struct linux_binprm *bprm);
 extern int prepare_bprm_creds(struct linux_binprm *bprm);
 extern void install_exec_creds(struct linux_binprm *bprm);
-extern void do_coredump(long signr, int exit_code, struct pt_regs *regs);
 extern void set_binfmt(struct linux_binfmt *new);
 extern void free_bprm(struct linux_binprm *);
 
diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h
index d021610efd6..cf6f4d998a7 100644
--- a/include/linux/ceph/ceph_fs.h
+++ b/include/linux/ceph/ceph_fs.h
@@ -12,8 +12,8 @@
 #ifndef CEPH_FS_H
 #define CEPH_FS_H
 
-#include "msgr.h"
-#include "rados.h"
+#include <linux/ceph/msgr.h>
+#include <linux/ceph/rados.h>
 
 /*
  * subprotocol versions.  when specific messages types or high-level
diff --git a/include/linux/ceph/debugfs.h b/include/linux/ceph/debugfs.h
index 2a79702e092..1df086d7882 100644
--- a/include/linux/ceph/debugfs.h
+++ b/include/linux/ceph/debugfs.h
@@ -1,8 +1,8 @@
 #ifndef _FS_CEPH_DEBUGFS_H
 #define _FS_CEPH_DEBUGFS_H
 
-#include "ceph_debug.h"
-#include "types.h"
+#include <linux/ceph/ceph_debug.h>
+#include <linux/ceph/types.h>
 
 #define CEPH_DEFINE_SHOW_FUNC(name)					\
 static int name##_open(struct inode *inode, struct file *file)		\
diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h
index 4bbf2db45f4..63d092822ba 100644
--- a/include/linux/ceph/decode.h
+++ b/include/linux/ceph/decode.h
@@ -6,7 +6,7 @@
 #include <linux/time.h>
 #include <asm/unaligned.h>
 
-#include "types.h"
+#include <linux/ceph/types.h>
 
 /*
  * in all cases,
diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
index 42624789b06..6470792b13d 100644
--- a/include/linux/ceph/libceph.h
+++ b/include/linux/ceph/libceph.h
@@ -1,7 +1,7 @@
 #ifndef _FS_CEPH_LIBCEPH_H
 #define _FS_CEPH_LIBCEPH_H
 
-#include "ceph_debug.h"
+#include <linux/ceph/ceph_debug.h>
 
 #include <asm/unaligned.h>
 #include <linux/backing-dev.h>
@@ -15,12 +15,12 @@
 #include <linux/writeback.h>
 #include <linux/slab.h>
 
-#include "types.h"
-#include "messenger.h"
-#include "msgpool.h"
-#include "mon_client.h"
-#include "osd_client.h"
-#include "ceph_fs.h"
+#include <linux/ceph/types.h>
+#include <linux/ceph/messenger.h>
+#include <linux/ceph/msgpool.h>
+#include <linux/ceph/mon_client.h>
+#include <linux/ceph/osd_client.h>
+#include <linux/ceph/ceph_fs.h>
 
 /*
  * mount options
diff --git a/include/linux/ceph/mdsmap.h b/include/linux/ceph/mdsmap.h
index 9935fac8c10..cb15b5d867c 100644
--- a/include/linux/ceph/mdsmap.h
+++ b/include/linux/ceph/mdsmap.h
@@ -2,7 +2,7 @@
 #define _FS_CEPH_MDSMAP_H
 
 #include <linux/bug.h>
-#include "types.h"
+#include <linux/ceph/types.h>
 
 /*
  * mds map - describe servers in the mds cluster.
diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 189ae063763..14ba5ee738a 100644
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -8,8 +8,8 @@
 #include <linux/uio.h>
 #include <linux/workqueue.h>
 
-#include "types.h"
-#include "buffer.h"
+#include <linux/ceph/types.h>
+#include <linux/ceph/buffer.h>
 
 struct ceph_msg;
 struct ceph_connection;
diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h
index 2113e3850a4..1fb93e9080b 100644
--- a/include/linux/ceph/mon_client.h
+++ b/include/linux/ceph/mon_client.h
@@ -5,7 +5,7 @@
 #include <linux/kref.h>
 #include <linux/rbtree.h>
 
-#include "messenger.h"
+#include <linux/ceph/messenger.h>
 
 struct ceph_client;
 struct ceph_mount_args;
diff --git a/include/linux/ceph/msgpool.h b/include/linux/ceph/msgpool.h
index 09fa96b4343..4b0d3896072 100644
--- a/include/linux/ceph/msgpool.h
+++ b/include/linux/ceph/msgpool.h
@@ -2,7 +2,7 @@
 #define _FS_CEPH_MSGPOOL
 
 #include <linux/mempool.h>
-#include "messenger.h"
+#include <linux/ceph/messenger.h>
 
 /*
  * we use memory pools for preallocating messages we may receive, to
diff --git a/include/linux/ceph/osdmap.h b/include/linux/ceph/osdmap.h
index 311ef8d6aa9..25b930bffea 100644
--- a/include/linux/ceph/osdmap.h
+++ b/include/linux/ceph/osdmap.h
@@ -2,8 +2,8 @@
 #define _FS_CEPH_OSDMAP_H
 
 #include <linux/rbtree.h>
-#include "types.h"
-#include "ceph_fs.h"
+#include <linux/ceph/types.h>
+#include <linux/ceph/ceph_fs.h>
 #include <linux/crush/crush.h>
 
 /*
diff --git a/include/linux/ceph/rados.h b/include/linux/ceph/rados.h
index 0a99099801a..de91fbdf127 100644
--- a/include/linux/ceph/rados.h
+++ b/include/linux/ceph/rados.h
@@ -6,7 +6,7 @@
  * (Reliable Autonomic Distributed Object Store).
  */
 
-#include "msgr.h"
+#include <linux/ceph/msgr.h>
 
 /*
  * osdmap encoding versions
diff --git a/include/linux/ceph/types.h b/include/linux/ceph/types.h
index 28b35a005ec..d3ff1cf2d27 100644
--- a/include/linux/ceph/types.h
+++ b/include/linux/ceph/types.h
@@ -7,9 +7,9 @@
 #include <linux/fcntl.h>
 #include <linux/string.h>
 
-#include "ceph_fs.h"
-#include "ceph_frag.h"
-#include "ceph_hash.h"
+#include <linux/ceph/ceph_fs.h>
+#include <linux/ceph/ceph_frag.h>
+#include <linux/ceph/ceph_hash.h>
 
 /*
  * Identify inodes by both their ino AND snapshot id (a u64).
diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h
index acba894374a..8a7096fcb01 100644
--- a/include/linux/clockchips.h
+++ b/include/linux/clockchips.h
@@ -97,6 +97,8 @@ struct clock_event_device {
 	void			(*broadcast)(const struct cpumask *mask);
 	void			(*set_mode)(enum clock_event_mode mode,
 					    struct clock_event_device *);
+	void			(*suspend)(struct clock_event_device *);
+	void			(*resume)(struct clock_event_device *);
 	unsigned long		min_delta_ticks;
 	unsigned long		max_delta_ticks;
 
@@ -156,6 +158,9 @@ clockevents_calc_mult_shift(struct clock_event_device *ce, u32 freq, u32 minsec)
 				      freq, minsec);
 }
 
+extern void clockevents_suspend(void);
+extern void clockevents_resume(void);
+
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
 extern void clockevents_notify(unsigned long reason, void *arg);
 #else
@@ -164,6 +169,9 @@ extern void clockevents_notify(unsigned long reason, void *arg);
 
 #else /* CONFIG_GENERIC_CLOCKEVENTS_BUILD */
 
+static inline void clockevents_suspend(void) {}
+static inline void clockevents_resume(void) {}
+
 #define clockevents_notify(reason, arg) do { } while (0)
 
 #endif
diff --git a/include/linux/compat.h b/include/linux/compat.h
index 09b28b7369d..3f53d002c7c 100644
--- a/include/linux/compat.h
+++ b/include/linux/compat.h
@@ -160,11 +160,6 @@ struct compat_ustat {
 	char			f_fpack[6];
 };
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
 #define COMPAT_SIGEV_PAD_SIZE	((SIGEV_MAX_SIZE/sizeof(int)) - 3)
 
 typedef struct compat_sigevent {
@@ -590,6 +585,9 @@ asmlinkage ssize_t compat_sys_process_vm_writev(compat_pid_t pid,
 		unsigned long liovcnt, const struct compat_iovec __user *rvec,
 		unsigned long riovcnt, unsigned long flags);
 
+asmlinkage long compat_sys_sendfile(int out_fd, int in_fd,
+				    compat_off_t __user *offset, compat_size_t count);
+
 #else
 
 #define is_compat_task() (0)
diff --git a/include/linux/coredump.h b/include/linux/coredump.h
index ba4b85a6d9b..1775eb8acc0 100644
--- a/include/linux/coredump.h
+++ b/include/linux/coredump.h
@@ -11,5 +11,10 @@
  */
 extern int dump_write(struct file *file, const void *addr, int nr);
 extern int dump_seek(struct file *file, loff_t off);
+#ifdef CONFIG_COREDUMP
+extern void do_coredump(siginfo_t *siginfo, struct pt_regs *regs);
+#else
+static inline void do_coredump(siginfo_t *siginfo, struct pt_regs *regs) {}
+#endif
 
 #endif /* _LINUX_COREDUMP_H */
diff --git a/include/linux/crush/mapper.h b/include/linux/crush/mapper.h
index 71d79f44a7d..5772dee3ecb 100644
--- a/include/linux/crush/mapper.h
+++ b/include/linux/crush/mapper.h
@@ -8,7 +8,7 @@
  * LGPL2
  */
 
-#include "crush.h"
+#include <linux/crush/crush.h>
 
 extern int crush_find_rule(const struct crush_map *map, int ruleset, int type, int size);
 extern int crush_do_rule(const struct crush_map *map,
diff --git a/include/linux/device.h b/include/linux/device.h
index af92883bb4a..86ef6ab553b 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -776,6 +776,13 @@ static inline void pm_suspend_ignore_children(struct device *dev, bool enable)
 	dev->power.ignore_children = enable;
 }
 
+static inline void dev_pm_syscore_device(struct device *dev, bool val)
+{
+#ifdef CONFIG_PM_SLEEP
+	dev->power.syscore = val;
+#endif
+}
+
 static inline void device_lock(struct device *dev)
 {
 	mutex_lock(&dev->mutex);
diff --git a/include/linux/drbd_tag_magic.h b/include/linux/drbd_tag_magic.h
index 81f52f2c572..82de1f9e48b 100644
--- a/include/linux/drbd_tag_magic.h
+++ b/include/linux/drbd_tag_magic.h
@@ -12,7 +12,7 @@ enum packet_types {
 #define NL_INT64(pn, pr, member)
 #define NL_BIT(pn, pr, member)
 #define NL_STRING(pn, pr, member, len)
-#include "drbd_nl.h"
+#include <linux/drbd_nl.h>
 	P_nl_after_last_packet,
 };
 
@@ -37,7 +37,7 @@ static const int tag_list_sizes[] = {
 #define NL_INT64(pn, pr, member)        + 4 + 8
 #define NL_BIT(pn, pr, member)          + 4 + 1
 #define NL_STRING(pn, pr, member, len)  + 4 + (len)
-#include "drbd_nl.h"
+#include <linux/drbd_nl.h>
 };
 
 /* The two highest bits are used for the tag type */
@@ -62,7 +62,7 @@ enum drbd_tags {
 #define NL_INT64(pn, pr, member)       T_ ## member = pn | TT_INT64   | pr ,
 #define NL_BIT(pn, pr, member)         T_ ## member = pn | TT_BIT     | pr ,
 #define NL_STRING(pn, pr, member, len) T_ ## member = pn | TT_STRING  | pr ,
-#include "drbd_nl.h"
+#include <linux/drbd_nl.h>
 };
 
 struct tag {
@@ -78,7 +78,7 @@ static const struct tag tag_descriptions[] = {
 #define NL_INT64(pn, pr, member)       [ pn ] = { #member, TT_INT64   | pr, sizeof(__u64) },
 #define NL_BIT(pn, pr, member)         [ pn ] = { #member, TT_BIT     | pr, sizeof(int)   },
 #define NL_STRING(pn, pr, member, len) [ pn ] = { #member, TT_STRING  | pr, (len)         },
-#include "drbd_nl.h"
+#include <linux/drbd_nl.h>
 };
 
 #endif
diff --git a/include/linux/elf.h b/include/linux/elf.h
index 0a05051a892..59ef40650e1 100644
--- a/include/linux/elf.h
+++ b/include/linux/elf.h
@@ -372,6 +372,12 @@ typedef struct elf64_shdr {
 #define NT_PRPSINFO	3
 #define NT_TASKSTRUCT	4
 #define NT_AUXV		6
+/*
+ * Note to userspace developers: size of NT_SIGINFO note may increase
+ * in the future to accomodate more fields, don't assume it is fixed!
+ */
+#define NT_SIGINFO      0x53494749
+#define NT_FILE         0x46494c45
 #define NT_PRXFPREG     0x46e62b7f      /* copied from gdb5.1/include/elf/common.h */
 #define NT_PPC_VMX	0x100		/* PowerPC Altivec/VMX registers */
 #define NT_PPC_SPE	0x101		/* PowerPC SPE/EVR registers */
diff --git a/include/linux/eventpoll.h b/include/linux/eventpoll.h
index f4bb378ccf6..41085d0f395 100644
--- a/include/linux/eventpoll.h
+++ b/include/linux/eventpoll.h
@@ -25,6 +25,7 @@
 #define EPOLL_CTL_ADD 1
 #define EPOLL_CTL_DEL 2
 #define EPOLL_CTL_MOD 3
+#define EPOLL_CTL_DISABLE 4
 
 /*
  * Request the handling of system wakeup events so as to prevent system suspends
diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h
index 158a41eed31..45052aa814c 100644
--- a/include/linux/fdtable.h
+++ b/include/linux/fdtable.h
@@ -30,31 +30,11 @@ struct fdtable {
 	struct fdtable *next;
 };
 
-static inline void __set_close_on_exec(int fd, struct fdtable *fdt)
-{
-	__set_bit(fd, fdt->close_on_exec);
-}
-
-static inline void __clear_close_on_exec(int fd, struct fdtable *fdt)
-{
-	__clear_bit(fd, fdt->close_on_exec);
-}
-
 static inline bool close_on_exec(int fd, const struct fdtable *fdt)
 {
 	return test_bit(fd, fdt->close_on_exec);
 }
 
-static inline void __set_open_fd(int fd, struct fdtable *fdt)
-{
-	__set_bit(fd, fdt->open_fds);
-}
-
-static inline void __clear_open_fd(int fd, struct fdtable *fdt)
-{
-	__clear_bit(fd, fdt->open_fds);
-}
-
 static inline bool fd_is_open(int fd, const struct fdtable *fdt)
 {
 	return test_bit(fd, fdt->open_fds);
@@ -93,15 +73,8 @@ struct file_operations;
 struct vfsmount;
 struct dentry;
 
-extern int expand_files(struct files_struct *, int nr);
-extern void free_fdtable_rcu(struct rcu_head *rcu);
 extern void __init files_defer_init(void);
 
-static inline void free_fdtable(struct fdtable *fdt)
-{
-	call_rcu(&fdt->rcu, free_fdtable_rcu);
-}
-
 static inline struct file * fcheck_files(struct files_struct *files, unsigned int fd)
 {
 	struct file * file = NULL;
@@ -122,8 +95,20 @@ struct task_struct;
 struct files_struct *get_files_struct(struct task_struct *);
 void put_files_struct(struct files_struct *fs);
 void reset_files_struct(struct files_struct *);
+void daemonize_descriptors(void);
 int unshare_files(struct files_struct **);
 struct files_struct *dup_fd(struct files_struct *, int *);
+void do_close_on_exec(struct files_struct *);
+int iterate_fd(struct files_struct *, unsigned,
+		int (*)(const void *, struct file *, unsigned),
+		const void *);
+
+extern int __alloc_fd(struct files_struct *files,
+		      unsigned start, unsigned end, unsigned flags);
+extern void __fd_install(struct files_struct *files,
+		      unsigned int fd, struct file *file);
+extern int __close_fd(struct files_struct *files,
+		      unsigned int fd);
 
 extern struct kmem_cache *files_cachep;
 
diff --git a/include/linux/file.h b/include/linux/file.h
index a22408bac0d..cbacf4faf44 100644
--- a/include/linux/file.h
+++ b/include/linux/file.h
@@ -26,15 +26,44 @@ static inline void fput_light(struct file *file, int fput_needed)
 		fput(file);
 }
 
+struct fd {
+	struct file *file;
+	int need_put;
+};
+
+static inline void fdput(struct fd fd)
+{
+	if (fd.need_put)
+		fput(fd.file);
+}
+
 extern struct file *fget(unsigned int fd);
 extern struct file *fget_light(unsigned int fd, int *fput_needed);
+
+static inline struct fd fdget(unsigned int fd)
+{
+	int b;
+	struct file *f = fget_light(fd, &b);
+	return (struct fd){f,b};
+}
+
 extern struct file *fget_raw(unsigned int fd);
 extern struct file *fget_raw_light(unsigned int fd, int *fput_needed);
+
+static inline struct fd fdget_raw(unsigned int fd)
+{
+	int b;
+	struct file *f = fget_raw_light(fd, &b);
+	return (struct fd){f,b};
+}
+
+extern int f_dupfd(unsigned int from, struct file *file, unsigned flags);
+extern int replace_fd(unsigned fd, struct file *file, unsigned flags);
 extern void set_close_on_exec(unsigned int fd, int flag);
+extern bool get_close_on_exec(unsigned int fd);
 extern void put_filp(struct file *);
-extern int alloc_fd(unsigned start, unsigned flags);
-extern int get_unused_fd(void);
-#define get_unused_fd_flags(flags) alloc_fd(0, (flags))
+extern int get_unused_fd_flags(unsigned flags);
+#define get_unused_fd() get_unused_fd_flags(0)
 extern void put_unused_fd(unsigned int fd);
 
 extern void fd_install(unsigned int fd, struct file *file);
diff --git a/include/linux/firewire.h b/include/linux/firewire.h
index db04ec5121c..191501afd7f 100644
--- a/include/linux/firewire.h
+++ b/include/linux/firewire.h
@@ -265,8 +265,16 @@ typedef void (*fw_transaction_callback_t)(struct fw_card *card, int rcode,
 					  void *data, size_t length,
 					  void *callback_data);
 /*
- * Important note:  Except for the FCP registers, the callback must guarantee
- * that either fw_send_response() or kfree() is called on the @request.
+ * This callback handles an inbound request subaction.  It is called in
+ * RCU read-side context, therefore must not sleep.
+ *
+ * The callback should not initiate outbound request subactions directly.
+ * Otherwise there is a danger of recursion of inbound and outbound
+ * transactions from and to the local node.
+ *
+ * The callback is responsible that either fw_send_response() or kfree()
+ * is called on the @request, except for FCP registers for which the core
+ * takes care of that.
  */
 typedef void (*fw_address_callback_t)(struct fw_card *card,
 				      struct fw_request *request,
diff --git a/include/linux/frontswap.h b/include/linux/frontswap.h
index 0e4e2eec5c1..30442547b9e 100644
--- a/include/linux/frontswap.h
+++ b/include/linux/frontswap.h
@@ -19,6 +19,8 @@ extern struct frontswap_ops
 extern void frontswap_shrink(unsigned long);
 extern unsigned long frontswap_curr_pages(void);
 extern void frontswap_writethrough(bool);
+#define FRONTSWAP_HAS_EXCLUSIVE_GETS
+extern void frontswap_tmem_exclusive_gets(bool);
 
 extern void __frontswap_init(unsigned type);
 extern int __frontswap_store(struct page *page);
diff --git a/include/linux/fs.h b/include/linux/fs.h
index aa110476a95..ca6d8c806f4 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1074,7 +1074,11 @@ struct file_handle {
 	unsigned char f_handle[0];
 };
 
-#define get_file(x)	atomic_long_inc(&(x)->f_count)
+static inline struct file *get_file(struct file *f)
+{
+	atomic_long_inc(&f->f_count);
+	return f;
+}
 #define fput_atomic(x)	atomic_long_add_unless(&(x)->f_count, -1, 1)
 #define file_count(x)	atomic_long_read(&(x)->f_count)
 
@@ -1126,9 +1130,9 @@ static inline int file_check_writeable(struct file *filp)
 /* Page cache limit. The filesystems should put that into their s_maxbytes 
    limits, otherwise bad things can happen in VM. */ 
 #if BITS_PER_LONG==32
-#define MAX_LFS_FILESIZE	(((u64)PAGE_CACHE_SIZE << (BITS_PER_LONG-1))-1) 
+#define MAX_LFS_FILESIZE	(((loff_t)PAGE_CACHE_SIZE << (BITS_PER_LONG-1))-1) 
 #elif BITS_PER_LONG==64
-#define MAX_LFS_FILESIZE 	0x7fffffffffffffffUL
+#define MAX_LFS_FILESIZE 	((loff_t)0x7fffffffffffffff)
 #endif
 
 #define FL_POSIX	1
diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h
index 5e98eeb2af3..dd7c569aaca 100644
--- a/include/linux/genalloc.h
+++ b/include/linux/genalloc.h
@@ -29,6 +29,20 @@
 
 #ifndef __GENALLOC_H__
 #define __GENALLOC_H__
+/**
+ * Allocation callback function type definition
+ * @map: Pointer to bitmap
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
+ * @nr: The number of zeroed bits we're looking for
+ * @data: optional additional data used by @genpool_algo_t
+ */
+typedef unsigned long (*genpool_algo_t)(unsigned long *map,
+			unsigned long size,
+			unsigned long start,
+			unsigned int nr,
+			void *data);
+
 /*
  *  General purpose special memory pool descriptor.
  */
@@ -36,6 +50,9 @@ struct gen_pool {
 	spinlock_t lock;
 	struct list_head chunks;	/* list of chunks in this pool */
 	int min_alloc_order;		/* minimum allocation order */
+
+	genpool_algo_t algo;		/* allocation function */
+	void *data;
 };
 
 /*
@@ -78,4 +95,14 @@ extern void gen_pool_for_each_chunk(struct gen_pool *,
 	void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *);
 extern size_t gen_pool_avail(struct gen_pool *);
 extern size_t gen_pool_size(struct gen_pool *);
+
+extern void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo,
+		void *data);
+
+extern unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size,
+		unsigned long start, unsigned int nr, void *data);
+
+extern unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
+		unsigned long start, unsigned int nr, void *data);
+
 #endif /* __GENALLOC_H__ */
diff --git a/include/linux/i2c/twl.h b/include/linux/i2c/twl.h
index a12a38107c1..1faa58f9b85 100644
--- a/include/linux/i2c/twl.h
+++ b/include/linux/i2c/twl.h
@@ -188,6 +188,7 @@ int twl_i2c_read(u8 mod_no, u8 *value, u8 reg, unsigned num_bytes);
 
 int twl_get_type(void);
 int twl_get_version(void);
+int twl_get_hfclk_rate(void);
 
 int twl6030_interrupt_unmask(u8 bit_mask, u8 offset);
 int twl6030_interrupt_mask(u8 bit_mask, u8 offset);
diff --git a/include/linux/idr.h b/include/linux/idr.h
index 255491cf522..87259a44c25 100644
--- a/include/linux/idr.h
+++ b/include/linux/idr.h
@@ -38,15 +38,15 @@
 #define IDR_SIZE (1 << IDR_BITS)
 #define IDR_MASK ((1 << IDR_BITS)-1)
 
-#define MAX_ID_SHIFT (sizeof(int)*8 - 1)
-#define MAX_ID_BIT (1U << MAX_ID_SHIFT)
-#define MAX_ID_MASK (MAX_ID_BIT - 1)
+#define MAX_IDR_SHIFT (sizeof(int)*8 - 1)
+#define MAX_IDR_BIT (1U << MAX_IDR_SHIFT)
+#define MAX_IDR_MASK (MAX_IDR_BIT - 1)
 
 /* Leave the possibility of an incomplete final layer */
-#define MAX_LEVEL (MAX_ID_SHIFT + IDR_BITS - 1) / IDR_BITS
+#define MAX_IDR_LEVEL ((MAX_IDR_SHIFT + IDR_BITS - 1) / IDR_BITS)
 
 /* Number of id_layer structs to leave in free list */
-#define IDR_FREE_MAX MAX_LEVEL + MAX_LEVEL
+#define MAX_IDR_FREE (MAX_IDR_LEVEL * 2)
 
 struct idr_layer {
 	unsigned long		 bitmap; /* A zero bit means "space here" */
diff --git a/include/linux/ima.h b/include/linux/ima.h
index 6ac8e50c6cf..2c7223d7e73 100644
--- a/include/linux/ima.h
+++ b/include/linux/ima.h
@@ -39,5 +39,32 @@ static inline int ima_file_mmap(struct file *file, unsigned long prot)
 {
 	return 0;
 }
+
 #endif /* CONFIG_IMA_H */
+
+#ifdef CONFIG_IMA_APPRAISE
+extern void ima_inode_post_setattr(struct dentry *dentry);
+extern int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name,
+		       const void *xattr_value, size_t xattr_value_len);
+extern int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name);
+#else
+static inline void ima_inode_post_setattr(struct dentry *dentry)
+{
+	return;
+}
+
+static inline int ima_inode_setxattr(struct dentry *dentry,
+				     const char *xattr_name,
+				     const void *xattr_value,
+				     size_t xattr_value_len)
+{
+	return 0;
+}
+
+static inline int ima_inode_removexattr(struct dentry *dentry,
+					const char *xattr_name)
+{
+	return 0;
+}
+#endif /* CONFIG_IMA_APPRAISE_H */
 #endif /* _LINUX_IMA_H */
diff --git a/include/linux/init.h b/include/linux/init.h
index 5e664f67161..e59041e21df 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -43,11 +43,22 @@
    discard it in modules) */
 #define __init		__section(.init.text) __cold notrace
 #define __initdata	__section(.init.data)
-#define __initconst	__section(.init.rodata)
+#define __initconst	__constsection(.init.rodata)
 #define __exitdata	__section(.exit.data)
 #define __exit_call	__used __section(.exitcall.exit)
 
 /*
+ * Some architecture have tool chains which do not handle rodata attributes
+ * correctly. For those disable special sections for const, so that other
+ * architectures can annotate correctly.
+ */
+#ifdef CONFIG_BROKEN_RODATA
+#define __constsection(x)
+#else
+#define __constsection(x) __section(x)
+#endif
+
+/*
  * modpost check for section mismatches during the kernel build.
  * A section mismatch happens when there are references from a
  * code or data section to an init section (both code or data).
@@ -66,7 +77,7 @@
  */
 #define __ref            __section(.ref.text) noinline
 #define __refdata        __section(.ref.data)
-#define __refconst       __section(.ref.rodata)
+#define __refconst       __constsection(.ref.rodata)
 
 /* compatibility defines */
 #define __init_refok     __ref
@@ -85,26 +96,26 @@
 /* Used for HOTPLUG */
 #define __devinit        __section(.devinit.text) __cold notrace
 #define __devinitdata    __section(.devinit.data)
-#define __devinitconst   __section(.devinit.rodata)
+#define __devinitconst   __constsection(.devinit.rodata)
 #define __devexit        __section(.devexit.text) __exitused __cold notrace
 #define __devexitdata    __section(.devexit.data)
-#define __devexitconst   __section(.devexit.rodata)
+#define __devexitconst   __constsection(.devexit.rodata)
 
 /* Used for HOTPLUG_CPU */
 #define __cpuinit        __section(.cpuinit.text) __cold notrace
 #define __cpuinitdata    __section(.cpuinit.data)
-#define __cpuinitconst   __section(.cpuinit.rodata)
+#define __cpuinitconst   __constsection(.cpuinit.rodata)
 #define __cpuexit        __section(.cpuexit.text) __exitused __cold notrace
 #define __cpuexitdata    __section(.cpuexit.data)
-#define __cpuexitconst   __section(.cpuexit.rodata)
+#define __cpuexitconst   __constsection(.cpuexit.rodata)
 
 /* Used for MEMORY_HOTPLUG */
 #define __meminit        __section(.meminit.text) __cold notrace
 #define __meminitdata    __section(.meminit.data)
-#define __meminitconst   __section(.meminit.rodata)
+#define __meminitconst   __constsection(.meminit.rodata)
 #define __memexit        __section(.memexit.text) __exitused __cold notrace
 #define __memexitdata    __section(.memexit.data)
-#define __memexitconst   __section(.memexit.rodata)
+#define __memexitconst   __constsection(.memexit.rodata)
 
 /* For assembly routines */
 #define __HEAD		.section	".head.text","ax"
diff --git a/include/linux/integrity.h b/include/linux/integrity.h
index a0c41256cb9..66c5fe9550a 100644
--- a/include/linux/integrity.h
+++ b/include/linux/integrity.h
@@ -22,13 +22,14 @@ enum integrity_status {
 
 /* List of EVM protected security xattrs */
 #ifdef CONFIG_INTEGRITY
-extern int integrity_inode_alloc(struct inode *inode);
+extern struct integrity_iint_cache *integrity_inode_get(struct inode *inode);
 extern void integrity_inode_free(struct inode *inode);
 
 #else
-static inline int integrity_inode_alloc(struct inode *inode)
+static inline struct integrity_iint_cache *
+				integrity_inode_get(struct inode *inode)
 {
-	return 0;
+	return NULL;
 }
 
 static inline void integrity_inode_free(struct inode *inode)
diff --git a/include/linux/ioport.h b/include/linux/ioport.h
index 589e0e75efa..85ac9b9b72a 100644
--- a/include/linux/ioport.h
+++ b/include/linux/ioport.h
@@ -29,8 +29,9 @@ struct resource {
 #define IORESOURCE_BITS		0x000000ff	/* Bus-specific bits */
 
 #define IORESOURCE_TYPE_BITS	0x00001f00	/* Resource type */
-#define IORESOURCE_IO		0x00000100
+#define IORESOURCE_IO		0x00000100	/* PCI/ISA I/O ports */
 #define IORESOURCE_MEM		0x00000200
+#define IORESOURCE_REG		0x00000300	/* Register offsets */
 #define IORESOURCE_IRQ		0x00000400
 #define IORESOURCE_DMA		0x00000800
 #define IORESOURCE_BUS		0x00001000
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 2ce09aa7d3b..0a6d6ba44c8 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -101,9 +101,13 @@ struct kvm_userspace_memory_region {
 	__u64 userspace_addr; /* start of the userspace allocated memory */
 };
 
-/* for kvm_memory_region::flags */
-#define KVM_MEM_LOG_DIRTY_PAGES  1UL
-#define KVM_MEMSLOT_INVALID      (1UL << 1)
+/*
+ * The bit 0 ~ bit 15 of kvm_memory_region::flags are visible for userspace,
+ * other bits are reserved for kvm internal use which are defined in
+ * include/linux/kvm_host.h.
+ */
+#define KVM_MEM_LOG_DIRTY_PAGES	(1UL << 0)
+#define KVM_MEM_READONLY	(1UL << 1)
 
 /* for KVM_IRQ_LINE */
 struct kvm_irq_level {
@@ -618,6 +622,10 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_GET_SMMU_INFO 78
 #define KVM_CAP_S390_COW 79
 #define KVM_CAP_PPC_ALLOC_HTAB 80
+#ifdef __KVM_HAVE_READONLY_MEM
+#define KVM_CAP_READONLY_MEM 81
+#endif
+#define KVM_CAP_IRQFD_RESAMPLE 82
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -683,12 +691,21 @@ struct kvm_xen_hvm_config {
 #endif
 
 #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)
+/*
+ * Available with KVM_CAP_IRQFD_RESAMPLE
+ *
+ * KVM_IRQFD_FLAG_RESAMPLE indicates resamplefd is valid and specifies
+ * the irqfd to operate in resampling mode for level triggered interrupt
+ * emlation.  See Documentation/virtual/kvm/api.txt.
+ */
+#define KVM_IRQFD_FLAG_RESAMPLE (1 << 1)
 
 struct kvm_irqfd {
 	__u32 fd;
 	__u32 gsi;
 	__u32 flags;
-	__u8  pad[20];
+	__u32 resamplefd;
+	__u8  pad[16];
 };
 
 struct kvm_clock_data {
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 8a59e0abe5f..93bfc9f9815 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -21,6 +21,7 @@
 #include <linux/slab.h>
 #include <linux/rcupdate.h>
 #include <linux/ratelimit.h>
+#include <linux/err.h>
 #include <asm/signal.h>
 
 #include <linux/kvm.h>
@@ -35,6 +36,13 @@
 #endif
 
 /*
+ * The bit 16 ~ bit 31 of kvm_memory_region::flags are internally used
+ * in kvm, other bits are visible for userspace which are defined in
+ * include/linux/kvm_h.
+ */
+#define KVM_MEMSLOT_INVALID	(1UL << 16)
+
+/*
  * If we support unaligned MMIO, at most one fragment will be split into two:
  */
 #ifdef KVM_UNALIGNED_MMIO
@@ -49,6 +57,47 @@
 	(KVM_MMIO_SIZE / KVM_USER_MMIO_SIZE + KVM_EXTRA_MMIO_FRAGMENTS)
 
 /*
+ * For the normal pfn, the highest 12 bits should be zero,
+ * so we can mask these bits to indicate the error.
+ */
+#define KVM_PFN_ERR_MASK	(0xfffULL << 52)
+
+#define KVM_PFN_ERR_FAULT	(KVM_PFN_ERR_MASK)
+#define KVM_PFN_ERR_HWPOISON	(KVM_PFN_ERR_MASK + 1)
+#define KVM_PFN_ERR_BAD		(KVM_PFN_ERR_MASK + 2)
+#define KVM_PFN_ERR_RO_FAULT	(KVM_PFN_ERR_MASK + 3)
+
+static inline bool is_error_pfn(pfn_t pfn)
+{
+	return !!(pfn & KVM_PFN_ERR_MASK);
+}
+
+static inline bool is_noslot_pfn(pfn_t pfn)
+{
+	return pfn == KVM_PFN_ERR_BAD;
+}
+
+static inline bool is_invalid_pfn(pfn_t pfn)
+{
+	return !is_noslot_pfn(pfn) && is_error_pfn(pfn);
+}
+
+#define KVM_HVA_ERR_BAD		(PAGE_OFFSET)
+#define KVM_HVA_ERR_RO_BAD	(PAGE_OFFSET + PAGE_SIZE)
+
+static inline bool kvm_is_error_hva(unsigned long addr)
+{
+	return addr >= PAGE_OFFSET;
+}
+
+#define KVM_ERR_PTR_BAD_PAGE	(ERR_PTR(-ENOENT))
+
+static inline bool is_error_page(struct page *page)
+{
+	return IS_ERR(page);
+}
+
+/*
  * vcpu->requests bit members
  */
 #define KVM_REQ_TLB_FLUSH          0
@@ -70,7 +119,8 @@
 #define KVM_REQ_PMU               16
 #define KVM_REQ_PMI               17
 
-#define KVM_USERSPACE_IRQ_SOURCE_ID	0
+#define KVM_USERSPACE_IRQ_SOURCE_ID		0
+#define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1
 
 struct kvm;
 struct kvm_vcpu;
@@ -183,6 +233,18 @@ struct kvm_vcpu {
 	} async_pf;
 #endif
 
+#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
+	/*
+	 * Cpu relax intercept or pause loop exit optimization
+	 * in_spin_loop: set when a vcpu does a pause loop exit
+	 *  or cpu relax intercepted.
+	 * dy_eligible: indicates whether vcpu is eligible for directed yield.
+	 */
+	struct {
+		bool in_spin_loop;
+		bool dy_eligible;
+	} spin_loop;
+#endif
 	struct kvm_vcpu_arch arch;
 };
 
@@ -201,7 +263,6 @@ struct kvm_memory_slot {
 	gfn_t base_gfn;
 	unsigned long npages;
 	unsigned long flags;
-	unsigned long *rmap;
 	unsigned long *dirty_bitmap;
 	struct kvm_arch_memory_slot arch;
 	unsigned long userspace_addr;
@@ -283,6 +344,8 @@ struct kvm {
 	struct {
 		spinlock_t        lock;
 		struct list_head  items;
+		struct list_head  resampler_list;
+		struct mutex      resampler_lock;
 	} irqfds;
 	struct list_head ioeventfds;
 #endif
@@ -348,7 +411,7 @@ static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
 
-void vcpu_load(struct kvm_vcpu *vcpu);
+int __must_check vcpu_load(struct kvm_vcpu *vcpu);
 void vcpu_put(struct kvm_vcpu *vcpu);
 
 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
@@ -378,23 +441,6 @@ id_to_memslot(struct kvm_memslots *slots, int id)
 	return slot;
 }
 
-#define HPA_MSB ((sizeof(hpa_t) * 8) - 1)
-#define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB)
-static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; }
-
-extern struct page *bad_page;
-extern struct page *fault_page;
-
-extern pfn_t bad_pfn;
-extern pfn_t fault_pfn;
-
-int is_error_page(struct page *page);
-int is_error_pfn(pfn_t pfn);
-int is_hwpoison_pfn(pfn_t pfn);
-int is_fault_pfn(pfn_t pfn);
-int is_noslot_pfn(pfn_t pfn);
-int is_invalid_pfn(pfn_t pfn);
-int kvm_is_error_hva(unsigned long addr);
 int kvm_set_memory_region(struct kvm *kvm,
 			  struct kvm_userspace_memory_region *mem,
 			  int user_alloc);
@@ -415,28 +461,33 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
 				int user_alloc);
 bool kvm_largepages_enabled(void);
 void kvm_disable_largepages(void);
-void kvm_arch_flush_shadow(struct kvm *kvm);
+/* flush all memory translations */
+void kvm_arch_flush_shadow_all(struct kvm *kvm);
+/* flush memory translations pointing to 'slot' */
+void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
+				   struct kvm_memory_slot *slot);
 
 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
 			    int nr_pages);
 
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
+unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
 void kvm_release_page_clean(struct page *page);
 void kvm_release_page_dirty(struct page *page);
 void kvm_set_page_dirty(struct page *page);
 void kvm_set_page_accessed(struct page *page);
 
-pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr);
 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn);
 pfn_t gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, bool *async,
 		       bool write_fault, bool *writable);
 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
 		      bool *writable);
-pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
-			 struct kvm_memory_slot *slot, gfn_t gfn);
-void kvm_release_pfn_dirty(pfn_t);
+pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn);
+pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn);
+
+void kvm_release_pfn_dirty(pfn_t pfn);
 void kvm_release_pfn_clean(pfn_t pfn);
 void kvm_set_pfn_dirty(pfn_t pfn);
 void kvm_set_pfn_accessed(pfn_t pfn);
@@ -494,6 +545,7 @@ int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 				   struct
 				   kvm_userspace_memory_region *mem,
 				   int user_alloc);
+int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level);
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg);
 
@@ -573,7 +625,7 @@ void kvm_arch_sync_events(struct kvm *kvm);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 
-int kvm_is_mmio_pfn(pfn_t pfn);
+bool kvm_is_mmio_pfn(pfn_t pfn);
 
 struct kvm_irq_ack_notifier {
 	struct hlist_node link;
@@ -728,6 +780,12 @@ __gfn_to_memslot(struct kvm_memslots *slots, gfn_t gfn)
 	return search_memslots(slots, gfn);
 }
 
+static inline unsigned long
+__gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
+{
+	return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
+}
+
 static inline int memslot_id(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn_to_memslot(kvm, gfn)->id;
@@ -740,10 +798,12 @@ static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
 		(base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
 }
 
-static inline unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
-					       gfn_t gfn)
+static inline gfn_t
+hva_to_gfn_memslot(unsigned long hva, struct kvm_memory_slot *slot)
 {
-	return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
+	gfn_t gfn_offset = (hva - slot->userspace_addr) >> PAGE_SHIFT;
+
+	return slot->base_gfn + gfn_offset;
 }
 
 static inline gpa_t gfn_to_gpa(gfn_t gfn)
@@ -899,5 +959,32 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu)
 	}
 }
 
+#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
+
+static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
+{
+	vcpu->spin_loop.in_spin_loop = val;
+}
+static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
+{
+	vcpu->spin_loop.dy_eligible = val;
+}
+
+#else /* !CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
+
+static inline void kvm_vcpu_set_in_spin_loop(struct kvm_vcpu *vcpu, bool val)
+{
+}
+
+static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
+{
+}
+
+static inline bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
+{
+	return true;
+}
+
+#endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
 #endif
 
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 64f90e17e51..77eeeda2b6e 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -162,6 +162,7 @@ enum {
 	ATA_DFLAG_DETACHED	= (1 << 25),
 
 	ATA_DFLAG_DA		= (1 << 26), /* device supports Device Attention */
+	ATA_DFLAG_DEVSLP	= (1 << 27), /* device supports Device Sleep */
 
 	ATA_DEV_UNKNOWN		= 0,	/* unknown device */
 	ATA_DEV_ATA		= 1,	/* ATA device */
@@ -184,6 +185,7 @@ enum {
 	ATA_LFLAG_DISABLED	= (1 << 6), /* link is disabled */
 	ATA_LFLAG_SW_ACTIVITY	= (1 << 7), /* keep activity stats */
 	ATA_LFLAG_NO_LPM	= (1 << 8), /* disable LPM on this link */
+	ATA_LFLAG_RST_ONCE	= (1 << 9), /* limit recovery to one reset */
 
 	/* struct ata_port flags */
 	ATA_FLAG_SLAVE_POSS	= (1 << 0), /* host supports slave dev */
@@ -649,6 +651,9 @@ struct ata_device {
 		u32		gscr[SATA_PMP_GSCR_DWORDS]; /* PMP GSCR block */
 	};
 
+	/* Identify Device Data Log (30h), SATA Settings (page 08h) */
+	u8			sata_settings[ATA_SECT_SIZE];
+
 	/* error history */
 	int			spdn_cnt;
 	/* ering is CLEAR_END, read comment above CLEAR_END */
@@ -986,8 +991,7 @@ extern int ata_host_activate(struct ata_host *host, int irq,
 			     irq_handler_t irq_handler, unsigned long irq_flags,
 			     struct scsi_host_template *sht);
 extern void ata_host_detach(struct ata_host *host);
-extern void ata_host_init(struct ata_host *, struct device *,
-			  unsigned long, struct ata_port_operations *);
+extern void ata_host_init(struct ata_host *, struct device *, struct ata_port_operations *);
 extern int ata_scsi_detect(struct scsi_host_template *sht);
 extern int ata_scsi_ioctl(struct scsi_device *dev, int cmd, void __user *arg);
 extern int ata_scsi_queuecmd(struct Scsi_Host *h, struct scsi_cmnd *cmd);
@@ -1012,6 +1016,17 @@ extern bool ata_link_offline(struct ata_link *link);
 #ifdef CONFIG_PM
 extern int ata_host_suspend(struct ata_host *host, pm_message_t mesg);
 extern void ata_host_resume(struct ata_host *host);
+extern int ata_sas_port_async_suspend(struct ata_port *ap, int *async);
+extern int ata_sas_port_async_resume(struct ata_port *ap, int *async);
+#else
+static inline int ata_sas_port_async_suspend(struct ata_port *ap, int *async)
+{
+	return 0;
+}
+static inline int ata_sas_port_async_resume(struct ata_port *ap, int *async)
+{
+	return 0;
+}
 #endif
 extern int ata_ratelimit(void);
 extern void ata_msleep(struct ata_port *ap, unsigned int msecs);
diff --git a/include/linux/mfd/88pm860x.h b/include/linux/mfd/88pm860x.h
index 7b24943779f..ef3e6b70117 100644
--- a/include/linux/mfd/88pm860x.h
+++ b/include/linux/mfd/88pm860x.h
@@ -34,22 +34,6 @@ enum {
 	PM8606_ID_MAX,
 };
 
-enum {
-	PM8606_BACKLIGHT1 = 0,
-	PM8606_BACKLIGHT2,
-	PM8606_BACKLIGHT3,
-};
-
-enum {
-	PM8606_LED1_RED = 0,
-	PM8606_LED1_GREEN,
-	PM8606_LED1_BLUE,
-	PM8606_LED2_RED,
-	PM8606_LED2_GREEN,
-	PM8606_LED2_BLUE,
-	PM8607_LED_VIBRATOR,
-};
-
 
 /* 8606 Registers */
 #define PM8606_DCM_BOOST		(0x00)
@@ -322,7 +306,7 @@ struct pm860x_chip {
 	struct regmap           *regmap_companion;
 
 	int			buck3_double;	/* DVC ramp slope double */
-	unsigned short		companion_addr;
+	int			companion_addr;
 	unsigned short		osc_vote;
 	int			id;
 	int			irq_mode;
@@ -340,16 +324,12 @@ enum {
 };
 
 struct pm860x_backlight_pdata {
-	int		id;
 	int		pwm;
 	int		iset;
-	unsigned long	flags;
 };
 
 struct pm860x_led_pdata {
-	int		id;
 	int		iset;
-	unsigned long	flags;
 };
 
 struct pm860x_rtc_pdata {
@@ -379,15 +359,29 @@ struct pm860x_platform_data {
 	struct pm860x_rtc_pdata		*rtc;
 	struct pm860x_touch_pdata	*touch;
 	struct pm860x_power_pdata	*power;
-	struct regulator_init_data	*regulator;
-
-	unsigned short	companion_addr;	/* I2C address of companion chip */
+	struct regulator_init_data	*buck1;
+	struct regulator_init_data	*buck2;
+	struct regulator_init_data	*buck3;
+	struct regulator_init_data	*ldo1;
+	struct regulator_init_data	*ldo2;
+	struct regulator_init_data	*ldo3;
+	struct regulator_init_data	*ldo4;
+	struct regulator_init_data	*ldo5;
+	struct regulator_init_data	*ldo6;
+	struct regulator_init_data	*ldo7;
+	struct regulator_init_data	*ldo8;
+	struct regulator_init_data	*ldo9;
+	struct regulator_init_data	*ldo10;
+	struct regulator_init_data	*ldo12;
+	struct regulator_init_data	*ldo_vibrator;
+	struct regulator_init_data	*ldo14;
+
+	int 		companion_addr;	/* I2C address of companion chip */
 	int		i2c_port;	/* Controlled by GI2C or PI2C */
 	int		irq_mode;	/* Clear interrupt by read/write(0/1) */
 	int		irq_base;	/* IRQ base number of 88pm860x */
 	int		num_leds;
 	int		num_backlights;
-	int		num_regulators;
 };
 
 extern int pm8606_osc_enable(struct pm860x_chip *, unsigned short);
@@ -408,8 +402,4 @@ extern int pm860x_page_bulk_write(struct i2c_client *, int, int,
 extern int pm860x_page_set_bits(struct i2c_client *, int, unsigned char,
 				unsigned char);
 
-extern int pm860x_device_init(struct pm860x_chip *chip,
-			      struct pm860x_platform_data *pdata) __devinit ;
-extern void pm860x_device_exit(struct pm860x_chip *chip) __devexit ;
-
 #endif /* __LINUX_MFD_88PM860X_H */
diff --git a/include/linux/mfd/ab3100.h b/include/linux/mfd/ab3100.h
new file mode 100644
index 00000000000..afd3080bde2
--- /dev/null
+++ b/include/linux/mfd/ab3100.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2007-2009 ST-Ericsson AB
+ * License terms: GNU General Public License (GPL) version 2
+ * AB3100 core access functions
+ * Author: Linus Walleij <linus.walleij@stericsson.com>
+ *
+ */
+
+#include <linux/regulator/machine.h>
+
+struct device;
+
+#ifndef MFD_AB3100_H
+#define MFD_AB3100_H
+
+
+#define AB3100_P1A	0xc0
+#define AB3100_P1B	0xc1
+#define AB3100_P1C	0xc2
+#define AB3100_P1D	0xc3
+#define AB3100_P1E	0xc4
+#define AB3100_P1F	0xc5
+#define AB3100_P1G	0xc6
+#define AB3100_R2A	0xc7
+#define AB3100_R2B	0xc8
+
+/*
+ * AB3100, EVENTA1, A2 and A3 event register flags
+ * these are catenated into a single 32-bit flag in the code
+ * for event notification broadcasts.
+ */
+#define AB3100_EVENTA1_ONSWA				(0x01<<16)
+#define AB3100_EVENTA1_ONSWB				(0x02<<16)
+#define AB3100_EVENTA1_ONSWC				(0x04<<16)
+#define AB3100_EVENTA1_DCIO				(0x08<<16)
+#define AB3100_EVENTA1_OVER_TEMP			(0x10<<16)
+#define AB3100_EVENTA1_SIM_OFF				(0x20<<16)
+#define AB3100_EVENTA1_VBUS				(0x40<<16)
+#define AB3100_EVENTA1_VSET_USB				(0x80<<16)
+
+#define AB3100_EVENTA2_READY_TX				(0x01<<8)
+#define AB3100_EVENTA2_READY_RX				(0x02<<8)
+#define AB3100_EVENTA2_OVERRUN_ERROR			(0x04<<8)
+#define AB3100_EVENTA2_FRAMING_ERROR			(0x08<<8)
+#define AB3100_EVENTA2_CHARG_OVERCURRENT		(0x10<<8)
+#define AB3100_EVENTA2_MIDR				(0x20<<8)
+#define AB3100_EVENTA2_BATTERY_REM			(0x40<<8)
+#define AB3100_EVENTA2_ALARM				(0x80<<8)
+
+#define AB3100_EVENTA3_ADC_TRIG5			(0x01)
+#define AB3100_EVENTA3_ADC_TRIG4			(0x02)
+#define AB3100_EVENTA3_ADC_TRIG3			(0x04)
+#define AB3100_EVENTA3_ADC_TRIG2			(0x08)
+#define AB3100_EVENTA3_ADC_TRIGVBAT			(0x10)
+#define AB3100_EVENTA3_ADC_TRIGVTX			(0x20)
+#define AB3100_EVENTA3_ADC_TRIG1			(0x40)
+#define AB3100_EVENTA3_ADC_TRIG0			(0x80)
+
+/* AB3100, STR register flags */
+#define AB3100_STR_ONSWA				(0x01)
+#define AB3100_STR_ONSWB				(0x02)
+#define AB3100_STR_ONSWC				(0x04)
+#define AB3100_STR_DCIO					(0x08)
+#define AB3100_STR_BOOT_MODE				(0x10)
+#define AB3100_STR_SIM_OFF				(0x20)
+#define AB3100_STR_BATT_REMOVAL				(0x40)
+#define AB3100_STR_VBUS					(0x80)
+
+/*
+ * AB3100 contains 8 regulators, one external regulator controller
+ * and a buck converter, further the LDO E and buck converter can
+ * have separate settings if they are in sleep mode, this is
+ * modeled as a separate regulator.
+ */
+#define AB3100_NUM_REGULATORS				10
+
+/**
+ * struct ab3100
+ * @access_mutex: lock out concurrent accesses to the AB3100 registers
+ * @dev: pointer to the containing device
+ * @i2c_client: I2C client for this chip
+ * @testreg_client: secondary client for test registers
+ * @chip_name: name of this chip variant
+ * @chip_id: 8 bit chip ID for this chip variant
+ * @event_subscribers: event subscribers are listed here
+ * @startup_events: a copy of the first reading of the event registers
+ * @startup_events_read: whether the first events have been read
+ *
+ * This struct is PRIVATE and devices using it should NOT
+ * access ANY fields. It is used as a token for calling the
+ * AB3100 functions.
+ */
+struct ab3100 {
+	struct mutex access_mutex;
+	struct device *dev;
+	struct i2c_client *i2c_client;
+	struct i2c_client *testreg_client;
+	char chip_name[32];
+	u8 chip_id;
+	struct blocking_notifier_head event_subscribers;
+	u8 startup_events[3];
+	bool startup_events_read;
+};
+
+/**
+ * struct ab3100_platform_data
+ * Data supplied to initialize board connections to the AB3100
+ * @reg_constraints: regulator constraints for target board
+ *     the order of these constraints are: LDO A, C, D, E,
+ *     F, G, H, K, EXT and BUCK.
+ * @reg_initvals: initial values for the regulator registers
+ *     plus two sleep settings for LDO E and the BUCK converter.
+ *     exactly AB3100_NUM_REGULATORS+2 values must be sent in.
+ *     Order: LDO A, C, E, E sleep, F, G, H, K, EXT, BUCK,
+ *     BUCK sleep, LDO D. (LDO D need to be initialized last.)
+ * @external_voltage: voltage level of the external regulator.
+ */
+struct ab3100_platform_data {
+	struct regulator_init_data reg_constraints[AB3100_NUM_REGULATORS];
+	u8 reg_initvals[AB3100_NUM_REGULATORS+2];
+	int external_voltage;
+};
+
+int ab3100_event_register(struct ab3100 *ab3100,
+			  struct notifier_block *nb);
+int ab3100_event_unregister(struct ab3100 *ab3100,
+			    struct notifier_block *nb);
+
+#endif /*  MFD_AB3100_H */
diff --git a/include/linux/mfd/abx500.h b/include/linux/mfd/abx500.h
index 1318ca62263..5d5298d5602 100644
--- a/include/linux/mfd/abx500.h
+++ b/include/linux/mfd/abx500.h
@@ -1,12 +1,9 @@
 /*
  * Copyright (C) 2007-2009 ST-Ericsson AB
  * License terms: GNU General Public License (GPL) version 2
- * AB3100 core access functions
- * Author: Linus Walleij <linus.walleij@stericsson.com>
  *
  * ABX500 core access functions.
- * The abx500 interface is used for the Analog Baseband chip
- * ab3100 and ab8500.
+ * The abx500 interface is used for the Analog Baseband chips.
  *
  * Author: Mattias Wallin <mattias.wallin@stericsson.com>
  * Author: Mattias Nilsson <mattias.i.nilsson@stericsson.com>
@@ -21,118 +18,6 @@ struct device;
 #ifndef MFD_ABX500_H
 #define MFD_ABX500_H
 
-#define AB3100_P1A	0xc0
-#define AB3100_P1B	0xc1
-#define AB3100_P1C	0xc2
-#define AB3100_P1D	0xc3
-#define AB3100_P1E	0xc4
-#define AB3100_P1F	0xc5
-#define AB3100_P1G	0xc6
-#define AB3100_R2A	0xc7
-#define AB3100_R2B	0xc8
-
-/*
- * AB3100, EVENTA1, A2 and A3 event register flags
- * these are catenated into a single 32-bit flag in the code
- * for event notification broadcasts.
- */
-#define AB3100_EVENTA1_ONSWA				(0x01<<16)
-#define AB3100_EVENTA1_ONSWB				(0x02<<16)
-#define AB3100_EVENTA1_ONSWC				(0x04<<16)
-#define AB3100_EVENTA1_DCIO				(0x08<<16)
-#define AB3100_EVENTA1_OVER_TEMP			(0x10<<16)
-#define AB3100_EVENTA1_SIM_OFF				(0x20<<16)
-#define AB3100_EVENTA1_VBUS				(0x40<<16)
-#define AB3100_EVENTA1_VSET_USB				(0x80<<16)
-
-#define AB3100_EVENTA2_READY_TX				(0x01<<8)
-#define AB3100_EVENTA2_READY_RX				(0x02<<8)
-#define AB3100_EVENTA2_OVERRUN_ERROR			(0x04<<8)
-#define AB3100_EVENTA2_FRAMING_ERROR			(0x08<<8)
-#define AB3100_EVENTA2_CHARG_OVERCURRENT		(0x10<<8)
-#define AB3100_EVENTA2_MIDR				(0x20<<8)
-#define AB3100_EVENTA2_BATTERY_REM			(0x40<<8)
-#define AB3100_EVENTA2_ALARM				(0x80<<8)
-
-#define AB3100_EVENTA3_ADC_TRIG5			(0x01)
-#define AB3100_EVENTA3_ADC_TRIG4			(0x02)
-#define AB3100_EVENTA3_ADC_TRIG3			(0x04)
-#define AB3100_EVENTA3_ADC_TRIG2			(0x08)
-#define AB3100_EVENTA3_ADC_TRIGVBAT			(0x10)
-#define AB3100_EVENTA3_ADC_TRIGVTX			(0x20)
-#define AB3100_EVENTA3_ADC_TRIG1			(0x40)
-#define AB3100_EVENTA3_ADC_TRIG0			(0x80)
-
-/* AB3100, STR register flags */
-#define AB3100_STR_ONSWA				(0x01)
-#define AB3100_STR_ONSWB				(0x02)
-#define AB3100_STR_ONSWC				(0x04)
-#define AB3100_STR_DCIO					(0x08)
-#define AB3100_STR_BOOT_MODE				(0x10)
-#define AB3100_STR_SIM_OFF				(0x20)
-#define AB3100_STR_BATT_REMOVAL				(0x40)
-#define AB3100_STR_VBUS					(0x80)
-
-/*
- * AB3100 contains 8 regulators, one external regulator controller
- * and a buck converter, further the LDO E and buck converter can
- * have separate settings if they are in sleep mode, this is
- * modeled as a separate regulator.
- */
-#define AB3100_NUM_REGULATORS				10
-
-/**
- * struct ab3100
- * @access_mutex: lock out concurrent accesses to the AB3100 registers
- * @dev: pointer to the containing device
- * @i2c_client: I2C client for this chip
- * @testreg_client: secondary client for test registers
- * @chip_name: name of this chip variant
- * @chip_id: 8 bit chip ID for this chip variant
- * @event_subscribers: event subscribers are listed here
- * @startup_events: a copy of the first reading of the event registers
- * @startup_events_read: whether the first events have been read
- *
- * This struct is PRIVATE and devices using it should NOT
- * access ANY fields. It is used as a token for calling the
- * AB3100 functions.
- */
-struct ab3100 {
-	struct mutex access_mutex;
-	struct device *dev;
-	struct i2c_client *i2c_client;
-	struct i2c_client *testreg_client;
-	char chip_name[32];
-	u8 chip_id;
-	struct blocking_notifier_head event_subscribers;
-	u8 startup_events[3];
-	bool startup_events_read;
-};
-
-/**
- * struct ab3100_platform_data
- * Data supplied to initialize board connections to the AB3100
- * @reg_constraints: regulator constraints for target board
- *     the order of these constraints are: LDO A, C, D, E,
- *     F, G, H, K, EXT and BUCK.
- * @reg_initvals: initial values for the regulator registers
- *     plus two sleep settings for LDO E and the BUCK converter.
- *     exactly AB3100_NUM_REGULATORS+2 values must be sent in.
- *     Order: LDO A, C, E, E sleep, F, G, H, K, EXT, BUCK,
- *     BUCK sleep, LDO D. (LDO D need to be initialized last.)
- * @external_voltage: voltage level of the external regulator.
- */
-struct ab3100_platform_data {
-	struct regulator_init_data reg_constraints[AB3100_NUM_REGULATORS];
-	u8 reg_initvals[AB3100_NUM_REGULATORS+2];
-	int external_voltage;
-};
-
-int ab3100_event_register(struct ab3100 *ab3100,
-			  struct notifier_block *nb);
-int ab3100_event_unregister(struct ab3100 *ab3100,
-			    struct notifier_block *nb);
-
 /**
  * struct abx500_init_setting
  * Initial value of the registers for driver to use during setup.
diff --git a/include/linux/mfd/abx500/ab8500.h b/include/linux/mfd/abx500/ab8500.h
index 3764cb6759e..1491044efa1 100644
--- a/include/linux/mfd/abx500/ab8500.h
+++ b/include/linux/mfd/abx500/ab8500.h
@@ -341,6 +341,4 @@ static inline int is_ab8500_2p0(struct ab8500 *ab)
 	return (is_ab8500(ab) && (ab->chip_id == AB8500_CUT2P0));
 }
 
-int ab8500_irq_get_virq(struct ab8500 *ab8500, int irq);
-
 #endif /* MFD_AB8500_H */
diff --git a/include/linux/mfd/anatop.h b/include/linux/mfd/anatop.h
deleted file mode 100644
index 7f92acf03d9..00000000000
--- a/include/linux/mfd/anatop.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * anatop.h - Anatop MFD driver
- *
- *  Copyright (C) 2012 Ying-Chun Liu (PaulLiu) <paul.liu@linaro.org>
- *  Copyright (C) 2012 Linaro
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-#ifndef __LINUX_MFD_ANATOP_H
-#define __LINUX_MFD_ANATOP_H
-
-#include <linux/spinlock.h>
-
-/**
- * anatop - MFD data
- * @ioreg: ioremap register
- * @reglock: spinlock for register read/write
- */
-struct anatop {
-	void *ioreg;
-	spinlock_t reglock;
-};
-
-extern u32 anatop_read_reg(struct anatop *, u32);
-extern void anatop_write_reg(struct anatop *, u32, u32, u32);
-
-#endif /*  __LINUX_MFD_ANATOP_H */
diff --git a/include/linux/mfd/da9055/core.h b/include/linux/mfd/da9055/core.h
new file mode 100644
index 00000000000..c96ad682c59
--- /dev/null
+++ b/include/linux/mfd/da9055/core.h
@@ -0,0 +1,94 @@
+/*
+ * da9055 declarations for DA9055 PMICs.
+ *
+ * Copyright(c) 2012 Dialog Semiconductor Ltd.
+ *
+ * Author: David Dajun Chen <dchen@diasemi.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#ifndef __DA9055_CORE_H
+#define __DA9055_CORE_H
+
+#include <linux/interrupt.h>
+#include <linux/regmap.h>
+
+/*
+ * PMIC IRQ
+ */
+#define DA9055_IRQ_ALARM	0x01
+#define DA9055_IRQ_TICK		0x02
+#define DA9055_IRQ_NONKEY	0x00
+#define DA9055_IRQ_REGULATOR	0x0B
+#define DA9055_IRQ_HWMON	0x03
+
+struct da9055_pdata;
+
+struct da9055 {
+	struct regmap *regmap;
+	struct regmap_irq_chip_data *irq_data;
+	struct device *dev;
+	struct i2c_client *i2c_client;
+
+	int irq_base;
+	int chip_irq;
+};
+
+/* Device I/O */
+static inline int da9055_reg_read(struct da9055 *da9055, unsigned char reg)
+{
+	int val, ret;
+
+	ret = regmap_read(da9055->regmap, reg, &val);
+	if (ret < 0)
+		return ret;
+
+	return val;
+}
+
+static inline int da9055_reg_write(struct da9055 *da9055, unsigned char reg,
+				    unsigned char val)
+{
+	return regmap_write(da9055->regmap, reg, val);
+}
+
+static inline int da9055_group_read(struct da9055 *da9055, unsigned char reg,
+				     unsigned reg_cnt, unsigned char *val)
+{
+	return regmap_bulk_read(da9055->regmap, reg, val, reg_cnt);
+}
+
+static inline int da9055_group_write(struct da9055 *da9055, unsigned char reg,
+				      unsigned reg_cnt, unsigned char *val)
+{
+	return regmap_raw_write(da9055->regmap, reg, val, reg_cnt);
+}
+
+static inline int da9055_reg_update(struct da9055 *da9055, unsigned char reg,
+				     unsigned char bit_mask,
+				     unsigned char reg_val)
+{
+	return regmap_update_bits(da9055->regmap, reg, bit_mask, reg_val);
+}
+
+/* Generic Device API */
+int da9055_device_init(struct da9055 *da9055);
+void da9055_device_exit(struct da9055 *da9055);
+
+extern struct regmap_config da9055_regmap_config;
+
+#endif /* __DA9055_CORE_H */
diff --git a/include/linux/mfd/da9055/pdata.h b/include/linux/mfd/da9055/pdata.h
new file mode 100644
index 00000000000..147293b4471
--- /dev/null
+++ b/include/linux/mfd/da9055/pdata.h
@@ -0,0 +1,32 @@
+/* Copyright (C) 2012 Dialog Semiconductor Ltd.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ */
+#ifndef __DA9055_PDATA_H
+#define __DA9055_PDATA_H
+
+#define DA9055_MAX_REGULATORS	8
+
+struct da9055;
+
+enum gpio_select {
+	NO_GPIO = 0,
+	GPIO_1,
+	GPIO_2
+};
+
+struct da9055_pdata {
+	int (*init) (struct da9055 *da9055);
+	int irq_base;
+	int gpio_base;
+
+	struct regulator_init_data *regulators[DA9055_MAX_REGULATORS];
+	bool reset_enable;		/* Enable RTC in RESET Mode */
+	enum gpio_select *gpio_rsel;	/* Select regulator set thru GPIO 1/2 */
+	enum gpio_select *gpio_ren;	/* Enable regulator thru GPIO 1/2 */
+};
+#endif /* __DA9055_PDATA_H */
diff --git a/include/linux/mfd/da9055/reg.h b/include/linux/mfd/da9055/reg.h
new file mode 100644
index 00000000000..df237ee5480
--- /dev/null
+++ b/include/linux/mfd/da9055/reg.h
@@ -0,0 +1,699 @@
+/*
+ * DA9055 declarations for DA9055 PMICs.
+ *
+ * Copyright(c) 2012 Dialog Semiconductor Ltd.
+ *
+ * Author: David Dajun Chen <dchen@diasemi.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#ifndef __DA9055_REG_H
+#define __DA9055_REG_H
+
+/*
+ * PMIC registers
+ */
+ /* PAGE0 */
+#define	DA9055_REG_PAGE_CON		0x00
+
+/* System Control and Event Registers */
+#define	DA9055_REG_STATUS_A		0x01
+#define	DA9055_REG_STATUS_B		0x02
+#define	DA9055_REG_FAULT_LOG		0x03
+#define	DA9055_REG_EVENT_A		0x04
+#define	DA9055_REG_EVENT_B		0x05
+#define	DA9055_REG_EVENT_C		0x06
+#define	DA9055_REG_IRQ_MASK_A		0x07
+#define	DA9055_REG_IRQ_MASK_B		0x08
+#define	DA9055_REG_IRQ_MASK_C		0x09
+#define	DA9055_REG_CONTROL_A		0x0A
+#define	DA9055_REG_CONTROL_B		0x0B
+#define	DA9055_REG_CONTROL_C		0x0C
+#define	DA9055_REG_CONTROL_D		0x0D
+#define	DA9055_REG_CONTROL_E		0x0E
+#define	DA9055_REG_PD_DIS		0x0F
+
+/* GPIO Control Registers */
+#define	DA9055_REG_GPIO0_1		0x10
+#define	DA9055_REG_GPIO2		0x11
+#define	DA9055_REG_GPIO_MODE0_2		0x12
+
+/* Regulator Control Registers */
+#define	DA9055_REG_BCORE_CONT		0x13
+#define	DA9055_REG_BMEM_CONT		0x14
+#define	DA9055_REG_LDO1_CONT		0x15
+#define	DA9055_REG_LDO2_CONT		0x16
+#define	DA9055_REG_LDO3_CONT		0x17
+#define	DA9055_REG_LDO4_CONT		0x18
+#define	DA9055_REG_LDO5_CONT		0x19
+#define	DA9055_REG_LDO6_CONT		0x1A
+
+/* GP-ADC Control Registers */
+#define	DA9055_REG_ADC_MAN		0x1B
+#define	DA9055_REG_ADC_CONT		0x1C
+#define	DA9055_REG_VSYS_MON		0x1D
+#define	DA9055_REG_ADC_RES_L		0x1E
+#define	DA9055_REG_ADC_RES_H		0x1F
+#define	DA9055_REG_VSYS_RES		0x20
+#define	DA9055_REG_ADCIN1_RES		0x21
+#define	DA9055_REG_ADCIN2_RES		0x22
+#define	DA9055_REG_ADCIN3_RES		0x23
+
+/* Sequencer Control Registers */
+#define	DA9055_REG_EN_32K		0x35
+
+/* Regulator Setting Registers */
+#define	DA9055_REG_BUCK_LIM		0x37
+#define	DA9055_REG_BCORE_MODE		0x38
+#define	DA9055_REG_VBCORE_A		0x39
+#define	DA9055_REG_VBMEM_A		0x3A
+#define	DA9055_REG_VLDO1_A		0x3B
+#define	DA9055_REG_VLDO2_A		0x3C
+#define	DA9055_REG_VLDO3_A		0x3D
+#define	DA9055_REG_VLDO4_A		0x3E
+#define	DA9055_REG_VLDO5_A		0x3F
+#define	DA9055_REG_VLDO6_A		0x40
+#define	DA9055_REG_VBCORE_B		0x41
+#define	DA9055_REG_VBMEM_B		0x42
+#define	DA9055_REG_VLDO1_B		0x43
+#define	DA9055_REG_VLDO2_B		0x44
+#define	DA9055_REG_VLDO3_B		0x45
+#define	DA9055_REG_VLDO4_B		0x46
+#define	DA9055_REG_VLDO5_B		0x47
+#define	DA9055_REG_VLDO6_B		0x48
+
+/* GP-ADC Threshold Registers */
+#define	DA9055_REG_AUTO1_HIGH		0x49
+#define	DA9055_REG_AUTO1_LOW		0x4A
+#define	DA9055_REG_AUTO2_HIGH		0x4B
+#define	DA9055_REG_AUTO2_LOW		0x4C
+#define	DA9055_REG_AUTO3_HIGH		0x4D
+#define	DA9055_REG_AUTO3_LOW		0x4E
+
+/* OTP */
+#define	DA9055_REG_OPT_COUNT		0x50
+#define	DA9055_REG_OPT_ADDR		0x51
+#define	DA9055_REG_OPT_DATA		0x52
+
+/* RTC Calendar and Alarm Registers */
+#define	DA9055_REG_COUNT_S		0x53
+#define	DA9055_REG_COUNT_MI		0x54
+#define	DA9055_REG_COUNT_H		0x55
+#define	DA9055_REG_COUNT_D		0x56
+#define	DA9055_REG_COUNT_MO		0x57
+#define	DA9055_REG_COUNT_Y		0x58
+#define	DA9055_REG_ALARM_MI		0x59
+#define	DA9055_REG_ALARM_H		0x5A
+#define	DA9055_REG_ALARM_D		0x5B
+#define	DA9055_REG_ALARM_MO		0x5C
+#define	DA9055_REG_ALARM_Y		0x5D
+#define	DA9055_REG_SECOND_A		0x5E
+#define	DA9055_REG_SECOND_B		0x5F
+#define	DA9055_REG_SECOND_C		0x60
+#define	DA9055_REG_SECOND_D		0x61
+
+/* Customer Trim and Configuration */
+#define	DA9055_REG_T_OFFSET		0x63
+#define	DA9055_REG_INTERFACE		0x64
+#define	DA9055_REG_CONFIG_A		0x65
+#define	DA9055_REG_CONFIG_B		0x66
+#define	DA9055_REG_CONFIG_C		0x67
+#define	DA9055_REG_CONFIG_D		0x68
+#define	DA9055_REG_CONFIG_E		0x69
+#define	DA9055_REG_TRIM_CLDR		0x6F
+
+/* General Purpose Registers */
+#define	DA9055_REG_GP_ID_0		0x70
+#define	DA9055_REG_GP_ID_1		0x71
+#define	DA9055_REG_GP_ID_2		0x72
+#define	DA9055_REG_GP_ID_3		0x73
+#define	DA9055_REG_GP_ID_4		0x74
+#define	DA9055_REG_GP_ID_5		0x75
+#define	DA9055_REG_GP_ID_6		0x76
+#define	DA9055_REG_GP_ID_7		0x77
+#define	DA9055_REG_GP_ID_8		0x78
+#define	DA9055_REG_GP_ID_9		0x79
+#define	DA9055_REG_GP_ID_10		0x7A
+#define	DA9055_REG_GP_ID_11		0x7B
+#define	DA9055_REG_GP_ID_12		0x7C
+#define	DA9055_REG_GP_ID_13		0x7D
+#define	DA9055_REG_GP_ID_14		0x7E
+#define	DA9055_REG_GP_ID_15		0x7F
+#define	DA9055_REG_GP_ID_16		0x80
+#define	DA9055_REG_GP_ID_17		0x81
+#define	DA9055_REG_GP_ID_18		0x82
+#define	DA9055_REG_GP_ID_19		0x83
+
+#define DA9055_MAX_REGISTER_CNT		DA9055_REG_GP_ID_19
+
+/*
+ * PMIC registers bits
+ */
+
+/* DA9055_REG_PAGE_CON (addr=0x00) */
+#define	DA9055_PAGE_WRITE_MODE		(0<<6)
+#define	DA9055_REPEAT_WRITE_MODE	(1<<6)
+
+/* DA9055_REG_STATUS_A (addr=0x01) */
+#define	DA9055_NOKEY_STS		0x01
+#define	DA9055_WAKE_STS			0x02
+#define	DA9055_DVC_BUSY_STS		0x04
+#define	DA9055_COMP1V2_STS		0x08
+#define	DA9055_NJIG_STS			0x10
+#define	DA9055_LDO5_LIM_STS		0x20
+#define	DA9055_LDO6_LIM_STS		0x40
+
+/* DA9055_REG_STATUS_B (addr=0x02) */
+#define	DA9055_GPI0_STS			0x01
+#define	DA9055_GPI1_STS			0x02
+#define	DA9055_GPI2_STS			0x04
+
+/* DA9055_REG_FAULT_LOG (addr=0x03) */
+#define	DA9055_TWD_ERROR_FLG		0x01
+#define	DA9055_POR_FLG			0x02
+#define	DA9055_VDD_FAULT_FLG		0x04
+#define	DA9055_VDD_START_FLG		0x08
+#define	DA9055_TEMP_CRIT_FLG		0x10
+#define	DA9055_KEY_RESET_FLG		0x20
+#define	DA9055_WAIT_SHUT_FLG		0x80
+
+/* DA9055_REG_EVENT_A (addr=0x04) */
+#define	DA9055_NOKEY_EINT		0x01
+#define	DA9055_ALARM_EINT		0x02
+#define	DA9055_TICK_EINT		0x04
+#define	DA9055_ADC_RDY_EINT		0x08
+#define	DA9055_SEQ_RDY_EINT		0x10
+#define	DA9055_EVENTS_B_EINT		0x20
+#define	DA9055_EVENTS_C_EINT		0x40
+
+/* DA9055_REG_EVENT_B (addr=0x05) */
+#define	DA9055_E_WAKE_EINT		0x01
+#define	DA9055_E_TEMP_EINT		0x02
+#define	DA9055_E_COMP1V2_EINT		0x04
+#define	DA9055_E_LDO_LIM_EINT		0x08
+#define	DA9055_E_NJIG_EINT		0x20
+#define	DA9055_E_VDD_MON_EINT		0x40
+#define	DA9055_E_VDD_WARN_EINT		0x80
+
+/* DA9055_REG_EVENT_C (addr=0x06) */
+#define	DA9055_E_GPI0_EINT		0x01
+#define	DA9055_E_GPI1_EINT		0x02
+#define	DA9055_E_GPI2_EINT		0x04
+
+/* DA9055_REG_IRQ_MASK_A (addr=0x07) */
+#define	DA9055_M_NONKEY_EINT		0x01
+#define	DA9055_M_ALARM_EINT		0x02
+#define	DA9055_M_TICK_EINT		0x04
+#define	DA9055_M_ADC_RDY_EINT		0x08
+#define	DA9055_M_SEQ_RDY_EINT		0x10
+
+/* DA9055_REG_IRQ_MASK_B (addr=0x08) */
+#define	DA9055_M_WAKE_EINT		0x01
+#define	DA9055_M_TEMP_EINT		0x02
+#define	DA9055_M_COMP_1V2_EINT		0x04
+#define	DA9055_M_LDO_LIM_EINT		0x08
+#define	DA9055_M_NJIG_EINT		0x20
+#define	DA9055_M_VDD_MON_EINT		0x40
+#define	DA9055_M_VDD_WARN_EINT		0x80
+
+/* DA9055_REG_IRQ_MASK_C (addr=0x09) */
+#define	DA9055_M_GPI0_EINT		0x01
+#define	DA9055_M_GPI1_EINT		0x02
+#define	DA9055_M_GPI2_EINT		0x04
+
+/* DA9055_REG_CONTROL_A (addr=0xA) */
+#define	DA9055_DEBOUNCING_SHIFT		0x00
+#define	DA9055_DEBOUNCING_MASK		0x07
+#define	DA9055_NRES_MODE_SHIFT		0x03
+#define	DA9055_NRES_MODE_MASK		0x08
+#define	DA9055_SLEW_RATE_SHIFT		0x04
+#define	DA9055_SLEW_RATE_MASK		0x30
+#define	DA9055_NOKEY_LOCK_SHIFT		0x06
+#define	DA9055_NOKEY_LOCK_MASK		0x40
+
+/* DA9055_REG_CONTROL_B (addr=0xB) */
+#define	DA9055_RTC_MODE_PD		0x01
+#define	DA9055_RTC_MODE_SD_SHIFT	0x01
+#define	DA9055_RTC_MODE_SD		0x02
+#define	DA9055_RTC_EN			0x04
+#define	DA9055_ECO_MODE_SHIFT		0x03
+#define	DA9055_ECO_MODE_MASK		0x08
+#define	DA9055_TWDSCALE_SHIFT		4
+#define	DA9055_TWDSCALE_MASK		0x70
+#define	DA9055_V_LOCK_SHIFT		0x07
+#define	DA9055_V_LOCK_MASK		0x80
+
+/* DA9055_REG_CONTROL_C (addr=0xC) */
+#define	DA9055_SYSTEM_EN_SHIFT		0x00
+#define	DA9055_SYSTEM_EN_MASK		0x01
+#define	DA9055_POWERN_EN_SHIFT		0x01
+#define	DA9055_POWERN_EN_MASK		0x02
+#define	DA9055_POWER1_EN_SHIFT		0x02
+#define	DA9055_POWER1_EN_MASK		0x04
+
+/* DA9055_REG_CONTROL_D (addr=0xD) */
+#define	DA9055_STANDBY_SHIFT		0x02
+#define	DA9055_STANDBY_MASK		0x08
+#define	DA9055_AUTO_BOOT_SHIFT		0x03
+#define	DA9055_AUTO_BOOT_MASK		0x04
+
+/* DA9055_REG_CONTROL_E (addr=0xE) */
+#define	DA9055_WATCHDOG_SHIFT		0x00
+#define	DA9055_WATCHDOG_MASK		0x01
+#define	DA9055_SHUTDOWN_SHIFT		0x01
+#define	DA9055_SHUTDOWN_MASK		0x02
+#define	DA9055_WAKE_UP_SHIFT		0x02
+#define	DA9055_WAKE_UP_MASK		0x04
+
+/* DA9055_REG_GPIO (addr=0x10/0x11) */
+#define	DA9055_GPIO0_PIN_SHIFT		0x00
+#define	DA9055_GPIO0_PIN_MASK		0x03
+#define	DA9055_GPIO0_TYPE_SHIFT		0x02
+#define	DA9055_GPIO0_TYPE_MASK		0x04
+#define	DA9055_GPIO0_WEN_SHIFT		0x03
+#define	DA9055_GPIO0_WEN_MASK		0x08
+#define	DA9055_GPIO1_PIN_SHIFT		0x04
+#define	DA9055_GPIO1_PIN_MASK		0x30
+#define	DA9055_GPIO1_TYPE_SHIFT		0x06
+#define	DA9055_GPIO1_TYPE_MASK		0x40
+#define	DA9055_GPIO1_WEN_SHIFT		0x07
+#define	DA9055_GPIO1_WEN_MASK		0x80
+#define	DA9055_GPIO2_PIN_SHIFT		0x00
+#define	DA9055_GPIO2_PIN_MASK		0x30
+#define	DA9055_GPIO2_TYPE_SHIFT		0x02
+#define	DA9055_GPIO2_TYPE_MASK		0x04
+#define	DA9055_GPIO2_WEN_SHIFT		0x03
+#define	DA9055_GPIO2_WEN_MASK		0x08
+
+/* DA9055_REG_GPIO_MODE (addr=0x12) */
+#define	DA9055_GPIO0_MODE_SHIFT		0x00
+#define	DA9055_GPIO0_MODE_MASK		0x01
+#define	DA9055_GPIO1_MODE_SHIFT		0x01
+#define	DA9055_GPIO1_MODE_MASK		0x02
+#define	DA9055_GPIO2_MODE_SHIFT		0x02
+#define	DA9055_GPIO2_MODE_MASK		0x04
+
+/* DA9055_REG_BCORE_CONT (addr=0x13) */
+#define	DA9055_BCORE_EN_SHIFT		0x00
+#define	DA9055_BCORE_EN_MASK		0x01
+#define	DA9055_BCORE_GPI_SHIFT		0x01
+#define	DA9055_BCORE_GPI_MASK		0x02
+#define	DA9055_BCORE_PD_DIS_SHIFT	0x03
+#define	DA9055_BCORE_PD_DIS_MASK	0x04
+#define	DA9055_VBCORE_SEL_SHIFT		0x04
+#define	DA9055_SEL_REG_A		0x0
+#define	DA9055_SEL_REG_B		0x10
+#define	DA9055_VBCORE_SEL_MASK		0x10
+#define DA9055_V_GPI_MASK		0x60
+#define DA9055_V_GPI_SHIFT		0x05
+#define DA9055_E_GPI_MASK		0x06
+#define DA9055_E_GPI_SHIFT		0x01
+#define	DA9055_VBCORE_GPI_SHIFT		0x05
+#define	DA9055_VBCORE_GPI_MASK		0x60
+#define	DA9055_BCORE_CONF_SHIFT		0x07
+#define	DA9055_BCORE_CONF_MASK		0x80
+
+/* DA9055_REG_BMEM_CONT (addr=0x14) */
+#define	DA9055_BMEM_EN_SHIFT		0x00
+#define	DA9055_BMEM_EN_MASK		0x01
+#define	DA9055_BMEM_GPI_SHIFT		0x01
+#define	DA9055_BMEM_GPI_MASK		0x06
+#define	DA9055_BMEM_PD_DIS_SHIFT	0x03
+#define	DA9055_BMEM_PD_DIS_MASK		0x08
+#define	DA9055_VBMEM_SEL_SHIT		0x04
+#define	DA9055_VBMEM_SEL_VBMEM_A	(0<<4)
+#define	DA9055_VBMEM_SEL_VBMEM_B	(1<<4)
+#define	DA9055_VBMEM_SEL_MASK		0x10
+#define	DA9055_VBMEM_GPI_SHIFT		0x05
+#define	DA9055_VBMEM_GPI_MASK		0x60
+#define	DA9055_BMEM_CONF_SHIFT		0x07
+#define	DA9055_BMEM_CONF_MASK		0x80
+
+/* DA9055_REG_LDO_CONT (addr=0x15-0x1A) */
+#define	DA9055_LDO_EN_SHIFT		0x00
+#define	DA9055_LDO_EN_MASK		0x01
+#define	DA9055_LDO_GPI_SHIFT		0x01
+#define	DA9055_LDO_GPI_MASK		0x06
+#define	DA9055_LDO_PD_DIS_SHIFT		0x03
+#define	DA9055_LDO_PD_DIS_MASK		0x08
+#define	DA9055_VLDO_SEL_SHIFT		0x04
+#define	DA9055_VLDO_SEL_MASK		0x10
+#define	DA9055_VLDO_SEL_VLDO_A		0x00
+#define	DA9055_VLDO_SEL_VLDO_B		0x01
+#define	DA9055_VLDO_GPI_SHIFT		0x05
+#define	DA9055_VLDO_GPI_MASK		0x60
+#define	DA9055_LDO_CONF_SHIFT		0x07
+#define	DA9055_LDO_CONF_MASK		0x80
+#define	DA9055_REGUALTOR_SET_A		0x00
+#define	DA9055_REGUALTOR_SET_B		0x10
+
+/* DA9055_REG_ADC_MAN (addr=0x1B) */
+#define	DA9055_ADC_MUX_SHIFT		0
+#define	DA9055_ADC_MUX_MASK		0xF
+#define	DA9055_ADC_MUX_VSYS		0x0
+#define	DA9055_ADC_MUX_ADCIN1		0x01
+#define	DA9055_ADC_MUX_ADCIN2		0x02
+#define	DA9055_ADC_MUX_ADCIN3		0x03
+#define	DA9055_ADC_MUX_T_SENSE		0x04
+#define	DA9055_ADC_MAN_SHIFT		0x04
+#define	DA9055_ADC_MAN_CONV		0x10
+#define DA9055_ADC_LSB_MASK		0X03
+#define DA9055_ADC_MODE_MASK		0x20
+#define	DA9055_ADC_MODE_SHIFT		5
+#define	DA9055_ADC_MODE_1MS		(1<<5)
+#define	DA9055_COMP1V2_EN_SHIFT		7
+
+/* DA9055_REG_ADC_CONT (addr=0x1C) */
+#define	DA9055_ADC_AUTO_VSYS_EN_SHIFT	0
+#define	DA9055_ADC_AUTO_AD1_EN_SHIFT	1
+#define	DA9055_ADC_AUTO_AD2_EN_SHIFT	2
+#define	DA9055_ADC_AUTO_AD3_EN_SHIFT	3
+#define	DA9055_ADC_ISRC_EN_SHIFT	4
+#define	DA9055_ADC_ADCIN1_DEB_SHIFT	5
+#define	DA9055_ADC_ADCIN2_DEB_SHIFT	6
+#define	DA9055_ADC_ADCIN3_DEB_SHIFT	7
+#define DA9055_AD1_ISRC_MASK		0x10
+#define DA9055_AD1_ISRC_SHIFT		4
+
+/* DA9055_REG_VSYS_MON (addr=0x1D) */
+#define	DA9055_VSYS_VAL_SHIFT		0
+#define	DA9055_VSYS_VAL_MASK		0xFF
+#define	DA9055_VSYS_VAL_BASE		0x00
+#define	DA9055_VSYS_VAL_MAX		DA9055_VSYS_VAL_MASK
+#define	DA9055_VSYS_VOLT_BASE		2500
+#define	DA9055_VSYS_VOLT_INC		10
+#define	DA9055_VSYS_STEPS		255
+#define	DA9055_VSYS_VOLT_MIN		2500
+
+/* DA9044_REG_XXX_RES (addr=0x20-0x23) */
+#define	DA9055_ADC_VAL_SHIFT		0
+#define	DA9055_ADC_VAL_MASK		0xFF
+#define	DA9055_ADC_VAL_BASE		0x00
+#define	DA9055_ADC_VAL_MAX		DA9055_ADC_VAL_MASK
+#define	DA9055_ADC_VOLT_BASE		0
+#define	DA9055_ADC_VSYS_VOLT_BASE	2500
+#define	DA9055_ADC_VOLT_INC		10
+#define	DA9055_ADC_VSYS_VOLT_INC	12
+#define	DA9055_ADC_STEPS		255
+
+/* DA9055_REG_EN_32K  (addr=0x35)*/
+#define	DA9055_STARTUP_TIME_MASK	0x07
+#define	DA9055_STARTUP_TIME_0S		0x0
+#define	DA9055_STARTUP_TIME_0_52S	0x1
+#define	DA9055_STARTUP_TIME_1S		0x2
+#define	DA9055_CRYSTAL_EN		0x08
+#define	DA9055_DELAY_MODE_EN		0x10
+#define	DA9055_OUT_CLCK_GATED		0x20
+#define	DA9055_RTC_CLOCK_GATED		0x40
+#define	DA9055_EN_32KOUT_BUF		0x80
+
+/* DA9055_REG_RESET (addr=0x36) */
+/* Timer up to 31.744 ms */
+#define	DA9055_RESET_TIMER_VAL_SHIFT	0
+#define	DA9055_RESET_LOW_VAL_MASK	0x3F
+#define	DA9055_RESET_LOW_VAL_BASE	0
+#define	DA9055_RESET_LOW_VAL_MAX	DA9055_RESET_LOW_VAL_MASK
+#define	DA9055_RESET_US_LOW_BASE	1024 /* min val in units of us */
+#define	DA9055_RESET_US_LOW_INC		1024 /* inc val in units of us */
+#define	DA9055_RESET_US_LOW_STEP	30
+
+/* Timer up to 1048.576ms */
+#define	DA9055_RESET_HIGH_VAL_MASK	0x3F
+#define	DA9055_RESET_HIGH_VAL_BASE	0
+#define	DA9055_RESET_HIGH_VAL_MAX	DA9055_RESET_HIGH_VAL_MASK
+#define	DA9055_RESET_US_HIGH_BASE	32768 /* min val in units of us */
+#define	DA9055_RESET_US_HIGH_INC	32768 /* inv val in units of us */
+#define	DA9055_RESET_US_HIGH_STEP	31
+
+/* DA9055_REG_BUCK_ILIM (addr=0x37)*/
+#define	DA9055_BMEM_ILIM_SHIFT		0
+#define	DA9055_ILIM_MASK		0x3
+#define	DA9055_ILIM_500MA		0x0
+#define	DA9055_ILIM_600MA		0x1
+#define	DA9055_ILIM_700MA		0x2
+#define	DA9055_ILIM_800MA		0x3
+#define	DA9055_BCORE_ILIM_SHIFT		2
+
+/* DA9055_REG_BCORE_MODE (addr=0x38) */
+#define	DA9055_BMEM_MODE_SHIFT		0
+#define	DA9055_MODE_MASK		0x3
+#define	DA9055_MODE_AB			0x0
+#define	DA9055_MODE_SLEEP		0x1
+#define	DA9055_MODE_SYNCHRO		0x2
+#define	DA9055_MODE_AUTO		0x3
+#define	DA9055_BCORE_MODE_SHIFT		2
+
+/* DA9055_REG_VBCORE_A/B (addr=0x39/0x41)*/
+#define	DA9055_VBCORE_VAL_SHIFT		0
+#define	DA9055_VBCORE_VAL_MASK		0x3F
+#define	DA9055_VBCORE_VAL_BASE		0x09
+#define	DA9055_VBCORE_VAL_MAX		DA9055_VBCORE_VAL_MASK
+#define	DA9055_VBCORE_VOLT_BASE		750
+#define	DA9055_VBCORE_VOLT_INC		25
+#define	DA9055_VBCORE_STEPS		53
+#define	DA9055_VBCORE_VOLT_MIN		DA9055_VBCORE_VOLT_BASE
+#define	DA9055_BCORE_SL_SYNCHRO		(0<<7)
+#define	DA9055_BCORE_SL_SLEEP		(1<<7)
+
+/* DA9055_REG_VBMEM_A/B (addr=0x3A/0x42)*/
+#define	DA9055_VBMEM_VAL_SHIFT		0
+#define	DA9055_VBMEM_VAL_MASK		0x3F
+#define	DA9055_VBMEM_VAL_BASE		0x00
+#define	DA9055_VBMEM_VAL_MAX		DA9055_VBMEM_VAL_MASK
+#define	DA9055_VBMEM_VOLT_BASE		925
+#define	DA9055_VBMEM_VOLT_INC		25
+#define	DA9055_VBMEM_STEPS		63
+#define	DA9055_VBMEM_VOLT_MIN		DA9055_VBMEM_VOLT_BASE
+#define	DA9055_BCMEM_SL_SYNCHRO		(0<<7)
+#define	DA9055_BCMEM_SL_SLEEP		(1<<7)
+
+
+/* DA9055_REG_VLDO (addr=0x3B-0x40/0x43-0x48)*/
+#define	DA9055_VLDO_VAL_SHIFT		0
+#define	DA9055_VLDO_VAL_MASK		0x3F
+#define	DA9055_VLDO6_VAL_MASK		0x7F
+#define	DA9055_VLDO_VAL_BASE		0x02
+#define	DA9055_VLDO2_VAL_BASE		0x03
+#define	DA9055_VLDO6_VAL_BASE		0x00
+#define	DA9055_VLDO_VAL_MAX		DA9055_VLDO_VAL_MASK
+#define	DA9055_VLDO6_VAL_MAX		DA9055_VLDO6_VAL_MASK
+#define	DA9055_VLDO_VOLT_BASE		900
+#define	DA9055_VLDO_VOLT_INC		50
+#define	DA9055_VLDO6_VOLT_INC		20
+#define	DA9055_VLDO_STEPS		48
+#define	DA9055_VLDO5_STEPS		37
+#define	DA9055_VLDO6_STEPS		120
+#define	DA9055_VLDO_VOLT_MIN		DA9055_VLDO_VOLT_BASE
+#define	DA9055_LDO_MODE_SHIFT		7
+#define	DA9055_LDO_SL_NORMAL		0
+#define	DA9055_LDO_SL_SLEEP		1
+
+/* DA9055_REG_OTP_CONT (addr=0x50) */
+#define	DA9055_OTP_TIM_NORMAL		(0<<0)
+#define	DA9055_OTP_TIM_MARGINAL		(1<<0)
+#define	DA9055_OTP_GP_RD_SHIFT		1
+#define	DA9055_OTP_APPS_RD_SHIFT	2
+#define	DA9055_PC_DONE_SHIFT		3
+#define	DA9055_OTP_GP_LOCK_SHIFT	4
+#define	DA9055_OTP_APPS_LOCK_SHIFT	5
+#define	DA9055_OTP_CONF_LOCK_SHIFT	6
+#define	DA9055_OTP_WRITE_DIS_SHIFT	7
+
+/* DA9055_REG_COUNT_S (addr=0x53) */
+#define	DA9055_RTC_SEC			0x3F
+#define	DA9055_RTC_MONITOR_EN		0x40
+#define	DA9055_RTC_READ			0x80
+
+/* DA9055_REG_COUNT_MI (addr=0x54) */
+#define	DA9055_RTC_MIN			0x3F
+
+/* DA9055_REG_COUNT_H (addr=0x55) */
+#define	DA9055_RTC_HOUR			0x1F
+
+/* DA9055_REG_COUNT_D (addr=0x56) */
+#define	DA9055_RTC_DAY			0x1F
+
+/* DA9055_REG_COUNT_MO (addr=0x57) */
+#define	DA9055_RTC_MONTH		0x0F
+
+/* DA9055_REG_COUNT_Y (addr=0x58) */
+#define	DA9055_RTC_YEAR			0x3F
+#define	DA9055_RTC_YEAR_BASE		2000
+
+/* DA9055_REG_ALARM_MI (addr=0x59) */
+#define	DA9055_RTC_ALM_MIN		0x3F
+#define	DA9055_ALARM_STATUS_SHIFT	6
+#define	DA9055_ALARM_STATUS_MASK	0x3
+#define	DA9055_ALARM_STATUS_NO_ALARM	0x0
+#define	DA9055_ALARM_STATUS_TICK	0x1
+#define	DA9055_ALARM_STATUS_TIMER_ALARM	0x2
+#define	DA9055_ALARM_STATUS_BOTH	0x3
+
+/* DA9055_REG_ALARM_H (addr=0x5A) */
+#define	DA9055_RTC_ALM_HOUR		0x1F
+
+/* DA9055_REG_ALARM_D (addr=0x5B) */
+#define	DA9055_RTC_ALM_DAY		0x1F
+
+/* DA9055_REG_ALARM_MO (addr=0x5C) */
+#define	DA9055_RTC_ALM_MONTH		0x0F
+#define	DA9055_RTC_TICK_WAKE_MASK	0x20
+#define	DA9055_RTC_TICK_WAKE_SHIFT	5
+#define	DA9055_RTC_TICK_TYPE		0x10
+#define	DA9055_RTC_TICK_TYPE_SHIFT	0x4
+#define	DA9055_RTC_TICK_SEC		0x0
+#define	DA9055_RTC_TICK_MIN		0x1
+#define	DA9055_ALARAM_TICK_WAKE		0x20
+
+/* DA9055_REG_ALARM_Y (addr=0x5D) */
+#define	DA9055_RTC_TICK_EN		0x80
+#define	DA9055_RTC_ALM_EN		0x40
+#define	DA9055_RTC_TICK_ALM_MASK	0xC0
+#define	DA9055_RTC_ALM_YEAR		0x3F
+
+/* DA9055_REG_TRIM_CLDR (addr=0x62) */
+#define	DA9055_TRIM_32K_SHIFT		0
+#define	DA9055_TRIM_32K_MASK		0x7F
+#define	DA9055_TRIM_DECREMENT		(1<<7)
+#define	DA9055_TRIM_INCREMENT		(0<<7)
+#define	DA9055_TRIM_VAL_BASE		0x0
+#define	DA9055_TRIM_PPM_BASE		0x0 /* min val in units of 0.1PPM */
+#define	DA9055_TRIM_PPM_INC		19 /* min inc in units of 0.1PPM */
+#define	DA9055_TRIM_STEPS		127
+
+/* DA9055_REG_CONFIG_A (addr=0x65) */
+#define	DA9055_PM_I_V_VDDCORE		(0<<0)
+#define	DA9055_PM_I_V_VDD_IO		(1<<0)
+#define	DA9055_VDD_FAULT_TYPE_ACT_LOW	(0<<1)
+#define	DA9055_VDD_FAULT_TYPE_ACT_HIGH	(1<<1)
+#define	DA9055_PM_O_TYPE_PUSH_PULL	(0<<2)
+#define	DA9055_PM_O_TYPE_OPEN_DRAIN	(1<<2)
+#define	DA9055_IRQ_TYPE_ACT_LOW		(0<<3)
+#define	DA9055_IRQ_TYPE_ACT_HIGH	(1<<3)
+#define	DA9055_NIRQ_MODE_IMM		(0<<4)
+#define	DA9055_NIRQ_MODE_ACTIVE		(1<<4)
+#define	DA9055_GPI_V_VDDCORE		(0<<5)
+#define	DA9055_GPI_V_VDD_IO		(1<<5)
+#define	DA9055_PM_IF_V_VDDCORE		(0<<6)
+#define	DA9055_PM_IF_V_VDD_IO		(1<<6)
+
+/* DA9055_REG_CONFIG_B (addr=0x66) */
+#define	DA9055_VDD_FAULT_VAL_SHIFT	0
+#define	DA9055_VDD_FAULT_VAL_MASK	0xF
+#define	DA9055_VDD_FAULT_VAL_BASE	0x0
+#define	DA9055_VDD_FAULT_VAL_MAX	DA9055_VDD_FAULT_VAL_MASK
+#define	DA9055_VDD_FAULT_VOLT_BASE	2500
+#define	DA9055_VDD_FAULT_VOLT_INC	50
+#define	DA9055_VDD_FAULT_STEPS		15
+
+#define	DA9055_VDD_HYST_VAL_SHIFT	4
+#define	DA9055_VDD_HYST_VAL_MASK	0x7
+#define	DA9055_VDD_HYST_VAL_BASE	0x0
+#define	DA9055_VDD_HYST_VAL_MAX		DA9055_VDD_HYST_VAL_MASK
+#define	DA9055_VDD_HYST_VOLT_BASE	100
+#define	DA9055_VDD_HYST_VOLT_INC	50
+#define	DA9055_VDD_HYST_STEPS		7
+#define	DA9055_VDD_HYST_VOLT_MIN	DA9055_VDD_HYST_VOLT_BASE
+
+#define	DA9055_VDD_FAULT_EN_SHIFT	7
+
+/* DA9055_REG_CONFIG_C (addr=0x67) */
+#define	DA9055_BCORE_CLK_INV_SHIFT	0
+#define	DA9055_BMEM_CLK_INV_SHIFT	1
+#define	DA9055_NFAULT_CONF_SHIFT	2
+#define	DA9055_LDO_SD_SHIFT		4
+#define	DA9055_LDO5_BYP_SHIFT		6
+#define	DA9055_LDO6_BYP_SHIFT		7
+
+/* DA9055_REG_CONFIG_D (addr=0x68) */
+#define	DA9055_NONKEY_PIN_SHIFT		0
+#define	DA9055_NONKEY_PIN_MASK		0x3
+#define	DA9055_NONKEY_PIN_PORT_MODE	0x0
+#define	DA9055_NONKEY_PIN_KEY_MODE	0x1
+#define	DA9055_NONKEY_PIN_MULTI_FUNC	0x2
+#define	DA9055_NONKEY_PIN_DEDICT	0x3
+#define	DA9055_NONKEY_SD_SHIFT		2
+#define	DA9055_KEY_DELAY_SHIFT		3
+#define	DA9055_KEY_DELAY_MASK		0x3
+#define	DA9055_KEY_DELAY_4S		0x0
+#define	DA9055_KEY_DELAY_6S		0x1
+#define	DA9055_KEY_DELAY_8S		0x2
+#define	DA9055_KEY_DELAY_10S		0x3
+
+/* DA9055_REG_CONFIG_E (addr=0x69) */
+#define	DA9055_GPIO_PUPD_PULL_UP	0x0
+#define	DA9055_GPIO_PUPD_OPEN_DRAIN	0x1
+#define	DA9055_GPIO0_PUPD_SHIFT		0
+#define	DA9055_GPIO1_PUPD_SHIFT		1
+#define	DA9055_GPIO2_PUPD_SHIFT		2
+#define	DA9055_UVOV_DELAY_SHIFT		4
+#define	DA9055_UVOV_DELAY_MASK		0x3
+#define	DA9055_RESET_DURATION_SHIFT	6
+#define	DA9055_RESET_DURATION_MASK	0x3
+#define	DA9055_RESET_DURATION_0MS	0x0
+#define	DA9055_RESET_DURATION_100MS	0x1
+#define	DA9055_RESET_DURATION_500MS	0x2
+#define	DA9055_RESET_DURATION_1000MS	0x3
+
+/* DA9055_REG_MON_REG_1 (addr=0x6A) */
+#define	DA9055_MON_THRES_SHIFT		0
+#define	DA9055_MON_THRES_MASK		0x3
+#define	DA9055_MON_RES_SHIFT		2
+#define	DA9055_MON_DEB_SHIFT		3
+#define	DA9055_MON_MODE_SHIFT		4
+#define	DA9055_MON_MODE_MASK		0x3
+#define	DA9055_START_MAX_SHIFT		6
+#define	DA9055_START_MAX_MASK		0x3
+
+/* DA9055_REG_MON_REG_2 (addr=0x6B) */
+#define	DA9055_LDO1_MON_EN_SHIFT	0
+#define	DA9055_LDO2_MON_EN_SHIFT	1
+#define	DA9055_LDO3_MON_EN_SHIFT	2
+#define	DA9055_LDO4_MON_EN_SHIFT	3
+#define	DA9055_LDO5_MON_EN_SHIFT	4
+#define	DA9055_LDO6_MON_EN_SHIFT	5
+#define	DA9055_BCORE_MON_EN_SHIFT	6
+#define	DA9055_BMEM_MON_EN_SHIFT	7
+
+/* DA9055_REG_CONFIG_F (addr=0x6C) */
+#define	DA9055_LDO1_DEF_SHIFT		0
+#define	DA9055_LDO2_DEF_SHIFT		1
+#define	DA9055_LDO3_DEF_SHIFT		2
+#define	DA9055_LDO4_DEF_SHIFT		3
+#define	DA9055_LDO5_DEF_SHIFT		4
+#define	DA9055_LDO6_DEF_SHIFT		5
+#define	DA9055_BCORE_DEF_SHIFT		6
+#define	DA9055_BMEM_DEF_SHIFT		7
+
+/* DA9055_REG_MON_REG_4 (addr=0x6D) */
+#define	DA9055_MON_A8_IDX_SHIFT		0
+#define	DA9055_MON_A89_IDX_MASK		0x3
+#define	DA9055_MON_A89_IDX_NONE		0x0
+#define	DA9055_MON_A89_IDX_BUCKCORE	0x1
+#define	DA9055_MON_A89_IDX_LDO3		0x2
+#define	DA9055_MON_A9_IDX_SHIFT		5
+
+/* DA9055_REG_MON_REG_5 (addr=0x6E) */
+#define	DA9055_MON_A10_IDX_SHIFT	0
+#define	DA9055_MON_A10_IDX_MASK		0x3
+#define	DA9055_MON_A10_IDX_NONE		0x0
+#define	DA9055_MON_A10_IDX_LDO1		0x1
+#define	DA9055_MON_A10_IDX_LDO2		0x2
+#define	DA9055_MON_A10_IDX_LDO5		0x3
+#define	DA9055_MON_A10_IDX_LDO6		0x4
+
+#endif /* __DA9055_REG_H */
diff --git a/include/linux/mfd/lp8788-isink.h b/include/linux/mfd/lp8788-isink.h
new file mode 100644
index 00000000000..f38262d21ff
--- /dev/null
+++ b/include/linux/mfd/lp8788-isink.h
@@ -0,0 +1,52 @@
+/*
+ * TI LP8788 MFD - common definitions for current sinks
+ *
+ * Copyright 2012 Texas Instruments
+ *
+ * Author: Milo(Woogyom) Kim <milo.kim@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __ISINK_LP8788_H__
+#define __ISINK_LP8788_H__
+
+/* register address */
+#define LP8788_ISINK_CTRL		0x99
+#define LP8788_ISINK12_IOUT		0x9A
+#define LP8788_ISINK3_IOUT		0x9B
+#define LP8788_ISINK1_PWM		0x9C
+#define LP8788_ISINK2_PWM		0x9D
+#define LP8788_ISINK3_PWM		0x9E
+
+/* mask bits */
+#define LP8788_ISINK1_IOUT_M		0x0F	/* Addr 9Ah */
+#define LP8788_ISINK2_IOUT_M		0xF0
+#define LP8788_ISINK3_IOUT_M		0x0F	/* Addr 9Bh */
+
+/* 6 bits used for PWM code : Addr 9C ~ 9Eh */
+#define LP8788_ISINK_MAX_PWM		63
+#define LP8788_ISINK_SCALE_OFFSET	3
+
+static const u8 lp8788_iout_addr[] = {
+	LP8788_ISINK12_IOUT,
+	LP8788_ISINK12_IOUT,
+	LP8788_ISINK3_IOUT,
+};
+
+static const u8 lp8788_iout_mask[] = {
+	LP8788_ISINK1_IOUT_M,
+	LP8788_ISINK2_IOUT_M,
+	LP8788_ISINK3_IOUT_M,
+};
+
+static const u8 lp8788_pwm_addr[] = {
+	LP8788_ISINK1_PWM,
+	LP8788_ISINK2_PWM,
+	LP8788_ISINK3_PWM,
+};
+
+#endif
diff --git a/include/linux/mfd/lp8788.h b/include/linux/mfd/lp8788.h
new file mode 100644
index 00000000000..cec364bdccf
--- /dev/null
+++ b/include/linux/mfd/lp8788.h
@@ -0,0 +1,364 @@
+/*
+ * TI LP8788 MFD Device
+ *
+ * Copyright 2012 Texas Instruments
+ *
+ * Author: Milo(Woogyom) Kim <milo.kim@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __MFD_LP8788_H__
+#define __MFD_LP8788_H__
+
+#include <linux/gpio.h>
+#include <linux/irqdomain.h>
+#include <linux/regmap.h>
+
+#define LP8788_DEV_BUCK		"lp8788-buck"
+#define LP8788_DEV_DLDO		"lp8788-dldo"
+#define LP8788_DEV_ALDO		"lp8788-aldo"
+#define LP8788_DEV_CHARGER	"lp8788-charger"
+#define LP8788_DEV_RTC		"lp8788-rtc"
+#define LP8788_DEV_BACKLIGHT	"lp8788-backlight"
+#define LP8788_DEV_VIBRATOR	"lp8788-vibrator"
+#define LP8788_DEV_KEYLED	"lp8788-keyled"
+#define LP8788_DEV_ADC		"lp8788-adc"
+
+#define LP8788_NUM_BUCKS	4
+#define LP8788_NUM_DLDOS	12
+#define LP8788_NUM_ALDOS	10
+#define LP8788_NUM_BUCK2_DVS	2
+
+#define LP8788_CHG_IRQ		"CHG_IRQ"
+#define LP8788_PRSW_IRQ		"PRSW_IRQ"
+#define LP8788_BATT_IRQ		"BATT_IRQ"
+#define LP8788_ALM_IRQ		"ALARM_IRQ"
+
+enum lp8788_int_id {
+	/* interrup register 1 : Addr 00h */
+	LP8788_INT_TSDL,
+	LP8788_INT_TSDH,
+	LP8788_INT_UVLO,
+	LP8788_INT_FLAGMON,
+	LP8788_INT_PWRON_TIME,
+	LP8788_INT_PWRON,
+	LP8788_INT_COMP1,
+	LP8788_INT_COMP2,
+
+	/* interrupt register 2 : Addr 01h */
+	LP8788_INT_CHG_INPUT_STATE,
+	LP8788_INT_CHG_STATE,
+	LP8788_INT_EOC,
+	LP8788_INT_CHG_RESTART,
+	LP8788_INT_RESTART_TIMEOUT,
+	LP8788_INT_FULLCHG_TIMEOUT,
+	LP8788_INT_PRECHG_TIMEOUT,
+
+	/* interrupt register 3 : Addr 02h */
+	LP8788_INT_RTC_ALARM1 = 17,
+	LP8788_INT_RTC_ALARM2,
+	LP8788_INT_ENTER_SYS_SUPPORT,
+	LP8788_INT_EXIT_SYS_SUPPORT,
+	LP8788_INT_BATT_LOW,
+	LP8788_INT_NO_BATT,
+
+	LP8788_INT_MAX = 24,
+};
+
+enum lp8788_dvs_sel {
+	DVS_SEL_V0,
+	DVS_SEL_V1,
+	DVS_SEL_V2,
+	DVS_SEL_V3,
+};
+
+enum lp8788_ext_ldo_en_id {
+	EN_ALDO1,
+	EN_ALDO234,
+	EN_ALDO5,
+	EN_ALDO7,
+	EN_DLDO7,
+	EN_DLDO911,
+	EN_LDOS_MAX,
+};
+
+enum lp8788_charger_event {
+	NO_CHARGER,
+	CHARGER_DETECTED,
+};
+
+enum lp8788_bl_ctrl_mode {
+	LP8788_BL_REGISTER_ONLY,
+	LP8788_BL_COMB_PWM_BASED,	/* PWM + I2C, changed by PWM input */
+	LP8788_BL_COMB_REGISTER_BASED,	/* PWM + I2C, changed by I2C */
+};
+
+enum lp8788_bl_dim_mode {
+	LP8788_DIM_EXPONENTIAL,
+	LP8788_DIM_LINEAR,
+};
+
+enum lp8788_bl_full_scale_current {
+	LP8788_FULLSCALE_5000uA,
+	LP8788_FULLSCALE_8500uA,
+	LP8788_FULLSCALE_1200uA,
+	LP8788_FULLSCALE_1550uA,
+	LP8788_FULLSCALE_1900uA,
+	LP8788_FULLSCALE_2250uA,
+	LP8788_FULLSCALE_2600uA,
+	LP8788_FULLSCALE_2950uA,
+};
+
+enum lp8788_bl_ramp_step {
+	LP8788_RAMP_8us,
+	LP8788_RAMP_1024us,
+	LP8788_RAMP_2048us,
+	LP8788_RAMP_4096us,
+	LP8788_RAMP_8192us,
+	LP8788_RAMP_16384us,
+	LP8788_RAMP_32768us,
+	LP8788_RAMP_65538us,
+};
+
+enum lp8788_bl_pwm_polarity {
+	LP8788_PWM_ACTIVE_HIGH,
+	LP8788_PWM_ACTIVE_LOW,
+};
+
+enum lp8788_isink_scale {
+	LP8788_ISINK_SCALE_100mA,
+	LP8788_ISINK_SCALE_120mA,
+};
+
+enum lp8788_isink_number {
+	LP8788_ISINK_1,
+	LP8788_ISINK_2,
+	LP8788_ISINK_3,
+};
+
+enum lp8788_alarm_sel {
+	LP8788_ALARM_1,
+	LP8788_ALARM_2,
+	LP8788_ALARM_MAX,
+};
+
+enum lp8788_adc_id {
+	LPADC_VBATT_5P5,
+	LPADC_VIN_CHG,
+	LPADC_IBATT,
+	LPADC_IC_TEMP,
+	LPADC_VBATT_6P0,
+	LPADC_VBATT_5P0,
+	LPADC_ADC1,
+	LPADC_ADC2,
+	LPADC_VDD,
+	LPADC_VCOIN,
+	LPADC_VDD_LDO,
+	LPADC_ADC3,
+	LPADC_ADC4,
+	LPADC_MAX,
+};
+
+struct lp8788;
+
+/*
+ * lp8788_buck1_dvs
+ * @gpio         : gpio pin number for dvs control
+ * @vsel         : dvs selector for buck v1 register
+ */
+struct lp8788_buck1_dvs {
+	int gpio;
+	enum lp8788_dvs_sel vsel;
+};
+
+/*
+ * lp8788_buck2_dvs
+ * @gpio         : two gpio pin numbers are used for dvs
+ * @vsel         : dvs selector for buck v2 register
+ */
+struct lp8788_buck2_dvs {
+	int gpio[LP8788_NUM_BUCK2_DVS];
+	enum lp8788_dvs_sel vsel;
+};
+
+/*
+ * struct lp8788_ldo_enable_pin
+ *
+ *   Basically, all LDOs are enabled through the I2C commands.
+ *   But ALDO 1 ~ 5, 7, DLDO 7, 9, 11 can be enabled by external gpio pins.
+ *
+ * @gpio         : gpio number which is used for enabling ldos
+ * @init_state   : initial gpio state (ex. GPIOF_OUT_INIT_LOW)
+ */
+struct lp8788_ldo_enable_pin {
+	int gpio;
+	int init_state;
+};
+
+/*
+ * struct lp8788_chg_param
+ * @addr         : charging control register address (range : 0x11 ~ 0x1C)
+ * @val          : charging parameter value
+ */
+struct lp8788_chg_param {
+	u8 addr;
+	u8 val;
+};
+
+/*
+ * struct lp8788_charger_platform_data
+ * @vbatt_adc         : adc selection id for battery voltage
+ * @batt_temp_adc     : adc selection id for battery temperature
+ * @max_vbatt_mv      : used for calculating battery capacity
+ * @chg_params        : initial charging parameters
+ * @num_chg_params    : numbers of charging parameters
+ * @charger_event     : the charger event can be reported to the platform side
+ */
+struct lp8788_charger_platform_data {
+	enum lp8788_adc_id vbatt_adc;
+	enum lp8788_adc_id batt_temp_adc;
+	unsigned int max_vbatt_mv;
+	struct lp8788_chg_param *chg_params;
+	int num_chg_params;
+	void (*charger_event) (struct lp8788 *lp,
+				enum lp8788_charger_event event);
+};
+
+/*
+ * struct lp8788_bl_pwm_data
+ * @pwm_set_intensity     : set duty of pwm
+ * @pwm_get_intensity     : get current duty of pwm
+ */
+struct lp8788_bl_pwm_data {
+	void (*pwm_set_intensity) (int brightness, int max_brightness);
+	int (*pwm_get_intensity) (int max_brightness);
+};
+
+/*
+ * struct lp8788_backlight_platform_data
+ * @name                  : backlight driver name. (default: "lcd-backlight")
+ * @initial_brightness    : initial value of backlight brightness
+ * @bl_mode               : brightness control by pwm or lp8788 register
+ * @dim_mode              : dimming mode selection
+ * @full_scale            : full scale current setting
+ * @rise_time             : brightness ramp up step time
+ * @fall_time             : brightness ramp down step time
+ * @pwm_pol               : pwm polarity setting when bl_mode is pwm based
+ * @pwm_data              : platform specific pwm generation functions
+ *                          only valid when bl_mode is pwm based
+ */
+struct lp8788_backlight_platform_data {
+	char *name;
+	int initial_brightness;
+	enum lp8788_bl_ctrl_mode bl_mode;
+	enum lp8788_bl_dim_mode dim_mode;
+	enum lp8788_bl_full_scale_current full_scale;
+	enum lp8788_bl_ramp_step rise_time;
+	enum lp8788_bl_ramp_step fall_time;
+	enum lp8788_bl_pwm_polarity pwm_pol;
+	struct lp8788_bl_pwm_data pwm_data;
+};
+
+/*
+ * struct lp8788_led_platform_data
+ * @name         : led driver name. (default: "keyboard-backlight")
+ * @scale        : current scale
+ * @num          : current sink number
+ * @iout_code    : current output value (Addr 9Ah ~ 9Bh)
+ */
+struct lp8788_led_platform_data {
+	char *name;
+	enum lp8788_isink_scale scale;
+	enum lp8788_isink_number num;
+	int iout_code;
+};
+
+/*
+ * struct lp8788_vib_platform_data
+ * @name         : vibrator driver name
+ * @scale        : current scale
+ * @num          : current sink number
+ * @iout_code    : current output value (Addr 9Ah ~ 9Bh)
+ * @pwm_code     : PWM code value (Addr 9Ch ~ 9Eh)
+ */
+struct lp8788_vib_platform_data {
+	char *name;
+	enum lp8788_isink_scale scale;
+	enum lp8788_isink_number num;
+	int iout_code;
+	int pwm_code;
+};
+
+/*
+ * struct lp8788_platform_data
+ * @init_func    : used for initializing registers
+ *                 before mfd driver is registered
+ * @buck_data    : regulator initial data for buck
+ * @dldo_data    : regulator initial data for digital ldo
+ * @aldo_data    : regulator initial data for analog ldo
+ * @buck1_dvs    : gpio configurations for buck1 dvs
+ * @buck2_dvs    : gpio configurations for buck2 dvs
+ * @ldo_pin      : gpio configurations for enabling LDOs
+ * @chg_pdata    : platform data for charger driver
+ * @alarm_sel    : rtc alarm selection (1 or 2)
+ * @bl_pdata     : configurable data for backlight driver
+ * @led_pdata    : configurable data for led driver
+ * @vib_pdata    : configurable data for vibrator driver
+ * @adc_pdata    : iio map data for adc driver
+ */
+struct lp8788_platform_data {
+	/* general system information */
+	int (*init_func) (struct lp8788 *lp);
+
+	/* regulators */
+	struct regulator_init_data *buck_data[LP8788_NUM_BUCKS];
+	struct regulator_init_data *dldo_data[LP8788_NUM_DLDOS];
+	struct regulator_init_data *aldo_data[LP8788_NUM_ALDOS];
+	struct lp8788_buck1_dvs *buck1_dvs;
+	struct lp8788_buck2_dvs *buck2_dvs;
+	struct lp8788_ldo_enable_pin *ldo_pin[EN_LDOS_MAX];
+
+	/* charger */
+	struct lp8788_charger_platform_data *chg_pdata;
+
+	/* rtc alarm */
+	enum lp8788_alarm_sel alarm_sel;
+
+	/* backlight */
+	struct lp8788_backlight_platform_data *bl_pdata;
+
+	/* current sinks */
+	struct lp8788_led_platform_data *led_pdata;
+	struct lp8788_vib_platform_data *vib_pdata;
+
+	/* adc iio map data */
+	struct iio_map *adc_pdata;
+};
+
+/*
+ * struct lp8788
+ * @dev          : parent device pointer
+ * @regmap       : used for i2c communcation on accessing registers
+ * @irqdm        : interrupt domain for handling nested interrupt
+ * @irq          : pin number of IRQ_N
+ * @pdata        : lp8788 platform specific data
+ */
+struct lp8788 {
+	struct device *dev;
+	struct regmap *regmap;
+	struct irq_domain *irqdm;
+	int irq;
+	struct lp8788_platform_data *pdata;
+};
+
+int lp8788_irq_init(struct lp8788 *lp, int chip_irq);
+void lp8788_irq_exit(struct lp8788 *lp);
+int lp8788_read_byte(struct lp8788 *lp, u8 reg, u8 *data);
+int lp8788_read_multi_bytes(struct lp8788 *lp, u8 reg, u8 *data, size_t count);
+int lp8788_write_byte(struct lp8788 *lp, u8 reg, u8 data);
+int lp8788_update_bits(struct lp8788 *lp, u8 reg, u8 mask, u8 data);
+#endif
diff --git a/include/linux/mfd/lpc_ich.h b/include/linux/mfd/lpc_ich.h
index fec5256c3f5..3e1df644c40 100644
--- a/include/linux/mfd/lpc_ich.h
+++ b/include/linux/mfd/lpc_ich.h
@@ -43,6 +43,7 @@ struct lpc_ich_info {
 	char name[32];
 	unsigned int iTCO_version;
 	unsigned int gpio_version;
+	u8 use_gpio;
 };
 
 #endif
diff --git a/include/linux/mfd/max8907.h b/include/linux/mfd/max8907.h
new file mode 100644
index 00000000000..b06f7a6a1e8
--- /dev/null
+++ b/include/linux/mfd/max8907.h
@@ -0,0 +1,252 @@
+/*
+ * Functions to access MAX8907 power management chip.
+ *
+ * Copyright (C) 2010 Gyungoh Yoo <jack.yoo@maxim-ic.com>
+ * Copyright (C) 2012, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __LINUX_MFD_MAX8907_H
+#define __LINUX_MFD_MAX8907_H
+
+#include <linux/mutex.h>
+#include <linux/pm.h>
+
+#define MAX8907_GEN_I2C_ADDR		(0x78 >> 1)
+#define MAX8907_ADC_I2C_ADDR		(0x8e >> 1)
+#define MAX8907_RTC_I2C_ADDR		(0xd0 >> 1)
+
+/* MAX8907 register map */
+#define MAX8907_REG_SYSENSEL		0x00
+#define MAX8907_REG_ON_OFF_IRQ1		0x01
+#define MAX8907_REG_ON_OFF_IRQ1_MASK	0x02
+#define MAX8907_REG_ON_OFF_STAT		0x03
+#define MAX8907_REG_SDCTL1		0x04
+#define MAX8907_REG_SDSEQCNT1		0x05
+#define MAX8907_REG_SDV1		0x06
+#define MAX8907_REG_SDCTL2		0x07
+#define MAX8907_REG_SDSEQCNT2		0x08
+#define MAX8907_REG_SDV2		0x09
+#define MAX8907_REG_SDCTL3		0x0A
+#define MAX8907_REG_SDSEQCNT3		0x0B
+#define MAX8907_REG_SDV3		0x0C
+#define MAX8907_REG_ON_OFF_IRQ2		0x0D
+#define MAX8907_REG_ON_OFF_IRQ2_MASK	0x0E
+#define MAX8907_REG_RESET_CNFG		0x0F
+#define MAX8907_REG_LDOCTL16		0x10
+#define MAX8907_REG_LDOSEQCNT16		0x11
+#define MAX8907_REG_LDO16VOUT		0x12
+#define MAX8907_REG_SDBYSEQCNT		0x13
+#define MAX8907_REG_LDOCTL17		0x14
+#define MAX8907_REG_LDOSEQCNT17		0x15
+#define MAX8907_REG_LDO17VOUT		0x16
+#define MAX8907_REG_LDOCTL1		0x18
+#define MAX8907_REG_LDOSEQCNT1		0x19
+#define MAX8907_REG_LDO1VOUT		0x1A
+#define MAX8907_REG_LDOCTL2		0x1C
+#define MAX8907_REG_LDOSEQCNT2		0x1D
+#define MAX8907_REG_LDO2VOUT		0x1E
+#define MAX8907_REG_LDOCTL3		0x20
+#define MAX8907_REG_LDOSEQCNT3		0x21
+#define MAX8907_REG_LDO3VOUT		0x22
+#define MAX8907_REG_LDOCTL4		0x24
+#define MAX8907_REG_LDOSEQCNT4		0x25
+#define MAX8907_REG_LDO4VOUT		0x26
+#define MAX8907_REG_LDOCTL5		0x28
+#define MAX8907_REG_LDOSEQCNT5		0x29
+#define MAX8907_REG_LDO5VOUT		0x2A
+#define MAX8907_REG_LDOCTL6		0x2C
+#define MAX8907_REG_LDOSEQCNT6		0x2D
+#define MAX8907_REG_LDO6VOUT		0x2E
+#define MAX8907_REG_LDOCTL7		0x30
+#define MAX8907_REG_LDOSEQCNT7		0x31
+#define MAX8907_REG_LDO7VOUT		0x32
+#define MAX8907_REG_LDOCTL8		0x34
+#define MAX8907_REG_LDOSEQCNT8		0x35
+#define MAX8907_REG_LDO8VOUT		0x36
+#define MAX8907_REG_LDOCTL9		0x38
+#define MAX8907_REG_LDOSEQCNT9		0x39
+#define MAX8907_REG_LDO9VOUT		0x3A
+#define MAX8907_REG_LDOCTL10		0x3C
+#define MAX8907_REG_LDOSEQCNT10		0x3D
+#define MAX8907_REG_LDO10VOUT		0x3E
+#define MAX8907_REG_LDOCTL11		0x40
+#define MAX8907_REG_LDOSEQCNT11		0x41
+#define MAX8907_REG_LDO11VOUT		0x42
+#define MAX8907_REG_LDOCTL12		0x44
+#define MAX8907_REG_LDOSEQCNT12		0x45
+#define MAX8907_REG_LDO12VOUT		0x46
+#define MAX8907_REG_LDOCTL13		0x48
+#define MAX8907_REG_LDOSEQCNT13		0x49
+#define MAX8907_REG_LDO13VOUT		0x4A
+#define MAX8907_REG_LDOCTL14		0x4C
+#define MAX8907_REG_LDOSEQCNT14		0x4D
+#define MAX8907_REG_LDO14VOUT		0x4E
+#define MAX8907_REG_LDOCTL15		0x50
+#define MAX8907_REG_LDOSEQCNT15		0x51
+#define MAX8907_REG_LDO15VOUT		0x52
+#define MAX8907_REG_OUT5VEN		0x54
+#define MAX8907_REG_OUT5VSEQ		0x55
+#define MAX8907_REG_OUT33VEN		0x58
+#define MAX8907_REG_OUT33VSEQ		0x59
+#define MAX8907_REG_LDOCTL19		0x5C
+#define MAX8907_REG_LDOSEQCNT19		0x5D
+#define MAX8907_REG_LDO19VOUT		0x5E
+#define MAX8907_REG_LBCNFG		0x60
+#define MAX8907_REG_SEQ1CNFG		0x64
+#define MAX8907_REG_SEQ2CNFG		0x65
+#define MAX8907_REG_SEQ3CNFG		0x66
+#define MAX8907_REG_SEQ4CNFG		0x67
+#define MAX8907_REG_SEQ5CNFG		0x68
+#define MAX8907_REG_SEQ6CNFG		0x69
+#define MAX8907_REG_SEQ7CNFG		0x6A
+#define MAX8907_REG_LDOCTL18		0x72
+#define MAX8907_REG_LDOSEQCNT18		0x73
+#define MAX8907_REG_LDO18VOUT		0x74
+#define MAX8907_REG_BBAT_CNFG		0x78
+#define MAX8907_REG_CHG_CNTL1		0x7C
+#define MAX8907_REG_CHG_CNTL2		0x7D
+#define MAX8907_REG_CHG_IRQ1		0x7E
+#define MAX8907_REG_CHG_IRQ2		0x7F
+#define MAX8907_REG_CHG_IRQ1_MASK	0x80
+#define MAX8907_REG_CHG_IRQ2_MASK	0x81
+#define MAX8907_REG_CHG_STAT		0x82
+#define MAX8907_REG_WLED_MODE_CNTL	0x84
+#define MAX8907_REG_ILED_CNTL		0x84
+#define MAX8907_REG_II1RR		0x8E
+#define MAX8907_REG_II2RR		0x8F
+#define MAX8907_REG_LDOCTL20		0x9C
+#define MAX8907_REG_LDOSEQCNT20		0x9D
+#define MAX8907_REG_LDO20VOUT		0x9E
+
+/* RTC register map */
+#define MAX8907_REG_RTC_SEC		0x00
+#define MAX8907_REG_RTC_MIN		0x01
+#define MAX8907_REG_RTC_HOURS		0x02
+#define MAX8907_REG_RTC_WEEKDAY		0x03
+#define MAX8907_REG_RTC_DATE		0x04
+#define MAX8907_REG_RTC_MONTH		0x05
+#define MAX8907_REG_RTC_YEAR1		0x06
+#define MAX8907_REG_RTC_YEAR2		0x07
+#define MAX8907_REG_ALARM0_SEC		0x08
+#define MAX8907_REG_ALARM0_MIN		0x09
+#define MAX8907_REG_ALARM0_HOURS	0x0A
+#define MAX8907_REG_ALARM0_WEEKDAY	0x0B
+#define MAX8907_REG_ALARM0_DATE		0x0C
+#define MAX8907_REG_ALARM0_MONTH	0x0D
+#define MAX8907_REG_ALARM0_YEAR1	0x0E
+#define MAX8907_REG_ALARM0_YEAR2	0x0F
+#define MAX8907_REG_ALARM1_SEC		0x10
+#define MAX8907_REG_ALARM1_MIN		0x11
+#define MAX8907_REG_ALARM1_HOURS	0x12
+#define MAX8907_REG_ALARM1_WEEKDAY	0x13
+#define MAX8907_REG_ALARM1_DATE		0x14
+#define MAX8907_REG_ALARM1_MONTH	0x15
+#define MAX8907_REG_ALARM1_YEAR1	0x16
+#define MAX8907_REG_ALARM1_YEAR2	0x17
+#define MAX8907_REG_ALARM0_CNTL		0x18
+#define MAX8907_REG_ALARM1_CNTL		0x19
+#define MAX8907_REG_RTC_STATUS		0x1A
+#define MAX8907_REG_RTC_CNTL		0x1B
+#define MAX8907_REG_RTC_IRQ		0x1C
+#define MAX8907_REG_RTC_IRQ_MASK	0x1D
+#define MAX8907_REG_MPL_CNTL		0x1E
+
+/* ADC and Touch Screen Controller register map */
+#define MAX8907_CTL			0
+#define MAX8907_SEQCNT			1
+#define MAX8907_VOUT			2
+
+/* mask bit fields */
+#define MAX8907_MASK_LDO_SEQ		0x1C
+#define MAX8907_MASK_LDO_EN		0x01
+#define MAX8907_MASK_VBBATTCV		0x03
+#define MAX8907_MASK_OUT5V_VINEN	0x10
+#define MAX8907_MASK_OUT5V_ENSRC	0x0E
+#define MAX8907_MASK_OUT5V_EN		0x01
+#define MAX8907_MASK_POWER_OFF		0x40
+
+/* Regulator IDs */
+#define MAX8907_MBATT	0
+#define MAX8907_SD1	1
+#define MAX8907_SD2	2
+#define MAX8907_SD3	3
+#define MAX8907_LDO1	4
+#define MAX8907_LDO2	5
+#define MAX8907_LDO3	6
+#define MAX8907_LDO4	7
+#define MAX8907_LDO5	8
+#define MAX8907_LDO6	9
+#define MAX8907_LDO7	10
+#define MAX8907_LDO8	11
+#define MAX8907_LDO9	12
+#define MAX8907_LDO10	13
+#define MAX8907_LDO11	14
+#define MAX8907_LDO12	15
+#define MAX8907_LDO13	16
+#define MAX8907_LDO14	17
+#define MAX8907_LDO15	18
+#define MAX8907_LDO16	19
+#define MAX8907_LDO17	20
+#define MAX8907_LDO18	21
+#define MAX8907_LDO19	22
+#define MAX8907_LDO20	23
+#define MAX8907_OUT5V	24
+#define MAX8907_OUT33V	25
+#define MAX8907_BBAT	26
+#define MAX8907_SDBY	27
+#define MAX8907_VRTC	28
+#define MAX8907_NUM_REGULATORS (MAX8907_VRTC + 1)
+
+/* IRQ definitions */
+enum {
+	MAX8907_IRQ_VCHG_DC_OVP = 0,
+	MAX8907_IRQ_VCHG_DC_F,
+	MAX8907_IRQ_VCHG_DC_R,
+	MAX8907_IRQ_VCHG_THM_OK_R,
+	MAX8907_IRQ_VCHG_THM_OK_F,
+	MAX8907_IRQ_VCHG_MBATTLOW_F,
+	MAX8907_IRQ_VCHG_MBATTLOW_R,
+	MAX8907_IRQ_VCHG_RST,
+	MAX8907_IRQ_VCHG_DONE,
+	MAX8907_IRQ_VCHG_TOPOFF,
+	MAX8907_IRQ_VCHG_TMR_FAULT,
+
+	MAX8907_IRQ_GPM_RSTIN = 0,
+	MAX8907_IRQ_GPM_MPL,
+	MAX8907_IRQ_GPM_SW_3SEC,
+	MAX8907_IRQ_GPM_EXTON_F,
+	MAX8907_IRQ_GPM_EXTON_R,
+	MAX8907_IRQ_GPM_SW_1SEC,
+	MAX8907_IRQ_GPM_SW_F,
+	MAX8907_IRQ_GPM_SW_R,
+	MAX8907_IRQ_GPM_SYSCKEN_F,
+	MAX8907_IRQ_GPM_SYSCKEN_R,
+
+	MAX8907_IRQ_RTC_ALARM1 = 0,
+	MAX8907_IRQ_RTC_ALARM0,
+};
+
+struct max8907_platform_data {
+	struct regulator_init_data *init_data[MAX8907_NUM_REGULATORS];
+	bool pm_off;
+};
+
+struct regmap_irq_chips_data;
+
+struct max8907 {
+	struct device			*dev;
+	struct mutex			irq_lock;
+	struct i2c_client		*i2c_gen;
+	struct i2c_client		*i2c_rtc;
+	struct regmap			*regmap_gen;
+	struct regmap			*regmap_rtc;
+	struct regmap_irq_chip_data	*irqc_chg;
+	struct regmap_irq_chip_data	*irqc_on_off;
+	struct regmap_irq_chip_data	*irqc_rtc;
+};
+
+#endif
diff --git a/include/linux/mfd/max8925.h b/include/linux/mfd/max8925.h
index 15b2392a56f..74d8e296963 100644
--- a/include/linux/mfd/max8925.h
+++ b/include/linux/mfd/max8925.h
@@ -158,8 +158,6 @@ enum {
 #define TSC_IRQ_MASK			(0x03)
 #define RTC_IRQ_MASK			(0x0c)
 
-#define MAX8925_MAX_REGULATOR		(23)
-
 #define MAX8925_NAME_SIZE		(32)
 
 /* IRQ definitions */
@@ -236,7 +234,29 @@ struct max8925_platform_data {
 	struct max8925_backlight_pdata	*backlight;
 	struct max8925_touch_pdata	*touch;
 	struct max8925_power_pdata	*power;
-	struct regulator_init_data	*regulator[MAX8925_MAX_REGULATOR];
+	struct regulator_init_data	*sd1;
+	struct regulator_init_data	*sd2;
+	struct regulator_init_data	*sd3;
+	struct regulator_init_data	*ldo1;
+	struct regulator_init_data	*ldo2;
+	struct regulator_init_data	*ldo3;
+	struct regulator_init_data	*ldo4;
+	struct regulator_init_data	*ldo5;
+	struct regulator_init_data	*ldo6;
+	struct regulator_init_data	*ldo7;
+	struct regulator_init_data	*ldo8;
+	struct regulator_init_data	*ldo9;
+	struct regulator_init_data	*ldo10;
+	struct regulator_init_data	*ldo11;
+	struct regulator_init_data	*ldo12;
+	struct regulator_init_data	*ldo13;
+	struct regulator_init_data	*ldo14;
+	struct regulator_init_data	*ldo15;
+	struct regulator_init_data	*ldo16;
+	struct regulator_init_data	*ldo17;
+	struct regulator_init_data	*ldo18;
+	struct regulator_init_data	*ldo19;
+	struct regulator_init_data	*ldo20;
 
 	int		irq_base;
 	int		tsc_irq;
diff --git a/include/linux/mfd/palmas.h b/include/linux/mfd/palmas.h
index 9cbc642d40a..29f6616e12f 100644
--- a/include/linux/mfd/palmas.h
+++ b/include/linux/mfd/palmas.h
@@ -23,6 +23,9 @@
 #define PALMAS_NUM_CLIENTS		3
 
 struct palmas_pmic;
+struct palmas_gpadc;
+struct palmas_resource;
+struct palmas_usb;
 
 struct palmas {
 	struct device *dev;
@@ -41,6 +44,9 @@ struct palmas {
 
 	/* Child Devices */
 	struct palmas_pmic *pmic;
+	struct palmas_gpadc *gpadc;
+	struct palmas_resource *resource;
+	struct palmas_usb *usb;
 
 	/* GPIO MUXing */
 	u8 gpio_muxed;
@@ -48,6 +54,23 @@ struct palmas {
 	u8 pwm_muxed;
 };
 
+struct palmas_gpadc_platform_data {
+	/* Channel 3 current source is only enabled during conversion */
+	int ch3_current;
+
+	/* Channel 0 current source can be used for battery detection.
+	 * If used for battery detection this will cause a permanent current
+	 * consumption depending on current level set here.
+	 */
+	int ch0_current;
+
+	/* default BAT_REMOVAL_DAT setting on device probe */
+	int bat_removal;
+
+	/* Sets the START_POLARITY bit in the RT_CTRL register */
+	int start_polarity;
+};
+
 struct palmas_reg_init {
 	/* warm_rest controls the voltage levels after a warm reset
 	 *
@@ -107,21 +130,94 @@ struct palmas_reg_init {
 
 };
 
+enum palmas_regulators {
+	/* SMPS regulators */
+	PALMAS_REG_SMPS12,
+	PALMAS_REG_SMPS123,
+	PALMAS_REG_SMPS3,
+	PALMAS_REG_SMPS45,
+	PALMAS_REG_SMPS457,
+	PALMAS_REG_SMPS6,
+	PALMAS_REG_SMPS7,
+	PALMAS_REG_SMPS8,
+	PALMAS_REG_SMPS9,
+	PALMAS_REG_SMPS10,
+	/* LDO regulators */
+	PALMAS_REG_LDO1,
+	PALMAS_REG_LDO2,
+	PALMAS_REG_LDO3,
+	PALMAS_REG_LDO4,
+	PALMAS_REG_LDO5,
+	PALMAS_REG_LDO6,
+	PALMAS_REG_LDO7,
+	PALMAS_REG_LDO8,
+	PALMAS_REG_LDO9,
+	PALMAS_REG_LDOLN,
+	PALMAS_REG_LDOUSB,
+	/* Total number of regulators */
+	PALMAS_NUM_REGS,
+};
+
 struct palmas_pmic_platform_data {
 	/* An array of pointers to regulator init data indexed by regulator
 	 * ID
 	 */
-	struct regulator_init_data **reg_data;
+	struct regulator_init_data *reg_data[PALMAS_NUM_REGS];
 
 	/* An array of pointers to structures containing sleep mode and DVS
 	 * configuration for regulators indexed by ID
 	 */
-	struct palmas_reg_init **reg_init;
+	struct palmas_reg_init *reg_init[PALMAS_NUM_REGS];
 
 	/* use LDO6 for vibrator control */
 	int ldo6_vibrator;
+};
 
+struct palmas_usb_platform_data {
+	/* Set this if platform wishes its own vbus control */
+	int no_control_vbus;
 
+	/* Do we enable the wakeup comparator on probe */
+	int wakeup;
+};
+
+struct palmas_resource_platform_data {
+	int regen1_mode_sleep;
+	int regen2_mode_sleep;
+	int sysen1_mode_sleep;
+	int sysen2_mode_sleep;
+
+	/* bitfield to be loaded to NSLEEP_RES_ASSIGN */
+	u8 nsleep_res;
+	/* bitfield to be loaded to NSLEEP_SMPS_ASSIGN */
+	u8 nsleep_smps;
+	/* bitfield to be loaded to NSLEEP_LDO_ASSIGN1 */
+	u8 nsleep_ldo1;
+	/* bitfield to be loaded to NSLEEP_LDO_ASSIGN2 */
+	u8 nsleep_ldo2;
+
+	/* bitfield to be loaded to ENABLE1_RES_ASSIGN */
+	u8 enable1_res;
+	/* bitfield to be loaded to ENABLE1_SMPS_ASSIGN */
+	u8 enable1_smps;
+	/* bitfield to be loaded to ENABLE1_LDO_ASSIGN1 */
+	u8 enable1_ldo1;
+	/* bitfield to be loaded to ENABLE1_LDO_ASSIGN2 */
+	u8 enable1_ldo2;
+
+	/* bitfield to be loaded to ENABLE2_RES_ASSIGN */
+	u8 enable2_res;
+	/* bitfield to be loaded to ENABLE2_SMPS_ASSIGN */
+	u8 enable2_smps;
+	/* bitfield to be loaded to ENABLE2_LDO_ASSIGN1 */
+	u8 enable2_ldo1;
+	/* bitfield to be loaded to ENABLE2_LDO_ASSIGN2 */
+	u8 enable2_ldo2;
+};
+
+struct palmas_clk_platform_data {
+	int clk32kg_mode_sleep;
+	int clk32kgaudio_mode_sleep;
 };
 
 struct palmas_platform_data {
@@ -138,8 +234,49 @@ struct palmas_platform_data {
 	u8 pad1, pad2;
 
 	struct palmas_pmic_platform_data *pmic_pdata;
+	struct palmas_gpadc_platform_data *gpadc_pdata;
+	struct palmas_usb_platform_data *usb_pdata;
+	struct palmas_resource_platform_data *resource_pdata;
+	struct palmas_clk_platform_data *clk_pdata;
+};
+
+struct palmas_gpadc_calibration {
+	s32 gain;
+	s32 gain_error;
+	s32 offset_error;
 };
 
+struct palmas_gpadc {
+	struct device *dev;
+	struct palmas *palmas;
+
+	int ch3_current;
+	int ch0_current;
+
+	int gpadc_force;
+
+	int bat_removal;
+
+	struct mutex reading_lock;
+	struct completion irq_complete;
+
+	int eoc_sw_irq;
+
+	struct palmas_gpadc_calibration *palmas_cal_tbl;
+
+	int conv0_channel;
+	int conv1_channel;
+	int rt_channel;
+};
+
+struct palmas_gpadc_result {
+	s32 raw_code;
+	s32 corrected_code;
+	s32 result;
+};
+
+#define PALMAS_MAX_CHANNELS 16
+
 /* Define the palmas IRQ numbers */
 enum palmas_irqs {
 	/* INT1 registers */
@@ -182,34 +319,6 @@ enum palmas_irqs {
 	PALMAS_NUM_IRQ,
 };
 
-enum palmas_regulators {
-	/* SMPS regulators */
-	PALMAS_REG_SMPS12,
-	PALMAS_REG_SMPS123,
-	PALMAS_REG_SMPS3,
-	PALMAS_REG_SMPS45,
-	PALMAS_REG_SMPS457,
-	PALMAS_REG_SMPS6,
-	PALMAS_REG_SMPS7,
-	PALMAS_REG_SMPS8,
-	PALMAS_REG_SMPS9,
-	PALMAS_REG_SMPS10,
-	/* LDO regulators */
-	PALMAS_REG_LDO1,
-	PALMAS_REG_LDO2,
-	PALMAS_REG_LDO3,
-	PALMAS_REG_LDO4,
-	PALMAS_REG_LDO5,
-	PALMAS_REG_LDO6,
-	PALMAS_REG_LDO7,
-	PALMAS_REG_LDO8,
-	PALMAS_REG_LDO9,
-	PALMAS_REG_LDOLN,
-	PALMAS_REG_LDOUSB,
-	/* Total number of regulators */
-	PALMAS_NUM_REGS,
-};
-
 struct palmas_pmic {
 	struct palmas *palmas;
 	struct device *dev;
@@ -223,6 +332,69 @@ struct palmas_pmic {
 	int range[PALMAS_REG_SMPS10];
 };
 
+struct palmas_resource {
+	struct palmas *palmas;
+	struct device *dev;
+};
+
+struct palmas_usb {
+	struct palmas *palmas;
+	struct device *dev;
+
+	/* for vbus reporting with irqs disabled */
+	spinlock_t lock;
+
+	struct regulator *vbus_reg;
+
+	/* used to set vbus, in atomic path */
+	struct work_struct set_vbus_work;
+
+	int irq1;
+	int irq2;
+	int irq3;
+	int irq4;
+
+	int vbus_enable;
+
+	u8 linkstat;
+};
+
+#define comparator_to_palmas(x) container_of((x), struct palmas_usb, comparator)
+
+enum usb_irq_events {
+	/* Wakeup events from INT3 */
+	PALMAS_USB_ID_WAKEPUP,
+	PALMAS_USB_VBUS_WAKEUP,
+
+	/* ID_OTG_EVENTS */
+	PALMAS_USB_ID_GND,
+	N_PALMAS_USB_ID_GND,
+	PALMAS_USB_ID_C,
+	N_PALMAS_USB_ID_C,
+	PALMAS_USB_ID_B,
+	N_PALMAS_USB_ID_B,
+	PALMAS_USB_ID_A,
+	N_PALMAS_USB_ID_A,
+	PALMAS_USB_ID_FLOAT,
+	N_PALMAS_USB_ID_FLOAT,
+
+	/* VBUS_OTG_EVENTS */
+	PALMAS_USB_VB_SESS_END,
+	N_PALMAS_USB_VB_SESS_END,
+	PALMAS_USB_VB_SESS_VLD,
+	N_PALMAS_USB_VB_SESS_VLD,
+	PALMAS_USB_VA_SESS_VLD,
+	N_PALMAS_USB_VA_SESS_VLD,
+	PALMAS_USB_VA_VBUS_VLD,
+	N_PALMAS_USB_VA_VBUS_VLD,
+	PALMAS_USB_VADP_SNS,
+	N_PALMAS_USB_VADP_SNS,
+	PALMAS_USB_VADP_PRB,
+	N_PALMAS_USB_VADP_PRB,
+	PALMAS_USB_VOTG_SESS_VLD,
+	N_PALMAS_USB_VOTG_SESS_VLD,
+};
+
 /* defines so we can store the mux settings */
 #define PALMAS_GPIO_0_MUXED					(1 << 0)
 #define PALMAS_GPIO_1_MUXED					(1 << 1)
diff --git a/include/linux/mfd/rc5t583.h b/include/linux/mfd/rc5t583.h
index 3661c59aa1e..36c242e52ef 100644
--- a/include/linux/mfd/rc5t583.h
+++ b/include/linux/mfd/rc5t583.h
@@ -146,6 +146,28 @@
 #define RC5T583_GPIO_MON_IOIN	0xAB
 #define RC5T583_GPIO_GPOFUNC	0xAC
 
+/* RTC registers */
+#define RC5T583_RTC_SEC		0xE0
+#define RC5T583_RTC_MIN		0xE1
+#define RC5T583_RTC_HOUR	0xE2
+#define RC5T583_RTC_WDAY	0xE3
+#define RC5T583_RTC_DAY		0xE4
+#define RC5T583_RTC_MONTH	0xE5
+#define RC5T583_RTC_YEAR	0xE6
+#define RC5T583_RTC_ADJ		0xE7
+#define RC5T583_RTC_AW_MIN	0xE8
+#define RC5T583_RTC_AW_HOUR	0xE9
+#define RC5T583_RTC_AW_WEEK	0xEA
+#define RC5T583_RTC_AD_MIN	0xEB
+#define RC5T583_RTC_AD_HOUR	0xEC
+#define RC5T583_RTC_CTL1	0xED
+#define RC5T583_RTC_CTL2	0xEE
+#define RC5T583_RTC_AY_MIN	0xF0
+#define RC5T583_RTC_AY_HOUR	0xF1
+#define RC5T583_RTC_AY_DAY	0xF2
+#define RC5T583_RTC_AY_MONTH 0xF3
+#define RC5T583_RTC_AY_YEAR	0xF4
+
 /* RICOH_RC5T583 IRQ definitions */
 enum {
 	RC5T583_IRQ_ONKEY,
diff --git a/include/linux/mfd/smsc.h b/include/linux/mfd/smsc.h
new file mode 100644
index 00000000000..9747b29f356
--- /dev/null
+++ b/include/linux/mfd/smsc.h
@@ -0,0 +1,109 @@
+/*
+ * SMSC ECE1099
+ *
+ * Copyright 2012 Texas Instruments Inc.
+ *
+ * Author: Sourav Poddar <sourav.poddar@ti.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under  the terms of the GNU General  Public License as published by the
+ *  Free Software Foundation;  either version 2 of the License, or (at your
+ *  option) any later version.
+ *
+ */
+
+#ifndef __LINUX_MFD_SMSC_H
+#define __LINUX_MFD_SMSC_H
+
+#include <linux/regmap.h>
+
+#define SMSC_ID_ECE1099			1
+#define SMSC_NUM_CLIENTS		2
+
+#define SMSC_BASE_ADDR			0x38
+#define OMAP_GPIO_SMSC_IRQ		151
+
+#define SMSC_MAXGPIO         32
+#define SMSC_BANK(offs)      ((offs) >> 3)
+#define SMSC_BIT(offs)       (1u << ((offs) & 0x7))
+
+struct smsc {
+	struct device *dev;
+	struct i2c_client *i2c_clients[SMSC_NUM_CLIENTS];
+	struct regmap *regmap;
+	int clk;
+	/* Stored chip id */
+	int id;
+};
+
+struct smsc_gpio;
+struct smsc_keypad;
+
+static inline int smsc_read(struct device *child, unsigned int reg,
+	unsigned int *dest)
+{
+	struct smsc     *smsc = dev_get_drvdata(child->parent);
+
+	return regmap_read(smsc->regmap, reg, dest);
+}
+
+static inline int smsc_write(struct device *child, unsigned int reg,
+	unsigned int value)
+{
+	struct smsc     *smsc = dev_get_drvdata(child->parent);
+
+	return regmap_write(smsc->regmap, reg, value);
+}
+
+/* Registers for SMSC */
+#define SMSC_RESET						0xF5
+#define SMSC_GRP_INT						0xF9
+#define SMSC_CLK_CTRL						0xFA
+#define SMSC_WKUP_CTRL						0xFB
+#define SMSC_DEV_ID						0xFC
+#define SMSC_DEV_REV						0xFD
+#define SMSC_VEN_ID_L						0xFE
+#define SMSC_VEN_ID_H						0xFF
+
+/* CLK VALUE */
+#define SMSC_CLK_VALUE						0x13
+
+/* Registers for function GPIO INPUT */
+#define SMSC_GPIO_DATA_IN_START					0x00
+
+/* Registers for function GPIO OUPUT */
+#define SMSC_GPIO_DATA_OUT_START                                       0x05
+
+/* Definitions for SMSC GPIO CONFIGURATION REGISTER*/
+#define SMSC_GPIO_INPUT_LOW					0x01
+#define SMSC_GPIO_INPUT_RISING					0x09
+#define SMSC_GPIO_INPUT_FALLING					0x11
+#define SMSC_GPIO_INPUT_BOTH_EDGE				0x19
+#define SMSC_GPIO_OUTPUT_PP					0x21
+#define SMSC_GPIO_OUTPUT_OP					0x31
+
+#define GRP_INT_STAT						0xf9
+#define	SMSC_GPI_INT						0x0f
+#define SMSC_CFG_START						0x0A
+
+/* Registers for SMSC GPIO INTERRUPT STATUS REGISTER*/
+#define SMSC_GPIO_INT_STAT_START                                  0x32
+
+/* Registers for SMSC GPIO INTERRUPT MASK REGISTER*/
+#define SMSC_GPIO_INT_MASK_START                               0x37
+
+/* Registers for SMSC function KEYPAD*/
+#define SMSC_KP_OUT						0x40
+#define SMSC_KP_IN						0x41
+#define SMSC_KP_INT_STAT					0x42
+#define SMSC_KP_INT_MASK					0x43
+
+/* Definitions for keypad */
+#define SMSC_KP_KSO           0x70
+#define SMSC_KP_KSI           0x51
+#define SMSC_KSO_ALL_LOW        0x20
+#define SMSC_KP_SET_LOW_PWR        0x0B
+#define SMSC_KP_SET_HIGH           0xFF
+#define SMSC_KSO_EVAL           0x00
+
+#endif /*  __LINUX_MFD_SMSC_H */
diff --git a/include/linux/mfd/syscon.h b/include/linux/mfd/syscon.h
new file mode 100644
index 00000000000..6aeb6b8da64
--- /dev/null
+++ b/include/linux/mfd/syscon.h
@@ -0,0 +1,23 @@
+/*
+ * System Control Driver
+ *
+ * Copyright (C) 2012 Freescale Semiconductor, Inc.
+ * Copyright (C) 2012 Linaro Ltd.
+ *
+ * Author: Dong Aisheng <dong.aisheng@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __LINUX_MFD_SYSCON_H__
+#define __LINUX_MFD_SYSCON_H__
+
+extern struct regmap *syscon_node_to_regmap(struct device_node *np);
+extern struct regmap *syscon_regmap_lookup_by_compatible(const char *s);
+extern struct regmap *syscon_regmap_lookup_by_phandle(
+					struct device_node *np,
+					const char *property);
+#endif /* __LINUX_MFD_SYSCON_H__ */
diff --git a/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h b/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h
new file mode 100644
index 00000000000..dab34a1deb2
--- /dev/null
+++ b/include/linux/mfd/syscon/imx6q-iomuxc-gpr.h
@@ -0,0 +1,319 @@
+/*
+ * Copyright (C) 2012 Freescale Semiconductor, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __LINUX_IMX6Q_IOMUXC_GPR_H
+#define __LINUX_IMX6Q_IOMUXC_GPR_H
+
+#include <linux/bitops.h>
+
+#define IOMUXC_GPR0	0x00
+#define IOMUXC_GPR1	0x04
+#define IOMUXC_GPR2	0x08
+#define IOMUXC_GPR3	0x0c
+#define IOMUXC_GPR4	0x10
+#define IOMUXC_GPR5	0x14
+#define IOMUXC_GPR6	0x18
+#define IOMUXC_GPR7	0x1c
+#define IOMUXC_GPR8	0x20
+#define IOMUXC_GPR9	0x24
+#define IOMUXC_GPR10	0x28
+#define IOMUXC_GPR11	0x2c
+#define IOMUXC_GPR12	0x30
+#define IOMUXC_GPR13	0x34
+
+#define IMX6Q_GPR0_CLOCK_8_MUX_SEL_MASK		(0x3 << 30)
+#define IMX6Q_GPR0_CLOCK_8_MUX_SEL_AUDMUX_RXCLK_P7_MUXED	(0x0 << 30)
+#define IMX6Q_GPR0_CLOCK_8_MUX_SEL_AUDMUX_RXCLK_P7	(0x1 << 30)
+#define IMX6Q_GPR0_CLOCK_8_MUX_SEL_SSI3_SSI_SRCK	(0x2 << 30)
+#define IMX6Q_GPR0_CLOCK_8_MUX_SEL_SSI3_RX_BIT_CLK	(0x3 << 30)
+#define IMX6Q_GPR0_CLOCK_0_MUX_SEL_MASK		(0x3 << 28)
+#define IMX6Q_GPR0_CLOCK_0_MUX_SEL_ESAI1_IPP_IND_SCKR_MUXED	(0x0 << 28)
+#define IMX6Q_GPR0_CLOCK_0_MUX_SEL_ESAI1_IPP_IND_SCKR	(0x1 << 28)
+#define IMX6Q_GPR0_CLOCK_0_MUX_SEL_ESAI1_IPP_DO_SCKR	(0x2 << 28)
+#define IMX6Q_GPR0_CLOCK_B_MUX_SEL_MASK		(0x3 << 26)
+#define IMX6Q_GPR0_CLOCK_B_MUX_SEL_AUDMUX_TXCLK_P7_MUXED	(0x0 << 26)
+#define IMX6Q_GPR0_CLOCK_B_MUX_SEL_AUDMUX_TXCLK_P7	(0x1 << 26)
+#define IMX6Q_GPR0_CLOCK_B_MUX_SEL_SSI3_SSI_STCK	(0x2 << 26)
+#define IMX6Q_GPR0_CLOCK_B_MUX_SEL_SSI3_TX_BIT_CLK	(0x3 << 26)
+#define IMX6Q_GPR0_CLOCK_3_MUX_SEL_MASK		(0x3 << 24)
+#define IMX6Q_GPR0_CLOCK_3_MUX_SEL_AUDMUX_RXCLK_P7_MUXED	(0x3 << 24)
+#define IMX6Q_GPR0_CLOCK_3_MUX_SEL_AUDMUX_RXCLK_P7	(0x3 << 24)
+#define IMX6Q_GPR0_CLOCK_3_MUX_SEL_SSI3_SSI_SRCK	(0x3 << 24)
+#define IMX6Q_GPR0_CLOCK_3_MUX_SEL_SSI3_RX_BIT_CLK	(0x3 << 24)
+#define IMX6Q_GPR0_CLOCK_A_MUX_SEL_MASK		(0x3 << 22)
+#define IMX6Q_GPR0_CLOCK_A_MUX_SEL_AUDMUX_TXCLK_P2_MUXED	(0x0 << 22)
+#define IMX6Q_GPR0_CLOCK_A_MUX_SEL_AUDMUX_TXCLK_P2	(0x1 << 22)
+#define IMX6Q_GPR0_CLOCK_A_MUX_SEL_SSI2_SSI_STCK	(0x2 << 22)
+#define IMX6Q_GPR0_CLOCK_A_MUX_SEL_SSI2_TX_BIT_CLK	(0x3 << 22)
+#define IMX6Q_GPR0_CLOCK_2_MUX_SEL_MASK		(0x3 << 20)
+#define IMX6Q_GPR0_CLOCK_2_MUX_SEL_AUDMUX_RXCLK_P2_MUXED	(0x0 << 20)
+#define IMX6Q_GPR0_CLOCK_2_MUX_SEL_AUDMUX_RXCLK_P2	(0x1 << 20)
+#define IMX6Q_GPR0_CLOCK_2_MUX_SEL_SSI2_SSI_SRCK	(0x2 << 20)
+#define IMX6Q_GPR0_CLOCK_2_MUX_SEL_SSI2_RX_BIT_CLK	(0x3 << 20)
+#define IMX6Q_GPR0_CLOCK_9_MUX_SEL_MASK		(0x3 << 18)
+#define IMX6Q_GPR0_CLOCK_9_MUX_SEL_AUDMUX_TXCLK_P1_MUXED	(0x0 << 18)
+#define IMX6Q_GPR0_CLOCK_9_MUX_SEL_AUDMUX_TXCLK_P1	(0x1 << 18)
+#define IMX6Q_GPR0_CLOCK_9_MUX_SEL_SSI1_SSI_STCK	(0x2 << 18)
+#define IMX6Q_GPR0_CLOCK_9_MUX_SEL_SSI1_SSI_TX_BIT_CLK	(0x3 << 18)
+#define IMX6Q_GPR0_CLOCK_1_MUX_SEL_MASK		(0x3 << 16)
+#define IMX6Q_GPR0_CLOCK_1_MUX_SEL_AUDMUX_RXCLK_P1_MUXED	(0x0 << 16)
+#define IMX6Q_GPR0_CLOCK_1_MUX_SEL_AUDMUX_RXCLK_P1	(0x1 << 16)
+#define IMX6Q_GPR0_CLOCK_1_MUX_SEL_SSI1_SSI_SRCK	(0x2 << 16)
+#define IMX6Q_GPR0_CLOCK_1_MUX_SEL_SSI1_SSI_RX_BIT_CLK	(0x3 << 16)
+#define IMX6Q_GPR0_TX_CLK2_MUX_SEL_MASK		(0x3 << 14)
+#define IMX6Q_GPR0_TX_CLK2_MUX_SEL_ASRCK_CLK1	(0x0 << 14)
+#define IMX6Q_GPR0_TX_CLK2_MUX_SEL_ASRCK_CLK2	(0x1 << 14)
+#define IMX6Q_GPR0_TX_CLK2_MUX_SEL_ASRCK_CLK3	(0x2 << 14)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL7_MASK		BIT(7)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL7_SPDIF	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL7_IOMUX	BIT(7)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL6_MASK		BIT(6)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL6_ESAI		0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL6_I2C3		BIT(6)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL5_MASK		BIT(5)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL5_ECSPI4	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL5_EPIT2	BIT(5)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL4_MASK		BIT(4)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL4_ECSPI4	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL4_I2C1		BIT(4)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL3_MASK		BIT(3)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL3_ECSPI2	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL3_I2C1		BIT(3)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL2_MASK		BIT(2)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL2_ECSPI1	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL2_I2C2		BIT(2)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL1_MASK		BIT(1)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL1_ECSPI1	0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL1_I2C3		BIT(1)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL0_MASK		BIT(0)
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL0_IPU1		0x0
+#define IMX6Q_GPR0_DMAREQ_MUX_SEL0_IOMUX	BIT(0)
+
+#define IMX6Q_GPR1_PCIE_REQ_MASK		(0x3 << 30)
+#define IMX6Q_GPR1_PCIE_EXIT_L1			BIT(28)
+#define IMX6Q_GPR1_PCIE_RDY_L23			BIT(27)
+#define IMX6Q_GPR1_PCIE_ENTER_L1		BIT(26)
+#define IMX6Q_GPR1_MIPI_COLOR_SW		BIT(25)
+#define IMX6Q_GPR1_DPI_OFF			BIT(24)
+#define IMX6Q_GPR1_EXC_MON_MASK			BIT(22)
+#define IMX6Q_GPR1_EXC_MON_OKAY			0x0
+#define IMX6Q_GPR1_EXC_MON_SLVE			BIT(22)
+#define IMX6Q_GPR1_MIPI_IPU2_SEL_MASK		BIT(21)
+#define IMX6Q_GPR1_MIPI_IPU2_SEL_GASKET		0x0
+#define IMX6Q_GPR1_MIPI_IPU2_SEL_IOMUX		BIT(21)
+#define IMX6Q_GPR1_MIPI_IPU1_MUX_MASK		BIT(20)
+#define IMX6Q_GPR1_MIPI_IPU1_MUX_GASKET		0x0
+#define IMX6Q_GPR1_MIPI_IPU1_MUX_IOMUX		BIT(20)
+#define IMX6Q_GPR1_MIPI_IPU2_MUX_MASK		BIT(19)
+#define IMX6Q_GPR1_MIPI_IPU2_MUX_GASKET		0x0
+#define IMX6Q_GPR1_MIPI_IPU2_MUX_IOMUX		BIT(19)
+#define IMX6Q_GPR1_PCIE_TEST_PD			BIT(18)
+#define IMX6Q_GPR1_IPU_VPU_MUX_MASK		BIT(17)
+#define IMX6Q_GPR1_IPU_VPU_MUX_IPU1		0x0
+#define IMX6Q_GPR1_IPU_VPU_MUX_IPU2		BIT(17)
+#define IMX6Q_GPR1_PCIE_REF_CLK_EN		BIT(16)
+#define IMX6Q_GPR1_USB_EXP_MODE			BIT(15)
+#define IMX6Q_GPR1_PCIE_INT			BIT(14)
+#define IMX6Q_GPR1_USB_OTG_ID_SEL_MASK		BIT(13)
+#define IMX6Q_GPR1_USB_OTG_ID_SEL_ENET_RX_ER	0x0
+#define IMX6Q_GPR1_USB_OTG_ID_SEL_GPIO_1	BIT(13)
+#define IMX6Q_GPR1_GINT				BIT(12)
+#define IMX6Q_GPR1_ADDRS3_MASK			(0x3 << 10)
+#define IMX6Q_GPR1_ADDRS3_32MB			(0x0 << 10)
+#define IMX6Q_GPR1_ADDRS3_64MB			(0x1 << 10)
+#define IMX6Q_GPR1_ADDRS3_128MB			(0x2 << 10)
+#define IMX6Q_GPR1_ACT_CS3			BIT(9)
+#define IMX6Q_GPR1_ADDRS2_MASK			(0x3 << 7)
+#define IMX6Q_GPR1_ACT_CS2			BIT(6)
+#define IMX6Q_GPR1_ADDRS1_MASK			(0x3 << 4)
+#define IMX6Q_GPR1_ACT_CS1			BIT(3)
+#define IMX6Q_GPR1_ADDRS0_MASK			(0x3 << 1)
+#define IMX6Q_GPR1_ACT_CS0			BIT(0)
+
+#define IMX6Q_GPR2_COUNTER_RESET_VAL_MASK	(0x3 << 20)
+#define IMX6Q_GPR2_COUNTER_RESET_VAL_5		(0x0 << 20)
+#define IMX6Q_GPR2_COUNTER_RESET_VAL_3		(0x1 << 20)
+#define IMX6Q_GPR2_COUNTER_RESET_VAL_4		(0x2 << 20)
+#define IMX6Q_GPR2_COUNTER_RESET_VAL_6		(0x3 << 20)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_MASK		(0x7 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_0		(0x0 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_1		(0x1 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_2		(0x2 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_3		(0x3 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_4		(0x4 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_5		(0x5 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_6		(0x6 << 16)
+#define IMX6Q_GPR2_LVDS_CLK_SHIFT_7		(0x7 << 16)
+#define IMX6Q_GPR2_BGREF_RRMODE_MASK		BIT(15)
+#define IMX6Q_GPR2_BGREF_RRMODE_EXT_RESISTOR	0x0
+#define IMX6Q_GPR2_BGREF_RRMODE_INT_RESISTOR	BIT(15)
+#define IMX6Q_GPR2_DI1_VS_POLARITY_MASK		BIT(10)
+#define IMX6Q_GPR2_DI1_VS_POLARITY_ACTIVE_H	0x0
+#define IMX6Q_GPR2_DI1_VS_POLARITY_ACTIVE_L	BIT(10)
+#define IMX6Q_GPR2_DI0_VS_POLARITY_MASK		BIT(9)
+#define IMX6Q_GPR2_DI0_VS_POLARITY_ACTIVE_H	0x0
+#define IMX6Q_GPR2_DI0_VS_POLARITY_ACTIVE_L	BIT(9)
+#define IMX6Q_GPR2_BIT_MAPPING_CH1_MASK		BIT(8)
+#define IMX6Q_GPR2_BIT_MAPPING_CH1_SPWG		0x0
+#define IMX6Q_GPR2_BIT_MAPPING_CH1_JEIDA	BIT(8)
+#define IMX6Q_GPR2_DATA_WIDTH_CH1_MASK		BIT(7)
+#define IMX6Q_GPR2_DATA_WIDTH_CH1_18BIT		0x0
+#define IMX6Q_GPR2_DATA_WIDTH_CH1_24BIT		BIT(7)
+#define IMX6Q_GPR2_BIT_MAPPING_CH0_MASK		BIT(6)
+#define IMX6Q_GPR2_BIT_MAPPING_CH0_SPWG		0x0
+#define IMX6Q_GPR2_BIT_MAPPING_CH0_JEIDA	BIT(6)
+#define IMX6Q_GPR2_DATA_WIDTH_CH0_MASK		BIT(5)
+#define IMX6Q_GPR2_DATA_WIDTH_CH0_18BIT		0x0
+#define IMX6Q_GPR2_DATA_WIDTH_CH0_24BIT		BIT(5)
+#define IMX6Q_GPR2_SPLIT_MODE_EN		BIT(4)
+#define IMX6Q_GPR2_CH1_MODE_MASK		(0x3 << 2)
+#define IMX6Q_GPR2_CH1_MODE_DISABLE		(0x0 << 2)
+#define IMX6Q_GPR2_CH1_MODE_EN_ROUTE_DI0	(0x1 << 2)
+#define IMX6Q_GPR2_CH1_MODE_EN_ROUTE_DI1	(0x3 << 2)
+#define IMX6Q_GPR2_CH0_MODE_MASK		(0x3 << 0)
+#define IMX6Q_GPR2_CH0_MODE_DISABLE		(0x0 << 0)
+#define IMX6Q_GPR2_CH0_MODE_EN_ROUTE_DI0	(0x1 << 0)
+#define IMX6Q_GPR2_CH0_MODE_EN_ROUTE_DI1	(0x3 << 0)
+
+#define IMX6Q_GPR3_GPU_DBG_MASK			(0x3 << 29)
+#define IMX6Q_GPR3_GPU_DBG_GPU3D		(0x0 << 29)
+#define IMX6Q_GPR3_GPU_DBG_GPU2D		(0x1 << 29)
+#define IMX6Q_GPR3_GPU_DBG_OPENVG		(0x2 << 29)
+#define IMX6Q_GPR3_BCH_WR_CACHE_CTL		BIT(28)
+#define IMX6Q_GPR3_BCH_RD_CACHE_CTL		BIT(27)
+#define IMX6Q_GPR3_USDHCX_WR_CACHE_CTL		BIT(26)
+#define IMX6Q_GPR3_USDHCX_RD_CACHE_CTL		BIT(25)
+#define IMX6Q_GPR3_OCRAM_CTL_MASK		(0xf << 21)
+#define IMX6Q_GPR3_OCRAM_STATUS_MASK		(0xf << 17)
+#define IMX6Q_GPR3_CORE3_DBG_ACK_EN		BIT(16)
+#define IMX6Q_GPR3_CORE2_DBG_ACK_EN		BIT(15)
+#define IMX6Q_GPR3_CORE1_DBG_ACK_EN		BIT(14)
+#define IMX6Q_GPR3_CORE0_DBG_ACK_EN		BIT(13)
+#define IMX6Q_GPR3_TZASC2_BOOT_LOCK		BIT(12)
+#define IMX6Q_GPR3_TZASC1_BOOT_LOCK		BIT(11)
+#define IMX6Q_GPR3_IPU_DIAG_MASK		BIT(10)
+#define IMX6Q_GPR3_LVDS1_MUX_CTL_MASK		(0x3 << 8)
+#define IMX6Q_GPR3_LVDS1_MUX_CTL_IPU1_DI0	(0x0 << 8)
+#define IMX6Q_GPR3_LVDS1_MUX_CTL_IPU1_DI1	(0x1 << 8)
+#define IMX6Q_GPR3_LVDS1_MUX_CTL_IPU2_DI0	(0x2 << 8)
+#define IMX6Q_GPR3_LVDS1_MUX_CTL_IPU2_DI1	(0x3 << 8)
+#define IMX6Q_GPR3_LVDS0_MUX_CTL_MASK		(0x3 << 6)
+#define IMX6Q_GPR3_LVDS0_MUX_CTL_IPU1_DI0	(0x0 << 6)
+#define IMX6Q_GPR3_LVDS0_MUX_CTL_IPU1_DI1	(0x1 << 6)
+#define IMX6Q_GPR3_LVDS0_MUX_CTL_IPU2_DI0	(0x2 << 6)
+#define IMX6Q_GPR3_LVDS0_MUX_CTL_IPU2_DI1	(0x3 << 6)
+#define IMX6Q_GPR3_MIPI_MUX_CTL_MASK		(0x3 << 4)
+#define IMX6Q_GPR3_MIPI_MUX_CTL_IPU1_DI0	(0x0 << 4)
+#define IMX6Q_GPR3_MIPI_MUX_CTL_IPU1_DI1	(0x1 << 4)
+#define IMX6Q_GPR3_MIPI_MUX_CTL_IPU2_DI0	(0x2 << 4)
+#define IMX6Q_GPR3_MIPI_MUX_CTL_IPU2_DI1	(0x3 << 4)
+#define IMX6Q_GPR3_HDMI_MUX_CTL_MASK		(0x3 << 2)
+#define IMX6Q_GPR3_HDMI_MUX_CTL_IPU1_DI0	(0x0 << 2)
+#define IMX6Q_GPR3_HDMI_MUX_CTL_IPU1_DI1	(0x1 << 2)
+#define IMX6Q_GPR3_HDMI_MUX_CTL_IPU2_DI0	(0x2 << 2)
+#define IMX6Q_GPR3_HDMI_MUX_CTL_IPU2_DI1	(0x3 << 2)
+
+#define IMX6Q_GPR4_VDOA_WR_CACHE_SEL		BIT(31)
+#define IMX6Q_GPR4_VDOA_RD_CACHE_SEL		BIT(30)
+#define IMX6Q_GPR4_VDOA_WR_CACHE_VAL		BIT(29)
+#define IMX6Q_GPR4_VDOA_RD_CACHE_VAL		BIT(28)
+#define IMX6Q_GPR4_PCIE_WR_CACHE_SEL		BIT(27)
+#define IMX6Q_GPR4_PCIE_RD_CACHE_SEL		BIT(26)
+#define IMX6Q_GPR4_PCIE_WR_CACHE_VAL		BIT(25)
+#define IMX6Q_GPR4_PCIE_RD_CACHE_VAL		BIT(24)
+#define IMX6Q_GPR4_SDMA_STOP_ACK		BIT(19)
+#define IMX6Q_GPR4_CAN2_STOP_ACK		BIT(18)
+#define IMX6Q_GPR4_CAN1_STOP_ACK		BIT(17)
+#define IMX6Q_GPR4_ENET_STOP_ACK		BIT(16)
+#define IMX6Q_GPR4_SOC_VERSION_MASK		(0xff << 8)
+#define IMX6Q_GPR4_SOC_VERSION_OFF		0x8
+#define IMX6Q_GPR4_VPU_WR_CACHE_SEL		BIT(7)
+#define IMX6Q_GPR4_VPU_RD_CACHE_SEL		BIT(6)
+#define IMX6Q_GPR4_VPU_P_WR_CACHE_VAL		BIT(3)
+#define IMX6Q_GPR4_VPU_P_RD_CACHE_VAL_MASK	BIT(2)
+#define IMX6Q_GPR4_IPU_WR_CACHE_CTL		BIT(1)
+#define IMX6Q_GPR4_IPU_RD_CACHE_CTL		BIT(0)
+
+#define IMX6Q_GPR5_L2_CLK_STOP			BIT(8)
+
+#define IMX6Q_GPR9_TZASC2_BYP			BIT(1)
+#define IMX6Q_GPR9_TZASC1_BYP			BIT(0)
+
+#define IMX6Q_GPR10_LOCK_DBG_EN			BIT(29)
+#define IMX6Q_GPR10_LOCK_DBG_CLK_EN		BIT(28)
+#define IMX6Q_GPR10_LOCK_SEC_ERR_RESP		BIT(27)
+#define IMX6Q_GPR10_LOCK_OCRAM_TZ_ADDR		(0x3f << 21)
+#define IMX6Q_GPR10_LOCK_OCRAM_TZ_EN		BIT(20)
+#define IMX6Q_GPR10_LOCK_DCIC2_MUX_MASK		(0x3 << 18)
+#define IMX6Q_GPR10_LOCK_DCIC1_MUX_MASK		(0x3 << 16)
+#define IMX6Q_GPR10_DBG_EN			BIT(13)
+#define IMX6Q_GPR10_DBG_CLK_EN			BIT(12)
+#define IMX6Q_GPR10_SEC_ERR_RESP_MASK		BIT(11)
+#define IMX6Q_GPR10_SEC_ERR_RESP_OKEY		0x0
+#define IMX6Q_GPR10_SEC_ERR_RESP_SLVE		BIT(11)
+#define IMX6Q_GPR10_OCRAM_TZ_ADDR_MASK		(0x3f << 5)
+#define IMX6Q_GPR10_OCRAM_TZ_EN_MASK		BIT(4)
+#define IMX6Q_GPR10_DCIC2_MUX_CTL_MASK		(0x3 << 2)
+#define IMX6Q_GPR10_DCIC2_MUX_CTL_IPU1_DI0	(0x0 << 2)
+#define IMX6Q_GPR10_DCIC2_MUX_CTL_IPU1_DI1	(0x1 << 2)
+#define IMX6Q_GPR10_DCIC2_MUX_CTL_IPU2_DI0	(0x2 << 2)
+#define IMX6Q_GPR10_DCIC2_MUX_CTL_IPU2_DI1	(0x3 << 2)
+#define IMX6Q_GPR10_DCIC1_MUX_CTL_MASK		(0x3 << 0)
+#define IMX6Q_GPR10_DCIC1_MUX_CTL_IPU1_DI0	(0x0 << 0)
+#define IMX6Q_GPR10_DCIC1_MUX_CTL_IPU1_DI1	(0x1 << 0)
+#define IMX6Q_GPR10_DCIC1_MUX_CTL_IPU2_DI0	(0x2 << 0)
+#define IMX6Q_GPR10_DCIC1_MUX_CTL_IPU2_DI1	(0x3 << 0)
+
+#define IMX6Q_GPR12_ARMP_IPG_CLK_EN		BIT(27)
+#define IMX6Q_GPR12_ARMP_AHB_CLK_EN		BIT(26)
+#define IMX6Q_GPR12_ARMP_ATB_CLK_EN		BIT(25)
+#define IMX6Q_GPR12_ARMP_APB_CLK_EN		BIT(24)
+#define IMX6Q_GPR12_PCIE_CTL_2			BIT(10)
+
+#define IMX6Q_GPR13_SDMA_STOP_REQ		BIT(30)
+#define IMX6Q_GPR13_CAN2_STOP_REQ		BIT(29)
+#define IMX6Q_GPR13_CAN1_STOP_REQ		BIT(28)
+#define IMX6Q_GPR13_ENET_STOP_REQ		BIT(27)
+#define IMX6Q_GPR13_SATA_PHY_8_MASK		(0x7 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_0_5_DB		(0x0 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_1_0_DB		(0x1 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_1_5_DB		(0x2 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_2_0_DB		(0x3 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_2_5_DB		(0x4 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_3_0_DB		(0x5 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_3_5_DB		(0x6 << 24)
+#define IMX6Q_GPR13_SATA_PHY_8_4_0_DB		(0x7 << 24)
+#define IMX6Q_GPR13_SATA_PHY_7_MASK		(0x1f << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA1I		(0x10 << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA1M		(0x10 << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA1X		(0x1a << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA2I		(0x12 << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA2M		(0x12 << 19)
+#define IMX6Q_GPR13_SATA_PHY_7_SATA2X		(0x1a << 19)
+#define IMX6Q_GPR13_SATA_PHY_6_MASK		(0x7 << 16)
+#define IMX6Q_GPR13_SATA_SPEED_MASK		BIT(15)
+#define IMX6Q_GPR13_SATA_SPEED_1P5G		0x0
+#define IMX6Q_GPR13_SATA_SPEED_3P0G		BIT(15)
+#define IMX6Q_GPR13_SATA_PHY_5			BIT(14)
+#define IMX6Q_GPR13_SATA_PHY_4_MASK		(0x7 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_16_16		(0x0 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_14_16		(0x1 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_12_16		(0x2 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_10_16		(0x3 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_9_16		(0x4 << 11)
+#define IMX6Q_GPR13_SATA_PHY_4_8_16		(0x5 << 11)
+#define IMX6Q_GPR13_SATA_PHY_3_MASK		(0xf << 7)
+#define IMX6Q_GPR13_SATA_PHY_3_OFF		0x7
+#define IMX6Q_GPR13_SATA_PHY_2_MASK		(0x1f << 2)
+#define IMX6Q_GPR13_SATA_PHY_2_OFF		0x2
+#define IMX6Q_GPR13_SATA_PHY_1_MASK		(0x3 << 0)
+#define IMX6Q_GPR13_SATA_PHY_1_FAST		(0x0 << 0)
+#define IMX6Q_GPR13_SATA_PHY_1_MED		(0x1 << 0)
+#define IMX6Q_GPR13_SATA_PHY_1_SLOW		(0x2 << 0)
+
+#endif /* __LINUX_IMX6Q_IOMUXC_GPR_H */
diff --git a/include/linux/mfd/tc3589x.h b/include/linux/mfd/tc3589x.h
index 3acb3a8e3af..6b8e1ff4672 100644
--- a/include/linux/mfd/tc3589x.h
+++ b/include/linux/mfd/tc3589x.h
@@ -117,6 +117,7 @@ struct tc3589x {
 	struct mutex lock;
 	struct device *dev;
 	struct i2c_client *i2c;
+	struct irq_domain *domain;
 
 	int irq_base;
 	int num_gpio;
diff --git a/include/linux/mfd/tps65217.h b/include/linux/mfd/tps65217.h
index 7cd83d826ed..290762f9393 100644
--- a/include/linux/mfd/tps65217.h
+++ b/include/linux/mfd/tps65217.h
@@ -213,6 +213,23 @@ enum tps65217_regulator_id {
 /* Number of total regulators available */
 #define TPS65217_NUM_REGULATOR		(TPS65217_NUM_DCDC + TPS65217_NUM_LDO)
 
+enum tps65217_bl_isel {
+	TPS65217_BL_ISET1 = 1,
+	TPS65217_BL_ISET2,
+};
+
+enum tps65217_bl_fdim {
+	TPS65217_BL_FDIM_100HZ,
+	TPS65217_BL_FDIM_200HZ,
+	TPS65217_BL_FDIM_500HZ,
+	TPS65217_BL_FDIM_1000HZ,
+};
+
+struct tps65217_bl_pdata {
+	enum tps65217_bl_isel isel;
+	enum tps65217_bl_fdim fdim;
+};
+
 /**
  * struct tps65217_board - packages regulator init data
  * @tps65217_regulator_data: regulator initialization values
@@ -222,6 +239,7 @@ enum tps65217_regulator_id {
 struct tps65217_board {
 	struct regulator_init_data *tps65217_init_data[TPS65217_NUM_REGULATOR];
 	struct device_node *of_node[TPS65217_NUM_REGULATOR];
+	struct tps65217_bl_pdata *bl_pdata;
 };
 
 /**
diff --git a/include/linux/mfd/tps6586x.h b/include/linux/mfd/tps6586x.h
index 94514710a03..2dd12319495 100644
--- a/include/linux/mfd/tps6586x.h
+++ b/include/linux/mfd/tps6586x.h
@@ -78,6 +78,7 @@ struct tps6586x_platform_data {
 
 	int gpio_base;
 	int irq_base;
+	bool pm_off;
 };
 
 /*
diff --git a/include/linux/mfd/tps65910.h b/include/linux/mfd/tps65910.h
index 9bf8767818b..02e894f3ff4 100644
--- a/include/linux/mfd/tps65910.h
+++ b/include/linux/mfd/tps65910.h
@@ -132,6 +132,16 @@
  *
  */
 
+/* RTC_CTRL_REG bitfields */
+#define TPS65910_RTC_CTRL_STOP_RTC			0x01 /*0=stop, 1=run */
+#define TPS65910_RTC_CTRL_GET_TIME			0x40
+
+/* RTC_STATUS_REG bitfields */
+#define TPS65910_RTC_STATUS_ALARM               0x40
+
+/* RTC_INTERRUPTS_REG bitfields */
+#define TPS65910_RTC_INTERRUPTS_EVERY           0x03
+#define TPS65910_RTC_INTERRUPTS_IT_ALARM        0x08
 
 /*Register BCK1  (0x80) register.RegisterDescription */
 #define BCK1_BCKUP_MASK					0xFF
@@ -366,6 +376,8 @@
 
 
 /*Register DEVCTRL  (0x80) register.RegisterDescription */
+#define DEVCTRL_PWR_OFF_MASK				0x80
+#define DEVCTRL_PWR_OFF_SHIFT				7
 #define DEVCTRL_RTC_PWDN_MASK				0x40
 #define DEVCTRL_RTC_PWDN_SHIFT				6
 #define DEVCTRL_CK32K_CTRL_MASK				0x20
@@ -809,6 +821,7 @@ struct tps65910_board {
 	int vmbch2_threshold;
 	bool en_ck32k_xtal;
 	bool en_dev_slp;
+	bool pm_off;
 	struct tps65910_sleep_keepon_data *slp_keepon;
 	bool en_gpio_sleep[TPS6591X_MAX_NUM_GPIO];
 	unsigned long regulator_ext_sleep_control[TPS65910_NUM_REGS];
diff --git a/include/linux/mfd/twl6040.h b/include/linux/mfd/twl6040.h
index ba43d4806b8..a8eff4ad9be 100644
--- a/include/linux/mfd/twl6040.h
+++ b/include/linux/mfd/twl6040.h
@@ -143,7 +143,7 @@
 
 #define TWL6040_GPO1			0x01
 #define TWL6040_GPO2			0x02
-#define TWL6040_GPO3			0x03
+#define TWL6040_GPO3			0x04
 
 /* ACCCTL (0x2D) fields */
 
@@ -158,7 +158,7 @@
 #define TWL6040_VIBROCDET		0x20
 #define TWL6040_TSHUTDET                0x40
 
-#define TWL6040_CELLS			2
+#define TWL6040_CELLS			3
 
 #define TWL6040_REV_ES1_0		0x00
 #define TWL6040_REV_ES1_1		0x01 /* Rev ES1.1 and ES1.2 */
@@ -176,6 +176,8 @@
 #define TWL6040_SYSCLK_SEL_LPPLL	0
 #define TWL6040_SYSCLK_SEL_HPPLL	1
 
+#define TWL6040_GPO_MAX	3
+
 struct twl6040_codec_data {
 	u16 hs_left_step;
 	u16 hs_right_step;
@@ -192,11 +194,16 @@ struct twl6040_vibra_data {
 	int vddvibr_uV;			/* VDDVIBR volt, set 0 for fixed reg */
 };
 
+struct twl6040_gpo_data {
+	int gpio_base;
+};
+
 struct twl6040_platform_data {
 	int audpwron_gpio;	/* audio power-on gpio */
 
 	struct twl6040_codec_data *codec;
 	struct twl6040_vibra_data *vibra;
+	struct twl6040_gpo_data *gpo;
 };
 
 struct regmap;
diff --git a/include/linux/mtd/partitions.h b/include/linux/mtd/partitions.h
index 2475228c115..1f8d24bdafd 100644
--- a/include/linux/mtd/partitions.h
+++ b/include/linux/mtd/partitions.h
@@ -79,9 +79,10 @@ struct mtd_part_parser {
 extern int register_mtd_parser(struct mtd_part_parser *parser);
 extern int deregister_mtd_parser(struct mtd_part_parser *parser);
 
-int mtd_is_partition(struct mtd_info *mtd);
+int mtd_is_partition(const struct mtd_info *mtd);
 int mtd_add_partition(struct mtd_info *master, char *name,
 		      long long offset, long long length);
 int mtd_del_partition(struct mtd_info *master, int partno);
+uint64_t mtd_get_device_size(const struct mtd_info *mtd);
 
 #endif
diff --git a/include/linux/nbd.h b/include/linux/nbd.h
index d146ca10c0f..5c86e2b33e2 100644
--- a/include/linux/nbd.h
+++ b/include/linux/nbd.h
@@ -27,13 +27,22 @@
 #define NBD_SET_SIZE_BLOCKS	_IO( 0xab, 7 )
 #define NBD_DISCONNECT  _IO( 0xab, 8 )
 #define NBD_SET_TIMEOUT _IO( 0xab, 9 )
+#define NBD_SET_FLAGS   _IO( 0xab, 10)
 
 enum {
 	NBD_CMD_READ = 0,
 	NBD_CMD_WRITE = 1,
-	NBD_CMD_DISC = 2
+	NBD_CMD_DISC = 2,
+	/* there is a gap here to match userspace */
+	NBD_CMD_TRIM = 4
 };
 
+/* values for flags field */
+#define NBD_FLAG_HAS_FLAGS    (1 << 0) /* nbd-server supports flags */
+#define NBD_FLAG_READ_ONLY    (1 << 1) /* device is read-only */
+/* there is a gap here to match userspace */
+#define NBD_FLAG_SEND_TRIM    (1 << 5) /* send trim/discard */
+
 #define nbd_cmd(req) ((req)->cmd[0])
 
 /* userspace doesn't need the nbd_device structure */
@@ -42,10 +51,6 @@ enum {
 #include <linux/wait.h>
 #include <linux/mutex.h>
 
-/* values for flags field */
-#define NBD_READ_ONLY 0x0001
-#define NBD_WRITE_NOCHK 0x0002
-
 struct request;
 
 struct nbd_device {
diff --git a/include/linux/net.h b/include/linux/net.h
index 99276c3dc89..6ab31cabef7 100644
--- a/include/linux/net.h
+++ b/include/linux/net.h
@@ -65,6 +65,7 @@ typedef enum {
 struct poll_table_struct;
 struct pipe_inode_info;
 struct inode;
+struct file;
 struct net;
 
 #define SOCK_ASYNC_NOSPACE	0
@@ -246,7 +247,7 @@ extern int   	     sock_sendmsg(struct socket *sock, struct msghdr *msg,
 				  size_t len);
 extern int	     sock_recvmsg(struct socket *sock, struct msghdr *msg,
 				  size_t size, int flags);
-extern int 	     sock_map_fd(struct socket *sock, int flags);
+extern struct file  *sock_alloc_file(struct socket *sock, int flags, const char *dname);
 extern struct socket *sockfd_lookup(int fd, int *err);
 extern struct socket *sock_from_file(struct file *file, int *err);
 #define		     sockfd_put(sock) fput(sock->file)
diff --git a/include/linux/netfilter/nf_conntrack_h323_asn1.h b/include/linux/netfilter/nf_conntrack_h323_asn1.h
index 8dab5968fc7..3176a277eed 100644
--- a/include/linux/netfilter/nf_conntrack_h323_asn1.h
+++ b/include/linux/netfilter/nf_conntrack_h323_asn1.h
@@ -40,7 +40,7 @@
 /*****************************************************************************
  * H.323 Types
  ****************************************************************************/
-#include "nf_conntrack_h323_types.h"
+#include <linux/netfilter/nf_conntrack_h323_types.h>
 
 typedef struct {
 	enum {
diff --git a/include/linux/nx842.h b/include/linux/nx842.h
new file mode 100644
index 00000000000..a4d324c6406
--- /dev/null
+++ b/include/linux/nx842.h
@@ -0,0 +1,11 @@
+#ifndef __NX842_H__
+#define __NX842_H__
+
+int nx842_get_workmem_size(void);
+int nx842_get_workmem_size_aligned(void);
+int nx842_compress(const unsigned char *in, unsigned int in_len,
+		unsigned char *out, unsigned int *out_len, void *wrkmem);
+int nx842_decompress(const unsigned char *in, unsigned int in_len,
+		unsigned char *out, unsigned int *out_len, void *wrkmem);
+
+#endif
diff --git a/include/linux/of.h b/include/linux/of.h
index 1b1163225f3..72843b72a2b 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -193,6 +193,8 @@ extern struct device_node *of_get_next_child(const struct device_node *node,
 extern struct device_node *of_get_next_available_child(
 	const struct device_node *node, struct device_node *prev);
 
+extern struct device_node *of_get_child_by_name(const struct device_node *node,
+					const char *name);
 #define for_each_child_of_node(parent, child) \
 	for (child = of_get_next_child(parent, NULL); child != NULL; \
 	     child = of_get_next_child(parent, child))
@@ -315,6 +317,12 @@ static inline const char* of_node_full_name(struct device_node *np)
 	return "<no-node>";
 }
 
+static inline struct device_node *of_find_node_by_name(struct device_node *from,
+	const char *name)
+{
+	return NULL;
+}
+
 static inline bool of_have_populated_dt(void)
 {
 	return false;
diff --git a/include/linux/of_address.h b/include/linux/of_address.h
index 01b925ad8d7..c3cdc1025c3 100644
--- a/include/linux/of_address.h
+++ b/include/linux/of_address.h
@@ -6,6 +6,7 @@
 
 #ifdef CONFIG_OF_ADDRESS
 extern u64 of_translate_address(struct device_node *np, const __be32 *addr);
+extern bool of_can_translate_address(struct device_node *dev);
 extern int of_address_to_resource(struct device_node *dev, int index,
 				  struct resource *r);
 extern struct device_node *of_find_matching_node_by_address(
diff --git a/include/linux/opp.h b/include/linux/opp.h
index 2a4e5faee90..214e0ebcb84 100644
--- a/include/linux/opp.h
+++ b/include/linux/opp.h
@@ -48,6 +48,14 @@ int opp_disable(struct device *dev, unsigned long freq);
 
 struct srcu_notifier_head *opp_get_notifier(struct device *dev);
 
+#ifdef CONFIG_OF
+int of_init_opp_table(struct device *dev);
+#else
+static inline int of_init_opp_table(struct device *dev)
+{
+	return -EINVAL;
+}
+#endif /* CONFIG_OF */
 #else
 static inline unsigned long opp_get_voltage(struct opp *opp)
 {
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index 8d3c4271938..33880f6f4e5 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -753,6 +753,7 @@
 #define PCI_DEVICE_ID_HP_CISSD		0x3238
 #define PCI_DEVICE_ID_HP_CISSE		0x323a
 #define PCI_DEVICE_ID_HP_CISSF		0x323b
+#define PCI_DEVICE_ID_HP_CISSH		0x323c
 #define PCI_DEVICE_ID_HP_ZX2_IOC	0x4031
 
 #define PCI_VENDOR_ID_PCTECH		0x1042
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 2b9f82c037c..cc88172c7d9 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -107,7 +107,7 @@ enum pcpu_fc {
 
 	PCPU_FC_NR,
 };
-extern const char *pcpu_fc_names[PCPU_FC_NR];
+extern const char * const pcpu_fc_names[PCPU_FC_NR];
 
 extern enum pcpu_fc pcpu_chosen_fc;
 
diff --git a/include/linux/pinctrl/consumer.h b/include/linux/pinctrl/consumer.h
index d2816454c26..4aad3cea69a 100644
--- a/include/linux/pinctrl/consumer.h
+++ b/include/linux/pinctrl/consumer.h
@@ -15,7 +15,7 @@
 #include <linux/err.h>
 #include <linux/list.h>
 #include <linux/seq_file.h>
-#include "pinctrl-state.h"
+#include <linux/pinctrl/pinctrl-state.h>
 
 /* This struct is private to the core and should be regarded as a cookie */
 struct pinctrl;
diff --git a/include/linux/pinctrl/machine.h b/include/linux/pinctrl/machine.h
index 7d22ab00343..e5b1716f98c 100644
--- a/include/linux/pinctrl/machine.h
+++ b/include/linux/pinctrl/machine.h
@@ -14,7 +14,7 @@
 
 #include <linux/bug.h>
 
-#include "pinctrl-state.h"
+#include <linux/pinctrl/pinctrl-state.h>
 
 enum pinctrl_map_type {
 	PIN_MAP_TYPE_INVALID,
diff --git a/include/linux/pinctrl/pinctrl.h b/include/linux/pinctrl/pinctrl.h
index 69393a66253..7d087f03e91 100644
--- a/include/linux/pinctrl/pinctrl.h
+++ b/include/linux/pinctrl/pinctrl.h
@@ -17,7 +17,7 @@
 #include <linux/radix-tree.h>
 #include <linux/list.h>
 #include <linux/seq_file.h>
-#include "pinctrl-state.h"
+#include <linux/pinctrl/pinctrl-state.h>
 
 struct device;
 struct pinctrl_dev;
diff --git a/include/linux/pinctrl/pinmux.h b/include/linux/pinctrl/pinmux.h
index 1818dcbdd9a..c15395031cb 100644
--- a/include/linux/pinctrl/pinmux.h
+++ b/include/linux/pinctrl/pinmux.h
@@ -14,7 +14,7 @@
 
 #include <linux/list.h>
 #include <linux/seq_file.h>
-#include "pinctrl.h"
+#include <linux/pinctrl/pinctrl.h>
 
 #ifdef CONFIG_PINMUX
 
diff --git a/include/linux/platform_data/lm3630_bl.h b/include/linux/platform_data/lm3630_bl.h
new file mode 100644
index 00000000000..9176dd3f2d6
--- /dev/null
+++ b/include/linux/platform_data/lm3630_bl.h
@@ -0,0 +1,57 @@
+/*
+* Simple driver for Texas Instruments LM3630 LED Flash driver chip
+* Copyright (C) 2012 Texas Instruments
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+*/
+
+#ifndef __LINUX_LM3630_H
+#define __LINUX_LM3630_H
+
+#define LM3630_NAME "lm3630_bl"
+
+enum lm3630_pwm_ctrl {
+	PWM_CTRL_DISABLE = 0,
+	PWM_CTRL_BANK_A,
+	PWM_CTRL_BANK_B,
+	PWM_CTRL_BANK_ALL,
+};
+
+enum lm3630_pwm_active {
+	PWM_ACTIVE_HIGH = 0,
+	PWM_ACTIVE_LOW,
+};
+
+enum lm3630_bank_a_ctrl {
+	BANK_A_CTRL_DISABLE = 0x0,
+	BANK_A_CTRL_LED1 = 0x4,
+	BANK_A_CTRL_LED2 = 0x1,
+	BANK_A_CTRL_ALL = 0x5,
+};
+
+enum lm3630_bank_b_ctrl {
+	BANK_B_CTRL_DISABLE = 0,
+	BANK_B_CTRL_LED2,
+};
+
+struct lm3630_platform_data {
+
+	/* maximum brightness */
+	int max_brt_led1;
+	int max_brt_led2;
+
+	/* initial on brightness */
+	int init_brt_led1;
+	int init_brt_led2;
+	enum lm3630_pwm_ctrl pwm_ctrl;
+	enum lm3630_pwm_active pwm_active;
+	enum lm3630_bank_a_ctrl bank_a_ctrl;
+	enum lm3630_bank_b_ctrl bank_b_ctrl;
+	unsigned int pwm_period;
+	void (*pwm_set_intensity) (int brightness, int max_brightness);
+};
+
+#endif /* __LINUX_LM3630_H */
diff --git a/include/linux/platform_data/lm3639_bl.h b/include/linux/platform_data/lm3639_bl.h
new file mode 100644
index 00000000000..5234cd5ed16
--- /dev/null
+++ b/include/linux/platform_data/lm3639_bl.h
@@ -0,0 +1,69 @@
+/*
+* Simple driver for Texas Instruments LM3630 LED Flash driver chip
+* Copyright (C) 2012 Texas Instruments
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License version 2 as
+* published by the Free Software Foundation.
+*
+*/
+
+#ifndef __LINUX_LM3639_H
+#define __LINUX_LM3639_H
+
+#define LM3639_NAME "lm3639_bl"
+
+enum lm3639_pwm {
+	LM3639_PWM_DISABLE = 0x00,
+	LM3639_PWM_EN_ACTLOW = 0x48,
+	LM3639_PWM_EN_ACTHIGH = 0x40,
+};
+
+enum lm3639_strobe {
+	LM3639_STROBE_DISABLE = 0x00,
+	LM3639_STROBE_EN_ACTLOW = 0x10,
+	LM3639_STROBE_EN_ACTHIGH = 0x30,
+};
+
+enum lm3639_txpin {
+	LM3639_TXPIN_DISABLE = 0x00,
+	LM3639_TXPIN_EN_ACTLOW = 0x04,
+	LM3639_TXPIN_EN_ACTHIGH = 0x0C,
+};
+
+enum lm3639_fleds {
+	LM3639_FLED_DIASBLE_ALL = 0x00,
+	LM3639_FLED_EN_1 = 0x40,
+	LM3639_FLED_EN_2 = 0x20,
+	LM3639_FLED_EN_ALL = 0x60,
+};
+
+enum lm3639_bleds {
+	LM3639_BLED_DIASBLE_ALL = 0x00,
+	LM3639_BLED_EN_1 = 0x10,
+	LM3639_BLED_EN_2 = 0x08,
+	LM3639_BLED_EN_ALL = 0x18,
+};
+enum lm3639_bled_mode {
+	LM3639_BLED_MODE_EXPONETIAL = 0x00,
+	LM3639_BLED_MODE_LINEAR = 0x10,
+};
+
+struct lm3639_platform_data {
+	unsigned int max_brt_led;
+	unsigned int init_brt_led;
+
+	/* input pins */
+	enum lm3639_pwm pin_pwm;
+	enum lm3639_strobe pin_strobe;
+	enum lm3639_txpin pin_tx;
+
+	/* output pins */
+	enum lm3639_fleds fled_pins;
+	enum lm3639_bleds bled_pins;
+	enum lm3639_bled_mode bled_mode;
+
+	void (*pwm_set_intensity) (int brightness, int max_brightness);
+	int (*pwm_get_intensity) (void);
+};
+#endif /* __LINUX_LM3639_H */
diff --git a/include/linux/platform_data/lp855x.h b/include/linux/platform_data/lp855x.h
index cc76f1f18f1..761f3175236 100644
--- a/include/linux/platform_data/lp855x.h
+++ b/include/linux/platform_data/lp855x.h
@@ -46,6 +46,8 @@
 #define LP8556_I2C_CONFIG	((ENABLE_BL << BL_CTL_SHFT) | \
 				(LP8556_I2C_ONLY << BRT_MODE_SHFT))
 #define LP8556_COMB2_CONFIG	(LP8556_COMBINED2 << BRT_MODE_SHFT)
+#define LP8556_FAST_CONFIG	BIT(7) /* use it if EPROMs should be maintained
+					  when exiting the low power mode */
 
 enum lp855x_chip_id {
 	LP8550,
diff --git a/include/linux/platform_data/remoteproc-omap.h b/include/linux/platform_data/remoteproc-omap.h
index b10eac89e2e..3c1c6444ec4 100644
--- a/include/linux/platform_data/remoteproc-omap.h
+++ b/include/linux/platform_data/remoteproc-omap.h
@@ -30,6 +30,7 @@ struct platform_device;
  * @ops: start/stop rproc handlers
  * @device_enable: omap-specific handler for enabling a device
  * @device_shutdown: omap-specific handler for shutting down a device
+ * @set_bootaddr: omap-specific handler for setting the rproc boot address
  */
 struct omap_rproc_pdata {
 	const char *name;
@@ -40,6 +41,7 @@ struct omap_rproc_pdata {
 	const struct rproc_ops *ops;
 	int (*device_enable) (struct platform_device *pdev);
 	int (*device_shutdown) (struct platform_device *pdev);
+	void(*set_bootaddr)(u32);
 };
 
 #if defined(CONFIG_OMAP_REMOTEPROC) || defined(CONFIG_OMAP_REMOTEPROC_MODULE)
diff --git a/include/linux/platform_data/shmob_drm.h b/include/linux/platform_data/shmob_drm.h
new file mode 100644
index 00000000000..7c686d335c1
--- /dev/null
+++ b/include/linux/platform_data/shmob_drm.h
@@ -0,0 +1,99 @@
+/*
+ * shmob_drm.h  --  SH Mobile DRM driver
+ *
+ * Copyright (C) 2012 Renesas Corporation
+ *
+ * Laurent Pinchart (laurent.pinchart@ideasonboard.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __SHMOB_DRM_H__
+#define __SHMOB_DRM_H__
+
+#include <linux/kernel.h>
+
+#include <drm/drm_mode.h>
+
+struct sh_mobile_meram_cfg;
+struct sh_mobile_meram_info;
+
+enum shmob_drm_clk_source {
+	SHMOB_DRM_CLK_BUS,
+	SHMOB_DRM_CLK_PERIPHERAL,
+	SHMOB_DRM_CLK_EXTERNAL,
+};
+
+enum shmob_drm_interface {
+	SHMOB_DRM_IFACE_RGB8,		/* 24bpp, 8:8:8 */
+	SHMOB_DRM_IFACE_RGB9,		/* 18bpp, 9:9 */
+	SHMOB_DRM_IFACE_RGB12A,		/* 24bpp, 12:12 */
+	SHMOB_DRM_IFACE_RGB12B,		/* 12bpp */
+	SHMOB_DRM_IFACE_RGB16,		/* 16bpp */
+	SHMOB_DRM_IFACE_RGB18,		/* 18bpp */
+	SHMOB_DRM_IFACE_RGB24,		/* 24bpp */
+	SHMOB_DRM_IFACE_YUV422,		/* 16bpp */
+	SHMOB_DRM_IFACE_SYS8A,		/* 24bpp, 8:8:8 */
+	SHMOB_DRM_IFACE_SYS8B,		/* 18bpp, 8:8:2 */
+	SHMOB_DRM_IFACE_SYS8C,		/* 18bpp, 2:8:8 */
+	SHMOB_DRM_IFACE_SYS8D,		/* 16bpp, 8:8 */
+	SHMOB_DRM_IFACE_SYS9,		/* 18bpp, 9:9 */
+	SHMOB_DRM_IFACE_SYS12,		/* 24bpp, 12:12 */
+	SHMOB_DRM_IFACE_SYS16A,		/* 16bpp */
+	SHMOB_DRM_IFACE_SYS16B,		/* 18bpp, 16:2 */
+	SHMOB_DRM_IFACE_SYS16C,		/* 18bpp, 2:16 */
+	SHMOB_DRM_IFACE_SYS18,		/* 18bpp */
+	SHMOB_DRM_IFACE_SYS24,		/* 24bpp */
+};
+
+struct shmob_drm_backlight_data {
+	const char *name;
+	int max_brightness;
+	int (*get_brightness)(void);
+	int (*set_brightness)(int brightness);
+};
+
+struct shmob_drm_panel_data {
+	unsigned int width_mm;		/* Panel width in mm */
+	unsigned int height_mm;		/* Panel height in mm */
+	struct drm_mode_modeinfo mode;
+};
+
+struct shmob_drm_sys_interface_data {
+	unsigned int read_latch:6;
+	unsigned int read_setup:8;
+	unsigned int read_cycle:8;
+	unsigned int read_strobe:8;
+	unsigned int write_setup:8;
+	unsigned int write_cycle:8;
+	unsigned int write_strobe:8;
+	unsigned int cs_setup:3;
+	unsigned int vsync_active_high:1;
+	unsigned int vsync_dir_input:1;
+};
+
+#define SHMOB_DRM_IFACE_FL_DWPOL (1 << 0) /* Rising edge dot clock data latch */
+#define SHMOB_DRM_IFACE_FL_DIPOL (1 << 1) /* Active low display enable */
+#define SHMOB_DRM_IFACE_FL_DAPOL (1 << 2) /* Active low display data */
+#define SHMOB_DRM_IFACE_FL_HSCNT (1 << 3) /* Disable HSYNC during VBLANK */
+#define SHMOB_DRM_IFACE_FL_DWCNT (1 << 4) /* Disable dotclock during blanking */
+
+struct shmob_drm_interface_data {
+	enum shmob_drm_interface interface;
+	struct shmob_drm_sys_interface_data sys;
+	unsigned int clk_div;
+	unsigned int flags;
+};
+
+struct shmob_drm_platform_data {
+	enum shmob_drm_clk_source clk_source;
+	struct shmob_drm_interface_data iface;
+	struct shmob_drm_panel_data panel;
+	struct shmob_drm_backlight_data backlight;
+	const struct sh_mobile_meram_cfg *meram;
+};
+
+#endif /* __SHMOB_DRM_H__ */
diff --git a/include/linux/pm.h b/include/linux/pm.h
index 88f034a23f2..007e687c4f6 100644
--- a/include/linux/pm.h
+++ b/include/linux/pm.h
@@ -510,12 +510,14 @@ struct dev_pm_info {
 	bool			is_prepared:1;	/* Owned by the PM core */
 	bool			is_suspended:1;	/* Ditto */
 	bool			ignore_children:1;
+	bool			early_init:1;	/* Owned by the PM core */
 	spinlock_t		lock;
 #ifdef CONFIG_PM_SLEEP
 	struct list_head	entry;
 	struct completion	completion;
 	struct wakeup_source	*wakeup;
 	bool			wakeup_path:1;
+	bool			syscore:1;
 #else
 	unsigned int		should_wakeup:1;
 #endif
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index a7d6172922d..7c1d252b20c 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -114,7 +114,6 @@ struct generic_pm_domain_data {
 	struct mutex lock;
 	unsigned int refcount;
 	bool need_restore;
-	bool always_on;
 };
 
 #ifdef CONFIG_PM_GENERIC_DOMAINS
@@ -139,36 +138,32 @@ extern int __pm_genpd_of_add_device(struct device_node *genpd_node,
 				    struct device *dev,
 				    struct gpd_timing_data *td);
 
-static inline int pm_genpd_add_device(struct generic_pm_domain *genpd,
-				      struct device *dev)
-{
-	return __pm_genpd_add_device(genpd, dev, NULL);
-}
-
-static inline int pm_genpd_of_add_device(struct device_node *genpd_node,
-					 struct device *dev)
-{
-	return __pm_genpd_of_add_device(genpd_node, dev, NULL);
-}
+extern int __pm_genpd_name_add_device(const char *domain_name,
+				      struct device *dev,
+				      struct gpd_timing_data *td);
 
 extern int pm_genpd_remove_device(struct generic_pm_domain *genpd,
 				  struct device *dev);
-extern void pm_genpd_dev_always_on(struct device *dev, bool val);
 extern void pm_genpd_dev_need_restore(struct device *dev, bool val);
 extern int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 				  struct generic_pm_domain *new_subdomain);
+extern int pm_genpd_add_subdomain_names(const char *master_name,
+					const char *subdomain_name);
 extern int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 				     struct generic_pm_domain *target);
 extern int pm_genpd_add_callbacks(struct device *dev,
 				  struct gpd_dev_ops *ops,
 				  struct gpd_timing_data *td);
 extern int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td);
-extern int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state);
-extern int genpd_detach_cpuidle(struct generic_pm_domain *genpd);
+extern int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state);
+extern int pm_genpd_name_attach_cpuidle(const char *name, int state);
+extern int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd);
+extern int pm_genpd_name_detach_cpuidle(const char *name);
 extern void pm_genpd_init(struct generic_pm_domain *genpd,
 			  struct dev_power_governor *gov, bool is_off);
 
 extern int pm_genpd_poweron(struct generic_pm_domain *genpd);
+extern int pm_genpd_name_poweron(const char *domain_name);
 
 extern bool default_stop_ok(struct device *dev);
 
@@ -189,8 +184,15 @@ static inline int __pm_genpd_add_device(struct generic_pm_domain *genpd,
 {
 	return -ENOSYS;
 }
-static inline int pm_genpd_add_device(struct generic_pm_domain *genpd,
-				      struct device *dev)
+static inline int __pm_genpd_of_add_device(struct device_node *genpd_node,
+					   struct device *dev,
+					   struct gpd_timing_data *td)
+{
+	return -ENOSYS;
+}
+static inline int __pm_genpd_name_add_device(const char *domain_name,
+					     struct device *dev,
+					     struct gpd_timing_data *td)
 {
 	return -ENOSYS;
 }
@@ -199,13 +201,17 @@ static inline int pm_genpd_remove_device(struct generic_pm_domain *genpd,
 {
 	return -ENOSYS;
 }
-static inline void pm_genpd_dev_always_on(struct device *dev, bool val) {}
 static inline void pm_genpd_dev_need_restore(struct device *dev, bool val) {}
 static inline int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
 					 struct generic_pm_domain *new_sd)
 {
 	return -ENOSYS;
 }
+static inline int pm_genpd_add_subdomain_names(const char *master_name,
+					       const char *subdomain_name)
+{
+	return -ENOSYS;
+}
 static inline int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
 					    struct generic_pm_domain *target)
 {
@@ -221,11 +227,19 @@ static inline int __pm_genpd_remove_callbacks(struct device *dev, bool clear_td)
 {
 	return -ENOSYS;
 }
-static inline int genpd_attach_cpuidle(struct generic_pm_domain *genpd, int st)
+static inline int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int st)
 {
 	return -ENOSYS;
 }
-static inline int genpd_detach_cpuidle(struct generic_pm_domain *genpd)
+static inline int pm_genpd_name_attach_cpuidle(const char *name, int state)
+{
+	return -ENOSYS;
+}
+static inline int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd)
+{
+	return -ENOSYS;
+}
+static inline int pm_genpd_name_detach_cpuidle(const char *name)
 {
 	return -ENOSYS;
 }
@@ -237,6 +251,10 @@ static inline int pm_genpd_poweron(struct generic_pm_domain *genpd)
 {
 	return -ENOSYS;
 }
+static inline int pm_genpd_name_poweron(const char *domain_name)
+{
+	return -ENOSYS;
+}
 static inline bool default_stop_ok(struct device *dev)
 {
 	return false;
@@ -245,6 +263,24 @@ static inline bool default_stop_ok(struct device *dev)
 #define pm_domain_always_on_gov NULL
 #endif
 
+static inline int pm_genpd_add_device(struct generic_pm_domain *genpd,
+				      struct device *dev)
+{
+	return __pm_genpd_add_device(genpd, dev, NULL);
+}
+
+static inline int pm_genpd_of_add_device(struct device_node *genpd_node,
+					 struct device *dev)
+{
+	return __pm_genpd_of_add_device(genpd_node, dev, NULL);
+}
+
+static inline int pm_genpd_name_add_device(const char *domain_name,
+					   struct device *dev)
+{
+	return __pm_genpd_name_add_device(domain_name, dev, NULL);
+}
+
 static inline int pm_genpd_remove_callbacks(struct device *dev)
 {
 	return __pm_genpd_remove_callbacks(dev, true);
@@ -258,4 +294,20 @@ static inline void genpd_queue_power_off_work(struct generic_pm_domain *gpd) {}
 static inline void pm_genpd_poweroff_unused(void) {}
 #endif
 
+#ifdef CONFIG_PM_GENERIC_DOMAINS_SLEEP
+extern void pm_genpd_syscore_switch(struct device *dev, bool suspend);
+#else
+static inline void pm_genpd_syscore_switch(struct device *dev, bool suspend) {}
+#endif
+
+static inline void pm_genpd_syscore_poweroff(struct device *dev)
+{
+	pm_genpd_syscore_switch(dev, true);
+}
+
+static inline void pm_genpd_syscore_poweron(struct device *dev)
+{
+	pm_genpd_syscore_switch(dev, false);
+}
+
 #endif /* _LINUX_PM_DOMAIN_H */
diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h
index 597e4fdb97f..3db698aee34 100644
--- a/include/linux/ptrace.h
+++ b/include/linux/ptrace.h
@@ -130,8 +130,6 @@ extern void exit_ptrace(struct task_struct *tracer);
 #define PTRACE_MODE_READ	0x01
 #define PTRACE_MODE_ATTACH	0x02
 #define PTRACE_MODE_NOAUDIT	0x04
-/* Returns 0 on success, -errno on denial. */
-extern int __ptrace_may_access(struct task_struct *task, unsigned int mode);
 /* Returns true on success, false on denial. */
 extern bool ptrace_may_access(struct task_struct *task, unsigned int mode);
 
diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h
index 131b53957b9..faf33324c78 100644
--- a/include/linux/remoteproc.h
+++ b/include/linux/remoteproc.h
@@ -361,6 +361,19 @@ enum rproc_state {
 };
 
 /**
+ * enum rproc_crash_type - remote processor crash types
+ * @RPROC_MMUFAULT:	iommu fault
+ *
+ * Each element of the enum is used as an array index. So that, the value of
+ * the elements should be always something sane.
+ *
+ * Feel free to add more types when needed.
+ */
+enum rproc_crash_type {
+	RPROC_MMUFAULT,
+};
+
+/**
  * struct rproc - represents a physical remote processor device
  * @node: klist node of this rproc object
  * @domain: iommu domain
@@ -383,6 +396,11 @@ enum rproc_state {
  * @rvdevs: list of remote virtio devices
  * @notifyids: idr for dynamically assigning rproc-wide unique notify ids
  * @index: index of this rproc device
+ * @crash_handler: workqueue for handling a crash
+ * @crash_cnt: crash counter
+ * @crash_comp: completion used to sync crash handler and the rproc reload
+ * @recovery_disabled: flag that state if recovery was disabled
+ * @max_notifyid: largest allocated notify id.
  */
 struct rproc {
 	struct klist_node node;
@@ -406,6 +424,11 @@ struct rproc {
 	struct list_head rvdevs;
 	struct idr notifyids;
 	int index;
+	struct work_struct crash_handler;
+	unsigned crash_cnt;
+	struct completion crash_comp;
+	bool recovery_disabled;
+	int max_notifyid;
 };
 
 /* we currently support only two vrings per rvdev */
@@ -460,6 +483,7 @@ int rproc_del(struct rproc *rproc);
 
 int rproc_boot(struct rproc *rproc);
 void rproc_shutdown(struct rproc *rproc);
+void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type);
 
 static inline struct rproc_vdev *vdev_to_rvdev(struct virtio_device *vdev)
 {
diff --git a/include/linux/rio.h b/include/linux/rio.h
index a90ebadd9da..d2dff22cf68 100644
--- a/include/linux/rio.h
+++ b/include/linux/rio.h
@@ -30,6 +30,7 @@
 #define RIO_MAX_MPORTS		8
 #define RIO_MAX_MPORT_RESOURCES	16
 #define RIO_MAX_DEV_RESOURCES	16
+#define RIO_MAX_MPORT_NAME	40
 
 #define RIO_GLOBAL_TABLE	0xff	/* Indicates access of a switch's
 					   global routing table if it
@@ -235,6 +236,7 @@ enum rio_phy_type {
  * @phys_efptr: RIO port extended features pointer
  * @name: Port name string
  * @priv: Master port private data
+ * @dma: DMA device associated with mport
  */
 struct rio_mport {
 	struct list_head dbells;	/* list of doorbell events */
@@ -255,13 +257,21 @@ struct rio_mport {
 				 */
 	enum rio_phy_type phy_type;	/* RapidIO phy type */
 	u32 phys_efptr;
-	unsigned char name[40];
+	unsigned char name[RIO_MAX_MPORT_NAME];
 	void *priv;		/* Master port private data */
 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 	struct dma_device	dma;
 #endif
 };
 
+struct rio_id_table {
+	u16 start;	/* logical minimal id */
+	u16 next;	/* hint for find */
+	u32 max;	/* max number of IDs in table */
+	spinlock_t lock;
+	unsigned long *table;
+};
+
 /**
  * struct rio_net - RIO network info
  * @node: Node in global list of RIO networks
@@ -273,9 +283,11 @@ struct rio_mport {
 struct rio_net {
 	struct list_head node;	/* node in list of networks */
 	struct list_head devices;	/* list of devices in this net */
+	struct list_head switches;	/* list of switches in this net */
 	struct list_head mports;	/* list of ports accessing net */
 	struct rio_mport *hport;	/* primary port for accessing net */
 	unsigned char id;	/* RIO network ID */
+	struct rio_id_table destid_table;  /* destID allocation table */
 };
 
 /* Definitions used by switch sysfs initialization callback */
@@ -299,6 +311,8 @@ struct rio_net {
  * @add_outb_message: Callback to add a message to an outbound mailbox queue.
  * @add_inb_buffer: Callback to	add a buffer to an inbound mailbox queue.
  * @get_inb_message: Callback to get a message from an inbound mailbox queue.
+ * @map_inb: Callback to map RapidIO address region into local memory space.
+ * @unmap_inb: Callback to unmap RapidIO address region mapped with map_inb().
  */
 struct rio_ops {
 	int (*lcread) (struct rio_mport *mport, int index, u32 offset, int len,
@@ -321,6 +335,9 @@ struct rio_ops {
 				 int mbox, void *buffer, size_t len);
 	int (*add_inb_buffer)(struct rio_mport *mport, int mbox, void *buf);
 	void *(*get_inb_message)(struct rio_mport *mport, int mbox);
+	int (*map_inb)(struct rio_mport *mport, dma_addr_t lstart,
+			u64 rstart, u32 size, u32 flags);
+	void (*unmap_inb)(struct rio_mport *mport, dma_addr_t lstart);
 };
 
 #define RIO_RESOURCE_MEM	0x00000100
@@ -403,7 +420,7 @@ union rio_pw_msg {
 
 #ifdef CONFIG_RAPIDIO_DMA_ENGINE
 
-/**
+/*
  * enum rio_write_type - RIO write transaction types used in DMA transfers
  *
  * Note: RapidIO specification defines write (NWRITE) and
diff --git a/include/linux/rio_drv.h b/include/linux/rio_drv.h
index 31ad146be31..b75c05920ab 100644
--- a/include/linux/rio_drv.h
+++ b/include/linux/rio_drv.h
@@ -365,6 +365,11 @@ void rio_release_regions(struct rio_dev *);
 int rio_request_region(struct rio_dev *, int, char *);
 void rio_release_region(struct rio_dev *, int);
 
+/* Memory mapping functions */
+extern int rio_map_inb_region(struct rio_mport *mport, dma_addr_t local,
+			u64 rbase, u32 size, u32 rflags);
+extern void rio_unmap_inb_region(struct rio_mport *mport, dma_addr_t lstart);
+
 /* Port-Write management */
 extern int rio_request_inb_pwrite(struct rio_dev *,
 			int (*)(struct rio_dev *, union rio_pw_msg*, int));
diff --git a/include/linux/rtc-ds2404.h b/include/linux/rtc-ds2404.h
new file mode 100644
index 00000000000..22c53825528
--- /dev/null
+++ b/include/linux/rtc-ds2404.h
@@ -0,0 +1,20 @@
+/*
+ * ds2404.h - platform data structure for the DS2404 RTC.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012 Sven Schnelle <svens@stackframe.org>
+ */
+
+#ifndef __LINUX_DS2404_H
+#define __LINUX_DS2404_H
+
+struct ds2404_platform_data {
+
+	unsigned int gpio_rst;
+	unsigned int gpio_clk;
+	unsigned int gpio_dq;
+};
+#endif
diff --git a/include/linux/rtc.h b/include/linux/rtc.h
index f071b3922c6..20ec4d3bed7 100644
--- a/include/linux/rtc.h
+++ b/include/linux/rtc.h
@@ -276,7 +276,7 @@ static inline bool is_leap_year(unsigned int year)
 	return (!(year % 4) && (year % 100)) || !(year % 400);
 }
 
-#ifdef CONFIG_RTC_HCTOSYS
+#ifdef CONFIG_RTC_HCTOSYS_DEVICE
 extern int rtc_hctosys_ret;
 #else
 #define rtc_hctosys_ret -ENODEV
diff --git a/include/linux/security.h b/include/linux/security.h
index 145accee923..5b50c4e1a7c 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -3022,5 +3022,36 @@ static inline void free_secdata(void *secdata)
 { }
 #endif /* CONFIG_SECURITY */
 
+#ifdef CONFIG_SECURITY_YAMA
+extern int yama_ptrace_access_check(struct task_struct *child,
+				    unsigned int mode);
+extern int yama_ptrace_traceme(struct task_struct *parent);
+extern void yama_task_free(struct task_struct *task);
+extern int yama_task_prctl(int option, unsigned long arg2, unsigned long arg3,
+			   unsigned long arg4, unsigned long arg5);
+#else
+static inline int yama_ptrace_access_check(struct task_struct *child,
+					   unsigned int mode)
+{
+	return 0;
+}
+
+static inline int yama_ptrace_traceme(struct task_struct *parent)
+{
+	return 0;
+}
+
+static inline void yama_task_free(struct task_struct *task)
+{
+}
+
+static inline int yama_task_prctl(int option, unsigned long arg2,
+				  unsigned long arg3, unsigned long arg4,
+				  unsigned long arg5)
+{
+	return -ENOSYS;
+}
+#endif /* CONFIG_SECURITY_YAMA */
+
 #endif /* ! __LINUX_SECURITY_H */
 
diff --git a/include/linux/ste_modem_shm.h b/include/linux/ste_modem_shm.h
new file mode 100644
index 00000000000..8444a4eff1b
--- /dev/null
+++ b/include/linux/ste_modem_shm.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) ST-Ericsson AB 2012
+ * Author: Sjur Brendeland / sjur.brandeland@stericsson.com
+ *
+ * License terms: GNU General Public License (GPL) version 2
+ */
+
+#ifndef __INC_MODEM_DEV_H
+#define __INC_MODEM_DEV_H
+#include <linux/types.h>
+#include <linux/platform_device.h>
+
+struct ste_modem_device;
+
+/**
+ * struct ste_modem_dev_cb - Callbacks for modem initiated events.
+ * @kick: Called when the modem kicks the host.
+ *
+ * This structure contains callbacks for actions triggered by the modem.
+ */
+struct ste_modem_dev_cb {
+	void (*kick)(struct ste_modem_device *mdev, int notify_id);
+};
+
+/**
+ * struct ste_modem_dev_ops - Functions to control modem and modem interface.
+ *
+ * @power:	Main power switch, used for cold-start or complete power off.
+ * @kick:	Kick the modem.
+ * @kick_subscribe: Subscribe for notifications from the modem.
+ * @setup:	Provide callback functions to modem device.
+ *
+ * This structure contains functions used by the ste remoteproc driver
+ * to manage the modem.
+ */
+struct ste_modem_dev_ops {
+	int (*power)(struct ste_modem_device *mdev, bool on);
+	int (*kick)(struct ste_modem_device *mdev, int notify_id);
+	int (*kick_subscribe)(struct ste_modem_device *mdev, int notify_id);
+	int (*setup)(struct ste_modem_device *mdev,
+		     struct ste_modem_dev_cb *cfg);
+};
+
+/**
+ * struct ste_modem_device - represent the STE modem device
+ * @pdev: Reference to platform device
+ * @ops: Operations used to manage the modem.
+ * @drv_data: Driver private data.
+ */
+struct ste_modem_device {
+	struct platform_device pdev;
+	struct ste_modem_dev_ops ops;
+	void *drv_data;
+};
+
+#endif /*INC_MODEM_DEV_H*/
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index e872526fdc5..8d08b3ed406 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -25,6 +25,7 @@ extern int swiotlb_force;
 extern void swiotlb_init(int verbose);
 extern void swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
 extern unsigned long swiotlb_nr_tbl(void);
+extern int swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs);
 
 /*
  * Enumeration for sync targets
diff --git a/include/linux/tpm.h b/include/linux/tpm.h
index fdc718abf83..fcb627ff8d3 100644
--- a/include/linux/tpm.h
+++ b/include/linux/tpm.h
@@ -32,6 +32,7 @@
 extern int tpm_pcr_read(u32 chip_num, int pcr_idx, u8 *res_buf);
 extern int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash);
 extern int tpm_send(u32 chip_num, void *cmd, size_t buflen);
+extern int tpm_get_random(u32 chip_num, u8 *data, size_t max);
 #else
 static inline int tpm_pcr_read(u32 chip_num, int pcr_idx, u8 *res_buf) {
 	return -ENODEV;
@@ -42,5 +43,8 @@ static inline int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash) {
 static inline int tpm_send(u32 chip_num, void *cmd, size_t buflen) {
 	return -ENODEV;
 }
+static inline int tpm_get_random(u32 chip_num, u8 *data, size_t max) {
+	return -ENODEV;
+}
 #endif
 #endif
diff --git a/include/linux/xattr.h b/include/linux/xattr.h
index 2ace7a60316..cc13e111597 100644
--- a/include/linux/xattr.h
+++ b/include/linux/xattr.h
@@ -33,6 +33,9 @@
 #define XATTR_EVM_SUFFIX "evm"
 #define XATTR_NAME_EVM XATTR_SECURITY_PREFIX XATTR_EVM_SUFFIX
 
+#define XATTR_IMA_SUFFIX "ima"
+#define XATTR_NAME_IMA XATTR_SECURITY_PREFIX XATTR_IMA_SUFFIX
+
 #define XATTR_SELINUX_SUFFIX "selinux"
 #define XATTR_NAME_SELINUX XATTR_SECURITY_PREFIX XATTR_SELINUX_SUFFIX
 
diff --git a/include/mtd/ubi-user.h b/include/mtd/ubi-user.h
index 8787349fbaf..53cae1e11e5 100644
--- a/include/mtd/ubi-user.h
+++ b/include/mtd/ubi-user.h
@@ -222,6 +222,7 @@ enum {
  * @ubi_num: UBI device number to create
  * @mtd_num: MTD device number to attach
  * @vid_hdr_offset: VID header offset (use defaults if %0)
+ * @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs
  * @padding: reserved for future, not used, has to be zeroed
  *
  * This data structure is used to specify MTD device UBI has to attach and the
@@ -245,12 +246,25 @@ enum {
  * be 2KiB-64 bytes = 1984. Note, that this position is not even 512-bytes
  * aligned, which is OK, as UBI is clever enough to realize this is 4th
  * sub-page of the first page and add needed padding.
+ *
+ * The @max_beb_per1024 is the maximum amount of bad PEBs UBI expects on the
+ * UBI device per 1024 eraseblocks.  This value is often given in an other form
+ * in the NAND datasheet (min NVB i.e. minimal number of valid blocks). The
+ * maximum expected bad eraseblocks per 1024 is then:
+ *    1024 * (1 - MinNVB / MaxNVB)
+ * Which gives 20 for most NAND devices.  This limit is used in order to derive
+ * amount of eraseblock UBI reserves for handling new bad blocks. If the device
+ * has more bad eraseblocks than this limit, UBI does not reserve any physical
+ * eraseblocks for new bad eraseblocks, but attempts to use available
+ * eraseblocks (if any). The accepted range is 0-768. If 0 is given, the
+ * default kernel value of %CONFIG_MTD_UBI_BEB_LIMIT will be used.
  */
 struct ubi_attach_req {
 	__s32 ubi_num;
 	__s32 mtd_num;
 	__s32 vid_hdr_offset;
-	__s8 padding[12];
+	__s16 max_beb_per1024;
+	__s8 padding[10];
 };
 
 /**
diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h
index 4faf6612eca..95e64664118 100644
--- a/include/net/net_namespace.h
+++ b/include/net/net_namespace.h
@@ -257,10 +257,12 @@ static inline struct net *read_pnet(struct net * const *pnet)
 #define __net_init
 #define __net_exit
 #define __net_initdata
+#define __net_initconst
 #else
 #define __net_init	__init
 #define __net_exit	__exit_refok
 #define __net_initdata	__initdata
+#define __net_initconst	__initconst
 #endif
 
 struct pernet_operations {
diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h
index ae33706afeb..ef937b56f9b 100644
--- a/include/scsi/libsas.h
+++ b/include/scsi/libsas.h
@@ -79,7 +79,8 @@ enum phy_event {
 	PHYE_OOB_DONE         = 1,
 	PHYE_OOB_ERROR        = 2,
 	PHYE_SPINUP_HOLD      = 3, /* hot plug SATA, no COMWAKE sent */
-	PHY_NUM_EVENTS        = 4,
+	PHYE_RESUME_TIMEOUT   = 4,
+	PHY_NUM_EVENTS        = 5,
 };
 
 enum discover_event {
@@ -87,8 +88,10 @@ enum discover_event {
 	DISCE_REVALIDATE_DOMAIN = 1,
 	DISCE_PORT_GONE         = 2,
 	DISCE_PROBE		= 3,
-	DISCE_DESTRUCT		= 4,
-	DISC_NUM_EVENTS		= 5,
+	DISCE_SUSPEND		= 4,
+	DISCE_RESUME		= 5,
+	DISCE_DESTRUCT		= 6,
+	DISC_NUM_EVENTS		= 7,
 };
 
 /* ---------- Expander Devices ---------- */
@@ -128,7 +131,7 @@ struct ex_phy {
 	u8   attached_sas_addr[SAS_ADDR_SIZE];
 	u8   attached_phy_id;
 
-	u8   phy_change_count;
+	int phy_change_count;
 	enum routing_attribute routing_attr;
 	u8   virtual:1;
 
@@ -141,7 +144,7 @@ struct ex_phy {
 struct expander_device {
 	struct list_head children;
 
-	u16    ex_change_count;
+	int    ex_change_count;
 	u16    max_route_indexes;
 	u8     num_phys;
 
@@ -169,6 +172,7 @@ struct sata_device {
         enum   ata_command_set command_set;
         struct smp_resp        rps_resp; /* report_phy_sata_resp */
         u8     port_no;        /* port number, if this is a PM (Port) */
+	int    pm_result;
 
 	struct ata_port *ap;
 	struct ata_host ata_host;
@@ -182,6 +186,7 @@ struct ssp_device {
 
 enum {
 	SAS_DEV_GONE,
+	SAS_DEV_FOUND, /* device notified to lldd */
 	SAS_DEV_DESTROY,
 	SAS_DEV_EH_PENDING,
 	SAS_DEV_LU_RESET,
@@ -273,6 +278,7 @@ struct asd_sas_port {
 	enum   sas_linkrate linkrate;
 
 	struct sas_work work;
+	int suspended;
 
 /* public: */
 	int id;
@@ -321,6 +327,7 @@ struct asd_sas_phy {
 	unsigned long phy_events_pending;
 
 	int error;
+	int suspended;
 
 	struct sas_phy *phy;
 
@@ -687,6 +694,9 @@ struct sas_domain_function_template {
 
 extern int sas_register_ha(struct sas_ha_struct *);
 extern int sas_unregister_ha(struct sas_ha_struct *);
+extern void sas_prep_resume_ha(struct sas_ha_struct *sas_ha);
+extern void sas_resume_ha(struct sas_ha_struct *sas_ha);
+extern void sas_suspend_ha(struct sas_ha_struct *sas_ha);
 
 int sas_set_phy_speed(struct sas_phy *phy,
 		      struct sas_phy_linkrates *rates);
diff --git a/include/scsi/osd_attributes.h b/include/scsi/osd_attributes.h
index 56e920ade32..303ba1118a4 100644
--- a/include/scsi/osd_attributes.h
+++ b/include/scsi/osd_attributes.h
@@ -1,7 +1,7 @@
 #ifndef __OSD_ATTRIBUTES_H__
 #define __OSD_ATTRIBUTES_H__
 
-#include "osd_protocol.h"
+#include <scsi/osd_protocol.h>
 
 /*
  * Contains types and constants that define attribute pages and attribute
diff --git a/include/scsi/osd_initiator.h b/include/scsi/osd_initiator.h
index 572fb549366..b2e85fdd2ae 100644
--- a/include/scsi/osd_initiator.h
+++ b/include/scsi/osd_initiator.h
@@ -14,8 +14,8 @@
 #ifndef __OSD_INITIATOR_H__
 #define __OSD_INITIATOR_H__
 
-#include "osd_protocol.h"
-#include "osd_types.h"
+#include <scsi/osd_protocol.h>
+#include <scsi/osd_types.h>
 
 #include <linux/blkdev.h>
 #include <scsi/scsi_device.h>
diff --git a/include/scsi/osd_sec.h b/include/scsi/osd_sec.h
index 4c09fee8ae1..f96151c9c9e 100644
--- a/include/scsi/osd_sec.h
+++ b/include/scsi/osd_sec.h
@@ -14,8 +14,8 @@
 #ifndef __OSD_SEC_H__
 #define __OSD_SEC_H__
 
-#include "osd_protocol.h"
-#include "osd_types.h"
+#include <scsi/osd_protocol.h>
+#include <scsi/osd_types.h>
 
 /*
  * Contains types and constants of osd capabilities and security
diff --git a/include/scsi/sas_ata.h b/include/scsi/sas_ata.h
index 2dfbdaa0b34..ff71a565468 100644
--- a/include/scsi/sas_ata.h
+++ b/include/scsi/sas_ata.h
@@ -45,6 +45,8 @@ void sas_ata_eh(struct Scsi_Host *shost, struct list_head *work_q,
 void sas_ata_schedule_reset(struct domain_device *dev);
 void sas_ata_wait_eh(struct domain_device *dev);
 void sas_probe_sata(struct asd_sas_port *port);
+void sas_suspend_sata(struct asd_sas_port *port);
+void sas_resume_sata(struct asd_sas_port *port);
 void sas_ata_end_eh(struct ata_port *ap);
 #else
 
@@ -82,6 +84,14 @@ static inline void sas_probe_sata(struct asd_sas_port *port)
 {
 }
 
+static inline void sas_suspend_sata(struct asd_sas_port *port)
+{
+}
+
+static inline void sas_resume_sata(struct asd_sas_port *port)
+{
+}
+
 static inline int sas_get_ata_info(struct domain_device *dev, struct ex_phy *phy)
 {
 	return 0;
diff --git a/include/scsi/scsi_bsg_fc.h b/include/scsi/scsi_bsg_fc.h
index 91a4e4ff9a9..3031b900b08 100644
--- a/include/scsi/scsi_bsg_fc.h
+++ b/include/scsi/scsi_bsg_fc.h
@@ -26,8 +26,6 @@
  * This file intended to be included by both kernel and user space
  */
 
-#include <scsi/scsi.h>
-
 /*
  * FC Transport SGIO v4 BSG Message Support
  */
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index 9895f69294f..88fae8d2015 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -156,6 +156,7 @@ struct scsi_device {
 	unsigned is_visible:1;	/* is the device visible in sysfs */
 	unsigned can_power_off:1; /* Device supports runtime power off */
 	unsigned wce_default_on:1;	/* Cache is ON by default */
+	unsigned no_dif:1;	/* T10 PI (DIF) should be disabled */
 
 	DECLARE_BITMAP(supported_events, SDEV_EVT_MAXBITS); /* supported events */
 	struct list_head event_list;	/* asserted events */
@@ -476,6 +477,9 @@ static inline int scsi_device_enclosure(struct scsi_device *sdev)
 
 static inline int scsi_device_protection(struct scsi_device *sdev)
 {
+	if (sdev->no_dif)
+		return 0;
+
 	return sdev->scsi_level > SCSI_2 && sdev->inquiry[5] & (1<<0);
 }
 
diff --git a/include/scsi/scsi_devinfo.h b/include/scsi/scsi_devinfo.h
index b4ddd3b18b4..cc1f3e786ad 100644
--- a/include/scsi/scsi_devinfo.h
+++ b/include/scsi/scsi_devinfo.h
@@ -30,4 +30,5 @@
 #define BLIST_RETRY_HWERROR	0x400000 /* retry HARDWARE_ERROR */
 #define BLIST_MAX_512		0x800000 /* maximum 512 sector cdb length */
 #define BLIST_ATTACH_PQ3	0x1000000 /* Scan: Attach to PQ3 devices */
+#define BLIST_NO_DIF		0x2000000 /* Disable T10 PI (DIF) */
 #endif
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 5f7d5b3b1c6..49084807eb6 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -873,6 +873,9 @@ static inline unsigned int scsi_host_dif_capable(struct Scsi_Host *shost, unsign
 				       SHOST_DIF_TYPE2_PROTECTION,
 				       SHOST_DIF_TYPE3_PROTECTION };
 
+	if (target_type > SHOST_DIF_TYPE3_PROTECTION)
+		return 0;
+
 	return shost->prot_capabilities & cap[target_type] ? target_type : 0;
 }
 
@@ -884,6 +887,9 @@ static inline unsigned int scsi_host_dix_capable(struct Scsi_Host *shost, unsign
 				       SHOST_DIX_TYPE2_PROTECTION,
 				       SHOST_DIX_TYPE3_PROTECTION };
 
+	if (target_type > SHOST_DIX_TYPE3_PROTECTION)
+		return 0;
+
 	return shost->prot_capabilities & cap[target_type];
 #endif
 	return 0;
diff --git a/include/sound/ac97_codec.h b/include/sound/ac97_codec.h
index 02cbb50225b..fdeb8dceec0 100644
--- a/include/sound/ac97_codec.h
+++ b/include/sound/ac97_codec.h
@@ -28,9 +28,9 @@
 #include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/workqueue.h>
-#include "pcm.h"
-#include "control.h"
-#include "info.h"
+#include <sound/pcm.h>
+#include <sound/control.h>
+#include <sound/info.h>
 
 /* maximum number of devices on the AC97 bus */
 #define	AC97_BUS_MAX_DEVICES	4
diff --git a/include/sound/ad1816a.h b/include/sound/ad1816a.h
index d010858c33c..a7d8dc782e7 100644
--- a/include/sound/ad1816a.h
+++ b/include/sound/ad1816a.h
@@ -20,9 +20,9 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */
 
-#include "control.h"
-#include "pcm.h"
-#include "timer.h"
+#include <sound/control.h>
+#include <sound/pcm.h>
+#include <sound/timer.h>
 
 #define AD1816A_REG(r)			(chip->port + r)
 
diff --git a/include/sound/ak4531_codec.h b/include/sound/ak4531_codec.h
index 575296cf798..85ea86ea35b 100644
--- a/include/sound/ak4531_codec.h
+++ b/include/sound/ak4531_codec.h
@@ -25,8 +25,8 @@
  *
  */
 
-#include "info.h"
-#include "control.h"
+#include <sound/info.h>
+#include <sound/control.h>
 
 /*
  *  ASAHI KASEI - AK4531 codec
diff --git a/include/sound/emu10k1_synth.h b/include/sound/emu10k1_synth.h
index 6ef61c42093..9f211e957bf 100644
--- a/include/sound/emu10k1_synth.h
+++ b/include/sound/emu10k1_synth.h
@@ -20,8 +20,8 @@
  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "emu10k1.h"
-#include "emux_synth.h"
+#include <sound/emu10k1.h>
+#include <sound/emux_synth.h>
 
 /* sequencer device id */
 #define SNDRV_SEQ_DEV_ID_EMU10K1_SYNTH	"emu10k1-synth"
diff --git a/include/sound/emu8000.h b/include/sound/emu8000.h
index c8f66bde6d9..c321302a914 100644
--- a/include/sound/emu8000.h
+++ b/include/sound/emu8000.h
@@ -21,8 +21,8 @@
  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "emux_synth.h"
-#include "seq_kernel.h"
+#include <sound/emux_synth.h>
+#include <sound/seq_kernel.h>
 
 /*
  * Hardware parameters.
diff --git a/include/sound/emux_legacy.h b/include/sound/emux_legacy.h
index 6fe3da2a5e1..baf43fc24d3 100644
--- a/include/sound/emux_legacy.h
+++ b/include/sound/emux_legacy.h
@@ -22,7 +22,7 @@
  *
  */
 
-#include "seq_oss_legacy.h"
+#include <sound/seq_oss_legacy.h>
 
 /*
  * awe hardware controls
diff --git a/include/sound/emux_synth.h b/include/sound/emux_synth.h
index d8cb51b86c2..fb81f3722b6 100644
--- a/include/sound/emux_synth.h
+++ b/include/sound/emux_synth.h
@@ -21,15 +21,15 @@
  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "seq_kernel.h"
-#include "seq_device.h"
-#include "soundfont.h"
-#include "seq_midi_emul.h"
+#include <sound/seq_kernel.h>
+#include <sound/seq_device.h>
+#include <sound/soundfont.h>
+#include <sound/seq_midi_emul.h>
 #ifdef CONFIG_SND_SEQUENCER_OSS
-#include "seq_oss.h"
+#include <sound/seq_oss.h>
 #endif
-#include "emux_legacy.h"
-#include "seq_virmidi.h"
+#include <sound/emux_legacy.h>
+#include <sound/seq_virmidi.h>
 
 /*
  * compile flags
diff --git a/include/sound/es1688.h b/include/sound/es1688.h
index f752dd33dfa..1d636a2d889 100644
--- a/include/sound/es1688.h
+++ b/include/sound/es1688.h
@@ -22,8 +22,8 @@
  *
  */
 
-#include "control.h"
-#include "pcm.h"
+#include <sound/control.h>
+#include <sound/pcm.h>
 #include <linux/interrupt.h>
 
 #define ES1688_HW_AUTO		0x0000
diff --git a/include/sound/gus.h b/include/sound/gus.h
index 841bb8df38c..42905d811da 100644
--- a/include/sound/gus.h
+++ b/include/sound/gus.h
@@ -22,11 +22,11 @@
  *
  */
 
-#include "pcm.h"
-#include "rawmidi.h"
-#include "timer.h"
-#include "seq_midi_emul.h"
-#include "seq_device.h"
+#include <sound/pcm.h>
+#include <sound/rawmidi.h>
+#include <sound/timer.h>
+#include <sound/seq_midi_emul.h>
+#include <sound/seq_device.h>
 #include <asm/io.h>
 
 /* IO ports */
diff --git a/include/sound/mpu401.h b/include/sound/mpu401.h
index 20230db00ef..e9420969251 100644
--- a/include/sound/mpu401.h
+++ b/include/sound/mpu401.h
@@ -22,7 +22,7 @@
  *
  */
 
-#include "rawmidi.h"
+#include <sound/rawmidi.h>
 #include <linux/interrupt.h>
 
 #define MPU401_HW_MPU401		1	/* native MPU401 */
diff --git a/include/sound/pcm.h b/include/sound/pcm.h
index cdca2ab1e71..d0711bc8c91 100644
--- a/include/sound/pcm.h
+++ b/include/sound/pcm.h
@@ -35,7 +35,7 @@
 #define snd_pcm_chip(pcm) ((pcm)->private_data)
 
 #if defined(CONFIG_SND_PCM_OSS) || defined(CONFIG_SND_PCM_OSS_MODULE)
-#include "pcm_oss.h"
+#include <sound/pcm_oss.h>
 #endif
 
 /*
diff --git a/include/sound/rawmidi.h b/include/sound/rawmidi.h
index 6b14359d9fe..adf0885153f 100644
--- a/include/sound/rawmidi.h
+++ b/include/sound/rawmidi.h
@@ -30,7 +30,7 @@
 #include <linux/workqueue.h>
 
 #if defined(CONFIG_SND_SEQUENCER) || defined(CONFIG_SND_SEQUENCER_MODULE)
-#include "seq_device.h"
+#include <sound/seq_device.h>
 #endif
 
 /*
diff --git a/include/sound/sb.h b/include/sound/sb.h
index 95353542256..ba396032964 100644
--- a/include/sound/sb.h
+++ b/include/sound/sb.h
@@ -22,8 +22,8 @@
  *
  */
 
-#include "pcm.h"
-#include "rawmidi.h"
+#include <sound/pcm.h>
+#include <sound/rawmidi.h>
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
diff --git a/include/sound/sb16_csp.h b/include/sound/sb16_csp.h
index af1b49e982d..7e950560e59 100644
--- a/include/sound/sb16_csp.h
+++ b/include/sound/sb16_csp.h
@@ -119,8 +119,8 @@ struct snd_sb_csp_info {
 #define SNDRV_SB_CSP_IOCTL_RESTART	_IO('H', 0x16)
 
 #ifdef __KERNEL__
-#include "sb.h"
-#include "hwdep.h"
+#include <sound/sb.h>
+#include <sound/hwdep.h>
 #include <linux/firmware.h>
 
 struct snd_sb_csp;
diff --git a/include/sound/seq_kernel.h b/include/sound/seq_kernel.h
index f352a98ce4f..2398521f099 100644
--- a/include/sound/seq_kernel.h
+++ b/include/sound/seq_kernel.h
@@ -22,7 +22,7 @@
  *
  */
 #include <linux/time.h>
-#include "asequencer.h"
+#include <sound/asequencer.h>
 
 typedef struct snd_seq_real_time snd_seq_real_time_t;
 typedef union snd_seq_timestamp snd_seq_timestamp_t;
diff --git a/include/sound/seq_midi_emul.h b/include/sound/seq_midi_emul.h
index d6c4615901b..8139d8c191e 100644
--- a/include/sound/seq_midi_emul.h
+++ b/include/sound/seq_midi_emul.h
@@ -22,7 +22,7 @@
  *
  */
 
-#include "seq_kernel.h"
+#include <sound/seq_kernel.h>
 
 /*
  * This structure is used to keep track of the current state on each
diff --git a/include/sound/seq_midi_event.h b/include/sound/seq_midi_event.h
index 5efab8b29c5..e40f43e6fc7 100644
--- a/include/sound/seq_midi_event.h
+++ b/include/sound/seq_midi_event.h
@@ -22,7 +22,7 @@
  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "asequencer.h"
+#include <sound/asequencer.h>
 
 #define MAX_MIDI_EVENT_BUF	256
 
diff --git a/include/sound/seq_oss.h b/include/sound/seq_oss.h
index 9b060bbd6e0..d0b27ec6f8b 100644
--- a/include/sound/seq_oss.h
+++ b/include/sound/seq_oss.h
@@ -21,8 +21,8 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "asequencer.h"
-#include "seq_kernel.h"
+#include <sound/asequencer.h>
+#include <sound/seq_kernel.h>
 
 /*
  * argument structure for synthesizer operations
diff --git a/include/sound/seq_virmidi.h b/include/sound/seq_virmidi.h
index d888433a309..a03acd0d398 100644
--- a/include/sound/seq_virmidi.h
+++ b/include/sound/seq_virmidi.h
@@ -22,8 +22,8 @@
  *
  */
 
-#include "rawmidi.h"
-#include "seq_midi_event.h"
+#include <sound/rawmidi.h>
+#include <sound/seq_midi_event.h>
 
 /*
  * device file instance:
diff --git a/include/sound/snd_wavefront.h b/include/sound/snd_wavefront.h
index fa149ca77e4..35e94b3d1ec 100644
--- a/include/sound/snd_wavefront.h
+++ b/include/sound/snd_wavefront.h
@@ -1,10 +1,10 @@
 #ifndef __SOUND_SND_WAVEFRONT_H__
 #define __SOUND_SND_WAVEFRONT_H__
 
-#include "mpu401.h"
-#include "hwdep.h"
-#include "rawmidi.h"
-#include "wavefront.h"  /* generic OSS/ALSA/user-level wavefront header */
+#include <sound/mpu401.h>
+#include <sound/hwdep.h>
+#include <sound/rawmidi.h>
+#include <sound/wavefront.h>  /* generic OSS/ALSA/user-level wavefront header */
 
 /* MIDI interface */
 
diff --git a/include/sound/soundfont.h b/include/sound/soundfont.h
index 679df057406..7c93efdba90 100644
--- a/include/sound/soundfont.h
+++ b/include/sound/soundfont.h
@@ -22,8 +22,8 @@
  *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
-#include "sfnt_info.h"
-#include "util_mem.h"
+#include <sound/sfnt_info.h>
+#include <sound/util_mem.h>
 
 #define SF_MAX_INSTRUMENTS	128	/* maximum instrument number */
 #define SF_MAX_PRESETS  256	/* drums are mapped from 128 to 256 */
diff --git a/include/sound/tea6330t.h b/include/sound/tea6330t.h
index 51b282b7689..e6beec23d7f 100644
--- a/include/sound/tea6330t.h
+++ b/include/sound/tea6330t.h
@@ -22,7 +22,7 @@
  *
  */
 
-#include "i2c.h"		/* generic i2c support */
+#include <sound/i2c.h>		/* generic i2c support */
 
 int snd_tea6330t_detect(struct snd_i2c_bus *bus, int equalizer);
 int snd_tea6330t_update_mixer(struct snd_card *card, struct snd_i2c_bus *bus,
diff --git a/include/sound/wss.h b/include/sound/wss.h
index fd01f22825c..0c7f034f1e8 100644
--- a/include/sound/wss.h
+++ b/include/sound/wss.h
@@ -22,11 +22,11 @@
  *
  */
 
-#include "control.h"
-#include "pcm.h"
-#include "timer.h"
+#include <sound/control.h>
+#include <sound/pcm.h>
+#include <sound/timer.h>
 
-#include "cs4231-regs.h"
+#include <sound/cs4231-regs.h>
 
 /* defines for codec.mode */
 
diff --git a/include/trace/events/compaction.h b/include/trace/events/compaction.h
index 388bcdd26d4..fde1b3e94c7 100644
--- a/include/trace/events/compaction.h
+++ b/include/trace/events/compaction.h
@@ -6,7 +6,7 @@
 
 #include <linux/types.h>
 #include <linux/tracepoint.h>
-#include "gfpflags.h"
+#include <trace/events/gfpflags.h>
 
 DECLARE_EVENT_CLASS(mm_compaction_isolate_template,
 
diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h
index 08fa27244da..6bc943ecb84 100644
--- a/include/trace/events/kmem.h
+++ b/include/trace/events/kmem.h
@@ -6,7 +6,7 @@
 
 #include <linux/types.h>
 #include <linux/tracepoint.h>
-#include "gfpflags.h"
+#include <trace/events/gfpflags.h>
 
 DECLARE_EVENT_CLASS(kmem_alloc,
 
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index bab3b87e406..63cfcccaebb 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -8,7 +8,7 @@
 #include <linux/tracepoint.h>
 #include <linux/mm.h>
 #include <linux/memcontrol.h>
-#include "gfpflags.h"
+#include <trace/events/gfpflags.h>
 
 #define RECLAIM_WB_ANON		0x0001u
 #define RECLAIM_WB_FILE		0x0002u
diff --git a/include/uapi/Kbuild b/include/uapi/Kbuild
new file mode 100644
index 00000000000..81d2106287f
--- /dev/null
+++ b/include/uapi/Kbuild
@@ -0,0 +1,14 @@
+# UAPI Header export list
+# Top-level Makefile calls into asm-$(ARCH)
+# List only non-arch directories below
+
+
+header-y += asm-generic/
+header-y += linux/
+header-y += sound/
+header-y += mtd/
+header-y += rdma/
+header-y += video/
+header-y += drm/
+header-y += xen/
+header-y += scsi/
diff --git a/include/uapi/asm-generic/Kbuild b/include/uapi/asm-generic/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/asm-generic/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/asm-generic/Kbuild.asm b/include/uapi/asm-generic/Kbuild.asm
new file mode 100644
index 00000000000..fcd50b75921
--- /dev/null
+++ b/include/uapi/asm-generic/Kbuild.asm
@@ -0,0 +1,49 @@
+#
+# Headers that are optional in usr/include/asm/
+#
+opt-header += kvm.h
+opt-header += kvm_para.h
+opt-header += a.out.h
+
+#
+# Headers that are mandatory in usr/include/asm/
+#
+header-y += auxvec.h
+header-y += bitsperlong.h
+header-y += byteorder.h
+header-y += errno.h
+header-y += fcntl.h
+header-y += ioctl.h
+header-y += ioctls.h
+header-y += ipcbuf.h
+header-y += mman.h
+header-y += msgbuf.h
+header-y += param.h
+header-y += poll.h
+header-y += posix_types.h
+header-y += ptrace.h
+header-y += resource.h
+header-y += sembuf.h
+header-y += setup.h
+header-y += shmbuf.h
+header-y += sigcontext.h
+header-y += siginfo.h
+header-y += signal.h
+header-y += socket.h
+header-y += sockios.h
+header-y += stat.h
+header-y += statfs.h
+header-y += swab.h
+header-y += termbits.h
+header-y += termios.h
+header-y += types.h
+header-y += unistd.h
+
+header-y += $(foreach hdr,$(opt-header), \
+	      $(if \
+		$(wildcard \
+			$(srctree)/arch/$(SRCARCH)/include/uapi/asm/$(hdr) \
+			$(srctree)/arch/$(SRCARCH)/include/asm/$(hdr) \
+		), \
+		$(hdr) \
+		))
diff --git a/include/uapi/drm/Kbuild b/include/uapi/drm/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/drm/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild
new file mode 100644
index 00000000000..b0fd4d03499
--- /dev/null
+++ b/include/uapi/linux/Kbuild
@@ -0,0 +1,24 @@
+# UAPI Header export list
+header-y += byteorder/
+header-y += can/
+header-y += caif/
+header-y += dvb/
+header-y += hdlc/
+header-y += hsi/
+header-y += isdn/
+header-y += mmc/
+header-y += nfsd/
+header-y += raid/
+header-y += spi/
+header-y += sunrpc/
+header-y += tc_act/
+header-y += tc_ematch/
+header-y += netfilter/
+header-y += netfilter_arp/
+header-y += netfilter_bridge/
+header-y += netfilter_ipv4/
+header-y += netfilter_ipv6/
+header-y += usb/
+header-y += wimax/
+
+genhdr-y += version.h
diff --git a/include/uapi/linux/byteorder/Kbuild b/include/uapi/linux/byteorder/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/byteorder/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/caif/Kbuild b/include/uapi/linux/caif/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/caif/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/can/Kbuild b/include/uapi/linux/can/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/can/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/dvb/Kbuild b/include/uapi/linux/dvb/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/dvb/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/hdlc/Kbuild b/include/uapi/linux/hdlc/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/hdlc/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/hsi/Kbuild b/include/uapi/linux/hsi/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/hsi/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/isdn/Kbuild b/include/uapi/linux/isdn/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/isdn/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/mmc/Kbuild b/include/uapi/linux/mmc/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/mmc/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/netfilter/Kbuild b/include/uapi/linux/netfilter/Kbuild
new file mode 100644
index 00000000000..4afbace8e86
--- /dev/null
+++ b/include/uapi/linux/netfilter/Kbuild
@@ -0,0 +1,2 @@
+# UAPI Header export list
+header-y += ipset/
diff --git a/include/uapi/linux/netfilter/ipset/Kbuild b/include/uapi/linux/netfilter/ipset/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/netfilter/ipset/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/netfilter_arp/Kbuild b/include/uapi/linux/netfilter_arp/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/netfilter_arp/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/netfilter_bridge/Kbuild b/include/uapi/linux/netfilter_bridge/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/netfilter_bridge/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/netfilter_ipv4/Kbuild b/include/uapi/linux/netfilter_ipv4/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/netfilter_ipv4/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/netfilter_ipv6/Kbuild b/include/uapi/linux/netfilter_ipv6/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/netfilter_ipv6/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/nfsd/Kbuild b/include/uapi/linux/nfsd/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/nfsd/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/raid/Kbuild b/include/uapi/linux/raid/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/raid/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/spi/Kbuild b/include/uapi/linux/spi/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/spi/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/sunrpc/Kbuild b/include/uapi/linux/sunrpc/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/sunrpc/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/tc_act/Kbuild b/include/uapi/linux/tc_act/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/tc_act/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/tc_ematch/Kbuild b/include/uapi/linux/tc_ematch/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/tc_ematch/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/usb/Kbuild b/include/uapi/linux/usb/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/usb/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/linux/wimax/Kbuild b/include/uapi/linux/wimax/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/linux/wimax/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/mtd/Kbuild b/include/uapi/mtd/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/mtd/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/rdma/Kbuild b/include/uapi/rdma/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/rdma/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/scsi/Kbuild b/include/uapi/scsi/Kbuild
new file mode 100644
index 00000000000..29a87dd26cf
--- /dev/null
+++ b/include/uapi/scsi/Kbuild
@@ -0,0 +1,2 @@
+# UAPI Header export list
+header-y += fc/
diff --git a/include/uapi/scsi/fc/Kbuild b/include/uapi/scsi/fc/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/scsi/fc/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/sound/Kbuild b/include/uapi/sound/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/sound/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/video/Kbuild b/include/uapi/video/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/video/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/uapi/xen/Kbuild b/include/uapi/xen/Kbuild
new file mode 100644
index 00000000000..aafaa5aa54d
--- /dev/null
+++ b/include/uapi/xen/Kbuild
@@ -0,0 +1 @@
+# UAPI Header export list
diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h
index f19fff8650e..aecee9d112c 100644
--- a/include/xen/grant_table.h
+++ b/include/xen/grant_table.h
@@ -190,4 +190,16 @@ int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 		      struct gnttab_map_grant_ref *kunmap_ops,
 		      struct page **pages, unsigned int count);
 
+/* Perform a batch of grant map/copy operations. Retry every batch slot
+ * for which the hypervisor returns GNTST_eagain. This is typically due
+ * to paged out target frames.
+ *
+ * Will retry for 1, 2, ... 255 ms, i.e. 256 times during 32 seconds.
+ *
+ * Return value in each iand every status field of the batch guaranteed
+ * to not be GNTST_eagain.
+ */
+void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count);
+void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count);
+
 #endif /* __ASM_GNTTAB_H__ */
diff --git a/include/xen/interface/callback.h b/include/xen/interface/callback.h
index 2ae3cd24326..8c5fa0e2015 100644
--- a/include/xen/interface/callback.h
+++ b/include/xen/interface/callback.h
@@ -27,7 +27,7 @@
 #ifndef __XEN_PUBLIC_CALLBACK_H__
 #define __XEN_PUBLIC_CALLBACK_H__
 
-#include "xen.h"
+#include <xen/interface/xen.h>
 
 /*
  * Prototype for this hypercall is:
diff --git a/include/xen/interface/grant_table.h b/include/xen/interface/grant_table.h
index a17d84433e6..f9f8b975ae7 100644
--- a/include/xen/interface/grant_table.h
+++ b/include/xen/interface/grant_table.h
@@ -338,7 +338,7 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_dump_table);
 #define GNTTABOP_transfer                4
 struct gnttab_transfer {
     /* IN parameters. */
-    unsigned long mfn;
+    xen_pfn_t mfn;
     domid_t       domid;
     grant_ref_t   ref;
     /* OUT parameters. */
@@ -375,7 +375,7 @@ struct gnttab_copy {
 	struct {
 		union {
 			grant_ref_t ref;
-			unsigned long   gmfn;
+			xen_pfn_t   gmfn;
 		} u;
 		domid_t  domid;
 		uint16_t offset;
@@ -519,7 +519,9 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_get_version);
 #define GNTST_no_device_space  (-7) /* Out of space in I/O MMU.              */
 #define GNTST_permission_denied (-8) /* Not enough privilege for operation.  */
 #define GNTST_bad_page         (-9) /* Specified page was invalid for op.    */
-#define GNTST_bad_copy_arg    (-10) /* copy arguments cross page boundary */
+#define GNTST_bad_copy_arg    (-10) /* copy arguments cross page boundary.   */
+#define GNTST_address_too_big (-11) /* transfer page address too large.      */
+#define GNTST_eagain          (-12) /* Operation not done; try again.        */
 
 #define GNTTABOP_error_msgs {                   \
     "okay",                                     \
@@ -532,7 +534,9 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_get_version);
     "no spare translation slot in the I/O MMU", \
     "permission denied",                        \
     "bad page",                                 \
-    "copy arguments cross page boundary"        \
+    "copy arguments cross page boundary",       \
+    "page address size too large",              \
+    "operation not done; try again"             \
 }
 
 #endif /* __XEN_PUBLIC_GRANT_TABLE_H__ */
diff --git a/include/xen/interface/hvm/params.h b/include/xen/interface/hvm/params.h
index 1b4f923d708..a6c79911e72 100644
--- a/include/xen/interface/hvm/params.h
+++ b/include/xen/interface/hvm/params.h
@@ -21,7 +21,7 @@
 #ifndef __XEN_PUBLIC_HVM_PARAMS_H__
 #define __XEN_PUBLIC_HVM_PARAMS_H__
 
-#include "hvm_op.h"
+#include <xen/interface/hvm/hvm_op.h>
 
 /*
  * Parameter space for HVMOP_{set,get}_param.
diff --git a/include/xen/interface/io/blkif.h b/include/xen/interface/io/blkif.h
index ee338bfde18..01c3d62436e 100644
--- a/include/xen/interface/io/blkif.h
+++ b/include/xen/interface/io/blkif.h
@@ -9,8 +9,8 @@
 #ifndef __XEN_PUBLIC_IO_BLKIF_H__
 #define __XEN_PUBLIC_IO_BLKIF_H__
 
-#include "ring.h"
-#include "../grant_table.h"
+#include <xen/interface/io/ring.h>
+#include <xen/interface/grant_table.h>
 
 /*
  * Front->back notifications: When enqueuing a new request, sending a
diff --git a/include/xen/interface/io/netif.h b/include/xen/interface/io/netif.h
index cb94668f6e9..9dfc1200098 100644
--- a/include/xen/interface/io/netif.h
+++ b/include/xen/interface/io/netif.h
@@ -9,8 +9,8 @@
 #ifndef __XEN_PUBLIC_IO_NETIF_H__
 #define __XEN_PUBLIC_IO_NETIF_H__
 
-#include "ring.h"
-#include "../grant_table.h"
+#include <xen/interface/io/ring.h>
+#include <xen/interface/grant_table.h>
 
 /*
  * Notifications after enqueuing any type of message should be conditional on
diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
index eac3ce15371..d8e33a93ea4 100644
--- a/include/xen/interface/memory.h
+++ b/include/xen/interface/memory.h
@@ -31,7 +31,7 @@ struct xen_memory_reservation {
      *   OUT: GMFN bases of extents that were allocated
      *   (NB. This command also updates the mach_to_phys translation table)
      */
-    GUEST_HANDLE(ulong) extent_start;
+    GUEST_HANDLE(xen_pfn_t) extent_start;
 
     /* Number of extents, and size/alignment of each (2^extent_order pages). */
     unsigned long  nr_extents;
@@ -130,7 +130,7 @@ struct xen_machphys_mfn_list {
      * any large discontiguities in the machine address space, 2MB gaps in
      * the machphys table will be represented by an MFN base of zero.
      */
-    GUEST_HANDLE(ulong) extent_start;
+    GUEST_HANDLE(xen_pfn_t) extent_start;
 
     /*
      * Number of extents written to the above array. This will be smaller
@@ -163,6 +163,9 @@ struct xen_add_to_physmap {
     /* Which domain to change the mapping for. */
     domid_t domid;
 
+    /* Number of pages to go through for gmfn_range */
+    uint16_t    size;
+
     /* Source mapping space. */
 #define XENMAPSPACE_shared_info 0 /* shared info page */
 #define XENMAPSPACE_grant_table 1 /* grant table page */
@@ -172,7 +175,7 @@ struct xen_add_to_physmap {
     unsigned long idx;
 
     /* GPFN where the source mapping page should appear. */
-    unsigned long gpfn;
+    xen_pfn_t gpfn;
 };
 DEFINE_GUEST_HANDLE_STRUCT(xen_add_to_physmap);
 
diff --git a/include/xen/interface/platform.h b/include/xen/interface/platform.h
index 61fa6616098..4755b5fac9c 100644
--- a/include/xen/interface/platform.h
+++ b/include/xen/interface/platform.h
@@ -27,7 +27,7 @@
 #ifndef __XEN_PUBLIC_PLATFORM_H__
 #define __XEN_PUBLIC_PLATFORM_H__
 
-#include "xen.h"
+#include <xen/interface/xen.h>
 
 #define XENPF_INTERFACE_VERSION 0x03000001
 
@@ -54,7 +54,7 @@ DEFINE_GUEST_HANDLE_STRUCT(xenpf_settime_t);
 #define XENPF_add_memtype         31
 struct xenpf_add_memtype {
 	/* IN variables. */
-	unsigned long mfn;
+	xen_pfn_t mfn;
 	uint64_t nr_mfns;
 	uint32_t type;
 	/* OUT variables. */
@@ -84,7 +84,7 @@ struct xenpf_read_memtype {
 	/* IN variables. */
 	uint32_t reg;
 	/* OUT variables. */
-	unsigned long mfn;
+	xen_pfn_t mfn;
 	uint64_t nr_mfns;
 	uint32_t type;
 };
@@ -112,6 +112,7 @@ DEFINE_GUEST_HANDLE_STRUCT(xenpf_platform_quirk_t);
 #define XEN_FW_DISK_INFO          1 /* from int 13 AH=08/41/48 */
 #define XEN_FW_DISK_MBR_SIGNATURE 2 /* from MBR offset 0x1b8 */
 #define XEN_FW_VBEDDC_INFO        3 /* from int 10 AX=4f15 */
+#define XEN_FW_KBD_SHIFT_FLAGS    5 /* Int16, Fn02: Get keyboard shift flags. */
 struct xenpf_firmware_info {
 	/* IN variables. */
 	uint32_t type;
@@ -142,6 +143,8 @@ struct xenpf_firmware_info {
 			/* must refer to 128-byte buffer */
 			GUEST_HANDLE(uchar) edid;
 		} vbeddc_info; /* XEN_FW_VBEDDC_INFO */
+
+		uint8_t kbd_shift_flags; /* XEN_FW_KBD_SHIFT_FLAGS */
 	} u;
 };
 DEFINE_GUEST_HANDLE_STRUCT(xenpf_firmware_info_t);
diff --git a/include/xen/interface/sched.h b/include/xen/interface/sched.h
index dd55dac340d..9ce083960a2 100644
--- a/include/xen/interface/sched.h
+++ b/include/xen/interface/sched.h
@@ -9,7 +9,7 @@
 #ifndef __XEN_PUBLIC_SCHED_H__
 #define __XEN_PUBLIC_SCHED_H__
 
-#include "event_channel.h"
+#include <xen/interface/event_channel.h>
 
 /*
  * The prototype for this hypercall is:
diff --git a/include/xen/interface/version.h b/include/xen/interface/version.h
index e8b6519d47e..5f5e551cf54 100644
--- a/include/xen/interface/version.h
+++ b/include/xen/interface/version.h
@@ -55,9 +55,12 @@ struct xen_feature_info {
 };
 
 /* Declares the features reported by XENVER_get_features. */
-#include "features.h"
+#include <xen/interface/features.h>
 
 /* arg == NULL; returns host memory page size. */
 #define XENVER_pagesize 7
 
+/* arg == xen_domain_handle_t. */
+#define XENVER_guest_handle 8
+
 #endif /* __XEN_PUBLIC_VERSION_H__ */
diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h
index 0801468f9ab..886a5d80a18 100644
--- a/include/xen/interface/xen.h
+++ b/include/xen/interface/xen.h
@@ -10,7 +10,6 @@
 #define __XEN_PUBLIC_XEN_H__
 
 #include <asm/xen/interface.h>
-#include <asm/pvclock-abi.h>
 
 /*
  * XEN "SYSTEM CALLS" (a.k.a. HYPERCALLS).
@@ -190,7 +189,7 @@ struct mmuext_op {
 	unsigned int cmd;
 	union {
 		/* [UN]PIN_TABLE, NEW_BASEPTR, NEW_USER_BASEPTR */
-		unsigned long mfn;
+		xen_pfn_t mfn;
 		/* INVLPG_LOCAL, INVLPG_ALL, SET_LDT */
 		unsigned long linear_addr;
 	} arg1;
@@ -430,11 +429,11 @@ struct start_info {
 	unsigned long nr_pages;     /* Total pages allocated to this domain.  */
 	unsigned long shared_info;  /* MACHINE address of shared info struct. */
 	uint32_t flags;             /* SIF_xxx flags.                         */
-	unsigned long store_mfn;    /* MACHINE page number of shared page.    */
+	xen_pfn_t store_mfn;        /* MACHINE page number of shared page.    */
 	uint32_t store_evtchn;      /* Event channel for store communication. */
 	union {
 		struct {
-			unsigned long mfn;  /* MACHINE page number of console page.   */
+			xen_pfn_t mfn;      /* MACHINE page number of console page.   */
 			uint32_t  evtchn;   /* Event channel for console page.        */
 		} domU;
 		struct {
@@ -455,6 +454,7 @@ struct dom0_vga_console_info {
 	uint8_t video_type;
 #define XEN_VGATYPE_TEXT_MODE_3 0x03
 #define XEN_VGATYPE_VESA_LFB    0x23
+#define XEN_VGATYPE_EFI_LFB     0x70
 
 	union {
 		struct {
diff --git a/include/xen/privcmd.h b/include/xen/privcmd.h
index 17857fb4d55..a85316811d7 100644
--- a/include/xen/privcmd.h
+++ b/include/xen/privcmd.h
@@ -35,8 +35,7 @@
 
 #include <linux/types.h>
 #include <linux/compiler.h>
-
-typedef unsigned long xen_pfn_t;
+#include <xen/interface/xen.h>
 
 struct privcmd_hypercall {
 	__u64 op;
@@ -59,13 +58,33 @@ struct privcmd_mmapbatch {
 	int num;     /* number of pages to populate */
 	domid_t dom; /* target domain */
 	__u64 addr;  /* virtual address */
-	xen_pfn_t __user *arr; /* array of mfns - top nibble set on err */
+	xen_pfn_t __user *arr; /* array of mfns - or'd with
+				  PRIVCMD_MMAPBATCH_*_ERROR on err */
+};
+
+#define PRIVCMD_MMAPBATCH_MFN_ERROR     0xf0000000U
+#define PRIVCMD_MMAPBATCH_PAGED_ERROR   0x80000000U
+
+struct privcmd_mmapbatch_v2 {
+	unsigned int num; /* number of pages to populate */
+	domid_t dom;      /* target domain */
+	__u64 addr;       /* virtual address */
+	const xen_pfn_t __user *arr; /* array of mfns */
+	int __user *err;  /* array of error codes */
 };
 
 /*
  * @cmd: IOCTL_PRIVCMD_HYPERCALL
  * @arg: &privcmd_hypercall_t
  * Return: Value returned from execution of the specified hypercall.
+ *
+ * @cmd: IOCTL_PRIVCMD_MMAPBATCH_V2
+ * @arg: &struct privcmd_mmapbatch_v2
+ * Return: 0 on success (i.e., arg->err contains valid error codes for
+ * each frame).  On an error other than a failed frame remap, -1 is
+ * returned and errno is set to EINVAL, EFAULT etc.  As an exception,
+ * if the operation was otherwise successful but any frame failed with
+ * -ENOENT, then -1 is returned and errno is set to ENOENT.
  */
 #define IOCTL_PRIVCMD_HYPERCALL					\
 	_IOC(_IOC_NONE, 'P', 0, sizeof(struct privcmd_hypercall))
@@ -73,5 +92,7 @@ struct privcmd_mmapbatch {
 	_IOC(_IOC_NONE, 'P', 2, sizeof(struct privcmd_mmap))
 #define IOCTL_PRIVCMD_MMAPBATCH					\
 	_IOC(_IOC_NONE, 'P', 3, sizeof(struct privcmd_mmapbatch))
+#define IOCTL_PRIVCMD_MMAPBATCH_V2				\
+	_IOC(_IOC_NONE, 'P', 4, sizeof(struct privcmd_mmapbatch_v2))
 
 #endif /* __LINUX_PUBLIC_PRIVCMD_H__ */
diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
index 4f4d449f00f..de8bcc641c4 100644
--- a/include/xen/swiotlb-xen.h
+++ b/include/xen/swiotlb-xen.h
@@ -3,7 +3,7 @@
 
 #include <linux/swiotlb.h>
 
-extern void xen_swiotlb_init(int verbose);
+extern int xen_swiotlb_init(int verbose, bool early);
 
 extern void
 *xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
@@ -23,15 +23,6 @@ extern dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
 extern void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
 				   size_t size, enum dma_data_direction dir,
 				   struct dma_attrs *attrs);
-/*
-extern int
-xen_swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
-		   enum dma_data_direction dir);
-
-extern void
-xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
-		     enum dma_data_direction dir);
-*/
 extern int
 xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
 			 int nelems, enum dma_data_direction dir,
diff --git a/init/Kconfig b/init/Kconfig
index cb003a3c912..ed6334dd5e7 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1199,6 +1199,7 @@ config BUG
           Just say Y.
 
 config ELF_CORE
+	depends on COREDUMP
 	default y
 	bool "Enable ELF core dumps" if EXPERT
 	help
@@ -1581,4 +1582,10 @@ config PADATA
 	depends on SMP
 	bool
 
+# Can be selected by architectures with broken toolchains
+# that get confused by correct const<->read_only section
+# mappings
+config BROKEN_RODATA
+	bool
+
 source "kernel/Kconfig.locks"
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 9a08acc9e64..6d255e535d0 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -944,7 +944,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
 		size_t, msg_len, unsigned int, msg_prio,
 		const struct timespec __user *, u_abs_timeout)
 {
-	struct file *filp;
+	struct fd f;
 	struct inode *inode;
 	struct ext_wait_queue wait;
 	struct ext_wait_queue *receiver;
@@ -967,21 +967,21 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
 
 	audit_mq_sendrecv(mqdes, msg_len, msg_prio, timeout ? &ts : NULL);
 
-	filp = fget(mqdes);
-	if (unlikely(!filp)) {
+	f = fdget(mqdes);
+	if (unlikely(!f.file)) {
 		ret = -EBADF;
 		goto out;
 	}
 
-	inode = filp->f_path.dentry->d_inode;
-	if (unlikely(filp->f_op != &mqueue_file_operations)) {
+	inode = f.file->f_path.dentry->d_inode;
+	if (unlikely(f.file->f_op != &mqueue_file_operations)) {
 		ret = -EBADF;
 		goto out_fput;
 	}
 	info = MQUEUE_I(inode);
-	audit_inode(NULL, filp->f_path.dentry);
+	audit_inode(NULL, f.file->f_path.dentry);
 
-	if (unlikely(!(filp->f_mode & FMODE_WRITE))) {
+	if (unlikely(!(f.file->f_mode & FMODE_WRITE))) {
 		ret = -EBADF;
 		goto out_fput;
 	}
@@ -1023,7 +1023,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
 	}
 
 	if (info->attr.mq_curmsgs == info->attr.mq_maxmsg) {
-		if (filp->f_flags & O_NONBLOCK) {
+		if (f.file->f_flags & O_NONBLOCK) {
 			ret = -EAGAIN;
 		} else {
 			wait.task = current;
@@ -1056,7 +1056,7 @@ out_free:
 	if (ret)
 		free_msg(msg_ptr);
 out_fput:
-	fput(filp);
+	fdput(f);
 out:
 	return ret;
 }
@@ -1067,7 +1067,7 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
 {
 	ssize_t ret;
 	struct msg_msg *msg_ptr;
-	struct file *filp;
+	struct fd f;
 	struct inode *inode;
 	struct mqueue_inode_info *info;
 	struct ext_wait_queue wait;
@@ -1084,21 +1084,21 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
 
 	audit_mq_sendrecv(mqdes, msg_len, 0, timeout ? &ts : NULL);
 
-	filp = fget(mqdes);
-	if (unlikely(!filp)) {
+	f = fdget(mqdes);
+	if (unlikely(!f.file)) {
 		ret = -EBADF;
 		goto out;
 	}
 
-	inode = filp->f_path.dentry->d_inode;
-	if (unlikely(filp->f_op != &mqueue_file_operations)) {
+	inode = f.file->f_path.dentry->d_inode;
+	if (unlikely(f.file->f_op != &mqueue_file_operations)) {
 		ret = -EBADF;
 		goto out_fput;
 	}
 	info = MQUEUE_I(inode);
-	audit_inode(NULL, filp->f_path.dentry);
+	audit_inode(NULL, f.file->f_path.dentry);
 
-	if (unlikely(!(filp->f_mode & FMODE_READ))) {
+	if (unlikely(!(f.file->f_mode & FMODE_READ))) {
 		ret = -EBADF;
 		goto out_fput;
 	}
@@ -1130,7 +1130,7 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
 	}
 
 	if (info->attr.mq_curmsgs == 0) {
-		if (filp->f_flags & O_NONBLOCK) {
+		if (f.file->f_flags & O_NONBLOCK) {
 			spin_unlock(&info->lock);
 			ret = -EAGAIN;
 		} else {
@@ -1160,7 +1160,7 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr,
 		free_msg(msg_ptr);
 	}
 out_fput:
-	fput(filp);
+	fdput(f);
 out:
 	return ret;
 }
@@ -1174,7 +1174,7 @@ SYSCALL_DEFINE2(mq_notify, mqd_t, mqdes,
 		const struct sigevent __user *, u_notification)
 {
 	int ret;
-	struct file *filp;
+	struct fd f;
 	struct sock *sock;
 	struct inode *inode;
 	struct sigevent notification;
@@ -1220,13 +1220,13 @@ SYSCALL_DEFINE2(mq_notify, mqd_t, mqdes,
 			skb_put(nc, NOTIFY_COOKIE_LEN);
 			/* and attach it to the socket */
 retry:
-			filp = fget(notification.sigev_signo);
-			if (!filp) {
+			f = fdget(notification.sigev_signo);
+			if (!f.file) {
 				ret = -EBADF;
 				goto out;
 			}
-			sock = netlink_getsockbyfilp(filp);
-			fput(filp);
+			sock = netlink_getsockbyfilp(f.file);
+			fdput(f);
 			if (IS_ERR(sock)) {
 				ret = PTR_ERR(sock);
 				sock = NULL;
@@ -1245,14 +1245,14 @@ retry:
 		}
 	}
 
-	filp = fget(mqdes);
-	if (!filp) {
+	f = fdget(mqdes);
+	if (!f.file) {
 		ret = -EBADF;
 		goto out;
 	}
 
-	inode = filp->f_path.dentry->d_inode;
-	if (unlikely(filp->f_op != &mqueue_file_operations)) {
+	inode = f.file->f_path.dentry->d_inode;
+	if (unlikely(f.file->f_op != &mqueue_file_operations)) {
 		ret = -EBADF;
 		goto out_fput;
 	}
@@ -1292,7 +1292,7 @@ retry:
 	}
 	spin_unlock(&info->lock);
 out_fput:
-	fput(filp);
+	fdput(f);
 out:
 	if (sock) {
 		netlink_detachskb(sock, nc);
@@ -1308,7 +1308,7 @@ SYSCALL_DEFINE3(mq_getsetattr, mqd_t, mqdes,
 {
 	int ret;
 	struct mq_attr mqstat, omqstat;
-	struct file *filp;
+	struct fd f;
 	struct inode *inode;
 	struct mqueue_inode_info *info;
 
@@ -1319,14 +1319,14 @@ SYSCALL_DEFINE3(mq_getsetattr, mqd_t, mqdes,
 			return -EINVAL;
 	}
 
-	filp = fget(mqdes);
-	if (!filp) {
+	f = fdget(mqdes);
+	if (!f.file) {
 		ret = -EBADF;
 		goto out;
 	}
 
-	inode = filp->f_path.dentry->d_inode;
-	if (unlikely(filp->f_op != &mqueue_file_operations)) {
+	inode = f.file->f_path.dentry->d_inode;
+	if (unlikely(f.file->f_op != &mqueue_file_operations)) {
 		ret = -EBADF;
 		goto out_fput;
 	}
@@ -1335,15 +1335,15 @@ SYSCALL_DEFINE3(mq_getsetattr, mqd_t, mqdes,
 	spin_lock(&info->lock);
 
 	omqstat = info->attr;
-	omqstat.mq_flags = filp->f_flags & O_NONBLOCK;
+	omqstat.mq_flags = f.file->f_flags & O_NONBLOCK;
 	if (u_mqstat) {
 		audit_mq_getsetattr(mqdes, &mqstat);
-		spin_lock(&filp->f_lock);
+		spin_lock(&f.file->f_lock);
 		if (mqstat.mq_flags & O_NONBLOCK)
-			filp->f_flags |= O_NONBLOCK;
+			f.file->f_flags |= O_NONBLOCK;
 		else
-			filp->f_flags &= ~O_NONBLOCK;
-		spin_unlock(&filp->f_lock);
+			f.file->f_flags &= ~O_NONBLOCK;
+		spin_unlock(&f.file->f_lock);
 
 		inode->i_atime = inode->i_ctime = CURRENT_TIME;
 	}
@@ -1356,7 +1356,7 @@ SYSCALL_DEFINE3(mq_getsetattr, mqd_t, mqdes,
 		ret = -EFAULT;
 
 out_fput:
-	fput(filp);
+	fdput(f);
 out:
 	return ret;
 }
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index ff4798fcb48..29e090cc0e4 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -1146,13 +1146,44 @@ error_path:
 
 EXPORT_SYMBOL(audit_log_task_context);
 
-static void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
+void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk)
 {
+	const struct cred *cred;
 	char name[sizeof(tsk->comm)];
 	struct mm_struct *mm = tsk->mm;
 	struct vm_area_struct *vma;
+	char *tty;
+
+	if (!ab)
+		return;
 
 	/* tsk == current */
+	cred = current_cred();
+
+	spin_lock_irq(&tsk->sighand->siglock);
+	if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name)
+		tty = tsk->signal->tty->name;
+	else
+		tty = "(none)";
+	spin_unlock_irq(&tsk->sighand->siglock);
+
+
+	audit_log_format(ab,
+			 " ppid=%ld pid=%d auid=%u uid=%u gid=%u"
+			 " euid=%u suid=%u fsuid=%u"
+			 " egid=%u sgid=%u fsgid=%u ses=%u tty=%s",
+			 sys_getppid(),
+			 tsk->pid,
+			 from_kuid(&init_user_ns, tsk->loginuid),
+			 from_kuid(&init_user_ns, cred->uid),
+			 from_kgid(&init_user_ns, cred->gid),
+			 from_kuid(&init_user_ns, cred->euid),
+			 from_kuid(&init_user_ns, cred->suid),
+			 from_kuid(&init_user_ns, cred->fsuid),
+			 from_kgid(&init_user_ns, cred->egid),
+			 from_kgid(&init_user_ns, cred->sgid),
+			 from_kgid(&init_user_ns, cred->fsgid),
+			 tsk->sessionid, tty);
 
 	get_task_comm(name, tsk);
 	audit_log_format(ab, " comm=");
@@ -1175,6 +1206,8 @@ static void audit_log_task_info(struct audit_buffer *ab, struct task_struct *tsk
 	audit_log_task_context(ab);
 }
 
+EXPORT_SYMBOL(audit_log_task_info);
+
 static int audit_log_pid_context(struct audit_context *context, pid_t pid,
 				 kuid_t auid, kuid_t uid, unsigned int sessionid,
 				 u32 sid, char *comm)
@@ -1580,26 +1613,12 @@ static void audit_log_name(struct audit_context *context, struct audit_names *n,
 
 static void audit_log_exit(struct audit_context *context, struct task_struct *tsk)
 {
-	const struct cred *cred;
 	int i, call_panic = 0;
 	struct audit_buffer *ab;
 	struct audit_aux_data *aux;
-	const char *tty;
 	struct audit_names *n;
 
 	/* tsk == current */
-	context->pid = tsk->pid;
-	if (!context->ppid)
-		context->ppid = sys_getppid();
-	cred = current_cred();
-	context->uid   = cred->uid;
-	context->gid   = cred->gid;
-	context->euid  = cred->euid;
-	context->suid  = cred->suid;
-	context->fsuid = cred->fsuid;
-	context->egid  = cred->egid;
-	context->sgid  = cred->sgid;
-	context->fsgid = cred->fsgid;
 	context->personality = tsk->personality;
 
 	ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL);
@@ -1614,37 +1633,13 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
 				 (context->return_valid==AUDITSC_SUCCESS)?"yes":"no",
 				 context->return_code);
 
-	spin_lock_irq(&tsk->sighand->siglock);
-	if (tsk->signal && tsk->signal->tty && tsk->signal->tty->name)
-		tty = tsk->signal->tty->name;
-	else
-		tty = "(none)";
-	spin_unlock_irq(&tsk->sighand->siglock);
-
 	audit_log_format(ab,
-		  " a0=%lx a1=%lx a2=%lx a3=%lx items=%d"
-		  " ppid=%d pid=%d auid=%u uid=%u gid=%u"
-		  " euid=%u suid=%u fsuid=%u"
-		  " egid=%u sgid=%u fsgid=%u tty=%s ses=%u",
-		  context->argv[0],
-		  context->argv[1],
-		  context->argv[2],
-		  context->argv[3],
-		  context->name_count,
-		  context->ppid,
-		  context->pid,
-		  from_kuid(&init_user_ns, tsk->loginuid),
-		  from_kuid(&init_user_ns, context->uid),
-		  from_kgid(&init_user_ns, context->gid),
-		  from_kuid(&init_user_ns, context->euid),
-		  from_kuid(&init_user_ns, context->suid),
-		  from_kuid(&init_user_ns, context->fsuid),
-		  from_kgid(&init_user_ns, context->egid),
-		  from_kgid(&init_user_ns, context->sgid),
-		  from_kgid(&init_user_ns, context->fsgid),
-		  tty,
-		  tsk->sessionid);
-
+			 " a0=%lx a1=%lx a2=%lx a3=%lx items=%d",
+			 context->argv[0],
+			 context->argv[1],
+			 context->argv[2],
+			 context->argv[3],
+			 context->name_count);
 
 	audit_log_task_info(ab, tsk);
 	audit_log_key(ab, context->filterkey);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index deec4e50eb3..f16f3c58f11 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -468,14 +468,13 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 {
 	struct perf_cgroup *cgrp;
 	struct cgroup_subsys_state *css;
-	struct file *file;
-	int ret = 0, fput_needed;
+	struct fd f = fdget(fd);
+	int ret = 0;
 
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	if (!f.file)
 		return -EBADF;
 
-	css = cgroup_css_from_dir(file, perf_subsys_id);
+	css = cgroup_css_from_dir(f.file, perf_subsys_id);
 	if (IS_ERR(css)) {
 		ret = PTR_ERR(css);
 		goto out;
@@ -501,7 +500,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 		ret = -EINVAL;
 	}
 out:
-	fput_light(file, fput_needed);
+	fdput(f);
 	return ret;
 }
 
@@ -3234,21 +3233,18 @@ unlock:
 
 static const struct file_operations perf_fops;
 
-static struct file *perf_fget_light(int fd, int *fput_needed)
+static inline int perf_fget_light(int fd, struct fd *p)
 {
-	struct file *file;
-
-	file = fget_light(fd, fput_needed);
-	if (!file)
-		return ERR_PTR(-EBADF);
+	struct fd f = fdget(fd);
+	if (!f.file)
+		return -EBADF;
 
-	if (file->f_op != &perf_fops) {
-		fput_light(file, *fput_needed);
-		*fput_needed = 0;
-		return ERR_PTR(-EBADF);
+	if (f.file->f_op != &perf_fops) {
+		fdput(f);
+		return -EBADF;
 	}
-
-	return file;
+	*p = f;
+	return 0;
 }
 
 static int perf_event_set_output(struct perf_event *event,
@@ -3280,22 +3276,19 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
 	case PERF_EVENT_IOC_SET_OUTPUT:
 	{
-		struct file *output_file = NULL;
-		struct perf_event *output_event = NULL;
-		int fput_needed = 0;
 		int ret;
-
 		if (arg != -1) {
-			output_file = perf_fget_light(arg, &fput_needed);
-			if (IS_ERR(output_file))
-				return PTR_ERR(output_file);
-			output_event = output_file->private_data;
+			struct perf_event *output_event;
+			struct fd output;
+			ret = perf_fget_light(arg, &output);
+			if (ret)
+				return ret;
+			output_event = output.file->private_data;
+			ret = perf_event_set_output(event, output_event);
+			fdput(output);
+		} else {
+			ret = perf_event_set_output(event, NULL);
 		}
-
-		ret = perf_event_set_output(event, output_event);
-		if (output_event)
-			fput_light(output_file, fput_needed);
-
 		return ret;
 	}
 
@@ -6443,12 +6436,11 @@ SYSCALL_DEFINE5(perf_event_open,
 	struct perf_event_attr attr;
 	struct perf_event_context *ctx;
 	struct file *event_file = NULL;
-	struct file *group_file = NULL;
+	struct fd group = {NULL, 0};
 	struct task_struct *task = NULL;
 	struct pmu *pmu;
 	int event_fd;
 	int move_group = 0;
-	int fput_needed = 0;
 	int err;
 
 	/* for future expandability... */
@@ -6478,17 +6470,15 @@ SYSCALL_DEFINE5(perf_event_open,
 	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
 		return -EINVAL;
 
-	event_fd = get_unused_fd_flags(O_RDWR);
+	event_fd = get_unused_fd();
 	if (event_fd < 0)
 		return event_fd;
 
 	if (group_fd != -1) {
-		group_file = perf_fget_light(group_fd, &fput_needed);
-		if (IS_ERR(group_file)) {
-			err = PTR_ERR(group_file);
+		err = perf_fget_light(group_fd, &group);
+		if (err)
 			goto err_fd;
-		}
-		group_leader = group_file->private_data;
+		group_leader = group.file->private_data;
 		if (flags & PERF_FLAG_FD_OUTPUT)
 			output_event = group_leader;
 		if (flags & PERF_FLAG_FD_NO_GROUP)
@@ -6664,7 +6654,7 @@ SYSCALL_DEFINE5(perf_event_open,
 	 * of the group leader will find the pointer to itself in
 	 * perf_group_detach().
 	 */
-	fput_light(group_file, fput_needed);
+	fdput(group);
 	fd_install(event_fd, event_file);
 	return event_fd;
 
@@ -6678,7 +6668,7 @@ err_task:
 	if (task)
 		put_task_struct(task);
 err_group_fd:
-	fput_light(group_file, fput_needed);
+	fdput(group);
 err_fd:
 	put_unused_fd(event_fd);
 	return err;
diff --git a/kernel/exit.c b/kernel/exit.c
index 42f25952edd..346616c0092 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -457,108 +457,13 @@ void daemonize(const char *name, ...)
 	/* Become as one with the init task */
 
 	daemonize_fs_struct();
-	exit_files(current);
-	current->files = init_task.files;
-	atomic_inc(&current->files->count);
+	daemonize_descriptors();
 
 	reparent_to_kthreadd();
 }
 
 EXPORT_SYMBOL(daemonize);
 
-static void close_files(struct files_struct * files)
-{
-	int i, j;
-	struct fdtable *fdt;
-
-	j = 0;
-
-	/*
-	 * It is safe to dereference the fd table without RCU or
-	 * ->file_lock because this is the last reference to the
-	 * files structure.  But use RCU to shut RCU-lockdep up.
-	 */
-	rcu_read_lock();
-	fdt = files_fdtable(files);
-	rcu_read_unlock();
-	for (;;) {
-		unsigned long set;
-		i = j * BITS_PER_LONG;
-		if (i >= fdt->max_fds)
-			break;
-		set = fdt->open_fds[j++];
-		while (set) {
-			if (set & 1) {
-				struct file * file = xchg(&fdt->fd[i], NULL);
-				if (file) {
-					filp_close(file, files);
-					cond_resched();
-				}
-			}
-			i++;
-			set >>= 1;
-		}
-	}
-}
-
-struct files_struct *get_files_struct(struct task_struct *task)
-{
-	struct files_struct *files;
-
-	task_lock(task);
-	files = task->files;
-	if (files)
-		atomic_inc(&files->count);
-	task_unlock(task);
-
-	return files;
-}
-
-void put_files_struct(struct files_struct *files)
-{
-	struct fdtable *fdt;
-
-	if (atomic_dec_and_test(&files->count)) {
-		close_files(files);
-		/*
-		 * Free the fd and fdset arrays if we expanded them.
-		 * If the fdtable was embedded, pass files for freeing
-		 * at the end of the RCU grace period. Otherwise,
-		 * you can free files immediately.
-		 */
-		rcu_read_lock();
-		fdt = files_fdtable(files);
-		if (fdt != &files->fdtab)
-			kmem_cache_free(files_cachep, files);
-		free_fdtable(fdt);
-		rcu_read_unlock();
-	}
-}
-
-void reset_files_struct(struct files_struct *files)
-{
-	struct task_struct *tsk = current;
-	struct files_struct *old;
-
-	old = tsk->files;
-	task_lock(tsk);
-	tsk->files = files;
-	task_unlock(tsk);
-	put_files_struct(old);
-}
-
-void exit_files(struct task_struct *tsk)
-{
-	struct files_struct * files = tsk->files;
-
-	if (files) {
-		task_lock(tsk);
-		tsk->files = NULL;
-		task_unlock(tsk);
-		put_files_struct(files);
-	}
-}
-
 #ifdef CONFIG_MM_OWNER
 /*
  * A task is exiting.   If it owned this mm, find a new owner for the mm.
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 43049192b5e..60f48fa0fd0 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -118,6 +118,7 @@ void jump_label_rate_limit(struct static_key_deferred *key,
 	key->timeout = rl;
 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
 }
+EXPORT_SYMBOL_GPL(jump_label_rate_limit);
 
 static int addr_conflict(struct jump_entry *entry, void *start, void *end)
 {
diff --git a/kernel/kexec.c b/kernel/kexec.c
index 0668d58d641..5e4bd7864c5 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -21,7 +21,6 @@
 #include <linux/hardirq.h>
 #include <linux/elf.h>
 #include <linux/elfcore.h>
-#include <generated/utsrelease.h>
 #include <linux/utsname.h>
 #include <linux/numa.h>
 #include <linux/suspend.h>
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index a70518c9d82..5dfdc9ea180 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -263,6 +263,10 @@ config PM_GENERIC_DOMAINS
 	bool
 	depends on PM
 
+config PM_GENERIC_DOMAINS_SLEEP
+	def_bool y
+	depends on PM_SLEEP && PM_GENERIC_DOMAINS
+
 config PM_GENERIC_DOMAINS_RUNTIME
 	def_bool y
 	depends on PM_RUNTIME && PM_GENERIC_DOMAINS
diff --git a/kernel/power/poweroff.c b/kernel/power/poweroff.c
index d52359374e8..68197a4e8fc 100644
--- a/kernel/power/poweroff.c
+++ b/kernel/power/poweroff.c
@@ -37,7 +37,7 @@ static struct sysrq_key_op	sysrq_poweroff_op = {
 	.enable_mask	= SYSRQ_ENABLE_BOOT,
 };
 
-static int pm_sysrq_init(void)
+static int __init pm_sysrq_init(void)
 {
 	register_sysrq_key('o', &sysrq_poweroff_op);
 	return 0;
diff --git a/kernel/power/process.c b/kernel/power/process.c
index 19db29f6755..87da817f9e1 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -79,7 +79,7 @@ static int try_to_freeze_tasks(bool user_only)
 
 		/*
 		 * We need to retry, but first give the freezing tasks some
-		 * time to enter the regrigerator.
+		 * time to enter the refrigerator.
 		 */
 		msleep(10);
 	}
diff --git a/kernel/power/qos.c b/kernel/power/qos.c
index 6a031e68402..846bd42c7ed 100644
--- a/kernel/power/qos.c
+++ b/kernel/power/qos.c
@@ -139,6 +139,7 @@ static inline int pm_qos_get_value(struct pm_qos_constraints *c)
 	default:
 		/* runtime check for not using enum */
 		BUG();
+		return PM_QOS_DEFAULT_VALUE;
 	}
 }
 
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index a232bb59d93..1f5e55dda95 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -180,7 +180,8 @@ static int ptrace_has_cap(struct user_namespace *ns, unsigned int mode)
 		return has_ns_capability(current, ns, CAP_SYS_PTRACE);
 }
 
-int __ptrace_may_access(struct task_struct *task, unsigned int mode)
+/* Returns 0 on success, -errno on denial. */
+static int __ptrace_may_access(struct task_struct *task, unsigned int mode)
 {
 	const struct cred *cred = current_cred(), *tcred;
 
diff --git a/kernel/resource.c b/kernel/resource.c
index 34d45886ee8..73f35d4b30b 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -763,6 +763,7 @@ static void __init __reserve_region_with_split(struct resource *root,
 	struct resource *parent = root;
 	struct resource *conflict;
 	struct resource *res = kzalloc(sizeof(*res), GFP_ATOMIC);
+	struct resource *next_res = NULL;
 
 	if (!res)
 		return;
@@ -772,21 +773,46 @@ static void __init __reserve_region_with_split(struct resource *root,
 	res->end = end;
 	res->flags = IORESOURCE_BUSY;
 
-	conflict = __request_resource(parent, res);
-	if (!conflict)
-		return;
+	while (1) {
 
-	/* failed, split and try again */
-	kfree(res);
+		conflict = __request_resource(parent, res);
+		if (!conflict) {
+			if (!next_res)
+				break;
+			res = next_res;
+			next_res = NULL;
+			continue;
+		}
 
-	/* conflict covered whole area */
-	if (conflict->start <= start && conflict->end >= end)
-		return;
+		/* conflict covered whole area */
+		if (conflict->start <= res->start &&
+				conflict->end >= res->end) {
+			kfree(res);
+			WARN_ON(next_res);
+			break;
+		}
+
+		/* failed, split and try again */
+		if (conflict->start > res->start) {
+			end = res->end;
+			res->end = conflict->start - 1;
+			if (conflict->end < end) {
+				next_res = kzalloc(sizeof(*next_res),
+						GFP_ATOMIC);
+				if (!next_res) {
+					kfree(res);
+					break;
+				}
+				next_res->name = name;
+				next_res->start = conflict->end + 1;
+				next_res->end = end;
+				next_res->flags = IORESOURCE_BUSY;
+			}
+		} else {
+			res->start = conflict->end + 1;
+		}
+	}
 
-	if (conflict->start > start)
-		__reserve_region_with_split(root, start, conflict->start-1, name);
-	if (conflict->end < end)
-		__reserve_region_with_split(root, conflict->end+1, end, name);
 }
 
 void __init reserve_region_with_split(struct resource *root,
diff --git a/kernel/signal.c b/kernel/signal.c
index 2c681f11b7d..0af8868525d 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -17,6 +17,7 @@
 #include <linux/fs.h>
 #include <linux/tty.h>
 #include <linux/binfmts.h>
+#include <linux/coredump.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/ptrace.h>
@@ -2359,7 +2360,7 @@ relock:
 			 * first and our do_group_exit call below will use
 			 * that value and ignore the one we pass it.
 			 */
-			do_coredump(info->si_signo, info->si_signo, regs);
+			do_coredump(info, regs);
 		}
 
 		/*
diff --git a/kernel/sys.c b/kernel/sys.c
index 241507f23ec..c5cb5b99cb8 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -368,6 +368,7 @@ EXPORT_SYMBOL(unregister_reboot_notifier);
 void kernel_restart(char *cmd)
 {
 	kernel_restart_prepare(cmd);
+	disable_nonboot_cpus();
 	if (!cmd)
 		printk(KERN_EMERG "Restarting system.\n");
 	else
@@ -1788,15 +1789,15 @@ SYSCALL_DEFINE1(umask, int, mask)
 #ifdef CONFIG_CHECKPOINT_RESTORE
 static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd)
 {
-	struct file *exe_file;
+	struct fd exe;
 	struct dentry *dentry;
 	int err;
 
-	exe_file = fget(fd);
-	if (!exe_file)
+	exe = fdget(fd);
+	if (!exe.file)
 		return -EBADF;
 
-	dentry = exe_file->f_path.dentry;
+	dentry = exe.file->f_path.dentry;
 
 	/*
 	 * Because the original mm->exe_file points to executable file, make
@@ -1805,7 +1806,7 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd)
 	 */
 	err = -EACCES;
 	if (!S_ISREG(dentry->d_inode->i_mode)	||
-	    exe_file->f_path.mnt->mnt_flags & MNT_NOEXEC)
+	    exe.file->f_path.mnt->mnt_flags & MNT_NOEXEC)
 		goto exit;
 
 	err = inode_permission(dentry->d_inode, MAY_EXEC);
@@ -1839,12 +1840,12 @@ static int prctl_set_mm_exe_file(struct mm_struct *mm, unsigned int fd)
 		goto exit_unlock;
 
 	err = 0;
-	set_mm_exe_file(mm, exe_file);
+	set_mm_exe_file(mm, exe.file);	/* this grabs a reference to exe.file */
 exit_unlock:
 	up_write(&mm->mmap_sem);
 
 exit:
-	fput(exe_file);
+	fdput(exe);
 	return err;
 }
 
@@ -2204,7 +2205,7 @@ static int __orderly_poweroff(void)
 		return -ENOMEM;
 	}
 
-	ret = call_usermodehelper_fns(argv[0], argv, envp, UMH_NO_WAIT,
+	ret = call_usermodehelper_fns(argv[0], argv, envp, UMH_WAIT_EXEC,
 				      NULL, argv_cleanup, NULL);
 	if (ret == -ENOMEM)
 		argv_free(argv);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 84c76a34e41..c2a2f8084ba 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -97,10 +97,12 @@
 extern int sysctl_overcommit_memory;
 extern int sysctl_overcommit_ratio;
 extern int max_threads;
-extern int core_uses_pid;
 extern int suid_dumpable;
+#ifdef CONFIG_COREDUMP
+extern int core_uses_pid;
 extern char core_pattern[];
 extern unsigned int core_pipe_limit;
+#endif
 extern int pid_max;
 extern int min_free_kbytes;
 extern int pid_max_min, pid_max_max;
@@ -177,8 +179,10 @@ static int proc_dointvec_minmax_sysadmin(struct ctl_table *table, int write,
 
 static int proc_dointvec_minmax_coredump(struct ctl_table *table, int write,
 		void __user *buffer, size_t *lenp, loff_t *ppos);
+#ifdef CONFIG_COREDUMP
 static int proc_dostring_coredump(struct ctl_table *table, int write,
 		void __user *buffer, size_t *lenp, loff_t *ppos);
+#endif
 
 #ifdef CONFIG_MAGIC_SYSRQ
 /* Note: sysrq code uses it's own private copy */
@@ -404,6 +408,7 @@ static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec,
 	},
+#ifdef CONFIG_COREDUMP
 	{
 		.procname	= "core_uses_pid",
 		.data		= &core_uses_pid,
@@ -425,6 +430,7 @@ static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec,
 	},
+#endif
 #ifdef CONFIG_PROC_SYSCTL
 	{
 		.procname	= "tainted",
@@ -2036,12 +2042,14 @@ int proc_dointvec_minmax(struct ctl_table *table, int write,
 
 static void validate_coredump_safety(void)
 {
+#ifdef CONFIG_COREDUMP
 	if (suid_dumpable == SUID_DUMPABLE_SAFE &&
 	    core_pattern[0] != '/' && core_pattern[0] != '|') {
 		printk(KERN_WARNING "Unsafe core_pattern used with "\
 			"suid_dumpable=2. Pipe handler or fully qualified "\
 			"core dump path required.\n");
 	}
+#endif
 }
 
 static int proc_dointvec_minmax_coredump(struct ctl_table *table, int write,
@@ -2053,6 +2061,7 @@ static int proc_dointvec_minmax_coredump(struct ctl_table *table, int write,
 	return error;
 }
 
+#ifdef CONFIG_COREDUMP
 static int proc_dostring_coredump(struct ctl_table *table, int write,
 		  void __user *buffer, size_t *lenp, loff_t *ppos)
 {
@@ -2061,6 +2070,7 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
 		validate_coredump_safety();
 	return error;
 }
+#endif
 
 static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, int write,
 				     void __user *buffer,
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 5eab1f3edfa..145bb4d3bd4 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -424,16 +424,15 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 	struct nlattr *na;
 	size_t size;
 	u32 fd;
-	struct file *file;
-	int fput_needed;
+	struct fd f;
 
 	na = info->attrs[CGROUPSTATS_CMD_ATTR_FD];
 	if (!na)
 		return -EINVAL;
 
 	fd = nla_get_u32(info->attrs[CGROUPSTATS_CMD_ATTR_FD]);
-	file = fget_light(fd, &fput_needed);
-	if (!file)
+	f = fdget(fd);
+	if (!f.file)
 		return 0;
 
 	size = nla_total_size(sizeof(struct cgroupstats));
@@ -446,6 +445,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 	na = nla_reserve(rep_skb, CGROUPSTATS_TYPE_CGROUP_STATS,
 				sizeof(struct cgroupstats));
 	if (na == NULL) {
+		nlmsg_free(rep_skb);
 		rc = -EMSGSIZE;
 		goto err;
 	}
@@ -453,7 +453,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 	stats = nla_data(na);
 	memset(stats, 0, sizeof(*stats));
 
-	rc = cgroupstats_build(stats, file->f_dentry);
+	rc = cgroupstats_build(stats, f.file->f_dentry);
 	if (rc < 0) {
 		nlmsg_free(rep_skb);
 		goto err;
@@ -462,7 +462,7 @@ static int cgroupstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
 	rc = send_reply(rep_skb, info);
 
 err:
-	fput_light(file, fput_needed);
+	fdput(f);
 	return rc;
 }
 
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 7e1ce012a85..30b6de0d977 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -397,6 +397,30 @@ void clockevents_exchange_device(struct clock_event_device *old,
 	local_irq_restore(flags);
 }
 
+/**
+ * clockevents_suspend - suspend clock devices
+ */
+void clockevents_suspend(void)
+{
+	struct clock_event_device *dev;
+
+	list_for_each_entry_reverse(dev, &clockevent_devices, list)
+		if (dev->suspend)
+			dev->suspend(dev);
+}
+
+/**
+ * clockevents_resume - resume clock devices
+ */
+void clockevents_resume(void)
+{
+	struct clock_event_device *dev;
+
+	list_for_each_entry(dev, &clockevent_devices, list)
+		if (dev->resume)
+			dev->resume(dev);
+}
+
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
 /**
  * clockevents_notify - notification about relevant events
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index d3b91e75cec..5ce06a3fa91 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -776,6 +776,7 @@ static void timekeeping_resume(void)
 
 	read_persistent_clock(&ts);
 
+	clockevents_resume();
 	clocksource_resume();
 
 	write_seqlock_irqsave(&tk->lock, flags);
@@ -835,6 +836,7 @@ static int timekeeping_suspend(void)
 
 	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
 	clocksource_suspend();
+	clockevents_suspend();
 
 	return 0;
 }
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 35c4565ee8f..7fba3a98967 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -196,12 +196,13 @@ config LOCKUP_DETECTOR
 	  thresholds can be controlled through the sysctl watchdog_thresh.
 
 config HARDLOCKUP_DETECTOR
-	def_bool LOCKUP_DETECTOR && PERF_EVENTS && HAVE_PERF_EVENTS_NMI && \
-		 !HAVE_NMI_WATCHDOG
+	def_bool y
+	depends on LOCKUP_DETECTOR && !HAVE_NMI_WATCHDOG
+	depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI
 
 config BOOTPARAM_HARDLOCKUP_PANIC
 	bool "Panic (Reboot) On Hard Lockups"
-	depends on LOCKUP_DETECTOR
+	depends on HARDLOCKUP_DETECTOR
 	help
 	  Say Y here to enable the kernel to panic on "hard lockups",
 	  which are bugs that cause the kernel to loop in kernel
@@ -212,7 +213,7 @@ config BOOTPARAM_HARDLOCKUP_PANIC
 
 config BOOTPARAM_HARDLOCKUP_PANIC_VALUE
 	int
-	depends on LOCKUP_DETECTOR
+	depends on HARDLOCKUP_DETECTOR
 	range 0 1
 	default 0 if !BOOTPARAM_HARDLOCKUP_PANIC
 	default 1 if BOOTPARAM_HARDLOCKUP_PANIC
diff --git a/lib/crc32.c b/lib/crc32.c
index 61774b8db4d..072fbd8234d 100644
--- a/lib/crc32.c
+++ b/lib/crc32.c
@@ -188,11 +188,13 @@ u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
 #else
 u32 __pure crc32_le(u32 crc, unsigned char const *p, size_t len)
 {
-	return crc32_le_generic(crc, p, len, crc32table_le, CRCPOLY_LE);
+	return crc32_le_generic(crc, p, len,
+			(const u32 (*)[256])crc32table_le, CRCPOLY_LE);
 }
 u32 __pure __crc32c_le(u32 crc, unsigned char const *p, size_t len)
 {
-	return crc32_le_generic(crc, p, len, crc32ctable_le, CRC32C_POLY_LE);
+	return crc32_le_generic(crc, p, len,
+			(const u32 (*)[256])crc32ctable_le, CRC32C_POLY_LE);
 }
 #endif
 EXPORT_SYMBOL(crc32_le);
@@ -253,7 +255,8 @@ u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
 #else
 u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len)
 {
-	return crc32_be_generic(crc, p, len, crc32table_be, CRCPOLY_BE);
+	return crc32_be_generic(crc, p, len,
+			(const u32 (*)[256])crc32table_be, CRCPOLY_BE);
 }
 #endif
 EXPORT_SYMBOL(crc32_be);
diff --git a/lib/decompress.c b/lib/decompress.c
index 3d766b7f60a..31a80427728 100644
--- a/lib/decompress.c
+++ b/lib/decompress.c
@@ -14,6 +14,7 @@
 
 #include <linux/types.h>
 #include <linux/string.h>
+#include <linux/init.h>
 
 #ifndef CONFIG_DECOMPRESS_GZIP
 # define gunzip NULL
@@ -31,11 +32,13 @@
 # define unlzo NULL
 #endif
 
-static const struct compress_format {
+struct compress_format {
 	unsigned char magic[2];
 	const char *name;
 	decompress_fn decompressor;
-} compressed_formats[] = {
+};
+
+static const struct compress_format compressed_formats[] __initdata = {
 	{ {037, 0213}, "gzip", gunzip },
 	{ {037, 0236}, "gzip", gunzip },
 	{ {0x42, 0x5a}, "bzip2", bunzip2 },
@@ -45,7 +48,7 @@ static const struct compress_format {
 	{ {0, 0}, NULL, NULL }
 };
 
-decompress_fn decompress_method(const unsigned char *inbuf, int len,
+decompress_fn __init decompress_method(const unsigned char *inbuf, int len,
 				const char **name)
 {
 	const struct compress_format *cf;
diff --git a/lib/gcd.c b/lib/gcd.c
index cce4f3cd14b..3657f129d7b 100644
--- a/lib/gcd.c
+++ b/lib/gcd.c
@@ -9,6 +9,9 @@ unsigned long gcd(unsigned long a, unsigned long b)
 
 	if (a < b)
 		swap(a, b);
+
+	if (!b)
+		return a;
 	while ((r = a % b) != 0) {
 		a = b;
 		b = r;
diff --git a/lib/gen_crc32table.c b/lib/gen_crc32table.c
index 8f8d5439e2d..71fcfcd9641 100644
--- a/lib/gen_crc32table.c
+++ b/lib/gen_crc32table.c
@@ -109,7 +109,7 @@ int main(int argc, char** argv)
 
 	if (CRC_LE_BITS > 1) {
 		crc32init_le();
-		printf("static const u32 __cacheline_aligned "
+		printf("static u32 __cacheline_aligned "
 		       "crc32table_le[%d][%d] = {",
 		       LE_TABLE_ROWS, LE_TABLE_SIZE);
 		output_table(crc32table_le, LE_TABLE_ROWS,
@@ -119,7 +119,7 @@ int main(int argc, char** argv)
 
 	if (CRC_BE_BITS > 1) {
 		crc32init_be();
-		printf("static const u32 __cacheline_aligned "
+		printf("static u32 __cacheline_aligned "
 		       "crc32table_be[%d][%d] = {",
 		       BE_TABLE_ROWS, BE_TABLE_SIZE);
 		output_table(crc32table_be, LE_TABLE_ROWS,
@@ -128,7 +128,7 @@ int main(int argc, char** argv)
 	}
 	if (CRC_LE_BITS > 1) {
 		crc32cinit_le();
-		printf("static const u32 __cacheline_aligned "
+		printf("static u32 __cacheline_aligned "
 		       "crc32ctable_le[%d][%d] = {",
 		       LE_TABLE_ROWS, LE_TABLE_SIZE);
 		output_table(crc32ctable_le, LE_TABLE_ROWS,
diff --git a/lib/genalloc.c b/lib/genalloc.c
index 6bc04aab6ec..ca208a92628 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -152,6 +152,8 @@ struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
 		spin_lock_init(&pool->lock);
 		INIT_LIST_HEAD(&pool->chunks);
 		pool->min_alloc_order = min_alloc_order;
+		pool->algo = gen_pool_first_fit;
+		pool->data = NULL;
 	}
 	return pool;
 }
@@ -255,8 +257,9 @@ EXPORT_SYMBOL(gen_pool_destroy);
  * @size: number of bytes to allocate from the pool
  *
  * Allocate the requested number of bytes from the specified pool.
- * Uses a first-fit algorithm. Can not be used in NMI handler on
- * architectures without NMI-safe cmpxchg implementation.
+ * Uses the pool allocation function (with first-fit algorithm by default).
+ * Can not be used in NMI handler on architectures without
+ * NMI-safe cmpxchg implementation.
  */
 unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
 {
@@ -280,8 +283,8 @@ unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
 
 		end_bit = (chunk->end_addr - chunk->start_addr) >> order;
 retry:
-		start_bit = bitmap_find_next_zero_area(chunk->bits, end_bit,
-						       start_bit, nbits, 0);
+		start_bit = pool->algo(chunk->bits, end_bit, start_bit, nbits,
+				pool->data);
 		if (start_bit >= end_bit)
 			continue;
 		remain = bitmap_set_ll(chunk->bits, start_bit, nbits);
@@ -400,3 +403,80 @@ size_t gen_pool_size(struct gen_pool *pool)
 	return size;
 }
 EXPORT_SYMBOL_GPL(gen_pool_size);
+
+/**
+ * gen_pool_set_algo - set the allocation algorithm
+ * @pool: pool to change allocation algorithm
+ * @algo: custom algorithm function
+ * @data: additional data used by @algo
+ *
+ * Call @algo for each memory allocation in the pool.
+ * If @algo is NULL use gen_pool_first_fit as default
+ * memory allocation function.
+ */
+void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo, void *data)
+{
+	rcu_read_lock();
+
+	pool->algo = algo;
+	if (!pool->algo)
+		pool->algo = gen_pool_first_fit;
+
+	pool->data = data;
+
+	rcu_read_unlock();
+}
+EXPORT_SYMBOL(gen_pool_set_algo);
+
+/**
+ * gen_pool_first_fit - find the first available region
+ * of memory matching the size requirement (no alignment constraint)
+ * @map: The address to base the search on
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
+ * @nr: The number of zeroed bits we're looking for
+ * @data: additional data - unused
+ */
+unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size,
+		unsigned long start, unsigned int nr, void *data)
+{
+	return bitmap_find_next_zero_area(map, size, start, nr, 0);
+}
+EXPORT_SYMBOL(gen_pool_first_fit);
+
+/**
+ * gen_pool_best_fit - find the best fitting region of memory
+ * macthing the size requirement (no alignment constraint)
+ * @map: The address to base the search on
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
+ * @nr: The number of zeroed bits we're looking for
+ * @data: additional data - unused
+ *
+ * Iterate over the bitmap to find the smallest free region
+ * which we can allocate the memory.
+ */
+unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
+		unsigned long start, unsigned int nr, void *data)
+{
+	unsigned long start_bit = size;
+	unsigned long len = size + 1;
+	unsigned long index;
+
+	index = bitmap_find_next_zero_area(map, size, start, nr, 0);
+
+	while (index < size) {
+		int next_bit = find_next_bit(map, size, index + nr);
+		if ((next_bit - index) < len) {
+			len = next_bit - index;
+			start_bit = index;
+			if (len == nr)
+				return start_bit;
+		}
+		index = bitmap_find_next_zero_area(map, size,
+						   next_bit + 1, nr, 0);
+	}
+
+	return start_bit;
+}
+EXPORT_SYMBOL(gen_pool_best_fit);
diff --git a/lib/idr.c b/lib/idr.c
index 4046e29c0a9..648239079dd 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -20,7 +20,7 @@
  * that id to this code and it returns your pointer.
 
  * You can release ids at any time. When all ids are released, most of
- * the memory is returned (we keep IDR_FREE_MAX) in a local pool so we
+ * the memory is returned (we keep MAX_IDR_FREE) in a local pool so we
  * don't need to go to the memory "store" during an id allocate, just
  * so you don't need to be too concerned about locking and conflicts
  * with the slab allocator.
@@ -122,7 +122,7 @@ static void idr_mark_full(struct idr_layer **pa, int id)
  */
 int idr_pre_get(struct idr *idp, gfp_t gfp_mask)
 {
-	while (idp->id_free_cnt < IDR_FREE_MAX) {
+	while (idp->id_free_cnt < MAX_IDR_FREE) {
 		struct idr_layer *new;
 		new = kmem_cache_zalloc(idr_layer_cache, gfp_mask);
 		if (new == NULL)
@@ -179,7 +179,7 @@ static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa)
 			sh = IDR_BITS*l;
 			id = ((id >> sh) ^ n ^ m) << sh;
 		}
-		if ((id >= MAX_ID_BIT) || (id < 0))
+		if ((id >= MAX_IDR_BIT) || (id < 0))
 			return IDR_NOMORE_SPACE;
 		if (l == 0)
 			break;
@@ -223,7 +223,7 @@ build_up:
 	 * Add a new layer to the top of the tree if the requested
 	 * id is larger than the currently allocated space.
 	 */
-	while ((layers < (MAX_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) {
+	while ((layers < (MAX_IDR_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) {
 		layers++;
 		if (!p->count) {
 			/* special case: if the tree is currently empty,
@@ -265,7 +265,7 @@ build_up:
 
 static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id)
 {
-	struct idr_layer *pa[MAX_LEVEL];
+	struct idr_layer *pa[MAX_IDR_LEVEL];
 	int id;
 
 	id = idr_get_empty_slot(idp, starting_id, pa);
@@ -357,7 +357,7 @@ static void idr_remove_warning(int id)
 static void sub_remove(struct idr *idp, int shift, int id)
 {
 	struct idr_layer *p = idp->top;
-	struct idr_layer **pa[MAX_LEVEL];
+	struct idr_layer **pa[MAX_IDR_LEVEL];
 	struct idr_layer ***paa = &pa[0];
 	struct idr_layer *to_free;
 	int n;
@@ -402,7 +402,7 @@ void idr_remove(struct idr *idp, int id)
 	struct idr_layer *to_free;
 
 	/* Mask off upper bits we don't use for the search. */
-	id &= MAX_ID_MASK;
+	id &= MAX_IDR_MASK;
 
 	sub_remove(idp, (idp->layers - 1) * IDR_BITS, id);
 	if (idp->top && idp->top->count == 1 && (idp->layers > 1) &&
@@ -420,7 +420,7 @@ void idr_remove(struct idr *idp, int id)
 		to_free->bitmap = to_free->count = 0;
 		free_layer(to_free);
 	}
-	while (idp->id_free_cnt >= IDR_FREE_MAX) {
+	while (idp->id_free_cnt >= MAX_IDR_FREE) {
 		p = get_from_free_list(idp);
 		/*
 		 * Note: we don't call the rcu callback here, since the only
@@ -451,7 +451,7 @@ void idr_remove_all(struct idr *idp)
 	int n, id, max;
 	int bt_mask;
 	struct idr_layer *p;
-	struct idr_layer *pa[MAX_LEVEL];
+	struct idr_layer *pa[MAX_IDR_LEVEL];
 	struct idr_layer **paa = &pa[0];
 
 	n = idp->layers * IDR_BITS;
@@ -517,7 +517,7 @@ void *idr_find(struct idr *idp, int id)
 	n = (p->layer+1) * IDR_BITS;
 
 	/* Mask off upper bits we don't use for the search. */
-	id &= MAX_ID_MASK;
+	id &= MAX_IDR_MASK;
 
 	if (id >= (1 << n))
 		return NULL;
@@ -555,7 +555,7 @@ int idr_for_each(struct idr *idp,
 {
 	int n, id, max, error = 0;
 	struct idr_layer *p;
-	struct idr_layer *pa[MAX_LEVEL];
+	struct idr_layer *pa[MAX_IDR_LEVEL];
 	struct idr_layer **paa = &pa[0];
 
 	n = idp->layers * IDR_BITS;
@@ -601,7 +601,7 @@ EXPORT_SYMBOL(idr_for_each);
  */
 void *idr_get_next(struct idr *idp, int *nextidp)
 {
-	struct idr_layer *p, *pa[MAX_LEVEL];
+	struct idr_layer *p, *pa[MAX_IDR_LEVEL];
 	struct idr_layer **paa = &pa[0];
 	int id = *nextidp;
 	int n, max;
@@ -659,7 +659,7 @@ void *idr_replace(struct idr *idp, void *ptr, int id)
 
 	n = (p->layer+1) * IDR_BITS;
 
-	id &= MAX_ID_MASK;
+	id &= MAX_IDR_MASK;
 
 	if (id >= (1 << n))
 		return ERR_PTR(-EINVAL);
@@ -780,7 +780,7 @@ EXPORT_SYMBOL(ida_pre_get);
  */
 int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
 {
-	struct idr_layer *pa[MAX_LEVEL];
+	struct idr_layer *pa[MAX_IDR_LEVEL];
 	struct ida_bitmap *bitmap;
 	unsigned long flags;
 	int idr_id = starting_id / IDA_BITMAP_BITS;
@@ -793,7 +793,7 @@ int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
 	if (t < 0)
 		return _idr_rc_to_errno(t);
 
-	if (t * IDA_BITMAP_BITS >= MAX_ID_BIT)
+	if (t * IDA_BITMAP_BITS >= MAX_IDR_BIT)
 		return -ENOSPC;
 
 	if (t != idr_id)
@@ -827,7 +827,7 @@ int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
 	}
 
 	id = idr_id * IDA_BITMAP_BITS + t;
-	if (id >= MAX_ID_BIT)
+	if (id >= MAX_IDR_BIT)
 		return -ENOSPC;
 
 	__set_bit(t, bitmap->bitmap);
diff --git a/lib/parser.c b/lib/parser.c
index c4341008483..52cfa69f73d 100644
--- a/lib/parser.c
+++ b/lib/parser.c
@@ -122,13 +122,14 @@ int match_token(char *s, const match_table_t table, substring_t args[])
  *
  * Description: Given a &substring_t and a base, attempts to parse the substring
  * as a number in that base. On success, sets @result to the integer represented
- * by the string and returns 0. Returns either -ENOMEM or -EINVAL on failure.
+ * by the string and returns 0. Returns -ENOMEM, -EINVAL, or -ERANGE on failure.
  */
 static int match_number(substring_t *s, int *result, int base)
 {
 	char *endp;
 	char *buf;
 	int ret;
+	long val;
 	size_t len = s->to - s->from;
 
 	buf = kmalloc(len + 1, GFP_KERNEL);
@@ -136,10 +137,15 @@ static int match_number(substring_t *s, int *result, int base)
 		return -ENOMEM;
 	memcpy(buf, s->from, len);
 	buf[len] = '\0';
-	*result = simple_strtol(buf, &endp, base);
+
 	ret = 0;
+	val = simple_strtol(buf, &endp, base);
 	if (endp == buf)
 		ret = -EINVAL;
+	else if (val < (long)INT_MIN || val > (long)INT_MAX)
+		ret = -ERANGE;
+	else
+		*result = (int) val;
 	kfree(buf);
 	return ret;
 }
diff --git a/lib/plist.c b/lib/plist.c
index 6ab0e521c48..1ebc95f7a46 100644
--- a/lib/plist.c
+++ b/lib/plist.c
@@ -175,7 +175,7 @@ static int  __init plist_test(void)
 	int nr_expect = 0, i, loop;
 	unsigned int r = local_clock();
 
-	printk(KERN_INFO "start plist test\n");
+	pr_debug("start plist test\n");
 	plist_head_init(&test_head);
 	for (i = 0; i < ARRAY_SIZE(test_node); i++)
 		plist_node_init(test_node + i, 0);
@@ -203,7 +203,7 @@ static int  __init plist_test(void)
 		plist_test_check(nr_expect);
 	}
 
-	printk(KERN_INFO "end plist test\n");
+	pr_debug("end plist test\n");
 	return 0;
 }
 
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index fadae774a20..e76d85cf317 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -404,14 +404,13 @@ EXPORT_SYMBOL(sg_miter_start);
  * @miter: sg mapping iter to proceed
  *
  * Description:
- *   Proceeds @miter@ to the next mapping.  @miter@ should have been
- *   started using sg_miter_start().  On successful return,
- *   @miter@->page, @miter@->addr and @miter@->length point to the
- *   current mapping.
+ *   Proceeds @miter to the next mapping.  @miter should have been started
+ *   using sg_miter_start().  On successful return, @miter->page,
+ *   @miter->addr and @miter->length point to the current mapping.
  *
  * Context:
- *   IRQ disabled if SG_MITER_ATOMIC.  IRQ must stay disabled till
- *   @miter@ is stopped.  May sleep if !SG_MITER_ATOMIC.
+ *   Preemption disabled if SG_MITER_ATOMIC.  Preemption must stay disabled
+ *   till @miter is stopped.  May sleep if !SG_MITER_ATOMIC.
  *
  * Returns:
  *   true if @miter contains the next mapping.  false if end of sg
@@ -465,7 +464,8 @@ EXPORT_SYMBOL(sg_miter_next);
  *   resources (kmap) need to be released during iteration.
  *
  * Context:
- *   IRQ disabled if the SG_MITER_ATOMIC is set.  Don't care otherwise.
+ *   Preemption disabled if the SG_MITER_ATOMIC is set.  Don't care
+ *   otherwise.
  */
 void sg_miter_stop(struct sg_mapping_iter *miter)
 {
@@ -479,7 +479,7 @@ void sg_miter_stop(struct sg_mapping_iter *miter)
 			flush_kernel_dcache_page(miter->page);
 
 		if (miter->__flags & SG_MITER_ATOMIC) {
-			WARN_ON(!irqs_disabled());
+			WARN_ON_ONCE(preemptible());
 			kunmap_atomic(miter->addr);
 		} else
 			kunmap(miter->page);
diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c
index eb10578ae05..0374a596cff 100644
--- a/lib/spinlock_debug.c
+++ b/lib/spinlock_debug.c
@@ -107,23 +107,27 @@ static void __spin_lock_debug(raw_spinlock_t *lock)
 {
 	u64 i;
 	u64 loops = loops_per_jiffy * HZ;
-	int print_once = 1;
 
-	for (;;) {
-		for (i = 0; i < loops; i++) {
-			if (arch_spin_trylock(&lock->raw_lock))
-				return;
-			__delay(1);
-		}
-		/* lockup suspected: */
-		if (print_once) {
-			print_once = 0;
-			spin_dump(lock, "lockup suspected");
+	for (i = 0; i < loops; i++) {
+		if (arch_spin_trylock(&lock->raw_lock))
+			return;
+		__delay(1);
+	}
+	/* lockup suspected: */
+	spin_dump(lock, "lockup suspected");
 #ifdef CONFIG_SMP
-			trigger_all_cpu_backtrace();
+	trigger_all_cpu_backtrace();
 #endif
-		}
-	}
+
+	/*
+	 * The trylock above was causing a livelock.  Give the lower level arch
+	 * specific lock code a chance to acquire the lock. We have already
+	 * printed a warning/backtrace at this point. The non-debug arch
+	 * specific code might actually succeed in acquiring the lock.  If it is
+	 * not successful, the end-result is the same - there is no forward
+	 * progress.
+	 */
+	arch_spin_lock(&lock->raw_lock);
 }
 
 void do_raw_spin_lock(raw_spinlock_t *lock)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 45bc1f83a5a..f114bf6a8e1 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -170,7 +170,7 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
  * Statically reserve bounce buffer space and initialize bounce buffer data
  * structures for the software IO TLB used to implement the DMA API.
  */
-void __init
+static void __init
 swiotlb_init_with_default_size(size_t default_size, int verbose)
 {
 	unsigned long bytes;
@@ -206,8 +206,9 @@ swiotlb_init(int verbose)
 int
 swiotlb_late_init_with_default_size(size_t default_size)
 {
-	unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
+	unsigned long bytes, req_nslabs = io_tlb_nslabs;
 	unsigned int order;
+	int rc = 0;
 
 	if (!io_tlb_nslabs) {
 		io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
@@ -229,16 +230,32 @@ swiotlb_late_init_with_default_size(size_t default_size)
 		order--;
 	}
 
-	if (!io_tlb_start)
-		goto cleanup1;
-
+	if (!io_tlb_start) {
+		io_tlb_nslabs = req_nslabs;
+		return -ENOMEM;
+	}
 	if (order != get_order(bytes)) {
 		printk(KERN_WARNING "Warning: only able to allocate %ld MB "
 		       "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
 		io_tlb_nslabs = SLABS_PER_PAGE << order;
-		bytes = io_tlb_nslabs << IO_TLB_SHIFT;
 	}
+	rc = swiotlb_late_init_with_tbl(io_tlb_start, io_tlb_nslabs);
+	if (rc)
+		free_pages((unsigned long)io_tlb_start, order);
+	return rc;
+}
+
+int
+swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
+{
+	unsigned long i, bytes;
+
+	bytes = nslabs << IO_TLB_SHIFT;
+
+	io_tlb_nslabs = nslabs;
+	io_tlb_start = tlb;
 	io_tlb_end = io_tlb_start + bytes;
+
 	memset(io_tlb_start, 0, bytes);
 
 	/*
@@ -288,10 +305,8 @@ cleanup3:
 	io_tlb_list = NULL;
 cleanup2:
 	io_tlb_end = NULL;
-	free_pages((unsigned long)io_tlb_start, order);
 	io_tlb_start = NULL;
-cleanup1:
-	io_tlb_nslabs = req_nslabs;
+	io_tlb_nslabs = 0;
 	return -ENOMEM;
 }
 
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 0e337541f00..39c99fea7c0 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -174,35 +174,25 @@ char *put_dec_trunc8(char *buf, unsigned r)
 	unsigned q;
 
 	/* Copy of previous function's body with added early returns */
-	q      = (r * (uint64_t)0x1999999a) >> 32;
-	*buf++ = (r - 10 * q) + '0'; /* 2 */
-	if (q == 0)
-		return buf;
-	r      = (q * (uint64_t)0x1999999a) >> 32;
-	*buf++ = (q - 10 * r) + '0'; /* 3 */
-	if (r == 0)
-		return buf;
-	q      = (r * (uint64_t)0x1999999a) >> 32;
-	*buf++ = (r - 10 * q) + '0'; /* 4 */
-	if (q == 0)
-		return buf;
-	r      = (q * (uint64_t)0x1999999a) >> 32;
-	*buf++ = (q - 10 * r) + '0'; /* 5 */
-	if (r == 0)
-		return buf;
-	q      = (r * 0x199a) >> 16;
-	*buf++ = (r - 10 * q)  + '0'; /* 6 */
+	while (r >= 10000) {
+		q = r + '0';
+		r  = (r * (uint64_t)0x1999999a) >> 32;
+		*buf++ = q - 10*r;
+	}
+
+	q      = (r * 0x199a) >> 16;	/* r <= 9999 */
+	*buf++ = (r - 10 * q)  + '0';
 	if (q == 0)
 		return buf;
-	r      = (q * 0xcd) >> 11;
-	*buf++ = (q - 10 * r)  + '0'; /* 7 */
+	r      = (q * 0xcd) >> 11;	/* q <= 999 */
+	*buf++ = (q - 10 * r)  + '0';
 	if (r == 0)
 		return buf;
-	q      = (r * 0xcd) >> 11;
-	*buf++ = (r - 10 * q) + '0'; /* 8 */
+	q      = (r * 0xcd) >> 11;	/* r <= 99 */
+	*buf++ = (r - 10 * q) + '0';
 	if (q == 0)
 		return buf;
-	*buf++ = q + '0'; /* 9 */
+	*buf++ = q + '0';		 /* q <= 9 */
 	return buf;
 }
 
@@ -243,18 +233,34 @@ char *put_dec(char *buf, unsigned long long n)
 
 /* Second algorithm: valid only for 64-bit long longs */
 
+/* See comment in put_dec_full9 for choice of constants */
 static noinline_for_stack
-char *put_dec_full4(char *buf, unsigned q)
+void put_dec_full4(char *buf, unsigned q)
 {
 	unsigned r;
-	r      = (q * 0xcccd) >> 19;
-	*buf++ = (q - 10 * r) + '0';
-	q      = (r * 0x199a) >> 16;
-	*buf++ = (r - 10 * q)  + '0';
+	r      = (q * 0xccd) >> 15;
+	buf[0] = (q - 10 * r) + '0';
+	q      = (r * 0xcd) >> 11;
+	buf[1] = (r - 10 * q)  + '0';
 	r      = (q * 0xcd) >> 11;
-	*buf++ = (q - 10 * r)  + '0';
-	*buf++ = r + '0';
-	return buf;
+	buf[2] = (q - 10 * r)  + '0';
+	buf[3] = r + '0';
+}
+
+/*
+ * Call put_dec_full4 on x % 10000, return x / 10000.
+ * The approximation x/10000 == (x * 0x346DC5D7) >> 43
+ * holds for all x < 1,128,869,999.  The largest value this
+ * helper will ever be asked to convert is 1,125,520,955.
+ * (d1 in the put_dec code, assuming n is all-ones).
+ */
+static
+unsigned put_dec_helper4(char *buf, unsigned x)
+{
+        uint32_t q = (x * (uint64_t)0x346DC5D7) >> 43;
+
+        put_dec_full4(buf, x - q * 10000);
+        return q;
 }
 
 /* Based on code by Douglas W. Jones found at
@@ -276,28 +282,19 @@ char *put_dec(char *buf, unsigned long long n)
 	d3  = (h >> 16); /* implicit "& 0xffff" */
 
 	q   = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
+	q = put_dec_helper4(buf, q);
+
+	q += 7671 * d3 + 9496 * d2 + 6 * d1;
+	q = put_dec_helper4(buf+4, q);
+
+	q += 4749 * d3 + 42 * d2;
+	q = put_dec_helper4(buf+8, q);
 
-	buf = put_dec_full4(buf, q % 10000);
-	q   = q / 10000;
-
-	d1  = q + 7671 * d3 + 9496 * d2 + 6 * d1;
-	buf = put_dec_full4(buf, d1 % 10000);
-	q   = d1 / 10000;
-
-	d2  = q + 4749 * d3 + 42 * d2;
-	buf = put_dec_full4(buf, d2 % 10000);
-	q   = d2 / 10000;
-
-	d3  = q + 281 * d3;
-	if (!d3)
-		goto done;
-	buf = put_dec_full4(buf, d3 % 10000);
-	q   = d3 / 10000;
-	if (!q)
-		goto done;
-	buf = put_dec_full4(buf, q);
- done:
-	while (buf[-1] == '0')
+	q += 281 * d3;
+	buf += 12;
+	if (q)
+		buf = put_dec_trunc8(buf, q);
+	else while (buf[-1] == '0')
 		--buf;
 
 	return buf;
@@ -990,7 +987,7 @@ int kptr_restrict __read_mostly;
  * - 'm' For a 6-byte MAC address, it prints the hex address without colons
  * - 'MF' For a 6-byte MAC FDDI address, it prints the address
  *       with a dash-separated hex notation
- * - '[mM]R For a 6-byte MAC address, Reverse order (Bluetooth)
+ * - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth)
  * - 'I' [46] for IPv4/IPv6 addresses printed in the usual way
  *       IPv4 uses dot-separated decimal without leading 0's (1.2.3.4)
  *       IPv6 uses colon separated network-order 16 bit hex with leading 0's
@@ -1341,7 +1338,10 @@ qualifier:
  * %pR output the address range in a struct resource with decoded flags
  * %pr output the address range in a struct resource with raw flags
  * %pM output a 6-byte MAC address with colons
+ * %pMR output a 6-byte MAC address with colons in reversed order
+ * %pMF output a 6-byte MAC address with dashes
  * %pm output a 6-byte MAC address without colons
+ * %pmR output a 6-byte MAC address without colons in reversed order
  * %pI4 print an IPv4 address without leading zeros
  * %pi4 print an IPv4 address with leading zeros
  * %pI6 print an IPv6 address with colons
@@ -2017,7 +2017,7 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
 	s16 field_width;
 	bool is_sign;
 
-	while (*fmt && *str) {
+	while (*fmt) {
 		/* skip any white space in format */
 		/* white space in format matchs any amount of
 		 * white space, including none, in the input.
@@ -2042,6 +2042,8 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
 		 * advance both strings to next white space
 		 */
 		if (*fmt == '*') {
+			if (!*str)
+				break;
 			while (!isspace(*fmt) && *fmt != '%' && *fmt)
 				fmt++;
 			while (!isspace(*str) && *str)
@@ -2070,7 +2072,17 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
 			}
 		}
 
-		if (!*fmt || !*str)
+		if (!*fmt)
+			break;
+
+		if (*fmt == 'n') {
+			/* return number of characters read so far */
+			*va_arg(args, int *) = str - buf;
+			++fmt;
+			continue;
+		}
+
+		if (!*str)
 			break;
 
 		base = 10;
@@ -2103,13 +2115,6 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
 			num++;
 		}
 		continue;
-		case 'n':
-			/* return number of characters read so far */
-		{
-			int *i = (int *)va_arg(args, int*);
-			*i = str - buf;
-		}
-		continue;
 		case 'o':
 			base = 8;
 			break;
@@ -2210,16 +2215,6 @@ int vsscanf(const char *buf, const char *fmt, va_list args)
 		str = next;
 	}
 
-	/*
-	 * Now we've come all the way through so either the input string or the
-	 * format ended. In the former case, there can be a %n at the current
-	 * position in the format that needs to be filled.
-	 */
-	if (*fmt == '%' && *(fmt + 1) == 'n') {
-		int *p = (int *)va_arg(args, int *);
-		*p = str - buf;
-	}
-
 	return num;
 }
 EXPORT_SYMBOL(vsscanf);
diff --git a/mm/fadvise.c b/mm/fadvise.c
index 9b75a045dbf..a47f0f50c89 100644
--- a/mm/fadvise.c
+++ b/mm/fadvise.c
@@ -26,7 +26,7 @@
  */
 SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
 {
-	struct file *file = fget(fd);
+	struct fd f = fdget(fd);
 	struct address_space *mapping;
 	struct backing_dev_info *bdi;
 	loff_t endbyte;			/* inclusive */
@@ -35,15 +35,15 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
 	unsigned long nrpages;
 	int ret = 0;
 
-	if (!file)
+	if (!f.file)
 		return -EBADF;
 
-	if (S_ISFIFO(file->f_path.dentry->d_inode->i_mode)) {
+	if (S_ISFIFO(f.file->f_path.dentry->d_inode->i_mode)) {
 		ret = -ESPIPE;
 		goto out;
 	}
 
-	mapping = file->f_mapping;
+	mapping = f.file->f_mapping;
 	if (!mapping || len < 0) {
 		ret = -EINVAL;
 		goto out;
@@ -76,21 +76,21 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
 
 	switch (advice) {
 	case POSIX_FADV_NORMAL:
-		file->f_ra.ra_pages = bdi->ra_pages;
-		spin_lock(&file->f_lock);
-		file->f_mode &= ~FMODE_RANDOM;
-		spin_unlock(&file->f_lock);
+		f.file->f_ra.ra_pages = bdi->ra_pages;
+		spin_lock(&f.file->f_lock);
+		f.file->f_mode &= ~FMODE_RANDOM;
+		spin_unlock(&f.file->f_lock);
 		break;
 	case POSIX_FADV_RANDOM:
-		spin_lock(&file->f_lock);
-		file->f_mode |= FMODE_RANDOM;
-		spin_unlock(&file->f_lock);
+		spin_lock(&f.file->f_lock);
+		f.file->f_mode |= FMODE_RANDOM;
+		spin_unlock(&f.file->f_lock);
 		break;
 	case POSIX_FADV_SEQUENTIAL:
-		file->f_ra.ra_pages = bdi->ra_pages * 2;
-		spin_lock(&file->f_lock);
-		file->f_mode &= ~FMODE_RANDOM;
-		spin_unlock(&file->f_lock);
+		f.file->f_ra.ra_pages = bdi->ra_pages * 2;
+		spin_lock(&f.file->f_lock);
+		f.file->f_mode &= ~FMODE_RANDOM;
+		spin_unlock(&f.file->f_lock);
 		break;
 	case POSIX_FADV_WILLNEED:
 		/* First and last PARTIAL page! */
@@ -106,7 +106,7 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
 		 * Ignore return value because fadvise() shall return
 		 * success even if filesystem can't retrieve a hint,
 		 */
-		force_page_cache_readahead(mapping, file, start_index,
+		force_page_cache_readahead(mapping, f.file, start_index,
 					   nrpages);
 		break;
 	case POSIX_FADV_NOREUSE:
@@ -128,7 +128,7 @@ SYSCALL_DEFINE(fadvise64_64)(int fd, loff_t offset, loff_t len, int advice)
 		ret = -EINVAL;
 	}
 out:
-	fput(file);
+	fdput(f);
 	return ret;
 }
 #ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
diff --git a/mm/fremap.c b/mm/fremap.c
index 9ed4fd43246..048659c0c03 100644
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -195,10 +195,9 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
 		 */
 		if (mapping_cap_account_dirty(mapping)) {
 			unsigned long addr;
-			struct file *file = vma->vm_file;
+			struct file *file = get_file(vma->vm_file);
 
 			flags &= MAP_NONBLOCK;
-			get_file(file);
 			addr = mmap_region(file, start, size,
 					flags, vma->vm_flags, pgoff);
 			fput(file);
diff --git a/mm/frontswap.c b/mm/frontswap.c
index 6b3e71a2cd4..2890e67d602 100644
--- a/mm/frontswap.c
+++ b/mm/frontswap.c
@@ -44,6 +44,13 @@ EXPORT_SYMBOL(frontswap_enabled);
  */
 static bool frontswap_writethrough_enabled __read_mostly;
 
+/*
+ * If enabled, the underlying tmem implementation is capable of doing
+ * exclusive gets, so frontswap_load, on a successful tmem_get must
+ * mark the page as no longer in frontswap AND mark it dirty.
+ */
+static bool frontswap_tmem_exclusive_gets_enabled __read_mostly;
+
 #ifdef CONFIG_DEBUG_FS
 /*
  * Counters available via /sys/kernel/debug/frontswap (if debugfs is
@@ -97,6 +104,15 @@ void frontswap_writethrough(bool enable)
 EXPORT_SYMBOL(frontswap_writethrough);
 
 /*
+ * Enable/disable frontswap exclusive gets (see above).
+ */
+void frontswap_tmem_exclusive_gets(bool enable)
+{
+	frontswap_tmem_exclusive_gets_enabled = enable;
+}
+EXPORT_SYMBOL(frontswap_tmem_exclusive_gets);
+
+/*
  * Called when a swap device is swapon'd.
  */
 void __frontswap_init(unsigned type)
@@ -174,8 +190,13 @@ int __frontswap_load(struct page *page)
 	BUG_ON(sis == NULL);
 	if (frontswap_test(sis, offset))
 		ret = frontswap_ops.load(type, offset, page);
-	if (ret == 0)
+	if (ret == 0) {
 		inc_frontswap_loads();
+		if (frontswap_tmem_exclusive_gets_enabled) {
+			SetPageDirty(page);
+			frontswap_clear(sis, offset);
+		}
+	}
 	return ret;
 }
 EXPORT_SYMBOL(__frontswap_load);
@@ -263,6 +284,11 @@ static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused,
 	return ret;
 }
 
+/*
+ * Used to check if it's necessory and feasible to unuse pages.
+ * Return 1 when nothing to do, 0 when need to shink pages,
+ * error code when there is an error.
+ */
 static int __frontswap_shrink(unsigned long target_pages,
 				unsigned long *pages_to_unuse,
 				int *type)
@@ -275,7 +301,7 @@ static int __frontswap_shrink(unsigned long target_pages,
 	if (total_pages <= target_pages) {
 		/* Nothing to do */
 		*pages_to_unuse = 0;
-		return 0;
+		return 1;
 	}
 	total_pages_to_unuse = total_pages - target_pages;
 	return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type);
@@ -292,7 +318,7 @@ static int __frontswap_shrink(unsigned long target_pages,
 void frontswap_shrink(unsigned long target_pages)
 {
 	unsigned long pages_to_unuse = 0;
-	int type, ret;
+	int uninitialized_var(type), ret;
 
 	/*
 	 * we don't want to hold swap_lock while doing a very
@@ -302,7 +328,7 @@ void frontswap_shrink(unsigned long target_pages)
 	spin_lock(&swap_lock);
 	ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type);
 	spin_unlock(&swap_lock);
-	if (ret == 0 && pages_to_unuse)
+	if (ret == 0)
 		try_to_unuse(type, true, pages_to_unuse);
 	return;
 }
diff --git a/mm/mmap.c b/mm/mmap.c
index ae18a48e7e4..872441e8191 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1301,8 +1301,7 @@ munmap_back:
 				goto free_vma;
 			correct_wcount = 1;
 		}
-		vma->vm_file = file;
-		get_file(file);
+		vma->vm_file = get_file(file);
 		error = file->f_op->mmap(file, vma);
 		if (error)
 			goto unmap_and_free_vma;
diff --git a/mm/nommu.c b/mm/nommu.c
index d4b0c10872d..dee2ff89fd5 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -1282,10 +1282,8 @@ unsigned long do_mmap_pgoff(struct file *file,
 	vma->vm_pgoff = pgoff;
 
 	if (file) {
-		region->vm_file = file;
-		get_file(file);
-		vma->vm_file = file;
-		get_file(file);
+		region->vm_file = get_file(file);
+		vma->vm_file = get_file(file);
 		if (vm_flags & VM_EXECUTABLE) {
 			added_exe_file_vma(current->mm);
 			vma->vm_mm = current->mm;
diff --git a/mm/percpu.c b/mm/percpu.c
index bb4be7435ce..ddc5efb9c5b 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1370,7 +1370,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
 #ifdef CONFIG_SMP
 
-const char *pcpu_fc_names[PCPU_FC_NR] __initdata = {
+const char * const pcpu_fc_names[PCPU_FC_NR] __initconst = {
 	[PCPU_FC_AUTO]	= "auto",
 	[PCPU_FC_EMBED]	= "embed",
 	[PCPU_FC_PAGE]	= "page",
diff --git a/mm/readahead.c b/mm/readahead.c
index ea8f8fa2164..7963f239123 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -579,19 +579,19 @@ do_readahead(struct address_space *mapping, struct file *filp,
 SYSCALL_DEFINE(readahead)(int fd, loff_t offset, size_t count)
 {
 	ssize_t ret;
-	struct file *file;
+	struct fd f;
 
 	ret = -EBADF;
-	file = fget(fd);
-	if (file) {
-		if (file->f_mode & FMODE_READ) {
-			struct address_space *mapping = file->f_mapping;
+	f = fdget(fd);
+	if (f.file) {
+		if (f.file->f_mode & FMODE_READ) {
+			struct address_space *mapping = f.file->f_mapping;
 			pgoff_t start = offset >> PAGE_CACHE_SHIFT;
 			pgoff_t end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
 			unsigned long len = end - start + 1;
-			ret = do_readahead(mapping, file, start, len);
+			ret = do_readahead(mapping, f.file, start, len);
 		}
-		fput(file);
+		fdput(f);
 	}
 	return ret;
 }
diff --git a/net/9p/trans_fd.c b/net/9p/trans_fd.c
index 505f0ce3f10..15656b8573f 100644
--- a/net/9p/trans_fd.c
+++ b/net/9p/trans_fd.c
@@ -793,30 +793,28 @@ static int p9_fd_open(struct p9_client *client, int rfd, int wfd)
 static int p9_socket_open(struct p9_client *client, struct socket *csocket)
 {
 	struct p9_trans_fd *p;
-	int ret, fd;
+	struct file *file;
+	int ret;
 
 	p = kmalloc(sizeof(struct p9_trans_fd), GFP_KERNEL);
 	if (!p)
 		return -ENOMEM;
 
 	csocket->sk->sk_allocation = GFP_NOIO;
-	fd = sock_map_fd(csocket, 0);
-	if (fd < 0) {
+	file = sock_alloc_file(csocket, 0, NULL);
+	if (IS_ERR(file)) {
 		pr_err("%s (%d): failed to map fd\n",
 		       __func__, task_pid_nr(current));
 		sock_release(csocket);
 		kfree(p);
-		return fd;
+		return PTR_ERR(file);
 	}
 
-	get_file(csocket->file);
-	get_file(csocket->file);
-	p->wr = p->rd = csocket->file;
+	get_file(file);
+	p->wr = p->rd = file;
 	client->trans = p;
 	client->status = Connected;
 
-	sys_close(fd);	/* still racy */
-
 	p->rd->f_flags |= O_NONBLOCK;
 
 	p->conn = p9_conn_create(client);
diff --git a/net/can/af_can.c b/net/can/af_can.c
index 821022a7214..ddac1ee2ed2 100644
--- a/net/can/af_can.c
+++ b/net/can/af_can.c
@@ -63,7 +63,7 @@
 
 #include "af_can.h"
 
-static __initdata const char banner[] = KERN_INFO
+static __initconst const char banner[] = KERN_INFO
 	"can: controller area network core (" CAN_VERSION_STRING ")\n";
 
 MODULE_DESCRIPTION("Controller Area Network PF_CAN core");
diff --git a/net/can/bcm.c b/net/can/bcm.c
index 151b7730c12..6f747582718 100644
--- a/net/can/bcm.c
+++ b/net/can/bcm.c
@@ -77,7 +77,7 @@
 		     (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG))
 
 #define CAN_BCM_VERSION CAN_VERSION
-static __initdata const char banner[] = KERN_INFO
+static __initconst const char banner[] = KERN_INFO
 	"can: broadcast manager protocol (rev " CAN_BCM_VERSION " t)\n";
 
 MODULE_DESCRIPTION("PF_CAN broadcast manager protocol");
diff --git a/net/can/gw.c b/net/can/gw.c
index 127879c55fb..1f5c9785a26 100644
--- a/net/can/gw.c
+++ b/net/can/gw.c
@@ -58,7 +58,7 @@
 #include <net/sock.h>
 
 #define CAN_GW_VERSION "20101209"
-static __initdata const char banner[] =
+static __initconst const char banner[] =
 	KERN_INFO "can: netlink gateway (rev " CAN_GW_VERSION ")\n";
 
 MODULE_DESCRIPTION("PF_CAN netlink gateway");
diff --git a/net/can/raw.c b/net/can/raw.c
index 3e9c89356a9..5b0e3e330d9 100644
--- a/net/can/raw.c
+++ b/net/can/raw.c
@@ -55,7 +55,7 @@
 #include <net/net_namespace.h>
 
 #define CAN_RAW_VERSION CAN_VERSION
-static __initdata const char banner[] =
+static __initconst const char banner[] =
 	KERN_INFO "can: raw protocol (rev " CAN_RAW_VERSION ")\n";
 
 MODULE_DESCRIPTION("PF_CAN raw protocol");
diff --git a/net/compat.c b/net/compat.c
index 74ed1d7a84a..79ae8848500 100644
--- a/net/compat.c
+++ b/net/compat.c
@@ -301,8 +301,7 @@ void scm_detach_fds_compat(struct msghdr *kmsg, struct scm_cookie *scm)
 			break;
 		}
 		/* Bump the usage count and install the file. */
-		get_file(fp[i]);
-		fd_install(new_fd, fp[i]);
+		fd_install(new_fd, get_file(fp[i]));
 	}
 
 	if (i > 0) {
diff --git a/net/core/netprio_cgroup.c b/net/core/netprio_cgroup.c
index 4a83fb3c8e8..79285a36035 100644
--- a/net/core/netprio_cgroup.c
+++ b/net/core/netprio_cgroup.c
@@ -239,38 +239,24 @@ out_free_devname:
 	return ret;
 }
 
+static int update_netprio(const void *v, struct file *file, unsigned n)
+{
+	int err;
+	struct socket *sock = sock_from_file(file, &err);
+	if (sock)
+		sock->sk->sk_cgrp_prioidx = (u32)(unsigned long)v;
+	return 0;
+}
+
 void net_prio_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
 {
 	struct task_struct *p;
+	void *v;
 
 	cgroup_taskset_for_each(p, cgrp, tset) {
-		unsigned int fd;
-		struct fdtable *fdt;
-		struct files_struct *files;
-
 		task_lock(p);
-		files = p->files;
-		if (!files) {
-			task_unlock(p);
-			continue;
-		}
-
-		spin_lock(&files->file_lock);
-		fdt = files_fdtable(files);
-		for (fd = 0; fd < fdt->max_fds; fd++) {
-			struct file *file;
-			struct socket *sock;
-			int err;
-
-			file = fcheck_files(files, fd);
-			if (!file)
-				continue;
-
-			sock = sock_from_file(file, &err);
-			if (sock)
-				sock_update_netprioidx(sock->sk, p);
-		}
-		spin_unlock(&files->file_lock);
+		v = (void *)(unsigned long)task_netprioidx(p);
+		iterate_fd(p->files, 0, update_netprio, v);
 		task_unlock(p);
 	}
 }
diff --git a/net/core/scm.c b/net/core/scm.c
index 9c1c63da3ca..ab570841a53 100644
--- a/net/core/scm.c
+++ b/net/core/scm.c
@@ -301,11 +301,10 @@ void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm)
 			break;
 		}
 		/* Bump the usage count and install the file. */
-		get_file(fp[i]);
 		sock = sock_from_file(fp[i], &err);
 		if (sock)
 			sock_update_netprioidx(sock->sk, current);
-		fd_install(new_fd, fp[i]);
+		fd_install(new_fd, get_file(fp[i]));
 	}
 
 	if (i > 0)
diff --git a/net/decnet/dn_rules.c b/net/decnet/dn_rules.c
index e65f2c856e0..faf7cc3483f 100644
--- a/net/decnet/dn_rules.c
+++ b/net/decnet/dn_rules.c
@@ -220,7 +220,7 @@ static void dn_fib_rule_flush_cache(struct fib_rules_ops *ops)
 	dn_rt_cache_flush(-1);
 }
 
-static const struct fib_rules_ops __net_initdata dn_fib_rules_ops_template = {
+static const struct fib_rules_ops __net_initconst dn_fib_rules_ops_template = {
 	.family		= AF_DECnet,
 	.rule_size	= sizeof(struct dn_fib_rule),
 	.addr_size	= sizeof(u16),
diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
index 274309d3ade..26aa65d1fce 100644
--- a/net/ipv4/fib_rules.c
+++ b/net/ipv4/fib_rules.c
@@ -262,7 +262,7 @@ static void fib4_rule_flush_cache(struct fib_rules_ops *ops)
 	rt_cache_flush(ops->fro_net);
 }
 
-static const struct fib_rules_ops __net_initdata fib4_rules_ops_template = {
+static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = {
 	.family		= AF_INET,
 	.rule_size	= sizeof(struct fib4_rule),
 	.addr_size	= sizeof(u32),
diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
index 1daa95c2a0b..6168c4dc58b 100644
--- a/net/ipv4/ipmr.c
+++ b/net/ipv4/ipmr.c
@@ -221,7 +221,7 @@ static int ipmr_rule_fill(struct fib_rule *rule, struct sk_buff *skb,
 	return 0;
 }
 
-static const struct fib_rules_ops __net_initdata ipmr_rules_ops_template = {
+static const struct fib_rules_ops __net_initconst ipmr_rules_ops_template = {
 	.family		= RTNL_FAMILY_IPMR,
 	.rule_size	= sizeof(struct ipmr_rule),
 	.addr_size	= sizeof(u32),
diff --git a/net/ipv6/addrlabel.c b/net/ipv6/addrlabel.c
index 4be23da32b8..ff76eecfd62 100644
--- a/net/ipv6/addrlabel.c
+++ b/net/ipv6/addrlabel.c
@@ -79,7 +79,7 @@ struct net *ip6addrlbl_net(const struct ip6addrlbl_entry *lbl)
 
 #define IPV6_ADDR_LABEL_DEFAULT	0xffffffffUL
 
-static const __net_initdata struct ip6addrlbl_init_table
+static const __net_initconst struct ip6addrlbl_init_table
 {
 	const struct in6_addr *prefix;
 	int prefixlen;
diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
index 0ff1cfd55bc..d9fb9110f60 100644
--- a/net/ipv6/fib6_rules.c
+++ b/net/ipv6/fib6_rules.c
@@ -238,7 +238,7 @@ static size_t fib6_rule_nlmsg_payload(struct fib_rule *rule)
 	       + nla_total_size(16); /* src */
 }
 
-static const struct fib_rules_ops __net_initdata fib6_rules_ops_template = {
+static const struct fib_rules_ops __net_initconst fib6_rules_ops_template = {
 	.family			= AF_INET6,
 	.rule_size		= sizeof(struct fib6_rule),
 	.addr_size		= sizeof(struct in6_addr),
diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c
index 08ea3f0b6e5..f7c7c631972 100644
--- a/net/ipv6/ip6mr.c
+++ b/net/ipv6/ip6mr.c
@@ -205,7 +205,7 @@ static int ip6mr_rule_fill(struct fib_rule *rule, struct sk_buff *skb,
 	return 0;
 }
 
-static const struct fib_rules_ops __net_initdata ip6mr_rules_ops_template = {
+static const struct fib_rules_ops __net_initconst ip6mr_rules_ops_template = {
 	.family		= RTNL_FAMILY_IP6MR,
 	.rule_size	= sizeof(struct ip6mr_rule),
 	.addr_size	= sizeof(struct in6_addr),
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index d37d24ff197..59d16ea927f 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -70,6 +70,7 @@
 #include <linux/init.h>
 #include <linux/crypto.h>
 #include <linux/slab.h>
+#include <linux/file.h>
 
 #include <net/ip.h>
 #include <net/icmp.h>
@@ -4292,6 +4293,7 @@ static int sctp_getsockopt_peeloff(struct sock *sk, int len, char __user *optval
 {
 	sctp_peeloff_arg_t peeloff;
 	struct socket *newsock;
+	struct file *newfile;
 	int retval = 0;
 
 	if (len < sizeof(sctp_peeloff_arg_t))
@@ -4305,22 +4307,35 @@ static int sctp_getsockopt_peeloff(struct sock *sk, int len, char __user *optval
 		goto out;
 
 	/* Map the socket to an unused fd that can be returned to the user.  */
-	retval = sock_map_fd(newsock, 0);
+	retval = get_unused_fd();
 	if (retval < 0) {
 		sock_release(newsock);
 		goto out;
 	}
 
+	newfile = sock_alloc_file(newsock, 0, NULL);
+	if (unlikely(IS_ERR(newfile))) {
+		put_unused_fd(retval);
+		sock_release(newsock);
+		return PTR_ERR(newfile);
+	}
+
 	SCTP_DEBUG_PRINTK("%s: sk: %p newsk: %p sd: %d\n",
 			  __func__, sk, newsock->sk, retval);
 
 	/* Return the fd mapped to the new socket.  */
+	if (put_user(len, optlen)) {
+		fput(newfile);
+		put_unused_fd(retval);
+		return -EFAULT;
+	}
 	peeloff.sd = retval;
-	if (put_user(len, optlen))
+	if (copy_to_user(optval, &peeloff, len)) {
+		fput(newfile);
+		put_unused_fd(retval);
 		return -EFAULT;
-	if (copy_to_user(optval, &peeloff, len))
-		retval = -EFAULT;
-
+	}
+	fd_install(retval, newfile);
 out:
 	return retval;
 }
diff --git a/net/socket.c b/net/socket.c
index 80dc7e84b04..d92c490e66f 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -347,17 +347,11 @@ static struct file_system_type sock_fs_type = {
  *	but we take care of internal coherence yet.
  */
 
-static int sock_alloc_file(struct socket *sock, struct file **f, int flags,
-			   const char *dname)
+struct file *sock_alloc_file(struct socket *sock, int flags, const char *dname)
 {
 	struct qstr name = { .name = "" };
 	struct path path;
 	struct file *file;
-	int fd;
-
-	fd = get_unused_fd_flags(flags);
-	if (unlikely(fd < 0))
-		return fd;
 
 	if (dname) {
 		name.name = dname;
@@ -367,10 +361,8 @@ static int sock_alloc_file(struct socket *sock, struct file **f, int flags,
 		name.len = strlen(name.name);
 	}
 	path.dentry = d_alloc_pseudo(sock_mnt->mnt_sb, &name);
-	if (unlikely(!path.dentry)) {
-		put_unused_fd(fd);
-		return -ENOMEM;
-	}
+	if (unlikely(!path.dentry))
+		return ERR_PTR(-ENOMEM);
 	path.mnt = mntget(sock_mnt);
 
 	d_instantiate(path.dentry, SOCK_INODE(sock));
@@ -382,30 +374,33 @@ static int sock_alloc_file(struct socket *sock, struct file **f, int flags,
 		/* drop dentry, keep inode */
 		ihold(path.dentry->d_inode);
 		path_put(&path);
-		put_unused_fd(fd);
-		return -ENFILE;
+		return ERR_PTR(-ENFILE);
 	}
 
 	sock->file = file;
 	file->f_flags = O_RDWR | (flags & O_NONBLOCK);
 	file->f_pos = 0;
 	file->private_data = sock;
-
-	*f = file;
-	return fd;
+	return file;
 }
+EXPORT_SYMBOL(sock_alloc_file);
 
-int sock_map_fd(struct socket *sock, int flags)
+static int sock_map_fd(struct socket *sock, int flags)
 {
 	struct file *newfile;
-	int fd = sock_alloc_file(sock, &newfile, flags, NULL);
+	int fd = get_unused_fd_flags(flags);
+	if (unlikely(fd < 0))
+		return fd;
 
-	if (likely(fd >= 0))
+	newfile = sock_alloc_file(sock, flags, NULL);
+	if (likely(!IS_ERR(newfile))) {
 		fd_install(fd, newfile);
+		return fd;
+	}
 
-	return fd;
+	put_unused_fd(fd);
+	return PTR_ERR(newfile);
 }
-EXPORT_SYMBOL(sock_map_fd);
 
 struct socket *sock_from_file(struct file *file, int *err)
 {
@@ -1466,17 +1461,32 @@ SYSCALL_DEFINE4(socketpair, int, family, int, type, int, protocol,
 	if (err < 0)
 		goto out_release_both;
 
-	fd1 = sock_alloc_file(sock1, &newfile1, flags, NULL);
+	fd1 = get_unused_fd_flags(flags);
 	if (unlikely(fd1 < 0)) {
 		err = fd1;
 		goto out_release_both;
 	}
-
-	fd2 = sock_alloc_file(sock2, &newfile2, flags, NULL);
+	fd2 = get_unused_fd_flags(flags);
 	if (unlikely(fd2 < 0)) {
 		err = fd2;
+		put_unused_fd(fd1);
+		goto out_release_both;
+	}
+
+	newfile1 = sock_alloc_file(sock1, flags, NULL);
+	if (unlikely(IS_ERR(newfile1))) {
+		err = PTR_ERR(newfile1);
+		put_unused_fd(fd1);
+		put_unused_fd(fd2);
+		goto out_release_both;
+	}
+
+	newfile2 = sock_alloc_file(sock2, flags, NULL);
+	if (IS_ERR(newfile2)) {
+		err = PTR_ERR(newfile2);
 		fput(newfile1);
 		put_unused_fd(fd1);
+		put_unused_fd(fd2);
 		sock_release(sock2);
 		goto out;
 	}
@@ -1608,13 +1618,19 @@ SYSCALL_DEFINE4(accept4, int, fd, struct sockaddr __user *, upeer_sockaddr,
 	 */
 	__module_get(newsock->ops->owner);
 
-	newfd = sock_alloc_file(newsock, &newfile, flags,
-				sock->sk->sk_prot_creator->name);
+	newfd = get_unused_fd_flags(flags);
 	if (unlikely(newfd < 0)) {
 		err = newfd;
 		sock_release(newsock);
 		goto out_put;
 	}
+	newfile = sock_alloc_file(newsock, flags, sock->sk->sk_prot_creator->name);
+	if (unlikely(IS_ERR(newfile))) {
+		err = PTR_ERR(newfile);
+		put_unused_fd(newfd);
+		sock_release(newsock);
+		goto out_put;
+	}
 
 	err = security_socket_accept(sock, newsock);
 	if (err)
diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile
index 16aa2d42498..bbbd276659b 100644
--- a/samples/seccomp/Makefile
+++ b/samples/seccomp/Makefile
@@ -18,14 +18,22 @@ HOSTCFLAGS_bpf-direct.o += -idirafter $(objtree)/include
 bpf-direct-objs := bpf-direct.o
 
 # Try to match the kernel target.
-ifeq ($(CONFIG_64BIT),)
-HOSTCFLAGS_bpf-direct.o += -m32
-HOSTCFLAGS_dropper.o += -m32
-HOSTCFLAGS_bpf-helper.o += -m32
-HOSTCFLAGS_bpf-fancy.o += -m32
-HOSTLOADLIBES_bpf-direct += -m32
-HOSTLOADLIBES_bpf-fancy += -m32
-HOSTLOADLIBES_dropper += -m32
+ifndef CONFIG_64BIT
+
+# s390 has -m31 flag to build 31 bit binaries
+ifndef CONFIG_S390
+MFLAG = -m32
+else
+MFLAG = -m31
+endif
+
+HOSTCFLAGS_bpf-direct.o += $(MFLAG)
+HOSTCFLAGS_dropper.o += $(MFLAG)
+HOSTCFLAGS_bpf-helper.o += $(MFLAG)
+HOSTCFLAGS_bpf-fancy.o += $(MFLAG)
+HOSTLOADLIBES_bpf-direct += $(MFLAG)
+HOSTLOADLIBES_bpf-fancy += $(MFLAG)
+HOSTLOADLIBES_dropper += $(MFLAG)
 endif
 
 # Tell kbuild to always build the programs
diff --git a/samples/seccomp/bpf-helper.h b/samples/seccomp/bpf-helper.h
index 643279dd30f..38ee70f3cd5 100644
--- a/samples/seccomp/bpf-helper.h
+++ b/samples/seccomp/bpf-helper.h
@@ -59,6 +59,16 @@ void seccomp_bpf_print(struct sock_filter *filter, size_t count);
 #define FIND_LABEL(labels, label) seccomp_bpf_label((labels), #label)
 
 #define EXPAND(...) __VA_ARGS__
+
+/* Ensure that we load the logically correct offset. */
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)])
+#elif __BYTE_ORDER == __BIG_ENDIAN
+#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + sizeof(__u32)
+#else
+#error "Unknown endianness"
+#endif
+
 /* Map all width-sensitive operations */
 #if __BITS_PER_LONG == 32
 
@@ -70,21 +80,16 @@ void seccomp_bpf_print(struct sock_filter *filter, size_t count);
 #define JLE(x, jt) JLE32(x, EXPAND(jt))
 #define JA(x, jt) JA32(x, EXPAND(jt))
 #define ARG(i) ARG_32(i)
-#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)])
 
 #elif __BITS_PER_LONG == 64
 
 /* Ensure that we load the logically correct offset. */
 #if __BYTE_ORDER == __LITTLE_ENDIAN
 #define ENDIAN(_lo, _hi) _lo, _hi
-#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)])
 #define HI_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + sizeof(__u32)
 #elif __BYTE_ORDER == __BIG_ENDIAN
 #define ENDIAN(_lo, _hi) _hi, _lo
-#define LO_ARG(idx) offsetof(struct seccomp_data, args[(idx)]) + sizeof(__u32)
 #define HI_ARG(idx) offsetof(struct seccomp_data, args[(idx)])
-#else
-#error "Unknown endianness"
 #endif
 
 union arg64 {
diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include
index 6a3ee981931..afa44595f34 100644
--- a/scripts/Kbuild.include
+++ b/scripts/Kbuild.include
@@ -209,7 +209,7 @@ endif
 # >$< substitution to preserve $ when reloading .cmd file
 # note: when using inline perl scripts [perl -e '...$$t=1;...']
 # in $(cmd_xxx) double $$ your perl vars
-make-cmd = $(subst \#,\\\#,$(subst $$,$$$$,$(call escsq,$(cmd_$(1)))))
+make-cmd = $(subst \\,\\\\,$(subst \#,\\\#,$(subst $$,$$$$,$(call escsq,$(cmd_$(1))))))
 
 # Find any prerequisites that is newer than target or that does not exist.
 # PHONY targets skipped in both cases.
diff --git a/scripts/Makefile.headersinst b/scripts/Makefile.headersinst
index d3bae5e7b60..06ba4a70bd4 100644
--- a/scripts/Makefile.headersinst
+++ b/scripts/Makefile.headersinst
@@ -3,13 +3,12 @@
 #
 # header-y  - list files to be installed. They are preprocessed
 #             to remove __KERNEL__ section of the file
-# objhdr-y  - Same as header-y but for generated files
-# genhdr-y  - Same as objhdr-y but in a generated/ directory
+# genhdr-y  - Same as header-y but in a generated/ directory
 #
 # ==========================================================================
 
 # called may set destination dir (when installing to asm/)
-_dst := $(if $(dst),$(dst),$(obj))
+_dst := $(or $(destination-y),$(dst),$(obj))
 
 # generated header directory
 gen := $(if $(gen),$(gen),$(subst include/,include/generated/,$(obj)))
@@ -17,49 +16,64 @@ gen := $(if $(gen),$(gen),$(subst include/,include/generated/,$(obj)))
 kbuild-file := $(srctree)/$(obj)/Kbuild
 include $(kbuild-file)
 
-_dst := $(if $(destination-y),$(destination-y),$(_dst))
+old-kbuild-file := $(srctree)/$(subst uapi/,,$(obj))/Kbuild
+ifneq ($(wildcard $(old-kbuild-file)),)
+include $(old-kbuild-file)
+endif
 
 include scripts/Kbuild.include
 
-install       := $(INSTALL_HDR_PATH)/$(_dst)
+installdir    := $(INSTALL_HDR_PATH)/$(subst uapi/,,$(_dst))
 
 header-y      := $(sort $(header-y))
 subdirs       := $(patsubst %/,%,$(filter %/, $(header-y)))
 header-y      := $(filter-out %/, $(header-y))
 
 # files used to track state of install/check
-install-file  := $(install)/.install
-check-file    := $(install)/.check
+install-file  := $(installdir)/.install
+check-file    := $(installdir)/.check
 
 # generic-y list all files an architecture uses from asm-generic
 # Use this to build a list of headers which require a wrapper
 wrapper-files := $(filter $(header-y), $(generic-y))
 
+srcdir        := $(srctree)/$(obj)
+gendir        := $(objtree)/$(gen)
+
+oldsrcdir     := $(srctree)/$(subst /uapi,,$(obj))
+
 # all headers files for this dir
 header-y      := $(filter-out $(generic-y), $(header-y))
-all-files     := $(header-y) $(objhdr-y) $(genhdr-y) $(wrapper-files)
-input-files   := $(addprefix $(srctree)/$(obj)/,$(header-y)) \
-                 $(addprefix $(objtree)/$(obj)/,$(objhdr-y)) \
-                 $(addprefix $(objtree)/$(gen)/,$(genhdr-y))
-output-files  := $(addprefix $(install)/, $(all-files))
+all-files     := $(header-y) $(genhdr-y) $(wrapper-files)
+output-files  := $(addprefix $(installdir)/, $(all-files))
+
+input-files   := $(foreach hdr, $(header-y), \
+		   $(or \
+			$(wildcard $(srcdir)/$(hdr)), \
+			$(wildcard $(oldsrcdir)/$(hdr)), \
+			$(error Missing UAPI file $(srcdir)/$(hdr)) \
+		   )) \
+		 $(foreach hdr, $(genhdr-y), \
+		   $(or \
+			$(wildcard $(gendir)/$(hdr)), \
+			$(error Missing generated UAPI file $(gendir)/$(hdr)) \
+		   ))
 
 # Work out what needs to be removed
-oldheaders    := $(patsubst $(install)/%,%,$(wildcard $(install)/*.h))
+oldheaders    := $(patsubst $(installdir)/%,%,$(wildcard $(installdir)/*.h))
 unwanted      := $(filter-out $(all-files),$(oldheaders))
 
 # Prefix unwanted with full paths to $(INSTALL_HDR_PATH)
-unwanted-file := $(addprefix $(install)/, $(unwanted))
+unwanted-file := $(addprefix $(installdir)/, $(unwanted))
 
 printdir = $(patsubst $(INSTALL_HDR_PATH)/%/,%,$(dir $@))
 
 quiet_cmd_install = INSTALL $(printdir) ($(words $(all-files))\
                             file$(if $(word 2, $(all-files)),s))
       cmd_install = \
-        $(PERL) $< $(srctree)/$(obj) $(install) $(SRCARCH) $(header-y); \
-        $(PERL) $< $(objtree)/$(obj) $(install) $(SRCARCH) $(objhdr-y); \
-        $(PERL) $< $(objtree)/$(gen) $(install) $(SRCARCH) $(genhdr-y); \
+        $(PERL) $< $(installdir) $(SRCARCH) $(input-files); \
         for F in $(wrapper-files); do                                   \
-                echo "\#include <asm-generic/$$F>" > $(install)/$$F;    \
+                echo "\#include <asm-generic/$$F>" > $(installdir)/$$F;    \
         done;                                                           \
         touch $@
 
@@ -70,7 +84,7 @@ quiet_cmd_check = CHECK   $(printdir) ($(words $(all-files)) files)
 # Headers list can be pretty long, xargs helps to avoid
 # the "Argument list too long" error.
       cmd_check = for f in $(all-files); do                          \
-                  echo "$(install)/$${f}"; done                      \
+                  echo "$(installdir)/$${f}"; done                      \
                   | xargs                                            \
                   $(PERL) $< $(INSTALL_HDR_PATH)/include $(SRCARCH); \
 	          touch $@
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index ca05ba217f5..21a9f5de0a2 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -421,7 +421,7 @@ sub top_of_kernel_tree {
 		}
 	}
 	return 1;
-    }
+}
 
 sub parse_email {
 	my ($formatted_email) = @_;
@@ -1386,6 +1386,8 @@ sub process {
 	my $in_header_lines = 1;
 	my $in_commit_log = 0;		#Scanning lines before patch
 
+	my $non_utf8_charset = 0;
+
 	our @report = ();
 	our $cnt_lines = 0;
 	our $cnt_error = 0;
@@ -1686,10 +1688,17 @@ sub process {
 			$in_commit_log = 1;
 		}
 
-# Still not yet in a patch, check for any UTF-8
-		if ($in_commit_log && $realfile =~ /^$/ &&
+# Check if there is UTF-8 in a commit log when a mail header has explicitly
+# declined it, i.e defined some charset where it is missing.
+		if ($in_header_lines &&
+		    $rawline =~ /^Content-Type:.+charset="(.+)".*$/ &&
+		    $1 !~ /utf-8/i) {
+			$non_utf8_charset = 1;
+		}
+
+		if ($in_commit_log && $non_utf8_charset && $realfile =~ /^$/ &&
 		    $rawline =~ /$NON_ASCII_UTF8/) {
-			CHK("UTF8_BEFORE_PATCH",
+			WARN("UTF8_BEFORE_PATCH",
 			    "8-bit UTF-8 used in possible commit log\n" . $herecurr);
 		}
 
@@ -1873,6 +1882,20 @@ sub process {
 			    "No space is necessary after a cast\n" . $hereprev);
 		}
 
+		if ($realfile =~ m@^(drivers/net/|net/)@ &&
+		    $rawline =~ /^\+[ \t]*\/\*[ \t]*$/ &&
+		    $prevrawline =~ /^\+[ \t]*$/) {
+			WARN("NETWORKING_BLOCK_COMMENT_STYLE",
+			     "networking block comments don't use an empty /* line, use /* Comment...\n" . $hereprev);
+		}
+
+		if ($realfile =~ m@^(drivers/net/|net/)@ &&
+		    $rawline !~ m@^\+[ \t]*(\/\*|\*\/)@ &&
+		    $rawline =~ m@^\+[ \t]*.+\*\/[ \t]*$@) {
+			WARN("NETWORKING_BLOCK_COMMENT_STYLE",
+			     "networking block comments put the trailing */ on a separate line\n" . $herecurr);
+		}
+
 # check for spaces at the beginning of a line.
 # Exceptions:
 #  1) within comments
@@ -2390,8 +2413,10 @@ sub process {
 			my $orig = $1;
 			my $level = lc($orig);
 			$level = "warn" if ($level eq "warning");
+			my $level2 = $level;
+			$level2 = "dbg" if ($level eq "debug");
 			WARN("PREFER_PR_LEVEL",
-			     "Prefer pr_$level(... to printk(KERN_$1, ...\n" . $herecurr);
+			     "Prefer netdev_$level2(netdev, ... then dev_$level2(dev, ... then pr_$level(...  to printk(KERN_$orig ...\n" . $herecurr);
 		}
 
 		if ($line =~ /\bpr_warning\s*\(/) {
@@ -2947,7 +2972,7 @@ sub process {
 			my $exceptions = qr{
 				$Declare|
 				module_param_named|
-				MODULE_PARAM_DESC|
+				MODULE_PARM_DESC|
 				DECLARE_PER_CPU|
 				DEFINE_PER_CPU|
 				__typeof__\(|
diff --git a/scripts/dtc/Makefile.dtc b/scripts/dtc/Makefile.dtc
index 6ddf9ecac66..bece49b3553 100644
--- a/scripts/dtc/Makefile.dtc
+++ b/scripts/dtc/Makefile.dtc
@@ -3,7 +3,16 @@
 # This is not a complete Makefile of itself.  Instead, it is designed to
 # be easily embeddable into other systems of Makefiles.
 #
-DTC_SRCS = dtc.c flattree.c fstree.c data.c livetree.c treesource.c srcpos.c \
-	checks.c
+DTC_SRCS = \
+	checks.c \
+	data.c \
+	dtc.c \
+	flattree.c \
+	fstree.c \
+	livetree.c \
+	srcpos.c \
+	treesource.c \
+	util.c
+
 DTC_GEN_SRCS = dtc-lexer.lex.c dtc-parser.tab.c
 DTC_OBJS = $(DTC_SRCS:%.c=%.o) $(DTC_GEN_SRCS:%.c=%.o)
diff --git a/scripts/dtc/checks.c b/scripts/dtc/checks.c
index a662a004479..ee96a2519ef 100644
--- a/scripts/dtc/checks.c
+++ b/scripts/dtc/checks.c
@@ -31,12 +31,6 @@
 #define TRACE(c, fmt, ...)	do { } while (0)
 #endif
 
-enum checklevel {
-	IGNORE = 0,
-	WARN = 1,
-	ERROR = 2,
-};
-
 enum checkstatus {
 	UNCHECKED = 0,
 	PREREQ,
@@ -57,14 +51,14 @@ struct check {
 	node_check_fn node_fn;
 	prop_check_fn prop_fn;
 	void *data;
-	enum checklevel level;
+	bool warn, error;
 	enum checkstatus status;
 	int inprogress;
 	int num_prereqs;
 	struct check **prereq;
 };
 
-#define CHECK(nm, tfn, nfn, pfn, d, lvl, ...) \
+#define CHECK_ENTRY(nm, tfn, nfn, pfn, d, w, e, ...)	       \
 	static struct check *nm##_prereqs[] = { __VA_ARGS__ }; \
 	static struct check nm = { \
 		.name = #nm, \
@@ -72,20 +66,37 @@ struct check {
 		.node_fn = (nfn), \
 		.prop_fn = (pfn), \
 		.data = (d), \
-		.level = (lvl), \
+		.warn = (w), \
+		.error = (e), \
 		.status = UNCHECKED, \
 		.num_prereqs = ARRAY_SIZE(nm##_prereqs), \
 		.prereq = nm##_prereqs, \
 	};
-
-#define TREE_CHECK(nm, d, lvl, ...) \
-	CHECK(nm, check_##nm, NULL, NULL, d, lvl, __VA_ARGS__)
-#define NODE_CHECK(nm, d, lvl, ...) \
-	CHECK(nm, NULL, check_##nm, NULL, d, lvl, __VA_ARGS__)
-#define PROP_CHECK(nm, d, lvl, ...) \
-	CHECK(nm, NULL, NULL, check_##nm, d, lvl, __VA_ARGS__)
-#define BATCH_CHECK(nm, lvl, ...) \
-	CHECK(nm, NULL, NULL, NULL, NULL, lvl, __VA_ARGS__)
+#define WARNING(nm, tfn, nfn, pfn, d, ...) \
+	CHECK_ENTRY(nm, tfn, nfn, pfn, d, true, false, __VA_ARGS__)
+#define ERROR(nm, tfn, nfn, pfn, d, ...) \
+	CHECK_ENTRY(nm, tfn, nfn, pfn, d, false, true, __VA_ARGS__)
+#define CHECK(nm, tfn, nfn, pfn, d, ...) \
+	CHECK_ENTRY(nm, tfn, nfn, pfn, d, false, false, __VA_ARGS__)
+
+#define TREE_WARNING(nm, d, ...) \
+	WARNING(nm, check_##nm, NULL, NULL, d, __VA_ARGS__)
+#define TREE_ERROR(nm, d, ...) \
+	ERROR(nm, check_##nm, NULL, NULL, d, __VA_ARGS__)
+#define TREE_CHECK(nm, d, ...) \
+	CHECK(nm, check_##nm, NULL, NULL, d, __VA_ARGS__)
+#define NODE_WARNING(nm, d, ...) \
+	WARNING(nm, NULL, check_##nm, NULL, d,  __VA_ARGS__)
+#define NODE_ERROR(nm, d, ...) \
+	ERROR(nm, NULL, check_##nm, NULL, d, __VA_ARGS__)
+#define NODE_CHECK(nm, d, ...) \
+	CHECK(nm, NULL, check_##nm, NULL, d, __VA_ARGS__)
+#define PROP_WARNING(nm, d, ...) \
+	WARNING(nm, NULL, NULL, check_##nm, d, __VA_ARGS__)
+#define PROP_ERROR(nm, d, ...) \
+	ERROR(nm, NULL, NULL, check_##nm, d, __VA_ARGS__)
+#define PROP_CHECK(nm, d, ...) \
+	CHECK(nm, NULL, NULL, check_##nm, d, __VA_ARGS__)
 
 #ifdef __GNUC__
 static inline void check_msg(struct check *c, const char *fmt, ...) __attribute__((format (printf, 2, 3)));
@@ -95,13 +106,13 @@ static inline void check_msg(struct check *c, const char *fmt, ...)
 	va_list ap;
 	va_start(ap, fmt);
 
-	if ((c->level < WARN) || (c->level <= quiet))
-		return; /* Suppress message */
-
-	fprintf(stderr, "%s (%s): ",
-		(c->level == ERROR) ? "ERROR" : "Warning", c->name);
-	vfprintf(stderr, fmt, ap);
-	fprintf(stderr, "\n");
+	if ((c->warn && (quiet < 1))
+	    || (c->error && (quiet < 2))) {
+		fprintf(stderr, "%s (%s): ",
+			(c->error) ? "ERROR" : "Warning", c->name);
+		vfprintf(stderr, fmt, ap);
+		fprintf(stderr, "\n");
+	}
 }
 
 #define FAIL(c, ...) \
@@ -167,7 +178,7 @@ static int run_check(struct check *c, struct node *dt)
 
 out:
 	c->inprogress = 0;
-	if ((c->status != PASSED) && (c->level == ERROR))
+	if ((c->status != PASSED) && (c->error))
 		error = 1;
 	return error;
 }
@@ -176,6 +187,13 @@ out:
  * Utility check functions
  */
 
+/* A check which always fails, for testing purposes only */
+static inline void check_always_fail(struct check *c, struct node *dt)
+{
+	FAIL(c, "always_fail check");
+}
+TREE_CHECK(always_fail, NULL);
+
 static void check_is_string(struct check *c, struct node *root,
 			    struct node *node)
 {
@@ -190,8 +208,10 @@ static void check_is_string(struct check *c, struct node *root,
 		FAIL(c, "\"%s\" property in %s is not a string",
 		     propname, node->fullpath);
 }
-#define CHECK_IS_STRING(nm, propname, lvl) \
-	CHECK(nm, NULL, check_is_string, NULL, (propname), (lvl))
+#define WARNING_IF_NOT_STRING(nm, propname) \
+	WARNING(nm, NULL, check_is_string, NULL, (propname))
+#define ERROR_IF_NOT_STRING(nm, propname) \
+	ERROR(nm, NULL, check_is_string, NULL, (propname))
 
 static void check_is_cell(struct check *c, struct node *root,
 			  struct node *node)
@@ -207,8 +227,10 @@ static void check_is_cell(struct check *c, struct node *root,
 		FAIL(c, "\"%s\" property in %s is not a single cell",
 		     propname, node->fullpath);
 }
-#define CHECK_IS_CELL(nm, propname, lvl) \
-	CHECK(nm, NULL, check_is_cell, NULL, (propname), (lvl))
+#define WARNING_IF_NOT_CELL(nm, propname) \
+	WARNING(nm, NULL, check_is_cell, NULL, (propname))
+#define ERROR_IF_NOT_CELL(nm, propname) \
+	ERROR(nm, NULL, check_is_cell, NULL, (propname))
 
 /*
  * Structural check functions
@@ -227,20 +249,24 @@ static void check_duplicate_node_names(struct check *c, struct node *dt,
 				FAIL(c, "Duplicate node name %s",
 				     child->fullpath);
 }
-NODE_CHECK(duplicate_node_names, NULL, ERROR);
+NODE_ERROR(duplicate_node_names, NULL);
 
 static void check_duplicate_property_names(struct check *c, struct node *dt,
 					   struct node *node)
 {
 	struct property *prop, *prop2;
 
-	for_each_property(node, prop)
-		for (prop2 = prop->next; prop2; prop2 = prop2->next)
+	for_each_property(node, prop) {
+		for (prop2 = prop->next; prop2; prop2 = prop2->next) {
+			if (prop2->deleted)
+				continue;
 			if (streq(prop->name, prop2->name))
 				FAIL(c, "Duplicate property name %s in %s",
 				     prop->name, node->fullpath);
+		}
+	}
 }
-NODE_CHECK(duplicate_property_names, NULL, ERROR);
+NODE_ERROR(duplicate_property_names, NULL);
 
 #define LOWERCASE	"abcdefghijklmnopqrstuvwxyz"
 #define UPPERCASE	"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -256,7 +282,7 @@ static void check_node_name_chars(struct check *c, struct node *dt,
 		FAIL(c, "Bad character '%c' in node %s",
 		     node->name[n], node->fullpath);
 }
-NODE_CHECK(node_name_chars, PROPNODECHARS "@", ERROR);
+NODE_ERROR(node_name_chars, PROPNODECHARS "@");
 
 static void check_node_name_format(struct check *c, struct node *dt,
 				   struct node *node)
@@ -265,7 +291,7 @@ static void check_node_name_format(struct check *c, struct node *dt,
 		FAIL(c, "Node %s has multiple '@' characters in name",
 		     node->fullpath);
 }
-NODE_CHECK(node_name_format, NULL, ERROR, &node_name_chars);
+NODE_ERROR(node_name_format, NULL, &node_name_chars);
 
 static void check_property_name_chars(struct check *c, struct node *dt,
 				      struct node *node, struct property *prop)
@@ -276,7 +302,7 @@ static void check_property_name_chars(struct check *c, struct node *dt,
 		FAIL(c, "Bad character '%c' in property name \"%s\", node %s",
 		     prop->name[n], prop->name, node->fullpath);
 }
-PROP_CHECK(property_name_chars, PROPNODECHARS, ERROR);
+PROP_ERROR(property_name_chars, PROPNODECHARS);
 
 #define DESCLABEL_FMT	"%s%s%s%s%s"
 #define DESCLABEL_ARGS(node,prop,mark)		\
@@ -331,8 +357,8 @@ static void check_duplicate_label_prop(struct check *c, struct node *dt,
 	for_each_marker_of_type(m, LABEL)
 		check_duplicate_label(c, dt, m->ref, node, prop, m);
 }
-CHECK(duplicate_label, NULL, check_duplicate_label_node,
-      check_duplicate_label_prop, NULL, ERROR);
+ERROR(duplicate_label, NULL, check_duplicate_label_node,
+      check_duplicate_label_prop, NULL);
 
 static void check_explicit_phandles(struct check *c, struct node *root,
 				    struct node *node, struct property *prop)
@@ -391,7 +417,7 @@ static void check_explicit_phandles(struct check *c, struct node *root,
 
 	node->phandle = phandle;
 }
-PROP_CHECK(explicit_phandles, NULL, ERROR);
+PROP_ERROR(explicit_phandles, NULL);
 
 static void check_name_properties(struct check *c, struct node *root,
 				  struct node *node)
@@ -420,8 +446,8 @@ static void check_name_properties(struct check *c, struct node *root,
 		free(prop);
 	}
 }
-CHECK_IS_STRING(name_is_string, "name", ERROR);
-NODE_CHECK(name_properties, NULL, ERROR, &name_is_string);
+ERROR_IF_NOT_STRING(name_is_string, "name");
+NODE_ERROR(name_properties, NULL, &name_is_string);
 
 /*
  * Reference fixup functions
@@ -448,7 +474,7 @@ static void fixup_phandle_references(struct check *c, struct node *dt,
 		*((cell_t *)(prop->val.val + m->offset)) = cpu_to_fdt32(phandle);
 	}
 }
-CHECK(phandle_references, NULL, NULL, fixup_phandle_references, NULL, ERROR,
+ERROR(phandle_references, NULL, NULL, fixup_phandle_references, NULL,
       &duplicate_node_names, &explicit_phandles);
 
 static void fixup_path_references(struct check *c, struct node *dt,
@@ -473,19 +499,19 @@ static void fixup_path_references(struct check *c, struct node *dt,
 						  strlen(path) + 1);
 	}
 }
-CHECK(path_references, NULL, NULL, fixup_path_references, NULL, ERROR,
+ERROR(path_references, NULL, NULL, fixup_path_references, NULL,
       &duplicate_node_names);
 
 /*
  * Semantic checks
  */
-CHECK_IS_CELL(address_cells_is_cell, "#address-cells", WARN);
-CHECK_IS_CELL(size_cells_is_cell, "#size-cells", WARN);
-CHECK_IS_CELL(interrupt_cells_is_cell, "#interrupt-cells", WARN);
+WARNING_IF_NOT_CELL(address_cells_is_cell, "#address-cells");
+WARNING_IF_NOT_CELL(size_cells_is_cell, "#size-cells");
+WARNING_IF_NOT_CELL(interrupt_cells_is_cell, "#interrupt-cells");
 
-CHECK_IS_STRING(device_type_is_string, "device_type", WARN);
-CHECK_IS_STRING(model_is_string, "model", WARN);
-CHECK_IS_STRING(status_is_string, "status", WARN);
+WARNING_IF_NOT_STRING(device_type_is_string, "device_type");
+WARNING_IF_NOT_STRING(model_is_string, "model");
+WARNING_IF_NOT_STRING(status_is_string, "status");
 
 static void fixup_addr_size_cells(struct check *c, struct node *dt,
 				  struct node *node)
@@ -503,8 +529,8 @@ static void fixup_addr_size_cells(struct check *c, struct node *dt,
 	if (prop)
 		node->size_cells = propval_cell(prop);
 }
-CHECK(addr_size_cells, NULL, fixup_addr_size_cells, NULL, NULL, WARN,
-      &address_cells_is_cell, &size_cells_is_cell);
+WARNING(addr_size_cells, NULL, fixup_addr_size_cells, NULL, NULL,
+	&address_cells_is_cell, &size_cells_is_cell);
 
 #define node_addr_cells(n) \
 	(((n)->addr_cells == -1) ? 2 : (n)->addr_cells)
@@ -538,7 +564,7 @@ static void check_reg_format(struct check *c, struct node *dt,
 		     "(#address-cells == %d, #size-cells == %d)",
 		     node->fullpath, prop->val.len, addr_cells, size_cells);
 }
-NODE_CHECK(reg_format, NULL, WARN, &addr_size_cells);
+NODE_WARNING(reg_format, NULL, &addr_size_cells);
 
 static void check_ranges_format(struct check *c, struct node *dt,
 				struct node *node)
@@ -579,7 +605,7 @@ static void check_ranges_format(struct check *c, struct node *dt,
 		     p_addr_cells, c_addr_cells, c_size_cells);
 	}
 }
-NODE_CHECK(ranges_format, NULL, WARN, &addr_size_cells);
+NODE_WARNING(ranges_format, NULL, &addr_size_cells);
 
 /*
  * Style checks
@@ -606,7 +632,7 @@ static void check_avoid_default_addr_size(struct check *c, struct node *dt,
 		FAIL(c, "Relying on default #size-cells value for %s",
 		     node->fullpath);
 }
-NODE_CHECK(avoid_default_addr_size, NULL, WARN, &addr_size_cells);
+NODE_WARNING(avoid_default_addr_size, NULL, &addr_size_cells);
 
 static void check_obsolete_chosen_interrupt_controller(struct check *c,
 						       struct node *dt)
@@ -623,7 +649,7 @@ static void check_obsolete_chosen_interrupt_controller(struct check *c,
 		FAIL(c, "/chosen has obsolete \"interrupt-controller\" "
 		     "property");
 }
-TREE_CHECK(obsolete_chosen_interrupt_controller, NULL, WARN);
+TREE_WARNING(obsolete_chosen_interrupt_controller, NULL);
 
 static struct check *check_table[] = {
 	&duplicate_node_names, &duplicate_property_names,
@@ -642,8 +668,71 @@ static struct check *check_table[] = {
 
 	&avoid_default_addr_size,
 	&obsolete_chosen_interrupt_controller,
+
+	&always_fail,
 };
 
+static void enable_warning_error(struct check *c, bool warn, bool error)
+{
+	int i;
+
+	/* Raising level, also raise it for prereqs */
+	if ((warn && !c->warn) || (error && !c->error))
+		for (i = 0; i < c->num_prereqs; i++)
+			enable_warning_error(c->prereq[i], warn, error);
+
+	c->warn = c->warn || warn;
+	c->error = c->error || error;
+}
+
+static void disable_warning_error(struct check *c, bool warn, bool error)
+{
+	int i;
+
+	/* Lowering level, also lower it for things this is the prereq
+	 * for */
+	if ((warn && c->warn) || (error && c->error)) {
+		for (i = 0; i < ARRAY_SIZE(check_table); i++) {
+			struct check *cc = check_table[i];
+			int j;
+
+			for (j = 0; j < cc->num_prereqs; j++)
+				if (cc->prereq[j] == c)
+					disable_warning_error(cc, warn, error);
+		}
+	}
+
+	c->warn = c->warn && !warn;
+	c->error = c->error && !error;
+}
+
+void parse_checks_option(bool warn, bool error, const char *optarg)
+{
+	int i;
+	const char *name = optarg;
+	bool enable = true;
+
+	if ((strncmp(optarg, "no-", 3) == 0)
+	    || (strncmp(optarg, "no_", 3) == 0)) {
+		name = optarg + 3;
+		enable = false;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(check_table); i++) {
+		struct check *c = check_table[i];
+
+		if (streq(c->name, name)) {
+			if (enable)
+				enable_warning_error(c, warn, error);
+			else
+				disable_warning_error(c, warn, error);
+			return;
+		}
+	}
+
+	die("Unrecognized check name \"%s\"\n", name);
+}
+
 void process_checks(int force, struct boot_info *bi)
 {
 	struct node *dt = bi->dt;
@@ -653,7 +742,7 @@ void process_checks(int force, struct boot_info *bi)
 	for (i = 0; i < ARRAY_SIZE(check_table); i++) {
 		struct check *c = check_table[i];
 
-		if (c->level != IGNORE)
+		if (c->warn || c->error)
 			error = error || run_check(c, dt);
 	}
 
diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c
index fe555e819bf..4a40c5b9247 100644
--- a/scripts/dtc/data.c
+++ b/scripts/dtc/data.c
@@ -68,40 +68,6 @@ struct data data_copy_mem(const char *mem, int len)
 	return d;
 }
 
-static char get_oct_char(const char *s, int *i)
-{
-	char x[4];
-	char *endx;
-	long val;
-
-	x[3] = '\0';
-	strncpy(x, s + *i, 3);
-
-	val = strtol(x, &endx, 8);
-
-	assert(endx > x);
-
-	(*i) += endx - x;
-	return val;
-}
-
-static char get_hex_char(const char *s, int *i)
-{
-	char x[3];
-	char *endx;
-	long val;
-
-	x[2] = '\0';
-	strncpy(x, s + *i, 2);
-
-	val = strtol(x, &endx, 16);
-	if (!(endx  > x))
-		die("\\x used with no following hex digits\n");
-
-	(*i) += endx - x;
-	return val;
-}
-
 struct data data_copy_escape_string(const char *s, int len)
 {
 	int i = 0;
@@ -114,53 +80,10 @@ struct data data_copy_escape_string(const char *s, int len)
 	while (i < len) {
 		char c = s[i++];
 
-		if (c != '\\') {
-			q[d.len++] = c;
-			continue;
-		}
-
-		c = s[i++];
-		assert(c);
-		switch (c) {
-		case 'a':
-			q[d.len++] = '\a';
-			break;
-		case 'b':
-			q[d.len++] = '\b';
-			break;
-		case 't':
-			q[d.len++] = '\t';
-			break;
-		case 'n':
-			q[d.len++] = '\n';
-			break;
-		case 'v':
-			q[d.len++] = '\v';
-			break;
-		case 'f':
-			q[d.len++] = '\f';
-			break;
-		case 'r':
-			q[d.len++] = '\r';
-			break;
-		case '0':
-		case '1':
-		case '2':
-		case '3':
-		case '4':
-		case '5':
-		case '6':
-		case '7':
-			i--; /* need to re-read the first digit as
-			      * part of the octal value */
-			q[d.len++] = get_oct_char(s, &i);
-			break;
-		case 'x':
-			q[d.len++] = get_hex_char(s, &i);
-			break;
-		default:
-			q[d.len++] = c;
-		}
+		if (c == '\\')
+			c = get_escape_char(s, &i);
+
+		q[d.len++] = c;
 	}
 
 	q[d.len++] = '\0';
@@ -245,11 +168,33 @@ struct data data_merge(struct data d1, struct data d2)
 	return d;
 }
 
-struct data data_append_cell(struct data d, cell_t word)
+struct data data_append_integer(struct data d, uint64_t value, int bits)
 {
-	cell_t beword = cpu_to_fdt32(word);
-
-	return data_append_data(d, &beword, sizeof(beword));
+	uint8_t value_8;
+	uint16_t value_16;
+	uint32_t value_32;
+	uint64_t value_64;
+
+	switch (bits) {
+	case 8:
+		value_8 = value;
+		return data_append_data(d, &value_8, 1);
+
+	case 16:
+		value_16 = cpu_to_fdt16(value);
+		return data_append_data(d, &value_16, 2);
+
+	case 32:
+		value_32 = cpu_to_fdt32(value);
+		return data_append_data(d, &value_32, 4);
+
+	case 64:
+		value_64 = cpu_to_fdt64(value);
+		return data_append_data(d, &value_64, 8);
+
+	default:
+		die("Invalid literal size (%d)\n", bits);
+	}
 }
 
 struct data data_append_re(struct data d, const struct fdt_reserve_entry *re)
@@ -262,11 +207,14 @@ struct data data_append_re(struct data d, const struct fdt_reserve_entry *re)
 	return data_append_data(d, &bere, sizeof(bere));
 }
 
-struct data data_append_addr(struct data d, uint64_t addr)
+struct data data_append_cell(struct data d, cell_t word)
 {
-	uint64_t beaddr = cpu_to_fdt64(addr);
+	return data_append_integer(d, word, sizeof(word) * 8);
+}
 
-	return data_append_data(d, &beaddr, sizeof(beaddr));
+struct data data_append_addr(struct data d, uint64_t addr)
+{
+	return data_append_integer(d, addr, sizeof(addr) * 8);
 }
 
 struct data data_append_byte(struct data d, uint8_t byte)
diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l
index e866ea5166a..254d5af8895 100644
--- a/scripts/dtc/dtc-lexer.l
+++ b/scripts/dtc/dtc-lexer.l
@@ -29,6 +29,7 @@ PROPNODECHAR	[a-zA-Z0-9,._+*#?@-]
 PATHCHAR	({PROPNODECHAR}|[/])
 LABEL		[a-zA-Z_][a-zA-Z0-9_]*
 STRING		\"([^\\"]|\\.)*\"
+CHAR_LITERAL	'([^']|\\')*'
 WS		[[:space:]]
 COMMENT		"/*"([^*]|\*+[^*/])*\*+"/"
 LINECOMMENT	"//".*\n
@@ -70,6 +71,27 @@ static int pop_input_file(void);
 			push_input_file(name);
 		}
 
+<*>^"#"(line)?{WS}+[0-9]+{WS}+{STRING}({WS}+[0-9]+)? {
+			char *line, *tmp, *fn;
+			/* skip text before line # */
+			line = yytext;
+			while (!isdigit(*line))
+				line++;
+			/* skip digits in line # */
+			tmp = line;
+			while (!isspace(*tmp))
+				tmp++;
+			/* "NULL"-terminate line # */
+			*tmp = '\0';
+			/* start of filename */
+			fn = strchr(tmp + 1, '"') + 1;
+			/* strip trailing " from filename */
+			tmp = strchr(fn, '"');
+			*tmp = 0;
+			/* -1 since #line is the number of the next line */
+			srcpos_set_line(xstrdup(fn), atoi(line) - 1);
+		}
+
 <*><<EOF>>		{
 			if (!pop_input_file()) {
 				yyterminate();
@@ -96,6 +118,26 @@ static int pop_input_file(void);
 			return DT_MEMRESERVE;
 		}
 
+<*>"/bits/"	{
+			DPRINT("Keyword: /bits/\n");
+			BEGIN_DEFAULT();
+			return DT_BITS;
+		}
+
+<*>"/delete-property/"	{
+			DPRINT("Keyword: /delete-property/\n");
+			DPRINT("<PROPNODENAME>\n");
+			BEGIN(PROPNODENAME);
+			return DT_DEL_PROP;
+		}
+
+<*>"/delete-node/"	{
+			DPRINT("Keyword: /delete-node/\n");
+			DPRINT("<PROPNODENAME>\n");
+			BEGIN(PROPNODENAME);
+			return DT_DEL_NODE;
+		}
+
 <*>{LABEL}:	{
 			DPRINT("Label: %s\n", yytext);
 			yylval.labelref = xstrdup(yytext);
@@ -103,12 +145,19 @@ static int pop_input_file(void);
 			return DT_LABEL;
 		}
 
-<V1>[0-9]+|0[xX][0-9a-fA-F]+      {
+<V1>([0-9]+|0[xX][0-9a-fA-F]+)(U|L|UL|LL|ULL)? {
 			yylval.literal = xstrdup(yytext);
 			DPRINT("Literal: '%s'\n", yylval.literal);
 			return DT_LITERAL;
 		}
 
+<*>{CHAR_LITERAL}	{
+			yytext[yyleng-1] = '\0';
+			yylval.literal = xstrdup(yytext+1);
+			DPRINT("Character literal: %s\n", yylval.literal);
+			return DT_CHAR_LITERAL;
+		}
+
 <*>\&{LABEL}	{	/* label reference */
 			DPRINT("Ref: %s\n", yytext+1);
 			yylval.labelref = xstrdup(yytext+1);
@@ -134,9 +183,10 @@ static int pop_input_file(void);
 			return ']';
 		}
 
-<PROPNODENAME>{PROPNODECHAR}+ {
+<PROPNODENAME>\\?{PROPNODECHAR}+ {
 			DPRINT("PropNodeName: %s\n", yytext);
-			yylval.propnodename = xstrdup(yytext);
+			yylval.propnodename = xstrdup((yytext[0] == '\\') ?
+							yytext + 1 : yytext);
 			BEGIN_DEFAULT();
 			return DT_PROPNODENAME;
 		}
@@ -150,6 +200,15 @@ static int pop_input_file(void);
 <*>{COMMENT}+	/* eat C-style comments */
 <*>{LINECOMMENT}+ /* eat C++-style comments */
 
+<*>"<<"		{ return DT_LSHIFT; };
+<*>">>"		{ return DT_RSHIFT; };
+<*>"<="		{ return DT_LE; };
+<*>">="		{ return DT_GE; };
+<*>"=="		{ return DT_EQ; };
+<*>"!="		{ return DT_NE; };
+<*>"&&"		{ return DT_AND; };
+<*>"||"		{ return DT_OR; };
+
 <*>.		{
 			DPRINT("Char: %c (\\x%02x)\n", yytext[0],
 				(unsigned)yytext[0]);
diff --git a/scripts/dtc/dtc-lexer.lex.c_shipped b/scripts/dtc/dtc-lexer.lex.c_shipped
index 8bbe1281705..a6c5fcdfc03 100644
--- a/scripts/dtc/dtc-lexer.lex.c_shipped
+++ b/scripts/dtc/dtc-lexer.lex.c_shipped
@@ -1,5 +1,6 @@
+#line 2 "dtc-lexer.lex.c"
 
-#line 3 "scripts/dtc/dtc-lexer.lex.c_shipped"
+#line 4 "dtc-lexer.lex.c"
 
 #define  YY_INT_ALIGNED short int
 
@@ -53,7 +54,6 @@ typedef int flex_int32_t;
 typedef unsigned char flex_uint8_t; 
 typedef unsigned short int flex_uint16_t;
 typedef unsigned int flex_uint32_t;
-#endif /* ! C99 */
 
 /* Limits of integral types. */
 #ifndef INT8_MIN
@@ -84,6 +84,8 @@ typedef unsigned int flex_uint32_t;
 #define UINT32_MAX             (4294967295U)
 #endif
 
+#endif /* ! C99 */
+
 #endif /* ! FLEXINT_H */
 
 #ifdef __cplusplus
@@ -140,7 +142,15 @@ typedef unsigned int flex_uint32_t;
 
 /* Size of default input buffer. */
 #ifndef YY_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k.
+ * Moreover, YY_BUF_SIZE is 2*YY_READ_BUF_SIZE in the general case.
+ * Ditto for the __ia64__ case accordingly.
+ */
+#define YY_BUF_SIZE 32768
+#else
 #define YY_BUF_SIZE 16384
+#endif /* __ia64__ */
 #endif
 
 /* The state buf must be large enough to hold one state per character in the main buffer.
@@ -362,8 +372,8 @@ static void yy_fatal_error (yyconst char msg[]  );
 	*yy_cp = '\0'; \
 	(yy_c_buf_p) = yy_cp;
 
-#define YY_NUM_RULES 17
-#define YY_END_OF_BUFFER 18
+#define YY_NUM_RULES 30
+#define YY_END_OF_BUFFER 31
 /* This struct is not used in this scanner,
    but its presence is necessary. */
 struct yy_trans_info
@@ -371,19 +381,25 @@ struct yy_trans_info
 	flex_int32_t yy_verify;
 	flex_int32_t yy_nxt;
 	};
-static yyconst flex_int16_t yy_accept[94] =
+static yyconst flex_int16_t yy_accept[161] =
     {   0,
         0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
-       18,   16,   13,   13,   16,   16,   16,   16,   16,   16,
-       16,   10,   11,   11,    6,    6,   13,    0,    2,    0,
-        7,    0,    0,    0,    0,    0,    0,    0,    5,    0,
-        9,    9,   11,   11,    6,    0,    7,    0,    0,    0,
-        0,   15,    0,    0,    0,    0,    6,    0,   14,    0,
-        0,    0,    0,    0,    8,    0,    0,    0,    0,    0,
-        0,    0,    0,    0,    0,    0,    0,    0,    3,   12,
-        0,    0,    0,    0,    0,    0,    0,    0,    1,    0,
-        0,    4,    0
-
+       31,   29,   18,   18,   29,   29,   29,   29,   29,   29,
+       29,   29,   29,   29,   29,   29,   29,   29,   15,   16,
+       16,   29,   16,   10,   10,   18,   26,    0,    3,    0,
+       27,   12,    0,    0,   11,    0,    0,    0,    0,    0,
+        0,    0,   21,   23,   25,   24,   22,    0,    9,   28,
+        0,    0,    0,   14,   14,   16,   16,   16,   10,   10,
+       10,    0,   12,    0,   11,    0,    0,    0,   20,    0,
+        0,    0,    0,    0,    0,    0,    0,   16,   10,   10,
+       10,    0,   19,    0,    0,    0,    0,    0,    0,    0,
+
+        0,    0,   16,   13,    0,    0,    0,    0,    0,    0,
+        0,    0,    0,   16,    6,    0,    0,    0,    0,    0,
+        0,    2,    0,    0,    0,    0,    0,    0,    0,    0,
+        4,   17,    0,    0,    2,    0,    0,    0,    0,    0,
+        0,    0,    0,    0,    0,    0,    0,    1,    0,    0,
+        0,    0,    5,    8,    0,    0,    0,    0,    7,    0
     } ;
 
 static yyconst flex_int32_t yy_ec[256] =
@@ -391,17 +407,17 @@ static yyconst flex_int32_t yy_ec[256] =
         1,    1,    1,    1,    1,    1,    1,    1,    2,    3,
         2,    2,    2,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    2,    1,    4,    5,    1,    1,    6,    1,    1,
-        1,    7,    5,    5,    8,    5,    9,   10,   11,   12,
-       12,   12,   12,   12,   12,   12,   12,   13,    1,    1,
-        1,    1,    5,    5,   14,   14,   14,   14,   14,   14,
-       15,   15,   15,   15,   15,   15,   15,   15,   15,   15,
-       15,   15,   15,   15,   15,   15,   15,   16,   15,   15,
-        1,   17,   18,    1,   15,    1,   14,   19,   20,   21,
-
-       22,   14,   15,   15,   23,   15,   15,   24,   25,   26,
-       15,   15,   15,   27,   28,   29,   30,   31,   15,   16,
-       15,   15,   32,    1,   33,    1,    1,    1,    1,    1,
+        1,    2,    4,    5,    6,    1,    1,    7,    8,    1,
+        1,    9,   10,   10,   11,   10,   12,   13,   14,   15,
+       15,   15,   15,   15,   15,   15,   15,   16,    1,   17,
+       18,   19,   10,   10,   20,   20,   20,   20,   20,   20,
+       21,   21,   21,   21,   21,   22,   21,   21,   21,   21,
+       21,   21,   21,   21,   23,   21,   21,   24,   21,   21,
+        1,   25,   26,    1,   21,    1,   20,   27,   28,   29,
+
+       30,   20,   21,   21,   31,   21,   21,   32,   33,   34,
+       35,   36,   21,   37,   38,   39,   40,   41,   21,   24,
+       42,   21,   43,   44,   45,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
@@ -418,112 +434,163 @@ static yyconst flex_int32_t yy_ec[256] =
         1,    1,    1,    1,    1
     } ;
 
-static yyconst flex_int32_t yy_meta[34] =
+static yyconst flex_int32_t yy_meta[46] =
     {   0,
-        1,    1,    1,    1,    2,    1,    2,    2,    3,    4,
-        4,    4,    5,    6,    7,    7,    1,    1,    6,    6,
-        6,    6,    7,    7,    7,    7,    7,    7,    7,    7,
-        7,    8,    1
+        1,    1,    1,    1,    1,    2,    3,    1,    2,    2,
+        2,    4,    5,    5,    5,    6,    1,    1,    1,    7,
+        8,    8,    8,    8,    1,    1,    7,    7,    7,    7,
+        8,    8,    8,    8,    8,    8,    8,    8,    8,    8,
+        8,    8,    3,    1,    1
     } ;
 
-static yyconst flex_int16_t yy_base[106] =
+static yyconst flex_int16_t yy_base[175] =
     {   0,
-        0,    0,  237,  236,   25,    0,   47,    0,   30,   71,
-      244,  247,   82,   84,   84,  211,   95,  229,  218,    0,
-      111,  247,    0,   84,   83,   95,  106,   86,  247,  237,
-        0,  230,  231,  234,  207,  209,  212,  220,  247,  206,
-      247,  218,    0,  106,  116,    0,    0,    0,  223,   89,
-      226,  219,  199,  206,  200,  204,    0,  190,  213,  212,
-      202,   91,  178,  161,  247,  172,  144,  150,  140,  130,
-      140,  124,  128,  120,  138,  137,  123,  122,  247,  247,
-      134,  114,  132,   86,  135,  125,   90,  136,  247,   97,
-       29,  247,  247,  153,  156,  161,  165,  170,  176,  180,
-
-      187,  195,  200,  205,  212
+        0,  388,  381,   40,   41,  386,   71,  385,   34,   44,
+      390,  395,   60,   62,  371,  112,  111,  111,  111,  104,
+      370,  106,  371,  342,  124,  119,    0,  144,  395,    0,
+      123,    0,  159,  153,  165,  167,  395,  130,  395,  382,
+      395,    0,  372,  122,  395,  157,  374,  379,  350,   21,
+      346,  349,  395,  395,  395,  395,  395,  362,  395,  395,
+      181,  346,  342,  395,  359,    0,  191,  343,  190,  351,
+      350,    0,    0,    0,  173,  362,  177,  367,  357,  329,
+      335,  328,  337,  331,  206,  329,  334,  327,  395,  338,
+      170,  314,  346,  345,  318,  325,  343,  158,  316,  212,
+
+      322,  319,  320,  395,  340,  336,  308,  305,  314,  304,
+      295,  138,  208,  220,  395,  292,  305,  265,  264,  254,
+      201,  222,  285,  275,  273,  270,  236,  235,  225,  115,
+      395,  395,  252,  216,  216,  217,  214,  230,  209,  220,
+      213,  239,  211,  217,  216,  209,  229,  395,  240,  225,
+      206,  169,  395,  395,  116,  106,   99,   54,  395,  395,
+      254,  260,  268,  272,  276,  282,  289,  293,  301,  309,
+      313,  319,  327,  335
     } ;
 
-static yyconst flex_int16_t yy_def[106] =
+static yyconst flex_int16_t yy_def[175] =
     {   0,
-       93,    1,    1,    1,    1,    5,   93,    7,    1,    1,
-       93,   93,   93,   93,   94,   95,   93,   96,   17,   97,
-       96,   93,   98,   99,   93,   93,   93,   94,   93,   94,
-      100,   93,  101,  102,   93,   93,   93,   96,   93,   93,
-       93,   96,   98,   99,   93,  103,  100,  104,  101,  101,
-      102,   93,   93,   93,   93,   93,  103,  104,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,  105,   93,  105,   93,  105,
-       93,   93,    0,   93,   93,   93,   93,   93,   93,   93,
-
-       93,   93,   93,   93,   93
+      160,    1,    1,    1,    1,    5,  160,    7,    1,    1,
+      160,  160,  160,  160,  160,  161,  162,  163,  160,  160,
+      160,  160,  164,  160,  160,  160,  165,  164,  160,  166,
+      167,  166,  166,  160,  160,  160,  160,  161,  160,  161,
+      160,  168,  160,  163,  160,  163,  169,  170,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  164,  160,  160,
+      160,  160,  160,  160,  164,  166,  167,  166,  160,  160,
+      160,  171,  168,  172,  163,  169,  169,  170,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  166,  160,  160,
+      171,  172,  160,  160,  160,  160,  160,  160,  160,  160,
+
+      160,  160,  166,  160,  160,  160,  160,  160,  160,  160,
+      160,  173,  160,  166,  160,  160,  160,  160,  160,  160,
+      173,  160,  173,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  174,  160,  160,  160,  174,  160,  174,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,    0,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160
     } ;
 
-static yyconst flex_int16_t yy_nxt[281] =
+static yyconst flex_int16_t yy_nxt[441] =
     {   0,
-       12,   13,   14,   15,   12,   16,   12,   12,   17,   12,
-       12,   12,   12,   18,   18,   18,   12,   12,   18,   18,
-       18,   18,   18,   18,   18,   18,   18,   18,   18,   18,
-       18,   12,   12,   19,   20,   20,   20,   92,   21,   25,
-       26,   26,   22,   21,   21,   21,   21,   12,   13,   14,
-       15,   23,   16,   23,   23,   19,   23,   23,   23,   12,
-       24,   24,   24,   12,   12,   24,   24,   24,   24,   24,
-       24,   24,   24,   24,   24,   24,   24,   24,   12,   12,
-       25,   26,   26,   27,   27,   27,   27,   29,   43,   29,
-       43,   43,   45,   45,   45,   50,   39,   59,   46,   93,
-
-       30,   33,   30,   34,   45,   45,   45,   27,   27,   68,
-       43,   91,   43,   43,   69,   35,   87,   36,   39,   37,
-       42,   42,   42,   39,   42,   45,   45,   45,   89,   42,
-       42,   42,   42,   85,   85,   86,   85,   85,   86,   89,
-       84,   90,   83,   82,   81,   80,   79,   78,   77,   76,
-       75,   74,   90,   28,   28,   28,   28,   28,   28,   28,
-       28,   31,   31,   31,   38,   38,   38,   38,   41,   73,
-       41,   43,   72,   43,   71,   43,   43,   44,   33,   44,
-       44,   44,   44,   47,   69,   47,   47,   49,   49,   49,
-       49,   49,   49,   49,   49,   51,   51,   51,   51,   51,
-
-       51,   51,   51,   57,   70,   57,   58,   58,   58,   67,
-       58,   58,   88,   88,   88,   88,   88,   88,   88,   88,
-       34,   66,   65,   64,   63,   62,   61,   60,   52,   50,
-       39,   56,   39,   55,   54,   53,   52,   50,   48,   93,
-       40,   39,   32,   93,   19,   19,   11,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93
+       12,   13,   14,   15,   16,   12,   17,   18,   12,   12,
+       12,   19,   12,   12,   12,   12,   20,   21,   22,   23,
+       23,   23,   23,   23,   12,   12,   23,   23,   23,   23,
+       23,   23,   23,   23,   23,   23,   23,   23,   23,   23,
+       23,   23,   12,   24,   12,   25,   34,   35,   35,   25,
+       81,   26,   26,   27,   27,   27,   34,   35,   35,   82,
+       28,   36,   36,   36,   36,  159,   29,   28,   28,   28,
+       28,   12,   13,   14,   15,   16,   30,   17,   18,   30,
+       30,   30,   26,   30,   30,   30,   12,   20,   21,   22,
+       31,   31,   31,   31,   31,   32,   12,   31,   31,   31,
+
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   12,   24,   12,   39,   41,   45,   47,
+       53,   54,   48,   56,   57,   61,   61,   47,   66,   45,
+       48,   66,   66,   66,   39,   46,   40,   49,   59,   50,
+      158,   51,  122,   52,  157,   49,   46,   50,  136,   63,
+      137,   52,  156,   43,   40,   62,   65,   65,   65,   59,
+       61,   61,  123,   65,   75,   69,   69,   69,   36,   36,
+       65,   65,   65,   65,   70,   71,   72,   69,   69,   69,
+       45,   46,   61,   61,  109,   77,   70,   71,   93,  110,
+       68,   70,   71,   85,   85,   85,   66,   46,  155,   66,
+
+       66,   66,   69,   69,   69,  122,   59,  100,  100,   61,
+       61,   70,   71,  100,  100,  148,  112,  154,   85,   85,
+       85,   61,   61,  129,  129,  123,  129,  129,  135,  135,
+      135,  142,  142,  148,  143,  149,  153,  135,  135,  135,
+      142,  142,  160,  143,  152,  151,  150,  146,  145,  144,
+      141,  140,  139,  149,   38,   38,   38,   38,   38,   38,
+       38,   38,   42,  138,  134,  133,   42,   42,   44,   44,
+       44,   44,   44,   44,   44,   44,   58,   58,   58,   58,
+       64,  132,   64,   66,  131,  130,   66,  160,   66,   66,
+       67,  128,  127,   67,   67,   67,   67,   73,  126,   73,
+
+       73,   76,   76,   76,   76,   76,   76,   76,   76,   78,
+       78,   78,   78,   78,   78,   78,   78,   91,  125,   91,
+       92,  124,   92,   92,  120,   92,   92,  121,  121,  121,
+      121,  121,  121,  121,  121,  147,  147,  147,  147,  147,
+      147,  147,  147,  119,  118,  117,  116,  115,   47,  114,
+      110,  113,  111,  108,  107,  106,   48,  105,  104,   89,
+      103,  102,  101,   99,   98,   97,   96,   95,   94,   79,
+       77,   90,   89,   88,   59,   87,   86,   59,   84,   83,
+       80,   79,   77,   74,  160,   60,   59,   55,   37,  160,
+       33,   25,   26,   25,   11,  160,  160,  160,  160,  160,
+
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160
     } ;
 
-static yyconst flex_int16_t yy_chk[281] =
+static yyconst flex_int16_t yy_chk[441] =
     {   0,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    5,    5,    5,    5,   91,    5,    9,
-        9,    9,    5,    5,    5,    5,    5,    7,    7,    7,
+        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        1,    1,    1,    1,    1,    4,    9,    9,    9,   10,
+       50,    4,    5,    5,    5,    5,   10,   10,   10,   50,
+        5,   13,   13,   14,   14,  158,    5,    5,    5,    5,
+        5,    7,    7,    7,    7,    7,    7,    7,    7,    7,
         7,    7,    7,    7,    7,    7,    7,    7,    7,    7,
         7,    7,    7,    7,    7,    7,    7,    7,    7,    7,
+
         7,    7,    7,    7,    7,    7,    7,    7,    7,    7,
-       10,   10,   10,   13,   13,   14,   14,   15,   24,   28,
-       24,   24,   25,   25,   25,   50,   24,   50,   25,   90,
-
-       15,   17,   28,   17,   26,   26,   26,   27,   27,   62,
-       44,   87,   44,   44,   62,   17,   84,   17,   44,   17,
-       21,   21,   21,   21,   21,   45,   45,   45,   86,   21,
-       21,   21,   21,   83,   83,   83,   85,   85,   85,   88,
-       82,   86,   81,   78,   77,   76,   75,   74,   73,   72,
-       71,   70,   88,   94,   94,   94,   94,   94,   94,   94,
-       94,   95,   95,   95,   96,   96,   96,   96,   97,   69,
-       97,   98,   68,   98,   67,   98,   98,   99,   66,   99,
-       99,   99,   99,  100,   64,  100,  100,  101,  101,  101,
-      101,  101,  101,  101,  101,  102,  102,  102,  102,  102,
-
-      102,  102,  102,  103,   63,  103,  104,  104,  104,   61,
-      104,  104,  105,  105,  105,  105,  105,  105,  105,  105,
-       60,   59,   58,   56,   55,   54,   53,   52,   51,   49,
-       42,   40,   38,   37,   36,   35,   34,   33,   32,   30,
-       19,   18,   16,   11,    4,    3,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93,
-       93,   93,   93,   93,   93,   93,   93,   93,   93,   93
+        7,    7,    7,    7,    7,    7,   16,   17,   18,   19,
+       20,   20,   19,   22,   22,   25,   25,   26,   31,   44,
+       26,   31,   31,   31,   38,   18,   16,   19,   31,   19,
+      157,   19,  112,   19,  156,   26,   44,   26,  130,   26,
+      130,   26,  155,   17,   38,   25,   28,   28,   28,   28,
+       33,   33,  112,   28,   46,   34,   34,   34,   36,   36,
+       28,   28,   28,   28,   34,   34,   34,   35,   35,   35,
+       75,   46,   61,   61,   98,   77,   35,   35,   77,   98,
+       33,   91,   91,   61,   61,   61,   67,   75,  152,   67,
+
+       67,   67,   69,   69,   69,  121,   67,   85,   85,  113,
+      113,   69,   69,  100,  100,  143,  100,  151,   85,   85,
+       85,  114,  114,  122,  122,  121,  129,  129,  135,  135,
+      135,  138,  138,  147,  138,  143,  150,  129,  129,  129,
+      142,  142,  149,  142,  146,  145,  144,  141,  140,  139,
+      137,  136,  134,  147,  161,  161,  161,  161,  161,  161,
+      161,  161,  162,  133,  128,  127,  162,  162,  163,  163,
+      163,  163,  163,  163,  163,  163,  164,  164,  164,  164,
+      165,  126,  165,  166,  125,  124,  166,  123,  166,  166,
+      167,  120,  119,  167,  167,  167,  167,  168,  118,  168,
+
+      168,  169,  169,  169,  169,  169,  169,  169,  169,  170,
+      170,  170,  170,  170,  170,  170,  170,  171,  117,  171,
+      172,  116,  172,  172,  111,  172,  172,  173,  173,  173,
+      173,  173,  173,  173,  173,  174,  174,  174,  174,  174,
+      174,  174,  174,  110,  109,  108,  107,  106,  105,  103,
+      102,  101,   99,   97,   96,   95,   94,   93,   92,   90,
+       88,   87,   86,   84,   83,   82,   81,   80,   79,   78,
+       76,   71,   70,   68,   65,   63,   62,   58,   52,   51,
+       49,   48,   47,   43,   40,   24,   23,   21,   15,   11,
+        8,    6,    3,    2,  160,  160,  160,  160,  160,  160,
+
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160,
+      160,  160,  160,  160,  160,  160,  160,  160,  160,  160
     } ;
 
 static yy_state_type yy_last_accepting_state;
@@ -540,6 +607,7 @@ int yy_flex_debug = 0;
 #define YY_MORE_ADJ 0
 #define YY_RESTORE_YY_MORE_OFFSET
 char *yytext;
+#line 1 "dtc-lexer.l"
 /*
  * (C) Copyright David Gibson <dwg@au1.ibm.com>, IBM Corporation.  2005.
  *
@@ -561,6 +629,10 @@ char *yytext;
  */
 #define YY_NO_INPUT 1
 
+
+
+
+#line 38 "dtc-lexer.l"
 #include "dtc.h"
 #include "srcpos.h"
 #include "dtc-parser.tab.h"
@@ -588,6 +660,7 @@ static int dts_version = 1;
 
 static void push_input_file(const char *filename);
 static int pop_input_file(void);
+#line 664 "dtc-lexer.lex.c"
 
 #define INITIAL 0
 #define INCLUDE 1
@@ -670,7 +743,12 @@ static int input (void );
 
 /* Amount of stuff to slurp up with each read. */
 #ifndef YY_READ_BUF_SIZE
+#ifdef __ia64__
+/* On IA-64, the buffer size is 16k, not 8k */
+#define YY_READ_BUF_SIZE 16384
+#else
 #define YY_READ_BUF_SIZE 8192
+#endif /* __ia64__ */
 #endif
 
 /* Copy whatever the last rule matched to the standard output. */
@@ -689,7 +767,7 @@ static int input (void );
 	if ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \
 		{ \
 		int c = '*'; \
-		unsigned n; \
+		size_t n; \
 		for ( n = 0; n < max_size && \
 			     (c = getc( yyin )) != EOF && c != '\n'; ++n ) \
 			buf[n] = (char) c; \
@@ -761,6 +839,9 @@ extern int yylex (void);
 #endif
 
 #define YY_RULE_SETUP \
+	if ( yyleng > 0 ) \
+		YY_CURRENT_BUFFER_LVALUE->yy_at_bol = \
+				(yytext[yyleng - 1] == '\n'); \
 	YY_USER_ACTION
 
 /** The main scanner function which does all the work.
@@ -771,6 +852,10 @@ YY_DECL
 	register char *yy_cp, *yy_bp;
 	register int yy_act;
     
+#line 67 "dtc-lexer.l"
+
+#line 858 "dtc-lexer.lex.c"
+
 	if ( !(yy_init) )
 		{
 		(yy_init) = 1;
@@ -810,6 +895,7 @@ YY_DECL
 		yy_bp = yy_cp;
 
 		yy_current_state = (yy_start);
+		yy_current_state += YY_AT_BOL();
 yy_match:
 		do
 			{
@@ -822,13 +908,13 @@ yy_match:
 			while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 				{
 				yy_current_state = (int) yy_def[yy_current_state];
-				if ( yy_current_state >= 94 )
+				if ( yy_current_state >= 161 )
 					yy_c = yy_meta[(unsigned int) yy_c];
 				}
 			yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
 			++yy_cp;
 			}
-		while ( yy_current_state != 93 );
+		while ( yy_current_state != 160 );
 		yy_cp = (yy_last_accepting_cpos);
 		yy_current_state = (yy_last_accepting_state);
 
@@ -851,26 +937,54 @@ do_action:	/* This label is used only to access EOF actions. */
 case 1:
 /* rule 1 can match eol */
 YY_RULE_SETUP
+#line 68 "dtc-lexer.l"
 {
 			char *name = strchr(yytext, '\"') + 1;
 			yytext[yyleng-1] = '\0';
 			push_input_file(name);
 		}
 	YY_BREAK
+case 2:
+/* rule 2 can match eol */
+YY_RULE_SETUP
+#line 74 "dtc-lexer.l"
+{
+			char *line, *tmp, *fn;
+			/* skip text before line # */
+			line = yytext;
+			while (!isdigit(*line))
+				line++;
+			/* skip digits in line # */
+			tmp = line;
+			while (!isspace(*tmp))
+				tmp++;
+			/* "NULL"-terminate line # */
+			*tmp = '\0';
+			/* start of filename */
+			fn = strchr(tmp + 1, '"') + 1;
+			/* strip trailing " from filename */
+			tmp = strchr(fn, '"');
+			*tmp = 0;
+			/* -1 since #line is the number of the next line */
+			srcpos_set_line(xstrdup(fn), atoi(line) - 1);
+		}
+	YY_BREAK
 case YY_STATE_EOF(INITIAL):
 case YY_STATE_EOF(INCLUDE):
 case YY_STATE_EOF(BYTESTRING):
 case YY_STATE_EOF(PROPNODENAME):
 case YY_STATE_EOF(V1):
+#line 95 "dtc-lexer.l"
 {
 			if (!pop_input_file()) {
 				yyterminate();
 			}
 		}
 	YY_BREAK
-case 2:
-/* rule 2 can match eol */
+case 3:
+/* rule 3 can match eol */
 YY_RULE_SETUP
+#line 101 "dtc-lexer.l"
 {
 			DPRINT("String: %s\n", yytext);
 			yylval.data = data_copy_escape_string(yytext+1,
@@ -878,8 +992,9 @@ YY_RULE_SETUP
 			return DT_STRING;
 		}
 	YY_BREAK
-case 3:
+case 4:
 YY_RULE_SETUP
+#line 108 "dtc-lexer.l"
 {
 			DPRINT("Keyword: /dts-v1/\n");
 			dts_version = 1;
@@ -887,16 +1002,47 @@ YY_RULE_SETUP
 			return DT_V1;
 		}
 	YY_BREAK
-case 4:
+case 5:
 YY_RULE_SETUP
+#line 115 "dtc-lexer.l"
 {
 			DPRINT("Keyword: /memreserve/\n");
 			BEGIN_DEFAULT();
 			return DT_MEMRESERVE;
 		}
 	YY_BREAK
-case 5:
+case 6:
+YY_RULE_SETUP
+#line 121 "dtc-lexer.l"
+{
+			DPRINT("Keyword: /bits/\n");
+			BEGIN_DEFAULT();
+			return DT_BITS;
+		}
+	YY_BREAK
+case 7:
 YY_RULE_SETUP
+#line 127 "dtc-lexer.l"
+{
+			DPRINT("Keyword: /delete-property/\n");
+			DPRINT("<PROPNODENAME>\n");
+			BEGIN(PROPNODENAME);
+			return DT_DEL_PROP;
+		}
+	YY_BREAK
+case 8:
+YY_RULE_SETUP
+#line 134 "dtc-lexer.l"
+{
+			DPRINT("Keyword: /delete-node/\n");
+			DPRINT("<PROPNODENAME>\n");
+			BEGIN(PROPNODENAME);
+			return DT_DEL_NODE;
+		}
+	YY_BREAK
+case 9:
+YY_RULE_SETUP
+#line 141 "dtc-lexer.l"
 {
 			DPRINT("Label: %s\n", yytext);
 			yylval.labelref = xstrdup(yytext);
@@ -904,24 +1050,38 @@ YY_RULE_SETUP
 			return DT_LABEL;
 		}
 	YY_BREAK
-case 6:
+case 10:
 YY_RULE_SETUP
+#line 148 "dtc-lexer.l"
 {
 			yylval.literal = xstrdup(yytext);
 			DPRINT("Literal: '%s'\n", yylval.literal);
 			return DT_LITERAL;
 		}
 	YY_BREAK
-case 7:
+case 11:
+/* rule 11 can match eol */
+YY_RULE_SETUP
+#line 154 "dtc-lexer.l"
+{
+			yytext[yyleng-1] = '\0';
+			yylval.literal = xstrdup(yytext+1);
+			DPRINT("Character literal: %s\n", yylval.literal);
+			return DT_CHAR_LITERAL;
+		}
+	YY_BREAK
+case 12:
 YY_RULE_SETUP
+#line 161 "dtc-lexer.l"
 {	/* label reference */
 			DPRINT("Ref: %s\n", yytext+1);
 			yylval.labelref = xstrdup(yytext+1);
 			return DT_REF;
 		}
 	YY_BREAK
-case 8:
+case 13:
 YY_RULE_SETUP
+#line 167 "dtc-lexer.l"
 {	/* new-style path reference */
 			yytext[yyleng-1] = '\0';
 			DPRINT("Ref: %s\n", yytext+2);
@@ -929,55 +1089,104 @@ YY_RULE_SETUP
 			return DT_REF;
 		}
 	YY_BREAK
-case 9:
+case 14:
 YY_RULE_SETUP
+#line 174 "dtc-lexer.l"
 {
 			yylval.byte = strtol(yytext, NULL, 16);
 			DPRINT("Byte: %02x\n", (int)yylval.byte);
 			return DT_BYTE;
 		}
 	YY_BREAK
-case 10:
+case 15:
 YY_RULE_SETUP
+#line 180 "dtc-lexer.l"
 {
 			DPRINT("/BYTESTRING\n");
 			BEGIN_DEFAULT();
 			return ']';
 		}
 	YY_BREAK
-case 11:
+case 16:
 YY_RULE_SETUP
+#line 186 "dtc-lexer.l"
 {
 			DPRINT("PropNodeName: %s\n", yytext);
-			yylval.propnodename = xstrdup(yytext);
+			yylval.propnodename = xstrdup((yytext[0] == '\\') ?
+							yytext + 1 : yytext);
 			BEGIN_DEFAULT();
 			return DT_PROPNODENAME;
 		}
 	YY_BREAK
-case 12:
+case 17:
 YY_RULE_SETUP
+#line 194 "dtc-lexer.l"
 {
 			DPRINT("Binary Include\n");
 			return DT_INCBIN;
 		}
 	YY_BREAK
-case 13:
-/* rule 13 can match eol */
+case 18:
+/* rule 18 can match eol */
 YY_RULE_SETUP
+#line 199 "dtc-lexer.l"
 /* eat whitespace */
 	YY_BREAK
-case 14:
-/* rule 14 can match eol */
+case 19:
+/* rule 19 can match eol */
 YY_RULE_SETUP
+#line 200 "dtc-lexer.l"
 /* eat C-style comments */
 	YY_BREAK
-case 15:
-/* rule 15 can match eol */
+case 20:
+/* rule 20 can match eol */
 YY_RULE_SETUP
+#line 201 "dtc-lexer.l"
 /* eat C++-style comments */
 	YY_BREAK
-case 16:
+case 21:
 YY_RULE_SETUP
+#line 203 "dtc-lexer.l"
+{ return DT_LSHIFT; };
+	YY_BREAK
+case 22:
+YY_RULE_SETUP
+#line 204 "dtc-lexer.l"
+{ return DT_RSHIFT; };
+	YY_BREAK
+case 23:
+YY_RULE_SETUP
+#line 205 "dtc-lexer.l"
+{ return DT_LE; };
+	YY_BREAK
+case 24:
+YY_RULE_SETUP
+#line 206 "dtc-lexer.l"
+{ return DT_GE; };
+	YY_BREAK
+case 25:
+YY_RULE_SETUP
+#line 207 "dtc-lexer.l"
+{ return DT_EQ; };
+	YY_BREAK
+case 26:
+YY_RULE_SETUP
+#line 208 "dtc-lexer.l"
+{ return DT_NE; };
+	YY_BREAK
+case 27:
+YY_RULE_SETUP
+#line 209 "dtc-lexer.l"
+{ return DT_AND; };
+	YY_BREAK
+case 28:
+YY_RULE_SETUP
+#line 210 "dtc-lexer.l"
+{ return DT_OR; };
+	YY_BREAK
+case 29:
+YY_RULE_SETUP
+#line 212 "dtc-lexer.l"
 {
 			DPRINT("Char: %c (\\x%02x)\n", yytext[0],
 				(unsigned)yytext[0]);
@@ -993,10 +1202,12 @@ YY_RULE_SETUP
 			return yytext[0];
 		}
 	YY_BREAK
-case 17:
+case 30:
 YY_RULE_SETUP
+#line 227 "dtc-lexer.l"
 ECHO;
 	YY_BREAK
+#line 1211 "dtc-lexer.lex.c"
 
 	case YY_END_OF_BUFFER:
 		{
@@ -1275,6 +1486,7 @@ static int yy_get_next_buffer (void)
 	register char *yy_cp;
     
 	yy_current_state = (yy_start);
+	yy_current_state += YY_AT_BOL();
 
 	for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp )
 		{
@@ -1287,7 +1499,7 @@ static int yy_get_next_buffer (void)
 		while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 			{
 			yy_current_state = (int) yy_def[yy_current_state];
-			if ( yy_current_state >= 94 )
+			if ( yy_current_state >= 161 )
 				yy_c = yy_meta[(unsigned int) yy_c];
 			}
 		yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -1315,11 +1527,11 @@ static int yy_get_next_buffer (void)
 	while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
 		{
 		yy_current_state = (int) yy_def[yy_current_state];
-		if ( yy_current_state >= 94 )
+		if ( yy_current_state >= 161 )
 			yy_c = yy_meta[(unsigned int) yy_c];
 		}
 	yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
-	yy_is_jam = (yy_current_state == 93);
+	yy_is_jam = (yy_current_state == 160);
 
 	return yy_is_jam ? 0 : yy_current_state;
 }
@@ -1394,6 +1606,8 @@ static int yy_get_next_buffer (void)
 	*(yy_c_buf_p) = '\0';	/* preserve yytext */
 	(yy_hold_char) = *++(yy_c_buf_p);
 
+	YY_CURRENT_BUFFER_LVALUE->yy_at_bol = (c == '\n');
+
 	return c;
 }
 #endif	/* ifndef YY_NO_INPUT */
@@ -1712,8 +1926,8 @@ YY_BUFFER_STATE yy_scan_string (yyconst char * yystr )
 
 /** Setup the input buffer state to scan the given bytes. The next call to yylex() will
  * scan from a @e copy of @a bytes.
- * @param bytes the byte buffer to scan
- * @param len the number of bytes in the buffer pointed to by @a bytes.
+ * @param yybytes the byte buffer to scan
+ * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes.
  * 
  * @return the newly allocated buffer state object.
  */
@@ -1952,6 +2166,10 @@ void yyfree (void * ptr )
 
 #define YYTABLES_NAME "yytables"
 
+#line 227 "dtc-lexer.l"
+
+
+
 static void push_input_file(const char *filename)
 {
 	assert(filename);
@@ -1963,6 +2181,7 @@ static void push_input_file(const char *filename)
 	yypush_buffer_state(yy_create_buffer(yyin,YY_BUF_SIZE));
 }
 
+
 static int pop_input_file(void)
 {
 	if (srcfile_pop() == 0)
diff --git a/scripts/dtc/dtc-parser.tab.c_shipped b/scripts/dtc/dtc-parser.tab.c_shipped
index b05921e1e84..4af55900a15 100644
--- a/scripts/dtc/dtc-parser.tab.c_shipped
+++ b/scripts/dtc/dtc-parser.tab.c_shipped
@@ -1,9 +1,10 @@
-/* A Bison parser, made by GNU Bison 2.4.3.  */
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
 
 /* Skeleton implementation for Bison's Yacc-like parsers in C
    
-      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2009, 2010 Free Software Foundation, Inc.
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
    
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -45,7 +46,7 @@
 #define YYBISON 1
 
 /* Bison version.  */
-#define YYBISON_VERSION "2.4.3"
+#define YYBISON_VERSION "2.4.1"
 
 /* Skeleton name.  */
 #define YYSKELETON_NAME "yacc.c"
@@ -66,6 +67,8 @@
 
 /* Copy the first part of user declarations.  */
 
+/* Line 189 of yacc.c  */
+#line 21 "dtc-parser.y"
 
 #include <stdio.h>
 
@@ -82,12 +85,15 @@ extern struct boot_info *the_boot_info;
 extern int treesource_error;
 
 static unsigned long long eval_literal(const char *s, int base, int bits);
+static unsigned char eval_char_literal(const char *s);
 
 
+/* Line 189 of yacc.c  */
+#line 93 "dtc-parser.tab.c"
 
 /* Enabling traces.  */
 #ifndef YYDEBUG
-# define YYDEBUG 1
+# define YYDEBUG 0
 #endif
 
 /* Enabling verbose error messages.  */
@@ -112,14 +118,26 @@ static unsigned long long eval_literal(const char *s, int base, int bits);
    enum yytokentype {
      DT_V1 = 258,
      DT_MEMRESERVE = 259,
-     DT_PROPNODENAME = 260,
-     DT_LITERAL = 261,
-     DT_BASE = 262,
-     DT_BYTE = 263,
-     DT_STRING = 264,
-     DT_LABEL = 265,
-     DT_REF = 266,
-     DT_INCBIN = 267
+     DT_LSHIFT = 260,
+     DT_RSHIFT = 261,
+     DT_LE = 262,
+     DT_GE = 263,
+     DT_EQ = 264,
+     DT_NE = 265,
+     DT_AND = 266,
+     DT_OR = 267,
+     DT_BITS = 268,
+     DT_DEL_PROP = 269,
+     DT_DEL_NODE = 270,
+     DT_PROPNODENAME = 271,
+     DT_LITERAL = 272,
+     DT_CHAR_LITERAL = 273,
+     DT_BASE = 274,
+     DT_BYTE = 275,
+     DT_STRING = 276,
+     DT_LABEL = 277,
+     DT_REF = 278,
+     DT_INCBIN = 279
    };
 #endif
 
@@ -129,6 +147,8 @@ static unsigned long long eval_literal(const char *s, int base, int bits);
 typedef union YYSTYPE
 {
 
+/* Line 214 of yacc.c  */
+#line 40 "dtc-parser.y"
 
 	char *propnodename;
 	char *literal;
@@ -137,16 +157,22 @@ typedef union YYSTYPE
 	uint8_t byte;
 	struct data data;
 
-	uint64_t addr;
-	cell_t cell;
+	struct {
+		struct data	data;
+		int		bits;
+	} array;
+
 	struct property *prop;
 	struct property *proplist;
 	struct node *node;
 	struct node *nodelist;
 	struct reserve_info *re;
+	uint64_t integer;
 
 
 
+/* Line 214 of yacc.c  */
+#line 176 "dtc-parser.tab.c"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
@@ -157,6 +183,8 @@ typedef union YYSTYPE
 /* Copy the second part of user declarations.  */
 
 
+/* Line 264 of yacc.c  */
+#line 188 "dtc-parser.tab.c"
 
 #ifdef short
 # undef short
@@ -206,7 +234,7 @@ typedef short int yytype_int16;
 #define YYSIZE_MAXIMUM ((YYSIZE_T) -1)
 
 #ifndef YY_
-# if defined YYENABLE_NLS && YYENABLE_NLS
+# if YYENABLE_NLS
 #  if ENABLE_NLS
 #   include <libintl.h> /* INFRINGES ON USER NAME SPACE */
 #   define YY_(msgid) dgettext ("bison-runtime", msgid)
@@ -371,20 +399,20 @@ union yyalloc
 /* YYFINAL -- State number of the termination state.  */
 #define YYFINAL  4
 /* YYLAST -- Last index in YYTABLE.  */
-#define YYLAST   56
+#define YYLAST   133
 
 /* YYNTOKENS -- Number of terminals.  */
-#define YYNTOKENS  25
+#define YYNTOKENS  48
 /* YYNNTS -- Number of nonterminals.  */
-#define YYNNTS  16
+#define YYNNTS  28
 /* YYNRULES -- Number of rules.  */
-#define YYNRULES  39
+#define YYNRULES  79
 /* YYNRULES -- Number of states.  */
-#define YYNSTATES  67
+#define YYNSTATES  141
 
 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
 #define YYUNDEFTOK  2
-#define YYMAXUTOK   267
+#define YYMAXUTOK   279
 
 #define YYTRANSLATE(YYX)						\
   ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
@@ -395,16 +423,16 @@ static const yytype_uint8 yytranslate[] =
        0,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-      22,    24,     2,     2,    23,     2,     2,    14,     2,     2,
-       2,     2,     2,     2,     2,     2,     2,     2,     2,    13,
-      18,    17,    19,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,    47,     2,     2,     2,    45,    41,     2,
+      33,    35,    44,    42,    34,    43,     2,    26,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,    38,    25,
+      36,    29,    30,    37,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,    20,     2,    21,     2,     2,     2,     2,     2,     2,
+       2,    31,     2,    32,    40,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-       2,     2,     2,    15,     2,    16,     2,     2,     2,     2,
+       2,     2,     2,    27,    39,    28,    46,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
@@ -418,45 +446,68 @@ static const yytype_uint8 yytranslate[] =
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
        2,     2,     2,     2,     2,     2,     1,     2,     3,     4,
-       5,     6,     7,     8,     9,    10,    11,    12
+       5,     6,     7,     8,     9,    10,    11,    12,    13,    14,
+      15,    16,    17,    18,    19,    20,    21,    22,    23,    24
 };
 
 #if YYDEBUG
 /* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in
    YYRHS.  */
-static const yytype_uint8 yyprhs[] =
+static const yytype_uint16 yyprhs[] =
 {
-       0,     0,     3,     8,     9,    12,    17,    20,    22,    25,
-      29,    33,    39,    40,    43,    48,    51,    54,    57,    62,
-      67,    70,    80,    86,    89,    90,    93,    96,    97,   100,
-     103,   106,   108,   109,   112,   115,   116,   119,   122,   125
+       0,     0,     3,     8,     9,    12,    17,    20,    23,    27,
+      31,    36,    42,    43,    46,    51,    54,    58,    61,    64,
+      68,    73,    76,    86,    92,    95,    96,    99,   102,   106,
+     108,   111,   114,   117,   119,   121,   125,   127,   129,   135,
+     137,   141,   143,   147,   149,   153,   155,   159,   161,   165,
+     167,   171,   175,   177,   181,   185,   189,   193,   197,   201,
+     203,   207,   211,   213,   217,   221,   225,   227,   229,   232,
+     235,   238,   239,   242,   245,   246,   249,   252,   255,   259
 };
 
 /* YYRHS -- A `-1'-separated list of the rules' RHS.  */
 static const yytype_int8 yyrhs[] =
 {
-      26,     0,    -1,     3,    13,    27,    30,    -1,    -1,    28,
-      27,    -1,     4,    29,    29,    13,    -1,    10,    28,    -1,
-       6,    -1,    14,    31,    -1,    30,    14,    31,    -1,    30,
-      11,    31,    -1,    15,    32,    39,    16,    13,    -1,    -1,
-      32,    33,    -1,     5,    17,    34,    13,    -1,     5,    13,
-      -1,    10,    33,    -1,    35,     9,    -1,    35,    18,    36,
-      19,    -1,    35,    20,    38,    21,    -1,    35,    11,    -1,
-      35,    12,    22,     9,    23,    29,    23,    29,    24,    -1,
-      35,    12,    22,     9,    24,    -1,    34,    10,    -1,    -1,
-      34,    23,    -1,    35,    10,    -1,    -1,    36,    37,    -1,
-      36,    11,    -1,    36,    10,    -1,     6,    -1,    -1,    38,
-       8,    -1,    38,    10,    -1,    -1,    40,    39,    -1,    40,
-      33,    -1,     5,    31,    -1,    10,    40,    -1
+      49,     0,    -1,     3,    25,    50,    52,    -1,    -1,    51,
+      50,    -1,     4,    59,    59,    25,    -1,    22,    51,    -1,
+      26,    53,    -1,    52,    26,    53,    -1,    52,    23,    53,
+      -1,    52,    15,    23,    25,    -1,    27,    54,    74,    28,
+      25,    -1,    -1,    54,    55,    -1,    16,    29,    56,    25,
+      -1,    16,    25,    -1,    14,    16,    25,    -1,    22,    55,
+      -1,    57,    21,    -1,    57,    58,    30,    -1,    57,    31,
+      73,    32,    -1,    57,    23,    -1,    57,    24,    33,    21,
+      34,    59,    34,    59,    35,    -1,    57,    24,    33,    21,
+      35,    -1,    56,    22,    -1,    -1,    56,    34,    -1,    57,
+      22,    -1,    13,    17,    36,    -1,    36,    -1,    58,    59,
+      -1,    58,    23,    -1,    58,    22,    -1,    17,    -1,    18,
+      -1,    33,    60,    35,    -1,    61,    -1,    62,    -1,    62,
+      37,    60,    38,    61,    -1,    63,    -1,    62,    12,    63,
+      -1,    64,    -1,    63,    11,    64,    -1,    65,    -1,    64,
+      39,    65,    -1,    66,    -1,    65,    40,    66,    -1,    67,
+      -1,    66,    41,    67,    -1,    68,    -1,    67,     9,    68,
+      -1,    67,    10,    68,    -1,    69,    -1,    68,    36,    69,
+      -1,    68,    30,    69,    -1,    68,     7,    69,    -1,    68,
+       8,    69,    -1,    69,     5,    70,    -1,    69,     6,    70,
+      -1,    70,    -1,    70,    42,    71,    -1,    70,    43,    71,
+      -1,    71,    -1,    71,    44,    72,    -1,    71,    26,    72,
+      -1,    71,    45,    72,    -1,    72,    -1,    59,    -1,    43,
+      72,    -1,    46,    72,    -1,    47,    72,    -1,    -1,    73,
+      20,    -1,    73,    22,    -1,    -1,    75,    74,    -1,    75,
+      55,    -1,    16,    53,    -1,    15,    16,    25,    -1,    22,
+      75,    -1
 };
 
 /* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
 static const yytype_uint16 yyrline[] =
 {
-       0,    86,    86,    95,    98,   105,   109,   117,   124,   128,
-     132,   145,   153,   156,   163,   167,   171,   179,   183,   187,
-     191,   195,   212,   222,   230,   233,   237,   245,   248,   252,
-     257,   264,   272,   275,   279,   287,   290,   294,   302,   306
+       0,   109,   109,   118,   121,   128,   132,   140,   144,   148,
+     158,   172,   180,   183,   190,   194,   198,   202,   210,   214,
+     218,   222,   226,   243,   253,   261,   264,   268,   275,   290,
+     295,   315,   329,   336,   340,   344,   351,   355,   356,   360,
+     361,   365,   366,   370,   371,   375,   376,   380,   381,   385,
+     386,   387,   391,   392,   393,   394,   395,   399,   400,   401,
+     405,   406,   407,   411,   412,   413,   414,   418,   419,   420,
+     421,   426,   429,   433,   441,   444,   448,   456,   460,   464
 };
 #endif
 
@@ -465,13 +516,19 @@ static const yytype_uint16 yyrline[] =
    First, the terminals, then, starting at YYNTOKENS, nonterminals.  */
 static const char *const yytname[] =
 {
-  "$end", "error", "$undefined", "DT_V1", "DT_MEMRESERVE",
-  "DT_PROPNODENAME", "DT_LITERAL", "DT_BASE", "DT_BYTE", "DT_STRING",
-  "DT_LABEL", "DT_REF", "DT_INCBIN", "';'", "'/'", "'{'", "'}'", "'='",
-  "'<'", "'>'", "'['", "']'", "'('", "','", "')'", "$accept", "sourcefile",
-  "memreserves", "memreserve", "addr", "devicetree", "nodedef", "proplist",
-  "propdef", "propdata", "propdataprefix", "celllist", "cellval",
-  "bytestring", "subnodes", "subnode", 0
+  "$end", "error", "$undefined", "DT_V1", "DT_MEMRESERVE", "DT_LSHIFT",
+  "DT_RSHIFT", "DT_LE", "DT_GE", "DT_EQ", "DT_NE", "DT_AND", "DT_OR",
+  "DT_BITS", "DT_DEL_PROP", "DT_DEL_NODE", "DT_PROPNODENAME", "DT_LITERAL",
+  "DT_CHAR_LITERAL", "DT_BASE", "DT_BYTE", "DT_STRING", "DT_LABEL",
+  "DT_REF", "DT_INCBIN", "';'", "'/'", "'{'", "'}'", "'='", "'>'", "'['",
+  "']'", "'('", "','", "')'", "'<'", "'?'", "':'", "'|'", "'^'", "'&'",
+  "'+'", "'-'", "'*'", "'%'", "'~'", "'!'", "$accept", "sourcefile",
+  "memreserves", "memreserve", "devicetree", "nodedef", "proplist",
+  "propdef", "propdata", "propdataprefix", "arrayprefix", "integer_prim",
+  "integer_expr", "integer_trinary", "integer_or", "integer_and",
+  "integer_bitor", "integer_bitxor", "integer_bitand", "integer_eq",
+  "integer_rela", "integer_shift", "integer_add", "integer_mul",
+  "integer_unary", "bytestring", "subnodes", "subnode", 0
 };
 #endif
 
@@ -481,27 +538,37 @@ static const char *const yytname[] =
 static const yytype_uint16 yytoknum[] =
 {
        0,   256,   257,   258,   259,   260,   261,   262,   263,   264,
-     265,   266,   267,    59,    47,   123,   125,    61,    60,    62,
-      91,    93,    40,    44,    41
+     265,   266,   267,   268,   269,   270,   271,   272,   273,   274,
+     275,   276,   277,   278,   279,    59,    47,   123,   125,    61,
+      62,    91,    93,    40,    44,    41,    60,    63,    58,   124,
+      94,    38,    43,    45,    42,    37,   126,    33
 };
 # endif
 
 /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives.  */
 static const yytype_uint8 yyr1[] =
 {
-       0,    25,    26,    27,    27,    28,    28,    29,    30,    30,
-      30,    31,    32,    32,    33,    33,    33,    34,    34,    34,
-      34,    34,    34,    34,    35,    35,    35,    36,    36,    36,
-      36,    37,    38,    38,    38,    39,    39,    39,    40,    40
+       0,    48,    49,    50,    50,    51,    51,    52,    52,    52,
+      52,    53,    54,    54,    55,    55,    55,    55,    56,    56,
+      56,    56,    56,    56,    56,    57,    57,    57,    58,    58,
+      58,    58,    58,    59,    59,    59,    60,    61,    61,    62,
+      62,    63,    63,    64,    64,    65,    65,    66,    66,    67,
+      67,    67,    68,    68,    68,    68,    68,    69,    69,    69,
+      70,    70,    70,    71,    71,    71,    71,    72,    72,    72,
+      72,    73,    73,    73,    74,    74,    74,    75,    75,    75
 };
 
 /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN.  */
 static const yytype_uint8 yyr2[] =
 {
-       0,     2,     4,     0,     2,     4,     2,     1,     2,     3,
-       3,     5,     0,     2,     4,     2,     2,     2,     4,     4,
-       2,     9,     5,     2,     0,     2,     2,     0,     2,     2,
-       2,     1,     0,     2,     2,     0,     2,     2,     2,     2
+       0,     2,     4,     0,     2,     4,     2,     2,     3,     3,
+       4,     5,     0,     2,     4,     2,     3,     2,     2,     3,
+       4,     2,     9,     5,     2,     0,     2,     2,     3,     1,
+       2,     2,     2,     1,     1,     3,     1,     1,     5,     1,
+       3,     1,     3,     1,     3,     1,     3,     1,     3,     1,
+       3,     3,     1,     3,     3,     3,     3,     3,     3,     1,
+       3,     3,     1,     3,     3,     3,     1,     1,     2,     2,
+       2,     0,     2,     2,     0,     2,     2,     2,     3,     2
 };
 
 /* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state
@@ -509,41 +576,59 @@ static const yytype_uint8 yyr2[] =
    means the default is an error.  */
 static const yytype_uint8 yydefact[] =
 {
-       0,     0,     0,     3,     1,     0,     0,     0,     3,     7,
-       0,     6,     0,     2,     4,     0,    12,     8,     0,     0,
-       5,    35,    10,     9,     0,     0,    13,     0,    35,    15,
-      24,    38,    16,    39,     0,    37,    36,     0,     0,    11,
-      23,    14,    25,    17,    26,    20,     0,    27,    32,     0,
-       0,     0,     0,    31,    30,    29,    18,    28,    33,    34,
-      19,     0,    22,     0,     0,     0,    21
+       0,     0,     0,     3,     1,     0,     0,     0,     3,    33,
+      34,     0,     0,     6,     0,     2,     4,     0,     0,     0,
+      67,     0,    36,    37,    39,    41,    43,    45,    47,    49,
+      52,    59,    62,    66,     0,    12,     7,     0,     0,     0,
+      68,    69,    70,    35,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     0,     0,     0,     0,     0,     0,     0,
+       0,     0,     0,     5,    74,     0,     9,     8,    40,     0,
+      42,    44,    46,    48,    50,    51,    55,    56,    54,    53,
+      57,    58,    60,    61,    64,    63,    65,     0,     0,     0,
+       0,    13,     0,    74,    10,     0,     0,     0,    15,    25,
+      77,    17,    79,     0,    76,    75,    38,    16,    78,     0,
+       0,    11,    24,    14,    26,     0,    18,    27,    21,     0,
+      71,    29,     0,     0,     0,     0,    32,    31,    19,    30,
+      28,     0,    72,    73,    20,     0,    23,     0,     0,     0,
+      22
 };
 
 /* YYDEFGOTO[NTERM-NUM].  */
 static const yytype_int8 yydefgoto[] =
 {
-      -1,     2,     7,     8,    10,    13,    17,    21,    26,    37,
-      38,    50,    57,    51,    27,    28
+      -1,     2,     7,     8,    15,    36,    64,    91,   109,   110,
+     122,    20,    21,    22,    23,    24,    25,    26,    27,    28,
+      29,    30,    31,    32,    33,   125,    92,    93
 };
 
 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
    STATE-NUM.  */
-#define YYPACT_NINF -12
+#define YYPACT_NINF -78
 static const yytype_int8 yypact[] =
 {
-      10,   -11,    18,    -1,   -12,    22,    -1,    15,    -1,   -12,
-      22,   -12,    20,     1,   -12,    17,   -12,   -12,    20,    20,
-     -12,     6,   -12,   -12,    21,     6,   -12,    23,     6,   -12,
-     -12,   -12,   -12,   -12,    28,   -12,   -12,    -6,    13,   -12,
-     -12,   -12,   -12,   -12,   -12,   -12,    24,   -12,   -12,    33,
-      -5,     0,    -4,   -12,   -12,   -12,   -12,   -12,   -12,   -12,
-     -12,    22,   -12,    25,    22,    19,   -12
+      22,    11,    51,    10,   -78,    23,    10,     2,    10,   -78,
+     -78,    -9,    23,   -78,    30,    38,   -78,    -9,    -9,    -9,
+     -78,    35,   -78,    -6,    52,    29,    48,    49,    33,     3,
+      71,    36,     0,   -78,    64,   -78,   -78,    68,    30,    30,
+     -78,   -78,   -78,   -78,    -9,    -9,    -9,    -9,    -9,    -9,
+      -9,    -9,    -9,    -9,    -9,    -9,    -9,    -9,    -9,    -9,
+      -9,    -9,    -9,   -78,    44,    67,   -78,   -78,    52,    55,
+      29,    48,    49,    33,     3,     3,    71,    71,    71,    71,
+      36,    36,     0,     0,   -78,   -78,   -78,    78,    79,    42,
+      44,   -78,    69,    44,   -78,    -9,    73,    74,   -78,   -78,
+     -78,   -78,   -78,    75,   -78,   -78,   -78,   -78,   -78,    -7,
+      -1,   -78,   -78,   -78,   -78,    84,   -78,   -78,   -78,    63,
+     -78,   -78,    32,    66,    82,    -3,   -78,   -78,   -78,   -78,
+     -78,    46,   -78,   -78,   -78,    23,   -78,    70,    23,    72,
+     -78
 };
 
 /* YYPGOTO[NTERM-NUM].  */
 static const yytype_int8 yypgoto[] =
 {
-     -12,   -12,    36,    39,   -10,   -12,     8,   -12,    12,   -12,
-     -12,   -12,   -12,   -12,    27,    31
+     -78,   -78,    97,   100,   -78,   -37,   -78,   -77,   -78,   -78,
+     -78,    -5,    65,    13,   -78,    76,    77,    62,    80,    83,
+      34,    20,    26,    28,   -14,   -78,    18,    24
 };
 
 /* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
@@ -553,35 +638,59 @@ static const yytype_int8 yypgoto[] =
 #define YYTABLE_NINF -1
 static const yytype_uint8 yytable[] =
 {
-      15,    53,     3,     5,    40,    54,    55,    41,    58,     6,
-      59,    24,    18,     1,    56,    19,    25,    42,     4,    61,
-      62,    60,    43,    44,    45,    46,    22,    23,     9,    12,
-      20,    47,    31,    48,    29,    16,    16,    32,    30,    34,
-      35,    39,    52,    66,    14,    11,    49,     0,    64,     0,
-       0,    63,     0,     0,    65,    36,    33
+      12,    66,    67,    40,    41,    42,    44,    34,     9,    10,
+      52,    53,   115,   101,     5,   112,   104,   132,   113,   133,
+     116,   117,   118,   119,    11,     1,    60,   114,    14,   134,
+     120,    45,     6,    54,    17,   121,     3,    18,    19,    55,
+       9,    10,    50,    51,    61,    62,    84,    85,    86,     9,
+      10,     4,   100,    37,   126,   127,    11,    35,    87,    88,
+      89,    38,   128,    46,    39,    11,    90,    98,    47,    35,
+      43,    99,    76,    77,    78,    79,    56,    57,    58,    59,
+     135,   136,    80,    81,    74,    75,    82,    83,    48,    63,
+      49,    65,    94,    95,    96,    97,   124,   103,   107,   108,
+     111,   123,   130,   131,   138,    16,    13,   140,   106,    71,
+      69,   105,     0,     0,   102,     0,     0,   129,     0,     0,
+      68,     0,     0,    70,     0,     0,     0,     0,    72,     0,
+     137,     0,    73,   139
 };
 
-static const yytype_int8 yycheck[] =
+static const yytype_int16 yycheck[] =
 {
-      10,     6,    13,     4,    10,    10,    11,    13,     8,    10,
-      10,     5,    11,     3,    19,    14,    10,    23,     0,    23,
-      24,    21,     9,    10,    11,    12,    18,    19,     6,    14,
-      13,    18,    24,    20,    13,    15,    15,    25,    17,    16,
-      28,    13,     9,    24,     8,     6,    22,    -1,    23,    -1,
-      -1,    61,    -1,    -1,    64,    28,    25
+       5,    38,    39,    17,    18,    19,    12,    12,    17,    18,
+       7,     8,    13,    90,     4,    22,    93,    20,    25,    22,
+      21,    22,    23,    24,    33,     3,    26,    34,    26,    32,
+      31,    37,    22,    30,    43,    36,    25,    46,    47,    36,
+      17,    18,     9,    10,    44,    45,    60,    61,    62,    17,
+      18,     0,    89,    15,    22,    23,    33,    27,    14,    15,
+      16,    23,    30,    11,    26,    33,    22,    25,    39,    27,
+      35,    29,    52,    53,    54,    55,     5,     6,    42,    43,
+      34,    35,    56,    57,    50,    51,    58,    59,    40,    25,
+      41,    23,    25,    38,    16,    16,    33,    28,    25,    25,
+      25,    17,    36,    21,    34,     8,     6,    35,    95,    47,
+      45,    93,    -1,    -1,    90,    -1,    -1,   122,    -1,    -1,
+      44,    -1,    -1,    46,    -1,    -1,    -1,    -1,    48,    -1,
+     135,    -1,    49,   138
 };
 
 /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
    symbol of state STATE-NUM.  */
 static const yytype_uint8 yystos[] =
 {
-       0,     3,    26,    13,     0,     4,    10,    27,    28,     6,
-      29,    28,    14,    30,    27,    29,    15,    31,    11,    14,
-      13,    32,    31,    31,     5,    10,    33,    39,    40,    13,
-      17,    31,    33,    40,    16,    33,    39,    34,    35,    13,
-      10,    13,    23,     9,    10,    11,    12,    18,    20,    22,
-      36,    38,     9,     6,    10,    11,    19,    37,     8,    10,
-      21,    23,    24,    29,    23,    29,    24
+       0,     3,    49,    25,     0,     4,    22,    50,    51,    17,
+      18,    33,    59,    51,    26,    52,    50,    43,    46,    47,
+      59,    60,    61,    62,    63,    64,    65,    66,    67,    68,
+      69,    70,    71,    72,    59,    27,    53,    15,    23,    26,
+      72,    72,    72,    35,    12,    37,    11,    39,    40,    41,
+       9,    10,     7,     8,    30,    36,     5,     6,    42,    43,
+      26,    44,    45,    25,    54,    23,    53,    53,    63,    60,
+      64,    65,    66,    67,    68,    68,    69,    69,    69,    69,
+      70,    70,    71,    71,    72,    72,    72,    14,    15,    16,
+      22,    55,    74,    75,    25,    38,    16,    16,    25,    29,
+      53,    55,    75,    28,    55,    74,    61,    25,    25,    56,
+      57,    25,    22,    25,    34,    13,    21,    22,    23,    24,
+      31,    36,    58,    17,    33,    73,    22,    23,    30,    59,
+      36,    21,    20,    22,    32,    34,    35,    59,    34,    59,
+      35
 };
 
 #define yyerrok		(yyerrstatus = 0)
@@ -596,18 +705,9 @@ static const yytype_uint8 yystos[] =
 
 /* Like YYERROR except do call yyerror.  This remains here temporarily
    to ease the transition to the new meaning of YYERROR, for GCC.
-   Once GCC version 2 has supplanted version 1, this can go.  However,
-   YYFAIL appears to be in use.  Nevertheless, it is formally deprecated
-   in Bison 2.4.2's NEWS entry, where a plan to phase it out is
-   discussed.  */
+   Once GCC version 2 has supplanted version 1, this can go.  */
 
 #define YYFAIL		goto yyerrlab
-#if defined YYFAIL
-  /* This is here to suppress warnings from the GCC cpp's
-     -Wunused-macros.  Normally we don't worry about that warning, but
-     some users do, and we want to make it easy for users to remove
-     YYFAIL uses, which will produce warnings from Bison 2.5.  */
-#endif
 
 #define YYRECOVERING()  (!!yyerrstatus)
 
@@ -664,7 +764,7 @@ while (YYID (0))
    we won't break user code: when these are the locations we know.  */
 
 #ifndef YY_LOCATION_PRINT
-# if defined YYLTYPE_IS_TRIVIAL && YYLTYPE_IS_TRIVIAL
+# if YYLTYPE_IS_TRIVIAL
 #  define YY_LOCATION_PRINT(File, Loc)			\
      fprintf (File, "%d.%d-%d.%d",			\
 	      (Loc).first_line, (Loc).first_column,	\
@@ -1403,6 +1503,8 @@ yyreduce:
     {
         case 2:
 
+/* Line 1455 of yacc.c  */
+#line 110 "dtc-parser.y"
     {
 			the_boot_info = build_boot_info((yyvsp[(3) - (4)].re), (yyvsp[(4) - (4)].node),
 							guess_boot_cpuid((yyvsp[(4) - (4)].node)));
@@ -1411,6 +1513,8 @@ yyreduce:
 
   case 3:
 
+/* Line 1455 of yacc.c  */
+#line 118 "dtc-parser.y"
     {
 			(yyval.re) = NULL;
 		;}
@@ -1418,6 +1522,8 @@ yyreduce:
 
   case 4:
 
+/* Line 1455 of yacc.c  */
+#line 122 "dtc-parser.y"
     {
 			(yyval.re) = chain_reserve_entry((yyvsp[(1) - (2)].re), (yyvsp[(2) - (2)].re));
 		;}
@@ -1425,13 +1531,17 @@ yyreduce:
 
   case 5:
 
+/* Line 1455 of yacc.c  */
+#line 129 "dtc-parser.y"
     {
-			(yyval.re) = build_reserve_entry((yyvsp[(2) - (4)].addr), (yyvsp[(3) - (4)].addr));
+			(yyval.re) = build_reserve_entry((yyvsp[(2) - (4)].integer), (yyvsp[(3) - (4)].integer));
 		;}
     break;
 
   case 6:
 
+/* Line 1455 of yacc.c  */
+#line 133 "dtc-parser.y"
     {
 			add_label(&(yyvsp[(2) - (2)].re)->labels, (yyvsp[(1) - (2)].labelref));
 			(yyval.re) = (yyvsp[(2) - (2)].re);
@@ -1440,40 +1550,57 @@ yyreduce:
 
   case 7:
 
+/* Line 1455 of yacc.c  */
+#line 141 "dtc-parser.y"
     {
-			(yyval.addr) = eval_literal((yyvsp[(1) - (1)].literal), 0, 64);
+			(yyval.node) = name_node((yyvsp[(2) - (2)].node), "");
 		;}
     break;
 
   case 8:
 
+/* Line 1455 of yacc.c  */
+#line 145 "dtc-parser.y"
     {
-			(yyval.node) = name_node((yyvsp[(2) - (2)].node), "");
+			(yyval.node) = merge_nodes((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node));
 		;}
     break;
 
   case 9:
 
+/* Line 1455 of yacc.c  */
+#line 149 "dtc-parser.y"
     {
-			(yyval.node) = merge_nodes((yyvsp[(1) - (3)].node), (yyvsp[(3) - (3)].node));
+			struct node *target = get_node_by_ref((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].labelref));
+
+			if (target)
+				merge_nodes(target, (yyvsp[(3) - (3)].node));
+			else
+				print_error("label or path, '%s', not found", (yyvsp[(2) - (3)].labelref));
+			(yyval.node) = (yyvsp[(1) - (3)].node);
 		;}
     break;
 
   case 10:
 
+/* Line 1455 of yacc.c  */
+#line 159 "dtc-parser.y"
     {
-			struct node *target = get_node_by_ref((yyvsp[(1) - (3)].node), (yyvsp[(2) - (3)].labelref));
+			struct node *target = get_node_by_ref((yyvsp[(1) - (4)].node), (yyvsp[(3) - (4)].labelref));
 
-			if (target)
-				merge_nodes(target, (yyvsp[(3) - (3)].node));
+			if (!target)
+				print_error("label or path, '%s', not found", (yyvsp[(3) - (4)].labelref));
 			else
-				print_error("label or path, '%s', not found", (yyvsp[(2) - (3)].labelref));
-			(yyval.node) = (yyvsp[(1) - (3)].node);
+				delete_node(target);
+
+			(yyval.node) = (yyvsp[(1) - (4)].node);
 		;}
     break;
 
   case 11:
 
+/* Line 1455 of yacc.c  */
+#line 173 "dtc-parser.y"
     {
 			(yyval.node) = build_node((yyvsp[(2) - (5)].proplist), (yyvsp[(3) - (5)].nodelist));
 		;}
@@ -1481,6 +1608,8 @@ yyreduce:
 
   case 12:
 
+/* Line 1455 of yacc.c  */
+#line 180 "dtc-parser.y"
     {
 			(yyval.proplist) = NULL;
 		;}
@@ -1488,6 +1617,8 @@ yyreduce:
 
   case 13:
 
+/* Line 1455 of yacc.c  */
+#line 184 "dtc-parser.y"
     {
 			(yyval.proplist) = chain_property((yyvsp[(2) - (2)].prop), (yyvsp[(1) - (2)].proplist));
 		;}
@@ -1495,6 +1626,8 @@ yyreduce:
 
   case 14:
 
+/* Line 1455 of yacc.c  */
+#line 191 "dtc-parser.y"
     {
 			(yyval.prop) = build_property((yyvsp[(1) - (4)].propnodename), (yyvsp[(3) - (4)].data));
 		;}
@@ -1502,6 +1635,8 @@ yyreduce:
 
   case 15:
 
+/* Line 1455 of yacc.c  */
+#line 195 "dtc-parser.y"
     {
 			(yyval.prop) = build_property((yyvsp[(1) - (2)].propnodename), empty_data);
 		;}
@@ -1509,62 +1644,85 @@ yyreduce:
 
   case 16:
 
+/* Line 1455 of yacc.c  */
+#line 199 "dtc-parser.y"
     {
-			add_label(&(yyvsp[(2) - (2)].prop)->labels, (yyvsp[(1) - (2)].labelref));
-			(yyval.prop) = (yyvsp[(2) - (2)].prop);
+			(yyval.prop) = build_property_delete((yyvsp[(2) - (3)].propnodename));
 		;}
     break;
 
   case 17:
 
+/* Line 1455 of yacc.c  */
+#line 203 "dtc-parser.y"
     {
-			(yyval.data) = data_merge((yyvsp[(1) - (2)].data), (yyvsp[(2) - (2)].data));
+			add_label(&(yyvsp[(2) - (2)].prop)->labels, (yyvsp[(1) - (2)].labelref));
+			(yyval.prop) = (yyvsp[(2) - (2)].prop);
 		;}
     break;
 
   case 18:
 
+/* Line 1455 of yacc.c  */
+#line 211 "dtc-parser.y"
     {
-			(yyval.data) = data_merge((yyvsp[(1) - (4)].data), (yyvsp[(3) - (4)].data));
+			(yyval.data) = data_merge((yyvsp[(1) - (2)].data), (yyvsp[(2) - (2)].data));
 		;}
     break;
 
   case 19:
 
+/* Line 1455 of yacc.c  */
+#line 215 "dtc-parser.y"
     {
-			(yyval.data) = data_merge((yyvsp[(1) - (4)].data), (yyvsp[(3) - (4)].data));
+			(yyval.data) = data_merge((yyvsp[(1) - (3)].data), (yyvsp[(2) - (3)].array).data);
 		;}
     break;
 
   case 20:
 
+/* Line 1455 of yacc.c  */
+#line 219 "dtc-parser.y"
     {
-			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), REF_PATH, (yyvsp[(2) - (2)].labelref));
+			(yyval.data) = data_merge((yyvsp[(1) - (4)].data), (yyvsp[(3) - (4)].data));
 		;}
     break;
 
   case 21:
 
+/* Line 1455 of yacc.c  */
+#line 223 "dtc-parser.y"
+    {
+			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), REF_PATH, (yyvsp[(2) - (2)].labelref));
+		;}
+    break;
+
+  case 22:
+
+/* Line 1455 of yacc.c  */
+#line 227 "dtc-parser.y"
     {
 			FILE *f = srcfile_relative_open((yyvsp[(4) - (9)].data).val, NULL);
 			struct data d;
 
-			if ((yyvsp[(6) - (9)].addr) != 0)
-				if (fseek(f, (yyvsp[(6) - (9)].addr), SEEK_SET) != 0)
+			if ((yyvsp[(6) - (9)].integer) != 0)
+				if (fseek(f, (yyvsp[(6) - (9)].integer), SEEK_SET) != 0)
 					print_error("Couldn't seek to offset %llu in \"%s\": %s",
-						     (unsigned long long)(yyvsp[(6) - (9)].addr),
+						     (unsigned long long)(yyvsp[(6) - (9)].integer),
 						     (yyvsp[(4) - (9)].data).val,
 						     strerror(errno));
 
-			d = data_copy_file(f, (yyvsp[(8) - (9)].addr));
+			d = data_copy_file(f, (yyvsp[(8) - (9)].integer));
 
 			(yyval.data) = data_merge((yyvsp[(1) - (9)].data), d);
 			fclose(f);
 		;}
     break;
 
-  case 22:
+  case 23:
 
+/* Line 1455 of yacc.c  */
+#line 244 "dtc-parser.y"
     {
 			FILE *f = srcfile_relative_open((yyvsp[(4) - (5)].data).val, NULL);
 			struct data d = empty_data;
@@ -1576,122 +1734,383 @@ yyreduce:
 		;}
     break;
 
-  case 23:
-
-    {
-			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
-		;}
-    break;
-
   case 24:
 
+/* Line 1455 of yacc.c  */
+#line 254 "dtc-parser.y"
     {
-			(yyval.data) = empty_data;
+			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
 		;}
     break;
 
   case 25:
 
+/* Line 1455 of yacc.c  */
+#line 261 "dtc-parser.y"
     {
-			(yyval.data) = (yyvsp[(1) - (2)].data);
+			(yyval.data) = empty_data;
 		;}
     break;
 
   case 26:
 
+/* Line 1455 of yacc.c  */
+#line 265 "dtc-parser.y"
     {
-			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
+			(yyval.data) = (yyvsp[(1) - (2)].data);
 		;}
     break;
 
   case 27:
 
+/* Line 1455 of yacc.c  */
+#line 269 "dtc-parser.y"
     {
-			(yyval.data) = empty_data;
+			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
 		;}
     break;
 
   case 28:
 
+/* Line 1455 of yacc.c  */
+#line 276 "dtc-parser.y"
     {
-			(yyval.data) = data_append_cell((yyvsp[(1) - (2)].data), (yyvsp[(2) - (2)].cell));
+			(yyval.array).data = empty_data;
+			(yyval.array).bits = eval_literal((yyvsp[(2) - (3)].literal), 0, 7);
+
+			if (((yyval.array).bits !=  8) &&
+			    ((yyval.array).bits != 16) &&
+			    ((yyval.array).bits != 32) &&
+			    ((yyval.array).bits != 64))
+			{
+				print_error("Only 8, 16, 32 and 64-bit elements"
+					    " are currently supported");
+				(yyval.array).bits = 32;
+			}
 		;}
     break;
 
   case 29:
 
+/* Line 1455 of yacc.c  */
+#line 291 "dtc-parser.y"
     {
-			(yyval.data) = data_append_cell(data_add_marker((yyvsp[(1) - (2)].data), REF_PHANDLE,
-							      (yyvsp[(2) - (2)].labelref)), -1);
+			(yyval.array).data = empty_data;
+			(yyval.array).bits = 32;
 		;}
     break;
 
   case 30:
 
+/* Line 1455 of yacc.c  */
+#line 296 "dtc-parser.y"
     {
-			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
+			if ((yyvsp[(1) - (2)].array).bits < 64) {
+				uint64_t mask = (1ULL << (yyvsp[(1) - (2)].array).bits) - 1;
+				/*
+				 * Bits above mask must either be all zero
+				 * (positive within range of mask) or all one
+				 * (negative and sign-extended). The second
+				 * condition is true if when we set all bits
+				 * within the mask to one (i.e. | in the
+				 * mask), all bits are one.
+				 */
+				if (((yyvsp[(2) - (2)].integer) > mask) && (((yyvsp[(2) - (2)].integer) | mask) != -1ULL))
+					print_error(
+						"integer value out of range "
+						"%016lx (%d bits)", (yyvsp[(1) - (2)].array).bits);
+			}
+
+			(yyval.array).data = data_append_integer((yyvsp[(1) - (2)].array).data, (yyvsp[(2) - (2)].integer), (yyvsp[(1) - (2)].array).bits);
 		;}
     break;
 
   case 31:
 
+/* Line 1455 of yacc.c  */
+#line 316 "dtc-parser.y"
     {
-			(yyval.cell) = eval_literal((yyvsp[(1) - (1)].literal), 0, 32);
+			uint64_t val = ~0ULL >> (64 - (yyvsp[(1) - (2)].array).bits);
+
+			if ((yyvsp[(1) - (2)].array).bits == 32)
+				(yyvsp[(1) - (2)].array).data = data_add_marker((yyvsp[(1) - (2)].array).data,
+							  REF_PHANDLE,
+							  (yyvsp[(2) - (2)].labelref));
+			else
+				print_error("References are only allowed in "
+					    "arrays with 32-bit elements.");
+
+			(yyval.array).data = data_append_integer((yyvsp[(1) - (2)].array).data, val, (yyvsp[(1) - (2)].array).bits);
 		;}
     break;
 
   case 32:
 
+/* Line 1455 of yacc.c  */
+#line 330 "dtc-parser.y"
     {
-			(yyval.data) = empty_data;
+			(yyval.array).data = data_add_marker((yyvsp[(1) - (2)].array).data, LABEL, (yyvsp[(2) - (2)].labelref));
 		;}
     break;
 
   case 33:
 
+/* Line 1455 of yacc.c  */
+#line 337 "dtc-parser.y"
     {
-			(yyval.data) = data_append_byte((yyvsp[(1) - (2)].data), (yyvsp[(2) - (2)].byte));
+			(yyval.integer) = eval_literal((yyvsp[(1) - (1)].literal), 0, 64);
 		;}
     break;
 
   case 34:
 
+/* Line 1455 of yacc.c  */
+#line 341 "dtc-parser.y"
     {
-			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
+			(yyval.integer) = eval_char_literal((yyvsp[(1) - (1)].literal));
 		;}
     break;
 
   case 35:
 
+/* Line 1455 of yacc.c  */
+#line 345 "dtc-parser.y"
+    {
+			(yyval.integer) = (yyvsp[(2) - (3)].integer);
+		;}
+    break;
+
+  case 38:
+
+/* Line 1455 of yacc.c  */
+#line 356 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (5)].integer) ? (yyvsp[(3) - (5)].integer) : (yyvsp[(5) - (5)].integer); ;}
+    break;
+
+  case 40:
+
+/* Line 1455 of yacc.c  */
+#line 361 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) || (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 42:
+
+/* Line 1455 of yacc.c  */
+#line 366 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) && (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 44:
+
+/* Line 1455 of yacc.c  */
+#line 371 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) | (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 46:
+
+/* Line 1455 of yacc.c  */
+#line 376 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) ^ (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 48:
+
+/* Line 1455 of yacc.c  */
+#line 381 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) & (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 50:
+
+/* Line 1455 of yacc.c  */
+#line 386 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) == (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 51:
+
+/* Line 1455 of yacc.c  */
+#line 387 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) != (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 53:
+
+/* Line 1455 of yacc.c  */
+#line 392 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) < (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 54:
+
+/* Line 1455 of yacc.c  */
+#line 393 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) > (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 55:
+
+/* Line 1455 of yacc.c  */
+#line 394 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) <= (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 56:
+
+/* Line 1455 of yacc.c  */
+#line 395 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) >= (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 57:
+
+/* Line 1455 of yacc.c  */
+#line 399 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) << (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 58:
+
+/* Line 1455 of yacc.c  */
+#line 400 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) >> (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 60:
+
+/* Line 1455 of yacc.c  */
+#line 405 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) + (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 61:
+
+/* Line 1455 of yacc.c  */
+#line 406 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) - (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 63:
+
+/* Line 1455 of yacc.c  */
+#line 411 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) * (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 64:
+
+/* Line 1455 of yacc.c  */
+#line 412 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) / (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 65:
+
+/* Line 1455 of yacc.c  */
+#line 413 "dtc-parser.y"
+    { (yyval.integer) = (yyvsp[(1) - (3)].integer) % (yyvsp[(3) - (3)].integer); ;}
+    break;
+
+  case 68:
+
+/* Line 1455 of yacc.c  */
+#line 419 "dtc-parser.y"
+    { (yyval.integer) = -(yyvsp[(2) - (2)].integer); ;}
+    break;
+
+  case 69:
+
+/* Line 1455 of yacc.c  */
+#line 420 "dtc-parser.y"
+    { (yyval.integer) = ~(yyvsp[(2) - (2)].integer); ;}
+    break;
+
+  case 70:
+
+/* Line 1455 of yacc.c  */
+#line 421 "dtc-parser.y"
+    { (yyval.integer) = !(yyvsp[(2) - (2)].integer); ;}
+    break;
+
+  case 71:
+
+/* Line 1455 of yacc.c  */
+#line 426 "dtc-parser.y"
+    {
+			(yyval.data) = empty_data;
+		;}
+    break;
+
+  case 72:
+
+/* Line 1455 of yacc.c  */
+#line 430 "dtc-parser.y"
+    {
+			(yyval.data) = data_append_byte((yyvsp[(1) - (2)].data), (yyvsp[(2) - (2)].byte));
+		;}
+    break;
+
+  case 73:
+
+/* Line 1455 of yacc.c  */
+#line 434 "dtc-parser.y"
+    {
+			(yyval.data) = data_add_marker((yyvsp[(1) - (2)].data), LABEL, (yyvsp[(2) - (2)].labelref));
+		;}
+    break;
+
+  case 74:
+
+/* Line 1455 of yacc.c  */
+#line 441 "dtc-parser.y"
     {
 			(yyval.nodelist) = NULL;
 		;}
     break;
 
-  case 36:
+  case 75:
 
+/* Line 1455 of yacc.c  */
+#line 445 "dtc-parser.y"
     {
 			(yyval.nodelist) = chain_node((yyvsp[(1) - (2)].node), (yyvsp[(2) - (2)].nodelist));
 		;}
     break;
 
-  case 37:
+  case 76:
 
+/* Line 1455 of yacc.c  */
+#line 449 "dtc-parser.y"
     {
 			print_error("syntax error: properties must precede subnodes");
 			YYERROR;
 		;}
     break;
 
-  case 38:
+  case 77:
 
+/* Line 1455 of yacc.c  */
+#line 457 "dtc-parser.y"
     {
 			(yyval.node) = name_node((yyvsp[(2) - (2)].node), (yyvsp[(1) - (2)].propnodename));
 		;}
     break;
 
-  case 39:
+  case 78:
+
+/* Line 1455 of yacc.c  */
+#line 461 "dtc-parser.y"
+    {
+			(yyval.node) = name_node(build_node_delete(), (yyvsp[(2) - (3)].propnodename));
+		;}
+    break;
+
+  case 79:
 
+/* Line 1455 of yacc.c  */
+#line 465 "dtc-parser.y"
     {
 			add_label(&(yyvsp[(2) - (2)].node)->labels, (yyvsp[(1) - (2)].labelref));
 			(yyval.node) = (yyvsp[(2) - (2)].node);
@@ -1700,6 +2119,8 @@ yyreduce:
 
 
 
+/* Line 1455 of yacc.c  */
+#line 2124 "dtc-parser.tab.c"
       default: break;
     }
   YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
@@ -1910,6 +2331,8 @@ yyreturn:
 
 
 
+/* Line 1675 of yacc.c  */
+#line 471 "dtc-parser.y"
 
 
 void print_error(char const *fmt, ...)
@@ -1934,9 +2357,12 @@ static unsigned long long eval_literal(const char *s, int base, int bits)
 
 	errno = 0;
 	val = strtoull(s, &e, base);
-	if (*e)
-		print_error("bad characters in literal");
-	else if ((errno == ERANGE)
+	if (*e) {
+		size_t uls = strspn(e, "UL");
+		if (e[uls])
+			print_error("bad characters in literal");
+	}
+	if ((errno == ERANGE)
 		 || ((bits < 64) && (val >= (1ULL << bits))))
 		print_error("literal out of range");
 	else if (errno != 0)
@@ -1944,3 +2370,29 @@ static unsigned long long eval_literal(const char *s, int base, int bits)
 	return val;
 }
 
+static unsigned char eval_char_literal(const char *s)
+{
+	int i = 1;
+	char c = s[0];
+
+	if (c == '\0')
+	{
+		print_error("empty character literal");
+		return 0;
+	}
+
+	/*
+	 * If the first character in the character literal is a \ then process
+	 * the remaining characters as an escape encoding. If the first
+	 * character is neither an escape or a terminator it should be the only
+	 * character in the literal and will be returned.
+	 */
+	if (c == '\\')
+		c = get_escape_char(s, &i);
+
+	if (s[i] != '\0')
+		print_error("malformed character literal");
+
+	return c;
+}
+
diff --git a/scripts/dtc/dtc-parser.tab.h_shipped b/scripts/dtc/dtc-parser.tab.h_shipped
index 4ee682bb7d3..9d2dce41211 100644
--- a/scripts/dtc/dtc-parser.tab.h_shipped
+++ b/scripts/dtc/dtc-parser.tab.h_shipped
@@ -1,9 +1,10 @@
-/* A Bison parser, made by GNU Bison 2.4.3.  */
+
+/* A Bison parser, made by GNU Bison 2.4.1.  */
 
 /* Skeleton interface for Bison's Yacc-like parsers in C
    
-      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-   2009, 2010 Free Software Foundation, Inc.
+      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
    
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -40,14 +41,26 @@
    enum yytokentype {
      DT_V1 = 258,
      DT_MEMRESERVE = 259,
-     DT_PROPNODENAME = 260,
-     DT_LITERAL = 261,
-     DT_BASE = 262,
-     DT_BYTE = 263,
-     DT_STRING = 264,
-     DT_LABEL = 265,
-     DT_REF = 266,
-     DT_INCBIN = 267
+     DT_LSHIFT = 260,
+     DT_RSHIFT = 261,
+     DT_LE = 262,
+     DT_GE = 263,
+     DT_EQ = 264,
+     DT_NE = 265,
+     DT_AND = 266,
+     DT_OR = 267,
+     DT_BITS = 268,
+     DT_DEL_PROP = 269,
+     DT_DEL_NODE = 270,
+     DT_PROPNODENAME = 271,
+     DT_LITERAL = 272,
+     DT_CHAR_LITERAL = 273,
+     DT_BASE = 274,
+     DT_BYTE = 275,
+     DT_STRING = 276,
+     DT_LABEL = 277,
+     DT_REF = 278,
+     DT_INCBIN = 279
    };
 #endif
 
@@ -57,6 +70,8 @@
 typedef union YYSTYPE
 {
 
+/* Line 1676 of yacc.c  */
+#line 40 "dtc-parser.y"
 
 	char *propnodename;
 	char *literal;
@@ -65,16 +80,22 @@ typedef union YYSTYPE
 	uint8_t byte;
 	struct data data;
 
-	uint64_t addr;
-	cell_t cell;
+	struct {
+		struct data	data;
+		int		bits;
+	} array;
+
 	struct property *prop;
 	struct property *proplist;
 	struct node *node;
 	struct node *nodelist;
 	struct reserve_info *re;
+	uint64_t integer;
 
 
 
+/* Line 1676 of yacc.c  */
+#line 99 "dtc-parser.tab.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
diff --git a/scripts/dtc/dtc-parser.y b/scripts/dtc/dtc-parser.y
index 5e84a67fc1d..f412460f94d 100644
--- a/scripts/dtc/dtc-parser.y
+++ b/scripts/dtc/dtc-parser.y
@@ -34,6 +34,7 @@ extern struct boot_info *the_boot_info;
 extern int treesource_error;
 
 static unsigned long long eval_literal(const char *s, int base, int bits);
+static unsigned char eval_char_literal(const char *s);
 %}
 
 %union {
@@ -44,19 +45,28 @@ static unsigned long long eval_literal(const char *s, int base, int bits);
 	uint8_t byte;
 	struct data data;
 
-	uint64_t addr;
-	cell_t cell;
+	struct {
+		struct data	data;
+		int		bits;
+	} array;
+
 	struct property *prop;
 	struct property *proplist;
 	struct node *node;
 	struct node *nodelist;
 	struct reserve_info *re;
+	uint64_t integer;
 }
 
 %token DT_V1
 %token DT_MEMRESERVE
+%token DT_LSHIFT DT_RSHIFT DT_LE DT_GE DT_EQ DT_NE DT_AND DT_OR
+%token DT_BITS
+%token DT_DEL_PROP
+%token DT_DEL_NODE
 %token <propnodename> DT_PROPNODENAME
 %token <literal> DT_LITERAL
+%token <literal> DT_CHAR_LITERAL
 %token <cbase> DT_BASE
 %token <byte> DT_BYTE
 %token <data> DT_STRING
@@ -68,9 +78,7 @@ static unsigned long long eval_literal(const char *s, int base, int bits);
 %type <data> propdataprefix
 %type <re> memreserve
 %type <re> memreserves
-%type <addr> addr
-%type <data> celllist
-%type <cell> cellval
+%type <array> arrayprefix
 %type <data> bytestring
 %type <prop> propdef
 %type <proplist> proplist
@@ -80,6 +88,21 @@ static unsigned long long eval_literal(const char *s, int base, int bits);
 %type <node> subnode
 %type <nodelist> subnodes
 
+%type <integer> integer_prim
+%type <integer> integer_unary
+%type <integer> integer_mul
+%type <integer> integer_add
+%type <integer> integer_shift
+%type <integer> integer_rela
+%type <integer> integer_eq
+%type <integer> integer_bitand
+%type <integer> integer_bitxor
+%type <integer> integer_bitor
+%type <integer> integer_and
+%type <integer> integer_or
+%type <integer> integer_trinary
+%type <integer> integer_expr
+
 %%
 
 sourcefile:
@@ -102,7 +125,7 @@ memreserves:
 	;
 
 memreserve:
-	  DT_MEMRESERVE addr addr ';'
+	  DT_MEMRESERVE integer_prim integer_prim ';'
 		{
 			$$ = build_reserve_entry($2, $3);
 		}
@@ -113,13 +136,6 @@ memreserve:
 		}
 	;
 
-addr:
-	  DT_LITERAL
-		{
-			$$ = eval_literal($1, 0, 64);
-		}
-	  ;
-
 devicetree:
 	  '/' nodedef
 		{
@@ -139,6 +155,17 @@ devicetree:
 				print_error("label or path, '%s', not found", $2);
 			$$ = $1;
 		}
+	| devicetree DT_DEL_NODE DT_REF ';'
+		{
+			struct node *target = get_node_by_ref($1, $3);
+
+			if (!target)
+				print_error("label or path, '%s', not found", $3);
+			else
+				delete_node(target);
+
+			$$ = $1;
+		}
 	;
 
 nodedef:
@@ -168,6 +195,10 @@ propdef:
 		{
 			$$ = build_property($1, empty_data);
 		}
+	| DT_DEL_PROP DT_PROPNODENAME ';'
+		{
+			$$ = build_property_delete($2);
+		}
 	| DT_LABEL propdef
 		{
 			add_label(&$2->labels, $1);
@@ -180,9 +211,9 @@ propdata:
 		{
 			$$ = data_merge($1, $2);
 		}
-	| propdataprefix '<' celllist '>'
+	| propdataprefix arrayprefix '>'
 		{
-			$$ = data_merge($1, $3);
+			$$ = data_merge($1, $2.data);
 		}
 	| propdataprefix '[' bytestring ']'
 		{
@@ -192,7 +223,7 @@ propdata:
 		{
 			$$ = data_add_marker($1, REF_PATH, $2);
 		}
-	| propdataprefix DT_INCBIN '(' DT_STRING ',' addr ',' addr ')'
+	| propdataprefix DT_INCBIN '(' DT_STRING ',' integer_prim ',' integer_prim ')'
 		{
 			FILE *f = srcfile_relative_open($4.val, NULL);
 			struct data d;
@@ -240,31 +271,154 @@ propdataprefix:
 		}
 	;
 
-celllist:
-	  /* empty */
+arrayprefix:
+	DT_BITS DT_LITERAL '<'
+		{
+			$$.data = empty_data;
+			$$.bits = eval_literal($2, 0, 7);
+
+			if (($$.bits !=  8) &&
+			    ($$.bits != 16) &&
+			    ($$.bits != 32) &&
+			    ($$.bits != 64))
+			{
+				print_error("Only 8, 16, 32 and 64-bit elements"
+					    " are currently supported");
+				$$.bits = 32;
+			}
+		}
+	| '<'
+		{
+			$$.data = empty_data;
+			$$.bits = 32;
+		}
+	| arrayprefix integer_prim
+		{
+			if ($1.bits < 64) {
+				uint64_t mask = (1ULL << $1.bits) - 1;
+				/*
+				 * Bits above mask must either be all zero
+				 * (positive within range of mask) or all one
+				 * (negative and sign-extended). The second
+				 * condition is true if when we set all bits
+				 * within the mask to one (i.e. | in the
+				 * mask), all bits are one.
+				 */
+				if (($2 > mask) && (($2 | mask) != -1ULL))
+					print_error(
+						"integer value out of range "
+						"%016lx (%d bits)", $1.bits);
+			}
+
+			$$.data = data_append_integer($1.data, $2, $1.bits);
+		}
+	| arrayprefix DT_REF
+		{
+			uint64_t val = ~0ULL >> (64 - $1.bits);
+
+			if ($1.bits == 32)
+				$1.data = data_add_marker($1.data,
+							  REF_PHANDLE,
+							  $2);
+			else
+				print_error("References are only allowed in "
+					    "arrays with 32-bit elements.");
+
+			$$.data = data_append_integer($1.data, val, $1.bits);
+		}
+	| arrayprefix DT_LABEL
 		{
-			$$ = empty_data;
+			$$.data = data_add_marker($1.data, LABEL, $2);
 		}
-	| celllist cellval
+	;
+
+integer_prim:
+	  DT_LITERAL
 		{
-			$$ = data_append_cell($1, $2);
+			$$ = eval_literal($1, 0, 64);
 		}
-	| celllist DT_REF
+	| DT_CHAR_LITERAL
 		{
-			$$ = data_append_cell(data_add_marker($1, REF_PHANDLE,
-							      $2), -1);
+			$$ = eval_char_literal($1);
 		}
-	| celllist DT_LABEL
+	| '(' integer_expr ')'
 		{
-			$$ = data_add_marker($1, LABEL, $2);
+			$$ = $2;
 		}
 	;
 
-cellval:
-	  DT_LITERAL
-		{
-			$$ = eval_literal($1, 0, 32);
-		}
+integer_expr:
+	integer_trinary
+	;
+
+integer_trinary:
+	  integer_or
+	| integer_or '?' integer_expr ':' integer_trinary { $$ = $1 ? $3 : $5; }
+	;
+
+integer_or:
+	  integer_and
+	| integer_or DT_OR integer_and { $$ = $1 || $3; }
+	;
+
+integer_and:
+	  integer_bitor
+	| integer_and DT_AND integer_bitor { $$ = $1 && $3; }
+	;
+
+integer_bitor:
+	  integer_bitxor
+	| integer_bitor '|' integer_bitxor { $$ = $1 | $3; }
+	;
+
+integer_bitxor:
+	  integer_bitand
+	| integer_bitxor '^' integer_bitand { $$ = $1 ^ $3; }
+	;
+
+integer_bitand:
+	  integer_eq
+	| integer_bitand '&' integer_eq { $$ = $1 & $3; }
+	;
+
+integer_eq:
+	  integer_rela
+	| integer_eq DT_EQ integer_rela { $$ = $1 == $3; }
+	| integer_eq DT_NE integer_rela { $$ = $1 != $3; }
+	;
+
+integer_rela:
+	  integer_shift
+	| integer_rela '<' integer_shift { $$ = $1 < $3; }
+	| integer_rela '>' integer_shift { $$ = $1 > $3; }
+	| integer_rela DT_LE integer_shift { $$ = $1 <= $3; }
+	| integer_rela DT_GE integer_shift { $$ = $1 >= $3; }
+	;
+
+integer_shift:
+	  integer_shift DT_LSHIFT integer_add { $$ = $1 << $3; }
+	| integer_shift DT_RSHIFT integer_add { $$ = $1 >> $3; }
+	| integer_add
+	;
+
+integer_add:
+	  integer_add '+' integer_mul { $$ = $1 + $3; }
+	| integer_add '-' integer_mul { $$ = $1 - $3; }
+	| integer_mul
+	;
+
+integer_mul:
+	  integer_mul '*' integer_unary { $$ = $1 * $3; }
+	| integer_mul '/' integer_unary { $$ = $1 / $3; }
+	| integer_mul '%' integer_unary { $$ = $1 % $3; }
+	| integer_unary
+	;
+
+integer_unary:
+	  integer_prim
+	| '-' integer_unary { $$ = -$2; }
+	| '~' integer_unary { $$ = ~$2; }
+	| '!' integer_unary { $$ = !$2; }
 	;
 
 bytestring:
@@ -303,6 +457,10 @@ subnode:
 		{
 			$$ = name_node($2, $1);
 		}
+	| DT_DEL_NODE DT_PROPNODENAME ';'
+		{
+			$$ = name_node(build_node_delete(), $2);
+		}
 	| DT_LABEL subnode
 		{
 			add_label(&$2->labels, $1);
@@ -334,12 +492,41 @@ static unsigned long long eval_literal(const char *s, int base, int bits)
 
 	errno = 0;
 	val = strtoull(s, &e, base);
-	if (*e)
-		print_error("bad characters in literal");
-	else if ((errno == ERANGE)
+	if (*e) {
+		size_t uls = strspn(e, "UL");
+		if (e[uls])
+			print_error("bad characters in literal");
+	}
+	if ((errno == ERANGE)
 		 || ((bits < 64) && (val >= (1ULL << bits))))
 		print_error("literal out of range");
 	else if (errno != 0)
 		print_error("bad literal");
 	return val;
 }
+
+static unsigned char eval_char_literal(const char *s)
+{
+	int i = 1;
+	char c = s[0];
+
+	if (c == '\0')
+	{
+		print_error("empty character literal");
+		return 0;
+	}
+
+	/*
+	 * If the first character in the character literal is a \ then process
+	 * the remaining characters as an escape encoding. If the first
+	 * character is neither an escape or a terminator it should be the only
+	 * character in the literal and will be returned.
+	 */
+	if (c == '\\')
+		c = get_escape_char(s, &i);
+
+	if (s[i] != '\0')
+		print_error("malformed character literal");
+
+	return c;
+}
diff --git a/scripts/dtc/dtc.c b/scripts/dtc/dtc.c
index 2ef5e2e3dd3..a375683c153 100644
--- a/scripts/dtc/dtc.c
+++ b/scripts/dtc/dtc.c
@@ -82,6 +82,8 @@ static void  __attribute__ ((noreturn)) usage(void)
 	fprintf(stderr, "\t\tSet the physical boot cpu\n");
 	fprintf(stderr, "\t-f\n");
 	fprintf(stderr, "\t\tForce - try to produce output even if the input tree has errors\n");
+	fprintf(stderr, "\t-i\n");
+	fprintf(stderr, "\t\tAdd a path to search for include files\n");
 	fprintf(stderr, "\t-s\n");
 	fprintf(stderr, "\t\tSort nodes and properties before outputting (only useful for\n\t\tcomparing trees)\n");
 	fprintf(stderr, "\t-v\n");
@@ -91,6 +93,9 @@ static void  __attribute__ ((noreturn)) usage(void)
 	fprintf(stderr, "\t\t\tlegacy - \"linux,phandle\" properties only\n");
 	fprintf(stderr, "\t\t\tepapr - \"phandle\" properties only\n");
 	fprintf(stderr, "\t\t\tboth - Both \"linux,phandle\" and \"phandle\" properties\n");
+	fprintf(stderr, "\t-W [no-]<checkname>\n");
+	fprintf(stderr, "\t-E [no-]<checkname>\n");
+	fprintf(stderr, "\t\t\tenable or disable warnings and errors\n");
 	exit(3);
 }
 
@@ -113,7 +118,7 @@ int main(int argc, char *argv[])
 	minsize    = 0;
 	padsize    = 0;
 
-	while ((opt = getopt(argc, argv, "hI:O:o:V:d:R:S:p:fcqb:vH:s"))
+	while ((opt = getopt(argc, argv, "hI:O:o:V:d:R:S:p:fqb:i:vH:sW:E:"))
 			!= EOF) {
 		switch (opt) {
 		case 'I':
@@ -149,6 +154,9 @@ int main(int argc, char *argv[])
 		case 'b':
 			cmdline_boot_cpuid = strtoll(optarg, NULL, 0);
 			break;
+		case 'i':
+			srcfile_add_search_path(optarg);
+			break;
 		case 'v':
 			printf("Version: %s\n", DTC_VERSION);
 			exit(0);
@@ -168,6 +176,14 @@ int main(int argc, char *argv[])
 			sort = 1;
 			break;
 
+		case 'W':
+			parse_checks_option(true, false, optarg);
+			break;
+
+		case 'E':
+			parse_checks_option(false, true, optarg);
+			break;
+
 		case 'h':
 		default:
 			usage();
@@ -188,9 +204,6 @@ int main(int argc, char *argv[])
 	if (minsize)
 		fprintf(stderr, "DTC: Use of \"-S\" is deprecated; it will be removed soon, use \"-p\" instead\n");
 
-	fprintf(stderr, "DTC: %s->%s  on file \"%s\"\n",
-		inform, outform, arg);
-
 	if (depname) {
 		depfile = fopen(depname, "w");
 		if (!depfile)
diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h
index f37c97eb3df..d501c8605f2 100644
--- a/scripts/dtc/dtc.h
+++ b/scripts/dtc/dtc.h
@@ -25,6 +25,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <stdarg.h>
 #include <assert.h>
 #include <ctype.h>
@@ -109,6 +110,7 @@ struct data data_insert_at_marker(struct data d, struct marker *m,
 				  const void *p, int len);
 struct data data_merge(struct data d1, struct data d2);
 struct data data_append_cell(struct data d, cell_t word);
+struct data data_append_integer(struct data d, uint64_t word, int bits);
 struct data data_append_re(struct data d, const struct fdt_reserve_entry *re);
 struct data data_append_addr(struct data d, uint64_t addr);
 struct data data_append_byte(struct data d, uint8_t byte);
@@ -126,11 +128,13 @@ int data_is_one_string(struct data d);
 
 /* Live trees */
 struct label {
+	int deleted;
 	char *label;
 	struct label *next;
 };
 
 struct property {
+	int deleted;
 	char *name;
 	struct data val;
 
@@ -140,6 +144,7 @@ struct property {
 };
 
 struct node {
+	int deleted;
 	char *name;
 	struct property *proplist;
 	struct node *children;
@@ -156,28 +161,71 @@ struct node {
 	struct label *labels;
 };
 
+static inline struct label *for_each_label_next(struct label *l)
+{
+	do {
+		l = l->next;
+	} while (l && l->deleted);
+
+	return l;
+}
+
 #define for_each_label(l0, l) \
+	for ((l) = (l0); (l); (l) = for_each_label_next(l))
+
+#define for_each_label_withdel(l0, l) \
 	for ((l) = (l0); (l); (l) = (l)->next)
 
+static inline struct property *for_each_property_next(struct property *p)
+{
+	do {
+		p = p->next;
+	} while (p && p->deleted);
+
+	return p;
+}
+
 #define for_each_property(n, p) \
+	for ((p) = (n)->proplist; (p); (p) = for_each_property_next(p))
+
+#define for_each_property_withdel(n, p) \
 	for ((p) = (n)->proplist; (p); (p) = (p)->next)
 
-#define for_each_child(n, c)	\
+static inline struct node *for_each_child_next(struct node *c)
+{
+	do {
+		c = c->next_sibling;
+	} while (c && c->deleted);
+
+	return c;
+}
+
+#define for_each_child(n, c) \
+	for ((c) = (n)->children; (c); (c) = for_each_child_next(c))
+
+#define for_each_child_withdel(n, c) \
 	for ((c) = (n)->children; (c); (c) = (c)->next_sibling)
 
 void add_label(struct label **labels, char *label);
+void delete_labels(struct label **labels);
 
 struct property *build_property(char *name, struct data val);
+struct property *build_property_delete(char *name);
 struct property *chain_property(struct property *first, struct property *list);
 struct property *reverse_properties(struct property *first);
 
 struct node *build_node(struct property *proplist, struct node *children);
+struct node *build_node_delete(void);
 struct node *name_node(struct node *node, char *name);
 struct node *chain_node(struct node *first, struct node *list);
 struct node *merge_nodes(struct node *old_node, struct node *new_node);
 
 void add_property(struct node *node, struct property *prop);
+void delete_property_by_name(struct node *node, char *name);
+void delete_property(struct property *prop);
 void add_child(struct node *parent, struct node *child);
+void delete_node_by_name(struct node *parent, char *name);
+void delete_node(struct node *node);
 
 const char *get_unitname(struct node *node);
 struct property *get_property(struct node *node, const char *propname);
@@ -224,6 +272,7 @@ void sort_tree(struct boot_info *bi);
 
 /* Checks */
 
+void parse_checks_option(bool warn, bool error, const char *optarg);
 void process_checks(int force, struct boot_info *bi);
 
 /* Flattened trees */
diff --git a/scripts/dtc/fdtdump.c b/scripts/dtc/fdtdump.c
new file mode 100644
index 00000000000..207a46d6486
--- /dev/null
+++ b/scripts/dtc/fdtdump.c
@@ -0,0 +1,162 @@
+/*
+ * fdtdump.c - Contributed by Pantelis Antoniou <pantelis.antoniou AT gmail.com>
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+
+#include <fdt.h>
+#include <libfdt_env.h>
+
+#include "util.h"
+
+#define ALIGN(x, a)	(((x) + ((a) - 1)) & ~((a) - 1))
+#define PALIGN(p, a)	((void *)(ALIGN((unsigned long)(p), (a))))
+#define GET_CELL(p)	(p += 4, *((const uint32_t *)(p-4)))
+
+static void print_data(const char *data, int len)
+{
+	int i;
+	const char *p = data;
+
+	/* no data, don't print */
+	if (len == 0)
+		return;
+
+	if (util_is_printable_string(data, len)) {
+		printf(" = \"%s\"", (const char *)data);
+	} else if ((len % 4) == 0) {
+		printf(" = <");
+		for (i = 0; i < len; i += 4)
+			printf("0x%08x%s", fdt32_to_cpu(GET_CELL(p)),
+			       i < (len - 4) ? " " : "");
+		printf(">");
+	} else {
+		printf(" = [");
+		for (i = 0; i < len; i++)
+			printf("%02x%s", *p++, i < len - 1 ? " " : "");
+		printf("]");
+	}
+}
+
+static void dump_blob(void *blob)
+{
+	struct fdt_header *bph = blob;
+	uint32_t off_mem_rsvmap = fdt32_to_cpu(bph->off_mem_rsvmap);
+	uint32_t off_dt = fdt32_to_cpu(bph->off_dt_struct);
+	uint32_t off_str = fdt32_to_cpu(bph->off_dt_strings);
+	struct fdt_reserve_entry *p_rsvmap =
+		(struct fdt_reserve_entry *)((char *)blob + off_mem_rsvmap);
+	const char *p_struct = (const char *)blob + off_dt;
+	const char *p_strings = (const char *)blob + off_str;
+	uint32_t version = fdt32_to_cpu(bph->version);
+	uint32_t totalsize = fdt32_to_cpu(bph->totalsize);
+	uint32_t tag;
+	const char *p, *s, *t;
+	int depth, sz, shift;
+	int i;
+	uint64_t addr, size;
+
+	depth = 0;
+	shift = 4;
+
+	printf("/dts-v1/;\n");
+	printf("// magic:\t\t0x%x\n", fdt32_to_cpu(bph->magic));
+	printf("// totalsize:\t\t0x%x (%d)\n", totalsize, totalsize);
+	printf("// off_dt_struct:\t0x%x\n", off_dt);
+	printf("// off_dt_strings:\t0x%x\n", off_str);
+	printf("// off_mem_rsvmap:\t0x%x\n", off_mem_rsvmap);
+	printf("// version:\t\t%d\n", version);
+	printf("// last_comp_version:\t%d\n",
+	       fdt32_to_cpu(bph->last_comp_version));
+	if (version >= 2)
+		printf("// boot_cpuid_phys:\t0x%x\n",
+		       fdt32_to_cpu(bph->boot_cpuid_phys));
+
+	if (version >= 3)
+		printf("// size_dt_strings:\t0x%x\n",
+		       fdt32_to_cpu(bph->size_dt_strings));
+	if (version >= 17)
+		printf("// size_dt_struct:\t0x%x\n",
+		       fdt32_to_cpu(bph->size_dt_struct));
+	printf("\n");
+
+	for (i = 0; ; i++) {
+		addr = fdt64_to_cpu(p_rsvmap[i].address);
+		size = fdt64_to_cpu(p_rsvmap[i].size);
+		if (addr == 0 && size == 0)
+			break;
+
+		printf("/memreserve/ %llx %llx;\n",
+		       (unsigned long long)addr, (unsigned long long)size);
+	}
+
+	p = p_struct;
+	while ((tag = fdt32_to_cpu(GET_CELL(p))) != FDT_END) {
+
+		/* printf("tag: 0x%08x (%d)\n", tag, p - p_struct); */
+
+		if (tag == FDT_BEGIN_NODE) {
+			s = p;
+			p = PALIGN(p + strlen(s) + 1, 4);
+
+			if (*s == '\0')
+				s = "/";
+
+			printf("%*s%s {\n", depth * shift, "", s);
+
+			depth++;
+			continue;
+		}
+
+		if (tag == FDT_END_NODE) {
+			depth--;
+
+			printf("%*s};\n", depth * shift, "");
+			continue;
+		}
+
+		if (tag == FDT_NOP) {
+			printf("%*s// [NOP]\n", depth * shift, "");
+			continue;
+		}
+
+		if (tag != FDT_PROP) {
+			fprintf(stderr, "%*s ** Unknown tag 0x%08x\n", depth * shift, "", tag);
+			break;
+		}
+		sz = fdt32_to_cpu(GET_CELL(p));
+		s = p_strings + fdt32_to_cpu(GET_CELL(p));
+		if (version < 16 && sz >= 8)
+			p = PALIGN(p, 8);
+		t = p;
+
+		p = PALIGN(p + sz, 4);
+
+		printf("%*s%s", depth * shift, "", s);
+		print_data(t, sz);
+		printf(";\n");
+	}
+}
+
+
+int main(int argc, char *argv[])
+{
+	char *buf;
+
+	if (argc < 2) {
+		fprintf(stderr, "supply input filename\n");
+		return 5;
+	}
+
+	buf = utilfdt_read(argv[1]);
+	if (buf)
+		dump_blob(buf);
+	else
+		return 10;
+
+	return 0;
+}
diff --git a/scripts/dtc/fdtget.c b/scripts/dtc/fdtget.c
new file mode 100644
index 00000000000..c2fbab2a547
--- /dev/null
+++ b/scripts/dtc/fdtget.c
@@ -0,0 +1,366 @@
+/*
+ * Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+ *
+ * Portions from U-Boot cmd_fdt.c (C) Copyright 2007
+ * Gerald Van Baren, Custom IDEAS, vanbaren@cideas.com
+ * Based on code written by:
+ *   Pantelis Antoniou <pantelis.antoniou@gmail.com> and
+ *   Matthew McClintock <msm@freescale.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <libfdt.h>
+
+#include "util.h"
+
+enum display_mode {
+	MODE_SHOW_VALUE,	/* show values for node properties */
+	MODE_LIST_PROPS,	/* list the properties for a node */
+	MODE_LIST_SUBNODES,	/* list the subnodes of a node */
+};
+
+/* Holds information which controls our output and options */
+struct display_info {
+	int type;		/* data type (s/i/u/x or 0 for default) */
+	int size;		/* data size (1/2/4) */
+	enum display_mode mode;	/* display mode that we are using */
+	const char *default_val; /* default value if node/property not found */
+};
+
+static void report_error(const char *where, int err)
+{
+	fprintf(stderr, "Error at '%s': %s\n", where, fdt_strerror(err));
+}
+
+/**
+ * Displays data of a given length according to selected options
+ *
+ * If a specific data type is provided in disp, then this is used. Otherwise
+ * we try to guess the data type / size from the contents.
+ *
+ * @param disp		Display information / options
+ * @param data		Data to display
+ * @param len		Maximum length of buffer
+ * @return 0 if ok, -1 if data does not match format
+ */
+static int show_data(struct display_info *disp, const char *data, int len)
+{
+	int i, size;
+	const uint8_t *p = (const uint8_t *)data;
+	const char *s;
+	int value;
+	int is_string;
+	char fmt[3];
+
+	/* no data, don't print */
+	if (len == 0)
+		return 0;
+
+	is_string = (disp->type) == 's' ||
+		(!disp->type && util_is_printable_string(data, len));
+	if (is_string) {
+		if (data[len - 1] != '\0') {
+			fprintf(stderr, "Unterminated string\n");
+			return -1;
+		}
+		for (s = data; s - data < len; s += strlen(s) + 1) {
+			if (s != data)
+				printf(" ");
+			printf("%s", (const char *)s);
+		}
+		return 0;
+	}
+	size = disp->size;
+	if (size == -1) {
+		size = (len % 4) == 0 ? 4 : 1;
+	} else if (len % size) {
+		fprintf(stderr, "Property length must be a multiple of "
+				"selected data size\n");
+		return -1;
+	}
+	fmt[0] = '%';
+	fmt[1] = disp->type ? disp->type : 'd';
+	fmt[2] = '\0';
+	for (i = 0; i < len; i += size, p += size) {
+		if (i)
+			printf(" ");
+		value = size == 4 ? fdt32_to_cpu(*(const uint32_t *)p) :
+			size == 2 ? (*p << 8) | p[1] : *p;
+		printf(fmt, value);
+	}
+	return 0;
+}
+
+/**
+ * List all properties in a node, one per line.
+ *
+ * @param blob		FDT blob
+ * @param node		Node to display
+ * @return 0 if ok, or FDT_ERR... if not.
+ */
+static int list_properties(const void *blob, int node)
+{
+	const struct fdt_property *data;
+	const char *name;
+	int prop;
+
+	prop = fdt_first_property_offset(blob, node);
+	do {
+		/* Stop silently when there are no more properties */
+		if (prop < 0)
+			return prop == -FDT_ERR_NOTFOUND ? 0 : prop;
+		data = fdt_get_property_by_offset(blob, prop, NULL);
+		name = fdt_string(blob, fdt32_to_cpu(data->nameoff));
+		if (name)
+			puts(name);
+		prop = fdt_next_property_offset(blob, prop);
+	} while (1);
+}
+
+#define MAX_LEVEL	32		/* how deeply nested we will go */
+
+/**
+ * List all subnodes in a node, one per line
+ *
+ * @param blob		FDT blob
+ * @param node		Node to display
+ * @return 0 if ok, or FDT_ERR... if not.
+ */
+static int list_subnodes(const void *blob, int node)
+{
+	int nextoffset;		/* next node offset from libfdt */
+	uint32_t tag;		/* current tag */
+	int level = 0;		/* keep track of nesting level */
+	const char *pathp;
+	int depth = 1;		/* the assumed depth of this node */
+
+	while (level >= 0) {
+		tag = fdt_next_tag(blob, node, &nextoffset);
+		switch (tag) {
+		case FDT_BEGIN_NODE:
+			pathp = fdt_get_name(blob, node, NULL);
+			if (level <= depth) {
+				if (pathp == NULL)
+					pathp = "/* NULL pointer error */";
+				if (*pathp == '\0')
+					pathp = "/";	/* root is nameless */
+				if (level == 1)
+					puts(pathp);
+			}
+			level++;
+			if (level >= MAX_LEVEL) {
+				printf("Nested too deep, aborting.\n");
+				return 1;
+			}
+			break;
+		case FDT_END_NODE:
+			level--;
+			if (level == 0)
+				level = -1;		/* exit the loop */
+			break;
+		case FDT_END:
+			return 1;
+		case FDT_PROP:
+			break;
+		default:
+			if (level <= depth)
+				printf("Unknown tag 0x%08X\n", tag);
+			return 1;
+		}
+		node = nextoffset;
+	}
+	return 0;
+}
+
+/**
+ * Show the data for a given node (and perhaps property) according to the
+ * display option provided.
+ *
+ * @param blob		FDT blob
+ * @param disp		Display information / options
+ * @param node		Node to display
+ * @param property	Name of property to display, or NULL if none
+ * @return 0 if ok, -ve on error
+ */
+static int show_data_for_item(const void *blob, struct display_info *disp,
+		int node, const char *property)
+{
+	const void *value = NULL;
+	int len, err = 0;
+
+	switch (disp->mode) {
+	case MODE_LIST_PROPS:
+		err = list_properties(blob, node);
+		break;
+
+	case MODE_LIST_SUBNODES:
+		err = list_subnodes(blob, node);
+		break;
+
+	default:
+		assert(property);
+		value = fdt_getprop(blob, node, property, &len);
+		if (value) {
+			if (show_data(disp, value, len))
+				err = -1;
+			else
+				printf("\n");
+		} else if (disp->default_val) {
+			puts(disp->default_val);
+		} else {
+			report_error(property, len);
+			err = -1;
+		}
+		break;
+	}
+
+	return err;
+}
+
+/**
+ * Run the main fdtget operation, given a filename and valid arguments
+ *
+ * @param disp		Display information / options
+ * @param filename	Filename of blob file
+ * @param arg		List of arguments to process
+ * @param arg_count	Number of arguments
+ * @param return 0 if ok, -ve on error
+ */
+static int do_fdtget(struct display_info *disp, const char *filename,
+		     char **arg, int arg_count, int args_per_step)
+{
+	char *blob;
+	const char *prop;
+	int i, node;
+
+	blob = utilfdt_read(filename);
+	if (!blob)
+		return -1;
+
+	for (i = 0; i + args_per_step <= arg_count; i += args_per_step) {
+		node = fdt_path_offset(blob, arg[i]);
+		if (node < 0) {
+			if (disp->default_val) {
+				puts(disp->default_val);
+				continue;
+			} else {
+				report_error(arg[i], node);
+				return -1;
+			}
+		}
+		prop = args_per_step == 1 ? NULL : arg[i + 1];
+
+		if (show_data_for_item(blob, disp, node, prop))
+			return -1;
+	}
+	return 0;
+}
+
+static const char *usage_msg =
+	"fdtget - read values from device tree\n"
+	"\n"
+	"Each value is printed on a new line.\n\n"
+	"Usage:\n"
+	"	fdtget <options> <dt file> [<node> <property>]...\n"
+	"	fdtget -p <options> <dt file> [<node> ]...\n"
+	"Options:\n"
+	"\t-t <type>\tType of data\n"
+	"\t-p\t\tList properties for each node\n"
+	"\t-l\t\tList subnodes for each node\n"
+	"\t-d\t\tDefault value to display when the property is "
+			"missing\n"
+	"\t-h\t\tPrint this help\n\n"
+	USAGE_TYPE_MSG;
+
+static void usage(const char *msg)
+{
+	if (msg)
+		fprintf(stderr, "Error: %s\n\n", msg);
+
+	fprintf(stderr, "%s", usage_msg);
+	exit(2);
+}
+
+int main(int argc, char *argv[])
+{
+	char *filename = NULL;
+	struct display_info disp;
+	int args_per_step = 2;
+
+	/* set defaults */
+	memset(&disp, '\0', sizeof(disp));
+	disp.size = -1;
+	disp.mode = MODE_SHOW_VALUE;
+	for (;;) {
+		int c = getopt(argc, argv, "d:hlpt:");
+		if (c == -1)
+			break;
+
+		switch (c) {
+		case 'h':
+		case '?':
+			usage(NULL);
+
+		case 't':
+			if (utilfdt_decode_type(optarg, &disp.type,
+					&disp.size))
+				usage("Invalid type string");
+			break;
+
+		case 'p':
+			disp.mode = MODE_LIST_PROPS;
+			args_per_step = 1;
+			break;
+
+		case 'l':
+			disp.mode = MODE_LIST_SUBNODES;
+			args_per_step = 1;
+			break;
+
+		case 'd':
+			disp.default_val = optarg;
+			break;
+		}
+	}
+
+	if (optind < argc)
+		filename = argv[optind++];
+	if (!filename)
+		usage("Missing filename");
+
+	argv += optind;
+	argc -= optind;
+
+	/* Allow no arguments, and silently succeed */
+	if (!argc)
+		return 0;
+
+	/* Check for node, property arguments */
+	if (args_per_step == 2 && (argc % 2))
+		usage("Must have an even number of arguments");
+
+	if (do_fdtget(&disp, filename, argv, argc, args_per_step))
+		return 1;
+	return 0;
+}
diff --git a/scripts/dtc/fdtput.c b/scripts/dtc/fdtput.c
new file mode 100644
index 00000000000..f2197f51930
--- /dev/null
+++ b/scripts/dtc/fdtput.c
@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <libfdt.h>
+
+#include "util.h"
+
+/* These are the operations we support */
+enum oper_type {
+	OPER_WRITE_PROP,		/* Write a property in a node */
+	OPER_CREATE_NODE,		/* Create a new node */
+};
+
+struct display_info {
+	enum oper_type oper;	/* operation to perform */
+	int type;		/* data type (s/i/u/x or 0 for default) */
+	int size;		/* data size (1/2/4) */
+	int verbose;		/* verbose output */
+	int auto_path;		/* automatically create all path components */
+};
+
+
+/**
+ * Report an error with a particular node.
+ *
+ * @param name		Node name to report error on
+ * @param namelen	Length of node name, or -1 to use entire string
+ * @param err		Error number to report (-FDT_ERR_...)
+ */
+static void report_error(const char *name, int namelen, int err)
+{
+	if (namelen == -1)
+		namelen = strlen(name);
+	fprintf(stderr, "Error at '%1.*s': %s\n", namelen, name,
+		fdt_strerror(err));
+}
+
+/**
+ * Encode a series of arguments in a property value.
+ *
+ * @param disp		Display information / options
+ * @param arg		List of arguments from command line
+ * @param arg_count	Number of arguments (may be 0)
+ * @param valuep	Returns buffer containing value
+ * @param *value_len	Returns length of value encoded
+ */
+static int encode_value(struct display_info *disp, char **arg, int arg_count,
+			char **valuep, int *value_len)
+{
+	char *value = NULL;	/* holding area for value */
+	int value_size = 0;	/* size of holding area */
+	char *ptr;		/* pointer to current value position */
+	int len;		/* length of this cell/string/byte */
+	int ival;
+	int upto;	/* the number of bytes we have written to buf */
+	char fmt[3];
+
+	upto = 0;
+
+	if (disp->verbose)
+		fprintf(stderr, "Decoding value:\n");
+
+	fmt[0] = '%';
+	fmt[1] = disp->type ? disp->type : 'd';
+	fmt[2] = '\0';
+	for (; arg_count > 0; arg++, arg_count--, upto += len) {
+		/* assume integer unless told otherwise */
+		if (disp->type == 's')
+			len = strlen(*arg) + 1;
+		else
+			len = disp->size == -1 ? 4 : disp->size;
+
+		/* enlarge our value buffer by a suitable margin if needed */
+		if (upto + len > value_size) {
+			value_size = (upto + len) + 500;
+			value = realloc(value, value_size);
+			if (!value) {
+				fprintf(stderr, "Out of mmory: cannot alloc "
+					"%d bytes\n", value_size);
+				return -1;
+			}
+		}
+
+		ptr = value + upto;
+		if (disp->type == 's') {
+			memcpy(ptr, *arg, len);
+			if (disp->verbose)
+				fprintf(stderr, "\tstring: '%s'\n", ptr);
+		} else {
+			int *iptr = (int *)ptr;
+			sscanf(*arg, fmt, &ival);
+			if (len == 4)
+				*iptr = cpu_to_fdt32(ival);
+			else
+				*ptr = (uint8_t)ival;
+			if (disp->verbose) {
+				fprintf(stderr, "\t%s: %d\n",
+					disp->size == 1 ? "byte" :
+					disp->size == 2 ? "short" : "int",
+					ival);
+			}
+		}
+	}
+	*value_len = upto;
+	*valuep = value;
+	if (disp->verbose)
+		fprintf(stderr, "Value size %d\n", upto);
+	return 0;
+}
+
+static int store_key_value(void *blob, const char *node_name,
+		const char *property, const char *buf, int len)
+{
+	int node;
+	int err;
+
+	node = fdt_path_offset(blob, node_name);
+	if (node < 0) {
+		report_error(node_name, -1, node);
+		return -1;
+	}
+
+	err = fdt_setprop(blob, node, property, buf, len);
+	if (err) {
+		report_error(property, -1, err);
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * Create paths as needed for all components of a path
+ *
+ * Any components of the path that do not exist are created. Errors are
+ * reported.
+ *
+ * @param blob		FDT blob to write into
+ * @param in_path	Path to process
+ * @return 0 if ok, -1 on error
+ */
+static int create_paths(void *blob, const char *in_path)
+{
+	const char *path = in_path;
+	const char *sep;
+	int node, offset = 0;
+
+	/* skip leading '/' */
+	while (*path == '/')
+		path++;
+
+	for (sep = path; *sep; path = sep + 1, offset = node) {
+		/* equivalent to strchrnul(), but it requires _GNU_SOURCE */
+		sep = strchr(path, '/');
+		if (!sep)
+			sep = path + strlen(path);
+
+		node = fdt_subnode_offset_namelen(blob, offset, path,
+				sep - path);
+		if (node == -FDT_ERR_NOTFOUND) {
+			node = fdt_add_subnode_namelen(blob, offset, path,
+						       sep - path);
+		}
+		if (node < 0) {
+			report_error(path, sep - path, node);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Create a new node in the fdt.
+ *
+ * This will overwrite the node_name string. Any error is reported.
+ *
+ * TODO: Perhaps create fdt_path_offset_namelen() so we don't need to do this.
+ *
+ * @param blob		FDT blob to write into
+ * @param node_name	Name of node to create
+ * @return new node offset if found, or -1 on failure
+ */
+static int create_node(void *blob, const char *node_name)
+{
+	int node = 0;
+	char *p;
+
+	p = strrchr(node_name, '/');
+	if (!p) {
+		report_error(node_name, -1, -FDT_ERR_BADPATH);
+		return -1;
+	}
+	*p = '\0';
+
+	if (p > node_name) {
+		node = fdt_path_offset(blob, node_name);
+		if (node < 0) {
+			report_error(node_name, -1, node);
+			return -1;
+		}
+	}
+
+	node = fdt_add_subnode(blob, node, p + 1);
+	if (node < 0) {
+		report_error(p + 1, -1, node);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int do_fdtput(struct display_info *disp, const char *filename,
+		    char **arg, int arg_count)
+{
+	char *value;
+	char *blob;
+	int len, ret = 0;
+
+	blob = utilfdt_read(filename);
+	if (!blob)
+		return -1;
+
+	switch (disp->oper) {
+	case OPER_WRITE_PROP:
+		/*
+		 * Convert the arguments into a single binary value, then
+		 * store them into the property.
+		 */
+		assert(arg_count >= 2);
+		if (disp->auto_path && create_paths(blob, *arg))
+			return -1;
+		if (encode_value(disp, arg + 2, arg_count - 2, &value, &len) ||
+			store_key_value(blob, *arg, arg[1], value, len))
+			ret = -1;
+		break;
+	case OPER_CREATE_NODE:
+		for (; ret >= 0 && arg_count--; arg++) {
+			if (disp->auto_path)
+				ret = create_paths(blob, *arg);
+			else
+				ret = create_node(blob, *arg);
+		}
+		break;
+	}
+	if (ret >= 0)
+		ret = utilfdt_write(filename, blob);
+
+	free(blob);
+	return ret;
+}
+
+static const char *usage_msg =
+	"fdtput - write a property value to a device tree\n"
+	"\n"
+	"The command line arguments are joined together into a single value.\n"
+	"\n"
+	"Usage:\n"
+	"	fdtput <options> <dt file> <node> <property> [<value>...]\n"
+	"	fdtput -c <options> <dt file> [<node>...]\n"
+	"Options:\n"
+	"\t-c\t\tCreate nodes if they don't already exist\n"
+	"\t-p\t\tAutomatically create nodes as needed for the node path\n"
+	"\t-t <type>\tType of data\n"
+	"\t-v\t\tVerbose: display each value decoded from command line\n"
+	"\t-h\t\tPrint this help\n\n"
+	USAGE_TYPE_MSG;
+
+static void usage(const char *msg)
+{
+	if (msg)
+		fprintf(stderr, "Error: %s\n\n", msg);
+
+	fprintf(stderr, "%s", usage_msg);
+	exit(2);
+}
+
+int main(int argc, char *argv[])
+{
+	struct display_info disp;
+	char *filename = NULL;
+
+	memset(&disp, '\0', sizeof(disp));
+	disp.size = -1;
+	disp.oper = OPER_WRITE_PROP;
+	for (;;) {
+		int c = getopt(argc, argv, "chpt:v");
+		if (c == -1)
+			break;
+
+		/*
+		 * TODO: add options to:
+		 * - delete property
+		 * - delete node (optionally recursively)
+		 * - rename node
+		 * - pack fdt before writing
+		 * - set amount of free space when writing
+		 * - expand fdt if value doesn't fit
+		 */
+		switch (c) {
+		case 'c':
+			disp.oper = OPER_CREATE_NODE;
+			break;
+		case 'h':
+		case '?':
+			usage(NULL);
+		case 'p':
+			disp.auto_path = 1;
+			break;
+		case 't':
+			if (utilfdt_decode_type(optarg, &disp.type,
+					&disp.size))
+				usage("Invalid type string");
+			break;
+
+		case 'v':
+			disp.verbose = 1;
+			break;
+		}
+	}
+
+	if (optind < argc)
+		filename = argv[optind++];
+	if (!filename)
+		usage("Missing filename");
+
+	argv += optind;
+	argc -= optind;
+
+	if (disp.oper == OPER_WRITE_PROP) {
+		if (argc < 1)
+			usage("Missing node");
+		if (argc < 2)
+			usage("Missing property");
+	}
+
+	if (do_fdtput(&disp, filename, argv, argc))
+		return 1;
+	return 0;
+}
diff --git a/scripts/dtc/flattree.c b/scripts/dtc/flattree.c
index 28d0b2381df..665dad7bb46 100644
--- a/scripts/dtc/flattree.c
+++ b/scripts/dtc/flattree.c
@@ -263,6 +263,9 @@ static void flatten_tree(struct node *tree, struct emitter *emit,
 	struct node *child;
 	int seen_name_prop = 0;
 
+	if (tree->deleted)
+		return;
+
 	emit->beginnode(etarget, tree->labels);
 
 	if (vi->flags & FTF_FULLPATH)
diff --git a/scripts/dtc/libfdt/Makefile.libfdt b/scripts/dtc/libfdt/Makefile.libfdt
index 6c42acfa21e..91126c000a1 100644
--- a/scripts/dtc/libfdt/Makefile.libfdt
+++ b/scripts/dtc/libfdt/Makefile.libfdt
@@ -3,6 +3,8 @@
 # This is not a complete Makefile of itself.  Instead, it is designed to
 # be easily embeddable into other systems of Makefiles.
 #
-LIBFDT_INCLUDES = fdt.h libfdt.h
-LIBFDT_SRCS = fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
+LIBFDT_soname = libfdt.$(SHAREDLIB_EXT).1
+LIBFDT_INCLUDES = fdt.h libfdt.h libfdt_env.h
+LIBFDT_VERSION = version.lds
+LIBFDT_SRCS = fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c fdt_empty_tree.c
 LIBFDT_OBJS = $(LIBFDT_SRCS:%.c=%.o)
diff --git a/scripts/dtc/libfdt/fdt.c b/scripts/dtc/libfdt/fdt.c
index 2acaec5923a..e56833ae9b6 100644
--- a/scripts/dtc/libfdt/fdt.c
+++ b/scripts/dtc/libfdt/fdt.c
@@ -74,7 +74,7 @@ int fdt_check_header(const void *fdt)
 	return 0;
 }
 
-const void *fdt_offset_ptr(const void *fdt, int offset, int len)
+const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int len)
 {
 	const char *p;
 
@@ -90,42 +90,53 @@ const void *fdt_offset_ptr(const void *fdt, int offset, int len)
 	return p;
 }
 
-uint32_t fdt_next_tag(const void *fdt, int offset, int *nextoffset)
+uint32_t fdt_next_tag(const void *fdt, int startoffset, int *nextoffset)
 {
 	const uint32_t *tagp, *lenp;
 	uint32_t tag;
+	int offset = startoffset;
 	const char *p;
 
-	if (offset % FDT_TAGSIZE)
-		return -1;
-
+	*nextoffset = -FDT_ERR_TRUNCATED;
 	tagp = fdt_offset_ptr(fdt, offset, FDT_TAGSIZE);
-	if (! tagp)
+	if (!tagp)
 		return FDT_END; /* premature end */
 	tag = fdt32_to_cpu(*tagp);
 	offset += FDT_TAGSIZE;
 
+	*nextoffset = -FDT_ERR_BADSTRUCTURE;
 	switch (tag) {
 	case FDT_BEGIN_NODE:
 		/* skip name */
 		do {
 			p = fdt_offset_ptr(fdt, offset++, 1);
 		} while (p && (*p != '\0'));
-		if (! p)
-			return FDT_END;
+		if (!p)
+			return FDT_END; /* premature end */
 		break;
+
 	case FDT_PROP:
 		lenp = fdt_offset_ptr(fdt, offset, sizeof(*lenp));
-		if (! lenp)
-			return FDT_END;
-		/* skip name offset, length and value */
-		offset += 2*FDT_TAGSIZE + fdt32_to_cpu(*lenp);
+		if (!lenp)
+			return FDT_END; /* premature end */
+		/* skip-name offset, length and value */
+		offset += sizeof(struct fdt_property) - FDT_TAGSIZE
+			+ fdt32_to_cpu(*lenp);
+		break;
+
+	case FDT_END:
+	case FDT_END_NODE:
+	case FDT_NOP:
 		break;
+
+	default:
+		return FDT_END;
 	}
 
-	if (nextoffset)
-		*nextoffset = FDT_TAGALIGN(offset);
+	if (!fdt_offset_ptr(fdt, startoffset, offset - startoffset))
+		return FDT_END; /* premature end */
 
+	*nextoffset = FDT_TAGALIGN(offset);
 	return tag;
 }
 
@@ -138,6 +149,15 @@ int _fdt_check_node_offset(const void *fdt, int offset)
 	return offset;
 }
 
+int _fdt_check_prop_offset(const void *fdt, int offset)
+{
+	if ((offset < 0) || (offset % FDT_TAGSIZE)
+	    || (fdt_next_tag(fdt, offset, &offset) != FDT_PROP))
+		return -FDT_ERR_BADOFFSET;
+
+	return offset;
+}
+
 int fdt_next_node(const void *fdt, int offset, int *depth)
 {
 	int nextoffset = 0;
@@ -162,15 +182,16 @@ int fdt_next_node(const void *fdt, int offset, int *depth)
 			break;
 
 		case FDT_END_NODE:
-			if (depth)
-				(*depth)--;
+			if (depth && ((--(*depth)) < 0))
+				return nextoffset;
 			break;
 
 		case FDT_END:
-			return -FDT_ERR_NOTFOUND;
-
-		default:
-			return -FDT_ERR_BADSTRUCTURE;
+			if ((nextoffset >= 0)
+			    || ((nextoffset == -FDT_ERR_TRUNCATED) && !depth))
+				return -FDT_ERR_NOTFOUND;
+			else
+				return nextoffset;
 		}
 	} while (tag != FDT_BEGIN_NODE);
 
diff --git a/scripts/dtc/libfdt/fdt_empty_tree.c b/scripts/dtc/libfdt/fdt_empty_tree.c
new file mode 100644
index 00000000000..f72d13b1d19
--- /dev/null
+++ b/scripts/dtc/libfdt/fdt_empty_tree.c
@@ -0,0 +1,84 @@
+/*
+ * libfdt - Flat Device Tree manipulation
+ * Copyright (C) 2012 David Gibson, IBM Corporation.
+ *
+ * libfdt is dual licensed: you can use it either under the terms of
+ * the GPL, or the BSD license, at your option.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public
+ *     License along with this library; if not, write to the Free
+ *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+ *     MA 02110-1301 USA
+ *
+ * Alternatively,
+ *
+ *  b) Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *     1. Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *     2. Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ *     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ *     CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ *     INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ *     MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ *     DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ *     CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *     NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ *     LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ *     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ *     OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ *     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "libfdt_env.h"
+
+#include <fdt.h>
+#include <libfdt.h>
+
+#include "libfdt_internal.h"
+
+int fdt_create_empty_tree(void *buf, int bufsize)
+{
+	int err;
+
+	err = fdt_create(buf, bufsize);
+	if (err)
+		return err;
+
+	err = fdt_finish_reservemap(buf);
+	if (err)
+		return err;
+
+	err = fdt_begin_node(buf, "");
+	if (err)
+		return err;
+
+	err =  fdt_end_node(buf);
+	if (err)
+		return err;
+
+	err = fdt_finish(buf);
+	if (err)
+		return err;
+
+	return fdt_open_into(buf, buf, bufsize);
+}
+
diff --git a/scripts/dtc/libfdt/fdt_ro.c b/scripts/dtc/libfdt/fdt_ro.c
index 22e692919ff..02b6d687537 100644
--- a/scripts/dtc/libfdt/fdt_ro.c
+++ b/scripts/dtc/libfdt/fdt_ro.c
@@ -80,6 +80,14 @@ const char *fdt_string(const void *fdt, int stroffset)
 	return (const char *)fdt + fdt_off_dt_strings(fdt) + stroffset;
 }
 
+static int _fdt_string_eq(const void *fdt, int stroffset,
+			  const char *s, int len)
+{
+	const char *p = fdt_string(fdt, stroffset);
+
+	return (strlen(p) == len) && (memcmp(p, s, len) == 0);
+}
+
 int fdt_get_mem_rsv(const void *fdt, int n, uint64_t *address, uint64_t *size)
 {
 	FDT_CHECK_HEADER(fdt);
@@ -97,6 +105,30 @@ int fdt_num_mem_rsv(const void *fdt)
 	return i;
 }
 
+static int _nextprop(const void *fdt, int offset)
+{
+	uint32_t tag;
+	int nextoffset;
+
+	do {
+		tag = fdt_next_tag(fdt, offset, &nextoffset);
+
+		switch (tag) {
+		case FDT_END:
+			if (nextoffset >= 0)
+				return -FDT_ERR_BADSTRUCTURE;
+			else
+				return nextoffset;
+
+		case FDT_PROP:
+			return offset;
+		}
+		offset = nextoffset;
+	} while (tag == FDT_NOP);
+
+	return -FDT_ERR_NOTFOUND;
+}
+
 int fdt_subnode_offset_namelen(const void *fdt, int offset,
 			       const char *name, int namelen)
 {
@@ -104,20 +136,16 @@ int fdt_subnode_offset_namelen(const void *fdt, int offset,
 
 	FDT_CHECK_HEADER(fdt);
 
-	for (depth = 0, offset = fdt_next_node(fdt, offset, &depth);
-	     (offset >= 0) && (depth > 0);
-	     offset = fdt_next_node(fdt, offset, &depth)) {
-		if (depth < 0)
-			return -FDT_ERR_NOTFOUND;
-		else if ((depth == 1)
-			 && _fdt_nodename_eq(fdt, offset, name, namelen))
+	for (depth = 0;
+	     (offset >= 0) && (depth >= 0);
+	     offset = fdt_next_node(fdt, offset, &depth))
+		if ((depth == 1)
+		    && _fdt_nodename_eq(fdt, offset, name, namelen))
 			return offset;
-	}
 
-	if (offset < 0)
-		return offset; /* error */
-	else
+	if (depth < 0)
 		return -FDT_ERR_NOTFOUND;
+	return offset; /* error */
 }
 
 int fdt_subnode_offset(const void *fdt, int parentoffset,
@@ -134,8 +162,20 @@ int fdt_path_offset(const void *fdt, const char *path)
 
 	FDT_CHECK_HEADER(fdt);
 
-	if (*path != '/')
-		return -FDT_ERR_BADPATH;
+	/* see if we have an alias */
+	if (*path != '/') {
+		const char *q = strchr(path, '/');
+
+		if (!q)
+			q = end;
+
+		p = fdt_get_alias_namelen(fdt, p, q - p);
+		if (!p)
+			return -FDT_ERR_BADPATH;
+		offset = fdt_path_offset(fdt, p);
+
+		p = q;
+	}
 
 	while (*p) {
 		const char *q;
@@ -178,93 +218,142 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *len)
 	return NULL;
 }
 
-const struct fdt_property *fdt_get_property(const void *fdt,
-					    int nodeoffset,
-					    const char *name, int *lenp)
+int fdt_first_property_offset(const void *fdt, int nodeoffset)
+{
+	int offset;
+
+	if ((offset = _fdt_check_node_offset(fdt, nodeoffset)) < 0)
+		return offset;
+
+	return _nextprop(fdt, offset);
+}
+
+int fdt_next_property_offset(const void *fdt, int offset)
+{
+	if ((offset = _fdt_check_prop_offset(fdt, offset)) < 0)
+		return offset;
+
+	return _nextprop(fdt, offset);
+}
+
+const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
+						      int offset,
+						      int *lenp)
 {
-	uint32_t tag;
-	const struct fdt_property *prop;
-	int namestroff;
-	int offset, nextoffset;
 	int err;
+	const struct fdt_property *prop;
 
-	if (((err = fdt_check_header(fdt)) != 0)
-	    || ((err = _fdt_check_node_offset(fdt, nodeoffset)) < 0))
-			goto fail;
+	if ((err = _fdt_check_prop_offset(fdt, offset)) < 0) {
+		if (lenp)
+			*lenp = err;
+		return NULL;
+	}
 
-	nextoffset = err;
-	do {
-		offset = nextoffset;
+	prop = _fdt_offset_ptr(fdt, offset);
 
-		tag = fdt_next_tag(fdt, offset, &nextoffset);
-		switch (tag) {
-		case FDT_END:
-			err = -FDT_ERR_TRUNCATED;
-			goto fail;
+	if (lenp)
+		*lenp = fdt32_to_cpu(prop->len);
 
-		case FDT_BEGIN_NODE:
-		case FDT_END_NODE:
-		case FDT_NOP:
-			break;
+	return prop;
+}
 
-		case FDT_PROP:
-			err = -FDT_ERR_BADSTRUCTURE;
-			prop = fdt_offset_ptr(fdt, offset, sizeof(*prop));
-			if (! prop)
-				goto fail;
-			namestroff = fdt32_to_cpu(prop->nameoff);
-			if (strcmp(fdt_string(fdt, namestroff), name) == 0) {
-				/* Found it! */
-				int len = fdt32_to_cpu(prop->len);
-				prop = fdt_offset_ptr(fdt, offset,
-						      sizeof(*prop)+len);
-				if (! prop)
-					goto fail;
-
-				if (lenp)
-					*lenp = len;
-
-				return prop;
-			}
-			break;
+const struct fdt_property *fdt_get_property_namelen(const void *fdt,
+						    int offset,
+						    const char *name,
+						    int namelen, int *lenp)
+{
+	for (offset = fdt_first_property_offset(fdt, offset);
+	     (offset >= 0);
+	     (offset = fdt_next_property_offset(fdt, offset))) {
+		const struct fdt_property *prop;
 
-		default:
-			err = -FDT_ERR_BADSTRUCTURE;
-			goto fail;
+		if (!(prop = fdt_get_property_by_offset(fdt, offset, lenp))) {
+			offset = -FDT_ERR_INTERNAL;
+			break;
 		}
-	} while ((tag != FDT_BEGIN_NODE) && (tag != FDT_END_NODE));
+		if (_fdt_string_eq(fdt, fdt32_to_cpu(prop->nameoff),
+				   name, namelen))
+			return prop;
+	}
 
-	err = -FDT_ERR_NOTFOUND;
- fail:
 	if (lenp)
-		*lenp = err;
+		*lenp = offset;
 	return NULL;
 }
 
-const void *fdt_getprop(const void *fdt, int nodeoffset,
-		  const char *name, int *lenp)
+const struct fdt_property *fdt_get_property(const void *fdt,
+					    int nodeoffset,
+					    const char *name, int *lenp)
+{
+	return fdt_get_property_namelen(fdt, nodeoffset, name,
+					strlen(name), lenp);
+}
+
+const void *fdt_getprop_namelen(const void *fdt, int nodeoffset,
+				const char *name, int namelen, int *lenp)
 {
 	const struct fdt_property *prop;
 
-	prop = fdt_get_property(fdt, nodeoffset, name, lenp);
+	prop = fdt_get_property_namelen(fdt, nodeoffset, name, namelen, lenp);
 	if (! prop)
 		return NULL;
 
 	return prop->data;
 }
 
+const void *fdt_getprop_by_offset(const void *fdt, int offset,
+				  const char **namep, int *lenp)
+{
+	const struct fdt_property *prop;
+
+	prop = fdt_get_property_by_offset(fdt, offset, lenp);
+	if (!prop)
+		return NULL;
+	if (namep)
+		*namep = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
+	return prop->data;
+}
+
+const void *fdt_getprop(const void *fdt, int nodeoffset,
+			const char *name, int *lenp)
+{
+	return fdt_getprop_namelen(fdt, nodeoffset, name, strlen(name), lenp);
+}
+
 uint32_t fdt_get_phandle(const void *fdt, int nodeoffset)
 {
 	const uint32_t *php;
 	int len;
 
-	php = fdt_getprop(fdt, nodeoffset, "linux,phandle", &len);
-	if (!php || (len != sizeof(*php)))
-		return 0;
+	/* FIXME: This is a bit sub-optimal, since we potentially scan
+	 * over all the properties twice. */
+	php = fdt_getprop(fdt, nodeoffset, "phandle", &len);
+	if (!php || (len != sizeof(*php))) {
+		php = fdt_getprop(fdt, nodeoffset, "linux,phandle", &len);
+		if (!php || (len != sizeof(*php)))
+			return 0;
+	}
 
 	return fdt32_to_cpu(*php);
 }
 
+const char *fdt_get_alias_namelen(const void *fdt,
+				  const char *name, int namelen)
+{
+	int aliasoffset;
+
+	aliasoffset = fdt_path_offset(fdt, "/aliases");
+	if (aliasoffset < 0)
+		return NULL;
+
+	return fdt_getprop_namelen(fdt, aliasoffset, name, namelen, NULL);
+}
+
+const char *fdt_get_alias(const void *fdt, const char *name)
+{
+	return fdt_get_alias_namelen(fdt, name, strlen(name));
+}
+
 int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 {
 	int pdepth = 0, p = 0;
@@ -279,9 +368,6 @@ int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 	for (offset = 0, depth = 0;
 	     (offset >= 0) && (offset <= nodeoffset);
 	     offset = fdt_next_node(fdt, offset, &depth)) {
-		if (pdepth < depth)
-			continue; /* overflowed buffer */
-
 		while (pdepth > depth) {
 			do {
 				p--;
@@ -289,14 +375,16 @@ int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 			pdepth--;
 		}
 
-		name = fdt_get_name(fdt, offset, &namelen);
-		if (!name)
-			return namelen;
-		if ((p + namelen + 1) <= buflen) {
-			memcpy(buf + p, name, namelen);
-			p += namelen;
-			buf[p++] = '/';
-			pdepth++;
+		if (pdepth >= depth) {
+			name = fdt_get_name(fdt, offset, &namelen);
+			if (!name)
+				return namelen;
+			if ((p + namelen + 1) <= buflen) {
+				memcpy(buf + p, name, namelen);
+				p += namelen;
+				buf[p++] = '/';
+				pdepth++;
+			}
 		}
 
 		if (offset == nodeoffset) {
@@ -306,7 +394,7 @@ int fdt_get_path(const void *fdt, int nodeoffset, char *buf, int buflen)
 			if (p > 1) /* special case so that root path is "/", not "" */
 				p--;
 			buf[p] = '\0';
-			return p;
+			return 0;
 		}
 	}
 
@@ -404,14 +492,31 @@ int fdt_node_offset_by_prop_value(const void *fdt, int startoffset,
 
 int fdt_node_offset_by_phandle(const void *fdt, uint32_t phandle)
 {
+	int offset;
+
 	if ((phandle == 0) || (phandle == -1))
 		return -FDT_ERR_BADPHANDLE;
-	phandle = cpu_to_fdt32(phandle);
-	return fdt_node_offset_by_prop_value(fdt, -1, "linux,phandle",
-					     &phandle, sizeof(phandle));
+
+	FDT_CHECK_HEADER(fdt);
+
+	/* FIXME: The algorithm here is pretty horrible: we
+	 * potentially scan each property of a node in
+	 * fdt_get_phandle(), then if that didn't find what
+	 * we want, we scan over them again making our way to the next
+	 * node.  Still it's the easiest to implement approach;
+	 * performance can come later. */
+	for (offset = fdt_next_node(fdt, -1, NULL);
+	     offset >= 0;
+	     offset = fdt_next_node(fdt, offset, NULL)) {
+		if (fdt_get_phandle(fdt, offset) == phandle)
+			return offset;
+	}
+
+	return offset; /* error from fdt_next_node() */
 }
 
-static int _stringlist_contains(const char *strlist, int listlen, const char *str)
+static int _fdt_stringlist_contains(const char *strlist, int listlen,
+				    const char *str)
 {
 	int len = strlen(str);
 	const char *p;
@@ -437,7 +542,7 @@ int fdt_node_check_compatible(const void *fdt, int nodeoffset,
 	prop = fdt_getprop(fdt, nodeoffset, "compatible", &len);
 	if (!prop)
 		return len;
-	if (_stringlist_contains(prop, len, compatible))
+	if (_fdt_stringlist_contains(prop, len, compatible))
 		return 0;
 	else
 		return 1;
diff --git a/scripts/dtc/libfdt/fdt_rw.c b/scripts/dtc/libfdt/fdt_rw.c
index 8e7ec4cb7bc..24437dfc32b 100644
--- a/scripts/dtc/libfdt/fdt_rw.c
+++ b/scripts/dtc/libfdt/fdt_rw.c
@@ -289,6 +289,33 @@ int fdt_setprop(void *fdt, int nodeoffset, const char *name,
 	return 0;
 }
 
+int fdt_appendprop(void *fdt, int nodeoffset, const char *name,
+		   const void *val, int len)
+{
+	struct fdt_property *prop;
+	int err, oldlen, newlen;
+
+	FDT_RW_CHECK_HEADER(fdt);
+
+	prop = fdt_get_property_w(fdt, nodeoffset, name, &oldlen);
+	if (prop) {
+		newlen = len + oldlen;
+		err = _fdt_splice_struct(fdt, prop->data,
+					 FDT_TAGALIGN(oldlen),
+					 FDT_TAGALIGN(newlen));
+		if (err)
+			return err;
+		prop->len = cpu_to_fdt32(newlen);
+		memcpy(prop->data + oldlen, val, len);
+	} else {
+		err = _fdt_add_property(fdt, nodeoffset, name, len, &prop);
+		if (err)
+			return err;
+		memcpy(prop->data, val, len);
+	}
+	return 0;
+}
+
 int fdt_delprop(void *fdt, int nodeoffset, const char *name)
 {
 	struct fdt_property *prop;
@@ -406,6 +433,8 @@ int fdt_open_into(const void *fdt, void *buf, int bufsize)
 		struct_size = 0;
 		while (fdt_next_tag(fdt, struct_size, &struct_size) != FDT_END)
 			;
+		if (struct_size < 0)
+			return struct_size;
 	}
 
 	if (!_fdt_blocks_misordered(fdt, mem_rsv_size, struct_size)) {
diff --git a/scripts/dtc/libfdt/fdt_sw.c b/scripts/dtc/libfdt/fdt_sw.c
index 698329e0cca..55ebebf1eb2 100644
--- a/scripts/dtc/libfdt/fdt_sw.c
+++ b/scripts/dtc/libfdt/fdt_sw.c
@@ -70,7 +70,7 @@ static int _fdt_sw_check_header(void *fdt)
 			return err; \
 	}
 
-static void *_fdt_grab_space(void *fdt, int len)
+static void *_fdt_grab_space(void *fdt, size_t len)
 {
 	int offset = fdt_size_dt_struct(fdt);
 	int spaceleft;
@@ -82,7 +82,7 @@ static void *_fdt_grab_space(void *fdt, int len)
 		return NULL;
 
 	fdt_set_size_dt_struct(fdt, offset + len);
-	return fdt_offset_ptr_w(fdt, offset, len);
+	return _fdt_offset_ptr_w(fdt, offset);
 }
 
 int fdt_create(void *buf, int bufsize)
@@ -237,18 +237,17 @@ int fdt_finish(void *fdt)
 	while ((tag = fdt_next_tag(fdt, offset, &nextoffset)) != FDT_END) {
 		if (tag == FDT_PROP) {
 			struct fdt_property *prop =
-				fdt_offset_ptr_w(fdt, offset, sizeof(*prop));
+				_fdt_offset_ptr_w(fdt, offset);
 			int nameoff;
 
-			if (! prop)
-				return -FDT_ERR_BADSTRUCTURE;
-
 			nameoff = fdt32_to_cpu(prop->nameoff);
 			nameoff += fdt_size_dt_strings(fdt);
 			prop->nameoff = cpu_to_fdt32(nameoff);
 		}
 		offset = nextoffset;
 	}
+	if (nextoffset < 0)
+		return nextoffset;
 
 	/* Finally, adjust the header */
 	fdt_set_totalsize(fdt, newstroffset + fdt_size_dt_strings(fdt));
diff --git a/scripts/dtc/libfdt/fdt_wip.c b/scripts/dtc/libfdt/fdt_wip.c
index a4652c6e787..6025fa1fe8f 100644
--- a/scripts/dtc/libfdt/fdt_wip.c
+++ b/scripts/dtc/libfdt/fdt_wip.c
@@ -94,41 +94,14 @@ int fdt_nop_property(void *fdt, int nodeoffset, const char *name)
 	return 0;
 }
 
-int _fdt_node_end_offset(void *fdt, int nodeoffset)
+int _fdt_node_end_offset(void *fdt, int offset)
 {
-	int level = 0;
-	uint32_t tag;
-	int offset, nextoffset;
-
-	tag = fdt_next_tag(fdt, nodeoffset, &nextoffset);
-	if (tag != FDT_BEGIN_NODE)
-		return -FDT_ERR_BADOFFSET;
-	do {
-		offset = nextoffset;
-		tag = fdt_next_tag(fdt, offset, &nextoffset);
-
-		switch (tag) {
-		case FDT_END:
-			return offset;
-
-		case FDT_BEGIN_NODE:
-			level++;
-			break;
-
-		case FDT_END_NODE:
-			level--;
-			break;
-
-		case FDT_PROP:
-		case FDT_NOP:
-			break;
-
-		default:
-			return -FDT_ERR_BADSTRUCTURE;
-		}
-	} while (level >= 0);
-
-	return nextoffset;
+	int depth = 0;
+
+	while ((offset >= 0) && (depth >= 0))
+		offset = fdt_next_node(fdt, offset, &depth);
+
+	return offset;
 }
 
 int fdt_nop_node(void *fdt, int nodeoffset)
diff --git a/scripts/dtc/libfdt/libfdt.h b/scripts/dtc/libfdt/libfdt.h
index ff6246f000c..73f49759a5e 100644
--- a/scripts/dtc/libfdt/libfdt.h
+++ b/scripts/dtc/libfdt/libfdt.h
@@ -61,7 +61,7 @@
 #define FDT_ERR_NOTFOUND	1
 	/* FDT_ERR_NOTFOUND: The requested node or property does not exist */
 #define FDT_ERR_EXISTS		2
-	/* FDT_ERR_EXISTS: Attempted to create a node or property which
+	/* FDT_ERR_EXISTS: Attemped to create a node or property which
 	 * already exists */
 #define FDT_ERR_NOSPACE		3
 	/* FDT_ERR_NOSPACE: Operation needed to expand the device
@@ -122,7 +122,7 @@
 /* Low-level functions (you probably don't need these)                */
 /**********************************************************************/
 
-const void *fdt_offset_ptr(const void *fdt, int offset, int checklen);
+const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int checklen);
 static inline void *fdt_offset_ptr_w(void *fdt, int offset, int checklen)
 {
 	return (void *)(uintptr_t)fdt_offset_ptr(fdt, offset, checklen);
@@ -156,7 +156,7 @@ int fdt_next_node(const void *fdt, int offset, int *depth);
 #define __fdt_set_hdr(name) \
 	static inline void fdt_set_##name(void *fdt, uint32_t val) \
 	{ \
-		struct fdt_header *fdth = fdt; \
+		struct fdt_header *fdth = (struct fdt_header*)fdt; \
 		fdth->name = cpu_to_fdt32(val); \
 	}
 __fdt_set_hdr(magic);
@@ -343,6 +343,91 @@ int fdt_path_offset(const void *fdt, const char *path);
 const char *fdt_get_name(const void *fdt, int nodeoffset, int *lenp);
 
 /**
+ * fdt_first_property_offset - find the offset of a node's first property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: structure block offset of a node
+ *
+ * fdt_first_property_offset() finds the first property of the node at
+ * the given structure block offset.
+ *
+ * returns:
+ *	structure block offset of the property (>=0), on success
+ *	-FDT_ERR_NOTFOUND, if the requested node has no properties
+ *	-FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_BEGIN_NODE tag
+ *      -FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_TRUNCATED, standard meanings.
+ */
+int fdt_first_property_offset(const void *fdt, int nodeoffset);
+
+/**
+ * fdt_next_property_offset - step through a node's properties
+ * @fdt: pointer to the device tree blob
+ * @offset: structure block offset of a property
+ *
+ * fdt_next_property_offset() finds the property immediately after the
+ * one at the given structure block offset.  This will be a property
+ * of the same node as the given property.
+ *
+ * returns:
+ *	structure block offset of the next property (>=0), on success
+ *	-FDT_ERR_NOTFOUND, if the given property is the last in its node
+ *	-FDT_ERR_BADOFFSET, if nodeoffset did not point to an FDT_PROP tag
+ *      -FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_TRUNCATED, standard meanings.
+ */
+int fdt_next_property_offset(const void *fdt, int offset);
+
+/**
+ * fdt_get_property_by_offset - retrieve the property at a given offset
+ * @fdt: pointer to the device tree blob
+ * @offset: offset of the property to retrieve
+ * @lenp: pointer to an integer variable (will be overwritten) or NULL
+ *
+ * fdt_get_property_by_offset() retrieves a pointer to the
+ * fdt_property structure within the device tree blob at the given
+ * offset.  If lenp is non-NULL, the length of the property value is
+ * also returned, in the integer pointed to by lenp.
+ *
+ * returns:
+ *	pointer to the structure representing the property
+ *		if lenp is non-NULL, *lenp contains the length of the property
+ *		value (>=0)
+ *	NULL, on error
+ *		if lenp is non-NULL, *lenp contains an error code (<0):
+ *		-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_PROP tag
+ *		-FDT_ERR_BADMAGIC,
+ *		-FDT_ERR_BADVERSION,
+ *		-FDT_ERR_BADSTATE,
+ *		-FDT_ERR_BADSTRUCTURE,
+ *		-FDT_ERR_TRUNCATED, standard meanings
+ */
+const struct fdt_property *fdt_get_property_by_offset(const void *fdt,
+						      int offset,
+						      int *lenp);
+
+/**
+ * fdt_get_property_namelen - find a property based on substring
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to find
+ * @name: name of the property to find
+ * @namelen: number of characters of name to consider
+ * @lenp: pointer to an integer variable (will be overwritten) or NULL
+ *
+ * Identical to fdt_get_property_namelen(), but only examine the first
+ * namelen characters of name for matching the property name.
+ */
+const struct fdt_property *fdt_get_property_namelen(const void *fdt,
+						    int nodeoffset,
+						    const char *name,
+						    int namelen, int *lenp);
+
+/**
  * fdt_get_property - find a given property in a given node
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to find
@@ -380,6 +465,54 @@ static inline struct fdt_property *fdt_get_property_w(void *fdt, int nodeoffset,
 }
 
 /**
+ * fdt_getprop_by_offset - retrieve the value of a property at a given offset
+ * @fdt: pointer to the device tree blob
+ * @ffset: offset of the property to read
+ * @namep: pointer to a string variable (will be overwritten) or NULL
+ * @lenp: pointer to an integer variable (will be overwritten) or NULL
+ *
+ * fdt_getprop_by_offset() retrieves a pointer to the value of the
+ * property at structure block offset 'offset' (this will be a pointer
+ * to within the device blob itself, not a copy of the value).  If
+ * lenp is non-NULL, the length of the property value is also
+ * returned, in the integer pointed to by lenp.  If namep is non-NULL,
+ * the property's namne will also be returned in the char * pointed to
+ * by namep (this will be a pointer to within the device tree's string
+ * block, not a new copy of the name).
+ *
+ * returns:
+ *	pointer to the property's value
+ *		if lenp is non-NULL, *lenp contains the length of the property
+ *		value (>=0)
+ *		if namep is non-NULL *namep contiains a pointer to the property
+ *		name.
+ *	NULL, on error
+ *		if lenp is non-NULL, *lenp contains an error code (<0):
+ *		-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_PROP tag
+ *		-FDT_ERR_BADMAGIC,
+ *		-FDT_ERR_BADVERSION,
+ *		-FDT_ERR_BADSTATE,
+ *		-FDT_ERR_BADSTRUCTURE,
+ *		-FDT_ERR_TRUNCATED, standard meanings
+ */
+const void *fdt_getprop_by_offset(const void *fdt, int offset,
+				  const char **namep, int *lenp);
+
+/**
+ * fdt_getprop_namelen - get property value based on substring
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to find
+ * @name: name of the property to find
+ * @namelen: number of characters of name to consider
+ * @lenp: pointer to an integer variable (will be overwritten) or NULL
+ *
+ * Identical to fdt_getprop(), but only examine the first namelen
+ * characters of name for matching the property name.
+ */
+const void *fdt_getprop_namelen(const void *fdt, int nodeoffset,
+				const char *name, int namelen, int *lenp);
+
+/**
  * fdt_getprop - retrieve the value of a given property
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to find
@@ -429,6 +562,32 @@ static inline void *fdt_getprop_w(void *fdt, int nodeoffset,
 uint32_t fdt_get_phandle(const void *fdt, int nodeoffset);
 
 /**
+ * fdt_get_alias_namelen - get alias based on substring
+ * @fdt: pointer to the device tree blob
+ * @name: name of the alias th look up
+ * @namelen: number of characters of name to consider
+ *
+ * Identical to fdt_get_alias(), but only examine the first namelen
+ * characters of name for matching the alias name.
+ */
+const char *fdt_get_alias_namelen(const void *fdt,
+				  const char *name, int namelen);
+
+/**
+ * fdt_get_alias - retreive the path referenced by a given alias
+ * @fdt: pointer to the device tree blob
+ * @name: name of the alias th look up
+ *
+ * fdt_get_alias() retrieves the value of a given alias.  That is, the
+ * value of the property named 'name' in the node /aliases.
+ *
+ * returns:
+ *	a pointer to the expansion of the alias named 'name', of it exists
+ *	NULL, if the given alias or the /aliases node does not exist
+ */
+const char *fdt_get_alias(const void *fdt, const char *name);
+
+/**
  * fdt_get_path - determine the full path of a node
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose path to find
@@ -693,17 +852,17 @@ int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name,
 			const void *val, int len);
 
 /**
- * fdt_setprop_inplace_cell - change the value of a single-cell property
+ * fdt_setprop_inplace_u32 - change the value of a 32-bit integer property
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to change
  * @name: name of the property to change
- * @val: cell (32-bit integer) value to replace the property with
+ * @val: 32-bit integer value to replace the property with
  *
- * fdt_setprop_inplace_cell() replaces the value of a given property
- * with the 32-bit integer cell value in val, converting val to
- * big-endian if necessary.  This function cannot change the size of a
- * property, and so will only work if the property already exists and
- * has length 4.
+ * fdt_setprop_inplace_u32() replaces the value of a given property
+ * with the 32-bit integer value in val, converting val to big-endian
+ * if necessary.  This function cannot change the size of a property,
+ * and so will only work if the property already exists and has length
+ * 4.
  *
  * This function will alter only the bytes in the blob which contain
  * the given property value, and will not alter or move any other part
@@ -712,7 +871,7 @@ int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name,
  * returns:
  *	0, on success
  *	-FDT_ERR_NOSPACE, if the property's length is not equal to 4
-  *	-FDT_ERR_NOTFOUND, node does not have the named property
+ *	-FDT_ERR_NOTFOUND, node does not have the named property
  *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
  *	-FDT_ERR_BADMAGIC,
  *	-FDT_ERR_BADVERSION,
@@ -720,14 +879,60 @@ int fdt_setprop_inplace(void *fdt, int nodeoffset, const char *name,
  *	-FDT_ERR_BADSTRUCTURE,
  *	-FDT_ERR_TRUNCATED, standard meanings
  */
-static inline int fdt_setprop_inplace_cell(void *fdt, int nodeoffset,
-					   const char *name, uint32_t val)
+static inline int fdt_setprop_inplace_u32(void *fdt, int nodeoffset,
+					  const char *name, uint32_t val)
 {
 	val = cpu_to_fdt32(val);
 	return fdt_setprop_inplace(fdt, nodeoffset, name, &val, sizeof(val));
 }
 
 /**
+ * fdt_setprop_inplace_u64 - change the value of a 64-bit integer property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to change
+ * @val: 64-bit integer value to replace the property with
+ *
+ * fdt_setprop_inplace_u64() replaces the value of a given property
+ * with the 64-bit integer value in val, converting val to big-endian
+ * if necessary.  This function cannot change the size of a property,
+ * and so will only work if the property already exists and has length
+ * 8.
+ *
+ * This function will alter only the bytes in the blob which contain
+ * the given property value, and will not alter or move any other part
+ * of the tree.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, if the property's length is not equal to 8
+ *	-FDT_ERR_NOTFOUND, node does not have the named property
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+static inline int fdt_setprop_inplace_u64(void *fdt, int nodeoffset,
+					  const char *name, uint64_t val)
+{
+	val = cpu_to_fdt64(val);
+	return fdt_setprop_inplace(fdt, nodeoffset, name, &val, sizeof(val));
+}
+
+/**
+ * fdt_setprop_inplace_cell - change the value of a single-cell property
+ *
+ * This is an alternative name for fdt_setprop_inplace_u32()
+ */
+static inline int fdt_setprop_inplace_cell(void *fdt, int nodeoffset,
+					   const char *name, uint32_t val)
+{
+	return fdt_setprop_inplace_u32(fdt, nodeoffset, name, val);
+}
+
+/**
  * fdt_nop_property - replace a property with nop tags
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to nop
@@ -786,11 +991,20 @@ int fdt_add_reservemap_entry(void *fdt, uint64_t addr, uint64_t size);
 int fdt_finish_reservemap(void *fdt);
 int fdt_begin_node(void *fdt, const char *name);
 int fdt_property(void *fdt, const char *name, const void *val, int len);
-static inline int fdt_property_cell(void *fdt, const char *name, uint32_t val)
+static inline int fdt_property_u32(void *fdt, const char *name, uint32_t val)
 {
 	val = cpu_to_fdt32(val);
 	return fdt_property(fdt, name, &val, sizeof(val));
 }
+static inline int fdt_property_u64(void *fdt, const char *name, uint64_t val)
+{
+	val = cpu_to_fdt64(val);
+	return fdt_property(fdt, name, &val, sizeof(val));
+}
+static inline int fdt_property_cell(void *fdt, const char *name, uint32_t val)
+{
+	return fdt_property_u32(fdt, name, val);
+}
 #define fdt_property_string(fdt, name, str) \
 	fdt_property(fdt, name, str, strlen(str)+1)
 int fdt_end_node(void *fdt);
@@ -800,6 +1014,7 @@ int fdt_finish(void *fdt);
 /* Read-write functions                                               */
 /**********************************************************************/
 
+int fdt_create_empty_tree(void *buf, int bufsize);
 int fdt_open_into(const void *fdt, void *buf, int bufsize);
 int fdt_pack(void *fdt);
 
@@ -909,14 +1124,14 @@ int fdt_setprop(void *fdt, int nodeoffset, const char *name,
 		const void *val, int len);
 
 /**
- * fdt_setprop_cell - set a property to a single cell value
+ * fdt_setprop_u32 - set a property to a 32-bit integer
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to change
  * @name: name of the property to change
  * @val: 32-bit integer value for the property (native endian)
  *
- * fdt_setprop_cell() sets the value of the named property in the
- * given node to the given cell value (converting to big-endian if
+ * fdt_setprop_u32() sets the value of the named property in the given
+ * node to the given 32-bit integer value (converting to big-endian if
  * necessary), or creates a new property with that value if it does
  * not already exist.
  *
@@ -936,14 +1151,60 @@ int fdt_setprop(void *fdt, int nodeoffset, const char *name,
  *	-FDT_ERR_BADLAYOUT,
  *	-FDT_ERR_TRUNCATED, standard meanings
  */
-static inline int fdt_setprop_cell(void *fdt, int nodeoffset, const char *name,
-				   uint32_t val)
+static inline int fdt_setprop_u32(void *fdt, int nodeoffset, const char *name,
+				  uint32_t val)
 {
 	val = cpu_to_fdt32(val);
 	return fdt_setprop(fdt, nodeoffset, name, &val, sizeof(val));
 }
 
 /**
+ * fdt_setprop_u64 - set a property to a 64-bit integer
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to change
+ * @val: 64-bit integer value for the property (native endian)
+ *
+ * fdt_setprop_u64() sets the value of the named property in the given
+ * node to the given 64-bit integer value (converting to big-endian if
+ * necessary), or creates a new property with that value if it does
+ * not already exist.
+ *
+ * This function may insert or delete data from the blob, and will
+ * therefore change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain the new property value
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+static inline int fdt_setprop_u64(void *fdt, int nodeoffset, const char *name,
+				  uint64_t val)
+{
+	val = cpu_to_fdt64(val);
+	return fdt_setprop(fdt, nodeoffset, name, &val, sizeof(val));
+}
+
+/**
+ * fdt_setprop_cell - set a property to a single cell value
+ *
+ * This is an alternative name for fdt_setprop_u32()
+ */
+static inline int fdt_setprop_cell(void *fdt, int nodeoffset, const char *name,
+				   uint32_t val)
+{
+	return fdt_setprop_u32(fdt, nodeoffset, name, val);
+}
+
+/**
  * fdt_setprop_string - set a property to a string value
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to change
@@ -975,6 +1236,147 @@ static inline int fdt_setprop_cell(void *fdt, int nodeoffset, const char *name,
 	fdt_setprop((fdt), (nodeoffset), (name), (str), strlen(str)+1)
 
 /**
+ * fdt_appendprop - append to or create a property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to append to
+ * @val: pointer to data to append to the property value
+ * @len: length of the data to append to the property value
+ *
+ * fdt_appendprop() appends the value to the named property in the
+ * given node, creating the property if it does not already exist.
+ *
+ * This function may insert data into the blob, and will therefore
+ * change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain the new property value
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+int fdt_appendprop(void *fdt, int nodeoffset, const char *name,
+		   const void *val, int len);
+
+/**
+ * fdt_appendprop_u32 - append a 32-bit integer value to a property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to change
+ * @val: 32-bit integer value to append to the property (native endian)
+ *
+ * fdt_appendprop_u32() appends the given 32-bit integer value
+ * (converting to big-endian if necessary) to the value of the named
+ * property in the given node, or creates a new property with that
+ * value if it does not already exist.
+ *
+ * This function may insert data into the blob, and will therefore
+ * change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain the new property value
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+static inline int fdt_appendprop_u32(void *fdt, int nodeoffset,
+				     const char *name, uint32_t val)
+{
+	val = cpu_to_fdt32(val);
+	return fdt_appendprop(fdt, nodeoffset, name, &val, sizeof(val));
+}
+
+/**
+ * fdt_appendprop_u64 - append a 64-bit integer value to a property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to change
+ * @val: 64-bit integer value to append to the property (native endian)
+ *
+ * fdt_appendprop_u64() appends the given 64-bit integer value
+ * (converting to big-endian if necessary) to the value of the named
+ * property in the given node, or creates a new property with that
+ * value if it does not already exist.
+ *
+ * This function may insert data into the blob, and will therefore
+ * change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain the new property value
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+static inline int fdt_appendprop_u64(void *fdt, int nodeoffset,
+				     const char *name, uint64_t val)
+{
+	val = cpu_to_fdt64(val);
+	return fdt_appendprop(fdt, nodeoffset, name, &val, sizeof(val));
+}
+
+/**
+ * fdt_appendprop_cell - append a single cell value to a property
+ *
+ * This is an alternative name for fdt_appendprop_u32()
+ */
+static inline int fdt_appendprop_cell(void *fdt, int nodeoffset,
+				      const char *name, uint32_t val)
+{
+	return fdt_appendprop_u32(fdt, nodeoffset, name, val);
+}
+
+/**
+ * fdt_appendprop_string - append a string to a property
+ * @fdt: pointer to the device tree blob
+ * @nodeoffset: offset of the node whose property to change
+ * @name: name of the property to change
+ * @str: string value to append to the property
+ *
+ * fdt_appendprop_string() appends the given string to the value of
+ * the named property in the given node, or creates a new property
+ * with that value if it does not already exist.
+ *
+ * This function may insert data into the blob, and will therefore
+ * change the offsets of some existing nodes.
+ *
+ * returns:
+ *	0, on success
+ *	-FDT_ERR_NOSPACE, there is insufficient free space in the blob to
+ *		contain the new property value
+ *	-FDT_ERR_BADOFFSET, nodeoffset did not point to FDT_BEGIN_NODE tag
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_BADMAGIC,
+ *	-FDT_ERR_BADVERSION,
+ *	-FDT_ERR_BADSTATE,
+ *	-FDT_ERR_BADSTRUCTURE,
+ *	-FDT_ERR_BADLAYOUT,
+ *	-FDT_ERR_TRUNCATED, standard meanings
+ */
+#define fdt_appendprop_string(fdt, nodeoffset, name, str) \
+	fdt_appendprop((fdt), (nodeoffset), (name), (str), strlen(str)+1)
+
+/**
  * fdt_delprop - delete a property
  * @fdt: pointer to the device tree blob
  * @nodeoffset: offset of the node whose property to nop
diff --git a/scripts/dtc/libfdt/libfdt_env.h b/scripts/dtc/libfdt/libfdt_env.h
index 449bf602daf..213d7fb81c4 100644
--- a/scripts/dtc/libfdt/libfdt_env.h
+++ b/scripts/dtc/libfdt/libfdt_env.h
@@ -5,19 +5,25 @@
 #include <stdint.h>
 #include <string.h>
 
-#define _B(n)	((unsigned long long)((uint8_t *)&x)[n])
+#define EXTRACT_BYTE(n)	((unsigned long long)((uint8_t *)&x)[n])
+static inline uint16_t fdt16_to_cpu(uint16_t x)
+{
+	return (EXTRACT_BYTE(0) << 8) | EXTRACT_BYTE(1);
+}
+#define cpu_to_fdt16(x) fdt16_to_cpu(x)
+
 static inline uint32_t fdt32_to_cpu(uint32_t x)
 {
-	return (_B(0) << 24) | (_B(1) << 16) | (_B(2) << 8) | _B(3);
+	return (EXTRACT_BYTE(0) << 24) | (EXTRACT_BYTE(1) << 16) | (EXTRACT_BYTE(2) << 8) | EXTRACT_BYTE(3);
 }
 #define cpu_to_fdt32(x) fdt32_to_cpu(x)
 
 static inline uint64_t fdt64_to_cpu(uint64_t x)
 {
-	return (_B(0) << 56) | (_B(1) << 48) | (_B(2) << 40) | (_B(3) << 32)
-		| (_B(4) << 24) | (_B(5) << 16) | (_B(6) << 8) | _B(7);
+	return (EXTRACT_BYTE(0) << 56) | (EXTRACT_BYTE(1) << 48) | (EXTRACT_BYTE(2) << 40) | (EXTRACT_BYTE(3) << 32)
+		| (EXTRACT_BYTE(4) << 24) | (EXTRACT_BYTE(5) << 16) | (EXTRACT_BYTE(6) << 8) | EXTRACT_BYTE(7);
 }
 #define cpu_to_fdt64(x) fdt64_to_cpu(x)
-#undef _B
+#undef EXTRACT_BYTE
 
 #endif /* _LIBFDT_ENV_H */
diff --git a/scripts/dtc/libfdt/libfdt_internal.h b/scripts/dtc/libfdt/libfdt_internal.h
index 46eb93e4af5..381133ba81d 100644
--- a/scripts/dtc/libfdt/libfdt_internal.h
+++ b/scripts/dtc/libfdt/libfdt_internal.h
@@ -62,8 +62,8 @@
 			return err; \
 	}
 
-uint32_t _fdt_next_tag(const void *fdt, int startoffset, int *nextoffset);
 int _fdt_check_node_offset(const void *fdt, int offset);
+int _fdt_check_prop_offset(const void *fdt, int offset);
 const char *_fdt_find_string(const char *strtab, int tabsize, const char *s);
 int _fdt_node_end_offset(void *fdt, int nodeoffset);
 
diff --git a/scripts/dtc/livetree.c b/scripts/dtc/livetree.c
index 26d0e1e60c0..b61465fb2f3 100644
--- a/scripts/dtc/livetree.c
+++ b/scripts/dtc/livetree.c
@@ -29,16 +29,27 @@ void add_label(struct label **labels, char *label)
 	struct label *new;
 
 	/* Make sure the label isn't already there */
-	for_each_label(*labels, new)
-		if (streq(new->label, label))
+	for_each_label_withdel(*labels, new)
+		if (streq(new->label, label)) {
+			new->deleted = 0;
 			return;
+		}
 
 	new = xmalloc(sizeof(*new));
+	memset(new, 0, sizeof(*new));
 	new->label = label;
 	new->next = *labels;
 	*labels = new;
 }
 
+void delete_labels(struct label **labels)
+{
+	struct label *label;
+
+	for_each_label(*labels, label)
+		label->deleted = 1;
+}
+
 struct property *build_property(char *name, struct data val)
 {
 	struct property *new = xmalloc(sizeof(*new));
@@ -51,6 +62,18 @@ struct property *build_property(char *name, struct data val)
 	return new;
 }
 
+struct property *build_property_delete(char *name)
+{
+	struct property *new = xmalloc(sizeof(*new));
+
+	memset(new, 0, sizeof(*new));
+
+	new->name = name;
+	new->deleted = 1;
+
+	return new;
+}
+
 struct property *chain_property(struct property *first, struct property *list)
 {
 	assert(first->next == NULL);
@@ -91,6 +114,17 @@ struct node *build_node(struct property *proplist, struct node *children)
 	return new;
 }
 
+struct node *build_node_delete(void)
+{
+	struct node *new = xmalloc(sizeof(*new));
+
+	memset(new, 0, sizeof(*new));
+
+	new->deleted = 1;
+
+	return new;
+}
+
 struct node *name_node(struct node *node, char *name)
 {
 	assert(node->name == NULL);
@@ -106,8 +140,10 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
 	struct node *new_child, *old_child;
 	struct label *l;
 
+	old_node->deleted = 0;
+
 	/* Add new node labels to old node */
-	for_each_label(new_node->labels, l)
+	for_each_label_withdel(new_node->labels, l)
 		add_label(&old_node->labels, l->label);
 
 	/* Move properties from the new node to the old node.  If there
@@ -118,14 +154,21 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
 		new_node->proplist = new_prop->next;
 		new_prop->next = NULL;
 
+		if (new_prop->deleted) {
+			delete_property_by_name(old_node, new_prop->name);
+			free(new_prop);
+			continue;
+		}
+
 		/* Look for a collision, set new value if there is */
-		for_each_property(old_node, old_prop) {
+		for_each_property_withdel(old_node, old_prop) {
 			if (streq(old_prop->name, new_prop->name)) {
 				/* Add new labels to old property */
-				for_each_label(new_prop->labels, l)
+				for_each_label_withdel(new_prop->labels, l)
 					add_label(&old_prop->labels, l->label);
 
 				old_prop->val = new_prop->val;
+				old_prop->deleted = 0;
 				free(new_prop);
 				new_prop = NULL;
 				break;
@@ -146,8 +189,14 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
 		new_child->parent = NULL;
 		new_child->next_sibling = NULL;
 
+		if (new_child->deleted) {
+			delete_node_by_name(old_node, new_child->name);
+			free(new_child);
+			continue;
+		}
+
 		/* Search for a collision.  Merge if there is */
-		for_each_child(old_node, old_child) {
+		for_each_child_withdel(old_node, old_child) {
 			if (streq(old_child->name, new_child->name)) {
 				merge_nodes(old_child, new_child);
 				new_child = NULL;
@@ -155,7 +204,7 @@ struct node *merge_nodes(struct node *old_node, struct node *new_node)
 			}
 		}
 
-		/* if no collision occurred, add child to the old node. */
+		/* if no collision occured, add child to the old node. */
 		if (new_child)
 			add_child(old_node, new_child);
 	}
@@ -188,6 +237,25 @@ void add_property(struct node *node, struct property *prop)
 	*p = prop;
 }
 
+void delete_property_by_name(struct node *node, char *name)
+{
+	struct property *prop = node->proplist;
+
+	while (prop) {
+		if (!strcmp(prop->name, name)) {
+			delete_property(prop);
+			return;
+		}
+		prop = prop->next;
+	}
+}
+
+void delete_property(struct property *prop)
+{
+	prop->deleted = 1;
+	delete_labels(&prop->labels);
+}
+
 void add_child(struct node *parent, struct node *child)
 {
 	struct node **p;
@@ -202,6 +270,32 @@ void add_child(struct node *parent, struct node *child)
 	*p = child;
 }
 
+void delete_node_by_name(struct node *parent, char *name)
+{
+	struct node *node = parent->children;
+
+	while (node) {
+		if (!strcmp(node->name, name)) {
+			delete_node(node);
+			return;
+		}
+		node = node->next_sibling;
+	}
+}
+
+void delete_node(struct node *node)
+{
+	struct property *prop;
+	struct node *child;
+
+	node->deleted = 1;
+	for_each_child(node, child)
+		delete_node(child);
+	for_each_property(node, prop)
+		delete_property(prop);
+	delete_labels(&node->labels);
+}
+
 struct reserve_info *build_reserve_entry(uint64_t address, uint64_t size)
 {
 	struct reserve_info *new = xmalloc(sizeof(*new));
@@ -353,8 +447,11 @@ struct node *get_node_by_path(struct node *tree, const char *path)
 	const char *p;
 	struct node *child;
 
-	if (!path || ! (*path))
+	if (!path || ! (*path)) {
+		if (tree->deleted)
+			return NULL;
 		return tree;
+	}
 
 	while (path[0] == '/')
 		path++;
@@ -397,8 +494,11 @@ struct node *get_node_by_phandle(struct node *tree, cell_t phandle)
 
 	assert((phandle != 0) && (phandle != -1));
 
-	if (tree->phandle == phandle)
+	if (tree->phandle == phandle) {
+		if (tree->deleted)
+			return NULL;
 		return tree;
+	}
 
 	for_each_child(tree, child) {
 		node = get_node_by_phandle(child, phandle);
@@ -535,7 +635,7 @@ static void sort_properties(struct node *node)
 	int n = 0, i = 0;
 	struct property *prop, **tbl;
 
-	for_each_property(node, prop)
+	for_each_property_withdel(node, prop)
 		n++;
 
 	if (n == 0)
@@ -543,7 +643,7 @@ static void sort_properties(struct node *node)
 
 	tbl = xmalloc(n * sizeof(*tbl));
 
-	for_each_property(node, prop)
+	for_each_property_withdel(node, prop)
 		tbl[i++] = prop;
 
 	qsort(tbl, n, sizeof(*tbl), cmp_prop);
@@ -571,7 +671,7 @@ static void sort_subnodes(struct node *node)
 	int n = 0, i = 0;
 	struct node *subnode, **tbl;
 
-	for_each_child(node, subnode)
+	for_each_child_withdel(node, subnode)
 		n++;
 
 	if (n == 0)
@@ -579,7 +679,7 @@ static void sort_subnodes(struct node *node)
 
 	tbl = xmalloc(n * sizeof(*tbl));
 
-	for_each_child(node, subnode)
+	for_each_child_withdel(node, subnode)
 		tbl[i++] = subnode;
 
 	qsort(tbl, n, sizeof(*tbl), cmp_subnode);
@@ -598,7 +698,7 @@ static void sort_node(struct node *node)
 
 	sort_properties(node);
 	sort_subnodes(node);
-	for_each_child(node, c)
+	for_each_child_withdel(node, c)
 		sort_node(c);
 }
 
diff --git a/scripts/dtc/srcpos.c b/scripts/dtc/srcpos.c
index 36a38e9f1a2..246ab4bc0d9 100644
--- a/scripts/dtc/srcpos.c
+++ b/scripts/dtc/srcpos.c
@@ -24,6 +24,15 @@
 #include "dtc.h"
 #include "srcpos.h"
 
+/* A node in our list of directories to search for source/include files */
+struct search_path {
+	struct search_path *next;	/* next node in list, NULL for end */
+	const char *dirname;		/* name of directory to search */
+};
+
+/* This is the list of directories that we search for source files */
+static struct search_path *search_path_head, **search_path_tail;
+
 
 static char *dirname(const char *path)
 {
@@ -47,6 +56,64 @@ struct srcfile_state *current_srcfile; /* = NULL */
 #define MAX_SRCFILE_DEPTH     (100)
 static int srcfile_depth; /* = 0 */
 
+
+/**
+ * Try to open a file in a given directory.
+ *
+ * If the filename is an absolute path, then dirname is ignored. If it is a
+ * relative path, then we look in that directory for the file.
+ *
+ * @param dirname	Directory to look in, or NULL for none
+ * @param fname		Filename to look for
+ * @param fp		Set to NULL if file did not open
+ * @return allocated filename on success (caller must free), NULL on failure
+ */
+static char *try_open(const char *dirname, const char *fname, FILE **fp)
+{
+	char *fullname;
+
+	if (!dirname || fname[0] == '/')
+		fullname = xstrdup(fname);
+	else
+		fullname = join_path(dirname, fname);
+
+	*fp = fopen(fullname, "r");
+	if (!*fp) {
+		free(fullname);
+		fullname = NULL;
+	}
+
+	return fullname;
+}
+
+/**
+ * Open a file for read access
+ *
+ * If it is a relative filename, we search the full search path for it.
+ *
+ * @param fname	Filename to open
+ * @param fp	Returns pointer to opened FILE, or NULL on failure
+ * @return pointer to allocated filename, which caller must free
+ */
+static char *fopen_any_on_path(const char *fname, FILE **fp)
+{
+	const char *cur_dir = NULL;
+	struct search_path *node;
+	char *fullname;
+
+	/* Try current directory first */
+	assert(fp);
+	if (current_srcfile)
+		cur_dir = current_srcfile->dir;
+	fullname = try_open(cur_dir, fname, fp);
+
+	/* Failing that, try each search path in turn */
+	for (node = search_path_head; !*fp && node; node = node->next)
+		fullname = try_open(node->dirname, fname, fp);
+
+	return fullname;
+}
+
 FILE *srcfile_relative_open(const char *fname, char **fullnamep)
 {
 	FILE *f;
@@ -56,13 +123,7 @@ FILE *srcfile_relative_open(const char *fname, char **fullnamep)
 		f = stdin;
 		fullname = xstrdup("<stdin>");
 	} else {
-		if (!current_srcfile || !current_srcfile->dir
-		    || (fname[0] == '/'))
-			fullname = xstrdup(fname);
-		else
-			fullname = join_path(current_srcfile->dir, fname);
-
-		f = fopen(fullname, "r");
+		fullname = fopen_any_on_path(fname, &f);
 		if (!f)
 			die("Couldn't open \"%s\": %s\n", fname,
 			    strerror(errno));
@@ -119,6 +180,23 @@ int srcfile_pop(void)
 	return current_srcfile ? 1 : 0;
 }
 
+void srcfile_add_search_path(const char *dirname)
+{
+	struct search_path *node;
+
+	/* Create the node */
+	node = xmalloc(sizeof(*node));
+	node->next = NULL;
+	node->dirname = xstrdup(dirname);
+
+	/* Add to the end of our list */
+	if (search_path_tail)
+		*search_path_tail = node;
+	else
+		search_path_head = node;
+	search_path_tail = &node->next;
+}
+
 /*
  * The empty source position.
  */
@@ -250,3 +328,9 @@ srcpos_warn(struct srcpos *pos, char const *fmt, ...)
 
 	va_end(va);
 }
+
+void srcpos_set_line(char *f, int l)
+{
+	current_srcfile->name = f;
+	current_srcfile->lineno = l;
+}
diff --git a/scripts/dtc/srcpos.h b/scripts/dtc/srcpos.h
index ce980cafe58..93a27123c2e 100644
--- a/scripts/dtc/srcpos.h
+++ b/scripts/dtc/srcpos.h
@@ -33,10 +33,39 @@ struct srcfile_state {
 extern FILE *depfile; /* = NULL */
 extern struct srcfile_state *current_srcfile; /* = NULL */
 
+/**
+ * Open a source file.
+ *
+ * If the source file is a relative pathname, then it is searched for in the
+ * current directory (the directory of the last source file read) and after
+ * that in the search path.
+ *
+ * We work through the search path in order from the first path specified to
+ * the last.
+ *
+ * If the file is not found, then this function does not return, but calls
+ * die().
+ *
+ * @param fname		Filename to search
+ * @param fullnamep	If non-NULL, it is set to the allocated filename of the
+ *			file that was opened. The caller is then responsible
+ *			for freeing the pointer.
+ * @return pointer to opened FILE
+ */
 FILE *srcfile_relative_open(const char *fname, char **fullnamep);
+
 void srcfile_push(const char *fname);
 int srcfile_pop(void);
 
+/**
+ * Add a new directory to the search path for input files
+ *
+ * The new path is added at the end of the list.
+ *
+ * @param dirname	Directory to add
+ */
+void srcfile_add_search_path(const char *dirname);
+
 struct srcpos {
     int first_line;
     int first_column;
@@ -84,4 +113,6 @@ extern void srcpos_error(struct srcpos *pos, char const *, ...)
 extern void srcpos_warn(struct srcpos *pos, char const *, ...)
      __attribute__((format(printf, 2, 3)));
 
+extern void srcpos_set_line(char *f, int l);
+
 #endif /* _SRCPOS_H_ */
diff --git a/scripts/dtc/treesource.c b/scripts/dtc/treesource.c
index c09aafade31..33eeba55fb4 100644
--- a/scripts/dtc/treesource.c
+++ b/scripts/dtc/treesource.c
@@ -23,6 +23,7 @@
 
 extern FILE *yyin;
 extern int yyparse(void);
+extern YYLTYPE yylloc;
 
 struct boot_info *the_boot_info;
 int treesource_error;
@@ -34,6 +35,7 @@ struct boot_info *dt_from_source(const char *fname)
 
 	srcfile_push(fname);
 	yyin = current_srcfile->f;
+	yylloc.file = current_srcfile;
 
 	if (yyparse() != 0)
 		die("Unable to parse input tree\n");
diff --git a/scripts/dtc/util.c b/scripts/dtc/util.c
index d7ac27d2ae1..2422c34e11d 100644
--- a/scripts/dtc/util.c
+++ b/scripts/dtc/util.c
@@ -1,6 +1,10 @@
 /*
+ * Copyright 2011 The Chromium Authors, All Rights Reserved.
  * Copyright 2008 Jon Loeliger, Freescale Semiconductor, Inc.
  *
+ * util_is_printable_string contributed by
+ *	Pantelis Antoniou <pantelis.antoniou AT gmail.com>
+ *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
@@ -17,11 +21,18 @@
  *                                                                   USA
  */
 
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
 #include <string.h>
+#include <assert.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
 
+#include "libfdt.h"
 #include "util.h"
 
 char *xstrdup(const char *s)
@@ -57,3 +68,264 @@ char *join_path(const char *path, const char *name)
 	memcpy(str+lenp, name, lenn+1);
 	return str;
 }
+
+int util_is_printable_string(const void *data, int len)
+{
+	const char *s = data;
+	const char *ss;
+
+	/* zero length is not */
+	if (len == 0)
+		return 0;
+
+	/* must terminate with zero */
+	if (s[len - 1] != '\0')
+		return 0;
+
+	ss = s;
+	while (*s && isprint(*s))
+		s++;
+
+	/* not zero, or not done yet */
+	if (*s != '\0' || (s + 1 - ss) < len)
+		return 0;
+
+	return 1;
+}
+
+/*
+ * Parse a octal encoded character starting at index i in string s.  The
+ * resulting character will be returned and the index i will be updated to
+ * point at the character directly after the end of the encoding, this may be
+ * the '\0' terminator of the string.
+ */
+static char get_oct_char(const char *s, int *i)
+{
+	char x[4];
+	char *endx;
+	long val;
+
+	x[3] = '\0';
+	strncpy(x, s + *i, 3);
+
+	val = strtol(x, &endx, 8);
+
+	assert(endx > x);
+
+	(*i) += endx - x;
+	return val;
+}
+
+/*
+ * Parse a hexadecimal encoded character starting at index i in string s.  The
+ * resulting character will be returned and the index i will be updated to
+ * point at the character directly after the end of the encoding, this may be
+ * the '\0' terminator of the string.
+ */
+static char get_hex_char(const char *s, int *i)
+{
+	char x[3];
+	char *endx;
+	long val;
+
+	x[2] = '\0';
+	strncpy(x, s + *i, 2);
+
+	val = strtol(x, &endx, 16);
+	if (!(endx  > x))
+		die("\\x used with no following hex digits\n");
+
+	(*i) += endx - x;
+	return val;
+}
+
+char get_escape_char(const char *s, int *i)
+{
+	char	c = s[*i];
+	int	j = *i + 1;
+	char	val;
+
+	assert(c);
+	switch (c) {
+	case 'a':
+		val = '\a';
+		break;
+	case 'b':
+		val = '\b';
+		break;
+	case 't':
+		val = '\t';
+		break;
+	case 'n':
+		val = '\n';
+		break;
+	case 'v':
+		val = '\v';
+		break;
+	case 'f':
+		val = '\f';
+		break;
+	case 'r':
+		val = '\r';
+		break;
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+		j--; /* need to re-read the first digit as
+		      * part of the octal value */
+		val = get_oct_char(s, &j);
+		break;
+	case 'x':
+		val = get_hex_char(s, &j);
+		break;
+	default:
+		val = c;
+	}
+
+	(*i) = j;
+	return val;
+}
+
+int utilfdt_read_err(const char *filename, char **buffp)
+{
+	int fd = 0;	/* assume stdin */
+	char *buf = NULL;
+	off_t bufsize = 1024, offset = 0;
+	int ret = 0;
+
+	*buffp = NULL;
+	if (strcmp(filename, "-") != 0) {
+		fd = open(filename, O_RDONLY);
+		if (fd < 0)
+			return errno;
+	}
+
+	/* Loop until we have read everything */
+	buf = malloc(bufsize);
+	do {
+		/* Expand the buffer to hold the next chunk */
+		if (offset == bufsize) {
+			bufsize *= 2;
+			buf = realloc(buf, bufsize);
+			if (!buf) {
+				ret = ENOMEM;
+				break;
+			}
+		}
+
+		ret = read(fd, &buf[offset], bufsize - offset);
+		if (ret < 0) {
+			ret = errno;
+			break;
+		}
+		offset += ret;
+	} while (ret != 0);
+
+	/* Clean up, including closing stdin; return errno on error */
+	close(fd);
+	if (ret)
+		free(buf);
+	else
+		*buffp = buf;
+	return ret;
+}
+
+char *utilfdt_read(const char *filename)
+{
+	char *buff;
+	int ret = utilfdt_read_err(filename, &buff);
+
+	if (ret) {
+		fprintf(stderr, "Couldn't open blob from '%s': %s\n", filename,
+			strerror(ret));
+		return NULL;
+	}
+	/* Successful read */
+	return buff;
+}
+
+int utilfdt_write_err(const char *filename, const void *blob)
+{
+	int fd = 1;	/* assume stdout */
+	int totalsize;
+	int offset;
+	int ret = 0;
+	const char *ptr = blob;
+
+	if (strcmp(filename, "-") != 0) {
+		fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
+		if (fd < 0)
+			return errno;
+	}
+
+	totalsize = fdt_totalsize(blob);
+	offset = 0;
+
+	while (offset < totalsize) {
+		ret = write(fd, ptr + offset, totalsize - offset);
+		if (ret < 0) {
+			ret = -errno;
+			break;
+		}
+		offset += ret;
+	}
+	/* Close the file/stdin; return errno on error */
+	if (fd != 1)
+		close(fd);
+	return ret < 0 ? -ret : 0;
+}
+
+
+int utilfdt_write(const char *filename, const void *blob)
+{
+	int ret = utilfdt_write_err(filename, blob);
+
+	if (ret) {
+		fprintf(stderr, "Couldn't write blob to '%s': %s\n", filename,
+			strerror(ret));
+	}
+	return ret ? -1 : 0;
+}
+
+int utilfdt_decode_type(const char *fmt, int *type, int *size)
+{
+	int qualifier = 0;
+
+	if (!*fmt)
+		return -1;
+
+	/* get the conversion qualifier */
+	*size = -1;
+	if (strchr("hlLb", *fmt)) {
+		qualifier = *fmt++;
+		if (qualifier == *fmt) {
+			switch (*fmt++) {
+/* TODO:		case 'l': qualifier = 'L'; break;*/
+			case 'h':
+				qualifier = 'b';
+				break;
+			}
+		}
+	}
+
+	/* we should now have a type */
+	if ((*fmt == '\0') || !strchr("iuxs", *fmt))
+		return -1;
+
+	/* convert qualifier (bhL) to byte size */
+	if (*fmt != 's')
+		*size = qualifier == 'b' ? 1 :
+				qualifier == 'h' ? 2 :
+				qualifier == 'l' ? 4 : -1;
+	*type = *fmt++;
+
+	/* that should be it! */
+	if (*fmt)
+		return -1;
+	return 0;
+}
diff --git a/scripts/dtc/util.h b/scripts/dtc/util.h
index 9cead842c11..c8eb45d9f04 100644
--- a/scripts/dtc/util.h
+++ b/scripts/dtc/util.h
@@ -1,7 +1,10 @@
 #ifndef _UTIL_H
 #define _UTIL_H
 
+#include <stdarg.h>
+
 /*
+ * Copyright 2011 The Chromium Authors, All Rights Reserved.
  * Copyright 2008 Jon Loeliger, Freescale Semiconductor, Inc.
  *
  * This program is free software; you can redistribute it and/or
@@ -53,4 +56,98 @@ static inline void *xrealloc(void *p, size_t len)
 extern char *xstrdup(const char *s);
 extern char *join_path(const char *path, const char *name);
 
+/**
+ * Check a string of a given length to see if it is all printable and
+ * has a valid terminator.
+ *
+ * @param data	The string to check
+ * @param len	The string length including terminator
+ * @return 1 if a valid printable string, 0 if not */
+int util_is_printable_string(const void *data, int len);
+
+/*
+ * Parse an escaped character starting at index i in string s.  The resulting
+ * character will be returned and the index i will be updated to point at the
+ * character directly after the end of the encoding, this may be the '\0'
+ * terminator of the string.
+ */
+char get_escape_char(const char *s, int *i);
+
+/**
+ * Read a device tree file into a buffer. This will report any errors on
+ * stderr.
+ *
+ * @param filename	The filename to read, or - for stdin
+ * @return Pointer to allocated buffer containing fdt, or NULL on error
+ */
+char *utilfdt_read(const char *filename);
+
+/**
+ * Read a device tree file into a buffer. Does not report errors, but only
+ * returns them. The value returned can be passed to strerror() to obtain
+ * an error message for the user.
+ *
+ * @param filename	The filename to read, or - for stdin
+ * @param buffp		Returns pointer to buffer containing fdt
+ * @return 0 if ok, else an errno value representing the error
+ */
+int utilfdt_read_err(const char *filename, char **buffp);
+
+
+/**
+ * Write a device tree buffer to a file. This will report any errors on
+ * stderr.
+ *
+ * @param filename	The filename to write, or - for stdout
+ * @param blob		Poiner to buffer containing fdt
+ * @return 0 if ok, -1 on error
+ */
+int utilfdt_write(const char *filename, const void *blob);
+
+/**
+ * Write a device tree buffer to a file. Does not report errors, but only
+ * returns them. The value returned can be passed to strerror() to obtain
+ * an error message for the user.
+ *
+ * @param filename	The filename to write, or - for stdout
+ * @param blob		Poiner to buffer containing fdt
+ * @return 0 if ok, else an errno value representing the error
+ */
+int utilfdt_write_err(const char *filename, const void *blob);
+
+/**
+ * Decode a data type string. The purpose of this string
+ *
+ * The string consists of an optional character followed by the type:
+ *	Modifier characters:
+ *		hh or b	1 byte
+ *		h	2 byte
+ *		l	4 byte, default
+ *
+ *	Type character:
+ *		s	string
+ *		i	signed integer
+ *		u	unsigned integer
+ *		x	hex
+ *
+ * TODO: Implement ll modifier (8 bytes)
+ * TODO: Implement o type (octal)
+ *
+ * @param fmt		Format string to process
+ * @param type		Returns type found(s/d/u/x), or 0 if none
+ * @param size		Returns size found(1,2,4,8) or 4 if none
+ * @return 0 if ok, -1 on error (no type given, or other invalid format)
+ */
+int utilfdt_decode_type(const char *fmt, int *type, int *size);
+
+/*
+ * This is a usage message fragment for the -t option. It is the format
+ * supported by utilfdt_decode_type.
+ */
+
+#define USAGE_TYPE_MSG \
+	"<type>\ts=string, i=int, u=unsigned, x=hex\n" \
+	"\tOptional modifier prefix:\n" \
+	"\t\thh or b=byte, h=2 byte, l=4 byte (default)\n";
+
 #endif /* _UTIL_H */
diff --git a/scripts/headers_install.pl b/scripts/headers_install.pl
index 48462be328b..239d22d4207 100644
--- a/scripts/headers_install.pl
+++ b/scripts/headers_install.pl
@@ -4,8 +4,7 @@
 # user space and copy the files to their destination.
 #
 # Usage: headers_install.pl readdir installdir arch [files...]
-# readdir:    dir to open files
-# installdir: dir to install the files
+# installdir: dir to install the files to
 # arch:       current architecture
 #             arch is used to force a reinstallation when the arch
 #             changes because kbuild then detect a command line change.
@@ -18,15 +17,18 @@
 
 use strict;
 
-my ($readdir, $installdir, $arch, @files) = @ARGV;
+my ($installdir, $arch, @files) = @ARGV;
 
 my $unifdef = "scripts/unifdef -U__KERNEL__ -D__EXPORTED_HEADERS__";
 
-foreach my $file (@files) {
+foreach my $filename (@files) {
+	my $file = $filename;
+	$file =~ s!^.*/!!;
+
 	my $tmpfile = "$installdir/$file.tmp";
 
-	open(my $in, '<', "$readdir/$file")
-	    or die "$readdir/$file: $!\n";
+	open(my $in, '<', $filename)
+	    or die "$filename: $!\n";
 	open(my $out, '>', $tmpfile)
 	    or die "$tmpfile: $!\n";
 	while (my $line = <$in>) {
diff --git a/scripts/kernel-doc b/scripts/kernel-doc
index 8fd107a3fac..01e8a8e2260 100755
--- a/scripts/kernel-doc
+++ b/scripts/kernel-doc
@@ -230,6 +230,7 @@ my $dohighlight = "";
 
 my $verbose = 0;
 my $output_mode = "man";
+my $output_preformatted = 0;
 my $no_doc_sections = 0;
 my %highlights = %highlights_man;
 my $blankline = $blankline_man;
@@ -280,9 +281,10 @@ my $doc_special = "\@\%\$\&";
 my $doc_start = '^/\*\*\s*$'; # Allow whitespace at end of comment start.
 my $doc_end = '\*/';
 my $doc_com = '\s*\*\s*';
+my $doc_com_body = '\s*\* ?';
 my $doc_decl = $doc_com . '(\w+)';
 my $doc_sect = $doc_com . '([' . $doc_special . ']?[\w\s]+):(.*)';
-my $doc_content = $doc_com . '(.*)';
+my $doc_content = $doc_com_body . '(.*)';
 my $doc_block = $doc_com . 'DOC:\s*(.*)?';
 
 my %constants;
@@ -459,8 +461,13 @@ sub output_highlight {
 #   print STDERR "contents af:$contents\n";
 
     foreach $line (split "\n", $contents) {
+	if (! $output_preformatted) {
+	    $line =~ s/^\s*//;
+	}
 	if ($line eq ""){
-	    print $lineprefix, local_unescape($blankline);
+	    if (! $output_preformatted) {
+		print $lineprefix, local_unescape($blankline);
+	    }
 	} else {
 	    $line =~ s/\\\\\\/\&/g;
 	    if ($output_mode eq "man" && substr($line, 0, 1) eq ".") {
@@ -643,10 +650,12 @@ sub output_section_xml(%) {
 	print "<title>$section</title>\n";
 	if ($section =~ m/EXAMPLE/i) {
 	    print "<informalexample><programlisting>\n";
+	    $output_preformatted = 1;
 	} else {
 	    print "<para>\n";
 	}
 	output_highlight($args{'sections'}{$section});
+	$output_preformatted = 0;
 	if ($section =~ m/EXAMPLE/i) {
 	    print "</programlisting></informalexample>\n";
 	} else {
@@ -949,10 +958,12 @@ sub output_blockhead_xml(%) {
 	}
 	if ($section =~ m/EXAMPLE/i) {
 	    print "<example><para>\n";
+	    $output_preformatted = 1;
 	} else {
 	    print "<para>\n";
 	}
 	output_highlight($args{'sections'}{$section});
+	$output_preformatted = 0;
 	if ($section =~ m/EXAMPLE/i) {
 	    print "</para></example>\n";
 	} else {
@@ -1028,10 +1039,12 @@ sub output_function_gnome {
 	print "<simplesect>\n <title>$section</title>\n";
 	if ($section =~ m/EXAMPLE/i) {
 	    print "<example><programlisting>\n";
+	    $output_preformatted = 1;
 	} else {
 	}
 	print "<para>\n";
 	output_highlight($args{'sections'}{$section});
+	$output_preformatted = 0;
 	print "</para>\n";
 	if ($section =~ m/EXAMPLE/i) {
 	    print "</programlisting></example>\n";
@@ -2046,6 +2059,9 @@ sub process_file($) {
 
     $section_counter = 0;
     while (<IN>) {
+	while (s/\\\s*$//) {
+	    $_ .= <IN>;
+	}
 	if ($state == 0) {
 	    if (/$doc_start/o) {
 		$state = 1;		# next line is always the function name
@@ -2073,7 +2089,7 @@ sub process_file($) {
 		    $descr= $1;
 		    $descr =~ s/^\s*//;
 		    $descr =~ s/\s*$//;
-		    $descr =~ s/\s+/ /;
+		    $descr =~ s/\s+/ /g;
 		    $declaration_purpose = xml_escape($descr);
 		    $in_purpose = 1;
 		} else {
@@ -2165,6 +2181,7 @@ sub process_file($) {
 		    # Continued declaration purpose
 		    chomp($declaration_purpose);
 		    $declaration_purpose .= " " . xml_escape($1);
+		    $declaration_purpose =~ s/\s+/ /g;
 		} else {
 		    $contents .= $1 . "\n";
 		}
diff --git a/security/device_cgroup.c b/security/device_cgroup.c
index 4b877a92a7e..44dfc415a37 100644
--- a/security/device_cgroup.c
+++ b/security/device_cgroup.c
@@ -26,12 +26,12 @@
 static DEFINE_MUTEX(devcgroup_mutex);
 
 /*
- * whitelist locking rules:
+ * exception list locking rules:
  * hold devcgroup_mutex for update/read.
  * hold rcu_read_lock() for read.
  */
 
-struct dev_whitelist_item {
+struct dev_exception_item {
 	u32 major, minor;
 	short type;
 	short access;
@@ -41,7 +41,8 @@ struct dev_whitelist_item {
 
 struct dev_cgroup {
 	struct cgroup_subsys_state css;
-	struct list_head whitelist;
+	struct list_head exceptions;
+	bool deny_all;
 };
 
 static inline struct dev_cgroup *css_to_devcgroup(struct cgroup_subsys_state *s)
@@ -74,12 +75,12 @@ static int devcgroup_can_attach(struct cgroup *new_cgrp,
 /*
  * called under devcgroup_mutex
  */
-static int dev_whitelist_copy(struct list_head *dest, struct list_head *orig)
+static int dev_exceptions_copy(struct list_head *dest, struct list_head *orig)
 {
-	struct dev_whitelist_item *wh, *tmp, *new;
+	struct dev_exception_item *ex, *tmp, *new;
 
-	list_for_each_entry(wh, orig, list) {
-		new = kmemdup(wh, sizeof(*wh), GFP_KERNEL);
+	list_for_each_entry(ex, orig, list) {
+		new = kmemdup(ex, sizeof(*ex), GFP_KERNEL);
 		if (!new)
 			goto free_and_exit;
 		list_add_tail(&new->list, dest);
@@ -88,64 +89,60 @@ static int dev_whitelist_copy(struct list_head *dest, struct list_head *orig)
 	return 0;
 
 free_and_exit:
-	list_for_each_entry_safe(wh, tmp, dest, list) {
-		list_del(&wh->list);
-		kfree(wh);
+	list_for_each_entry_safe(ex, tmp, dest, list) {
+		list_del(&ex->list);
+		kfree(ex);
 	}
 	return -ENOMEM;
 }
 
-/* Stupid prototype - don't bother combining existing entries */
 /*
  * called under devcgroup_mutex
  */
-static int dev_whitelist_add(struct dev_cgroup *dev_cgroup,
-			struct dev_whitelist_item *wh)
+static int dev_exception_add(struct dev_cgroup *dev_cgroup,
+			     struct dev_exception_item *ex)
 {
-	struct dev_whitelist_item *whcopy, *walk;
+	struct dev_exception_item *excopy, *walk;
 
-	whcopy = kmemdup(wh, sizeof(*wh), GFP_KERNEL);
-	if (!whcopy)
+	excopy = kmemdup(ex, sizeof(*ex), GFP_KERNEL);
+	if (!excopy)
 		return -ENOMEM;
 
-	list_for_each_entry(walk, &dev_cgroup->whitelist, list) {
-		if (walk->type != wh->type)
+	list_for_each_entry(walk, &dev_cgroup->exceptions, list) {
+		if (walk->type != ex->type)
 			continue;
-		if (walk->major != wh->major)
+		if (walk->major != ex->major)
 			continue;
-		if (walk->minor != wh->minor)
+		if (walk->minor != ex->minor)
 			continue;
 
-		walk->access |= wh->access;
-		kfree(whcopy);
-		whcopy = NULL;
+		walk->access |= ex->access;
+		kfree(excopy);
+		excopy = NULL;
 	}
 
-	if (whcopy != NULL)
-		list_add_tail_rcu(&whcopy->list, &dev_cgroup->whitelist);
+	if (excopy != NULL)
+		list_add_tail_rcu(&excopy->list, &dev_cgroup->exceptions);
 	return 0;
 }
 
 /*
  * called under devcgroup_mutex
  */
-static void dev_whitelist_rm(struct dev_cgroup *dev_cgroup,
-			struct dev_whitelist_item *wh)
+static void dev_exception_rm(struct dev_cgroup *dev_cgroup,
+			     struct dev_exception_item *ex)
 {
-	struct dev_whitelist_item *walk, *tmp;
+	struct dev_exception_item *walk, *tmp;
 
-	list_for_each_entry_safe(walk, tmp, &dev_cgroup->whitelist, list) {
-		if (walk->type == DEV_ALL)
-			goto remove;
-		if (walk->type != wh->type)
+	list_for_each_entry_safe(walk, tmp, &dev_cgroup->exceptions, list) {
+		if (walk->type != ex->type)
 			continue;
-		if (walk->major != ~0 && walk->major != wh->major)
+		if (walk->major != ex->major)
 			continue;
-		if (walk->minor != ~0 && walk->minor != wh->minor)
+		if (walk->minor != ex->minor)
 			continue;
 
-remove:
-		walk->access &= ~wh->access;
+		walk->access &= ~ex->access;
 		if (!walk->access) {
 			list_del_rcu(&walk->list);
 			kfree_rcu(walk, rcu);
@@ -153,6 +150,22 @@ remove:
 	}
 }
 
+/**
+ * dev_exception_clean - frees all entries of the exception list
+ * @dev_cgroup: dev_cgroup with the exception list to be cleaned
+ *
+ * called under devcgroup_mutex
+ */
+static void dev_exception_clean(struct dev_cgroup *dev_cgroup)
+{
+	struct dev_exception_item *ex, *tmp;
+
+	list_for_each_entry_safe(ex, tmp, &dev_cgroup->exceptions, list) {
+		list_del(&ex->list);
+		kfree(ex);
+	}
+}
+
 /*
  * called from kernel/cgroup.c with cgroup_lock() held.
  */
@@ -165,25 +178,17 @@ static struct cgroup_subsys_state *devcgroup_create(struct cgroup *cgroup)
 	dev_cgroup = kzalloc(sizeof(*dev_cgroup), GFP_KERNEL);
 	if (!dev_cgroup)
 		return ERR_PTR(-ENOMEM);
-	INIT_LIST_HEAD(&dev_cgroup->whitelist);
+	INIT_LIST_HEAD(&dev_cgroup->exceptions);
 	parent_cgroup = cgroup->parent;
 
-	if (parent_cgroup == NULL) {
-		struct dev_whitelist_item *wh;
-		wh = kmalloc(sizeof(*wh), GFP_KERNEL);
-		if (!wh) {
-			kfree(dev_cgroup);
-			return ERR_PTR(-ENOMEM);
-		}
-		wh->minor = wh->major = ~0;
-		wh->type = DEV_ALL;
-		wh->access = ACC_MASK;
-		list_add(&wh->list, &dev_cgroup->whitelist);
-	} else {
+	if (parent_cgroup == NULL)
+		dev_cgroup->deny_all = false;
+	else {
 		parent_dev_cgroup = cgroup_to_devcgroup(parent_cgroup);
 		mutex_lock(&devcgroup_mutex);
-		ret = dev_whitelist_copy(&dev_cgroup->whitelist,
-				&parent_dev_cgroup->whitelist);
+		ret = dev_exceptions_copy(&dev_cgroup->exceptions,
+					  &parent_dev_cgroup->exceptions);
+		dev_cgroup->deny_all = parent_dev_cgroup->deny_all;
 		mutex_unlock(&devcgroup_mutex);
 		if (ret) {
 			kfree(dev_cgroup);
@@ -197,13 +202,9 @@ static struct cgroup_subsys_state *devcgroup_create(struct cgroup *cgroup)
 static void devcgroup_destroy(struct cgroup *cgroup)
 {
 	struct dev_cgroup *dev_cgroup;
-	struct dev_whitelist_item *wh, *tmp;
 
 	dev_cgroup = cgroup_to_devcgroup(cgroup);
-	list_for_each_entry_safe(wh, tmp, &dev_cgroup->whitelist, list) {
-		list_del(&wh->list);
-		kfree(wh);
-	}
+	dev_exception_clean(dev_cgroup);
 	kfree(dev_cgroup);
 }
 
@@ -249,59 +250,87 @@ static int devcgroup_seq_read(struct cgroup *cgroup, struct cftype *cft,
 				struct seq_file *m)
 {
 	struct dev_cgroup *devcgroup = cgroup_to_devcgroup(cgroup);
-	struct dev_whitelist_item *wh;
+	struct dev_exception_item *ex;
 	char maj[MAJMINLEN], min[MAJMINLEN], acc[ACCLEN];
 
 	rcu_read_lock();
-	list_for_each_entry_rcu(wh, &devcgroup->whitelist, list) {
-		set_access(acc, wh->access);
-		set_majmin(maj, wh->major);
-		set_majmin(min, wh->minor);
-		seq_printf(m, "%c %s:%s %s\n", type_to_char(wh->type),
+	/*
+	 * To preserve the compatibility:
+	 * - Only show the "all devices" when the default policy is to allow
+	 * - List the exceptions in case the default policy is to deny
+	 * This way, the file remains as a "whitelist of devices"
+	 */
+	if (devcgroup->deny_all == false) {
+		set_access(acc, ACC_MASK);
+		set_majmin(maj, ~0);
+		set_majmin(min, ~0);
+		seq_printf(m, "%c %s:%s %s\n", type_to_char(DEV_ALL),
 			   maj, min, acc);
+	} else {
+		list_for_each_entry_rcu(ex, &devcgroup->exceptions, list) {
+			set_access(acc, ex->access);
+			set_majmin(maj, ex->major);
+			set_majmin(min, ex->minor);
+			seq_printf(m, "%c %s:%s %s\n", type_to_char(ex->type),
+				   maj, min, acc);
+		}
 	}
 	rcu_read_unlock();
 
 	return 0;
 }
 
-/*
- * may_access_whitelist:
- * does the access granted to dev_cgroup c contain the access
- * requested in whitelist item refwh.
- * return 1 if yes, 0 if no.
- * call with devcgroup_mutex held
+/**
+ * may_access - verifies if a new exception is part of what is allowed
+ *		by a dev cgroup based on the default policy +
+ *		exceptions. This is used to make sure a child cgroup
+ *		won't have more privileges than its parent or to
+ *		verify if a certain access is allowed.
+ * @dev_cgroup: dev cgroup to be tested against
+ * @refex: new exception
  */
-static int may_access_whitelist(struct dev_cgroup *c,
-				       struct dev_whitelist_item *refwh)
+static int may_access(struct dev_cgroup *dev_cgroup,
+		      struct dev_exception_item *refex)
 {
-	struct dev_whitelist_item *whitem;
+	struct dev_exception_item *ex;
+	bool match = false;
 
-	list_for_each_entry(whitem, &c->whitelist, list) {
-		if (whitem->type & DEV_ALL)
-			return 1;
-		if ((refwh->type & DEV_BLOCK) && !(whitem->type & DEV_BLOCK))
+	list_for_each_entry(ex, &dev_cgroup->exceptions, list) {
+		if ((refex->type & DEV_BLOCK) && !(ex->type & DEV_BLOCK))
 			continue;
-		if ((refwh->type & DEV_CHAR) && !(whitem->type & DEV_CHAR))
+		if ((refex->type & DEV_CHAR) && !(ex->type & DEV_CHAR))
 			continue;
-		if (whitem->major != ~0 && whitem->major != refwh->major)
+		if (ex->major != ~0 && ex->major != refex->major)
 			continue;
-		if (whitem->minor != ~0 && whitem->minor != refwh->minor)
+		if (ex->minor != ~0 && ex->minor != refex->minor)
 			continue;
-		if (refwh->access & (~whitem->access))
+		if (refex->access & (~ex->access))
 			continue;
-		return 1;
+		match = true;
+		break;
 	}
+
+	/*
+	 * In two cases we'll consider this new exception valid:
+	 * - the dev cgroup has its default policy to allow + exception list:
+	 *   the new exception should *not* match any of the exceptions
+	 *   (!deny_all, !match)
+	 * - the dev cgroup has its default policy to deny + exception list:
+	 *   the new exception *should* match the exceptions
+	 *   (deny_all, match)
+	 */
+	if (dev_cgroup->deny_all == match)
+		return 1;
 	return 0;
 }
 
 /*
  * parent_has_perm:
- * when adding a new allow rule to a device whitelist, the rule
+ * when adding a new allow rule to a device exception list, the rule
  * must be allowed in the parent device
  */
 static int parent_has_perm(struct dev_cgroup *childcg,
-				  struct dev_whitelist_item *wh)
+				  struct dev_exception_item *ex)
 {
 	struct cgroup *pcg = childcg->css.cgroup->parent;
 	struct dev_cgroup *parent;
@@ -309,17 +338,17 @@ static int parent_has_perm(struct dev_cgroup *childcg,
 	if (!pcg)
 		return 1;
 	parent = cgroup_to_devcgroup(pcg);
-	return may_access_whitelist(parent, wh);
+	return may_access(parent, ex);
 }
 
 /*
- * Modify the whitelist using allow/deny rules.
+ * Modify the exception list using allow/deny rules.
  * CAP_SYS_ADMIN is needed for this.  It's at least separate from CAP_MKNOD
  * so we can give a container CAP_MKNOD to let it create devices but not
- * modify the whitelist.
+ * modify the exception list.
  * It seems likely we'll want to add a CAP_CONTAINER capability to allow
  * us to also grant CAP_SYS_ADMIN to containers without giving away the
- * device whitelist controls, but for now we'll stick with CAP_SYS_ADMIN
+ * device exception list controls, but for now we'll stick with CAP_SYS_ADMIN
  *
  * Taking rules away is always allowed (given CAP_SYS_ADMIN).  Granting
  * new access is only allowed if you're in the top-level cgroup, or your
@@ -331,26 +360,36 @@ static int devcgroup_update_access(struct dev_cgroup *devcgroup,
 	const char *b;
 	char *endp;
 	int count;
-	struct dev_whitelist_item wh;
+	struct dev_exception_item ex;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	memset(&wh, 0, sizeof(wh));
+	memset(&ex, 0, sizeof(ex));
 	b = buffer;
 
 	switch (*b) {
 	case 'a':
-		wh.type = DEV_ALL;
-		wh.access = ACC_MASK;
-		wh.major = ~0;
-		wh.minor = ~0;
-		goto handle;
+		switch (filetype) {
+		case DEVCG_ALLOW:
+			if (!parent_has_perm(devcgroup, &ex))
+				return -EPERM;
+			dev_exception_clean(devcgroup);
+			devcgroup->deny_all = false;
+			break;
+		case DEVCG_DENY:
+			dev_exception_clean(devcgroup);
+			devcgroup->deny_all = true;
+			break;
+		default:
+			return -EINVAL;
+		}
+		return 0;
 	case 'b':
-		wh.type = DEV_BLOCK;
+		ex.type = DEV_BLOCK;
 		break;
 	case 'c':
-		wh.type = DEV_CHAR;
+		ex.type = DEV_CHAR;
 		break;
 	default:
 		return -EINVAL;
@@ -360,10 +399,10 @@ static int devcgroup_update_access(struct dev_cgroup *devcgroup,
 		return -EINVAL;
 	b++;
 	if (*b == '*') {
-		wh.major = ~0;
+		ex.major = ~0;
 		b++;
 	} else if (isdigit(*b)) {
-		wh.major = simple_strtoul(b, &endp, 10);
+		ex.major = simple_strtoul(b, &endp, 10);
 		b = endp;
 	} else {
 		return -EINVAL;
@@ -374,10 +413,10 @@ static int devcgroup_update_access(struct dev_cgroup *devcgroup,
 
 	/* read minor */
 	if (*b == '*') {
-		wh.minor = ~0;
+		ex.minor = ~0;
 		b++;
 	} else if (isdigit(*b)) {
-		wh.minor = simple_strtoul(b, &endp, 10);
+		ex.minor = simple_strtoul(b, &endp, 10);
 		b = endp;
 	} else {
 		return -EINVAL;
@@ -387,13 +426,13 @@ static int devcgroup_update_access(struct dev_cgroup *devcgroup,
 	for (b++, count = 0; count < 3; count++, b++) {
 		switch (*b) {
 		case 'r':
-			wh.access |= ACC_READ;
+			ex.access |= ACC_READ;
 			break;
 		case 'w':
-			wh.access |= ACC_WRITE;
+			ex.access |= ACC_WRITE;
 			break;
 		case 'm':
-			wh.access |= ACC_MKNOD;
+			ex.access |= ACC_MKNOD;
 			break;
 		case '\n':
 		case '\0':
@@ -404,15 +443,31 @@ static int devcgroup_update_access(struct dev_cgroup *devcgroup,
 		}
 	}
 
-handle:
 	switch (filetype) {
 	case DEVCG_ALLOW:
-		if (!parent_has_perm(devcgroup, &wh))
+		if (!parent_has_perm(devcgroup, &ex))
 			return -EPERM;
-		return dev_whitelist_add(devcgroup, &wh);
+		/*
+		 * If the default policy is to allow by default, try to remove
+		 * an matching exception instead. And be silent about it: we
+		 * don't want to break compatibility
+		 */
+		if (devcgroup->deny_all == false) {
+			dev_exception_rm(devcgroup, &ex);
+			return 0;
+		}
+		return dev_exception_add(devcgroup, &ex);
 	case DEVCG_DENY:
-		dev_whitelist_rm(devcgroup, &wh);
-		break;
+		/*
+		 * If the default policy is to deny by default, try to remove
+		 * an matching exception instead. And be silent about it: we
+		 * don't want to break compatibility
+		 */
+		if (devcgroup->deny_all == true) {
+			dev_exception_rm(devcgroup, &ex);
+			return 0;
+		}
+		return dev_exception_add(devcgroup, &ex);
 	default:
 		return -EINVAL;
 	}
@@ -468,73 +523,71 @@ struct cgroup_subsys devices_subsys = {
 	.broken_hierarchy = true,
 };
 
-int __devcgroup_inode_permission(struct inode *inode, int mask)
+/**
+ * __devcgroup_check_permission - checks if an inode operation is permitted
+ * @dev_cgroup: the dev cgroup to be tested against
+ * @type: device type
+ * @major: device major number
+ * @minor: device minor number
+ * @access: combination of ACC_WRITE, ACC_READ and ACC_MKNOD
+ *
+ * returns 0 on success, -EPERM case the operation is not permitted
+ */
+static int __devcgroup_check_permission(struct dev_cgroup *dev_cgroup,
+					short type, u32 major, u32 minor,
+				        short access)
 {
-	struct dev_cgroup *dev_cgroup;
-	struct dev_whitelist_item *wh;
-
-	rcu_read_lock();
+	struct dev_exception_item ex;
+	int rc;
 
-	dev_cgroup = task_devcgroup(current);
+	memset(&ex, 0, sizeof(ex));
+	ex.type = type;
+	ex.major = major;
+	ex.minor = minor;
+	ex.access = access;
 
-	list_for_each_entry_rcu(wh, &dev_cgroup->whitelist, list) {
-		if (wh->type & DEV_ALL)
-			goto found;
-		if ((wh->type & DEV_BLOCK) && !S_ISBLK(inode->i_mode))
-			continue;
-		if ((wh->type & DEV_CHAR) && !S_ISCHR(inode->i_mode))
-			continue;
-		if (wh->major != ~0 && wh->major != imajor(inode))
-			continue;
-		if (wh->minor != ~0 && wh->minor != iminor(inode))
-			continue;
+	rcu_read_lock();
+	rc = may_access(dev_cgroup, &ex);
+	rcu_read_unlock();
 
-		if ((mask & MAY_WRITE) && !(wh->access & ACC_WRITE))
-			continue;
-		if ((mask & MAY_READ) && !(wh->access & ACC_READ))
-			continue;
-found:
-		rcu_read_unlock();
-		return 0;
-	}
+	if (!rc)
+		return -EPERM;
 
-	rcu_read_unlock();
+	return 0;
+}
 
-	return -EPERM;
+int __devcgroup_inode_permission(struct inode *inode, int mask)
+{
+	struct dev_cgroup *dev_cgroup = task_devcgroup(current);
+	short type, access = 0;
+
+	if (S_ISBLK(inode->i_mode))
+		type = DEV_BLOCK;
+	if (S_ISCHR(inode->i_mode))
+		type = DEV_CHAR;
+	if (mask & MAY_WRITE)
+		access |= ACC_WRITE;
+	if (mask & MAY_READ)
+		access |= ACC_READ;
+
+	return __devcgroup_check_permission(dev_cgroup, type, imajor(inode),
+					    iminor(inode), access);
 }
 
 int devcgroup_inode_mknod(int mode, dev_t dev)
 {
-	struct dev_cgroup *dev_cgroup;
-	struct dev_whitelist_item *wh;
+	struct dev_cgroup *dev_cgroup = task_devcgroup(current);
+	short type;
 
 	if (!S_ISBLK(mode) && !S_ISCHR(mode))
 		return 0;
 
-	rcu_read_lock();
-
-	dev_cgroup = task_devcgroup(current);
-
-	list_for_each_entry_rcu(wh, &dev_cgroup->whitelist, list) {
-		if (wh->type & DEV_ALL)
-			goto found;
-		if ((wh->type & DEV_BLOCK) && !S_ISBLK(mode))
-			continue;
-		if ((wh->type & DEV_CHAR) && !S_ISCHR(mode))
-			continue;
-		if (wh->major != ~0 && wh->major != MAJOR(dev))
-			continue;
-		if (wh->minor != ~0 && wh->minor != MINOR(dev))
-			continue;
-
-		if (!(wh->access & ACC_MKNOD))
-			continue;
-found:
-		rcu_read_unlock();
-		return 0;
-	}
+	if (S_ISBLK(mode))
+		type = DEV_BLOCK;
+	else
+		type = DEV_CHAR;
 
-	rcu_read_unlock();
+	return __devcgroup_check_permission(dev_cgroup, type, MAJOR(dev),
+					    MINOR(dev), ACC_MKNOD);
 
-	return -EPERM;
 }
diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c
index 8901501425f..eb5484504f5 100644
--- a/security/integrity/evm/evm_main.c
+++ b/security/integrity/evm/evm_main.c
@@ -34,6 +34,9 @@ char *evm_config_xattrnames[] = {
 #ifdef CONFIG_SECURITY_SMACK
 	XATTR_NAME_SMACK,
 #endif
+#ifdef CONFIG_IMA_APPRAISE
+	XATTR_NAME_IMA,
+#endif
 	XATTR_NAME_CAPS,
 	NULL
 };
diff --git a/security/integrity/iint.c b/security/integrity/iint.c
index 399641c3e84..d82a5a13d85 100644
--- a/security/integrity/iint.c
+++ b/security/integrity/iint.c
@@ -22,7 +22,7 @@
 #include "integrity.h"
 
 static struct rb_root integrity_iint_tree = RB_ROOT;
-static DEFINE_SPINLOCK(integrity_iint_lock);
+static DEFINE_RWLOCK(integrity_iint_lock);
 static struct kmem_cache *iint_cache __read_mostly;
 
 int iint_initialized;
@@ -35,8 +35,6 @@ static struct integrity_iint_cache *__integrity_iint_find(struct inode *inode)
 	struct integrity_iint_cache *iint;
 	struct rb_node *n = integrity_iint_tree.rb_node;
 
-	assert_spin_locked(&integrity_iint_lock);
-
 	while (n) {
 		iint = rb_entry(n, struct integrity_iint_cache, rb_node);
 
@@ -63,9 +61,9 @@ struct integrity_iint_cache *integrity_iint_find(struct inode *inode)
 	if (!IS_IMA(inode))
 		return NULL;
 
-	spin_lock(&integrity_iint_lock);
+	read_lock(&integrity_iint_lock);
 	iint = __integrity_iint_find(inode);
-	spin_unlock(&integrity_iint_lock);
+	read_unlock(&integrity_iint_lock);
 
 	return iint;
 }
@@ -74,59 +72,53 @@ static void iint_free(struct integrity_iint_cache *iint)
 {
 	iint->version = 0;
 	iint->flags = 0UL;
+	iint->ima_status = INTEGRITY_UNKNOWN;
 	iint->evm_status = INTEGRITY_UNKNOWN;
 	kmem_cache_free(iint_cache, iint);
 }
 
 /**
- * integrity_inode_alloc - allocate an iint associated with an inode
+ * integrity_inode_get - find or allocate an iint associated with an inode
  * @inode: pointer to the inode
+ * @return: allocated iint
+ *
+ * Caller must lock i_mutex
  */
-int integrity_inode_alloc(struct inode *inode)
+struct integrity_iint_cache *integrity_inode_get(struct inode *inode)
 {
 	struct rb_node **p;
-	struct rb_node *new_node, *parent = NULL;
-	struct integrity_iint_cache *new_iint, *test_iint;
-	int rc;
+	struct rb_node *node, *parent = NULL;
+	struct integrity_iint_cache *iint, *test_iint;
 
-	new_iint = kmem_cache_alloc(iint_cache, GFP_NOFS);
-	if (!new_iint)
-		return -ENOMEM;
+	iint = integrity_iint_find(inode);
+	if (iint)
+		return iint;
 
-	new_iint->inode = inode;
-	new_node = &new_iint->rb_node;
+	iint = kmem_cache_alloc(iint_cache, GFP_NOFS);
+	if (!iint)
+		return NULL;
 
-	mutex_lock(&inode->i_mutex);	/* i_flags */
-	spin_lock(&integrity_iint_lock);
+	write_lock(&integrity_iint_lock);
 
 	p = &integrity_iint_tree.rb_node;
 	while (*p) {
 		parent = *p;
 		test_iint = rb_entry(parent, struct integrity_iint_cache,
 				     rb_node);
-		rc = -EEXIST;
 		if (inode < test_iint->inode)
 			p = &(*p)->rb_left;
-		else if (inode > test_iint->inode)
-			p = &(*p)->rb_right;
 		else
-			goto out_err;
+			p = &(*p)->rb_right;
 	}
 
+	iint->inode = inode;
+	node = &iint->rb_node;
 	inode->i_flags |= S_IMA;
-	rb_link_node(new_node, parent, p);
-	rb_insert_color(new_node, &integrity_iint_tree);
+	rb_link_node(node, parent, p);
+	rb_insert_color(node, &integrity_iint_tree);
 
-	spin_unlock(&integrity_iint_lock);
-	mutex_unlock(&inode->i_mutex);	/* i_flags */
-
-	return 0;
-out_err:
-	spin_unlock(&integrity_iint_lock);
-	mutex_unlock(&inode->i_mutex);	/* i_flags */
-	iint_free(new_iint);
-
-	return rc;
+	write_unlock(&integrity_iint_lock);
+	return iint;
 }
 
 /**
@@ -142,10 +134,10 @@ void integrity_inode_free(struct inode *inode)
 	if (!IS_IMA(inode))
 		return;
 
-	spin_lock(&integrity_iint_lock);
+	write_lock(&integrity_iint_lock);
 	iint = __integrity_iint_find(inode);
 	rb_erase(&iint->rb_node, &integrity_iint_tree);
-	spin_unlock(&integrity_iint_lock);
+	write_unlock(&integrity_iint_lock);
 
 	iint_free(iint);
 }
@@ -157,7 +149,7 @@ static void init_once(void *foo)
 	memset(iint, 0, sizeof *iint);
 	iint->version = 0;
 	iint->flags = 0UL;
-	mutex_init(&iint->mutex);
+	iint->ima_status = INTEGRITY_UNKNOWN;
 	iint->evm_status = INTEGRITY_UNKNOWN;
 }
 
diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig
index b9c1219924f..d232c73647a 100644
--- a/security/integrity/ima/Kconfig
+++ b/security/integrity/ima/Kconfig
@@ -11,6 +11,7 @@ config IMA
 	select CRYPTO_SHA1
 	select TCG_TPM if HAS_IOMEM && !UML
 	select TCG_TIS if TCG_TPM && X86
+	select TCG_IBMVTPM if TCG_TPM && PPC64
 	help
 	  The Trusted Computing Group(TCG) runtime Integrity
 	  Measurement Architecture(IMA) maintains a list of hash
@@ -55,3 +56,18 @@ config IMA_LSM_RULES
 	default y
 	help
 	  Disabling this option will disregard LSM based policy rules.
+
+config IMA_APPRAISE
+	bool "Appraise integrity measurements"
+	depends on IMA
+	default n
+	help
+	  This option enables local measurement integrity appraisal.
+	  It requires the system to be labeled with a security extended
+	  attribute containing the file hash measurement.  To protect
+	  the security extended attributes from offline attack, enable
+	  and configure EVM.
+
+	  For more information on integrity appraisal refer to:
+	  <http://linux-ima.sourceforge.net>
+	  If unsure, say N.
diff --git a/security/integrity/ima/Makefile b/security/integrity/ima/Makefile
index 5f740f6971e..3f2ca6bdc38 100644
--- a/security/integrity/ima/Makefile
+++ b/security/integrity/ima/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_IMA) += ima.o
 ima-y := ima_fs.o ima_queue.o ima_init.o ima_main.o ima_crypto.o ima_api.o \
 	 ima_policy.o
 ima-$(CONFIG_IMA_AUDIT) += ima_audit.o
+ima-$(CONFIG_IMA_APPRAISE) += ima_appraise.o
diff --git a/security/integrity/ima/ima.h b/security/integrity/ima/ima.h
index e7c99fd0d22..8180adde10b 100644
--- a/security/integrity/ima/ima.h
+++ b/security/integrity/ima/ima.h
@@ -40,6 +40,7 @@ enum tpm_pcrs { TPM_PCR0 = 0, TPM_PCR8 = 8 };
 extern int ima_initialized;
 extern int ima_used_chip;
 extern char *ima_hash;
+extern int ima_appraise;
 
 /* IMA inode template definition */
 struct ima_template_data {
@@ -107,11 +108,14 @@ static inline unsigned long ima_hash_key(u8 *digest)
 }
 
 /* LIM API function definitions */
+int ima_get_action(struct inode *inode, int mask, int function);
 int ima_must_measure(struct inode *inode, int mask, int function);
 int ima_collect_measurement(struct integrity_iint_cache *iint,
 			    struct file *file);
 void ima_store_measurement(struct integrity_iint_cache *iint, struct file *file,
 			   const unsigned char *filename);
+void ima_audit_measurement(struct integrity_iint_cache *iint,
+			   const unsigned char *filename);
 int ima_store_template(struct ima_template_entry *entry, int violation,
 		       struct inode *inode);
 void ima_template_show(struct seq_file *m, void *e, enum ima_show_type show);
@@ -123,14 +127,45 @@ struct integrity_iint_cache *integrity_iint_insert(struct inode *inode);
 struct integrity_iint_cache *integrity_iint_find(struct inode *inode);
 
 /* IMA policy related functions */
-enum ima_hooks { FILE_CHECK = 1, FILE_MMAP, BPRM_CHECK };
+enum ima_hooks { FILE_CHECK = 1, FILE_MMAP, BPRM_CHECK, POST_SETATTR };
 
-int ima_match_policy(struct inode *inode, enum ima_hooks func, int mask);
+int ima_match_policy(struct inode *inode, enum ima_hooks func, int mask,
+		     int flags);
 void ima_init_policy(void);
 void ima_update_policy(void);
 ssize_t ima_parse_add_rule(char *);
 void ima_delete_rules(void);
 
+/* Appraise integrity measurements */
+#define IMA_APPRAISE_ENFORCE	0x01
+#define IMA_APPRAISE_FIX	0x02
+
+#ifdef CONFIG_IMA_APPRAISE
+int ima_appraise_measurement(struct integrity_iint_cache *iint,
+			     struct file *file, const unsigned char *filename);
+int ima_must_appraise(struct inode *inode, enum ima_hooks func, int mask);
+void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file);
+
+#else
+static inline int ima_appraise_measurement(struct integrity_iint_cache *iint,
+					   struct file *file,
+					   const unsigned char *filename)
+{
+	return INTEGRITY_UNKNOWN;
+}
+
+static inline int ima_must_appraise(struct inode *inode,
+				    enum ima_hooks func, int mask)
+{
+	return 0;
+}
+
+static inline void ima_update_xattr(struct integrity_iint_cache *iint,
+				    struct file *file)
+{
+}
+#endif
+
 /* LSM based policy rules require audit */
 #ifdef CONFIG_IMA_LSM_RULES
 
diff --git a/security/integrity/ima/ima_api.c b/security/integrity/ima/ima_api.c
index 032ff03ad90..b356884fb3e 100644
--- a/security/integrity/ima/ima_api.c
+++ b/security/integrity/ima/ima_api.c
@@ -9,13 +9,17 @@
  * License.
  *
  * File: ima_api.c
- *	Implements must_measure, collect_measurement, store_measurement,
- *	and store_template.
+ *	Implements must_appraise_or_measure, collect_measurement,
+ *	appraise_measurement, store_measurement and store_template.
  */
 #include <linux/module.h>
 #include <linux/slab.h>
-
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/xattr.h>
+#include <linux/evm.h>
 #include "ima.h"
+
 static const char *IMA_TEMPLATE_NAME = "ima";
 
 /*
@@ -93,7 +97,7 @@ err_out:
 }
 
 /**
- * ima_must_measure - measure decision based on policy.
+ * ima_get_action - appraise & measure decision based on policy.
  * @inode: pointer to inode to measure
  * @mask: contains the permission mask (MAY_READ, MAY_WRITE, MAY_EXECUTE)
  * @function: calling function (FILE_CHECK, BPRM_CHECK, FILE_MMAP)
@@ -105,15 +109,22 @@ err_out:
  * 	mask: contains the permission mask
  *	fsmagic: hex value
  *
- * Return 0 to measure. For matching a DONT_MEASURE policy, no policy,
- * or other error, return an error code.
-*/
-int ima_must_measure(struct inode *inode, int mask, int function)
+ * Returns IMA_MEASURE, IMA_APPRAISE mask.
+ *
+ */
+int ima_get_action(struct inode *inode, int mask, int function)
 {
-	int must_measure;
+	int flags = IMA_MEASURE | IMA_AUDIT | IMA_APPRAISE;
+
+	if (!ima_appraise)
+		flags &= ~IMA_APPRAISE;
 
-	must_measure = ima_match_policy(inode, function, mask);
-	return must_measure ? 0 : -EACCES;
+	return ima_match_policy(inode, function, mask, flags);
+}
+
+int ima_must_measure(struct inode *inode, int mask, int function)
+{
+	return ima_match_policy(inode, function, mask, IMA_MEASURE);
 }
 
 /*
@@ -129,16 +140,24 @@ int ima_must_measure(struct inode *inode, int mask, int function)
 int ima_collect_measurement(struct integrity_iint_cache *iint,
 			    struct file *file)
 {
-	int result = -EEXIST;
+	struct inode *inode = file->f_dentry->d_inode;
+	const char *filename = file->f_dentry->d_name.name;
+	int result = 0;
 
-	if (!(iint->flags & IMA_MEASURED)) {
+	if (!(iint->flags & IMA_COLLECTED)) {
 		u64 i_version = file->f_dentry->d_inode->i_version;
 
-		memset(iint->digest, 0, IMA_DIGEST_SIZE);
-		result = ima_calc_hash(file, iint->digest);
-		if (!result)
+		iint->ima_xattr.type = IMA_XATTR_DIGEST;
+		result = ima_calc_hash(file, iint->ima_xattr.digest);
+		if (!result) {
 			iint->version = i_version;
+			iint->flags |= IMA_COLLECTED;
+		}
 	}
+	if (result)
+		integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode,
+				    filename, "collect_data", "failed",
+				    result, 0);
 	return result;
 }
 
@@ -167,6 +186,9 @@ void ima_store_measurement(struct integrity_iint_cache *iint,
 	struct ima_template_entry *entry;
 	int violation = 0;
 
+	if (iint->flags & IMA_MEASURED)
+		return;
+
 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 	if (!entry) {
 		integrity_audit_msg(AUDIT_INTEGRITY_PCR, inode, filename,
@@ -174,7 +196,7 @@ void ima_store_measurement(struct integrity_iint_cache *iint,
 		return;
 	}
 	memset(&entry->template, 0, sizeof(entry->template));
-	memcpy(entry->template.digest, iint->digest, IMA_DIGEST_SIZE);
+	memcpy(entry->template.digest, iint->ima_xattr.digest, IMA_DIGEST_SIZE);
 	strcpy(entry->template.file_name,
 	       (strlen(filename) > IMA_EVENT_NAME_LEN_MAX) ?
 	       file->f_dentry->d_name.name : filename);
@@ -185,3 +207,33 @@ void ima_store_measurement(struct integrity_iint_cache *iint,
 	if (result < 0)
 		kfree(entry);
 }
+
+void ima_audit_measurement(struct integrity_iint_cache *iint,
+			   const unsigned char *filename)
+{
+	struct audit_buffer *ab;
+	char hash[(IMA_DIGEST_SIZE * 2) + 1];
+	int i;
+
+	if (iint->flags & IMA_AUDITED)
+		return;
+
+	for (i = 0; i < IMA_DIGEST_SIZE; i++)
+		hex_byte_pack(hash + (i * 2), iint->ima_xattr.digest[i]);
+	hash[i * 2] = '\0';
+
+	ab = audit_log_start(current->audit_context, GFP_KERNEL,
+			     AUDIT_INTEGRITY_RULE);
+	if (!ab)
+		return;
+
+	audit_log_format(ab, "file=");
+	audit_log_untrustedstring(ab, filename);
+	audit_log_format(ab, " hash=");
+	audit_log_untrustedstring(ab, hash);
+
+	audit_log_task_info(ab, current);
+	audit_log_end(ab);
+
+	iint->flags |= IMA_AUDITED;
+}
diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c
new file mode 100644
index 00000000000..0aa43bde441
--- /dev/null
+++ b/security/integrity/ima/ima_appraise.c
@@ -0,0 +1,263 @@
+/*
+ * Copyright (C) 2011 IBM Corporation
+ *
+ * Author:
+ * Mimi Zohar <zohar@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 2 of the License.
+ */
+#include <linux/module.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/xattr.h>
+#include <linux/magic.h>
+#include <linux/ima.h>
+#include <linux/evm.h>
+
+#include "ima.h"
+
+static int __init default_appraise_setup(char *str)
+{
+	if (strncmp(str, "off", 3) == 0)
+		ima_appraise = 0;
+	else if (strncmp(str, "fix", 3) == 0)
+		ima_appraise = IMA_APPRAISE_FIX;
+	return 1;
+}
+
+__setup("ima_appraise=", default_appraise_setup);
+
+/*
+ * ima_must_appraise - set appraise flag
+ *
+ * Return 1 to appraise
+ */
+int ima_must_appraise(struct inode *inode, enum ima_hooks func, int mask)
+{
+	if (!ima_appraise)
+		return 0;
+
+	return ima_match_policy(inode, func, mask, IMA_APPRAISE);
+}
+
+static void ima_fix_xattr(struct dentry *dentry,
+			  struct integrity_iint_cache *iint)
+{
+	iint->ima_xattr.type = IMA_XATTR_DIGEST;
+	__vfs_setxattr_noperm(dentry, XATTR_NAME_IMA, (u8 *)&iint->ima_xattr,
+			      sizeof iint->ima_xattr, 0);
+}
+
+/*
+ * ima_appraise_measurement - appraise file measurement
+ *
+ * Call evm_verifyxattr() to verify the integrity of 'security.ima'.
+ * Assuming success, compare the xattr hash with the collected measurement.
+ *
+ * Return 0 on success, error code otherwise
+ */
+int ima_appraise_measurement(struct integrity_iint_cache *iint,
+			     struct file *file, const unsigned char *filename)
+{
+	struct dentry *dentry = file->f_dentry;
+	struct inode *inode = dentry->d_inode;
+	struct evm_ima_xattr_data *xattr_value = NULL;
+	enum integrity_status status = INTEGRITY_UNKNOWN;
+	const char *op = "appraise_data";
+	char *cause = "unknown";
+	int rc;
+
+	if (!ima_appraise)
+		return 0;
+	if (!inode->i_op->getxattr)
+		return INTEGRITY_UNKNOWN;
+
+	if (iint->flags & IMA_APPRAISED)
+		return iint->ima_status;
+
+	rc = vfs_getxattr_alloc(dentry, XATTR_NAME_IMA, (char **)&xattr_value,
+				0, GFP_NOFS);
+	if (rc <= 0) {
+		if (rc && rc != -ENODATA)
+			goto out;
+
+		cause = "missing-hash";
+		status =
+		    (inode->i_size == 0) ? INTEGRITY_PASS : INTEGRITY_NOLABEL;
+		goto out;
+	}
+
+	status = evm_verifyxattr(dentry, XATTR_NAME_IMA, xattr_value, rc, iint);
+	if ((status != INTEGRITY_PASS) && (status != INTEGRITY_UNKNOWN)) {
+		if ((status == INTEGRITY_NOLABEL)
+		    || (status == INTEGRITY_NOXATTRS))
+			cause = "missing-HMAC";
+		else if (status == INTEGRITY_FAIL)
+			cause = "invalid-HMAC";
+		goto out;
+	}
+
+	switch (xattr_value->type) {
+	case IMA_XATTR_DIGEST:
+		rc = memcmp(xattr_value->digest, iint->ima_xattr.digest,
+			    IMA_DIGEST_SIZE);
+		if (rc) {
+			cause = "invalid-hash";
+			status = INTEGRITY_FAIL;
+			print_hex_dump_bytes("security.ima: ", DUMP_PREFIX_NONE,
+					     xattr_value, sizeof(*xattr_value));
+			print_hex_dump_bytes("collected: ", DUMP_PREFIX_NONE,
+					     (u8 *)&iint->ima_xattr,
+					     sizeof iint->ima_xattr);
+			break;
+		}
+		status = INTEGRITY_PASS;
+		break;
+	case EVM_IMA_XATTR_DIGSIG:
+		iint->flags |= IMA_DIGSIG;
+		rc = integrity_digsig_verify(INTEGRITY_KEYRING_IMA,
+					     xattr_value->digest, rc - 1,
+					     iint->ima_xattr.digest,
+					     IMA_DIGEST_SIZE);
+		if (rc == -EOPNOTSUPP) {
+			status = INTEGRITY_UNKNOWN;
+		} else if (rc) {
+			cause = "invalid-signature";
+			status = INTEGRITY_FAIL;
+		} else {
+			status = INTEGRITY_PASS;
+		}
+		break;
+	default:
+		status = INTEGRITY_UNKNOWN;
+		cause = "unknown-ima-data";
+		break;
+	}
+
+out:
+	if (status != INTEGRITY_PASS) {
+		if ((ima_appraise & IMA_APPRAISE_FIX) &&
+		    (!xattr_value ||
+		     xattr_value->type != EVM_IMA_XATTR_DIGSIG)) {
+			ima_fix_xattr(dentry, iint);
+			status = INTEGRITY_PASS;
+		}
+		integrity_audit_msg(AUDIT_INTEGRITY_DATA, inode, filename,
+				    op, cause, rc, 0);
+	} else {
+		iint->flags |= IMA_APPRAISED;
+	}
+	iint->ima_status = status;
+	kfree(xattr_value);
+	return status;
+}
+
+/*
+ * ima_update_xattr - update 'security.ima' hash value
+ */
+void ima_update_xattr(struct integrity_iint_cache *iint, struct file *file)
+{
+	struct dentry *dentry = file->f_dentry;
+	int rc = 0;
+
+	/* do not collect and update hash for digital signatures */
+	if (iint->flags & IMA_DIGSIG)
+		return;
+
+	rc = ima_collect_measurement(iint, file);
+	if (rc < 0)
+		return;
+
+	ima_fix_xattr(dentry, iint);
+}
+
+/**
+ * ima_inode_post_setattr - reflect file metadata changes
+ * @dentry: pointer to the affected dentry
+ *
+ * Changes to a dentry's metadata might result in needing to appraise.
+ *
+ * This function is called from notify_change(), which expects the caller
+ * to lock the inode's i_mutex.
+ */
+void ima_inode_post_setattr(struct dentry *dentry)
+{
+	struct inode *inode = dentry->d_inode;
+	struct integrity_iint_cache *iint;
+	int must_appraise, rc;
+
+	if (!ima_initialized || !ima_appraise || !S_ISREG(inode->i_mode)
+	    || !inode->i_op->removexattr)
+		return;
+
+	must_appraise = ima_must_appraise(inode, MAY_ACCESS, POST_SETATTR);
+	iint = integrity_iint_find(inode);
+	if (iint) {
+		if (must_appraise)
+			iint->flags |= IMA_APPRAISE;
+		else
+			iint->flags &= ~(IMA_APPRAISE | IMA_APPRAISED);
+	}
+	if (!must_appraise)
+		rc = inode->i_op->removexattr(dentry, XATTR_NAME_IMA);
+	return;
+}
+
+/*
+ * ima_protect_xattr - protect 'security.ima'
+ *
+ * Ensure that not just anyone can modify or remove 'security.ima'.
+ */
+static int ima_protect_xattr(struct dentry *dentry, const char *xattr_name,
+			     const void *xattr_value, size_t xattr_value_len)
+{
+	if (strcmp(xattr_name, XATTR_NAME_IMA) == 0) {
+		if (!capable(CAP_SYS_ADMIN))
+			return -EPERM;
+		return 1;
+	}
+	return 0;
+}
+
+static void ima_reset_appraise_flags(struct inode *inode)
+{
+	struct integrity_iint_cache *iint;
+
+	if (!ima_initialized || !ima_appraise || !S_ISREG(inode->i_mode))
+		return;
+
+	iint = integrity_iint_find(inode);
+	if (!iint)
+		return;
+
+	iint->flags &= ~IMA_DONE_MASK;
+	return;
+}
+
+int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name,
+		       const void *xattr_value, size_t xattr_value_len)
+{
+	int result;
+
+	result = ima_protect_xattr(dentry, xattr_name, xattr_value,
+				   xattr_value_len);
+	if (result == 1) {
+		ima_reset_appraise_flags(dentry->d_inode);
+		result = 0;
+	}
+	return result;
+}
+
+int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name)
+{
+	int result;
+
+	result = ima_protect_xattr(dentry, xattr_name, NULL, 0);
+	if (result == 1) {
+		ima_reset_appraise_flags(dentry->d_inode);
+		result = 0;
+	}
+	return result;
+}
diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c
index 9b3ade7468b..b21ee5b5495 100644
--- a/security/integrity/ima/ima_crypto.c
+++ b/security/integrity/ima/ima_crypto.c
@@ -48,7 +48,7 @@ int ima_calc_hash(struct file *file, char *digest)
 	struct scatterlist sg[1];
 	loff_t i_size, offset = 0;
 	char *rbuf;
-	int rc;
+	int rc, read = 0;
 
 	rc = init_desc(&desc);
 	if (rc != 0)
@@ -59,6 +59,10 @@ int ima_calc_hash(struct file *file, char *digest)
 		rc = -ENOMEM;
 		goto out;
 	}
+	if (!(file->f_mode & FMODE_READ)) {
+		file->f_mode |= FMODE_READ;
+		read = 1;
+	}
 	i_size = i_size_read(file->f_dentry->d_inode);
 	while (offset < i_size) {
 		int rbuf_len;
@@ -80,6 +84,8 @@ int ima_calc_hash(struct file *file, char *digest)
 	kfree(rbuf);
 	if (!rc)
 		rc = crypto_hash_final(&desc, digest);
+	if (read)
+		file->f_mode &= ~FMODE_READ;
 out:
 	crypto_free_hash(desc.tfm);
 	return rc;
diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c
index be8294915cf..73c9a268253 100644
--- a/security/integrity/ima/ima_main.c
+++ b/security/integrity/ima/ima_main.c
@@ -22,12 +22,19 @@
 #include <linux/mount.h>
 #include <linux/mman.h>
 #include <linux/slab.h>
+#include <linux/xattr.h>
 #include <linux/ima.h>
 
 #include "ima.h"
 
 int ima_initialized;
 
+#ifdef CONFIG_IMA_APPRAISE
+int ima_appraise = IMA_APPRAISE_ENFORCE;
+#else
+int ima_appraise;
+#endif
+
 char *ima_hash = "sha1";
 static int __init hash_setup(char *str)
 {
@@ -52,7 +59,7 @@ static void ima_rdwr_violation_check(struct file *file)
 	struct dentry *dentry = file->f_path.dentry;
 	struct inode *inode = dentry->d_inode;
 	fmode_t mode = file->f_mode;
-	int rc;
+	int must_measure;
 	bool send_tomtou = false, send_writers = false;
 	unsigned char *pathname = NULL, *pathbuf = NULL;
 
@@ -67,8 +74,8 @@ static void ima_rdwr_violation_check(struct file *file)
 		goto out;
 	}
 
-	rc = ima_must_measure(inode, MAY_READ, FILE_CHECK);
-	if (rc < 0)
+	must_measure = ima_must_measure(inode, MAY_READ, FILE_CHECK);
+	if (!must_measure)
 		goto out;
 
 	if (atomic_read(&inode->i_writecount) > 0)
@@ -100,17 +107,21 @@ out:
 }
 
 static void ima_check_last_writer(struct integrity_iint_cache *iint,
-				  struct inode *inode,
-				  struct file *file)
+				  struct inode *inode, struct file *file)
 {
 	fmode_t mode = file->f_mode;
 
-	mutex_lock(&iint->mutex);
-	if (mode & FMODE_WRITE &&
-	    atomic_read(&inode->i_writecount) == 1 &&
-	    iint->version != inode->i_version)
-		iint->flags &= ~IMA_MEASURED;
-	mutex_unlock(&iint->mutex);
+	if (!(mode & FMODE_WRITE))
+		return;
+
+	mutex_lock(&inode->i_mutex);
+	if (atomic_read(&inode->i_writecount) == 1 &&
+	    iint->version != inode->i_version) {
+		iint->flags &= ~IMA_DONE_MASK;
+		if (iint->flags & IMA_APPRAISE)
+			ima_update_xattr(iint, file);
+	}
+	mutex_unlock(&inode->i_mutex);
 }
 
 /**
@@ -140,28 +151,37 @@ static int process_measurement(struct file *file, const unsigned char *filename,
 	struct inode *inode = file->f_dentry->d_inode;
 	struct integrity_iint_cache *iint;
 	unsigned char *pathname = NULL, *pathbuf = NULL;
-	int rc = 0;
+	int rc = -ENOMEM, action, must_appraise;
 
 	if (!ima_initialized || !S_ISREG(inode->i_mode))
 		return 0;
 
-	rc = ima_must_measure(inode, mask, function);
-	if (rc != 0)
-		return rc;
-retry:
-	iint = integrity_iint_find(inode);
-	if (!iint) {
-		rc = integrity_inode_alloc(inode);
-		if (!rc || rc == -EEXIST)
-			goto retry;
-		return rc;
-	}
+	/* Determine if in appraise/audit/measurement policy,
+	 * returns IMA_MEASURE, IMA_APPRAISE, IMA_AUDIT bitmask.  */
+	action = ima_get_action(inode, mask, function);
+	if (!action)
+		return 0;
 
-	mutex_lock(&iint->mutex);
+	must_appraise = action & IMA_APPRAISE;
 
-	rc = iint->flags & IMA_MEASURED ? 1 : 0;
-	if (rc != 0)
+	mutex_lock(&inode->i_mutex);
+
+	iint = integrity_inode_get(inode);
+	if (!iint)
+		goto out;
+
+	/* Determine if already appraised/measured based on bitmask
+	 * (IMA_MEASURE, IMA_MEASURED, IMA_APPRAISE, IMA_APPRAISED,
+	 *  IMA_AUDIT, IMA_AUDITED) */
+	iint->flags |= action;
+	action &= ~((iint->flags & IMA_DONE_MASK) >> 1);
+
+	/* Nothing to do, just return existing appraised status */
+	if (!action) {
+		if (iint->flags & IMA_APPRAISED)
+			rc = iint->ima_status;
 		goto out;
+	}
 
 	rc = ima_collect_measurement(iint, file);
 	if (rc != 0)
@@ -177,11 +197,18 @@ retry:
 				pathname = NULL;
 		}
 	}
-	ima_store_measurement(iint, file, !pathname ? filename : pathname);
+	if (action & IMA_MEASURE)
+		ima_store_measurement(iint, file,
+				      !pathname ? filename : pathname);
+	if (action & IMA_APPRAISE)
+		rc = ima_appraise_measurement(iint, file,
+					      !pathname ? filename : pathname);
+	if (action & IMA_AUDIT)
+		ima_audit_measurement(iint, !pathname ? filename : pathname);
 	kfree(pathbuf);
 out:
-	mutex_unlock(&iint->mutex);
-	return rc;
+	mutex_unlock(&inode->i_mutex);
+	return (rc && must_appraise) ? -EACCES : 0;
 }
 
 /**
@@ -197,14 +224,14 @@ out:
  */
 int ima_file_mmap(struct file *file, unsigned long prot)
 {
-	int rc;
+	int rc = 0;
 
 	if (!file)
 		return 0;
 	if (prot & PROT_EXEC)
 		rc = process_measurement(file, file->f_dentry->d_name.name,
 					 MAY_EXEC, FILE_MMAP);
-	return 0;
+	return (ima_appraise & IMA_APPRAISE_ENFORCE) ? rc : 0;
 }
 
 /**
@@ -228,7 +255,7 @@ int ima_bprm_check(struct linux_binprm *bprm)
 				 (strcmp(bprm->filename, bprm->interp) == 0) ?
 				 bprm->filename : bprm->interp,
 				 MAY_EXEC, BPRM_CHECK);
-	return 0;
+	return (ima_appraise & IMA_APPRAISE_ENFORCE) ? rc : 0;
 }
 
 /**
@@ -249,7 +276,7 @@ int ima_file_check(struct file *file, int mask)
 	rc = process_measurement(file, file->f_dentry->d_name.name,
 				 mask & (MAY_READ | MAY_WRITE | MAY_EXEC),
 				 FILE_CHECK);
-	return 0;
+	return (ima_appraise & IMA_APPRAISE_ENFORCE) ? rc : 0;
 }
 EXPORT_SYMBOL_GPL(ima_file_check);
 
diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c
index c84df05180c..c7dacd2eab7 100644
--- a/security/integrity/ima/ima_policy.c
+++ b/security/integrity/ima/ima_policy.c
@@ -24,22 +24,29 @@
 #define IMA_MASK 	0x0002
 #define IMA_FSMAGIC	0x0004
 #define IMA_UID		0x0008
+#define IMA_FOWNER	0x0010
 
-enum ima_action { UNKNOWN = -1, DONT_MEASURE = 0, MEASURE };
+#define UNKNOWN		0
+#define MEASURE		0x0001	/* same as IMA_MEASURE */
+#define DONT_MEASURE	0x0002
+#define APPRAISE	0x0004	/* same as IMA_APPRAISE */
+#define DONT_APPRAISE	0x0008
+#define AUDIT		0x0040
 
 #define MAX_LSM_RULES 6
 enum lsm_rule_types { LSM_OBJ_USER, LSM_OBJ_ROLE, LSM_OBJ_TYPE,
 	LSM_SUBJ_USER, LSM_SUBJ_ROLE, LSM_SUBJ_TYPE
 };
 
-struct ima_measure_rule_entry {
+struct ima_rule_entry {
 	struct list_head list;
-	enum ima_action action;
+	int action;
 	unsigned int flags;
 	enum ima_hooks func;
 	int mask;
 	unsigned long fsmagic;
 	kuid_t uid;
+	kuid_t fowner;
 	struct {
 		void *rule;	/* LSM file metadata specific */
 		int type;	/* audit type */
@@ -48,7 +55,7 @@ struct ima_measure_rule_entry {
 
 /*
  * Without LSM specific knowledge, the default policy can only be
- * written in terms of .action, .func, .mask, .fsmagic, and .uid
+ * written in terms of .action, .func, .mask, .fsmagic, .uid, and .fowner
  */
 
 /*
@@ -57,7 +64,7 @@ struct ima_measure_rule_entry {
  * normal users can easily run the machine out of memory simply building
  * and running executables.
  */
-static struct ima_measure_rule_entry default_rules[] = {
+static struct ima_rule_entry default_rules[] = {
 	{.action = DONT_MEASURE,.fsmagic = PROC_SUPER_MAGIC,.flags = IMA_FSMAGIC},
 	{.action = DONT_MEASURE,.fsmagic = SYSFS_MAGIC,.flags = IMA_FSMAGIC},
 	{.action = DONT_MEASURE,.fsmagic = DEBUGFS_MAGIC,.flags = IMA_FSMAGIC},
@@ -75,19 +82,41 @@ static struct ima_measure_rule_entry default_rules[] = {
 	 .flags = IMA_FUNC | IMA_MASK | IMA_UID},
 };
 
-static LIST_HEAD(measure_default_rules);
-static LIST_HEAD(measure_policy_rules);
-static struct list_head *ima_measure;
+static struct ima_rule_entry default_appraise_rules[] = {
+	{.action = DONT_APPRAISE,.fsmagic = PROC_SUPER_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = SYSFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = DEBUGFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = TMPFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = RAMFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = DEVPTS_SUPER_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = BINFMTFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = SECURITYFS_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = SELINUX_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = DONT_APPRAISE,.fsmagic = CGROUP_SUPER_MAGIC,.flags = IMA_FSMAGIC},
+	{.action = APPRAISE,.fowner = GLOBAL_ROOT_UID,.flags = IMA_FOWNER},
+};
+
+static LIST_HEAD(ima_default_rules);
+static LIST_HEAD(ima_policy_rules);
+static struct list_head *ima_rules;
 
-static DEFINE_MUTEX(ima_measure_mutex);
+static DEFINE_MUTEX(ima_rules_mutex);
 
 static bool ima_use_tcb __initdata;
-static int __init default_policy_setup(char *str)
+static int __init default_measure_policy_setup(char *str)
 {
 	ima_use_tcb = 1;
 	return 1;
 }
-__setup("ima_tcb", default_policy_setup);
+__setup("ima_tcb", default_measure_policy_setup);
+
+static bool ima_use_appraise_tcb __initdata;
+static int __init default_appraise_policy_setup(char *str)
+{
+	ima_use_appraise_tcb = 1;
+	return 1;
+}
+__setup("ima_appraise_tcb", default_appraise_policy_setup);
 
 /**
  * ima_match_rules - determine whether an inode matches the measure rule.
@@ -98,7 +127,7 @@ __setup("ima_tcb", default_policy_setup);
  *
  * Returns true on rule match, false on failure.
  */
-static bool ima_match_rules(struct ima_measure_rule_entry *rule,
+static bool ima_match_rules(struct ima_rule_entry *rule,
 			    struct inode *inode, enum ima_hooks func, int mask)
 {
 	struct task_struct *tsk = current;
@@ -114,6 +143,8 @@ static bool ima_match_rules(struct ima_measure_rule_entry *rule,
 		return false;
 	if ((rule->flags & IMA_UID) && !uid_eq(rule->uid, cred->uid))
 		return false;
+	if ((rule->flags & IMA_FOWNER) && !uid_eq(rule->fowner, inode->i_uid))
+		return false;
 	for (i = 0; i < MAX_LSM_RULES; i++) {
 		int rc = 0;
 		u32 osid, sid;
@@ -163,39 +194,61 @@ static bool ima_match_rules(struct ima_measure_rule_entry *rule,
  * as elements in the list are never deleted, nor does the list
  * change.)
  */
-int ima_match_policy(struct inode *inode, enum ima_hooks func, int mask)
+int ima_match_policy(struct inode *inode, enum ima_hooks func, int mask,
+		     int flags)
 {
-	struct ima_measure_rule_entry *entry;
+	struct ima_rule_entry *entry;
+	int action = 0, actmask = flags | (flags << 1);
+
+	list_for_each_entry(entry, ima_rules, list) {
+
+		if (!(entry->action & actmask))
+			continue;
+
+		if (!ima_match_rules(entry, inode, func, mask))
+			continue;
 
-	list_for_each_entry(entry, ima_measure, list) {
-		bool rc;
+		action |= entry->action & IMA_DO_MASK;
+		if (entry->action & IMA_DO_MASK)
+			actmask &= ~(entry->action | entry->action << 1);
+		else
+			actmask &= ~(entry->action | entry->action >> 1);
 
-		rc = ima_match_rules(entry, inode, func, mask);
-		if (rc)
-			return entry->action;
+		if (!actmask)
+			break;
 	}
-	return 0;
+
+	return action;
 }
 
 /**
  * ima_init_policy - initialize the default measure rules.
  *
- * ima_measure points to either the measure_default_rules or the
- * the new measure_policy_rules.
+ * ima_rules points to either the ima_default_rules or the
+ * the new ima_policy_rules.
  */
 void __init ima_init_policy(void)
 {
-	int i, entries;
+	int i, measure_entries, appraise_entries;
 
 	/* if !ima_use_tcb set entries = 0 so we load NO default rules */
-	if (ima_use_tcb)
-		entries = ARRAY_SIZE(default_rules);
-	else
-		entries = 0;
-
-	for (i = 0; i < entries; i++)
-		list_add_tail(&default_rules[i].list, &measure_default_rules);
-	ima_measure = &measure_default_rules;
+	measure_entries = ima_use_tcb ? ARRAY_SIZE(default_rules) : 0;
+	appraise_entries = ima_use_appraise_tcb ?
+			 ARRAY_SIZE(default_appraise_rules) : 0;
+	
+	for (i = 0; i < measure_entries + appraise_entries; i++) {
+		if (i < measure_entries)
+			list_add_tail(&default_rules[i].list,
+				      &ima_default_rules);
+		else {
+			int j = i - measure_entries;
+
+			list_add_tail(&default_appraise_rules[j].list,
+				      &ima_default_rules);
+		}
+	}
+
+	ima_rules = &ima_default_rules;
 }
 
 /**
@@ -212,8 +265,8 @@ void ima_update_policy(void)
 	int result = 1;
 	int audit_info = 0;
 
-	if (ima_measure == &measure_default_rules) {
-		ima_measure = &measure_policy_rules;
+	if (ima_rules == &ima_default_rules) {
+		ima_rules = &ima_policy_rules;
 		cause = "complete";
 		result = 0;
 	}
@@ -224,14 +277,19 @@ void ima_update_policy(void)
 enum {
 	Opt_err = -1,
 	Opt_measure = 1, Opt_dont_measure,
+	Opt_appraise, Opt_dont_appraise,
+	Opt_audit,
 	Opt_obj_user, Opt_obj_role, Opt_obj_type,
 	Opt_subj_user, Opt_subj_role, Opt_subj_type,
-	Opt_func, Opt_mask, Opt_fsmagic, Opt_uid
+	Opt_func, Opt_mask, Opt_fsmagic, Opt_uid, Opt_fowner
 };
 
 static match_table_t policy_tokens = {
 	{Opt_measure, "measure"},
 	{Opt_dont_measure, "dont_measure"},
+	{Opt_appraise, "appraise"},
+	{Opt_dont_appraise, "dont_appraise"},
+	{Opt_audit, "audit"},
 	{Opt_obj_user, "obj_user=%s"},
 	{Opt_obj_role, "obj_role=%s"},
 	{Opt_obj_type, "obj_type=%s"},
@@ -242,10 +300,11 @@ static match_table_t policy_tokens = {
 	{Opt_mask, "mask=%s"},
 	{Opt_fsmagic, "fsmagic=%s"},
 	{Opt_uid, "uid=%s"},
+	{Opt_fowner, "fowner=%s"},
 	{Opt_err, NULL}
 };
 
-static int ima_lsm_rule_init(struct ima_measure_rule_entry *entry,
+static int ima_lsm_rule_init(struct ima_rule_entry *entry,
 			     char *args, int lsm_rule, int audit_type)
 {
 	int result;
@@ -269,7 +328,7 @@ static void ima_log_string(struct audit_buffer *ab, char *key, char *value)
 	audit_log_format(ab, " ");
 }
 
-static int ima_parse_rule(char *rule, struct ima_measure_rule_entry *entry)
+static int ima_parse_rule(char *rule, struct ima_rule_entry *entry)
 {
 	struct audit_buffer *ab;
 	char *p;
@@ -278,6 +337,7 @@ static int ima_parse_rule(char *rule, struct ima_measure_rule_entry *entry)
 	ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_INTEGRITY_RULE);
 
 	entry->uid = INVALID_UID;
+	entry->fowner = INVALID_UID;
 	entry->action = UNKNOWN;
 	while ((p = strsep(&rule, " \t")) != NULL) {
 		substring_t args[MAX_OPT_ARGS];
@@ -306,11 +366,35 @@ static int ima_parse_rule(char *rule, struct ima_measure_rule_entry *entry)
 
 			entry->action = DONT_MEASURE;
 			break;
+		case Opt_appraise:
+			ima_log_string(ab, "action", "appraise");
+
+			if (entry->action != UNKNOWN)
+				result = -EINVAL;
+
+			entry->action = APPRAISE;
+			break;
+		case Opt_dont_appraise:
+			ima_log_string(ab, "action", "dont_appraise");
+
+			if (entry->action != UNKNOWN)
+				result = -EINVAL;
+
+			entry->action = DONT_APPRAISE;
+			break;
+		case Opt_audit:
+			ima_log_string(ab, "action", "audit");
+
+			if (entry->action != UNKNOWN)
+				result = -EINVAL;
+
+			entry->action = AUDIT;
+			break;
 		case Opt_func:
 			ima_log_string(ab, "func", args[0].from);
 
 			if (entry->func)
-				result  = -EINVAL;
+				result = -EINVAL;
 
 			if (strcmp(args[0].from, "FILE_CHECK") == 0)
 				entry->func = FILE_CHECK;
@@ -375,6 +459,23 @@ static int ima_parse_rule(char *rule, struct ima_measure_rule_entry *entry)
 					entry->flags |= IMA_UID;
 			}
 			break;
+		case Opt_fowner:
+			ima_log_string(ab, "fowner", args[0].from);
+
+			if (uid_valid(entry->fowner)) {
+				result = -EINVAL;
+				break;
+			}
+
+			result = strict_strtoul(args[0].from, 10, &lnum);
+			if (!result) {
+				entry->fowner = make_kuid(current_user_ns(), (uid_t)lnum);
+				if (!uid_valid(entry->fowner) || (((uid_t)lnum) != lnum))
+					result = -EINVAL;
+				else
+					entry->flags |= IMA_FOWNER;
+			}
+			break;
 		case Opt_obj_user:
 			ima_log_string(ab, "obj_user", args[0].from);
 			result = ima_lsm_rule_init(entry, args[0].from,
@@ -426,7 +527,7 @@ static int ima_parse_rule(char *rule, struct ima_measure_rule_entry *entry)
 }
 
 /**
- * ima_parse_add_rule - add a rule to measure_policy_rules
+ * ima_parse_add_rule - add a rule to ima_policy_rules
  * @rule - ima measurement policy rule
  *
  * Uses a mutex to protect the policy list from multiple concurrent writers.
@@ -436,12 +537,12 @@ ssize_t ima_parse_add_rule(char *rule)
 {
 	const char *op = "update_policy";
 	char *p;
-	struct ima_measure_rule_entry *entry;
+	struct ima_rule_entry *entry;
 	ssize_t result, len;
 	int audit_info = 0;
 
 	/* Prevent installed policy from changing */
-	if (ima_measure != &measure_default_rules) {
+	if (ima_rules != &ima_default_rules) {
 		integrity_audit_msg(AUDIT_INTEGRITY_STATUS, NULL,
 				    NULL, op, "already exists",
 				    -EACCES, audit_info);
@@ -474,9 +575,9 @@ ssize_t ima_parse_add_rule(char *rule)
 		return result;
 	}
 
-	mutex_lock(&ima_measure_mutex);
-	list_add_tail(&entry->list, &measure_policy_rules);
-	mutex_unlock(&ima_measure_mutex);
+	mutex_lock(&ima_rules_mutex);
+	list_add_tail(&entry->list, &ima_policy_rules);
+	mutex_unlock(&ima_rules_mutex);
 
 	return len;
 }
@@ -484,12 +585,12 @@ ssize_t ima_parse_add_rule(char *rule)
 /* ima_delete_rules called to cleanup invalid policy */
 void ima_delete_rules(void)
 {
-	struct ima_measure_rule_entry *entry, *tmp;
+	struct ima_rule_entry *entry, *tmp;
 
-	mutex_lock(&ima_measure_mutex);
-	list_for_each_entry_safe(entry, tmp, &measure_policy_rules, list) {
+	mutex_lock(&ima_rules_mutex);
+	list_for_each_entry_safe(entry, tmp, &ima_policy_rules, list) {
 		list_del(&entry->list);
 		kfree(entry);
 	}
-	mutex_unlock(&ima_measure_mutex);
+	mutex_unlock(&ima_rules_mutex);
 }
diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h
index 7a25ecec5aa..e9db763a875 100644
--- a/security/integrity/integrity.h
+++ b/security/integrity/integrity.h
@@ -15,8 +15,22 @@
 #include <linux/integrity.h>
 #include <crypto/sha.h>
 
+/* iint action cache flags */
+#define IMA_MEASURE		0x0001
+#define IMA_MEASURED		0x0002
+#define IMA_APPRAISE		0x0004
+#define IMA_APPRAISED		0x0008
+/*#define IMA_COLLECT		0x0010  do not use this flag */
+#define IMA_COLLECTED		0x0020
+#define IMA_AUDIT		0x0040
+#define IMA_AUDITED		0x0080
+
 /* iint cache flags */
-#define IMA_MEASURED		0x01
+#define IMA_DIGSIG		0x0100
+
+#define IMA_DO_MASK		(IMA_MEASURE | IMA_APPRAISE | IMA_AUDIT)
+#define IMA_DONE_MASK		(IMA_MEASURED | IMA_APPRAISED | IMA_AUDITED \
+				 | IMA_COLLECTED)
 
 enum evm_ima_xattr_type {
 	IMA_XATTR_DIGEST = 0x01,
@@ -34,9 +48,9 @@ struct integrity_iint_cache {
 	struct rb_node rb_node; /* rooted in integrity_iint_tree */
 	struct inode *inode;	/* back pointer to inode in question */
 	u64 version;		/* track inode changes */
-	unsigned char flags;
-	u8 digest[SHA1_DIGEST_SIZE];
-	struct mutex mutex;	/* protects: version, flags, digest */
+	unsigned short flags;
+	struct evm_ima_xattr_data ima_xattr;
+	enum integrity_status ima_status;
 	enum integrity_status evm_status;
 };
 
diff --git a/security/keys/trusted.c b/security/keys/trusted.c
index 2d5d041f204..3f163d0489a 100644
--- a/security/keys/trusted.c
+++ b/security/keys/trusted.c
@@ -369,38 +369,6 @@ static int trusted_tpm_send(const u32 chip_num, unsigned char *cmd,
 }
 
 /*
- * get a random value from TPM
- */
-static int tpm_get_random(struct tpm_buf *tb, unsigned char *buf, uint32_t len)
-{
-	int ret;
-
-	INIT_BUF(tb);
-	store16(tb, TPM_TAG_RQU_COMMAND);
-	store32(tb, TPM_GETRANDOM_SIZE);
-	store32(tb, TPM_ORD_GETRANDOM);
-	store32(tb, len);
-	ret = trusted_tpm_send(TPM_ANY_NUM, tb->data, sizeof tb->data);
-	if (!ret)
-		memcpy(buf, tb->data + TPM_GETRANDOM_SIZE, len);
-	return ret;
-}
-
-static int my_get_random(unsigned char *buf, int len)
-{
-	struct tpm_buf *tb;
-	int ret;
-
-	tb = kmalloc(sizeof *tb, GFP_KERNEL);
-	if (!tb)
-		return -ENOMEM;
-	ret = tpm_get_random(tb, buf, len);
-
-	kfree(tb);
-	return ret;
-}
-
-/*
  * Lock a trusted key, by extending a selected PCR.
  *
  * Prevents a trusted key that is sealed to PCRs from being accessed.
@@ -413,8 +381,8 @@ static int pcrlock(const int pcrnum)
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
-	ret = my_get_random(hash, SHA1_DIGEST_SIZE);
-	if (ret < 0)
+	ret = tpm_get_random(TPM_ANY_NUM, hash, SHA1_DIGEST_SIZE);
+	if (ret != SHA1_DIGEST_SIZE)
 		return ret;
 	return tpm_pcr_extend(TPM_ANY_NUM, pcrnum, hash) ? -EINVAL : 0;
 }
@@ -429,8 +397,8 @@ static int osap(struct tpm_buf *tb, struct osapsess *s,
 	unsigned char ononce[TPM_NONCE_SIZE];
 	int ret;
 
-	ret = tpm_get_random(tb, ononce, TPM_NONCE_SIZE);
-	if (ret < 0)
+	ret = tpm_get_random(TPM_ANY_NUM, ononce, TPM_NONCE_SIZE);
+	if (ret != TPM_NONCE_SIZE)
 		return ret;
 
 	INIT_BUF(tb);
@@ -524,8 +492,8 @@ static int tpm_seal(struct tpm_buf *tb, uint16_t keytype,
 	if (ret < 0)
 		goto out;
 
-	ret = tpm_get_random(tb, td->nonceodd, TPM_NONCE_SIZE);
-	if (ret < 0)
+	ret = tpm_get_random(TPM_ANY_NUM, td->nonceodd, TPM_NONCE_SIZE);
+	if (ret != TPM_NONCE_SIZE)
 		goto out;
 	ordinal = htonl(TPM_ORD_SEAL);
 	datsize = htonl(datalen);
@@ -634,8 +602,8 @@ static int tpm_unseal(struct tpm_buf *tb,
 
 	ordinal = htonl(TPM_ORD_UNSEAL);
 	keyhndl = htonl(SRKHANDLE);
-	ret = tpm_get_random(tb, nonceodd, TPM_NONCE_SIZE);
-	if (ret < 0) {
+	ret = tpm_get_random(TPM_ANY_NUM, nonceodd, TPM_NONCE_SIZE);
+	if (ret != TPM_NONCE_SIZE) {
 		pr_info("trusted_key: tpm_get_random failed (%d)\n", ret);
 		return ret;
 	}
@@ -935,6 +903,7 @@ static int trusted_instantiate(struct key *key, const void *data,
 	char *datablob;
 	int ret = 0;
 	int key_cmd;
+	size_t key_len;
 
 	if (datalen <= 0 || datalen > 32767 || !data)
 		return -EINVAL;
@@ -974,8 +943,9 @@ static int trusted_instantiate(struct key *key, const void *data,
 			pr_info("trusted_key: key_unseal failed (%d)\n", ret);
 		break;
 	case Opt_new:
-		ret = my_get_random(payload->key, payload->key_len);
-		if (ret < 0) {
+		key_len = payload->key_len;
+		ret = tpm_get_random(TPM_ANY_NUM, payload->key, key_len);
+		if (ret != key_len) {
 			pr_info("trusted_key: key_create failed (%d)\n", ret);
 			goto out;
 		}
diff --git a/security/security.c b/security/security.c
index f9a2f2ef245..3724029d0f6 100644
--- a/security/security.c
+++ b/security/security.c
@@ -136,11 +136,23 @@ int __init register_security(struct security_operations *ops)
 
 int security_ptrace_access_check(struct task_struct *child, unsigned int mode)
 {
+#ifdef CONFIG_SECURITY_YAMA_STACKED
+	int rc;
+	rc = yama_ptrace_access_check(child, mode);
+	if (rc)
+		return rc;
+#endif
 	return security_ops->ptrace_access_check(child, mode);
 }
 
 int security_ptrace_traceme(struct task_struct *parent)
 {
+#ifdef CONFIG_SECURITY_YAMA_STACKED
+	int rc;
+	rc = yama_ptrace_traceme(parent);
+	if (rc)
+		return rc;
+#endif
 	return security_ops->ptrace_traceme(parent);
 }
 
@@ -561,6 +573,9 @@ int security_inode_setxattr(struct dentry *dentry, const char *name,
 	ret = security_ops->inode_setxattr(dentry, name, value, size, flags);
 	if (ret)
 		return ret;
+	ret = ima_inode_setxattr(dentry, name, value, size);
+	if (ret)
+		return ret;
 	return evm_inode_setxattr(dentry, name, value, size);
 }
 
@@ -596,6 +611,9 @@ int security_inode_removexattr(struct dentry *dentry, const char *name)
 	ret = security_ops->inode_removexattr(dentry, name);
 	if (ret)
 		return ret;
+	ret = ima_inode_removexattr(dentry, name);
+	if (ret)
+		return ret;
 	return evm_inode_removexattr(dentry, name);
 }
 
@@ -761,6 +779,9 @@ int security_task_create(unsigned long clone_flags)
 
 void security_task_free(struct task_struct *task)
 {
+#ifdef CONFIG_SECURITY_YAMA_STACKED
+	yama_task_free(task);
+#endif
 	security_ops->task_free(task);
 }
 
@@ -876,6 +897,12 @@ int security_task_wait(struct task_struct *p)
 int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
 			 unsigned long arg4, unsigned long arg5)
 {
+#ifdef CONFIG_SECURITY_YAMA_STACKED
+	int rc;
+	rc = yama_task_prctl(option, arg2, arg3, arg4, arg5);
+	if (rc != -ENOSYS)
+		return rc;
+#endif
 	return security_ops->task_prctl(option, arg2, arg3, arg4, arg5);
 }
 
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 6c77f63c759..651d8456611 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -2088,15 +2088,19 @@ static int selinux_bprm_secureexec(struct linux_binprm *bprm)
 	return (atsecure || cap_bprm_secureexec(bprm));
 }
 
+static int match_file(const void *p, struct file *file, unsigned fd)
+{
+	return file_has_perm(p, file, file_to_av(file)) ? fd + 1 : 0;
+}
+
 /* Derived from fs/exec.c:flush_old_files. */
 static inline void flush_unauthorized_files(const struct cred *cred,
 					    struct files_struct *files)
 {
 	struct file *file, *devnull = NULL;
 	struct tty_struct *tty;
-	struct fdtable *fdt;
-	long j = -1;
 	int drop_tty = 0;
+	unsigned n;
 
 	tty = get_current_tty();
 	if (tty) {
@@ -2123,58 +2127,23 @@ static inline void flush_unauthorized_files(const struct cred *cred,
 		no_tty();
 
 	/* Revalidate access to inherited open files. */
-	spin_lock(&files->file_lock);
-	for (;;) {
-		unsigned long set, i;
-		int fd;
-
-		j++;
-		i = j * BITS_PER_LONG;
-		fdt = files_fdtable(files);
-		if (i >= fdt->max_fds)
-			break;
-		set = fdt->open_fds[j];
-		if (!set)
-			continue;
-		spin_unlock(&files->file_lock);
-		for ( ; set ; i++, set >>= 1) {
-			if (set & 1) {
-				file = fget(i);
-				if (!file)
-					continue;
-				if (file_has_perm(cred,
-						  file,
-						  file_to_av(file))) {
-					sys_close(i);
-					fd = get_unused_fd();
-					if (fd != i) {
-						if (fd >= 0)
-							put_unused_fd(fd);
-						fput(file);
-						continue;
-					}
-					if (devnull) {
-						get_file(devnull);
-					} else {
-						devnull = dentry_open(
-							&selinux_null,
-							O_RDWR, cred);
-						if (IS_ERR(devnull)) {
-							devnull = NULL;
-							put_unused_fd(fd);
-							fput(file);
-							continue;
-						}
-					}
-					fd_install(fd, devnull);
-				}
-				fput(file);
-			}
-		}
-		spin_lock(&files->file_lock);
+	n = iterate_fd(files, 0, match_file, cred);
+	if (!n) /* none found? */
+		return;
 
+	devnull = dentry_open(&selinux_null, O_RDWR, cred);
+	if (!IS_ERR(devnull)) {
+		/* replace all the matching ones with this */
+		do {
+			replace_fd(n - 1, get_file(devnull), 0);
+		} while ((n = iterate_fd(files, n, match_file, cred)) != 0);
+		fput(devnull);
+	} else {
+		/* just close all the matching ones */
+		do {
+			replace_fd(n - 1, NULL, 0);
+		} while ((n = iterate_fd(files, n, match_file, cred)) != 0);
 	}
-	spin_unlock(&files->file_lock);
 }
 
 /*
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 8221514cc99..2874c731678 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -1691,40 +1691,19 @@ static int smack_task_kill(struct task_struct *p, struct siginfo *info,
  * smack_task_wait - Smack access check for waiting
  * @p: task to wait for
  *
- * Returns 0 if current can wait for p, error code otherwise
+ * Returns 0
  */
 static int smack_task_wait(struct task_struct *p)
 {
-	struct smk_audit_info ad;
-	char *sp = smk_of_current();
-	char *tsp = smk_of_forked(task_security(p));
-	int rc;
-
-	/* we don't log here, we can be overriden */
-	rc = smk_access(tsp, sp, MAY_WRITE, NULL);
-	if (rc == 0)
-		goto out_log;
-
 	/*
-	 * Allow the operation to succeed if either task
-	 * has privilege to perform operations that might
-	 * account for the smack labels having gotten to
-	 * be different in the first place.
-	 *
-	 * This breaks the strict subject/object access
-	 * control ideal, taking the object's privilege
-	 * state into account in the decision as well as
-	 * the smack value.
+	 * Allow the operation to succeed.
+	 * Zombies are bad.
+	 * In userless environments (e.g. phones) programs
+	 * get marked with SMACK64EXEC and even if the parent
+	 * and child shouldn't be talking the parent still
+	 * may expect to know when the child exits.
 	 */
-	if (smack_privileged(CAP_MAC_OVERRIDE) ||
-	    has_capability(p, CAP_MAC_OVERRIDE))
-		rc = 0;
-	/* we log only if we didn't get overriden */
- out_log:
-	smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_TASK);
-	smk_ad_setfield_u_tsk(&ad, p);
-	smack_log(tsp, sp, MAY_WRITE, rc, &ad);
-	return rc;
+	return 0;
 }
 
 /**
@@ -2705,9 +2684,7 @@ static int smack_getprocattr(struct task_struct *p, char *name, char **value)
 static int smack_setprocattr(struct task_struct *p, char *name,
 			     void *value, size_t size)
 {
-	int rc;
 	struct task_smack *tsp;
-	struct task_smack *oldtsp;
 	struct cred *new;
 	char *newsmack;
 
@@ -2737,21 +2714,13 @@ static int smack_setprocattr(struct task_struct *p, char *name,
 	if (newsmack == smack_known_web.smk_known)
 		return -EPERM;
 
-	oldtsp = p->cred->security;
 	new = prepare_creds();
 	if (new == NULL)
 		return -ENOMEM;
 
-	tsp = new_task_smack(newsmack, oldtsp->smk_forked, GFP_KERNEL);
-	if (tsp == NULL) {
-		kfree(new);
-		return -ENOMEM;
-	}
-	rc = smk_copy_rules(&tsp->smk_rules, &oldtsp->smk_rules, GFP_KERNEL);
-	if (rc != 0)
-		return rc;
+	tsp = new->security;
+	tsp->smk_task = newsmack;
 
-	new->security = tsp;
 	commit_creds(new);
 	return size;
 }
diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
index b1b768e4049..99929a50093 100644
--- a/security/smack/smackfs.c
+++ b/security/smack/smackfs.c
@@ -49,6 +49,7 @@ enum smk_inos {
 	SMK_LOAD_SELF2	= 15,	/* load task specific rules with long labels */
 	SMK_ACCESS2	= 16,	/* make an access check with long labels */
 	SMK_CIPSO2	= 17,	/* load long label -> CIPSO mapping */
+	SMK_REVOKE_SUBJ	= 18,	/* set rules with subject label to '-' */
 };
 
 /*
@@ -1992,6 +1993,77 @@ static const struct file_operations smk_access2_ops = {
 };
 
 /**
+ * smk_write_revoke_subj - write() for /smack/revoke-subject
+ * @file: file pointer
+ * @buf: data from user space
+ * @count: bytes sent
+ * @ppos: where to start - must be 0
+ */
+static ssize_t smk_write_revoke_subj(struct file *file, const char __user *buf,
+				size_t count, loff_t *ppos)
+{
+	char *data = NULL;
+	const char *cp = NULL;
+	struct smack_known *skp;
+	struct smack_rule *sp;
+	struct list_head *rule_list;
+	struct mutex *rule_lock;
+	int rc = count;
+
+	if (*ppos != 0)
+		return -EINVAL;
+
+	if (!smack_privileged(CAP_MAC_ADMIN))
+		return -EPERM;
+
+	if (count == 0 || count > SMK_LONGLABEL)
+		return -EINVAL;
+
+	data = kzalloc(count, GFP_KERNEL);
+	if (data == NULL)
+		return -ENOMEM;
+
+	if (copy_from_user(data, buf, count) != 0) {
+		rc = -EFAULT;
+		goto free_out;
+	}
+
+	cp = smk_parse_smack(data, count);
+	if (cp == NULL) {
+		rc = -EINVAL;
+		goto free_out;
+	}
+
+	skp = smk_find_entry(cp);
+	if (skp == NULL) {
+		rc = -EINVAL;
+		goto free_out;
+	}
+
+	rule_list = &skp->smk_rules;
+	rule_lock = &skp->smk_rules_lock;
+
+	mutex_lock(rule_lock);
+
+	list_for_each_entry_rcu(sp, rule_list, list)
+		sp->smk_access = 0;
+
+	mutex_unlock(rule_lock);
+
+free_out:
+	kfree(data);
+	kfree(cp);
+	return rc;
+}
+
+static const struct file_operations smk_revoke_subj_ops = {
+	.write		= smk_write_revoke_subj,
+	.read		= simple_transaction_read,
+	.release	= simple_transaction_release,
+	.llseek		= generic_file_llseek,
+};
+
+/**
  * smk_fill_super - fill the /smackfs superblock
  * @sb: the empty superblock
  * @data: unused
@@ -2037,6 +2109,9 @@ static int smk_fill_super(struct super_block *sb, void *data, int silent)
 			"access2", &smk_access2_ops, S_IRUGO|S_IWUGO},
 		[SMK_CIPSO2] = {
 			"cipso2", &smk_cipso2_ops, S_IRUGO|S_IWUSR},
+		[SMK_REVOKE_SUBJ] = {
+			"revoke-subject", &smk_revoke_subj_ops,
+			S_IRUGO|S_IWUSR},
 		/* last one */
 			{""}
 	};
diff --git a/security/yama/Kconfig b/security/yama/Kconfig
index 51d6709d8bb..20ef5143c0c 100644
--- a/security/yama/Kconfig
+++ b/security/yama/Kconfig
@@ -11,3 +11,11 @@ config SECURITY_YAMA
 	  Further information can be found in Documentation/security/Yama.txt.
 
 	  If you are unsure how to answer this question, answer N.
+
+config SECURITY_YAMA_STACKED
+	bool "Yama stacked with other LSMs"
+	depends on SECURITY_YAMA
+	default n
+	help
+	  When Yama is built into the kernel, force it to stack with the
+	  selected primary LSM.
diff --git a/security/yama/yama_lsm.c b/security/yama/yama_lsm.c
index 0cc99a3ea42..b4c29848b49 100644
--- a/security/yama/yama_lsm.c
+++ b/security/yama/yama_lsm.c
@@ -100,7 +100,7 @@ static void yama_ptracer_del(struct task_struct *tracer,
  * yama_task_free - check for task_pid to remove from exception list
  * @task: task being removed
  */
-static void yama_task_free(struct task_struct *task)
+void yama_task_free(struct task_struct *task)
 {
 	yama_ptracer_del(task, task);
 }
@@ -116,7 +116,7 @@ static void yama_task_free(struct task_struct *task)
  * Return 0 on success, -ve on error.  -ENOSYS is returned when Yama
  * does not handle the given option.
  */
-static int yama_task_prctl(int option, unsigned long arg2, unsigned long arg3,
+int yama_task_prctl(int option, unsigned long arg2, unsigned long arg3,
 			   unsigned long arg4, unsigned long arg5)
 {
 	int rc;
@@ -143,7 +143,7 @@ static int yama_task_prctl(int option, unsigned long arg2, unsigned long arg3,
 		if (arg2 == 0) {
 			yama_ptracer_del(NULL, myself);
 			rc = 0;
-		} else if (arg2 == PR_SET_PTRACER_ANY) {
+		} else if (arg2 == PR_SET_PTRACER_ANY || (int)arg2 == -1) {
 			rc = yama_ptracer_add(NULL, myself);
 		} else {
 			struct task_struct *tracer;
@@ -243,7 +243,7 @@ static int ptracer_exception_found(struct task_struct *tracer,
  *
  * Returns 0 if following the ptrace is allowed, -ve on error.
  */
-static int yama_ptrace_access_check(struct task_struct *child,
+int yama_ptrace_access_check(struct task_struct *child,
 				    unsigned int mode)
 {
 	int rc;
@@ -293,7 +293,7 @@ static int yama_ptrace_access_check(struct task_struct *child,
  *
  * Returns 0 if following the ptrace is allowed, -ve on error.
  */
-static int yama_ptrace_traceme(struct task_struct *parent)
+int yama_ptrace_traceme(struct task_struct *parent)
 {
 	int rc;
 
@@ -324,6 +324,7 @@ static int yama_ptrace_traceme(struct task_struct *parent)
 	return rc;
 }
 
+#ifndef CONFIG_SECURITY_YAMA_STACKED
 static struct security_operations yama_ops = {
 	.name =			"yama",
 
@@ -332,6 +333,7 @@ static struct security_operations yama_ops = {
 	.task_prctl =		yama_task_prctl,
 	.task_free =		yama_task_free,
 };
+#endif
 
 #ifdef CONFIG_SYSCTL
 static int yama_dointvec_minmax(struct ctl_table *table, int write,
@@ -378,13 +380,17 @@ static struct ctl_table yama_sysctl_table[] = {
 
 static __init int yama_init(void)
 {
+#ifndef CONFIG_SECURITY_YAMA_STACKED
 	if (!security_module_enable(&yama_ops))
 		return 0;
+#endif
 
 	printk(KERN_INFO "Yama: becoming mindful.\n");
 
+#ifndef CONFIG_SECURITY_YAMA_STACKED
 	if (register_security(&yama_ops))
 		panic("Yama: kernel registration failed.\n");
+#endif
 
 #ifdef CONFIG_SYSCTL
 	if (!register_sysctl_paths(yama_sysctl_path, yama_sysctl_table))
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index 53b5ada8f7c..20554eff5a2 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -1563,25 +1563,25 @@ static int snd_pcm_drop(struct snd_pcm_substream *substream)
 
 
 /* WARNING: Don't forget to fput back the file */
-static struct file *snd_pcm_file_fd(int fd)
+static struct file *snd_pcm_file_fd(int fd, int *fput_needed)
 {
 	struct file *file;
 	struct inode *inode;
 	unsigned int minor;
 
-	file = fget(fd);
+	file = fget_light(fd, fput_needed);
 	if (!file)
 		return NULL;
 	inode = file->f_path.dentry->d_inode;
 	if (!S_ISCHR(inode->i_mode) ||
 	    imajor(inode) != snd_major) {
-		fput(file);
+		fput_light(file, *fput_needed);
 		return NULL;
 	}
 	minor = iminor(inode);
 	if (!snd_lookup_minor_data(minor, SNDRV_DEVICE_TYPE_PCM_PLAYBACK) &&
 	    !snd_lookup_minor_data(minor, SNDRV_DEVICE_TYPE_PCM_CAPTURE)) {
-		fput(file);
+		fput_light(file, *fput_needed);
 		return NULL;
 	}
 	return file;
@@ -1597,8 +1597,9 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd)
 	struct snd_pcm_file *pcm_file;
 	struct snd_pcm_substream *substream1;
 	struct snd_pcm_group *group;
+	int fput_needed;
 
-	file = snd_pcm_file_fd(fd);
+	file = snd_pcm_file_fd(fd, &fput_needed);
 	if (!file)
 		return -EBADFD;
 	pcm_file = file->private_data;
@@ -1633,7 +1634,7 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd)
 	write_unlock_irq(&snd_pcm_link_rwlock);
 	up_write(&snd_pcm_link_rwsem);
  _nolock:
-	fput(file);
+	fput_light(file, fput_needed);
 	if (res < 0)
 		kfree(group);
 	return res;
diff --git a/sound/soc/codecs/wm5100.c b/sound/soc/codecs/wm5100.c
index f4817292ef4..aa62c0e44cb 100644
--- a/sound/soc/codecs/wm5100.c
+++ b/sound/soc/codecs/wm5100.c
@@ -1233,7 +1233,7 @@ static const struct snd_soc_dapm_route wm5100_dapm_routes[] = {
 	{ "PWM2", NULL, "PWM2 Driver" },
 };
 
-static const __devinitdata struct reg_default wm5100_reva_patches[] = {
+static const __devinitconst struct reg_default wm5100_reva_patches[] = {
 	{ WM5100_AUDIO_IF_1_10, 0 },
 	{ WM5100_AUDIO_IF_1_11, 1 },
 	{ WM5100_AUDIO_IF_1_12, 2 },
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 85baf11e2ac..43480149119 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -1,4 +1,4 @@
-TARGETS = breakpoints kcmp mqueue vm cpu-hotplug memory-hotplug
+TARGETS = breakpoints kcmp mqueue vm cpu-hotplug memory-hotplug epoll
 
 all:
 	for TARGET in $(TARGETS); do \
diff --git a/tools/testing/selftests/epoll/Makefile b/tools/testing/selftests/epoll/Makefile
new file mode 100644
index 00000000000..19806ed62f5
--- /dev/null
+++ b/tools/testing/selftests/epoll/Makefile
@@ -0,0 +1,11 @@
+# Makefile for epoll selftests
+
+all: test_epoll
+%: %.c
+	gcc -pthread -g -o $@ $^
+
+run_tests: all
+	./test_epoll
+
+clean:
+	$(RM) test_epoll
diff --git a/tools/testing/selftests/epoll/test_epoll.c b/tools/testing/selftests/epoll/test_epoll.c
new file mode 100644
index 00000000000..e0fcff1e833
--- /dev/null
+++ b/tools/testing/selftests/epoll/test_epoll.c
@@ -0,0 +1,344 @@
+/*
+ *  tools/testing/selftests/epoll/test_epoll.c
+ *
+ *  Copyright 2012 Adobe Systems Incorporated
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Paton J. Lewis <palewis@adobe.com>
+ *
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+
+/*
+ * A pointer to an epoll_item_private structure will be stored in the epoll
+ * item's event structure so that we can get access to the epoll_item_private
+ * data after calling epoll_wait:
+ */
+struct epoll_item_private {
+	int index;  /* Position of this struct within the epoll_items array. */
+	int fd;
+	uint32_t events;
+	pthread_mutex_t mutex;  /* Guards the following variables... */
+	int stop;
+	int status;  /* Stores any error encountered while handling item. */
+	/* The following variable allows us to test whether we have encountered
+	   a problem while attempting to cancel and delete the associated
+	   event. When the test program exits, 'deleted' should be exactly
+	   one. If it is greater than one, then the failed test reflects a real
+	   world situation where we would have tried to access the epoll item's
+	   private data after deleting it: */
+	int deleted;
+};
+
+struct epoll_item_private *epoll_items;
+
+/*
+ * Delete the specified item from the epoll set. In a real-world secneario this
+ * is where we would free the associated data structure, but in this testing
+ * environment we retain the structure so that we can test for double-deletion:
+ */
+void delete_item(int index)
+{
+	__sync_fetch_and_add(&epoll_items[index].deleted, 1);
+}
+
+/*
+ * A pointer to a read_thread_data structure will be passed as the argument to
+ * each read thread:
+ */
+struct read_thread_data {
+	int stop;
+	int status;  /* Indicates any error encountered by the read thread. */
+	int epoll_set;
+};
+
+/*
+ * The function executed by the read threads:
+ */
+void *read_thread_function(void *function_data)
+{
+	struct read_thread_data *thread_data =
+		(struct read_thread_data *)function_data;
+	struct epoll_event event_data;
+	struct epoll_item_private *item_data;
+	char socket_data;
+
+	/* Handle events until we encounter an error or this thread's 'stop'
+	   condition is set: */
+	while (1) {
+		int result = epoll_wait(thread_data->epoll_set,
+					&event_data,
+					1,	/* Number of desired events */
+					1000);  /* Timeout in ms */
+		if (result < 0) {
+			/* Breakpoints signal all threads. Ignore that while
+			   debugging: */
+			if (errno == EINTR)
+				continue;
+			thread_data->status = errno;
+			return 0;
+		} else if (thread_data->stop)
+			return 0;
+		else if (result == 0)  /* Timeout */
+			continue;
+
+		/* We need the mutex here because checking for the stop
+		   condition and re-enabling the epoll item need to be done
+		   together as one atomic operation when EPOLL_CTL_DISABLE is
+		   available: */
+		item_data = (struct epoll_item_private *)event_data.data.ptr;
+		pthread_mutex_lock(&item_data->mutex);
+
+		/* Remove the item from the epoll set if we want to stop
+		   handling that event: */
+		if (item_data->stop)
+			delete_item(item_data->index);
+		else {
+			/* Clear the data that was written to the other end of
+			   our non-blocking socket: */
+			do {
+				if (read(item_data->fd, &socket_data, 1) < 1) {
+					if ((errno == EAGAIN) ||
+					    (errno == EWOULDBLOCK))
+						break;
+					else
+						goto error_unlock;
+				}
+			} while (item_data->events & EPOLLET);
+
+			/* The item was one-shot, so re-enable it: */
+			event_data.events = item_data->events;
+			if (epoll_ctl(thread_data->epoll_set,
+						  EPOLL_CTL_MOD,
+						  item_data->fd,
+						  &event_data) < 0)
+				goto error_unlock;
+		}
+
+		pthread_mutex_unlock(&item_data->mutex);
+	}
+
+error_unlock:
+	thread_data->status = item_data->status = errno;
+	pthread_mutex_unlock(&item_data->mutex);
+	return 0;
+}
+
+/*
+ * A pointer to a write_thread_data structure will be passed as the argument to
+ * the write thread:
+ */
+struct write_thread_data {
+	int stop;
+	int status;  /* Indicates any error encountered by the write thread. */
+	int n_fds;
+	int *fds;
+};
+
+/*
+ * The function executed by the write thread. It writes a single byte to each
+ * socket in turn until the stop condition for this thread is set. If writing to
+ * a socket would block (i.e. errno was EAGAIN), we leave that socket alone for
+ * the moment and just move on to the next socket in the list. We don't care
+ * about the order in which we deliver events to the epoll set. In fact we don't
+ * care about the data we're writing to the pipes at all; we just want to
+ * trigger epoll events:
+ */
+void *write_thread_function(void *function_data)
+{
+	const char data = 'X';
+	int index;
+	struct write_thread_data *thread_data =
+		(struct write_thread_data *)function_data;
+	while (!write_thread_data->stop)
+		for (index = 0;
+		     !thread_data->stop && (index < thread_data->n_fds);
+		     ++index)
+			if ((write(thread_data->fds[index], &data, 1) < 1) &&
+				(errno != EAGAIN) &&
+				(errno != EWOULDBLOCK)) {
+				write_thread_data->status = errno;
+				return;
+			}
+}
+
+/*
+ * Arguments are currently ignored:
+ */
+int main(int argc, char **argv)
+{
+	const int n_read_threads = 100;
+	const int n_epoll_items = 500;
+	int index;
+	int epoll_set = epoll_create1(0);
+	struct write_thread_data write_thread_data = {
+		0, 0, n_epoll_items, malloc(n_epoll_items * sizeof(int))
+	};
+	struct read_thread_data *read_thread_data =
+		malloc(n_read_threads * sizeof(struct read_thread_data));
+	pthread_t *read_threads = malloc(n_read_threads * sizeof(pthread_t));
+	pthread_t write_thread;
+
+	printf("-----------------\n");
+	printf("Runing test_epoll\n");
+	printf("-----------------\n");
+
+	epoll_items = malloc(n_epoll_items * sizeof(struct epoll_item_private));
+
+	if (epoll_set < 0 || epoll_items == 0 || write_thread_data.fds == 0 ||
+		read_thread_data == 0 || read_threads == 0)
+		goto error;
+
+	if (sysconf(_SC_NPROCESSORS_ONLN) < 2) {
+		printf("Error: please run this test on a multi-core system.\n");
+		goto error;
+	}
+
+	/* Create the socket pairs and epoll items: */
+	for (index = 0; index < n_epoll_items; ++index) {
+		int socket_pair[2];
+		struct epoll_event event_data;
+		if (socketpair(AF_UNIX,
+			       SOCK_STREAM | SOCK_NONBLOCK,
+			       0,
+			       socket_pair) < 0)
+			goto error;
+		write_thread_data.fds[index] = socket_pair[0];
+		epoll_items[index].index = index;
+		epoll_items[index].fd = socket_pair[1];
+		if (pthread_mutex_init(&epoll_items[index].mutex, NULL) != 0)
+			goto error;
+		/* We always use EPOLLONESHOT because this test is currently
+		   structured to demonstrate the need for EPOLL_CTL_DISABLE,
+		   which only produces useful information in the EPOLLONESHOT
+		   case (without EPOLLONESHOT, calling epoll_ctl with
+		   EPOLL_CTL_DISABLE will never return EBUSY). If support for
+		   testing events without EPOLLONESHOT is desired, it should
+		   probably be implemented in a separate unit test. */
+		epoll_items[index].events = EPOLLIN | EPOLLONESHOT;
+		if (index < n_epoll_items / 2)
+			epoll_items[index].events |= EPOLLET;
+		epoll_items[index].stop = 0;
+		epoll_items[index].status = 0;
+		epoll_items[index].deleted = 0;
+		event_data.events = epoll_items[index].events;
+		event_data.data.ptr = &epoll_items[index];
+		if (epoll_ctl(epoll_set,
+			      EPOLL_CTL_ADD,
+			      epoll_items[index].fd,
+			      &event_data) < 0)
+			goto error;
+	}
+
+	/* Create and start the read threads: */
+	for (index = 0; index < n_read_threads; ++index) {
+		read_thread_data[index].stop = 0;
+		read_thread_data[index].status = 0;
+		read_thread_data[index].epoll_set = epoll_set;
+		if (pthread_create(&read_threads[index],
+				   NULL,
+				   read_thread_function,
+				   &read_thread_data[index]) != 0)
+			goto error;
+	}
+
+	if (pthread_create(&write_thread,
+			   NULL,
+			   write_thread_function,
+			   &write_thread_data) != 0)
+		goto error;
+
+	/* Cancel all event pollers: */
+#ifdef EPOLL_CTL_DISABLE
+	for (index = 0; index < n_epoll_items; ++index) {
+		pthread_mutex_lock(&epoll_items[index].mutex);
+		++epoll_items[index].stop;
+		if (epoll_ctl(epoll_set,
+			      EPOLL_CTL_DISABLE,
+			      epoll_items[index].fd,
+			      NULL) == 0)
+			delete_item(index);
+		else if (errno != EBUSY) {
+			pthread_mutex_unlock(&epoll_items[index].mutex);
+			goto error;
+		}
+		/* EBUSY means events were being handled; allow the other thread
+		   to delete the item. */
+		pthread_mutex_unlock(&epoll_items[index].mutex);
+	}
+#else
+	for (index = 0; index < n_epoll_items; ++index) {
+		pthread_mutex_lock(&epoll_items[index].mutex);
+		++epoll_items[index].stop;
+		pthread_mutex_unlock(&epoll_items[index].mutex);
+		/* Wait in case a thread running read_thread_function is
+		   currently executing code between epoll_wait and
+		   pthread_mutex_lock with this item. Note that a longer delay
+		   would make double-deletion less likely (at the expense of
+		   performance), but there is no guarantee that any delay would
+		   ever be sufficient. Note also that we delete all event
+		   pollers at once for testing purposes, but in a real-world
+		   environment we are likely to want to be able to cancel event
+		   pollers at arbitrary times. Therefore we can't improve this
+		   situation by just splitting this loop into two loops
+		   (i.e. signal 'stop' for all items, sleep, and then delete all
+		   items). We also can't fix the problem via EPOLL_CTL_DEL
+		   because that command can't prevent the case where some other
+		   thread is executing read_thread_function within the region
+		   mentioned above: */
+		usleep(1);
+		pthread_mutex_lock(&epoll_items[index].mutex);
+		if (!epoll_items[index].deleted)
+			delete_item(index);
+		pthread_mutex_unlock(&epoll_items[index].mutex);
+	}
+#endif
+
+	/* Shut down the read threads: */
+	for (index = 0; index < n_read_threads; ++index)
+		__sync_fetch_and_add(&read_thread_data[index].stop, 1);
+	for (index = 0; index < n_read_threads; ++index) {
+		if (pthread_join(read_threads[index], NULL) != 0)
+			goto error;
+		if (read_thread_data[index].status)
+			goto error;
+	}
+
+	/* Shut down the write thread: */
+	__sync_fetch_and_add(&write_thread_data.stop, 1);
+	if ((pthread_join(write_thread, NULL) != 0) || write_thread_data.status)
+		goto error;
+
+	/* Check for final error conditions: */
+	for (index = 0; index < n_epoll_items; ++index) {
+		if (epoll_items[index].status != 0)
+			goto error;
+		if (pthread_mutex_destroy(&epoll_items[index].mutex) < 0)
+			goto error;
+	}
+	for (index = 0; index < n_epoll_items; ++index)
+		if (epoll_items[index].deleted != 1) {
+			printf("Error: item data deleted %1d times.\n",
+				   epoll_items[index].deleted);
+			goto error;
+		}
+
+	printf("[PASS]\n");
+	return 0;
+
+ error:
+	printf("[FAIL]\n");
+	return errno;
+}
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 28694f4a913..d01b24b72c6 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -21,3 +21,6 @@ config KVM_ASYNC_PF
 
 config HAVE_KVM_MSI
        bool
+
+config HAVE_KVM_CPU_RELAX_INTERCEPT
+       bool
diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
index 74268b4c2ee..ea475cd0351 100644
--- a/virt/kvm/async_pf.c
+++ b/virt/kvm/async_pf.c
@@ -111,8 +111,8 @@ void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
 			list_entry(vcpu->async_pf.done.next,
 				   typeof(*work), link);
 		list_del(&work->link);
-		if (work->page)
-			put_page(work->page);
+		if (!is_error_page(work->page))
+			kvm_release_page_clean(work->page);
 		kmem_cache_free(async_pf_cache, work);
 	}
 	spin_unlock(&vcpu->async_pf.lock);
@@ -138,8 +138,8 @@ void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
 
 		list_del(&work->queue);
 		vcpu->async_pf.queued--;
-		if (work->page)
-			put_page(work->page);
+		if (!is_error_page(work->page))
+			kvm_release_page_clean(work->page);
 		kmem_cache_free(async_pf_cache, work);
 	}
 }
@@ -203,8 +203,7 @@ int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu)
 	if (!work)
 		return -ENOMEM;
 
-	work->page = bad_page;
-	get_page(bad_page);
+	work->page = KVM_ERR_PTR_BAD_PAGE;
 	INIT_LIST_HEAD(&work->queue); /* for list_del to work */
 
 	spin_lock(&vcpu->async_pf.lock);
diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c
index 67a35e90384..9718e98d6d2 100644
--- a/virt/kvm/eventfd.c
+++ b/virt/kvm/eventfd.c
@@ -43,6 +43,31 @@
  * --------------------------------------------------------------------
  */
 
+/*
+ * Resampling irqfds are a special variety of irqfds used to emulate
+ * level triggered interrupts.  The interrupt is asserted on eventfd
+ * trigger.  On acknowledgement through the irq ack notifier, the
+ * interrupt is de-asserted and userspace is notified through the
+ * resamplefd.  All resamplers on the same gsi are de-asserted
+ * together, so we don't need to track the state of each individual
+ * user.  We can also therefore share the same irq source ID.
+ */
+struct _irqfd_resampler {
+	struct kvm *kvm;
+	/*
+	 * List of resampling struct _irqfd objects sharing this gsi.
+	 * RCU list modified under kvm->irqfds.resampler_lock
+	 */
+	struct list_head list;
+	struct kvm_irq_ack_notifier notifier;
+	/*
+	 * Entry in list of kvm->irqfd.resampler_list.  Use for sharing
+	 * resamplers among irqfds on the same gsi.
+	 * Accessed and modified under kvm->irqfds.resampler_lock
+	 */
+	struct list_head link;
+};
+
 struct _irqfd {
 	/* Used for MSI fast-path */
 	struct kvm *kvm;
@@ -52,6 +77,12 @@ struct _irqfd {
 	/* Used for level IRQ fast-path */
 	int gsi;
 	struct work_struct inject;
+	/* The resampler used by this irqfd (resampler-only) */
+	struct _irqfd_resampler *resampler;
+	/* Eventfd notified on resample (resampler-only) */
+	struct eventfd_ctx *resamplefd;
+	/* Entry in list of irqfds for a resampler (resampler-only) */
+	struct list_head resampler_link;
 	/* Used for setup/shutdown */
 	struct eventfd_ctx *eventfd;
 	struct list_head list;
@@ -67,8 +98,58 @@ irqfd_inject(struct work_struct *work)
 	struct _irqfd *irqfd = container_of(work, struct _irqfd, inject);
 	struct kvm *kvm = irqfd->kvm;
 
-	kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1);
-	kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0);
+	if (!irqfd->resampler) {
+		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1);
+		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0);
+	} else
+		kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
+			    irqfd->gsi, 1);
+}
+
+/*
+ * Since resampler irqfds share an IRQ source ID, we de-assert once
+ * then notify all of the resampler irqfds using this GSI.  We can't
+ * do multiple de-asserts or we risk racing with incoming re-asserts.
+ */
+static void
+irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian)
+{
+	struct _irqfd_resampler *resampler;
+	struct _irqfd *irqfd;
+
+	resampler = container_of(kian, struct _irqfd_resampler, notifier);
+
+	kvm_set_irq(resampler->kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
+		    resampler->notifier.gsi, 0);
+
+	rcu_read_lock();
+
+	list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link)
+		eventfd_signal(irqfd->resamplefd, 1);
+
+	rcu_read_unlock();
+}
+
+static void
+irqfd_resampler_shutdown(struct _irqfd *irqfd)
+{
+	struct _irqfd_resampler *resampler = irqfd->resampler;
+	struct kvm *kvm = resampler->kvm;
+
+	mutex_lock(&kvm->irqfds.resampler_lock);
+
+	list_del_rcu(&irqfd->resampler_link);
+	synchronize_rcu();
+
+	if (list_empty(&resampler->list)) {
+		list_del(&resampler->link);
+		kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);
+		kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
+			    resampler->notifier.gsi, 0);
+		kfree(resampler);
+	}
+
+	mutex_unlock(&kvm->irqfds.resampler_lock);
 }
 
 /*
@@ -92,6 +173,11 @@ irqfd_shutdown(struct work_struct *work)
 	 */
 	flush_work(&irqfd->inject);
 
+	if (irqfd->resampler) {
+		irqfd_resampler_shutdown(irqfd);
+		eventfd_ctx_put(irqfd->resamplefd);
+	}
+
 	/*
 	 * It is now safe to release the object's resources
 	 */
@@ -203,7 +289,7 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
 	struct kvm_irq_routing_table *irq_rt;
 	struct _irqfd *irqfd, *tmp;
 	struct file *file = NULL;
-	struct eventfd_ctx *eventfd = NULL;
+	struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL;
 	int ret;
 	unsigned int events;
 
@@ -231,6 +317,54 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
 
 	irqfd->eventfd = eventfd;
 
+	if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {
+		struct _irqfd_resampler *resampler;
+
+		resamplefd = eventfd_ctx_fdget(args->resamplefd);
+		if (IS_ERR(resamplefd)) {
+			ret = PTR_ERR(resamplefd);
+			goto fail;
+		}
+
+		irqfd->resamplefd = resamplefd;
+		INIT_LIST_HEAD(&irqfd->resampler_link);
+
+		mutex_lock(&kvm->irqfds.resampler_lock);
+
+		list_for_each_entry(resampler,
+				    &kvm->irqfds.resampler_list, list) {
+			if (resampler->notifier.gsi == irqfd->gsi) {
+				irqfd->resampler = resampler;
+				break;
+			}
+		}
+
+		if (!irqfd->resampler) {
+			resampler = kzalloc(sizeof(*resampler), GFP_KERNEL);
+			if (!resampler) {
+				ret = -ENOMEM;
+				mutex_unlock(&kvm->irqfds.resampler_lock);
+				goto fail;
+			}
+
+			resampler->kvm = kvm;
+			INIT_LIST_HEAD(&resampler->list);
+			resampler->notifier.gsi = irqfd->gsi;
+			resampler->notifier.irq_acked = irqfd_resampler_ack;
+			INIT_LIST_HEAD(&resampler->link);
+
+			list_add(&resampler->link, &kvm->irqfds.resampler_list);
+			kvm_register_irq_ack_notifier(kvm,
+						      &resampler->notifier);
+			irqfd->resampler = resampler;
+		}
+
+		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);
+		synchronize_rcu();
+
+		mutex_unlock(&kvm->irqfds.resampler_lock);
+	}
+
 	/*
 	 * Install our own custom wake-up handling so we are notified via
 	 * a callback whenever someone signals the underlying eventfd
@@ -276,6 +410,12 @@ kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
 	return 0;
 
 fail:
+	if (irqfd->resampler)
+		irqfd_resampler_shutdown(irqfd);
+
+	if (resamplefd && !IS_ERR(resamplefd))
+		eventfd_ctx_put(resamplefd);
+
 	if (eventfd && !IS_ERR(eventfd))
 		eventfd_ctx_put(eventfd);
 
@@ -291,6 +431,8 @@ kvm_eventfd_init(struct kvm *kvm)
 {
 	spin_lock_init(&kvm->irqfds.lock);
 	INIT_LIST_HEAD(&kvm->irqfds.items);
+	INIT_LIST_HEAD(&kvm->irqfds.resampler_list);
+	mutex_init(&kvm->irqfds.resampler_lock);
 	INIT_LIST_HEAD(&kvm->ioeventfds);
 }
 
@@ -340,7 +482,7 @@ kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
 int
 kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
 {
-	if (args->flags & ~KVM_IRQFD_FLAG_DEASSIGN)
+	if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))
 		return -EINVAL;
 
 	if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)
diff --git a/virt/kvm/ioapic.c b/virt/kvm/ioapic.c
index ef61d529a6c..cfb7e4d52dc 100644
--- a/virt/kvm/ioapic.c
+++ b/virt/kvm/ioapic.c
@@ -197,28 +197,29 @@ int kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int irq_source_id,
 	u32 old_irr;
 	u32 mask = 1 << irq;
 	union kvm_ioapic_redirect_entry entry;
-	int ret = 1;
+	int ret, irq_level;
+
+	BUG_ON(irq < 0 || irq >= IOAPIC_NUM_PINS);
 
 	spin_lock(&ioapic->lock);
 	old_irr = ioapic->irr;
-	if (irq >= 0 && irq < IOAPIC_NUM_PINS) {
-		int irq_level = __kvm_irq_line_state(&ioapic->irq_states[irq],
-						     irq_source_id, level);
-		entry = ioapic->redirtbl[irq];
-		irq_level ^= entry.fields.polarity;
-		if (!irq_level)
-			ioapic->irr &= ~mask;
-		else {
-			int edge = (entry.fields.trig_mode == IOAPIC_EDGE_TRIG);
-			ioapic->irr |= mask;
-			if ((edge && old_irr != ioapic->irr) ||
-			    (!edge && !entry.fields.remote_irr))
-				ret = ioapic_service(ioapic, irq);
-			else
-				ret = 0; /* report coalesced interrupt */
-		}
-		trace_kvm_ioapic_set_irq(entry.bits, irq, ret == 0);
+	irq_level = __kvm_irq_line_state(&ioapic->irq_states[irq],
+					 irq_source_id, level);
+	entry = ioapic->redirtbl[irq];
+	irq_level ^= entry.fields.polarity;
+	if (!irq_level) {
+		ioapic->irr &= ~mask;
+		ret = 1;
+	} else {
+		int edge = (entry.fields.trig_mode == IOAPIC_EDGE_TRIG);
+		ioapic->irr |= mask;
+		if ((edge && old_irr != ioapic->irr) ||
+		    (!edge && !entry.fields.remote_irr))
+			ret = ioapic_service(ioapic, irq);
+		else
+			ret = 0; /* report coalesced interrupt */
 	}
+	trace_kvm_ioapic_set_irq(entry.bits, irq, ret == 0);
 	spin_unlock(&ioapic->lock);
 
 	return ret;
diff --git a/virt/kvm/iommu.c b/virt/kvm/iommu.c
index e9fff9830bf..037cb6730e6 100644
--- a/virt/kvm/iommu.c
+++ b/virt/kvm/iommu.c
@@ -42,13 +42,13 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm);
 static void kvm_iommu_put_pages(struct kvm *kvm,
 				gfn_t base_gfn, unsigned long npages);
 
-static pfn_t kvm_pin_pages(struct kvm *kvm, struct kvm_memory_slot *slot,
-			   gfn_t gfn, unsigned long size)
+static pfn_t kvm_pin_pages(struct kvm_memory_slot *slot, gfn_t gfn,
+			   unsigned long size)
 {
 	gfn_t end_gfn;
 	pfn_t pfn;
 
-	pfn     = gfn_to_pfn_memslot(kvm, slot, gfn);
+	pfn     = gfn_to_pfn_memslot(slot, gfn);
 	end_gfn = gfn + (size >> PAGE_SHIFT);
 	gfn    += 1;
 
@@ -56,7 +56,7 @@ static pfn_t kvm_pin_pages(struct kvm *kvm, struct kvm_memory_slot *slot,
 		return pfn;
 
 	while (gfn < end_gfn)
-		gfn_to_pfn_memslot(kvm, slot, gfn++);
+		gfn_to_pfn_memslot(slot, gfn++);
 
 	return pfn;
 }
@@ -105,7 +105,7 @@ int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
 		 * Pin all pages we are about to map in memory. This is
 		 * important because we unmap and unpin in 4kb steps later.
 		 */
-		pfn = kvm_pin_pages(kvm, slot, gfn, page_size);
+		pfn = kvm_pin_pages(slot, gfn, page_size);
 		if (is_error_pfn(pfn)) {
 			gfn += 1;
 			continue;
@@ -300,6 +300,12 @@ static void kvm_iommu_put_pages(struct kvm *kvm,
 
 		/* Get physical address */
 		phys = iommu_iova_to_phys(domain, gfn_to_gpa(gfn));
+
+		if (!phys) {
+			gfn++;
+			continue;
+		}
+
 		pfn  = phys >> PAGE_SHIFT;
 
 		/* Unmap address from IO address space */
diff --git a/virt/kvm/irq_comm.c b/virt/kvm/irq_comm.c
index 83402d74a76..2eb58af7ee9 100644
--- a/virt/kvm/irq_comm.c
+++ b/virt/kvm/irq_comm.c
@@ -68,8 +68,13 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
 	struct kvm_vcpu *vcpu, *lowest = NULL;
 
 	if (irq->dest_mode == 0 && irq->dest_id == 0xff &&
-			kvm_is_dm_lowest_prio(irq))
+			kvm_is_dm_lowest_prio(irq)) {
 		printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n");
+		irq->delivery_mode = APIC_DM_FIXED;
+	}
+
+	if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r))
+		return r;
 
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (!kvm_apic_present(vcpu))
@@ -223,6 +228,9 @@ int kvm_request_irq_source_id(struct kvm *kvm)
 	}
 
 	ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
+#ifdef CONFIG_X86
+	ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
+#endif
 	set_bit(irq_source_id, bitmap);
 unlock:
 	mutex_unlock(&kvm->irq_lock);
@@ -233,6 +241,9 @@ unlock:
 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
 {
 	ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
+#ifdef CONFIG_X86
+	ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
+#endif
 
 	mutex_lock(&kvm->irq_lock);
 	if (irq_source_id < 0 ||
@@ -321,11 +332,11 @@ static int setup_routing_entry(struct kvm_irq_routing_table *rt,
 		switch (ue->u.irqchip.irqchip) {
 		case KVM_IRQCHIP_PIC_MASTER:
 			e->set = kvm_set_pic_irq;
-			max_pin = 16;
+			max_pin = PIC_NUM_PINS;
 			break;
 		case KVM_IRQCHIP_PIC_SLAVE:
 			e->set = kvm_set_pic_irq;
-			max_pin = 16;
+			max_pin = PIC_NUM_PINS;
 			delta = 8;
 			break;
 		case KVM_IRQCHIP_IOAPIC:
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index d617f69131d..e59bb63cb08 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -100,13 +100,7 @@ EXPORT_SYMBOL_GPL(kvm_rebooting);
 
 static bool largepages_enabled = true;
 
-static struct page *hwpoison_page;
-static pfn_t hwpoison_pfn;
-
-struct page *fault_page;
-pfn_t fault_pfn;
-
-inline int kvm_is_mmio_pfn(pfn_t pfn)
+bool kvm_is_mmio_pfn(pfn_t pfn)
 {
 	if (pfn_valid(pfn)) {
 		int reserved;
@@ -137,11 +131,12 @@ inline int kvm_is_mmio_pfn(pfn_t pfn)
 /*
  * Switches to specified vcpu, until a matching vcpu_put()
  */
-void vcpu_load(struct kvm_vcpu *vcpu)
+int vcpu_load(struct kvm_vcpu *vcpu)
 {
 	int cpu;
 
-	mutex_lock(&vcpu->mutex);
+	if (mutex_lock_killable(&vcpu->mutex))
+		return -EINTR;
 	if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
 		/* The thread running this VCPU changed. */
 		struct pid *oldpid = vcpu->pid;
@@ -154,6 +149,7 @@ void vcpu_load(struct kvm_vcpu *vcpu)
 	preempt_notifier_register(&vcpu->preempt_notifier);
 	kvm_arch_vcpu_load(vcpu, cpu);
 	put_cpu();
+	return 0;
 }
 
 void vcpu_put(struct kvm_vcpu *vcpu)
@@ -236,6 +232,9 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 	}
 	vcpu->run = page_address(page);
 
+	kvm_vcpu_set_in_spin_loop(vcpu, false);
+	kvm_vcpu_set_dy_eligible(vcpu, false);
+
 	r = kvm_arch_vcpu_init(vcpu);
 	if (r < 0)
 		goto fail_free_run;
@@ -332,8 +331,7 @@ static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 	 * count is also read inside the mmu_lock critical section.
 	 */
 	kvm->mmu_notifier_count++;
-	for (; start < end; start += PAGE_SIZE)
-		need_tlb_flush |= kvm_unmap_hva(kvm, start);
+	need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
 	need_tlb_flush |= kvm->tlbs_dirty;
 	/* we've to flush the tlb before the pages can be freed */
 	if (need_tlb_flush)
@@ -412,7 +410,7 @@ static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 	int idx;
 
 	idx = srcu_read_lock(&kvm->srcu);
-	kvm_arch_flush_shadow(kvm);
+	kvm_arch_flush_shadow_all(kvm);
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 
@@ -551,16 +549,12 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 				  struct kvm_memory_slot *dont)
 {
-	if (!dont || free->rmap != dont->rmap)
-		vfree(free->rmap);
-
 	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 		kvm_destroy_dirty_bitmap(free);
 
 	kvm_arch_free_memslot(free, dont);
 
 	free->npages = 0;
-	free->rmap = NULL;
 }
 
 void kvm_free_physmem(struct kvm *kvm)
@@ -590,7 +584,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 #else
-	kvm_arch_flush_shadow(kvm);
+	kvm_arch_flush_shadow_all(kvm);
 #endif
 	kvm_arch_destroy_vm(kvm);
 	kvm_free_physmem(kvm);
@@ -686,6 +680,20 @@ void update_memslots(struct kvm_memslots *slots, struct kvm_memory_slot *new)
 	slots->generation++;
 }
 
+static int check_memory_region_flags(struct kvm_userspace_memory_region *mem)
+{
+	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
+
+#ifdef KVM_CAP_READONLY_MEM
+	valid_flags |= KVM_MEM_READONLY;
+#endif
+
+	if (mem->flags & ~valid_flags)
+		return -EINVAL;
+
+	return 0;
+}
+
 /*
  * Allocate some memory and give it an address in the guest physical address
  * space.
@@ -706,6 +714,10 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	struct kvm_memory_slot old, new;
 	struct kvm_memslots *slots, *old_memslots;
 
+	r = check_memory_region_flags(mem);
+	if (r)
+		goto out;
+
 	r = -EINVAL;
 	/* General sanity checks */
 	if (mem->memory_size & (PAGE_SIZE - 1))
@@ -769,11 +781,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	if (npages && !old.npages) {
 		new.user_alloc = user_alloc;
 		new.userspace_addr = mem->userspace_addr;
-#ifndef CONFIG_S390
-		new.rmap = vzalloc(npages * sizeof(*new.rmap));
-		if (!new.rmap)
-			goto out_free;
-#endif /* not defined CONFIG_S390 */
+
 		if (kvm_arch_create_memslot(&new, npages))
 			goto out_free;
 	}
@@ -785,7 +793,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
 		/* destroy any largepage mappings for dirty tracking */
 	}
 
-	if (!npages) {
+	if (!npages || base_gfn != old.base_gfn) {
 		struct kvm_memory_slot *slot;
 
 		r = -ENOMEM;
@@ -801,14 +809,14 @@ int __kvm_set_memory_region(struct kvm *kvm,
 		old_memslots = kvm->memslots;
 		rcu_assign_pointer(kvm->memslots, slots);
 		synchronize_srcu_expedited(&kvm->srcu);
-		/* From this point no new shadow pages pointing to a deleted
-		 * memslot will be created.
+		/* From this point no new shadow pages pointing to a deleted,
+		 * or moved, memslot will be created.
 		 *
 		 * validation of sp->gfn happens in:
 		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 		 * 	- kvm_is_visible_gfn (mmu_check_roots)
 		 */
-		kvm_arch_flush_shadow(kvm);
+		kvm_arch_flush_shadow_memslot(kvm, slot);
 		kfree(old_memslots);
 	}
 
@@ -832,7 +840,6 @@ int __kvm_set_memory_region(struct kvm *kvm,
 
 	/* actual memory is freed via old in kvm_free_physmem_slot below */
 	if (!npages) {
-		new.rmap = NULL;
 		new.dirty_bitmap = NULL;
 		memset(&new.arch, 0, sizeof(new.arch));
 	}
@@ -844,13 +851,6 @@ int __kvm_set_memory_region(struct kvm *kvm,
 
 	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
 
-	/*
-	 * If the new memory slot is created, we need to clear all
-	 * mmio sptes.
-	 */
-	if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT)
-		kvm_arch_flush_shadow(kvm);
-
 	kvm_free_physmem_slot(&old, &new);
 	kfree(old_memslots);
 
@@ -932,53 +932,6 @@ void kvm_disable_largepages(void)
 }
 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
 
-int is_error_page(struct page *page)
-{
-	return page == bad_page || page == hwpoison_page || page == fault_page;
-}
-EXPORT_SYMBOL_GPL(is_error_page);
-
-int is_error_pfn(pfn_t pfn)
-{
-	return pfn == bad_pfn || pfn == hwpoison_pfn || pfn == fault_pfn;
-}
-EXPORT_SYMBOL_GPL(is_error_pfn);
-
-int is_hwpoison_pfn(pfn_t pfn)
-{
-	return pfn == hwpoison_pfn;
-}
-EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
-
-int is_fault_pfn(pfn_t pfn)
-{
-	return pfn == fault_pfn;
-}
-EXPORT_SYMBOL_GPL(is_fault_pfn);
-
-int is_noslot_pfn(pfn_t pfn)
-{
-	return pfn == bad_pfn;
-}
-EXPORT_SYMBOL_GPL(is_noslot_pfn);
-
-int is_invalid_pfn(pfn_t pfn)
-{
-	return pfn == hwpoison_pfn || pfn == fault_pfn;
-}
-EXPORT_SYMBOL_GPL(is_invalid_pfn);
-
-static inline unsigned long bad_hva(void)
-{
-	return PAGE_OFFSET;
-}
-
-int kvm_is_error_hva(unsigned long addr)
-{
-	return addr == bad_hva();
-}
-EXPORT_SYMBOL_GPL(kvm_is_error_hva);
-
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 {
 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
@@ -1021,28 +974,62 @@ out:
 	return size;
 }
 
-static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
-				     gfn_t *nr_pages)
+static bool memslot_is_readonly(struct kvm_memory_slot *slot)
+{
+	return slot->flags & KVM_MEM_READONLY;
+}
+
+static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
+				       gfn_t *nr_pages, bool write)
 {
 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
-		return bad_hva();
+		return KVM_HVA_ERR_BAD;
+
+	if (memslot_is_readonly(slot) && write)
+		return KVM_HVA_ERR_RO_BAD;
 
 	if (nr_pages)
 		*nr_pages = slot->npages - (gfn - slot->base_gfn);
 
-	return gfn_to_hva_memslot(slot, gfn);
+	return __gfn_to_hva_memslot(slot, gfn);
 }
 
+static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
+				     gfn_t *nr_pages)
+{
+	return __gfn_to_hva_many(slot, gfn, nr_pages, true);
+}
+
+unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
+				 gfn_t gfn)
+{
+	return gfn_to_hva_many(slot, gfn, NULL);
+}
+EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
+
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
 }
 EXPORT_SYMBOL_GPL(gfn_to_hva);
 
-static pfn_t get_fault_pfn(void)
+/*
+ * The hva returned by this function is only allowed to be read.
+ * It should pair with kvm_read_hva() or kvm_read_hva_atomic().
+ */
+static unsigned long gfn_to_hva_read(struct kvm *kvm, gfn_t gfn)
+{
+	return __gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL, false);
+}
+
+static int kvm_read_hva(void *data, void __user *hva, int len)
 {
-	get_page(fault_page);
-	return fault_pfn;
+	return __copy_from_user(data, hva, len);
+}
+
+static int kvm_read_hva_atomic(void *data, void __user *hva, int len)
+{
+	return __copy_from_user_inatomic(data, hva, len);
 }
 
 int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
@@ -1065,108 +1052,186 @@ static inline int check_user_page_hwpoison(unsigned long addr)
 	return rc == -EHWPOISON;
 }
 
-static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr, bool atomic,
-			bool *async, bool write_fault, bool *writable)
+/*
+ * The atomic path to get the writable pfn which will be stored in @pfn,
+ * true indicates success, otherwise false is returned.
+ */
+static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
+			    bool write_fault, bool *writable, pfn_t *pfn)
 {
 	struct page *page[1];
-	int npages = 0;
-	pfn_t pfn;
+	int npages;
 
-	/* we can do it either atomically or asynchronously, not both */
-	BUG_ON(atomic && async);
+	if (!(async || atomic))
+		return false;
 
-	BUG_ON(!write_fault && !writable);
+	/*
+	 * Fast pin a writable pfn only if it is a write fault request
+	 * or the caller allows to map a writable pfn for a read fault
+	 * request.
+	 */
+	if (!(write_fault || writable))
+		return false;
 
-	if (writable)
-		*writable = true;
+	npages = __get_user_pages_fast(addr, 1, 1, page);
+	if (npages == 1) {
+		*pfn = page_to_pfn(page[0]);
 
-	if (atomic || async)
-		npages = __get_user_pages_fast(addr, 1, 1, page);
+		if (writable)
+			*writable = true;
+		return true;
+	}
 
-	if (unlikely(npages != 1) && !atomic) {
-		might_sleep();
+	return false;
+}
 
-		if (writable)
-			*writable = write_fault;
+/*
+ * The slow path to get the pfn of the specified host virtual address,
+ * 1 indicates success, -errno is returned if error is detected.
+ */
+static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
+			   bool *writable, pfn_t *pfn)
+{
+	struct page *page[1];
+	int npages = 0;
 
-		if (async) {
-			down_read(&current->mm->mmap_sem);
-			npages = get_user_page_nowait(current, current->mm,
-						     addr, write_fault, page);
-			up_read(&current->mm->mmap_sem);
-		} else
-			npages = get_user_pages_fast(addr, 1, write_fault,
-						     page);
-
-		/* map read fault as writable if possible */
-		if (unlikely(!write_fault) && npages == 1) {
-			struct page *wpage[1];
-
-			npages = __get_user_pages_fast(addr, 1, 1, wpage);
-			if (npages == 1) {
-				*writable = true;
-				put_page(page[0]);
-				page[0] = wpage[0];
-			}
-			npages = 1;
+	might_sleep();
+
+	if (writable)
+		*writable = write_fault;
+
+	if (async) {
+		down_read(&current->mm->mmap_sem);
+		npages = get_user_page_nowait(current, current->mm,
+					      addr, write_fault, page);
+		up_read(&current->mm->mmap_sem);
+	} else
+		npages = get_user_pages_fast(addr, 1, write_fault,
+					     page);
+	if (npages != 1)
+		return npages;
+
+	/* map read fault as writable if possible */
+	if (unlikely(!write_fault) && writable) {
+		struct page *wpage[1];
+
+		npages = __get_user_pages_fast(addr, 1, 1, wpage);
+		if (npages == 1) {
+			*writable = true;
+			put_page(page[0]);
+			page[0] = wpage[0];
 		}
+
+		npages = 1;
 	}
+	*pfn = page_to_pfn(page[0]);
+	return npages;
+}
 
-	if (unlikely(npages != 1)) {
-		struct vm_area_struct *vma;
+static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
+{
+	if (unlikely(!(vma->vm_flags & VM_READ)))
+		return false;
 
-		if (atomic)
-			return get_fault_pfn();
+	if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
+		return false;
 
-		down_read(&current->mm->mmap_sem);
-		if (npages == -EHWPOISON ||
-			(!async && check_user_page_hwpoison(addr))) {
-			up_read(&current->mm->mmap_sem);
-			get_page(hwpoison_page);
-			return page_to_pfn(hwpoison_page);
-		}
+	return true;
+}
 
-		vma = find_vma_intersection(current->mm, addr, addr+1);
-
-		if (vma == NULL)
-			pfn = get_fault_pfn();
-		else if ((vma->vm_flags & VM_PFNMAP)) {
-			pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
-				vma->vm_pgoff;
-			BUG_ON(!kvm_is_mmio_pfn(pfn));
-		} else {
-			if (async && (vma->vm_flags & VM_WRITE))
-				*async = true;
-			pfn = get_fault_pfn();
-		}
-		up_read(&current->mm->mmap_sem);
-	} else
-		pfn = page_to_pfn(page[0]);
+/*
+ * Pin guest page in memory and return its pfn.
+ * @addr: host virtual address which maps memory to the guest
+ * @atomic: whether this function can sleep
+ * @async: whether this function need to wait IO complete if the
+ *         host page is not in the memory
+ * @write_fault: whether we should get a writable host page
+ * @writable: whether it allows to map a writable host page for !@write_fault
+ *
+ * The function will map a writable host page for these two cases:
+ * 1): @write_fault = true
+ * 2): @write_fault = false && @writable, @writable will tell the caller
+ *     whether the mapping is writable.
+ */
+static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
+			bool write_fault, bool *writable)
+{
+	struct vm_area_struct *vma;
+	pfn_t pfn = 0;
+	int npages;
+
+	/* we can do it either atomically or asynchronously, not both */
+	BUG_ON(atomic && async);
 
+	if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
+		return pfn;
+
+	if (atomic)
+		return KVM_PFN_ERR_FAULT;
+
+	npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
+	if (npages == 1)
+		return pfn;
+
+	down_read(&current->mm->mmap_sem);
+	if (npages == -EHWPOISON ||
+	      (!async && check_user_page_hwpoison(addr))) {
+		pfn = KVM_PFN_ERR_HWPOISON;
+		goto exit;
+	}
+
+	vma = find_vma_intersection(current->mm, addr, addr + 1);
+
+	if (vma == NULL)
+		pfn = KVM_PFN_ERR_FAULT;
+	else if ((vma->vm_flags & VM_PFNMAP)) {
+		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
+			vma->vm_pgoff;
+		BUG_ON(!kvm_is_mmio_pfn(pfn));
+	} else {
+		if (async && vma_is_valid(vma, write_fault))
+			*async = true;
+		pfn = KVM_PFN_ERR_FAULT;
+	}
+exit:
+	up_read(&current->mm->mmap_sem);
 	return pfn;
 }
 
-pfn_t hva_to_pfn_atomic(struct kvm *kvm, unsigned long addr)
+static pfn_t
+__gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
+		     bool *async, bool write_fault, bool *writable)
 {
-	return hva_to_pfn(kvm, addr, true, NULL, true, NULL);
+	unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
+
+	if (addr == KVM_HVA_ERR_RO_BAD)
+		return KVM_PFN_ERR_RO_FAULT;
+
+	if (kvm_is_error_hva(addr))
+		return KVM_PFN_ERR_BAD;
+
+	/* Do not map writable pfn in the readonly memslot. */
+	if (writable && memslot_is_readonly(slot)) {
+		*writable = false;
+		writable = NULL;
+	}
+
+	return hva_to_pfn(addr, atomic, async, write_fault,
+			  writable);
 }
-EXPORT_SYMBOL_GPL(hva_to_pfn_atomic);
 
 static pfn_t __gfn_to_pfn(struct kvm *kvm, gfn_t gfn, bool atomic, bool *async,
 			  bool write_fault, bool *writable)
 {
-	unsigned long addr;
+	struct kvm_memory_slot *slot;
 
 	if (async)
 		*async = false;
 
-	addr = gfn_to_hva(kvm, gfn);
-	if (kvm_is_error_hva(addr)) {
-		get_page(bad_page);
-		return page_to_pfn(bad_page);
-	}
+	slot = gfn_to_memslot(kvm, gfn);
 
-	return hva_to_pfn(kvm, addr, atomic, async, write_fault, writable);
+	return __gfn_to_pfn_memslot(slot, gfn, atomic, async, write_fault,
+				    writable);
 }
 
 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
@@ -1195,12 +1260,16 @@ pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
 }
 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
 
-pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
-			 struct kvm_memory_slot *slot, gfn_t gfn)
+pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
+{
+	return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
+}
+
+pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
 {
-	unsigned long addr = gfn_to_hva_memslot(slot, gfn);
-	return hva_to_pfn(kvm, addr, false, NULL, true, NULL);
+	return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
 }
+EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
 
 int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
 								  int nr_pages)
@@ -1219,30 +1288,42 @@ int gfn_to_page_many_atomic(struct kvm *kvm, gfn_t gfn, struct page **pages,
 }
 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
 
+static struct page *kvm_pfn_to_page(pfn_t pfn)
+{
+	if (is_error_pfn(pfn))
+		return KVM_ERR_PTR_BAD_PAGE;
+
+	if (kvm_is_mmio_pfn(pfn)) {
+		WARN_ON(1);
+		return KVM_ERR_PTR_BAD_PAGE;
+	}
+
+	return pfn_to_page(pfn);
+}
+
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 {
 	pfn_t pfn;
 
 	pfn = gfn_to_pfn(kvm, gfn);
-	if (!kvm_is_mmio_pfn(pfn))
-		return pfn_to_page(pfn);
-
-	WARN_ON(kvm_is_mmio_pfn(pfn));
 
-	get_page(bad_page);
-	return bad_page;
+	return kvm_pfn_to_page(pfn);
 }
 
 EXPORT_SYMBOL_GPL(gfn_to_page);
 
 void kvm_release_page_clean(struct page *page)
 {
+	WARN_ON(is_error_page(page));
+
 	kvm_release_pfn_clean(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
 
 void kvm_release_pfn_clean(pfn_t pfn)
 {
+	WARN_ON(is_error_pfn(pfn));
+
 	if (!kvm_is_mmio_pfn(pfn))
 		put_page(pfn_to_page(pfn));
 }
@@ -1250,6 +1331,8 @@ EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
 
 void kvm_release_page_dirty(struct page *page)
 {
+	WARN_ON(is_error_page(page));
+
 	kvm_release_pfn_dirty(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
@@ -1305,10 +1388,10 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 	int r;
 	unsigned long addr;
 
-	addr = gfn_to_hva(kvm, gfn);
+	addr = gfn_to_hva_read(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
-	r = __copy_from_user(data, (void __user *)addr + offset, len);
+	r = kvm_read_hva(data, (void __user *)addr + offset, len);
 	if (r)
 		return -EFAULT;
 	return 0;
@@ -1343,11 +1426,11 @@ int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int offset = offset_in_page(gpa);
 
-	addr = gfn_to_hva(kvm, gfn);
+	addr = gfn_to_hva_read(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	pagefault_disable();
-	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
+	r = kvm_read_hva_atomic(data, (void __user *)addr + offset, len);
 	pagefault_enable();
 	if (r)
 		return -EFAULT;
@@ -1485,8 +1568,7 @@ void mark_page_dirty_in_slot(struct kvm *kvm, struct kvm_memory_slot *memslot,
 	if (memslot && memslot->dirty_bitmap) {
 		unsigned long rel_gfn = gfn - memslot->base_gfn;
 
-		/* TODO: introduce set_bit_le() and use it */
-		test_and_set_bit_le(rel_gfn, memslot->dirty_bitmap);
+		set_bit_le(rel_gfn, memslot->dirty_bitmap);
 	}
 }
 
@@ -1580,6 +1662,43 @@ bool kvm_vcpu_yield_to(struct kvm_vcpu *target)
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
 
+#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
+/*
+ * Helper that checks whether a VCPU is eligible for directed yield.
+ * Most eligible candidate to yield is decided by following heuristics:
+ *
+ *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
+ *  (preempted lock holder), indicated by @in_spin_loop.
+ *  Set at the beiginning and cleared at the end of interception/PLE handler.
+ *
+ *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
+ *  chance last time (mostly it has become eligible now since we have probably
+ *  yielded to lockholder in last iteration. This is done by toggling
+ *  @dy_eligible each time a VCPU checked for eligibility.)
+ *
+ *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
+ *  to preempted lock-holder could result in wrong VCPU selection and CPU
+ *  burning. Giving priority for a potential lock-holder increases lock
+ *  progress.
+ *
+ *  Since algorithm is based on heuristics, accessing another VCPU data without
+ *  locking does not harm. It may result in trying to yield to  same VCPU, fail
+ *  and continue with next VCPU and so on.
+ */
+bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
+{
+	bool eligible;
+
+	eligible = !vcpu->spin_loop.in_spin_loop ||
+			(vcpu->spin_loop.in_spin_loop &&
+			 vcpu->spin_loop.dy_eligible);
+
+	if (vcpu->spin_loop.in_spin_loop)
+		kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
+
+	return eligible;
+}
+#endif
 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 {
 	struct kvm *kvm = me->kvm;
@@ -1589,6 +1708,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 	int pass;
 	int i;
 
+	kvm_vcpu_set_in_spin_loop(me, true);
 	/*
 	 * We boost the priority of a VCPU that is runnable but not
 	 * currently running, because it got preempted by something
@@ -1607,6 +1727,8 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 				continue;
 			if (waitqueue_active(&vcpu->wq))
 				continue;
+			if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
+				continue;
 			if (kvm_vcpu_yield_to(vcpu)) {
 				kvm->last_boosted_vcpu = i;
 				yielded = 1;
@@ -1614,6 +1736,10 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me)
 			}
 		}
 	}
+	kvm_vcpu_set_in_spin_loop(me, false);
+
+	/* Ensure vcpu is not eligible during next spinloop */
+	kvm_vcpu_set_dy_eligible(me, false);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
 
@@ -1766,7 +1892,9 @@ static long kvm_vcpu_ioctl(struct file *filp,
 #endif
 
 
-	vcpu_load(vcpu);
+	r = vcpu_load(vcpu);
+	if (r)
+		return r;
 	switch (ioctl) {
 	case KVM_RUN:
 		r = -EINVAL;
@@ -2094,6 +2222,29 @@ static long kvm_vm_ioctl(struct file *filp,
 		break;
 	}
 #endif
+#ifdef __KVM_HAVE_IRQ_LINE
+	case KVM_IRQ_LINE_STATUS:
+	case KVM_IRQ_LINE: {
+		struct kvm_irq_level irq_event;
+
+		r = -EFAULT;
+		if (copy_from_user(&irq_event, argp, sizeof irq_event))
+			goto out;
+
+		r = kvm_vm_ioctl_irq_line(kvm, &irq_event);
+		if (r)
+			goto out;
+
+		r = -EFAULT;
+		if (ioctl == KVM_IRQ_LINE_STATUS) {
+			if (copy_to_user(argp, &irq_event, sizeof irq_event))
+				goto out;
+		}
+
+		r = 0;
+		break;
+	}
+#endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
 		if (r == -ENOTTY)
@@ -2698,9 +2849,6 @@ static struct syscore_ops kvm_syscore_ops = {
 	.resume = kvm_resume,
 };
 
-struct page *bad_page;
-pfn_t bad_pfn;
-
 static inline
 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
 {
@@ -2732,33 +2880,6 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
 	if (r)
 		goto out_fail;
 
-	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-
-	if (bad_page == NULL) {
-		r = -ENOMEM;
-		goto out;
-	}
-
-	bad_pfn = page_to_pfn(bad_page);
-
-	hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-
-	if (hwpoison_page == NULL) {
-		r = -ENOMEM;
-		goto out_free_0;
-	}
-
-	hwpoison_pfn = page_to_pfn(hwpoison_page);
-
-	fault_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-
-	if (fault_page == NULL) {
-		r = -ENOMEM;
-		goto out_free_0;
-	}
-
-	fault_pfn = page_to_pfn(fault_page);
-
 	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
 		r = -ENOMEM;
 		goto out_free_0;
@@ -2833,12 +2954,6 @@ out_free_1:
 out_free_0a:
 	free_cpumask_var(cpus_hardware_enabled);
 out_free_0:
-	if (fault_page)
-		__free_page(fault_page);
-	if (hwpoison_page)
-		__free_page(hwpoison_page);
-	__free_page(bad_page);
-out:
 	kvm_arch_exit();
 out_fail:
 	return r;
@@ -2858,8 +2973,5 @@ void kvm_exit(void)
 	kvm_arch_hardware_unsetup();
 	kvm_arch_exit();
 	free_cpumask_var(cpus_hardware_enabled);
-	__free_page(fault_page);
-	__free_page(hwpoison_page);
-	__free_page(bad_page);
 }
 EXPORT_SYMBOL_GPL(kvm_exit);